Commit 817f2395 authored by Yoni Fogel's avatar Yoni Fogel

Import make.mysql.bash@HEAD

parent 2b1d690e
#!/usr/bin/env bash
shopt -s compat31 2> /dev/null
function usage() {
echo "make.mysql.bash - build mysql with the fractal tree"
echo "[--branch=$branch] [--revision=$revision] [--suffix=$suffix]"
echo "[--mysql=$mysql]"
echo "[--tokudb=$tokudb] [--tokudbengine=$tokudbengine]"
echo "[--jemalloc=$jemalloc]"
echo "[--cc=$cc --cxx=$cxx] [--ftcc=$ftcc --ftcxx=$ftcxx] [--debugbuild=$debugbuild] [--staticft=$staticft]"
echo "[--rpm=$rpm] [--do_s3=$do_s3] [--do_make_check=$do_make_check]"
return 1
}
function retry() {
local cmd
local retries
local exitcode
cmd=$*
let retries=0
while [ $retries -le 10 ] ; do
echo `date` $cmd
bash -c "$cmd"
exitcode=$?
echo `date` $cmd $exitcode $retries
let retries=retries+1
if [ $exitcode -eq 0 ] ; then break; fi
sleep 10
done
test $exitcode = 0
}
function get_ncpus() {
if [ -f /proc/cpuinfo ]; then
grep bogomips /proc/cpuinfo | wc -l
elif [ $system = darwin ] ; then
sysctl -n hw.ncpu
else
echo 1
fi
}
function get_branchrevision() {
local branch=$1; local revision=$2; local suffix=$3
local branchrevision
if [ $branch != "." ] ; then
branchrevision=$(basename $branch)-$revision
else
branchrevision=$revision
fi
if [ "$suffix" != "." ] ; then
branchrevision=$branchrevision-$suffix
fi
echo $branchrevision
}
function mysql_version() {
local mysql=$1
if [[ $mysql =~ (mysql|mysqlcom|mariadb)-(.*) ]] ; then
echo ${BASH_REMATCH[2]}
fi
}
function build_jemalloc() {
local jemalloc=$1; local branch=$2; local revision=$3; local branchrevision=$4
if [ ! -d $jemalloc ] ; then
retry svn export -q -r $revision $svnserver/$branch/$jemalloc $jemalloc
if [ $? != 0 ] ; then exit 1; fi
pushd $jemalloc
if [ $? = 0 ] ; then
CC=$cc ./configure --with-private-namespace=jemalloc_
if [ $? != 0 ] ; then exit 1; fi
make
if [ $? != 0 ] ; then exit 1; fi
popd
fi
fi
}
# check out the fractal tree source from subversion, build it, and make the fractal tree tarballs
function build_fractal_tree() {
if [ ! -d $tokufractaltreedir ] ; then
mkdir $tokufractaltreedir
if [ $branch = "." ] ; then tokudb_branch=toku; else tokudb_branch=$branch; fi
retry svn export -q -r $revision $svnserver/$tokudb_branch/$tokudb
exitcode=$?; if [ $exitcode != 0 ] ; then exit 1; fi
if [[ $ftcc =~ icc ]] ; then
if [ -d /opt/intel/bin ] ; then
export PATH=$PATH:/opt/intel/bin
. /opt/intel/bin/compilervars.sh intel64
fi
fi
retry svn export -q -r $revision $svnserver/$branch/$xz
exitcode=$?; if [ $exitcode != 0 ] ; then exit 1; fi
pushd $tokudb
echo `date` make $tokudb $ftcc $($ftcc --version)
local intel_cc=""
if [[ $ftcc = "icc" ]]; then
intel_cc="ON"
cmake_env=""
else
intel_cc="OFF"
cmake_env="CC=$ftcc CXX=$ftcxx"
fi
local use_valgrind=""
local debug_paranoid=""
if [[ $debugbuild = 1 ]]; then
use_valgrind="ON"
debug_paranoid="ON"
else
use_valgrind="OFF"
debug_paranoid="OFF"
fi
mkdir -p build
cd build
# only use FT Release or Debug build types
local ft_build_type=$build_type
if [ $ft_build_type = RelWithDebInfo ] ; then ft_build_type=Release; fi
eval $cmake_env cmake \
-D LIBTOKUDB=$tokufractaltree \
-D LIBTOKUPORTABILITY=$tokuportability \
-D CMAKE_TOKUDB_REVISION=$revision \
-D CMAKE_BUILD_TYPE=$ft_build_type \
-D TOKU_SVNROOT=$basedir \
-D JEMALLOC_SOURCE_DIR=../../$jemalloc \
-D CMAKE_INSTALL_PREFIX=../../$tokufractaltreedir \
-D BUILD_TESTING=OFF \
-D USE_GTAGS=OFF \
-D USE_CTAGS=OFF \
-D USE_ETAGS=OFF \
-D USE_CSCOPE=OFF \
-D USE_VALGRIND=$use_valgrind \
-D TOKU_DEBUG_PARANOID=$debug_paranoid \
..
make install -j$makejobs
exitcode=$?; if [ $exitcode != 0 ] ; then exit 1; fi
popd
pushd $tokufractaltreedir/examples
# test the examples
sed -ie "s/LIBTOKUDB = tokudb/LIBTOKUDB = $tokufractaltree/" Makefile
sed -ie "s/LIBTOKUPORTABILITY = tokuportability/LIBTOKUPORTABILITY = $tokuportability/" Makefile
if [ $system = darwin ] ; then
DYLD_LIBRARY_PATH=$PWD/../lib:$DYLD_LIBRARY_PATH make check CC=$ftcc
exitcode=$?
else
if [ $do_make_check != 0 ] ; then
make check CC=$ftcc
exitcode=$?
else
exitcode=0
fi
fi
echo `date` make check examples $tokufractaltree $exitcode
if [ $exitcode != 0 ] ; then exit 1; fi
make clean
popd
# add jemalloc.so
if [ $system = darwin ] ; then
cp $jemalloc/lib/libjemalloc.1.dylib $tokufractaltreedir/lib/lib$jemalloc-$branchrevision.dylib
else
cp $jemalloc/lib/libjemalloc.so.1 $tokufractaltreedir/lib/lib$jemalloc-$branchrevision.so
fi
cp $jemalloc/lib/libjemalloc.a $tokufractaltreedir/lib/libjemalloc.a
# make tarballs
tar czf $tokufractaltreedir.tar.gz $tokufractaltreedir
md5sum $tokufractaltreedir.tar.gz >$tokufractaltreedir.tar.gz.md5
md5sum --check $tokufractaltreedir.tar.gz.md5
fi
}
function generate_cmake_cmd() {
local mysqlsrc=$1; shift
local extra_flags=$*
if [[ $mysqlsrc =~ ^Percona ]] ; then extra_flags=-DWITH_EMBEDDED_SERVER=OFF; fi
echo -n cmake .. $extra_flags -DBUILD_CONFIG=mysql_release -DCMAKE_BUILD_TYPE=$build_type
if [ $system = linux ] ; then
echo -n " " \
-DCMAKE_EXE_LINKER_FLAGS=\"-Wl,--whole-archive \$TOKUFRACTALTREE/lib/libjemalloc.a -Wl,-no-whole-archive\"
elif [ $system = darwin ] ; then
echo -n " " \
-DWITH_SAFEMALLOC=OFF -DWITH_SSL=system \
-DCMAKE_EXE_LINKER_FLAGS=\"-Wl,-force_load -Wl,\$TOKUFRACTALTREE/lib/libjemalloc.a\"
else
exit 1
fi
if [ $link_enterprise_hot_backup != 0 ] ; then
echo -n " " -DCOMPILATION_COMMENT=\"TokuDB Enterprise Server \(GPL\)\"
fi
echo
}
# generate a script that builds from the mysql src tarball and the fractal tree tarball
function generate_build_from_src() {
local tokumysqldir=$1; local mysqlsrc=$2; local tokufractaltreedir=$3
pushd $mysqlsrc >/dev/null 2>&1
if [ $? = 0 ] ; then
echo '#/usr/bin/env bash'
# defaults
echo 'makejobs=1'
echo 'cc=gcc47'
echo 'cxx=g++47'
# parse arguments
echo 'while [ $# -gt 0 ] ; do'
echo ' arg=$1; shift'
echo ' if [[ $arg =~ --(.*)=(.*) ]] ; then'
echo ' eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}'
echo ' else'
echo ' break'
echo ' fi'
echo 'done'
# check tarballs
echo md5sum --check $mysqlsrc.tar.gz.md5
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo md5sum --check $tokufractaltreedir.tar.gz.md5
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo "if [ ! -d $mysqlsrc ] ; then"
echo " tar xzf $mysqlsrc.tar.gz"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo fi
echo "if [ ! -d $tokufractaltreedir ] ;then"
echo " tar xzf $tokufractaltreedir.tar.gz"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo fi
# setup environment variables
echo export TOKUDB_VERSION=$branchrevision
echo export TOKUFRACTALTREE='$PWD'/$tokufractaltreedir
echo export TOKUDB_PATCHES=$tokudb_patches
if [ $staticft != 0 ] ; then
echo export TOKUFRACTALTREE_LIBNAME=${tokufractaltree}_static
echo export TOKUPORTABILITY_LIBNAME=${tokuportability}_static
else
echo export TOKUFRACTALTREE_LIBNAME=${tokufractaltree}
echo export TOKUPORTABILITY_LIBNAME=${tokuportability}
fi
echo 'export CC=$cc; export CXX=$cxx'
echo "pushd $mysqlsrc"
echo 'if [ $? != 0 ] ; then exit 1; fi'
if [ ! -f CMakeLists.txt ] ; then exit 1; fi
if [ $build_tarball != 0 ] ; then
echo "mkdir build.$build_type; pushd build.$build_type"
echo 'if [ $? != 0 ] ; then exit 1; fi'
cmd=$(generate_cmake_cmd $mysqlsrc)
echo $cmd
echo 'if [ $? != 0 ] ; then exit 1; fi'
# make binary distribution package
echo 'make package -j$makejobs'
echo 'if [ $? != 0 ] ; then exit 1; fi'
# fixup tarball name mismatch (if any)
echo "if [ ! -f $mysqldir.tar.gz ] ; then"
echo ' oldtb=$(ls *.gz)'
echo ' if [[ $oldtb =~ (.*).tar.gz ]] ; then oldmysqldir=${BASH_REMATCH[1]}; fi'
echo ' tar xzf $oldtb'
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo ' mv $oldmysqldir' $mysqldir
echo " tar czf $mysqldir.tar.gz $mysqldir"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo 'fi'
# add ft files to the mysql tarball
echo tar xzf $mysqldir.tar.gz
echo 'if [ $? != 0 ] ; then exit 1; fi'
if [ $staticft = 0 ] ; then
# add fractal tree libs to the mysql tarball only if we are linking to shared libs
echo "if [ -d $mysqldir/lib/mysql ] ; then"
echo " cp ../../$tokufractaltreedir/lib/lib* $mysqldir/lib/mysql"
echo else
echo " cp ../../$tokufractaltreedir/lib/lib* $mysqldir/lib"
echo fi
echo 'if [ $? != 0 ] ; then exit 1; fi'
fi
# add ftdump to mysql tarball
echo "cp ../../$tokufractaltreedir/bin/ftdump $mysqldir/bin/tokuftdump"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# add tokustat to the mysql tarball
echo "cp ../../$mysqlsrc/scripts/tokustat.py $mysqldir/bin"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# add tokufilecheck to the mysql tarball
echo "cp ../../$mysqlsrc/scripts/tokufilecheck.py $mysqldir/bin"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# cleanup
echo rm $mysqldir.tar.gz
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo tar czf $mysqldir.tar.gz $mysqldir
echo 'if [ $? != 0 ] ; then exit 1; fi'
# generate the md5 file
echo "if [ -f $mysqldir.tar.gz ] ; then"
echo " md5sum $mysqldir.tar.gz >$mysqldir.tar.gz.md5"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo 'fi'
echo popd
fi
# make RPMs
if [ $system = linux -a $rpm != 0 ]; then
echo "mkdir build.${build_type}.rpms; pushd build.${build_type}.rpms"
echo 'if [ $? != 0 ] ; then exit 1; fi'
if [[ $mysqlsrc =~ ^mariadb ]] ; then
echo linux_distro=linux
echo 'if [ -f /etc/issue ] ; then'
echo ' if [[ "$(head -1 /etc/issue)" =~ Red\ Hat\ Enterprise\ Linux\ Server\ release\ ([56]) ]] ; then linux_distro=rhel${BASH_REMATCH[1]}; fi'
echo ' if [[ "$(head -1 /etc/issue)" =~ CentOS\ release\ ([56]) ]] ; then linux_distro=centos${BASH_REMATCH[1]}; fi'
echo fi
cmd=$(generate_cmake_cmd $mysqlsrc -DRPM=\$linux_distro)
echo $cmd
echo 'if [ $? != 0 ] ; then exit 1; fi'
# make binary distribution package
echo 'make package -j$makejobs'
echo 'if [ $? != 0 ] ; then exit 1; fi'
else
echo 'mkdir -p {tmp,BUILD,RPMS,SOURCES,SPECS,SRPMS}'
# split myslqsrc into components
echo "if [[ $mysqlsrc =~ (.*)-([0-9]+\.[0-9]+\.[0-9]+.*)-tokudb-(.*) ]] ; then"
echo ' mysql_distro=${BASH_REMATCH[1]}'
echo ' mysql_version=${BASH_REMATCH[2]}'
echo ' mysql_distro_version=${mysql_distro}-${mysql_version}'
echo ' tokudb_distro=tokudb'
echo ' tokudb_version=${BASH_REMATCH[3]}'
echo ' tokudb_distro_version=${tokudb_distro}-${tokudb_version}'
echo else
echo ' exit 1'
echo fi
# copy the source tarball
echo "cp ../../$mysqlsrc.tar.gz SOURCES/$mysqlsrc.tar.gz"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# hack on the spec file
echo 'specfile=${mysql_distro}.${mysql_version}.spec'
echo 'cp ../support-files/$specfile SPECS/$specfile'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'sed -i -e"s/^%define mysql_version.*/&-${tokudb_distro_version}/" SPECS/$specfile'
echo 'sed -i -e"s/^%define release.*/%define release $(echo ${tokudb_distro_version}|tr - .)/" SPECS/$specfile'
# echo 'sed -i -e"s/^\(.*-DMYSQL_SERVER_SUFFIX=.*\)$/& -DEXTRA_VERSION=-${tokudb_distro_version}/" SPECS/$specfile'
# add jemalloc to the linker flags
echo 'sed -i -e"s/^\(.*-DMYSQL_SERVER_SUFFIX=.*\)$/& -DCMAKE_EXE_LINKER_FLAGS=\"-Wl,--whole-archive \$\{TOKUFRACTALTREE\}\/lib\/libjemalloc.a -Wl,-no-whole-archive\"/" SPECS/$specfile'
# add the hot backup lib to the server package file list
echo 'sed -i -e"s/%{_datadir}\/mysql\/$/&\n%attr(755, root, root) %{_libdir}\/libHotBackup*.so/" SPECS/$specfile'
# build generic linux RPMs
echo 'export MYSQL_BUILD_MAKE_JFLAG=-j$makejobs'
echo 'rpmbuild -v --define="_topdir $PWD" --define="_tmppath $PWD/tmp" --define="src_base $mysql_distro" -ba SPECS/$specfile'
echo 'if [ $? != 0 ] ; then exit 1; fi'
# move the rpms
echo 'pushd RPMS/$(uname -m)'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'mv *.rpm ../..'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'popd'
fi
# generate the md5 file
echo 'rpmfiles=$(ls *.rpm)'
echo 'for x in $rpmfiles; do'
echo ' md5sum $x >$x.md5'
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo 'done'
echo popd
fi
echo popd
popd >/dev/null 2>&1
fi
}
# checkout the mysql source from subversion, generate a build script, and make the mysql source tarball
function build_mysql_src() {
if [ ! -d $mysqlsrc ] ; then
# get the mysql source
if [ $branch = "." ] ; then mysql_branch=mysql.com; else mysql_branch=$branch; fi
retry svn export -q -r $revision $svnserver/$mysql_branch/$mysql $mysqlsrc
if [ $? != 0 ] ; then exit 1; fi
# get the tokudb handlerton source
if [ $branch = "." ] ; then tokumysql_branch=mysql; else tokumysql_branch=$branch/mysql; fi
pushd $mysqlsrc/storage
retry svn export -q -r $revision $svnserver/$tokumysql_branch/tokudb-engine/$tokudbengine tokudb
if [ $? != 0 ] ; then exit 1; fi
popd
pushd $mysqlsrc/scripts
retry svn export -q -r $revision $svnserver/$tokumysql_branch/scripts/tokustat.py tokustat.py
if [ $? != 0 ] ; then exit 1; fi
popd
pushd $mysqlsrc/scripts
retry svn export -q -r $revision $svnserver/$tokumysql_branch/scripts/tokufilecheck.py tokufilecheck.py
if [ $? != 0 ] ; then exit 1; fi
popd
# append the tokudb version to the MYSQL_VERSION_EXTRA variable in the VERSION file
pushd $mysqlsrc
if [ $? = 0 ] ; then
sed -i "" -e"s/^MYSQL_VERSION_EXTRA=.*/&-tokudb-${branchrevision}/" VERSION
popd
fi
# merge the common mysql tests into the source
pushd $mysqlsrc/mysql-test
if [ $? = 0 ] ; then
mkdir ../mysql-test-save
mv * ../mysql-test-save
retry svn export -q -r $revision $svnserver/$tokumysql_branch/tests/mysql-test
mv mysql-test/* .
rmdir mysql-test
cp -r ../mysql-test-save/* .
rm -rf ../mysql-test-save
popd
fi
# get the hot backup source
pushd $mysqlsrc
if [ $? = 0 ] ; then
if [ $link_enterprise_hot_backup != 0 ] ; then
retry svn export -q -r $revision $svnserver/backup-restore/backup/ toku_backup
else
retry svn export -q -r $revision $svnserver/backup-restore/backup-community/ toku_backup
fi
fi
popd
generate_build_from_src $tokumysql_branch $mysqlsrc $tokufractaltreedir >$mysqlsrc/scripts/tokudb.build.bash
# make the mysql src tarball
tar czf $mysqlsrc.tar.gz $mysqlsrc
if [ $? != 0 ] ; then exit 1; fi
md5sum $mysqlsrc.tar.gz >$mysqlsrc.tar.gz.md5
if [ $? != 0 ] ; then exit 1; fi
fi
}
# make the mysql release tarball
# by executing the build script in the mysql source tarball and compiling with the fractal tree tarball
function build_mysql_release() {
if [ ! -d $mysqlbuild ] ; then
mkdir $mysqlbuild
pushd $mysqlbuild
# create links to the mysql source and fractal tree tarballs
ln $basedir/$mysqlsrc.tar.gz
ln $basedir/$mysqlsrc.tar.gz.md5
ln $basedir/$tokufractaltreedir.tar.gz
ln $basedir/$tokufractaltreedir.tar.gz.md5
# extract the build script
tar xzf $mysqlsrc.tar.gz $mysqlsrc/scripts/tokudb.build.bash
if [ $? != 0 ] ; then exit 1; fi
cp $mysqlsrc/scripts/tokudb.build.bash .
if [ $? != 0 ] ; then exit 1; fi
rm -rf $mysqlsrc
# execute the build script
bash -x tokudb.build.bash --makejobs=$makejobs --cc=$cc --cxx=$cxx
if [ $? != 0 ] ; then exit 1; fi
# move the build files
pushd $mysqlsrc/build.${build_type}
if [ $? = 0 ] ; then
cp $mysqldir.tar.gz* $basedir; if [ $? != 0 ] ; then exit 1; fi
popd
fi
if [ $rpm != 0 ] ; then
pushd $mysqlsrc/build.${build_type}.rpms
if [ $? = 0 ] ; then
rpmfiles=$(ls *.rpm *.rpm.md5)
for x in $rpmfiles; do cp $x $basedir; if [ $? != 0 ] ; then exit 1; fi; done
popd
fi
fi
popd
fi
}
PATH=$HOME/bin:$PATH
branch=.
revision=0
suffix=.
mysql=mysql-5.5.28
tokudb=tokudb
tokudbengine=tokudb-engine
link_enterprise_hot_backup=0
do_s3=1
do_make_check=1
cc=gcc47
cxx=g++47
ftcc=gcc47
ftcxx=g++47
system=`uname -s | tr '[:upper:]' '[:lower:]'`
arch=`uname -m | tr '[:upper:]' '[:lower:]'`
svnserver=https://svn.tokutek.com/tokudb
jemalloc=jemalloc-3.3.0
xz=xz-4.999.9beta
makejobs=$(get_ncpus)
debugbuild=0
staticft=1
tokudb_patches=1
build_type=RelWithDebInfo
build_tarball=1
rpm=0
while [ $# -gt 0 ] ; do
arg=$1; shift
if [ $arg = "--gcc44" ] ; then
cc=gcc44; cxx=g++44
elif [[ $arg =~ --(.*)=(.*) ]] ; then
eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
else
usage; exit 1;
fi
done
if [ $debugbuild != 0 ] ; then
if [ $build_type = RelWithDebInfo ] ; then build_type=Debug; fi
if [ $suffix = "." ] ; then suffix=debug; fi
fi
if [ $link_enterprise_hot_backup != 0 ] ; then
if [ $suffix = "." ] ; then suffix="e"; else suffix="$suffix-e"; fi
fi
branchrevision=$(get_branchrevision $branch $revision $suffix)
builddir=build-$tokudb-$branchrevision
if [ ! -d $builddir ] ; then mkdir $builddir; fi
pushd $builddir
basedir=$PWD
# build the jemalloc library
build_jemalloc $jemalloc $branch $revision $branchrevision
# build the fractal tree tarball
tokufractaltree=tokufractaltreeindex-$branchrevision
tokuportability=tokuportability-$branchrevision
tokufractaltreedir=$tokufractaltree-$system-$arch
build_fractal_tree
# build the mysql source tarball
mysqlversion=$(mysql_version $mysql)
mysqldir=$mysql-tokudb-$branchrevision-$system-$arch
mysqlsrc=$mysql-tokudb-$branchrevision
build_mysql_src
# build the mysql release tarball
mysqlbuild=$mysqldir-build
build_mysql_release
# copy to s3
if [ $do_s3 != 0 ] ; then
files=$(ls $tokufractaltreedir.tar.gz* $mysqldir.tar.gz* $mysqlsrc.tar.gz* *.rpm*)
for f in $files; do
# add the file to the tokutek-mysql-build bucket
echo `date` s3put $f
s3put tokutek-mysql-build $f $f
exitcode=$?
# index the file by date
echo `date` s3put tokutek-mysql-build $f $exitcode
d=$(date +%Y%m%d)
s3put tokutek-mysql-build-date $d/$f /dev/null
exitcode=$?
echo `date` s3put tokutek-mysql-build-date $d/$f $exitcode
done
if [ $branch != '.' ] ; then
branch=$(basename $branch)
s3mkbucket tokutek-mysql-$branch
if [ $? = 0 ] ; then
files=$(ls $tokufractaltreedir.tar.gz* $mysqldir.tar.gz* $mysqlsrc.tar.gz* *.rpm*)
for f in $files; do
echo `date` s3copykey $branch $f
s3copykey tokutek-mysql-$branch $f tokutek-mysql-build $f
exitcode=$?
echo `date` s3copykey $branch $f $exitcode
done
fi
fi
fi
popd
exit 0
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment