Skip to content

M
mariadb
Commit c7b79a59 authored by Rich Prohaska's avatar Rich Prohaska
Browse files
Options

build mysql from github

parent 1d80daf1
Show whitespace changes
Inline Side-by-side
Showing with 228 additions and 232 deletions
scripts/make.mysql.bash
View file @ c7b79a59
......@@ -4,12 +4,12 @@ shopt -s compat31 2> /dev/null
function usage() {
echo "make.mysql.bash - build mysql with the fractal tree"
echo "[--branch=$branch] [--revision=$revision] [--suffix=$suffix]"
echo "[--mysql=$mysql]"
echo "[--tokudb=$tokudb] [--tokudbengine=$tokudbengine]"
echo "[--jemalloc=$jemalloc]"
echo "[--cc=$cc --cxx=$cxx] [--ftcc=$ftcc --ftcxx=$ftcxx] [--debugbuild=$debugbuild] [--staticft=$staticft]"
echo "[--rpm=$rpm] [--do_s3=$do_s3] [--do_make_check=$do_make_check]"
echo "--git_tag=$git_tag --git_server=$git_server"
echo "--mysql=$mysql"
echo "--build_type=$build_type --cmake_build_type=$cmake_build_type"
echo "--cc=$cc --cxx=$cxx"
echo "--build_debug=$build_debug --build_tgz=$build_tgz --build_rpm=$build_rpm"
echo "--do_s3=$do_s3 --do_make_check=$do_make_check"
return 1
}
......@@ -31,6 +31,50 @@ function retry() {
test $exitcode = 0
}
function github_download() {
repo=$1; shift
rev=$1; shift
dest=$1; shift
mkdir $dest
if [ ! -z $github_token ] ; then
retry curl \
--header "Authorization:\\ token\\ $github_token" \
--location https://api.github.com/repos/$repo/tarball/$rev \
--output $dest.tar.gz
tar --extract \
--gzip \
--directory=$dest \
--strip-components=1 \
--file $dest.tar.gz
rm -f $dest.tar.gz
elif [ ! -z $github_user ] ; then
retry curl \
--user $github_user \
--location https://api.github.com/repos/$repo/tarball/$rev \
--output $dest.tar.gz
tar --extract \
--gzip \
--directory=$dest \
--strip-components=1 \
--file $dest.tar.gz
rm -f $dest.tar.gz
else
tempdir=$(mktemp -d -p $PWD)
retry git clone git@github.com:${repo}.git $tempdir
# export the right branch or tag
(cd $tempdir ;
git archive \
--format=tar \
$rev) | \
tar --extract \
--directory $dest
rm -rf $tempdir
fi
}
function get_ncpus() {
if [ -f /proc/cpuinfo ]; then
grep bogomips /proc/cpuinfo | wc -l
......@@ -41,99 +85,58 @@ function get_ncpus() {
fi
}
function get_branchrevision() {
local branch=$1; local revision=$2; local suffix=$3
local branchrevision
if [ $branch != "." ] ; then
branchrevision=$(basename $branch)-$revision
else
branchrevision=$revision
fi
if [ "$suffix" != "." ] ; then
branchrevision=$branchrevision-$suffix
fi
echo $branchrevision
}
function mysql_version() {
local mysql=$1
if [[ $mysql =~ (mysql|mysqlcom|mariadb)-(.*) ]] ; then
echo ${BASH_REMATCH[2]}
fi
}
function build_jemalloc() {
local jemalloc=$1; local branch=$2; local revision=$3; local branchrevision=$4
if [ ! -d $jemalloc ] ; then
retry svn export -q -r $revision $svnserver/$branch/$jemalloc $jemalloc
if [ ! -d jemalloc ] ; then
# get the jemalloc repo jemalloc
github_download Tokutek/jemalloc $git_tag jemalloc
if [ $? != 0 ] ; then exit 1; fi
pushd jemalloc
if [ $? != 0 ] ; then exit 1; fi
pushd $jemalloc
if [ $? = 0 ] ; then
CC=$cc ./configure --with-private-namespace=jemalloc_
if [ $? != 0 ] ; then exit 1; fi
make
if [ $? != 0 ] ; then exit 1; fi
popd
fi
fi
}
# check out the fractal tree source from subversion, build it, and make the fractal tree tarballs
function build_fractal_tree() {
if [ ! -d $tokufractaltreedir ] ; then
mkdir $tokufractaltreedir
if [ $branch = "." ] ; then tokudb_branch=toku; else tokudb_branch=$branch; fi
retry svn export -q -r $revision $svnserver/$tokudb_branch/$tokudb
exitcode=$?; if [ $exitcode != 0 ] ; then exit 1; fi
if [ ! -d ft-index ] ; then
if [[ $ftcc =~ icc ]] ; then
if [ -d /opt/intel/bin ] ; then
export PATH=$PATH:/opt/intel/bin
. /opt/intel/bin/compilervars.sh intel64
fi
fi
# get the ft-index repo
github_download Tokutek/ft-index $git_tag ft-index
if [ $? != 0 ] ; then exit 1; fi
retry svn export -q -r $revision $svnserver/$branch/$xz
exitcode=$?; if [ $exitcode != 0 ] ; then exit 1; fi
pushd ft-index
if [ $? != 0 ] ; then exit 1; fi
pushd $tokudb
echo `date` make $tokudb $cc $($cc --version)
echo `date` make $tokudb $ftcc $($ftcc --version)
local intel_cc=""
if [[ $ftcc = "icc" ]]; then
intel_cc="ON"
cmake_env=""
else
intel_cc="OFF"
cmake_env="CC=$ftcc CXX=$ftcxx"
fi
local ft_build_type=""
local use_valgrind=""
local debug_paranoid=""
if [[ $debugbuild = 1 ]]; then
if [[ $build_debug = 1 ]]; then
ft_build_type=Debug
use_valgrind="ON"
debug_paranoid="ON"
else
ft_build_type=Release
use_valgrind="OFF"
debug_paranoid="OFF"
fi
mkdir -p build
cd build
# only use FT Release or Debug build types
local ft_build_type=$build_type
if [ $ft_build_type = RelWithDebInfo ] ; then ft_build_type=Release; fi
pushd build
if [ $? != 0 ] ; then exit 1; fi
eval $cmake_env cmake \
CC=$cc CXX=$cxx cmake \
-D LIBTOKUDB=$tokufractaltree \
-D LIBTOKUPORTABILITY=$tokuportability \
-D CMAKE_TOKUDB_REVISION=$revision \
-D CMAKE_TOKUDB_REVISION=0 \
-D CMAKE_BUILD_TYPE=$ft_build_type \
-D TOKU_SVNROOT=$basedir \
-D JEMALLOC_SOURCE_DIR=../../$jemalloc \
-D JEMALLOC_SOURCE_DIR=../../jemalloc \
-D CMAKE_INSTALL_PREFIX=../../$tokufractaltreedir \
-D BUILD_TESTING=OFF \
-D USE_GTAGS=OFF \
......@@ -144,19 +147,24 @@ function build_fractal_tree() {
-D TOKU_DEBUG_PARANOID=$debug_paranoid \
..
make install -j$makejobs
exitcode=$?; if [ $exitcode != 0 ] ; then exit 1; fi
popd
if [ $? != 0 ] ; then exit 1; fi
popd # build
popd # ft-index
pushd $tokufractaltreedir/examples
if [ $? != 0 ] ; then exit 1; fi
# test the examples
sed -ie "s/LIBTOKUDB = tokudb/LIBTOKUDB = $tokufractaltree/" Makefile
sed -ie "s/LIBTOKUPORTABILITY = tokuportability/LIBTOKUPORTABILITY = $tokuportability/" Makefile
if [ $system = darwin ] ; then
DYLD_LIBRARY_PATH=$PWD/../lib:$DYLD_LIBRARY_PATH make check CC=$ftcc
DYLD_LIBRARY_PATH=$PWD/../lib:$DYLD_LIBRARY_PATH make check CC=$cc
exitcode=$?
else
if [ $do_make_check != 0 ] ; then
make check CC=$ftcc
make check CC=$cc
exitcode=$?
else
exitcode=0
......@@ -168,12 +176,12 @@ function build_fractal_tree() {
popd
# add jemalloc.so
if [ $system = darwin ] ; then
cp $jemalloc/lib/libjemalloc.1.dylib $tokufractaltreedir/lib/lib$jemalloc-$branchrevision.dylib
else
cp $jemalloc/lib/libjemalloc.so.1 $tokufractaltreedir/lib/lib$jemalloc-$branchrevision.so
if [ $system = linux ] ; then
cp jemalloc/lib/libjemalloc.so.1 $tokufractaltreedir/lib
elif [ $system = darwin ] ; then
cp jemalloc/lib/libjemalloc.1.dylib $tokufractaltreedir/lib
fi
cp $jemalloc/lib/libjemalloc.a $tokufractaltreedir/lib/libjemalloc.a
cp jemalloc/lib/libjemalloc.a $tokufractaltreedir/lib/libjemalloc.a
# make tarballs
tar czf $tokufractaltreedir.tar.gz $tokufractaltreedir
......@@ -187,7 +195,7 @@ function generate_cmake_cmd() {
local extra_flags=$*
if [[ $mysqlsrc =~ ^Percona ]] ; then extra_flags=-DWITH_EMBEDDED_SERVER=OFF; fi
echo -n cmake .. $extra_flags -DBUILD_CONFIG=mysql_release -DCMAKE_BUILD_TYPE=$build_type
echo -n cmake .. $extra_flags -DBUILD_CONFIG=mysql_release -DCMAKE_BUILD_TYPE=$cmake_build_type
if [ $system = linux ] ; then
echo -n " " \
-DCMAKE_EXE_LINKER_FLAGS=\"-Wl,--whole-archive \$TOKUFRACTALTREE/lib/libjemalloc.a -Wl,-no-whole-archive\"
......@@ -198,7 +206,7 @@ function generate_cmake_cmd() {
else
exit 1
fi
if [ $link_enterprise_hot_backup != 0 ] ; then
if [ $build_type = enterprise ] ; then
echo -n " " -DCOMPILATION_COMMENT=\"TokuDB Enterprise Server \(GPL\)\"
fi
echo
......@@ -206,7 +214,7 @@ function generate_cmake_cmd() {
# generate a script that builds from the mysql src tarball and the fractal tree tarball
function generate_build_from_src() {
local tokumysqldir=$1; local mysqlsrc=$2; local tokufractaltreedir=$3
local mysqlsrc=$1; local tokufractaltreedir=$2
pushd $mysqlsrc >/dev/null 2>&1
if [ $? = 0 ] ; then
echo '#/usr/bin/env bash'
......@@ -242,16 +250,11 @@ function generate_build_from_src() {
echo fi
# setup environment variables
echo export TOKUDB_VERSION=$branchrevision
echo export TOKUDB_VERSION=$tokudb_version
echo export TOKUFRACTALTREE='$PWD'/$tokufractaltreedir
echo export TOKUDB_PATCHES=$tokudb_patches
if [ $staticft != 0 ] ; then
echo export TOKUFRACTALTREE_LIBNAME=${tokufractaltree}_static
echo export TOKUPORTABILITY_LIBNAME=${tokuportability}_static
else
echo export TOKUFRACTALTREE_LIBNAME=${tokufractaltree}
echo export TOKUPORTABILITY_LIBNAME=${tokuportability}
fi
echo 'export CC=$cc; export CXX=$cxx'
echo "pushd $mysqlsrc"
......@@ -259,8 +262,8 @@ function generate_build_from_src() {
if [ ! -f CMakeLists.txt ] ; then exit 1; fi
if [ $build_tarball != 0 ] ; then
echo "mkdir build.$build_type; pushd build.$build_type"
if [ $build_tgz != 0 ] ; then
echo "mkdir build.$cmake_build_type; pushd build.$cmake_build_type"
echo 'if [ $? != 0 ] ; then exit 1; fi'
cmd=$(generate_cmake_cmd $mysqlsrc)
......@@ -285,24 +288,9 @@ function generate_build_from_src() {
# add ft files to the mysql tarball
echo tar xzf $mysqldir.tar.gz
echo 'if [ $? != 0 ] ; then exit 1; fi'
if [ $staticft = 0 ] ; then
# add fractal tree libs to the mysql tarball only if we are linking to shared libs
echo "if [ -d $mysqldir/lib/mysql ] ; then"
echo " cp ../../$tokufractaltreedir/lib/lib* $mysqldir/lib/mysql"
echo else
echo " cp ../../$tokufractaltreedir/lib/lib* $mysqldir/lib"
echo fi
echo 'if [ $? != 0 ] ; then exit 1; fi'
fi
# add ftdump to mysql tarball
echo "cp ../../$tokufractaltreedir/bin/ftdump $mysqldir/bin/tokuftdump"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# add tokustat to the mysql tarball
echo "cp ../../$mysqlsrc/scripts/tokustat.py $mysqldir/bin"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# add tokufilecheck to the mysql tarball
echo "cp ../../$mysqlsrc/scripts/tokufilecheck.py $mysqldir/bin"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# cleanup
echo rm $mysqldir.tar.gz
......@@ -320,8 +308,8 @@ function generate_build_from_src() {
fi
# make RPMs
if [ $system = linux -a $rpm != 0 ]; then
echo "mkdir build.${build_type}.rpms; pushd build.${build_type}.rpms"
if [ $system = linux -a $build_rpm != 0 ]; then
echo "mkdir build.${cmake_build_type}.rpms; pushd build.${cmake_build_type}.rpms"
echo 'if [ $? != 0 ] ; then exit 1; fi'
if [[ $mysqlsrc =~ ^mariadb ]] ; then
......@@ -382,6 +370,11 @@ function generate_build_from_src() {
echo 'mv *.rpm ../..'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'popd'
echo 'pushd SRPMS'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'mv *.rpm ..'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'popd'
fi
# generate the md5 file
......@@ -400,63 +393,60 @@ function generate_build_from_src() {
fi
}
# checkout the mysql source from subversion, generate a build script, and make the mysql source tarball
# checkout the mysql source, generate a build script, and make the mysql source tarball
function build_mysql_src() {
if [ ! -d $mysqlsrc ] ; then
# get the mysql source
if [ $branch = "." ] ; then mysql_branch=mysql.com; else mysql_branch=$branch; fi
# get the mysql repo
if [ ! -d $mysql_repo ] ; then
github_download Tokutek/$mysql_repo $git_tag $mysqlsrc
if [ $? != 0 ] ; then exit 1; fi
fi
retry svn export -q -r $revision $svnserver/$mysql_branch/$mysql $mysqlsrc
# get the hot backup source
if [ ! -d backup-$build_type ] ; then
github_download Tokutek/backup-$build_type $git_tag backup-$build_type
if [ $? != 0 ] ; then exit 1; fi
fi
# get the tokudb handlerton source
if [ $branch = "." ] ; then tokumysql_branch=mysql; else tokumysql_branch=$branch/mysql; fi
pushd $mysqlsrc/storage
retry svn export -q -r $revision $svnserver/$tokumysql_branch/tokudb-engine/$tokudbengine tokudb
# get the ft-engine repo
if [ ! -d ft-engine ] ; then
github_download Tokutek/ft-engine $git_tag ft-engine
if [ $? != 0 ] ; then exit 1; fi
popd
pushd $mysqlsrc/scripts
retry svn export -q -r $revision $svnserver/$tokumysql_branch/scripts/tokustat.py tokustat.py
fi
# append the tokudb version to the MYSQL_VERSION_EXTRA variable in the VERSION file
sed -i "" -e"s/^MYSQL_VERSION_EXTRA=.*/&-tokudb-${tokudb_version}/" $mysqlsrc/VERSION
# add the backup source
mkdir $mysqlsrc/toku_backup
pushd backup-$build_type/backup
if [ $? != 0 ] ; then exit 1; fi
popd
pushd $mysqlsrc/scripts
retry svn export -q -r $revision $svnserver/$tokumysql_branch/scripts/tokufilecheck.py tokufilecheck.py
copy_files=$(ls CMakeLists.txt *.h *.cc *.cmake export.map)
cp $copy_files ../../$mysqlsrc/toku_backup
if [ $? != 0 ] ; then exit 1; fi
popd
# append the tokudb version to the MYSQL_VERSION_EXTRA variable in the VERSION file
pushd $mysqlsrc
if [ $? = 0 ] ; then
sed -i "" -e"s/^MYSQL_VERSION_EXTRA=.*/&-tokudb-${branchrevision}/" VERSION
# add the tokudb storage engine source
mkdir -p $mysqlsrc/storage/tokudb
pushd ft-engine/storage/tokudb
if [ $? != 0 ] ; then exit 1; fi
copy_files=$(ls CMakeLists.txt *.h *.cc)
cp $copy_files ../../../$mysqlsrc/storage/tokudb
if [ $? != 0 ] ; then exit 1; fi
popd
fi
# merge the common mysql tests into the source
pushd $mysqlsrc/mysql-test
if [ $? = 0 ] ; then
mkdir ../mysql-test-save
mv * ../mysql-test-save
retry svn export -q -r $revision $svnserver/$tokumysql_branch/tests/mysql-test
mv mysql-test/* .
rmdir mysql-test
cp -r ../mysql-test-save/* .
rm -rf ../mysql-test-save
popd
fi
mv $mysqlsrc/mysql-test $mysqlsrc/mysql-test-save
cp -r ft-engine/mysql-test $mysqlsrc
cp -r $mysqlsrc/mysql-test-save/* $mysqlsrc/mysql-test
rm -rf $mysqlsrc/mysql-test-save
# get the hot backup source
pushd $mysqlsrc
if [ $? = 0 ] ; then
if [ $link_enterprise_hot_backup != 0 ] ; then
retry svn export -q -r $revision $svnserver/backup-restore/backup/ toku_backup
else
retry svn export -q -r $revision $svnserver/backup-restore/backup-community/ toku_backup
fi
fi
popd
# add the common scripts into the source
cp -r ft-engine/scripts $mysqlsrc
generate_build_from_src $tokumysql_branch $mysqlsrc $tokufractaltreedir >$mysqlsrc/scripts/tokudb.build.bash
# generate the tokudb.build.bash script from the mysql src and the fractal tree tarballs
generate_build_from_src $mysqlsrc $tokufractaltreedir >$mysqlsrc/scripts/tokudb.build.bash
# make the mysql src tarball
tar czf $mysqlsrc.tar.gz $mysqlsrc
......@@ -492,13 +482,13 @@ function build_mysql_release() {
if [ $? != 0 ] ; then exit 1; fi
# move the build files
pushd $mysqlsrc/build.${build_type}
pushd $mysqlsrc/build.${cmake_build_type}
if [ $? = 0 ] ; then
cp $mysqldir.tar.gz* $basedir; if [ $? != 0 ] ; then exit 1; fi
popd
fi
if [ $rpm != 0 ] ; then
pushd $mysqlsrc/build.${build_type}.rpms
if [ $build_rpm != 0 ] ; then
pushd $mysqlsrc/build.${cmake_build_type}.rpms
if [ $? = 0 ] ; then
rpmfiles=$(ls *.rpm *.rpm.md5)
for x in $rpmfiles; do cp $x $basedir; if [ $? != 0 ] ; then exit 1; fi; done
......@@ -512,31 +502,24 @@ function build_mysql_release() {
PATH=$HOME/bin:$PATH
branch=.
revision=0
suffix=.
mysql=mysql-5.5.28
git_server=git@github.com
git_tag=
mysql=mysql-5.5.30
tokudb=tokudb
tokudbengine=tokudb-engine
link_enterprise_hot_backup=0
do_s3=1
do_make_check=1
cc=gcc47
cxx=g++47
ftcc=gcc47
ftcxx=g++47
system=`uname -s | tr '[:upper:]' '[:lower:]'`
arch=`uname -m | tr '[:upper:]' '[:lower:]'`
svnserver=https://svn.tokutek.com/tokudb
jemalloc=jemalloc-3.3.0
xz=xz-4.999.9beta
makejobs=$(get_ncpus)
debugbuild=0
staticft=1
build_debug=0
build_type=community
tokudb_patches=1
build_type=RelWithDebInfo
build_tarball=1
rpm=0
cmake_build_type=RelWithDebInfo
build_tgz=1
build_rpm=0
while [ $# -gt 0 ] ; do
arg=$1; shift
......@@ -549,35 +532,49 @@ while [ $# -gt 0 ] ; do
fi
done
if [ $debugbuild != 0 ] ; then
if [ $build_type = RelWithDebInfo ] ; then build_type=Debug; fi
if [ $suffix = "." ] ; then suffix=debug; fi
if [ -z $git_tag ] ; then exit 1; fi
# set tokudb version
if [[ $git_tag =~ tokudb-(.*) ]] ; then
tokudb_version=${BASH_REMATCH[1]}
else
tokudb_version=$git_tag
fi
if [ $build_debug != 0 ] ; then
if [ $cmake_build_type = RelWithDebInfo ] ; then cmake_build_type=Debug; fi
tokudb_version=$tokudb_version-debug
fi
if [ $build_type == enterprise ] ; then
tokudb_version=$tokudb_version-e
fi
if [ $link_enterprise_hot_backup != 0 ] ; then
if [ $suffix = "." ] ; then suffix="e"; else suffix="$suffix-e"; fi
# split mysql into mysql_repo and mysql_version
if [[ $mysql =~ ^(mysql|mariadb)-(.*)$ ]] ; then
mysql_repo=${BASH_REMATCH[1]}
mysql_version=${BASH_REMATCH[2]}
else
exit 1
fi
branchrevision=$(get_branchrevision $branch $revision $suffix)
builddir=build-$tokudb-$branchrevision
# set build dir
builddir=build-$tokudb-$tokudb_version
if [ ! -d $builddir ] ; then mkdir $builddir; fi
pushd $builddir
basedir=$PWD
# build the jemalloc library
build_jemalloc $jemalloc $branch $revision $branchrevision
build_jemalloc
# build the fractal tree tarball
tokufractaltree=tokufractaltreeindex-$branchrevision
tokuportability=tokuportability-$branchrevision
tokufractaltree=tokufractaltreeindex-$tokudb_version
tokuportability=tokuportability-$tokudb_version
tokufractaltreedir=$tokufractaltree-$system-$arch
build_fractal_tree
# build the mysql source tarball
mysqlversion=$(mysql_version $mysql)
mysqldir=$mysql-tokudb-$branchrevision-$system-$arch
mysqlsrc=$mysql-tokudb-$branchrevision
mysqldir=$mysql-tokudb-$tokudb_version-$system-$arch
mysqlsrc=$mysql-tokudb-$tokudb_version
build_mysql_src
# build the mysql release tarball
......@@ -599,16 +596,15 @@ if [ $do_s3 != 0 ] ; then
exitcode=$?
echo `date` s3put tokutek-mysql-build-date $d/$f $exitcode
done
if [ $branch != '.' ] ; then
branch=$(basename $branch)
s3mkbucket tokutek-mysql-$branch
if [ ! -z $git_tag ] ; then
s3mkbucket tokutek-mysql-$git_tag
if [ $? = 0 ] ; then
files=$(ls $tokufractaltreedir.tar.gz* $mysqldir.tar.gz* $mysqlsrc.tar.gz* *.rpm*)
for f in $files; do
echo `date` s3copykey $branch $f
s3copykey tokutek-mysql-$branch $f tokutek-mysql-build $f
echo `date` s3copykey $git_tag $f
s3copykey tokutek-mysql-$git_tag $f tokutek-mysql-build $f
exitcode=$?
echo `date` s3copykey $branch $f $exitcode
echo `date` s3copykey $git_tag $f $exitcode
done
fi
fi
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7