buildout.cfg 13 KB
Newer Older
Boxiang Sun's avatar
Boxiang Sun committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
[buildout]
# slaposcookbook_recipe_location = ${:parts-directory}/slaposcookbook

extends =
  ../curl/buildout.cfg
  ../git/buildout.cfg
  ../python3/buildout.cfg
  ../ccache/buildout.cfg
  ../nodejs/buildout.cfg
  ../wget/buildout.cfg
  ../tar/buildout.cfg
  ../libyaml/buildout.cfg
  ../python-PyYAML/buildout.cfg
  ../cmake/buildout.cfg
  ../bzip2/buildout.cfg
  ../file/buildout.cfg
  ../gcc/buildout.cfg
  ../gdbm/buildout.cfg
  ../gettext/buildout.cfg
  ../libexpat/buildout.cfg
  ../libffi/buildout.cfg
  ../libpng/buildout.cfg
  ../freetype/buildout.cfg
  ../pkgconfig/buildout.cfg
  ../ncurses/buildout.cfg
  ../openssl/buildout.cfg
  ../patch/buildout.cfg
  ../readline/buildout.cfg
  ../sqlite3/buildout.cfg
  ../xz-utils/buildout.cfg
  ../zlib/buildout.cfg
  ../f2c/buildout.cfg

parts =
  pyodide
  pyodide-script-test
  nodejs
  npm
  python3.7-PyYAML

[node-less]
recipe = slapos.recipe.build:npm
packages = less
node = nodejs
environment =
  PATH=${nodejs:location}/bin:%(PATH)s

[node-uglify-js]
recipe = slapos.recipe.build:npm
packages = uglify-js
node = nodejs
environment =
  PATH=${nodejs:location}/bin:%(PATH)s

[pyodide-src]
recipe = slapos.recipe.build:gitclone

git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/Daetalus/pyodide.git
location = ${buildout:parts-directory}/${:_buildout_section_name_}
branch = master

63 64 65 66 67 68 69 70
[get-pip]
recipe = slapos.recipe.build:download
# version 19.2.3
url = https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/${:filename}
md5sum = 6f489c199cb5be8a4b84c6bd7ad6e051
filename = get-pip.py
mode = 0755

Boxiang Sun's avatar
Boxiang Sun committed
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
[pyodide-script]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
build_dir =  ${buildout:parts-directory}/pyodide-script-test
part_dir = ${buildout:parts-directory}
git-executable = ${git:location}/bin/git
emsdk_set_env_script = ${:build_dir}/emsdk/emsdk/emsdk_set_env.sh
repository = https://lab.nexedi.com/Daetalus/pyodide.git
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig
PATH=${curl:location}/bin:${git:location}/bin:${pkgconfig:location}/bin:${freetype:location}/bin:${nodejs:location}/bin/:${f2c:location}:${python3.7:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:${xz-utils:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${libffi:location}/include -I${gettext:location}/include
LD_LIBRARY_PATH=${zlib:location}/lib:${libpng:location}/lib:${openssl:location}/lib:${readline:location}/lib:${libexpat:location}/lib:${xz-utils:location}/lib:${sqlite3:location}/lib:${gdbm:location}/lib:${bzip2:location}/lib:${ncurses:location}/lib:${libffi:location}/lib
LDFLAGS= -L${libffi:location}/lib -L${zlib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
CPATH=${f2c:location}/include:${zlib:location}/include:${xz-utils:location}/include:${readline:location}/include:${libexpat:location}/include:${libffi:location}/include:${ncurses:location}/include:${ncurses:location}/include/ncursesw:${bzip2:location}/include:${gdbm:location}/include:${openssl:location}/include:${sqlite3:location}/include:${gettext:location}/include
openssl_location = ${openssl:location}
gcc_bin_dir = ${gcc-8.2:location}/bin
python_bin_dir = ${python3.7:location}/bin
md5sum = 47ec6a091e503349d948760dc03bed51
89
pip_script = ${get-pip:target}
Boxiang Sun's avatar
Boxiang Sun committed
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
script =
  import subprocess, os, shutil

  env = {
    'PKG_CONFIG_PATH':self.options['PKG_CONFIG_PATH'],
    'PATH':self.options['PATH'] + ':' + os.environ['PATH'],
    'CPATH':self.options['CPATH'],
    'CPPFLAGS':self.options['CPPFLAGS'],
    'LDFLAGS':self.options['LDFLAGS'],
  }

  if not os.path.isdir(self.options['build_dir']):
    command_list_0 = [{'cmd': self.options['git-executable'] + ' clone ' + self.options['repository'] + ' pyodide-script-test', 'cwd': self.options['part_dir']}]
    # put the Popen in the loop to let the pipe.wait() work.
    for command_0 in command_list_0:
      pipe = subprocess.Popen(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
      pipe.wait()
  # else:
    # shutil.rmtree(self.options['build_dir'], ignore_errors=True)

  cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + self.options['openssl_location'] + ' --prefix@" ' + self.options['build_dir'] + '/cpython/Makefile'
  pip_script = self.options['pip_script']
  install_pip = "python3 " + pip_script
  install_pytest = "pip install pytest selenium pytest-instafail pytest-timeout"
  # Hack: create a gfortran symlink inside the python bin dir.
  # We have to build the pyodide cpython along with SlapOS host gcc, not the custom gcc in the parts dir.
  # Because if we use custom gcc to build the pyodide cpython, later we build pyodide python extension, the distutil going to
  # expand the gcc which used by the pyodide cpython to full path instead plain `gcc`.
  # Pyodide did some hacks to make distutil support crossing compile, which require use the plain `gcc` when building python extension.
  # So we are not going to provide the custom gcc bin path in the PATH environment variable.
  # But scipy needs gfortran. The workaround is just give a symlink in the cpython bin dir, which points to the custom gcc gfortran.
  # Note: the parts/cpython is for run the pyodide building script. The pyodide/cpython is for compile to webassembly module which will loads to the browser.
  create_gfortran_symlink = 'ln -s ' + self.options['gcc_bin_dir'] + '/gfortran ' + self.options['python_bin_dir'] + '/gfortran'

  command_list = [
    {'cmd': cpython_makefile_command, 'cwd': self.options['build_dir']},
    {'cmd': install_pip, 'cwd': self.options['build_dir']},
    {'cmd': install_pytest, 'cwd': self.options['build_dir']},
    {'cmd': create_gfortran_symlink, 'cwd': self.options['build_dir']},
    {'cmd': 'make -C emsdk', 'cwd': self.options['build_dir']},
    {'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': self.options['build_dir']},
    {'cmd': 'make -C lz4', 'cwd': self.options['build_dir']},
    {'cmd': 'make -C cpython', 'cwd': self.options['build_dir']},
    {'cmd': 'make -C CLAPACK', 'cwd': self.options['build_dir']},
    {'cmd': 'make -C packages', 'cwd': self.options['build_dir']},
  ]

  for command in command_list:
    pipe = subprocess.Popen(command['cmd'], cwd=command['cwd'], env=env, shell=True)
    pipe.wait()
  
  # We need to execute the command "./emsdk/emsdk/emsdk_env.sh" to add the emsdk related environment variables.
  # Run "emsdk_env.sh" first, then read the content of "emsdk_set_env.sh" to get the environment variables
  emsdk_env_content = open(self.options['emsdk_set_env_script'])
  for content in emsdk_env_content:
    # remove "export " prefix
    content = content[7:]
    key, value = content.split('=')
    value = value.rstrip().strip('"')
    try:
      env[key] += ':' + value
    except KeyError:
      env[key] = value
  env['CPATH'] += env['EMSCRIPTEN'] + '/system/include/'

  command_list_2 = [
    {'cmd': 'make build/pyodide.asm.js', 'cwd': self.options['build_dir']},
    {'cmd': 'make -e', 'cwd': self.options['build_dir']},
  ]
  
  for command_2 in command_list_2:
    pipe = subprocess.Popen(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
    pipe.wait()

[python3.7-PyYAML]
recipe = slapos.recipe.cmmi
python_bin = ${python3.7:location}/bin/python3.7
url = https://pyyaml.org/download/pyyaml/PyYAML-3.13.tar.gz
configure-command = true
make-binary = ${:python_bin} setup.py install
environment =
  C_INCLUDE_PATH=${libyaml:location}/include
location = ${buildout:parts-directory}/${:_buildout_section_name_}

[pyodide-emsdk]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}/emsdk
configure-command= alias python=${python3.7:location}/bin/python3 && make clean
make-options = -e
environment =
  PATH=${f2c:location}/bin:${python3.7:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${gcc-8.2:location}/bin:%(PATH)s
  CC=${gcc-8.2:location}/bin/gcc
  CXX=${gcc-8.2:location}/bin/g++
  LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
  PYTHON_EXECUTABLE=${python3.7:location}/bin/python3.7
make-targets =

# Build pyodide cpython with custom configuration. Otherwise we will get a cpython without ssl support
# The pyodide cpython **must** been build **AFTER** emsdk
[pyodide-cpython]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}/cpython
configure-command = sed -ie "s@./configure --prefix@./configure --with-openssl=${openssl:location} --prefix@" ${pyodide-src:location}/cpython/Makefile && source ${pyodide-emsdk:path}/emsdk/emsdk_env.sh
make-options = -e
environment =
  PATH=${xz-utils:location}/bin:${gcc-8.2:location}/bin:%(PATH)s
  CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
  LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
  CC=${gcc-8.2:location}/bin/gcc
  CXX=${gcc-8.2:location}/bin/g++
  LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
make-targets =

[pyodide-packages]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}
configure-command = alias python3=${pyodide-cpython:path}/build/3.7.0/host/bin/python3 && source ${pyodide-emsdk:path}/emsdk/emsdk_env.sh
make-options = -e
make-targets =
environment =
  PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:$PKG_CONFIG_PATH
  PATH=${gcc-8.2:location}/bin:${f2c:location}/bin:${pkgconfig:location}/bin:${pyodide-cpython:path}/build/3.7.0/host/bin:${nodejs:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:%(PATH)s
  LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64

[pyodide]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}
configure-command = make -e
make-binary =
make-targets = cp -r ./* ${buildout:parts-directory}/${:_buildout_section_name_} 
# configure-command = echo ${python3.7-PyYAML:location} && echo ${pyodide-cpython:path} && echo ${pyodide-emsdk:path} && echo ${node-uglify-js:location} && echo ${node-less:location}# ${pyodide-packages:path}
configure-command = echo ${python3.7-PyYAML:location} && echo ${pyodide-script:location}
environment =
  PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:$PKG_CONFIG_PATH
  PATH=${pkgconfig:location}/bin:${freetype:location}/bin:${nodejs:location}/bin/:${f2c:location}/bin:${python3.7:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${gcc-8.2:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:${xz-utils:location}/bin:%(PATH)s
  CC=${gcc-8.2:location}/bin/gcc
  CXX=${gcc-8.2:location}/bin/g++
  CPPFLAGS=-I${f2c:location}/include -I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
  LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64:${openssl:location}/lib:${readline:location}/lib:${libexpat:location}/lib:${xz-utils:location}/lib:${sqlite3:location}/lib:${gdbm:location}/lib:${bzip2:location}/lib:${ncurses:location}/lib:${libffi:location}/lib
  LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
  PYTHON_EXECUTABLE=${python3.7:location}/bin/python3.7