Commit c8c81baa authored by Kirill Smelkov's avatar Kirill Smelkov

slaprunner: Build NumPy/friends & Ruby gems in parallel

Slaprunner tries to leverage multicore, and spawns multiple jobs when compiling software, based on `cpu-usage-ratio` parameter. But this currently have effect only on Make-based projects (via setting `MAKEFLAGS=-j<n>`) and does not affect software with different build systems.

Let's also provide support for parallel building for NumPy-based software and Ruby gems out of the box.

/cc @cedric.leninivin, @kazuhiko, @alain.takoudjou
/reviewed-by @jerome, @rafael  (on !22)
parents 2a3644dd fc40878c
master 1.0 ZODB3 alain-wrapper-recipe apache-lower-reload apache-promise armin arnau arnau-RD-future arnau-RD-py3-master arnau-zope4py2 aurelien-test-wendelin backup bbu bug/20170414-1800C9F cert-auto check_system_gfortran cherry-pick-bbfbf262 cleanup/erp5_drop_erp5-util-develop cloudooo_test_with_client_certificate cmf_upgrade_versions cythonplus debug-resiliency default_software_type_default django-tutorial draft_cloudooo_py3 drop_mt_index_in_BTreeFolder2 drupal-lamp e2e e2e-ors erp5 erp5-ca erp5-component erp5-frontend-request-fix erp5-request erp5-zope2-tests erp5_kernel_jupyter erp5testnode-max_timeout erp5testnode-userhosts exportsh extend-erp5-wcfs-integration-tests failed-request-log-warning failover-backend feat/buildout_testing_python3_new feat/debian11 feat/erp5-incremenatal-repozo feat/fix_kvm_wipe feat/gitea feat/inkscape-1.3 feat/ipv6_range feat/kvm-import-nice feat/libreoffice-24.2 feat/libreoffice-7.6 feat/mariadb-10.11 feat/mariadb-10.11-unsafe feat/mariadb-10.11bis feat/mariadb-10.11ter feat/mariadb-10.5 feat/mariadb-10.6 feat/mariadb-11.4 feat/mariadb-11.4-unsafe feat/mgroonga feat/new_vps_software feat/nxdbom feat/obs_test feat/olapy feat/openssl-3.0 feat/openssl-3.0-bis feat/openssl-3.0-py2 feat/run-tests-at-compile-time feat/seleniumrunner-firefox-60 feat/switch-software-type feat/telegraf-version-up feat/testcase feat/theia-1.24.0 feat/theia-bash-completions feat/version_up_ldap_graphviz feat/zope-testrunner feat/zope5.11-pygolang-zodbtools feature/service-recipe feature/wrapper-update firefox-version-up fix-1.0.106.1 fix-1.0.135.1 fix-1.0.135.2 fix-1.0.190.1 fix-1.0.207.1 fix-1.0.240.1 fix-1.0.56 fix-1.0.95 fix-1.0.95.1 fix-libdb fix/cloudooo-rpath fix/erp5-haproxy-wait-reload fix/erp5_remove_unused_eggs fix/gcc_min_version fix/golang-20 fix/haproxy-up fix/jinja2_version_up fix/netrc-parse-error fix/perl-dbd-mysql_link_against_system_libraries fix/re6st-token-manager fix/resilient_test fix/slapos.toolbox-py2 fix/slaprunner_missing_slapos_scripts fix/version_up fix_kvm_monitor for_testrunner_1 gitlab-fixup gitlab-test gitlab-test-8.6.1 gitlab-upgrade gitlab-v14.10 goodbye-openssl-1.0.x haproxy-config-header html5as html5as-base html5as-test html5as-tutorial improve-monitor-httpd inkscape-cmake intentially_broken_cloudooo jm json-api jupyter_renderjs_extension jupyter_wendelin keep-around/f67bd37e5e2a04b3dd0d04d67016ffbe9a73afa4 kvm-testing kvm_auto_reboot lab-resilent lle-bout/ppc64le logrotate/truncate lte-standalone mariadb-10.1.26 master+ZODB4-wc2 matomo-tutorial monitor-stack moodle my2to3 nayuos-anonimity nayuos-cleanup nayuos-cleanup-squash nayuos-devserver nayuos-kato nayuos-lorenzo nayuos-stable nbextension_jiocontents nbextension_jiocontents_standalone neo-reflink netframe nextcloud nextcloud-fix-tests nginx_with_auth nirina_tutorial nofile old-mail-server onlyoffice-core openblas-make-ext-options ors ors-amarisoft/aw2s ors-dev ors-dev2 ors-dev3 ors-dev3-bak ors-dnsmasq ors-fake-lteenb ors-handover ors-oran-ru ors-oran-ru-toolbox ors-promise ors-ptt ors-publish ors-ssb-nr-arfcn ors-test ors-tests ors-ue osc patrowl-dev php-7 proview-r public-deltachat-core push-to-wendolin py_magic_workaround python3.12 rafael-py3 remove_inkscape remove_tempstorage repman-1.0 repman-test resilience_all_parameters resiliencefix restore-from-backup revert-e7b48c0b ru-alarms runner-multi-sr scalability_crash_mariadb selenium_test shebang_limitation simpleran-dev slap-configuration slapos-master-cluster slapos-node_python3.9 slapos_master_load_balancer_test sozu-dev stack/supervisord systemd-python tdd_config_5ms_6ul_3dl telecom-matomo-hotfix test-fix-erp5-resiliency textsynth tomo_fix_erp5 tomo_fix_kvm tomo_old_slapos_to_test_testnode tomo_openradio_e2e_testing top-stand1 ttrm-novel ttrm-novel-debian10 ttrm-novel-fix-testnode ttrn upgrade_erp5_test_updated upgrade_responses vanilla-slapos webdav wipapi wrapper wsgi-gevent xy/lte-multiru y/wc2-next zope2py2-reorder-versions-priority zope2zope4py2 zope4 zope4py2-faketime zope4py3 zope4py3-bstr zope4py3-kaz 1.0.396 1.0.395 1.0.394 1.0.393 1.0.392 1.0.391 1.0.390 1.0.389 1.0.388 1.0.387 1.0.386 1.0.385 1.0.384 1.0.383 1.0.382 1.0.381 1.0.380 1.0.379 1.0.378 1.0.377 1.0.376 1.0.375 1.0.374 1.0.373 1.0.372 1.0.371 1.0.370 1.0.369 1.0.368 1.0.367 1.0.366 1.0.365 1.0.364.3 1.0.364.2 1.0.364.1 1.0.364 1.0.363 1.0.362 1.0.361 1.0.360 1.0.359 1.0.358 1.0.357 1.0.356 1.0.355 1.0.354 1.0.354-mariadb-replication-efc8bfe3c 1.0.354-mariadb-replication-8b12a75b5 1.0.353 1.0.352 1.0.351 1.0.350 1.0.349 1.0.348 1.0.347 1.0.346 1.0.345 1.0.344.1 1.0.344 1.0.343 1.0.342 1.0.341 1.0.340 1.0.339 1.0.338 1.0.337 1.0.336 1.0.335 1.0.334 1.0.333 1.0.332 1.0.331 1.0.330 1.0.329 1.0.328 1.0.327 1.0.326 1.0.325 1.0.324 1.0.323 1.0.322 1.0.321 1.0.320 1.0.319 1.0.318 1.0.317 1.0.316 1.0.315 1.0.314 1.0.313 1.0.312 1.0.311 1.0.310 1.0.309 1.0.308 1.0.307 1.0.306 1.0.305 1.0.304 1.0.303 1.0.302 1.0.301 1.0.300 1.0.299 1.0.298 1.0.297 1.0.296 1.0.295 1.0.294 1.0.293 1.0.292 1.0.291 1.0.290 1.0.289 1.0.288 1.0.287 1.0.286 1.0.285 1.0.284 1.0.283 1.0.282 1.0.281 1.0.280 1.0.279 1.0.278 1.0.277 1.0.276 1.0.275 1.0.274 1.0.273 1.0.272 1.0.271 1.0.270 1.0.269 1.0.268 1.0.267 1.0.266 1.0.265 1.0.264 1.0.263 1.0.262 1.0.261 1.0.260 1.0.259 1.0.258 1.0.257 1.0.256 1.0.255 1.0.254 1.0.253 1.0.252 1.0.251 1.0.250 1.0.249 1.0.248 1.0.247 1.0.246 1.0.245 1.0.244 1.0.243 1.0.242 1.0.241 1.0.240.1 1.0.240 1.0.239 1.0.238 1.0.237 1.0.236 1.0.235 1.0.234 1.0.233 1.0.232 1.0.231 1.0.230 1.0.229 1.0.228 1.0.227 1.0.226 1.0.225 1.0.224 1.0.223 1.0.222 1.0.221 1.0.220 1.0.219 1.0.218 1.0.217 1.0.216 1.0.215 1.0.214 1.0.213 1.0.212 1.0.211 1.0.210 1.0.209 1.0.208 1.0.207.1 1.0.207 1.0.206 1.0.205 1.0.204 1.0.203 1.0.202 1.0.201 1.0.200 1.0.199 1.0.198 1.0.197 1.0.196 1.0.195 1.0.194 1.0.193 1.0.192 1.0.191 1.0.190.1 1.0.190 1.0.189 1.0.188 1.0.187 1.0.186 1.0.185 1.0.184 1.0.183 1.0.182 1.0.181 1.0.180 1.0.179 1.0.178 1.0.177 1.0.176 1.0.175 1.0.174 1.0.173 1.0.172 1.0.171 1.0.170 1.0.169 1.0.168 1.0.167.10 1.0.167.9 1.0.167.8 1.0.167.7 1.0.167.6 1.0.167.5 1.0.167.4 1.0.167.3 1.0.167.2 1.0.167.1 1.0.167 1.0.166 1.0.165 1.0.164 1.0.163 1.0.162 1.0.161.1 1.0.161 1.0.160 1.0.159 1.0.158 1.0.157 1.0.156 1.0.155 1.0.154 1.0.153 1.0.152 1.0.151 1.0.150 1.0.149 1.0.148 1.0.147 1.0.146 1.0.145 1.0.144 1.0.143 1.0.142 1.0.141 1.0.140 1.0.137 1.0.136 1.0.135.2 1.0.135.1 1.0.135 1.0.134 1.0.133 1.0.132 1.0.131 1.0.130 1.0.129 1.0.128 1.0.127 1.0.126 1.0.125 1.0.124 1.0.123 1.0.122 1.0.121 1.0.120 1.0.119 1.0.118 1.0.117 1.0.116 1.0.115 1.0.114 1.0.113 1.0.112 1.0.111 1.0.110 1.0.109 1.0.108 1.0.107 1.0.106.1 1.0.106 1.0.105 1.0.104 1.0.103 1.0.102 1.0.101 1.0.100 1.0.99 1.0.98 1.0.97 1.0.96 1.0.95.2 1.0.95.1 1.0.95 1.0.94 1.0.93 1.0.92 1.0.91 1.0.90 1.0.89 1.0.88 1.0.87 1.0.86 1.0.85 1.0.84 1.0.83.1 1.0.83 1.0.82 1.0.81 1.0.80 1.0.79 1.0.78 1.0.77 1.0.76 1.0.75 1.0.74 1.0.73 1.0.72 1.0.71 1.0.70 1.0.69 1.0.68 1.0.67 1.0.66 1.0.65 1.0.64 1.0.63 1.0.62 1.0.60.1 1.0.60 1.0.59 1.0.58 1.0.57 1.0.56.1 1.0.56 1.0.55 1.0.54 1.0.53 1.0.52 1.0.51 1.0.50.1 1.0.50 1.0.49 1.0.48 1.0.47 1.0.46.1 1.0.46 1.0.45 1.0.44 1.0.43 1.0.42 1.0.41 1.0.40.1 1.0.40 1.0.39 1.0.38.1 1.0.38 1.0.37 1.0.36 1.0.35 1.0.34 1.0.33 1.0.32 1.0.31 1.0.30 1.0.29 1.0.28 1.0.27 1.0.26 1.0.25 1.0.24 1.0.23 1.0.22 1.0.21 1.0.20 1.0.19 1.0.18 1.0.17 slapos.cookbook-1.0.124 seleniumrunner-firefox60 re6stnet-0.551+2 re6stnet-0.548+3 re6stnet-0.541+1 re6stnet-0.539+3 re6stnet-0.539+2 re6stnet-0.539+1 re6stnet-0.536+2 phase-1.00 mr80_review_2 frontend_parameter_doc_review1 erp5testnode-scalability-1-rc2 erp5testnode-scalability-1-rc1
1 merge request!32Update 1.0 branch for Next Release
......@@ -156,7 +156,7 @@ mode = 0644
[template-supervisord]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/${:filename}
md5sum = 069e593e50204b227bdb08d29d7292fd
md5sum = d294d0dafd265048399de6da8c96345f
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = supervisord.conf.in
download-only = true
......
......@@ -26,7 +26,9 @@ autorestart = {{ supervisord['autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{- '%d' % builtin.max(1, (multiprocessing.cpu_count() / builtin.int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -}}"
{# how many parallel build jobs to spawn when compiling software -#}
{% set njobs = builtin.max(1, (multiprocessing.cpu_count() // builtin.int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%}
environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{ njobs }}",NPY_NUM_BUILD_JOBS="{{ njobs }}",BUNDLE_JOBS="{{ njobs }}"
[program:{{- supervisord['slapgrid-cp'] -}}]
command = {{ supervisord['slapgrid-cp-command'] }}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment