Merge pull request #202 from mdboom/fix-benchmarks

Fix benchmarks in light of recent changes

.circleci/config.yml

@@ -19,21 +19,29 @@ jobs:
             # Set up the Debian testing repo, and then install g++ from there...
             sudo bash -c "echo \"deb http://ftp.us.debian.org/debian testing main contrib non-free\" >> /etc/apt/sources.list"
             sudo apt-get update
-            sudo apt-get install node-less cmake build-essential clang-format-6.0 flake8 uglifyjs python3-yaml chromium ccache
+            sudo apt-get install node-less cmake build-essential clang-format-6.0 uglifyjs chromium ccache
             sudo apt-get install -t testing g++-8
             sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-6 60 --slave /usr/bin/g++ g++ /usr/bin/g++-6
             sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8
             sudo update-alternatives --set gcc /usr/bin/gcc-8
             sudo ln -s /usr/bin/clang-format-6.0 /usr/bin/clang-format
 
+            # Install Python dependencies
             sudo pip install virtualenv
-
             virtualenv pyodide-env
-
             source pyodide-env/bin/activate
+            pip install pytest pytest-xdist pytest-instafail selenium PyYAML flake8
 
-            pip install pytest pytest-xdist pytest-instafail selenium PyYAML
+            # Get recent version of Firefox and geckodriver
+            wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-latest-ssl\&os\=linux64\&lang\=en-US
+            tar jxf firefox.tar.bz2
+            wget https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-linux64.tar.gz
+            tar zxf geckodriver-v0.21.0-linux64.tar.gz -C firefox
 
+            # Get recent version of chromedriver
+            wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip
+            unzip chromedriver_linux64.zip
+            mv chromedriver pyodide-env/bin/
 
       - run:
           name: lint
@@ -69,6 +77,8 @@ jobs:
           paths:
             - ./build
             - ./pyodide-env
+            - ./cpython/build/3.7.0/host
+            - ./firefox
 
       - store_artifacts:
           path: /home/circleci/repo/build/
@@ -86,12 +96,6 @@ jobs:
             # causes Firefox to complain when loading it.  Let's just add the new mime type.
             sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types"
 
-            # Get recent version of Firefox and geckodriver
-            wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-latest-ssl\&os\=linux64\&lang\=en-US
-            tar jxf firefox.tar.bz2
-            wget https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-linux64.tar.gz
-            tar zxf geckodriver-v0.21.0-linux64.tar.gz -C firefox
-
             source pyodide-env/bin/activate
             export PATH=$PWD/firefox:$PATH
             pytest test -v -k firefox
@@ -109,14 +113,35 @@ jobs:
             # causes Firefox to complain when loading it.  Let's just add the new mime type.
             sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types"
 
-            # Get recent version of chromedriver
-            wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip
-            unzip chromedriver_linux64.zip
-            mv chromedriver pyodide-env/bin/
-
             source pyodide-env/bin/activate
             pytest test -v -k chrome
 
+  benchmark:
+    <<: *defaults
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: dependencies
+          command: |
+            sudo bash -c "echo \"deb http://ftp.us.debian.org/debian testing main contrib non-free\" >> /etc/apt/sources.list"
+            sudo apt-get update
+            sudo apt-get install -t testing g++-8
+            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-6 60 --slave /usr/bin/g++ g++ /usr/bin/g++-6
+            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8
+            sudo update-alternatives --set gcc /usr/bin/gcc-8
+      - run:
+          name: benchmark
+          command: |
+            sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types"
+
+            source pyodide-env/bin/activate
+            export PATH=$PWD/firefox:$PATH
+            python benchmark/benchmark.py cpython/build/3.7.0/host/bin/python3 build/benchmarks.json
+      - store_artifacts:
+          path: /home/circleci/repo/build
+
   deploy:
     machine:
       enabled: true
@@ -141,6 +166,9 @@ workflows:
       - test-firefox:
           requires:
             - build
+      - benchmark:
+          requires:
+            - build
       - deploy:
           requires:
             - test-chrome

Makefile

@@ -122,7 +122,7 @@ lint:
 	clang-format -output-replacements-xml src/*.c src/*.h src/*.js | (! grep '<replacement ')
 
 
-benchmark: all build/test.html
+benchmark: all
 	python benchmark/benchmark.py $(HOSTPYTHON) build/benchmarks.json
 	python benchmark/plot_benchmark.py build/benchmarks.json build/benchmarks.png
 

benchmark/benchmark.py

@@ -10,7 +10,7 @@ sys.path.insert(
 import conftest  # noqa
 
 
-SKIP = set(['fft', 'hyantes'])
+SKIP = set(['fft', 'hyantes', 'README'])
 
 
 def run_native(hostpython, code):
@@ -25,27 +25,27 @@ def run_native(hostpython, code):
     return float(output.strip().split()[-1])
 
 
-def run_wasm(code, cls):
-    s = cls()
+def run_wasm(code, cls, port):
+    s = cls(port)
     try:
         s.load_package('numpy')
         s.run(code)
         try:
-            runtime = float(s.logs[-1])
+            runtime = float(s.logs.split('\n')[-1])
         except ValueError:
-            print('\n'.join(s.logs))
+            print(s.logs)
             raise
     finally:
         s.driver.quit()
     return runtime
 
 
-def run_all(hostpython, code):
+def run_all(hostpython, port, code):
     a = run_native(hostpython, code)
     print("native:", a)
-    b = run_wasm(code, conftest.FirefoxWrapper)
+    b = run_wasm(code, conftest.FirefoxWrapper, port)
     print("firefox:", b)
-    c = run_wasm(code, conftest.ChromeWrapper)
+    c = run_wasm(code, conftest.ChromeWrapper, port)
     print("chrome:", c)
     result = {
         'native': a,
@@ -74,9 +74,9 @@ def parse_numpy_benchmark(filename):
 
 
 def get_numpy_benchmarks():
-    root = Path('../numpy-benchmarks/benchmarks')
+    root = Path(__file__).resolve().parent / 'benchmarks'
     for filename in root.iterdir():
-        name = filename.name
+        name = filename.stem
         if name in SKIP:
             continue
         content = parse_numpy_benchmark(filename)
@@ -87,6 +87,8 @@ def get_numpy_benchmarks():
             "from timeit import Timer\n"
             "t = Timer(run, setup)\n"
             "r = t.repeat(11, 40)\n"
+            "r.remove(min(r))\n"
+            "r.remove(max(r))\n"
             "print(np.mean(r))\n".format(name))
         yield name, content
 
@@ -97,10 +99,11 @@ def get_benchmarks():
 
 
 def main(hostpython):
-    results = {}
-    for k, v in get_benchmarks():
-        print(k)
-        results[k] = run_all(hostpython, v)
+    with conftest.spawn_web_server() as (hostname, port, log_path):
+        results = {}
+        for k, v in get_benchmarks():
+            print(k)
+            results[k] = run_all(hostpython, port, v)
     return results
 
 

benchmark/plot_benchmark.py

@@ -28,7 +28,7 @@ ax.invert_yaxis()
 ax.set_xlabel('Slowdown factor (WebAssembly:Native)')
 ax.set_title('Python benchmarks')
 ax.axvline(1.0, color='red')
-ax.grid()
+ax.grid(axis='x')
 ax.legend(loc='lower right')
 
 plt.savefig(sys.argv[-1])