From 365fb14568f36579db194b275f32a6c98ef4e143 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Thu, 29 Oct 2020 16:53:35 +0100
Subject: [PATCH 01/15] Add pre-commit workflow, rename test workflow

---
 .github/workflows/pre-commit.yaml             | 21 +++++++++++++++++++
 .../{python-app.yml => ubuntu-test.yml}       |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/pre-commit.yaml
 rename .github/workflows/{python-app.yml => ubuntu-test.yml} (99%)

diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml
new file mode 100644
index 000000000..25a138bda
--- /dev/null
+++ b/.github/workflows/pre-commit.yaml
@@ -0,0 +1,21 @@
+name: pre-commit
+
+on: [push]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: install pre-commit
+      run: |
+        pip install pre-commit
+        pre-commit install
+    - name: run pre-commit
+      run: |
+        set -e
+        pre-commit run --all-files
diff --git a/.github/workflows/python-app.yml b/.github/workflows/ubuntu-test.yml
similarity index 99%
rename from .github/workflows/python-app.yml
rename to .github/workflows/ubuntu-test.yml
index 7719af353..d409743d1 100644
--- a/.github/workflows/python-app.yml
+++ b/.github/workflows/ubuntu-test.yml
@@ -1,4 +1,4 @@
-name: CI
+name: ubuntu
 
 on: [push, pull_request]
 

From 43209c5b3fa2ea7ce9a0e13e1a50041b056ea457 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Thu, 29 Oct 2020 17:05:24 +0100
Subject: [PATCH 02/15] Fix formatting

---
 openml/datasets/functions.py                  | 9 ++++-----
 tests/test_datasets/test_dataset_functions.py | 5 +----
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py
index c2eb8ee75..b508626e8 100644
--- a/openml/datasets/functions.py
+++ b/openml/datasets/functions.py
@@ -183,7 +183,7 @@ def list_datasets(
     status: Optional[str] = None,
     tag: Optional[str] = None,
     output_format: str = "dict",
-    **kwargs
+    **kwargs,
 ) -> Union[Dict, pd.DataFrame]:
 
     """
@@ -251,7 +251,7 @@ def list_datasets(
         size=size,
         status=status,
         tag=tag,
-        **kwargs
+        **kwargs,
     )
 
 
@@ -334,8 +334,7 @@ def _load_features_from_file(features_file: str) -> Dict:
 
 
 def check_datasets_active(
-    dataset_ids: List[int],
-    raise_error_if_not_exist: bool = True,
+    dataset_ids: List[int], raise_error_if_not_exist: bool = True,
 ) -> Dict[int, bool]:
     """
     Check if the dataset ids provided are active.
@@ -363,7 +362,7 @@ def check_datasets_active(
         dataset = dataset_list.get(did, None)
         if dataset is None:
             if raise_error_if_not_exist:
-                raise ValueError(f'Could not find dataset {did} in OpenML dataset list.')
+                raise ValueError(f"Could not find dataset {did} in OpenML dataset list.")
         else:
             active[did] = dataset["status"] == "active"
 
diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py
index 707b6f9c5..ac7c9f862 100644
--- a/tests/test_datasets/test_dataset_functions.py
+++ b/tests/test_datasets/test_dataset_functions.py
@@ -227,10 +227,7 @@ def test_list_datasets_empty(self):
     def test_check_datasets_active(self):
         # Have to test on live because there is no deactivated dataset on the test server.
         openml.config.server = self.production_server
-        active = openml.datasets.check_datasets_active(
-            [2, 17, 79],
-            raise_error_if_not_exist=False,
-        )
+        active = openml.datasets.check_datasets_active([2, 17, 79], raise_error_if_not_exist=False,)
         self.assertTrue(active[2])
         self.assertFalse(active[17])
         self.assertIsNone(active.get(79))

From eb60b2c6f76dc607f86b6ebf062b5d5d32b4c85d Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Thu, 29 Oct 2020 18:13:44 +0100
Subject: [PATCH 03/15] Add a dist check workflow

Checks the dist can be built and installed and the long description is
rendered correctly on pypi
---
 .github/workflows/dist.yaml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 .github/workflows/dist.yaml

diff --git a/.github/workflows/dist.yaml b/.github/workflows/dist.yaml
new file mode 100644
index 000000000..f7bdd8bd4
--- /dev/null
+++ b/.github/workflows/dist.yaml
@@ -0,0 +1,24 @@
+name: dist-check
+
+on: [push, pull_request]
+
+jobs:
+  dist:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: build dist
+      run: |
+        python setup.py sdist
+        last_dist="dist/$(ls -t dist/openml-*.tar.gz | head -n 1)"
+    - name: twine check
+      run: |
+        pip install twine
+        twine check $last_dist
+    - name: install dist
+      run: |
+        pip install $last_dist

From 9cb3c15c587e2b26a00725db461d9b3810f9bc2c Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 30 Oct 2020 13:29:51 +0100
Subject: [PATCH 04/15] Determine $last_dist at each step

---
 .github/workflows/dist.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dist.yaml b/.github/workflows/dist.yaml
index f7bdd8bd4..8920ffb09 100644
--- a/.github/workflows/dist.yaml
+++ b/.github/workflows/dist.yaml
@@ -14,11 +14,12 @@ jobs:
     - name: build dist
       run: |
         python setup.py sdist
-        last_dist="dist/$(ls -t dist/openml-*.tar.gz | head -n 1)"
     - name: twine check
       run: |
         pip install twine
+        last_dist="dist/$(ls -t dist/openml-*.tar.gz | head -n 1)"
         twine check $last_dist
     - name: install dist
       run: |
+        last_dist="dist/$(ls -t dist/openml-*.tar.gz | head -n 1)"
         pip install $last_dist

From 768b0b8d95a20d6364056fadaca2be240e885f6c Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 30 Oct 2020 13:35:02 +0100
Subject: [PATCH 05/15] Remove duplicate "dist/"

---
 .github/workflows/dist.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dist.yaml b/.github/workflows/dist.yaml
index 8920ffb09..b14379b39 100644
--- a/.github/workflows/dist.yaml
+++ b/.github/workflows/dist.yaml
@@ -17,9 +17,9 @@ jobs:
     - name: twine check
       run: |
         pip install twine
-        last_dist="dist/$(ls -t dist/openml-*.tar.gz | head -n 1)"
+        last_dist=$(ls -t dist/openml-*.tar.gz | head -n 1)
         twine check $last_dist
     - name: install dist
       run: |
-        last_dist="dist/$(ls -t dist/openml-*.tar.gz | head -n 1)"
+        last_dist=$(ls -t dist/openml-*.tar.gz | head -n 1)
         pip install $last_dist

From f709a4048985de7f23bb0d7427eb386e49c57d76 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 30 Oct 2020 13:58:12 +0100
Subject: [PATCH 06/15] Downgrade to Python 3.8 since no wheel for 3.9

Not all wheels are available for Py3.9 yet, so CI stalls on installing
some packages (see also
https://github.com/docker-library/python/issues/540)
---
 .github/workflows/dist.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dist.yaml b/.github/workflows/dist.yaml
index b14379b39..7d7094b81 100644
--- a/.github/workflows/dist.yaml
+++ b/.github/workflows/dist.yaml
@@ -10,7 +10,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.9
+        python-version: 3.8
     - name: build dist
       run: |
         python setup.py sdist

From 76d5c7c4e2a29b6b9ee93cd7c7623ec4d7221b8e Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 30 Oct 2020 20:17:45 +0100
Subject: [PATCH 07/15] Add PEP 561 compliance check

---
 .github/workflows/dist.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/dist.yaml b/.github/workflows/dist.yaml
index 7d7094b81..afcc6fe08 100644
--- a/.github/workflows/dist.yaml
+++ b/.github/workflows/dist.yaml
@@ -23,3 +23,8 @@ jobs:
       run: |
         last_dist=$(ls -t dist/openml-*.tar.gz | head -n 1)
         pip install $last_dist
+    - name: PEP 561 Compliance
+      run: |
+        pip install mypy
+        cd ..  # required to use the installed version of openml
+        if ! python -m mypy -c "import openml"; then exit 1; fi

From 658c47c25761b5d90d3a3fc303299159d9969b4d Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Sat, 31 Oct 2020 22:07:49 +0100
Subject: [PATCH 08/15] Add workflow to build and deploy docs

---
 .github/workflows/docs.yaml | 44 +++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 .github/workflows/docs.yaml

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
new file mode 100644
index 000000000..a536920cf
--- /dev/null
+++ b/.github/workflows/docs.yaml
@@ -0,0 +1,44 @@
+name: Docs
+on: [pull_request, push]
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: install dependencies
+      run: |
+        pip install -e .[examples,examples_unix]
+        pip install sphinx-gallery sphinx_bootstrap_theme numpydoc
+    - name: make docs
+      run: |
+        cd doc
+        make html
+    - name: pull latest gh-pages
+      if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
+      run: |
+        cd ..
+        git clone https://github.com/openml/openml-python.git --branch gh-pages --single-branch gh-pages
+    - name: copy new doc into gh-pages
+      if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
+      run: |
+        branch_name=${GITHUB_REF##*/}
+        cd ../gh-pages
+        rm -rf $branch_name
+        cp -r ../openml-python/doc/build/html $branch_name
+    - name: push to gh-pages
+      if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
+      run: |
+        last_commit=$(git log --pretty=format:"%an: %s")
+        cd ../gh-pages
+        branch_name=${GITHUB_REF##*/}
+        git add $branch_name/
+        git config --global user.name 'Github Actions'
+        git config --global user.email 'not@mail.com'
+        git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
+        git commit -am "$last_commit"
+        git push

From b072f62f2ce9846645c44cf83e0e2cc549bf86ab Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Sun, 1 Nov 2020 15:30:39 +0100
Subject: [PATCH 09/15] Add code coverage reporting to py3.8/sk0.23.1

---
 .github/workflows/ubuntu-test.yml | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ubuntu-test.yml b/.github/workflows/ubuntu-test.yml
index d409743d1..fb68beac2 100644
--- a/.github/workflows/ubuntu-test.yml
+++ b/.github/workflows/ubuntu-test.yml
@@ -15,15 +15,18 @@ jobs:
             scikit-learn: 0.21.2
         include:
           - python-version: 3.6
-            scikit-learn: 0.18.2               
-            scipy: 1.2.0       
+            scikit-learn: 0.18.2
+            scipy: 1.2.0
           - python-version: 3.6
-            scikit-learn: 0.19.2           
+            scikit-learn: 0.19.2
           - python-version: 3.6
             scikit-learn: 0.20.2
+          - python-version: 3.8
+            scikit-learn: 0.23.1
+            code-cov: true
       fail-fast:  false
       max-parallel: 4
-    
+
     steps:
     - uses: actions/checkout@v2
     - name: CI Python ${{ matrix.python-version }} scikit-learn ${{ matrix.scikit-learn }}
@@ -38,6 +41,13 @@ jobs:
       run: |
         pip install scikit-learn==${{ matrix.scikit-learn }}
         if [ ${{ matrix.scipy }} ]; then pip install scipy==${{ matrix.scipy }}; fi
-    - name: Pytest
+    - name: Run Tests
       run: |
-        pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread -sv $PYTEST_ARGS $test_dir
+        if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long  --cov-report=xml'; fi
+        pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread -sv $codecov
+    - name: Upload Coverage
+      if: ${{ matrix.code-cov }} && ${{ always() }}
+      uses: codecov/codecov-action@v1
+      with:
+        fail_ci_if_error: true
+        verbose: true

From 601732de066770da4e8568fc57ff18bf276db05d Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Sun, 1 Nov 2020 15:45:36 +0100
Subject: [PATCH 10/15] Improve naming, make it consistent

Step names start with a capital. Clarified names to indicate what is
done.
---
 .github/workflows/dist.yaml       |  6 +++---
 .github/workflows/docs.yaml       | 12 ++++++------
 .github/workflows/pre-commit.yaml |  9 ++++-----
 .github/workflows/ubuntu-test.yml | 17 ++++++++++-------
 4 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/dist.yaml b/.github/workflows/dist.yaml
index afcc6fe08..51ffe03d5 100644
--- a/.github/workflows/dist.yaml
+++ b/.github/workflows/dist.yaml
@@ -11,15 +11,15 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    - name: build dist
+    - name: Build dist
       run: |
         python setup.py sdist
-    - name: twine check
+    - name: Twine check
       run: |
         pip install twine
         last_dist=$(ls -t dist/openml-*.tar.gz | head -n 1)
         twine check $last_dist
-    - name: install dist
+    - name: Install dist
       run: |
         last_dist=$(ls -t dist/openml-*.tar.gz | head -n 1)
         pip install $last_dist
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index a536920cf..61bc45473 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -2,7 +2,7 @@ name: Docs
 on: [pull_request, push]
 
 jobs:
-  docs:
+  build-and-deploy:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -10,27 +10,27 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    - name: install dependencies
+    - name: Install dependencies
       run: |
         pip install -e .[examples,examples_unix]
         pip install sphinx-gallery sphinx_bootstrap_theme numpydoc
-    - name: make docs
+    - name: Make docs
       run: |
         cd doc
         make html
-    - name: pull latest gh-pages
+    - name: Pull latest gh-pages
       if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
       run: |
         cd ..
         git clone https://github.com/openml/openml-python.git --branch gh-pages --single-branch gh-pages
-    - name: copy new doc into gh-pages
+    - name: Copy new doc into gh-pages
       if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
       run: |
         branch_name=${GITHUB_REF##*/}
         cd ../gh-pages
         rm -rf $branch_name
         cp -r ../openml-python/doc/build/html $branch_name
-    - name: push to gh-pages
+    - name: Push to gh-pages
       if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
       run: |
         last_commit=$(git log --pretty=format:"%an: %s")
diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml
index 25a138bda..6132b2de2 100644
--- a/.github/workflows/pre-commit.yaml
+++ b/.github/workflows/pre-commit.yaml
@@ -3,19 +3,18 @@ name: pre-commit
 on: [push]
 
 jobs:
-  pre-commit:
+  run-all-files:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Setup Python
+    - name: Setup Python 3.7
       uses: actions/setup-python@v2
       with:
         python-version: 3.7
-    - name: install pre-commit
+    - name: Install pre-commit
       run: |
         pip install pre-commit
         pre-commit install
-    - name: run pre-commit
+    - name: Run pre-commit
       run: |
-        set -e
         pre-commit run --all-files
diff --git a/.github/workflows/ubuntu-test.yml b/.github/workflows/ubuntu-test.yml
index fb68beac2..af02948b0 100644
--- a/.github/workflows/ubuntu-test.yml
+++ b/.github/workflows/ubuntu-test.yml
@@ -1,9 +1,9 @@
-name: ubuntu
+name: Tests
 
 on: [push, pull_request]
 
 jobs:
-  unittest:
+  ubuntu:
 
     runs-on: ubuntu-latest
     strategy:
@@ -29,23 +29,26 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: CI Python ${{ matrix.python-version }} scikit-learn ${{ matrix.scikit-learn }}
+    - name: Setup Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies (.[test])
+    - name: Install test dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e .[test]
     - name: Install scikit-learn ${{ matrix.scikit-learn }}
       run: |
         pip install scikit-learn==${{ matrix.scikit-learn }}
-        if [ ${{ matrix.scipy }} ]; then pip install scipy==${{ matrix.scipy }}; fi
-    - name: Run Tests
+    - name: Install scipy ${{ matrix.scipy }}
+      if: ${{ matrix.scipy }}
+      run: |
+        pip install scipy==${{ matrix.scipy }}
+    - name: Run tests
       run: |
         if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long  --cov-report=xml'; fi
         pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread -sv $codecov
-    - name: Upload Coverage
+    - name: Upload coverage
       if: ${{ matrix.code-cov }} && ${{ always() }}
       uses: codecov/codecov-action@v1
       with:

From faba2d682e1353cd38a95c3b6352a43167830149 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Sun, 1 Nov 2020 18:21:15 +0100
Subject: [PATCH 11/15] Check no files left behind after test

---
 .github/workflows/ubuntu-test.yml | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ubuntu-test.yml b/.github/workflows/ubuntu-test.yml
index af02948b0..5e2429c12 100644
--- a/.github/workflows/ubuntu-test.yml
+++ b/.github/workflows/ubuntu-test.yml
@@ -10,7 +10,7 @@ jobs:
       matrix:
         python-version: [3.6, 3.7, 3.8]
         scikit-learn: [0.21.2, 0.22.2, 0.23.1]
-        exclude:
+        exclude:  # no scikit-learn 0.21.2 release for Python 3.8
           - python-version: 3.8
             scikit-learn: 0.21.2
         include:
@@ -44,10 +44,25 @@ jobs:
       if: ${{ matrix.scipy }}
       run: |
         pip install scipy==${{ matrix.scipy }}
+    - name: Store repository status
+      id: status-before
+      run: |
+        echo "::set-output name=BEFORE::$(git status --porcelain -b)"
     - name: Run tests
       run: |
         if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long  --cov-report=xml'; fi
         pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread -sv $codecov
+    - name: Check for files left behind by test
+      if: ${{ always() }}
+      run: |
+        before="${{ steps.status-before.outputs.BEFORE }}"
+        after="$(git status --porcelain -b)"
+        if [[ "$before" != "$after" ]]; then
+            echo "git status from before: $before"
+            echo "git status from after: $after"
+            echo "Not all generated files have been deleted!"
+            exit 1
+        fi
     - name: Upload coverage
       if: ${{ matrix.code-cov }} && ${{ always() }}
       uses: codecov/codecov-action@v1

From 22760226b78e5d764ac708a66b5142548b7e557f Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Sun, 1 Nov 2020 18:39:23 +0100
Subject: [PATCH 12/15] Avoid upload coverage on non-coverage job

---
 .github/workflows/ubuntu-test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ubuntu-test.yml b/.github/workflows/ubuntu-test.yml
index 5e2429c12..9cc7b0f18 100644
--- a/.github/workflows/ubuntu-test.yml
+++ b/.github/workflows/ubuntu-test.yml
@@ -10,6 +10,7 @@ jobs:
       matrix:
         python-version: [3.6, 3.7, 3.8]
         scikit-learn: [0.21.2, 0.22.2, 0.23.1]
+        code-cov: [false]
         exclude:  # no scikit-learn 0.21.2 release for Python 3.8
           - python-version: 3.8
             scikit-learn: 0.21.2

From a9a84716e715591fad9b15abcbd2972882cdb98a Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Sun, 1 Nov 2020 19:02:22 +0100
Subject: [PATCH 13/15] Fix the conditional for codecov upload

---
 .github/workflows/ubuntu-test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/ubuntu-test.yml b/.github/workflows/ubuntu-test.yml
index 9cc7b0f18..c78de6445 100644
--- a/.github/workflows/ubuntu-test.yml
+++ b/.github/workflows/ubuntu-test.yml
@@ -10,7 +10,6 @@ jobs:
       matrix:
         python-version: [3.6, 3.7, 3.8]
         scikit-learn: [0.21.2, 0.22.2, 0.23.1]
-        code-cov: [false]
         exclude:  # no scikit-learn 0.21.2 release for Python 3.8
           - python-version: 3.8
             scikit-learn: 0.21.2
@@ -65,7 +64,7 @@ jobs:
             exit 1
         fi
     - name: Upload coverage
-      if: ${{ matrix.code-cov }} && ${{ always() }}
+      if: matrix.code-cov && always()
       uses: codecov/codecov-action@v1
       with:
         fail_ci_if_error: true

From d889726f777586440273b18011f483f77d59acca Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Sun, 1 Nov 2020 19:18:19 +0100
Subject: [PATCH 14/15] Remove Travis files

---
 .travis.yml              | 44 ----------------------
 ci_scripts/create_doc.sh | 61 ------------------------------
 ci_scripts/install.sh    | 81 ----------------------------------------
 ci_scripts/success.sh    | 15 --------
 ci_scripts/test.sh       | 48 ------------------------
 5 files changed, 249 deletions(-)
 delete mode 100644 .travis.yml
 delete mode 100644 ci_scripts/create_doc.sh
 delete mode 100755 ci_scripts/install.sh
 delete mode 100644 ci_scripts/success.sh
 delete mode 100644 ci_scripts/test.sh

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index ac9c067c1..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-language: python
-
-sudo: false
-
-cache:
-  apt: true
-  # We use three different cache directory
-  # to work around a Travis bug with multi-platform cache
-  directories:
-  - $HOME/.cache/pip
-  - $HOME/download
-env:
-  global:
-  # Directory where tests are run from
-  - TEST_DIR=/tmp/test_dir/
-  - MODULE=openml
-  matrix:
-  - DISTRIB="conda" PYTHON_VERSION="3.6" SKLEARN_VERSION="0.23.1" COVERAGE="true" DOCPUSH="true" SKIP_TESTS="true"
-  - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.23.1" RUN_FLAKE8="true" SKIP_TESTS="true"
-# Travis issue
-# https://github.com/travis-ci/travis-ci/issues/8920
-before_install:
- - python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)"
-
-install: source ci_scripts/install.sh
-script: bash ci_scripts/test.sh
-after_success: source ci_scripts/success.sh && source ci_scripts/create_doc.sh $TRAVIS_BRANCH "doc_result"
-
-# travis will check the deploy on condition, before actually running before_deploy
-# before_deploy: source ci_scripts/create_doc.sh $TRAVIS_BRANCH "doc_result"
-
-# For more info regarding the deploy process and the github token look at:
-# https://docs.travis-ci.com/user/deployment/pages/
-
-deploy:
-  provider: pages
-  skip_cleanup: true
-  github_token: $GITHUB_TOKEN
-  keep-history: true
-  committer-from-gh: true
-  on:
-    all_branches: true
-    condition: $doc_result = "success"
-  local_dir: doc/$TRAVIS_BRANCH
diff --git a/ci_scripts/create_doc.sh b/ci_scripts/create_doc.sh
deleted file mode 100644
index 83afaa26b..000000000
--- a/ci_scripts/create_doc.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-# License: BSD 3-Clause
-
-set -euo pipefail
-
-# Check if DOCPUSH is set
-if ! [[ -z ${DOCPUSH+x} ]]; then
-
-    if [[ "$DOCPUSH" == "true" ]]; then
-
-        # install documentation building dependencies
-        pip install matplotlib seaborn sphinx pillow sphinx-gallery sphinx_bootstrap_theme cython numpydoc nbformat nbconvert
-
-        # $1 is the branch name
-        # $2 is the global variable where we set the script status
-
-        if ! { [ $1 = "master" ] || [ $1 = "develop" ]; }; then
-            { echo "Not one of the allowed branches"; exit 0; }
-        fi
-
-        # delete any previous documentation folder
-        if [ -d doc/$1 ]; then
-            rm -rf doc/$1
-        fi
-
-        # create the documentation
-        cd doc && make html 2>&1
-
-        # create directory with branch name
-        # the documentation for dev/stable from git will be stored here
-        mkdir $1
-
-        # get previous documentation from github
-        git clone https://github.com/openml/openml-python.git --branch gh-pages --single-branch
-
-        # copy previous documentation
-        cp -r openml-python/. $1
-        rm -rf openml-python
-
-        # if the documentation for the branch exists, remove it
-        if [ -d $1/$1 ]; then
-            rm -rf $1/$1
-        fi
-
-        # copy the updated documentation for this branch
-        mkdir $1/$1
-        cp -r build/html/. $1/$1
-
-        # takes a variable name as an argument and assigns the script outcome to a
-        # variable with the given name. If it got this far, the script was successful
-        function set_return() {
-            # $1 is the variable where we save the script outcome
-            local __result=$1
-            local  status='success'
-            eval $__result="'$status'"
-        }
-
-        set_return "$2"
-    fi
-fi
-# Workaround for travis failure
-set +u
diff --git a/ci_scripts/install.sh b/ci_scripts/install.sh
deleted file mode 100755
index 67530af53..000000000
--- a/ci_scripts/install.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-# License: BSD 3-Clause
-
-set -e
-
-# Deactivate the travis-provided virtual environment and setup a
-# conda-based environment instead
-deactivate
-
-# Use the miniconda installer for faster download / install of conda
-# itself
-pushd .
-cd
-mkdir -p download
-cd download
-echo "Cached in $HOME/download :"
-ls -l
-echo
-if [[ ! -f miniconda.sh ]]
-   then
-   wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-       -O miniconda.sh
-   fi
-chmod +x miniconda.sh && ./miniconda.sh -b -p $HOME/miniconda
-cd ..
-export PATH=/home/travis/miniconda/bin:$PATH
-conda update --yes conda
-popd
-
-# Configure the conda environment and put it in the path using the
-# provided versions
-conda create -n testenv --yes python=$PYTHON_VERSION pip
-source activate testenv
-
-if [[ -v SCIPY_VERSION ]]; then
-    conda install --yes scipy=$SCIPY_VERSION
-fi
-python --version
-
-if [[ "$TEST_DIST" == "true" ]]; then
-    pip install twine nbconvert jupyter_client matplotlib pyarrow pytest pytest-xdist pytest-timeout \
-        nbformat oslo.concurrency flaky mypy
-    python setup.py sdist
-    # Find file which was modified last as done in https://stackoverflow.com/a/4561987
-    dist=`find dist -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" "`
-    echo "Installing $dist"
-    pip install "$dist"
-    twine check "$dist"
-else
-    pip install -e '.[test]'
-fi
-
-python -c "import numpy; print('numpy %s' % numpy.__version__)"
-python -c "import scipy; print('scipy %s' % scipy.__version__)"
-
-
-if [[ "$DOCPUSH" == "true" ]]; then
-    conda install --yes gxx_linux-64 gcc_linux-64 swig
-    pip install -e '.[examples,examples_unix]'
-fi
-if [[ "$COVERAGE" == "true" ]]; then
-    pip install codecov pytest-cov
-fi
-if [[ "$RUN_FLAKE8" == "true" ]]; then
-    pip install pre-commit
-    pre-commit install
-fi
-
-# PEP 561 compliance check
-# Assumes mypy relies solely on the PEP 561 standard
-if ! python -m mypy -c "import openml"; then
-   echo "Failed: PEP 561 compliance"
-   exit 1
-else
-   echo "Success: PEP 561 compliant"
-fi
-
-# Install scikit-learn last to make sure the openml package installation works
-# from a clean environment without scikit-learn.
-pip install scikit-learn==$SKLEARN_VERSION
-
-conda list
diff --git a/ci_scripts/success.sh b/ci_scripts/success.sh
deleted file mode 100644
index dad97d54e..000000000
--- a/ci_scripts/success.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-# License: BSD 3-Clause
-
-set -e
-
-if [[ "$COVERAGE" == "true" ]]; then
-    # Need to run coveralls from a git checkout, so we copy .coverage
-    # from TEST_DIR where pytest has been run
-    cp $TEST_DIR/.coverage $TRAVIS_BUILD_DIR
-    cd $TRAVIS_BUILD_DIR
-    # Ignore coveralls failures as the coveralls server is not
-    # very reliable but we don't want travis to report a failure
-    # in the github UI just because the coverage report failed to
-    # be published.
-    codecov || echo "Codecov upload failed"
-fi
diff --git a/ci_scripts/test.sh b/ci_scripts/test.sh
deleted file mode 100644
index 504d15bbd..000000000
--- a/ci_scripts/test.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-# License: BSD 3-Clause
-
-set -e
-
-# check status and branch before running the unit tests
-before="`git status --porcelain -b`"
-before="$before"
-# storing current working directory
-curr_dir=`pwd`
-
-run_tests() {
-    # Get into a temp directory to run test from the installed scikit learn and
-    # check if we  do not leave artifacts
-    mkdir -p $TEST_DIR
-
-    cwd=`pwd`
-    test_dir=$cwd/tests
-
-    cd $TEST_DIR
-
-    if [[ "$COVERAGE" == "true" ]]; then
-        PYTEST_ARGS='--cov=openml --long'
-    else
-        PYTEST_ARGS=''
-    fi
-
-    pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread -sv $PYTEST_ARGS $test_dir
-}
-
-if [[ "$RUN_FLAKE8" == "true" ]]; then
-    pre-commit run --all-files
-fi
-
-if [[ "$SKIP_TESTS" != "true" ]]; then
-    run_tests
-fi
-
-# changing directory to stored working directory
-cd $curr_dir
-# check status and branch after running the unit tests
-# compares with $before to check for remaining files
-after="`git status --porcelain -b`"
-if [[ "$before" != "$after" ]]; then
-    echo 'git status from before: '$before
-    echo 'git status from after: '$after
-    echo "All generated files have not been deleted!"
-    exit 1
-fi

From 76bef2536ae46121079229e4578600e64a518ff2 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 2 Nov 2020 09:22:55 +0100
Subject: [PATCH 15/15] Add optional dependencies for building docs

---
 .github/workflows/docs.yaml | 3 +--
 CONTRIBUTING.md             | 9 ++-------
 setup.py                    | 3 ++-
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index 61bc45473..2219c7fac 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -12,8 +12,7 @@ jobs:
         python-version: 3.8
     - name: Install dependencies
       run: |
-        pip install -e .[examples,examples_unix]
-        pip install sphinx-gallery sphinx_bootstrap_theme numpydoc
+        pip install -e .[docs,examples,examples_unix]
     - name: Make docs
       run: |
         cd doc
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6b7cffad3..6fe4fd605 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -260,14 +260,9 @@ The resulting HTML files will be placed in ``build/html/`` and are viewable in
 a web browser. See the ``README`` file in the ``doc/`` directory for more
 information.
 
-For building the documentation, you will need
-[sphinx](http://sphinx.pocoo.org/),
-[sphinx-bootstrap-theme](https://ryan-roemer.github.io/sphinx-bootstrap-theme/),
-[sphinx-gallery](https://sphinx-gallery.github.io/)
-and
-[numpydoc](https://numpydoc.readthedocs.io/en/latest/).
+For building the documentation, you will need to install a few additional dependencies:
 ```bash
-$ pip install sphinx sphinx-bootstrap-theme sphinx-gallery numpydoc
+$ pip install -e .[docs]
 ```
 When dependencies are installed, run
 ```bash
diff --git a/setup.py b/setup.py
index 9e9a093e4..b386f5829 100644
--- a/setup.py
+++ b/setup.py
@@ -81,7 +81,8 @@
             "ipykernel",
             "seaborn",
         ],
-        "examples_unix": ["fanova",],
+        "examples_unix": ["fanova"],
+        "docs": ["sphinx", "sphinx-gallery", "sphinx_bootstrap_theme", "numpydoc"],
     },
     test_suite="pytest",
     classifiers=[