[SYSTEMDS-3901] Updated Python CI Test setup
- split down of python test runs: w/ env, w/o env, federated
- separate python formatting test
These changes reduce the runtime of the single tests, and also make the Python test without environment to actually run without the SYSTEMDS_HOME installation.
Closes #2300.
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 465df22..273ca6f 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -53,8 +53,9 @@
os: [ubuntu-24.04]
java: ['17']
javadist: ['adopt']
+ test_mode: [env, noenv, federated]
- name: ${{ matrix.os }} Java ${{ matrix.java }} ${{ matrix.javadist }} Python ${{ matrix.python-version }}
+ name: ${{ matrix.os }} Java ${{ matrix.java }} ${{ matrix.javadist }} Python ${{ matrix.python-version }}/ ${{ matrix.test_mode}}
steps:
- name: Checkout Repository
uses: actions/checkout@v4
@@ -116,7 +117,6 @@
librosa \
h5py \
gensim \
- black \
opt-einsum \
nltk
@@ -125,7 +125,8 @@
cd src/main/python
python create_python_dist.py
- - name: Run all python tests
+ - name: Run tests with env
+ if: ${{ matrix.test_mode == 'env' }}
run: |
export SYSTEMDS_ROOT=$(pwd)
export PATH=$SYSTEMDS_ROOT/bin:$PATH
@@ -133,30 +134,20 @@
export LOG4JPROP=$SYSTEMDS_ROOT/src/test/resources/log4j.properties
cd src/main/python
unittest-parallel -t . -s tests -v
- # python -m unittest discover -s tests -p 'test_*.py'
- echo "Exit Status: " $?
- - name: Run all python tests no environment
+ - name: Run tests no env
+ if: ${{ matrix.test_mode == 'noenv' }}
run: |
export LOG4JPROP=$(pwd)/src/test/resources/log4j.properties
cd src/main/python
unittest-parallel -t . -s tests -v
- # python -m unittest discover -s tests -p 'test_*.py'
- echo "Exit Status: " $?
- name: Run Federated Python Tests
+ if: ${{ matrix.test_mode == 'federated' }}
run: |
export SYSTEMDS_ROOT=$(pwd)
export PATH=$SYSTEMDS_ROOT/bin:$PATH
cd src/main/python
./tests/federated/runFedTest.sh
- - name: Check formatting according to Black (src/main/python/systemds)
- run: |
- black --check --exclude operator/algorithm src/main/python/systemds
-
- - name: Check formatting according to Black (src/main/python/tests)
- run: |
- black --check src/main/python/tests
-
diff --git a/.github/workflows/pythonFormatting.yml b/.github/workflows/pythonFormatting.yml
new file mode 100644
index 0000000..7ecc325
--- /dev/null
+++ b/.github/workflows/pythonFormatting.yml
@@ -0,0 +1,56 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+name: Python Black Format Check
+
+on:
+ push:
+ paths:
+ - 'src/main/python/**'
+ branches:
+ - main
+ pull_request:
+ paths:
+ - 'src/main/python/**'
+ branches:
+ - main
+
+jobs:
+ black:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout Repository
+ uses: actions/checkout@v4
+
+ - name: Setup Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.x'
+
+ - name: Install Black
+ run: |
+ pip install --upgrade pip
+ pip install black
+
+ - name: Run Black Check
+ run: |
+ black --check --exclude operator/algorithm \
+ src/main/python/systemds src/main/python/tests
\ No newline at end of file
diff --git a/src/main/python/systemds/context/systemds_context.py b/src/main/python/systemds/context/systemds_context.py
index 440c51c..00aa693 100644
--- a/src/main/python/systemds/context/systemds_context.py
+++ b/src/main/python/systemds/context/systemds_context.py
@@ -965,7 +965,7 @@
def __init__(self, level=logging.NOTSET, formatter=None):
# Avoid setting stream directly to sys.stderr, we will do that dynamically
# For more info see test case: tests/basics/test_context_creation/test_random_port_debug3
- # where we redirect the err channel in between different contexts
+ # where we redirect the err channel between different contexts
super().__init__(stream=None)
self.setLevel(level)
if formatter:
