From c793a40dbf62e8f60cf2f83f77a6d1d69339d36c Mon Sep 17 00:00:00 2001 From: bennahugo Date: Tue, 26 Mar 2024 10:12:33 +0200 Subject: [PATCH 1/8] fix failing test --- docker/python38.docker | 2 ++ tricolour/tests/test_flagging.py | 34 ++++++++++++++++---------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/docker/python38.docker b/docker/python38.docker index 6221771..8281665 100644 --- a/docker/python38.docker +++ b/docker/python38.docker @@ -17,3 +17,5 @@ WORKDIR /code # Install base + testing packages RUN pip3 install .[testing] + +RUN py.test -s -vvv --flake8 tricolour/ diff --git a/tricolour/tests/test_flagging.py b/tricolour/tests/test_flagging.py index 476ce41..6b27fa2 100644 --- a/tricolour/tests/test_flagging.py +++ b/tricolour/tests/test_flagging.py @@ -5,11 +5,11 @@ import scipy.interpolate from scipy.ndimage import gaussian_filter1d, gaussian_filter import pytest - from tricolour import flagging +import unittest -class TestAsbool(object): +class TestAsbool(unittest.TestCase): def _test(self, dtype, expect_view): a = np.array([0, 1, 1, 0, 1, 0, 0, 1], dtype) expected = a.astype(np.bool_) @@ -33,8 +33,8 @@ def test_bool(self): self._test(np.bool_, True) -class TestAverageFreq(object): - def setup(self): +class TestAverageFreq(unittest.TestCase): + def setUp(self): self.small_data = np.arange(30, dtype=np.float32).reshape(1, 5, 6) self.small_data = self.small_data.repeat(2, axis=0) self.small_flags = np.zeros(self.small_data.shape, np.bool_) @@ -151,11 +151,11 @@ def test_time_median(): np.testing.assert_array_equal(expected_flags, out_flags) -class TestMedianAbs(object): +class TestMedianAbs(unittest.TestCase): """Tests for :func:`katsdpsigproc.rfi.flagging._median_abs` and :func:`katsdpsigproc.rfi.flagging._median_abs_axis0`.""" - def setup(self): + def setUp(self): self.data = np.array([[-2.0, -6.0, 4.5], [1.5, 3.3, 0.5]], np.float32) self.flags = np.array([[0, 0, 0], [0, 1, 0]], np.uint8) @@ -179,12 +179,12 @@ def test_axis0_all_flagged(self): np.testing.assert_array_equal(expected, out) -class TestLinearlyInterpolateNans(object): +class TestLinearlyInterpolateNans(unittest.TestCase): """ Tests for :func:`katsdpsigproc.rfi.flagging._linearly_interpolate_nans`. """ - def setup(self): + def setUp(self): self.y = np.array([np.nan, np.nan, 4.0, np.nan, np.nan, 10.0, np.nan, -2.0, np.nan, np.nan]) self.expected = np.array([4.0, 4.0, 4.0, 6.0, 8.0, @@ -224,7 +224,7 @@ def test_2d(self): np.testing.assert_allclose(expected, y) -class TestBoxGaussianFilter(object): +class TestBoxGaussianFilter(unittest.TestCase): def test_one_pass(self): """Test that _box_gaussian_filter1d places the box correctly""" a = np.array([50.0, 10.0, 60.0, -70.0, 30.0, 20.0, -15.0], np.float32) @@ -289,8 +289,8 @@ def test_edge(self): np.testing.assert_allclose(fdata[:, 80:120], fcore, rtol=1e-5) -class TestMaskedGaussianFilter(object): - def setup(self): +class TestMaskedGaussianFilter(unittest.TestCase): + def setUp(self): self.rs = np.random.RandomState(seed=1) shape = (77, 53) self.data = self.rs.uniform(size=shape).astype(np.float32) @@ -332,7 +332,7 @@ def test_nan(self): assert 0 < np.sum(np.isnan(expected)) -class TestGetBackground2D(object): +class TestGetBackground2D(unittest.TestCase): """Tests for :func:`katsdpsigproc.rfi.flagging._get_background2d`. This is a difficult function to test, because it's not really practical to @@ -340,7 +340,7 @@ class TestGetBackground2D(object): where large regions are flagged. """ - def setup(self): + def setUp(self): self.shape = (95, 86) self.data = np.ones(self.shape, np.float32) * 7.5 self.flags = np.zeros(self.shape, np.uint8) @@ -421,8 +421,8 @@ def test_iterations(self): np.testing.assert_allclose(expected, background, rtol=1e-2) -class TestSumThreshold(object): - def setup(self): +class TestSumThreshold(unittest.TestCase): + def setUp(self): self.small_data = np.arange(30, dtype=np.float32).reshape(5, 6) self.small_flags = np.zeros(self.small_data.shape, np.bool_) self.small_flags[3, :] = 1 @@ -501,10 +501,10 @@ def test_sum_threshold_existing(self): out_flags[70, :4]) -class TestSumThresholdFlagger(object): +class TestSumThresholdFlagger(unittest.TestCase): """Tests for :class:`katsdpsigproc.rfi.flagging.SumThresholdFlagger`.""" - def setup(self): + def setUp(self): self.flagger = flagging.SumThresholdFlagger() def _make_background(self, shape, rs): From 741e1375ff7f7b7ae283c49088c1311d3525eb2e Mon Sep 17 00:00:00 2001 From: bennahugo Date: Tue, 26 Mar 2024 14:08:20 +0200 Subject: [PATCH 2/8] remove test from directly being called in the build --- docker/python36.docker | 2 +- docker/python38.docker | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/python36.docker b/docker/python36.docker index d998e3c..899b5d5 100644 --- a/docker/python36.docker +++ b/docker/python36.docker @@ -16,4 +16,4 @@ ADD . /code WORKDIR /code # Install base + testing packages -RUN pip3 install .[testing] +RUN pip3 install .[testing] \ No newline at end of file diff --git a/docker/python38.docker b/docker/python38.docker index 8281665..6221771 100644 --- a/docker/python38.docker +++ b/docker/python38.docker @@ -17,5 +17,3 @@ WORKDIR /code # Install base + testing packages RUN pip3 install .[testing] - -RUN py.test -s -vvv --flake8 tricolour/ From 5c2be7ce652d9a0183b92210ab056288a72a1ae1 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Tue, 26 Mar 2024 14:09:22 +0200 Subject: [PATCH 3/8] remove 3.6 and 3.7 from test matrix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07aaa65..af2eab0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8"] + python-version: ["3.8"] steps: - name: Set up Python ${{ matrix.python-version }} From 8e609b95c3df565f84f1459dbd86e0b42f568922 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Tue, 26 Mar 2024 14:47:56 +0200 Subject: [PATCH 4/8] migrate to newer numpy --- docker/python36.docker | 4 +++- docker/python38.docker | 2 ++ setup.py | 16 ++++++++-------- tricolour/apps/tricolour/app.py | 2 +- tricolour/mask.py | 2 +- tricolour/packing.py | 4 ++-- tricolour/tests/test_flagging_additional.py | 8 ++++---- tricolour/window_statistics.py | 8 ++++---- 8 files changed, 25 insertions(+), 21 deletions(-) diff --git a/docker/python36.docker b/docker/python36.docker index 899b5d5..750d5db 100644 --- a/docker/python36.docker +++ b/docker/python36.docker @@ -16,4 +16,6 @@ ADD . /code WORKDIR /code # Install base + testing packages -RUN pip3 install .[testing] \ No newline at end of file +RUN pip3 install .[testing] + +RUN python3 -m pytest --flake8 -s -vvv . \ No newline at end of file diff --git a/docker/python38.docker b/docker/python38.docker index 6221771..7e104a9 100644 --- a/docker/python38.docker +++ b/docker/python38.docker @@ -17,3 +17,5 @@ WORKDIR /code # Install base + testing packages RUN pip3 install .[testing] + +RUN python3 -m pytest --flake8 -s -vvv . \ No newline at end of file diff --git a/setup.py b/setup.py index 4d95568..5312bff 100644 --- a/setup.py +++ b/setup.py @@ -9,14 +9,14 @@ readme = readme_file.read() requirements = [ - 'dask[array] == 2021.2.0', - 'donfig >= 0.4.0', - 'numpy >= 1.14.0, <= 1.19.5', # breakage in newer numpy + numerical errors - 'numba >= 0.43.0', - 'scipy >= 1.2.0', - 'threadpoolctl >= 1.0.0', - 'dask-ms == 0.2.6', - 'zarr >= 2.3.1' + 'dask[array] >= 2021.2.0, <=2024.3.1', + 'donfig >= 0.4.0, <0.8.2', + 'numpy >= 1.14.0, <= 1.22.0', # breakage in newer numpy + numerical errors + 'numba >= 0.43.0, <= 0.59.1', + 'scipy >= 1.2.0, <=1.12.0', + 'threadpoolctl >= 1.0.0, <=3.4.0', + 'dask-ms >= 0.2.6, <=0.2.20', + 'zarr >= 2.3.1, <=2.17.1' ] extras_require = {'testing': ['pytest', diff --git a/tricolour/apps/tricolour/app.py b/tricolour/apps/tricolour/app.py index e420a7f..28de366 100644 --- a/tricolour/apps/tricolour/app.py +++ b/tricolour/apps/tricolour/app.py @@ -386,7 +386,7 @@ def _main(args): # Generate unflagged defaults if we should ignore existing flags # otherwise take flags from the dataset if args.ignore_flags is True: - flags = da.full_like(vis, False, dtype=np.bool) + flags = da.full_like(vis, False, dtype=bool) log.critical("Completely ignoring measurement set " "flags as per '-if' request. " "Strategy WILL NOT or with original flags, even if " diff --git a/tricolour/mask.py b/tricolour/mask.py index da1ab02..27273b1 100644 --- a/tricolour/mask.py +++ b/tricolour/mask.py @@ -60,7 +60,7 @@ def load_mask(filename, dilate): # Load mask mask = np.load(filename) - if mask.dtype[0] != np.bool or mask.dtype[1] != np.float64: + if mask.dtype[0] != bool or mask.dtype[1] != np.float64: raise ValueError("Mask %s is not a valid static mask " "with labelled channel axis " "[dtype == (bool, float64)]" % filename) diff --git a/tricolour/packing.py b/tricolour/packing.py index 6b009b6..11d25de 100644 --- a/tricolour/packing.py +++ b/tricolour/packing.py @@ -90,7 +90,7 @@ def _create_window_dask(name, ntime, nchan, nbl, ncorr, token, graph = HighLevelGraph.from_collections(collection_name, layers, ()) chunks = ((0,),) # One chunk containing single zarr array object - return da.Array(graph, collection_name, chunks, dtype=np.object) + return da.Array(graph, collection_name, chunks, dtype=object) def create_vis_windows(ntime, nchan, nbl, ncorr, token, @@ -343,7 +343,7 @@ def pack_data(time_inv, ubl, flags, ("row", "chan", "corr"), vis_win_obj, ("windim",), flag_win_obj, ("windim",), - dtype=np.bool) + dtype=bool) # Expose visibility data at it's full resolution vis_windows = da.blockwise(_packed_windows, _WINDOW_SCHEMA, diff --git a/tricolour/tests/test_flagging_additional.py b/tricolour/tests/test_flagging_additional.py index 53303f9..3680a0e 100644 --- a/tricolour/tests/test_flagging_additional.py +++ b/tricolour/tests/test_flagging_additional.py @@ -131,7 +131,7 @@ def test_apply_static_mask(wsrt_ants, unique_baselines, accumulation_mode="or") # Check that first mask's flags are applied - chan_sel = np.zeros(chan_freqs.shape[0], dtype=np.bool) + chan_sel = np.zeros(chan_freqs.shape[0], dtype=bool) chan_sel[[2, 10]] = True assert np.all(new_flags[:, :, :, chan_sel] == 1) @@ -144,7 +144,7 @@ def test_apply_static_mask(wsrt_ants, unique_baselines, accumulation_mode="or") # Check that both mask's flags have been applied - chan_sel = np.zeros(chan_freqs.shape[0], dtype=np.bool) + chan_sel = np.zeros(chan_freqs.shape[0], dtype=bool) chan_sel[[2, 10, 4, 11, 5]] = True assert np.all(new_flags[:, :, :, chan_sel] == 1) @@ -157,7 +157,7 @@ def test_apply_static_mask(wsrt_ants, unique_baselines, accumulation_mode="override") # Check that only last mask's flags applied - chan_sel = np.zeros(chan_freqs.shape[0], dtype=np.bool) + chan_sel = np.zeros(chan_freqs.shape[0], dtype=bool) chan_sel[[4, 11, 5]] = True assert np.all(new_flags[:, :, :, chan_sel] == 1) @@ -176,7 +176,7 @@ def test_apply_static_mask(wsrt_ants, unique_baselines, uvrange=uvrange) # Check that both mask's flags have been applied - chan_sel = np.zeros(chan_freqs.shape[0], dtype=np.bool) + chan_sel = np.zeros(chan_freqs.shape[0], dtype=bool) chan_sel[[2, 10, 4, 11, 5]] = True # Select baselines based on the uvrange diff --git a/tricolour/window_statistics.py b/tricolour/window_statistics.py index 6057e70..20d1843 100644 --- a/tricolour/window_statistics.py +++ b/tricolour/window_statistics.py @@ -123,7 +123,7 @@ def window_stats(flag_window, ubls, chan_freqs, field_name, None, ddid, None, nchanbins, None, - meta=np.empty((0,), dtype=np.object)) + meta=np.empty((0,), dtype=object)) # Create an empty stats object if the user hasn't supplied one if prev_stats is None: @@ -131,13 +131,13 @@ def _window_stat_creator(): return WindowStatistics(nchanbins) prev_stats = da.blockwise(_window_stat_creator, (), - meta=np.empty((), dtype=np.object)) + meta=np.empty((), dtype=object)) # Combine per-baseline stats into a single stats object return da.blockwise(_combine_baseline_window_stats, (), stats, ("bl",), prev_stats, (), - meta=np.empty((), dtype=np.object)) + meta=np.empty((), dtype=object)) def _combine_window_stats(*args): @@ -167,7 +167,7 @@ def combine_window_stats(window_stats): args = (v for ws in window_stats for v in (ws, ())) return da.blockwise(_combine_window_stats, (), - *args, dtype=np.object) + *args, dtype=object) class WindowStatistics(object): From f31661fb8096983a9b512648178735b3c2590bd2 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Tue, 26 Mar 2024 18:10:17 +0200 Subject: [PATCH 5/8] Add Python 3.10 test and replace generator_jit with overload --- .github/workflows/ci.yml | 2 +- Jenkinsfile.sh | 11 +++++++++++ docker/python310.docker | 22 ++++++++++++++++++++++ docker/python36.docker | 4 +--- docker/python38.docker | 4 +--- setup.py | 4 ++-- tricolour/flagging.py | 16 ++++++++++++++-- 7 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 docker/python310.docker diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af2eab0..cced2e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8"] + python-version: ["3.8", "3.9", "3.10"] steps: - name: Set up Python ${{ matrix.python-version }} diff --git a/Jenkinsfile.sh b/Jenkinsfile.sh index c72fb01..710acdc 100644 --- a/Jenkinsfile.sh +++ b/Jenkinsfile.sh @@ -12,6 +12,7 @@ TEST_DATA_DIR="$WORKSPACE/../../../test-data" # build docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python36.docker -t tricolour.1804.py36:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/ docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python38.docker -t tricolour.2004.py38:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/ +docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python310.docker -t tricolour.2204.py310:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/ #run tests tar xvf $TEST_DATA_DIR/acceptance_test_data.tar.gz -C $TEST_OUTPUT_DIR @@ -38,4 +39,14 @@ docker run \ --workdir /code \ --entrypoint /bin/sh \ tricolour.2004.py38:${BUILD_NUMBER} \ + -c "python3 -m pytest --flake8 -s -vvv ." + +# test 3.10 +docker run \ + --rm \ + -v $TEST_OUTPUT_DIR:/testdata \ + --env TRICOLOUR_TEST_MS=/testdata/$TEST_MS_REL \ + --workdir /code \ + --entrypoint /bin/sh \ + tricolour.2204.py310:${BUILD_NUMBER} \ -c "python3 -m pytest --flake8 -s -vvv ." \ No newline at end of file diff --git a/docker/python310.docker b/docker/python310.docker new file mode 100644 index 0000000..00d6af6 --- /dev/null +++ b/docker/python310.docker @@ -0,0 +1,22 @@ +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_PRIORITY critical + +# Install base requirements +RUN apt update && apt install -y python3-pip \ + build-essential \ + cmake \ + casacore-dev \ + python3-numpy \ + python3-setuptools \ + libboost-python-dev \ + libcfitsio-dev \ + wcslib-dev + +RUN pip3 install --upgrade pip setuptools +ADD . /code +WORKDIR /code + +# Install base + testing packages +RUN pip3 install .[testing] \ No newline at end of file diff --git a/docker/python36.docker b/docker/python36.docker index 750d5db..899b5d5 100644 --- a/docker/python36.docker +++ b/docker/python36.docker @@ -16,6 +16,4 @@ ADD . /code WORKDIR /code # Install base + testing packages -RUN pip3 install .[testing] - -RUN python3 -m pytest --flake8 -s -vvv . \ No newline at end of file +RUN pip3 install .[testing] \ No newline at end of file diff --git a/docker/python38.docker b/docker/python38.docker index 7e104a9..d413cec 100644 --- a/docker/python38.docker +++ b/docker/python38.docker @@ -16,6 +16,4 @@ ADD . /code WORKDIR /code # Install base + testing packages -RUN pip3 install .[testing] - -RUN python3 -m pytest --flake8 -s -vvv . \ No newline at end of file +RUN pip3 install .[testing] \ No newline at end of file diff --git a/setup.py b/setup.py index 5312bff..0bf88ee 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ requirements = [ 'dask[array] >= 2021.2.0, <=2024.3.1', 'donfig >= 0.4.0, <0.8.2', - 'numpy >= 1.14.0, <= 1.22.0', # breakage in newer numpy + numerical errors + 'numpy >= 1.14.0, <= 1.22.0', 'numba >= 0.43.0, <= 0.59.1', 'scipy >= 1.2.0, <=1.12.0', 'threadpoolctl >= 1.0.0, <=3.4.0', @@ -21,7 +21,7 @@ extras_require = {'testing': ['pytest', 'pytest-flake8', - 'flake8 >= 4.0.0, <5.0.0', + "flake8 >= 4.0.0, <5.0.0", 'requests', 'gdown']} setup( diff --git a/tricolour/flagging.py b/tricolour/flagging.py index 04fb045..710ba72 100644 --- a/tricolour/flagging.py +++ b/tricolour/flagging.py @@ -26,7 +26,7 @@ """ # noqa -@numba.njit(nogil=True, cache=True) +@numba.jit(nopython=True, nogil=True, cache=True) def flag_nans_and_zeros(vis_windows, flag_windows): """ Flag nan and zero visibilities. @@ -190,8 +190,20 @@ def _as_min_dtype(value): return np.array(value, dtype) -@numba.generated_jit(nopython=True, nogil=True, cache=True) +@numba.jit(nopython=True, nogil=True, cache=True) def _asbool(data): + return _asbool_impl(data) + + +def _asbool_impl(data): + if data.dtype.itemsize == 1: + return data.view(np.bool_) + else: + return data.astype(np.bool_) + + +@numba.extending.overload(_asbool_impl, nopython=True, nogil=True, cache=True) +def _asbool_impl_jit(data): """Create a boolean array with the same values as `data`. The `data` contain only 0's and 1's. If possible, a view is returned, From 7ede714854b4f35bb68acb822a48fb0f0d70ff99 Mon Sep 17 00:00:00 2001 From: Benjamin Hugo Date: Wed, 27 Mar 2024 15:55:29 +0200 Subject: [PATCH 6/8] Drop failing Python3.6 support --- Jenkinsfile.sh | 16 +++------------- docker/python36.docker | 19 ------------------- 2 files changed, 3 insertions(+), 32 deletions(-) delete mode 100644 docker/python36.docker diff --git a/Jenkinsfile.sh b/Jenkinsfile.sh index 710acdc..870f2b6 100644 --- a/Jenkinsfile.sh +++ b/Jenkinsfile.sh @@ -18,29 +18,19 @@ docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python310.docker -t tar xvf $TEST_DATA_DIR/acceptance_test_data.tar.gz -C $TEST_OUTPUT_DIR TEST_MS_REL=1519747221.subset.ms -# test 3.6 +# test 3.8 docker run \ --rm \ -v $TEST_OUTPUT_DIR:/testdata \ --env TRICOLOUR_TEST_MS=/testdata/$TEST_MS_REL \ --workdir /code \ --entrypoint /bin/sh \ - tricolour.1804.py36:${BUILD_NUMBER} \ + tricolour.2004.py38:${BUILD_NUMBER} \ -c "python3 -m pytest --flake8 -s -vvv ." rm -rf $TEST_OUTPUT_DIR/TEST_MS_REL tar xvf $TEST_DATA_DIR/acceptance_test_data.tar.gz -C $TEST_OUTPUT_DIR -# test 3.8 -docker run \ - --rm \ - -v $TEST_OUTPUT_DIR:/testdata \ - --env TRICOLOUR_TEST_MS=/testdata/$TEST_MS_REL \ - --workdir /code \ - --entrypoint /bin/sh \ - tricolour.2004.py38:${BUILD_NUMBER} \ - -c "python3 -m pytest --flake8 -s -vvv ." - # test 3.10 docker run \ --rm \ @@ -49,4 +39,4 @@ docker run \ --workdir /code \ --entrypoint /bin/sh \ tricolour.2204.py310:${BUILD_NUMBER} \ - -c "python3 -m pytest --flake8 -s -vvv ." \ No newline at end of file + -c "python3 -m pytest --flake8 -s -vvv ." diff --git a/docker/python36.docker b/docker/python36.docker deleted file mode 100644 index 899b5d5..0000000 --- a/docker/python36.docker +++ /dev/null @@ -1,19 +0,0 @@ -FROM kernsuite/base:6 - -# Install base requirements -RUN docker-apt-install python3-pip \ - build-essential \ - cmake \ - casacore-dev \ - python-numpy \ - python-setuptools \ - libboost-python-dev \ - libcfitsio-dev \ - wcslib-dev - -RUN pip3 install --upgrade pip setuptools -ADD . /code -WORKDIR /code - -# Install base + testing packages -RUN pip3 install .[testing] \ No newline at end of file From adc5c786954d7542abc1c281d765f2ba6052792b Mon Sep 17 00:00:00 2001 From: bennahugo Date: Thu, 28 Mar 2024 07:42:56 +0200 Subject: [PATCH 7/8] remove python3.6 build from jenkins script --- Jenkinsfile.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile.sh b/Jenkinsfile.sh index 870f2b6..502f6c7 100644 --- a/Jenkinsfile.sh +++ b/Jenkinsfile.sh @@ -10,7 +10,6 @@ TEST_DATA_DIR="$WORKSPACE/../../../test-data" # build -docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python36.docker -t tricolour.1804.py36:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/ docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python38.docker -t tricolour.2004.py38:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/ docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python310.docker -t tricolour.2204.py310:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/ From 3aff237a9b2e393bd2be0e0ce3ad270e8fd84e65 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Thu, 28 Mar 2024 16:26:50 +0200 Subject: [PATCH 8/8] Fixes #92 - Make logfile path configurable - Timestamp log - Gracefully bail if logfile cannot be written --- tricolour/apps/tricolour/app.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tricolour/apps/tricolour/app.py b/tricolour/apps/tricolour/app.py index 28de366..654427e 100644 --- a/tricolour/apps/tricolour/app.py +++ b/tricolour/apps/tricolour/app.py @@ -37,6 +37,7 @@ from tricolour.window_statistics import (window_stats, combine_window_stats, summarise_stats) +from datetime import datetime ############################################################## # Initialize Post Mortem debugger @@ -61,18 +62,30 @@ def create_logger(): log = logging.getLogger("tricolour") cfmt = logging.Formatter(u'%(name)s - %(asctime)s ' '%(levelname)s - %(message)s') - log.setLevel(logging.DEBUG) - filehandler = logging.FileHandler("tricolour.log") - filehandler.setFormatter(cfmt) - log.addHandler(filehandler) log.setLevel(logging.INFO) - console = logging.StreamHandler() console.setLevel(logging.INFO) console.setFormatter(cfmt) - log.addHandler(console) + # add an optional file handler + logger_path = os.environ.get("TRICOLOUR_LOGPATH", os.getcwd()) + nowT = int(np.ceil(datetime.timestamp(datetime.now()))) + logfile = os.path.join(logger_path, + f"tricolour.{nowT}.log") + try: + with open(logfile, "w") as f: + f.write("") + filehandler = logging.FileHandler(logfile) + filehandler.setFormatter(cfmt) + log.addHandler(filehandler) + if logger_path != os.getcwd(): + log.info(f"A copy of this log is available at {logfile}") + except PermissionError: + log.warning(f"Failed to initialize logfile for this run. " + f"Check your permissions and available space on " + f"'{logger_path}'. Proceeding without writing " + f"a logfile.") return log