diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d56cdf062fc..3413eb5c391 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -133,7 +133,7 @@ stages: key: $(testing_version) path: /home/vsts/mne_data displayName: 'Cache testing data' - - script: python -c "import mne; mne.datasets.testing.data_path(verbose=True)" + - bash: ./tools/github_actions_download.sh displayName: 'Get test data' - script: pytest -m "ultraslowtest or pgtest" --tb=short --cov=mne --cov-report=xml -vv mne displayName: 'slow and mne-qt-browser tests' @@ -188,7 +188,7 @@ stages: key: $(testing_version) path: /home/vsts/mne_data displayName: 'Cache testing data' - - script: python -c "import mne; mne.datasets.testing.data_path(verbose=True)" + - bash: ./tools/github_actions_download.sh displayName: 'Get test data' - bash: | set -eo pipefail @@ -285,7 +285,7 @@ stages: key: $(testing_version) path: C:\Users\VssAdministrator\mne_data displayName: 'Cache testing data' - - script: python -c "import mne; mne.datasets.testing.data_path(verbose=True)" + - bash: ./tools/github_actions_download.sh displayName: 'Get test data' - script: pytest -m "not (slowtest or pgtest)" --tb=short --cov=mne --cov-report=xml -vv mne displayName: 'Run tests' diff --git a/mne/commands/tests/test_commands.py b/mne/commands/tests/test_commands.py index a5d1ef32d7d..41347cec21b 100644 --- a/mne/commands/tests/test_commands.py +++ b/mne/commands/tests/test_commands.py @@ -49,6 +49,7 @@ from mne.io import read_info, read_raw_fif, show_fiff from mne.utils import ( ArgvSetter, + _chmod_rw_R, _record_warnings, _stamp_to_dt, requires_freesurfer, @@ -207,6 +208,7 @@ def test_make_scalp_surfaces(tmp_path, monkeypatch): shutil.copy(t1_path, t1_path_new) shutil.copy(headseg_path, headseg_path_new) shutil.copy(surf_path, surf_path_new) + _chmod_rw_R(tmp_path) cmd = ( "-s", diff --git a/mne/conftest.py b/mne/conftest.py index 6425d95b7ed..85a1e72e9f8 100644 --- a/mne/conftest.py +++ b/mne/conftest.py @@ -39,6 +39,7 @@ Bunch, _assert_no_instances, _check_qt_version, + _chmod_rw_R, _pl, _record_warnings, _TempDir, @@ -771,6 +772,7 @@ def subjects_dir_tmp(tmp_path): """Copy MNE-testing-data subjects_dir to a temp dir for manipulation.""" for key in ("sample", "fsaverage"): shutil.copytree(op.join(subjects_dir, key), str(tmp_path / key)) + _chmod_rw_R(tmp_path) return str(tmp_path) @@ -788,6 +790,7 @@ def subjects_dir_tmp_few(tmp_path): shutil.copytree( test_path / "subjects" / "sample" / dirname, sample_path / dirname ) + _chmod_rw_R(subjects_path) return subjects_path diff --git a/mne/io/ctf/tests/test_ctf.py b/mne/io/ctf/tests/test_ctf.py index 448ea90baba..53aaf6abcd8 100644 --- a/mne/io/ctf/tests/test_ctf.py +++ b/mne/io/ctf/tests/test_ctf.py @@ -32,7 +32,13 @@ from mne.io.tests.test_raw import _test_raw_reader from mne.tests.test_annotations import _assert_annotations_equal from mne.transforms import apply_trans -from mne.utils import _clean_names, _record_warnings, _stamp_to_dt, catch_logging +from mne.utils import ( + _clean_names, + _record_warnings, + _stamp_to_dt, + catch_logging, + copytree_rw, +) ctf_dir = testing.data_path(download=False) / "CTF" ctf_fname_continuous = "testdata_ctf.ds" @@ -75,7 +81,7 @@ def test_read_ctf(tmp_path): # Create a dummy .eeg file so we can test our reading/application of it os.mkdir(op.join(temp_dir, "randpos")) ctf_eeg_fname = op.join(temp_dir, "randpos", ctf_fname_catch) - shutil.copytree(op.join(ctf_dir, ctf_fname_catch), ctf_eeg_fname) + copytree_rw(op.join(ctf_dir, ctf_fname_catch), ctf_eeg_fname) with pytest.warns(RuntimeWarning, match="RMSP .* changed to a MISC ch"): raw = _test_raw_reader(read_raw_ctf, directory=ctf_eeg_fname) picks = pick_types(raw.info, meg=False, eeg=True) @@ -689,7 +695,7 @@ def _bad_res4_grad_comp(dsdir): def test_missing_res4(tmp_path): """Test that res4 missing is handled gracefully.""" use_ds = tmp_path / ctf_fname_continuous - shutil.copytree(ctf_dir / ctf_fname_continuous, tmp_path / ctf_fname_continuous) + copytree_rw(ctf_dir / ctf_fname_continuous, tmp_path / ctf_fname_continuous) read_raw_ctf(use_ds) os.remove(use_ds / (ctf_fname_continuous[:-2] + "meg4")) with pytest.raises(OSError, match="could not find the following"): diff --git a/mne/io/egi/tests/test_egi.py b/mne/io/egi/tests/test_egi.py index 261a9c80da3..09d1946e108 100644 --- a/mne/io/egi/tests/test_egi.py +++ b/mne/io/egi/tests/test_egi.py @@ -4,7 +4,6 @@ import os -import shutil from copy import deepcopy from datetime import datetime, timezone from pathlib import Path @@ -20,7 +19,7 @@ from mne.io import read_evokeds_mff, read_raw_egi, read_raw_fif from mne.io.egi.egi import _combine_triggers from mne.io.tests.test_raw import _test_raw_reader -from mne.utils import object_diff +from mne.utils import copytree_rw, object_diff base_dir = Path(__file__).parent / "data" egi_fname = base_dir / "test_egi.raw" @@ -337,7 +336,7 @@ def test_io_egi_pns_mff(tmp_path): # EEG missing new_mff = tmp_path / "temp.mff" - shutil.copytree(egi_mff_pns_fname, new_mff) + copytree_rw(egi_mff_pns_fname, new_mff) read_raw_egi(new_mff, verbose="error") os.remove(new_mff / "info1.xml") os.remove(new_mff / "signal1.bin") @@ -593,7 +592,7 @@ def test_set_standard_montage_mff(fname, standard_montage): def test_egi_mff_bad_xml(tmp_path): """Test that corrupt XML files are gracefully handled.""" pytest.importorskip("defusedxml") - mff_fname = shutil.copytree(egi_mff_fname, tmp_path / "test_egi_bad_xml.mff") + mff_fname = copytree_rw(egi_mff_fname, tmp_path / "test_egi_bad_xml.mff") bad_xml = mff_fname / "bad.xml" bad_xml.write_text("", encoding="utf-8") # Missing coordinate file diff --git a/mne/io/fil/tests/test_fil.py b/mne/io/fil/tests/test_fil.py index df15dd13353..af1a63303dd 100644 --- a/mne/io/fil/tests/test_fil.py +++ b/mne/io/fil/tests/test_fil.py @@ -2,7 +2,6 @@ # License: BSD-3-Clause # Copyright the MNE-Python contributors. -import shutil from os import remove import pytest @@ -14,6 +13,7 @@ from mne.datasets import testing from mne.io import read_raw_fil from mne.io.fil.sensors import _get_pos_units +from mne.utils import copytree_rw fil_path = testing.data_path(download=False) / "FIL" @@ -161,7 +161,7 @@ def test_fil_complete(): def test_fil_no_positions(tmp_path): """Test FIL reader in cases where a position file is missing.""" test_path = tmp_path / "FIL" - shutil.copytree(fil_path, test_path) + copytree_rw(fil_path, test_path) posname = test_path / "sub-noise_ses-001_task-noise220622_run-001_positions.tsv" binname = test_path / "sub-noise_ses-001_task-noise220622_run-001_meg.bin" @@ -179,7 +179,7 @@ def test_fil_no_positions(tmp_path): def test_fil_bad_channel_spec(tmp_path): """Test FIL reader when a bad channel is specified in channels.tsv.""" test_path = tmp_path / "FIL" - shutil.copytree(fil_path, test_path) + copytree_rw(fil_path, test_path) channame = test_path / "sub-noise_ses-001_task-noise220622_run-001_channels.tsv" binname = test_path / "sub-noise_ses-001_task-noise220622_run-001_meg.bin" diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 16d81c55e78..e9a5a117480 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -4,7 +4,6 @@ import datetime as dt import os -import shutil import numpy as np import pytest @@ -22,6 +21,7 @@ source_detector_distances, ) from mne.transforms import _get_trans, apply_trans +from mne.utils import copytree_rw testing_path = data_path(download=False) fname_nirx_15_0 = testing_path / "NIRx" / "nirscout" / "nirx_15_0_recording" @@ -246,7 +246,7 @@ def test_nirx_missing_warn(): @requires_testing_data def test_nirx_missing_evt(tmp_path): """Test reading NIRX files when missing data.""" - shutil.copytree(fname_nirx_15_2_short, str(tmp_path) + "/data/") + copytree_rw(fname_nirx_15_2_short, str(tmp_path) + "/data/") os.rename( tmp_path / "data" / "NIRS-2019-08-23_001.evt", tmp_path / "data" / "NIRS-2019-08-23_001.xxx", @@ -259,7 +259,7 @@ def test_nirx_missing_evt(tmp_path): @requires_testing_data def test_nirx_dat_warn(tmp_path): """Test reading NIRX files when missing data.""" - shutil.copytree(fname_nirx_15_2_short, str(tmp_path) + "/data/") + copytree_rw(fname_nirx_15_2_short, str(tmp_path) + "/data/") os.rename( tmp_path / "data" / "NIRS-2019-08-23_001.dat", tmp_path / "data" / "NIRS-2019-08-23_001.tmp", @@ -461,7 +461,7 @@ def test_nirx_15_3_short(): def test_locale_encoding(tmp_path): """Test NIRx encoding.""" fname = tmp_path / "latin" - shutil.copytree(fname_nirx_15_2, fname) + copytree_rw(fname_nirx_15_2, fname) hdr_fname = fname / "NIRS-2019-10-02_003.hdr" hdr = list() with open(hdr_fname, "rb") as fid: diff --git a/mne/io/persyst/tests/test_persyst.py b/mne/io/persyst/tests/test_persyst.py index 986a37b9dbb..b46804e1e09 100644 --- a/mne/io/persyst/tests/test_persyst.py +++ b/mne/io/persyst/tests/test_persyst.py @@ -12,6 +12,7 @@ from mne.datasets.testing import data_path, requires_testing_data from mne.io import read_raw_persyst from mne.io.tests.test_raw import _test_raw_reader +from mne.utils import _chmod_rw_R testing_path = data_path(download=False) fname_lay = ( @@ -144,6 +145,7 @@ def test_persyst_moved_file(tmp_path): new_fname_lay = tmp_path / fname_lay.name new_fname_dat = tmp_path / fname_dat.name shutil.copy(fname_lay, new_fname_lay) + _chmod_rw_R(tmp_path) # original file read should work read_raw_persyst(fname_lay) diff --git a/mne/io/snirf/tests/test_snirf.py b/mne/io/snirf/tests/test_snirf.py index aaec3ea20cf..40f72cf6778 100644 --- a/mne/io/snirf/tests/test_snirf.py +++ b/mne/io/snirf/tests/test_snirf.py @@ -28,7 +28,7 @@ source_detector_distances, ) from mne.transforms import _get_trans, apply_trans -from mne.utils import catch_logging +from mne.utils import _chmod_rw_R, catch_logging testing_path = data_path(download=False) # SfNIRS files @@ -252,8 +252,9 @@ def test_snirf_against_nirx(): @requires_testing_data def test_snirf_nonstandard(tmp_path): """Test custom tags.""" - shutil.copy(sfnirs_homer_103_wShort, str(tmp_path) + "/mod.snirf") fname = str(tmp_path) + "/mod.snirf" + shutil.copy(sfnirs_homer_103_wShort, fname) + _chmod_rw_R(tmp_path) # Manually mark up the file to match MNE-NIRS custom tags with h5py.File(fname, "r+") as f: f.create_dataset("nirs/metaDataTags/middleName", data=[b"X"]) @@ -287,8 +288,9 @@ def test_snirf_nonstandard(tmp_path): @requires_testing_data def test_snirf_empty_landmark_labels(tmp_path): """Test reading SNIRF files with empty landmarkLabels (gh-13627).""" - shutil.copy(sfnirs_homer_103_wShort, tmp_path / "empty_labels.snirf") fname = tmp_path / "empty_labels.snirf" + shutil.copy(sfnirs_homer_103_wShort, fname) + _chmod_rw_R(tmp_path) # Modify file to have landmarkPos3D but empty/scalar landmarkLabels with h5py.File(fname, "r+") as f: @@ -586,6 +588,7 @@ def test_sample_rate_jitter(tmp_path): # Create a clean copy and ensure it loads without error new_file = tmp_path / "snirf_nirsport2_2019.snirf" copy2(snirf_nirsport2_20219, new_file) + _chmod_rw_R(tmp_path) read_raw_snirf(new_file) # Edit the file and add jitter within tolerance (0.99%) diff --git a/mne/source_space/tests/test_source_space.py b/mne/source_space/tests/test_source_space.py index cb5f5c9d567..347e3887c10 100644 --- a/mne/source_space/tests/test_source_space.py +++ b/mne/source_space/tests/test_source_space.py @@ -3,7 +3,6 @@ # Copyright the MNE-Python contributors. from pathlib import Path -from shutil import copytree import numpy as np import pytest @@ -43,7 +42,7 @@ ) from mne.source_space._source_space import _compare_source_spaces from mne.surface import _accumulate_normals, _triangle_neighbors -from mne.utils import _record_warnings, requires_mne, run_subprocess +from mne.utils import _record_warnings, copytree_rw, requires_mne, run_subprocess data_path = testing.data_path(download=False) subjects_dir = data_path / "subjects" @@ -563,7 +562,7 @@ def test_setup_source_space(tmp_path): def test_setup_source_space_spacing(tmp_path, spacing, monkeypatch): """Test setting up surface source spaces using a given spacing.""" pytest.importorskip("nibabel") - copytree(subjects_dir / "sample", tmp_path / "sample") + copytree_rw(subjects_dir / "sample", tmp_path / "sample") args = [] if spacing == 7 else ["--spacing", str(spacing)] monkeypatch.setenv("SUBJECTS_DIR", str(tmp_path)) monkeypatch.setenv("SUBJECT", "sample") diff --git a/mne/tests/test_bem.py b/mne/tests/test_bem.py index 87f9b32a30d..cb65767fc33 100644 --- a/mne/tests/test_bem.py +++ b/mne/tests/test_bem.py @@ -44,6 +44,7 @@ from mne.surface import _get_ico_surface, read_surface from mne.transforms import translation from mne.utils import ( + _chmod_rw_R, _record_warnings, catch_logging, check_version, @@ -231,6 +232,7 @@ def test_bem_model_topology(tmp_path): subjects_dir / "sample" / "bem" / fname, tmp_path / "foo" / "bem" / fname, ) + _chmod_rw_R(tmp_path) outer_fname = tmp_path / "foo" / "bem" / "outer_skull.surf" rr, tris = read_surface(outer_fname) tris = tris[:-1] diff --git a/mne/utils/__init__.pyi b/mne/utils/__init__.pyi index 6e207423846..aeff12f21f7 100644 --- a/mne/utils/__init__.pyi +++ b/mne/utils/__init__.pyi @@ -56,6 +56,7 @@ __all__ = [ "_check_stc_units", "_check_subject", "_check_time_format", + "_chmod_rw_R", "_clean_names", "_click_ch_name", "_compute_row_norms", @@ -133,6 +134,7 @@ __all__ = [ "compute_corr", "copy_doc", "copy_function_doc_to_method_doc", + "copytree_rw", "create_slices", "deprecated", "deprecated_alias", @@ -202,6 +204,7 @@ from ._logging import ( ) from ._testing import ( ArgvSetter, + _chmod_rw_R, _click_ch_name, _raw_annot, _TempDir, @@ -212,6 +215,7 @@ from ._testing import ( assert_snr, assert_stcs_equal, buggy_mkl_svd, + copytree_rw, has_freesurfer, has_mne_c, requires_freesurfer, diff --git a/mne/utils/_testing.py b/mne/utils/_testing.py index 189d6877134..f1076d423e4 100644 --- a/mne/utils/_testing.py +++ b/mne/utils/_testing.py @@ -6,6 +6,7 @@ import inspect import os +import shutil import sys import tempfile import traceback @@ -417,3 +418,23 @@ def assert_trans_allclose(actual, desired, dist_tol=0.0, angle_tol=0.0): f"{1000 * dist:0.3f} > {1000 * dist_tol:0.3f} mm translation" ) assert angle <= angle_tol, f"{angle:0.3f} > {angle_tol:0.3f}° rotation" + + +def _chmod_rw_R(path): + assert os.path.isdir(path), f"Expected a directory, got {path}" + os.chmod(path, 0o700 | os.stat(path).st_mode) + for root, dirs, files in os.walk(path): + for name in files: + this_name = os.path.join(root, name) + os.chmod(this_name, 0o600 | os.stat(this_name).st_mode) + for name in dirs: + this_name = os.path.join(root, name) + os.chmod(this_name, 0o770 | os.stat(this_name).st_mode) + + +def copytree_rw(src, dst): + """Copy a directory tree and make it read/write.""" + assert os.path.isdir(src), f"Expected a directory, got {src}" + shutil.copytree(src, dst) + _chmod_rw_R(dst) + return dst diff --git a/tools/github_actions_download.sh b/tools/github_actions_download.sh index 638cea44327..a503b562068 100755 --- a/tools/github_actions_download.sh +++ b/tools/github_actions_download.sh @@ -3,4 +3,8 @@ if [ "${MNE_CI_KIND}" != "minimal" ]; then python -c 'import mne; mne.datasets.testing.data_path(verbose=True)'; python -c "import mne; mne.datasets.misc.data_path(verbose=True)"; + # Make read-only to make sure we don't modify its contents + TESTING_PATH=$(python -c "import mne; print(mne.datasets.testing.data_path(verbose=False))") + echo "Testing data path: $TESTING_PATH" + chmod -R a-w "$TESTING_PATH" fi