diff --git a/.github/workflows/pythonpackage.yaml b/.github/workflows/pythonpackage.yaml index 0fe18285..bf70c9d6 100644 --- a/.github/workflows/pythonpackage.yaml +++ b/.github/workflows/pythonpackage.yaml @@ -35,9 +35,9 @@ jobs: - name: Format check with black run: | black --check . - - name: Format check with flake8 + - name: Format ruff check run: | - flake8 + ruff check . test: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index b6e47617..fabc8c22 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.ruff_cache/ # Translations *.mo diff --git a/CHANGELOG.md b/CHANGELOG.md index c5aca562..3f5a96c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fix bad release header links in CHANGELOG.md. ([PR #105](https://github.com/ClimateImpactLab/dscim/pull/105), [@brews](https://github.com/brews)) +- Fixed broken code quality checks in CI. Now using `ruff` instead of `flake8`. ([PR #107](https://github.com/ClimateImpactLab/dscim/pull/107), [@brews](https://github.com/brews)) ## [0.4.0] - 2023-07-06 diff --git a/pyproject.toml b/pyproject.toml index 894a0b52..83ab8957 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,3 +28,24 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] fallback_version = "999" + +[tool.ruff] +exclude = [ + ".eggs", + "docs", +] +# E402: module level import not at top of file +# E501: line too long - let black worry about that +ignore = [ + "E402", + "E501", +] +select = [ + # Pyflakes + "F", + # Pycodestyle + "E", + "W", + # Pyupgrade + "UP", +] diff --git a/requirements.txt b/requirements.txt index 2297e46f..99193a47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ dask-jobqueue==0.8.2 seaborn==0.12.2 geopandas==0.13.2 black==23.7.0 -flake8==6.0.0 +ruff==0.0.278 netcdf4==1.6.4 h5netcdf==1.2.0 impactlab-tools==0.4.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 76220c36..00000000 --- a/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[flake8] -exclude = docs,build,dist -ignore = E203,E266,E402,E501,W503,F401,C901 -max-line-length = 88 -max-complexity = 18 -select = B,C,E,F,W,T4,B9 diff --git a/src/dscim/__init__.py b/src/dscim/__init__.py index fdbba3f9..998197a4 100644 --- a/src/dscim/__init__.py +++ b/src/dscim/__init__.py @@ -1,7 +1,6 @@ import logging import sys import os -import imp import yaml import xarray import pathlib diff --git a/src/dscim/descriptors.py b/src/dscim/descriptors.py index 5b38c27b..d80a50d7 100644 --- a/src/dscim/descriptors.py +++ b/src/dscim/descriptors.py @@ -1,7 +1,7 @@ # from https://github.com/dssg/dickens/blob/master/src/descriptors.py -class cachedproperty(object): +class cachedproperty: """Non-data descriptor decorator implementing a read-only property which overrides itself on the instance with an entry in the instance's data dictionary, caching the result of the decorated diff --git a/src/dscim/menu/baseline.py b/src/dscim/menu/baseline.py index 7fa51fe0..892c3cc7 100644 --- a/src/dscim/menu/baseline.py +++ b/src/dscim/menu/baseline.py @@ -1,6 +1,3 @@ -import pandas as pd -import numpy as np -import xarray as xr from dscim.menu.main_recipe import MainRecipe diff --git a/src/dscim/menu/main_recipe.py b/src/dscim/menu/main_recipe.py index 27174016..b7adafb2 100644 --- a/src/dscim/menu/main_recipe.py +++ b/src/dscim/menu/main_recipe.py @@ -1,5 +1,4 @@ import os -import dask import logging import subprocess from subprocess import CalledProcessError @@ -10,7 +9,7 @@ from dscim.descriptors import cachedproperty from itertools import product from dscim.menu.decorators import save -from dscim.menu.simple_storage import StackedDamages, EconVars +from dscim.menu.simple_storage import StackedDamages from dscim.utils.utils import ( model_outputs, compute_damages, @@ -400,8 +399,6 @@ def output_attrs(self): A dict Class metadata """ - import dscim - # find machine name machine_name = os.getenv("HOSTNAME") if machine_name is None: @@ -894,7 +891,7 @@ def weitzman_min(self, no_cc_consumption, cc_consumption, parameter): if self.eta == 1: w_utility = np.log(parameter) bottom_utility = np.power(parameter, -1) * (parameter - cc_consumption) - bottom_coded_cons = np.exp((w_utility - bottom_utility)) + bottom_coded_cons = np.exp(w_utility - bottom_utility) clipped_cons = xr.where( cc_consumption > parameter, cc_consumption, bottom_coded_cons @@ -1048,10 +1045,8 @@ def marginal_damages(self): md = self.median_params_marginal_damages else: raise NotImplementedError( - ( - f"{agg} is not available. Enter list including" - '["ce", "fair", "median", "median_params"]' - ) + f"{agg} is not available. Enter list including" + '["ce", "fair", "median", "median_params"]' ) md = md.assign_coords({"fair_aggregation": agg}).expand_dims( diff --git a/src/dscim/menu/risk_aversion.py b/src/dscim/menu/risk_aversion.py index 2f5e274c..a06f9030 100644 --- a/src/dscim/menu/risk_aversion.py +++ b/src/dscim/menu/risk_aversion.py @@ -1,4 +1,3 @@ -import numpy as np import pandas as pd import xarray as xr from dscim.menu.main_recipe import MainRecipe diff --git a/src/dscim/menu/simple_storage.py b/src/dscim/menu/simple_storage.py index 68c17957..adfe4ef7 100644 --- a/src/dscim/menu/simple_storage.py +++ b/src/dscim/menu/simple_storage.py @@ -1,6 +1,5 @@ import logging import os -import sys import xarray as xr import numpy as np import pandas as pd diff --git a/src/dscim/preprocessing/climate/all_masks_rff.py b/src/dscim/preprocessing/climate/all_masks_rff.py index 959a82e0..19ec0b66 100644 --- a/src/dscim/preprocessing/climate/all_masks_rff.py +++ b/src/dscim/preprocessing/climate/all_masks_rff.py @@ -1,9 +1,6 @@ import xarray as xr -import pandas as pd -import numpy as np from dscim.menu.simple_storage import Climate import os -import sys import yaml from p_tqdm import p_map diff --git a/src/dscim/preprocessing/climate/ssp_clipped_rff.py b/src/dscim/preprocessing/climate/ssp_clipped_rff.py index 66622aa0..0e94dd63 100644 --- a/src/dscim/preprocessing/climate/ssp_clipped_rff.py +++ b/src/dscim/preprocessing/climate/ssp_clipped_rff.py @@ -1,6 +1,4 @@ import xarray as xr -import numpy as np -import pandas as pd from p_tqdm import p_map from itertools import product diff --git a/src/dscim/preprocessing/climate/stack_iterations.py b/src/dscim/preprocessing/climate/stack_iterations.py index 4ae35614..1bf93f83 100644 --- a/src/dscim/preprocessing/climate/stack_iterations.py +++ b/src/dscim/preprocessing/climate/stack_iterations.py @@ -1,8 +1,5 @@ import xarray as xr -import pandas as pd -import numpy as np import os -import sys USER = os.getenv("USER") from p_tqdm import p_map diff --git a/src/dscim/preprocessing/climate/stack_masks.py b/src/dscim/preprocessing/climate/stack_masks.py index 21cab5aa..6fea466b 100644 --- a/src/dscim/preprocessing/climate/stack_masks.py +++ b/src/dscim/preprocessing/climate/stack_masks.py @@ -1,11 +1,6 @@ import xarray as xr -import pandas as pd -import numpy as np import os -import sys -import yaml from p_tqdm import p_map -from itertools import product USER = os.getenv("USER") diff --git a/src/dscim/preprocessing/input_damages.py b/src/dscim/preprocessing/input_damages.py index ab368ff0..0dc99972 100644 --- a/src/dscim/preprocessing/input_damages.py +++ b/src/dscim/preprocessing/input_damages.py @@ -5,7 +5,6 @@ import os import re import logging -import warnings import multiprocessing import numpy as np import pandas as pd diff --git a/src/dscim/preprocessing/midprocessing.py b/src/dscim/preprocessing/midprocessing.py index 72ab903d..2b93b627 100644 --- a/src/dscim/preprocessing/midprocessing.py +++ b/src/dscim/preprocessing/midprocessing.py @@ -1,5 +1,4 @@ import os -import sys import shutil USER = os.getenv("USER") diff --git a/src/dscim/preprocessing/misc/check_nans.py b/src/dscim/preprocessing/misc/check_nans.py index 2528f17f..992d005f 100644 --- a/src/dscim/preprocessing/misc/check_nans.py +++ b/src/dscim/preprocessing/misc/check_nans.py @@ -1,11 +1,8 @@ # Pass a directory, output log files detailing number of nans each file has -import os -import numpy as np import pandas as pd import xarray as xr import glob -import shutil import csv from p_tqdm import p_uimap diff --git a/src/dscim/preprocessing/misc/convert_float32.py b/src/dscim/preprocessing/misc/convert_float32.py index 1c8c326f..355f37c6 100644 --- a/src/dscim/preprocessing/misc/convert_float32.py +++ b/src/dscim/preprocessing/misc/convert_float32.py @@ -1,6 +1,5 @@ import os import numpy as np -import pandas as pd import xarray as xr import glob import shutil diff --git a/src/dscim/preprocessing/preprocessing.py b/src/dscim/preprocessing/preprocessing.py index a253b389..8ac0e10f 100644 --- a/src/dscim/preprocessing/preprocessing.py +++ b/src/dscim/preprocessing/preprocessing.py @@ -1,19 +1,12 @@ -from math import ceil from pathlib import Path import os -import sys USER = os.getenv("USER") -import dask import dask.array as da -import dask.config import numpy as np import xarray as xr -from dask.distributed import Client, progress from dscim.utils.functions import ce_func, mean_func import yaml -import time -import argparse def ce_from_chunk( @@ -87,7 +80,7 @@ def reduce_damages( ), "Adding up does not take an eta argument. Please set to None." # client = Client(n_workers=35, memory_limit="9G", threads_per_worker=1) - with open(config, "r") as stream: + with open(config) as stream: c = yaml.safe_load(stream) params = c["sectors"][sector] @@ -209,7 +202,7 @@ def sum_AMEL( AMEL, ): # load config - with open(config, "r") as stream: + with open(config) as stream: loaded_config = yaml.safe_load(stream) params = loaded_config["sectors"] @@ -385,7 +378,7 @@ def clip_damages( """ # load config - with open(config, "r") as stream: + with open(config) as stream: loaded_config = yaml.safe_load(stream) params = loaded_config["sectors"][sector] diff --git a/src/dscim/utils/generate_yaml.py b/src/dscim/utils/generate_yaml.py index 2f4e65f7..964245fe 100644 --- a/src/dscim/utils/generate_yaml.py +++ b/src/dscim/utils/generate_yaml.py @@ -1,5 +1,4 @@ import os -import sys import yaml import dscim from dscim import Waiter diff --git a/src/dscim/utils/menu_runs.py b/src/dscim/utils/menu_runs.py index cfc7df49..c8a70845 100644 --- a/src/dscim/utils/menu_runs.py +++ b/src/dscim/utils/menu_runs.py @@ -5,18 +5,12 @@ from pathlib import Path import os -import gc -import time import yaml USER = os.getenv("USER") -import numpy as np -import pandas as pd -import xarray as xr import dask dask.config.set(**{"array.slicing.split_large_chunks": False}) -from dscim import ProWaiter from itertools import product MENU_OPTIONS = { @@ -44,7 +38,7 @@ def run_ssps( if fair_dims_list is None: fair_dims_list = [["simulation"]] - with open(config, "r") as stream: + with open(config) as stream: conf = yaml.safe_load(stream) for sector, pulse_year, menu_disc, eta_rho, mask, fair_dims in product( @@ -132,7 +126,7 @@ def run_rff( order="scc", regenerate_sccs=True, ): - with open(config, "r") as stream: + with open(config) as stream: conf = yaml.safe_load(stream) for sector, pulse_year, menu_disc, eta_rho in product( diff --git a/src/dscim/utils/plotting_utils.py b/src/dscim/utils/plotting_utils.py index 82d97470..5faf497a 100644 --- a/src/dscim/utils/plotting_utils.py +++ b/src/dscim/utils/plotting_utils.py @@ -1,4 +1,4 @@ -from matplotlib.ticker import SymmetricalLogLocator, FuncFormatter +from matplotlib.ticker import SymmetricalLogLocator import numpy as np diff --git a/src/dscim/utils/rff.py b/src/dscim/utils/rff.py index f97af2dc..2e45d2ab 100644 --- a/src/dscim/utils/rff.py +++ b/src/dscim/utils/rff.py @@ -5,13 +5,10 @@ from itertools import product from functools import partial import os -import sys from numpy.testing import assert_allclose from datetime import datetime import fsspec -import pyarrow import gurobipy as gp -import time from scipy.sparse import coo_matrix from gurobipy import GRB diff --git a/src/dscim/utils/utils.py b/src/dscim/utils/utils.py index 00612b6e..fce23a58 100644 --- a/src/dscim/utils/utils.py +++ b/src/dscim/utils/utils.py @@ -1,14 +1,10 @@ -import os import warnings -import dask import logging import xarray as xr import pandas as pd import numpy as np -from pathlib import Path import statsmodels.formula.api as smf import impactlab_tools.utils.weighting -import dask.dataframe as dd from itertools import product logger = logging.getLogger(__name__) diff --git a/tests/__init__.py b/tests/__init__.py index da046b13..94d831da 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,6 @@ import os import dask import zipfile -import pytest import pandas as pd import xarray as xr diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index e17acc36..9bdfb47b 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -1,10 +1,6 @@ import xarray as xr import numpy as np -import math import pytest -import copy -import dscim.utils.utils as estimations -from dscim.menu.risk_aversion import RiskAversionRecipe from dscim.preprocessing.preprocessing import ( subset_USA_ssp_econ, subset_USA_reduced_damages, @@ -12,7 +8,6 @@ reduce_damages, ce_from_chunk, ) -from pathlib import Path import yaml