diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 67c6268..d5e4ec7 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - PYSPARK_VERSION: ["3.1.3", "3.2", "3.3"] + PYSPARK_VERSION: ["3.1.3", "3.2", "3.3", "3.5"] steps: - uses: actions/checkout@v3 diff --git a/Dockerfile b/Dockerfile index bdd9099..da30d1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ COPY poetry.lock /python-deequ WORKDIR python-deequ RUN poetry install -vvv -RUN poetry add pyspark==3.3 -vvv +RUN poetry add pyspark==3.5.0 -vvv -ENV SPARK_VERSION=3.3 +ENV SPARK_VERSION=3.5 COPY . /python-deequ CMD poetry run python -m pytest -s tests diff --git a/pydeequ/configs.py b/pydeequ/configs.py index ea4d0e8..d4d4b31 100644 --- a/pydeequ/configs.py +++ b/pydeequ/configs.py @@ -5,6 +5,7 @@ SPARK_TO_DEEQU_COORD_MAPPING = { + "3.5": "com.amazon.deequ:deequ:2.0.7-spark-3.5", "3.3": "com.amazon.deequ:deequ:2.0.7-spark-3.3", "3.2": "com.amazon.deequ:deequ:2.0.7-spark-3.2", "3.1": "com.amazon.deequ:deequ:2.0.7-spark-3.1" diff --git a/pyproject.toml b/pyproject.toml index c9fa38e..e4728dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ [tool.poetry.dependencies] -python = ">=3.7,<4" +python = ">=3.8,<4" numpy = ">=1.14.1" pandas = ">=0.23.0" pyspark = { version = ">=2.4.7, <3.3.0", optional = true }