diff --git a/README.md b/README.md index e53250f0..e5a3dbec 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ pip install saagieapi== | >= 2023.02 | >= 2.6.0 | | >= 2023.03 | >= 2.7.0 | | >= 2023.04 | >= 2.9.0 | +| >= 2023.05 | >= 2.10.0 | ## Contributing diff --git a/docs/index.rst b/docs/index.rst index 0dbd1a23..c87f7922 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,6 +24,7 @@ Compatibility with your Saagie platform >= 2023.02 >= 2.6.0 >= 2023.03 >= 2.7.0 >= 2023.04 >= 2.9.0 +>= 2023.05 >= 2.10.0 =========================== ====================== Usage diff --git a/saagieapi/pipelines/gql_queries.py b/saagieapi/pipelines/gql_queries.py index fbed55db..50154e32 100644 --- a/saagieapi/pipelines/gql_queries.py +++ b/saagieapi/pipelines/gql_queries.py @@ -516,3 +516,12 @@ ) } """ + +GQL_DUPLICATE_PIPELINE = """ +mutation duplicatePipeline($pipelineId: UUID!) { + duplicatePipeline(originalPipelineId: $pipelineId) { + id + name + } +} +""" diff --git a/saagieapi/pipelines/pipelines.py b/saagieapi/pipelines/pipelines.py index 3055fb12..7891b212 100644 --- a/saagieapi/pipelines/pipelines.py +++ b/saagieapi/pipelines/pipelines.py @@ -1778,3 +1778,32 @@ def delete_instances_by_date( result = self.saagie_api.client.execute(query=gql(GQL_DELETE_PIPELINE_INSTANCE_BY_DATE), variable_values=params) logging.info("✅ Instances of pipeline [%s] successfully deleted", pipeline_id) return result + + def duplicate(self, pipeline_id): + """Duplicate a given pipeline + + Parameters + ---------- + pipeline_id : str + UUID of your pipeline (see README on how to find it) + + Returns + ------- + dict + Dict of duplicate pipeline with its id and name + + Examples + -------- + >>> saagie_api.pipelines.duplicate(pipeline_id=pipeline_id) + { + 'duplicatePipeline': { + 'id': '29cf1b80-6b9c-47bc-a06c-c20897257097', + 'name': 'Copy of my_pipeline 2' + } + } + """ + result = self.saagie_api.client.execute( + query=gql(GQL_DUPLICATE_PIPELINE), variable_values={"pipelineId": pipeline_id} + ) + logging.info("✅ Pipeline [%s] successfully duplicated", pipeline_id) + return result diff --git a/saagieapi/storages/gql_queries.py b/saagieapi/storages/gql_queries.py index 23afdf3a..99308ced 100644 --- a/saagieapi/storages/gql_queries.py +++ b/saagieapi/storages/gql_queries.py @@ -32,6 +32,7 @@ volumes { id name + projectId ...volumeInformation @skip(if: $minimal) } } @@ -43,6 +44,7 @@ createVolume(volume: $volume) { id name + projectId description size creator @@ -79,3 +81,43 @@ } } """ + +GQL_MOVE_STORAGE = """ +mutation moveVolumeMutation($volumeId: UUID!, $targetPlatformId: Int!, $targetProjectId: UUID!) { + moveVolume(volumeId: $volumeId, targetPlatformId: $targetPlatformId, targetProjectId: $targetProjectId) +} +""" + +GQL_GET_STORAGE_INFO = """ +fragment appVersionFieldInformation on AppVersion { + number + volumesWithPath { + path + volume { + id + } + } +} + +query volumeInfoQuery($volumeId: UUID!) { + volume(id: $volumeId) { + id + name + projectId + size + description + creationDate + creator + linkedApp { + id + name + versions { + ...appVersionFieldInformation + } + currentVersion { + ...appVersionFieldInformation + } + } + } +} +""" diff --git a/saagieapi/storages/storages.py b/saagieapi/storages/storages.py index 45dad891..0f1611d1 100644 --- a/saagieapi/storages/storages.py +++ b/saagieapi/storages/storages.py @@ -1,6 +1,7 @@ import logging from typing import Dict, Optional +import deprecation from gql import gql from .gql_queries import * @@ -120,6 +121,11 @@ def list_for_project( query=gql(GQL_LIST_STORAGE_FOR_PROJECT), variable_values=params, pprint_result=pprint_result ) + @deprecation.deprecated( + details="This function is deprecated and will be removed in a future version. " + "Please use :func:`get()` instead.", + deprecated_in="2.10.0", + ) def get_info(self, project_id: str, storage_id: str) -> Dict: """Get storage information. @@ -144,7 +150,7 @@ def get_info(self, project_id: str, storage_id: str) -> Dict: -------- >>> saagieapi.storages.get_info( ... project_id="860b8dc8-e634-4c98-b2e7-f9ec32ab4771", - ... job_id="f5fce22d-2152-4a01-8c6a-4c2eb4808b6d" + ... storage_id="f5fce22d-2152-4a01-8c6a-4c2eb4808b6d" ... ) { "id":"89bf5f86-3fc3-4bf6-879b-7ca8eafe6c4f", @@ -183,6 +189,65 @@ def get_info(self, project_id: str, storage_id: str) -> Dict: raise ValueError(f"❌ Storage '{storage_id}' not found in project '{project_id}'") + def get(self, storage_id: str) -> Dict: + """Get storage information. + + Note + ---- + You can only get storage information if you have at least the viewer role on + the project. + + Parameters + ---------- + storage_id : str + UUID of your storage + + Returns + ------- + dict + Dict of storage information + + Examples + -------- + >>> saagieapi.storages.get( + ... storage_id="f5fce22d-2152-4a01-8c6a-4c2eb4808b6d" + ... ) + { + "volume": + { + "id":"89bf5f86-3fc3-4bf6-879b-7ca8eafe6c4f", + "name":"storage Jupyter Notebook (0)", + "size":"64 MB", + "description":"Autogenerated storage from app installation for \"/notebooks-dir\" path in \"Jupyter Notebook\" app.", + "creationDate":"2022-08-26T13:12:05.363Z", + "creator":"user.name", + "linkedApp":{ + "id":"6871e9a2-2c06-45fe-bf8d-6356090f1d1d", + "name":"Jupyter Notebook", + "versions":[ + { + "number":1, + "volumesWithPath":[ + { + "path":"/notebooks-dir", + "volume":{ + "id":"89bf5f86-3fc3-4bf6-879b-7ca8eafe6c4f" + } + } + ] + } + ], + "currentVersion":{ + "number":1, + "volumesWithPath":[] + } + } + } + } + """ # pylint: disable=line-too-long + + return self.saagie_api.client.execute(query=gql(GQL_GET_STORAGE_INFO), variable_values={"volumeId": storage_id}) + def create( self, project_id: str, @@ -337,7 +402,7 @@ def delete(self, storage_id: str, project_id: str) -> Dict: } } """ - storage_info = self.get_info(project_id, storage_id) + storage_info = self.get(storage_id=storage_id)["volume"] if storage_info["linkedApp"] is not None and "currentVersion" in storage_info["linkedApp"]: for volume in storage_info["linkedApp"]["currentVersion"]["volumesWithPath"]: if volume["volume"]["id"] == storage_id: @@ -380,7 +445,7 @@ def unlink(self, storage_id: str, project_id: str) -> Dict: } } """ - storage_info = self.get_info(project_id, storage_id) + storage_info = self.get(storage_id=storage_id)["volume"] if storage_info["linkedApp"] is not None and "currentVersion" in storage_info["linkedApp"].keys(): for volume in storage_info["linkedApp"]["currentVersion"]["volumesWithPath"]: if volume["volume"]["id"] == storage_id: @@ -389,3 +454,36 @@ def unlink(self, storage_id: str, project_id: str) -> Dict: result = self.saagie_api.client.execute(query=gql(GQL_UNLINK_STORAGE), variable_values={"id": storage_id}) logging.info("✅ Storage [%s] successfully unlinked", storage_id) return result + + def move(self, storage_id: str, target_platform_id: int, target_project_id: str): + """Move a storage to another project in the same platform or another one + + Parameters + ---------- + storage_id : str + UUID of your storage (see README on how to find it) + target_platform_id : int + Id of the platform to move the storage to + target_project_id : str + UUID of the project to move the storage to + + Returns + ------- + dict + Dict of the moved storage with its new id + + Examples + -------- + >>> saagie_api.storages.move(storage_id=storage_id, target_platform_id=1, target_project_id=project_id) + { + 'moveVolume': '29cf1b80-6b9c-47bc-a06c-c20897257097', + } + """ + return self.saagie_api.client.execute( + query=gql(GQL_MOVE_STORAGE), + variable_values={ + "volumeId": storage_id, + "targetPlatformId": target_platform_id, + "targetProjectId": target_project_id, + }, + ) diff --git a/tests/integration/pipelines_integration_test.py b/tests/integration/pipelines_integration_test.py index 7060465d..3db21a89 100644 --- a/tests/integration/pipelines_integration_test.py +++ b/tests/integration/pipelines_integration_test.py @@ -406,3 +406,14 @@ def test_get_info_by_name(create_global_project, create_then_delete_graph_pipeli result = conf.saagie_api.pipelines.get_info_by_name(pipeline_name="TEST_VIA_API", project_id=conf.project_id) assert result["graphPipelineByName"]["id"] == pipeline_id + + @staticmethod + def test_duplicate_pipeline(create_global_project, create_then_delete_graph_pipeline): + conf = create_global_project + pipeline_id, _ = create_then_delete_graph_pipeline + + result = conf.saagie_api.pipelines.duplicate(pipeline_id=pipeline_id) + + conf.saagie_api.pipelines.delete(pipeline_id=result["duplicatePipeline"]["id"]) + + assert "duplicatePipeline" in result diff --git a/tests/integration/storages_integration_test.py b/tests/integration/storages_integration_test.py index 871a18e1..664bc5de 100644 --- a/tests/integration/storages_integration_test.py +++ b/tests/integration/storages_integration_test.py @@ -1,3 +1,4 @@ +import time from datetime import datetime import pytest @@ -166,3 +167,59 @@ def test_unlink_used_storage(create_global_project): with pytest.raises(ValueError) as vale: conf.saagie_api.storages.unlink(storage_id, conf.project_id) assert str(vale.value).startswith(f"❌ Storage '{storage_id}' is currently used by an App. Unlink impossible.") + + @staticmethod + def test_move_storage(create_global_project, create_storage): + conf = create_global_project + + storage = create_storage + + res_proj = conf.saagie_api.projects.create( + name="test_move_storage", + group=conf.group, + role="Manager", + jobs_technologies_allowed={"saagie": ["python", "spark", "bash"]}, + description="test_move_storage", + ) + + project_id = res_proj["createProject"]["id"] + + # Waiting for the project to be ready + project_status = conf.saagie_api.projects.get_info(project_id=project_id)["project"]["status"] + waiting_time = 0 + + # Safety: wait for 5min max for project initialisation + project_creation_timeout = 400 + while project_status != "READY" and waiting_time <= project_creation_timeout: + time.sleep(10) + project_status = conf.saagie_api.projects.get_info(project_id)["project"]["status"] + waiting_time += 10 + if project_status != "READY": + raise TimeoutError( + f"Project creation is taking longer than usual, " + f"aborting integration tests after {project_creation_timeout} seconds" + ) + + result = conf.saagie_api.storages.move( + storage_id=storage["id"], + target_platform_id=1, + target_project_id=project_id, + ) + + res = conf.saagie_api.storages.get(storage_id=result["moveVolume"]) + + conf.saagie_api.projects.delete(project_id) + + # assert result == {"moveVolume": storage["id"]} + assert res["volume"]["name"] == storage["name"] + + @staticmethod + def test_get_storage_info(create_then_delete_storage, create_global_project): + conf = create_global_project + + storage = create_then_delete_storage + + result = conf.saagie_api.storages.get(storage_id=storage["id"]) + del result["volume"]["creationDate"] + + assert storage == result["volume"] diff --git a/tests/unit/pipelines_unit_test.py b/tests/unit/pipelines_unit_test.py index 6fce5880..0f3ceced 100644 --- a/tests/unit/pipelines_unit_test.py +++ b/tests/unit/pipelines_unit_test.py @@ -81,3 +81,7 @@ def test_count_deletable_instances_by_status(self): def test_count_deletable_instances_by_date(self): query = gql(GQL_COUNT_DELETABLE_PIPELINE_INSTANCE_BY_DATE) self.client.validate(query) + + def test_duplicate_pipeline(self): + query = gql(GQL_DUPLICATE_PIPELINE) + self.client.validate(query) diff --git a/tests/unit/resources/schema.graphqls b/tests/unit/resources/schema.graphqls index b009b802..779eb544 100644 --- a/tests/unit/resources/schema.graphqls +++ b/tests/unit/resources/schema.graphqls @@ -76,6 +76,11 @@ type Query { # Get app stats (uptime and downtime percentages and recovered count). appStats(appHistoryId: UUID!, versionNumber: Int!, startTime: DateTime!): AppStats + # Get an volume's details from its UUID. + # + # You can only get an volume's details if you have at least the viewer role in the project associated to the volume or in all projects. + volume(id: UUID!): Volume + # List environment variables in a given project. # # You can only list environment variables if you have at least the viewer role in this project or in all projects. @@ -603,6 +608,25 @@ type Mutation { # You can only delete a pipeline if you have the global Super-Admin role. deletePipeline(id: UUID!): Boolean! + # Duplicate a pipeline in the same project. Its jobs is not duplicated and only current version of pipeline is duplicated. + # + # You can only duplicate a pipeline if you have at least role editor on the project associated to the pipeline or on all projects. + duplicatePipeline(originalPipelineId: UUID!): Pipeline! + + # Move pipeline with its jobs from a project to another. The target can be in another platform. + # Also move all jobs associated with this pipeline. + # Move all versions of all jobs of pipeline and all versions of pipeline. + # + # You can only move pipeline if you have at least the editor role in both the source and target projects. + # You can only move pipeline if all its jobs (and all versions) is only in this pipeline and not other. + # All job's technologies must exist in the target project. + # + # **Fields validation:** + # - pipelineId: must exist, must be deletable, must not be in migration state. + # - targetPlatformId: must exist, must not be the same as the source project, must be ready. + # - targetProjectId: must not be the same as the source project, must exist and have the appropriate technology in the same category or target category if it is given. + movePipeline(pipelineId: UUID!, targetPlatformId: Int!, targetProjectId: UUID!): UUID! + # Add a new pipeline version in linear format to an existing pipeline. # # You can only add a version to a pipeline if you have at least the editor role in the project associated to the pipeline or in all projects. @@ -701,6 +725,19 @@ type Mutation { unlinkVolume(id: UUID!): Volume + # Move volume from a project to another. The target can be in another platform. + # + # You can only move volume if you have at least the editor role in both the source and target projects. + # + # **Fields validation:** + # - volumeId: must exist, must be link with an app + # - targetPlatformId: must exist, must not be the same as the source project, must be ready. + # - targetProjectId: must not be the same as the source project and must exist + # + # **Returns:** + # UUID of the volume moved in the target + moveVolume(volumeId: UUID!, targetPlatformId: Int!, targetProjectId: UUID!): UUID! + # Create a Connection related to a project. # **Fields validation:** # - name: must be unique (case insensitive) in the project, shorter than 255 characters and cannot be empty. @@ -1043,6 +1080,7 @@ input VolumeEditionInput { id: UUID! name: String description: String + size: DataSize } input EntryInput { @@ -1134,6 +1172,10 @@ type Volume { projectId: UUID! creationDate: DateTime! linkedApp: App + migrationStatus: MigrationStatus + migrationPlatformId: Int + migrationProjectId: UUID + migrationVolumeId: UUID } type Job { @@ -1212,6 +1254,7 @@ type AppHistory { startTime: DateTime stopTime: DateTime author: String + triggerType: TriggerType } type StatusHistoryByAction { @@ -1245,35 +1288,40 @@ type RunAction implements AppEvent { recordAt: DateTime! executionId: UUID! versionNumber: Int! - author: String! + author: String + triggerType: TriggerType } type RollbackAction implements AppEvent { recordAt: DateTime! executionId: UUID! versionNumber: Int! - author: String! + author: String + triggerType: TriggerType } type UpgradeAction implements AppEvent { recordAt: DateTime! executionId: UUID! versionNumber: Int! - author: String! + author: String + triggerType: TriggerType } type StopAction implements AppEvent { recordAt: DateTime! executionId: UUID! versionNumber: Int! - author: String! + author: String + triggerType: TriggerType } type RestartAction implements AppEvent { recordAt: DateTime! executionId: UUID! versionNumber: Int! - author: String! + author: String + triggerType: TriggerType } enum AppStatus { @@ -1408,6 +1456,12 @@ type JobInstance { sparkuiUrl: String runTrigger: Trigger stopTrigger: AuthoredTrigger + deletableState: JobInstanceDeletableState +} + +type JobInstanceDeletableState { + deletable: Boolean! + reasons: [JobInstanceDeletableStateReason!] } type InstanceHistory { @@ -1622,8 +1676,45 @@ type Pipeline { cronScheduling: Cron scheduleStatus: ScheduleStatus scheduleTimezone: TimeZone + migrationStatus: MigrationStatus + sourcesMigration: [MigrationData] + targetMigration: MigrationData # Indicates whether this pipeline version is a linear pipeline or not isLegacyPipeline: Boolean + # Preview: use with precaution this field, it can be removed or changed in the future + allVersionsJobTechnologiesByCategory: [TechnologyByCategory] + movableState: PipelineMovableState +} + +type MigrationData { + platformId: Int! + projectId: UUID! + migrationDate: DateTime! + pipelineId: UUID +} + +type PipelineMovableState { + movable: Boolean! + reasons: [PipelineMovableStateError!] +} + +type PipelineMovableStateError { + reason: PipelineMovableStateErrorReason! + offendingId: UUID + offendingName: String + sharedJobInManyPipelinesError: SharedJobInManyPipelines +} + +type SharedJobInManyPipelines { + jobId: UUID! + jobName: String! + otherPipelinesLinked: [OtherPipelineWithSharedJob!]! +} + +type OtherPipelineWithSharedJob { + pipelineId: UUID! + pipelineName: String! + pipelineVersions: [Int]! } type PipelineVersion { @@ -1997,6 +2088,8 @@ enum MigrationStatus { enum TriggerType { MANUAL_JOB, + MANUAL_APP, + EXPAND_VOLUME, CRON_JOB, CRON_PIPELINE, MANUAL_PIPELINE @@ -2010,6 +2103,13 @@ enum JobInstanceSelector { UNKNOWN } +enum JobInstanceDeletableStateReason { + INSTANCE_NOT_TERMINATED, + INSTANCE_NOT_YET_CLEANED, + INSTANCE_LINKED_TO_PIPELINE, + INSTANCE_DOES_NOT_MATCH_JOB +} + enum JobVersionDeletableStateReason { CURRENT_VERSION VERSION_HAS_UNFINISHED_INSTANCES @@ -2054,3 +2154,15 @@ enum PipelineVersionDeletableStateReason { VERSION_INSTANCES_NOT_YET_CLEANED } +enum PipelineMovableStateErrorReason { + PIPELINE_HAS_UNFINISHED_INSTANCES, + PIPELINE_HAS_INSTANCES_NOT_YET_CLEANED, + PIPELINE_ALREADY_IN_MIGRATION_STATE, + PIPELINE_SCHEDULED, + JOB_HAS_UNFINISHED_INSTANCES, + JOB_HAS_INSTANCES_NOT_YET_CLEANED, + JOB_ALREADY_IN_MIGRATION_STATE, + JOB_SCHEDULED, + JOBS_EXISTS_IN_MANY_PIPELINES +} + diff --git a/tests/unit/storages_unit_test.py b/tests/unit/storages_unit_test.py index 9b34f8b5..1d37601d 100644 --- a/tests/unit/storages_unit_test.py +++ b/tests/unit/storages_unit_test.py @@ -29,3 +29,11 @@ def test_delete_storage(self): def test_unlink_storage(self): query = gql(GQL_UNLINK_STORAGE) self.client.validate(query) + + def test_get_storage(self): + query = gql(GQL_GET_STORAGE_INFO) + self.client.validate(query) + + def test_move_storage(self): + query = gql(GQL_MOVE_STORAGE) + self.client.validate(query)