diff --git a/.gitignore b/.gitignore index fea46f2e..099135aa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ deploy.cfg .coverage test/test_temp/ lib/SampleService/SampleServiceImpl.py.bak* +kb-sdk \ No newline at end of file diff --git a/Makefile b/Makefile index 2ad6251a..93af0369 100644 --- a/Makefile +++ b/Makefile @@ -21,20 +21,20 @@ default: compile all: compile build build-startup-script build-executable-script build-test-script -compile: +compile: # Don't compile server automatically, overwrites fixes to error handling # Temporarily add the next line to the command line args if recompiliation is needed to add # methods. # --pysrvname $(SERVICE_CAPS).$(SERVICE_CAPS)Server \ - kb-sdk compile $(SPEC_FILE) \ + @PATH=${PWD}:${PATH} kb-sdk compile $(SPEC_FILE) \ --out $(LIB_DIR) \ --pyclname $(SERVICE_CAPS).$(SERVICE_CAPS)Client \ --dynservver release \ --pyimplname $(SERVICE_CAPS).$(SERVICE_CAPS)Impl; - - rm $(LIB_DIR)/$(SERVICE_CAPS)Server.py + @- rm $(LIB_DIR)/$(SERVICE_CAPS)Server.py - kb-sdk compile $(SPEC_FILE) \ + @PATH=${PWD}:${PATH} kb-sdk compile $(SPEC_FILE) \ --out . \ --html \ @@ -51,6 +51,13 @@ test-sdkless: clean: rm -rfv $(LBIN_DIR) +install-sdk: + scripts/install-sdk.sh + +format: + black lib + black test + # Managing development container orchestration host-start-dev-server: diff --git a/Pipfile.lock b/Pipfile.lock index 7ee9bc28..6e69865e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1049,4 +1049,4 @@ "version": "==3.7.0" } } -} +} \ No newline at end of file diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index a74cdbc1..c0a589d8 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,6 +2,7 @@ ## Unreleased +* Apply black to all python source in `lib` and `test` ## 0.2.1 diff --git a/SampleService.html b/SampleService.html index f14a2026..0b2f0d68 100644 --- a/SampleService.html +++ b/SampleService.html @@ -1 +1 @@ -SampleService
/*
*A KBase module: SampleService
*
*Handles creating, updating, retriving samples and linking data to samples.
*
*Note that usage of the administration flags will be logged by the service.
*/
moduleSampleService{

/*
*A boolean value, 0 for false, 1 for true.
*/
typedefintboolean;

/*
*A timestamp in epoch milliseconds.
*/
typedefinttimestamp;

/*
*A user's username.
*/
typedefstringuser;

/*
*A SampleNode ID. Must be unique within a Sample and be less than 255 characters.
*/
typedefstringnode_id;

/*
*The type of a sample node. One of:
*BioReplicate - a biological replicate. Always at the top of the sample tree.
*TechReplicate - a technical replicate.
*SubSample - a sub sample that is not a technical replicate.
*/
typedefstringsamplenode_type;

/*
*A Sample ID. Must be globally unique. Always assigned by the Sample service.
*/
typedefstringsample_id;

/*
*A link ID. Must be globally unique. Always assigned by the Sample service.
*Typically only of use to service admins.
*/
typedefstringlink_id;

/*
*A sample name. Must be less than 255 characters.
*/
typedefstringsample_name;

/*
*The version of a sample. Always > 0.
*/
typedefintversion;

/*
*A key in a metadata key/value pair. Less than 1000 unicode characters.
*/
typedefstringmetadata_key;

/*
*A key for a value associated with a piece of metadata. Less than 1000 unicode characters.
*Examples: units, value, species
*/
typedefstringmetadata_value_key;

/*
*A workspace type string.
*Specifies the workspace data type a single string in the format
*[module].[typename]:
*
*module - a string. The module name of the typespec containing the type.
*typename - a string. The name of the type as assigned by the typedef
*statement.
*
*Example: KBaseSets.SampleSet
*/
typedefstringws_type_string;

/*
*A metadata value, represented by a mapping of value keys to primitive values. An example for
*a location metadata key might be:
*{
*"name": "Castle Geyser",
*"lat": 44.463816,
*"long": -110.836471
*}
*"primitive values" means an int, float, string, or equivalent typedefs. Including any
*collection types is an error.
*/
typedefmapping<metadata_value_key,UnspecifiedObject>metadata_value;

/*
*Metadata attached to a sample.
*/
typedefmapping<metadata_key,metadata_value>metadata;

/*
*Information about a metadata key as it appeared at the data source.
*The source key and value represents the original state of the metadata before it was
*tranformed for ingestion by the sample service.
*
*key - the metadata key.
*skey - the key as it appeared at the data source.
*svalue - the value as it appeared at the data source.
*/
typedefstructure{
metadata_keykey;
metadata_keyskey;
metadata_valuesvalue;
}SourceMetadata;

/*
*A KBase Workspace service Unique Permanent Address (UPA). E.g. 5/6/7 where 5 is the
*workspace ID, 6 the object ID, and 7 the object version.
*/
typedefstringws_upa;

/*
*An id for a unit of data within a KBase Workspace object. A single object may contain
*many data units. A dataid is expected to be unique within a single object. Must be less
*than 255 characters.
*/
typedefstringdata_id;

/*
*A node in a sample tree.
*id - the ID of the node.
*parent - the id of the parent node for the current node. BioReplicate nodes, and only
*BioReplicate nodes, do not have a parent.
*type - the type of the node.
*meta_controlled - metadata restricted by the sample controlled vocabulary and validators.
*source_meta - the pre-transformation keys and values of the controlled metadata at the
*data source for controlled metadata keys. In some cases the source metadata
*may be transformed prior to ingestion by the Sample Service; the contents of this
*data structure allows for reconstructing the original representation. The metadata
*here is not validated other than basic size checks and is provided on an
*informational basis only. The metadata keys in the SourceMetadata data structure
*must be a subset of the meta_controlled mapping keys.
*meta_user - unrestricted metadata.
*/
typedefstructure{
node_idid;
node_idparent;
samplenode_typetype;
metadatameta_controlled;
list<SourceMetadata>source_meta;
metadatameta_user;
}SampleNode;

/*
*A Sample, consisting of a tree of subsamples and replicates.
*id - the ID of the sample.
*user - the user that saved the sample.
*node_tree - the tree(s) of sample nodes in the sample. The the roots of all trees must
*be BioReplicate nodes. All the BioReplicate nodes must be at the start of the list,
*and all child nodes must occur after their parents in the list.
*name - the name of the sample. Must be less than 255 characters.
*save_date - the date the sample version was saved.
*version - the version of the sample.
*/
typedefstructure{
sample_idid;
useruser;
list<SampleNode>node_tree;
sample_namename;
timestampsave_date;
versionversion;
}Sample;

/*
*Access control lists for a sample. Access levels include the privileges of the lower
*access levels.
*
*owner - the user that created and owns the sample.
*admin - users that can administrate (e.g. alter ACLs) the sample.
*write - users that can write (e.g. create a new version) to the sample.
*read - users that can view the sample.
*public_read - whether any user can read the sample, regardless of permissions.
*/
typedefstructure{
userowner;
list<user>admin;
list<user>write;
list<user>read;
booleanpublic_read;
}SampleACLs;

/*
*A Sample ID and version.
*id - the ID of the sample.
*version - the version of the sample.
*/
typedefstructure{
sample_idid;
versionversion;
}SampleAddress;

/*
*Parameters for creating a sample.
*If Sample.id is null, a new Sample is created along with a new ID.
*Otherwise, a new version of Sample.id is created. If Sample.id does not exist, an error
*is returned.
*Any incoming user, version or timestamp in the incoming sample is ignored.
*
*sample - the sample to save.
*prior_version - if non-null, ensures that no other sample version is saved between
*prior_version and the version that is created by this save. If this is not the case,
*the sample will fail to save.
*as_admin - run the method as a service administrator. The user must have full
*administration permissions.
*as_user - create the sample as a different user. Ignored if as_admin is not true. Neither
*the administrator nor the impersonated user need have permissions to the sample if a
*new version is saved.
*/
typedefstructure{
Samplesample;
intprior_version;
booleanas_admin;
useras_user;
}CreateSampleParams;

/*
*Create a new sample or a sample version.
*/
funcdefcreate_sample(CreateSampleParamsparams)returns(SampleAddressaddress)authenticationrequired;

/*
*get_sample parameters.
*id - the ID of the sample to retrieve.
*version - the version of the sample to retrieve, or the most recent sample if omitted.
*as_admin - get the sample regardless of ACLs as long as the user has administration read
*permissions.
*/
typedefstructure{
sample_idid;
versionversion;
booleanas_admin;
}GetSampleParams;

/*
*Get a sample. If the version is omitted the most recent sample is returned.
*/
funcdefget_sample(GetSampleParamsparams)returns(Samplesample)authenticationoptional;

typedefstructure{
sample_idid;
versionversion;
}SampleIdentifier;

typedefstructure{
list<SampleIdentifier>samples;
booleanas_admin;
}GetSamplesParams;

funcdefget_samples(GetSamplesParamsparams)returns(list<Sample>samples)authenticationoptional;

/*
*get_sample_acls parameters.
*id - the ID of the sample to retrieve.
*as_admin - get the sample acls regardless of ACL contents as long as the user has
*administration read permissions.
*/
typedefstructure{
sample_idid;
booleanas_admin;
}GetSampleACLsParams;

/*
*Get a sample's ACLs.
*/
funcdefget_sample_acls(GetSampleACLsParamsparams)returns(SampleACLsacls)authenticationoptional;

/*
*update_sample_acls parameters.
*
*id - the ID of the sample to modify.
*admin - a list of users that will receive admin privileges. Default none.
*write - a list of users that will receive write privileges. Default none.
*read - a list of users that will receive read privileges. Default none.
*remove - a list of users that will have all privileges removed. Default none.
*public_read - an integer that determines whether the sample will be set to publicly
*readable:
*> 0: public read.
*0: No change (the default).
*< 0: private.
*at_least - false, the default, indicates that the users should get the exact permissions
*as specified in the user lists, which may mean a reduction in permissions. If true,
*users that already exist in the sample ACLs will not have their permissions reduced
*as part of the ACL update unless they are in the remove list. E.g. if a user has
*write permissions and read permissions are specified in the update, no changes will
*be made to the user's permission.
*as_admin - update the sample acls regardless of sample ACL contents as long as the user has
*full service administration permissions.
*/
typedefstructure{
sample_idid;
list<user>admin;
list<user>write;
list<user>read;
list<user>remove;
intpublic_read;
booleanat_least;
booleanas_admin;
}UpdateSampleACLsParams;

/*
*Update a sample's ACLs.
*/
funcdefupdate_sample_acls(UpdateSampleACLsParamsparams)returns()authenticationrequired;

/*
*update_samples_acls parameters.
*
*These parameters are the same as update_sample_acls, except:
*ids - a list of IDs of samples to modify.
*/
typedefstructure{
list<sample_id>ids;
list<user>admin;
list<user>write;
list<user>read;
list<user>remove;
intpublic_read;
booleanat_least;
booleanas_admin;
}UpdateSamplesACLsParams;

/*
*Update the ACLs of many samples.
*/
funcdefupdate_samples_acls(UpdateSamplesACLsParamsparams)returns()authenticationrequired;

/*
*replace_sample_acls parameters.
*
*id - the ID of the sample to modify.
*acls - the ACLs to set on the sample.
*as_admin - replace the sample acls regardless of sample ACL contents as long as the user
*has full service administration permissions.
*/
typedefstructure{
sample_idid;
SampleACLsacls;
booleanas_admin;
}ReplaceSampleACLsParams;

/*
*Completely overwrite a sample's ACLs. Any current ACLs are replaced by the provided
*ACLs, even if empty, and gone forever.
*
*The sample owner cannot be changed via this method.
*/
funcdefreplace_sample_acls(ReplaceSampleACLsParamsparams)returns()authenticationrequired;

/*
*get_metadata_key_static_metadata parameters.
*
*keys - the list of metadata keys to interrogate.
*prefix -
*0 (the default) to interrogate standard metadata keys.
*1 to interrogate prefix metadata keys, but require an exact match to the prefix key.
*2 to interrogate prefix metadata keys, but any keys which are a prefix of the
*provided keys will be included in the results.
*/
typedefstructure{
list<metadata_key>keys;
intprefix;
}GetMetadataKeyStaticMetadataParams;

/*
*get_metadata_key_static_metadata results.
*
*static_metadata - the static metadata for the requested keys.
*/
typedefstructure{
metadatastatic_metadata;
}GetMetadataKeyStaticMetadataResults;

/*
*Get static metadata for one or more metadata keys.
*
*The static metadata for a metadata key is metadata *about* the key - e.g. it may
*define the key's semantics or denote that the key is linked to an ontological ID.
*
*The static metadata does not change without the service being restarted. Client caching is
*recommended to improve performance.
*/
funcdefget_metadata_key_static_metadata(GetMetadataKeyStaticMetadataParamsparams)returns(GetMetadataKeyStaticMetadataResultsresults)authenticationnone;

/*
*create_data_link parameters.
*
*upa - the workspace UPA of the object to be linked.
*dataid - the dataid of the data to be linked, if any, within the object. If omitted the
*entire object is linked to the sample.
*id - the sample id.
*version - the sample version.
*node - the sample node.
*update - if false (the default), fail if a link already exists from the data unit (the
*combination of the UPA and dataid). if true, expire the old link and create the new
*link unless the link is already to the requested sample node, in which case the
*operation is a no-op.
*as_admin - run the method as a service administrator. The user must have full
*administration permissions.
*as_user - create the link as a different user. Ignored if as_admin is not true. Neither
*the administrator nor the impersonated user need have permissions to the data or
*sample.
*/
typedefstructure{
ws_upaupa;
data_iddataid;
sample_idid;
versionversion;
node_idnode;
booleanupdate;
booleanas_admin;
useras_user;
}CreateDataLinkParams;

/*
*A data link from a KBase workspace object to a sample.
*
*upa - the workspace UPA of the linked object.
*dataid - the dataid of the linked data, if any, within the object. If omitted the
*entire object is linked to the sample.
*id - the sample id.
*version - the sample version.
*node - the sample node.
*createdby - the user that created the link.
*created - the time the link was created.
*expiredby - the user that expired the link, if any.
*expired - the time the link was expired, if at all.
*/
typedefstructure{
link_idlinkid;
ws_upaupa;
data_iddataid;
sample_idid;
versionversion;
node_idnode;
usercreatedby;
timestampcreated;
userexpiredby;
timestampexpired;
}DataLink;

/*
*create_data_link results.
*
*new_link - the new link.
*/
typedefstructure{
DataLinknew_link;
}CreateDataLinkResults;

/*
*Create a link from a KBase Workspace object to a sample.
*
*The user must have admin permissions for the sample and write permissions for the
*Workspace object.
*/
funcdefcreate_data_link(CreateDataLinkParamsparams)returns(CreateDataLinkResultsresults)authenticationrequired;

/*
*propagate_data_links parameters.
*
*id - the sample id.
*version - the sample version. (data links are propagated to)
*previous_version - the previouse sample version. (data links are propagated from)
*ignore_types - the workspace data type ignored from propagating. default empty.
*update - if false (the default), fail if a link already exists from the data unit (the
*combination of the UPA and dataid). if true, expire the old link and create the new
*link unless the link is already to the requested sample node, in which case the
*operation is a no-op.
*effective_time - the effective time at which the query should be run - the default is
*the current time. Providing a time allows for reproducibility of previous results.
*as_admin - run the method as a service administrator. The user must have full
*administration permissions.
*as_user - create the link as a different user. Ignored if as_admin is not true. Neither
*the administrator nor the impersonated user need have permissions to the data or
*sample.
*/
typedefstructure{
sample_idid;
versionversion;
versionprevious_version;
list<ws_type_string>ignore_types;
booleanupdate;
timestampeffective_time;
booleanas_admin;
useras_user;
}PropagateDataLinkParams;

/*
*propagate_data_links results.
*
*links - the links.
*/
typedefstructure{
list<DataLink>links;
}PropagateDataLinkResults;

/*
*Propagates data links from a previous sample to the current (latest) version
*
*The user must have admin permissions for the sample and write permissions for the
*Workspace object.
*/
funcdefpropagate_data_links(PropagateDataLinkParamsparams)returns(PropagateDataLinkResultsresults)authenticationrequired;

/*
*expire_data_link parameters.
*
*upa - the workspace upa of the object from which the link originates.
*dataid - the dataid, if any, of the data within the object from which the link originates.
*Omit for links where the link is from the entire object.
*as_admin - run the method as a service administrator. The user must have full
*administration permissions.
*as_user - expire the link as a different user. Ignored if as_admin is not true. Neither
*the administrator nor the impersonated user need have permissions to the link if a
*new version is saved.
*/
typedefstructure{
ws_upaupa;
data_iddataid;
booleanas_admin;
useras_user;
}ExpireDataLinkParams;

/*
*Expire a link from a KBase Workspace object.
*
*The user must have admin permissions for the sample and write permissions for the
*Workspace object.
*/
funcdefexpire_data_link(ExpireDataLinkParamsparams)returns()authenticationrequired;

/*
*get_data_links_from_sample parameters.
*
*id - the sample ID.
*version - the sample version.
*effective_time - the effective time at which the query should be run - the default is
*the current time. Providing a time allows for reproducibility of previous results.
*as_admin - run the method as a service administrator. The user must have read
*administration permissions.
*/
typedefstructure{
sample_idid;
versionversion;
timestampeffective_time;
booleanas_admin;
}GetDataLinksFromSampleParams;

/*
*get_data_links_from_sample results.
*
*links - the links.
*effective_time - the time at which the query was run. This timestamp, if saved, can be
*used when running the method again to ensure reproducible results. Note that changes
*to workspace permissions may cause results to change over time.
*/
typedefstructure{
list<DataLink>links;
timestampeffective_time;
}GetDataLinksFromSampleResults;

/*
*Get data links to Workspace objects originating from a sample.
*
*The user must have read permissions to the sample. Only Workspace objects the user
*can read are returned.
*/
funcdefget_data_links_from_sample(GetDataLinksFromSampleParamsparams)returns(GetDataLinksFromSampleResultsresults)authenticationoptional;

/*
*get_data_links_from_sample_set parameters.
*sample_ids - a list of sample ids and versions
*effective_time - the time at which the query was run. This timestamp, if saved, can be
*used when running the method again to enqure reproducible results. Note that changes
*to workspace permissions may cause results to change over time.
*as_admin - run the method as a service administrator. The user must have read
*administration permissions.
*/
typedefstructure{
list<SampleIdentifier>sample_ids;
timestampeffective_time;
booleanas_admin;
}GetDataLinksFromSampleSetParams;

/*
*Get all workspace object metadata linked to samples in a list of samples or sample set
*refs. Returns metadata about links to data objects. A batch version of
*get_data_links_from_sample.
*
*The user must have read permissions to the sample. A permissions error is thrown when a
*sample is found that the user has no access to.
*/
funcdefget_data_links_from_sample_set(GetDataLinksFromSampleSetParamsparams)returns(GetDataLinksFromSampleResultsresults)authenticationoptional;

/*
*get_data_links_from_data parameters.
*
*upa - the data UPA.
*effective_time - the effective time at which the query should be run - the default is
*the current time. Providing a time allows for reproducibility of previous results.
*as_admin - run the method as a service administrator. The user must have read
*administration permissions.
*/
typedefstructure{
ws_upaupa;
timestampeffective_time;
booleanas_admin;
}GetDataLinksFromDataParams;

/*
*get_data_links_from_data results.
*
*links - the links.
*effective_time - the time at which the query was run. This timestamp, if saved, can be
*used when running the method again to ensure reproducible results.
*/
typedefstructure{
list<DataLink>links;
timestampeffective_time;
}GetDataLinksFromDataResults;

/*
*Get data links to samples originating from Workspace data.
*
*The user must have read permissions to the workspace data.
*/
funcdefget_data_links_from_data(GetDataLinksFromDataParamsparams)returns(GetDataLinksFromDataResultsresults)authenticationoptional;

/*
*get_sample_via_data parameters.
*
*upa - the workspace UPA of the target object.
*id - the target sample id.
*version - the target sample version.
*/
typedefstructure{
ws_upaupa;
sample_idid;
versionversion;
}GetSampleViaDataParams;

/*
*Get a sample via a workspace object. Read permissions to a workspace object grants
*read permissions to all versions of any linked samples, whether the links are expired or
*not. This method allows for fetching samples when the user does not have explicit
*read access to the sample.
*/
funcdefget_sample_via_data(GetSampleViaDataParamsparams)returns(Samplesample)authenticationoptional;

/*
*get_data_link parameters.
*
*linkid - the link ID.
*/
typedefstructure{
link_idlinkid;
}GetDataLinkParams;

/*
*Get a link, expired or not, by its ID. This method requires read administration privileges
*for the service.
*/
funcdefget_data_link(GetDataLinkParamsparams)returns(DataLinklink)authenticationrequired;

/*
*Provide sample and run through the validation steps, but without saving them. Allows all the samples to be evaluated for validity first so potential errors can be addressed.
*/
typedefstructure{
list<Sample>samples;
}ValidateSamplesParams;

typedefstructure{
stringmessage;
stringdev_message;
sample_namesample_name;
node_idnode;
metadata_keykey;
stringsubkey;
}ValidateSamplesError;

typedefstructure{
list<ValidateSamplesError>errors;
}ValidateSamplesResults;

funcdefvalidate_samples(ValidateSamplesParamsparams)returns(ValidateSamplesResultsresults)authenticationrequired;
};

Function Index

create_data_link
create_sample
expire_data_link
get_data_link
get_data_links_from_data
get_data_links_from_sample
get_data_links_from_sample_set
get_metadata_key_static_metadata
get_sample
get_sample_acls
get_sample_via_data
get_samples
propagate_data_links
replace_sample_acls
update_sample_acls
update_samples_acls
validate_samples

Type Index

boolean
CreateDataLinkParams
CreateDataLinkResults
CreateSampleParams
data_id
DataLink
ExpireDataLinkParams
GetDataLinkParams
GetDataLinksFromDataParams
GetDataLinksFromDataResults
GetDataLinksFromSampleParams
GetDataLinksFromSampleResults
GetDataLinksFromSampleSetParams
GetMetadataKeyStaticMetadataParams
GetMetadataKeyStaticMetadataResults
GetSampleACLsParams
GetSampleParams
GetSamplesParams
GetSampleViaDataParams
link_id
metadata
metadata_key
metadata_value
metadata_value_key
node_id
PropagateDataLinkParams
PropagateDataLinkResults
ReplaceSampleACLsParams
Sample
sample_id
sample_name
SampleACLs
SampleAddress
SampleIdentifier
SampleNode
samplenode_type
SourceMetadata
timestamp
UpdateSampleACLsParams
UpdateSamplesACLsParams
user
ValidateSamplesError
ValidateSamplesParams
ValidateSamplesResults
version
ws_type_string
ws_upa
\ No newline at end of file +SampleService
/*
*A KBase module: SampleService
*
*Handles creating, updating, retriving samples and linking data to samples.
*
*Note that usage of the administration flags will be logged by the service.
*/
moduleSampleService{

/*
*A boolean value, 0 for false, 1 for true.
*/
typedefintboolean;

/*
*A timestamp in epoch milliseconds.
*/
typedefinttimestamp;

/*
*A user's username.
*/
typedefstringuser;

/*
*A SampleNode ID. Must be unique within a Sample and be less than 255 characters.
*/
typedefstringnode_id;

/*
*The type of a sample node. One of:
*BioReplicate - a biological replicate. Always at the top of the sample tree.
*TechReplicate - a technical replicate.
*SubSample - a sub sample that is not a technical replicate.
*/
typedefstringsamplenode_type;

/*
*A Sample ID. Must be globally unique. Always assigned by the Sample service.
*/
typedefstringsample_id;

/*
*A link ID. Must be globally unique. Always assigned by the Sample service.
*Typically only of use to service admins.
*/
typedefstringlink_id;

/*
*A sample name. Must be less than 255 characters.
*/
typedefstringsample_name;

/*
*The version of a sample. Always > 0.
*/
typedefintversion;

/*
*A key in a metadata key/value pair. Less than 1000 unicode characters.
*/
typedefstringmetadata_key;

/*
*A key for a value associated with a piece of metadata. Less than 1000 unicode characters.
*Examples: units, value, species
*/
typedefstringmetadata_value_key;

/*
*A workspace type string.
*Specifies the workspace data type a single string in the format
*[module].[typename]:
*
*module - a string. The module name of the typespec containing the type.
*typename - a string. The name of the type as assigned by the typedef
*statement.
*
*Example: KBaseSets.SampleSet
*/
typedefstringws_type_string;

/*
*A metadata value, represented by a mapping of value keys to primitive values. An example for
*a location metadata key might be:
*{
*"name": "Castle Geyser",
*"lat": 44.463816,
*"long": -110.836471
*}
*"primitive values" means an int, float, string, or equivalent typedefs. Including any
*collection types is an error.
*/
typedefmapping<metadata_value_key,UnspecifiedObject>metadata_value;

/*
*Metadata attached to a sample.
*/
typedefmapping<metadata_key,metadata_value>metadata;

/*
*Information about a metadata key as it appeared at the data source.
*The source key and value represents the original state of the metadata before it was
*tranformed for ingestion by the sample service.
*
*key - the metadata key.
*skey - the key as it appeared at the data source.
*svalue - the value as it appeared at the data source.
*/
typedefstructure{
metadata_keykey;
metadata_keyskey;
metadata_valuesvalue;
}SourceMetadata;

/*
*A KBase Workspace service Unique Permanent Address (UPA). E.g. 5/6/7 where 5 is the
*workspace ID, 6 the object ID, and 7 the object version.
*/
typedefstringws_upa;

/*
*An id for a unit of data within a KBase Workspace object. A single object may contain
*many data units. A dataid is expected to be unique within a single object. Must be less
*than 255 characters.
*/
typedefstringdata_id;

/*
*A node in a sample tree.
*id - the ID of the node.
*parent - the id of the parent node for the current node. BioReplicate nodes, and only
*BioReplicate nodes, do not have a parent.
*type - the type of the node.
*meta_controlled - metadata restricted by the sample controlled vocabulary and validators.
*source_meta - the pre-transformation keys and values of the controlled metadata at the
*data source for controlled metadata keys. In some cases the source metadata
*may be transformed prior to ingestion by the Sample Service; the contents of this
*data structure allows for reconstructing the original representation. The metadata
*here is not validated other than basic size checks and is provided on an
*informational basis only. The metadata keys in the SourceMetadata data structure
*must be a subset of the meta_controlled mapping keys.
*meta_user - unrestricted metadata.
*/
typedefstructure{
node_idid;
node_idparent;
samplenode_typetype;
metadatameta_controlled;
list<SourceMetadata>source_meta;
metadatameta_user;
}SampleNode;

/*
*A Sample, consisting of a tree of subsamples and replicates.
*id - the ID of the sample.
*user - the user that saved the sample.
*node_tree - the tree(s) of sample nodes in the sample. The the roots of all trees must
*be BioReplicate nodes. All the BioReplicate nodes must be at the start of the list,
*and all child nodes must occur after their parents in the list.
*name - the name of the sample. Must be less than 255 characters.
*save_date - the date the sample version was saved.
*version - the version of the sample.
*/
typedefstructure{
sample_idid;
useruser;
list<SampleNode>node_tree;
sample_namename;
timestampsave_date;
versionversion;
}Sample;

/*
*Access control lists for a sample. Access levels include the privileges of the lower
*access levels.
*
*owner - the user that created and owns the sample.
*admin - users that can administrate (e.g. alter ACLs) the sample.
*write - users that can write (e.g. create a new version) to the sample.
*read - users that can view the sample.
*public_read - whether any user can read the sample, regardless of permissions.
*/
typedefstructure{
userowner;
list<user>admin;
list<user>write;
list<user>read;
booleanpublic_read;
}SampleACLs;

/*
*A Sample ID and version.
*id - the ID of the sample.
*version - the version of the sample.
*/
typedefstructure{
sample_idid;
versionversion;
}SampleAddress;

/*
*Parameters for creating a sample.
*If Sample.id is null, a new Sample is created along with a new ID.
*Otherwise, a new version of Sample.id is created. If Sample.id does not exist, an error
*is returned.
*Any incoming user, version or timestamp in the incoming sample is ignored.
*
*sample - the sample to save.
*prior_version - if non-null, ensures that no other sample version is saved between
*prior_version and the version that is created by this save. If this is not the case,
*the sample will fail to save.
*as_admin - run the method as a service administrator. The user must have full
*administration permissions.
*as_user - create the sample as a different user. Ignored if as_admin is not true. Neither
*the administrator nor the impersonated user need have permissions to the sample if a
*new version is saved.
*/
typedefstructure{
Samplesample;
intprior_version;
booleanas_admin;
useras_user;
}CreateSampleParams;

/*
*Create a new sample or a sample version.
*/
funcdefcreate_sample(CreateSampleParamsparams)returns(SampleAddressaddress)authenticationrequired;

/*
*get_sample parameters.
*id - the ID of the sample to retrieve.
*version - the version of the sample to retrieve, or the most recent sample if omitted.
*as_admin - get the sample regardless of ACLs as long as the user has administration read
*permissions.
*/
typedefstructure{
sample_idid;
versionversion;
booleanas_admin;
}GetSampleParams;

/*
*Get a sample. If the version is omitted the most recent sample is returned.
*/
funcdefget_sample(GetSampleParamsparams)returns(Samplesample)authenticationoptional;

typedefstructure{
sample_idid;
versionversion;
}SampleIdentifier;

typedefstructure{
list<SampleIdentifier>samples;
booleanas_admin;
}GetSamplesParams;

funcdefget_samples(GetSamplesParamsparams)returns(list<Sample>samples)authenticationoptional;

/*
*get_sample_acls parameters.
*id - the ID of the sample to retrieve.
*as_admin - get the sample acls regardless of ACL contents as long as the user has
*administration read permissions.
*/
typedefstructure{
sample_idid;
booleanas_admin;
}GetSampleACLsParams;

/*
*Get a sample's ACLs.
*/
funcdefget_sample_acls(GetSampleACLsParamsparams)returns(SampleACLsacls)authenticationoptional;

/*
*update_sample_acls parameters.
*
*id - the ID of the sample to modify.
*admin - a list of users that will receive admin privileges. Default none.
*write - a list of users that will receive write privileges. Default none.
*read - a list of users that will receive read privileges. Default none.
*remove - a list of users that will have all privileges removed. Default none.
*public_read - an integer that determines whether the sample will be set to publicly
*readable:
*> 0: public read.
*0: No change (the default).
*< 0: private.
*at_least - false, the default, indicates that the users should get the exact permissions
*as specified in the user lists, which may mean a reduction in permissions. If true,
*users that already exist in the sample ACLs will not have their permissions reduced
*as part of the ACL update unless they are in the remove list. E.g. if a user has
*write permissions and read permissions are specified in the update, no changes will
*be made to the user's permission.
*as_admin - update the sample acls regardless of sample ACL contents as long as the user has
*full service administration permissions.
*/
typedefstructure{
sample_idid;
list<user>admin;
list<user>write;
list<user>read;
list<user>remove;
intpublic_read;
booleanat_least;
booleanas_admin;
}UpdateSampleACLsParams;

/*
*Update a sample's ACLs.
*/
funcdefupdate_sample_acls(UpdateSampleACLsParamsparams)returns()authenticationrequired;

/*
*update_samples_acls parameters.
*
*These parameters are the same as update_sample_acls, except:
*ids - a list of IDs of samples to modify.
*/
typedefstructure{
list<sample_id>ids;
list<user>admin;
list<user>write;
list<user>read;
list<user>remove;
intpublic_read;
booleanat_least;
booleanas_admin;
}UpdateSamplesACLsParams;

/*
*Update the ACLs of many samples.
*/
funcdefupdate_samples_acls(UpdateSamplesACLsParamsparams)returns()authenticationrequired;

/*
*replace_sample_acls parameters.
*
*id - the ID of the sample to modify.
*acls - the ACLs to set on the sample.
*as_admin - replace the sample acls regardless of sample ACL contents as long as the user
*has full service administration permissions.
*/
typedefstructure{
sample_idid;
SampleACLsacls;
booleanas_admin;
}ReplaceSampleACLsParams;

/*
*Completely overwrite a sample's ACLs. Any current ACLs are replaced by the provided
*ACLs, even if empty, and gone forever.
*
*The sample owner cannot be changed via this method.
*/
funcdefreplace_sample_acls(ReplaceSampleACLsParamsparams)returns()authenticationrequired;

/*
*get_metadata_key_static_metadata parameters.
*
*keys - the list of metadata keys to interrogate.
*prefix -
*0 (the default) to interrogate standard metadata keys.
*1 to interrogate prefix metadata keys, but require an exact match to the prefix key.
*2 to interrogate prefix metadata keys, but any keys which are a prefix of the
*provided keys will be included in the results.
*/
typedefstructure{
list<metadata_key>keys;
intprefix;
}GetMetadataKeyStaticMetadataParams;

/*
*get_metadata_key_static_metadata results.
*
*static_metadata - the static metadata for the requested keys.
*/
typedefstructure{
metadatastatic_metadata;
}GetMetadataKeyStaticMetadataResults;

/*
*Get static metadata for one or more metadata keys.
*
*The static metadata for a metadata key is metadata *about* the key - e.g. it may
*define the key's semantics or denote that the key is linked to an ontological ID.
*
*The static metadata does not change without the service being restarted. Client caching is
*recommended to improve performance.
*/
funcdefget_metadata_key_static_metadata(GetMetadataKeyStaticMetadataParamsparams)returns(GetMetadataKeyStaticMetadataResultsresults)authenticationnone;

/*
*create_data_link parameters.
*
*upa - the workspace UPA of the object to be linked.
*dataid - the dataid of the data to be linked, if any, within the object. If omitted the
*entire object is linked to the sample.
*id - the sample id.
*version - the sample version.
*node - the sample node.
*update - if false (the default), fail if a link already exists from the data unit (the
*combination of the UPA and dataid). if true, expire the old link and create the new
*link unless the link is already to the requested sample node, in which case the
*operation is a no-op.
*as_admin - run the method as a service administrator. The user must have full
*administration permissions.
*as_user - create the link as a different user. Ignored if as_admin is not true. Neither
*the administrator nor the impersonated user need have permissions to the data or
*sample.
*/
typedefstructure{
ws_upaupa;
data_iddataid;
sample_idid;
versionversion;
node_idnode;
booleanupdate;
booleanas_admin;
useras_user;
}CreateDataLinkParams;

/*
*A data link from a KBase workspace object to a sample.
*
*upa - the workspace UPA of the linked object.
*dataid - the dataid of the linked data, if any, within the object. If omitted the
*entire object is linked to the sample.
*id - the sample id.
*version - the sample version.
*node - the sample node.
*createdby - the user that created the link.
*created - the time the link was created.
*expiredby - the user that expired the link, if any.
*expired - the time the link was expired, if at all.
*/
typedefstructure{
link_idlinkid;
ws_upaupa;
data_iddataid;
sample_idid;
versionversion;
node_idnode;
usercreatedby;
timestampcreated;
userexpiredby;
timestampexpired;
}DataLink;

/*
*create_data_link results.
*
*new_link - the new link.
*/
typedefstructure{
DataLinknew_link;
}CreateDataLinkResults;

/*
*Create a link from a KBase Workspace object to a sample.
*
*The user must have admin permissions for the sample and write permissions for the
*Workspace object.
*/
funcdefcreate_data_link(CreateDataLinkParamsparams)returns(CreateDataLinkResultsresults)authenticationrequired;

/*
*propagate_data_links parameters.
*
*id - the sample id.
*version - the sample version. (data links are propagated to)
*previous_version - the previouse sample version. (data links are propagated from)
*ignore_types - the workspace data type ignored from propagating. default empty.
*update - if false (the default), fail if a link already exists from the data unit (the
*combination of the UPA and dataid). if true, expire the old link and create the new
*link unless the link is already to the requested sample node, in which case the
*operation is a no-op.
*effective_time - the effective time at which the query should be run - the default is
*the current time. Providing a time allows for reproducibility of previous results.
*as_admin - run the method as a service administrator. The user must have full
*administration permissions.
*as_user - create the link as a different user. Ignored if as_admin is not true. Neither
*the administrator nor the impersonated user need have permissions to the data or
*sample.
*/
typedefstructure{
sample_idid;
versionversion;
versionprevious_version;
list<ws_type_string>ignore_types;
booleanupdate;
timestampeffective_time;
booleanas_admin;
useras_user;
}PropagateDataLinkParams;

/*
*propagate_data_links results.
*
*links - the links.
*/
typedefstructure{
list<DataLink>links;
}PropagateDataLinkResults;

/*
*Propagates data links from a previous sample to the current (latest) version
*
*The user must have admin permissions for the sample and write permissions for the
*Workspace object.
*/
funcdefpropagate_data_links(PropagateDataLinkParamsparams)returns(PropagateDataLinkResultsresults)authenticationrequired;

/*
*expire_data_link parameters.
*
*upa - the workspace upa of the object from which the link originates.
*dataid - the dataid, if any, of the data within the object from which the link originates.
*Omit for links where the link is from the entire object.
*as_admin - run the method as a service administrator. The user must have full
*administration permissions.
*as_user - expire the link as a different user. Ignored if as_admin is not true. Neither
*the administrator nor the impersonated user need have permissions to the link if a
*new version is saved.
*/
typedefstructure{
ws_upaupa;
data_iddataid;
booleanas_admin;
useras_user;
}ExpireDataLinkParams;

/*
*Expire a link from a KBase Workspace object.
*
*The user must have admin permissions for the sample and write permissions for the
*Workspace object.
*/
funcdefexpire_data_link(ExpireDataLinkParamsparams)returns()authenticationrequired;

/*
*get_data_links_from_sample parameters.
*
*id - the sample ID.
*version - the sample version.
*effective_time - the effective time at which the query should be run - the default is
*the current time. Providing a time allows for reproducibility of previous results.
*as_admin - run the method as a service administrator. The user must have read
*administration permissions.
*/
typedefstructure{
sample_idid;
versionversion;
timestampeffective_time;
booleanas_admin;
}GetDataLinksFromSampleParams;

/*
*get_data_links_from_sample results.
*
*links - the links.
*effective_time - the time at which the query was run. This timestamp, if saved, can be
*used when running the method again to ensure reproducible results. Note that changes
*to workspace permissions may cause results to change over time.
*/
typedefstructure{
list<DataLink>links;
timestampeffective_time;
}GetDataLinksFromSampleResults;

/*
*Get data links to Workspace objects originating from a sample.
*
*The user must have read permissions to the sample. Only Workspace objects the user
*can read are returned.
*/
funcdefget_data_links_from_sample(GetDataLinksFromSampleParamsparams)returns(GetDataLinksFromSampleResultsresults)authenticationoptional;

/*
*get_data_links_from_sample_set parameters.
*sample_ids - a list of sample ids and versions
*effective_time - the time at which the query was run. This timestamp, if saved, can be
*used when running the method again to enqure reproducible results. Note that changes
*to workspace permissions may cause results to change over time.
*as_admin - run the method as a service administrator. The user must have read
*administration permissions.
*/
typedefstructure{
list<SampleIdentifier>sample_ids;
timestampeffective_time;
booleanas_admin;
}GetDataLinksFromSampleSetParams;

/*
*Get all workspace object metadata linked to samples in a list of samples or sample set
*refs. Returns metadata about links to data objects. A batch version of
*get_data_links_from_sample.
*
*The user must have read permissions to the sample. A permissions error is thrown when a
*sample is found that the user has no access to.
*/
funcdefget_data_links_from_sample_set(GetDataLinksFromSampleSetParamsparams)returns(GetDataLinksFromSampleResultsresults)authenticationoptional;

/*
*get_data_links_from_data parameters.
*
*upa - the data UPA.
*effective_time - the effective time at which the query should be run - the default is
*the current time. Providing a time allows for reproducibility of previous results.
*as_admin - run the method as a service administrator. The user must have read
*administration permissions.
*/
typedefstructure{
ws_upaupa;
timestampeffective_time;
booleanas_admin;
}GetDataLinksFromDataParams;

/*
*get_data_links_from_data results.
*
*links - the links.
*effective_time - the time at which the query was run. This timestamp, if saved, can be
*used when running the method again to ensure reproducible results.
*/
typedefstructure{
list<DataLink>links;
timestampeffective_time;
}GetDataLinksFromDataResults;

/*
*Get data links to samples originating from Workspace data.
*
*The user must have read permissions to the workspace data.
*/
funcdefget_data_links_from_data(GetDataLinksFromDataParamsparams)returns(GetDataLinksFromDataResultsresults)authenticationoptional;

/*
*get_sample_via_data parameters.
*
*upa - the workspace UPA of the target object.
*id - the target sample id.
*version - the target sample version.
*/
typedefstructure{
ws_upaupa;
sample_idid;
versionversion;
}GetSampleViaDataParams;

/*
*Get a sample via a workspace object. Read permissions to a workspace object grants
*read permissions to all versions of any linked samples, whether the links are expired or
*not. This method allows for fetching samples when the user does not have explicit
*read access to the sample.
*/
funcdefget_sample_via_data(GetSampleViaDataParamsparams)returns(Samplesample)authenticationoptional;

/*
*get_data_link parameters.
*
*linkid - the link ID.
*/
typedefstructure{
link_idlinkid;
}GetDataLinkParams;

/*
*Get a link, expired or not, by its ID. This method requires read administration privileges
*for the service.
*/
funcdefget_data_link(GetDataLinkParamsparams)returns(DataLinklink)authenticationrequired;

/*
*Provide sample and run through the validation steps, but without saving them. Allows all the samples to be evaluated for validity first so potential errors can be addressed.
*/
typedefstructure{
list<Sample>samples;
}ValidateSamplesParams;

typedefstructure{
stringmessage;
stringdev_message;
sample_namesample_name;
node_idnode;
metadata_keykey;
stringsubkey;
}ValidateSamplesError;

typedefstructure{
list<ValidateSamplesError>errors;
}ValidateSamplesResults;

funcdefvalidate_samples(ValidateSamplesParamsparams)returns(ValidateSamplesResultsresults)authenticationrequired;
};

Function Index

create_data_link
create_sample
expire_data_link
get_data_link
get_data_links_from_data
get_data_links_from_sample
get_data_links_from_sample_set
get_metadata_key_static_metadata
get_sample
get_sample_acls
get_sample_via_data
get_samples
propagate_data_links
replace_sample_acls
update_sample_acls
update_samples_acls
validate_samples

Type Index

boolean
CreateDataLinkParams
CreateDataLinkResults
CreateSampleParams
data_id
DataLink
ExpireDataLinkParams
GetDataLinkParams
GetDataLinksFromDataParams
GetDataLinksFromDataResults
GetDataLinksFromSampleParams
GetDataLinksFromSampleResults
GetDataLinksFromSampleSetParams
GetMetadataKeyStaticMetadataParams
GetMetadataKeyStaticMetadataResults
GetSampleACLsParams
GetSampleParams
GetSamplesParams
GetSampleViaDataParams
link_id
metadata
metadata_key
metadata_value
metadata_value_key
node_id
PropagateDataLinkParams
PropagateDataLinkResults
ReplaceSampleACLsParams
Sample
sample_id
sample_name
SampleACLs
SampleAddress
SampleIdentifier
SampleNode
samplenode_type
SourceMetadata
timestamp
UpdateSampleACLsParams
UpdateSamplesACLsParams
user
ValidateSamplesError
ValidateSamplesParams
ValidateSamplesResults
version
ws_type_string
ws_upa
diff --git a/SampleService.spec b/SampleService.spec index 3e5a28be..63bf697b 100644 --- a/SampleService.spec +++ b/SampleService.spec @@ -339,8 +339,8 @@ module SampleService { The static metadata for a metadata key is metadata *about* the key - e.g. it may define the key's semantics or denote that the key is linked to an ontological ID. - The static metadata does not change without the service being restarted. Client caching is - recommended to improve performance. + The static metadata does not change without the service being restarted. + Client caching is recommended to improve performance. */ funcdef get_metadata_key_static_metadata(GetMetadataKeyStaticMetadataParams params) diff --git a/docs/using-kb-sdk.md b/docs/using-kb-sdk.md new file mode 100644 index 00000000..ca60fd91 --- /dev/null +++ b/docs/using-kb-sdk.md @@ -0,0 +1,57 @@ +# Using `kb_sdk` with this repo + +The `sample_service` is based on the code layout and structure established by initializing and managing a codebase with the [KBase App SDK (`kb_sdk`)](https://github.com/kbase/kb_sdk). + +It does not adhere strictly to `kb_sdk` practices. + +This document describes, in a nutshell, current practice. + +> Note that the "KBase App SDK" is known, interchangeably, as `kb_sdk`, which is the name of the repo, and `kb-sdk`, which is the name of the command and the image. + +## Differences + +### Compilation + +This is a Python project, so there is no real code compilation. However, the "compile" command, as configured in the project's `Makefile` and implemented in `kb-sdk`, does more than compile a project which requires compilation. It also generates code. + +Typically, a `kb_sdk` service is maintained by updating the "spec file" (`SampleService.spec`) and then running `make compile`. This would: + +1. create core utility library files `lib/SampleService/authclient.py`, `lib/SampleService/baseclient.py` +2. add, modify, or remove method stubs to `lib/SampleService/SampleServiceImpl.py`, depending on changes to the spec file +3. create the main entrypoint `lib/SampleServiceServer.py`, overwriting the existing file if any +4. create the spec documentation page `SampleService.html`, overwriting the existing file if any + +In this repo, (3) is omitted. See the `Makefile` for how this was done. The Server file needs to be maintained by hand. + +The main server entrypoint is `lib/SampleService/SampleServiceServer.py`, rather than `lib/SampleServiceServer.py`. + +### Testing + +Typically a `kb-sdk` project runs tests through `kb-sdk`, which manages invocation of the tests via a container using an image built from the project's `Dockerfile` and utilizes conventions for directories, configuration, etc. + +In this project, the limitations of that test environment preclude using that test framework. + +Rather, tests are run via `make test-sdkless`. + +## Using `kb-sdk` + +The `kb-sdk` command is made available through a docker container, which uses an image based on the latest release of `kb_sdk`. + +You can consult the [KBase SDK Docs](https://kbase.github.io/kb_sdk_docs/tutorial/2_install.html) for the official installation procedure, or follow these somewhat simpler and less intrusive instructions: + +- install `kb-sdk`: + - `make install-sdk` +- following the instructions printed to the console, adjust the path in your shell: + - export PATH=$PWD:$PATH + +The typical workflow does not use `kb-sdk` directly. + +The primary current usage of `kb-sdk` in this project is compilation, as briefly described above. + +The following command: + +```sh +make compile +``` + +will run the compilation. diff --git a/docs/using-pipenv.md b/docs/using-pipenv.md index 60bcd0a2..6cfd9962 100644 --- a/docs/using-pipenv.md +++ b/docs/using-pipenv.md @@ -38,6 +38,8 @@ This section describes how to get `pipenv` up and running on macOS. These instru `pipenv` will be installed by `pip` into `~/Library/Python/3.7/lib/python/site-packages` (module) and `~/Library/Python/3.7/bin` (executable). + It will be available to your host user account only. + - ensure `pipenv` is runnable: If you look for `pipenv` from the command line, you'll find it is not found: @@ -134,4 +136,4 @@ within the directory in which you ran `pipenv` previously. - Paste in the copied path and hit Return - +- You should have have a fully functioning Python development setup for VSC! diff --git a/lib/SampleService/SampleServiceImpl.py b/lib/SampleService/SampleServiceImpl.py index b93cd943..bd849d47 100644 --- a/lib/SampleService/SampleServiceImpl.py +++ b/lib/SampleService/SampleServiceImpl.py @@ -2,17 +2,21 @@ #BEGIN_HEADER import datetime as _datetime -from collections import defaultdict from SampleService.core.config import build_samples as _build_samples -from SampleService.core.api_translation import (get_sample_address_from_object as - _get_sample_address_from_object) +from SampleService.core.api_translation import ( + get_sample_address_from_object as _get_sample_address_from_object, +) from SampleService.core.api_translation import get_id_from_object as _get_id_from_object from SampleService.core.api_translation import acls_from_dict as _acls_from_dict from SampleService.core.api_translation import acls_to_dict as _acls_to_dict from SampleService.core.api_translation import sample_to_dict as _sample_to_dict -from SampleService.core.api_translation import create_sample_params as _create_sample_params -from SampleService.core.api_translation import validate_samples_params as _validate_samples_params +from SampleService.core.api_translation import ( + create_sample_params as _create_sample_params, +) +from SampleService.core.api_translation import ( + validate_samples_params as _validate_samples_params, +) from SampleService.core.api_translation import check_admin as _check_admin from SampleService.core.api_translation import ( get_static_key_metadata_params as _get_static_key_metadata_params, @@ -33,8 +37,8 @@ update_samples_acls as _update_samples_acls ) -_CTX_USER = 'user_id' -_CTX_TOKEN = 'token' +_CTX_USER = "user_id" +_CTX_TOKEN = "token" #END_HEADER @@ -68,7 +72,9 @@ class SampleService: # be found def __init__(self, config): #BEGIN_CONSTRUCTOR - self._samples, self._user_lookup, self._read_exempt_roles = _build_samples(config) + self._samples, self._user_lookup, self._read_exempt_roles = _build_samples( + config + ) #END_CONSTRUCTOR pass @@ -183,14 +189,25 @@ def create_sample(self, ctx, params): # return variables are: address #BEGIN create_sample sample, id_, prev_ver = _create_sample_params(params) - as_admin, user = _get_admin_request_from_object(params, 'as_admin', 'as_user') + as_admin, user = _get_admin_request_from_object(params, "as_admin", "as_user") _check_admin( - self._user_lookup, ctx[_CTX_TOKEN], _AdminPermission.FULL, + self._user_lookup, + ctx[_CTX_TOKEN], + _AdminPermission.FULL, # pretty annoying to test ctx.log_info is working, do it manually - 'create_sample', ctx.log_info, as_user=user, skip_check=not as_admin) + "create_sample", + ctx.log_info, + as_user=user, + skip_check=not as_admin, + ) ret = self._samples.save_sample( - sample, user if user else _UserID(ctx[_CTX_USER]), id_, prev_ver, as_admin=as_admin) - address = {'id': str(ret[0]), 'version': ret[1]} + sample, + user if user else _UserID(ctx[_CTX_USER]), + id_, + prev_ver, + as_admin=as_admin, + ) + address = {"id": str(ret[0]), "version": ret[1]} #END create_sample # At some point might do deeper type checking... @@ -297,11 +314,18 @@ def get_sample(self, ctx, params): # return variables are: sample #BEGIN get_sample id_, ver = _get_sample_address_from_object(params) - admin = _check_admin(self._user_lookup, ctx.get(_CTX_TOKEN), _AdminPermission.READ, - # pretty annoying to test ctx.log_info is working, do it manually - 'get_sample', ctx.log_info, skip_check=not params.get('as_admin')) + admin = _check_admin( + self._user_lookup, + ctx.get(_CTX_TOKEN), + _AdminPermission.READ, + # pretty annoying to test ctx.log_info is working, do it manually + "get_sample", + ctx.log_info, + skip_check=not params.get("as_admin"), + ) s = self._samples.get_sample( - id_, _get_user_from_object(ctx, _CTX_USER), ver, as_admin=admin) + id_, _get_user_from_object(ctx, _CTX_USER), ver, as_admin=admin + ) sample = _sample_to_dict(s) #END get_sample @@ -404,18 +428,25 @@ def get_samples(self, ctx, params): # ctx is the context object # return variables are: samples #BEGIN get_samples - if not params.get('samples'): - raise ValueError(f"") + if not params.get("samples"): + raise ValueError("The 'samples' parameter is required") ids_ = [] - for samp_obj in params['samples']: - id_, ver = _get_sample_address_from_object(samp_obj) - ids_.append({'id': id_, 'version': ver}) + for samp_obj in params["samples"]: + id_, ver = _get_sample_address_from_object(samp_obj) + ids_.append({"id": id_, "version": ver}) # ids_ = _get_sample_addresses_from_object(params) - admin = _check_admin(self._user_lookup, ctx.get(_CTX_TOKEN), _AdminPermission.READ, - # pretty annoying to test ctx.log_info is working, do it manually - 'get_sample', ctx.log_info, skip_check=not params.get('as_admin')) + admin = _check_admin( + self._user_lookup, + ctx.get(_CTX_TOKEN), + _AdminPermission.READ, + # pretty annoying to test ctx.log_info is working, do it manually + "get_sample", + ctx.log_info, + skip_check=not params.get("as_admin"), + ) samples = self._samples.get_samples( - ids_, _get_user_from_object(ctx, _CTX_USER), as_admin=admin) + ids_, _get_user_from_object(ctx, _CTX_USER), as_admin=admin + ) samples = [_sample_to_dict(s) for s in samples] #END get_samples @@ -454,13 +485,19 @@ def get_sample_acls(self, ctx, params): # ctx is the context object # return variables are: acls #BEGIN get_sample_acls - id_ = _get_id_from_object(params, 'id', required=True) + id_ = _get_id_from_object(params, "id", required=True) admin = _check_admin( - self._user_lookup, ctx.get(_CTX_TOKEN), _AdminPermission.READ, + self._user_lookup, + ctx.get(_CTX_TOKEN), + _AdminPermission.READ, # pretty annoying to test ctx.log_info is working, do it manually - 'get_sample_acls', ctx.log_info, skip_check=not params.get('as_admin')) + "get_sample_acls", + ctx.log_info, + skip_check=not params.get("as_admin"), + ) acls_ret = self._samples.get_sample_acls( - id_, _get_user_from_object(ctx, _CTX_USER), as_admin=admin) + id_, _get_user_from_object(ctx, _CTX_USER), as_admin=admin + ) acls = _acls_to_dict(acls_ret, read_exempt_roles=self._read_exempt_roles) #END get_sample_acls @@ -505,13 +542,20 @@ def update_sample_acls(self, ctx, params): """ # ctx is the context object #BEGIN update_sample_acls - id_ = _get_id_from_object(params, 'id', required=True) + id_ = _get_id_from_object(params, "id", required=True) acldelta = _acl_delta_from_dict(params) admin = _check_admin( - self._user_lookup, ctx[_CTX_TOKEN], _AdminPermission.FULL, + self._user_lookup, + ctx[_CTX_TOKEN], + _AdminPermission.FULL, # pretty annoying to test ctx.log_info is working, do it manually - 'update_sample_acls', ctx.log_info, skip_check=not params.get('as_admin')) - self._samples.update_sample_acls(id_, _UserID(ctx[_CTX_USER]), acldelta, as_admin=admin) + "update_sample_acls", + ctx.log_info, + skip_check=not params.get("as_admin"), + ) + self._samples.update_sample_acls( + id_, _UserID(ctx[_CTX_USER]), acldelta, as_admin=admin + ) #END update_sample_acls pass @@ -575,13 +619,20 @@ def replace_sample_acls(self, ctx, params): """ # ctx is the context object #BEGIN replace_sample_acls - id_ = _get_id_from_object(params, 'id', required=True) + id_ = _get_id_from_object(params, "id", required=True) acls = _acls_from_dict(params) admin = _check_admin( - self._user_lookup, ctx[_CTX_TOKEN], _AdminPermission.FULL, + self._user_lookup, + ctx[_CTX_TOKEN], + _AdminPermission.FULL, # pretty annoying to test ctx.log_info is working, do it manually - 'replace_sample_acls', ctx.log_info, skip_check=not params.get('as_admin')) - self._samples.replace_sample_acls(id_, _UserID(ctx[_CTX_USER]), acls, as_admin=admin) + "replace_sample_acls", + ctx.log_info, + skip_check=not params.get("as_admin"), + ) + self._samples.replace_sample_acls( + id_, _UserID(ctx[_CTX_USER]), acls, as_admin=admin + ) #END replace_sample_acls pass @@ -621,7 +672,11 @@ def get_metadata_key_static_metadata(self, ctx, params): # return variables are: results #BEGIN get_metadata_key_static_metadata keys, prefix = _get_static_key_metadata_params(params) - results = {'static_metadata': self._samples.get_key_static_metadata(keys, prefix=prefix)} + results = { + "static_metadata": self._samples.get_key_static_metadata( + keys, prefix=prefix + ) + } #END get_metadata_key_static_metadata # At some point might do deeper type checking... @@ -699,18 +754,25 @@ def create_data_link(self, ctx, params): # return variables are: results #BEGIN create_data_link duid, sna, update = _create_data_link_params(params) - as_admin, user = _get_admin_request_from_object(params, 'as_admin', 'as_user') + as_admin, user = _get_admin_request_from_object(params, "as_admin", "as_user") _check_admin( - self._user_lookup, ctx[_CTX_TOKEN], _AdminPermission.FULL, + self._user_lookup, + ctx[_CTX_TOKEN], + _AdminPermission.FULL, # pretty annoying to test ctx.log_info is working, do it manually - 'create_data_link', ctx.log_info, as_user=user, skip_check=not as_admin) + "create_data_link", + ctx.log_info, + as_user=user, + skip_check=not as_admin, + ) link = self._samples.create_data_link( user if user else _UserID(ctx[_CTX_USER]), duid, sna, update, - as_admin=as_admin) - results = {'new_link': _links_to_dicts([link])[0]} + as_admin=as_admin, + ) + results = {"new_link": _links_to_dicts([link])[0]} #END create_data_link # At some point might do deeper type checking... @@ -791,43 +853,53 @@ def propagate_data_links(self, ctx, params): # ctx is the context object # return variables are: results #BEGIN propagate_data_links - sid = params.get('id') - ver = params.get('version') + sid = params.get("id") + ver = params.get("version") - get_links_params = {'id': sid, - 'version': params.get('previous_version'), - 'effective_time': params.get('effective_time'), - 'as_admin': params.get('as_admin')} - data_links = self.get_data_links_from_sample(ctx, get_links_params)[0].get('links') + get_links_params = { + "id": sid, + "version": params.get("previous_version"), + "effective_time": params.get("effective_time"), + "as_admin": params.get("as_admin"), + } + data_links = self.get_data_links_from_sample(ctx, get_links_params)[0].get( + "links" + ) links = list() - ignore_types = params.get('ignore_types', list()) + ignore_types = params.get("ignore_types", list()) for data_link in data_links: - upa = data_link['upa'] + upa = data_link["upa"] ignored = False if ignore_types: wsClient = self._samples._ws._ws - ret = wsClient.administer({'command': 'getObjectInfo', - 'params': {'objects': [{'ref': str(upa)}], - 'ignoreErrors': 1}}) + ret = wsClient.administer( + { + "command": "getObjectInfo", + "params": {"objects": [{"ref": str(upa)}], "ignoreErrors": 1}, + } + ) - if ret['infos'][0][2].split('-')[0] in ignore_types: + if ret["infos"][0][2].split("-")[0] in ignore_types: ignored = True if not ignored: - create_link_params = {'upa': upa, - 'dataid': data_link.get('dataid') + '_' + str(ver), - 'node': data_link.get('node'), - 'id': sid, - 'version': ver, - 'update': params.get('update'), - 'as_admin': params.get('as_admin'), - 'as_user': params.get('as_user') - } - new_link = self.create_data_link(ctx, create_link_params)[0].get('new_link') + create_link_params = { + "upa": upa, + "dataid": data_link.get("dataid") + "_" + str(ver), + "node": data_link.get("node"), + "id": sid, + "version": ver, + "update": params.get("update"), + "as_admin": params.get("as_admin"), + "as_user": params.get("as_user"), + } + new_link = self.create_data_link(ctx, create_link_params)[0].get( + "new_link" + ) links.append(new_link) - results = {'links': links} + results = {"links": links} #END propagate_data_links # At some point might do deeper type checking... @@ -865,15 +937,20 @@ def expire_data_link(self, ctx, params): # ctx is the context object #BEGIN expire_data_link duid = _get_data_unit_id_from_object(params) - as_admin, user = _get_admin_request_from_object(params, 'as_admin', 'as_user') + as_admin, user = _get_admin_request_from_object(params, "as_admin", "as_user") _check_admin( - self._user_lookup, ctx[_CTX_TOKEN], _AdminPermission.FULL, + self._user_lookup, + ctx[_CTX_TOKEN], + _AdminPermission.FULL, # pretty annoying to test ctx.log_info is working, do it manually - 'expire_data_link', ctx.log_info, as_user=user, skip_check=not as_admin) + "expire_data_link", + ctx.log_info, + as_user=user, + skip_check=not as_admin, + ) self._samples.expire_data_link( - user if user else _UserID(ctx[_CTX_USER]), - duid, - as_admin=as_admin) + user if user else _UserID(ctx[_CTX_USER]), duid, as_admin=as_admin + ) #END expire_data_link pass @@ -934,16 +1011,26 @@ def get_data_links_from_sample(self, ctx, params): # return variables are: results #BEGIN get_data_links_from_sample sid, ver = _get_sample_address_from_object(params, version_required=True) - dt = _get_datetime_from_epochmillseconds_in_object(params, 'effective_time') + dt = _get_datetime_from_epochmillseconds_in_object(params, "effective_time") admin = _check_admin( - self._user_lookup, ctx.get(_CTX_TOKEN), _AdminPermission.READ, + self._user_lookup, + ctx.get(_CTX_TOKEN), + _AdminPermission.READ, # pretty annoying to test ctx.log_info is working, do it manually - 'get_data_links_from_sample', ctx.log_info, skip_check=not params.get('as_admin')) + "get_data_links_from_sample", + ctx.log_info, + skip_check=not params.get("as_admin"), + ) links, ts = self._samples.get_links_from_sample( - _get_user_from_object(ctx, _CTX_USER), _SampleAddress(sid, ver), dt, as_admin=admin) - results = {'links': _links_to_dicts(links), - 'effective_time': _datetime_to_epochmilliseconds(ts) - } + _get_user_from_object(ctx, _CTX_USER), + _SampleAddress(sid, ver), + dt, + as_admin=admin, + ) + results = { + "links": _links_to_dicts(links), + "effective_time": _datetime_to_epochmilliseconds(ts), + } #END get_data_links_from_sample # At some point might do deeper type checking... @@ -1112,16 +1199,23 @@ def get_data_links_from_data(self, ctx, params): # return variables are: results #BEGIN get_data_links_from_data upa = _get_upa_from_object(params) - dt = _get_datetime_from_epochmillseconds_in_object(params, 'effective_time') + dt = _get_datetime_from_epochmillseconds_in_object(params, "effective_time") admin = _check_admin( - self._user_lookup, ctx.get(_CTX_TOKEN), _AdminPermission.READ, + self._user_lookup, + ctx.get(_CTX_TOKEN), + _AdminPermission.READ, # pretty annoying to test ctx.log_info is working, do it manually - 'get_data_links_from_data', ctx.log_info, skip_check=not params.get('as_admin')) + "get_data_links_from_data", + ctx.log_info, + skip_check=not params.get("as_admin"), + ) links, ts = self._samples.get_links_from_data( - _get_user_from_object(ctx, _CTX_USER), upa, dt, as_admin=admin) - results = {'links': _links_to_dicts(links), - 'effective_time': _datetime_to_epochmilliseconds(ts) - } + _get_user_from_object(ctx, _CTX_USER), upa, dt, as_admin=admin + ) + results = { + "links": _links_to_dicts(links), + "effective_time": _datetime_to_epochmilliseconds(ts), + } #END get_data_links_from_data # At some point might do deeper type checking... @@ -1233,7 +1327,8 @@ def get_sample_via_data(self, ctx, params): upa = _get_upa_from_object(params) sid, ver = _get_sample_address_from_object(params, version_required=True) sample = self._samples.get_sample_via_data( - _get_user_from_object(ctx, _CTX_USER), upa, _SampleAddress(sid, ver)) + _get_user_from_object(ctx, _CTX_USER), upa, _SampleAddress(sid, ver) + ) sample = _sample_to_dict(sample) #END get_sample_via_data @@ -1284,11 +1379,15 @@ def get_data_link(self, ctx, params): # ctx is the context object # return variables are: link #BEGIN get_data_link - id_ = _get_id_from_object(params, 'linkid', required=True) + id_ = _get_id_from_object(params, "linkid", required=True) _check_admin( - self._user_lookup, ctx[_CTX_TOKEN], _AdminPermission.READ, + self._user_lookup, + ctx[_CTX_TOKEN], + _AdminPermission.READ, # pretty annoying to test ctx.log_info is working, do it manually - 'get_data_link', ctx.log_info) + "get_data_link", + ctx.log_info, + ) dl = self._samples.get_data_link_admin(id_) link = _links_to_dicts([dl])[0] #END get_data_link @@ -1401,9 +1500,9 @@ def validate_samples(self, ctx, params): samples = _validate_samples_params(params) errors = [] for sample in samples: - error_detail = self._samples.validate_sample(sample) - errors.extend(error_detail) - results = {'errors': errors} + error_detail = self._samples.validate_sample(sample) + errors.extend(error_detail) + results = {"errors": errors} #END validate_samples # At some point might do deeper type checking... @@ -1414,12 +1513,15 @@ def validate_samples(self, ctx, params): return [results] def status(self, ctx): #BEGIN_STATUS - returnVal = {'state': "OK", - 'message': "", - 'version': self.VERSION, - 'git_url': self.GIT_URL, - 'git_commit_hash': self.GIT_COMMIT_HASH, - 'servertime': _datetime_to_epochmilliseconds(_datetime.datetime.now( - tz=_datetime.timezone.utc))} + returnVal = { + "state": "OK", + "message": "", + "version": self.VERSION, + "git_url": self.GIT_URL, + "git_commit_hash": self.GIT_COMMIT_HASH, + "servertime": _datetime_to_epochmilliseconds( + _datetime.datetime.now(tz=_datetime.timezone.utc) + ), + } #END_STATUS return [returnVal] diff --git a/lib/SampleService/SampleServiceServer.py b/lib/SampleService/SampleServiceServer.py index 92adca08..2f907339 100644 --- a/lib/SampleService/SampleServiceServer.py +++ b/lib/SampleService/SampleServiceServer.py @@ -12,8 +12,13 @@ from wsgiref.simple_server import make_server import requests as _requests -from jsonrpcbase import JSONRPCService, InvalidParamsError, KeywordError, \ - JSONRPCError, InvalidRequestError +from jsonrpcbase import ( + JSONRPCService, + InvalidParamsError, + KeywordError, + JSONRPCError, + InvalidRequestError, +) from jsonrpcbase import ServerError as JSONServerError from biokbase import log @@ -24,9 +29,9 @@ except ImportError: from configparser import ConfigParser -DEPLOY = 'KB_DEPLOYMENT_CONFIG' -SERVICE = 'KB_SERVICE_NAME' -AUTH = 'auth-service-url' +DEPLOY = "KB_DEPLOYMENT_CONFIG" +SERVICE = "KB_SERVICE_NAME" +AUTH = "auth-service-url" # Note that the error fields do not match the 2.0 JSONRPC spec @@ -45,30 +50,29 @@ def get_config(): retconfig = {} config = ConfigParser() config.read(get_config_file()) - for nameval in config.items(get_service_name() or 'SampleService'): + for nameval in config.items(get_service_name() or "SampleService"): retconfig[nameval[0]] = nameval[1] return retconfig config = get_config() from SampleService.SampleServiceImpl import SampleService # noqa @IgnorePep8 + impl_SampleService = SampleService(config) class JSONObjectEncoder(json.JSONEncoder): - def default(self, obj): if isinstance(obj, set): return list(obj) if isinstance(obj, frozenset): return list(obj) - if hasattr(obj, 'toJSONable'): + if hasattr(obj, "toJSONable"): return obj.toJSONable() return json.JSONEncoder.default(self, obj) class JSONRPCServiceCustom(JSONRPCService): - def call(self, ctx, jsondata): """ Calls jsonrpc service's method and returns its return value in a JSON @@ -85,24 +89,23 @@ def call(self, ctx, jsondata): def _call_method(self, ctx, request): """Calls given method with given params and returns it value.""" - method = self.method_data[request['method']]['method'] - params = request['params'] + method = self.method_data[request["method"]]["method"] + params = request["params"] result = None try: if isinstance(params, list): # Does it have enough arguments? if len(params) < self._man_args(method) - 1: - raise InvalidParamsError('not enough arguments') + raise InvalidParamsError("not enough arguments") # Does it have too many arguments? - if(not self._vargs(method) and len(params) > - self._max_args(method) - 1): - raise InvalidParamsError('too many arguments') + if not self._vargs(method) and len(params) > self._max_args(method) - 1: + raise InvalidParamsError("too many arguments") result = method(ctx, *params) elif isinstance(params, dict): # Do not accept keyword arguments if the jsonrpc version is # not >=1.1. - if request['jsonrpc'] < 11: + if request["jsonrpc"] < 11: raise KeywordError result = method(ctx, **params) @@ -139,10 +142,10 @@ def call_py(self, ctx, jsondata): rdata = jsondata # we already deserialize the json string earlier in the server code, no # need to do it again -# try: -# rdata = json.loads(jsondata) -# except ValueError: -# raise ParseError + # try: + # rdata = json.loads(jsondata) + # except ValueError: + # raise ParseError # set some default values for error handling request = self._get_default_vals() @@ -185,36 +188,35 @@ def call_py(self, ctx, jsondata): def _handle_request(self, ctx, request): """Handles given request and returns its response.""" - if 'types' in self.method_data[request['method']]: - self._validate_params_types(request['method'], request['params']) + if "types" in self.method_data[request["method"]]: + self._validate_params_types(request["method"], request["params"]) result = self._call_method(ctx, request) # Do not respond to notifications. - if request['id'] is None: + if request["id"] is None: return None respond = {} - self._fill_ver(request['jsonrpc'], respond) - respond['result'] = result - respond['id'] = request['id'] + self._fill_ver(request["jsonrpc"], respond) + respond["result"] = result + respond["id"] = request["id"] return respond class MethodContext(dict): - def __init__(self, logger): - self['client_ip'] = None - self['user_id'] = None - self['authenticated'] = None - self['token'] = None - self['module'] = None - self['method'] = None - self['call_id'] = None - self['rpc_context'] = None - self['provenance'] = None - self._debug_levels = set([7, 8, 9, 'DEBUG', 'DEBUG2', 'DEBUG3']) + self["client_ip"] = None + self["user_id"] = None + self["authenticated"] = None + self["token"] = None + self["module"] = None + self["method"] = None + self["call_id"] = None + self["rpc_context"] = None + self["provenance"] = None + self._debug_levels = set([7, 8, 9, "DEBUG", "DEBUG2", "DEBUG3"]) self._logger = logger def log_err(self, message): @@ -243,79 +245,85 @@ def clear_log_level(self): self._logger.clear_user_log_level() def _log(self, level, message): - self._logger.log_message(level, message, self['client_ip'], - self['user_id'], self['module'], - self['method'], self['call_id']) + self._logger.log_message( + level, + message, + self["client_ip"], + self["user_id"], + self["module"], + self["method"], + self["call_id"], + ) def provenance(self): - callbackURL = os.environ.get('SDK_CALLBACK_URL') + callbackURL = os.environ.get("SDK_CALLBACK_URL") if callbackURL: # OK, there's a callback server from which we can get provenance - arg_hash = {'method': 'CallbackServer.get_provenance', - 'params': [], - 'version': '1.1', - 'id': str(_random.random())[2:] - } + arg_hash = { + "method": "CallbackServer.get_provenance", + "params": [], + "version": "1.1", + "id": str(_random.random())[2:], + } body = json.dumps(arg_hash) - response = _requests.post(callbackURL, data=body, - timeout=60) - response.encoding = 'utf-8' + response = _requests.post(callbackURL, data=body, timeout=60) + response.encoding = "utf-8" if response.status_code == 500: - if ('content-type' in response.headers and - response.headers['content-type'] == - 'application/json'): + if ( + "content-type" in response.headers + and response.headers["content-type"] == "application/json" + ): err = response.json() - if 'error' in err: - raise ServerError(**err['error']) + if "error" in err: + raise ServerError(**err["error"]) else: - raise ServerError('Unknown', 0, response.text) + raise ServerError("Unknown", 0, response.text) else: - raise ServerError('Unknown', 0, response.text) + raise ServerError("Unknown", 0, response.text) if not response.ok: response.raise_for_status() resp = response.json() - if 'result' not in resp: - raise ServerError('Unknown', 0, - 'An unknown server error occurred') - return resp['result'][0] + if "result" not in resp: + raise ServerError("Unknown", 0, "An unknown server error occurred") + return resp["result"][0] else: - return self.get('provenance') + return self.get("provenance") class ServerError(Exception): - ''' + """ The call returned an error. Fields: name - the name of the error. code - the error code. message - a human readable error message. data - the server side stacktrace. - ''' + """ def __init__(self, name, code, message, data=None, error=None): super(Exception, self).__init__(message) self.name = name self.code = code - self.message = message if message else '' - self.data = data or error or '' + self.message = message if message else "" + self.data = data or error or "" # data = JSON RPC 2.0, error = 1.1 def __str__(self): - return self.name + ': ' + str(self.code) + '. ' + self.message + \ - '\n' + self.data + return ( + self.name + ": " + str(self.code) + ". " + self.message + "\n" + self.data + ) def getIPAddress(environ): - xFF = environ.get('HTTP_X_FORWARDED_FOR') - realIP = environ.get('HTTP_X_REAL_IP') - trustXHeaders = config is None or \ - config.get('dont_trust_x_ip_headers') != 'true' - - if (trustXHeaders): - if (xFF): - return xFF.split(',')[0].strip() - if (realIP): + xFF = environ.get("HTTP_X_FORWARDED_FOR") + realIP = environ.get("HTTP_X_REAL_IP") + trustXHeaders = config is None or config.get("dont_trust_x_ip_headers") != "true" + + if trustXHeaders: + if xFF: + return xFF.split(",")[0].strip() + if realIP: return realIP.strip() - return environ.get('REMOTE_ADDR') + return environ.get("REMOTE_ADDR") class Application(object): @@ -327,19 +335,37 @@ def logcallback(self): self.serverlog.set_log_file(self.userlog.get_log_file()) def log(self, level, context, message): - self.serverlog.log_message(level, message, context['client_ip'], - context['user_id'], context['module'], - context['method'], context['call_id']) + self.serverlog.log_message( + level, + message, + context["client_ip"], + context["user_id"], + context["module"], + context["method"], + context["call_id"], + ) def __init__(self): - submod = get_service_name() or 'SampleService' + submod = get_service_name() or "SampleService" self.userlog = log.log( - submod, ip_address=True, authuser=True, module=True, method=True, - call_id=True, changecallback=self.logcallback, - config=get_config_file()) + submod, + ip_address=True, + authuser=True, + module=True, + method=True, + call_id=True, + changecallback=self.logcallback, + config=get_config_file(), + ) self.serverlog = log.log( - submod, ip_address=True, authuser=True, module=True, method=True, - call_id=True, logfile=self.userlog.get_log_file()) + submod, + ip_address=True, + authuser=True, + module=True, + method=True, + call_id=True, + logfile=self.userlog.get_log_file(), + ) self.serverlog.set_log_level(6) self.rpc_service = JSONRPCServiceCustom() self.method_authentication = dict() @@ -420,94 +446,101 @@ def __init__(self): def __call__(self, environ, start_response): # Context object, equivalent to the perl impl CallContext ctx = MethodContext(self.userlog) - ctx['client_ip'] = getIPAddress(environ) - status = '500 Internal Server Error' + ctx["client_ip"] = getIPAddress(environ) + status = "500 Internal Server Error" try: - body_size = int(environ.get('CONTENT_LENGTH', 0)) + body_size = int(environ.get("CONTENT_LENGTH", 0)) except (ValueError): body_size = 0 - if environ['REQUEST_METHOD'] == 'OPTIONS': + if environ["REQUEST_METHOD"] == "OPTIONS": # we basically do nothing and just return headers - status = '200 OK' + status = "200 OK" rpc_result = "" else: - request_body = environ['wsgi.input'].read(body_size) + request_body = environ["wsgi.input"].read(body_size) try: req = json.loads(request_body) except ValueError as ve: - err = {'error': {'code': -32700, - 'name': "Parse error", - 'message': str(ve), - } - } - rpc_result = self.process_error(err, ctx, {'version': '1.1'}) + err = { + "error": { + "code": -32700, + "name": "Parse error", + "message": str(ve), + } + } + rpc_result = self.process_error(err, ctx, {"version": "1.1"}) else: - ctx['module'], ctx['method'] = req['method'].split('.') - ctx['call_id'] = req['id'] - ctx['rpc_context'] = { - 'call_stack': [{'time': self.now_in_utc(), - 'method': req['method']} - ] + ctx["module"], ctx["method"] = req["method"].split(".") + ctx["call_id"] = req["id"] + ctx["rpc_context"] = { + "call_stack": [{"time": self.now_in_utc(), "method": req["method"]}] + } + prov_action = { + "service": ctx["module"], + "method": ctx["method"], + "method_params": req["params"], } - prov_action = {'service': ctx['module'], - 'method': ctx['method'], - 'method_params': req['params'] - } - ctx['provenance'] = [prov_action] + ctx["provenance"] = [prov_action] try: - token = environ.get('HTTP_AUTHORIZATION') + token = environ.get("HTTP_AUTHORIZATION") # parse out the method being requested and check if it # has an authentication requirement - method_name = req['method'] - auth_req = self.method_authentication.get( - method_name, 'none') - if auth_req != 'none': - if token is None and auth_req == 'required': + method_name = req["method"] + auth_req = self.method_authentication.get(method_name, "none") + if auth_req != "none": + if token is None and auth_req == "required": err = JSONServerError() err.data = ( - 'Authentication required for ' + - 'SampleService ' + - 'but no authentication header was passed') + "Authentication required for " + + "SampleService " + + "but no authentication header was passed" + ) raise err - elif token is None and auth_req == 'optional': + elif token is None and auth_req == "optional": pass else: try: user = self.auth_client.get_user(token) - ctx['user_id'] = user - ctx['authenticated'] = 1 - ctx['token'] = token + ctx["user_id"] = user + ctx["authenticated"] = 1 + ctx["token"] = token except Exception as e: - if auth_req == 'required': + if auth_req == "required": err = JSONServerError() - err.data = \ - "Token validation failed: %s" % e + err.data = "Token validation failed: %s" % e raise err - if (environ.get('HTTP_X_FORWARDED_FOR')): - self.log(log.INFO, ctx, 'X-Forwarded-For: ' + - environ.get('HTTP_X_FORWARDED_FOR')) - self.log(log.INFO, ctx, 'start method') + if environ.get("HTTP_X_FORWARDED_FOR"): + self.log( + log.INFO, + ctx, + "X-Forwarded-For: " + environ.get("HTTP_X_FORWARDED_FOR"), + ) + self.log(log.INFO, ctx, "start method") rpc_result = self.rpc_service.call(ctx, req) - self.log(log.INFO, ctx, 'end method') - status = '200 OK' + self.log(log.INFO, ctx, "end method") + status = "200 OK" except JSONRPCError as jre: - err = {'error': {'code': jre.code, - 'name': jre.message, - 'message': jre.data - } - } - trace = jre.trace if hasattr(jre, 'trace') else None + err = { + "error": { + "code": jre.code, + "name": jre.message, + "message": jre.data, + } + } + trace = jre.trace if hasattr(jre, "trace") else None rpc_result = self.process_error(err, ctx, req, trace) except Exception: - err = {'error': {'code': 0, - 'name': 'Unexpected Server Error', - 'message': 'An unexpected server error ' + - 'occurred', - } - } - rpc_result = self.process_error(err, ctx, req, - traceback.format_exc()) + err = { + "error": { + "code": 0, + "name": "Unexpected Server Error", + "message": "An unexpected server error " + "occurred", + } + } + rpc_result = self.process_error( + err, ctx, req, traceback.format_exc() + ) # print('Request method was %s\n' % environ['REQUEST_METHOD']) # print('Environment dictionary is:\n%s\n' % pprint.pformat(environ)) @@ -518,33 +551,36 @@ def __call__(self, environ, start_response): if rpc_result: response_body = rpc_result else: - response_body = '' + response_body = "" response_headers = [ - ('Access-Control-Allow-Origin', '*'), - ('Access-Control-Allow-Headers', environ.get( - 'HTTP_ACCESS_CONTROL_REQUEST_HEADERS', 'authorization')), - ('content-type', 'application/json'), - ('content-length', str(len(response_body)))] + ("Access-Control-Allow-Origin", "*"), + ( + "Access-Control-Allow-Headers", + environ.get("HTTP_ACCESS_CONTROL_REQUEST_HEADERS", "authorization"), + ), + ("content-type", "application/json"), + ("content-length", str(len(response_body))), + ] start_response(status, response_headers) - return [response_body.encode('utf8')] + return [response_body.encode("utf8")] def process_error(self, error, context, request, trace=None): if trace: - self.log(log.ERR, context, trace.split('\n')[0:-1]) - if 'id' in request: - error['id'] = request['id'] - if 'version' in request: - error['version'] = request['version'] - e = error['error'].get('error') + self.log(log.ERR, context, trace.split("\n")[0:-1]) + if "id" in request: + error["id"] = request["id"] + if "version" in request: + error["version"] = request["version"] + e = error["error"].get("error") if not e: - error['error']['error'] = trace - elif 'jsonrpc' in request: - error['jsonrpc'] = request['jsonrpc'] - error['error']['data'] = trace + error["error"]["error"] = trace + elif "jsonrpc" in request: + error["jsonrpc"] = request["jsonrpc"] + error["error"]["data"] = trace else: - error['version'] = '1.0' - error['error']['error'] = trace + error["version"] = "1.0" + error["error"]["error"] = trace return json.dumps(error) def now_in_utc(self): @@ -552,8 +588,7 @@ def now_in_utc(self): dtnow = datetime.datetime.now() dtutcnow = datetime.datetime.utcnow() delta = dtnow - dtutcnow - hh, mm = divmod((delta.days * 24 * 60 * 60 + delta.seconds + 30) // 60, - 60) + hh, mm = divmod((delta.days * 24 * 60 * 60 + delta.seconds + 30) // 60, 60) return "%s%+02d:%02d" % (dtnow.isoformat(), hh, mm) application = Application() @@ -573,16 +608,18 @@ def now_in_utc(self): # try: import uwsgi -# Before we do anything with the application, see if the -# configs specify patching all std routines to be asynch -# *ONLY* use this if you are going to wrap the service in -# a wsgi container that has enabled gevent, such as -# uwsgi with the --gevent option - if config is not None and config.get('gevent_monkeypatch_all', False): + + # Before we do anything with the application, see if the + # configs specify patching all std routines to be asynch + # *ONLY* use this if you are going to wrap the service in + # a wsgi container that has enabled gevent, such as + # uwsgi with the --gevent option + if config is not None and config.get("gevent_monkeypatch_all", False): print("Monkeypatching std libraries for async") from gevent import monkey + monkey.patch_all() - uwsgi.applications = {'': application} + uwsgi.applications = {"": application} except ImportError: # Not available outside of wsgi, ignore pass @@ -590,17 +627,17 @@ def now_in_utc(self): _proc = None -def start_server(host='localhost', port=0, newprocess=False): - ''' +def start_server(host="localhost", port=0, newprocess=False): + """ By default, will start the server on localhost on a system assigned port in the main thread. Excecution of the main thread will stay in the server main loop until interrupted. To run the server in a separate process, and thus allow the stop_server method to be called, set newprocess = True. This - will also allow returning of the port number.''' + will also allow returning of the port number.""" global _proc if _proc: - raise RuntimeError('server is already running') + raise RuntimeError("server is already running") httpd = make_server(host, port, application) port = httpd.server_address[1] print("Listening on port %s" % port) @@ -623,53 +660,61 @@ def process_async_cli(input_file_path, output_file_path, token): exit_code = 0 with open(input_file_path) as data_file: req = json.load(data_file) - if 'version' not in req: - req['version'] = '1.1' - if 'id' not in req: - req['id'] = str(_random.random())[2:] + if "version" not in req: + req["version"] = "1.1" + if "id" not in req: + req["id"] = str(_random.random())[2:] ctx = MethodContext(application.userlog) if token: user = application.auth_client.get_user(token) - ctx['user_id'] = user - ctx['authenticated'] = 1 - ctx['token'] = token - if 'context' in req: - ctx['rpc_context'] = req['context'] - ctx['CLI'] = 1 - ctx['module'], ctx['method'] = req['method'].split('.') - prov_action = {'service': ctx['module'], 'method': ctx['method'], - 'method_params': req['params']} - ctx['provenance'] = [prov_action] + ctx["user_id"] = user + ctx["authenticated"] = 1 + ctx["token"] = token + if "context" in req: + ctx["rpc_context"] = req["context"] + ctx["CLI"] = 1 + ctx["module"], ctx["method"] = req["method"].split(".") + prov_action = { + "service": ctx["module"], + "method": ctx["method"], + "method_params": req["params"], + } + ctx["provenance"] = [prov_action] resp = None try: resp = application.rpc_service.call_py(ctx, req) except JSONRPCError as jre: - trace = jre.trace if hasattr(jre, 'trace') else None - resp = {'id': req['id'], - 'version': req['version'], - 'error': {'code': jre.code, - 'name': jre.message, - 'message': jre.data, - 'error': trace} - } + trace = jre.trace if hasattr(jre, "trace") else None + resp = { + "id": req["id"], + "version": req["version"], + "error": { + "code": jre.code, + "name": jre.message, + "message": jre.data, + "error": trace, + }, + } except Exception: trace = traceback.format_exc() - resp = {'id': req['id'], - 'version': req['version'], - 'error': {'code': 0, - 'name': 'Unexpected Server Error', - 'message': 'An unexpected server error occurred', - 'error': trace} - } - if 'error' in resp: + resp = { + "id": req["id"], + "version": req["version"], + "error": { + "code": 0, + "name": "Unexpected Server Error", + "message": "An unexpected server error occurred", + "error": trace, + }, + } + if "error" in resp: exit_code = 500 with open(output_file_path, "w") as f: f.write(json.dumps(resp, cls=JSONObjectEncoder)) return exit_code if __name__ == "__main__": - if (len(sys.argv) >= 3 and len(sys.argv) <= 4 and - os.path.isfile(sys.argv[1])): + if len(sys.argv) >= 3 and len(sys.argv) <= 4 and os.path.isfile(sys.argv[1]): token = None if len(sys.argv) == 4: if os.path.isfile(sys.argv[3]): @@ -685,11 +730,11 @@ def process_async_cli(input_file_path, output_file_path, token): print(str(err)) # will print something like "option -a not recognized" sys.exit(2) port = 9999 - host = 'localhost' + host = "localhost" for o, a in opts: - if o == '--port': + if o == "--port": port = int(a) - elif o == '--host': + elif o == "--host": host = a print("Host set to %s" % host) else: diff --git a/lib/SampleService/core/acls.py b/lib/SampleService/core/acls.py index e4f2ce4f..77355164 100644 --- a/lib/SampleService/core/acls.py +++ b/lib/SampleService/core/acls.py @@ -1,6 +1,6 @@ -''' +""" Classes and methods for working with sample ACLs. -''' +""" import datetime @@ -10,7 +10,7 @@ from SampleService.core.arg_checkers import ( not_falsy as _not_falsy, not_falsy_in_iterable as _not_falsy_in_iterable, - check_timestamp as _check_timestamp + check_timestamp as _check_timestamp, ) from SampleService.core.errors import ( IllegalParameterError as _IllegalParameterError, @@ -22,9 +22,10 @@ class SampleAccessType(IntEnum): - ''' + """ The different levels of sample access. - ''' + """ + NONE = 1 READ = 2 WRITE = 3 @@ -33,16 +34,17 @@ class SampleAccessType(IntEnum): class AdminPermission(IntEnum): - ''' + """ The different levels of admin permissions. - ''' + """ + NONE = 1 READ = 2 FULL = 3 class SampleACLOwnerless: - ''' + """ An Access Control List for a sample, consisting of user names for various privileges, but without an owner. @@ -50,15 +52,16 @@ class SampleACLOwnerless: :ivar write: the list of usernames with write privileges. :ivar read: the list of usernames with read privileges. :ivar public_read: a boolean designating whether the sample is publically readable. - ''' + """ def __init__( - self, - admin: Sequence[UserID] = None, - write: Sequence[UserID] = None, - read: Sequence[UserID] = None, - public_read: bool = False): - ''' + self, + admin: Sequence[UserID] = None, + write: Sequence[UserID] = None, + read: Sequence[UserID] = None, + public_read: bool = False, + ): + """ Create the ACLs. :param admin: the list of admin usernames. @@ -67,19 +70,21 @@ def __init__( :param public_read: a boolean designating whether the sample is publically readable. None is considered false. :raises IllegalParameterError: if a user appears in more than one ACL. - ''' - self.admin = _to_tuple(admin, 'admin') - self.write = _to_tuple(write, 'write') - self.read = _to_tuple(read, 'read') + """ + self.admin = _to_tuple(admin, "admin") + self.write = _to_tuple(write, "write") + self.read = _to_tuple(read, "read") self.public_read = bool(public_read) # deal with None inputs _check_acl_duplicates(self.admin, self.write, self.read) def __eq__(self, other): if type(other) is type(self): - return (other.admin == self.admin - and other.write == self.write - and other.read == self.read - and other.public_read == self.public_read) + return ( + other.admin == self.admin + and other.write == self.write + and other.read == self.read + and other.public_read == self.public_read + ) return NotImplemented def __hash__(self): @@ -88,23 +93,30 @@ def __hash__(self): def _to_tuple(seq, name) -> _Tuple[UserID, ...]: # dict.fromkeys removes dupes - return tuple(dict.fromkeys( - sorted( # sort to make equals and hash consistent - _cast(Sequence[UserID], _not_falsy_in_iterable([] if seq is None else seq, name)), - key=lambda u: u.id))) # add comparison methods to user? + return tuple( + dict.fromkeys( + sorted( # sort to make equals and hash consistent + _cast( + Sequence[UserID], + _not_falsy_in_iterable([] if seq is None else seq, name), + ), + key=lambda u: u.id, + ) + ) + ) # add comparison methods to user? def _check_acl_duplicates(admin, write, read): for u in admin: if u in write or u in read: - raise _IllegalParameterError(f'User {u} appears in two ACLs') + raise _IllegalParameterError(f"User {u} appears in two ACLs") for u in write: if u in read: - raise _IllegalParameterError(f'User {u} appears in two ACLs') + raise _IllegalParameterError(f"User {u} appears in two ACLs") -class SampleACLDelta(): - ''' +class SampleACLDelta: + """ An Access Control Sequence delta for a sample, consisting of user names that should be added for various privileges and and list of usernames that should be removed for all privileges. @@ -117,18 +129,20 @@ class SampleACLDelta(): :ivar at_least: True signifies that the provided user's permissions should not be downgraded if they are greater than the permission in the delta ACL. If False, the user's permission will be set to exactly the permission in the delta ACL. - ''' + """ + # hmm, this is pretty similar to SampleACLOwnerless... semantics are different though. def __init__( - self, - admin: Sequence[UserID] = None, - write: Sequence[UserID] = None, - read: Sequence[UserID] = None, - remove: Sequence[UserID] = None, - public_read: Optional[bool] = None, - at_least: bool = False): - ''' + self, + admin: Sequence[UserID] = None, + write: Sequence[UserID] = None, + read: Sequence[UserID] = None, + remove: Sequence[UserID] = None, + public_read: Optional[bool] = None, + at_least: bool = False, + ): + """ Create the ACLs. :param admin: the list of usernames to be granted admin privileges. @@ -142,36 +156,48 @@ def __init__( user's permission will be set to exactly the permission in the delta ACL. None is treated as False. :raises IllegalParameterError: If a user appears in more than one ACL - ''' - self.admin = _to_tuple(admin, 'admin') - self.write = _to_tuple(write, 'write') - self.read = _to_tuple(read, 'read') - self.remove = _to_tuple(remove, 'remove') + """ + self.admin = _to_tuple(admin, "admin") + self.write = _to_tuple(write, "write") + self.read = _to_tuple(read, "read") + self.remove = _to_tuple(remove, "remove") self.public_read = public_read self.at_least = bool(at_least) # handle None _check_acl_duplicates(self.admin, self.write, self.read) all_ = set(self.admin + self.write + self.read) for r in self.remove: if r in all_: - raise _IllegalParameterError('Users in the remove list cannot be in any other ACL') + raise _IllegalParameterError( + "Users in the remove list cannot be in any other ACL" + ) def __eq__(self, other): if type(other) is type(self): - return (other.admin == self.admin - and other.write == self.write - and other.read == self.read - and other.remove == self.remove - and other.public_read is self.public_read - and other.at_least is self.at_least) + return ( + other.admin == self.admin + and other.write == self.write + and other.read == self.read + and other.remove == self.remove + and other.public_read is self.public_read + and other.at_least is self.at_least + ) return NotImplemented def __hash__(self): - return hash((self.admin, self.write, self.read, self.remove, self.public_read, - self.at_least)) + return hash( + ( + self.admin, + self.write, + self.read, + self.remove, + self.public_read, + self.at_least, + ) + ) class SampleACL(SampleACLOwnerless): - ''' + """ An Access Control Sequence for a sample, consisting of user names for various privileges. :ivar owner: the owner username. @@ -180,17 +206,18 @@ class SampleACL(SampleACLOwnerless): :ivar read: the list of usernames with read privileges. :ivar public_read: a boolean designating whether the sample is publically readable. :ivar lastupdate: the date the last time the ACLs were updated. - ''' + """ def __init__( - self, - owner: UserID, - lastupdate: datetime.datetime, - admin: Sequence[UserID] = None, - write: Sequence[UserID] = None, - read: Sequence[UserID] = None, - public_read: bool = False): - ''' + self, + owner: UserID, + lastupdate: datetime.datetime, + admin: Sequence[UserID] = None, + write: Sequence[UserID] = None, + read: Sequence[UserID] = None, + public_read: bool = False, + ): + """ Create the ACLs. :param owner: the owner username. @@ -201,17 +228,17 @@ def __init__( :param public_read: a boolean designating whether the sample is publically readable. None is considered false. :raises IllegalParameterError: If a user appears in more than one ACL - ''' - self.owner = _not_falsy(owner, 'owner') - self.lastupdate = _check_timestamp(lastupdate, 'lastupdate') + """ + self.owner = _not_falsy(owner, "owner") + self.lastupdate = _check_timestamp(lastupdate, "lastupdate") super().__init__(admin, write, read, public_read) all_ = (self.admin, self.write, self.read) for i in range(len(all_)): if self.owner in all_[i]: - raise _IllegalParameterError('The owner cannot be in any other ACL') + raise _IllegalParameterError("The owner cannot be in any other ACL") def is_update(self, update: SampleACLDelta) -> bool: - ''' + """ Check if an acl delta update is actually an update or a noop for the sample. :param update: the update. @@ -219,13 +246,14 @@ def is_update(self, update: SampleACLDelta) -> bool: considered. :raises UnauthorizedError: if the update would affect the owner and update.at_least is not true, or if the owner is in the remove list regardless of at_least. - ''' - _not_falsy(update, 'update') + """ + _not_falsy(update, "update") o = self.owner ownerchange = o in update.admin or o in update.write or o in update.read if (ownerchange and not update.at_least) or o in update.remove: raise _UnauthorizedError( - f'ACLs for the sample owner {o.id} may not be modified by a delta update.') + f"ACLs for the sample owner {o.id} may not be modified by a delta update." + ) rem = set(update.remove) admin = set(self.admin) @@ -233,11 +261,18 @@ def is_update(self, update: SampleACLDelta) -> bool: read = set(self.read) # check if users are removed - if not rem.isdisjoint(admin) or not rem.isdisjoint(write) or not rem.isdisjoint(read): + if ( + not rem.isdisjoint(admin) + or not rem.isdisjoint(write) + or not rem.isdisjoint(read) + ): return True # check if public read is changed - if update.public_read is not None and update.public_read is not self.public_read: + if ( + update.public_read is not None + and update.public_read is not self.public_read + ): return True uadmin = set(update.admin) @@ -247,27 +282,41 @@ def is_update(self, update: SampleACLDelta) -> bool: # check if users' permission is changed if update.at_least: - return (not uadmin.issubset(admin | owner) or - not uwrite.issubset(write | admin | owner) or - not uread.issubset(read | write | admin | owner)) + return ( + not uadmin.issubset(admin | owner) + or not uwrite.issubset(write | admin | owner) + or not uread.issubset(read | write | admin | owner) + ) else: - return (not uadmin.issubset(admin) or - not uwrite.issubset(write) or - not uread.issubset(read)) + return ( + not uadmin.issubset(admin) + or not uwrite.issubset(write) + or not uread.issubset(read) + ) def __eq__(self, other): if type(other) is type(self): - return (other.owner == self.owner - and other.lastupdate == self.lastupdate - and other.admin == self.admin - and other.write == self.write - and other.read == self.read - and other.public_read is self.public_read) + return ( + other.owner == self.owner + and other.lastupdate == self.lastupdate + and other.admin == self.admin + and other.write == self.write + and other.read == self.read + and other.public_read is self.public_read + ) return NotImplemented def __hash__(self): - return hash((self.owner, self.lastupdate, self.admin, self.write, self.read, - self.public_read)) + return hash( + ( + self.owner, + self.lastupdate, + self.admin, + self.write, + self.read, + self.public_read, + ) + ) # def __repr__(self): # return (f'SampleACL[{self.owner}, {self.lastupdate}, {self.admin}, {self.write}, ' + diff --git a/lib/SampleService/core/api_translation.py b/lib/SampleService/core/api_translation.py index 14b73fde..4583d79d 100644 --- a/lib/SampleService/core/api_translation.py +++ b/lib/SampleService/core/api_translation.py @@ -1,6 +1,6 @@ -''' +""" Contains helper functions for translating between the SDK API and the core Samples code. -''' +""" from uuid import UUID @@ -17,49 +17,55 @@ SubSampleType as _SubSampleType, SourceMetadata as _SourceMetadata, ) -from SampleService.core.acls import SampleACLOwnerless, SampleACL, AdminPermission, SampleACLDelta +from SampleService.core.acls import ( + SampleACLOwnerless, + SampleACL, + AdminPermission, + SampleACLDelta, +) from SampleService.core.user_lookup import KBaseUserLookup from SampleService.core.arg_checkers import ( not_falsy as _not_falsy, not_falsy_in_iterable as _not_falsy_in_iterable, - check_string as _check_string + check_string as _check_string, ) from SampleService.core.data_link import DataLink from SampleService.core.errors import ( IllegalParameterError as _IllegalParameterError, MissingParameterError as _MissingParameterError, UnauthorizedError as _UnauthorizedError, - NoSuchUserError as _NoSuchUserError + NoSuchUserError as _NoSuchUserError, ) from SampleService.core.user import UserID from SampleService.core.workspace import DataUnitID, UPA -ID = 'id' -''' The ID of a sample. ''' +ID = "id" +""" The ID of a sample. """ def get_user_from_object(params: Dict[str, Any], key: str) -> Optional[UserID]: - ''' + """ Get a user ID from a key in an object. :param params: the dict containing the user. :param key: the key in the dict where the value is the username. :returns: the user ID or None if the user is not present. :raises IllegalParameterError: if the user if invalid. - ''' + """ _check_params(params) - u = params.get(_cast(str, _check_string(key, 'key'))) + u = params.get(_cast(str, _check_string(key, "key"))) if u is None: return None if type(u) is not str: - raise _IllegalParameterError(f'{key} must be a string if present') + raise _IllegalParameterError(f"{key} must be a string if present") else: return UserID(u) def get_admin_request_from_object( - params: Dict[str, Any], as_admin: str, as_user: str) -> Tuple[bool, Optional[UserID]]: - ''' + params: Dict[str, Any], as_admin: str, as_user: str +) -> Tuple[bool, Optional[UserID]]: + """ Get information about a request for administration mode from an object. :param params: the dict containing the information. @@ -70,18 +76,20 @@ def get_admin_request_from_object( :returns: A tuple where the first element is a boolean denoting whether the user wishes to be recognized as an administrator and the second element is the user that admin wishes to impersonate, if any. The user is always None if the boolean is False. - ''' + """ _check_params(params) - as_ad = bool(params.get(_cast(str, _check_string(as_admin, 'as_admin')))) - _check_string(as_user, 'as_user') + as_ad = bool(params.get(_cast(str, _check_string(as_admin, "as_admin")))) + _check_string(as_user, "as_user") if not as_ad: return (as_ad, None) user = get_user_from_object(params, as_user) return (as_ad, user) -def get_id_from_object(obj: Dict[str, Any], key, name=None, required=False) -> Optional[UUID]: - ''' +def get_id_from_object( + obj: Dict[str, Any], key, name=None, required=False +) -> Optional[UUID]: + """ Given a dict, get a sample ID from the dict if it exists. If None or an empty dict is passed to the method, None is returned. @@ -93,9 +101,9 @@ def get_id_from_object(obj: Dict[str, Any], key, name=None, required=False) -> O :returns: the ID, if it exists, or None. :raises MissingParameterError: if the ID is required but not present. :raises IllegalParameterError: if the ID is provided but is invalid. - ''' + """ id_ = None - _check_string(key, 'key') + _check_string(key, "key") name = name if name else key if required and (not obj or not obj.get(key)): raise _MissingParameterError(name) @@ -103,37 +111,42 @@ def get_id_from_object(obj: Dict[str, Any], key, name=None, required=False) -> O id_ = validate_sample_id(obj[key], name) return id_ + def datetime_to_epochmilliseconds(d: datetime.datetime) -> int: - ''' + """ Convert a datetime object to epoch milliseconds. :param d: The datetime. :returns: The date in epoch milliseconds. - ''' - return round(_not_falsy(d, 'd').timestamp() * 1000) + """ + return round(_not_falsy(d, "d").timestamp() * 1000) def get_datetime_from_epochmilliseconds_in_object( - params: Dict[str, Any], key: str) -> Optional[datetime.datetime]: - ''' + params: Dict[str, Any], key: str +) -> Optional[datetime.datetime]: + """ Get a non-naive datetime from a field in an object where the value is epoch milliseconds. :param params: the object. :param key: the key in the object for which the value is an epoch millisecond timestamp. :returns: the datatime or None if none was provided. :raises IllegalParameterError: if the timestamp is illegal. - ''' + """ t = _check_params(params).get(key) if t is None: return None if type(t) != int: raise _IllegalParameterError( - f"key '{key}' value of '{t}' is not a valid epoch millisecond timestamp") + f"key '{key}' value of '{t}' is not a valid epoch millisecond timestamp" + ) return datetime.datetime.fromtimestamp(t / 1000, tz=datetime.timezone.utc) -def create_sample_params(params: Dict[str, Any]) -> Tuple[Sample, Optional[UUID], Optional[int]]: - ''' +def create_sample_params( + params: Dict[str, Any] +) -> Tuple[Sample, Optional[UUID], Optional[int]]: + """ Process the input from the create_sample API call and translate it into standard types. :param params: The unmarshalled JSON recieved from the API as part of the create_sample @@ -142,27 +155,29 @@ def create_sample_params(params: Dict[str, Any]) -> Tuple[Sample, Optional[UUID] be created or None if an entirely new sample should be created, and the previous version of the sample expected when saving a new version. :raises IllegalParameterError: if any of the arguments are illegal. - ''' + """ _check_params(params) - if type(params.get('sample')) != dict: - raise _IllegalParameterError('params must contain sample key that maps to a structure') - s = params['sample'] - if type(s.get('node_tree')) != list: - raise _IllegalParameterError('sample node tree must be present and a list') - if s.get('name') is not None and type(s.get('name')) != str: - raise _IllegalParameterError('sample name must be omitted or a string') + if type(params.get("sample")) != dict: + raise _IllegalParameterError( + "params must contain sample key that maps to a structure" + ) + s = params["sample"] + if type(s.get("node_tree")) != list: + raise _IllegalParameterError("sample node tree must be present and a list") + if s.get("name") is not None and type(s.get("name")) != str: + raise _IllegalParameterError("sample name must be omitted or a string") nodes = _check_nodes(s) - id_ = get_id_from_object(s, ID, name='sample.id') + id_ = get_id_from_object(s, ID, name="sample.id") - pv = params.get('prior_version') + pv = params.get("prior_version") if pv is not None and type(pv) != int: - raise _IllegalParameterError('prior_version must be an integer if supplied') - s = Sample(nodes, s.get('name')) + raise _IllegalParameterError("prior_version must be an integer if supplied") + s = Sample(nodes, s.get("name")) return (s, id_, pv) def validate_samples_params(params: Dict[str, Any]) -> List[Sample]: - ''' + """ Process the input from the validate_samples API call and translate it into standard types. :param params: The unmarshalled JSON recieved from the API as part of the create_sample @@ -171,47 +186,54 @@ def validate_samples_params(params: Dict[str, Any]) -> List[Sample]: be created or None if an entirely new sample should be created, and the previous version of the sample expected when saving a new version. :raises IllegalParameterError: if any of the arguments are illegal. - ''' + """ _check_params(params) - if type(params.get('samples')) != list or len(params.get('samples', [])) == 0: - raise _IllegalParameterError('params must contain list of `samples`') + if type(params.get("samples")) != list or len(params.get("samples", [])) == 0: + raise _IllegalParameterError("params must contain list of `samples`") samples = [] - for s in params['samples']: - if type(s.get('node_tree')) != list: - raise _IllegalParameterError('sample node tree must be present and a list') - if type(s.get('name')) != str or len(s.get('name')) <= 0: - raise _IllegalParameterError('sample name must be included as non-empty string') + for s in params["samples"]: + if type(s.get("node_tree")) != list: + raise _IllegalParameterError("sample node tree must be present and a list") + if type(s.get("name")) != str or len(s.get("name")) <= 0: + raise _IllegalParameterError( + "sample name must be included as non-empty string" + ) nodes = _check_nodes(s) - s = Sample(nodes, s.get('name')) + s = Sample(nodes, s.get("name")) samples.append(s) return samples + def _check_nodes(s): nodes = [] - for i, n in enumerate(s['node_tree']): + for i, n in enumerate(s["node_tree"]): if type(n) != dict: - raise _IllegalParameterError(f'Node at index {i} is not a structure') - if type(n.get('id')) != str: + raise _IllegalParameterError(f"Node at index {i} is not a structure") + if type(n.get("id")) != str: raise _IllegalParameterError( - f'Node at index {i} must have an id key that maps to a string') + f"Node at index {i} must have an id key that maps to a string" + ) try: - type_ = _SubSampleType(n.get('type')) + type_ = _SubSampleType(n.get("type")) except ValueError: raise _IllegalParameterError( - f'Node at index {i} has an invalid sample type: {n.get("type")}') - if n.get('parent') and type(n.get('parent')) != str: + f'Node at index {i} has an invalid sample type: {n.get("type")}' + ) + if n.get("parent") and type(n.get("parent")) != str: raise _IllegalParameterError( - f'Node at index {i} has a parent entry that is not a string') - mc = _check_meta(n.get('meta_controlled'), i, 'controlled metadata') - mu = _check_meta(n.get('meta_user'), i, 'user metadata') - sm = _check_source_meta(n.get('source_meta'), i) + f"Node at index {i} has a parent entry that is not a string" + ) + mc = _check_meta(n.get("meta_controlled"), i, "controlled metadata") + mu = _check_meta(n.get("meta_user"), i, "user metadata") + sm = _check_source_meta(n.get("source_meta"), i) try: - nodes.append(_SampleNode(n.get('id'), type_, n.get('parent'), mc, mu, sm)) + nodes.append(_SampleNode(n.get("id"), type_, n.get("parent"), mc, mu, sm)) # already checked for the missing param error above, for id except _IllegalParameterError as e: raise _IllegalParameterError( - f'Error for node at index {i}: ' + _cast(str, e.message)) from e + f"Error for node at index {i}: " + _cast(str, e.message) + ) from e return nodes @@ -219,23 +241,36 @@ def _check_meta(m, index, name) -> Optional[Dict[str, Dict[str, PrimitiveType]]] if not m: return None if type(m) != dict: - raise _IllegalParameterError(f"Node at index {index}'s {name} entry must be a mapping") + raise _IllegalParameterError( + f"Node at index {index}'s {name} entry must be a mapping" + ) for k1 in m: if type(k1) != str: raise _IllegalParameterError( - f"Node at index {index}'s {name} entry contains a non-string key") + f"Node at index {index}'s {name} entry contains a non-string key" + ) if type(m[k1]) != dict: - raise _IllegalParameterError(f"Node at index {index}'s {name} entry does " + - f"not have a dict as a value at key {k1}") + raise _IllegalParameterError( + f"Node at index {index}'s {name} entry does " + + f"not have a dict as a value at key {k1}" + ) for k2 in m[k1]: if type(k2) != str: - raise _IllegalParameterError(f"Node at index {index}'s {name} entry contains " + - f'a non-string key under key {k1}') + raise _IllegalParameterError( + f"Node at index {index}'s {name} entry contains " + + f"a non-string key under key {k1}" + ) v = m[k1][k2] - if type(v) != str and type(v) != int and type(v) != float and type(v) != bool: + if ( + type(v) != str + and type(v) != int + and type(v) != float + and type(v) != bool + ): raise _IllegalParameterError( - f"Node at index {index}'s {name} entry does " + - f"not have a primitive type as the value at {k1}/{k2}") + f"Node at index {index}'s {name} entry does " + + f"not have a primitive type as the value at {k1}/{k2}" + ) return m @@ -243,49 +278,64 @@ def _check_source_meta(m, index) -> List[_SourceMetadata]: if not m: return [] if type(m) != list: - raise _IllegalParameterError(f"Node at index {index}'s source metadata must be a list") + raise _IllegalParameterError( + f"Node at index {index}'s source metadata must be a list" + ) ret = [] for i, sm in enumerate(m): errprefix = f"Node at index {index}'s source metadata has an entry at index {i}" if type(sm) != dict: - raise _IllegalParameterError(f'{errprefix} that is not a dict') - if type(sm.get('key')) != str: + raise _IllegalParameterError(f"{errprefix} that is not a dict") + if type(sm.get("key")) != str: raise _IllegalParameterError( - f'{errprefix} where the required key field is not a string') + f"{errprefix} where the required key field is not a string" + ) # there's some duplicate code here, but I find getting the error messages right # is too difficult when DRYing up code like this. They're kind of sucky as is - if type(sm.get('skey')) != str: + if type(sm.get("skey")) != str: raise _IllegalParameterError( - f'{errprefix} where the required skey field is not a string') - if type(sm.get('svalue')) != dict: + f"{errprefix} where the required skey field is not a string" + ) + if type(sm.get("svalue")) != dict: raise _IllegalParameterError( - f'{errprefix} where the required svalue field is not a mapping') - for vk in sm['svalue']: + f"{errprefix} where the required svalue field is not a mapping" + ) + for vk in sm["svalue"]: if type(vk) != str: raise _IllegalParameterError( - f'{errprefix} with a value mapping key that is not a string') - v = sm['svalue'][vk] - if type(v) != str and type(v) != int and type(v) != float and type(v) != bool: + f"{errprefix} with a value mapping key that is not a string" + ) + v = sm["svalue"][vk] + if ( + type(v) != str + and type(v) != int + and type(v) != float + and type(v) != bool + ): raise _IllegalParameterError( - f'{errprefix} with a value in the value mapping under key {vk} ' + - 'that is not a primitive type') + f"{errprefix} with a value in the value mapping under key {vk} " + + "that is not a primitive type" + ) try: - ret.append(_SourceMetadata(sm['key'], sm['skey'], sm['svalue'])) + ret.append(_SourceMetadata(sm["key"], sm["skey"], sm["svalue"])) except _IllegalParameterError as e: raise _IllegalParameterError( - f"Node at index {index}'s source metadata has an error at index {i}: " + - f'{e.message}') from e + f"Node at index {index}'s source metadata has an error at index {i}: " + + f"{e.message}" + ) from e return ret def _check_params(params): if params is None: - raise ValueError('params cannot be None') + raise ValueError("params cannot be None") return params -def get_version_from_object(params: Dict[str, Any], required: bool = False) -> Optional[int]: - ''' +def get_version_from_object( + params: Dict[str, Any], required: bool = False +) -> Optional[int]: + """ Given a dict, get a sample version from the dict if it exists, using the key 'version'. :param params: the unmarshalled JSON recieved from the API as part of the API call. @@ -293,19 +343,20 @@ def get_version_from_object(params: Dict[str, Any], required: bool = False) -> O :returns: the version or None if no version was provided. :raises MissingParameterError: if the version is required and not present. :raises IllegalParameterError: if the version is not an integer or < 1. - ''' + """ _check_params(params) - ver = params.get('version') + ver = params.get("version") if ver is None and required: - raise _MissingParameterError('version') + raise _MissingParameterError("version") if ver is not None and (type(ver) != int or ver < 1): - raise _IllegalParameterError(f'Illegal version argument: {ver}') + raise _IllegalParameterError(f"Illegal version argument: {ver}") return ver def get_sample_address_from_object( - params: Dict[str, Any], version_required: bool = False) -> Tuple[UUID, Optional[int]]: - ''' + params: Dict[str, Any], version_required: bool = False +) -> Tuple[UUID, Optional[int]]: + """ Given a dict, get a sample ID and version from the dict. The sample ID is required but the version is not. The keys 'id' and 'version' are used. @@ -315,33 +366,39 @@ def get_sample_address_from_object( :raises MissingParameterError: if the ID is missing or the version is required and not present. :raises IllegalParameterError: if the ID is malformed or if the version is not an integer or < 1. - ''' - return (_cast(UUID, get_id_from_object(params, ID, required=True)), - get_version_from_object(params, version_required)) + """ + return ( + _cast(UUID, get_id_from_object(params, ID, required=True)), + get_version_from_object(params, version_required), + ) def sample_to_dict(sample: SavedSample) -> Dict[str, Any]: - ''' + """ Convert a sample to a JSONable structure to return to the SDK API. :param sample: The sample to convert. :return: The sample as a dict. - ''' - nodes = [{ID: n.name, - 'type': n.type.value, - 'parent': n.parent, - 'meta_controlled': _unfreeze_meta(n.controlled_metadata), - 'meta_user': _unfreeze_meta(n.user_metadata), - 'source_meta': _source_meta_to_list(n.source_metadata) - } - for n in _not_falsy(sample, 'sample').nodes] - return {ID: str(sample.id), - 'user': sample.user.id, - 'name': sample.name, - 'node_tree': nodes, - 'save_date': datetime_to_epochmilliseconds(sample.savetime), - 'version': sample.version - } + """ + nodes = [ + { + ID: n.name, + "type": n.type.value, + "parent": n.parent, + "meta_controlled": _unfreeze_meta(n.controlled_metadata), + "meta_user": _unfreeze_meta(n.user_metadata), + "source_meta": _source_meta_to_list(n.source_metadata), + } + for n in _not_falsy(sample, "sample").nodes + ] + return { + ID: str(sample.id), + "user": sample.user.id, + "name": sample.name, + "node_tree": nodes, + "save_date": datetime_to_epochmilliseconds(sample.savetime), + "version": sample.version, + } def _unfreeze_meta(m): @@ -352,69 +409,77 @@ def _unfreeze_meta(m): def _source_meta_to_list(m): - return [{'key': sm.key, 'skey': sm.sourcekey, 'svalue': dict(sm.sourcevalue)} for sm in m] + return [ + {"key": sm.key, "skey": sm.sourcekey, "svalue": dict(sm.sourcevalue)} + for sm in m + ] def acls_to_dict(acls: SampleACL, read_exempt_roles: List[str] = []) -> Dict[str, Any]: - ''' + """ Convert sample ACLs to a JSONable structure to return to the SDK API. :param acls: The ACLs to convert. :return: the ACLs as a dict. - ''' + """ # don't expose mod time for now, could do later - return {'owner': _not_falsy(acls, 'acls').owner.id, - 'admin': tuple(u.id for u in acls.admin), - 'write': tuple(u.id for u in acls.write), - 'read': tuple(u.id for u in acls.read if u.id not in read_exempt_roles), - 'public_read': 1 if acls.public_read else 0, - } + return { + "owner": _not_falsy(acls, "acls").owner.id, + "admin": tuple(u.id for u in acls.admin), + "write": tuple(u.id for u in acls.write), + "read": tuple(u.id for u in acls.read if u.id not in read_exempt_roles), + "public_read": 1 if acls.public_read else 0, + } def acls_from_dict(d: Dict[str, Any]) -> SampleACLOwnerless: - ''' + """ Given a dict, create a SampleACLOwnerless object from the contents of the acls key. :param params: The dict containing the ACLs. :returns: the ACLs. :raises IllegalParameterError: if any of the arguments are illegal. - ''' - _not_falsy(d, 'd') - if d.get('acls') is None or type(d['acls']) != dict: - raise _IllegalParameterError('ACLs must be supplied in the acls key and must be a mapping') - acls = d['acls'] + """ + _not_falsy(d, "d") + if d.get("acls") is None or type(d["acls"]) != dict: + raise _IllegalParameterError( + "ACLs must be supplied in the acls key and must be a mapping" + ) + acls = d["acls"] return SampleACLOwnerless( - _get_acl(acls, 'admin'), - _get_acl(acls, 'write'), - _get_acl(acls, 'read'), - bool(acls.get('public_read'))) + _get_acl(acls, "admin"), + _get_acl(acls, "write"), + _get_acl(acls, "read"), + bool(acls.get("public_read")), + ) def acl_delta_from_dict(d: Dict[str, Any]) -> SampleACLDelta: - ''' + """ Given a dict, create a SampleACLDelta object from the contents of the dict. :param params: The dict containing the ACL delta. :returns: the ACL delta. :raises IllegalParameterError: if any of the arguments are illegal. - ''' + """ # since we're translating from SDK server data structures, we assume this is a dict - incpub = d.get('public_read') + incpub = d.get("public_read") if incpub is None or incpub == 0: pub = None elif type(incpub) != int: - raise _IllegalParameterError('public_read must be an integer if present') + raise _IllegalParameterError("public_read must be an integer if present") else: pub = incpub > 0 return SampleACLDelta( - _get_acl(d, 'admin'), - _get_acl(d, 'write'), - _get_acl(d, 'read'), - _get_acl(d, 'remove'), + _get_acl(d, "admin"), + _get_acl(d, "write"), + _get_acl(d, "read"), + _get_acl(d, "remove"), pub, - bool(d.get('at_least'))) + bool(d.get("at_least")), + ) def _get_acl(acls, type_): @@ -422,23 +487,29 @@ def _get_acl(acls, type_): if acls.get(type_) is not None: acl = acls[type_] if type(acl) != list: - raise _IllegalParameterError(f'{type_} ACL must be a list') - for i, item, in enumerate(acl): + raise _IllegalParameterError(f"{type_} ACL must be a list") + for ( + i, + item, + ) in enumerate(acl): if type(item) != str: - raise _IllegalParameterError(f'Index {i} of {type_} ACL does not contain a string') + raise _IllegalParameterError( + f"Index {i} of {type_} ACL does not contain a string" + ) ret.append(UserID(item)) return ret def check_admin( - user_lookup: KBaseUserLookup, - token: Optional[str], - perm: AdminPermission, - method: str, - log_fn: Callable[[str], None], - as_user: UserID = None, - skip_check: bool = False) -> bool: - ''' + user_lookup: KBaseUserLookup, + token: Optional[str], + perm: AdminPermission, + method: str, + log_fn: Callable[[str], None], + as_user: UserID = None, + skip_check: bool = False, +) -> bool: + """ Check whether a user has admin privileges. The request is logged. @@ -456,33 +527,43 @@ def check_admin( :raises UnauthorizedError: if the user does not have the permission required. :raises InvalidUserError: if any of the user names are invalid. :raises UnauthorizedError: if any of the users names are valid but do not exist in the system. - ''' + """ if skip_check: return False if not token: - raise _UnauthorizedError('Anonymous users may not act as service administrators.') - _not_falsy(method, 'method') - _not_falsy(log_fn, 'log_fn') - if _not_falsy(perm, 'perm') == AdminPermission.NONE: - raise ValueError('what are you doing calling this method with no permission ' + - 'requirement? That totally makes no sense. Get a brain moran') + raise _UnauthorizedError( + "Anonymous users may not act as service administrators." + ) + _not_falsy(method, "method") + _not_falsy(log_fn, "log_fn") + if _not_falsy(perm, "perm") == AdminPermission.NONE: + raise ValueError( + "what are you doing calling this method with no permission " + + "requirement? That totally makes no sense. Get a brain moran" + ) if as_user and perm != AdminPermission.FULL: - raise ValueError('as_user is supplied, but permission is not FULL') - p, user = _not_falsy(user_lookup, 'user_lookup').is_admin(token) + raise ValueError("as_user is supplied, but permission is not FULL") + p, user = _not_falsy(user_lookup, "user_lookup").is_admin(token) if p < perm: - err = (f'User {user} does not have the necessary administration ' + - f'privileges to run method {method}') + err = ( + f"User {user} does not have the necessary administration " + + f"privileges to run method {method}" + ) log_fn(err) raise _UnauthorizedError(err) if as_user and user_lookup.invalid_users([as_user]): # returns list of bad users raise _NoSuchUserError(as_user.id) - log_fn(f'User {user} is running method {method} with administration permission {p.name}' + - (f' as user {as_user}' if as_user else '')) + log_fn( + f"User {user} is running method {method} with administration permission {p.name}" + + (f" as user {as_user}" if as_user else "") + ) return True -def get_static_key_metadata_params(params: Dict[str, Any]) -> Tuple[List[str], Optional[bool]]: - ''' +def get_static_key_metadata_params( + params: Dict[str, Any] +) -> Tuple[List[str], Optional[bool]]: + """ Given a dict, extract the parameters to interrogate metadata key static metadata. :param params: The parameters. @@ -492,15 +573,15 @@ def get_static_key_metadata_params(params: Dict[str, Any]) -> Tuple[List[str], O considered, and True indicates that prefix keys should be interrogated but prefix matches should be included in the results. :raises IllegalParameterError: if any of the arguments are illegal. - ''' + """ _check_params(params) - keys = params.get('keys') + keys = params.get("keys") if type(keys) != list: - raise _IllegalParameterError('keys must be a list') + raise _IllegalParameterError("keys must be a list") for i, k in enumerate(_cast(List[Any], keys)): if type(k) != str: - raise _IllegalParameterError(f'index {i} of keys is not a string') - prefix = params.get('prefix') + raise _IllegalParameterError(f"index {i} of keys is not a string") + prefix = params.get("prefix") pre: Optional[bool] if not prefix: pre = False @@ -509,12 +590,14 @@ def get_static_key_metadata_params(params: Dict[str, Any]) -> Tuple[List[str], O elif prefix == 2: pre = True else: - raise _IllegalParameterError(f'Unexpected value for prefix: {prefix}') + raise _IllegalParameterError(f"Unexpected value for prefix: {prefix}") return _cast(List[str], keys), pre -def create_data_link_params(params: Dict[str, Any]) -> Tuple[DataUnitID, SampleNodeAddress, bool]: - ''' +def create_data_link_params( + params: Dict[str, Any] +) -> Tuple[DataUnitID, SampleNodeAddress, bool]: + """ Given a dict, extract the parameters to create parameters for creating a data link. Expected keys: @@ -532,20 +615,21 @@ def create_data_link_params(params: Dict[str, Any]) -> Tuple[DataUnitID, SampleN 3) A boolean that indicates whether the link should be updated if it already exists. :raises MissingParameterError: if any of the required arguments are missing. :raises IllegalParameterError: if any of the arguments are illegal. - ''' + """ _check_params(params) sna = SampleNodeAddress( _SampleAddress( _cast(UUID, get_id_from_object(params, ID, required=True)), - _cast(int, get_version_from_object(params, required=True))), - _cast(str, _check_string_int(params, 'node', True)) + _cast(int, get_version_from_object(params, required=True)), + ), + _cast(str, _check_string_int(params, "node", True)), ) duid = get_data_unit_id_from_object(params) - return (duid, sna, bool(params.get('update'))) + return (duid, sna, bool(params.get("update"))) def get_data_unit_id_from_object(params: Dict[str, Any]) -> DataUnitID: - ''' + """ Get a Data Unit ID from a parameter object. Expects an UPA in the key 'upa' and a data unit ID, if any, in the key dataID. @@ -553,25 +637,27 @@ def get_data_unit_id_from_object(params: Dict[str, Any]) -> DataUnitID: :returns: the DUID. :raises MissingParameterError: if the UPA is missing. :raises IllegalParameterError: if the UPA or data ID are illegal. - ''' + """ _check_params(params) - return DataUnitID(get_upa_from_object(params), _check_string_int(params, 'dataid')) + return DataUnitID(get_upa_from_object(params), _check_string_int(params, "dataid")) def get_upa_from_object(params: Dict[str, Any]) -> UPA: - ''' + """ Get an UPA from a parameter object. Expects the UPA in the key 'upa'. :param params: the parameters. :returns: the UPA. :raises MissingParameterError: if the UPA is missing. :raises IllegalParameterError: if the UPA is illegal. - ''' + """ _check_params(params) - return UPA(_cast(str, _check_string_int(params, 'upa', True))) + return UPA(_cast(str, _check_string_int(params, "upa", True))) -def _check_string_int(params: Dict[str, Any], key: str, required=False) -> Optional[str]: +def _check_string_int( + params: Dict[str, Any], key: str, required=False +) -> Optional[str]: v = params.get(key) if v is None: if required: @@ -579,44 +665,47 @@ def _check_string_int(params: Dict[str, Any], key: str, required=False) -> Optio else: return None if type(v) != str: - raise _IllegalParameterError(f'{key} key is not a string as required') + raise _IllegalParameterError(f"{key} key is not a string as required") return _cast(str, v) def links_to_dicts(links: List[DataLink]) -> List[Dict[str, Any]]: - ''' + """ Translate a list of link objects to a list of dicts suitable for tranlating to JSON. :param links: the links. :returns: the list of dicts. - ''' + """ ret = [] - for link in _cast(List[DataLink], _not_falsy_in_iterable(links, 'links')): + for link in _cast(List[DataLink], _not_falsy_in_iterable(links, "links")): ex = datetime_to_epochmilliseconds(link.expired) if link.expired else None - ret.append({ - 'linkid': str(link.id), - 'upa': str(link.duid.upa), - 'dataid': link.duid.dataid, - 'id': str(link.sample_node_address.sampleid), - 'version': link.sample_node_address.version, - 'node': link.sample_node_address.node, - 'createdby': str(link.created_by), - 'created': datetime_to_epochmilliseconds(link.created), - 'expiredby': str(link.expired_by) if link.expired_by else None, - 'expired': ex - }) + ret.append( + { + "linkid": str(link.id), + "upa": str(link.duid.upa), + "dataid": link.duid.dataid, + "id": str(link.sample_node_address.sampleid), + "version": link.sample_node_address.version, + "node": link.sample_node_address.node, + "createdby": str(link.created_by), + "created": datetime_to_epochmilliseconds(link.created), + "expiredby": str(link.expired_by) if link.expired_by else None, + "expired": ex, + } + ) return ret + def validate_sample_id(id_, name=None): - ''' + """ Given a string, validate the sample ID. :param id_: the sample's ID. :param name: the name of the ID to use in an exception, defaulting to the key. :returns: the ID, if it is a valid UUID :raises IllegalParameterError: if the ID is provided but is invalid. - ''' - err = _IllegalParameterError(f'{name} {id_} must be a UUID string') + """ + err = _IllegalParameterError(f"{name} {id_} must be a UUID string") if type(id_) != str: raise err try: diff --git a/lib/SampleService/core/arg_checkers.py b/lib/SampleService/core/arg_checkers.py index 47a31405..27ba2f42 100644 --- a/lib/SampleService/core/arg_checkers.py +++ b/lib/SampleService/core/arg_checkers.py @@ -1,33 +1,32 @@ -''' +""" Contains various miscellaneous utilies such as argument checkers. -''' +""" import datetime import unicodedata from typing import Optional, Iterable, TypeVar from SampleService.core.errors import IllegalParameterError, MissingParameterError -T = TypeVar('T') +T = TypeVar("T") def not_falsy(item: T, item_name: str) -> T: - ''' + """ Check if a value is falsy and throw and exception if so. :param item: the item to check for falsiness. :param item_name: the name of the item to include in any exception. :raises ValueError: if the item is falsy. :returns: the item. - ''' + """ if not item: - raise ValueError(f'{item_name} cannot be a value that evaluates to false') + raise ValueError(f"{item_name} cannot be a value that evaluates to false") return item def not_falsy_in_iterable( - iterable: Optional[Iterable[T]], - name: str, - allow_none: bool = False) -> Optional[Iterable[T]]: - ''' + iterable: Optional[Iterable[T]], name: str, allow_none: bool = False +) -> Optional[Iterable[T]]: + """ Check that an iterable is not None and contains no falsy items. Empty iterables are accepted. :param iterable: the iterable to check. @@ -36,50 +35,51 @@ def not_falsy_in_iterable( the iterable may not be None. :returns: the iterable. :raises ValueError: if the iterable is None or contains falsy items. - ''' + """ # probably need to allow for 0 as an option if iterable is None: if allow_none: return None - raise ValueError(f'{name} cannot be None') + raise ValueError(f"{name} cannot be None") for i, item in enumerate(iterable): - not_falsy(item, f'Index {i} of iterable {name}') + not_falsy(item, f"Index {i} of iterable {name}") return iterable def _contains_control_characters(string: str) -> bool: - ''' + """ Check if a string contains control characters, as denoted by the Unicode character category starting with a C. :param string: the string to check. :returns: True if the string contains control characters, False otherwise. - ''' + """ # make public if needed # See https://stackoverflow.com/questions/4324790/removing-control-characters-from-a-string-in-python # noqa: E501 for c in string: - if unicodedata.category(c)[0] == 'C': + if unicodedata.category(c)[0] == "C": return True return False def _no_control_characters(string: str, name: str) -> str: - ''' + """ Checks that a string contains no control characters and throws an exception if it does. See :meth:`contains_control_characters` for more information. :param string: The string to check. :param name: the name of the string to include in any exception. :raises IllegalParameterError: if the string contains control characters. :returns: the string. - ''' + """ # make public if needed if _contains_control_characters(string): - raise IllegalParameterError(name + ' contains control characters') + raise IllegalParameterError(name + " contains control characters") return string -def check_string(string: Optional[str], name: str, max_len: int = None, optional: bool = False - ) -> Optional[str]: - ''' +def check_string( + string: Optional[str], name: str, max_len: int = None, optional: bool = False +) -> Optional[str]: + """ Check that a string meets a set of criteria: - it is not None or whitespace only (unless the optional parameter is specified) - it contains no control characters @@ -92,11 +92,11 @@ def check_string(string: Optional[str], name: str, max_len: int = None, optional :returns: the stripped string or None if the string was optional and None or whitespace only. :raises MissingParameterError: if the string is None or whitespace only. :raises IllegalParameterError: if the string is too long or contains illegal characters. - ''' + """ # See the IDMapping service if character classes are needed. # Maybe package this stuff if max_len is not None and max_len < 1: - raise ValueError('max_len must be > 0 if provided') + raise ValueError("max_len must be > 0 if provided") if not string or not string.strip(): if optional: return None @@ -104,12 +104,14 @@ def check_string(string: Optional[str], name: str, max_len: int = None, optional string = string.strip() _no_control_characters(string, name) if max_len and len(string) > max_len: - raise IllegalParameterError('{} exceeds maximum length of {}'.format(name, max_len)) + raise IllegalParameterError( + "{} exceeds maximum length of {}".format(name, max_len) + ) return string def check_timestamp(timestamp: datetime.datetime, name: str): - ''' + """ Check that a timestamp is not None and not naive. See https://docs.python.org/3.8/library/datetime.html#aware-and-naive-objects @@ -117,10 +119,10 @@ def check_timestamp(timestamp: datetime.datetime, name: str): :param name: the name of the variable to use in thrown errors. :returns: the timestamp. :raises ValueError: if the check fails. - ''' + """ if not_falsy(timestamp, name).tzinfo is None: # The docs say you should also check savetime.tzinfo.utcoffset(savetime) is not None, # but initializing a datetime with a tzinfo subclass that returns None for that method # causes the constructor to throw an error - raise ValueError(f'{name} cannot be a naive datetime') + raise ValueError(f"{name} cannot be a naive datetime") return timestamp diff --git a/lib/SampleService/core/config.py b/lib/SampleService/core/config.py index 30e3de19..b17e3474 100644 --- a/lib/SampleService/core/config.py +++ b/lib/SampleService/core/config.py @@ -1,6 +1,6 @@ -''' +""" Configuration parsing and creation for the sample service. -''' +""" # Because creating the samples instance involves contacting arango and the auth service, # this code is mostly tested in the integration tests. @@ -16,10 +16,13 @@ import arango as _arango from SampleService.core.validator.metadata_validator import MetadataValidatorSet -from SampleService.core.validator.metadata_validator import MetadataValidator as _MetadataValidator +from SampleService.core.validator.metadata_validator import ( + MetadataValidator as _MetadataValidator, +) from SampleService.core.samples import Samples -from SampleService.core.storage.arango_sample_storage import ArangoSampleStorage \ - as _ArangoSampleStorage +from SampleService.core.storage.arango_sample_storage import ( + ArangoSampleStorage as _ArangoSampleStorage, +) from SampleService.core.arg_checkers import check_string as _check_string from SampleService.core.notification import KafkaNotifier as _KafkaNotifer from SampleService.core.user_lookup import KBaseUserLookup @@ -29,60 +32,84 @@ def build_samples(config: Dict[str, str]) -> Tuple[Samples, KBaseUserLookup, List[str]]: - ''' + """ Build the sample service instance from the SDK server provided parameters. :param cfg: The SDK generated configuration. :returns: A samples instance. - ''' + """ if not config: - raise ValueError('config is empty, cannot start service') - arango_url = _check_string_req(config.get('arango-url'), 'config param arango-url') - arango_db = _check_string_req(config.get('arango-db'), 'config param arango-db') - arango_user = _check_string_req(config.get('arango-user'), 'config param arango-user') - arango_pwd = _check_string_req(config.get('arango-pwd'), 'config param arango-pwd') - - col_sample = _check_string_req(config.get('sample-collection'), - 'config param sample-collection') + raise ValueError("config is empty, cannot start service") + arango_url = _check_string_req(config.get("arango-url"), "config param arango-url") + arango_db = _check_string_req(config.get("arango-db"), "config param arango-db") + arango_user = _check_string_req( + config.get("arango-user"), "config param arango-user" + ) + arango_pwd = _check_string_req(config.get("arango-pwd"), "config param arango-pwd") + + col_sample = _check_string_req( + config.get("sample-collection"), "config param sample-collection" + ) col_version = _check_string_req( - config.get('version-collection'), 'config param version-collection') + config.get("version-collection"), "config param version-collection" + ) col_ver_edge = _check_string_req( - config.get('version-edge-collection'), 'config param version-edge-collection') - col_node = _check_string_req(config.get('node-collection'), 'config param node-collection') + config.get("version-edge-collection"), "config param version-edge-collection" + ) + col_node = _check_string_req( + config.get("node-collection"), "config param node-collection" + ) col_node_edge = _check_string_req( - config.get('node-edge-collection'), 'config param node-edge-collection') + config.get("node-edge-collection"), "config param node-edge-collection" + ) col_data_link = _check_string_req( - config.get('data-link-collection'), 'config param data-link-collection') + config.get("data-link-collection"), "config param data-link-collection" + ) col_ws_obj_ver = _check_string_req( - config.get('workspace-object-version-shadow-collection'), - 'config param workspace-object-version-shadow-collection') - col_schema = _check_string_req(config.get('schema-collection'), - 'config param schema-collection') - - auth_root_url = _check_string_req(config.get('auth-root-url'), 'config param auth-root-url') - auth_token = _check_string_req(config.get('auth-token'), 'config param auth-token') - full_roles = split_value(config, 'auth-full-admin-roles') - read_roles = split_value(config, 'auth-read-admin-roles') - read_exempt_roles = split_value(config, 'auth-read-exempt-roles') - - ws_url = _check_string_req(config.get('workspace-url'), 'config param workspace-url') - ws_token = _check_string_req(config.get('workspace-read-admin-token'), - 'config param workspace-read-admin-token') - - kafka_servers = _check_string(config.get('kafka-bootstrap-servers'), - 'config param kafka-bootstrap-servers', - optional=True) + config.get("workspace-object-version-shadow-collection"), + "config param workspace-object-version-shadow-collection", + ) + col_schema = _check_string_req( + config.get("schema-collection"), "config param schema-collection" + ) + + auth_root_url = _check_string_req( + config.get("auth-root-url"), "config param auth-root-url" + ) + auth_token = _check_string_req(config.get("auth-token"), "config param auth-token") + full_roles = split_value(config, "auth-full-admin-roles") + read_roles = split_value(config, "auth-read-admin-roles") + read_exempt_roles = split_value(config, "auth-read-exempt-roles") + + ws_url = _check_string_req( + config.get("workspace-url"), "config param workspace-url" + ) + ws_token = _check_string_req( + config.get("workspace-read-admin-token"), + "config param workspace-read-admin-token", + ) + + kafka_servers = _check_string( + config.get("kafka-bootstrap-servers"), + "config param kafka-bootstrap-servers", + optional=True, + ) kafka_topic = None if kafka_servers: # have to start the server twice to test no kafka scenario - kafka_topic = _check_string(config.get('kafka-topic'), 'config param kafka-topic') + kafka_topic = _check_string( + config.get("kafka-topic"), "config param kafka-topic" + ) - metaval_url = _check_string(config.get('metadata-validator-config-url'), - 'config param metadata-validator-config-url', - optional=True) + metaval_url = _check_string( + config.get("metadata-validator-config-url"), + "config param metadata-validator-config-url", + optional=True, + ) # meta params may have info that shouldn't be logged so don't log any for now. # Add code to deal with this later if needed - print(f''' + print( + f""" Starting server with config: arango-url: {arango_url} arango-db: {arango_db} @@ -106,14 +133,16 @@ def build_samples(config: Dict[str, str]) -> Tuple[Samples, KBaseUserLookup, Lis kafka-bootstrap-servers: {kafka_servers} kafka-topic: {kafka_topic} metadata-validators-config-url: {metaval_url} - ''') + """ + ) # build the validators before trying to connect to arango metaval = get_validators(metaval_url) if metaval_url else MetadataValidatorSet() arangoclient = _arango.ArangoClient(hosts=arango_url) arango_db = arangoclient.db( - arango_db, username=arango_user, password=arango_pwd, verify=True) + arango_db, username=arango_user, password=arango_pwd, verify=True + ) storage = _ArangoSampleStorage( arango_db, col_sample, @@ -126,27 +155,33 @@ def build_samples(config: Dict[str, str]) -> Tuple[Samples, KBaseUserLookup, Lis col_schema, ) storage.start_consistency_checker() - kafka = _KafkaNotifer(kafka_servers, _cast(str, kafka_topic)) if kafka_servers else None + kafka = ( + _KafkaNotifer(kafka_servers, _cast(str, kafka_topic)) if kafka_servers else None + ) user_lookup = KBaseUserLookup(auth_root_url, auth_token, full_roles, read_roles) ws = _WS(_Workspace(ws_url, token=ws_token)) - return Samples(storage, user_lookup, metaval, ws, kafka), user_lookup, read_exempt_roles + return ( + Samples(storage, user_lookup, metaval, ws, kafka), + user_lookup, + read_exempt_roles, + ) def split_value(d: Dict[str, str], key: str): - ''' + """ Get a list of comma separated values given a string taken from a configuration dict. :param config: The configuration dict containing the string to be processed as a value. :param key: The key in the dict containing the value. :returns: a list of strings split from the source comma separated string, or an empty list if the key does not exist or contains only whitespace. :raises ValueError: if the value contains control characters. - ''' + """ if d is None: - raise ValueError('d cannot be None') - rstr = _check_string(d.get(key), 'config param ' + key, optional=True) + raise ValueError("d cannot be None") + rstr = _check_string(d.get(key), "config param " + key, optional=True) if not rstr: return [] - return [x.strip() for x in rstr.split(',') if x.strip()] + return [x.strip() for x in rstr.split(",") if x.strip()] def _check_string_req(s: Optional[str], name: str) -> str: @@ -156,93 +191,105 @@ def _check_string_req(s: Optional[str], name: str) -> str: # TODO may need a versioning scheme # If this structure is updated, please update the README file. _META_VAL_JSONSCHEMA = { - 'type': 'object', - 'definitions': { - 'validator_set': { - 'type': 'object', + "type": "object", + "definitions": { + "validator_set": { + "type": "object", # validate values only - 'additionalProperties': { - 'type': 'object', - 'properties': { - 'key_metadata': { - 'type': 'object', - 'additionalProperties': { - 'type': ['array', 'object', 'number', 'boolean', 'string', 'null'] - } + "additionalProperties": { + "type": "object", + "properties": { + "key_metadata": { + "type": "object", + "additionalProperties": { + "type": [ + "array", + "object", + "number", + "boolean", + "string", + "null", + ] + }, }, - 'validators': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'module': {'type': 'string'}, - 'callable_builder': {'type': 'string'}, - 'parameters': {'type': 'object'} + "validators": { + "type": "array", + "items": { + "type": "object", + "properties": { + "module": {"type": "string"}, + "callable_builder": {"type": "string"}, + "parameters": {"type": "object"}, }, - 'additionalProperties': False, - 'required': ['module', 'callable_builder'] - } - - } + "additionalProperties": False, + "required": ["module", "callable_builder"], + }, + }, }, - 'required': ['validators'] - } + "required": ["validators"], + }, }, - 'additionalProperties': False, + "additionalProperties": False, }, - 'properties': { - 'validators': {'$ref': '#/definitions/validator_set'}, - 'prefix_validators': {'$ref': '#/definitions/validator_set'}, + "properties": { + "validators": {"$ref": "#/definitions/validator_set"}, + "prefix_validators": {"$ref": "#/definitions/validator_set"}, }, - 'additionalProperties': False + "additionalProperties": False, } def get_validators(url: str) -> MetadataValidatorSet: - ''' + """ Given a url pointing to a config file, initialize any metadata validators present in the configuration. :param url: The URL for a config file for the metadata validators. :returns: A set of metadata validators. - ''' + """ # TODO VALIDATOR make validator CLI try: with _urllib.request.urlopen(url) as res: cfg = _yaml.safe_load(res) except _URLError as e: raise ValueError( - f'Failed to open validator configuration file at {url}: {str(e.reason)}') from e + f"Failed to open validator configuration file at {url}: {str(e.reason)}" + ) from e except _ParserError as e: raise ValueError( - f'Failed to open validator configuration file at {url}: {str(e)}') from e + f"Failed to open validator configuration file at {url}: {str(e)}" + ) from e _validate(instance=cfg, schema=_META_VAL_JSONSCHEMA) mvals = _get_validators( - cfg.get('validators', {}), - 'Metadata', - lambda k, v, m: _MetadataValidator(k, v, metadata=m)) - mvals.extend(_get_validators( - cfg.get('prefix_validators', {}), - 'Prefix metadata', - lambda k, v, m: _MetadataValidator(k, prefix_validators=v, metadata=m))) + cfg.get("validators", {}), + "Metadata", + lambda k, v, m: _MetadataValidator(k, v, metadata=m), + ) + mvals.extend( + _get_validators( + cfg.get("prefix_validators", {}), + "Prefix metadata", + lambda k, v, m: _MetadataValidator(k, prefix_validators=v, metadata=m), + ) + ) return MetadataValidatorSet(mvals) def _get_validators(cfg, name_, metaval_func) -> List[_MetadataValidator]: mvals = [] for key, keyval in cfg.items(): - meta = keyval.get('key_metadata') + meta = keyval.get("key_metadata") lvals = [] - for i, val in enumerate(keyval['validators']): - m = importlib.import_module(val['module']) - p = val.get('parameters', {}) + for i, val in enumerate(keyval["validators"]): + m = importlib.import_module(val["module"]) + p = val.get("parameters", {}) try: - build_func = getattr(m, val['callable_builder']) + build_func = getattr(m, val["callable_builder"]) lvals.append(build_func(p)) except Exception as e: raise ValueError( - f'{name_} validator callable build #{i} failed for key {key}: {e.args[0]}' - ) from e + f"{name_} validator callable build #{i} failed for key {key}: {e.args[0]}" + ) from e mvals.append(metaval_func(key, lvals, meta)) return mvals diff --git a/lib/SampleService/core/core_types.py b/lib/SampleService/core/core_types.py index bb89b7cc..ccb3c83c 100644 --- a/lib/SampleService/core/core_types.py +++ b/lib/SampleService/core/core_types.py @@ -1,6 +1,6 @@ -''' +""" Contains type definitions that are used in multiple places. -''' +""" from typing import Union diff --git a/lib/SampleService/core/data_link.py b/lib/SampleService/core/data_link.py index a8895c71..d223cd81 100644 --- a/lib/SampleService/core/data_link.py +++ b/lib/SampleService/core/data_link.py @@ -1,7 +1,7 @@ -''' +""" Contains classes relevant to linking data from outside sources (e.g. the KBase workspace service) to samples. -''' +""" from __future__ import annotations @@ -16,7 +16,7 @@ class DataLink: - ''' + """ A link from a workspace object to a sample node. :ivar duid: the data ID. @@ -25,18 +25,19 @@ class DataLink: :ivar created_by: the user that created the link. :ivar expired: the expiration time or None if the link is not expired. :ivar expired_by: the user that expired the link or None if the link is not expired. - ''' + """ def __init__( - self, - id_: uuid.UUID, - duid: DataUnitID, - sample_node_address: SampleNodeAddress, - created: datetime.datetime, - created_by: UserID, - expired: datetime.datetime = None, - expired_by: UserID = None): - ''' + self, + id_: uuid.UUID, + duid: DataUnitID, + sample_node_address: SampleNodeAddress, + created: datetime.datetime, + created_by: UserID, + expired: datetime.datetime = None, + expired_by: UserID = None, + ): + """ Create the link. If expired is provided expired_by must also be provided. If expired is falsy expired_by is ignored. @@ -47,49 +48,78 @@ def __init__( :param created_by: the user that created the link. :param expired: the expiration time for the link or None if the link is not expired. :param expired_by: the user that expired the link or None if the link is not expired. - ''' + """ # may need to make this non ws specific. YAGNI for now. - self.id = _not_falsy(id_, 'id_') - self.duid = _not_falsy(duid, 'duid') - self.sample_node_address = _not_falsy(sample_node_address, 'sample_node_address') - self.created = _check_timestamp(created, 'created') - self.created_by = _not_falsy(created_by, 'created_by') + self.id = _not_falsy(id_, "id_") + self.duid = _not_falsy(duid, "duid") + self.sample_node_address = _not_falsy( + sample_node_address, "sample_node_address" + ) + self.created = _check_timestamp(created, "created") + self.created_by = _not_falsy(created_by, "created_by") self.expired = None self.expired_by = None if expired: - self.expired = _check_timestamp(expired, 'expired') + self.expired = _check_timestamp(expired, "expired") if expired < created: - raise ValueError('link cannot expire before it is created') - self.expired_by = _not_falsy(expired_by, 'expired_by') + raise ValueError("link cannot expire before it is created") + self.expired_by = _not_falsy(expired_by, "expired_by") def is_equivalent(self, link: DataLink): - ''' + """ Check whether this link is equivalent to another link. Equivalent means that the DUID and sample node address are identical. :param link: The link to check. :returns: True if the links are equivalent, False otherwise. - ''' - _not_falsy(link, 'link') - return (self.duid, self.sample_node_address) == (link.duid, link.sample_node_address) + """ + _not_falsy(link, "link") + return (self.duid, self.sample_node_address) == ( + link.duid, + link.sample_node_address, + ) def __str__(self): - return (f'id={self.id} ' + - f'duid=[{self.duid}] ' + - f'sample_node_address=[{self.sample_node_address}] ' + - f'created={self.created.timestamp()} ' + - f'created_by={self.created_by} ' + - f'expired={self.expired.timestamp() if self.expired else None} ' + - f'expired_by={self.expired_by}') + return ( + f"id={self.id} " + + f"duid=[{self.duid}] " + + f"sample_node_address=[{self.sample_node_address}] " + + f"created={self.created.timestamp()} " + + f"created_by={self.created_by} " + + f"expired={self.expired.timestamp() if self.expired else None} " + + f"expired_by={self.expired_by}" + ) def __eq__(self, other): if type(self) == type(other): - return (self.id, self.duid, self.sample_node_address, - self.created, self.created_by, self.expired, self.expired_by) == ( - other.id, other.duid, other.sample_node_address, - other.created, other.created_by, other.expired, other.expired_by) + return ( + self.id, + self.duid, + self.sample_node_address, + self.created, + self.created_by, + self.expired, + self.expired_by, + ) == ( + other.id, + other.duid, + other.sample_node_address, + other.created, + other.created_by, + other.expired, + other.expired_by, + ) return False def __hash__(self): - return hash((self.id, self.duid, self.sample_node_address, - self.created, self.created_by, self.expired, self.expired_by)) + return hash( + ( + self.id, + self.duid, + self.sample_node_address, + self.created, + self.created_by, + self.expired, + self.expired_by, + ) + ) diff --git a/lib/SampleService/core/errors.py b/lib/SampleService/core/errors.py index 4be9f7d4..2de46d2d 100644 --- a/lib/SampleService/core/errors.py +++ b/lib/SampleService/core/errors.py @@ -13,57 +13,60 @@ class ErrorType(Enum): :ivar error_type: a brief string describing the error type. """ -# These should be handled by the SDK code but keeping them around for future use if we -# add a rest-ish endpoint. -# AUTHENTICATION_FAILED = (10000, "Authentication failed") # noqa: E222 @IgnorePep8 -# """ A general authentication error. """ + # These should be handled by the SDK code but keeping them around for future use if we + # add a rest-ish endpoint. + # AUTHENTICATION_FAILED = (10000, "Authentication failed") # noqa: E222 @IgnorePep8 + # """ A general authentication error. """ -# NO_TOKEN = (10010, "No authentication token") # noqa: E222 @IgnorePep8 -# """ No token was provided when required. """ + # NO_TOKEN = (10010, "No authentication token") # noqa: E222 @IgnorePep8 + # """ No token was provided when required. """ -# INVALID_TOKEN = (10020, "Invalid token") # noqa: E222 @IgnorePep8 -# """ The token provided is not valid. """ + # INVALID_TOKEN = (10020, "Invalid token") # noqa: E222 @IgnorePep8 + # """ The token provided is not valid. """ - UNAUTHORIZED = (20000, "Unauthorized") # noqa: E222 @IgnorePep8 + UNAUTHORIZED = (20000, "Unauthorized") # noqa: E222 @IgnorePep8 """ The user is not authorized to perform the requested action. """ - MISSING_PARAMETER = (30000, "Missing input parameter") # noqa: E222 @IgnorePep8 + MISSING_PARAMETER = (30000, "Missing input parameter") # noqa: E222 @IgnorePep8 """ A required input parameter was not provided. """ - ILLEGAL_PARAMETER = (30001, "Illegal input parameter") # noqa: E222 @IgnorePep8 + ILLEGAL_PARAMETER = (30001, "Illegal input parameter") # noqa: E222 @IgnorePep8 """ An input parameter had an illegal value. """ - METADATA_VALIDATION = (30010, "Metadata validation failed") # noqa: E222 @IgnorePep8 + METADATA_VALIDATION = ( + 30010, + "Metadata validation failed", + ) # noqa: E222 @IgnorePep8 """ Metadata failed validation. """ - SAMPLE_CONCURRENCY = (40000, "Concurrency violation") # noqa: E222 @IgnorePep8 + SAMPLE_CONCURRENCY = (40000, "Concurrency violation") # noqa: E222 @IgnorePep8 """ A concurrency check failed and the operation could not continue. """ - NO_SUCH_USER = (50000, "No such user") # noqa: E222 @IgnorePep8 + NO_SUCH_USER = (50000, "No such user") # noqa: E222 @IgnorePep8 """ The requested user does not exist. """ - NO_SUCH_SAMPLE = (50010, "No such sample") # noqa: E222 @IgnorePep8 + NO_SUCH_SAMPLE = (50010, "No such sample") # noqa: E222 @IgnorePep8 """ The requested sample does not exist. """ NO_SUCH_SAMPLE_VERSION = (50020, "No such sample version") # noqa: E222 @IgnorePep8 """ The requested sample version does not exist. """ - NO_SUCH_SAMPLE_NODE = (50030, "No such sample node") # noqa: E222 @IgnorePep8 + NO_SUCH_SAMPLE_NODE = (50030, "No such sample node") # noqa: E222 @IgnorePep8 """ The requested sample node does not exist. """ NO_SUCH_WORKSPACE_DATA = (50040, "No such workspace data") # noqa: E222 @IgnorePep8 """ The requested workspace or workspace data does not exist. """ - NO_SUCH_DATA_LINK = (50050, "No such data link") # noqa: E222 @IgnorePep8 + NO_SUCH_DATA_LINK = (50050, "No such data link") # noqa: E222 @IgnorePep8 """ The requested data link does not exist. """ - DATA_LINK_EXISTS = (60000, "Data link exists for data ID") # noqa: E222 @IgnorePep8 + DATA_LINK_EXISTS = (60000, "Data link exists for data ID") # noqa: E222 @IgnorePep8 """ A link from the provided data ID already exists. """ - TOO_MANY_DATA_LINKS = (60010, "Too many data links") # noqa: E222 @IgnorePep8 + TOO_MANY_DATA_LINKS = (60010, "Too many data links") # noqa: E222 @IgnorePep8 """ Too many links from the sample version or workspace object version already exist. """ - UNSUPPORTED_OP = (100000, "Unsupported operation") # noqa: E222 @IgnorePep8 + UNSUPPORTED_OP = (100000, "Unsupported operation") # noqa: E222 @IgnorePep8 """ The requested operation is not supported. """ def __init__(self, error_code, error_type): @@ -80,20 +83,20 @@ class SampleError(Exception): """ def __init__(self, error_type: ErrorType, message: Optional[str] = None) -> None: - ''' + """ Create a Sample error. :param error_type: the error type of this error. :param message: an error message. :raises TypeError: if error_type is None - ''' + """ if not error_type: # don't use not_falsy here, causes circular import - raise TypeError('error_type cannot be None') + raise TypeError("error_type cannot be None") et = error_type - msg = f'Sample service error code {et.error_code} {et.error_type}' + msg = f"Sample service error code {et.error_code} {et.error_type}" message = message.strip() if message and message.strip() else None if message: - msg += ': ' + message + msg += ": " + message super().__init__(msg) self.error_type = error_type self.message: Optional[str] = message diff --git a/lib/SampleService/core/notification.py b/lib/SampleService/core/notification.py index 3afef558..dcf6a55c 100644 --- a/lib/SampleService/core/notification.py +++ b/lib/SampleService/core/notification.py @@ -12,7 +12,7 @@ from SampleService.core.arg_checkers import ( not_falsy as _not_falsy, - check_string as _check_string + check_string as _check_string, ) @@ -20,6 +20,7 @@ class KafkaNotifier: """ A notifier that sends JSON messages to Kafka. """ + # Unfortunately kafka-python does not yet support exactly once guarantees: # https://github.com/dpkp/kafka-python/issues/1063 # TODO LATER KAFKA enable idempotence when kafka-python supports it. @@ -55,16 +56,16 @@ class KafkaNotifier: # we do some really gross monkey patching). As such, it's only tested in the intergration # tests. - _EVENT_TYPE = 'event_type' - _SAMPLE_ID = 'sample_id' - _SAMPLE_VERSION = 'sample_ver' - _NEW_SAMPLE = 'NEW_SAMPLE' - _ACL_CHANGE = 'ACL_CHANGE' - _LINK_ID = 'link_id' - _NEW_LINK = 'NEW_LINK' - _EXPIRED_LINK = 'EXPIRED_LINK' + _EVENT_TYPE = "event_type" + _SAMPLE_ID = "sample_id" + _SAMPLE_VERSION = "sample_ver" + _NEW_SAMPLE = "NEW_SAMPLE" + _ACL_CHANGE = "ACL_CHANGE" + _LINK_ID = "link_id" + _NEW_LINK = "NEW_LINK" + _EXPIRED_LINK = "EXPIRED_LINK" - _KAFKA_TOPIC_ILLEGAL_CHARS_RE = _re.compile('[^a-zA-Z0-9-]+') + _KAFKA_TOPIC_ILLEGAL_CHARS_RE = _re.compile("[^a-zA-Z0-9-]+") def __init__(self, bootstrap_servers: str, topic: str): """ @@ -75,11 +76,13 @@ def __init__(self, bootstrap_servers: str, topic: str): name to consist of ASCII alphanumeric values and the hyphen to avoid Kafka issues around ambiguity between period and underscore values. """ - _check_string(bootstrap_servers, 'bootstrap_servers') - self._topic = _check_string(topic, 'topic', max_len=249) + _check_string(bootstrap_servers, "bootstrap_servers") + self._topic = _check_string(topic, "topic", max_len=249) match = self._KAFKA_TOPIC_ILLEGAL_CHARS_RE.search(_cast(str, self._topic)) if match: - raise ValueError(f'Illegal character in Kafka topic {self._topic}: {match.group()}') + raise ValueError( + f"Illegal character in Kafka topic {self._topic}: {match.group()}" + ) # TODO LATER KAFKA support delivery.timeout.ms when the client supports it # https://github.com/dpkp/kafka-python/issues/1723 @@ -88,8 +91,8 @@ def __init__(self, bootstrap_servers: str, topic: str): # this will fail if it can't connect self._prod = _KafkaProducer( # can't test multiple servers without a massive PITA - bootstrap_servers=bootstrap_servers.split(','), - acks='all', + bootstrap_servers=bootstrap_servers.split(","), + acks="all", # retries can occur from 100-30000 ms by default. If we allow 300 retries, that means # the send can take from 30s to 150m. Presumably another server timeout will kill # the request before then. @@ -100,7 +103,7 @@ def __init__(self, bootstrap_servers: str, topic: str): request_timeout_ms=30000, # default is 30000 # presumably this can be removed once idempotence is supported max_in_flight_requests_per_connection=1, - ) + ) self._closed = False def notify_new_sample_version(self, sample_id: UUID, sample_ver: int): @@ -111,12 +114,14 @@ def notify_new_sample_version(self, sample_id: UUID, sample_ver: int): :param sample_ver: the version of the sample. """ if sample_ver < 1: - raise ValueError('sample_ver must be > 0') - self._send_message({ - self._EVENT_TYPE: self._NEW_SAMPLE, - self._SAMPLE_ID: str(_not_falsy(sample_id, 'sample_id')), - self._SAMPLE_VERSION: sample_ver - }) + raise ValueError("sample_ver must be > 0") + self._send_message( + { + self._EVENT_TYPE: self._NEW_SAMPLE, + self._SAMPLE_ID: str(_not_falsy(sample_id, "sample_id")), + self._SAMPLE_VERSION: sample_ver, + } + ) def notify_sample_acl_change(self, sample_id: UUID): """ @@ -124,10 +129,12 @@ def notify_sample_acl_change(self, sample_id: UUID): :param sample_id: the sample ID. """ - self._send_message({ - self._EVENT_TYPE: self._ACL_CHANGE, - self._SAMPLE_ID: str(_not_falsy(sample_id, 'sample_id')) - }) + self._send_message( + { + self._EVENT_TYPE: self._ACL_CHANGE, + self._SAMPLE_ID: str(_not_falsy(sample_id, "sample_id")), + } + ) def notify_new_link(self, link_id: UUID): """ @@ -135,10 +142,12 @@ def notify_new_link(self, link_id: UUID): :param link_id: the link ID. """ - self._send_message({ - self._EVENT_TYPE: self._NEW_LINK, - self._LINK_ID: str(_not_falsy(link_id, 'link_id')) - }) + self._send_message( + { + self._EVENT_TYPE: self._NEW_LINK, + self._LINK_ID: str(_not_falsy(link_id, "link_id")), + } + ) def notify_expired_link(self, link_id: UUID): """ @@ -146,15 +155,17 @@ def notify_expired_link(self, link_id: UUID): :param link_id: the link ID. """ - self._send_message({ - self._EVENT_TYPE: self._EXPIRED_LINK, - self._LINK_ID: str(_not_falsy(link_id, 'link_id')) - }) + self._send_message( + { + self._EVENT_TYPE: self._EXPIRED_LINK, + self._LINK_ID: str(_not_falsy(link_id, "link_id")), + } + ) def _send_message(self, message): if self._closed: - raise ValueError('client is closed') - future = self._prod.send(self._topic, _json.dumps(message).encode('utf-8')) + raise ValueError("client is closed") + future = self._prod.send(self._topic, _json.dumps(message).encode("utf-8")) # ensure the message was send correctly, or if not throw an exeption in the correct thread future.get(timeout=35) # this is very difficult to test diff --git a/lib/SampleService/core/sample.py b/lib/SampleService/core/sample.py index 5d984769..9b9c0cd9 100644 --- a/lib/SampleService/core/sample.py +++ b/lib/SampleService/core/sample.py @@ -1,6 +1,6 @@ -''' +""" Contains classes related to samples. -''' +""" import datetime from enum import Enum as _Enum, unique as _unique @@ -34,22 +34,22 @@ @_unique class SubSampleType(_Enum): - ''' + """ The type of a SampleNode. - ''' + """ # do not change the enum constant variable names, they may be saved in DBs - BIOLOGICAL_REPLICATE = 'BioReplicate' - ''' A biological replicate. ''' - TECHNICAL_REPLICATE = 'TechReplicate' # noqa: E222 @IgnorePep8 - ''' A technical replicate. ''' - SUB_SAMPLE = 'SubSample' # noqa: E222 @IgnorePep8 - ''' A subsample that is not a biological or technical replicate.''' + BIOLOGICAL_REPLICATE = "BioReplicate" + """ A biological replicate. """ + TECHNICAL_REPLICATE = "TechReplicate" # noqa: E222 @IgnorePep8 + """ A technical replicate. """ + SUB_SAMPLE = "SubSample" # noqa: E222 @IgnorePep8 + """ A subsample that is not a biological or technical replicate.""" -_T = _TypeVar('_T') -_V = _TypeVar('_V') +_T = _TypeVar("_T") +_V = _TypeVar("_V") def _fz(d: Dict[_T, _V]) -> Dict[_T, _V]: @@ -57,16 +57,16 @@ def _fz(d: Dict[_T, _V]) -> Dict[_T, _V]: class SourceMetadata: - ''' + """ Sample metadata as it existed at the source. :ivar key: the controlled metadata key. :ivar sourcekey: the metadata key as it existed at the source. :ivar sourcevalue: the metadata value as it existed at the source. - ''' + """ def __init__(self, key: str, sourcekey: str, sourcevalue: Dict[str, PrimitiveType]): - ''' + """ Create the source metadata entry. :param key: the controlled metadata key. @@ -74,17 +74,18 @@ def __init__(self, key: str, sourcekey: str, sourcevalue: Dict[str, PrimitiveTyp :param sourcevalue: the metadata value as it existed at the source. :raises IllegalParameterError: if any of the various keys or values are missing, too long, or contain control characters. - ''' - self.key = _check_metadata_key(key, 'Controlled') - self.sourcekey = _check_metadata_key(sourcekey, 'Source') - self.sourcevalue = _fz(_check_metadata_value(key, sourcevalue, 'Source')) + """ + self.key = _check_metadata_key(key, "Controlled") + self.sourcekey = _check_metadata_key(sourcekey, "Source") + self.sourcevalue = _fz(_check_metadata_value(key, sourcevalue, "Source")) def __eq__(self, other): if type(other) is type(self): - return (other.key == self.key - and other.sourcekey == self.sourcekey - and other.sourcevalue == self.sourcevalue - ) + return ( + other.key == self.key + and other.sourcekey == self.sourcekey + and other.sourcevalue == self.sourcevalue + ) return NotImplemented def __hash__(self): @@ -95,7 +96,7 @@ def __hash__(self): class SampleNode: - ''' + """ A node in the sample tree. :ivar name: The name of the sample node. :ivar type: The type of this sample nde. @@ -105,18 +106,18 @@ class SampleNode: :ivar user_metadata: Unrestricted sample metadata. :ivar source_metadata: Information about the controlled metadata as it existed at the data source, prior to any possible transformations for ingest. - ''' + """ def __init__( - self, - name: str, - type_: SubSampleType = SubSampleType.BIOLOGICAL_REPLICATE, - parent: Optional[str] = None, - controlled_metadata: Optional[Dict[str, Dict[str, PrimitiveType]]] = None, - user_metadata: Optional[Dict[str, Dict[str, PrimitiveType]]] = None, - source_metadata: Optional[List[SourceMetadata]] = None - ): - ''' + self, + name: str, + type_: SubSampleType = SubSampleType.BIOLOGICAL_REPLICATE, + parent: Optional[str] = None, + controlled_metadata: Optional[Dict[str, Dict[str, PrimitiveType]]] = None, + user_metadata: Optional[Dict[str, Dict[str, PrimitiveType]]] = None, + source_metadata: Optional[List[SourceMetadata]] = None, + ): + """ Create a sample node. :param name: The name of the sample node. :param type_: The type of this sample nde. @@ -131,11 +132,15 @@ def __init__( :raises IllegalParameterError: if the name or parent is too long or contains illegal characters, the parent is missing and the node type is not BIOLOGICAL_REPLICATE, or basic metadata constraints are violated. - ''' + """ # could make a bioreplicate class... meh for now - self.name = _cast(str, _check_string(name, 'subsample name', max_len=_MAX_SAMPLE_NAME_LEN)) - self.type = _not_falsy(type_, 'type') - self.parent = _check_string(parent, 'parent', max_len=_MAX_SAMPLE_NAME_LEN, optional=True) + self.name = _cast( + str, _check_string(name, "subsample name", max_len=_MAX_SAMPLE_NAME_LEN) + ) + self.type = _not_falsy(type_, "type") + self.parent = _check_string( + parent, "parent", max_len=_MAX_SAMPLE_NAME_LEN, optional=True + ) cm = controlled_metadata if controlled_metadata else {} _check_meta(cm, True) self.controlled_metadata = _fz(cm) @@ -148,25 +153,35 @@ def __init__( isbiorep = type_ == SubSampleType.BIOLOGICAL_REPLICATE if not _xor(bool(parent), isbiorep): raise IllegalParameterError( - f'Node {self.name} is of type {type_.value} and therefore ' + - f'{"cannot" if isbiorep else "must"} have a parent') + f"Node {self.name} is of type {type_.value} and therefore " + + f'{"cannot" if isbiorep else "must"} have a parent' + ) # TODO description def __eq__(self, other): if type(other) is type(self): - return (other.name == self.name - and other.type == self.type - and other.parent == self.parent - and other.controlled_metadata == self.controlled_metadata - and other.user_metadata == self.user_metadata - and other.source_metadata == self.source_metadata - ) + return ( + other.name == self.name + and other.type == self.type + and other.parent == self.parent + and other.controlled_metadata == self.controlled_metadata + and other.user_metadata == self.user_metadata + and other.source_metadata == self.source_metadata + ) return NotImplemented def __hash__(self): - return hash((self.name, self.type, self.parent, self.controlled_metadata, - self.user_metadata, self.source_metadata)) + return hash( + ( + self.name, + self.type, + self.parent, + self.controlled_metadata, + self.user_metadata, + self.source_metadata, + ) + ) # def __repr__(self): # return (f'{self.name}, {self.type}, {self.parent}, {self.controlled_metadata}, ' + @@ -174,109 +189,124 @@ def __hash__(self): def _check_meta(m: Dict[str, Dict[str, PrimitiveType]], controlled: bool): - c = 'Controlled' if controlled else 'User' + c = "Controlled" if controlled else "User" for k in m: _check_metadata_key(k, c) _check_metadata_value(k, m[k], c) - if len(_json.dumps(m, ensure_ascii=False).encode('utf-8')) > _META_MAX_SIZE_B: + if len(_json.dumps(m, ensure_ascii=False).encode("utf-8")) > _META_MAX_SIZE_B: # would be nice if that could be streamed so we don't make a new byte array raise IllegalParameterError( - f'{c} metadata is larger than maximum of {_META_MAX_SIZE_B}B') + f"{c} metadata is larger than maximum of {_META_MAX_SIZE_B}B" + ) def _check_metadata_key(key: str, name: str) -> str: if not key or not key.strip(): - raise IllegalParameterError(f'{name} metadata keys may not be null or whitespace only') + raise IllegalParameterError( + f"{name} metadata keys may not be null or whitespace only" + ) if len(key) > _META_MAX_KEY_SIZE: raise IllegalParameterError( - f'{name} metadata has key starting with {key[:_META_MAX_KEY_SIZE]} that ' + - f'exceeds maximum length of {_META_MAX_KEY_SIZE}') + f"{name} metadata has key starting with {key[:_META_MAX_KEY_SIZE]} that " + + f"exceeds maximum length of {_META_MAX_KEY_SIZE}" + ) cc = _control_char_first_pos(key) if cc >= 0: raise IllegalParameterError( - f"{name} metadata key {key}'s character at index {cc} is a control character.") + f"{name} metadata key {key}'s character at index {cc} is a control character." + ) return key def _check_metadata_value( - key: str, value: Dict[str, PrimitiveType], name: str) -> Dict[str, PrimitiveType]: + key: str, value: Dict[str, PrimitiveType], name: str +) -> Dict[str, PrimitiveType]: if not value: raise IllegalParameterError( - f'{name} metadata value associated with metadata key {key} is null or empty') + f"{name} metadata value associated with metadata key {key} is null or empty" + ) for vk in value: cc = _control_char_first_pos(vk) if cc >= 0: raise IllegalParameterError( - f"{name} metadata value key {vk} associated with metadata key {key} has a " + - f'character at index {cc} that is a control character.') + f"{name} metadata value key {vk} associated with metadata key {key} has a " + + f"character at index {cc} that is a control character." + ) if len(vk) > _META_MAX_KEY_SIZE: raise IllegalParameterError( - f'{name} metadata has a value key associated with metadata key {key} starting ' + - f'with {vk[:_META_MAX_KEY_SIZE]} that exceeds maximum length of ' + - f'{_META_MAX_KEY_SIZE}') + f"{name} metadata has a value key associated with metadata key {key} starting " + + f"with {vk[:_META_MAX_KEY_SIZE]} that exceeds maximum length of " + + f"{_META_MAX_KEY_SIZE}" + ) val = value[vk] if type(val) == str: cc = _control_char_first_pos(_cast(str, val), allow_tabs_and_lf=True) if cc >= 0: raise IllegalParameterError( - f"{name} metadata value associated with metadata key {key} and " + - f'value key {vk} has a character at index {cc} that is a control character.') + f"{name} metadata value associated with metadata key {key} and " + + f"value key {vk} has a character at index {cc} that is a control character." + ) if len(_cast(str, val)) > _META_MAX_VALUE_SIZE: raise IllegalParameterError( - f'{name} metadata has a value associated with metadata key {key} ' + - f'and value key {vk} starting with {_cast(str, val)[:_META_MAX_KEY_SIZE]} ' + - f'that exceeds maximum length of {_META_MAX_VALUE_SIZE}') + f"{name} metadata has a value associated with metadata key {key} " + + f"and value key {vk} starting with {_cast(str, val)[:_META_MAX_KEY_SIZE]} " + + f"that exceeds maximum length of {_META_MAX_VALUE_SIZE}" + ) return value def _control_char_first_pos(string: str, allow_tabs_and_lf: bool = False): for i, c in enumerate(string): if _unicodedata.category(c)[0] == "C": - if not allow_tabs_and_lf or (c != '\n' and c != '\t'): + if not allow_tabs_and_lf or (c != "\n" and c != "\t"): return i return -1 def _check_source_meta(m: List[SourceMetadata], controlled_metadata): - _not_falsy_in_iterable(m, 'source_metadata') + _not_falsy_in_iterable(m, "source_metadata") # it doesn't make sense to turn the whole thing into json as a rough measure of size as the # user is not responsible for the field names. total_bytes = 0 seen = set() for sm in m: if sm.key in seen: - raise IllegalParameterError(f'Duplicate source metadata key: {sm.key}') + raise IllegalParameterError(f"Duplicate source metadata key: {sm.key}") seen.add(sm.key) # could check for duplicate sm.sourcekeys too, but possibly the source data is split into # two metadata keys? if sm.key not in controlled_metadata: raise IllegalParameterError( - f'Source metadata key {sm.key} does not appear in the controlled metadata') - total_bytes += len(sm.key.encode('utf-8')) + len(sm.sourcekey.encode('utf-8')) - total_bytes += len(_json.dumps(dict(sm.sourcevalue), ensure_ascii=False).encode('utf-8')) + f"Source metadata key {sm.key} does not appear in the controlled metadata" + ) + total_bytes += len(sm.key.encode("utf-8")) + len(sm.sourcekey.encode("utf-8")) + total_bytes += len( + _json.dumps(dict(sm.sourcevalue), ensure_ascii=False).encode("utf-8") + ) # Would be nice if that could be streamed so we don't make a new byte array # This calculation is more convoluted than I would like and hard to reproduce for users # but it's really unlikely the limit is ever going to be hit so YAGNI re improvements, # at least for now. if total_bytes > _META_MAX_SIZE_B: raise IllegalParameterError( - f'Source metadata is larger than maximum of {_META_MAX_SIZE_B}B') + f"Source metadata is larger than maximum of {_META_MAX_SIZE_B}B" + ) class Sample: - ''' + """ A sample containing biological replicates, technical replicates, and sub samples. Do NOT mutate the instance variables post creation. :ivar nodes: The nodes in this sample. :ivar name: The name of the sample. - ''' + """ def __init__( - self, - nodes: List[SampleNode], - name: Optional[str] = None, - ): - ''' + self, + nodes: List[SampleNode], + name: Optional[str] = None, + ): + """ Create the the sample. :param nodes: The tree nodes in the sample. BIOLOGICAL_REPLICATES must come first in the list, and parents must come before children in the list. @@ -287,30 +317,37 @@ def __init__( the first node in the list is not a BIOLOGICAL_REPLICATE, all the BIOLOGICAL_REPLICATES are not at the start of this list, node names are not unique, or parent nodes do not appear in the list prior to their children. - ''' - self.name = _check_string(name, 'name', max_len=_MAX_SAMPLE_NAME_LEN, optional=True) + """ + self.name = _check_string( + name, "name", max_len=_MAX_SAMPLE_NAME_LEN, optional=True + ) if not nodes: - raise MissingParameterError('At least one node per sample is required') + raise MissingParameterError("At least one node per sample is required") if len(nodes) > _MAX_SAMPLE_NODES: raise IllegalParameterError( - f'At most {_MAX_SAMPLE_NODES} nodes are allowed per sample') + f"At most {_MAX_SAMPLE_NODES} nodes are allowed per sample" + ) if nodes[0].type != SubSampleType.BIOLOGICAL_REPLICATE: raise IllegalParameterError( - f'The first node in a sample must be a {SubSampleType.BIOLOGICAL_REPLICATE.value}') + f"The first node in a sample must be a {SubSampleType.BIOLOGICAL_REPLICATE.value}" + ) no_more_bio = False seen_names: _Set[str] = set() for n in nodes: if no_more_bio and n.type == SubSampleType.BIOLOGICAL_REPLICATE: raise IllegalParameterError( - f'{SubSampleType.BIOLOGICAL_REPLICATE.value}s must be the first ' + - 'nodes in the list of sample nodes.') + f"{SubSampleType.BIOLOGICAL_REPLICATE.value}s must be the first " + + "nodes in the list of sample nodes." + ) if n.type != SubSampleType.BIOLOGICAL_REPLICATE: no_more_bio = True if n.name in seen_names: - raise IllegalParameterError(f'Duplicate sample node name: {n.name}') + raise IllegalParameterError(f"Duplicate sample node name: {n.name}") if n.parent and n.parent not in seen_names: - raise IllegalParameterError(f'Parent {n.parent} of node {n.name} does not ' + - 'appear in node list prior to node.') + raise IllegalParameterError( + f"Parent {n.parent} of node {n.name} does not " + + "appear in node list prior to node." + ) seen_names.add(n.name) self.nodes = tuple(nodes) # make hashable @@ -327,7 +364,7 @@ def __hash__(self): class SavedSample(Sample): - ''' + """ A sample that has been, or is about to be, saved to persistent storage and therefore has a unique ID and a user associated with it. Do NOT mutate the instance variables post creation. :ivar id: The ID of the sample. @@ -337,17 +374,18 @@ class SavedSample(Sample): :ivar name: The name of the sample. :ivar version: The version of the sample. This may be None if the version has not yet been determined. - ''' + """ def __init__( - self, - id_: UUID, - user: UserID, - nodes: List[SampleNode], - savetime: datetime.datetime, - name: Optional[str] = None, - version: Optional[int] = None): - ''' + self, + id_: UUID, + user: UserID, + nodes: List[SampleNode], + savetime: datetime.datetime, + name: Optional[str] = None, + version: Optional[int] = None, + ): + """ Create the sample. :param id_: The ID of the sample. :param user: The user who saved the sample. @@ -362,29 +400,33 @@ def __init__( the first node in the list is not a BIOLOGICAL_REPLICATE, all the BIOLOGICAL_REPLICATES are not at the start of this list, node names are not unique, or parent nodes do not appear in the list prior to their children. - ''' + """ # having None as a possible version doesn't sit well with me, but that means we need # yet another class, so... super().__init__(nodes, name) - self.id = _not_falsy(id_, 'id_') - self.user = _not_falsy(user, 'user') - self.savetime = _check_timestamp(savetime, 'savetime') + self.id = _not_falsy(id_, "id_") + self.user = _not_falsy(user, "user") + self.savetime = _check_timestamp(savetime, "savetime") if version is not None and version < 1: - raise ValueError('version must be > 0') + raise ValueError("version must be > 0") self.version = version def __eq__(self, other): if type(other) is type(self): - return (other.id == self.id - and other.user == self.user - and other.name == self.name - and other.savetime == self.savetime - and other.version == self.version - and other.nodes == self.nodes) + return ( + other.id == self.id + and other.user == self.user + and other.name == self.name + and other.savetime == self.savetime + and other.version == self.version + and other.nodes == self.nodes + ) return NotImplemented def __hash__(self): - return hash((self.id, self.user, self.name, self.savetime, self.version, self.nodes)) + return hash( + (self.id, self.user, self.name, self.savetime, self.version, self.nodes) + ) # def __repr__(self): # return (f'{self.id}, {self.user}, {self.name}, {self.savetime}, {self.version}, ' + @@ -396,27 +438,27 @@ def _xor(bool1: bool, bool2: bool): class SampleAddress: - ''' + """ A persistent address for a version of a sample. :ivar sampleid: The ID of the sample. :ivar version: The version of the sample. - ''' + """ def __init__(self, sampleid: UUID, version: int): - ''' + """ Create the address. :param sampleid: The ID of the sample. :param version: The version of the sample. - ''' - self.sampleid = _not_falsy(sampleid, 'sampleid') + """ + self.sampleid = _not_falsy(sampleid, "sampleid") if version is None or version < 1: - raise IllegalParameterError('version must be > 0') + raise IllegalParameterError("version must be > 0") self.version = version def __str__(self): - return f'{self.sampleid}:{self.version}' + return f"{self.sampleid}:{self.version}" def __eq__(self, other): if type(self) is type(other): @@ -428,33 +470,38 @@ def __hash__(self): class SampleNodeAddress: - ''' + """ A persistent address for a node of a sample. :param sampleid: The ID of the sample. :param version: The version of the sample. :ivar node: The ID of the node. - ''' + """ def __init__(self, sample: SampleAddress, node: str): - ''' + """ Create the address. :param sample: The sample address. :param node: The ID of the sample node. - ''' + """ - self.sampleid = _not_falsy(sample, 'sample').sampleid + self.sampleid = _not_falsy(sample, "sample").sampleid self.version = sample.version - self.node = _cast(str, _check_string(node, 'node', max_len=_MAX_SAMPLE_NAME_LEN)) + self.node = _cast( + str, _check_string(node, "node", max_len=_MAX_SAMPLE_NAME_LEN) + ) def __str__(self): - return f'{self.sampleid}:{self.version}:{self.node}' + return f"{self.sampleid}:{self.version}:{self.node}" def __eq__(self, other): if type(self) is type(other): return (self.sampleid, self.version, self.node) == ( - other.sampleid, other.version, other.node) + other.sampleid, + other.version, + other.node, + ) return False def __hash__(self): diff --git a/lib/SampleService/core/samples.py b/lib/SampleService/core/samples.py index 3e755a1a..82c20363 100644 --- a/lib/SampleService/core/samples.py +++ b/lib/SampleService/core/samples.py @@ -1,6 +1,6 @@ -''' +""" Core class for saving and getting samples. -''' +""" import datetime import uuid as _uuid # lgtm [py/import-and-import-from] @@ -19,10 +19,15 @@ IllegalParameterError as _IllegalParameterError, MetadataValidationError as _MetadataValidationError, NoSuchUserError as _NoSuchUserError, - NoSuchLinkError as _NoSuchLinkError + NoSuchLinkError as _NoSuchLinkError, ) from SampleService.core.notification import KafkaNotifier -from SampleService.core.sample import Sample, SavedSample, SampleAddress, SampleNodeAddress +from SampleService.core.sample import ( + Sample, + SavedSample, + SampleAddress, + SampleNodeAddress, +) from SampleService.core.user_lookup import KBaseUserLookup from SampleService.core import user_lookup as _user_lookup_mod from SampleService.core.validator.metadata_validator import MetadataValidatorSet @@ -35,49 +40,53 @@ # TODO remove own acls. + class Samples: - ''' + """ Class implementing sample manipulation operations. - ''' + """ def __init__( - self, - storage: ArangoSampleStorage, - user_lookup: KBaseUserLookup, # make an interface? YAGNI - metadata_validator: MetadataValidatorSet, - workspace: WS, - # may want to support multiple notifiers. YAGNI for now - notifier: Optional[KafkaNotifier] = None, - now: Callable[[], datetime.datetime] = lambda: datetime.datetime.now( - tz=datetime.timezone.utc), - uuid_gen: Callable[[], UUID] = lambda: _uuid.uuid4()): - ''' + self, + storage: ArangoSampleStorage, + user_lookup: KBaseUserLookup, # make an interface? YAGNI + metadata_validator: MetadataValidatorSet, + workspace: WS, + # may want to support multiple notifiers. YAGNI for now + notifier: Optional[KafkaNotifier] = None, + now: Callable[[], datetime.datetime] = lambda: datetime.datetime.now( + tz=datetime.timezone.utc + ), + uuid_gen: Callable[[], UUID] = lambda: _uuid.uuid4(), + ): + """ Create the class. :param storage: the storage system to use. :param user_lookup: a service to verify usernames are valid and exist. :param metadata_validator: A validator for metadata. - ''' + """ # don't publicize these params # :param now: A callable that returns the current time. Primarily used for testing. # :param uuid_gen: A callable that returns a random UUID. Primarily used for testing. # extract an interface from ASS if needed. - self._storage = _not_falsy(storage, 'storage') - self._user_lookup = _not_falsy(user_lookup, 'user_lookup') - self._metaval = _not_falsy(metadata_validator, 'metadata_validator') - self._ws = _not_falsy(workspace, 'workspace') + self._storage = _not_falsy(storage, "storage") + self._user_lookup = _not_falsy(user_lookup, "user_lookup") + self._metaval = _not_falsy(metadata_validator, "metadata_validator") + self._ws = _not_falsy(workspace, "workspace") self._kafka = notifier # can be None - self._now = _not_falsy(now, 'now') - self._uuid_gen = _not_falsy(uuid_gen, 'uuid_gen') + self._now = _not_falsy(now, "now") + self._uuid_gen = _not_falsy(uuid_gen, "uuid_gen") def save_sample( - self, - sample: Sample, - user: UserID, - id_: UUID = None, - prior_version: Optional[int] = None, - as_admin: bool = False) -> Tuple[UUID, int]: - ''' + self, + sample: Sample, + user: UserID, + id_: UUID = None, + prior_version: Optional[int] = None, + as_admin: bool = False, + ) -> Tuple[UUID, int]: + """ Save a sample. :param sample: the sample to save. @@ -95,13 +104,13 @@ def save_sample( :raises SampleStorageError: if the sample could not be retrieved when saving a new version or if the sample fails to save. :raises ConcurrencyError: if the sample's version is not equal to prior_version. - ''' - _not_falsy(sample, 'sample') - _not_falsy(user, 'user') + """ + _not_falsy(sample, "sample") + _not_falsy(user, "user") _ = self._validate_metadata(sample) if id_: if prior_version is not None and prior_version < 1: - raise _IllegalParameterError('Prior version must be > 0') + raise _IllegalParameterError("Prior version must be > 0") self._check_perms(id_, user, _SampleAccessType.WRITE, as_admin=as_admin) swid = SavedSample(id_, user, list(sample.nodes), self._now(), sample.name) ver = self._storage.save_sample_version(swid, prior_version) @@ -115,44 +124,49 @@ def save_sample( self._kafka.notify_new_sample_version(id_, ver) return (id_, ver) - def _validate_metadata(self, sample: Sample, return_error_detail: bool=False): - ''' + def _validate_metadata(self, sample: Sample, return_error_detail: bool = False): + """ :params sample: sample to be validated :params return_exception: default=False, whether to return all errors found as a list exceptions. :returns: list of excpetions - ''' + """ for i, n in enumerate(sample.nodes): try: - error_detail = self._metaval.validate_metadata(n.controlled_metadata, return_error_detail) + error_detail = self._metaval.validate_metadata( + n.controlled_metadata, return_error_detail + ) if return_error_detail: for e in error_detail: - e['node'] = n.name + e["node"] = n.name return error_detail except _MetadataValidationError as e: - raise _MetadataValidationError(f'Node at index {i}: {e.message}') from e + raise _MetadataValidationError(f"Node at index {i}: {e.message}") from e def _check_perms( - self, - id_: UUID, - user: Optional[UserID], - access: _SampleAccessType, - acls: SampleACL = None, - as_admin: bool = False): + self, + id_: UUID, + user: Optional[UserID], + access: _SampleAccessType, + acls: SampleACL = None, + as_admin: bool = False, + ): if as_admin: return if not acls: acls = self._storage.get_sample_acls(id_) level = self._get_access_level(acls, user) if level < access: - uerr = f'User {user}' if user else 'Anonymous users' - errmsg = f'{uerr} {self._unauth_errmap[access]} sample {id_}' + uerr = f"User {user}" if user else "Anonymous users" + errmsg = f"{uerr} {self._unauth_errmap[access]} sample {id_}" raise _UnauthorizedError(errmsg) - _unauth_errmap = {_SampleAccessType.OWNER: 'does not own', - _SampleAccessType.ADMIN: 'cannot administrate', - _SampleAccessType.WRITE: 'cannot write to', - _SampleAccessType.READ: 'cannot read'} + _unauth_errmap = { + _SampleAccessType.OWNER: "does not own", + _SampleAccessType.ADMIN: "cannot administrate", + _SampleAccessType.WRITE: "cannot write to", + _SampleAccessType.READ: "cannot read", + } def _get_access_level(self, acls: SampleACL, user: Optional[UserID]): if user == acls.owner: @@ -166,12 +180,13 @@ def _get_access_level(self, acls: SampleACL, user: Optional[UserID]): return _SampleAccessType.NONE def get_sample( - self, - id_: UUID, - user: Optional[UserID], - version: int = None, - as_admin: bool = False) -> SavedSample: - ''' + self, + id_: UUID, + user: Optional[UserID], + version: int = None, + as_admin: bool = False, + ) -> SavedSample: + """ Get a sample. :param id_: the ID of the sample. :param user: the username of the user getting the sample, or None for an anonymous user. @@ -183,26 +198,31 @@ def get_sample( :raises NoSuchSampleError: if the sample does not exist. :raises NoSuchSampleVersionError: if the sample version does not exist. :raises SampleStorageError: if the sample could not be retrieved. - ''' + """ if version is not None and version < 1: - raise _IllegalParameterError('Version must be > 0') - self._check_perms(_not_falsy(id_, 'id_'), user, _SampleAccessType.READ, as_admin=as_admin) + raise _IllegalParameterError("Version must be > 0") + self._check_perms( + _not_falsy(id_, "id_"), user, _SampleAccessType.READ, as_admin=as_admin + ) return self._storage.get_sample(id_, version) def get_samples( - self, - ids_: List[Dict[str, Any]], - user: Optional[UserID], - as_admin: bool = False) -> List[SavedSample]: - ''' - ''' + self, ids_: List[Dict[str, Any]], user: Optional[UserID], as_admin: bool = False + ) -> List[SavedSample]: + """ """ for id_ in ids_: - self._check_perms(_not_falsy(id_['id'], 'id_'), user, _SampleAccessType.READ, as_admin=as_admin) + self._check_perms( + _not_falsy(id_["id"], "id_"), + user, + _SampleAccessType.READ, + as_admin=as_admin, + ) return self._storage.get_samples(ids_) def get_sample_acls( - self, id_: UUID, user: Optional[UserID], as_admin: bool = False) -> SampleACL: - ''' + self, id_: UUID, user: Optional[UserID], as_admin: bool = False + ) -> SampleACL: + """ Get a sample's acls. :param id_: the ID of the sample. :param user: the username of the user getting the acls or None if the user is anonymous. @@ -211,18 +231,19 @@ def get_sample_acls( :raises UnauthorizedError: if the user does not have read permission for the sample. :raises NoSuchSampleError: if the sample does not exist. :raises SampleStorageError: if the sample could not be retrieved. - ''' - acls = self._storage.get_sample_acls(_not_falsy(id_, 'id_')) + """ + acls = self._storage.get_sample_acls(_not_falsy(id_, "id_")) self._check_perms(id_, user, _SampleAccessType.READ, acls, as_admin=as_admin) return acls def replace_sample_acls( - self, - id_: UUID, - user: UserID, - new_acls: SampleACLOwnerless, - as_admin: bool = False) -> None: - ''' + self, + id_: UUID, + user: UserID, + new_acls: SampleACLOwnerless, + as_admin: bool = False, + ) -> None: + """ Completely replace a sample's ACLs. The owner cannot be changed. :param id_: the sample's ID. @@ -234,25 +255,34 @@ def replace_sample_acls( :raises UnauthorizedError: if the user does not have admin permission for the sample or the request attempts to change the owner. :raises SampleStorageError: if the sample could not be retrieved. - ''' - _not_falsy(id_, 'id_') - _not_falsy(user, 'user') - _not_falsy(new_acls, 'new_acls') - self._check_for_bad_users(_cast(List[UserID], []) + list(new_acls.admin) + - list(new_acls.write) + list(new_acls.read)) + """ + _not_falsy(id_, "id_") + _not_falsy(user, "user") + _not_falsy(new_acls, "new_acls") + self._check_for_bad_users( + _cast(List[UserID], []) + + list(new_acls.admin) + + list(new_acls.write) + + list(new_acls.read) + ) count = 0 while count >= 0: if count >= 5: - raise ValueError(f'Failed setting ACLs after 5 attempts for sample {id_}') + raise ValueError( + f"Failed setting ACLs after 5 attempts for sample {id_}" + ) acls = self._storage.get_sample_acls(id_) - self._check_perms(id_, user, _SampleAccessType.ADMIN, acls, as_admin=as_admin) + self._check_perms( + id_, user, _SampleAccessType.ADMIN, acls, as_admin=as_admin + ) new_acls = SampleACL( acls.owner, self._now(), new_acls.admin, new_acls.write, new_acls.read, - new_acls.public_read) + new_acls.public_read, + ) try: self._storage.replace_sample_acls(id_, new_acls) count = -1 @@ -262,12 +292,9 @@ def replace_sample_acls( self._kafka.notify_sample_acl_change(id_) def update_sample_acls( - self, - id_: UUID, - user: UserID, - update: SampleACLDelta, - as_admin: bool = False) -> None: - ''' + self, id_: UUID, user: UserID, update: SampleACLDelta, as_admin: bool = False + ) -> None: + """ Completely replace a sample's ACLs. :param id_: the sample's ID. @@ -280,14 +307,19 @@ def update_sample_acls( :raises UnauthorizedError: if the user does not have admin permission for the sample or the request attempts to alter the owner. :raises SampleStorageError: if the sample could not be retrieved. - ''' + """ # could make yet another ACL class that's a delta w/o an update time - probably not # worth it. If people get confused do it. - _not_falsy(id_, 'id_') - _not_falsy(user, 'user') - _not_falsy(update, 'update') - self._check_for_bad_users(_cast(List[UserID], []) + list(update.admin) + - list(update.write) + list(update.read) + list(update.remove)) + _not_falsy(id_, "id_") + _not_falsy(user, "user") + _not_falsy(update, "update") + self._check_for_bad_users( + _cast(List[UserID], []) + + list(update.admin) + + list(update.write) + + list(update.read) + + list(update.remove) + ) self._check_perms(id_, user, _SampleAccessType.ADMIN, as_admin=as_admin) @@ -305,38 +337,39 @@ def _check_for_bad_users(self, users: List[UserID]): except _user_lookup_mod.InvalidUserError as e: raise _NoSuchUserError(e.args[0]) from e except _user_lookup_mod.InvalidTokenError: - raise ValueError('user lookup token for KBase auth server is invalid, cannot continue') + raise ValueError( + "user lookup token for KBase auth server is invalid, cannot continue" + ) if bad_users: - raise _NoSuchUserError(', '.join([u.id for u in bad_users[:5]])) + raise _NoSuchUserError(", ".join([u.id for u in bad_users[:5]])) def get_key_static_metadata( - self, - keys: List[str], - prefix: Union[bool, None] = False - ) -> Dict[str, Dict[str, PrimitiveType]]: - ''' + self, keys: List[str], prefix: Union[bool, None] = False + ) -> Dict[str, Dict[str, PrimitiveType]]: + """ Get any static metadata associated with the provided list of keys. :param keys: The keys to query. :param prefix: True to query prefix keys, None to query prefix keys but only match exactly, False for standard keys. :returns: A mapping of key to key metadata. - ''' + """ if keys is None: - raise ValueError('keys cannot be None') + raise ValueError("keys cannot be None") if prefix is False: return self._metaval.key_metadata(keys) else: return self._metaval.prefix_key_metadata(keys, exact_match=not bool(prefix)) def create_data_link( - self, - user: UserID, - duid: DataUnitID, - sna: SampleNodeAddress, - update: bool = False, - as_admin: bool = False) -> DataLink: - ''' + self, + user: UserID, + duid: DataUnitID, + sna: SampleNodeAddress, + update: bool = False, + as_admin: bool = False, + ) -> DataLink: + """ Create a link from a data unit to a sample. The user must have admin access to the sample, since linking data grants permissions: once linked, if a user has access to the data unit, the user also has access to the sample. The user must have @@ -363,23 +396,31 @@ def create_data_link( :raises TooManyDataLinksError: if there are too many links from the sample version or the workspace object version. :raises SampleStorageError: if the sample could not be retrieved. - ''' - _not_falsy(user, 'user') - _not_falsy(duid, 'duid') + """ + _not_falsy(user, "user") + _not_falsy(duid, "duid") self._check_perms( - _not_falsy(sna, 'sna').sampleid, user, _SampleAccessType.ADMIN, as_admin=as_admin) + _not_falsy(sna, "sna").sampleid, + user, + _SampleAccessType.ADMIN, + as_admin=as_admin, + ) wsperm = _WorkspaceAccessType.NONE if as_admin else _WorkspaceAccessType.WRITE self._ws.has_permission(user, wsperm, upa=duid.upa) dl = DataLink(self._uuid_gen(), duid, sna, self._now(), user) expired_id = self._storage.create_data_link(dl, update=update) if self._kafka: self._kafka.notify_new_link(dl.id) - if expired_id: # maybe make the notifier accept both notifications & send both? + if ( + expired_id + ): # maybe make the notifier accept both notifications & send both? self._kafka.notify_expired_link(expired_id) return dl - def expire_data_link(self, user: UserID, duid: DataUnitID, as_admin: bool = False) -> None: - ''' + def expire_data_link( + self, user: UserID, duid: DataUnitID, as_admin: bool = False + ) -> None: + """ Expire a data link, ensuring that it will not show up in link queries without an effective timestamp in the past. The user must have admin access to the sample and write access to the data. The data may @@ -392,9 +433,9 @@ def expire_data_link(self, user: UserID, duid: DataUnitID, as_admin: bool = Fals :raises UnauthorizedError: if the user does not have acceptable permissions. :raises NoSuchWorkspaceDataError: if the workspace doesn't exist. :raises NoSuchLinkError: if there is no link from the data unit. - ''' - _not_falsy(user, 'user') - _not_falsy(duid, 'duid') + """ + _not_falsy(user, "user") + _not_falsy(duid, "duid") # allow expiring links for deleted objects. It should be impossible to have a link # for an object that has never existed. wsperm = _WorkspaceAccessType.NONE if as_admin else _WorkspaceAccessType.WRITE @@ -403,7 +444,11 @@ def expire_data_link(self, user: UserID, duid: DataUnitID, as_admin: bool = Fals # was concerned about exposing the sample ID, but if the user has write access to the # UPA then they can get the link with the sample ID, so don't worry about it. self._check_perms( - link.sample_node_address.sampleid, user, _SampleAccessType.ADMIN, as_admin=as_admin) + link.sample_node_address.sampleid, + user, + _SampleAccessType.ADMIN, + as_admin=as_admin, + ) # Use the ID here to prevent a race condition expiring a new link to a different sample # since the user may not have perms # There's a chance the link could be expired between db fetch and update, but that @@ -414,12 +459,13 @@ def expire_data_link(self, user: UserID, duid: DataUnitID, as_admin: bool = Fals self._kafka.notify_expired_link(link.id) def get_links_from_sample( - self, - user: Optional[UserID], - sample: SampleAddress, - timestamp: datetime.datetime = None, - as_admin: bool = False) -> Tuple[List[DataLink], datetime.datetime]: - ''' + self, + user: Optional[UserID], + sample: SampleAddress, + timestamp: datetime.datetime = None, + as_admin: bool = False, + ) -> Tuple[List[DataLink], datetime.datetime]: + """ Get a set of data links originating from a sample at a particular time. :param user: the user requesting the links or None if the user is anonymous. @@ -433,21 +479,24 @@ def get_links_from_sample( :raises NoSuchSampleError: if the sample does not exist. :raises NoSuchSampleVersionError: if the sample version does not exist. :raises NoSuchUserError: if the user does not exist. - ''' - _not_falsy(sample, 'sample') + """ + _not_falsy(sample, "sample") timestamp = self._resolve_timestamp(timestamp) - self._check_perms(sample.sampleid, user, _SampleAccessType.READ, as_admin=as_admin) + self._check_perms( + sample.sampleid, user, _SampleAccessType.READ, as_admin=as_admin + ) wsids = None if as_admin else self._ws.get_user_workspaces(user) # TODO DATALINK what about deleted objects? Currently not handled return self._storage.get_links_from_sample(sample, wsids, timestamp), timestamp def get_batch_links_from_sample_set( - self, - user: Optional[UserID], - samples: List[SampleAddress], - timestamp: datetime.datetime = None, - as_admin: bool = False) -> Tuple[List[DataLink], datetime.datetime]: - ''' + self, + user: Optional[UserID], + samples: List[SampleAddress], + timestamp: datetime.datetime = None, + as_admin: bool = False, + ) -> Tuple[List[DataLink], datetime.datetime]: + """ A batch version of get_links_from_sample. Gets a set of data links originating from multiple samples in a given sampleset at a particular time. @@ -462,8 +511,8 @@ def get_batch_links_from_sample_set( :raises NoSuchSampleError: if the sample does not exist. :raises NoSuchSampleVersionError: if the sample version does not exist. :raises NoSuchUserError: if the user does not exist. - ''' - _not_falsy(samples, 'samples') + """ + _not_falsy(samples, "samples") timestamp = self._resolve_timestamp(timestamp) wsids = None if as_admin else self._ws.get_user_workspaces(user) @@ -473,25 +522,32 @@ def get_batch_links_from_sample_set( # TODO: consider adding support for sample set refs, so that we don't have to check # perms for every individual sample. It does not appear to create a significant # bottleneck as implemented but it could help with a lot of samples at once - self._check_perms(sample.sampleid, user, _SampleAccessType.READ, as_admin=as_admin) - return_links.extend(self._storage.get_links_from_sample(sample, wsids, timestamp)) + self._check_perms( + sample.sampleid, user, _SampleAccessType.READ, as_admin=as_admin + ) + return_links.extend( + self._storage.get_links_from_sample(sample, wsids, timestamp) + ) return return_links, timestamp - def _resolve_timestamp(self, timestamp: datetime.datetime = None) -> datetime.datetime: + def _resolve_timestamp( + self, timestamp: datetime.datetime = None + ) -> datetime.datetime: if timestamp: - _check_timestamp(timestamp, 'timestamp') + _check_timestamp(timestamp, "timestamp") else: timestamp = self._now() return timestamp def get_links_from_data( - self, - user: Optional[UserID], - upa: UPA, - timestamp: datetime.datetime = None, - as_admin: bool = False) -> Tuple[List[DataLink], datetime.datetime]: - ''' + self, + user: Optional[UserID], + upa: UPA, + timestamp: datetime.datetime = None, + as_admin: bool = False, + ) -> Tuple[List[DataLink], datetime.datetime]: + """ Get a set of data links originating from a workspace object at a particular time. :param user: the user requesting the links, or None for an anonymous user. @@ -503,10 +559,10 @@ def get_links_from_data( :returns: a tuple consisting of a list of links and the timestamp used to query the links. :raises UnauthorizedError: if the user does not have read permission for the data. :raises NoSuchWorkspaceDataError: if the data does not exist. - ''' + """ # may need to make this independent of the workspace. YAGNI. # handle ref path? - _not_falsy(upa, 'upa') + _not_falsy(upa, "upa") timestamp = self._resolve_timestamp(timestamp) # NONE still checks that WS/obj exists. If it's deleted this method should fail wsperm = _WorkspaceAccessType.NONE if as_admin else _WorkspaceAccessType.READ @@ -514,11 +570,9 @@ def get_links_from_data( return self._storage.get_links_from_data(upa, timestamp), timestamp def get_sample_via_data( - self, - user: Optional[UserID], - upa: UPA, - sample_address: SampleAddress) -> SavedSample: - ''' + self, user: Optional[UserID], upa: UPA, sample_address: SampleAddress + ) -> SavedSample: + """ Given a workspace object, get a sample linked to that object. The user must have read permissions for the object, but not necessarily the sample. The link may be expired. @@ -530,23 +584,24 @@ def get_sample_via_data( :raises NoSuchWorkspaceDataError: if the workspace object does not exist. :raises NoSuchLinkError: if there is no link from the object to the sample. :raises NoSuchSampleVersionError: if the sample version does not exist. - ''' + """ # no admin mode needed - use get_links or get sample # may need to make this independent of the workspace. YAGNI. # handle ref path? - _not_falsy(upa, 'upa') - _not_falsy(sample_address, 'sample_address') + _not_falsy(upa, "upa") + _not_falsy(sample_address, "sample_address") # the order of these checks is important, check read first, then we know link & sample # access is ok self._ws.has_permission(user, _WorkspaceAccessType.READ, upa=upa) if not self._storage.has_data_link(upa, sample_address.sampleid): raise _NoSuchLinkError( - f'There is no link from UPA {upa} to sample {sample_address.sampleid}') + f"There is no link from UPA {upa} to sample {sample_address.sampleid}" + ) # can't raise no sample error since a link exists return self._storage.get_sample(sample_address.sampleid, sample_address.version) def get_data_link_admin(self, link_id: UUID) -> DataLink: - ''' + """ This method is intended for admin use and should not be exposed in a public API. Get a link by its ID. The link may be expired. @@ -554,18 +609,18 @@ def get_data_link_admin(self, link_id: UUID) -> DataLink: :param link_id: the link ID. :returns: the link. :raises NoSuchLinkError: if the link does not exist. - ''' + """ # if we expose this to users need to add ACL checking. Don't see a use case ATM. - return self._storage.get_data_link(_not_falsy(link_id, 'link_id')) + return self._storage.get_data_link(_not_falsy(link_id, "link_id")) def validate_sample(self, sample: Sample): - ''' + """ This method performs only the validation steps on a sample :param sample: the sample to validate - ''' - _not_falsy(sample, 'sample') + """ + _not_falsy(sample, "sample") error_detail = self._validate_metadata(sample, return_error_detail=True) for e in error_detail: - e['sample_name'] = sample.name + e["sample_name"] = sample.name return error_detail diff --git a/lib/SampleService/core/storage/arango_sample_storage.py b/lib/SampleService/core/storage/arango_sample_storage.py index 6a66f847..b5ae637c 100644 --- a/lib/SampleService/core/storage/arango_sample_storage.py +++ b/lib/SampleService/core/storage/arango_sample_storage.py @@ -1,6 +1,6 @@ -''' +""" An ArangoDB based storage system for the Sample service. -''' +""" # READ BEFORE CHANGING STUFF @@ -74,7 +74,9 @@ from typing import List, Tuple, Callable, cast as _cast, Optional, Sequence as _Sequence from typing import Dict as _Dict, Any as _Any -from apscheduler.schedulers.background import BackgroundScheduler as _BackgroundScheduler +from apscheduler.schedulers.background import ( + BackgroundScheduler as _BackgroundScheduler, +) from arango.database import StandardDatabase from SampleService.core.acls import SampleACL, SampleACLDelta @@ -109,95 +111,97 @@ from SampleService.core.user import UserID from SampleService.core.workspace import DataUnitID, UPA -_FLD_ARANGO_KEY = '_key' -_FLD_ARANGO_FROM = '_from' -_FLD_ARANGO_TO = '_to' -_FLD_ID = 'id' -_FLD_UUID_VER = 'uuidver' -_FLD_VER = 'ver' +_FLD_ARANGO_KEY = "_key" +_FLD_ARANGO_FROM = "_from" +_FLD_ARANGO_TO = "_to" +_FLD_ID = "id" +_FLD_UUID_VER = "uuidver" +_FLD_VER = "ver" _VAL_NO_VER = -1 -_FLD_NAME = 'name' -_FLD_USER = 'user' -_FLD_SAVE_TIME = 'saved' -_FLD_ACL_UPDATE_TIME = 'aclupdate' - -_FLD_NODE_NAME = 'name' -_FLD_NODE_TYPE = 'type' -_FLD_NODE_PARENT = 'parent' -_FLD_NODE_SAMPLE_ID = 'id' -_FLD_NODE_VER = 'ver' -_FLD_NODE_UUID_VER = 'uuidver' -_FLD_NODE_INDEX = 'index' -_FLD_NODE_CONTROLLED_METADATA = 'cmeta' -_FLD_NODE_UNCONTROLLED_METADATA = 'ucmeta' -_FLD_NODE_SOURCE_METADATA = 'smeta' -_FLD_NODE_META_OUTER_KEY = 'ok' -_FLD_NODE_META_KEY = 'k' -_FLD_NODE_META_SOURCE_KEY = 'sk' -_FLD_NODE_META_VALUE = 'v' - - -_FLD_ACLS = 'acls' -_FLD_OWNER = 'owner' -_FLD_READ = 'read' -_FLD_WRITE = 'write' -_FLD_ADMIN = 'admin' -_FLD_PUBLIC_READ = 'pubread' - -_FLD_VERSIONS = 'vers' - -_FLD_LINK_ID = 'id' -_FLD_LINK_WORKSPACE_ID = 'wsid' -_FLD_LINK_OBJECT_ID = 'objid' -_FLD_LINK_OBJECT_VERSION = 'objver' -_FLD_LINK_OBJECT_DATA_UNIT = 'dataid' -_FLD_LINK_SAMPLE_ID = 'sampleid' -_FLD_LINK_SAMPLE_UUID_VERSION = 'samuuidver' -_FLD_LINK_SAMPLE_INT_VERSION = 'samintver' -_FLD_LINK_SAMPLE_NODE = 'node' -_FLD_LINK_CREATED = 'created' -_FLD_LINK_CREATED_BY = 'createby' -_FLD_LINK_EXPIRED = 'expired' -_FLD_LINK_EXPIRED_BY = 'expireby' +_FLD_NAME = "name" +_FLD_USER = "user" +_FLD_SAVE_TIME = "saved" +_FLD_ACL_UPDATE_TIME = "aclupdate" + +_FLD_NODE_NAME = "name" +_FLD_NODE_TYPE = "type" +_FLD_NODE_PARENT = "parent" +_FLD_NODE_SAMPLE_ID = "id" +_FLD_NODE_VER = "ver" +_FLD_NODE_UUID_VER = "uuidver" +_FLD_NODE_INDEX = "index" +_FLD_NODE_CONTROLLED_METADATA = "cmeta" +_FLD_NODE_UNCONTROLLED_METADATA = "ucmeta" +_FLD_NODE_SOURCE_METADATA = "smeta" +_FLD_NODE_META_OUTER_KEY = "ok" +_FLD_NODE_META_KEY = "k" +_FLD_NODE_META_SOURCE_KEY = "sk" +_FLD_NODE_META_VALUE = "v" + + +_FLD_ACLS = "acls" +_FLD_OWNER = "owner" +_FLD_READ = "read" +_FLD_WRITE = "write" +_FLD_ADMIN = "admin" +_FLD_PUBLIC_READ = "pubread" + +_FLD_VERSIONS = "vers" + +_FLD_LINK_ID = "id" +_FLD_LINK_WORKSPACE_ID = "wsid" +_FLD_LINK_OBJECT_ID = "objid" +_FLD_LINK_OBJECT_VERSION = "objver" +_FLD_LINK_OBJECT_DATA_UNIT = "dataid" +_FLD_LINK_SAMPLE_ID = "sampleid" +_FLD_LINK_SAMPLE_UUID_VERSION = "samuuidver" +_FLD_LINK_SAMPLE_INT_VERSION = "samintver" +_FLD_LINK_SAMPLE_NODE = "node" +_FLD_LINK_CREATED = "created" +_FLD_LINK_CREATED_BY = "createby" +_FLD_LINK_EXPIRED = "expired" +_FLD_LINK_EXPIRED_BY = "expireby" # see https://www.arangodb.com/2018/07/time-traveling-with-graph-databases/ -_ARANGO_MAX_INTEGER = 2**53 - 1 +_ARANGO_MAX_INTEGER = 2 ** 53 - 1 -_JOB_ID = 'consistencyjob' +_JOB_ID = "consistencyjob" # schema version checking constants. # the current version of the database schema. _SCHEMA_VERSION = 1 # the value for the schema key. -_SCHEMA_VALUE = 'schema' +_SCHEMA_VALUE = "schema" # whether the schema is in the process of an update. Value is a boolean. -_FLD_SCHEMA_UPDATE = 'inupdate' +_FLD_SCHEMA_UPDATE = "inupdate" # the version of the schema. Value is _SCHEMA_VERSION. -_FLD_SCHEMA_VERSION = 'schemaver' +_FLD_SCHEMA_VERSION = "schemaver" class ArangoSampleStorage: - ''' + """ The ArangoDB storage wrapper. - ''' + """ def __init__( - self, - db: StandardDatabase, - sample_collection: str, - version_collection: str, - version_edge_collection: str, - node_collection: str, - node_edge_collection: str, - workspace_object_version_shadow_collection: str, - data_link_collection: str, - schema_collection: str, - # See https://kbase.slack.com/archives/CNRT78G66/p1583967289053500 for justification - max_links: int = 10000, - now: Callable[[], datetime.datetime] = lambda: datetime.datetime.now( - tz=datetime.timezone.utc)): - ''' + self, + db: StandardDatabase, + sample_collection: str, + version_collection: str, + version_edge_collection: str, + node_collection: str, + node_edge_collection: str, + workspace_object_version_shadow_collection: str, + data_link_collection: str, + schema_collection: str, + # See https://kbase.slack.com/archives/CNRT78G66/p1583967289053500 for justification + max_links: int = 10000, + now: Callable[[], datetime.datetime] = lambda: datetime.datetime.now( + tz=datetime.timezone.utc + ), + ): + """ Create the wrapper. Note that the database consistency checker will not start until the start_consistency_checker() method is called. @@ -218,36 +222,55 @@ def __init__( object version to sample nodes will be stored, indicating data links. :schema_collection: the name of the collection in which information about the database schema will be stored. - ''' + """ # Don't publicize these params, for testing only # :param max_links: The maximum links any one sample version or workspace object version # can have. # :param now: A callable that returns the current time. Primarily used for testing. # Maybe make a configuration class...? - self._db = _not_falsy(db, 'db') - self._now = _not_falsy(now, 'now') + self._db = _not_falsy(db, "db") + self._now = _not_falsy(now, "now") self._max_links = max_links self._col_sample = _init_collection( - db, sample_collection, 'sample collection', 'sample_collection') + db, sample_collection, "sample collection", "sample_collection" + ) self._col_version = _init_collection( - db, version_collection, 'version collection', 'version_collection') + db, version_collection, "version collection", "version_collection" + ) self._col_ver_edge = _init_collection( - db, version_edge_collection, 'version edge collection', 'version_edge_collection', - edge=True) + db, + version_edge_collection, + "version edge collection", + "version_edge_collection", + edge=True, + ) self._col_nodes = _init_collection( - db, node_collection, 'node collection', 'node_collection') + db, node_collection, "node collection", "node_collection" + ) self._col_node_edge = _init_collection( - db, node_edge_collection, 'node edge collection', 'node_edge_collection', edge=True) + db, + node_edge_collection, + "node edge collection", + "node_edge_collection", + edge=True, + ) self._col_ws = _init_collection( db, workspace_object_version_shadow_collection, - 'workspace object version shadow collection', - 'workspace_object_version_shadow_collection') + "workspace object version shadow collection", + "workspace_object_version_shadow_collection", + ) self._col_data_link = _init_collection( - db, data_link_collection, 'data link collection', 'data_link_collection', edge=True) + db, + data_link_collection, + "data link collection", + "data_link_collection", + edge=True, + ) self._col_schema = _init_collection( - db, schema_collection, 'schema collection', 'schema_collection') + db, schema_collection, "schema collection", "schema_collection" + ) self._ensure_indexes() self._check_schema() self._deletion_delay = datetime.timedelta(hours=1) # make configurable? @@ -259,51 +282,65 @@ def _ensure_indexes(self): self._col_node_edge.add_persistent_index([_FLD_UUID_VER]) self._col_ver_edge.add_persistent_index([_FLD_UUID_VER]) self._col_version.add_persistent_index([_FLD_UUID_VER]) - self._col_version.add_persistent_index([_FLD_VER]) # partial index would be useful + self._col_version.add_persistent_index( + [_FLD_VER] + ) # partial index would be useful self._col_nodes.add_persistent_index([_FLD_UUID_VER]) - self._col_nodes.add_persistent_index([_FLD_VER]) # partial index would be useful + self._col_nodes.add_persistent_index( + [_FLD_VER] + ) # partial index would be useful # find links by ID self._col_data_link.add_persistent_index([_FLD_LINK_ID]) # find links from objects self._col_data_link.add_persistent_index( - [_FLD_LINK_WORKSPACE_ID, _FLD_LINK_OBJECT_ID, _FLD_LINK_OBJECT_VERSION]) + [_FLD_LINK_WORKSPACE_ID, _FLD_LINK_OBJECT_ID, _FLD_LINK_OBJECT_VERSION] + ) # find links from sample versions self._col_data_link.add_persistent_index([_FLD_LINK_SAMPLE_UUID_VERSION]) # find links from samples self._col_data_link.add_persistent_index([_FLD_LINK_SAMPLE_ID]) except _arango.exceptions.IndexCreateError as e: # this is a real pain to test. - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def _check_schema(self): col = self._col_schema try: - col.insert({_FLD_ARANGO_KEY: _SCHEMA_VALUE, - _FLD_SCHEMA_UPDATE: False, - _FLD_SCHEMA_VERSION: _SCHEMA_VERSION}) + col.insert( + { + _FLD_ARANGO_KEY: _SCHEMA_VALUE, + _FLD_SCHEMA_UPDATE: False, + _FLD_SCHEMA_VERSION: _SCHEMA_VERSION, + } + ) except _arango.exceptions.DocumentInsertError as e: if e.error_code != 1210: # unique constraint violation code # this is a real pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError( + "Connection to database failed: " + str(e) + ) from e # ok, the schema version document is already there, this isn't the first time this # database as been used. Now check the document is ok. try: if col.count() != 1: raise _StorageInitError( - 'Multiple config objects found in the database. ' + - 'This should not happen, something is very wrong.') + "Multiple config objects found in the database. " + + "This should not happen, something is very wrong." + ) cfgdoc = col.get(_SCHEMA_VALUE) if cfgdoc[_FLD_SCHEMA_VERSION] != _SCHEMA_VERSION: raise _StorageInitError( - f'Incompatible database schema. Server is v{_SCHEMA_VERSION}, ' + - f'DB is v{cfgdoc[_FLD_SCHEMA_VERSION]}') + f"Incompatible database schema. Server is v{_SCHEMA_VERSION}, " + + f"DB is v{cfgdoc[_FLD_SCHEMA_VERSION]}" + ) if cfgdoc[_FLD_SCHEMA_UPDATE]: raise _StorageInitError( - 'The database is in the middle of an update from ' + - f'v{cfgdoc[_FLD_SCHEMA_VERSION]} of the schema. Aborting startup.') + "The database is in the middle of an update from " + + f"v{cfgdoc[_FLD_SCHEMA_VERSION]} of the schema. Aborting startup." + ) except _arango.exceptions.ArangoServerError as e: # this is a real pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def _check_db_updated(self): self._check_col_updated(self._col_version) @@ -316,20 +353,24 @@ def _check_col_updated(self, col): for doc in cur: id_ = UUID(doc[_FLD_ID]) uver = UUID(doc[_FLD_UUID_VER]) - ts = self._timestamp_to_datetime(self._timestamp_milliseconds_to_seconds(doc[_FLD_SAVE_TIME])) + ts = self._timestamp_to_datetime( + self._timestamp_milliseconds_to_seconds(doc[_FLD_SAVE_TIME]) + ) sampledoc = self._get_sample_doc(id_, exception=False) if not sampledoc: # the sample document was never saved for this version doc self._delete_version_and_node_docs(uver, ts) else: - version = self._get_int_version_from_sample_doc(sampledoc, str(uver)) + version = self._get_int_version_from_sample_doc( + sampledoc, str(uver) + ) if version: self._update_version_and_node_docs_with_find(id_, uver, version) else: self._delete_version_and_node_docs(uver, ts) except _arango.exceptions.DocumentGetError as e: # this is a real pain to test. - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def _get_int_version_from_sample_doc(self, sampledoc, uuidverstr): for i, v in enumerate(sampledoc[_FLD_VERSIONS]): @@ -339,7 +380,7 @@ def _get_int_version_from_sample_doc(self, sampledoc, uuidverstr): def _delete_version_and_node_docs(self, uuidver, savedate): if self._now() - savedate > self._deletion_delay: - print('deleting docs', self._now(), savedate, self._deletion_delay) + print("deleting docs", self._now(), savedate, self._deletion_delay) try: # TODO logging # delete edge docs first to ensure we don't orphan them @@ -349,34 +390,38 @@ def _delete_version_and_node_docs(self, uuidver, savedate): self._col_nodes.delete_match({_FLD_UUID_VER: str(uuidver)}) except _arango.exceptions.DocumentDeleteError as e: # this is a real pain to test. - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError( + "Connection to database failed: " + str(e) + ) from e def _build_scheduler(self): schd = _BackgroundScheduler() - schd.add_job(self._check_db_updated, 'interval', seconds=1, id=_JOB_ID) + schd.add_job(self._check_db_updated, "interval", seconds=1, id=_JOB_ID) schd.start(paused=True) return schd def start_consistency_checker(self, interval_sec=60): - ''' + """ Start the database consistency checker. In production use the consistency checker should always be on. :param interval_ms: How frequently to run the scheduler in seconds. Defaults to one minute. - ''' + """ if interval_sec < 1: - raise ValueError('interval_sec must be > 0') - self._scheduler.reschedule_job(_JOB_ID, trigger='interval', seconds=interval_sec) + raise ValueError("interval_sec must be > 0") + self._scheduler.reschedule_job( + _JOB_ID, trigger="interval", seconds=interval_sec + ) self._scheduler.resume() def stop_consistency_checker(self): - ''' + """ Stop the consistency checker. - ''' + """ self._scheduler.pause() def save_sample(self, sample: SavedSample) -> bool: - ''' + """ Save a new sample. The version in the sample object, if any, is ignored. The timestamp in the sample is expected to be accurate - the database may become corrupted @@ -386,8 +431,8 @@ def save_sample(self, sample: SavedSample) -> bool: :param sample: The sample to save. :returns: True if the sample saved successfully, False if the same ID already exists. :raises SampleStorageError: if the sample fails to save. - ''' - _not_falsy(sample, 'sample') + """ + _not_falsy(sample, "sample") if self._get_sample_doc(sample.id, exception=False): return False # bail early return self._save_sample_pt2(sample) @@ -401,18 +446,20 @@ def _save_sample_pt2(self, sample: SavedSample) -> bool: self._save_version_and_node_docs(sample, versionid) # create sample document, adding uuid to version list - tosave = {_FLD_ARANGO_KEY: str(sample.id), - # yes, this is redundant. It'll match the ver & node collectons though - _FLD_ID: str(sample.id), # TODO test this is saved - _FLD_VERSIONS: [str(versionid)], - _FLD_ACL_UPDATE_TIME: sample.savetime.timestamp(), - _FLD_ACLS: {_FLD_OWNER: sample.user.id, - _FLD_ADMIN: [], - _FLD_WRITE: [], - _FLD_READ: [], - _FLD_PUBLIC_READ: False - } - } + tosave = { + _FLD_ARANGO_KEY: str(sample.id), + # yes, this is redundant. It'll match the ver & node collectons though + _FLD_ID: str(sample.id), # TODO test this is saved + _FLD_VERSIONS: [str(versionid)], + _FLD_ACL_UPDATE_TIME: sample.savetime.timestamp(), + _FLD_ACLS: { + _FLD_OWNER: sample.user.id, + _FLD_ADMIN: [], + _FLD_WRITE: [], + _FLD_READ: [], + _FLD_PUBLIC_READ: False, + }, + } try: self._col_sample.insert(tosave) except _arango.exceptions.DocumentInsertError as e: @@ -420,28 +467,37 @@ def _save_sample_pt2(self, sample: SavedSample) -> bool: if e.error_code == 1210: # unique constraint violation code return False else: # this is a real pain to test. - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError( + "Connection to database failed: " + str(e) + ) from e self._update_version_and_node_docs(sample, versionid, 1) return True - def _update_version_and_node_docs(self, sample: SavedSample, versionid: UUID, version: int): + def _update_version_and_node_docs( + self, sample: SavedSample, versionid: UUID, version: int + ): nodeupdates: List[dict] = [] for n in sample.nodes: - ndoc = {_FLD_ARANGO_KEY: self._get_node_id(sample.id, versionid, n.name), - _FLD_NODE_VER: version, - } + ndoc = { + _FLD_ARANGO_KEY: self._get_node_id(sample.id, versionid, n.name), + _FLD_NODE_VER: version, + } nodeupdates.append(ndoc) self._update_many(self._col_nodes, nodeupdates) verdocid = self._get_version_id(sample.id, versionid) self._update(self._col_version, {_FLD_ARANGO_KEY: verdocid, _FLD_VER: version}) - def _update_version_and_node_docs_with_find(self, id_: UUID, versionid: UUID, version: int): + def _update_version_and_node_docs_with_find( + self, id_: UUID, versionid: UUID, version: int + ): try: - self._col_nodes.update_match({_FLD_UUID_VER: str(versionid)}, {_FLD_VER: version}) + self._col_nodes.update_match( + {_FLD_UUID_VER: str(versionid)}, {_FLD_VER: version} + ) except _arango.exceptions.DocumentUpdateError as e: # this is a real pain to test. - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e verdocid = self._get_version_id(id_, versionid) self._update(self._col_version, {_FLD_ARANGO_KEY: verdocid, _FLD_VER: version}) @@ -453,29 +509,35 @@ def _save_version_and_node_docs(self, sample: SavedSample, versionid: UUID): nodeedgedocs: List[dict] = [] for index, n in enumerate(sample.nodes): key = self._get_node_id(sample.id, versionid, n.name) - ndoc = {_FLD_ARANGO_KEY: key, - _FLD_NODE_SAMPLE_ID: str(sample.id), - _FLD_NODE_UUID_VER: str(versionid), - _FLD_NODE_VER: _VAL_NO_VER, - _FLD_SAVE_TIME: self._timestamp_seconds_to_milliseconds(sample.savetime.timestamp()), - _FLD_NODE_NAME: n.name, - _FLD_NODE_TYPE: n.type.name, - _FLD_NODE_PARENT: n.parent, - _FLD_NODE_INDEX: index, - _FLD_NODE_CONTROLLED_METADATA: self._meta_to_list(n.controlled_metadata), - _FLD_NODE_UNCONTROLLED_METADATA: self._meta_to_list(n.user_metadata), - _FLD_NODE_SOURCE_METADATA: self._source_meta_to_list(n.source_metadata), - } + ndoc = { + _FLD_ARANGO_KEY: key, + _FLD_NODE_SAMPLE_ID: str(sample.id), + _FLD_NODE_UUID_VER: str(versionid), + _FLD_NODE_VER: _VAL_NO_VER, + _FLD_SAVE_TIME: self._timestamp_seconds_to_milliseconds( + sample.savetime.timestamp() + ), + _FLD_NODE_NAME: n.name, + _FLD_NODE_TYPE: n.type.name, + _FLD_NODE_PARENT: n.parent, + _FLD_NODE_INDEX: index, + _FLD_NODE_CONTROLLED_METADATA: self._meta_to_list( + n.controlled_metadata + ), + _FLD_NODE_UNCONTROLLED_METADATA: self._meta_to_list(n.user_metadata), + _FLD_NODE_SOURCE_METADATA: self._source_meta_to_list(n.source_metadata), + } if n.type == _SubSampleType.BIOLOGICAL_REPLICATE: - to = f'{self._col_version.name}/{verdocid}' + to = f"{self._col_version.name}/{verdocid}" else: parentid = self._get_node_id(sample.id, versionid, _cast(str, n.parent)) - to = f'{self._col_nodes.name}/{parentid}' - nedoc = {_FLD_ARANGO_KEY: key, - _FLD_UUID_VER: str(versionid), - _FLD_ARANGO_FROM: f'{self._col_nodes.name}/{key}', - _FLD_ARANGO_TO: to - } + to = f"{self._col_nodes.name}/{parentid}" + nedoc = { + _FLD_ARANGO_KEY: key, + _FLD_UUID_VER: str(versionid), + _FLD_ARANGO_FROM: f"{self._col_nodes.name}/{key}", + _FLD_ARANGO_TO: to, + } nodedocs.append(ndoc) nodeedgedocs.append(nedoc) self._insert_many(self._col_nodes, nodedocs) @@ -484,24 +546,28 @@ def _save_version_and_node_docs(self, sample: SavedSample, versionid: UUID): self._insert_many(self._col_node_edge, nodeedgedocs) # save version document - verdoc = {_FLD_ARANGO_KEY: verdocid, - _FLD_ID: str(sample.id), - _FLD_USER: sample.user.id, - _FLD_VER: _VAL_NO_VER, - _FLD_UUID_VER: str(versionid), - _FLD_SAVE_TIME: self._timestamp_seconds_to_milliseconds(sample.savetime.timestamp()), - _FLD_NAME: sample.name - # TODO description - } + verdoc = { + _FLD_ARANGO_KEY: verdocid, + _FLD_ID: str(sample.id), + _FLD_USER: sample.user.id, + _FLD_VER: _VAL_NO_VER, + _FLD_UUID_VER: str(versionid), + _FLD_SAVE_TIME: self._timestamp_seconds_to_milliseconds( + sample.savetime.timestamp() + ), + _FLD_NAME: sample.name + # TODO description + } self._insert(self._col_version, verdoc) # TODO this actually isn't tested by anything since we're not doing traversals yet, but # it will be - veredgedoc = {_FLD_ARANGO_KEY: verdocid, - _FLD_UUID_VER: str(versionid), - _FLD_ARANGO_FROM: f'{self._col_version.name}/{verdocid}', - _FLD_ARANGO_TO: f'{self._col_sample.name}/{sample.id}', - } + veredgedoc = { + _FLD_ARANGO_KEY: verdocid, + _FLD_UUID_VER: str(versionid), + _FLD_ARANGO_FROM: f"{self._col_version.name}/{verdocid}", + _FLD_ARANGO_TO: f"{self._col_sample.name}/{sample.id}", + } self._insert(self._col_ver_edge, veredgedoc) # TODO may need to make a meta collection. See below. @@ -510,65 +576,91 @@ def _save_version_and_node_docs(self, sample: SavedSample, versionid: UUID): # Maybe need a doc for each metadata value, which implies going through the whole # save / version update routine the other docs go through # But that means you can't query for metadata on traversals - def _meta_to_list(self, m: _Dict[str, _Dict[str, _PrimitiveType]]) -> List[_Dict[str, _Any]]: + def _meta_to_list( + self, m: _Dict[str, _Dict[str, _PrimitiveType]] + ) -> List[_Dict[str, _Any]]: ret = [] for k in m: - ret.extend([{_FLD_NODE_META_OUTER_KEY: k, - _FLD_NODE_META_KEY: ik, - _FLD_NODE_META_VALUE: m[k][ik]} - for ik in m[k]] - ) + ret.extend( + [ + { + _FLD_NODE_META_OUTER_KEY: k, + _FLD_NODE_META_KEY: ik, + _FLD_NODE_META_VALUE: m[k][ik], + } + for ik in m[k] + ] + ) return ret def _list_to_meta( - self, list_: List[_Dict[str, _Any]]) -> _Dict[str, _Dict[str, _PrimitiveType]]: + self, list_: List[_Dict[str, _Any]] + ) -> _Dict[str, _Dict[str, _PrimitiveType]]: ret: _Dict[str, _Dict[str, _PrimitiveType]] = defaultdict(dict) for m in list_: - ret[m[_FLD_NODE_META_OUTER_KEY]][m[_FLD_NODE_META_KEY]] = m[_FLD_NODE_META_VALUE] - return dict(ret) # some libs don't play nice with default dict, in particular maps + ret[m[_FLD_NODE_META_OUTER_KEY]][m[_FLD_NODE_META_KEY]] = m[ + _FLD_NODE_META_VALUE + ] + return dict( + ret + ) # some libs don't play nice with default dict, in particular maps # source metadata is informational only and is not expected to be queryable - def _source_meta_to_list(self, sm: _Sequence[_SourceMetadata]) -> List[_Dict[str, _Any]]: - return [{_FLD_NODE_META_KEY: m.key, - _FLD_NODE_META_SOURCE_KEY: m.sourcekey, - _FLD_NODE_META_VALUE: dict(m.sourcevalue)} for m in sm] - - def _list_to_source_meta(self, list_: List[_Dict[str, _Any]]) -> List[_SourceMetadata]: + def _source_meta_to_list( + self, sm: _Sequence[_SourceMetadata] + ) -> List[_Dict[str, _Any]]: + return [ + { + _FLD_NODE_META_KEY: m.key, + _FLD_NODE_META_SOURCE_KEY: m.sourcekey, + _FLD_NODE_META_VALUE: dict(m.sourcevalue), + } + for m in sm + ] + + def _list_to_source_meta( + self, list_: List[_Dict[str, _Any]] + ) -> List[_SourceMetadata]: # allow for compatibility with old samples without a source meta field if not list_: return [] - return [_SourceMetadata( - sm[_FLD_NODE_META_KEY], - sm[_FLD_NODE_META_SOURCE_KEY], - sm[_FLD_NODE_META_VALUE] - ) for sm in list_] + return [ + _SourceMetadata( + sm[_FLD_NODE_META_KEY], + sm[_FLD_NODE_META_SOURCE_KEY], + sm[_FLD_NODE_META_VALUE], + ) + for sm in list_ + ] def _insert(self, col, doc, upsert=False): try: col.insert(doc, silent=True, overwrite=upsert) except _arango.exceptions.DocumentInsertError as e: # this is a real pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def _insert_many(self, col, docs): try: col.insert_many(docs, silent=True) except _arango.exceptions.DocumentInsertError as e: # this is a real pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def _update(self, col, doc): try: col.update(doc, silent=True, keep_none=True) except _arango.exceptions.DocumentUpdateError as e: # this is a real pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def _update_many(self, col, docs): try: col.update_many(docs, silent=True) except _arango.exceptions.DocumentUpdateError as e: # this is a real pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e - def save_sample_version(self, sample: SavedSample, prior_version: int = None) -> int: - ''' + def save_sample_version( + self, sample: SavedSample, prior_version: int = None + ) -> int: + """ Save a new version of a sample. The sample must already exist in the DB. Any version in the provided sample object is ignored. @@ -581,17 +673,19 @@ def save_sample_version(self, sample: SavedSample, prior_version: int = None) -> :return: the version of the saved sample. :raises SampleStorageError: if the sample fails to save. :raises ConcurrencyError: if the sample's version is not equal to prior_version. - ''' - _not_falsy(sample, 'sample') + """ + _not_falsy(sample, "sample") if prior_version is not None and prior_version < 1: - raise ValueError('prior_version must be > 0') + raise ValueError("prior_version must be > 0") sampledoc = self._get_sample_doc(sample.id, exception=False) if not sampledoc: raise _NoSuchSampleError(str(sample.id)) # bail early version = len(sampledoc[_FLD_VERSIONS]) if prior_version and version != prior_version: - raise _ConcurrencyError(f'Version required for sample {sample.id} is ' + - f'{prior_version}, but current version is {version}') + raise _ConcurrencyError( + f"Version required for sample {sample.id} is " + + f"{prior_version}, but current version is {version}" + ) return self._save_sample_version_pt2(sample, prior_version) @@ -603,26 +697,27 @@ def _save_sample_version_pt2(self, sample, prior_version) -> int: self._save_version_and_node_docs(sample, versionid) - aql = f''' + aql = f""" FOR s IN @@col - FILTER s.{_FLD_ARANGO_KEY} == @sampleid''' + FILTER s.{_FLD_ARANGO_KEY} == @sampleid""" if prior_version: - aql += f''' - FILTER LENGTH(s.{_FLD_VERSIONS}) == @version_count''' - aql += f''' + aql += f""" + FILTER LENGTH(s.{_FLD_VERSIONS}) == @version_count""" + aql += f""" UPDATE s WITH {{{_FLD_VERSIONS}: PUSH(s.{_FLD_VERSIONS}, @verid)}} IN @@col RETURN NEW - ''' + """ try: # we checked that the doc existed above, so it must exist now. # We assume here that you cannot delete samples from the DB. That's the plan as of now. - bind_vars = {'@col': self._col_sample.name, - 'sampleid': str(sample.id), - 'verid': str(versionid), - } + bind_vars = { + "@col": self._col_sample.name, + "sampleid": str(sample.id), + "verid": str(versionid), + } if prior_version: - bind_vars['version_count'] = prior_version + bind_vars["version_count"] = prior_version cur = self._db.aql.execute(aql, bind_vars=bind_vars) if not cur.empty(): version = len(cur.next()[_FLD_VERSIONS]) @@ -633,18 +728,20 @@ def _save_sample_version_pt2(self, sample, prior_version) -> int: # that the aql doesn't find the doc, then the version gets incremented, and the # version is ok here. That'll take millisecond timing though and the result is # one spurious error so we don't worry about it for now. - raise _ConcurrencyError(f'Version required for sample {sample.id} is ' + - f'{prior_version}, but current version is {version}') + raise _ConcurrencyError( + f"Version required for sample {sample.id} is " + + f"{prior_version}, but current version is {version}" + ) except _arango.exceptions.AQLQueryExecuteError as e: # let the reaper clean up any left over docs # this is a real pain to test. - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e self._update_version_and_node_docs(sample, versionid, version) return version def get_sample(self, id_: UUID, version: int = None) -> SavedSample: - ''' + """ Get a sample from the database. :param id_: the ID of the sample. :param version: The version of the sample to retrieve. Defaults to the latest version. @@ -652,74 +749,102 @@ def get_sample(self, id_: UUID, version: int = None) -> SavedSample: :raises NoSuchSampleError: if the sample does not exist. :raises NoSuchSampleVersionError: if the sample version does not exist. :raises SampleStorageError: if the sample could not be retrieved. - ''' + """ doc, verdoc, version = self._get_sample_and_version_doc(id_, version) nodes = self._get_nodes(id_, UUID(verdoc[_FLD_NODE_UUID_VER]), version) - dt = self._timestamp_to_datetime(self._timestamp_milliseconds_to_seconds(verdoc[_FLD_SAVE_TIME])) + dt = self._timestamp_to_datetime( + self._timestamp_milliseconds_to_seconds(verdoc[_FLD_SAVE_TIME]) + ) return SavedSample( - UUID(doc[_FLD_ID]), UserID(verdoc[_FLD_USER]), nodes, dt, verdoc[_FLD_NAME], version) + UUID(doc[_FLD_ID]), + UserID(verdoc[_FLD_USER]), + nodes, + dt, + verdoc[_FLD_NAME], + version, + ) def get_samples(self, ids_: List[_Dict[str, _Any]]) -> List[SavedSample]: - ''' + """ ids_: list of dictionaries containing "id" and "version" field. - ''' + """ docs, verdocs, versions = self._get_many_sample_and_version_doc(ids_) samples = [] for id_tup in ids_: - id_ = str(id_tup['id']) + id_ = str(id_tup["id"]) verdoc = verdocs[id_] - nodes = self._get_nodes(UUID(id_), UUID(verdoc[_FLD_NODE_UUID_VER]), versions[id_]) + nodes = self._get_nodes( + UUID(id_), UUID(verdoc[_FLD_NODE_UUID_VER]), versions[id_] + ) doc = docs[id_] version = versions[id_] dt = self._timestamp_to_datetime( self._timestamp_milliseconds_to_seconds(verdoc[_FLD_SAVE_TIME]) ) - samples.append(SavedSample( - UUID(doc[_FLD_ID]), - UserID(verdoc[_FLD_USER]), - nodes, dt, verdoc[_FLD_NAME], version - )) + samples.append( + SavedSample( + UUID(doc[_FLD_ID]), + UserID(verdoc[_FLD_USER]), + nodes, + dt, + verdoc[_FLD_NAME], + version, + ) + ) return samples def _get_sample_and_version_doc( - self, id_: UUID, version: Optional[int] = None) -> Tuple[dict, dict, int]: + self, id_: UUID, version: Optional[int] = None + ) -> Tuple[dict, dict, int]: doc, uuidversion, version = self._get_sample_doc_and_versions(id_, version) verdoc = self._get_version_doc(id_, uuidversion) if verdoc[_FLD_VER] == _VAL_NO_VER: # since the version id came from the sample doc, the implication # is that the db or server lost connection before the version could be updated # and the reaper hasn't caught it yet, so we go ahead and fix it. - self._update_version_and_node_docs_with_find(id_, verdoc[_FLD_UUID_VER], version) + self._update_version_and_node_docs_with_find( + id_, verdoc[_FLD_UUID_VER], version + ) return (doc, verdoc, version) def _get_sample_doc_and_versions( - self, id_: UUID, version: Optional[int] = None) -> Tuple[dict, UUID, int]: + self, id_: UUID, version: Optional[int] = None + ) -> Tuple[dict, UUID, int]: doc = _cast(dict, self._get_sample_doc(id_)) maxver = len(doc[_FLD_VERSIONS]) version = version if version else maxver if version > maxver: - raise _NoSuchSampleVersionError(f'{id_} ver {version}') + raise _NoSuchSampleVersionError(f"{id_} ver {version}") return doc, UUID(doc[_FLD_VERSIONS][version - 1]), version - def _get_many_sample_and_version_doc(self, ids_: List[_Dict[str, _Any]]) -> Tuple[_Dict[str, dict], _Dict[str, dict], _Dict[str, int]]: + def _get_many_sample_and_version_doc( + self, ids_: List[_Dict[str, _Any]] + ) -> Tuple[_Dict[str, dict], _Dict[str, dict], _Dict[str, int]]: docs, versions = self._get_many_sample_doc_and_versions(ids_) - verdocs = self._get_many_version_docs([(id_, UUID(doc[_FLD_VERSIONS][versions[id_] - 1])) for id_, doc in docs.items()]) # sends id and version id + verdocs = self._get_many_version_docs( + [ + (id_, UUID(doc[_FLD_VERSIONS][versions[id_] - 1])) + for id_, doc in docs.items() + ] + ) # sends id and version id return (docs, verdocs, versions) - def _get_many_sample_doc_and_versions(self, ids_: List[_Dict[str, _Any]]) -> Tuple[_Dict[str, dict], _Dict[str, int]]: + def _get_many_sample_doc_and_versions( + self, ids_: List[_Dict[str, _Any]] + ) -> Tuple[_Dict[str, dict], _Dict[str, int]]: docs = [_cast(dict, doc) for doc in self._get_many_sample_doc(ids_)] ret_docs = {} ret_versions = {} for id_ver in ids_: - id_ = id_ver['id'] - version = id_ver['version'] + id_ = id_ver["id"] + version = id_ver["version"] # match to document for doc in docs: # get doc id - id_ = doc['id'] + id_ = doc["id"] maxver = len(doc[_FLD_VERSIONS]) version = version if version else maxver if version > maxver: @@ -738,13 +863,13 @@ def _timestamp_milliseconds_to_seconds(self, ts: int) -> float: return ts / 1000 def _get_version_id(self, id_: UUID, ver: UUID): - return f'{id_}_{ver}' + return f"{id_}_{ver}" def _get_node_id(self, id_: UUID, ver: UUID, node_id: str): # arango keys can be at most 254B and only a few characters are allowed, so we MD5 # the node name for length and safe characters # https://www.arangodb.com/docs/stable/data-modeling-naming-conventions-document-keys.html - return f'{id_}_{ver}_{self._md5(node_id)}' + return f"{id_}_{ver}_{self._md5(node_id)}" def _md5(self, string: str): return _hashlib.md5(string.encode("utf-8")).hexdigest() @@ -754,19 +879,23 @@ def _get_version_doc(self, id_: UUID, ver: UUID) -> dict: try: doc = self._col_version.get(self._get_version_id(id_, ver)) except _arango.exceptions.DocumentGetError as e: # this is a pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e if not doc: - raise _SampleStorageError(f'Corrupt DB: Missing version {ver} for sample {id_}') + raise _SampleStorageError( + f"Corrupt DB: Missing version {ver} for sample {id_}" + ) return doc - def _get_many_version_docs(self, ids_, exception: bool=True) -> _Dict[str, dict]: + def _get_many_version_docs(self, ids_, exception: bool = True) -> _Dict[str, dict]: version_ids = [self._get_version_id(id_, ver) for id_, ver in ids_] ver_docs = self._get_many_docs(self._col_version, version_ids) if not ver_docs: if exception: - raise _NoSuchSampleError(f"Could not complete search for samples: {[str(id_['id']) for id_ in ids_]}") + raise _NoSuchSampleError( + f"Could not complete search for samples: {[str(id_['id']) for id_ in ids_]}" + ) # return None - return {ver_doc['id']: ver_doc for ver_doc in ver_docs} + return {ver_doc["id"]: ver_doc for ver_doc in ver_docs} # assumes ver came from the sample doc in the db. def _get_nodes(self, id_: UUID, ver: UUID, version: int) -> List[_SampleNode]: @@ -776,7 +905,8 @@ def _get_nodes(self, id_: UUID, ver: UUID, version: int) -> List[_SampleNode]: nodedocs = self._col_nodes.find({_FLD_NODE_UUID_VER: str(ver)}) if not nodedocs: raise _SampleStorageError( - f'Corrupt DB: Missing nodes for version {ver} of sample {id_}') + f"Corrupt DB: Missing nodes for version {ver} of sample {id_}" + ) index_to_node = {} for n in nodedocs: if n[_FLD_VER] == _VAL_NO_VER: @@ -792,9 +922,9 @@ def _get_nodes(self, id_: UUID, ver: UUID, version: int) -> List[_SampleNode]: self._list_to_meta(n[_FLD_NODE_UNCONTROLLED_METADATA]), # allow for compatatibility with old samples without a source meta field self._list_to_source_meta(n.get(_FLD_NODE_SOURCE_METADATA)), - ) + ) except _arango.exceptions.DocumentGetError as e: # this is a pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e # could check for keyerror here if nodes were deleted, but db is corrupt either way # so YAGNI. # Could add a node count to the version... but how about we just assume the db works @@ -802,18 +932,24 @@ def _get_nodes(self, id_: UUID, ver: UUID, version: int) -> List[_SampleNode]: return nodes def _get_sample_doc(self, id_: UUID, exception: bool = True) -> Optional[dict]: - doc = self._get_doc(self._col_sample, str(_not_falsy(id_, 'id_'))) + doc = self._get_doc(self._col_sample, str(_not_falsy(id_, "id_"))) if not doc: if exception: raise _NoSuchSampleError(str(id_)) return None return doc - def _get_many_sample_doc(self, ids_: List[_Dict[str, _Any]], exception: bool=True) -> List[dict]: - docs = self._get_many_docs(self._col_sample, [str(_not_falsy(id_['id'], 'id_')) for id_ in ids_]) + def _get_many_sample_doc( + self, ids_: List[_Dict[str, _Any]], exception: bool = True + ) -> List[dict]: + docs = self._get_many_docs( + self._col_sample, [str(_not_falsy(id_["id"], "id_")) for id_ in ids_] + ) if not docs: if exception: - raise _NoSuchSampleError(f"Could not complete search for samples: {[str(id_['id']) for id_ in ids_]}") + raise _NoSuchSampleError( + f"Could not complete search for samples: {[str(id_['id']) for id_ in ids_]}" + ) # return [{}] return docs @@ -821,22 +957,22 @@ def _get_doc(self, col, id_: str) -> Optional[dict]: try: return col.get(id_) except _arango.exceptions.DocumentGetError as e: # this is a pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e - def _get_many_docs(self, col, ids_:List[str]) -> List[dict]: + def _get_many_docs(self, col, ids_: List[str]) -> List[dict]: try: return col.get_many(ids_) except _arango.exceptions.DocumentGetError as e: # this is a pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def get_sample_acls(self, id_: UUID) -> SampleACL: - ''' + """ Get a sample's acls from the database. :param id_: the ID of the sample. :returns: the sample acls. :raises NoSuchSampleError: if the sample does not exist. :raises SampleStorageError: if the sample could not be retrieved. - ''' + """ # return no class for now, might need later doc = _cast(dict, self._get_sample_doc(id_)) acls = doc[_FLD_ACLS] @@ -847,10 +983,11 @@ def get_sample_acls(self, id_: UUID) -> SampleACL: [UserID(u) for u in acls[_FLD_WRITE]], [UserID(u) for u in acls[_FLD_READ]], # allow None for backwards compability with DB entries missing the key - acls.get(_FLD_PUBLIC_READ)) + acls.get(_FLD_PUBLIC_READ), + ) def replace_sample_acls(self, id_: UUID, acls: SampleACL): - ''' + """ Completely replace a sample's ACLs. The owner may not be changed via this method, but is required to ensure the owner has @@ -864,14 +1001,14 @@ def replace_sample_acls(self, id_: UUID, acls: SampleACL): :raises SampleStorageError: if the sample could not be retrieved. :raises OwnerChangedException: if the owner in the database is not the same as the owner in the provided ACLs. - ''' - _not_falsy(id_, 'id_') - _not_falsy(acls, 'acls') + """ + _not_falsy(id_, "id_") + _not_falsy(acls, "acls") # Could return a subset of s to save bandwith # This will update the timestamp even for a noop. Maybe that's ok? # Detecting a noop would make the query a lot more complicated. Don't worry about it for # now. - aql = f''' + aql = f""" FOR s in @@col FILTER s.{_FLD_ARANGO_KEY} == @id FILTER s.{_FLD_ACLS}.{_FLD_OWNER} == @owner @@ -879,29 +1016,34 @@ def replace_sample_acls(self, id_: UUID, acls: SampleACL): {_FLD_ACL_UPDATE_TIME}: @ts }} IN @@col RETURN s - ''' - bind_vars = {'@col': self._col_sample.name, - 'id': str(id_), - 'owner': acls.owner.id, - 'ts': acls.lastupdate.timestamp(), - 'acls': {_FLD_ADMIN: [u.id for u in acls.admin], - _FLD_WRITE: [u.id for u in acls.write], - _FLD_READ: [u.id for u in acls.read], - _FLD_PUBLIC_READ: acls.public_read - } - } + """ + bind_vars = { + "@col": self._col_sample.name, + "id": str(id_), + "owner": acls.owner.id, + "ts": acls.lastupdate.timestamp(), + "acls": { + _FLD_ADMIN: [u.id for u in acls.admin], + _FLD_WRITE: [u.id for u in acls.write], + _FLD_READ: [u.id for u in acls.read], + _FLD_PUBLIC_READ: acls.public_read, + }, + } try: cur = self._db.aql.execute(aql, bind_vars=bind_vars, count=True) if not cur.count(): # assume cur.count() is never > 1 as we're filtering on _key - self._get_sample_doc(id_) # will raise exception if document does not exist + self._get_sample_doc( + id_ + ) # will raise exception if document does not exist raise _OwnerChangedError() except _arango.exceptions.AQLQueryExecuteError as e: # this is a real pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def update_sample_acls( - self, id_: UUID, update: SampleACLDelta, update_time: datetime.datetime) -> None: - ''' + self, id_: UUID, update: SampleACLDelta, update_time: datetime.datetime + ) -> None: + """ Update a sample's ACLs via a delta specification. :param id_: the sample ID. @@ -910,11 +1052,11 @@ def update_sample_acls( :raises NoSuchSampleError: if the sample does not exist. :raises SampleStorageError: if the sample could not be retrieved. :raises UnauthorizedError: if the update attempts to alter the sample owner. - ''' + """ # Needs to ensure owner is not added to another ACL # could make an option to just ignore the update to the owner? YAGNI for now. - _not_falsy(update, 'update') - _check_timestamp(update_time, 'update_time') + _not_falsy(update, "update") + _check_timestamp(update_time, "update_time") s = self.get_sample_acls(id_) if not s.is_update(update): # noop. Theoretically the values in the DB may have changed since we pulled the ACLs, @@ -923,7 +1065,7 @@ def update_sample_acls( return self._update_sample_acls_pt2(id_, update, s.owner, update_time) - _UPDATE_ACLS_AQL = f''' + _UPDATE_ACLS_AQL = f""" FOR s in @@col FILTER s.{_FLD_ARANGO_KEY} == @id FILTER s.{_FLD_ACLS}.{_FLD_OWNER} == @owner @@ -939,9 +1081,9 @@ def update_sample_acls( {_FLD_READ}: REMOVE_VALUES( UNION_DISTINCT(s.{_FLD_ACLS}.{_FLD_READ}, @read), @read_remove) - ''' + """ - _UPDATE_ACLS_AT_LEAST_AQL = f''' + _UPDATE_ACLS_AT_LEAST_AQL = f""" FOR s in @@col FILTER s.{_FLD_ARANGO_KEY} == @id FILTER s.{_FLD_ACLS}.{_FLD_OWNER} == @owner @@ -958,7 +1100,7 @@ def update_sample_acls( REMOVE_VALUES(s.{_FLD_ACLS}.{_FLD_READ}, @read_remove), REMOVE_VALUES(@read, UNION_DISTINCT( s.{_FLD_ACLS}.{_FLD_ADMIN}, s.{_FLD_ACLS}.{_FLD_WRITE}))) - ''' + """ def _update_sample_acls_pt2(self, id_, update, owner, update_time): # this method is split solely to allow testing the owner change case. @@ -979,33 +1121,36 @@ def _update_sample_acls_pt2(self, id_, update, owner, update_time): r = [u.id for u in update.read if u != owner] rem = [u.id for u in update.remove] - bind_vars = {'@col': self._col_sample.name, - 'id': str(id_), - 'owner': owner.id, - 'ts': update_time.timestamp(), - 'admin': a, - 'write': w, - 'read': r, - 'read_remove': a + w + rem, - } + bind_vars = { + "@col": self._col_sample.name, + "id": str(id_), + "owner": owner.id, + "ts": update_time.timestamp(), + "admin": a, + "write": w, + "read": r, + "read_remove": a + w + rem, + } if update.at_least: - bind_vars['admin_remove'] = rem - bind_vars['write_remove'] = a + rem + bind_vars["admin_remove"] = rem + bind_vars["write_remove"] = a + rem else: - bind_vars['admin_remove'] = w + r + rem - bind_vars['write_remove'] = a + r + rem + bind_vars["admin_remove"] = w + r + rem + bind_vars["write_remove"] = a + r + rem # Could return a subset of s to save bandwith (see query text) # ensures the owner hasn't changed since we pulled the acls above (see query text). - aql = self._UPDATE_ACLS_AT_LEAST_AQL if update.at_least else self._UPDATE_ACLS_AQL + aql = ( + self._UPDATE_ACLS_AT_LEAST_AQL if update.at_least else self._UPDATE_ACLS_AQL + ) if update.public_read is not None: - aql += f''', - {_FLD_PUBLIC_READ}: @pubread''' - bind_vars['pubread'] = update.public_read - aql += ''' + aql += f""", + {_FLD_PUBLIC_READ}: @pubread""" + bind_vars["pubread"] = update.public_read + aql += """ } } IN @@col RETURN s - ''' + """ try: cur = self._db.aql.execute(aql, bind_vars=bind_vars, count=True) @@ -1014,13 +1159,14 @@ def _update_sample_acls_pt2(self, id_, update, owner, update_time): # We already know the sample exists, and samples at this point can't be # deleted, so just raise. raise _OwnerChangedError( # if this happens a lot make a retry loop. - 'The sample owner unexpectedly changed during the operation. Please retry. ' + - 'If this error occurs frequently, code changes may be necessary.') + "The sample owner unexpectedly changed during the operation. Please retry. " + + "If this error occurs frequently, code changes may be necessary." + ) except _arango.exceptions.AQLQueryExecuteError as e: # this is a real pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def create_data_link(self, link: DataLink, update: bool = False) -> Optional[UUID]: - ''' + """ Link data in the workspace to a sample. Each data unit can be linked to only one sample at a time. Expired links may exist to other samples. @@ -1047,7 +1193,7 @@ def create_data_link(self, link: DataLink, update: bool = False) -> Optional[UUI :raises DataLinkExistsError: if a link already exists from the data unit. :raises TooManyDataLinksError: if there are too many links from the sample version or the workspace object version. - ''' + """ # may want to link non-ws data at some point, would need a data source ID? YAGNI for now # Using the REST streaming api for the transaction. Might be faster with javascript @@ -1065,9 +1211,9 @@ def create_data_link(self, link: DataLink, update: bool = False) -> Optional[UUI # https://www.arangodb.com/docs/stable/data-modeling-naming-conventions-document-keys.html # TODO CODE this method is too long, try to split up - _not_falsy(link, 'link') + _not_falsy(link, "link") if link.expired: - raise ValueError('link cannot be expired') + raise ValueError("link cannot be expired") sna = link.sample_node_address # need to get the version doc to ensure the documents have been updated appropriately # as well as getting the uuid version, see comments at beginning of file @@ -1075,7 +1221,7 @@ def create_data_link(self, link: DataLink, update: bool = False) -> Optional[UUI samplever = UUID(versiondoc[_FLD_UUID_VER]) nodeid = self._get_node_id(sna.sampleid, samplever, sna.node) if not self._get_doc(self._col_nodes, nodeid): - raise _NoSuchSampleNodeError(f'{sna.sampleid} ver {sna.version} {sna.node}') + raise _NoSuchSampleNodeError(f"{sna.sampleid} ver {sna.version} {sna.node}") # makes a db specifically for this transaction tdb = self._db.begin_transaction( @@ -1083,14 +1229,17 @@ def create_data_link(self, link: DataLink, update: bool = False) -> Optional[UUI # Need exclusive as we're counting docs and making decisions based on that number # Write only checks for write collisions on specific docs, so count could change # during transaction - exclusive=self._col_data_link.name) + exclusive=self._col_data_link.name, + ) try: # makes a collection specifically for this transaction tdlc = tdb.collection(self._col_data_link.name) oldlinkdoc = self._get_doc(tdlc, self._create_link_key(link)) if oldlinkdoc: - if not update: # maybe want to move this after the noop check? or add noop option + if ( + not update + ): # maybe want to move this after the noop check? or add noop option raise _DataLinkExistsError(str(link.duid)) oldlink = self._doc_to_link(oldlinkdoc) if link.is_equivalent(oldlink): @@ -1106,8 +1255,12 @@ def create_data_link(self, link: DataLink, update: bool = False) -> Optional[UUI # I'm not a fan of this, but a millisecond gap seems safe and most systems # should have millisecond resolution. # Consider rounding to millisecond resolution for consistency? Make a class? - oldlinkdoc[_FLD_LINK_EXPIRED] = self._timestamp_seconds_to_milliseconds(link.created.timestamp() - 0.001) - oldlinkdoc[_FLD_ARANGO_KEY] = self._create_link_key_from_link_doc(oldlinkdoc) + oldlinkdoc[_FLD_LINK_EXPIRED] = self._timestamp_seconds_to_milliseconds( + link.created.timestamp() - 0.001 + ) + oldlinkdoc[_FLD_ARANGO_KEY] = self._create_link_key_from_link_doc( + oldlinkdoc + ) # since we're replacing a link we don't need to worry about counting links from # ws object. Not true for the sample, which could be different. @@ -1141,124 +1294,158 @@ def _commit_transaction(self, transaction_db): transaction_db.commit_transaction() except _arango.exceptions.TransactionCommitError as e: # dunno how to test this # TODO DATALINK if the transaction fails we may want to retry. pretty complicated - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def _abort_transaction(self, transaction_db): - if transaction_db.transaction_status() != 'committed': + if transaction_db.transaction_status() != "committed": try: transaction_db.abort_transaction() except _arango.exceptions.TransactionAbortError as e: # dunno how to test this # this will mask the previous error, but if this fails probably the DB # connection is hosed - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError( + "Connection to database failed: " + str(e) + ) from e def _check_link_count_from_ws_object(self, db, link: DataLink): - if self._count_links_from_ws_object( - db, link.duid.upa, link.created, link.expired) >= self._max_links: + if ( + self._count_links_from_ws_object( + db, link.duid.upa, link.created, link.expired + ) + >= self._max_links + ): raise _TooManyDataLinksError( - f'More than {self._max_links} links from workspace object {link.duid.upa}') + f"More than {self._max_links} links from workspace object {link.duid.upa}" + ) def _count_links_from_ws_object( - self, - db, - upa: UPA, - created: datetime.datetime, - expired: Optional[datetime.datetime]): + self, + db, + upa: UPA, + created: datetime.datetime, + expired: Optional[datetime.datetime], + ): wsc = self._count_links( db, - f''' + f""" FILTER d.{_FLD_LINK_WORKSPACE_ID} == @wsid FILTER d.{_FLD_LINK_OBJECT_ID} == @objid - FILTER d.{_FLD_LINK_OBJECT_VERSION} == @ver''', - {'wsid': upa.wsid, 'objid': upa.objid, 'ver': upa.version}, + FILTER d.{_FLD_LINK_OBJECT_VERSION} == @ver""", + {"wsid": upa.wsid, "objid": upa.objid, "ver": upa.version}, created, - expired) + expired, + ) return wsc def _check_link_count_from_sample_ver(self, db, samplever: UUID, link: DataLink): - if self._count_links_from_sample_ver( - db, samplever, link.created, link.expired) >= self._max_links: + if ( + self._count_links_from_sample_ver(db, samplever, link.created, link.expired) + >= self._max_links + ): sna = link.sample_node_address raise _TooManyDataLinksError( - f'More than {self._max_links} links from sample {sna.sampleid} ' + - f'version {sna.version}') + f"More than {self._max_links} links from sample {sna.sampleid} " + + f"version {sna.version}" + ) def _count_links_from_sample_ver( - self, - db, - version: UUID, - created: datetime.datetime, - expired: Optional[datetime.datetime]): + self, + db, + version: UUID, + created: datetime.datetime, + expired: Optional[datetime.datetime], + ): sv = self._count_links( db, - f''' - FILTER d.{_FLD_LINK_SAMPLE_UUID_VERSION} == @sver''', - {'sver': str(version)}, + f""" + FILTER d.{_FLD_LINK_SAMPLE_UUID_VERSION} == @sver""", + {"sver": str(version)}, created, - expired) + expired, + ) return sv def _count_links(self, db, filters: str, bind_vars, created, expired): - bind_vars['@col'] = self._col_data_link.name - bind_vars['created'] = self._timestamp_seconds_to_milliseconds(created.timestamp()) - bind_vars['expired'] = self._timestamp_seconds_to_milliseconds(expired.timestamp()) if expired else _ARANGO_MAX_INTEGER + bind_vars["@col"] = self._col_data_link.name + bind_vars["created"] = self._timestamp_seconds_to_milliseconds( + created.timestamp() + ) + bind_vars["expired"] = ( + self._timestamp_seconds_to_milliseconds(expired.timestamp()) + if expired + else _ARANGO_MAX_INTEGER + ) # might need to include created / expired in compound indexes if we get a ton of expired # links. Might not work in a NOT though. Alternate formulation is # (d.creatd >= @created AND d.created <= @expired) OR # (d.expired >= @created AND d.expired <= @expired) - q = (''' + q = ( + """ FOR d in @@col - ''' + - filters + - f''' + """ + + filters + + f""" FILTER NOT (d.{_FLD_LINK_EXPIRED} < @created OR d.{_FLD_LINK_CREATED} > @expired) COLLECT WITH COUNT INTO linkcount RETURN linkcount - ''') + """ + ) try: cur = db.aql.execute(q, bind_vars=bind_vars) return cur.next() except _arango.exceptions.AQLQueryExecuteError as e: # this is a pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def _create_link_key(self, link: DataLink): - cr = f'_{link.created.timestamp()}' if link.expired else '' + cr = f"_{link.created.timestamp()}" if link.expired else "" upa = link.duid.upa - dataid = f'_{self._md5(link.duid.dataid)}' if link.duid.dataid else '' - return f'{upa.wsid}_{upa.objid}_{upa.version}{dataid}{cr}' + dataid = f"_{self._md5(link.duid.dataid)}" if link.duid.dataid else "" + return f"{upa.wsid}_{upa.objid}_{upa.version}{dataid}{cr}" # only creates keys for unexpired links. def _create_link_key_from_duid(self, duid: DataUnitID): - dataid = f'_{self._md5(duid.dataid)}' if duid.dataid else '' - return f'{duid.upa.wsid}_{duid.upa.objid}_{duid.upa.version}{dataid}' + dataid = f"_{self._md5(duid.dataid)}" if duid.dataid else "" + return f"{duid.upa.wsid}_{duid.upa.objid}_{duid.upa.version}{dataid}" def _create_link_key_from_link_doc(self, link: dict): # arango sometimes removes trailing decimals and zeros from the number so we reformat # with datetime to ensure consistency - created=self._timestamp_milliseconds_to_seconds(link[_FLD_LINK_CREATED]) - cr = (f'_{self._timestamp_to_datetime(created).timestamp()}' - if link[_FLD_LINK_EXPIRED] != _ARANGO_MAX_INTEGER else '') - dataid = (f'_{self._md5(link[_FLD_LINK_OBJECT_DATA_UNIT])}' - if link[_FLD_LINK_OBJECT_DATA_UNIT] else '') + created = self._timestamp_milliseconds_to_seconds(link[_FLD_LINK_CREATED]) + cr = ( + f"_{self._timestamp_to_datetime(created).timestamp()}" + if link[_FLD_LINK_EXPIRED] != _ARANGO_MAX_INTEGER + else "" + ) + dataid = ( + f"_{self._md5(link[_FLD_LINK_OBJECT_DATA_UNIT])}" + if link[_FLD_LINK_OBJECT_DATA_UNIT] + else "" + ) wsid = link[_FLD_LINK_WORKSPACE_ID] objid = link[_FLD_LINK_OBJECT_ID] version = link[_FLD_LINK_OBJECT_VERSION] - return f'{wsid}_{objid}_{version}{dataid}{cr}' + return f"{wsid}_{objid}_{version}{dataid}{cr}" def _create_link_doc(self, link: DataLink, samplever: UUID): sna = link.sample_node_address upa = link.duid.upa nodeid = self._get_node_id(sna.sampleid, samplever, sna.node) # see https://github.com/kbase/relation_engine_spec/blob/4a9dc6df2088763a9df88f0b018fa5c64f2935aa/schemas/ws/ws_object_version.yaml#L17 # noqa - from_ = f'{self._col_ws.name}/{upa.wsid}:{upa.objid}:{upa.version}' + from_ = f"{self._col_ws.name}/{upa.wsid}:{upa.objid}:{upa.version}" return { _FLD_ARANGO_KEY: self._create_link_key(link), _FLD_ARANGO_FROM: from_, - _FLD_ARANGO_TO: f'{self._col_nodes.name}/{nodeid}', - _FLD_LINK_CREATED: self._timestamp_seconds_to_milliseconds(link.created.timestamp()), + _FLD_ARANGO_TO: f"{self._col_nodes.name}/{nodeid}", + _FLD_LINK_CREATED: self._timestamp_seconds_to_milliseconds( + link.created.timestamp() + ), _FLD_LINK_CREATED_BY: link.created_by.id, - _FLD_LINK_EXPIRED: self._timestamp_seconds_to_milliseconds(link.expired.timestamp()) if link.expired else _ARANGO_MAX_INTEGER, + _FLD_LINK_EXPIRED: self._timestamp_seconds_to_milliseconds( + link.expired.timestamp() + ) + if link.expired + else _ARANGO_MAX_INTEGER, _FLD_LINK_EXPIRED_BY: link.expired_by.id if link.expired_by else None, _FLD_LINK_ID: str(link.id), _FLD_LINK_WORKSPACE_ID: upa.wsid, @@ -1270,22 +1457,24 @@ def _create_link_doc(self, link: DataLink, samplever: UUID): # recording the integer version saves looking it up in the version doc and it's # immutable so denormalization is ok here _FLD_LINK_SAMPLE_INT_VERSION: sna.version, - _FLD_LINK_SAMPLE_NODE: sna.node + _FLD_LINK_SAMPLE_NODE: sna.node, } def _get_link_doc_from_link_id(self, id_): # if delete/hide samples added may need some more logic here try: - cur = self._col_data_link.find({_FLD_LINK_ID: str(_not_falsy(id_, 'id_'))}, limit=2) + cur = self._col_data_link.find( + {_FLD_LINK_ID: str(_not_falsy(id_, "id_"))}, limit=2 + ) if cur.count() == 0: raise _NoSuchLinkError(str(id_)) if cur.count() > 1: - raise _SampleStorageError(f'More than one data link found for ID {id_}') + raise _SampleStorageError(f"More than one data link found for ID {id_}") doc = cur.next() cur.close(True) return doc except _arango.exceptions.DocumentGetError as e: # this is a pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError("Connection to database failed: " + str(e)) from e def _get_link_doc_from_duid(self, duid): key = self._create_link_key_from_duid(duid) @@ -1295,12 +1484,13 @@ def _get_link_doc_from_duid(self, duid): return linkdoc def expire_data_link( - self, - expired: datetime.datetime, - expired_by: UserID, - id_: UUID = None, - duid: DataUnitID = None) -> DataLink: - ''' + self, + expired: datetime.datetime, + expired_by: UserID, + id_: UUID = None, + duid: DataUnitID = None, + ) -> DataLink: + """ Expire a data link. The link can be addressed by its ID or the DUID, since for a non- expired link the DUID is a unique identifier. Providing both IDs is an error. @@ -1311,12 +1501,12 @@ def expire_data_link( :param duid: the data unit ID from which the link originates. :returns: the updated link. :raises NoSuchLinkError: if the link does not exist or is already expired. - ''' + """ # See notes for creating links re the transaction approach. - _check_timestamp(expired, 'expired') - _not_falsy(expired_by, 'expired_by') + _check_timestamp(expired, "expired") + _not_falsy(expired_by, "expired_by") if not bool(id_) ^ bool(duid): # xor - raise ValueError('exactly one of id_ or duid must be provided') + raise ValueError("exactly one of id_ or duid must be provided") if id_: linkdoc = self._get_link_doc_from_link_id(id_) txtid = str(id_) @@ -1326,8 +1516,13 @@ def expire_data_link( linkdoc = self._get_link_doc_from_duid(duid) txtid = str(duid) - if self._timestamp_seconds_to_milliseconds(expired.timestamp()) < linkdoc[_FLD_LINK_CREATED]: - raise ValueError(f'expired is < link created time: {linkdoc[_FLD_LINK_CREATED]}') + if ( + self._timestamp_seconds_to_milliseconds(expired.timestamp()) + < linkdoc[_FLD_LINK_CREATED] + ): + raise ValueError( + f"expired is < link created time: {linkdoc[_FLD_LINK_CREATED]}" + ) return self._expire_data_link_pt2(linkdoc, expired, expired_by, txtid) @@ -1337,7 +1532,9 @@ def _expire_data_link_pt2(self, linkdoc, expired, expired_by, txtid) -> DataLink oldkey = self._create_link_key_from_link_doc(linkdoc) - linkdoc[_FLD_LINK_EXPIRED] = self._timestamp_seconds_to_milliseconds(expired.timestamp()) + linkdoc[_FLD_LINK_EXPIRED] = self._timestamp_seconds_to_milliseconds( + expired.timestamp() + ) linkdoc[_FLD_LINK_EXPIRED_BY] = expired_by.id linkdoc[_FLD_ARANGO_KEY] = self._create_link_key_from_link_doc(linkdoc) @@ -1348,8 +1545,8 @@ def _expire_data_link_pt2(self, linkdoc, expired, expired_by, txtid) -> DataLink # makes a db specifically for this transaction tdb = self._db.begin_transaction( - read=self._col_data_link.name, - write=self._col_data_link.name) + read=self._col_data_link.name, write=self._col_data_link.name + ) # TODO DATALINK Transactions in arango can allow some ops to succeed and others to fail. # What do? @@ -1365,7 +1562,9 @@ def _expire_data_link_pt2(self, linkdoc, expired, expired_by, txtid) -> DataLink # should *NOT* expire that. raise _NoSuchLinkError(txtid) else: # this is a real pain to test. - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError( + "Connection to database failed: " + str(e) + ) from e try: tdlc.delete(oldkey, silent=True) except _arango.exceptions.DocumentDeleteError as e: @@ -1379,14 +1578,16 @@ def _expire_data_link_pt2(self, linkdoc, expired, expired_by, txtid) -> DataLink # error and document potential failure modes for expire transaction # this is really hard to test - maybe impossible? - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e + raise _SampleStorageError( + "Connection to database failed: " + str(e) + ) from e self._commit_transaction(tdb) return self._doc_to_link(linkdoc) finally: self._abort_transaction(tdb) def get_data_link(self, id_: UUID = None, duid: DataUnitID = None) -> DataLink: - ''' + """ Get a link by its ID or Data Unit ID. The latter can only retrieve non-expired links. Exactly one of the ID or DUID must be specified. @@ -1394,9 +1595,9 @@ def get_data_link(self, id_: UUID = None, duid: DataUnitID = None) -> DataLink: :param duid: the link DUID. :returns: the link. :raises NoSuchLinkError: if the link does not exist. - ''' + """ if not bool(id_) ^ bool(duid): # xor - raise ValueError('exactly one of id_ or duid must be provided') + raise ValueError("exactly one of id_ or duid must be provided") if id_: return self._doc_to_link(self._get_link_doc_from_link_id(id_)) else: @@ -1409,28 +1610,39 @@ def _doc_to_link(self, doc) -> DataLink: self._doc_to_dataunit_id(doc), _SampleNodeAddress( SampleAddress( - UUID(doc[_FLD_LINK_SAMPLE_ID]), - doc[_FLD_LINK_SAMPLE_INT_VERSION]), - doc[_FLD_LINK_SAMPLE_NODE]), - self._timestamp_to_datetime(self._timestamp_milliseconds_to_seconds(doc[_FLD_LINK_CREATED])), + UUID(doc[_FLD_LINK_SAMPLE_ID]), doc[_FLD_LINK_SAMPLE_INT_VERSION] + ), + doc[_FLD_LINK_SAMPLE_NODE], + ), + self._timestamp_to_datetime( + self._timestamp_milliseconds_to_seconds(doc[_FLD_LINK_CREATED]) + ), UserID(doc[_FLD_LINK_CREATED_BY]), - None if ex == _ARANGO_MAX_INTEGER else self._timestamp_to_datetime(self._timestamp_milliseconds_to_seconds(ex)), - UserID(doc[_FLD_LINK_EXPIRED_BY]) if doc[_FLD_LINK_EXPIRED_BY] else None + None + if ex == _ARANGO_MAX_INTEGER + else self._timestamp_to_datetime( + self._timestamp_milliseconds_to_seconds(ex) + ), + UserID(doc[_FLD_LINK_EXPIRED_BY]) if doc[_FLD_LINK_EXPIRED_BY] else None, ) def _doc_to_dataunit_id(self, doc) -> DataUnitID: return DataUnitID( - UPA(wsid=doc[_FLD_LINK_WORKSPACE_ID], + UPA( + wsid=doc[_FLD_LINK_WORKSPACE_ID], objid=doc[_FLD_LINK_OBJECT_ID], - version=doc[_FLD_LINK_OBJECT_VERSION]), - doc[_FLD_LINK_OBJECT_DATA_UNIT]) + version=doc[_FLD_LINK_OBJECT_VERSION], + ), + doc[_FLD_LINK_OBJECT_DATA_UNIT], + ) def get_links_from_sample( - self, - sample: SampleAddress, - readable_wsids: Optional[List[int]], - timestamp: datetime.datetime) -> List[DataLink]: - ''' + self, + sample: SampleAddress, + readable_wsids: Optional[List[int]], + timestamp: datetime.datetime, + ) -> List[DataLink]: + """ Get the links from a sample at a particular time. :param sample: the sample of interest. @@ -1440,33 +1652,37 @@ def get_links_from_sample( :returns: a list of links. :raises NoSuchSampleError: if the sample does not exist. :raises NoSuchSampleVersionError: if the sample version does not exist. - ''' + """ # may want to make this work on non-ws objects at some point. YAGNI for now. - _not_falsy(sample, 'sample') - _check_timestamp(timestamp, 'timestamp') - _not_falsy_in_iterable(readable_wsids, 'readable_wsids', allow_none=True) + _not_falsy(sample, "sample") + _check_timestamp(timestamp, "timestamp") + _not_falsy_in_iterable(readable_wsids, "readable_wsids", allow_none=True) if readable_wsids is not None and not readable_wsids: return [] # need to get the version doc to ensure the documents have been updated appropriately # as well as getting the uuid version, see comments at beginning of file # note that testing version updating has been done for at least 2 other methods # the tests are not repeated here - _, versiondoc, _ = self._get_sample_and_version_doc(sample.sampleid, sample.version) - bind_vars = {'@col': self._col_data_link.name, - 'samplever': versiondoc[_FLD_UUID_VER], - 'ts': self._timestamp_seconds_to_milliseconds(timestamp.timestamp())} - wsidfilter = '' + _, versiondoc, _ = self._get_sample_and_version_doc( + sample.sampleid, sample.version + ) + bind_vars = { + "@col": self._col_data_link.name, + "samplever": versiondoc[_FLD_UUID_VER], + "ts": self._timestamp_seconds_to_milliseconds(timestamp.timestamp()), + } + wsidfilter = "" if readable_wsids: - bind_vars['wsids'] = readable_wsids - wsidfilter = f'FILTER d.{_FLD_LINK_WORKSPACE_ID} IN @wsids' - q = f''' + bind_vars["wsids"] = readable_wsids + wsidfilter = f"FILTER d.{_FLD_LINK_WORKSPACE_ID} IN @wsids" + q = f""" FOR d in @@col FILTER d.{_FLD_LINK_SAMPLE_UUID_VERSION} == @samplever {wsidfilter} FILTER d.{_FLD_LINK_CREATED} <= @ts FILTER d.{_FLD_LINK_EXPIRED} >= @ts RETURN d - ''' + """ # may need an index on version + created and expired? Assume for now links aren't # expired very often. # may also want a sample ver / wsid index? Max 10k items per version though, and @@ -1479,21 +1695,25 @@ def _find_links_via_aql(self, query, bind_vars): for link in self._db.aql.execute(query, bind_vars=bind_vars): duids.append(self._doc_to_link(link)) except _arango.exceptions.AQLQueryExecuteError as e: # this is a pain to test - raise _SampleStorageError('Connection to database failed: ' + str(e)) from e - return duids # a maxium of 10k can be returned based on the link creation function + raise _SampleStorageError("Connection to database failed: " + str(e)) from e + return ( + duids # a maxium of 10k can be returned based on the link creation function + ) - def get_links_from_data(self, upa: UPA, timestamp: datetime.datetime) -> List[DataLink]: - ''' + def get_links_from_data( + self, upa: UPA, timestamp: datetime.datetime + ) -> List[DataLink]: + """ Get links originating from a data object. The data object is not checked for existence. :param upa: the address of the data object. :param timestamp: the time to use to determine which links are active. :returns: a list of links. - ''' + """ # the UPA makes it workspace specific, may need to make it generic later. YAGNI for now. - _not_falsy(upa, 'upa') - _check_timestamp(timestamp, 'timestamp') - q = f''' + _not_falsy(upa, "upa") + _check_timestamp(timestamp, "timestamp") + q = f""" FOR d in @@col FILTER d.{_FLD_LINK_WORKSPACE_ID} == @wsid FILTER d.{_FLD_LINK_OBJECT_ID} == @objid @@ -1501,18 +1721,20 @@ def get_links_from_data(self, upa: UPA, timestamp: datetime.datetime) -> List[Da FILTER d.{_FLD_LINK_CREATED} <= @ts FILTER d.{_FLD_LINK_EXPIRED} >= @ts RETURN d - ''' - bind_vars = {'@col': self._col_data_link.name, - 'wsid': upa.wsid, - 'objid': upa.objid, - 'ver': upa.version, - 'ts': self._timestamp_seconds_to_milliseconds(timestamp.timestamp())} + """ + bind_vars = { + "@col": self._col_data_link.name, + "wsid": upa.wsid, + "objid": upa.objid, + "ver": upa.version, + "ts": self._timestamp_seconds_to_milliseconds(timestamp.timestamp()), + } # may need an index on upa + created and expired? Assume for now links aren't # expired very often. return self._find_links_via_aql(q, bind_vars) def has_data_link(self, upa: UPA, sample: UUID) -> bool: - ''' + """ Check if a link exists or has ever existed between an object and a sample. The sample and object are not checked for existence. @@ -1520,12 +1742,12 @@ def has_data_link(self, upa: UPA, sample: UUID) -> bool: :param sample: the sample to check. :returns: True if a link has ever existed between the sample and the object, or False otherwise. - ''' + """ # Again, this is fairly workspace specific. May want to generalize at some point. # YAGNI for now. - _not_falsy(upa, 'upa') - _not_falsy(sample, 'sample') - q = f''' + _not_falsy(upa, "upa") + _not_falsy(sample, "sample") + q = f""" FOR d in @@col FILTER d.{_FLD_LINK_SAMPLE_ID} == @sampleid FILTER d.{_FLD_LINK_WORKSPACE_ID} == @wsid @@ -1533,12 +1755,14 @@ def has_data_link(self, upa: UPA, sample: UUID) -> bool: FILTER d.{_FLD_LINK_OBJECT_VERSION} == @ver LIMIT 1 RETURN d - ''' - bind_vars = {'@col': self._col_data_link.name, - 'sampleid': str(sample), - 'wsid': upa.wsid, - 'objid': upa.objid, - 'ver': upa.version} + """ + bind_vars = { + "@col": self._col_data_link.name, + "sampleid": str(sample), + "wsid": upa.wsid, + "objid": upa.objid, + "ver": upa.version, + } # may want a sample / wsid index? Max 10k items per version though, and # probably much less. YAGNI for now. # not super efficient - could save some bandwidth by not returning the link. @@ -1547,9 +1771,13 @@ def has_data_link(self, upa: UPA, sample: UUID) -> bool: # if an edge is inserted into a non-edge collection _from and _to are silently dropped -def _init_collection(database, collection, collection_name, collection_variable_name, edge=False): +def _init_collection( + database, collection, collection_name, collection_variable_name, edge=False +): c = database.collection(_check_string(collection, collection_variable_name)) - if not c.properties()['edge'] is edge: # this is a http call - ctype = 'an edge' if edge else 'a vertex' - raise _StorageInitError(f'{collection_name} {collection} is not {ctype} collection') + if not c.properties()["edge"] is edge: # this is a http call + ctype = "an edge" if edge else "a vertex" + raise _StorageInitError( + f"{collection_name} {collection} is not {ctype} collection" + ) return c diff --git a/lib/SampleService/core/user.py b/lib/SampleService/core/user.py index a97d2bd1..39fbd5f1 100644 --- a/lib/SampleService/core/user.py +++ b/lib/SampleService/core/user.py @@ -1,25 +1,25 @@ -''' User oriented classes and functions. ''' +""" User oriented classes and functions. """ from SampleService.core.arg_checkers import check_string as _check_string class UserID: - ''' + """ A users's unique name / identifier. The ID is expected to be checked against a user registry and so only minimal constraints are enforced here. :ivar id: the user id. - ''' + """ def __init__(self, userid): - ''' + """ Create the user id. :param id: the user's id, a maximum of 256 unicode characters. - ''' - self.id = _check_string(userid, 'userid', max_len=256) + """ + self.id = _check_string(userid, "userid", max_len=256) def __str__(self): return self.id diff --git a/lib/SampleService/core/user_lookup.py b/lib/SampleService/core/user_lookup.py index 85b25342..563fc49b 100644 --- a/lib/SampleService/core/user_lookup.py +++ b/lib/SampleService/core/user_lookup.py @@ -1,4 +1,4 @@ -''' Look up user names in the KBase Auth service to determine if they represent existing users. ''' +""" Look up user names in the KBase Auth service to determine if they represent existing users. """ # this is tested in the integration tests to avoid starting up the auth server again, as # it takes a few seconds @@ -8,27 +8,30 @@ from typing import List, Sequence, Tuple from SampleService.core.arg_checkers import not_falsy as _not_falsy -from SampleService.core.arg_checkers import not_falsy_in_iterable as _no_falsy_in_iterable +from SampleService.core.arg_checkers import ( + not_falsy_in_iterable as _no_falsy_in_iterable, +) from SampleService.core.acls import AdminPermission from SampleService.core.user import UserID import time -from cacheout.lru import LRUCache # type: ignore +from cacheout.lru import LRUCache # type: ignore class KBaseUserLookup: - ''' A client for contacting the KBase authentication server to verify user names. ''' + """A client for contacting the KBase authentication server to verify user names.""" def __init__( - self, - auth_url: str, - auth_token: str, - full_admin_roles: List[str] = None, - read_admin_roles: List[str] = None, - cache_max_size: int=10000, - cache_admin_expiration: int=300, - cache_valid_expiration: int=3600): - ''' + self, + auth_url: str, + auth_token: str, + full_admin_roles: List[str] = None, + read_admin_roles: List[str] = None, + cache_max_size: int = 10000, + cache_admin_expiration: int = 300, + cache_valid_expiration: int = 3600, + ): + """ Create the client. :param auth_url: The root url of the authentication service. :param auth_token: A valid token for the authentication service. @@ -39,20 +42,22 @@ def __init__( seconds. This time can be overridden by a user handler on a per token basis. :param cache_valid_expiration: the default expiration time for the username -> validity cache. This time can be overridden by a user handler on a per user basis. - ''' - self._url = _not_falsy(auth_url, 'auth_url') - if not self._url.endswith('/'): - self._url += '/' - self._user_url = self._url + 'api/V2/users?list=' - self._me_url = self._url + 'api/V2/me' - self._token = _not_falsy(auth_token, 'auth_token') + """ + self._url = _not_falsy(auth_url, "auth_url") + if not self._url.endswith("/"): + self._url += "/" + self._user_url = self._url + "api/V2/users?list=" + self._me_url = self._url + "api/V2/me" + self._token = _not_falsy(auth_token, "auth_token") self._full_roles = set(full_admin_roles) if full_admin_roles else set() self._read_roles = set(read_admin_roles) if read_admin_roles else set() self._cache_timer = time.time - self._admin_cache = LRUCache(timer=self._cache_timer, maxsize=cache_max_size, - ttl=cache_admin_expiration) - self._valid_cache = LRUCache(timer=self._cache_timer, maxsize=cache_max_size, - ttl=cache_valid_expiration) + self._admin_cache = LRUCache( + timer=self._cache_timer, maxsize=cache_max_size, ttl=cache_admin_expiration + ) + self._valid_cache = LRUCache( + timer=self._cache_timer, maxsize=cache_max_size, ttl=cache_valid_expiration + ) # Auth 0.4.1 needs to be deployed before this will work # r = requests.get(self._url, headers={'Accept': 'application/json'}) @@ -65,7 +70,9 @@ def __init__( # check token is valid r = requests.get( - self._user_url, headers={'Accept': 'application/json', 'authorization': self._token}) + self._user_url, + headers={"Accept": "application/json", "authorization": self._token}, + ) self._check_error(r) # need to test this with a mock. YAGNI for now. # if r.json() != {}: @@ -76,24 +83,27 @@ def _check_error(self, r): try: j = r.json() except Exception: - err = ('Non-JSON response from KBase auth server, status code: ' + - str(r.status_code)) - logging.getLogger(__name__).info('%s, response:\n%s', err, r.text) + err = "Non-JSON response from KBase auth server, status code: " + str( + r.status_code + ) + logging.getLogger(__name__).info("%s, response:\n%s", err, r.text) raise IOError(err) # assume that if we get json then at least this is the auth server and we can # rely on the error structure. - if j['error'].get('appcode') == 10020: # Invalid token - raise InvalidTokenError('KBase auth server reported token is invalid.') - if j['error'].get('appcode') == 30010: # Invalid username + if j["error"].get("appcode") == 10020: # Invalid token + raise InvalidTokenError("KBase auth server reported token is invalid.") + if j["error"].get("appcode") == 30010: # Invalid username raise InvalidUserError( - 'The KBase auth server is being very assertive about ' + - 'one of the usernames being illegal: ' + j['error']['message']) + "The KBase auth server is being very assertive about " + + "one of the usernames being illegal: " + + j["error"]["message"] + ) # don't really see any other error codes we need to worry about - maybe disabled? # worry about it later. - raise IOError('Error from KBase auth server: ' + j['error']['message']) + raise IOError("Error from KBase auth server: " + j["error"]["message"]) def invalid_users(self, usernames: Sequence[UserID]) -> List[UserID]: - ''' + """ Check whether users exist in the authentication service. :param users: the users to check. @@ -101,19 +111,23 @@ def invalid_users(self, usernames: Sequence[UserID]) -> List[UserID]: service. :raises InvalidTokenError: if the token has expired :raises InvalidUserError: if any of the user names are illegal user names. - ''' + """ if usernames is None: - raise ValueError('usernames cannot be None') + raise ValueError("usernames cannot be None") if not usernames: return [] - _no_falsy_in_iterable(usernames, 'usernames') + _no_falsy_in_iterable(usernames, "usernames") - bad_usernames = [u for u in usernames if not self._valid_cache.get(u.id, default=False)] + bad_usernames = [ + u for u in usernames if not self._valid_cache.get(u.id, default=False) + ] if len(bad_usernames) == 0: return [] - r = requests.get(self._user_url + ','.join([u.id for u in bad_usernames]), - headers={'Authorization': self._token}) + r = requests.get( + self._user_url + ",".join([u.id for u in bad_usernames]), + headers={"Authorization": self._token}, + ) self._check_error(r) good_users = r.json() for u in bad_usernames: @@ -123,23 +137,23 @@ def invalid_users(self, usernames: Sequence[UserID]) -> List[UserID]: return [u for u in bad_usernames if u.id not in good_users] def is_admin(self, token: str) -> Tuple[AdminPermission, str]: - ''' + """ Check whether a user is a service administrator. :param token: The user's token. :returns: A tuple consisting of an enum indicating the user's administration permissions, if any, and the username. - ''' + """ # TODO CODE should regex the token to check for \n etc., but the SDK has already checked it - _not_falsy(token, 'token') + _not_falsy(token, "token") admin_cache = self._admin_cache.get(token, default=False) if admin_cache: - return admin_cache - r = requests.get(self._me_url, headers={'Authorization': token}) + return admin_cache + r = requests.get(self._me_url, headers={"Authorization": token}) self._check_error(r) j = r.json() - v = (self._get_role(j['customroles']), j['user']) + v = (self._get_role(j["customroles"]), j["user"]) self._admin_cache.set(token, v) return v @@ -153,12 +167,12 @@ def _get_role(self, roles): class AuthenticationError(Exception): - ''' An error thrown from the authentication service. ''' + """An error thrown from the authentication service.""" class InvalidTokenError(AuthenticationError): - ''' An error thrown when a token is invalid. ''' + """An error thrown when a token is invalid.""" class InvalidUserError(AuthenticationError): - ''' An error thrown when a user name is invalid. ''' + """An error thrown when a user name is invalid.""" diff --git a/lib/SampleService/core/validator/builtin.py b/lib/SampleService/core/validator/builtin.py index cac9a1a6..6fd51ca4 100644 --- a/lib/SampleService/core/validator/builtin.py +++ b/lib/SampleService/core/validator/builtin.py @@ -1,4 +1,4 @@ -''' +""" Contains metadata validation callable builder functions for the Sample service. The builder functions are expected to take a dict of configuration parameters @@ -11,7 +11,7 @@ thrown. If an exception is not thrown, and a falsy value is returned, the validation succeeds. -''' +""" import os import ranges @@ -29,45 +29,55 @@ _CACHE_MAX_SIZE = 10000 _CACHE_EXPIRATION = 3600 -_TOKEN_SEP = '::' -_ontology_terms_cache = LRUCache(timer=time.time, maxsize=_CACHE_MAX_SIZE, ttl=_CACHE_EXPIRATION) -_ontology_ancestors_cache = LRUCache(timer=time.time, maxsize=_CACHE_MAX_SIZE, ttl=_CACHE_EXPIRATION) +_TOKEN_SEP = "::" +_ontology_terms_cache = LRUCache( + timer=time.time, maxsize=_CACHE_MAX_SIZE, ttl=_CACHE_EXPIRATION +) +_ontology_ancestors_cache = LRUCache( + timer=time.time, maxsize=_CACHE_MAX_SIZE, ttl=_CACHE_EXPIRATION +) srv_wizard_url = None -if 'KB_DEPLOYMENT_CONFIG' in os.environ: - with open(os.environ['KB_DEPLOYMENT_CONFIG']) as f: +if "KB_DEPLOYMENT_CONFIG" in os.environ: + with open(os.environ["KB_DEPLOYMENT_CONFIG"]) as f: for line in f: - if line.startswith('srv-wiz-url'): - srv_wizard_url = line.split('=')[1].strip() + if line.startswith("srv-wiz-url"): + srv_wizard_url = line.split("=")[1].strip() def _check_unknown_keys(d, expected): if type(d) != dict: - raise ValueError('d must be a dict') + raise ValueError("d must be a dict") d2 = {k for k in d if k not in expected} if d2: - raise ValueError(f'Unexpected configuration parameter: {sorted(d2)[0]}') + raise ValueError(f"Unexpected configuration parameter: {sorted(d2)[0]}") + class ValidatorMessage(TypedDict): subkey: str message: str -def noop(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: - ''' +def noop( + d: Dict[str, Any] +) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: + """ Build a validation callable that allows any value for the metadata key. :params d: The configuration parameters for the callable. Unused. :returns: a callable that validates metadata maps. - ''' + """ _check_unknown_keys(d, []) def f(key: str, val: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]: return None + return f # mypy had trouble detecting the lambda type -def string(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: - ''' +def string( + d: Dict[str, Any] +) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: + """ Build a validation callable that performs string checking based on the following rules: If the 'keys' parameter is specified it must contain a string or a list of strings. The @@ -84,49 +94,72 @@ def string(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Optio :param d: the configuration map for the callable. :returns: a callable that validates metadata maps. - ''' + """ # no reason to require max-len, could just check all values are strings. YAGNI for now - _check_unknown_keys(d, {'max-len', 'required', 'keys'}) - if 'max-len' not in d: + _check_unknown_keys(d, {"max-len", "required", "keys"}) + if "max-len" not in d: maxlen = None else: try: - maxlen = int(d['max-len']) + maxlen = int(d["max-len"]) except ValueError: - raise ValueError('max-len must be an integer') + raise ValueError("max-len must be an integer") if maxlen < 1: - raise ValueError('max-len must be > 0') + raise ValueError("max-len must be > 0") - required = d.get('required') + required = d.get("required") keys = _get_keys(d) if keys: - def strlen(key: str, d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]: + def strlen( + key: str, d1: Dict[str, PrimitiveType] + ) -> Optional[ValidatorMessage]: for k in keys: if required and k not in d1: - return {'subkey':str(k), 'message':f'Required key {k} is missing'} + return {"subkey": str(k), "message": f"Required key {k} is missing"} v = d1.get(k) if v is not None and type(v) != str: - return {'subkey':str(k), 'message':f'Metadata value at key {k} is not a string'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is not a string", + } if v and maxlen and len(_cast(str, v)) > maxlen: - return {'subkey':str(k), 'message':f'Metadata value at key {k} is longer than max length of {maxlen}'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is longer than max length of {maxlen}", + } return None + elif maxlen: - def strlen(key: str, d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]: + + def strlen( + key: str, d1: Dict[str, PrimitiveType] + ) -> Optional[ValidatorMessage]: for k, v in d1.items(): if len(k) > _cast(int, maxlen): - return {'subkey':str(k), 'message':f'Metadata contains key longer than max length of {maxlen}'} + return { + "subkey": str(k), + "message": f"Metadata contains key longer than max length of {maxlen}", + } if type(v) == str: if len(_cast(str, v)) > _cast(int, maxlen): - return {'subkey':str(k), 'message':f'Metadata value at key {k} is longer than max length of {maxlen}'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is longer than max length of {maxlen}", + } return None + else: - raise ValueError('If the keys parameter is not specified, max-len must be specified') + raise ValueError( + "If the keys parameter is not specified, max-len must be specified" + ) return strlen -def enum(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: - ''' +def enum( + d: Dict[str, Any] +) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: + """ Build a validation callable that checks that values are one of a set of specified values. The 'allowed-values' parameter is required and is a list of the allowed values for the @@ -138,65 +171,83 @@ def enum(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Optiona :param d: the configuration map for the callable. :returns: a callable that validates metadata maps. - ''' - _check_unknown_keys(d, {'allowed-values', 'keys'}) - tmpallowed = d.get('allowed-values') + """ + _check_unknown_keys(d, {"allowed-values", "keys"}) + tmpallowed = d.get("allowed-values") if not tmpallowed: - raise ValueError('allowed-values is a required parameter') + raise ValueError("allowed-values is a required parameter") if type(tmpallowed) != list: - raise ValueError('allowed-values parameter must be a list') + raise ValueError("allowed-values parameter must be a list") for i, a in enumerate(tmpallowed): if _not_primitive(a): raise ValueError( - f'allowed-values parameter contains a non-primitive type entry at index {i}') + f"allowed-values parameter contains a non-primitive type entry at index {i}" + ) allowed: _Set[PrimitiveType] = set(tmpallowed) keys = _get_keys(d) if keys: - def enumval(key: str, d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]: + def enumval( + key: str, d1: Dict[str, PrimitiveType] + ) -> Optional[ValidatorMessage]: for k in keys: if d1.get(k) not in allowed: - return {'subkey':str(k), 'message':f'Metadata value at key {k} is not in the allowed list of values'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is not in the allowed list of values", + } return None + else: - def enumval(key: str, d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]: + def enumval( + key: str, d1: Dict[str, PrimitiveType] + ) -> Optional[ValidatorMessage]: for k, v in d1.items(): if v not in allowed: - return {'subkey':str(k), 'message':f'Metadata value at key {k} is not in the allowed list of values'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is not in the allowed list of values", + } return None + return enumval def _not_primitive(value): - return (type(value) != str and type(value) != int and - type(value) != float and type(value) != bool) + return ( + type(value) != str + and type(value) != int + and type(value) != float + and type(value) != bool + ) def _get_keys(d): - keys = d.get('keys') + keys = d.get("keys") if keys: if type(keys) == str: keys = [keys] elif type(keys) != list: - raise ValueError('keys parameter must be a string or list') + raise ValueError("keys parameter must be a string or list") for i, k in enumerate(keys): if type(k) != str: - raise ValueError(f'keys parameter contains a non-string entry at index {i}') + raise ValueError( + f"keys parameter contains a non-string entry at index {i}" + ) return keys _UNIT_REG = _UnitRegistry() _UNIT_REG.load_definitions( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "unit_definitions.txt" - ) + os.path.join(os.path.dirname(os.path.abspath(__file__)), "unit_definitions.txt") ) -def units(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: - ''' +def units( + d: Dict[str, Any] +) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: + """ Build a validation callable that checks that values are equivalent to a provided example unit. E.g., if the example units are N, lb * ft / s^2 is also accepted. @@ -205,18 +256,18 @@ def units(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Option :param d: the configuration map for the callable. :returns: a callable that validates metadata maps. - ''' - _check_unknown_keys(d, {'key', 'units'}) - k = d.get('key') + """ + _check_unknown_keys(d, {"key", "units"}) + k = d.get("key") if not k: - raise ValueError('key is a required parameter') + raise ValueError("key is a required parameter") if type(k) != str: - raise ValueError('the key parameter must be a string') - u = d.get('units') + raise ValueError("the key parameter must be a string") + u = d.get("units") if not u: - raise ValueError('units is a required parameter') + raise ValueError("units is a required parameter") if type(u) != str: - raise ValueError('the units parameter must be a string') + raise ValueError("the units parameter must be a string") try: req_units = _UNIT_REG.parse_expression(u) # looks like you just need to catch these two. I wish all the pint errors inherited @@ -230,30 +281,44 @@ def units(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Option def unitval(key: str, d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]: unitstr = d1.get(_cast(str, k)) if not unitstr: - return {'subkey':str(k), 'message':f'metadata value key {k} is required'} + return {"subkey": str(k), "message": f"metadata value key {k} is required"} if type(unitstr) != str: - return {'subkey':str(k), 'message':f'metadata value key {k} must be a string'} + return { + "subkey": str(k), + "message": f"metadata value key {k} must be a string", + } try: units = _UNIT_REG.parse_expression(unitstr) except _UndefinedUnitError as e: - return {'subkey':str(k), 'message':f'unable to parse units \'{u}\' at key {k}: undefined unit: {e.args[0]}'} + return { + "subkey": str(k), + "message": f"unable to parse units '{u}' at key {k}: undefined unit: {e.args[0]}", + } except _DefinitionSyntaxError as e: - return {'subkey':str(k), 'message':f'unable to parse units \'{u}\' at key {k}: syntax error: {e.args[0]}'} + return { + "subkey": str(k), + "message": f"unable to parse units '{u}' at key {k}: syntax error: {e.args[0]}", + } try: - # Here we attempt to convert a quantity of "1" in the provided unit to + # Here we attempt to convert a quantity of "1" in the provided unit to # the canonical (also referred to as "example") unit provided in the # validation spec. pint.quantity.Quantity(1, units).to(req_units) except _DimensionalityError as e: - msg = (f"Units at key {k}, '{unitstr}', are not equivalent to " + - f"required units, '{u}': {e}") - return {'subkey':str(k), 'message':msg} + msg = ( + f"Units at key {k}, '{unitstr}', are not equivalent to " + + f"required units, '{u}': {e}" + ) + return {"subkey": str(k), "message": msg} return None + return unitval -def number(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: - ''' +def number( + d: Dict[str, Any] +) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: + """ Build a validation callable that checks that values are numerical, and optionally within a given range. @@ -275,81 +340,104 @@ def number(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Optio :param d: the configuration map for the callable. :returns: a callable that validates metadata maps. - ''' + """ # hold off on complex numbers for now - _check_unknown_keys(d, {'required', 'keys', 'type', 'lt', 'gt', 'lte', 'gte'}) - required = d.get('required') + _check_unknown_keys(d, {"required", "keys", "type", "lt", "gt", "lte", "gte"}) + required = d.get("required") keys = _get_keys(d) types = _get_types(d) # range checker range_ = _get_range(d) if keys: - def strlen(key: str, d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]: + + def strlen( + key: str, d1: Dict[str, PrimitiveType] + ) -> Optional[ValidatorMessage]: for k in keys: if required and k not in d1: - return {'subkey':str(k), 'message':f'Required key {k} is missing'} + return {"subkey": str(k), "message": f"Required key {k} is missing"} v = d1.get(k) if v is not None and type(v) not in types: - return {'subkey':str(k), 'message':f'Metadata value at key {k} is not an accepted number type'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is not an accepted number type", + } if v is not None and v not in range_: - return {'subkey':str(k), 'message':f'Metadata value at key {k} is not within the range {range_}'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is not within the range {range_}", + } return None + else: - def strlen(key: str, d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]: + + def strlen( + key: str, d1: Dict[str, PrimitiveType] + ) -> Optional[ValidatorMessage]: for k, v in d1.items(): # duplicate of above, meh. if v is not None and type(v) not in types: - return {'subkey':str(k), 'message':f'Metadata value at key {k} is not an accepted number type'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is not an accepted number type", + } if v is not None and v not in range_: - return {'subkey':str(k), 'message':f'Metadata value at key {k} is not within the range {range_}'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is not within the range {range_}", + } return None + return strlen def _get_types(d): types = [float, int] - t = d.get('type') - if t == 'int': + t = d.get("type") + if t == "int": types = [int] - elif t is not None and t != 'float': + elif t is not None and t != "float": raise ValueError(f"Illegal value for type parameter: {d.get('type')}") return types def _get_range(d): - gte = _is_num('gte', d.get('gte')) - gt = _is_num('gt', d.get('gt')) - lte = _is_num('lte', d.get('lte')) - lt = _is_num('lt', d.get('lt')) + gte = _is_num("gte", d.get("gte")) + gt = _is_num("gt", d.get("gt")) + lte = _is_num("lte", d.get("lte")) + lt = _is_num("lt", d.get("lt")) # zero is ok here, so check vs. None if gte is not None and gt is not None: - raise ValueError('Cannot specify both gt and gte') + raise ValueError("Cannot specify both gt and gte") if lte is not None and lt is not None: - raise ValueError('Cannot specify both lt and lte') + raise ValueError("Cannot specify both lt and lte") rangevals = {} if gte is not None: - rangevals['start'] = gte - rangevals['include_start'] = True + rangevals["start"] = gte + rangevals["include_start"] = True if gt is not None: - rangevals['start'] = gt - rangevals['include_start'] = False + rangevals["start"] = gt + rangevals["include_start"] = False if lte is not None: - rangevals['end'] = lte - rangevals['include_end'] = True + rangevals["end"] = lte + rangevals["include_end"] = True if lt is not None: - rangevals['end'] = lt - rangevals['include_end'] = False + rangevals["end"] = lt + rangevals["include_end"] = False return ranges.Range(**rangevals) def _is_num(name, val): if val is not None and type(val) != float and type(val) != int: - raise ValueError(f'Value for {name} parameter is not a number') + raise ValueError(f"Value for {name} parameter is not a number") return val -def ontology_has_ancestor(d: Dict[str, Any]) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: - ''' + +def ontology_has_ancestor( + d: Dict[str, Any] +) -> Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]: + """ Build a validation callable that checks that value has valid ontology ancestor term provided The 'ontology' parameter is required and must be a string. It is the ontology name. @@ -358,20 +446,20 @@ def ontology_has_ancestor(d: Dict[str, Any]) -> Callable[[str, Dict[str, Primiti :param d: the configuration map for the callable. :returns: a callable that validates metadata maps. - ''' - _check_unknown_keys(d, {'ontology', 'ancestor_term'}) + """ + _check_unknown_keys(d, {"ontology", "ancestor_term"}) - ontology = d.get('ontology') + ontology = d.get("ontology") if not ontology: - raise ValueError('ontology is a required parameter') + raise ValueError("ontology is a required parameter") if type(ontology) != str: - raise ValueError('ontology must be a string') + raise ValueError("ontology must be a string") - ancestor_term = d.get('ancestor_term') + ancestor_term = d.get("ancestor_term") if not ancestor_term: - raise ValueError('ancestor_term is a required parameter') + raise ValueError("ancestor_term is a required parameter") if type(ancestor_term) != str: - raise ValueError('ancestor_term must be a string') + raise ValueError("ancestor_term must be a string") oac = None try: @@ -381,18 +469,22 @@ def ontology_has_ancestor(d: Dict[str, Any]) -> Callable[[str, Dict[str, Primiti if not ontology_terms_cache: ret = oac.get_terms({"ids": [ancestor_term], "ns": ontology}) if len(ret["results"]) == 0 or ret["results"][0] is None: - raise ValueError(f"ancestor_term {ancestor_term} is not found in {ontology}") + raise ValueError( + f"ancestor_term {ancestor_term} is not found in {ontology}" + ) else: _ontology_terms_cache.set(terms_key, True) except Exception as err: - if 'Parameter validation error' in str(err): - raise ValueError(f'ontology {ontology} doesn\'t exist') + if "Parameter validation error" in str(err): + raise ValueError(f"ontology {ontology} doesn't exist") else: raise def _get_ontology_ancestors(ontology, val): ancestors_key = _TOKEN_SEP.join([ontology, val]) - ontology_ancestors_cache = _ontology_ancestors_cache.get(ancestors_key, default=False) + ontology_ancestors_cache = _ontology_ancestors_cache.get( + ancestors_key, default=False + ) retval = None if ontology_ancestors_cache: retval = ontology_ancestors_cache @@ -402,12 +494,21 @@ def _get_ontology_ancestors(ontology, val): _ontology_ancestors_cache.set(ancestors_key, retval) return retval - def ontology_has_ancestor_val(key: str, d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]: + def ontology_has_ancestor_val( + key: str, d1: Dict[str, PrimitiveType] + ) -> Optional[ValidatorMessage]: for k, v in d1.items(): if v is None: - return {'subkey':str(k), 'message':f'Metadata value at key {k} is None'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} is None", + } ancestors = _get_ontology_ancestors(ontology, v) if ancestor_term not in ancestors: - return {'subkey':str(k), 'message':f'Metadata value at key {k} does not have {ontology} ancestor term {ancestor_term}'} + return { + "subkey": str(k), + "message": f"Metadata value at key {k} does not have {ontology} ancestor term {ancestor_term}", + } return None + return ontology_has_ancestor_val diff --git a/lib/SampleService/core/validator/metadata_validator.py b/lib/SampleService/core/validator/metadata_validator.py index 76f98127..c2f04905 100644 --- a/lib/SampleService/core/validator/metadata_validator.py +++ b/lib/SampleService/core/validator/metadata_validator.py @@ -1,19 +1,21 @@ -''' +""" Contains a Sample Service metadata validator class. -''' +""" import maps as _maps from typing import Dict, List, Callable, Optional, Tuple as _Tuple from pygtrie import CharTrie as _CharTrie from SampleService.core.arg_checkers import not_falsy as _not_falsy from SampleService.core.core_types import PrimitiveType -from SampleService.core.errors import MetadataValidationError as _MetadataValidationError +from SampleService.core.errors import ( + MetadataValidationError as _MetadataValidationError, +) from SampleService.core.errors import IllegalParameterError as _IllegalParameterError from SampleService.core.validator.builtin import ValidatorMessage class MetadataValidator: - ''' + """ A validator for a unit of metadata. Only one of validators and prefix_validators will be populated; the other will be an @@ -35,16 +37,20 @@ class MetadataValidator: any matching key. :ivar metadata: Metadata about the key; e.g. defining the key semantics or relation to an ontology. - ''' + """ def __init__( - self, - key: str, - validators: List[Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]] = None, - prefix_validators: List[ - Callable[[str, str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]]] = None, - metadata: Dict[str, PrimitiveType] = None): - ''' + self, + key: str, + validators: List[ + Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]] + ] = None, + prefix_validators: List[ + Callable[[str, str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]] + ] = None, + metadata: Dict[str, PrimitiveType] = None, + ): + """ Create the validator. Exactly one of the validators or prefix_validators arguments must be supplied and must contain at least one validator. @@ -64,96 +70,109 @@ def __init__( run on any matching key. :param metadata: Metadata about the key; e.g. defining the key semantics or relation to an ontology. - ''' + """ # may want a builder for this? - self.key = _not_falsy(key, 'key') + self.key = _not_falsy(key, "key") if not (bool(validators) ^ bool(prefix_validators)): # xor - raise ValueError('Exactly one of validators or prefix_validators must be supplied ' + - 'and must contain at least one validator') + raise ValueError( + "Exactly one of validators or prefix_validators must be supplied " + + "and must contain at least one validator" + ) self.validators = tuple(validators if validators else []) self.prefix_validators = tuple(prefix_validators if prefix_validators else []) self.metadata = metadata if metadata else {} def is_prefix_validator(self): - ''' + """ Check if this validator is a prefix validator. :returns: True if this validator is a prefix validator, False otherwise. - ''' + """ return bool(self.prefix_validators) class MetadataValidatorSet: - ''' + """ A set of validators of metadata. - ''' + """ def __init__(self, validators: List[MetadataValidator] = None): - ''' + """ Create the validator set. :param validators: The validators. - ''' - vals: Dict[str, _Tuple[Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]], ...]] = {} + """ + vals: Dict[ + str, + _Tuple[ + Callable[[str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]], + ..., + ], + ] = {} pvals: Dict[ - str, _Tuple[Callable[[str, str, Dict[str, PrimitiveType]], Optional[ValidatorMessage]], ...]] = {} + str, + _Tuple[ + Callable[ + [str, str, Dict[str, PrimitiveType]], Optional[ValidatorMessage] + ], + ..., + ], + ] = {} self._vals_meta = {} self._prefix_vals_meta = {} - for v in (validators if validators else []): + for v in validators if validators else []: if v.is_prefix_validator(): if v.key in pvals: - raise ValueError(f'Duplicate prefix validator: {v.key}') + raise ValueError(f"Duplicate prefix validator: {v.key}") pvals[v.key] = v.prefix_validators self._prefix_vals_meta[v.key] = v.metadata else: if v.key in vals: - raise ValueError(f'Duplicate validator: {v.key}') + raise ValueError(f"Duplicate validator: {v.key}") vals[v.key] = v.validators self._vals_meta[v.key] = v.metadata self._vals = dict(vals) self._prefix_vals = _CharTrie(pvals) def keys(self): - ''' + """ Get the keys with assigned metadata validators. :returns: the metadata keys. - ''' + """ return list(self._vals.keys()) def prefix_keys(self): - ''' + """ Get the keys with assigned prefix metadata validators. :returns: the metadata keys. - ''' + """ return self._prefix_vals.keys() def key_metadata(self, keys: List[str]) -> Dict[str, Dict[str, PrimitiveType]]: - ''' + """ Get any metdata associated with the specified keys. :param keys: The keys to query. :returns: A mapping of keys to their metadata. :raises IllegalParameterError: if one of the provided keys does not exist in this validator. - ''' - return self._key_metadata(keys, self._vals_meta, '') + """ + return self._key_metadata(keys, self._vals_meta, "") def _key_metadata(self, keys, meta, name_): if keys is None: - raise ValueError('keys cannot be None') + raise ValueError("keys cannot be None") ret = {} for k in keys: if k not in meta: - raise _IllegalParameterError(f'No such {name_}metadata key: {k}') + raise _IllegalParameterError(f"No such {name_}metadata key: {k}") ret[k] = meta[k] return ret def prefix_key_metadata( - self, - keys: List[str], - exact_match: bool = True - ) -> Dict[str, Dict[str, PrimitiveType]]: - ''' + self, keys: List[str], exact_match: bool = True + ) -> Dict[str, Dict[str, PrimitiveType]]: + """ Get any metdata associated with the specified prefix keys. :param keys: The keys to query. @@ -162,110 +181,109 @@ def prefix_key_metadata( :returns: A mapping of keys to their metadata. :raises IllegalParameterError: if one of the provided keys does not exist in this validator. - ''' + """ if exact_match: - return self._key_metadata(keys, self._prefix_vals_meta, 'prefix ') + return self._key_metadata(keys, self._prefix_vals_meta, "prefix ") else: if keys is None: - raise ValueError('keys cannot be None') + raise ValueError("keys cannot be None") ret = {} for k in keys: if not self._prefix_vals.shortest_prefix(k): - raise _IllegalParameterError(f'No prefix metadata keys matching key {k}') + raise _IllegalParameterError( + f"No prefix metadata keys matching key {k}" + ) for p in self._prefix_vals.prefixes(k): ret[p.key] = self._prefix_vals_meta[p.key] return ret def validator_count(self, key: str): - ''' + """ Get the number of validators assigned to a key. :param key: the key to query. :returns: the number of validators assigned to the key. - ''' + """ if not self._vals.get(key): - raise ValueError(f'No validators for key {key}') + raise ValueError(f"No validators for key {key}") return len(self._vals[key]) def prefix_validator_count(self, prefix: str): - ''' + """ Get the number of validators assigned to a prefix key. :param prefix: the key to query. :returns: the number of validators assigned to the key. - ''' + """ if not self._prefix_vals.get(prefix): - raise ValueError(f'No prefix validators for prefix {prefix}') + raise ValueError(f"No prefix validators for prefix {prefix}") return len(self._prefix_vals[prefix]) def call_validator( - self, - key: str, - index: int, - value: Dict[str, PrimitiveType] - ) -> Optional[ValidatorMessage]: - ''' + self, key: str, index: int, value: Dict[str, PrimitiveType] + ) -> Optional[ValidatorMessage]: + """ Call a particular validator for a metadata key. :param key: the key for which a validator should be called. :param index: the index of the validator in the list of validators. :param value: the metdata value to pass to the validator as an argument. :returns: the return value of the validator. - ''' + """ if not self._vals.get(key): - raise ValueError(f'No validators for key {key}') + raise ValueError(f"No validators for key {key}") if index >= len(self._vals[key]): raise IndexError( - f'Requested validator index {index} for key {key} but maximum index ' + - f'is {len(self._vals[key]) - 1}') + f"Requested validator index {index} for key {key} but maximum index " + + f"is {len(self._vals[key]) - 1}" + ) return self._vals[key][index](key, value) def call_prefix_validator( - self, - prefix: str, - index: int, - key: str, - value: Dict[str, PrimitiveType] - ) -> Optional[ValidatorMessage]: - ''' + self, prefix: str, index: int, key: str, value: Dict[str, PrimitiveType] + ) -> Optional[ValidatorMessage]: + """ Call a particular validator for a metadata key prefix. :param prefix: the prefix for which a valiator should be called. :param index: the index of the validator in the list of validators. :param key: the key to to pass to the validator as an argument. :param value: the metdata value to pass to the validator as an argument. :returns: the return value of the validator. - ''' + """ if not self._prefix_vals.get(prefix): - raise ValueError(f'No prefix validators for prefix {prefix}') + raise ValueError(f"No prefix validators for prefix {prefix}") if index >= len(self._prefix_vals[prefix]): raise IndexError( - f'Requested validator index {index} for prefix {prefix} but maximum ' + - f'index is {len(self._prefix_vals[prefix]) - 1}') + f"Requested validator index {index} for prefix {prefix} but maximum " + + f"index is {len(self._prefix_vals[prefix]) - 1}" + ) return self._prefix_vals[prefix][index](prefix, key, value) - def build_error_detail(self, message, dev_message=None, node=None, key=None, subkey=None, sample=None): + def build_error_detail( + self, message, dev_message=None, node=None, key=None, subkey=None, sample=None + ): return { - 'message': message, - 'dev_message': dev_message if dev_message!=None else message, - 'key': key, - 'subkey': subkey, - 'node': node, - 'sample_name': sample + "message": message, + "dev_message": dev_message if dev_message != None else message, + "key": key, + "subkey": subkey, + "node": node, + "sample_name": sample, } def validate_metadata( - self, - metadata: Dict[str, Dict[str, PrimitiveType]], - return_error_detail: bool=False - ): - ''' + self, + metadata: Dict[str, Dict[str, PrimitiveType]], + return_error_detail: bool = False, + ): + """ Validate a set of metadata key/value pairs. :param metadata: the metadata. :raises MetadataValidationError: if the metadata is invalid. :returns: list of errors raised during validation - ''' + """ # if not isinstance(metadata, dict): # doesn't work if type(metadata) != dict and type(metadata) != _maps.FrozenMap: - raise ValueError('metadata must be a dict') + raise ValueError("metadata must be a dict") errors = [] for k in metadata: sp = self._prefix_vals.shortest_prefix(k) @@ -274,18 +292,19 @@ def validate_metadata( errors.append( self.build_error_detail( f'Cannot validate controlled field "{k}", no matching validator found', - key=k + key=k, ) ) else: raise _MetadataValidationError( - f'No validator available for metadata key {k}') + f"No validator available for metadata key {k}" + ) for valfunc in self._vals.get(k, []): ret = valfunc(k, metadata[k]) if ret: try: - msg: str = ret['message'] - subkey: Optional[str] = ret['subkey'] + msg: str = ret["message"] + subkey: Optional[str] = ret["subkey"] except: msg = str(ret) subkey = None @@ -293,13 +312,13 @@ def validate_metadata( errors.append( self.build_error_detail( f'Validation failed: "{msg}"', - dev_message=f'Key {k}: {msg}', + dev_message=f"Key {k}: {msg}", key=k, - subkey=subkey + subkey=subkey, ) ) else: - raise _MetadataValidationError(f'Key {k}: ' + msg) + raise _MetadataValidationError(f"Key {k}: " + msg) for p in self._prefix_vals.prefixes(k): for f in p.value: error = f(p.key, k, metadata[k]) @@ -308,11 +327,12 @@ def validate_metadata( errors.append( self.build_error_detail( f'Validation failed: "{error}" from validator for prefix "{p.key}"', - dev_message=f'Prefix validator {p.key}, key {k}: {error}', - key=k + dev_message=f"Prefix validator {p.key}, key {k}: {error}", + key=k, ) ) else: raise _MetadataValidationError( - f'Prefix validator {p.key}, key {k}: {error}') + f"Prefix validator {p.key}, key {k}: {error}" + ) return errors diff --git a/lib/SampleService/core/workspace.py b/lib/SampleService/core/workspace.py index a3e73219..b5426d76 100644 --- a/lib/SampleService/core/workspace.py +++ b/lib/SampleService/core/workspace.py @@ -1,6 +1,6 @@ -''' +""" Methods for accessing workspace data. -''' +""" from enum import IntEnum from typing import List, Optional, Dict as _Dict, Set as _Set @@ -11,15 +11,18 @@ from SampleService.core.arg_checkers import check_string as _check_string from SampleService.core.errors import IllegalParameterError as _IllegalParameterError from SampleService.core.errors import UnauthorizedError as _UnauthorizedError -from SampleService.core.errors import NoSuchWorkspaceDataError as _NoSuchWorkspaceDataError +from SampleService.core.errors import ( + NoSuchWorkspaceDataError as _NoSuchWorkspaceDataError, +) from SampleService.core.errors import NoSuchUserError as _NoSuchUserError from SampleService.core.user import UserID class WorkspaceAccessType(IntEnum): - ''' + """ The different levels of workspace service access. - ''' + """ + NONE = 1 READ = 2 WRITE = 3 @@ -28,19 +31,20 @@ class WorkspaceAccessType(IntEnum): _PERM_TO_PERM_SET: _Dict[WorkspaceAccessType, _Set[str]] = { WorkspaceAccessType.NONE: set(), - WorkspaceAccessType.READ: {'r', 'w', 'a'}, - WorkspaceAccessType.WRITE: {'w', 'a'}, - WorkspaceAccessType.ADMIN: {'a'} - } + WorkspaceAccessType.READ: {"r", "w", "a"}, + WorkspaceAccessType.WRITE: {"w", "a"}, + WorkspaceAccessType.ADMIN: {"a"}, +} -_PERM_TO_PERM_TEXT = {WorkspaceAccessType.READ: 'read', - WorkspaceAccessType.WRITE: 'write to', - WorkspaceAccessType.ADMIN: 'administrate' - } +_PERM_TO_PERM_TEXT = { + WorkspaceAccessType.READ: "read", + WorkspaceAccessType.WRITE: "write to", + WorkspaceAccessType.ADMIN: "administrate", +} class UPA: - ''' + """ A Unique Permanent Address for a workspace object, consisting of the string 'X/Y/Z' where X, Y and Z are integers greater than 0 and respectively the workspace ID, the object ID, and the object version of the object. @@ -50,10 +54,12 @@ class UPA: :ivar wsid: The workspace ID. :ivar objid: The object ID. :ivar version: The object version. - ''' + """ - def __init__(self, upa: str = None, wsid: int = None, objid: int = None, version: int = None): - ''' + def __init__( + self, upa: str = None, wsid: int = None, objid: int = None, version: int = None + ): + """ Create the UPA. Requires either the upa parameter or all of the wsid, objid, and version parameters. If upa is supplied the other arguments are ignored. @@ -62,44 +68,50 @@ def __init__(self, upa: str = None, wsid: int = None, objid: int = None, version :param objid: The object ID. :param version: The object version. :raises IllegalParameterError: if the UPA is invalid. - ''' + """ if upa: self.wsid, self.objid, self.version = self._check_upa(upa) else: for num, name in ( - (wsid, 'workspace ID'), - (objid, 'object ID'), - (version, 'object version')): + (wsid, "workspace ID"), + (objid, "object ID"), + (version, "object version"), + ): if not num or num < 1: - raise _IllegalParameterError(f'Illegal {name}: {num}') + raise _IllegalParameterError(f"Illegal {name}: {num}") self.wsid = wsid self.objid = objid self.version = version def _check_upa(self, upa): - upastr = upa.split('/') + upastr = upa.split("/") if len(upastr) != 3: - raise _IllegalParameterError(f'{upa} is not a valid UPA') - return (self._get_ws_num(upastr[0], upa), - self._get_ws_num(upastr[1], upa), - self._get_ws_num(upastr[2], upa)) + raise _IllegalParameterError(f"{upa} is not a valid UPA") + return ( + self._get_ws_num(upastr[0], upa), + self._get_ws_num(upastr[1], upa), + self._get_ws_num(upastr[2], upa), + ) def _get_ws_num(self, int_: str, upa): try: i = int(int_) if i < 1: - raise _IllegalParameterError(f'{upa} is not a valid UPA') + raise _IllegalParameterError(f"{upa} is not a valid UPA") return i except ValueError: - raise _IllegalParameterError(f'{upa} is not a valid UPA') + raise _IllegalParameterError(f"{upa} is not a valid UPA") def __str__(self) -> str: - return f'{self.wsid}/{self.objid}/{self.version}' + return f"{self.wsid}/{self.objid}/{self.version}" def __eq__(self, other) -> bool: if type(self) is type(other): return (self.wsid, self.objid, self.version) == ( - other.wsid, other.objid, other.version) + other.wsid, + other.objid, + other.version, + ) return False def __hash__(self): @@ -107,7 +119,7 @@ def __hash__(self): class DataUnitID: - ''' + """ Represents a unit of data in the workspace, which may be a subpart of a workspace object. A single workspace object may have many data units. @@ -116,22 +128,22 @@ class DataUnitID: :ivar upa: The object UPA. :ivar dataid: The ID of the data within the object, if any. - ''' + """ def __init__(self, upa: UPA, dataid: str = None): - ''' + """ Create the DUID. :param upa: The workspace object's UPA. :param dataid: The id of the data within the object that this DUID references with a maximum of 256 characters. None if the data unit is the entire object. - ''' - self.upa = _not_falsy(upa, 'upa') - self.dataid = _check_string(dataid, 'dataid', max_len=256, optional=True) + """ + self.upa = _not_falsy(upa, "upa") + self.dataid = _check_string(dataid, "dataid", max_len=256, optional=True) def __str__(self): if self.dataid: - return f'{self.upa}:{self.dataid}' + return f"{self.upa}:{self.dataid}" else: return str(self.upa) @@ -145,30 +157,31 @@ def __hash__(self): class WS: - ''' + """ The workspace class. - ''' + """ def __init__(self, client: Workspace): - ''' + """ Create the workspace class. Attempts to contact the endpoint of the workspace in administration mode and does not catch any exceptions encountered. :param client: An SDK workspace client with administrator read permissions. - ''' - self._ws = _not_falsy(client, 'client') + """ + self._ws = _not_falsy(client, "client") # check token is a valid admin token - self._ws.administer({'command': 'listModRequests'}) + self._ws.administer({"command": "listModRequests"}) def has_permission( - self, - user: Optional[UserID], - perm: WorkspaceAccessType, - workspace_id: int = None, - upa: UPA = None): - ''' + self, + user: Optional[UserID], + perm: WorkspaceAccessType, + workspace_id: int = None, + upa: UPA = None, + ): + """ Check if a user can access a workspace resource. Exactly one of workspace_id or upa must be supplied - if both are supplied workspace_id takes precedence. @@ -184,72 +197,82 @@ def has_permission( :raises IllegalParameterError: if the wsid is illegal. :raises UnauthorizedError: if the user doesn't have the requested permission. :raises NoSuchWorkspaceDataError: if the workspace or UPA doesn't exist. - ''' - _not_falsy(perm, 'perm') + """ + _not_falsy(perm, "perm") if workspace_id is not None: wsid = workspace_id - name = 'workspace' + name = "workspace" target = str(workspace_id) upa = None elif upa: wsid = upa.wsid - name = 'upa' + name = "upa" target = str(upa) else: - raise ValueError('Either an UPA or a workpace ID must be supplied') + raise ValueError("Either an UPA or a workpace ID must be supplied") if wsid < 1: - raise _IllegalParameterError(f'{wsid} is not a valid workspace ID') + raise _IllegalParameterError(f"{wsid} is not a valid workspace ID") try: - p = self._ws.administer({'command': 'getPermissionsMass', - 'params': {'workspaces': [{'id': wsid}]} - } - )['perms'][0] + p = self._ws.administer( + { + "command": "getPermissionsMass", + "params": {"workspaces": [{"id": wsid}]}, + } + )["perms"][0] except _ServerError as se: # this is pretty ugly, need error codes - if 'No workspace' in se.args[0] or 'is deleted' in se.args[0]: + if "No workspace" in se.args[0] or "is deleted" in se.args[0]: raise _NoSuchWorkspaceDataError(se.args[0]) from se else: raise - publicaccess = p.get('*') == 'r' and perm == WorkspaceAccessType.READ + publicaccess = p.get("*") == "r" and perm == WorkspaceAccessType.READ hasaccess = p.get(user.id) in _PERM_TO_PERM_SET[perm] if user else False # could optimize a bit if NONE and upa but not worth the code complication most likely - if (perm != WorkspaceAccessType.NONE and not hasaccess and not publicaccess): - u = f'User {user}' if user else 'Anonymous users' - raise _UnauthorizedError(f'{u} cannot {_PERM_TO_PERM_TEXT[perm]} {name} {target}') + if perm != WorkspaceAccessType.NONE and not hasaccess and not publicaccess: + u = f"User {user}" if user else "Anonymous users" + raise _UnauthorizedError( + f"{u} cannot {_PERM_TO_PERM_TEXT[perm]} {name} {target}" + ) if upa: # Allow any server errors to percolate upwards # theoretically the workspace could've been deleted between the last call and this # one, but that'll just result in a different error and is extremely unlikely to # happen, so don't worry about it - ret = self._ws.administer({'command': 'getObjectInfo', - 'params': {'objects': [{'ref': str(upa)}], - 'ignoreErrors': 1} - }) - if not ret['infos'][0]: - raise _NoSuchWorkspaceDataError(f'Object {upa} does not exist') + ret = self._ws.administer( + { + "command": "getObjectInfo", + "params": {"objects": [{"ref": str(upa)}], "ignoreErrors": 1}, + } + ) + if not ret["infos"][0]: + raise _NoSuchWorkspaceDataError(f"Object {upa} does not exist") def get_user_workspaces(self, user: Optional[UserID]) -> List[int]: - ''' + """ Get a list of IDs of workspaces a user can read, including public workspaces. :param user: The username of the user whose workspaces will be returned, or null for an anonymous user. :returns: A list of workspace IDs. :raises NoSuchUserError: if the user does not exist. - ''' + """ # May also want write / admin / no public ws try: if user: - ids = self._ws.administer({'command': 'listWorkspaceIDs', - 'user': user.id, - 'params': {'perm': 'r', 'excludeGlobal': 0}}) + ids = self._ws.administer( + { + "command": "listWorkspaceIDs", + "user": user.id, + "params": {"perm": "r", "excludeGlobal": 0}, + } + ) else: - ids = self._ws.list_workspace_ids({'onlyGlobal': 1}) + ids = self._ws.list_workspace_ids({"onlyGlobal": 1}) except _ServerError as se: # this is pretty ugly, need error codes - if 'not a valid user' in se.args[0]: + if "not a valid user" in se.args[0]: raise _NoSuchUserError(se.args[0]) from se else: raise - return sorted(ids['workspaces'] + ids['pub']) + return sorted(ids["workspaces"] + ids["pub"]) diff --git a/lib/SampleService/impl_methods.py b/lib/SampleService/impl_methods.py index 7e10ad60..3141c04e 100644 --- a/lib/SampleService/impl_methods.py +++ b/lib/SampleService/impl_methods.py @@ -1,8 +1,8 @@ -''' +""" Module containing SampleService methods. Use this module to write methods for use in SampleServiceImpl.py. -''' +""" from SampleService.core.api_translation import ( acl_delta_from_dict as _acl_delta_from_dict, check_admin as _check_admin, @@ -14,7 +14,7 @@ def update_samples_acls( params, samples_client, user_lookup, user, token, perms, log_info ): - ''' + """ Completely replace the ACLs for a list of samples. :param id_: the sample's ID. @@ -37,19 +37,17 @@ def update_samples_acls( required. :raises UnauthorizedError: if the user does not have admin permission for the sample or the request attempts to alter the owner. - ''' + """ acldelta = _acl_delta_from_dict(params) admin = _check_admin( user_lookup, token, perms, - 'update_sample_acls', + "update_sample_acls", log_info, - skip_check=not params.get('as_admin'), + skip_check=not params.get("as_admin"), ) - ids = params.get('ids') + ids = params.get("ids") for id_ in ids: - _validate_sample_id(id_, '') - samples_client.update_sample_acls( - id_, _UserID(user), acldelta, as_admin=admin - ) + _validate_sample_id(id_, "") + samples_client.update_sample_acls(id_, _UserID(user), acldelta, as_admin=admin) diff --git a/lib/biokbase/log.py b/lib/biokbase/log.py index 5626ac03..25b3cb29 100644 --- a/lib/biokbase/log.py +++ b/lib/biokbase/log.py @@ -77,15 +77,15 @@ from configparser import ConfigParser as _ConfigParser import time -MLOG_ENV_FILE = 'MLOG_CONFIG_FILE' -_GLOBAL = 'global' -MLOG_LOG_LEVEL = 'mlog_log_level' -MLOG_API_URL = 'mlog_api_url' -MLOG_LOG_FILE = 'mlog_log_file' +MLOG_ENV_FILE = "MLOG_CONFIG_FILE" +_GLOBAL = "global" +MLOG_LOG_LEVEL = "mlog_log_level" +MLOG_API_URL = "mlog_api_url" +MLOG_LOG_FILE = "mlog_log_file" DEFAULT_LOG_LEVEL = 6 -#MSG_CHECK_COUNT = 100 -#MSG_CHECK_INTERVAL = 300 # 300s = 5min +# MSG_CHECK_COUNT = 100 +# MSG_CHECK_INTERVAL = 300 # 300s = 5min MSG_FACILITY = _syslog.LOG_LOCAL1 EMERG_FACILITY = _syslog.LOG_LOCAL0 @@ -99,22 +99,31 @@ DEBUG = 7 DEBUG2 = 8 DEBUG3 = 9 -_MLOG_TEXT_TO_LEVEL = {'EMERG': EMERG, - 'ALERT': ALERT, - 'CRIT': CRIT, - 'ERR': ERR, - 'WARNING': WARNING, - 'NOTICE': NOTICE, - 'INFO': INFO, - 'DEBUG': DEBUG, - 'DEBUG2': DEBUG2, - 'DEBUG3': DEBUG3, - } -_MLOG_TO_SYSLOG = [_syslog.LOG_EMERG, _syslog.LOG_ALERT, _syslog.LOG_CRIT, - _syslog.LOG_ERR, _syslog.LOG_WARNING, _syslog.LOG_NOTICE, - _syslog.LOG_INFO, _syslog.LOG_DEBUG, _syslog.LOG_DEBUG, - _syslog.LOG_DEBUG] -#ALLOWED_LOG_LEVELS = set(_MLOG_TEXT_TO_LEVEL.values()) +_MLOG_TEXT_TO_LEVEL = { + "EMERG": EMERG, + "ALERT": ALERT, + "CRIT": CRIT, + "ERR": ERR, + "WARNING": WARNING, + "NOTICE": NOTICE, + "INFO": INFO, + "DEBUG": DEBUG, + "DEBUG2": DEBUG2, + "DEBUG3": DEBUG3, +} +_MLOG_TO_SYSLOG = [ + _syslog.LOG_EMERG, + _syslog.LOG_ALERT, + _syslog.LOG_CRIT, + _syslog.LOG_ERR, + _syslog.LOG_WARNING, + _syslog.LOG_NOTICE, + _syslog.LOG_INFO, + _syslog.LOG_DEBUG, + _syslog.LOG_DEBUG, + _syslog.LOG_DEBUG, +] +# ALLOWED_LOG_LEVELS = set(_MLOG_TEXT_TO_LEVEL.values()) _MLOG_LEVEL_TO_TEXT = {} for k, v in _MLOG_TEXT_TO_LEVEL.items(): _MLOG_LEVEL_TO_TEXT[v] = k @@ -128,15 +137,24 @@ class log(object): This class contains the methods necessary for sending log messages. """ - def __init__(self, subsystem, constraints=None, config=None, logfile=None, - ip_address=False, authuser=False, module=False, - method=False, call_id=False, changecallback=None): + def __init__( + self, + subsystem, + constraints=None, + config=None, + logfile=None, + ip_address=False, + authuser=False, + module=False, + method=False, + call_id=False, + changecallback=None, + ): if not subsystem: raise ValueError("Subsystem must be supplied") self.user = _getpass.getuser() - self.parentfile = _os.path.abspath(_inspect.getfile( - _inspect.stack()[1][0])) + self.parentfile = _os.path.abspath(_inspect.getfile(_inspect.stack()[1][0])) self.ip_address = ip_address self.authuser = authuser self.module = module @@ -171,11 +189,11 @@ def _get_time_since_start(self): return time_diff def get_log_level(self): - if(self._user_log_level != -1): + if self._user_log_level != -1: return self._user_log_level - elif(self._config_log_level != -1): + elif self._config_log_level != -1: return self._config_log_level - elif(self._api_log_level != -1): + elif self._api_log_level != -1: return self._api_log_level else: return DEFAULT_LOG_LEVEL @@ -207,35 +225,37 @@ def update_config(self): self._config_log_level = int(cfgitems[MLOG_LOG_LEVEL]) except: _warnings.warn( - 'Cannot parse log level {} from file {} to int'.format( - cfgitems[MLOG_LOG_LEVEL], self._mlog_config_file) - + '. Keeping current log level.') + "Cannot parse log level {} from file {} to int".format( + cfgitems[MLOG_LOG_LEVEL], self._mlog_config_file + ) + + ". Keeping current log level." + ) if MLOG_API_URL in cfgitems: api_url = cfgitems[MLOG_API_URL] if MLOG_LOG_FILE in cfgitems: self._config_log_file = cfgitems[MLOG_LOG_FILE] elif self._mlog_config_file: - _warnings.warn('Cannot read config file ' + self._mlog_config_file) + _warnings.warn("Cannot read config file " + self._mlog_config_file) - if (api_url): + if api_url: subsystem_api_url = api_url + "/" + self._subsystem try: - data = _json.load(_urllib2.urlopen(subsystem_api_url, - timeout=5)) + data = _json.load(_urllib2.urlopen(subsystem_api_url, timeout=5)) except _urllib2.URLError as e: code_ = None - if hasattr(e, 'code'): - code_ = ' ' + str(e.code) + if hasattr(e, "code"): + code_ = " " + str(e.code) _warnings.warn( - 'Could not connect to mlog api server at ' + - '{}:{} {}. Using default log level {}.'.format( - subsystem_api_url, code_, str(e.reason), - str(DEFAULT_LOG_LEVEL))) + "Could not connect to mlog api server at " + + "{}:{} {}. Using default log level {}.".format( + subsystem_api_url, code_, str(e.reason), str(DEFAULT_LOG_LEVEL) + ) + ) else: max_matching_level = -1 - for constraint_set in data['log_levels']: - level = constraint_set['level'] - constraints = constraint_set['constraints'] + for constraint_set in data["log_levels"]: + level = constraint_set["level"] + constraints = constraint_set["constraints"] if level <= max_matching_level: continue @@ -243,23 +263,25 @@ def update_config(self): for constraint in constraints: if constraint not in self._log_constraints: matches = 0 - elif (self._log_constraints[constraint] != - constraints[constraint]): + elif ( + self._log_constraints[constraint] != constraints[constraint] + ): matches = 0 if matches == 1: max_matching_level = level self._api_log_level = max_matching_level - if ((self.get_log_level() != loglevel or - self.get_log_file() != logfile) and not self._init): + if ( + self.get_log_level() != loglevel or self.get_log_file() != logfile + ) and not self._init: self._callback() def _resolve_log_level(self, level): - if(level in _MLOG_TEXT_TO_LEVEL): + if level in _MLOG_TEXT_TO_LEVEL: level = _MLOG_TEXT_TO_LEVEL[level] - elif(level not in _MLOG_LEVEL_TO_TEXT): - raise ValueError('Illegal log level') + elif level not in _MLOG_LEVEL_TO_TEXT: + raise ValueError("Illegal log level") return level def set_log_level(self, level): @@ -280,33 +302,40 @@ def set_log_file(self, filename): def set_log_msg_check_count(self, count): count = int(count) if count < 0: - raise ValueError('Cannot check a negative number of messages') + raise ValueError("Cannot check a negative number of messages") self._recheck_api_msg = count def set_log_msg_check_interval(self, interval): interval = int(interval) if interval < 0: - raise ValueError('interval must be positive') + raise ValueError("interval must be positive") self._recheck_api_time = interval def clear_user_log_level(self): self._user_log_level = -1 self._callback() - def _get_ident(self, level, user, parentfile, ip_address, authuser, module, - method, call_id): - infos = [self._subsystem, _MLOG_LEVEL_TO_TEXT[level], - repr(time.time()), user, parentfile, str(_os.getpid())] + def _get_ident( + self, level, user, parentfile, ip_address, authuser, module, method, call_id + ): + infos = [ + self._subsystem, + _MLOG_LEVEL_TO_TEXT[level], + repr(time.time()), + user, + parentfile, + str(_os.getpid()), + ] if self.ip_address: - infos.append(str(ip_address) if ip_address else '-') + infos.append(str(ip_address) if ip_address else "-") if self.authuser: - infos.append(str(authuser) if authuser else '-') + infos.append(str(authuser) if authuser else "-") if self.module: - infos.append(str(module) if module else '-') + infos.append(str(module) if module else "-") if self.method: - infos.append(str(method) if method else '-') + infos.append(str(method) if method else "-") if self.call_id: - infos.append(str(call_id) if call_id else '-') + infos.append(str(call_id) if call_id else "-") return "[" + "] [".join(infos) + "]" def _syslog(self, facility, level, ident, message): @@ -322,47 +351,75 @@ def _syslog(self, facility, level, ident, message): _syslog.closelog() def _log(self, ident, message): - ident = ' '.join([str(time.strftime( - "%Y-%m-%d %H:%M:%S", time.localtime())), - _platform.node(), ident + ': ']) + ident = " ".join( + [ + str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())), + _platform.node(), + ident + ": ", + ] + ) try: - with open(self.get_log_file(), 'a') as log: + with open(self.get_log_file(), "a") as log: if isinstance(message, str): - log.write(ident + message + '\n') + log.write(ident + message + "\n") else: try: for m in message: - log.write(ident + m + '\n') + log.write(ident + m + "\n") except TypeError: - log.write(ident + str(message) + '\n') + log.write(ident + str(message) + "\n") except Exception as e: - err = 'Could not write to log file ' + str(self.get_log_file()) + \ - ': ' + str(e) + '.' + err = ( + "Could not write to log file " + + str(self.get_log_file()) + + ": " + + str(e) + + "." + ) _warnings.warn(err) - def log_message(self, level, message, ip_address=None, authuser=None, - module=None, method=None, call_id=None): -# message = str(message) + def log_message( + self, + level, + message, + ip_address=None, + authuser=None, + module=None, + method=None, + call_id=None, + ): + # message = str(message) level = self._resolve_log_level(level) self.msg_count += 1 self._msgs_since_config_update += 1 - if(self._msgs_since_config_update >= self._recheck_api_msg - or self._get_time_since_start() >= self._recheck_api_time): + if ( + self._msgs_since_config_update >= self._recheck_api_msg + or self._get_time_since_start() >= self._recheck_api_time + ): self.update_config() - ident = self._get_ident(level, self.user, self.parentfile, ip_address, - authuser, module, method, call_id) + ident = self._get_ident( + level, + self.user, + self.parentfile, + ip_address, + authuser, + module, + method, + call_id, + ) # If this message is an emergency, send a copy to the emergency # facility first. - if(level == 0): + if level == 0: self._syslog(EMERG_FACILITY, level, ident, message) - if(level <= self.get_log_level()): + if level <= self.get_log_level(): self._syslog(MSG_FACILITY, level, ident, message) if self.get_log_file(): self._log(ident, message) -if __name__ == '__main__': + +if __name__ == "__main__": pass diff --git a/lib/installed_clients/OntologyAPIClient.py b/lib/installed_clients/OntologyAPIClient.py index 7bb8bf16..5f1095ae 100644 --- a/lib/installed_clients/OntologyAPIClient.py +++ b/lib/installed_clients/OntologyAPIClient.py @@ -11,22 +11,32 @@ class OntologyAPI(object): - def __init__( - self, url=None, timeout=30 * 60, user_id=None, - password=None, token=None, ignore_authrc=False, - trust_all_ssl_certificates=False, - auth_svc='https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login', - service_ver='dev'): + self, + url=None, + timeout=30 * 60, + user_id=None, + password=None, + token=None, + ignore_authrc=False, + trust_all_ssl_certificates=False, + auth_svc="https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login", + service_ver="dev", + ): if url is None: - url = 'https://kbase.us/services/service_wizard' + url = "https://kbase.us/services/service_wizard" self._service_ver = service_ver self._client = _BaseClient( - url, timeout=timeout, user_id=user_id, password=password, - token=token, ignore_authrc=ignore_authrc, + url, + timeout=timeout, + user_id=user_id, + password=password, + token=token, + ignore_authrc=ignore_authrc, trust_all_ssl_certificates=trust_all_ssl_certificates, auth_svc=auth_svc, - lookup_url=True) + lookup_url=True, + ) def get_descendants(self, GenericParams, context=None): """ @@ -49,8 +59,9 @@ def get_descendants(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_descendants', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_descendants", [GenericParams], self._service_ver, context + ) def get_ancestors(self, GenericParams, context=None): """ @@ -73,8 +84,9 @@ def get_ancestors(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_ancestors', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_ancestors", [GenericParams], self._service_ver, context + ) def get_children(self, GenericParams, context=None): """ @@ -97,8 +109,9 @@ def get_children(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_children', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_children", [GenericParams], self._service_ver, context + ) def get_parents(self, GenericParams, context=None): """ @@ -121,8 +134,9 @@ def get_parents(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_parents', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_parents", [GenericParams], self._service_ver, context + ) def get_related(self, GenericParams, context=None): """ @@ -145,8 +159,9 @@ def get_related(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_related', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_related", [GenericParams], self._service_ver, context + ) def get_siblings(self, GenericParams, context=None): """ @@ -169,8 +184,9 @@ def get_siblings(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_siblings', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_siblings", [GenericParams], self._service_ver, context + ) def get_terms(self, GetTermsParams, context=None): """ @@ -191,8 +207,9 @@ def get_terms(self, GetTermsParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_terms', - [GetTermsParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_terms", [GetTermsParams], self._service_ver, context + ) def get_hierarchical_ancestors(self, GenericParams, context=None): """ @@ -215,8 +232,12 @@ def get_hierarchical_ancestors(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_hierarchical_ancestors', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_hierarchical_ancestors", + [GenericParams], + self._service_ver, + context, + ) def get_hierarchical_children(self, GenericParams, context=None): """ @@ -239,8 +260,12 @@ def get_hierarchical_children(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_hierarchical_children', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_hierarchical_children", + [GenericParams], + self._service_ver, + context, + ) def get_hierarchical_descendants(self, GenericParams, context=None): """ @@ -263,8 +288,12 @@ def get_hierarchical_descendants(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_hierarchical_descendants', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_hierarchical_descendants", + [GenericParams], + self._service_ver, + context, + ) def get_hierarchical_parents(self, GenericParams, context=None): """ @@ -287,8 +316,12 @@ def get_hierarchical_parents(self, GenericParams, context=None): unspecified object, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_hierarchical_parents', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_hierarchical_parents", + [GenericParams], + self._service_ver, + context, + ) def get_associated_ws_objects(self, GenericParams, context=None): """ @@ -319,8 +352,12 @@ def get_associated_ws_objects(self, GenericParams, context=None): "feature_id" of String, parameter "updated_at" of Long, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_associated_ws_objects', - [GenericParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_associated_ws_objects", + [GenericParams], + self._service_ver, + context, + ) def get_terms_from_ws_feature(self, GetTermsFromWSFeatureParams, context=None): """ @@ -355,8 +392,12 @@ def get_terms_from_ws_feature(self, GetTermsFromWSFeatureParams, context=None): "workspace_id" of Long, parameter "object_id" of Long, parameter "version" of Long, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_terms_from_ws_feature', - [GetTermsFromWSFeatureParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_terms_from_ws_feature", + [GetTermsFromWSFeatureParams], + self._service_ver, + context, + ) def get_terms_from_ws_obj(self, GetTermsFromWSObjParams, context=None): """ @@ -389,9 +430,14 @@ def get_terms_from_ws_obj(self, GetTermsFromWSObjParams, context=None): "workspace_id" of Long, parameter "object_id" of Long, parameter "version" of Long, parameter "ts" of Long, parameter "ns" of String """ - return self._client.call_method('OntologyAPI.get_terms_from_ws_obj', - [GetTermsFromWSObjParams], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.get_terms_from_ws_obj", + [GetTermsFromWSObjParams], + self._service_ver, + context, + ) def status(self, context=None): - return self._client.call_method('OntologyAPI.status', - [], self._service_ver, context) + return self._client.call_method( + "OntologyAPI.status", [], self._service_ver, context + ) diff --git a/lib/installed_clients/WorkspaceClient.py b/lib/installed_clients/WorkspaceClient.py index e6f203d6..88ddf100 100644 --- a/lib/installed_clients/WorkspaceClient.py +++ b/lib/installed_clients/WorkspaceClient.py @@ -11,28 +11,37 @@ class Workspace(object): - def __init__( - self, url=None, timeout=30 * 60, user_id=None, - password=None, token=None, ignore_authrc=False, - trust_all_ssl_certificates=False, - auth_svc='https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login'): + self, + url=None, + timeout=30 * 60, + user_id=None, + password=None, + token=None, + ignore_authrc=False, + trust_all_ssl_certificates=False, + auth_svc="https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login", + ): if url is None: - raise ValueError('A url is required') + raise ValueError("A url is required") self._service_ver = None self._client = _BaseClient( - url, timeout=timeout, user_id=user_id, password=password, - token=token, ignore_authrc=ignore_authrc, + url, + timeout=timeout, + user_id=user_id, + password=password, + token=token, + ignore_authrc=ignore_authrc, trust_all_ssl_certificates=trust_all_ssl_certificates, - auth_svc=auth_svc) + auth_svc=auth_svc, + ) def ver(self, context=None): """ Returns the version of the workspace service. :returns: instance of String """ - return self._client.call_method('Workspace.ver', - [], self._service_ver, context) + return self._client.call_method("Workspace.ver", [], self._service_ver, context) def create_workspace(self, params, context=None): """ @@ -96,8 +105,9 @@ def create_workspace(self, params, context=None): object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.create_workspace', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.create_workspace", [params], self._service_ver, context + ) def alter_workspace_metadata(self, params, context=None): """ @@ -123,8 +133,9 @@ def alter_workspace_metadata(self, params, context=None): Arbitrary key-value pairs provided by the user.) -> mapping from String to String, parameter "remove" of list of String """ - return self._client.call_method('Workspace.alter_workspace_metadata', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.alter_workspace_metadata", [params], self._service_ver, context + ) def clone_workspace(self, params, context=None): """ @@ -235,15 +246,16 @@ def clone_workspace(self, params, context=None): object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.clone_workspace', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.clone_workspace", [params], self._service_ver, context + ) def lock_workspace(self, wsi, context=None): """ Lock a workspace, preventing further changes. WARNING: Locking a workspace is permanent. A workspace, once locked, cannot be unlocked. - + The only changes allowed for a locked workspace are changing user based permissions or making a private workspace globally readable, thus permanently publishing the workspace. A locked, globally readable @@ -298,13 +310,14 @@ def lock_workspace(self, wsi, context=None): object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.lock_workspace', - [wsi], self._service_ver, context) + return self._client.call_method( + "Workspace.lock_workspace", [wsi], self._service_ver, context + ) def get_workspacemeta(self, params, context=None): """ Retrieves the metadata associated with the specified workspace. - Provided for backwards compatibility. + Provided for backwards compatibility. @deprecated Workspace.get_workspace_info :param params: instance of type "get_workspacemeta_params" (DEPRECATED Input parameters for the "get_workspacemeta" function. @@ -353,8 +366,9 @@ def get_workspacemeta(self, params, context=None): read. 'n' - no permissions.), parameter "num_id" of type "ws_id" (The unique, permanent numerical ID of a workspace.) """ - return self._client.call_method('Workspace.get_workspacemeta', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.get_workspacemeta", [params], self._service_ver, context + ) def get_workspace_info(self, wsi, context=None): """ @@ -409,8 +423,9 @@ def get_workspace_info(self, wsi, context=None): object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.get_workspace_info', - [wsi], self._service_ver, context) + return self._client.call_method( + "Workspace.get_workspace_info", [wsi], self._service_ver, context + ) def get_workspace_description(self, wsi, context=None): """ @@ -428,8 +443,9 @@ def get_workspace_description(self, wsi, context=None): of a workspace.) :returns: instance of String """ - return self._client.call_method('Workspace.get_workspace_description', - [wsi], self._service_ver, context) + return self._client.call_method( + "Workspace.get_workspace_description", [wsi], self._service_ver, context + ) def set_permissions(self, params, context=None): """ @@ -453,8 +469,9 @@ def set_permissions(self, params, context=None): permissions.), parameter "users" of list of type "username" (Login name of a KBase user account.) """ - return self._client.call_method('Workspace.set_permissions', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.set_permissions", [params], self._service_ver, context + ) def set_global_permission(self, params, context=None): """ @@ -478,8 +495,9 @@ def set_global_permission(self, params, context=None): administrator. All operations allowed. 'w' - read/write. 'r' - read. 'n' - no permissions.) """ - return self._client.call_method('Workspace.set_global_permission', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.set_global_permission", [params], self._service_ver, context + ) def set_workspace_description(self, params, context=None): """ @@ -500,8 +518,9 @@ def set_workspace_description(self, params, context=None): unique, permanent numerical ID of a workspace.), parameter "description" of String """ - return self._client.call_method('Workspace.set_workspace_description', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.set_workspace_description", [params], self._service_ver, context + ) def get_permissions_mass(self, mass, context=None): """ @@ -528,8 +547,9 @@ def get_permissions_mass(self, mass, context=None): workspace: 'a' - administrator. All operations allowed. 'w' - read/write. 'r' - read. 'n' - no permissions.) """ - return self._client.call_method('Workspace.get_permissions_mass', - [mass], self._service_ver, context) + return self._client.call_method( + "Workspace.get_permissions_mass", [mass], self._service_ver, context + ) def get_permissions(self, wsi, context=None): """ @@ -552,8 +572,9 @@ def get_permissions(self, wsi, context=None): administrator. All operations allowed. 'w' - read/write. 'r' - read. 'n' - no permissions.) """ - return self._client.call_method('Workspace.get_permissions', - [wsi], self._service_ver, context) + return self._client.call_method( + "Workspace.get_permissions", [wsi], self._service_ver, context + ) def save_object(self, params, context=None): """ @@ -647,8 +668,9 @@ def save_object(self, params, context=None): String, parameter "objid" of type "obj_id" (The unique, permanent numerical ID of an object.) """ - return self._client.call_method('Workspace.save_object', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.save_object", [params], self._service_ver, context + ) def save_objects(self, params, context=None): """ @@ -873,8 +895,9 @@ def save_objects(self, params, context=None): metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.save_objects', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.save_objects", [params], self._service_ver, context + ) def get_object(self, params, context=None): """ @@ -955,8 +978,9 @@ def get_object(self, params, context=None): String, parameter "objid" of type "obj_id" (The unique, permanent numerical ID of an object.) """ - return self._client.call_method('Workspace.get_object', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.get_object", [params], self._service_ver, context + ) def get_object_provenance(self, object_ids, context=None): """ @@ -1222,8 +1246,9 @@ def get_object_provenance(self, object_ids, context=None): from an object.), parameter "handle_error" of String, parameter "handle_stacktrace" of String """ - return self._client.call_method('Workspace.get_object_provenance', - [object_ids], self._service_ver, context) + return self._client.call_method( + "Workspace.get_object_provenance", [object_ids], self._service_ver, context + ) def get_objects(self, object_ids, context=None): """ @@ -1502,8 +1527,9 @@ def get_objects(self, object_ids, context=None): from an object.), parameter "handle_error" of String, parameter "handle_stacktrace" of String """ - return self._client.call_method('Workspace.get_objects', - [object_ids], self._service_ver, context) + return self._client.call_method( + "Workspace.get_objects", [object_ids], self._service_ver, context + ) def get_objects2(self, params, context=None): """ @@ -1949,8 +1975,9 @@ def get_objects2(self, params, context=None): from an object.), parameter "handle_error" of String, parameter "handle_stacktrace" of String """ - return self._client.call_method('Workspace.get_objects2', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.get_objects2", [params], self._service_ver, context + ) def get_object_subset(self, sub_object_ids, context=None): """ @@ -2263,8 +2290,9 @@ def get_object_subset(self, sub_object_ids, context=None): from an object.), parameter "handle_error" of String, parameter "handle_stacktrace" of String """ - return self._client.call_method('Workspace.get_object_subset', - [sub_object_ids], self._service_ver, context) + return self._client.call_method( + "Workspace.get_object_subset", [sub_object_ids], self._service_ver, context + ) def get_object_history(self, object, context=None): """ @@ -2345,8 +2373,9 @@ def get_object_history(self, object, context=None): metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.get_object_history', - [object], self._service_ver, context) + return self._client.call_method( + "Workspace.get_object_history", [object], self._service_ver, context + ) def list_referencing_objects(self, object_ids, context=None): """ @@ -2427,8 +2456,12 @@ def list_referencing_objects(self, object_ids, context=None): metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.list_referencing_objects', - [object_ids], self._service_ver, context) + return self._client.call_method( + "Workspace.list_referencing_objects", + [object_ids], + self._service_ver, + context, + ) def list_referencing_object_counts(self, object_ids, context=None): """ @@ -2470,26 +2503,30 @@ def list_referencing_object_counts(self, object_ids, context=None): latest version of the object is assumed.) :returns: instance of list of Long """ - return self._client.call_method('Workspace.list_referencing_object_counts', - [object_ids], self._service_ver, context) + return self._client.call_method( + "Workspace.list_referencing_object_counts", + [object_ids], + self._service_ver, + context, + ) def get_referenced_objects(self, ref_chains, context=None): """ DEPRECATED Get objects by references from other objects. NOTE: In the vast majority of cases, this method is not necessary and - get_objects should be used instead. - + get_objects should be used instead. + get_referenced_objects guarantees that a user that has access to an object can always see a) objects that are referenced inside the object and b) objects that are referenced in the object's provenance. This ensures that the user has visibility into the entire provenance of the object and the object's object dependencies (e.g. references). - + The user must have at least read access to the first object in each reference chain, but need not have access to any further objects in the chain, and those objects may be deleted. - + @deprecated Workspace.get_objects2 :param ref_chains: instance of list of type "ref_chain" (A chain of objects with references to one another. An object reference chain @@ -2767,8 +2804,9 @@ def get_referenced_objects(self, ref_chains, context=None): from an object.), parameter "handle_error" of String, parameter "handle_stacktrace" of String """ - return self._client.call_method('Workspace.get_referenced_objects', - [ref_chains], self._service_ver, context) + return self._client.call_method( + "Workspace.get_referenced_objects", [ref_chains], self._service_ver, context + ) def list_workspaces(self, params, context=None): """ @@ -2818,8 +2856,9 @@ def list_workspaces(self, params, context=None): read. 'n' - no permissions.), parameter "num_id" of type "ws_id" (The unique, permanent numerical ID of a workspace.) """ - return self._client.call_method('Workspace.list_workspaces', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.list_workspaces", [params], self._service_ver, context + ) def list_workspace_info(self, params, context=None): """ @@ -2906,8 +2945,9 @@ def list_workspace_info(self, params, context=None): object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.list_workspace_info', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.list_workspace_info", [params], self._service_ver, context + ) def list_workspace_ids(self, params, context=None): """ @@ -2935,8 +2975,9 @@ def list_workspace_ids(self, params, context=None): globally readable.) -> structure: parameter "workspaces" of list of Long, parameter "pub" of list of Long """ - return self._client.call_method('Workspace.list_workspace_ids', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.list_workspace_ids", [params], self._service_ver, context + ) def list_workspace_objects(self, params, context=None): """ @@ -3024,8 +3065,9 @@ def list_workspace_objects(self, params, context=None): String, parameter "objid" of type "obj_id" (The unique, permanent numerical ID of an object.) """ - return self._client.call_method('Workspace.list_workspace_objects', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.list_workspace_objects", [params], self._service_ver, context + ) def list_objects(self, params, context=None): """ @@ -3163,8 +3205,9 @@ def list_objects(self, params, context=None): metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.list_objects', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.list_objects", [params], self._service_ver, context + ) def get_objectmeta(self, params, context=None): """ @@ -3241,8 +3284,9 @@ def get_objectmeta(self, params, context=None): String, parameter "objid" of type "obj_id" (The unique, permanent numerical ID of an object.) """ - return self._client.call_method('Workspace.get_objectmeta', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.get_objectmeta", [params], self._service_ver, context + ) def get_object_info(self, object_ids, includeMetadata, context=None): """ @@ -3329,8 +3373,12 @@ def get_object_info(self, object_ids, includeMetadata, context=None): metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.get_object_info', - [object_ids, includeMetadata], self._service_ver, context) + return self._client.call_method( + "Workspace.get_object_info", + [object_ids, includeMetadata], + self._service_ver, + context, + ) def get_object_info_new(self, params, context=None): """ @@ -3577,8 +3625,9 @@ def get_object_info_new(self, params, context=None): metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.get_object_info_new', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.get_object_info_new", [params], self._service_ver, context + ) def get_object_info3(self, params, context=None): """ @@ -3838,8 +3887,9 @@ def get_object_info3(self, params, context=None): workspace.If the version number is omitted, the latest version of the object is assumed.) """ - return self._client.call_method('Workspace.get_object_info3', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.get_object_info3", [params], self._service_ver, context + ) def rename_workspace(self, params, context=None): """ @@ -3903,8 +3953,9 @@ def rename_workspace(self, params, context=None): object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.rename_workspace', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.rename_workspace", [params], self._service_ver, context + ) def rename_object(self, params, context=None): """ @@ -3991,8 +4042,9 @@ def rename_object(self, params, context=None): metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.rename_object', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.rename_object", [params], self._service_ver, context + ) def copy_object(self, params, context=None): """ @@ -4110,8 +4162,9 @@ def copy_object(self, params, context=None): metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.copy_object', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.copy_object", [params], self._service_ver, context + ) def revert_object(self, object, context=None): """ @@ -4193,8 +4246,9 @@ def revert_object(self, object, context=None): metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ - return self._client.call_method('Workspace.revert_object', - [object], self._service_ver, context) + return self._client.call_method( + "Workspace.revert_object", [object], self._service_ver, context + ) def get_names_by_prefix(self, params, context=None): """ @@ -4229,8 +4283,9 @@ def get_names_by_prefix(self, params, context=None): string consisting of alphanumeric characters and the characters |._- that is not an integer is acceptable.) """ - return self._client.call_method('Workspace.get_names_by_prefix', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.get_names_by_prefix", [params], self._service_ver, context + ) def hide_objects(self, object_ids, context=None): """ @@ -4268,8 +4323,9 @@ def hide_objects(self, object_ids, context=None): in the Towel workspace.If the version number is omitted, the latest version of the object is assumed.) """ - return self._client.call_method('Workspace.hide_objects', - [object_ids], self._service_ver, context) + return self._client.call_method( + "Workspace.hide_objects", [object_ids], self._service_ver, context + ) def unhide_objects(self, object_ids, context=None): """ @@ -4306,8 +4362,9 @@ def unhide_objects(self, object_ids, context=None): in the Towel workspace.If the version number is omitted, the latest version of the object is assumed.) """ - return self._client.call_method('Workspace.unhide_objects', - [object_ids], self._service_ver, context) + return self._client.call_method( + "Workspace.unhide_objects", [object_ids], self._service_ver, context + ) def delete_objects(self, object_ids, context=None): """ @@ -4344,8 +4401,9 @@ def delete_objects(self, object_ids, context=None): in the Towel workspace.If the version number is omitted, the latest version of the object is assumed.) """ - return self._client.call_method('Workspace.delete_objects', - [object_ids], self._service_ver, context) + return self._client.call_method( + "Workspace.delete_objects", [object_ids], self._service_ver, context + ) def undelete_objects(self, object_ids, context=None): """ @@ -4383,8 +4441,9 @@ def undelete_objects(self, object_ids, context=None): in the Towel workspace.If the version number is omitted, the latest version of the object is assumed.) """ - return self._client.call_method('Workspace.undelete_objects', - [object_ids], self._service_ver, context) + return self._client.call_method( + "Workspace.undelete_objects", [object_ids], self._service_ver, context + ) def delete_workspace(self, wsi, context=None): """ @@ -4401,8 +4460,9 @@ def delete_workspace(self, wsi, context=None): parameter "id" of type "ws_id" (The unique, permanent numerical ID of a workspace.) """ - return self._client.call_method('Workspace.delete_workspace', - [wsi], self._service_ver, context) + return self._client.call_method( + "Workspace.delete_workspace", [wsi], self._service_ver, context + ) def request_module_ownership(self, mod, context=None): """ @@ -4411,8 +4471,9 @@ def request_module_ownership(self, mod, context=None): :param mod: instance of type "modulename" (A module name defined in a KIDL typespec.) """ - return self._client.call_method('Workspace.request_module_ownership', - [mod], self._service_ver, context) + return self._client.call_method( + "Workspace.request_module_ownership", [mod], self._service_ver, context + ) def register_typespec(self, params, context=None): """ @@ -4469,8 +4530,9 @@ def register_typespec(self, params, context=None): MyModule.MyType-3.1) to type "jsonschema" (The JSON Schema (v4) representation of a type definition.) """ - return self._client.call_method('Workspace.register_typespec', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.register_typespec", [params], self._service_ver, context + ) def register_typespec_copy(self, params, context=None): """ @@ -4480,7 +4542,7 @@ def register_typespec_copy(self, params, context=None): Also see the release_types function. :param params: instance of type "RegisterTypespecCopyParams" (Parameters for the register_typespec_copy function. Required - arguments: string external_workspace_url - the URL of the + arguments: string external_workspace_url - the URL of the workspace server from which to copy a typespec. modulename mod - the name of the module in the workspace server Optional arguments: spec_version version - the version of the module in the workspace @@ -4491,8 +4553,9 @@ def register_typespec_copy(self, params, context=None): :returns: instance of type "spec_version" (The version of a typespec file.) """ - return self._client.call_method('Workspace.register_typespec_copy', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.register_typespec_copy", [params], self._service_ver, context + ) def release_module(self, mod, context=None): """ @@ -4504,7 +4567,7 @@ def release_module(self, mod, context=None): backwards incompatible changes from minor version to minor version. Once a type is released, backwards incompatible changes always cause a major version increment. - 2) This version of the type becomes the default version, and if a + 2) This version of the type becomes the default version, and if a specific version is not supplied in a function call, this version will be used. This means that newer, unreleased versions of the type may be skipped. @@ -4524,8 +4587,9 @@ def release_module(self, mod, context=None): not provided the most recent version will be used. Example: MyModule.MyType-3.1) """ - return self._client.call_method('Workspace.release_module', - [mod], self._service_ver, context) + return self._client.call_method( + "Workspace.release_module", [mod], self._service_ver, context + ) def list_modules(self, params, context=None): """ @@ -4537,8 +4601,9 @@ def list_modules(self, params, context=None): :returns: instance of list of type "modulename" (A module name defined in a KIDL typespec.) """ - return self._client.call_method('Workspace.list_modules', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.list_modules", [params], self._service_ver, context + ) def list_module_versions(self, params, context=None): """ @@ -4572,8 +4637,9 @@ def list_module_versions(self, params, context=None): version of a typespec file.), parameter "released_vers" of list of type "spec_version" (The version of a typespec file.) """ - return self._client.call_method('Workspace.list_module_versions', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.list_module_versions", [params], self._service_ver, context + ) def get_module_info(self, params, context=None): """ @@ -4636,8 +4702,9 @@ def get_module_info(self, params, context=None): parameter "is_released" of type "boolean" (A boolean. 0 = false, other = true.) """ - return self._client.call_method('Workspace.get_module_info', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.get_module_info", [params], self._service_ver, context + ) def get_jsonschema(self, type, context=None): """ @@ -4658,8 +4725,9 @@ def get_jsonschema(self, type, context=None): :returns: instance of type "jsonschema" (The JSON Schema (v4) representation of a type definition.) """ - return self._client.call_method('Workspace.get_jsonschema', - [type], self._service_ver, context) + return self._client.call_method( + "Workspace.get_jsonschema", [type], self._service_ver, context + ) def translate_from_MD5_types(self, md5_types, context=None): """ @@ -4703,8 +4771,12 @@ def translate_from_MD5_types(self, md5_types, context=None): not provided the most recent version will be used. Example: MyModule.MyType-3.1) """ - return self._client.call_method('Workspace.translate_from_MD5_types', - [md5_types], self._service_ver, context) + return self._client.call_method( + "Workspace.translate_from_MD5_types", + [md5_types], + self._service_ver, + context, + ) def translate_to_MD5_types(self, sem_types, context=None): """ @@ -4748,8 +4820,9 @@ def translate_to_MD5_types(self, sem_types, context=None): not provided the most recent version will be used. Example: MyModule.MyType-3.1) """ - return self._client.call_method('Workspace.translate_to_MD5_types', - [sem_types], self._service_ver, context) + return self._client.call_method( + "Workspace.translate_to_MD5_types", [sem_types], self._service_ver, context + ) def get_type_info(self, type, context=None): """ @@ -4868,8 +4941,9 @@ def get_type_info(self, type, context=None): provided the most recent version will be used. Example: MyModule.MyType-3.1) """ - return self._client.call_method('Workspace.get_type_info', - [type], self._service_ver, context) + return self._client.call_method( + "Workspace.get_type_info", [type], self._service_ver, context + ) def get_all_type_info(self, mod, context=None): """ @@ -4977,8 +5051,9 @@ def get_all_type_info(self, mod, context=None): provided the most recent version will be used. Example: MyModule.MyType-3.1) """ - return self._client.call_method('Workspace.get_all_type_info', - [mod], self._service_ver, context) + return self._client.call_method( + "Workspace.get_all_type_info", [mod], self._service_ver, context + ) def get_func_info(self, func, context=None): """ @@ -5069,8 +5144,9 @@ def get_func_info(self, func, context=None): not provided the most recent version will be used. Example: MyModule.MyType-3.1) """ - return self._client.call_method('Workspace.get_func_info', - [func], self._service_ver, context) + return self._client.call_method( + "Workspace.get_func_info", [func], self._service_ver, context + ) def get_all_func_info(self, mod, context=None): """ @@ -5150,8 +5226,9 @@ def get_all_func_info(self, mod, context=None): not provided the most recent version will be used. Example: MyModule.MyType-3.1) """ - return self._client.call_method('Workspace.get_all_func_info', - [mod], self._service_ver, context) + return self._client.call_method( + "Workspace.get_all_func_info", [mod], self._service_ver, context + ) def grant_module_ownership(self, params, context=None): """ @@ -5168,8 +5245,9 @@ def grant_module_ownership(self, params, context=None): user account.), parameter "with_grant_option" of type "boolean" (A boolean. 0 = false, other = true.) """ - return self._client.call_method('Workspace.grant_module_ownership', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.grant_module_ownership", [params], self._service_ver, context + ) def remove_module_ownership(self, params, context=None): """ @@ -5183,8 +5261,9 @@ def remove_module_ownership(self, params, context=None): defined in a KIDL typespec.), parameter "old_owner" of type "username" (Login name of a KBase user account.) """ - return self._client.call_method('Workspace.remove_module_ownership', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.remove_module_ownership", [params], self._service_ver, context + ) def list_all_types(self, params, context=None): """ @@ -5208,8 +5287,9 @@ def list_all_types(self, params, context=None): version implies that the type has changed in a way that is backwards compatible with previous type definitions.) """ - return self._client.call_method('Workspace.list_all_types', - [params], self._service_ver, context) + return self._client.call_method( + "Workspace.list_all_types", [params], self._service_ver, context + ) def administer(self, command, context=None): """ @@ -5217,9 +5297,11 @@ def administer(self, command, context=None): :param command: instance of unspecified object :returns: instance of unspecified object """ - return self._client.call_method('Workspace.administer', - [command], self._service_ver, context) + return self._client.call_method( + "Workspace.administer", [command], self._service_ver, context + ) def status(self, context=None): - return self._client.call_method('Workspace.status', - [], self._service_ver, context) + return self._client.call_method( + "Workspace.status", [], self._service_ver, context + ) diff --git a/lib/installed_clients/authclient.py b/lib/installed_clients/authclient.py index 844f9b0c..3602d257 100644 --- a/lib/installed_clients/authclient.py +++ b/lib/installed_clients/authclient.py @@ -1,10 +1,10 @@ -''' +""" Created on Aug 1, 2016 A very basic KBase auth client for the Python server. @author: gaprice@lbl.gov -''' +""" import time as _time import requests as _requests import threading as _threading @@ -12,7 +12,7 @@ class TokenCache(object): - ''' A basic cache for tokens. ''' + """A basic cache for tokens.""" _MAX_TIME_SEC = 5 * 60 # 5 min @@ -24,7 +24,7 @@ def __init__(self, maxsize=2000): self._halfmax = maxsize / 2 # int division to round down def get_user(self, token): - token = hashlib.sha256(token.encode('utf-8')).hexdigest() + token = hashlib.sha256(token.encode("utf-8")).hexdigest() with self._lock: usertime = self._cache.get(token) if not usertime: @@ -37,16 +37,15 @@ def get_user(self, token): def add_valid_token(self, token, user): if not token: - raise ValueError('Must supply token') + raise ValueError("Must supply token") if not user: - raise ValueError('Must supply user') - token = hashlib.sha256(token.encode('utf-8')).hexdigest() + raise ValueError("Must supply user") + token = hashlib.sha256(token.encode("utf-8")).hexdigest() with self._lock: self._cache[token] = [user, _time.time()] if len(self._cache) > self._maxsize: sorted_items = sorted( - list(self._cache.items()), - key=(lambda v: v[1][1]) + list(self._cache.items()), key=(lambda v: v[1][1]) ) for i, (t, _) in enumerate(sorted_items): if i <= self._halfmax: @@ -56,16 +55,16 @@ def add_valid_token(self, token, user): class KBaseAuth(object): - ''' + """ A very basic KBase auth client for the Python server. - ''' + """ - _LOGIN_URL = 'https://kbase.us/services/auth/api/legacy/KBase/Sessions/Login' + _LOGIN_URL = "https://kbase.us/services/auth/api/legacy/KBase/Sessions/Login" def __init__(self, auth_url=None): - ''' + """ Constructor - ''' + """ self._authurl = auth_url if not self._authurl: self._authurl = self._LOGIN_URL @@ -73,22 +72,24 @@ def __init__(self, auth_url=None): def get_user(self, token): if not token: - raise ValueError('Must supply token') + raise ValueError("Must supply token") user = self._cache.get_user(token) if user: return user - d = {'token': token, 'fields': 'user_id'} + d = {"token": token, "fields": "user_id"} ret = _requests.post(self._authurl, data=d) if not ret.ok: try: err = ret.json() except Exception as e: ret.raise_for_status() - raise ValueError('Error connecting to auth service: {} {}\n{}' - .format(ret.status_code, ret.reason, - err['error']['message'])) + raise ValueError( + "Error connecting to auth service: {} {}\n{}".format( + ret.status_code, ret.reason, err["error"]["message"] + ) + ) - user = ret.json()['user_id'] + user = ret.json()["user_id"] self._cache.add_valid_token(token, user) return user diff --git a/lib/installed_clients/baseclient.py b/lib/installed_clients/baseclient.py index c6c0ef1c..30e32686 100644 --- a/lib/installed_clients/baseclient.py +++ b/lib/installed_clients/baseclient.py @@ -19,9 +19,9 @@ from urllib.parse import urlparse as _urlparse import time -_CT = 'content-type' -_AJ = 'application/json' -_URL_SCHEME = frozenset(['http', 'https']) +_CT = "content-type" +_AJ = "application/json" +_URL_SCHEME = frozenset(["http", "https"]) _CHECK_JOB_RETRYS = 3 @@ -31,23 +31,32 @@ def _get_token(user_id, password, auth_svc): # note that currently globus usernames, and therefore kbase usernames, # cannot contain non-ascii characters. In python 2, quote doesn't handle # unicode, so if this changes this client will need to change. - body = ('user_id=' + _requests.utils.quote(user_id) + '&password=' + - _requests.utils.quote(password) + '&fields=token') + body = ( + "user_id=" + + _requests.utils.quote(user_id) + + "&password=" + + _requests.utils.quote(password) + + "&fields=token" + ) ret = _requests.post(auth_svc, data=body, allow_redirects=True) status = ret.status_code if status >= 200 and status <= 299: tok = _json.loads(ret.text) elif status == 403: - raise Exception('Authentication failed: Bad user_id/password ' + - 'combination for user %s' % (user_id)) + raise Exception( + "Authentication failed: Bad user_id/password " + + "combination for user %s" % (user_id) + ) else: raise Exception(ret.text) - return tok['token'] + return tok["token"] -def _read_inifile(file=_os.environ.get( # @ReservedAssignment - 'KB_DEPLOYMENT_CONFIG', _os.environ['HOME'] + - '/.kbase_config')): +def _read_inifile( + file=_os.environ.get( # @ReservedAssignment + "KB_DEPLOYMENT_CONFIG", _os.environ["HOME"] + "/.kbase_config" + ) +): # Another bandaid to read in the ~/.kbase_config file if one is present authdata = None if _os.path.exists(file): @@ -55,33 +64,40 @@ def _read_inifile(file=_os.environ.get( # @ReservedAssignment config = _ConfigParser() config.read(file) # strip down whatever we read to only what is legit - authdata = {x: config.get('authentication', x) - if config.has_option('authentication', x) - else None for x in ('user_id', 'token', - 'client_secret', 'keyfile', - 'keyfile_passphrase', 'password')} + authdata = { + x: config.get("authentication", x) + if config.has_option("authentication", x) + else None + for x in ( + "user_id", + "token", + "client_secret", + "keyfile", + "keyfile_passphrase", + "password", + ) + } except Exception as e: - print('Error while reading INI file {}: {}'.format(file, e)) + print("Error while reading INI file {}: {}".format(file, e)) return authdata class ServerError(Exception): - def __init__(self, name, code, message, data=None, error=None): super(Exception, self).__init__(message) self.name = name self.code = code - self.message = '' if message is None else message - self.data = data or error or '' + self.message = "" if message is None else message + self.data = data or error or "" # data = JSON RPC 2.0, error = 1.1 def __str__(self): - return self.name + ': ' + str(self.code) + '. ' + self.message + \ - '\n' + self.data + return ( + self.name + ": " + str(self.code) + ". " + self.message + "\n" + self.data + ) class _JSONObjectEncoder(_json.JSONEncoder): - def default(self, obj): if isinstance(obj, set): return list(obj) @@ -91,7 +107,7 @@ def default(self, obj): class BaseClient(object): - ''' + """ The KBase base client. Required initialization arguments (positional): url - the url of the the service to contact: @@ -113,18 +129,25 @@ class BaseClient(object): lookup_url - set to true when contacting KBase dynamic services. async_job_check_time_ms - the wait time between checking job state for asynchronous jobs run with the run_job method. - ''' + """ + def __init__( - self, url=None, timeout=30 * 60, user_id=None, - password=None, token=None, ignore_authrc=False, - trust_all_ssl_certificates=False, - auth_svc='https://kbase.us/services/auth/api/legacy/KBase/Sessions/Login', - lookup_url=False, - async_job_check_time_ms=100, - async_job_check_time_scale_percent=150, - async_job_check_max_time_ms=300000): + self, + url=None, + timeout=30 * 60, + user_id=None, + password=None, + token=None, + ignore_authrc=False, + trust_all_ssl_certificates=False, + auth_svc="https://kbase.us/services/auth/api/legacy/KBase/Sessions/Login", + lookup_url=False, + async_job_check_time_ms=100, + async_job_check_time_scale_percent=150, + async_job_check_max_time_ms=300000, + ): if url is None: - raise ValueError('A url is required') + raise ValueError("A url is required") scheme, _, _, _, _, _ = _urlparse(url) if scheme not in _URL_SCHEME: raise ValueError(url + " isn't a valid http url") @@ -134,93 +157,99 @@ def __init__( self.trust_all_ssl_certificates = trust_all_ssl_certificates self.lookup_url = lookup_url self.async_job_check_time = async_job_check_time_ms / 1000.0 - self.async_job_check_time_scale_percent = ( - async_job_check_time_scale_percent) + self.async_job_check_time_scale_percent = async_job_check_time_scale_percent self.async_job_check_max_time = async_job_check_max_time_ms / 1000.0 # token overrides user_id and password if token is not None: - self._headers['AUTHORIZATION'] = token + self._headers["AUTHORIZATION"] = token elif user_id is not None and password is not None: - self._headers['AUTHORIZATION'] = _get_token( - user_id, password, auth_svc) - elif 'KB_AUTH_TOKEN' in _os.environ: - self._headers['AUTHORIZATION'] = _os.environ.get('KB_AUTH_TOKEN') + self._headers["AUTHORIZATION"] = _get_token(user_id, password, auth_svc) + elif "KB_AUTH_TOKEN" in _os.environ: + self._headers["AUTHORIZATION"] = _os.environ.get("KB_AUTH_TOKEN") elif not ignore_authrc: authdata = _read_inifile() if authdata is not None: - if authdata.get('token') is not None: - self._headers['AUTHORIZATION'] = authdata['token'] - elif(authdata.get('user_id') is not None and - authdata.get('password') is not None): - self._headers['AUTHORIZATION'] = _get_token( - authdata['user_id'], authdata['password'], auth_svc) + if authdata.get("token") is not None: + self._headers["AUTHORIZATION"] = authdata["token"] + elif ( + authdata.get("user_id") is not None + and authdata.get("password") is not None + ): + self._headers["AUTHORIZATION"] = _get_token( + authdata["user_id"], authdata["password"], auth_svc + ) if self.timeout < 1: - raise ValueError('Timeout value must be at least 1 second') + raise ValueError("Timeout value must be at least 1 second") def _call(self, url, method, params, context=None): - arg_hash = {'method': method, - 'params': params, - 'version': '1.1', - 'id': str(_random.random())[2:] - } + arg_hash = { + "method": method, + "params": params, + "version": "1.1", + "id": str(_random.random())[2:], + } if context: if type(context) is not dict: - raise ValueError('context is not type dict as required.') - arg_hash['context'] = context + raise ValueError("context is not type dict as required.") + arg_hash["context"] = context body = _json.dumps(arg_hash, cls=_JSONObjectEncoder) - ret = _requests.post(url, data=body, headers=self._headers, - timeout=self.timeout, - verify=not self.trust_all_ssl_certificates) - ret.encoding = 'utf-8' + ret = _requests.post( + url, + data=body, + headers=self._headers, + timeout=self.timeout, + verify=not self.trust_all_ssl_certificates, + ) + ret.encoding = "utf-8" if ret.status_code == 500: if ret.headers.get(_CT) == _AJ: err = ret.json() - if 'error' in err: - raise ServerError(**err['error']) + if "error" in err: + raise ServerError(**err["error"]) else: - raise ServerError('Unknown', 0, ret.text) + raise ServerError("Unknown", 0, ret.text) else: - raise ServerError('Unknown', 0, ret.text) + raise ServerError("Unknown", 0, ret.text) if not ret.ok: ret.raise_for_status() resp = ret.json() - if 'result' not in resp: - raise ServerError('Unknown', 0, 'An unknown server error occurred') - if not resp['result']: + if "result" not in resp: + raise ServerError("Unknown", 0, "An unknown server error occurred") + if not resp["result"]: return - if len(resp['result']) == 1: - return resp['result'][0] - return resp['result'] + if len(resp["result"]) == 1: + return resp["result"][0] + return resp["result"] def _get_service_url(self, service_method, service_version): if not self.lookup_url: return self.url - service, _ = service_method.split('.') + service, _ = service_method.split(".") service_status_ret = self._call( - self.url, 'ServiceWizard.get_service_status', - [{'module_name': service, 'version': service_version}]) - return service_status_ret['url'] + self.url, + "ServiceWizard.get_service_status", + [{"module_name": service, "version": service_version}], + ) + return service_status_ret["url"] def _set_up_context(self, service_ver=None, context=None): if service_ver: if not context: context = {} - context['service_ver'] = service_ver + context["service_ver"] = service_ver return context def _check_job(self, service, job_id): - return self._call(self.url, service + '._check_job', [job_id]) + return self._call(self.url, service + "._check_job", [job_id]) - def _submit_job(self, service_method, args, service_ver=None, - context=None): + def _submit_job(self, service_method, args, service_ver=None, context=None): context = self._set_up_context(service_ver, context) - mod, meth = service_method.split('.') - return self._call(self.url, mod + '._' + meth + '_submit', - args, context) + mod, meth = service_method.split(".") + return self._call(self.url, mod + "._" + meth + "_submit", args, context) def run_job(self, service_method, args, service_ver=None, context=None): - ''' + """ Run a SDK method asynchronously. Required arguments: service_method - the service and method to run, e.g. myserv.mymeth. @@ -229,16 +258,16 @@ def run_job(self, service_method, args, service_ver=None, context=None): service_ver - the version of the service to run, e.g. a git hash or dev/beta/release. context - the rpc context dict. - ''' - mod, _ = service_method.split('.') + """ + mod, _ = service_method.split(".") job_id = self._submit_job(service_method, args, service_ver, context) async_job_check_time = self.async_job_check_time check_job_failures = 0 while check_job_failures < _CHECK_JOB_RETRYS: time.sleep(async_job_check_time) - async_job_check_time = (async_job_check_time * - self.async_job_check_time_scale_percent / - 100.0) + async_job_check_time = ( + async_job_check_time * self.async_job_check_time_scale_percent / 100.0 + ) if async_job_check_time > self.async_job_check_max_time: async_job_check_time = self.async_job_check_max_time @@ -249,18 +278,18 @@ def run_job(self, service_method, args, service_ver=None, context=None): check_job_failures += 1 continue - if job_state['finished']: - if not job_state['result']: + if job_state["finished"]: + if not job_state["result"]: return - if len(job_state['result']) == 1: - return job_state['result'][0] - return job_state['result'] - raise RuntimeError("_check_job failed {} times and exceeded limit".format( - check_job_failures)) + if len(job_state["result"]) == 1: + return job_state["result"][0] + return job_state["result"] + raise RuntimeError( + "_check_job failed {} times and exceeded limit".format(check_job_failures) + ) - def call_method(self, service_method, args, service_ver=None, - context=None): - ''' + def call_method(self, service_method, args, service_ver=None, context=None): + """ Call a standard or dynamic service synchronously. Required arguments: service_method - the service and method to run, e.g. myserv.mymeth. @@ -269,7 +298,7 @@ def call_method(self, service_method, args, service_ver=None, service_ver - the version of the service to run, e.g. a git hash or dev/beta/release. context - the rpc context dict. - ''' + """ url = self._get_service_url(service_method, service_ver) context = self._set_up_context(service_ver, context) return self._call(url, service_method, args, context) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..d735cc41 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.black] +target-version = ['py37'] +extend-exclude = ''' +( + authclient\.py +| baseclient\.py +| SampleServiceImpl\.py +| SampleServiceServer\.py +| SampleServiceClient\.py +) +''' \ No newline at end of file diff --git a/scripts/install-sdk.sh b/scripts/install-sdk.sh new file mode 100755 index 00000000..38ba80ba --- /dev/null +++ b/scripts/install-sdk.sh @@ -0,0 +1,10 @@ +echo Installing "kb-sdk"... +echo +docker pull kbase/kb-sdk +docker run kbase/kb-sdk genscript > kb-sdk +chmod u+x kb-sdk +echo +echo To make the installed kb-sdk available, enter the following into your shell: +echo +echo export PATH=\$PWD:\$PATH +echo diff --git a/test/SampleService_test.py b/test/SampleService_test.py index 56493f36..8a0bb169 100644 --- a/test/SampleService_test.py +++ b/test/SampleService_test.py @@ -22,7 +22,10 @@ from SampleService.SampleServiceImpl import SampleService from SampleService.core.errors import ( - MissingParameterError, NoSuchWorkspaceDataError, IllegalParameterError) + MissingParameterError, + NoSuchWorkspaceDataError, + IllegalParameterError, +) from SampleService.core.notification import KafkaNotifier from SampleService.core.user_lookup import KBaseUserLookup, AdminPermission from SampleService.core.user_lookup import InvalidTokenError, InvalidUserError @@ -36,7 +39,7 @@ from core.test_utils import ( assert_ms_epoch_close_to_now, assert_exception_correct, - find_free_port + find_free_port, ) from arango_controller import ArangoController from mongo_controller import MongoController @@ -47,147 +50,167 @@ # TODO should really test a start up for the case where the metadata validation config is not # supplied, but that's almost never going to be the case and the code is trivial, so YAGNI -VER = '0.2.1' - -_AUTH_DB = 'test_auth_db' -_WS_DB = 'test_ws_db' -_WS_TYPE_DB = 'test_ws_type_db' - -TEST_DB_NAME = 'test_sample_service' -TEST_COL_SAMPLE = 'samples' -TEST_COL_VERSION = 'versions' -TEST_COL_VER_EDGE = 'ver_to_sample' -TEST_COL_NODES = 'nodes' -TEST_COL_NODE_EDGE = 'node_edges' -TEST_COL_DATA_LINK = 'data_link' -TEST_COL_WS_OBJ_VER = 'ws_obj_ver_shadow' -TEST_COL_SCHEMA = 'schema' -TEST_USER = 'user1' -TEST_PWD = 'password1' - -USER_WS_READ_ADMIN = 'wsreadadmin' +VER = "0.2.1" + +_AUTH_DB = "test_auth_db" +_WS_DB = "test_ws_db" +_WS_TYPE_DB = "test_ws_type_db" + +TEST_DB_NAME = "test_sample_service" +TEST_COL_SAMPLE = "samples" +TEST_COL_VERSION = "versions" +TEST_COL_VER_EDGE = "ver_to_sample" +TEST_COL_NODES = "nodes" +TEST_COL_NODE_EDGE = "node_edges" +TEST_COL_DATA_LINK = "data_link" +TEST_COL_WS_OBJ_VER = "ws_obj_ver_shadow" +TEST_COL_SCHEMA = "schema" +TEST_USER = "user1" +TEST_PWD = "password1" + +USER_WS_READ_ADMIN = "wsreadadmin" TOKEN_WS_READ_ADMIN = None -USER_WS_FULL_ADMIN = 'wsfulladmin' +USER_WS_FULL_ADMIN = "wsfulladmin" TOKEN_WS_FULL_ADMIN = None -WS_READ_ADMIN = 'WS_READ_ADMIN' -WS_FULL_ADMIN = 'WS_FULL_ADMIN' +WS_READ_ADMIN = "WS_READ_ADMIN" +WS_FULL_ADMIN = "WS_FULL_ADMIN" -USER_SERVICE = 'serviceuser' +USER_SERVICE = "serviceuser" TOKEN_SERVICE = None -USER1 = 'user1' +USER1 = "user1" TOKEN1 = None -USER2 = 'user2' +USER2 = "user2" TOKEN2 = None -USER3 = 'user3' +USER3 = "user3" TOKEN3 = None -USER4 = 'user4' +USER4 = "user4" TOKEN4 = None -USER5 = 'user5' +USER5 = "user5" TOKEN5 = None -USER_NO_TOKEN1 = 'usernt1' -USER_NO_TOKEN2 = 'usernt2' -USER_NO_TOKEN3 = 'usernt3' +USER_NO_TOKEN1 = "usernt1" +USER_NO_TOKEN2 = "usernt2" +USER_NO_TOKEN3 = "usernt3" -KAFKA_TOPIC = 'sampleservice' +KAFKA_TOPIC = "sampleservice" def create_deploy_cfg(auth_port, arango_port, workspace_port, kafka_port): cfg = ConfigParser() - ss = 'SampleService' + ss = "SampleService" cfg.add_section(ss) - cfg[ss]['auth-service-url'] = (f'http://localhost:{auth_port}/testmode/' + - 'api/legacy/KBase/Sessions/Login') - cfg[ss]['auth-service-url-allow-insecure'] = 'true' + cfg[ss]["auth-service-url"] = ( + f"http://localhost:{auth_port}/testmode/" + "api/legacy/KBase/Sessions/Login" + ) + cfg[ss]["auth-service-url-allow-insecure"] = "true" - cfg[ss]['auth-root-url'] = f'http://localhost:{auth_port}/testmode' - cfg[ss]['auth-token'] = TOKEN_SERVICE - cfg[ss]['auth-read-admin-roles'] = 'readadmin1' - cfg[ss]['auth-full-admin-roles'] = 'fulladmin2' + cfg[ss]["auth-root-url"] = f"http://localhost:{auth_port}/testmode" + cfg[ss]["auth-token"] = TOKEN_SERVICE + cfg[ss]["auth-read-admin-roles"] = "readadmin1" + cfg[ss]["auth-full-admin-roles"] = "fulladmin2" - cfg[ss]['arango-url'] = f'http://localhost:{arango_port}' - cfg[ss]['arango-db'] = TEST_DB_NAME - cfg[ss]['arango-user'] = TEST_USER - cfg[ss]['arango-pwd'] = TEST_PWD + cfg[ss]["arango-url"] = f"http://localhost:{arango_port}" + cfg[ss]["arango-db"] = TEST_DB_NAME + cfg[ss]["arango-user"] = TEST_USER + cfg[ss]["arango-pwd"] = TEST_PWD - cfg[ss]['workspace-url'] = f'http://localhost:{workspace_port}' - cfg[ss]['workspace-read-admin-token'] = TOKEN_WS_READ_ADMIN + cfg[ss]["workspace-url"] = f"http://localhost:{workspace_port}" + cfg[ss]["workspace-read-admin-token"] = TOKEN_WS_READ_ADMIN - cfg[ss]['kafka-bootstrap-servers'] = f'localhost:{kafka_port}' - cfg[ss]['kafka-topic'] = KAFKA_TOPIC + cfg[ss]["kafka-bootstrap-servers"] = f"localhost:{kafka_port}" + cfg[ss]["kafka-topic"] = KAFKA_TOPIC - cfg[ss]['sample-collection'] = TEST_COL_SAMPLE - cfg[ss]['version-collection'] = TEST_COL_VERSION - cfg[ss]['version-edge-collection'] = TEST_COL_VER_EDGE - cfg[ss]['node-collection'] = TEST_COL_NODES - cfg[ss]['node-edge-collection'] = TEST_COL_NODE_EDGE - cfg[ss]['data-link-collection'] = TEST_COL_DATA_LINK - cfg[ss]['workspace-object-version-shadow-collection'] = TEST_COL_WS_OBJ_VER - cfg[ss]['schema-collection'] = TEST_COL_SCHEMA + cfg[ss]["sample-collection"] = TEST_COL_SAMPLE + cfg[ss]["version-collection"] = TEST_COL_VERSION + cfg[ss]["version-edge-collection"] = TEST_COL_VER_EDGE + cfg[ss]["node-collection"] = TEST_COL_NODES + cfg[ss]["node-edge-collection"] = TEST_COL_NODE_EDGE + cfg[ss]["data-link-collection"] = TEST_COL_DATA_LINK + cfg[ss]["workspace-object-version-shadow-collection"] = TEST_COL_WS_OBJ_VER + cfg[ss]["schema-collection"] = TEST_COL_SCHEMA metacfg = { - 'validators': { - 'foo': {'validators': [{'module': 'SampleService.core.validator.builtin', - 'callable_builder': 'noop' - }], - 'key_metadata': {'a': 'b', 'c': 'd'} + "validators": { + "foo": { + "validators": [ + { + "module": "SampleService.core.validator.builtin", + "callable_builder": "noop", + } + ], + "key_metadata": {"a": "b", "c": "d"}, + }, + "stringlentest": { + "validators": [ + { + "module": "SampleService.core.validator.builtin", + "callable_builder": "string", + "parameters": {"max-len": 5}, }, - 'stringlentest': {'validators': [{'module': 'SampleService.core.validator.builtin', - 'callable_builder': 'string', - 'parameters': {'max-len': 5} - }, - {'module': 'SampleService.core.validator.builtin', - 'callable_builder': 'string', - 'parameters': {'keys': 'spcky', 'max-len': 2} - }], - 'key_metadata': {'h': 'i', 'j': 'k'} - } - }, - 'prefix_validators': { - 'pre': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'prefix_validator_test_builder', - 'parameters': {'fail_on_arg': 'fail_plz'} - }], - 'key_metadata': {'1': '2'} + { + "module": "SampleService.core.validator.builtin", + "callable_builder": "string", + "parameters": {"keys": "spcky", "max-len": 2}, + }, + ], + "key_metadata": {"h": "i", "j": "k"}, + }, + }, + "prefix_validators": { + "pre": { + "validators": [ + { + "module": "core.config_test_vals", + "callable_builder": "prefix_validator_test_builder", + "parameters": {"fail_on_arg": "fail_plz"}, } - } + ], + "key_metadata": {"1": "2"}, + } + }, } - metaval = tempfile.mkstemp('.cfg', 'metaval-', dir=test_utils.get_temp_dir(), text=True) + metaval = tempfile.mkstemp( + ".cfg", "metaval-", dir=test_utils.get_temp_dir(), text=True + ) os.close(metaval[0]) - with open(metaval[1], 'w') as handle: + with open(metaval[1], "w") as handle: yaml.dump(metacfg, handle) - cfg[ss]['metadata-validator-config-url'] = f'file://{metaval[1]}' + cfg[ss]["metadata-validator-config-url"] = f"file://{metaval[1]}" - deploy = tempfile.mkstemp('.cfg', 'deploy-', dir=test_utils.get_temp_dir(), text=True) + deploy = tempfile.mkstemp( + ".cfg", "deploy-", dir=test_utils.get_temp_dir(), text=True + ) os.close(deploy[0]) - with open(deploy[1], 'w') as handle: + with open(deploy[1], "w") as handle: cfg.write(handle) return deploy[1] -@fixture(scope='module') +@fixture(scope="module") def mongo(): mongoexe = test_utils.get_mongo_exe() tempdir = test_utils.get_temp_dir() wt = test_utils.get_use_wired_tiger() mongo = MongoController(mongoexe, tempdir, wt) - wttext = ' with WiredTiger' if wt else '' - print(f'running mongo {mongo.db_version}{wttext} on port {mongo.port} in dir {mongo.temp_dir}') + wttext = " with WiredTiger" if wt else "" + print( + f"running mongo {mongo.db_version}{wttext} on port {mongo.port} in dir {mongo.temp_dir}" + ) yield mongo del_temp = test_utils.get_delete_temp_files() - print(f'shutting down mongo, delete_temp_files={del_temp}') + print(f"shutting down mongo, delete_temp_files={del_temp}") mongo.destroy(del_temp) -@fixture(scope='module') +@fixture(scope="module") def auth(mongo): global TOKEN_SERVICE global TOKEN_WS_FULL_ADMIN @@ -199,60 +222,62 @@ def auth(mongo): global TOKEN5 jd = test_utils.get_jars_dir() tempdir = test_utils.get_temp_dir() - auth = AuthController(jd, f'localhost:{mongo.port}', _AUTH_DB, tempdir) - print(f'Started KBase Auth2 {auth.version} on port {auth.port} ' + - f'in dir {auth.temp_dir} in {auth.startup_count}s') - url = f'http://localhost:{auth.port}' - - test_utils.create_auth_role(url, 'fulladmin1', 'fa1') - test_utils.create_auth_role(url, 'fulladmin2', 'fa2') - test_utils.create_auth_role(url, 'readadmin1', 'ra1') - test_utils.create_auth_role(url, 'readadmin2', 'ra2') - test_utils.create_auth_role(url, WS_READ_ADMIN, 'wsr') - test_utils.create_auth_role(url, WS_FULL_ADMIN, 'wsf') - - test_utils.create_auth_user(url, USER_SERVICE, 'serv') + auth = AuthController(jd, f"localhost:{mongo.port}", _AUTH_DB, tempdir) + print( + f"Started KBase Auth2 {auth.version} on port {auth.port} " + + f"in dir {auth.temp_dir} in {auth.startup_count}s" + ) + url = f"http://localhost:{auth.port}" + + test_utils.create_auth_role(url, "fulladmin1", "fa1") + test_utils.create_auth_role(url, "fulladmin2", "fa2") + test_utils.create_auth_role(url, "readadmin1", "ra1") + test_utils.create_auth_role(url, "readadmin2", "ra2") + test_utils.create_auth_role(url, WS_READ_ADMIN, "wsr") + test_utils.create_auth_role(url, WS_FULL_ADMIN, "wsf") + + test_utils.create_auth_user(url, USER_SERVICE, "serv") TOKEN_SERVICE = test_utils.create_auth_login_token(url, USER_SERVICE) - test_utils.create_auth_user(url, USER_WS_READ_ADMIN, 'wsra') + test_utils.create_auth_user(url, USER_WS_READ_ADMIN, "wsra") TOKEN_WS_READ_ADMIN = test_utils.create_auth_login_token(url, USER_WS_READ_ADMIN) test_utils.set_custom_roles(url, USER_WS_READ_ADMIN, [WS_READ_ADMIN]) - test_utils.create_auth_user(url, USER_WS_FULL_ADMIN, 'wsrf') + test_utils.create_auth_user(url, USER_WS_FULL_ADMIN, "wsrf") TOKEN_WS_FULL_ADMIN = test_utils.create_auth_login_token(url, USER_WS_FULL_ADMIN) test_utils.set_custom_roles(url, USER_WS_FULL_ADMIN, [WS_FULL_ADMIN]) - test_utils.create_auth_user(url, USER1, 'display1') + test_utils.create_auth_user(url, USER1, "display1") TOKEN1 = test_utils.create_auth_login_token(url, USER1) - test_utils.set_custom_roles(url, USER1, ['fulladmin1']) + test_utils.set_custom_roles(url, USER1, ["fulladmin1"]) - test_utils.create_auth_user(url, USER2, 'display2') + test_utils.create_auth_user(url, USER2, "display2") TOKEN2 = test_utils.create_auth_login_token(url, USER2) - test_utils.set_custom_roles(url, USER2, ['fulladmin1', 'fulladmin2', 'readadmin2']) + test_utils.set_custom_roles(url, USER2, ["fulladmin1", "fulladmin2", "readadmin2"]) - test_utils.create_auth_user(url, USER3, 'display3') + test_utils.create_auth_user(url, USER3, "display3") TOKEN3 = test_utils.create_auth_login_token(url, USER3) - test_utils.set_custom_roles(url, USER3, ['readadmin1']) + test_utils.set_custom_roles(url, USER3, ["readadmin1"]) - test_utils.create_auth_user(url, USER4, 'display4') + test_utils.create_auth_user(url, USER4, "display4") TOKEN4 = test_utils.create_auth_login_token(url, USER4) - test_utils.create_auth_user(url, USER5, 'display5') + test_utils.create_auth_user(url, USER5, "display5") TOKEN5 = test_utils.create_auth_login_token(url, USER5) - test_utils.set_custom_roles(url, USER5, ['fulladmin2']) + test_utils.set_custom_roles(url, USER5, ["fulladmin2"]) - test_utils.create_auth_user(url, USER_NO_TOKEN1, 'displaynt1') - test_utils.create_auth_user(url, USER_NO_TOKEN2, 'displaynt2') - test_utils.create_auth_user(url, USER_NO_TOKEN3, 'displaynt3') + test_utils.create_auth_user(url, USER_NO_TOKEN1, "displaynt1") + test_utils.create_auth_user(url, USER_NO_TOKEN2, "displaynt2") + test_utils.create_auth_user(url, USER_NO_TOKEN3, "displaynt3") yield auth del_temp = test_utils.get_delete_temp_files() - print(f'shutting down auth, delete_temp_files={del_temp}') + print(f"shutting down auth, delete_temp_files={del_temp}") auth.destroy(del_temp) -@fixture(scope='module') +@fixture(scope="module") def workspace(auth, mongo): jd = test_utils.get_jars_dir() tempdir = test_utils.get_temp_dir() @@ -261,16 +286,20 @@ def workspace(auth, mongo): mongo, _WS_DB, _WS_TYPE_DB, - f'http://localhost:{auth.port}/testmode', - tempdir) - print(f'Started KBase Workspace {ws.version} on port {ws.port} ' + - f'in dir {ws.temp_dir} in {ws.startup_count}s') - - wsc = Workspace(f'http://localhost:{ws.port}', token=TOKEN_WS_FULL_ADMIN) - wsc.request_module_ownership('Trivial') - wsc.administer({'command': 'approveModRequest', 'module': 'Trivial'}) - wsc.register_typespec({ - 'spec': ''' + f"http://localhost:{auth.port}/testmode", + tempdir, + ) + print( + f"Started KBase Workspace {ws.version} on port {ws.port} " + + f"in dir {ws.temp_dir} in {ws.startup_count}s" + ) + + wsc = Workspace(f"http://localhost:{ws.port}", token=TOKEN_WS_FULL_ADMIN) + wsc.request_module_ownership("Trivial") + wsc.administer({"command": "approveModRequest", "module": "Trivial"}) + wsc.register_typespec( + { + "spec": """ module Trivial { /* @optional dontusethisfieldorifyoudomakesureitsastring */ @@ -283,37 +312,40 @@ def workspace(auth, mongo): string dontusethisfieldorifyoudomakesureitsastring; } Object2; }; - ''', - 'dryrun': 0, - 'new_types': ['Object', 'Object2'] - }) - wsc.release_module('Trivial') + """, + "dryrun": 0, + "new_types": ["Object", "Object2"], + } + ) + wsc.release_module("Trivial") yield ws del_temp = test_utils.get_delete_temp_files() - print(f'shutting down workspace, delete_temp_files={del_temp}') + print(f"shutting down workspace, delete_temp_files={del_temp}") ws.destroy(del_temp, False) -@fixture(scope='module') +@fixture(scope="module") def arango(): arangoexe = test_utils.get_arango_exe() arangojs = test_utils.get_arango_js() tempdir = test_utils.get_temp_dir() arango = ArangoController(arangoexe, arangojs, tempdir) create_test_db(arango) - print('running arango on port {} in dir {}'.format(arango.port, arango.temp_dir)) + print("running arango on port {} in dir {}".format(arango.port, arango.temp_dir)) yield arango del_temp = test_utils.get_delete_temp_files() - print('shutting down arango, delete_temp_files={}'.format(del_temp)) + print("shutting down arango, delete_temp_files={}".format(del_temp)) arango.destroy(del_temp) def create_test_db(arango): systemdb = arango.client.db(verify=True) # default access to _system db - systemdb.create_database(TEST_DB_NAME, [{'username': TEST_USER, 'password': TEST_PWD}]) + systemdb.create_database( + TEST_DB_NAME, [{"username": TEST_USER, "password": TEST_PWD}] + ) return arango.client.db(TEST_DB_NAME, TEST_USER, TEST_PWD) @@ -331,32 +363,35 @@ def clear_db_and_recreate(arango): return db -@fixture(scope='module') +@fixture(scope="module") def kafka(): kafka_bin_dir = test_utils.get_kafka_bin_dir() tempdir = test_utils.get_temp_dir() kc = KafkaController(kafka_bin_dir, tempdir) - print('running kafka on port {} in dir {}'.format(kc.port, kc.temp_dir)) + print("running kafka on port {} in dir {}".format(kc.port, kc.temp_dir)) yield kc del_temp = test_utils.get_delete_temp_files() - print('shutting down kafka, delete_temp_files={}'.format(del_temp)) + print("shutting down kafka, delete_temp_files={}".format(del_temp)) kc.destroy(del_temp, dump_logs_to_stdout=False) -@fixture(scope='module') +@fixture(scope="module") def service(auth, arango, workspace, kafka): portint = test_utils.find_free_port() clear_db_and_recreate(arango) # this is completely stupid. The state is calculated on import so there's no way to # test the state creation normally. cfgpath = create_deploy_cfg(auth.port, arango.port, workspace.port, kafka.port) - os.environ['KB_DEPLOYMENT_CONFIG'] = cfgpath + os.environ["KB_DEPLOYMENT_CONFIG"] = cfgpath from SampleService import SampleServiceServer - Thread(target=SampleServiceServer.start_server, kwargs={'port': portint}, daemon=True).start() + + Thread( + target=SampleServiceServer.start_server, kwargs={"port": portint}, daemon=True + ).start() time.sleep(0.05) port = str(portint) - print('running sample service at localhost:' + port) + print("running sample service at localhost:" + port) yield port # shutdown the server @@ -375,50 +410,58 @@ def sample_port(service, arango, workspace, kafka): def test_init_fail(): # init success is tested via starting the server - init_fail(None, ValueError('config is empty, cannot start service')) + init_fail(None, ValueError("config is empty, cannot start service")) cfg = {} - init_fail(cfg, ValueError('config is empty, cannot start service')) - cfg['arango-url'] = None - init_fail(cfg, MissingParameterError('config param arango-url')) - cfg['arango-url'] = 'crap' - init_fail(cfg, MissingParameterError('config param arango-db')) - cfg['arango-db'] = 'crap' - init_fail(cfg, MissingParameterError('config param arango-user')) - cfg['arango-user'] = 'crap' - init_fail(cfg, MissingParameterError('config param arango-pwd')) - cfg['arango-pwd'] = 'crap' - init_fail(cfg, MissingParameterError('config param sample-collection')) - cfg['sample-collection'] = 'crap' - init_fail(cfg, MissingParameterError('config param version-collection')) - cfg['version-collection'] = 'crap' - init_fail(cfg, MissingParameterError('config param version-edge-collection')) - cfg['version-edge-collection'] = 'crap' - init_fail(cfg, MissingParameterError('config param node-collection')) - cfg['node-collection'] = 'crap' - init_fail(cfg, MissingParameterError('config param node-edge-collection')) - cfg['node-edge-collection'] = 'crap' - init_fail(cfg, MissingParameterError('config param data-link-collection')) - cfg['data-link-collection'] = 'crap' - init_fail(cfg, MissingParameterError( - 'config param workspace-object-version-shadow-collection')) - cfg['workspace-object-version-shadow-collection'] = 'crap' - init_fail(cfg, MissingParameterError('config param schema-collection')) - cfg['schema-collection'] = 'crap' - init_fail(cfg, MissingParameterError('config param auth-root-url')) - cfg['auth-root-url'] = 'crap' - init_fail(cfg, MissingParameterError('config param auth-token')) - cfg['auth-token'] = 'crap' - init_fail(cfg, MissingParameterError('config param workspace-url')) - cfg['workspace-url'] = 'crap' - init_fail(cfg, MissingParameterError('config param workspace-read-admin-token')) - cfg['workspace-read-admin-token'] = 'crap' - cfg['kafka-bootstrap-servers'] = 'crap' - init_fail(cfg, MissingParameterError('config param kafka-topic')) - cfg['kafka-topic'] = 'crap' + init_fail(cfg, ValueError("config is empty, cannot start service")) + cfg["arango-url"] = None + init_fail(cfg, MissingParameterError("config param arango-url")) + cfg["arango-url"] = "crap" + init_fail(cfg, MissingParameterError("config param arango-db")) + cfg["arango-db"] = "crap" + init_fail(cfg, MissingParameterError("config param arango-user")) + cfg["arango-user"] = "crap" + init_fail(cfg, MissingParameterError("config param arango-pwd")) + cfg["arango-pwd"] = "crap" + init_fail(cfg, MissingParameterError("config param sample-collection")) + cfg["sample-collection"] = "crap" + init_fail(cfg, MissingParameterError("config param version-collection")) + cfg["version-collection"] = "crap" + init_fail(cfg, MissingParameterError("config param version-edge-collection")) + cfg["version-edge-collection"] = "crap" + init_fail(cfg, MissingParameterError("config param node-collection")) + cfg["node-collection"] = "crap" + init_fail(cfg, MissingParameterError("config param node-edge-collection")) + cfg["node-edge-collection"] = "crap" + init_fail(cfg, MissingParameterError("config param data-link-collection")) + cfg["data-link-collection"] = "crap" + init_fail( + cfg, + MissingParameterError( + "config param workspace-object-version-shadow-collection" + ), + ) + cfg["workspace-object-version-shadow-collection"] = "crap" + init_fail(cfg, MissingParameterError("config param schema-collection")) + cfg["schema-collection"] = "crap" + init_fail(cfg, MissingParameterError("config param auth-root-url")) + cfg["auth-root-url"] = "crap" + init_fail(cfg, MissingParameterError("config param auth-token")) + cfg["auth-token"] = "crap" + init_fail(cfg, MissingParameterError("config param workspace-url")) + cfg["workspace-url"] = "crap" + init_fail(cfg, MissingParameterError("config param workspace-read-admin-token")) + cfg["workspace-read-admin-token"] = "crap" + cfg["kafka-bootstrap-servers"] = "crap" + init_fail(cfg, MissingParameterError("config param kafka-topic")) + cfg["kafka-topic"] = "crap" # get_validators is tested elsewhere, just make sure it'll error out - cfg['metadata-validator-config-url'] = 'https://kbase.us/services' - init_fail(cfg, ValueError( - 'Failed to open validator configuration file at https://kbase.us/services: Not Found')) + cfg["metadata-validator-config-url"] = "https://kbase.us/services" + init_fail( + cfg, + ValueError( + "Failed to open validator configuration file at https://kbase.us/services: Not Found" + ), + ) def init_fail(config, expected): @@ -428,36 +471,40 @@ def init_fail(config, expected): def test_status(sample_port): - res = requests.post('http://localhost:' + sample_port, json={ - 'method': 'SampleService.status', - 'params': [], - 'version': 1.1, - 'id': 1 # don't do this. This is bad practice - }) + res = requests.post( + "http://localhost:" + sample_port, + json={ + "method": "SampleService.status", + "params": [], + "version": 1.1, + "id": 1, # don't do this. This is bad practice + }, + ) assert res.status_code == 200 s = res.json() # print(s) - assert len(s['result']) == 1 # results are always in a list - assert_ms_epoch_close_to_now(s['result'][0]['servertime']) - assert s['result'][0]['state'] == 'OK' - assert s['result'][0]['message'] == "" - assert s['result'][0]['version'] == VER + assert len(s["result"]) == 1 # results are always in a list + assert_ms_epoch_close_to_now(s["result"][0]["servertime"]) + assert s["result"][0]["state"] == "OK" + assert s["result"][0]["message"] == "" + assert s["result"][0]["version"] == VER # ignore git url and hash, can change def get_authorized_headers(token): - headers = {'accept': 'application/json'} + headers = {"accept": "application/json"} if token is not None: - headers['authorization'] = token + headers["authorization"] = token return headers def _check_kafka_messages(kafka, expected_msgs, topic=KAFKA_TOPIC, print_res=False): kc = KafkaConsumer( topic, - bootstrap_servers=f'localhost:{kafka.port}', - auto_offset_reset='earliest', - group_id='foo') # quiets warnings + bootstrap_servers=f"localhost:{kafka.port}", + auto_offset_reset="earliest", + group_id="foo", + ) # quiets warnings try: res = kc.poll(timeout_ms=2000) # 1s not enough? Seems like a lot @@ -477,9 +524,10 @@ def _check_kafka_messages(kafka, expected_msgs, topic=KAFKA_TOPIC, print_res=Fal def _clear_kafka_messages(kafka, topic=KAFKA_TOPIC): kc = KafkaConsumer( topic, - bootstrap_servers=f'localhost:{kafka.port}', - auto_offset_reset='earliest', - group_id='foo') # quiets warnings + bootstrap_servers=f"localhost:{kafka.port}", + auto_offset_reset="earliest", + group_id="foo", + ) # quiets warnings try: kc.poll(timeout_ms=2000) # 1s not enough? Seems like a lot @@ -490,245 +538,328 @@ def _clear_kafka_messages(kafka, topic=KAFKA_TOPIC): def test_create_and_get_sample_with_version(sample_port, kafka): _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" # version 1 - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'}, - 'stringlentest': {'foooo': 'barrr', - 'spcky': 'fa'}, - 'prefixed': {'safe': 'args'} - }, - 'meta_user': {'a': {'b': 'c'}}, - 'source_meta': [ - {'key': 'foo', 'skey': 'bar', 'svalue': {'whee': 'whoo'}}, - {'key': 'stringlentest', - 'skey': 'ya fer sure', - 'svalue': {'just': 'some', 'data': 42}} - ] - } - ] - } - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "meta_controlled": { + "foo": {"bar": "baz"}, + "stringlentest": {"foooo": "barrr", "spcky": "fa"}, + "prefixed": {"safe": "args"}, + }, + "meta_user": {"a": {"b": "c"}}, + "source_meta": [ + { + "key": "foo", + "skey": "bar", + "svalue": {"whee": "whoo"}, + }, + { + "key": "stringlentest", + "skey": "ya fer sure", + "svalue": {"just": "some", "data": 42}, + }, + ], + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] # version 2 - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '68', - 'params': [{ - 'sample': {'name': 'mysample2', - 'id': id_, - 'node_tree': [{'id': 'root2', - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'bat'}}, - 'meta_user': {'a': {'b': 'd'}} - } - ] - }, - 'prior_version': 1 - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "68", + "params": [ + { + "sample": { + "name": "mysample2", + "id": id_, + "node_tree": [ + { + "id": "root2", + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "bat"}}, + "meta_user": {"a": {"b": "d"}}, + } + ], + }, + "prior_version": 1, + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 2 + assert ret.json()["result"][0]["version"] == 2 # get version 1 - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - j = ret.json()['result'][0] - assert_ms_epoch_close_to_now(j['save_date']) - del j['save_date'] + j = ret.json()["result"][0] + assert_ms_epoch_close_to_now(j["save_date"]) + del j["save_date"] assert j == { - 'id': id_, - 'version': 1, - 'user': USER1, - 'name': 'mysample', - 'node_tree': [{'id': 'root', - 'parent': None, - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'}, - 'stringlentest': {'foooo': 'barrr', - 'spcky': 'fa'}, - 'prefixed': {'safe': 'args'} - }, - 'meta_user': {'a': {'b': 'c'}}, - 'source_meta': [ - {'key': 'foo', 'skey': 'bar', 'svalue': {'whee': 'whoo'}}, - {'key': 'stringlentest', - 'skey': 'ya fer sure', - 'svalue': {'just': 'some', 'data': 42}} - ], - }] + "id": id_, + "version": 1, + "user": USER1, + "name": "mysample", + "node_tree": [ + { + "id": "root", + "parent": None, + "type": "BioReplicate", + "meta_controlled": { + "foo": {"bar": "baz"}, + "stringlentest": {"foooo": "barrr", "spcky": "fa"}, + "prefixed": {"safe": "args"}, + }, + "meta_user": {"a": {"b": "c"}}, + "source_meta": [ + {"key": "foo", "skey": "bar", "svalue": {"whee": "whoo"}}, + { + "key": "stringlentest", + "skey": "ya fer sure", + "svalue": {"just": "some", "data": 42}, + }, + ], + } + ], } # get version 2 - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '43', - 'params': [{'id': id_}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "43", + "params": [{"id": id_}], + }, + ) # print(ret.text) assert ret.ok is True - j = ret.json()['result'][0] - assert_ms_epoch_close_to_now(j['save_date']) - del j['save_date'] + j = ret.json()["result"][0] + assert_ms_epoch_close_to_now(j["save_date"]) + del j["save_date"] assert j == { - 'id': id_, - 'version': 2, - 'user': USER1, - 'name': 'mysample2', - 'node_tree': [{'id': 'root2', - 'parent': None, - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'bat'}}, - 'meta_user': {'a': {'b': 'd'}}, - 'source_meta': [], - }] + "id": id_, + "version": 2, + "user": USER1, + "name": "mysample2", + "node_tree": [ + { + "id": "root2", + "parent": None, + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "bat"}}, + "meta_user": {"a": {"b": "d"}}, + "source_meta": [], + } + ], } _check_kafka_messages( kafka, [ - {'event_type': 'NEW_SAMPLE', 'sample_id': id_, 'sample_ver': 1}, - {'event_type': 'NEW_SAMPLE', 'sample_id': id_, 'sample_ver': 2} - ]) + {"event_type": "NEW_SAMPLE", "sample_id": id_, "sample_ver": 1}, + {"event_type": "NEW_SAMPLE", "sample_id": id_, "sample_ver": 2}, + ], + ) def test_create_and_get_samples(sample_port, kafka): _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" # first sample - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'}, - 'stringlentest': {'foooo': 'barrr', - 'spcky': 'fa'}, - 'prefixed': {'safe': 'args'} - }, - 'meta_user': {'a': {'b': 'c'}}, - 'source_meta': [ - {'key': 'foo', 'skey': 'bar', 'svalue': {'whee': 'whoo'}}, - {'key': 'stringlentest', - 'skey': 'ya fer sure', - 'svalue': {'just': 'some', 'data': 42}} - ] - } - ] - } - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "meta_controlled": { + "foo": {"bar": "baz"}, + "stringlentest": {"foooo": "barrr", "spcky": "fa"}, + "prefixed": {"safe": "args"}, + }, + "meta_user": {"a": {"b": "c"}}, + "source_meta": [ + { + "key": "foo", + "skey": "bar", + "svalue": {"whee": "whoo"}, + }, + { + "key": "stringlentest", + "skey": "ya fer sure", + "svalue": {"just": "some", "data": 42}, + }, + ], + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id1_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id1_ = ret.json()["result"][0]["id"] # second sample - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '68', - 'params': [{ - 'sample': {'name': 'mysample2', - 'node_tree': [{'id': 'root2', - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'bat'}}, - 'meta_user': {'a': {'b': 'd'}} - } - ] - } - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "68", + "params": [ + { + "sample": { + "name": "mysample2", + "node_tree": [ + { + "id": "root2", + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "bat"}}, + "meta_user": {"a": {"b": "d"}}, + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id2_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id2_ = ret.json()["result"][0]["id"] # get both samples - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.get_samples', - 'version': '1.1', - 'id': '42', - 'params': [{'samples': [{'id': id1_, 'version': 1}, {'id': id2_, 'version': 1}]}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.get_samples", + "version": "1.1", + "id": "42", + "params": [ + {"samples": [{"id": id1_, "version": 1}, {"id": id2_, "version": 1}]} + ], + }, + ) # print(ret.text) assert ret.ok is True - j = ret.json()['result'][0] + j = ret.json()["result"][0] for s in j: - assert_ms_epoch_close_to_now(s['save_date']) - del s['save_date'] - print('-'*80) + assert_ms_epoch_close_to_now(s["save_date"]) + del s["save_date"] + print("-" * 80) import json + print(json.dumps(j)) - print('-'*80) - - assert j == [{ - 'id': id1_, - 'version': 1, - 'user': USER1, - 'name': 'mysample', - 'node_tree': [{ - 'id': 'root', - 'parent': None, - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'}, - 'stringlentest': {'foooo': 'barrr', - 'spcky': 'fa'}, - 'prefixed': {'safe': 'args'} - }, - 'meta_user': {'a': {'b': 'c'}}, - 'source_meta': [ - {'key': 'foo', 'skey': 'bar', 'svalue': {'whee': 'whoo'}}, - {'key': 'stringlentest', - 'skey': 'ya fer sure', - 'svalue': {'just': 'some', 'data': 42}} - ], - }] - }, { - 'id': id2_, - 'version': 1, - 'user': USER1, - 'name': 'mysample2', - 'node_tree': [{'id': 'root2', - 'parent': None, - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'bat'}}, - 'meta_user': {'a': {'b': 'd'}}, - 'source_meta': [] - }] - }] + print("-" * 80) + + assert j == [ + { + "id": id1_, + "version": 1, + "user": USER1, + "name": "mysample", + "node_tree": [ + { + "id": "root", + "parent": None, + "type": "BioReplicate", + "meta_controlled": { + "foo": {"bar": "baz"}, + "stringlentest": {"foooo": "barrr", "spcky": "fa"}, + "prefixed": {"safe": "args"}, + }, + "meta_user": {"a": {"b": "c"}}, + "source_meta": [ + {"key": "foo", "skey": "bar", "svalue": {"whee": "whoo"}}, + { + "key": "stringlentest", + "skey": "ya fer sure", + "svalue": {"just": "some", "data": 42}, + }, + ], + } + ], + }, + { + "id": id2_, + "version": 1, + "user": USER1, + "name": "mysample2", + "node_tree": [ + { + "id": "root2", + "parent": None, + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "bat"}}, + "meta_user": {"a": {"b": "d"}}, + "source_meta": [], + } + ], + }, + ] _check_kafka_messages( kafka, [ - {'event_type': 'NEW_SAMPLE', 'sample_id': id1_, 'sample_ver': 1}, - {'event_type': 'NEW_SAMPLE', 'sample_id': id2_, 'sample_ver': 1} - ]) + {"event_type": "NEW_SAMPLE", "sample_id": id1_, "sample_ver": 1}, + {"event_type": "NEW_SAMPLE", "sample_id": id2_, "sample_ver": 1}, + ], + ) def test_create_sample_as_admin(sample_port): @@ -736,61 +867,75 @@ def test_create_sample_as_admin(sample_port): def test_create_sample_as_admin_impersonate_user(sample_port): - _create_sample_as_admin(sample_port, ' ' + USER4 + ' ', TOKEN4, USER4) + _create_sample_as_admin(sample_port, " " + USER4 + " ", TOKEN4, USER4) def _create_sample_as_admin(sample_port, as_user, get_token, expected_user): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" # verison 1 - ret = requests.post(url, headers=get_authorized_headers(TOKEN2), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'} - }, - 'meta_user': {'a': {'b': 'c'}} - } - ] - }, - 'as_admin': 1, - 'as_user': as_user - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN2), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "baz"}}, + "meta_user": {"a": {"b": "c"}}, + } + ], + }, + "as_admin": 1, + "as_user": as_user, + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] # get - ret = requests.post(url, headers=get_authorized_headers(get_token), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(get_token), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - j = ret.json()['result'][0] - assert_ms_epoch_close_to_now(j['save_date']) - del j['save_date'] + j = ret.json()["result"][0] + assert_ms_epoch_close_to_now(j["save_date"]) + del j["save_date"] assert j == { - 'id': id_, - 'version': 1, - 'user': expected_user, - 'name': 'mysample', - 'node_tree': [{'id': 'root', - 'parent': None, - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'} - }, - 'meta_user': {'a': {'b': 'c'}}, - 'source_meta': [], - }] + "id": id_, + "version": 1, + "user": expected_user, + "name": "mysample", + "node_tree": [ + { + "id": "root", + "parent": None, + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "baz"}}, + "meta_user": {"a": {"b": "c"}}, + "source_meta": [], + } + ], } @@ -803,961 +948,1267 @@ def test_create_sample_version_as_admin_impersonate_user(sample_port): def _create_sample_version_as_admin(sample_port, as_user, expected_user): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" # verison 1 - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'}, - 'stringlentest': {'foooo': 'barrr', - 'spcky': 'fa'}, - 'prefixed': {'safe': 'args'} - }, - 'meta_user': {'a': {'b': 'c'}} - } - ] - } - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "meta_controlled": { + "foo": {"bar": "baz"}, + "stringlentest": {"foooo": "barrr", "spcky": "fa"}, + "prefixed": {"safe": "args"}, + }, + "meta_user": {"a": {"b": "c"}}, + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] # version 2 - ret = requests.post(url, headers=get_authorized_headers(TOKEN2), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '68', - 'params': [{ - 'sample': {'name': 'mysample2', - 'id': id_, - 'node_tree': [{'id': 'root2', - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'bat'}}, - 'meta_user': {'a': {'b': 'd'}} - } - ] - }, - 'as_admin': 1, - 'as_user': as_user - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN2), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "68", + "params": [ + { + "sample": { + "name": "mysample2", + "id": id_, + "node_tree": [ + { + "id": "root2", + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "bat"}}, + "meta_user": {"a": {"b": "d"}}, + } + ], + }, + "as_admin": 1, + "as_user": as_user, + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 2 + assert ret.json()["result"][0]["version"] == 2 # get version 2 - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '43', - 'params': [{'id': id_}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "43", + "params": [{"id": id_}], + }, + ) # print(ret.text) assert ret.ok is True - j = ret.json()['result'][0] - assert_ms_epoch_close_to_now(j['save_date']) - del j['save_date'] + j = ret.json()["result"][0] + assert_ms_epoch_close_to_now(j["save_date"]) + del j["save_date"] assert j == { - 'id': id_, - 'version': 2, - 'user': expected_user, - 'name': 'mysample2', - 'node_tree': [{'id': 'root2', - 'parent': None, - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'bat'}}, - 'meta_user': {'a': {'b': 'd'}}, - 'source_meta': [], - }] + "id": id_, + "version": 2, + "user": expected_user, + "name": "mysample2", + "node_tree": [ + { + "id": "root2", + "parent": None, + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "bat"}}, + "meta_user": {"a": {"b": "d"}}, + "source_meta": [], + } + ], } def test_get_sample_public_read(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - _replace_acls(url, id_, TOKEN1, {'public_read': 1}) + _replace_acls(url, id_, TOKEN1, {"public_read": 1}) for token in [TOKEN4, None]: # unauthed user and anonymous user s = _get_sample(url, token, id_) - assert_ms_epoch_close_to_now(s['save_date']) - del s['save_date'] + assert_ms_epoch_close_to_now(s["save_date"]) + del s["save_date"] assert s == { - 'id': id_, - 'version': 1, - 'user': 'user1', - 'name': 'mysample', - 'node_tree': [{'id': 'root', - 'parent': None, - 'type': 'BioReplicate', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - }, - {'id': 'foo', - 'parent': 'root', - 'type': 'TechReplicate', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - } - ] + "id": id_, + "version": 1, + "user": "user1", + "name": "mysample", + "node_tree": [ + { + "id": "root", + "parent": None, + "type": "BioReplicate", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + }, + { + "id": "foo", + "parent": "root", + "type": "TechReplicate", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + }, + ], } def _get_sample(url, token, id_): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '43', - 'params': [{'id': str(id_)}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "43", + "params": [{"id": str(id_)}], + }, + ) # print(ret.text) assert ret.ok is True - return ret.json()['result'][0] + return ret.json()["result"][0] def test_get_sample_as_admin(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" # verison 1 - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'} - }, - 'meta_user': {'a': {'b': 'c'}} - } - ] - } - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "baz"}}, + "meta_user": {"a": {"b": "c"}}, + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] # token3 has read admin but not full admin - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'version': 1, 'as_admin': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "version": 1, "as_admin": 1}], + }, + ) print(ret.text) assert ret.ok is True - j = ret.json()['result'][0] - assert_ms_epoch_close_to_now(j['save_date']) - del j['save_date'] + j = ret.json()["result"][0] + assert_ms_epoch_close_to_now(j["save_date"]) + del j["save_date"] assert j == { - 'id': id_, - 'version': 1, - 'user': USER1, - 'name': 'mysample', - 'node_tree': [{'id': 'root', - 'parent': None, - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'}, - }, - 'meta_user': {'a': {'b': 'c'}}, - 'source_meta': [], - }] + "id": id_, + "version": 1, + "user": USER1, + "name": "mysample", + "node_tree": [ + { + "id": "root", + "parent": None, + "type": "BioReplicate", + "meta_controlled": { + "foo": {"bar": "baz"}, + }, + "meta_user": {"a": {"b": "c"}}, + "source_meta": [], + } + ], } def test_create_sample_fail_no_nodes(sample_port): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': None - } - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [{"sample": {"name": "mysample", "node_tree": None}}], + }, + ) # print(ret.text) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 30001 Illegal input parameter: sample node tree ' + - 'must be present and a list') + assert ret.json()["error"]["message"] == ( + "Sample service error code 30001 Illegal input parameter: sample node tree " + + "must be present and a list" + ) def test_create_sample_fail_bad_metadata(sample_port): _create_sample_fail_bad_metadata( - sample_port, {'stringlentest': {}}, - 'Sample service error code 30001 Illegal input parameter: Error for node at index 0: ' + - 'Controlled metadata value associated with metadata key stringlentest is null or empty') + sample_port, + {"stringlentest": {}}, + "Sample service error code 30001 Illegal input parameter: Error for node at index 0: " + + "Controlled metadata value associated with metadata key stringlentest is null or empty", + ) _create_sample_fail_bad_metadata( - sample_port, {'stringlentest': {'foooo': 'barrrr'}}, - 'Sample service error code 30010 Metadata validation failed: Node at index 0: ' + - 'Key stringlentest: Metadata value at key foooo is longer than max length of 5') + sample_port, + {"stringlentest": {"foooo": "barrrr"}}, + "Sample service error code 30010 Metadata validation failed: Node at index 0: " + + "Key stringlentest: Metadata value at key foooo is longer than max length of 5", + ) _create_sample_fail_bad_metadata( - sample_port, {'stringlentest': {'foooo': 'barrr', 'spcky': 'baz'}}, - 'Sample service error code 30010 Metadata validation failed: Node at index 0: Key ' + - 'stringlentest: Metadata value at key spcky is longer than max length of 2') + sample_port, + {"stringlentest": {"foooo": "barrr", "spcky": "baz"}}, + "Sample service error code 30010 Metadata validation failed: Node at index 0: Key " + + "stringlentest: Metadata value at key spcky is longer than max length of 2", + ) _create_sample_fail_bad_metadata( - sample_port, {'prefix': {'fail_plz': 'yes, or principal sayof'}}, - "Sample service error code 30010 Metadata validation failed: Node at index 0: " + - "Prefix validator pre, key prefix: pre, prefix, {'fail_plz': 'yes, or principal sayof'}") + sample_port, + {"prefix": {"fail_plz": "yes, or principal sayof"}}, + "Sample service error code 30010 Metadata validation failed: Node at index 0: " + + "Prefix validator pre, key prefix: pre, prefix, {'fail_plz': 'yes, or principal sayof'}", + ) _create_sample_fail_bad_metadata( - sample_port, {'prefix': {'foo': 'bar'}}, - 'Sample service error code 30001 Illegal input parameter: Error for node at ' + - 'index 0: Duplicate source metadata key: prefix', + sample_port, + {"prefix": {"foo": "bar"}}, + "Sample service error code 30001 Illegal input parameter: Error for node at " + + "index 0: Duplicate source metadata key: prefix", sourcemeta=[ - {'key': 'prefix', 'skey': 'a', 'svalue': {'a': 'b'}}, - {'key': 'prefix', 'skey': 'b', 'svalue': {'c': 'd'}} - ]) + {"key": "prefix", "skey": "a", "svalue": {"a": "b"}}, + {"key": "prefix", "skey": "b", "svalue": {"c": "d"}}, + ], + ) def _create_sample_fail_bad_metadata(sample_port, meta, expected, sourcemeta=None): - url = f'http://localhost:{sample_port}' - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'meta_controlled': meta, - 'source_meta': sourcemeta - } - ] - } - }] - }) + url = f"http://localhost:{sample_port}" + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "meta_controlled": meta, + "source_meta": sourcemeta, + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def test_create_sample_fail_permissions(sample_port): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - } - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] - - _replace_acls(url, id_, TOKEN1, {'read': [USER2]}) - - ret = requests.post(url, headers=get_authorized_headers(TOKEN2), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'id': id_, - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - } - }] - }) + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] + + _replace_acls(url, id_, TOKEN1, {"read": [USER2]}) + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN2), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "id": id_, + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - f'Sample service error code 20000 Unauthorized: User user2 cannot write to sample {id_}') + assert ret.json()["error"]["message"] == ( + f"Sample service error code 20000 Unauthorized: User user2 cannot write to sample {id_}" + ) def test_create_sample_fail_admin_bad_user_name(sample_port): _create_sample_fail_admin_as_user( - sample_port, 'bad\tuser', - 'Sample service error code 30001 Illegal input parameter: userid contains ' + - 'control characters') + sample_port, + "bad\tuser", + "Sample service error code 30001 Illegal input parameter: userid contains " + + "control characters", + ) def test_create_sample_fail_admin_no_such_user(sample_port): _create_sample_fail_admin_as_user( - sample_port, USER4 + 'impostor', - 'Sample service error code 50000 No such user: user4impostor') + sample_port, + USER4 + "impostor", + "Sample service error code 50000 No such user: user4impostor", + ) def _create_sample_fail_admin_as_user(sample_port, user, expected): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN2), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - }, - 'as_admin': 'true', - 'as_user': user - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN2), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + }, + "as_admin": "true", + "as_user": user, + } + ], + }, + ) # print(ret.text) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def test_create_sample_fail_admin_permissions(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" # token 3 only has read permissions - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - }, - 'as_admin': 1, - 'as_user': USER4 - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + }, + "as_admin": 1, + "as_user": USER4, + } + ], + }, + ) # print(ret.text) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 20000 Unauthorized: User user3 does not have the ' + - 'necessary administration privileges to run method create_sample') + assert ret.json()["error"]["message"] == ( + "Sample service error code 20000 Unauthorized: User user3 does not have the " + + "necessary administration privileges to run method create_sample" + ) def test_get_sample_fail_bad_id(sample_port): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - } - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] - ret = requests.post(url, headers=get_authorized_headers(TOKEN2), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_[:-1]}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN2), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id_[:-1]}], + }, + ) # print(ret.text) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 30001 Illegal input parameter: ' + - f'id {id_[:-1]} must be a UUID string') + assert ret.json()["error"]["message"] == ( + "Sample service error code 30001 Illegal input parameter: " + + f"id {id_[:-1]} must be a UUID string" + ) def test_get_sample_fail_permissions(sample_port): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - } - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] _get_sample_fail( - url, TOKEN2, {'id': id_}, - f'Sample service error code 20000 Unauthorized: User user2 cannot read sample {id_}') + url, + TOKEN2, + {"id": id_}, + f"Sample service error code 20000 Unauthorized: User user2 cannot read sample {id_}", + ) _get_sample_fail( - url, None, {'id': id_}, - f'Sample service error code 20000 Unauthorized: Anonymous users cannot read sample {id_}') + url, + None, + {"id": id_}, + f"Sample service error code 20000 Unauthorized: Anonymous users cannot read sample {id_}", + ) _get_sample_fail( - url, None, {'id': id_, 'as_admin': 1}, - 'Sample service error code 20000 Unauthorized: Anonymous users ' + - 'may not act as service administrators.') + url, + None, + {"id": id_, "as_admin": 1}, + "Sample service error code 20000 Unauthorized: Anonymous users " + + "may not act as service administrators.", + ) def test_get_sample_fail_admin_permissions(sample_port): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - } - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] _get_sample_fail( - url, TOKEN4, {'id': id_, 'as_admin': 1}, - 'Sample service error code 20000 Unauthorized: User user4 does not have the ' + - 'necessary administration privileges to run method get_sample') + url, + TOKEN4, + {"id": id_, "as_admin": 1}, + "Sample service error code 20000 Unauthorized: User user4 does not have the " + + "necessary administration privileges to run method get_sample", + ) def _get_sample_fail(url, token, params, expected): # user 4 has no admin permissions - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) # print(ret.text) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def test_get_and_replace_acls(sample_port, kafka): _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - } - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] - - _assert_acl_contents(url, id_, TOKEN1, { - 'owner': USER1, - 'admin': [], - 'write': [], - 'read': [], - 'public_read': 0 - }) - - _replace_acls(url, id_, TOKEN1, { - 'admin': [USER2], - 'write': [USER_NO_TOKEN1, USER_NO_TOKEN2, USER3], - 'read': [USER_NO_TOKEN3, USER4] - }) + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] + + _assert_acl_contents( + url, + id_, + TOKEN1, + {"owner": USER1, "admin": [], "write": [], "read": [], "public_read": 0}, + ) + + _replace_acls( + url, + id_, + TOKEN1, + { + "admin": [USER2], + "write": [USER_NO_TOKEN1, USER_NO_TOKEN2, USER3], + "read": [USER_NO_TOKEN3, USER4], + }, + ) # test that people in the acls can read for token in [TOKEN2, TOKEN3, TOKEN4]: - _assert_acl_contents(url, id_, token, { - 'owner': USER1, - 'admin': [USER2], - 'write': [USER3, USER_NO_TOKEN1, USER_NO_TOKEN2], - 'read': [USER4, USER_NO_TOKEN3], - 'public_read': 0 - }) - - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_}] - }) + _assert_acl_contents( + url, + id_, + token, + { + "owner": USER1, + "admin": [USER2], + "write": [USER3, USER_NO_TOKEN1, USER_NO_TOKEN2], + "read": [USER4, USER_NO_TOKEN3], + "public_read": 0, + }, + ) + + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id_}], + }, + ) # print(ret.text) assert ret.ok is True - j = ret.json()['result'][0] - del j['save_date'] + j = ret.json()["result"][0] + del j["save_date"] assert j == { - 'id': id_, - 'version': 1, - 'user': USER1, - 'name': 'mysample', - 'node_tree': [{ - 'id': 'root', - 'type': 'BioReplicate', - 'parent': None, - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - }] - } + "id": id_, + "version": 1, + "user": USER1, + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "parent": None, + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + } + ], + } # test admins and writers can write for token, version in ((TOKEN2, 2), (TOKEN3, 3)): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '68', - 'params': [{ - 'sample': {'name': f'mysample{version}', - 'id': id_, - 'node_tree': [{'id': f'root{version}', - 'type': 'BioReplicate', - } - ] - } - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "68", + "params": [ + { + "sample": { + "name": f"mysample{version}", + "id": id_, + "node_tree": [ + { + "id": f"root{version}", + "type": "BioReplicate", + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == version + assert ret.json()["result"][0]["version"] == version # check one of the writes - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.get_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'version': 2}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.get_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "version": 2}], + }, + ) # print(ret.text) assert ret.ok is True - j = ret.json()['result'][0] - assert_ms_epoch_close_to_now(j['save_date']) - del j['save_date'] + j = ret.json()["result"][0] + assert_ms_epoch_close_to_now(j["save_date"]) + del j["save_date"] assert j == { - 'id': id_, - 'version': 2, - 'user': USER2, - 'name': 'mysample2', - 'node_tree': [{'id': 'root2', - 'parent': None, - 'type': 'BioReplicate', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - }] + "id": id_, + "version": 2, + "user": USER2, + "name": "mysample2", + "node_tree": [ + { + "id": "root2", + "parent": None, + "type": "BioReplicate", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + } + ], } # test that an admin can replace ACLs - _replace_acls(url, id_, TOKEN2, { - 'admin': [USER_NO_TOKEN2], - 'write': [], - 'read': [USER2], - 'public_read': 1 - }) - - _assert_acl_contents(url, id_, TOKEN1, { - 'owner': USER1, - 'admin': [USER_NO_TOKEN2], - 'write': [], - 'read': [USER2], - 'public_read': 1 - }) + _replace_acls( + url, + id_, + TOKEN2, + {"admin": [USER_NO_TOKEN2], "write": [], "read": [USER2], "public_read": 1}, + ) + + _assert_acl_contents( + url, + id_, + TOKEN1, + { + "owner": USER1, + "admin": [USER_NO_TOKEN2], + "write": [], + "read": [USER2], + "public_read": 1, + }, + ) _check_kafka_messages( kafka, [ - {'event_type': 'NEW_SAMPLE', 'sample_id': id_, 'sample_ver': 1}, - {'event_type': 'ACL_CHANGE', 'sample_id': id_}, - {'event_type': 'NEW_SAMPLE', 'sample_id': id_, 'sample_ver': 2}, - {'event_type': 'NEW_SAMPLE', 'sample_id': id_, 'sample_ver': 3}, - {'event_type': 'ACL_CHANGE', 'sample_id': id_}, - ]) + {"event_type": "NEW_SAMPLE", "sample_id": id_, "sample_ver": 1}, + {"event_type": "ACL_CHANGE", "sample_id": id_}, + {"event_type": "NEW_SAMPLE", "sample_id": id_, "sample_ver": 2}, + {"event_type": "NEW_SAMPLE", "sample_id": id_, "sample_ver": 3}, + {"event_type": "ACL_CHANGE", "sample_id": id_}, + ], + ) def test_get_acls_public_read(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - _replace_acls(url, id_, TOKEN1, {'public_read': 1}) + _replace_acls(url, id_, TOKEN1, {"public_read": 1}) for token in [TOKEN4, None]: # user with no explicit perms and anon user - _assert_acl_contents(url, id_, token, { - 'owner': USER1, - 'admin': [], - 'write': [], - 'read': [], - 'public_read': 1 - }) + _assert_acl_contents( + url, + id_, + token, + {"owner": USER1, "admin": [], "write": [], "read": [], "public_read": 1}, + ) def test_get_acls_as_admin(sample_port): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - } - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] # user 3 has admin read rights only - _assert_acl_contents(url, id_, TOKEN3, { - 'owner': USER1, - 'admin': [], - 'write': [], - 'read': [], - 'public_read': 0 - }, - as_admin=1) + _assert_acl_contents( + url, + id_, + TOKEN3, + {"owner": USER1, "admin": [], "write": [], "read": [], "public_read": 0}, + as_admin=1, + ) def test_replace_acls_as_admin(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - _assert_acl_contents(url, id_, TOKEN1, { - 'owner': USER1, - 'admin': [], - 'write': [], - 'read': [], - 'public_read': 0 - }) + _assert_acl_contents( + url, + id_, + TOKEN1, + {"owner": USER1, "admin": [], "write": [], "read": [], "public_read": 0}, + ) - _replace_acls(url, id_, TOKEN2, { - 'admin': [USER2], - 'write': [USER_NO_TOKEN1, USER_NO_TOKEN2, USER3], - 'read': [USER_NO_TOKEN3, USER4], - 'public_read': 1 + _replace_acls( + url, + id_, + TOKEN2, + { + "admin": [USER2], + "write": [USER_NO_TOKEN1, USER_NO_TOKEN2, USER3], + "read": [USER_NO_TOKEN3, USER4], + "public_read": 1, }, - as_admin=1) + as_admin=1, + ) - _assert_acl_contents(url, id_, TOKEN1, { - 'owner': USER1, - 'admin': [USER2], - 'write': [USER3, USER_NO_TOKEN1, USER_NO_TOKEN2], - 'read': [USER4, USER_NO_TOKEN3], - 'public_read': 1 - }) + _assert_acl_contents( + url, + id_, + TOKEN1, + { + "owner": USER1, + "admin": [USER2], + "write": [USER3, USER_NO_TOKEN1, USER_NO_TOKEN2], + "read": [USER4, USER_NO_TOKEN3], + "public_read": 1, + }, + ) def _replace_acls(url, id_, token, acls, as_admin=0, print_resp=False): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.replace_sample_acls', - 'version': '1.1', - 'id': '67', - 'params': [{'id': id_, 'acls': acls, 'as_admin': as_admin}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.replace_sample_acls", + "version": "1.1", + "id": "67", + "params": [{"id": id_, "acls": acls, "as_admin": as_admin}], + }, + ) if print_resp: print(ret.text) assert ret.ok is True - assert ret.json() == {'version': '1.1', 'id': '67', 'result': None} + assert ret.json() == {"version": "1.1", "id": "67", "result": None} def _assert_acl_contents(url, id_, token, expected, as_admin=0, print_resp=False): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_sample_acls', - 'version': '1.1', - 'id': '47', - 'params': [{'id': id_, 'as_admin': as_admin}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_sample_acls", + "version": "1.1", + "id": "47", + "params": [{"id": id_, "as_admin": as_admin}], + }, + ) if print_resp: print(ret.text) assert ret.ok is True - assert ret.json()['result'][0] == expected + assert ret.json()["result"][0] == expected def test_get_acls_fail_no_id(sample_port): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - } - ] - } - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + } + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - id_ = ret.json()['result'][0]['id'] - - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.get_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [{'ids': id_}] - }) + assert ret.json()["result"][0]["version"] == 1 + id_ = ret.json()["result"][0]["id"] + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.get_sample_acls", + "version": "1.1", + "id": "42", + "params": [{"ids": id_}], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 30000 Missing input parameter: id') + assert ret.json()["error"]["message"] == ( + "Sample service error code 30000 Missing input parameter: id" + ) def test_get_acls_fail_permissions(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) _get_acls_fail_permissions( - url, TOKEN2, {'id': id_}, - f'Sample service error code 20000 Unauthorized: User user2 cannot read sample {id_}') + url, + TOKEN2, + {"id": id_}, + f"Sample service error code 20000 Unauthorized: User user2 cannot read sample {id_}", + ) _get_acls_fail_permissions( - url, None, {'id': id_}, - f'Sample service error code 20000 Unauthorized: Anonymous users cannot read sample {id_}') + url, + None, + {"id": id_}, + f"Sample service error code 20000 Unauthorized: Anonymous users cannot read sample {id_}", + ) _get_acls_fail_permissions( - url, None, {'id': id_, 'as_admin': 1}, - 'Sample service error code 20000 Unauthorized: Anonymous users ' + - 'may not act as service administrators.') + url, + None, + {"id": id_, "as_admin": 1}, + "Sample service error code 20000 Unauthorized: Anonymous users " + + "may not act as service administrators.", + ) def _get_acls_fail_permissions(url, token, params, expected): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_sample_acls", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def test_get_acls_fail_admin_permissions(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) # user 4 has no admin perms - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.get_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'as_admin': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.get_sample_acls", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "as_admin": 1}], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 20000 Unauthorized: User user4 does not have the ' + - 'necessary administration privileges to run method get_sample_acls') + assert ret.json()["error"]["message"] == ( + "Sample service error code 20000 Unauthorized: User user4 does not have the " + + "necessary administration privileges to run method get_sample_acls" + ) def _create_generic_sample(url, token): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'sample': {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - }, - {'id': 'foo', - 'parent': 'root', - 'type': 'TechReplicate', - } - ] - } - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [ + { + "sample": { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + }, + { + "id": "foo", + "parent": "root", + "type": "TechReplicate", + }, + ], + } + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == 1 - return ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == 1 + return ret.json()["result"][0]["id"] def test_replace_acls_fail_no_id(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.replace_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [{'ids': id_}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.replace_sample_acls", + "version": "1.1", + "id": "42", + "params": [{"ids": id_}], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 30000 Missing input parameter: id') + assert ret.json()["error"]["message"] == ( + "Sample service error code 30000 Missing input parameter: id" + ) def test_replace_acls_fail_bad_acls(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.replace_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'acls': ['foo']}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.replace_sample_acls", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "acls": ["foo"]}], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 30001 Illegal input parameter: ' + - 'ACLs must be supplied in the acls key and must be a mapping') + assert ret.json()["error"]["message"] == ( + "Sample service error code 30001 Illegal input parameter: " + + "ACLs must be supplied in the acls key and must be a mapping" + ) def test_replace_acls_fail_permissions(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - _replace_acls(url, id_, TOKEN1, { - 'admin': [USER2], - 'write': [USER3], - 'read': [USER4] - }) + _replace_acls( + url, id_, TOKEN1, {"admin": [USER2], "write": [USER3], "read": [USER4]} + ) for user, token in ((USER3, TOKEN3), (USER4, TOKEN4)): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.replace_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'acls': {}}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.replace_sample_acls", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "acls": {}}], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - f'Sample service error code 20000 Unauthorized: User {user} cannot ' + - f'administrate sample {id_}') + assert ret.json()["error"]["message"] == ( + f"Sample service error code 20000 Unauthorized: User {user} cannot " + + f"administrate sample {id_}" + ) def test_replace_acls_fail_admin_permissions(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) for user, token in ((USER1, TOKEN1), (USER3, TOKEN3), (USER4, TOKEN4)): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.replace_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'acls': {}, 'as_admin': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.replace_sample_acls", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "acls": {}, "as_admin": 1}], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - f'Sample service error code 20000 Unauthorized: User {user} does not have the ' + - 'necessary administration privileges to run method replace_sample_acls') + assert ret.json()["error"]["message"] == ( + f"Sample service error code 20000 Unauthorized: User {user} does not have the " + + "necessary administration privileges to run method replace_sample_acls" + ) def test_replace_acls_fail_bad_user(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.replace_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, - 'acls': { - 'admin': [USER2, 'a'], - 'write': [USER3], - 'read': [USER4, 'philbin_j_montgomery_iii'] - } - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.replace_sample_acls", + "version": "1.1", + "id": "42", + "params": [ + { + "id": id_, + "acls": { + "admin": [USER2, "a"], + "write": [USER3], + "read": [USER4, "philbin_j_montgomery_iii"], + }, + } + ], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 50000 No such user: a, philbin_j_montgomery_iii') + assert ret.json()["error"]["message"] == ( + "Sample service error code 50000 No such user: a, philbin_j_montgomery_iii" + ) def test_replace_acls_fail_user_in_2_acls(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.replace_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'acls': {'write': [USER2, USER3], 'read': [USER2]}}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.replace_sample_acls", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "acls": {"write": [USER2, USER3], "read": [USER2]}}], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 30001 Illegal input parameter: ' + - f'User {USER2} appears in two ACLs') + assert ret.json()["error"]["message"] == ( + "Sample service error code 30001 Illegal input parameter: " + + f"User {USER2} appears in two ACLs" + ) def test_replace_acls_fail_owner_in_another_acl(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.replace_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'acls': {'write': [USER1]}}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.replace_sample_acls", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "acls": {"write": [USER1]}}], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == ( - 'Sample service error code 30001 Illegal input parameter: ' + - 'The owner cannot be in any other ACL') + assert ret.json()["error"]["message"] == ( + "Sample service error code 30001 Illegal input parameter: " + + "The owner cannot be in any other ACL" + ) def test_update_acls(sample_port, kafka): @@ -1765,44 +2216,60 @@ def test_update_acls(sample_port, kafka): _update_acls_tst(sample_port, kafka, TOKEN2, False) # admin _update_acls_tst(sample_port, kafka, TOKEN5, True) # as_admin = True + def _update_acls_tst(sample_port, kafka, token, as_admin): _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - _replace_acls(url, id_, TOKEN1, { - 'admin': [USER2], - 'write': [USER_NO_TOKEN1, USER_NO_TOKEN2, USER3], - 'read': [USER_NO_TOKEN3, USER4], - 'public_read': 0 - }) - - _update_acls(url, token, { - 'id': str(id_), - 'admin': [USER4], - 'write': [USER2], - 'read': [USER_NO_TOKEN2], - 'remove': [USER3], - 'public_read': 390, - 'as_admin': 1 if as_admin else 0, - }) - - _assert_acl_contents(url, id_, TOKEN1, { - 'owner': USER1, - 'admin': [USER4], - 'write': [USER2, USER_NO_TOKEN1], - 'read': [USER_NO_TOKEN2, USER_NO_TOKEN3], - 'public_read': 1 - }) + _replace_acls( + url, + id_, + TOKEN1, + { + "admin": [USER2], + "write": [USER_NO_TOKEN1, USER_NO_TOKEN2, USER3], + "read": [USER_NO_TOKEN3, USER4], + "public_read": 0, + }, + ) + + _update_acls( + url, + token, + { + "id": str(id_), + "admin": [USER4], + "write": [USER2], + "read": [USER_NO_TOKEN2], + "remove": [USER3], + "public_read": 390, + "as_admin": 1 if as_admin else 0, + }, + ) + + _assert_acl_contents( + url, + id_, + TOKEN1, + { + "owner": USER1, + "admin": [USER4], + "write": [USER2, USER_NO_TOKEN1], + "read": [USER_NO_TOKEN2, USER_NO_TOKEN3], + "public_read": 1, + }, + ) _check_kafka_messages( kafka, [ - {'event_type': 'NEW_SAMPLE', 'sample_id': id_, 'sample_ver': 1}, - {'event_type': 'ACL_CHANGE', 'sample_id': id_}, - {'event_type': 'ACL_CHANGE', 'sample_id': id_}, - ]) + {"event_type": "NEW_SAMPLE", "sample_id": id_, "sample_ver": 1}, + {"event_type": "ACL_CHANGE", "sample_id": id_}, + {"event_type": "ACL_CHANGE", "sample_id": id_}, + ], + ) def test_update_acls_with_at_least(sample_port, kafka): @@ -1813,355 +2280,429 @@ def test_update_acls_with_at_least(sample_port, kafka): def _update_acls_tst_with_at_least(sample_port, kafka, token, as_admin): _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - _replace_acls(url, id_, TOKEN1, { - 'admin': [USER2], - 'write': [USER_NO_TOKEN1, USER_NO_TOKEN2, USER3], - 'read': [USER_NO_TOKEN3, USER4], - 'public_read': 0 - }) - - _update_acls(url, token, { - 'id': str(id_), - 'admin': [USER4], - 'write': [USER2, USER_NO_TOKEN3], - 'read': [USER_NO_TOKEN2, USER5], - 'remove': [USER3], - 'public_read': 390, - 'as_admin': 1 if as_admin else 0, - 'at_least': 1, - }) - - _assert_acl_contents(url, id_, TOKEN1, { - 'owner': USER1, - 'admin': [USER2, USER4], - 'write': [USER_NO_TOKEN1, USER_NO_TOKEN2, USER_NO_TOKEN3], - 'read': [USER5], - 'public_read': 1 - }, print_resp=True) + _replace_acls( + url, + id_, + TOKEN1, + { + "admin": [USER2], + "write": [USER_NO_TOKEN1, USER_NO_TOKEN2, USER3], + "read": [USER_NO_TOKEN3, USER4], + "public_read": 0, + }, + ) + + _update_acls( + url, + token, + { + "id": str(id_), + "admin": [USER4], + "write": [USER2, USER_NO_TOKEN3], + "read": [USER_NO_TOKEN2, USER5], + "remove": [USER3], + "public_read": 390, + "as_admin": 1 if as_admin else 0, + "at_least": 1, + }, + ) + + _assert_acl_contents( + url, + id_, + TOKEN1, + { + "owner": USER1, + "admin": [USER2, USER4], + "write": [USER_NO_TOKEN1, USER_NO_TOKEN2, USER_NO_TOKEN3], + "read": [USER5], + "public_read": 1, + }, + print_resp=True, + ) _check_kafka_messages( kafka, [ - {'event_type': 'NEW_SAMPLE', 'sample_id': id_, 'sample_ver': 1}, - {'event_type': 'ACL_CHANGE', 'sample_id': id_}, - {'event_type': 'ACL_CHANGE', 'sample_id': id_}, - ]) + {"event_type": "NEW_SAMPLE", "sample_id": id_, "sample_ver": 1}, + {"event_type": "ACL_CHANGE", "sample_id": id_}, + {"event_type": "ACL_CHANGE", "sample_id": id_}, + ], + ) def test_update_acls_fail_no_id(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) _update_acls_fail( - url, TOKEN1, {'ids': id_}, - 'Sample service error code 30000 Missing input parameter: id') + url, + TOKEN1, + {"ids": id_}, + "Sample service error code 30000 Missing input parameter: id", + ) def test_update_acls_fail_bad_pub(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) _update_acls_fail( - url, TOKEN1, {'id': id_, 'public_read': 'thingy'}, - 'Sample service error code 30001 Illegal input parameter: ' + - 'public_read must be an integer if present') + url, + TOKEN1, + {"id": id_, "public_read": "thingy"}, + "Sample service error code 30001 Illegal input parameter: " + + "public_read must be an integer if present", + ) def test_update_acls_fail_permissions(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) - _replace_acls(url, id_, TOKEN1, { - 'admin': [USER2], - 'write': [USER3], - 'read': [USER4] - }) + _replace_acls( + url, id_, TOKEN1, {"admin": [USER2], "write": [USER3], "read": [USER4]} + ) for user, token in ((USER3, TOKEN3), (USER4, TOKEN4)): - _update_acls_fail(url, token, {'id': id_}, 'Sample service error code 20000 ' + - f'Unauthorized: User {user} cannot administrate sample {id_}') + _update_acls_fail( + url, + token, + {"id": id_}, + "Sample service error code 20000 " + + f"Unauthorized: User {user} cannot administrate sample {id_}", + ) def test_update_acls_fail_admin_permissions(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) for user, token in ((USER1, TOKEN1), (USER3, TOKEN3), (USER4, TOKEN4)): _update_acls_fail( - url, token, {'id': id_, 'as_admin': 1}, - f'Sample service error code 20000 Unauthorized: User {user} does not have the ' + - 'necessary administration privileges to run method update_sample_acls') + url, + token, + {"id": id_, "as_admin": 1}, + f"Sample service error code 20000 Unauthorized: User {user} does not have the " + + "necessary administration privileges to run method update_sample_acls", + ) def test_update_acls_fail_bad_user(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) _update_acls_fail( url, TOKEN1, - {'id': id_, - 'admin': [USER2, 'a'], - 'write': [USER3], - 'read': [USER4, 'philbin_j_montgomery_iii'], - 'remove': ['someguy'] - }, - 'Sample service error code 50000 No such user: a, philbin_j_montgomery_iii, someguy') + { + "id": id_, + "admin": [USER2, "a"], + "write": [USER3], + "read": [USER4, "philbin_j_montgomery_iii"], + "remove": ["someguy"], + }, + "Sample service error code 50000 No such user: a, philbin_j_montgomery_iii, someguy", + ) def test_update_acls_fail_user_2_acls(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) _update_acls_fail( url, TOKEN1, - {'id': id_, - 'admin': [USER2], - 'write': [USER3], - 'read': [USER4, USER2], - }, - 'Sample service error code 30001 Illegal input parameter: User user2 appears in two ACLs') + { + "id": id_, + "admin": [USER2], + "write": [USER3], + "read": [USER4, USER2], + }, + "Sample service error code 30001 Illegal input parameter: User user2 appears in two ACLs", + ) def test_update_acls_fail_user_in_acl_and_remove(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) _update_acls_fail( url, TOKEN1, - {'id': id_, - 'admin': [USER2], - 'write': [USER3], - 'read': [USER4], - 'remove': [USER2] - }, - 'Sample service error code 30001 Illegal input parameter: Users in the remove list ' + - 'cannot be in any other ACL') + { + "id": id_, + "admin": [USER2], + "write": [USER3], + "read": [USER4], + "remove": [USER2], + }, + "Sample service error code 30001 Illegal input parameter: Users in the remove list " + + "cannot be in any other ACL", + ) def test_update_acls_fail_owner_in_another_acl(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) _update_acls_fail( - url, TOKEN1, {'id': id_, 'write': [USER1]}, - 'Sample service error code 20000 Unauthorized: ' + - 'ACLs for the sample owner user1 may not be modified by a delta update.') + url, + TOKEN1, + {"id": id_, "write": [USER1]}, + "Sample service error code 20000 Unauthorized: " + + "ACLs for the sample owner user1 may not be modified by a delta update.", + ) def test_update_acls_fail_owner_in_remove_acl(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN1) _update_acls_fail( - url, TOKEN1, {'id': id_, 'remove': [USER1]}, - 'Sample service error code 20000 Unauthorized: ' + - 'ACLs for the sample owner user1 may not be modified by a delta update.') + url, + TOKEN1, + {"id": id_, "remove": [USER1]}, + "Sample service error code 20000 Unauthorized: " + + "ACLs for the sample owner user1 may not be modified by a delta update.", + ) def _update_acls_fail(url, token, params, expected): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.update_sample_acls', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.update_sample_acls", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def _update_acls(url, token, params, print_resp=False): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.update_sample_acls', - 'version': '1.1', - 'id': '67', - 'params': [params] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.update_sample_acls", + "version": "1.1", + "id": "67", + "params": [params], + }, + ) if print_resp: print(ret.text) assert ret.ok is True - assert ret.json() == {'version': '1.1', 'id': '67', 'result': None} + assert ret.json() == {"version": "1.1", "id": "67", "result": None} def _update_samples_acls(url, token, params, print_resp=False): - resp = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.update_samples_acls', - 'version': '1.1', - 'id': '1729', - 'params': [params] - }) + resp = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.update_samples_acls", + "version": "1.1", + "id": "1729", + "params": [params], + }, + ) if print_resp: print(resp.text) return resp def test_update_acls_many(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" # create samples - n_samples = 2 # 1000 + n_samples = 2 # 1000 ids = _create_samples(url, TOKEN1, n_samples, 1) for id_ in ids: _update_acls( url, TOKEN1, { - 'id': str(id_), - 'admin': [], - 'write': [], - 'read': [USER2], - 'remove': [], - 'public_read': 1, - 'as_admin': 0, + "id": str(id_), + "admin": [], + "write": [], + "read": [USER2], + "remove": [], + "public_read": 1, + "as_admin": 0, }, print_resp=True, ) def test_update_acls_many_bulk(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" # create samples - n_samples = 2 # 1000 + n_samples = 2 # 1000 ids = _create_samples(url, TOKEN1, n_samples, 1) resp = _update_samples_acls( url, TOKEN1, { - 'ids': ids, - 'admin': [], - 'write': [], - 'read': [USER2], - 'remove': [], - 'public_read': 1, - 'as_admin': 0, + "ids": ids, + "admin": [], + "write": [], + "read": [USER2], + "remove": [], + "public_read": 1, + "as_admin": 0, }, print_resp=True, ) assert resp.ok - assert resp.json()['result'] is None + assert resp.json()["result"] is None + def test_update_acls_many_bulk_fail(sample_port): - url = f'http://localhost:{sample_port}' - sample_bad_id = str(uuid.UUID('0'*32)) + url = f"http://localhost:{sample_port}" + sample_bad_id = str(uuid.UUID("0" * 32)) resp = _update_samples_acls( url, TOKEN1, { - 'ids': [sample_bad_id], - 'admin': [], - 'write': [], - 'read': [USER2], - 'remove': [], - 'public_read': 1, - 'as_admin': 0, + "ids": [sample_bad_id], + "admin": [], + "write": [], + "read": [USER2], + "remove": [], + "public_read": 1, + "as_admin": 0, }, print_resp=True, ) assert resp.status_code == 500 msg = f"Sample service error code 50010 No such sample: {sample_bad_id}" - assert resp.json()['error']['message'] == msg + assert resp.json()["error"]["message"] == msg + def test_get_metadata_key_static_metadata(sample_port): _get_metadata_key_static_metadata( - sample_port, {'keys': ['foo']}, {'foo': {'a': 'b', 'c': 'd'}}) + sample_port, {"keys": ["foo"]}, {"foo": {"a": "b", "c": "d"}} + ) _get_metadata_key_static_metadata( sample_port, - {'keys': ['foo', 'stringlentest'], 'prefix': 0}, - {'foo': {'a': 'b', 'c': 'd'}, 'stringlentest': {'h': 'i', 'j': 'k'}}) + {"keys": ["foo", "stringlentest"], "prefix": 0}, + {"foo": {"a": "b", "c": "d"}, "stringlentest": {"h": "i", "j": "k"}}, + ) _get_metadata_key_static_metadata( - sample_port, {'keys': ['pre'], 'prefix': 1}, {'pre': {'1': '2'}}) + sample_port, {"keys": ["pre"], "prefix": 1}, {"pre": {"1": "2"}} + ) _get_metadata_key_static_metadata( - sample_port, {'keys': ['premature'], 'prefix': 2}, {'pre': {'1': '2'}}) + sample_port, {"keys": ["premature"], "prefix": 2}, {"pre": {"1": "2"}} + ) def _get_metadata_key_static_metadata(sample_port, params, expected): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, json={ - 'method': 'SampleService.get_metadata_key_static_metadata', - 'version': '1.1', - 'id': '67', - 'params': [params] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + json={ + "method": "SampleService.get_metadata_key_static_metadata", + "version": "1.1", + "id": "67", + "params": [params], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0] == {'static_metadata': expected} + assert ret.json()["result"][0] == {"static_metadata": expected} def test_get_metadata_key_static_metadata_fail_bad_args(sample_port): _get_metadata_key_static_metadata_fail( sample_port, {}, - 'Sample service error code 30001 Illegal input parameter: keys must be a list') + "Sample service error code 30001 Illegal input parameter: keys must be a list", + ) _get_metadata_key_static_metadata_fail( sample_port, - {'keys': ['foo', 'stringlentestage'], 'prefix': 0}, - 'Sample service error code 30001 Illegal input parameter: No such metadata key: ' + - 'stringlentestage') + {"keys": ["foo", "stringlentestage"], "prefix": 0}, + "Sample service error code 30001 Illegal input parameter: No such metadata key: " + + "stringlentestage", + ) _get_metadata_key_static_metadata_fail( sample_port, - {'keys': ['premature'], 'prefix': 1}, - 'Sample service error code 30001 Illegal input parameter: No such prefix metadata key: ' + - 'premature') + {"keys": ["premature"], "prefix": 1}, + "Sample service error code 30001 Illegal input parameter: No such prefix metadata key: " + + "premature", + ) _get_metadata_key_static_metadata_fail( sample_port, - {'keys': ['somekey'], 'prefix': 2}, - 'Sample service error code 30001 Illegal input parameter: No prefix metadata keys ' + - 'matching key somekey') + {"keys": ["somekey"], "prefix": 2}, + "Sample service error code 30001 Illegal input parameter: No prefix metadata keys " + + "matching key somekey", + ) def _get_metadata_key_static_metadata_fail(sample_port, params, error): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, json={ - 'method': 'SampleService.get_metadata_key_static_metadata', - 'version': '1.1', - 'id': '67', - 'params': [params] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + json={ + "method": "SampleService.get_metadata_key_static_metadata", + "version": "1.1", + "id": "67", + "params": [params], + }, + ) # print(ret.text) assert ret.status_code == 500 - assert ret.json()['error']['message'] == error + assert ret.json()["error"]["message"] == error def _create_sample(url, token, sample, expected_version): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [{'sample': sample}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [{"sample": sample}], + }, + ) # print(ret.text) assert ret.ok is True - assert ret.json()['result'][0]['version'] == expected_version - return ret.json()['result'][0]['id'] + assert ret.json()["result"][0]["version"] == expected_version + return ret.json()["result"][0]["id"] + def _sample_factory(name): return { "sample": { "name": name, - "node_tree": [{ + "node_tree": [ + { "id": "root", "type": "BioReplicate", }, @@ -2169,8 +2710,8 @@ def _sample_factory(name): "id": "foo", "parent": "root", "type": "TechReplicate", - } - ] + }, + ], } } @@ -2182,12 +2723,16 @@ def _create_samples(url, token, n, expected_version, sample_factory=None): ids = [] for i in range(n): sample = sample_factory(f"sample-{i}") - resp = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.create_sample', - 'version': '1.1', - 'id': '67', - 'params': [sample] - }) + resp = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.create_sample", + "version": "1.1", + "id": "67", + "params": [sample], + }, + ) assert resp.ok data = resp.json()["result"][0] assert data["version"] == expected_version @@ -2196,31 +2741,35 @@ def _create_samples(url, token, n, expected_version, sample_factory=None): def _create_link(url, token, expected_user, params, print_resp=False): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.create_data_link', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.create_data_link", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) if print_resp: print(ret.text) assert ret.ok is True - link = ret.json()['result'][0]['new_link'] - id_ = link['linkid'] + link = ret.json()["result"][0]["new_link"] + id_ = link["linkid"] uuid.UUID(id_) # check the ID is a valid UUID - del link['linkid'] - created = link['created'] + del link["linkid"] + created = link["created"] assert_ms_epoch_close_to_now(created) - del link['created'] + del link["created"] assert link == { - 'id': params['id'], - 'version': params['version'], - 'node': params['node'], - 'upa': params['upa'], - 'dataid': params.get('dataid'), - 'createdby': expected_user, - 'expiredby': None, - 'expired': None + "id": params["id"], + "version": params["version"], + "node": params["node"], + "upa": params["upa"], + "dataid": params.get("dataid"), + "createdby": expected_user, + "expiredby": None, + "expired": None, } return id_ @@ -2230,33 +2779,43 @@ def _create_sample_and_links_for_propagate_links(url, token, user): sid = _create_sample( url, token, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) # ver 2 _create_sample( url, token, - {'id': sid, - 'name': 'mysample2', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 2 - ) + { + "id": sid, + "name": "mysample2", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 2, + ) # create links lid1 = _create_link( - url, token, user, - {'id': sid, 'version': 1, 'node': 'root', 'upa': '1/1/1', 'dataid': 'column1'}) + url, + token, + user, + {"id": sid, "version": 1, "node": "root", "upa": "1/1/1", "dataid": "column1"}, + ) lid2 = _create_link( - url, token, user, - {'id': sid, 'version': 1, 'node': 'root', 'upa': '1/2/1', 'dataid': 'column2'}) + url, + token, + user, + {"id": sid, "version": 1, "node": "root", "upa": "1/2/1", "dataid": "column2"}, + ) return sid, lid1, lid2 @@ -2265,8 +2824,8 @@ def _check_data_links(links, expected_links): assert len(links) == len(expected_links) for link in links: - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] for link in expected_links: assert link in links @@ -2274,19 +2833,23 @@ def _check_data_links(links, expected_links): def _check_sample_data_links(url, sample_id, version, expected_links, token): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': sample_id, 'version': version}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [{"id": sample_id, "version": version}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - links = ret.json()['result'][0]['links'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + links = ret.json()["result"][0]["links"] _check_data_links(links, expected_links) @@ -2295,69 +2858,78 @@ def test_create_and_propagate_data_links(sample_port, workspace, kafka): _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - {'name': 'baz', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + {"name": "baz", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) sid, lid1, lid2 = _create_sample_and_links_for_propagate_links(url, TOKEN3, USER3) # check initial links for both version expected_links = [ { - 'linkid': lid1, - 'id': sid, - 'version': 1, - 'node': 'root', - 'upa': '1/1/1', - 'dataid': 'column1', - 'createdby': USER3, - 'expiredby': None, - 'expired': None + "linkid": lid1, + "id": sid, + "version": 1, + "node": "root", + "upa": "1/1/1", + "dataid": "column1", + "createdby": USER3, + "expiredby": None, + "expired": None, }, { - 'linkid': lid2, - 'id': sid, - 'version': 1, - 'node': 'root', - 'upa': '1/2/1', - 'dataid': 'column2', - 'createdby': USER3, - 'expiredby': None, - 'expired': None - } + "linkid": lid2, + "id": sid, + "version": 1, + "node": "root", + "upa": "1/2/1", + "dataid": "column2", + "createdby": USER3, + "expiredby": None, + "expired": None, + }, ] _check_sample_data_links(url, sid, 1, expected_links, TOKEN3) _check_sample_data_links(url, sid, 2, [], TOKEN3) # propagate data links from sample version 1 to version 2 - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.propagate_data_links', - 'version': '1.1', - 'id': '38', - 'params': [{'id': sid, 'version': 2, 'previous_version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.propagate_data_links", + "version": "1.1", + "id": "38", + "params": [{"id": sid, "version": 2, "previous_version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 1 - links = ret.json()['result'][0]['links'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 1 + links = ret.json()["result"][0]["links"] - new_link_ids = [i['linkid'] for i in links] + new_link_ids = [i["linkid"] for i in links] expected_new_links = copy.deepcopy(expected_links) # propagated links should have new link id, dataid and version for idx, expected_link in enumerate(expected_new_links): - expected_link['linkid'] = new_link_ids[idx] - expected_link['dataid'] = expected_link['dataid'] + '_2' - expected_link['version'] = 2 + expected_link["linkid"] = new_link_ids[idx] + expected_link["dataid"] = expected_link["dataid"] + "_2" + expected_link["version"] = 2 _check_data_links(links, expected_new_links) @@ -2370,72 +2942,87 @@ def test_create_and_propagate_data_links_type_specific(sample_port, workspace, k _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - {'name': 'baz', 'data': {}, 'type': 'Trivial.Object2-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + {"name": "baz", "data": {}, "type": "Trivial.Object2-1.0"}, + ], + } + ) sid, lid1, lid2 = _create_sample_and_links_for_propagate_links(url, TOKEN3, USER3) # check initial links for both version expected_links = [ { - 'linkid': lid1, - 'id': sid, - 'version': 1, - 'node': 'root', - 'upa': '1/1/1', - 'dataid': 'column1', - 'createdby': USER3, - 'expiredby': None, - 'expired': None + "linkid": lid1, + "id": sid, + "version": 1, + "node": "root", + "upa": "1/1/1", + "dataid": "column1", + "createdby": USER3, + "expiredby": None, + "expired": None, }, { - 'linkid': lid2, - 'id': sid, - 'version': 1, - 'node': 'root', - 'upa': '1/2/1', - 'dataid': 'column2', - 'createdby': USER3, - 'expiredby': None, - 'expired': None - } + "linkid": lid2, + "id": sid, + "version": 1, + "node": "root", + "upa": "1/2/1", + "dataid": "column2", + "createdby": USER3, + "expiredby": None, + "expired": None, + }, ] _check_sample_data_links(url, sid, 1, expected_links, TOKEN3) _check_sample_data_links(url, sid, 2, [], TOKEN3) # propagate data links from sample version 1 to version 2 - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.propagate_data_links', - 'version': '1.1', - 'id': '38', - 'params': [{'id': sid, 'version': 2, 'previous_version': 1, - 'ignore_types': ['Trivial.Object2']}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.propagate_data_links", + "version": "1.1", + "id": "38", + "params": [ + { + "id": sid, + "version": 2, + "previous_version": 1, + "ignore_types": ["Trivial.Object2"], + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 1 - links = ret.json()['result'][0]['links'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 1 + links = ret.json()["result"][0]["links"] - new_link_ids = [i['linkid'] for i in links] + new_link_ids = [i["linkid"] for i in links] expected_new_links = copy.deepcopy(expected_links) expected_new_links.pop() assert len(expected_new_links) == 1 # propagated links should have new link id, dataid and version for idx, expected_link in enumerate(expected_new_links): - expected_link['linkid'] = new_link_ids[idx] - expected_link['dataid'] = expected_link['dataid'] + '_2' - expected_link['version'] = 2 + expected_link["linkid"] = new_link_ids[idx] + expected_link["dataid"] = expected_link["dataid"] + "_2" + expected_link["version"] = 2 _check_data_links(links, expected_new_links) @@ -2445,207 +3032,252 @@ def test_create_and_propagate_data_links_type_specific(sample_port, workspace, k def test_create_links_and_get_links_from_sample_basic(sample_port, workspace, kafka): - ''' + """ Also tests that the 'as_user' key is ignored if 'as_admin' is falsy. - ''' + """ _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - {'name': 'baz', 'data': {}, 'type': 'Trivial.Object-1.0'}, - {'name': 'baz', 'data': {}, 'type': 'Trivial.Object-1.0'} - ]}) - wscli.set_permissions({'id': 1, 'new_permission': 'w', 'users': [USER4]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + {"name": "baz", "data": {}, "type": "Trivial.Object-1.0"}, + {"name": "baz", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_permissions({"id": 1, "new_permission": "w", "users": [USER4]}) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) id2 = _create_sample( url, TOKEN4, - {'name': 'myothersample', - 'node_tree': [{'id': 'root2', 'type': 'BioReplicate'}, - {'id': 'foo2', 'type': 'TechReplicate', 'parent': 'root2'} - ] - }, - 1 - ) + { + "name": "myothersample", + "node_tree": [ + {"id": "root2", "type": "BioReplicate"}, + {"id": "foo2", "type": "TechReplicate", "parent": "root2"}, + ], + }, + 1, + ) # ver 2 _create_sample( url, TOKEN4, - {'id': id2, - 'name': 'myothersample3', - 'node_tree': [{'id': 'root3', 'type': 'BioReplicate'}, - {'id': 'foo3', 'type': 'TechReplicate', 'parent': 'root3'} - ] - }, - 2 - ) + { + "id": id2, + "name": "myothersample3", + "node_tree": [ + {"id": "root3", "type": "BioReplicate"}, + {"id": "foo3", "type": "TechReplicate", "parent": "root3"}, + ], + }, + 2, + ) # create links # as_user should be ignored unless as_admin is true - lid1 = _create_link(url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/2/2', 'as_user': USER1}) + lid1 = _create_link( + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "foo", "upa": "1/2/2", "as_user": USER1}, + ) lid2 = _create_link( - url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'root', 'upa': '1/1/1', 'dataid': 'column1'}) + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "root", "upa": "1/1/1", "dataid": "column1"}, + ) lid3 = _create_link( - url, TOKEN4, USER4, - {'id': id2, 'version': 1, 'node': 'foo2', 'upa': '1/2/1', 'dataid': 'column2'}) + url, + TOKEN4, + USER4, + {"id": id2, "version": 1, "node": "foo2", "upa": "1/2/1", "dataid": "column2"}, + ) # get links from sample 1 - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id1, 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id1, "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - res = ret.json()['result'][0]['links'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + res = ret.json()["result"][0]["links"] expected_links = [ { - 'linkid': lid1, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/2/2', - 'dataid': None, - 'createdby': USER3, - 'expiredby': None, - 'expired': None - }, + "linkid": lid1, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/2/2", + "dataid": None, + "createdby": USER3, + "expiredby": None, + "expired": None, + }, { - 'linkid': lid2, - 'id': id1, - 'version': 1, - 'node': 'root', - 'upa': '1/1/1', - 'dataid': 'column1', - 'createdby': USER3, - 'expiredby': None, - 'expired': None - } + "linkid": lid2, + "id": id1, + "version": 1, + "node": "root", + "upa": "1/1/1", + "dataid": "column1", + "createdby": USER3, + "expiredby": None, + "expired": None, + }, ] assert len(res) == len(expected_links) for link in res: - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] for link in expected_links: assert link in res # get links from sample 2 - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id2, 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id2, "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - res = ret.json()['result'][0]['links'] - assert_ms_epoch_close_to_now(res[0]['created']) - del res[0]['created'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + res = ret.json()["result"][0]["links"] + assert_ms_epoch_close_to_now(res[0]["created"]) + del res[0]["created"] assert res == [ { - 'linkid': lid3, - 'id': id2, - 'version': 1, - 'node': 'foo2', - 'upa': '1/2/1', - 'dataid': 'column2', - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } + "linkid": lid3, + "id": id2, + "version": 1, + "node": "foo2", + "upa": "1/2/1", + "dataid": "column2", + "createdby": USER4, + "expiredby": None, + "expired": None, + } ] # get links from ver 2 of sample 2 - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id2, 'version': 2}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id2, "version": 2}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - assert ret.json()['result'][0]['links'] == [] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + assert ret.json()["result"][0]["links"] == [] _check_kafka_messages( kafka, [ - {'event_type': 'NEW_SAMPLE', 'sample_id': id1, 'sample_ver': 1}, - {'event_type': 'NEW_SAMPLE', 'sample_id': id2, 'sample_ver': 1}, - {'event_type': 'NEW_SAMPLE', 'sample_id': id2, 'sample_ver': 2}, - {'event_type': 'NEW_LINK', 'link_id': lid1}, - {'event_type': 'NEW_LINK', 'link_id': lid2}, - {'event_type': 'NEW_LINK', 'link_id': lid3}, - ]) + {"event_type": "NEW_SAMPLE", "sample_id": id1, "sample_ver": 1}, + {"event_type": "NEW_SAMPLE", "sample_id": id2, "sample_ver": 1}, + {"event_type": "NEW_SAMPLE", "sample_id": id2, "sample_ver": 2}, + {"event_type": "NEW_LINK", "link_id": lid1}, + {"event_type": "NEW_LINK", "link_id": lid2}, + {"event_type": "NEW_LINK", "link_id": lid3}, + ], + ) def test_update_and_get_links_from_sample(sample_port, workspace, kafka): - ''' + """ Also tests getting links from a sample using an effective time - ''' + """ _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_permissions({'id': 1, 'new_permission': 'w', 'users': [USER4]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_permissions({"id": 1, "new_permission": "w", "users": [USER4]}) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) - _replace_acls(url, id1, TOKEN3, {'admin': [USER4]}) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) + _replace_acls(url, id1, TOKEN3, {"admin": [USER4]}) # create links - lid1 = _create_link(url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}) + lid1 = _create_link( + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + ) oldlinkactive = datetime.datetime.now() time.sleep(1) @@ -2655,459 +3287,564 @@ def test_update_and_get_links_from_sample(sample_port, workspace, kafka): url, TOKEN4, USER4, - {'id': id1, - 'version': 1, - 'node': 'root', - 'upa': '1/1/1', - 'dataid': 'yay', - 'update': 1}) + { + "id": id1, + "version": 1, + "node": "root", + "upa": "1/1/1", + "dataid": "yay", + "update": 1, + }, + ) # get current link - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id1, 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id1, "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - res = ret.json()['result'][0] + assert len(ret.json()["result"]) == 1 + res = ret.json()["result"][0] assert len(res) == 2 - assert_ms_epoch_close_to_now(res['effective_time']) - del res['effective_time'] - created = res['links'][0]['created'] + assert_ms_epoch_close_to_now(res["effective_time"]) + del res["effective_time"] + created = res["links"][0]["created"] assert_ms_epoch_close_to_now(created) - del res['links'][0]['created'] - assert res == {'links': [ - { - 'linkid': lid2, - 'id': id1, - 'version': 1, - 'node': 'root', - 'upa': '1/1/1', - 'dataid': 'yay', - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } - ]} + del res["links"][0]["created"] + assert res == { + "links": [ + { + "linkid": lid2, + "id": id1, + "version": 1, + "node": "root", + "upa": "1/1/1", + "dataid": "yay", + "createdby": USER4, + "expiredby": None, + "expired": None, + } + ] + } # get expired link - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [{ - 'id': id1, - 'version': 1, - 'effective_time': round(oldlinkactive.timestamp() * 1000)}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [ + { + "id": id1, + "version": 1, + "effective_time": round(oldlinkactive.timestamp() * 1000), + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - res = ret.json()['result'][0] - assert res['links'][0]['expired'] == created - 1 - assert_ms_epoch_close_to_now(res['links'][0]['created'] + 1000) - del res['links'][0]['created'] - del res['links'][0]['expired'] + assert len(ret.json()["result"]) == 1 + res = ret.json()["result"][0] + assert res["links"][0]["expired"] == created - 1 + assert_ms_epoch_close_to_now(res["links"][0]["created"] + 1000) + del res["links"][0]["created"] + del res["links"][0]["expired"] assert res == { - 'effective_time': round(oldlinkactive.timestamp() * 1000), - 'links': [ + "effective_time": round(oldlinkactive.timestamp() * 1000), + "links": [ { - 'linkid': lid1, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': 'yay', - 'createdby': USER3, - 'expiredby': USER4, + "linkid": lid1, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": "yay", + "createdby": USER3, + "expiredby": USER4, } - ]} + ], + } _check_kafka_messages( kafka, [ - {'event_type': 'NEW_SAMPLE', 'sample_id': id1, 'sample_ver': 1}, - {'event_type': 'ACL_CHANGE', 'sample_id': id1}, - {'event_type': 'NEW_LINK', 'link_id': lid1}, - {'event_type': 'NEW_LINK', 'link_id': lid2}, - {'event_type': 'EXPIRED_LINK', 'link_id': lid1}, - ]) + {"event_type": "NEW_SAMPLE", "sample_id": id1, "sample_ver": 1}, + {"event_type": "ACL_CHANGE", "sample_id": id1}, + {"event_type": "NEW_LINK", "link_id": lid1}, + {"event_type": "NEW_LINK", "link_id": lid2}, + {"event_type": "EXPIRED_LINK", "link_id": lid1}, + ], + ) def test_create_data_link_as_admin(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) # create links lid1 = _create_link( url, TOKEN2, USER2, - {'id': id1, - 'version': 1, - 'node': 'root', - 'upa': '1/1/1', - 'dataid': 'yeet', - 'as_admin': 1}) + { + "id": id1, + "version": 1, + "node": "root", + "upa": "1/1/1", + "dataid": "yeet", + "as_admin": 1, + }, + ) lid2 = _create_link( url, TOKEN2, USER4, - {'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'as_admin': 1, - 'as_user': f' {USER4} '}) + { + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "as_admin": 1, + "as_user": f" {USER4} ", + }, + ) # get link - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id1, 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id1, "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - res = ret.json()['result'][0]['links'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + res = ret.json()["result"][0]["links"] expected_links = [ { - 'linkid': lid1, - 'id': id1, - 'version': 1, - 'node': 'root', - 'upa': '1/1/1', - 'dataid': 'yeet', - 'createdby': USER2, - 'expiredby': None, - 'expired': None - }, + "linkid": lid1, + "id": id1, + "version": 1, + "node": "root", + "upa": "1/1/1", + "dataid": "yeet", + "createdby": USER2, + "expiredby": None, + "expired": None, + }, { - 'linkid': lid2, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': None, - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } + "linkid": lid2, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": None, + "createdby": USER4, + "expiredby": None, + "expired": None, + }, ] assert len(res) == len(expected_links) for link in res: - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] for link in expected_links: assert link in res def test_get_links_from_sample_exclude_workspaces(sample_port, workspace): - ''' + """ Tests that unreadable workspaces are excluded from link results - ''' - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + """ + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli3 = Workspace(wsurl, token=TOKEN3) wscli4 = Workspace(wsurl, token=TOKEN4) # create workspace & objects - wscli3.create_workspace({'workspace': 'foo'}) - wscli3.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - - wscli4.create_workspace({'workspace': 'bar'}) - wscli4.save_objects({'id': 2, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli4.set_permissions({'id': 2, 'new_permission': 'r', 'users': [USER3]}) - - wscli4.create_workspace({'workspace': 'baz'}) - wscli4.save_objects({'id': 3, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli4.set_global_permission({'id': 3, 'new_permission': 'r'}) - - wscli4.create_workspace({'workspace': 'bat'}) # unreadable - wscli4.save_objects({'id': 4, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli3.create_workspace({"workspace": "foo"}) + wscli3.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + + wscli4.create_workspace({"workspace": "bar"}) + wscli4.save_objects( + { + "id": 2, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli4.set_permissions({"id": 2, "new_permission": "r", "users": [USER3]}) + + wscli4.create_workspace({"workspace": "baz"}) + wscli4.save_objects( + { + "id": 3, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli4.set_global_permission({"id": 3, "new_permission": "r"}) + + wscli4.create_workspace({"workspace": "bat"}) # unreadable + wscli4.save_objects( + { + "id": 4, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) # create sample id_ = _create_generic_sample(url, TOKEN3) - _replace_acls(url, id_, TOKEN3, {'admin': [USER4]}) + _replace_acls(url, id_, TOKEN3, {"admin": [USER4]}) # create links lid1 = _create_link( - url, TOKEN3, USER3, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}) + url, TOKEN3, USER3, {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1"} + ) lid2 = _create_link( - url, TOKEN4, USER4, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '2/1/1'}) - lid3 = _create_link(url, TOKEN4, USER4, - {'id': id_, 'version': 1, 'node': 'foo', 'upa': '3/1/1', 'dataid': 'whee'}) + url, TOKEN4, USER4, {"id": id_, "version": 1, "node": "foo", "upa": "2/1/1"} + ) + lid3 = _create_link( + url, + TOKEN4, + USER4, + {"id": id_, "version": 1, "node": "foo", "upa": "3/1/1", "dataid": "whee"}, + ) _create_link( - url, TOKEN4, USER4, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '4/1/1'}) + url, TOKEN4, USER4, {"id": id_, "version": 1, "node": "foo", "upa": "4/1/1"} + ) # check correct links are returned - ret = _get_links_from_sample(url, TOKEN3, {'id': id_, 'version': 1}) + ret = _get_links_from_sample(url, TOKEN3, {"id": id_, "version": 1}) - assert_ms_epoch_close_to_now(ret['effective_time']) - res = ret['links'] + assert_ms_epoch_close_to_now(ret["effective_time"]) + res = ret["links"] expected_links = [ { - 'linkid': lid1, - 'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': None, - 'createdby': USER3, - 'expiredby': None, - 'expired': None - }, + "linkid": lid1, + "id": id_, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": None, + "createdby": USER3, + "expiredby": None, + "expired": None, + }, { - 'linkid': lid2, - 'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': '2/1/1', - 'dataid': None, - 'createdby': USER4, - 'expiredby': None, - 'expired': None - }, + "linkid": lid2, + "id": id_, + "version": 1, + "node": "foo", + "upa": "2/1/1", + "dataid": None, + "createdby": USER4, + "expiredby": None, + "expired": None, + }, { - 'linkid': lid3, - 'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': '3/1/1', - 'dataid': 'whee', - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } + "linkid": lid3, + "id": id_, + "version": 1, + "node": "foo", + "upa": "3/1/1", + "dataid": "whee", + "createdby": USER4, + "expiredby": None, + "expired": None, + }, ] assert len(res) == len(expected_links) for link in res: - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] for link in expected_links: assert link in res # test with anon user - _replace_acls(url, id_, TOKEN3, {'public_read': 1}) - ret = _get_links_from_sample(url, None, {'id': id_, 'version': 1}) + _replace_acls(url, id_, TOKEN3, {"public_read": 1}) + ret = _get_links_from_sample(url, None, {"id": id_, "version": 1}) - assert_ms_epoch_close_to_now(ret['effective_time']) - res = ret['links'] + assert_ms_epoch_close_to_now(ret["effective_time"]) + res = ret["links"] expected_links = [ { - 'linkid': lid3, - 'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': '3/1/1', - 'dataid': 'whee', - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } + "linkid": lid3, + "id": id_, + "version": 1, + "node": "foo", + "upa": "3/1/1", + "dataid": "whee", + "createdby": USER4, + "expiredby": None, + "expired": None, + } ] assert len(res) == len(expected_links) for link in res: - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] for link in expected_links: assert link in res def _get_links_from_sample(url, token, params, print_resp=False): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) if print_resp: print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - return ret.json()['result'][0] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + return ret.json()["result"][0] def test_get_links_from_sample_as_admin(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN4) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) # create sample id_ = _create_generic_sample(url, TOKEN4) # create links - lid = _create_link(url, TOKEN4, USER4, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}) + lid = _create_link( + url, TOKEN4, USER4, {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1"} + ) # check correct links are returned, user 3 has read admin perms, but not full - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'version': 1, 'as_admin': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "version": 1, "as_admin": 1}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - assert len(ret.json()['result'][0]['links']) == 1 - link = ret.json()['result'][0]['links'][0] - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + assert len(ret.json()["result"][0]["links"]) == 1 + link = ret.json()["result"][0]["links"][0] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] assert link == { - 'linkid': lid, - 'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': None, - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } + "linkid": lid, + "id": id_, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": None, + "createdby": USER4, + "expiredby": None, + "expired": None, + } def test_get_links_from_sample_public_read(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN1) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_global_permission({'id': 1, 'new_permission': 'r'}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_global_permission({"id": 1, "new_permission": "r"}) # create sample id_ = _create_generic_sample(url, TOKEN1) # create links - lid = _create_link(url, TOKEN1, USER1, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}) + lid = _create_link( + url, TOKEN1, USER1, {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1"} + ) - _replace_acls(url, id_, TOKEN1, {'public_read': 1}) + _replace_acls(url, id_, TOKEN1, {"public_read": 1}) for token in [None, TOKEN4]: # anon user & user without explicit permission # check correct links are returned - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [{'id': id_, 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [{"id": id_, "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - assert len(ret.json()['result'][0]['links']) == 1 - link = ret.json()['result'][0]['links'][0] - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + assert len(ret.json()["result"][0]["links"]) == 1 + link = ret.json()["result"][0]["links"][0] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] assert link == { - 'linkid': lid, - 'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': None, - 'createdby': USER1, - 'expiredby': None, - 'expired': None - } + "linkid": lid, + "id": id_, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": None, + "createdby": USER1, + "expiredby": None, + "expired": None, + } + def test_get_links_from_sample_set(sample_port, workspace): """ - test timing for fetching batch of links from list of samples + test timing for fetching batch of links from list of samples """ - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN1) N_SAMPLES = 100 # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'} for _ in range(N_SAMPLES) - ]}) - wscli.set_global_permission({'id': 1, 'new_permission': 'r'}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"} + for _ in range(N_SAMPLES) + ], + } + ) + wscli.set_global_permission({"id": 1, "new_permission": "r"}) ids_ = [_create_generic_sample(url, TOKEN1) for _ in range(N_SAMPLES)] - lids = [_create_link(url, TOKEN1, USER1, { - 'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': f'1/1/{i+1}'}) for i, id_ in enumerate(ids_)] + lids = [ + _create_link( + url, + TOKEN1, + USER1, + {"id": id_, "version": 1, "node": "foo", "upa": f"1/1/{i+1}"}, + ) + for i, id_ in enumerate(ids_) + ] start = time.time() - ret = requests.post(url, headers=get_authorized_headers(TOKEN1), json={ - 'method': 'SampleService.get_data_links_from_sample_set', - 'version': '1.1', - 'id': '42', - 'params': [{ - 'sample_ids': [{'id': id_, 'version': 1} for id_ in ids_], - 'as_admin': False, - 'effective_time': _get_current_epochmillis() - }] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN1), + json={ + "method": "SampleService.get_data_links_from_sample_set", + "version": "1.1", + "id": "42", + "params": [ + { + "sample_ids": [{"id": id_, "version": 1} for id_ in ids_], + "as_admin": False, + "effective_time": _get_current_epochmillis(), + } + ], + }, + ) end = time.time() elapsed = end - start # getting 500 sample links should take about 5 seconds (1 second per 100 samples) @@ -3115,236 +3852,348 @@ def test_get_links_from_sample_set(sample_port, workspace): assert ret.ok # assuming twice the amound of expected time elasped should raise concern assert elapsed < 10 - assert len(ret.json()['result'][0]['links']) == N_SAMPLES + assert len(ret.json()["result"][0]["links"]) == N_SAMPLES + def test_create_link_fail(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) id_ = _create_generic_sample(url, TOKEN3) _create_link_fail( - sample_port, TOKEN3, {'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}, - 'Sample service error code 30000 Missing input parameter: id') + sample_port, + TOKEN3, + {"version": 1, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + "Sample service error code 30000 Missing input parameter: id", + ) _create_link_fail( - sample_port, TOKEN3, {'id': id_, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}, - 'Sample service error code 30000 Missing input parameter: version') + sample_port, + TOKEN3, + {"id": id_, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + "Sample service error code 30000 Missing input parameter: version", + ) _create_link_fail( - sample_port, TOKEN3, - {'id': id_, 'version': 1, 'node': 'foo', 'upa': 'upalupa', 'dataid': 'yay'}, - 'Sample service error code 30001 Illegal input parameter: upalupa is not a valid UPA') + sample_port, + TOKEN3, + {"id": id_, "version": 1, "node": "foo", "upa": "upalupa", "dataid": "yay"}, + "Sample service error code 30001 Illegal input parameter: upalupa is not a valid UPA", + ) _create_link_fail( - sample_port, TOKEN3, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}, - 'Sample service error code 50040 No such workspace data: No workspace with id 1 exists') + sample_port, + TOKEN3, + {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1"}, + "Sample service error code 50040 No such workspace data: No workspace with id 1 exists", + ) - wscli.create_workspace({'workspace': 'foo'}) + wscli.create_workspace({"workspace": "foo"}) _create_link_fail( - sample_port, TOKEN3, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}, - 'Sample service error code 50040 No such workspace data: Object 1/1/1 does not exist') + sample_port, + TOKEN3, + {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1"}, + "Sample service error code 50040 No such workspace data: Object 1/1/1 does not exist", + ) - _replace_acls(url, id_, TOKEN3, {'write': [USER4]}) + _replace_acls(url, id_, TOKEN3, {"write": [USER4]}) _create_link_fail( # fails if permission granted is admin - sample_port, TOKEN4, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}, - 'Sample service error code 20000 Unauthorized: User user4 cannot ' + - f'administrate sample {id_}') + sample_port, + TOKEN4, + {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1"}, + "Sample service error code 20000 Unauthorized: User user4 cannot " + + f"administrate sample {id_}", + ) - _replace_acls(url, id_, TOKEN3, {'admin': [USER4]}) - wscli.set_permissions({'id': 1, 'new_permission': 'r', 'users': [USER4]}) + _replace_acls(url, id_, TOKEN3, {"admin": [USER4]}) + wscli.set_permissions({"id": 1, "new_permission": "r", "users": [USER4]}) _create_link_fail( # fails if permission granted is write - sample_port, TOKEN4, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}, - 'Sample service error code 20000 Unauthorized: User user4 cannot write to upa 1/1/1') + sample_port, + TOKEN4, + {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1"}, + "Sample service error code 20000 Unauthorized: User user4 cannot write to upa 1/1/1", + ) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) _create_link_fail( - sample_port, TOKEN3, {'id': id_, 'version': 1, 'node': 'fake', 'upa': '1/1/1'}, - f'Sample service error code 50030 No such sample node: {id_} ver 1 fake') + sample_port, + TOKEN3, + {"id": id_, "version": 1, "node": "fake", "upa": "1/1/1"}, + f"Sample service error code 50030 No such sample node: {id_} ver 1 fake", + ) # admin tests _create_link_fail( - sample_port, TOKEN2, - {'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'as_admin': 1, - 'as_user': 'foo\bbar'}, - 'Sample service error code 30001 Illegal input parameter: ' + - 'userid contains control characters') + sample_port, + TOKEN2, + { + "id": id_, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "as_admin": 1, + "as_user": "foo\bbar", + }, + "Sample service error code 30001 Illegal input parameter: " + + "userid contains control characters", + ) _create_link_fail( - sample_port, TOKEN3, - {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'as_user': USER4, 'as_admin': 'f'}, - 'Sample service error code 20000 Unauthorized: User user3 does not have ' + - 'the necessary administration privileges to run method create_data_link') + sample_port, + TOKEN3, + { + "id": id_, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "as_user": USER4, + "as_admin": "f", + }, + "Sample service error code 20000 Unauthorized: User user3 does not have " + + "the necessary administration privileges to run method create_data_link", + ) _create_link_fail( sample_port, TOKEN2, - {'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'as_user': 'fake', - 'as_admin': 'f'}, - 'Sample service error code 50000 No such user: fake') + { + "id": id_, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "as_user": "fake", + "as_admin": "f", + }, + "Sample service error code 50000 No such user: fake", + ) def test_create_link_fail_link_exists(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) id_ = _create_generic_sample(url, TOKEN3) - _create_link(url, TOKEN3, USER3, - {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}) + _create_link( + url, + TOKEN3, + USER3, + {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + ) _create_link_fail( - sample_port, TOKEN3, - {'id': id_, 'version': 1, 'node': 'root', 'upa': '1/1/1', 'dataid': 'yay'}, - 'Sample service error code 60000 Data link exists for data ID: 1/1/1:yay') + sample_port, + TOKEN3, + {"id": id_, "version": 1, "node": "root", "upa": "1/1/1", "dataid": "yay"}, + "Sample service error code 60000 Data link exists for data ID: 1/1/1:yay", + ) def _create_link_fail(sample_port, token, params, expected): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.create_data_link', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.create_data_link", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def test_get_links_from_sample_fail(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN3) _get_link_from_sample_fail( - sample_port, TOKEN3, {}, - 'Sample service error code 30000 Missing input parameter: id') + sample_port, + TOKEN3, + {}, + "Sample service error code 30000 Missing input parameter: id", + ) _get_link_from_sample_fail( - sample_port, TOKEN3, {'id': id_}, - 'Sample service error code 30000 Missing input parameter: version') + sample_port, + TOKEN3, + {"id": id_}, + "Sample service error code 30000 Missing input parameter: version", + ) _get_link_from_sample_fail( - sample_port, TOKEN3, {'id': id_, 'version': 1, 'effective_time': 'foo'}, - "Sample service error code 30001 Illegal input parameter: key 'effective_time' " + - "value of 'foo' is not a valid epoch millisecond timestamp") + sample_port, + TOKEN3, + {"id": id_, "version": 1, "effective_time": "foo"}, + "Sample service error code 30001 Illegal input parameter: key 'effective_time' " + + "value of 'foo' is not a valid epoch millisecond timestamp", + ) _get_link_from_sample_fail( - sample_port, TOKEN4, {'id': id_, 'version': 1}, - f'Sample service error code 20000 Unauthorized: User user4 cannot read sample {id_}') + sample_port, + TOKEN4, + {"id": id_, "version": 1}, + f"Sample service error code 20000 Unauthorized: User user4 cannot read sample {id_}", + ) _get_link_from_sample_fail( - sample_port, None, {'id': id_, 'version': 1}, - f'Sample service error code 20000 Unauthorized: Anonymous users cannot read sample {id_}') + sample_port, + None, + {"id": id_, "version": 1}, + f"Sample service error code 20000 Unauthorized: Anonymous users cannot read sample {id_}", + ) badid = uuid.uuid4() _get_link_from_sample_fail( - sample_port, TOKEN3, {'id': str(badid), 'version': 1}, - f'Sample service error code 50010 No such sample: {badid}') + sample_port, + TOKEN3, + {"id": str(badid), "version": 1}, + f"Sample service error code 50010 No such sample: {badid}", + ) # admin tests _get_link_from_sample_fail( - sample_port, TOKEN4, {'id': id_, 'version': 1, 'as_admin': 1}, - 'Sample service error code 20000 Unauthorized: User user4 does not have the ' + - 'necessary administration privileges to run method get_data_links_from_sample') + sample_port, + TOKEN4, + {"id": id_, "version": 1, "as_admin": 1}, + "Sample service error code 20000 Unauthorized: User user4 does not have the " + + "necessary administration privileges to run method get_data_links_from_sample", + ) _get_link_from_sample_fail( - sample_port, None, {'id': id_, 'version': 1, 'as_admin': 1}, - 'Sample service error code 20000 Unauthorized: Anonymous users ' + - 'may not act as service administrators.') + sample_port, + None, + {"id": id_, "version": 1, "as_admin": 1}, + "Sample service error code 20000 Unauthorized: Anonymous users " + + "may not act as service administrators.", + ) def _get_link_from_sample_fail(sample_port, token, params, expected): - url = f'http://localhost:{sample_port}' - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_data_links_from_sample', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + url = f"http://localhost:{sample_port}" + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_data_links_from_sample", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def test_get_links_from_sample_set_fail(sample_port): - url = f'http://localhost:{sample_port}' + url = f"http://localhost:{sample_port}" id_ = _create_generic_sample(url, TOKEN3) _get_links_from_sample_set_fail( - sample_port, TOKEN3, {}, - 'Missing "sample_ids" field - Must provide a list of valid sample ids.') + sample_port, + TOKEN3, + {}, + 'Missing "sample_ids" field - Must provide a list of valid sample ids.', + ) _get_links_from_sample_set_fail( - sample_port, TOKEN3, { - 'sample_ids': [{'id': id_}] - }, - "Malformed sample accessor - each sample must provide both an id and a version.") + sample_port, + TOKEN3, + {"sample_ids": [{"id": id_}]}, + "Malformed sample accessor - each sample must provide both an id and a version.", + ) _get_links_from_sample_set_fail( - sample_port, TOKEN3, { - 'sample_ids': [{'id': id_, 'version': 1}] - }, - 'Missing "effective_time" parameter.') + sample_port, + TOKEN3, + {"sample_ids": [{"id": id_, "version": 1}]}, + 'Missing "effective_time" parameter.', + ) _get_links_from_sample_set_fail( - sample_port, TOKEN3, { - 'sample_ids': [{'id': id_, 'version': 1}], - 'effective_time': 'foo' - }, - "Sample service error code 30001 Illegal input parameter: key 'effective_time' " + - "value of 'foo' is not a valid epoch millisecond timestamp") + sample_port, + TOKEN3, + {"sample_ids": [{"id": id_, "version": 1}], "effective_time": "foo"}, + "Sample service error code 30001 Illegal input parameter: key 'effective_time' " + + "value of 'foo' is not a valid epoch millisecond timestamp", + ) _get_links_from_sample_set_fail( - sample_port, TOKEN4, { - 'sample_ids': [{'id': id_, 'version': 1}], - 'effective_time': _get_current_epochmillis() - 500 + sample_port, + TOKEN4, + { + "sample_ids": [{"id": id_, "version": 1}], + "effective_time": _get_current_epochmillis() - 500, }, - f'Sample service error code 20000 Unauthorized: User user4 cannot read sample {id_}') + f"Sample service error code 20000 Unauthorized: User user4 cannot read sample {id_}", + ) _get_links_from_sample_set_fail( - sample_port, None, { - 'sample_ids': [{'id': id_, 'version': 1}], - 'effective_time': _get_current_epochmillis() - 500 + sample_port, + None, + { + "sample_ids": [{"id": id_, "version": 1}], + "effective_time": _get_current_epochmillis() - 500, }, - f'Sample service error code 20000 Unauthorized: Anonymous users cannot read sample {id_}') + f"Sample service error code 20000 Unauthorized: Anonymous users cannot read sample {id_}", + ) badid = uuid.uuid4() _get_links_from_sample_set_fail( - sample_port, TOKEN3, { - 'sample_ids': [{'id': str(badid), 'version': 1}], - 'effective_time': _get_current_epochmillis() - 500 + sample_port, + TOKEN3, + { + "sample_ids": [{"id": str(badid), "version": 1}], + "effective_time": _get_current_epochmillis() - 500, }, - f'Sample service error code 50010 No such sample: {badid}') + f"Sample service error code 50010 No such sample: {badid}", + ) # admin tests _get_links_from_sample_set_fail( - sample_port, TOKEN4, { - 'sample_ids': [{'id': id_, 'version': 1}], - 'effective_time': _get_current_epochmillis() - 500, - 'as_admin': 1, + sample_port, + TOKEN4, + { + "sample_ids": [{"id": id_, "version": 1}], + "effective_time": _get_current_epochmillis() - 500, + "as_admin": 1, }, - 'Sample service error code 20000 Unauthorized: User user4 does not have the ' + - 'necessary administration privileges to run method get_data_links_from_sample') + "Sample service error code 20000 Unauthorized: User user4 does not have the " + + "necessary administration privileges to run method get_data_links_from_sample", + ) _get_links_from_sample_set_fail( - sample_port, None, { - 'sample_ids': [{'id': id_, 'version': 1}], - 'effective_time': _get_current_epochmillis() - 500, - 'as_admin': 1 + sample_port, + None, + { + "sample_ids": [{"id": id_, "version": 1}], + "effective_time": _get_current_epochmillis() - 500, + "as_admin": 1, }, - 'Sample service error code 20000 Unauthorized: Anonymous users ' + - 'may not act as service administrators.') + "Sample service error code 20000 Unauthorized: Anonymous users " + + "may not act as service administrators.", + ) def _get_links_from_sample_set_fail(sample_port, token, params, expected): - url = f'http://localhost:{sample_port}' - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_data_links_from_sample_set', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + url = f"http://localhost:{sample_port}" + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_data_links_from_sample_set", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def _get_current_epochmillis(): @@ -3356,116 +4205,142 @@ def test_expire_data_link(sample_port, workspace, kafka): def test_expire_data_link_with_data_id(sample_port, workspace, kafka): - _expire_data_link(sample_port, workspace, 'whee', kafka) + _expire_data_link(sample_port, workspace, "whee", kafka) def _expire_data_link(sample_port, workspace, dataid, kafka): - ''' also tests that 'as_user' is ignored if 'as_admin' is false ''' + """also tests that 'as_user' is ignored if 'as_admin' is false""" _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_permissions({'id': 1, 'new_permission': 'w', 'users': [USER4]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_permissions({"id": 1, "new_permission": "w", "users": [USER4]}) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'}, - {'id': 'bar', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) - _replace_acls(url, id1, TOKEN3, {'admin': [USER4]}) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + {"id": "bar", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) + _replace_acls(url, id1, TOKEN3, {"admin": [USER4]}) # create links - lid1 = _create_link(url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': dataid}) - lid2 = _create_link(url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'bar', 'upa': '1/1/1', 'dataid': 'fake'}) + lid1 = _create_link( + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": dataid}, + ) + lid2 = _create_link( + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "bar", "upa": "1/1/1", "dataid": "fake"}, + ) time.sleep(1) # need to be able to set a resonable effective time to fetch links # expire link - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.expire_data_link', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'dataid': dataid, 'as_user': USER1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.expire_data_link", + "version": "1.1", + "id": "42", + "params": [{"upa": "1/1/1", "dataid": dataid, "as_user": USER1}], + }, + ) # print(ret.text) assert ret.ok is True # check links - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.get_data_links_from_data', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'effective_time': _get_current_epochmillis() - 500}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.get_data_links_from_data", + "version": "1.1", + "id": "42", + "params": [ + {"upa": "1/1/1", "effective_time": _get_current_epochmillis() - 500} + ], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - links = ret.json()['result'][0]['links'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + links = ret.json()["result"][0]["links"] assert len(links) == 2 for link in links: - if link['dataid'] == 'fake': + if link["dataid"] == "fake": current_link = link else: expired_link = link - assert_ms_epoch_close_to_now(expired_link['expired']) - assert_ms_epoch_close_to_now(expired_link['created'] + 1000) - del expired_link['created'] - del expired_link['expired'] + assert_ms_epoch_close_to_now(expired_link["expired"]) + assert_ms_epoch_close_to_now(expired_link["created"] + 1000) + del expired_link["created"] + del expired_link["expired"] assert expired_link == { - 'linkid': lid1, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': dataid, - 'createdby': USER3, - 'expiredby': USER4, - } - - assert_ms_epoch_close_to_now(current_link['created'] + 1000) - del current_link['created'] + "linkid": lid1, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": dataid, + "createdby": USER3, + "expiredby": USER4, + } + + assert_ms_epoch_close_to_now(current_link["created"] + 1000) + del current_link["created"] assert current_link == { - 'linkid': lid2, - 'id': id1, - 'version': 1, - 'node': 'bar', - 'upa': '1/1/1', - 'dataid': 'fake', - 'createdby': USER3, - 'expiredby': None, - 'expired': None - } + "linkid": lid2, + "id": id1, + "version": 1, + "node": "bar", + "upa": "1/1/1", + "dataid": "fake", + "createdby": USER3, + "expiredby": None, + "expired": None, + } _check_kafka_messages( kafka, [ - {'event_type': 'NEW_SAMPLE', 'sample_id': id1, 'sample_ver': 1}, - {'event_type': 'ACL_CHANGE', 'sample_id': id1}, - {'event_type': 'NEW_LINK', 'link_id': lid1}, - {'event_type': 'NEW_LINK', 'link_id': lid2}, - {'event_type': 'EXPIRED_LINK', 'link_id': lid1}, - ]) + {"event_type": "NEW_SAMPLE", "sample_id": id1, "sample_ver": 1}, + {"event_type": "ACL_CHANGE", "sample_id": id1}, + {"event_type": "NEW_LINK", "link_id": lid1}, + {"event_type": "NEW_LINK", "link_id": lid2}, + {"event_type": "EXPIRED_LINK", "link_id": lid1}, + ], + ) def test_expire_data_link_as_admin(sample_port, workspace, kafka): @@ -3479,1111 +4354,1463 @@ def test_expire_data_link_as_admin_impersonate_user(sample_port, workspace, kafk def _expire_data_link_as_admin(sample_port, workspace, user, expected_user, kafka): _clear_kafka_messages(kafka) - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_permissions({'id': 1, 'new_permission': 'w', 'users': [USER4]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_permissions({"id": 1, "new_permission": "w", "users": [USER4]}) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'}, - {'id': 'bar', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + {"id": "bar", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) # create links - lid = _create_link(url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'duidy'}) + lid = _create_link( + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": "duidy"}, + ) time.sleep(1) # need to be able to set a resonable effective time to fetch links # expire link - ret = requests.post(url, headers=get_authorized_headers(TOKEN2), json={ - 'method': 'SampleService.expire_data_link', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'dataid': 'duidy', 'as_admin': 1, 'as_user': user}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN2), + json={ + "method": "SampleService.expire_data_link", + "version": "1.1", + "id": "42", + "params": [ + {"upa": "1/1/1", "dataid": "duidy", "as_admin": 1, "as_user": user} + ], + }, + ) # print(ret.text) assert ret.ok is True # check links - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.get_data_links_from_data', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'effective_time': _get_current_epochmillis() - 500}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.get_data_links_from_data", + "version": "1.1", + "id": "42", + "params": [ + {"upa": "1/1/1", "effective_time": _get_current_epochmillis() - 500} + ], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - links = ret.json()['result'][0]['links'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + links = ret.json()["result"][0]["links"] assert len(links) == 1 link = links[0] - assert_ms_epoch_close_to_now(link['expired']) - assert_ms_epoch_close_to_now(link['created'] + 1000) - del link['created'] - del link['expired'] + assert_ms_epoch_close_to_now(link["expired"]) + assert_ms_epoch_close_to_now(link["created"] + 1000) + del link["created"] + del link["expired"] assert link == { - 'linkid': lid, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': 'duidy', - 'createdby': USER3, - 'expiredby': expected_user, - } + "linkid": lid, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": "duidy", + "createdby": USER3, + "expiredby": expected_user, + } _check_kafka_messages( kafka, [ - {'event_type': 'NEW_SAMPLE', 'sample_id': id1, 'sample_ver': 1}, - {'event_type': 'NEW_LINK', 'link_id': lid}, - {'event_type': 'EXPIRED_LINK', 'link_id': lid}, - ]) + {"event_type": "NEW_SAMPLE", "sample_id": id1, "sample_ver": 1}, + {"event_type": "NEW_LINK", "link_id": lid}, + {"event_type": "EXPIRED_LINK", "link_id": lid}, + ], + ) def test_expire_data_link_fail(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) # create links - _create_link(url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}) + _create_link( + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + ) _expire_data_link_fail( - sample_port, TOKEN3, {}, 'Sample service error code 30000 Missing input parameter: upa') + sample_port, + TOKEN3, + {}, + "Sample service error code 30000 Missing input parameter: upa", + ) _expire_data_link_fail( - sample_port, TOKEN3, {'upa': '1/0/1'}, - 'Sample service error code 30001 Illegal input parameter: 1/0/1 is not a valid UPA') + sample_port, + TOKEN3, + {"upa": "1/0/1"}, + "Sample service error code 30001 Illegal input parameter: 1/0/1 is not a valid UPA", + ) _expire_data_link_fail( - sample_port, TOKEN3, {'upa': '1/1/1', 'dataid': 'foo\nbar'}, - 'Sample service error code 30001 Illegal input parameter: ' + - 'dataid contains control characters') + sample_port, + TOKEN3, + {"upa": "1/1/1", "dataid": "foo\nbar"}, + "Sample service error code 30001 Illegal input parameter: " + + "dataid contains control characters", + ) _expire_data_link_fail( - sample_port, TOKEN4, {'upa': '1/1/1', 'dataid': 'yay'}, - 'Sample service error code 20000 Unauthorized: User user4 cannot write to workspace 1') + sample_port, + TOKEN4, + {"upa": "1/1/1", "dataid": "yay"}, + "Sample service error code 20000 Unauthorized: User user4 cannot write to workspace 1", + ) - wscli.delete_workspace({'id': 1}) + wscli.delete_workspace({"id": 1}) _expire_data_link_fail( - sample_port, TOKEN3, {'upa': '1/1/1', 'dataid': 'yay'}, - 'Sample service error code 50040 No such workspace data: Workspace 1 is deleted') + sample_port, + TOKEN3, + {"upa": "1/1/1", "dataid": "yay"}, + "Sample service error code 50040 No such workspace data: Workspace 1 is deleted", + ) wsadmin = Workspace(wsurl, token=TOKEN_WS_FULL_ADMIN) - wsadmin.administer({'command': 'undeleteWorkspace', 'params': {'id': 1}}) + wsadmin.administer({"command": "undeleteWorkspace", "params": {"id": 1}}) _expire_data_link_fail( - sample_port, TOKEN3, {'upa': '1/1/2', 'dataid': 'yay'}, - 'Sample service error code 50050 No such data link: 1/1/2:yay') + sample_port, + TOKEN3, + {"upa": "1/1/2", "dataid": "yay"}, + "Sample service error code 50050 No such data link: 1/1/2:yay", + ) _expire_data_link_fail( - sample_port, TOKEN3, {'upa': '1/1/1', 'dataid': 'yee'}, - 'Sample service error code 50050 No such data link: 1/1/1:yee') + sample_port, + TOKEN3, + {"upa": "1/1/1", "dataid": "yee"}, + "Sample service error code 50050 No such data link: 1/1/1:yee", + ) - wscli.set_permissions({'id': 1, 'new_permission': 'w', 'users': [USER4]}) + wscli.set_permissions({"id": 1, "new_permission": "w", "users": [USER4]}) _expire_data_link_fail( - sample_port, TOKEN4, {'upa': '1/1/1', 'dataid': 'yay'}, - 'Sample service error code 20000 Unauthorized: User user4 cannot ' + - f'administrate sample {id1}') + sample_port, + TOKEN4, + {"upa": "1/1/1", "dataid": "yay"}, + "Sample service error code 20000 Unauthorized: User user4 cannot " + + f"administrate sample {id1}", + ) # admin tests _expire_data_link_fail( - sample_port, TOKEN2, - {'upa': '1/1/1', 'dataid': 'yay', 'as_admin': ['t'], 'as_user': 'foo\tbar'}, - 'Sample service error code 30001 Illegal input parameter: ' + - 'userid contains control characters') + sample_port, + TOKEN2, + {"upa": "1/1/1", "dataid": "yay", "as_admin": ["t"], "as_user": "foo\tbar"}, + "Sample service error code 30001 Illegal input parameter: " + + "userid contains control characters", + ) _expire_data_link_fail( - sample_port, TOKEN3, - {'upa': '1/1/1', 'dataid': 'yay', 'as_admin': ['t'], 'as_user': USER4}, - 'Sample service error code 20000 Unauthorized: User user3 does not have ' + - 'the necessary administration privileges to run method expire_data_link') + sample_port, + TOKEN3, + {"upa": "1/1/1", "dataid": "yay", "as_admin": ["t"], "as_user": USER4}, + "Sample service error code 20000 Unauthorized: User user3 does not have " + + "the necessary administration privileges to run method expire_data_link", + ) _expire_data_link_fail( - sample_port, TOKEN2, - {'upa': '1/1/1', 'dataid': 'yay', 'as_admin': ['t'], 'as_user': 'fake'}, - 'Sample service error code 50000 No such user: fake') + sample_port, + TOKEN2, + {"upa": "1/1/1", "dataid": "yay", "as_admin": ["t"], "as_user": "fake"}, + "Sample service error code 50000 No such user: fake", + ) def _expire_data_link_fail(sample_port, token, params, expected): - _request_fail(sample_port, 'expire_data_link', token, params, expected) + _request_fail(sample_port, "expire_data_link", token, params, expected) def _request_fail(sample_port, method, token, params, expected): - url = f'http://localhost:{sample_port}' - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.' + method, - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + url = f"http://localhost:{sample_port}" + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService." + method, + "version": "1.1", + "id": "42", + "params": [params], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def test_get_links_from_data(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - {'name': 'baz', 'data': {}, 'type': 'Trivial.Object-1.0'}, - {'name': 'baz', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_permissions({'id': 1, 'new_permission': 'w', 'users': [USER4]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + {"name": "baz", "data": {}, "type": "Trivial.Object-1.0"}, + {"name": "baz", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_permissions({"id": 1, "new_permission": "w", "users": [USER4]}) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) id2 = _create_sample( url, TOKEN4, - {'name': 'myothersample', - 'node_tree': [{'id': 'root2', 'type': 'BioReplicate'}, - {'id': 'foo2', 'type': 'TechReplicate', 'parent': 'root2'} - ] - }, - 1 - ) + { + "name": "myothersample", + "node_tree": [ + {"id": "root2", "type": "BioReplicate"}, + {"id": "foo2", "type": "TechReplicate", "parent": "root2"}, + ], + }, + 1, + ) # ver 2 _create_sample( url, TOKEN4, - {'id': id2, - 'name': 'myothersample3', - 'node_tree': [{'id': 'root3', 'type': 'BioReplicate'}, - {'id': 'foo3', 'type': 'TechReplicate', 'parent': 'root3'} - ] - }, - 2 - ) + { + "id": id2, + "name": "myothersample3", + "node_tree": [ + {"id": "root3", "type": "BioReplicate"}, + {"id": "foo3", "type": "TechReplicate", "parent": "root3"}, + ], + }, + 2, + ) # create links lid1 = _create_link( - url, TOKEN3, USER3, {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/2/2'}) + url, TOKEN3, USER3, {"id": id1, "version": 1, "node": "foo", "upa": "1/2/2"} + ) lid2 = _create_link( - url, TOKEN4, USER4, - {'id': id2, 'version': 1, 'node': 'root2', 'upa': '1/1/1', 'dataid': 'column1'}) + url, + TOKEN4, + USER4, + {"id": id2, "version": 1, "node": "root2", "upa": "1/1/1", "dataid": "column1"}, + ) lid3 = _create_link( - url, TOKEN4, USER4, - {'id': id2, 'version': 2, 'node': 'foo3', 'upa': '1/2/2', 'dataid': 'column2'}) + url, + TOKEN4, + USER4, + {"id": id2, "version": 2, "node": "foo3", "upa": "1/2/2", "dataid": "column2"}, + ) # get links from object 1/2/2 - ret = _get_links_from_data(url, TOKEN3, {'upa': '1/2/2'}) + ret = _get_links_from_data(url, TOKEN3, {"upa": "1/2/2"}) - assert_ms_epoch_close_to_now(ret['effective_time']) - res = ret['links'] + assert_ms_epoch_close_to_now(ret["effective_time"]) + res = ret["links"] expected_links = [ { - 'linkid': lid1, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/2/2', - 'dataid': None, - 'createdby': USER3, - 'expiredby': None, - 'expired': None - }, + "linkid": lid1, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/2/2", + "dataid": None, + "createdby": USER3, + "expiredby": None, + "expired": None, + }, { - 'linkid': lid3, - 'id': id2, - 'version': 2, - 'node': 'foo3', - 'upa': '1/2/2', - 'dataid': 'column2', - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } + "linkid": lid3, + "id": id2, + "version": 2, + "node": "foo3", + "upa": "1/2/2", + "dataid": "column2", + "createdby": USER4, + "expiredby": None, + "expired": None, + }, ] assert len(res) == len(expected_links) for link in res: - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] for link in expected_links: assert link in res # get links from object 1/1/1 - ret = _get_links_from_data(url, TOKEN3, {'upa': '1/1/1'}) + ret = _get_links_from_data(url, TOKEN3, {"upa": "1/1/1"}) - assert_ms_epoch_close_to_now(ret['effective_time']) - res = ret['links'] - assert_ms_epoch_close_to_now(res[0]['created']) - del res[0]['created'] + assert_ms_epoch_close_to_now(ret["effective_time"]) + res = ret["links"] + assert_ms_epoch_close_to_now(res[0]["created"]) + del res[0]["created"] assert res == [ { - 'linkid': lid2, - 'id': id2, - 'version': 1, - 'node': 'root2', - 'upa': '1/1/1', - 'dataid': 'column1', - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } + "linkid": lid2, + "id": id2, + "version": 1, + "node": "root2", + "upa": "1/1/1", + "dataid": "column1", + "createdby": USER4, + "expiredby": None, + "expired": None, + } ] # get links from object 1/2/1 - ret = _get_links_from_data(url, TOKEN3, {'upa': '1/2/1'}) + ret = _get_links_from_data(url, TOKEN3, {"upa": "1/2/1"}) - assert_ms_epoch_close_to_now(ret['effective_time']) - assert ret['links'] == [] + assert_ms_epoch_close_to_now(ret["effective_time"]) + assert ret["links"] == [] def _get_links_from_data(url, token, params, print_resp=False): - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_data_links_from_data', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_data_links_from_data", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) if print_resp: print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - return ret.json()['result'][0] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + return ret.json()["result"][0] def test_get_links_from_data_expired(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_permissions({'id': 1, 'new_permission': 'w', 'users': [USER4]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_permissions({"id": 1, "new_permission": "w", "users": [USER4]}) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) - _replace_acls(url, id1, TOKEN3, {'admin': [USER4]}) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) + _replace_acls(url, id1, TOKEN3, {"admin": [USER4]}) # create links - lid1 = _create_link(url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}) + lid1 = _create_link( + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + ) oldlinkactive = datetime.datetime.now() time.sleep(1) # update link node - lid2 = _create_link(url, TOKEN4, USER4, { - 'id': id1, - 'version': 1, - 'node': 'root', - 'upa': '1/1/1', - 'dataid': 'yay', - 'update': 1 - }) + lid2 = _create_link( + url, + TOKEN4, + USER4, + { + "id": id1, + "version": 1, + "node": "root", + "upa": "1/1/1", + "dataid": "yay", + "update": 1, + }, + ) # get current link - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_data_links_from_data', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1'}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_data_links_from_data", + "version": "1.1", + "id": "42", + "params": [{"upa": "1/1/1"}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - res = ret.json()['result'][0] + assert len(ret.json()["result"]) == 1 + res = ret.json()["result"][0] assert len(res) == 2 - assert_ms_epoch_close_to_now(res['effective_time']) - del res['effective_time'] - created = res['links'][0]['created'] + assert_ms_epoch_close_to_now(res["effective_time"]) + del res["effective_time"] + created = res["links"][0]["created"] assert_ms_epoch_close_to_now(created) - del res['links'][0]['created'] - assert res == {'links': [ - { - 'linkid': lid2, - 'id': id1, - 'version': 1, - 'node': 'root', - 'upa': '1/1/1', - 'dataid': 'yay', - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } - ]} + del res["links"][0]["created"] + assert res == { + "links": [ + { + "linkid": lid2, + "id": id1, + "version": 1, + "node": "root", + "upa": "1/1/1", + "dataid": "yay", + "createdby": USER4, + "expiredby": None, + "expired": None, + } + ] + } # get expired link - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_data_links_from_data', - 'version': '1.1', - 'id': '42', - 'params': [{ - 'upa': '1/1/1', - 'effective_time': round(oldlinkactive.timestamp() * 1000)}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_data_links_from_data", + "version": "1.1", + "id": "42", + "params": [ + { + "upa": "1/1/1", + "effective_time": round(oldlinkactive.timestamp() * 1000), + } + ], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - res = ret.json()['result'][0] - assert res['links'][0]['expired'] == created - 1 - assert_ms_epoch_close_to_now(res['links'][0]['created'] + 1000) - del res['links'][0]['created'] - del res['links'][0]['expired'] + assert len(ret.json()["result"]) == 1 + res = ret.json()["result"][0] + assert res["links"][0]["expired"] == created - 1 + assert_ms_epoch_close_to_now(res["links"][0]["created"] + 1000) + del res["links"][0]["created"] + del res["links"][0]["expired"] assert res == { - 'effective_time': round(oldlinkactive.timestamp() * 1000), - 'links': [ + "effective_time": round(oldlinkactive.timestamp() * 1000), + "links": [ { - 'linkid': lid1, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': 'yay', - 'createdby': USER3, - 'expiredby': USER4, + "linkid": lid1, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": "yay", + "createdby": USER3, + "expiredby": USER4, } - ]} + ], + } def test_get_links_from_data_public_read(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN1) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_global_permission({'id': 1, 'new_permission': 'r'}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_global_permission({"id": 1, "new_permission": "r"}) # create samples id_ = _create_generic_sample(url, TOKEN1) # create links - lid = _create_link(url, TOKEN1, USER1, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}) + lid = _create_link( + url, TOKEN1, USER1, {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1"} + ) for token in [None, TOKEN4]: # anon user, user 4 has no explicit perms - ret = _get_links_from_data(url, token, {'upa': '1/1/1'}) + ret = _get_links_from_data(url, token, {"upa": "1/1/1"}) - assert_ms_epoch_close_to_now(ret['effective_time']) - assert len(ret['links']) == 1 - link = ret['links'][0] - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert_ms_epoch_close_to_now(ret["effective_time"]) + assert len(ret["links"]) == 1 + link = ret["links"][0] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] assert link == { - 'linkid': lid, - 'id': id_, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': None, - 'createdby': USER1, - 'expiredby': None, - 'expired': None - } + "linkid": lid, + "id": id_, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": None, + "createdby": USER1, + "expiredby": None, + "expired": None, + } def test_get_links_from_data_as_admin(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN4) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) # create samples id1 = _create_sample( url, TOKEN4, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) # create links - lid = _create_link(url, TOKEN4, USER4, {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}) + lid = _create_link( + url, TOKEN4, USER4, {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1"} + ) # get links from object, user 3 has admin read perms - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_data_links_from_data', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'as_admin': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_data_links_from_data", + "version": "1.1", + "id": "42", + "params": [{"upa": "1/1/1", "as_admin": 1}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - assert len(ret.json()['result'][0]) == 2 - assert_ms_epoch_close_to_now(ret.json()['result'][0]['effective_time']) - assert len(ret.json()['result'][0]['links']) == 1 - link = ret.json()['result'][0]['links'][0] - assert_ms_epoch_close_to_now(link['created']) - del link['created'] + assert len(ret.json()["result"]) == 1 + assert len(ret.json()["result"][0]) == 2 + assert_ms_epoch_close_to_now(ret.json()["result"][0]["effective_time"]) + assert len(ret.json()["result"][0]["links"]) == 1 + link = ret.json()["result"][0]["links"][0] + assert_ms_epoch_close_to_now(link["created"]) + del link["created"] assert link == { - 'linkid': lid, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': None, - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } + "linkid": lid, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": None, + "createdby": USER4, + "expiredby": None, + "expired": None, + } def test_get_links_from_data_fail(sample_port, workspace): - wsurl = f'http://localhost:{workspace.port}' + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) _get_link_from_data_fail( - sample_port, TOKEN3, {}, - 'Sample service error code 30000 Missing input parameter: upa') + sample_port, + TOKEN3, + {}, + "Sample service error code 30000 Missing input parameter: upa", + ) _get_link_from_data_fail( - sample_port, TOKEN3, {'upa': '1/1/1', 'effective_time': 'foo'}, - "Sample service error code 30001 Illegal input parameter: key 'effective_time' " + - "value of 'foo' is not a valid epoch millisecond timestamp") + sample_port, + TOKEN3, + {"upa": "1/1/1", "effective_time": "foo"}, + "Sample service error code 30001 Illegal input parameter: key 'effective_time' " + + "value of 'foo' is not a valid epoch millisecond timestamp", + ) _get_link_from_data_fail( - sample_port, TOKEN4, {'upa': '1/1/1'}, - 'Sample service error code 20000 Unauthorized: User user4 cannot read upa 1/1/1') + sample_port, + TOKEN4, + {"upa": "1/1/1"}, + "Sample service error code 20000 Unauthorized: User user4 cannot read upa 1/1/1", + ) _get_link_from_data_fail( - sample_port, None, {'upa': '1/1/1'}, - 'Sample service error code 20000 Unauthorized: Anonymous users cannot read upa 1/1/1') + sample_port, + None, + {"upa": "1/1/1"}, + "Sample service error code 20000 Unauthorized: Anonymous users cannot read upa 1/1/1", + ) _get_link_from_data_fail( - sample_port, TOKEN3, {'upa': '1/2/1'}, - 'Sample service error code 50040 No such workspace data: Object 1/2/1 does not exist') + sample_port, + TOKEN3, + {"upa": "1/2/1"}, + "Sample service error code 50040 No such workspace data: Object 1/2/1 does not exist", + ) # admin tests (also tests missing / deleted objects) _get_link_from_data_fail( - sample_port, TOKEN4, {'upa': '1/1/1', 'as_admin': 1}, - 'Sample service error code 20000 Unauthorized: User user4 does not have the necessary ' + - 'administration privileges to run method get_data_links_from_data') + sample_port, + TOKEN4, + {"upa": "1/1/1", "as_admin": 1}, + "Sample service error code 20000 Unauthorized: User user4 does not have the necessary " + + "administration privileges to run method get_data_links_from_data", + ) _get_link_from_data_fail( - sample_port, None, {'upa': '1/1/1', 'as_admin': 1}, - 'Sample service error code 20000 Unauthorized: Anonymous users may not act ' + - 'as service administrators.') + sample_port, + None, + {"upa": "1/1/1", "as_admin": 1}, + "Sample service error code 20000 Unauthorized: Anonymous users may not act " + + "as service administrators.", + ) _get_link_from_data_fail( - sample_port, TOKEN3, {'upa': '1/1/2', 'as_admin': 1}, - 'Sample service error code 50040 No such workspace data: Object 1/1/2 does not exist') + sample_port, + TOKEN3, + {"upa": "1/1/2", "as_admin": 1}, + "Sample service error code 50040 No such workspace data: Object 1/1/2 does not exist", + ) _get_link_from_data_fail( - sample_port, TOKEN3, {'upa': '2/1/1', 'as_admin': 1}, - 'Sample service error code 50040 No such workspace data: No workspace with id 2 exists') + sample_port, + TOKEN3, + {"upa": "2/1/1", "as_admin": 1}, + "Sample service error code 50040 No such workspace data: No workspace with id 2 exists", + ) - wscli.delete_objects([{'ref': '1/1'}]) + wscli.delete_objects([{"ref": "1/1"}]) _get_link_from_data_fail( - sample_port, TOKEN3, {'upa': '1/1/1', 'as_admin': 1}, - 'Sample service error code 50040 No such workspace data: Object 1/1/1 does not exist') + sample_port, + TOKEN3, + {"upa": "1/1/1", "as_admin": 1}, + "Sample service error code 50040 No such workspace data: Object 1/1/1 does not exist", + ) - wscli.delete_workspace({'id': 1}) + wscli.delete_workspace({"id": 1}) _get_link_from_data_fail( - sample_port, TOKEN3, {'upa': '1/1/1', 'as_admin': 1}, - 'Sample service error code 50040 No such workspace data: Workspace 1 is deleted') + sample_port, + TOKEN3, + {"upa": "1/1/1", "as_admin": 1}, + "Sample service error code 50040 No such workspace data: Workspace 1 is deleted", + ) def _get_link_from_data_fail(sample_port, token, params, expected): - url = f'http://localhost:{sample_port}' - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_data_links_from_data', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + url = f"http://localhost:{sample_port}" + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_data_links_from_data", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def test_get_sample_via_data(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_permissions({'id': 1, 'new_permission': 'r', 'users': [USER4]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_permissions({"id": 1, "new_permission": "r", "users": [USER4]}) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'meta_user': {'a': {'b': 'f', 'e': 'g'}, 'c': {'d': 'h'}}, - 'meta_controlled': {'foo': {'bar': 'baz'}, 'premature': {'e': 'fakeout'}}, - 'source_meta': [{'key': 'foo', 'skey': 'b', 'svalue': {'x': 'y'}}] - }, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "meta_user": {"a": {"b": "f", "e": "g"}, "c": {"d": "h"}}, + "meta_controlled": { + "foo": {"bar": "baz"}, + "premature": {"e": "fakeout"}, + }, + "source_meta": [{"key": "foo", "skey": "b", "svalue": {"x": "y"}}], + }, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) id2 = _create_sample( url, TOKEN3, - {'name': 'unused', 'node_tree': [{'id': 'unused', 'type': 'BioReplicate'}]}, - 1 - ) + {"name": "unused", "node_tree": [{"id": "unused", "type": "BioReplicate"}]}, + 1, + ) # ver 2 _create_sample( url, TOKEN3, - {'id': id2, - 'name': 'myothersample3', - 'node_tree': [{'id': 'root3', 'type': 'BioReplicate'}, - {'id': 'foo3', 'type': 'TechReplicate', 'parent': 'root3'} - ] - }, - 2 - ) + { + "id": id2, + "name": "myothersample3", + "node_tree": [ + {"id": "root3", "type": "BioReplicate"}, + {"id": "foo3", "type": "TechReplicate", "parent": "root3"}, + ], + }, + 2, + ) # create links - _create_link(url, TOKEN3, USER3, {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}) _create_link( - url, TOKEN3, USER3, - {'id': id2, 'version': 2, 'node': 'root3', 'upa': '1/1/1', 'dataid': 'column1'}) + url, TOKEN3, USER3, {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1"} + ) + _create_link( + url, + TOKEN3, + USER3, + {"id": id2, "version": 2, "node": "root3", "upa": "1/1/1", "dataid": "column1"}, + ) # get first sample via link from object 1/1/1 using a token that has no access - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.get_sample_via_data', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'id': str(id1), 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.get_sample_via_data", + "version": "1.1", + "id": "42", + "params": [{"upa": "1/1/1", "id": str(id1), "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - res = ret.json()['result'][0] - assert_ms_epoch_close_to_now(res['save_date']) - del res['save_date'] + res = ret.json()["result"][0] + assert_ms_epoch_close_to_now(res["save_date"]) + del res["save_date"] expected = { - 'id': id1, - 'version': 1, - 'name': 'mysample', - 'user': USER3, - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'parent': None, - 'meta_user': {'a': {'b': 'f', 'e': 'g'}, 'c': {'d': 'h'}}, - 'meta_controlled': {'foo': {'bar': 'baz'}, 'premature': {'e': 'fakeout'}}, - 'source_meta': [{'key': 'foo', 'skey': 'b', 'svalue': {'x': 'y'}}], - }, - {'id': 'foo', - 'type': 'TechReplicate', - 'parent': 'root', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - }, - ] - } + "id": id1, + "version": 1, + "name": "mysample", + "user": USER3, + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "parent": None, + "meta_user": {"a": {"b": "f", "e": "g"}, "c": {"d": "h"}}, + "meta_controlled": { + "foo": {"bar": "baz"}, + "premature": {"e": "fakeout"}, + }, + "source_meta": [{"key": "foo", "skey": "b", "svalue": {"x": "y"}}], + }, + { + "id": "foo", + "type": "TechReplicate", + "parent": "root", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + }, + ], + } assert res == expected # get second sample via link from object 1/1/1 using a token that has no access - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.get_sample_via_data', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'id': str(id2), 'version': 2}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.get_sample_via_data", + "version": "1.1", + "id": "42", + "params": [{"upa": "1/1/1", "id": str(id2), "version": 2}], + }, + ) # print(ret.text) assert ret.ok is True - res = ret.json()['result'][0] - assert_ms_epoch_close_to_now(res['save_date']) - del res['save_date'] + res = ret.json()["result"][0] + assert_ms_epoch_close_to_now(res["save_date"]) + del res["save_date"] expected = { - 'id': id2, - 'version': 2, - 'name': 'myothersample3', - 'user': USER3, - 'node_tree': [{'id': 'root3', - 'type': 'BioReplicate', - 'parent': None, - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - }, - {'id': 'foo3', - 'type': 'TechReplicate', - 'parent': 'root3', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - }, - ] - } + "id": id2, + "version": 2, + "name": "myothersample3", + "user": USER3, + "node_tree": [ + { + "id": "root3", + "type": "BioReplicate", + "parent": None, + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + }, + { + "id": "foo3", + "type": "TechReplicate", + "parent": "root3", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + }, + ], + } assert res == expected def test_get_sample_via_data_expired_with_anon_user(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_global_permission({'id': 1, 'new_permission': 'r'}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_global_permission({"id": 1, "new_permission": "r"}) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) id2 = _create_sample( url, TOKEN3, - {'name': 'myothersample', - 'node_tree': [{'id': 'root2', 'type': 'BioReplicate'}, - {'id': 'foo2', 'type': 'TechReplicate', 'parent': 'root2'} - ] - }, - 1 - ) + { + "name": "myothersample", + "node_tree": [ + {"id": "root2", "type": "BioReplicate"}, + {"id": "foo2", "type": "TechReplicate", "parent": "root2"}, + ], + }, + 1, + ) # create links - _create_link(url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}) + _create_link( + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + ) # update link node - _create_link(url, TOKEN3, USER3, { - 'id': id2, - 'version': 1, - 'node': 'root2', - 'upa': '1/1/1', - 'dataid': 'yay', - 'update': 1, - }) + _create_link( + url, + TOKEN3, + USER3, + { + "id": id2, + "version": 1, + "node": "root2", + "upa": "1/1/1", + "dataid": "yay", + "update": 1, + }, + ) # pulled link from server to check the old link was expired # get sample via current link - ret = requests.post(url, headers=get_authorized_headers(None), json={ - 'method': 'SampleService.get_sample_via_data', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'id': str(id2), 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(None), + json={ + "method": "SampleService.get_sample_via_data", + "version": "1.1", + "id": "42", + "params": [{"upa": "1/1/1", "id": str(id2), "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - res = ret.json()['result'][0] - assert_ms_epoch_close_to_now(res['save_date']) - del res['save_date'] + res = ret.json()["result"][0] + assert_ms_epoch_close_to_now(res["save_date"]) + del res["save_date"] expected = { - 'id': id2, - 'version': 1, - 'name': 'myothersample', - 'user': USER3, - 'node_tree': [{'id': 'root2', - 'type': 'BioReplicate', - 'parent': None, - 'meta_user': {}, - 'meta_controlled': {}, - 'source_meta': [], - }, - {'id': 'foo2', - 'type': 'TechReplicate', - 'parent': 'root2', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - }, - ] - } + "id": id2, + "version": 1, + "name": "myothersample", + "user": USER3, + "node_tree": [ + { + "id": "root2", + "type": "BioReplicate", + "parent": None, + "meta_user": {}, + "meta_controlled": {}, + "source_meta": [], + }, + { + "id": "foo2", + "type": "TechReplicate", + "parent": "root2", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + }, + ], + } assert res == expected # get sample via expired link - ret = requests.post(url, headers=get_authorized_headers(None), json={ - 'method': 'SampleService.get_sample_via_data', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'id': str(id1), 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(None), + json={ + "method": "SampleService.get_sample_via_data", + "version": "1.1", + "id": "42", + "params": [{"upa": "1/1/1", "id": str(id1), "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - res = ret.json()['result'][0] - assert_ms_epoch_close_to_now(res['save_date']) - del res['save_date'] + res = ret.json()["result"][0] + assert_ms_epoch_close_to_now(res["save_date"]) + del res["save_date"] expected = { - 'id': id1, - 'version': 1, - 'name': 'mysample', - 'user': USER3, - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'parent': None, - 'meta_user': {}, - 'meta_controlled': {}, - 'source_meta': [], - }, - {'id': 'foo', - 'type': 'TechReplicate', - 'parent': 'root', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - }, - ] - } + "id": id1, + "version": 1, + "name": "mysample", + "user": USER3, + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "parent": None, + "meta_user": {}, + "meta_controlled": {}, + "source_meta": [], + }, + { + "id": "foo", + "type": "TechReplicate", + "parent": "root", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + }, + ], + } assert res == expected def test_get_sample_via_data_public_read(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN1) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) - wscli.set_global_permission({'id': 1, 'new_permission': 'r'}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) + wscli.set_global_permission({"id": 1, "new_permission": "r"}) # create samples id_ = _create_generic_sample(url, TOKEN1) # create links - _create_link(url, TOKEN1, USER1, {'id': id_, 'version': 1, 'node': 'foo', 'upa': '1/1/1'}) + _create_link( + url, TOKEN1, USER1, {"id": id_, "version": 1, "node": "foo", "upa": "1/1/1"} + ) # get sample via link from object 1/1/1 using a token that has no explicit access - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.get_sample_via_data', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'id': str(id_), 'version': 1}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.get_sample_via_data", + "version": "1.1", + "id": "42", + "params": [{"upa": "1/1/1", "id": str(id_), "version": 1}], + }, + ) # print(ret.text) assert ret.ok is True - res = ret.json()['result'][0] - assert_ms_epoch_close_to_now(res['save_date']) - del res['save_date'] + res = ret.json()["result"][0] + assert_ms_epoch_close_to_now(res["save_date"]) + del res["save_date"] expected = { - 'id': id_, - 'version': 1, - 'name': 'mysample', - 'user': USER1, - 'node_tree': [{'id': 'root', - 'type': 'BioReplicate', - 'parent': None, - 'meta_user': {}, - 'meta_controlled': {}, - 'source_meta': [], - }, - {'id': 'foo', - 'type': 'TechReplicate', - 'parent': 'root', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - }, - ] - } + "id": id_, + "version": 1, + "name": "mysample", + "user": USER1, + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "parent": None, + "meta_user": {}, + "meta_controlled": {}, + "source_meta": [], + }, + { + "id": "foo", + "type": "TechReplicate", + "parent": "root", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + }, + ], + } assert res == expected def test_get_sample_via_data_fail(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN3) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) # create samples id1 = _create_sample( url, TOKEN3, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) # create links - _create_link(url, TOKEN3, USER3, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}) + _create_link( + url, + TOKEN3, + USER3, + {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + ) _get_sample_via_data_fail( - sample_port, TOKEN3, {}, - 'Sample service error code 30000 Missing input parameter: upa') + sample_port, + TOKEN3, + {}, + "Sample service error code 30000 Missing input parameter: upa", + ) _get_sample_via_data_fail( - sample_port, TOKEN3, {'upa': '1/1/1'}, - 'Sample service error code 30000 Missing input parameter: id') + sample_port, + TOKEN3, + {"upa": "1/1/1"}, + "Sample service error code 30000 Missing input parameter: id", + ) _get_sample_via_data_fail( - sample_port, TOKEN3, {'upa': '1/1/1', 'id': id1}, - 'Sample service error code 30000 Missing input parameter: version') + sample_port, + TOKEN3, + {"upa": "1/1/1", "id": id1}, + "Sample service error code 30000 Missing input parameter: version", + ) _get_sample_via_data_fail( - sample_port, TOKEN4, {'upa': '1/1/1', 'id': id1, 'version': 1}, - 'Sample service error code 20000 Unauthorized: User user4 cannot read upa 1/1/1') + sample_port, + TOKEN4, + {"upa": "1/1/1", "id": id1, "version": 1}, + "Sample service error code 20000 Unauthorized: User user4 cannot read upa 1/1/1", + ) _get_sample_via_data_fail( - sample_port, None, {'upa': '1/1/1', 'id': id1, 'version': 1}, - 'Sample service error code 20000 Unauthorized: Anonymous users cannot read upa 1/1/1') + sample_port, + None, + {"upa": "1/1/1", "id": id1, "version": 1}, + "Sample service error code 20000 Unauthorized: Anonymous users cannot read upa 1/1/1", + ) _get_sample_via_data_fail( - sample_port, TOKEN3, {'upa': '1/2/1', 'id': id1, 'version': 1}, - 'Sample service error code 50040 No such workspace data: Object 1/2/1 does not exist') + sample_port, + TOKEN3, + {"upa": "1/2/1", "id": id1, "version": 1}, + "Sample service error code 50040 No such workspace data: Object 1/2/1 does not exist", + ) badid = uuid.uuid4() _get_sample_via_data_fail( - sample_port, TOKEN3, {'upa': '1/1/1', 'id': str(badid), 'version': 1}, - 'Sample service error code 50050 No such data link: There is no link from UPA 1/1/1 ' + - f'to sample {badid}') + sample_port, + TOKEN3, + {"upa": "1/1/1", "id": str(badid), "version": 1}, + "Sample service error code 50050 No such data link: There is no link from UPA 1/1/1 " + + f"to sample {badid}", + ) _get_sample_via_data_fail( - sample_port, TOKEN3, {'upa': '1/1/1', 'id': str(id1), 'version': 2}, - f'Sample service error code 50020 No such sample version: {id1} ver 2') + sample_port, + TOKEN3, + {"upa": "1/1/1", "id": str(id1), "version": 2}, + f"Sample service error code 50020 No such sample version: {id1} ver 2", + ) def _get_sample_via_data_fail(sample_port, token, params, expected): # could make a single method that just takes the service method name to DRY things up a bit - url = f'http://localhost:{sample_port}' - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_sample_via_data', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + url = f"http://localhost:{sample_port}" + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_sample_via_data", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected def test_get_data_link(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN4) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) # create samples id1 = _create_sample( url, TOKEN4, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) # create link - lid = _create_link(url, TOKEN4, USER4, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}) + lid = _create_link( + url, + TOKEN4, + USER4, + {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + ) # get link, user 3 has admin read perms - ret = requests.post(url, headers=get_authorized_headers(TOKEN3), json={ - 'method': 'SampleService.get_data_link', - 'version': '1.1', - 'id': '42', - 'params': [{'linkid': lid}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN3), + json={ + "method": "SampleService.get_data_link", + "version": "1.1", + "id": "42", + "params": [{"linkid": lid}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - link = ret.json()['result'][0] - created = link.pop('created') + assert len(ret.json()["result"]) == 1 + link = ret.json()["result"][0] + created = link.pop("created") assert_ms_epoch_close_to_now(created) assert link == { - 'linkid': lid, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': 'yay', - 'createdby': USER4, - 'expiredby': None, - 'expired': None - } + "linkid": lid, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": "yay", + "createdby": USER4, + "expiredby": None, + "expired": None, + } # expire link - ret = requests.post(url, headers=get_authorized_headers(TOKEN4), json={ - 'method': 'SampleService.expire_data_link', - 'version': '1.1', - 'id': '42', - 'params': [{'upa': '1/1/1', 'dataid': 'yay'}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN4), + json={ + "method": "SampleService.expire_data_link", + "version": "1.1", + "id": "42", + "params": [{"upa": "1/1/1", "dataid": "yay"}], + }, + ) # print(ret.text) assert ret.ok is True # get link, user 5 has full perms - ret = requests.post(url, headers=get_authorized_headers(TOKEN5), json={ - 'method': 'SampleService.get_data_link', - 'version': '1.1', - 'id': '42', - 'params': [{'linkid': lid}] - }) + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN5), + json={ + "method": "SampleService.get_data_link", + "version": "1.1", + "id": "42", + "params": [{"linkid": lid}], + }, + ) # print(ret.text) assert ret.ok is True - assert len(ret.json()['result']) == 1 - link = ret.json()['result'][0] - assert_ms_epoch_close_to_now(link['expired']) - del link['expired'] + assert len(ret.json()["result"]) == 1 + link = ret.json()["result"][0] + assert_ms_epoch_close_to_now(link["expired"]) + del link["expired"] assert link == { - 'linkid': lid, - 'id': id1, - 'version': 1, - 'node': 'foo', - 'upa': '1/1/1', - 'dataid': 'yay', - 'created': created, - 'createdby': USER4, - 'expiredby': USER4, - } + "linkid": lid, + "id": id1, + "version": 1, + "node": "foo", + "upa": "1/1/1", + "dataid": "yay", + "created": created, + "createdby": USER4, + "expiredby": USER4, + } def test_get_data_link_fail(sample_port, workspace): - url = f'http://localhost:{sample_port}' - wsurl = f'http://localhost:{workspace.port}' + url = f"http://localhost:{sample_port}" + wsurl = f"http://localhost:{workspace.port}" wscli = Workspace(wsurl, token=TOKEN4) # create workspace & objects - wscli.create_workspace({'workspace': 'foo'}) - wscli.save_objects({'id': 1, 'objects': [ - {'name': 'bar', 'data': {}, 'type': 'Trivial.Object-1.0'}, - ]}) + wscli.create_workspace({"workspace": "foo"}) + wscli.save_objects( + { + "id": 1, + "objects": [ + {"name": "bar", "data": {}, "type": "Trivial.Object-1.0"}, + ], + } + ) # create samples id1 = _create_sample( url, TOKEN4, - {'name': 'mysample', - 'node_tree': [{'id': 'root', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 'root'} - ] - }, - 1 - ) + { + "name": "mysample", + "node_tree": [ + {"id": "root", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": "root"}, + ], + }, + 1, + ) # create link - lid = _create_link(url, TOKEN4, USER4, - {'id': id1, 'version': 1, 'node': 'foo', 'upa': '1/1/1', 'dataid': 'yay'}) + lid = _create_link( + url, + TOKEN4, + USER4, + {"id": id1, "version": 1, "node": "foo", "upa": "1/1/1", "dataid": "yay"}, + ) _get_data_link_fail( - sample_port, TOKEN3, {}, 'Sample service error code 30000 Missing input parameter: linkid') + sample_port, + TOKEN3, + {}, + "Sample service error code 30000 Missing input parameter: linkid", + ) _get_data_link_fail( - sample_port, TOKEN4, {'linkid': lid}, - 'Sample service error code 20000 Unauthorized: User user4 does not have the necessary ' + - 'administration privileges to run method get_data_link') + sample_port, + TOKEN4, + {"linkid": lid}, + "Sample service error code 20000 Unauthorized: User user4 does not have the necessary " + + "administration privileges to run method get_data_link", + ) oid = uuid.uuid4() _get_data_link_fail( - sample_port, TOKEN3, {'linkid': str(oid)}, - f'Sample service error code 50050 No such data link: {oid}') + sample_port, + TOKEN3, + {"linkid": str(oid)}, + f"Sample service error code 50050 No such data link: {oid}", + ) def _get_data_link_fail(sample_port, token, params, expected): # could make a single method that just takes the service method name to DRY things up a bit - url = f'http://localhost:{sample_port}' - ret = requests.post(url, headers=get_authorized_headers(token), json={ - 'method': 'SampleService.get_data_link', - 'version': '1.1', - 'id': '42', - 'params': [params] - }) + url = f"http://localhost:{sample_port}" + ret = requests.post( + url, + headers=get_authorized_headers(token), + json={ + "method": "SampleService.get_data_link", + "version": "1.1", + "id": "42", + "params": [params], + }, + ) assert ret.status_code == 500 - assert ret.json()['error']['message'] == expected + assert ret.json()["error"]["message"] == expected # ########################### @@ -4593,32 +5820,41 @@ def _get_data_link_fail(sample_port, token, params, expected): # for some reason having sample_port along with auth in the test fn args prevents a tear down # error, not quite sure why + def test_user_lookup_build_fail_bad_args(): _user_lookup_build_fail( - '', 'foo', ValueError('auth_url cannot be a value that evaluates to false')) + "", "foo", ValueError("auth_url cannot be a value that evaluates to false") + ) _user_lookup_build_fail( - 'http://foo.com', '', ValueError('auth_token cannot be a value that evaluates to false')) + "http://foo.com", + "", + ValueError("auth_token cannot be a value that evaluates to false"), + ) def test_user_lookup_build_fail_bad_token(sample_port, auth): _user_lookup_build_fail( - f'http://localhost:{auth.port}/testmode', - 'tokentokentoken!', - InvalidTokenError('KBase auth server reported token is invalid.')) + f"http://localhost:{auth.port}/testmode", + "tokentokentoken!", + InvalidTokenError("KBase auth server reported token is invalid."), + ) def test_user_lookup_build_fail_bad_auth_url(sample_port, auth): _user_lookup_build_fail( - f'http://localhost:{auth.port}/testmode/foo', + f"http://localhost:{auth.port}/testmode/foo", TOKEN1, - IOError('Error from KBase auth server: HTTP 404 Not Found')) + IOError("Error from KBase auth server: HTTP 404 Not Found"), + ) def test_user_lookup_build_fail_not_auth_url(auth): _user_lookup_build_fail( - 'https://httpbin.org/status/404', + "https://httpbin.org/status/404", TOKEN1, - IOError('Non-JSON response from KBase auth server, status code: 404')) + IOError("Non-JSON response from KBase auth server, status code: 404"), + ) + def _user_lookup_build_fail(url, token, expected): with raises(Exception) as got: @@ -4627,40 +5863,61 @@ def _user_lookup_build_fail(url, token, expected): def test_user_lookup(sample_port, auth): - ul = KBaseUserLookup(f'http://localhost:{auth.port}/testmode', TOKEN1) + ul = KBaseUserLookup(f"http://localhost:{auth.port}/testmode", TOKEN1) assert ul.invalid_users([]) == [] assert ul.invalid_users([UserID(USER1), UserID(USER2), UserID(USER3)]) == [] + def test_user_lookup_cache(sample_port, auth): - ul = KBaseUserLookup(f'http://localhost:{auth.port}/testmode', TOKEN1) + ul = KBaseUserLookup(f"http://localhost:{auth.port}/testmode", TOKEN1) assert ul._valid_cache.get(USER1, default=False) is False assert ul._valid_cache.get(USER2, default=False) is False ul.invalid_users([UserID(USER1)]) assert ul._valid_cache.get(USER1, default=False) is True assert ul._valid_cache.get(USER2, default=False) is False + def test_user_lookup_bad_users(sample_port, auth): - ul = KBaseUserLookup(f'http://localhost:{auth.port}/testmode/', TOKEN1) - assert ul.invalid_users( - [UserID('nouserhere'), UserID(USER1), UserID(USER2), UserID('whooptydoo'), - UserID(USER3)]) == [UserID('nouserhere'), UserID('whooptydoo')] + ul = KBaseUserLookup(f"http://localhost:{auth.port}/testmode/", TOKEN1) + assert ( + ul.invalid_users( + [ + UserID("nouserhere"), + UserID(USER1), + UserID(USER2), + UserID("whooptydoo"), + UserID(USER3), + ] + ) + == [UserID("nouserhere"), UserID("whooptydoo")] + ) def test_user_lookup_fail_bad_args(sample_port, auth): - ul = KBaseUserLookup(f'http://localhost:{auth.port}/testmode/', TOKEN1) - _user_lookup_fail(ul, None, ValueError('usernames cannot be None')) - _user_lookup_fail(ul, [UserID('foo'), UserID('bar'), None], ValueError( - 'Index 2 of iterable usernames cannot be a value that evaluates to false')) + ul = KBaseUserLookup(f"http://localhost:{auth.port}/testmode/", TOKEN1) + _user_lookup_fail(ul, None, ValueError("usernames cannot be None")) + _user_lookup_fail( + ul, + [UserID("foo"), UserID("bar"), None], + ValueError( + "Index 2 of iterable usernames cannot be a value that evaluates to false" + ), + ) def test_user_lookup_fail_bad_username(sample_port, auth): - ul = KBaseUserLookup(f'http://localhost:{auth.port}/testmode/', TOKEN1) + ul = KBaseUserLookup(f"http://localhost:{auth.port}/testmode/", TOKEN1) # maybe possibly this error should be shortened # definitely clear the user name is illegal though, there's no question about that - _user_lookup_fail(ul, [UserID('1')], InvalidUserError( - 'The KBase auth server is being very assertive about one of the usernames being ' + - 'illegal: 30010 Illegal user name: Illegal user name [1]: 30010 Illegal user name: ' + - 'Username must start with a letter')) + _user_lookup_fail( + ul, + [UserID("1")], + InvalidUserError( + "The KBase auth server is being very assertive about one of the usernames being " + + "illegal: 30010 Illegal user name: Illegal user name [1]: 30010 Illegal user name: " + + "Username must start with a letter" + ), + ) def _user_lookup_fail(userlookup, users, expected): @@ -4675,44 +5932,53 @@ def test_is_admin(sample_port, auth): f = AdminPermission.FULL _check_is_admin(auth.port, [n, n, n, n]) - _check_is_admin(auth.port, [f, f, n, n], ['fulladmin1']) - _check_is_admin(auth.port, [n, f, n, n], ['fulladmin2']) - _check_is_admin(auth.port, [n, n, r, n], None, ['readadmin1']) - _check_is_admin(auth.port, [n, r, n, n], None, ['readadmin2']) - _check_is_admin(auth.port, [n, f, n, n], ['fulladmin2'], ['readadmin2']) - _check_is_admin(auth.port, [n, f, r, n], ['fulladmin2'], ['readadmin1']) + _check_is_admin(auth.port, [f, f, n, n], ["fulladmin1"]) + _check_is_admin(auth.port, [n, f, n, n], ["fulladmin2"]) + _check_is_admin(auth.port, [n, n, r, n], None, ["readadmin1"]) + _check_is_admin(auth.port, [n, r, n, n], None, ["readadmin2"]) + _check_is_admin(auth.port, [n, f, n, n], ["fulladmin2"], ["readadmin2"]) + _check_is_admin(auth.port, [n, f, r, n], ["fulladmin2"], ["readadmin1"]) def _check_is_admin(port, results, full_roles=None, read_roles=None): ul = KBaseUserLookup( - f'http://localhost:{port}/testmode/', - TOKEN_SERVICE, - full_roles, - read_roles) + f"http://localhost:{port}/testmode/", TOKEN_SERVICE, full_roles, read_roles + ) - for t, u, r in zip([TOKEN1, TOKEN2, TOKEN3, TOKEN4], [USER1, USER2, USER3, USER4], results): + for t, u, r in zip( + [TOKEN1, TOKEN2, TOKEN3, TOKEN4], [USER1, USER2, USER3, USER4], results + ): assert ul.is_admin(t) == (r, u) + def test_is_admin_cache(sample_port, auth): - ul = KBaseUserLookup(f'http://localhost:{auth.port}/testmode/', TOKEN_SERVICE) + ul = KBaseUserLookup(f"http://localhost:{auth.port}/testmode/", TOKEN_SERVICE) assert ul._admin_cache.get(TOKEN1, default=False) is False assert ul._admin_cache.get(TOKEN2, default=False) is False ul.is_admin(TOKEN1) assert ul._admin_cache.get(TOKEN1, default=False) is not False assert ul._admin_cache.get(TOKEN2, default=False) is False + def test_is_admin_fail_bad_input(sample_port, auth): - ul = KBaseUserLookup(f'http://localhost:{auth.port}/testmode/', TOKEN_SERVICE) + ul = KBaseUserLookup(f"http://localhost:{auth.port}/testmode/", TOKEN_SERVICE) - _is_admin_fail(ul, None, ValueError('token cannot be a value that evaluates to false')) - _is_admin_fail(ul, '', ValueError('token cannot be a value that evaluates to false')) + _is_admin_fail( + ul, None, ValueError("token cannot be a value that evaluates to false") + ) + _is_admin_fail( + ul, "", ValueError("token cannot be a value that evaluates to false") + ) def test_is_admin_fail_bad_token(sample_port, auth): - ul = KBaseUserLookup(f'http://localhost:{auth.port}/testmode/', TOKEN_SERVICE) + ul = KBaseUserLookup(f"http://localhost:{auth.port}/testmode/", TOKEN_SERVICE) - _is_admin_fail(ul, 'bad token here', InvalidTokenError( - 'KBase auth server reported token is invalid.')) + _is_admin_fail( + ul, + "bad token here", + InvalidTokenError("KBase auth server reported token is invalid."), + ) def _is_admin_fail(userlookup, user, expected): @@ -4727,72 +5993,139 @@ def _is_admin_fail(userlookup, user, expected): def test_workspace_wrapper_has_permission(sample_port, workspace): - url = f'http://localhost:{workspace.port}' + url = f"http://localhost:{workspace.port}" wscli = Workspace(url, token=TOKEN_WS_READ_ADMIN) ws = WS(wscli) wscli2 = Workspace(url, token=TOKEN2) - wscli2.create_workspace({'workspace': 'foo'}) - wscli2.save_objects({'id': 1, - 'objects': [{'name': 'bar', 'type': 'Trivial.Object-1.0', 'data': {}}]}) - wscli2.save_objects({'id': 1, - 'objects': [{'name': 'foo', 'type': 'Trivial.Object-1.0', 'data': {}}]}) - wscli2.save_objects({'id': 1, - 'objects': [{'name': 'foo', 'type': 'Trivial.Object-1.0', 'data': {}}]}) + wscli2.create_workspace({"workspace": "foo"}) + wscli2.save_objects( + { + "id": 1, + "objects": [{"name": "bar", "type": "Trivial.Object-1.0", "data": {}}], + } + ) + wscli2.save_objects( + { + "id": 1, + "objects": [{"name": "foo", "type": "Trivial.Object-1.0", "data": {}}], + } + ) + wscli2.save_objects( + { + "id": 1, + "objects": [{"name": "foo", "type": "Trivial.Object-1.0", "data": {}}], + } + ) ws.has_permission(UserID(USER2), WorkspaceAccessType.ADMIN, 1) # Shouldn't fail - ws.has_permission(UserID(USER2), WorkspaceAccessType.ADMIN, upa=UPA('1/2/2')) # Shouldn't fail + ws.has_permission( + UserID(USER2), WorkspaceAccessType.ADMIN, upa=UPA("1/2/2") + ) # Shouldn't fail def test_workspace_wrapper_has_permission_fail_bad_args(sample_port, workspace): - url = f'http://localhost:{workspace.port}' + url = f"http://localhost:{workspace.port}" wscli2 = Workspace(url, token=TOKEN2) - wscli2.create_workspace({'workspace': 'foo'}) - wscli2.save_objects({'id': 1, - 'objects': [{'name': 'bar', 'type': 'Trivial.Object-1.0', 'data': {}}]}) - wscli2.save_objects({'id': 1, - 'objects': [{'name': 'foo', 'type': 'Trivial.Object-1.0', 'data': {}}]}) + wscli2.create_workspace({"workspace": "foo"}) + wscli2.save_objects( + { + "id": 1, + "objects": [{"name": "bar", "type": "Trivial.Object-1.0", "data": {}}], + } + ) + wscli2.save_objects( + { + "id": 1, + "objects": [{"name": "foo", "type": "Trivial.Object-1.0", "data": {}}], + } + ) _workspace_wrapper_has_permission_fail( - workspace.port, UserID(USER1), 1, None, UnauthorizedError( - 'User user1 cannot read workspace 1')) + workspace.port, + UserID(USER1), + 1, + None, + UnauthorizedError("User user1 cannot read workspace 1"), + ) _workspace_wrapper_has_permission_fail( - workspace.port, UserID(USER1), None, UPA('1/2/1'), - UnauthorizedError('User user1 cannot read upa 1/2/1')) + workspace.port, + UserID(USER1), + None, + UPA("1/2/1"), + UnauthorizedError("User user1 cannot read upa 1/2/1"), + ) _workspace_wrapper_has_permission_fail( - workspace.port, UserID('fakeuser'), 1, None, UnauthorizedError( - 'User fakeuser cannot read workspace 1')) + workspace.port, + UserID("fakeuser"), + 1, + None, + UnauthorizedError("User fakeuser cannot read workspace 1"), + ) _workspace_wrapper_has_permission_fail( - workspace.port, UserID('fakeuser'), None, UPA('1/2/1'), - UnauthorizedError('User fakeuser cannot read upa 1/2/1')) + workspace.port, + UserID("fakeuser"), + None, + UPA("1/2/1"), + UnauthorizedError("User fakeuser cannot read upa 1/2/1"), + ) _workspace_wrapper_has_permission_fail( - workspace.port, UserID(USER2), 2, None, - NoSuchWorkspaceDataError('No workspace with id 2 exists')) + workspace.port, + UserID(USER2), + 2, + None, + NoSuchWorkspaceDataError("No workspace with id 2 exists"), + ) _workspace_wrapper_has_permission_fail( - workspace.port, UserID(USER2), None, UPA('2/1/1'), - NoSuchWorkspaceDataError('No workspace with id 2 exists')) + workspace.port, + UserID(USER2), + None, + UPA("2/1/1"), + NoSuchWorkspaceDataError("No workspace with id 2 exists"), + ) _workspace_wrapper_has_permission_fail( - workspace.port, UserID(USER2), None, UPA('1/2/2'), - NoSuchWorkspaceDataError('Object 1/2/2 does not exist')) + workspace.port, + UserID(USER2), + None, + UPA("1/2/2"), + NoSuchWorkspaceDataError("Object 1/2/2 does not exist"), + ) _workspace_wrapper_has_permission_fail( - workspace.port, UserID(USER2), None, UPA('1/3/1'), - NoSuchWorkspaceDataError('Object 1/3/1 does not exist')) + workspace.port, + UserID(USER2), + None, + UPA("1/3/1"), + NoSuchWorkspaceDataError("Object 1/3/1 does not exist"), + ) - wscli2.delete_objects([{'ref': '1/2'}]) + wscli2.delete_objects([{"ref": "1/2"}]) _workspace_wrapper_has_permission_fail( - workspace.port, UserID(USER2), None, UPA('1/2/1'), - NoSuchWorkspaceDataError('Object 1/2/1 does not exist')) + workspace.port, + UserID(USER2), + None, + UPA("1/2/1"), + NoSuchWorkspaceDataError("Object 1/2/1 does not exist"), + ) - wscli2.delete_workspace({'id': 1}) + wscli2.delete_workspace({"id": 1}) _workspace_wrapper_has_permission_fail( - workspace.port, UserID(USER2), None, UPA('1/1/1'), - NoSuchWorkspaceDataError('Workspace 1 is deleted')) + workspace.port, + UserID(USER2), + None, + UPA("1/1/1"), + NoSuchWorkspaceDataError("Workspace 1 is deleted"), + ) _workspace_wrapper_has_permission_fail( - workspace.port, UserID(USER2), 1, None, NoSuchWorkspaceDataError('Workspace 1 is deleted')) + workspace.port, + UserID(USER2), + 1, + None, + NoSuchWorkspaceDataError("Workspace 1 is deleted"), + ) def _workspace_wrapper_has_permission_fail(ws_port, user, wsid, upa, expected): - url = f'http://localhost:{ws_port}' + url = f"http://localhost:{ws_port}" wscli = Workspace(url, token=TOKEN_WS_READ_ADMIN) ws = WS(wscli) @@ -4802,52 +6135,66 @@ def _workspace_wrapper_has_permission_fail(ws_port, user, wsid, upa, expected): def test_workspace_wrapper_get_workspaces(sample_port, workspace): - url = f'http://localhost:{workspace.port}' + url = f"http://localhost:{workspace.port}" wscli = Workspace(url, token=TOKEN_WS_READ_ADMIN) ws = WS(wscli) wscli1 = Workspace(url, token=TOKEN1) - wscli1.create_workspace({'workspace': 'baz'}) + wscli1.create_workspace({"workspace": "baz"}) wscli2 = Workspace(url, token=TOKEN2) - wscli2.create_workspace({'workspace': 'foo'}) - wscli2.set_global_permission({'id': 2, 'new_permission': 'r'}) + wscli2.create_workspace({"workspace": "foo"}) + wscli2.set_global_permission({"id": 2, "new_permission": "r"}) wscli3 = Workspace(url, token=TOKEN3) - wscli3.create_workspace({'workspace': 'bar'}) - wscli3.set_permissions({'id': 3, 'users': [USER1], 'new_permission': 'r'}) - wscli3.create_workspace({'workspace': 'invisible'}) + wscli3.create_workspace({"workspace": "bar"}) + wscli3.set_permissions({"id": 3, "users": [USER1], "new_permission": "r"}) + wscli3.create_workspace({"workspace": "invisible"}) assert ws.get_user_workspaces(UserID(USER1)) == [1, 2, 3] # not 4 def test_workspace_wrapper_get_workspaces_fail_no_user(sample_port, workspace): - url = f'http://localhost:{workspace.port}' + url = f"http://localhost:{workspace.port}" wscli = Workspace(url, token=TOKEN_WS_READ_ADMIN) ws = WS(wscli) with raises(Exception) as got: - ws.get_user_workspaces(UserID('fakeuser')) - assert_exception_correct(got.value, NoSuchUserError('User fakeuser is not a valid user')) + ws.get_user_workspaces(UserID("fakeuser")) + assert_exception_correct( + got.value, NoSuchUserError("User fakeuser is not a valid user") + ) # ########################### # Kafka notifier tests # ########################### + def test_kafka_notifier_init_fail(): - _kafka_notifier_init_fail(None, 't', MissingParameterError('bootstrap_servers')) - _kafka_notifier_init_fail(' \t ', 't', MissingParameterError('bootstrap_servers')) - _kafka_notifier_init_fail('localhost:10000', None, MissingParameterError('topic')) - _kafka_notifier_init_fail('localhost:10000', ' \t ', MissingParameterError('topic')) + _kafka_notifier_init_fail(None, "t", MissingParameterError("bootstrap_servers")) + _kafka_notifier_init_fail( + " \t ", "t", MissingParameterError("bootstrap_servers") + ) + _kafka_notifier_init_fail("localhost:10000", None, MissingParameterError("topic")) + _kafka_notifier_init_fail( + "localhost:10000", " \t ", MissingParameterError("topic") + ) + _kafka_notifier_init_fail( + "localhost:10000", + "mytopic" + 243 * "a", + IllegalParameterError("topic exceeds maximum length of 249"), + ) _kafka_notifier_init_fail( - 'localhost:10000', 'mytopic' + 243 * 'a', - IllegalParameterError('topic exceeds maximum length of 249')) - _kafka_notifier_init_fail(f'localhost:{find_free_port()}', 'mytopic', NoBrokersAvailable()) + f"localhost:{find_free_port()}", "mytopic", NoBrokersAvailable() + ) - for c in ['Ѽ', '_', '.', '*']: - _kafka_notifier_init_fail('localhost:10000', f'topic{c}topic', ValueError( - f'Illegal character in Kafka topic topic{c}topic: {c}')) + for c in ["Ѽ", "_", ".", "*"]: + _kafka_notifier_init_fail( + "localhost:10000", + f"topic{c}topic", + ValueError(f"Illegal character in Kafka topic topic{c}topic: {c}"), + ) def _kafka_notifier_init_fail(servers, topic, expected): @@ -4857,8 +6204,10 @@ def _kafka_notifier_init_fail(servers, topic, expected): def test_kafka_notifier_new_sample(sample_port, kafka): - topic = 'abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-' + 186 * 'a' - kn = KafkaNotifier(f'localhost:{kafka.port}', topic) + topic = ( + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-" + 186 * "a" + ) + kn = KafkaNotifier(f"localhost:{kafka.port}", topic) try: id_ = uuid.uuid4() @@ -4866,25 +6215,30 @@ def test_kafka_notifier_new_sample(sample_port, kafka): _check_kafka_messages( kafka, - [{'event_type': 'NEW_SAMPLE', 'sample_id': str(id_), 'sample_ver': 6}], - topic) + [{"event_type": "NEW_SAMPLE", "sample_id": str(id_), "sample_ver": 6}], + topic, + ) finally: kn.close() def test_kafka_notifier_notify_new_sample_version_fail(sample_port, kafka): - kn = KafkaNotifier(f'localhost:{kafka.port}', 'mytopic') + kn = KafkaNotifier(f"localhost:{kafka.port}", "mytopic") - _kafka_notifier_notify_new_sample_version_fail(kn, None, 1, ValueError( - 'sample_id cannot be a value that evaluates to false')) - _kafka_notifier_notify_new_sample_version_fail(kn, uuid.uuid4(), 0, ValueError( - 'sample_ver must be > 0')) - _kafka_notifier_notify_new_sample_version_fail(kn, uuid.uuid4(), -3, ValueError( - 'sample_ver must be > 0')) + _kafka_notifier_notify_new_sample_version_fail( + kn, None, 1, ValueError("sample_id cannot be a value that evaluates to false") + ) + _kafka_notifier_notify_new_sample_version_fail( + kn, uuid.uuid4(), 0, ValueError("sample_ver must be > 0") + ) + _kafka_notifier_notify_new_sample_version_fail( + kn, uuid.uuid4(), -3, ValueError("sample_ver must be > 0") + ) kn.close() - _kafka_notifier_notify_new_sample_version_fail(kn, uuid.uuid4(), 1, ValueError( - 'client is closed')) + _kafka_notifier_notify_new_sample_version_fail( + kn, uuid.uuid4(), 1, ValueError("client is closed") + ) def _kafka_notifier_notify_new_sample_version_fail(notifier, sample, version, expected): @@ -4894,29 +6248,30 @@ def _kafka_notifier_notify_new_sample_version_fail(notifier, sample, version, ex def test_kafka_notifier_acl_change(sample_port, kafka): - kn = KafkaNotifier(f'localhost:{kafka.port}', 'topictopic') + kn = KafkaNotifier(f"localhost:{kafka.port}", "topictopic") try: id_ = uuid.uuid4() kn.notify_sample_acl_change(id_) _check_kafka_messages( - kafka, - [{'event_type': 'ACL_CHANGE', 'sample_id': str(id_)}], - 'topictopic') + kafka, [{"event_type": "ACL_CHANGE", "sample_id": str(id_)}], "topictopic" + ) finally: kn.close() def test_kafka_notifier_notify_acl_change_fail(sample_port, kafka): - kn = KafkaNotifier(f'localhost:{kafka.port}', 'mytopic') + kn = KafkaNotifier(f"localhost:{kafka.port}", "mytopic") - _kafka_notifier_notify_acl_change_fail(kn, None, ValueError( - 'sample_id cannot be a value that evaluates to false')) + _kafka_notifier_notify_acl_change_fail( + kn, None, ValueError("sample_id cannot be a value that evaluates to false") + ) kn.close() - _kafka_notifier_notify_acl_change_fail(kn, uuid.uuid4(), ValueError( - 'client is closed')) + _kafka_notifier_notify_acl_change_fail( + kn, uuid.uuid4(), ValueError("client is closed") + ) def _kafka_notifier_notify_acl_change_fail(notifier, sample, expected): @@ -4926,29 +6281,28 @@ def _kafka_notifier_notify_acl_change_fail(notifier, sample, expected): def test_kafka_notifier_new_link(sample_port, kafka): - kn = KafkaNotifier(f'localhost:{kafka.port}', 'topictopic') + kn = KafkaNotifier(f"localhost:{kafka.port}", "topictopic") try: id_ = uuid.uuid4() kn.notify_new_link(id_) _check_kafka_messages( - kafka, - [{'event_type': 'NEW_LINK', 'link_id': str(id_)}], - 'topictopic') + kafka, [{"event_type": "NEW_LINK", "link_id": str(id_)}], "topictopic" + ) finally: kn.close() def test_kafka_notifier_new_link_fail(sample_port, kafka): - kn = KafkaNotifier(f'localhost:{kafka.port}', 'mytopic') + kn = KafkaNotifier(f"localhost:{kafka.port}", "mytopic") - _kafka_notifier_new_link_fail(kn, None, ValueError( - 'link_id cannot be a value that evaluates to false')) + _kafka_notifier_new_link_fail( + kn, None, ValueError("link_id cannot be a value that evaluates to false") + ) kn.close() - _kafka_notifier_new_link_fail(kn, uuid.uuid4(), ValueError( - 'client is closed')) + _kafka_notifier_new_link_fail(kn, uuid.uuid4(), ValueError("client is closed")) def _kafka_notifier_new_link_fail(notifier, sample, expected): @@ -4958,29 +6312,28 @@ def _kafka_notifier_new_link_fail(notifier, sample, expected): def test_kafka_notifier_expired_link(sample_port, kafka): - kn = KafkaNotifier(f'localhost:{kafka.port}', 'topictopic') + kn = KafkaNotifier(f"localhost:{kafka.port}", "topictopic") try: id_ = uuid.uuid4() kn.notify_expired_link(id_) _check_kafka_messages( - kafka, - [{'event_type': 'EXPIRED_LINK', 'link_id': str(id_)}], - 'topictopic') + kafka, [{"event_type": "EXPIRED_LINK", "link_id": str(id_)}], "topictopic" + ) finally: kn.close() def test_kafka_notifier_expired_link_fail(sample_port, kafka): - kn = KafkaNotifier(f'localhost:{kafka.port}', 'mytopic') + kn = KafkaNotifier(f"localhost:{kafka.port}", "mytopic") - _kafka_notifier_expired_link_fail(kn, None, ValueError( - 'link_id cannot be a value that evaluates to false')) + _kafka_notifier_expired_link_fail( + kn, None, ValueError("link_id cannot be a value that evaluates to false") + ) kn.close() - _kafka_notifier_expired_link_fail(kn, uuid.uuid4(), ValueError( - 'client is closed')) + _kafka_notifier_expired_link_fail(kn, uuid.uuid4(), ValueError("client is closed")) def _kafka_notifier_expired_link_fail(notifier, sample, expected): @@ -4994,25 +6347,35 @@ def test_validate_sample(sample_port): def _validate_sample_as_admin(sample_port, as_user, get_token, expected_user): - url = f'http://localhost:{sample_port}' - - ret = requests.post(url, headers=get_authorized_headers(TOKEN2), json={ - 'method': 'SampleService.validate_samples', - 'version': '1.1', - 'id': '67', - 'params': [{ - 'samples': [{ - 'name': 'mysample', - 'node_tree': [{ - 'id': 'root', - 'type': 'BioReplicate', - 'meta_controlled': {'foo': {'bar': 'baz'}}, - 'meta_user': {'a': {'b': 'c'}} - }] - }] - }] - }) + url = f"http://localhost:{sample_port}" + + ret = requests.post( + url, + headers=get_authorized_headers(TOKEN2), + json={ + "method": "SampleService.validate_samples", + "version": "1.1", + "id": "67", + "params": [ + { + "samples": [ + { + "name": "mysample", + "node_tree": [ + { + "id": "root", + "type": "BioReplicate", + "meta_controlled": {"foo": {"bar": "baz"}}, + "meta_user": {"a": {"b": "c"}}, + } + ], + } + ] + } + ], + }, + ) # print(ret.text) assert ret.ok is True - ret_json = ret.json()['result'][0] - assert 'mysample' not in ret_json['errors'] + ret_json = ret.json()["result"][0] + assert "mysample" not in ret_json["errors"] diff --git a/test/arango_controller.py b/test/arango_controller.py index 78b11335..8da17ed4 100644 --- a/test/arango_controller.py +++ b/test/arango_controller.py @@ -24,7 +24,7 @@ class ArangoController: """ def __init__(self, arangoexe: Path, arangojs: Path, root_temp_dir: Path) -> None: - ''' + """ Create and start a new ArangoDB database. An unused port will be selected for the server. :param arangoexe: The path to the ArangoDB server executable (e.g. arangod) to run. @@ -32,42 +32,58 @@ def __init__(self, arangoexe: Path, arangojs: Path, root_temp_dir: Path) -> None (e.g. --javascript.startup-directory). :param root_temp_dir: A temporary directory in which to store ArangoDB data and log files. The files will be stored inside a child directory that is unique per invocation. - ''' + """ arangoexe = Path(os.path.expanduser(arangoexe)) arangojs = Path(os.path.expanduser(arangojs)) if not arangoexe or not os.access(arangoexe, os.X_OK): - raise TestException('arangodb executable path {} does not exist or is not executable.' - .format(arangoexe)) + raise TestException( + "arangodb executable path {} does not exist or is not executable.".format( + arangoexe + ) + ) if not arangojs or not os.path.isdir(arangojs): - raise TestException('arangodb javascript path {} does not exist or is not a directory.' - .format(arangoexe)) + raise TestException( + "arangodb javascript path {} does not exist or is not a directory.".format( + arangoexe + ) + ) if not root_temp_dir: - raise ValueError('root_temp_dir is None') + raise ValueError("root_temp_dir is None") # make temp dirs root_temp_dir = root_temp_dir.absolute() os.makedirs(root_temp_dir, exist_ok=True) - self.temp_dir = Path(tempfile.mkdtemp(prefix='ArangoController-', dir=str(root_temp_dir))) - data_dir = self.temp_dir.joinpath('data') + self.temp_dir = Path( + tempfile.mkdtemp(prefix="ArangoController-", dir=str(root_temp_dir)) + ) + data_dir = self.temp_dir.joinpath("data") os.makedirs(data_dir) self.port = find_free_port() command = [ str(arangoexe), - '--server.endpoint', f'tcp://localhost:{self.port}', - '--configuration', 'none', - '--database.directory', str(data_dir), - '--javascript.startup-directory', str(arangojs), - '--javascript.app-path', str(data_dir / 'apps'), - '--log.file', str(self.temp_dir / 'arango.log') - ] - - self._outfile = open(self.temp_dir.joinpath('arango.out'), 'w') - - self._proc = subprocess.Popen(command, stdout=self._outfile, stderr=subprocess.STDOUT) + "--server.endpoint", + f"tcp://localhost:{self.port}", + "--configuration", + "none", + "--database.directory", + str(data_dir), + "--javascript.startup-directory", + str(arangojs), + "--javascript.app-path", + str(data_dir / "apps"), + "--log.file", + str(self.temp_dir / "arango.log"), + ] + + self._outfile = open(self.temp_dir.joinpath("arango.out"), "w") + + self._proc = subprocess.Popen( + command, stdout=self._outfile, stderr=subprocess.STDOUT + ) time.sleep(3) # wait for server to start up - self.client = arango.ArangoClient(hosts=f'http://localhost:{self.port}') + self.client = arango.ArangoClient(hosts=f"http://localhost:{self.port}") self.client.db(verify=True) # connect to the _system db with default creds def destroy(self, delete_temp_files: bool) -> None: @@ -85,38 +101,38 @@ def destroy(self, delete_temp_files: bool) -> None: shutil.rmtree(self.temp_dir) def clear_database(self, db_name, drop_indexes=False): - ''' + """ Remove all data from a database. :param db_name: the name of the db to clear. :param drop_indexes: drop all indexes if true, retain indexes (which will be empty) if false. - ''' + """ if drop_indexes: self.client.db().delete_database(db_name) else: db = self.client.db(db_name) for c in db.collections(): - if not c['name'].startswith('_'): + if not c["name"].startswith("_"): # don't drop collection since that drops indexes - db.collection(c['name']).delete_match({}) + db.collection(c["name"]).delete_match({}) def main(): - arangoexe = Path('~/arango/3.5.0/usr/sbin/arangod') - arangojs = Path('~/arango/3.5.0/usr/share/arangodb3/js/') + arangoexe = Path("~/arango/3.5.0/usr/sbin/arangod") + arangojs = Path("~/arango/3.5.0/usr/share/arangodb3/js/") - ac = ArangoController(arangoexe, arangojs, Path('./test_temp_can_delete')) - print('port: ' + str(ac.port)) - print('temp_dir: ' + str(ac.temp_dir)) + ac = ArangoController(arangoexe, arangojs, Path("./test_temp_can_delete")) + print("port: " + str(ac.port)) + print("temp_dir: " + str(ac.temp_dir)) db = ac.client.db() # _system db - db.create_database('foo') - db = ac.client.db('foo') - db.create_collection('bar') - ac.clear_database('foo', True) - input('press enter to shut down') + db.create_database("foo") + db = ac.client.db("foo") + db.create_collection("bar") + ac.clear_database("foo", True) + input("press enter to shut down") ac.destroy(True) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/test/auth_controller.py b/test/auth_controller.py index c5c0991e..47fe26b1 100644 --- a/test/auth_controller.py +++ b/test/auth_controller.py @@ -1,7 +1,7 @@ -''' +""" A controller for the KBase Auth2 service (https://github.com/kbase/auth2) for use in testing auth-enabled applications. -''' +""" # Ported from: # https://github.com/kbase/auth2/blob/master/src/us/kbase/test/auth2/authcontroller/AuthController.java @@ -18,8 +18,8 @@ from core.test_utils import TestException from core import test_utils -_AUTH_CLASS = 'us.kbase.test.auth2.StandaloneAuthServer' -_JARS_FILE = Path(__file__).resolve().parent.joinpath('authjars') +_AUTH_CLASS = "us.kbase.test.auth2.StandaloneAuthServer" +_JARS_FILE = Path(__file__).resolve().parent.joinpath("authjars") class AuthController: @@ -32,8 +32,10 @@ class AuthController: temp_dir - the location of the Auth data and logs. """ - def __init__(self, jars_dir: Path, mongo_host: str, mongo_db: str, root_temp_dir: Path): - ''' + def __init__( + self, jars_dir: Path, mongo_host: str, mongo_db: str, root_temp_dir: Path + ): + """ Create and start a new Auth service. An unused port will be selected for the server. :param jars_dir: The path to the lib/jars dir of the KBase Jars repo @@ -43,16 +45,17 @@ def __init__(self, jars_dir: Path, mongo_host: str, mongo_db: str, root_temp_dir :param mongo_db: The database in which to store Auth data. :param root_temp_dir: A temporary directory in which to store Auth data and log files. The files will be stored inside a child directory that is unique per invocation. - ''' + """ if not jars_dir or not os.access(jars_dir, os.X_OK): - raise TestException('jars_dir {} does not exist or is not executable.' - .format(jars_dir)) + raise TestException( + "jars_dir {} does not exist or is not executable.".format(jars_dir) + ) if not mongo_host: - raise TestException('mongo_host must be provided') + raise TestException("mongo_host must be provided") if not mongo_db: - raise TestException('mongo_db must be provided') + raise TestException("mongo_db must be provided") if not root_temp_dir: - raise TestException('root_temp_dir is None') + raise TestException("root_temp_dir is None") jars_dir = jars_dir.resolve() class_path = self._get_class_path(jars_dir) @@ -60,34 +63,42 @@ def __init__(self, jars_dir: Path, mongo_host: str, mongo_db: str, root_temp_dir # make temp dirs root_temp_dir = root_temp_dir.absolute() os.makedirs(root_temp_dir, exist_ok=True) - self.temp_dir = Path(tempfile.mkdtemp(prefix='AuthController-', dir=str(root_temp_dir))) + self.temp_dir = Path( + tempfile.mkdtemp(prefix="AuthController-", dir=str(root_temp_dir)) + ) self.port = test_utils.find_free_port() - template_dir = self.temp_dir.joinpath('templates') + template_dir = self.temp_dir.joinpath("templates") self._install_templates(jars_dir, template_dir) - command = ['java', - '-classpath', class_path, - '-DAUTH2_TEST_MONGOHOST=' + mongo_host, - '-DAUTH2_TEST_MONGODB=' + mongo_db, - '-DAUTH2_TEST_TEMPLATE_DIR=' + str(template_dir), - _AUTH_CLASS, - str(self.port) - ] + command = [ + "java", + "-classpath", + class_path, + "-DAUTH2_TEST_MONGOHOST=" + mongo_host, + "-DAUTH2_TEST_MONGODB=" + mongo_db, + "-DAUTH2_TEST_TEMPLATE_DIR=" + str(template_dir), + _AUTH_CLASS, + str(self.port), + ] - self._outfile = open(self.temp_dir.joinpath('auth.log'), 'w') + self._outfile = open(self.temp_dir.joinpath("auth.log"), "w") - self._proc = subprocess.Popen(command, stdout=self._outfile, stderr=subprocess.STDOUT) + self._proc = subprocess.Popen( + command, stdout=self._outfile, stderr=subprocess.STDOUT + ) for count in range(40): err = None time.sleep(1) # wait for server to start try: res = requests.get( - f'http://localhost:{self.port}', headers={'accept': 'application/json'}) + f"http://localhost:{self.port}", + headers={"accept": "application/json"}, + ) if res.ok: - self.version = res.json()['version'] + self.version = res.json()["version"] break err = TestException(res.text) except requests.exceptions.ConnectionError as e: @@ -98,12 +109,12 @@ def __init__(self, jars_dir: Path, mongo_host: str, mongo_db: str, root_temp_dir self.startup_count = count + 1 def destroy(self, delete_temp_files: bool = True): - ''' + """ Shut down the server and optionally delete any files generated. :param delete_temp_files: if true, delete all the temporary files generated as part of running the server. - ''' + """ if self._proc: self._proc.terminate() if self._outfile: @@ -124,9 +135,9 @@ def _get_class_path(self, jars_dir: Path): with open(_JARS_FILE) as jf: jf.readline() # 1st line is template file for l in jf: - if l.strip() and not l.startswith('#'): + if l.strip() and not l.startswith("#"): p = jars_dir.joinpath(l.strip()) if not p.is_file(): - raise TestException(f'Required jar does not exist: {p}') + raise TestException(f"Required jar does not exist: {p}") cp.append(str(p)) - return ':'.join(cp) + return ":".join(cp) diff --git a/test/core/acls_test.py b/test/core/acls_test.py index 3e737deb..fd1c89bb 100644 --- a/test/core/acls_test.py +++ b/test/core/acls_test.py @@ -25,13 +25,14 @@ def test_build_ownerless(): # test duplicates are removed and order maintained a = SampleACLOwnerless( - [u('baz'), u('baz')], - read=[u('wheee'), u('wheee'), u('c')], - write=[u('wugga'), u('a'), u('b'), u('a')], - public_read=True) - assert a.admin == (u('baz'),) - assert a.write == (u('a'), u('b'), u('wugga')) - assert a.read == (u('c'), u('wheee')) + [u("baz"), u("baz")], + read=[u("wheee"), u("wheee"), u("c")], + write=[u("wugga"), u("a"), u("b"), u("a")], + public_read=True, + ) + assert a.admin == (u("baz"),) + assert a.write == (u("a"), u("b"), u("wugga")) + assert a.read == (u("c"), u("wheee")) assert a.public_read is True # test None input for public read @@ -43,23 +44,50 @@ def test_build_ownerless(): def test_build_fail_ownerless(): - _build_fail_ownerless([u('a'), None], None, None, ValueError( - 'Index 1 of iterable admin cannot be a value that evaluates to false')) - _build_fail_ownerless(None, [None, None], None, ValueError( - 'Index 0 of iterable write cannot be a value that evaluates to false')) - _build_fail_ownerless(None, None, [u('a'), u('b'), None], ValueError( - 'Index 2 of iterable read cannot be a value that evaluates to false')) + _build_fail_ownerless( + [u("a"), None], + None, + None, + ValueError( + "Index 1 of iterable admin cannot be a value that evaluates to false" + ), + ) + _build_fail_ownerless( + None, + [None, None], + None, + ValueError( + "Index 0 of iterable write cannot be a value that evaluates to false" + ), + ) + _build_fail_ownerless( + None, + None, + [u("a"), u("b"), None], + ValueError( + "Index 2 of iterable read cannot be a value that evaluates to false" + ), + ) # test that you cannot have a user in 2 acls _build_fail_ownerless( - [u('a'), u('z')], [u('a'), u('c')], [u('w'), u('b')], - IllegalParameterError('User a appears in two ACLs')) + [u("a"), u("z")], + [u("a"), u("c")], + [u("w"), u("b")], + IllegalParameterError("User a appears in two ACLs"), + ) _build_fail_ownerless( - [u('a'), u('z')], [u('b'), u('c')], [u('w'), u('a')], - IllegalParameterError('User a appears in two ACLs')) + [u("a"), u("z")], + [u("b"), u("c")], + [u("w"), u("a")], + IllegalParameterError("User a appears in two ACLs"), + ) _build_fail_ownerless( - [u('x'), u('z')], [u('b'), u('c'), u('w')], [u('w'), u('a')], - IllegalParameterError('User w appears in two ACLs')) + [u("x"), u("z")], + [u("b"), u("c"), u("w")], + [u("w"), u("a")], + IllegalParameterError("User w appears in two ACLs"), + ) def _build_fail_ownerless(admin, write, read, expected): @@ -69,20 +97,20 @@ def _build_fail_ownerless(admin, write, read, expected): def test_eq_ownerless(): - assert SampleACLOwnerless([u('bar')]) == SampleACLOwnerless([u('bar')]) - assert SampleACLOwnerless([u('bar')]) != SampleACLOwnerless([u('baz')]) + assert SampleACLOwnerless([u("bar")]) == SampleACLOwnerless([u("bar")]) + assert SampleACLOwnerless([u("bar")]) != SampleACLOwnerless([u("baz")]) - assert SampleACLOwnerless(write=[u('bar')]) == SampleACLOwnerless(write=[u('bar')]) - assert SampleACLOwnerless(write=[u('bar')]) != SampleACLOwnerless(write=[u('baz')]) + assert SampleACLOwnerless(write=[u("bar")]) == SampleACLOwnerless(write=[u("bar")]) + assert SampleACLOwnerless(write=[u("bar")]) != SampleACLOwnerless(write=[u("baz")]) - assert SampleACLOwnerless(read=[u('bar')]) == SampleACLOwnerless(read=[u('bar')]) - assert SampleACLOwnerless(read=[u('bar')]) != SampleACLOwnerless(read=[u('baz')]) + assert SampleACLOwnerless(read=[u("bar")]) == SampleACLOwnerless(read=[u("bar")]) + assert SampleACLOwnerless(read=[u("bar")]) != SampleACLOwnerless(read=[u("baz")]) assert SampleACLOwnerless(public_read=True) == SampleACLOwnerless(public_read=True) assert SampleACLOwnerless(public_read=True) != SampleACLOwnerless(public_read=False) - assert SampleACLOwnerless([u('foo')]) != 1 - assert u('foo') != SampleACLOwnerless([u('foo')]) + assert SampleACLOwnerless([u("foo")]) != 1 + assert u("foo") != SampleACLOwnerless([u("foo")]) def test_hash_ownerless(): @@ -90,22 +118,34 @@ def test_hash_ownerless(): # tests can't be written that directly test the hash value. See # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - assert hash(SampleACLOwnerless([u('bar')])) == hash(SampleACLOwnerless([u('bar')])) - assert hash(SampleACLOwnerless([u('bar')])) != hash(SampleACLOwnerless([u('baz')])) + assert hash(SampleACLOwnerless([u("bar")])) == hash(SampleACLOwnerless([u("bar")])) + assert hash(SampleACLOwnerless([u("bar")])) != hash(SampleACLOwnerless([u("baz")])) - assert hash(SampleACLOwnerless(write=[u('bar')])) == hash(SampleACLOwnerless(write=[u('bar')])) - assert hash(SampleACLOwnerless(write=[u('bar')])) != hash(SampleACLOwnerless(write=[u('baz')])) + assert hash(SampleACLOwnerless(write=[u("bar")])) == hash( + SampleACLOwnerless(write=[u("bar")]) + ) + assert hash(SampleACLOwnerless(write=[u("bar")])) != hash( + SampleACLOwnerless(write=[u("baz")]) + ) - assert hash(SampleACLOwnerless(read=[u('bar')])) == hash(SampleACLOwnerless(read=[u('bar')])) - assert hash(SampleACLOwnerless(read=[u('bar')])) != hash(SampleACLOwnerless(read=[u('baz')])) + assert hash(SampleACLOwnerless(read=[u("bar")])) == hash( + SampleACLOwnerless(read=[u("bar")]) + ) + assert hash(SampleACLOwnerless(read=[u("bar")])) != hash( + SampleACLOwnerless(read=[u("baz")]) + ) - assert hash(SampleACLOwnerless(public_read=True)) == hash(SampleACLOwnerless(public_read=True)) - assert hash(SampleACLOwnerless(public_read=True)) != hash(SampleACLOwnerless(public_read=False)) + assert hash(SampleACLOwnerless(public_read=True)) == hash( + SampleACLOwnerless(public_read=True) + ) + assert hash(SampleACLOwnerless(public_read=True)) != hash( + SampleACLOwnerless(public_read=False) + ) def test_build(): - a = SampleACL(u('foo'), dt(30)) - assert a.owner == u('foo') + a = SampleACL(u("foo"), dt(30)) + assert a.owner == u("foo") assert a.lastupdate == dt(30) assert a.admin == () assert a.write == () @@ -113,21 +153,22 @@ def test_build(): assert a.public_read is False a = SampleACL( - u('foo'), + u("foo"), dt(-56), - [u('baz'), u('bat'), u('baz')], - read=[u('wheee'), u('wheee')], - write=[u('wugga'), u('a'), u('b'), u('wugga')], - public_read=True) - assert a.owner == u('foo') + [u("baz"), u("bat"), u("baz")], + read=[u("wheee"), u("wheee")], + write=[u("wugga"), u("a"), u("b"), u("wugga")], + public_read=True, + ) + assert a.owner == u("foo") assert a.lastupdate == dt(-56) - assert a.admin == (u('bat'), u('baz')) - assert a.write == (u('a'), u('b'), u('wugga')) - assert a.read == (u('wheee'),) + assert a.admin == (u("bat"), u("baz")) + assert a.write == (u("a"), u("b"), u("wugga")) + assert a.read == (u("wheee"),) assert a.public_read is True - a = SampleACL(u('foo'), dt(30), public_read=None) - assert a.owner == u('foo') + a = SampleACL(u("foo"), dt(30), public_read=None) + assert a.owner == u("foo") assert a.lastupdate == dt(30) assert a.admin == () assert a.write == () @@ -137,40 +178,112 @@ def test_build(): def test_build_fail(): t = dt(1) - _build_fail(None, t, None, None, None, ValueError( - 'owner cannot be a value that evaluates to false')) - _build_fail(u('f'), None, None, None, None, ValueError( - 'lastupdate cannot be a value that evaluates to false')) - _build_fail(u('f'), datetime.datetime.fromtimestamp(1), None, None, None, ValueError( - 'lastupdate cannot be a naive datetime')) - _build_fail(u('foo'), t, [u('a'), None], None, None, ValueError( - 'Index 1 of iterable admin cannot be a value that evaluates to false')) - _build_fail(u('foo'), t, None, [None, None], None, ValueError( - 'Index 0 of iterable write cannot be a value that evaluates to false')) - _build_fail(u('foo'), t, None, None, [u('a'), u('b'), None], ValueError( - 'Index 2 of iterable read cannot be a value that evaluates to false')) + _build_fail( + None, + t, + None, + None, + None, + ValueError("owner cannot be a value that evaluates to false"), + ) + _build_fail( + u("f"), + None, + None, + None, + None, + ValueError("lastupdate cannot be a value that evaluates to false"), + ) + _build_fail( + u("f"), + datetime.datetime.fromtimestamp(1), + None, + None, + None, + ValueError("lastupdate cannot be a naive datetime"), + ) + _build_fail( + u("foo"), + t, + [u("a"), None], + None, + None, + ValueError( + "Index 1 of iterable admin cannot be a value that evaluates to false" + ), + ) + _build_fail( + u("foo"), + t, + None, + [None, None], + None, + ValueError( + "Index 0 of iterable write cannot be a value that evaluates to false" + ), + ) + _build_fail( + u("foo"), + t, + None, + None, + [u("a"), u("b"), None], + ValueError( + "Index 2 of iterable read cannot be a value that evaluates to false" + ), + ) # test that you cannot have an owner in another ACL _build_fail( - u('foo'), t, [u('c'), u('d'), u('foo')], None, [u('a'), u('b'), u('x')], - IllegalParameterError('The owner cannot be in any other ACL')) + u("foo"), + t, + [u("c"), u("d"), u("foo")], + None, + [u("a"), u("b"), u("x")], + IllegalParameterError("The owner cannot be in any other ACL"), + ) _build_fail( - u('foo'), t, None, [u('a'), u('b'), u('foo')], [u('x')], - IllegalParameterError('The owner cannot be in any other ACL')) + u("foo"), + t, + None, + [u("a"), u("b"), u("foo")], + [u("x")], + IllegalParameterError("The owner cannot be in any other ACL"), + ) _build_fail( - u('foo'), t, [u('y')], None, [u('a'), u('b'), u('foo')], - IllegalParameterError('The owner cannot be in any other ACL')) + u("foo"), + t, + [u("y")], + None, + [u("a"), u("b"), u("foo")], + IllegalParameterError("The owner cannot be in any other ACL"), + ) # test that you cannot have a user in 2 acls _build_fail( - u('foo'), t, [u('a'), u('z')], [u('a'), u('c')], [u('w'), u('b')], - IllegalParameterError('User a appears in two ACLs')) + u("foo"), + t, + [u("a"), u("z")], + [u("a"), u("c")], + [u("w"), u("b")], + IllegalParameterError("User a appears in two ACLs"), + ) _build_fail( - u('foo'), t, [u('a'), u('z')], [u('b'), u('c')], [u('w'), u('a')], - IllegalParameterError('User a appears in two ACLs')) + u("foo"), + t, + [u("a"), u("z")], + [u("b"), u("c")], + [u("w"), u("a")], + IllegalParameterError("User a appears in two ACLs"), + ) _build_fail( - u('foo'), t, [u('x'), u('z')], [u('b'), u('c'), u('w')], [u('w'), u('a')], - IllegalParameterError('User w appears in two ACLs')) + u("foo"), + t, + [u("x"), u("z")], + [u("b"), u("c"), u("w")], + [u("w"), u("a")], + IllegalParameterError("User w appears in two ACLs"), + ) def _build_fail(owner, lastchanged, admin, write, read, expected): @@ -181,47 +294,43 @@ def _build_fail(owner, lastchanged, admin, write, read, expected): def test_is_update(): s = SampleACL( - u('o'), - dt(1), - [u('a1'), u('a2')], - [u('w1'), u('w2')], - [u('r1'), u('r2')], - True) + u("o"), dt(1), [u("a1"), u("a2")], [u("w1"), u("w2")], [u("r1"), u("r2")], True + ) assert s.is_update(SampleACLDelta()) is False - assert s.is_update(SampleACLDelta([u('a1')])) is False - assert s.is_update(SampleACLDelta([u('a1')], at_least=True)) is False - assert s.is_update(SampleACLDelta([u('o')], at_least=True)) is False - assert s.is_update(SampleACLDelta([u('a3')])) is True - - assert s.is_update(SampleACLDelta(write=[u('w2')])) is False - assert s.is_update(SampleACLDelta(write=[u('w2')], at_least=True)) is False - assert s.is_update(SampleACLDelta(write=[u('o')], at_least=True)) is False - assert s.is_update(SampleACLDelta(write=[u('a1')])) is True - assert s.is_update(SampleACLDelta(write=[u('a1')], at_least=True)) is False - assert s.is_update(SampleACLDelta(write=[u('w4')])) is True - - assert s.is_update(SampleACLDelta(read=[u('r1')])) is False - assert s.is_update(SampleACLDelta(read=[u('r1')], at_least=True)) is False - assert s.is_update(SampleACLDelta(read=[u('o')], at_least=True)) is False - assert s.is_update(SampleACLDelta(read=[u('a1')])) is True - assert s.is_update(SampleACLDelta(read=[u('a1')], at_least=True)) is False - assert s.is_update(SampleACLDelta(read=[u('w1')])) is True - assert s.is_update(SampleACLDelta(read=[u('w1')], at_least=True)) is False - assert s.is_update(SampleACLDelta(read=[u('r3')])) is True - - assert s.is_update(SampleACLDelta(remove=[u('a1')])) is True - assert s.is_update(SampleACLDelta(remove=[u('a1')], at_least=True)) is True - assert s.is_update(SampleACLDelta(remove=[u('a3')])) is False - - assert s.is_update(SampleACLDelta(remove=[u('w2')])) is True - assert s.is_update(SampleACLDelta(remove=[u('w2')], at_least=True)) is True - assert s.is_update(SampleACLDelta(remove=[u('w4')])) is False - - assert s.is_update(SampleACLDelta(remove=[u('r1')])) is True - assert s.is_update(SampleACLDelta(remove=[u('r1')], at_least=True)) is True - assert s.is_update(SampleACLDelta(remove=[u('r3')])) is False + assert s.is_update(SampleACLDelta([u("a1")])) is False + assert s.is_update(SampleACLDelta([u("a1")], at_least=True)) is False + assert s.is_update(SampleACLDelta([u("o")], at_least=True)) is False + assert s.is_update(SampleACLDelta([u("a3")])) is True + + assert s.is_update(SampleACLDelta(write=[u("w2")])) is False + assert s.is_update(SampleACLDelta(write=[u("w2")], at_least=True)) is False + assert s.is_update(SampleACLDelta(write=[u("o")], at_least=True)) is False + assert s.is_update(SampleACLDelta(write=[u("a1")])) is True + assert s.is_update(SampleACLDelta(write=[u("a1")], at_least=True)) is False + assert s.is_update(SampleACLDelta(write=[u("w4")])) is True + + assert s.is_update(SampleACLDelta(read=[u("r1")])) is False + assert s.is_update(SampleACLDelta(read=[u("r1")], at_least=True)) is False + assert s.is_update(SampleACLDelta(read=[u("o")], at_least=True)) is False + assert s.is_update(SampleACLDelta(read=[u("a1")])) is True + assert s.is_update(SampleACLDelta(read=[u("a1")], at_least=True)) is False + assert s.is_update(SampleACLDelta(read=[u("w1")])) is True + assert s.is_update(SampleACLDelta(read=[u("w1")], at_least=True)) is False + assert s.is_update(SampleACLDelta(read=[u("r3")])) is True + + assert s.is_update(SampleACLDelta(remove=[u("a1")])) is True + assert s.is_update(SampleACLDelta(remove=[u("a1")], at_least=True)) is True + assert s.is_update(SampleACLDelta(remove=[u("a3")])) is False + + assert s.is_update(SampleACLDelta(remove=[u("w2")])) is True + assert s.is_update(SampleACLDelta(remove=[u("w2")], at_least=True)) is True + assert s.is_update(SampleACLDelta(remove=[u("w4")])) is False + + assert s.is_update(SampleACLDelta(remove=[u("r1")])) is True + assert s.is_update(SampleACLDelta(remove=[u("r1")], at_least=True)) is True + assert s.is_update(SampleACLDelta(remove=[u("r3")])) is False assert s.is_update(SampleACLDelta(public_read=False)) is True assert s.is_update(SampleACLDelta(public_read=None)) is False @@ -229,24 +338,46 @@ def test_is_update(): def test_is_update_fail(): - s = SampleACL(u('u'), dt(1)) + s = SampleACL(u("u"), dt(1)) - _is_update_fail(s, None, ValueError('update cannot be a value that evaluates to false')) _is_update_fail( - s, SampleACLDelta([u('a'), u('u')], [u('v')]), - UnauthorizedError('ACLs for the sample owner u may not be modified by a delta update.')) + s, None, ValueError("update cannot be a value that evaluates to false") + ) + _is_update_fail( + s, + SampleACLDelta([u("a"), u("u")], [u("v")]), + UnauthorizedError( + "ACLs for the sample owner u may not be modified by a delta update." + ), + ) _is_update_fail( - s, SampleACLDelta([u('a')], write=[u('v'), u('u')]), - UnauthorizedError('ACLs for the sample owner u may not be modified by a delta update.')) + s, + SampleACLDelta([u("a")], write=[u("v"), u("u")]), + UnauthorizedError( + "ACLs for the sample owner u may not be modified by a delta update." + ), + ) _is_update_fail( - s, SampleACLDelta([u('a')], read=[u('v'), u('u')]), - UnauthorizedError('ACLs for the sample owner u may not be modified by a delta update.')) + s, + SampleACLDelta([u("a")], read=[u("v"), u("u")]), + UnauthorizedError( + "ACLs for the sample owner u may not be modified by a delta update." + ), + ) _is_update_fail( - s, SampleACLDelta([u('a')], remove=[u('v'), u('u')]), - UnauthorizedError('ACLs for the sample owner u may not be modified by a delta update.')) + s, + SampleACLDelta([u("a")], remove=[u("v"), u("u")]), + UnauthorizedError( + "ACLs for the sample owner u may not be modified by a delta update." + ), + ) _is_update_fail( - s, SampleACLDelta([u('a')], remove=[u('v'), u('u')], at_least=True), - UnauthorizedError('ACLs for the sample owner u may not be modified by a delta update.')) + s, + SampleACLDelta([u("a")], remove=[u("v"), u("u")], at_least=True), + UnauthorizedError( + "ACLs for the sample owner u may not be modified by a delta update." + ), + ) def _is_update_fail(sample, delta, expected): @@ -257,24 +388,36 @@ def _is_update_fail(sample, delta, expected): def test_eq(): t = dt(3) - assert SampleACL(u('foo'), t) == SampleACL(u('foo'), t) - assert SampleACL(u('foo'), t) != SampleACL(u('bar'), t) - assert SampleACL(u('foo'), t) != SampleACL(u('foo'), dt(7)) - - assert SampleACL(u('foo'), t, [u('bar')]) == SampleACL(u('foo'), t, [u('bar')]) - assert SampleACL(u('foo'), t, [u('bar')]) != SampleACL(u('foo'), t, [u('baz')]) - - assert SampleACL(u('foo'), t, write=[u('bar')]) == SampleACL(u('foo'), t, write=[u('bar')]) - assert SampleACL(u('foo'), t, write=[u('bar')]) != SampleACL(u('foo'), t, write=[u('baz')]) - - assert SampleACL(u('foo'), t, read=[u('bar')]) == SampleACL(u('foo'), t, read=[u('bar')]) - assert SampleACL(u('foo'), t, read=[u('bar')]) != SampleACL(u('foo'), t, read=[u('baz')]) - - assert SampleACL(u('foo'), t, public_read=True) == SampleACL(u('foo'), t, public_read=True) - assert SampleACL(u('foo'), t, public_read=True) != SampleACL(u('foo'), t, public_read=False) - - assert SampleACL(u('foo'), t) != 1 - assert u('foo') != SampleACL(u('foo'), t) + assert SampleACL(u("foo"), t) == SampleACL(u("foo"), t) + assert SampleACL(u("foo"), t) != SampleACL(u("bar"), t) + assert SampleACL(u("foo"), t) != SampleACL(u("foo"), dt(7)) + + assert SampleACL(u("foo"), t, [u("bar")]) == SampleACL(u("foo"), t, [u("bar")]) + assert SampleACL(u("foo"), t, [u("bar")]) != SampleACL(u("foo"), t, [u("baz")]) + + assert SampleACL(u("foo"), t, write=[u("bar")]) == SampleACL( + u("foo"), t, write=[u("bar")] + ) + assert SampleACL(u("foo"), t, write=[u("bar")]) != SampleACL( + u("foo"), t, write=[u("baz")] + ) + + assert SampleACL(u("foo"), t, read=[u("bar")]) == SampleACL( + u("foo"), t, read=[u("bar")] + ) + assert SampleACL(u("foo"), t, read=[u("bar")]) != SampleACL( + u("foo"), t, read=[u("baz")] + ) + + assert SampleACL(u("foo"), t, public_read=True) == SampleACL( + u("foo"), t, public_read=True + ) + assert SampleACL(u("foo"), t, public_read=True) != SampleACL( + u("foo"), t, public_read=False + ) + + assert SampleACL(u("foo"), t) != 1 + assert u("foo") != SampleACL(u("foo"), t) def test_hash(): @@ -284,28 +427,38 @@ def test_hash(): t = dt(56) - assert hash(SampleACL(u('foo'), t)) == hash(SampleACL(u('foo'), t)) - assert hash(SampleACL(u('bar'), dt(5))) == hash(SampleACL(u('bar'), dt(5))) - assert hash(SampleACL(u('foo'), t)) != hash(SampleACL(u('bar'), t)) - assert hash(SampleACL(u('foo'), t)) != hash(SampleACL(u('foo'), dt(55))) - - assert hash(SampleACL(u('foo'), t, [u('bar')])) == hash(SampleACL(u('foo'), t, [u('bar')])) - assert hash(SampleACL(u('foo'), t, [u('bar')])) != hash(SampleACL(u('foo'), t, [u('baz')])) - - assert hash(SampleACL(u('foo'), t, write=[u('bar')])) == hash( - SampleACL(u('foo'), t, write=[u('bar')])) - assert hash(SampleACL(u('foo'), t, write=[u('bar')])) != hash( - SampleACL(u('foo'), t, write=[u('baz')])) - - assert hash(SampleACL(u('foo'), t, read=[u('bar')])) == hash( - SampleACL(u('foo'), t, read=[u('bar')])) - assert hash(SampleACL(u('foo'), t, read=[u('bar')])) != hash( - SampleACL(u('foo'), t, read=[u('baz')])) - - assert hash(SampleACL(u('foo'), t, public_read=True)) == hash( - SampleACL(u('foo'), t, public_read=True)) - assert hash(SampleACL(u('foo'), t, public_read=True)) != hash( - SampleACL(u('foo'), t, public_read=False)) + assert hash(SampleACL(u("foo"), t)) == hash(SampleACL(u("foo"), t)) + assert hash(SampleACL(u("bar"), dt(5))) == hash(SampleACL(u("bar"), dt(5))) + assert hash(SampleACL(u("foo"), t)) != hash(SampleACL(u("bar"), t)) + assert hash(SampleACL(u("foo"), t)) != hash(SampleACL(u("foo"), dt(55))) + + assert hash(SampleACL(u("foo"), t, [u("bar")])) == hash( + SampleACL(u("foo"), t, [u("bar")]) + ) + assert hash(SampleACL(u("foo"), t, [u("bar")])) != hash( + SampleACL(u("foo"), t, [u("baz")]) + ) + + assert hash(SampleACL(u("foo"), t, write=[u("bar")])) == hash( + SampleACL(u("foo"), t, write=[u("bar")]) + ) + assert hash(SampleACL(u("foo"), t, write=[u("bar")])) != hash( + SampleACL(u("foo"), t, write=[u("baz")]) + ) + + assert hash(SampleACL(u("foo"), t, read=[u("bar")])) == hash( + SampleACL(u("foo"), t, read=[u("bar")]) + ) + assert hash(SampleACL(u("foo"), t, read=[u("bar")])) != hash( + SampleACL(u("foo"), t, read=[u("baz")]) + ) + + assert hash(SampleACL(u("foo"), t, public_read=True)) == hash( + SampleACL(u("foo"), t, public_read=True) + ) + assert hash(SampleACL(u("foo"), t, public_read=True)) != hash( + SampleACL(u("foo"), t, public_read=False) + ) def test_delta_build(): @@ -318,16 +471,17 @@ def test_delta_build(): assert a.at_least is False a = SampleACLDelta( - [u('baz'), u('bat'), u('baz')], - read=[u('wheee'), u('wheee')], - write=[u('wugga'), u('a'), u('b'), u('wugga')], - remove=[u('bleah'), u('ffs'), u('c')], + [u("baz"), u("bat"), u("baz")], + read=[u("wheee"), u("wheee")], + write=[u("wugga"), u("a"), u("b"), u("wugga")], + remove=[u("bleah"), u("ffs"), u("c")], public_read=True, - at_least=True) - assert a.admin == (u('bat'), u('baz')) - assert a.write == (u('a'), u('b'), u('wugga')) - assert a.read == (u('wheee'),) - assert a.remove == (u('bleah'), u('c'), u('ffs')) + at_least=True, + ) + assert a.admin == (u("bat"), u("baz")) + assert a.write == (u("a"), u("b"), u("wugga")) + assert a.read == (u("wheee"),) + assert a.remove == (u("bleah"), u("c"), u("ffs")) assert a.public_read is True assert a.at_least is True @@ -349,36 +503,88 @@ def test_delta_build(): def test_delta_build_fail(): - _build_delta_fail([u('a'), None], None, None, None, ValueError( - 'Index 1 of iterable admin cannot be a value that evaluates to false')) - _build_delta_fail(None, [None, None], None, None, ValueError( - 'Index 0 of iterable write cannot be a value that evaluates to false')) - _build_delta_fail(None, None, [u('a'), u('b'), None], None, ValueError( - 'Index 2 of iterable read cannot be a value that evaluates to false')) - _build_delta_fail(None, None, None, [None], ValueError( - 'Index 0 of iterable remove cannot be a value that evaluates to false')) + _build_delta_fail( + [u("a"), None], + None, + None, + None, + ValueError( + "Index 1 of iterable admin cannot be a value that evaluates to false" + ), + ) + _build_delta_fail( + None, + [None, None], + None, + None, + ValueError( + "Index 0 of iterable write cannot be a value that evaluates to false" + ), + ) + _build_delta_fail( + None, + None, + [u("a"), u("b"), None], + None, + ValueError( + "Index 2 of iterable read cannot be a value that evaluates to false" + ), + ) + _build_delta_fail( + None, + None, + None, + [None], + ValueError( + "Index 0 of iterable remove cannot be a value that evaluates to false" + ), + ) # test that you cannot have a user in 2 acls _build_delta_fail( - [u('a'), u('z')], [u('a'), u('c')], [u('w'), u('b')], None, - IllegalParameterError('User a appears in two ACLs')) + [u("a"), u("z")], + [u("a"), u("c")], + [u("w"), u("b")], + None, + IllegalParameterError("User a appears in two ACLs"), + ) _build_delta_fail( - [u('a'), u('z')], [u('b'), u('c')], [u('w'), u('a')], None, - IllegalParameterError('User a appears in two ACLs')) + [u("a"), u("z")], + [u("b"), u("c")], + [u("w"), u("a")], + None, + IllegalParameterError("User a appears in two ACLs"), + ) _build_delta_fail( - [u('x'), u('z')], [u('b'), u('c'), u('w')], [u('w'), u('a')], None, - IllegalParameterError('User w appears in two ACLs')) + [u("x"), u("z")], + [u("b"), u("c"), u("w")], + [u("w"), u("a")], + None, + IllegalParameterError("User w appears in two ACLs"), + ) # test that you cannot have a user in the remove list and an acl _build_delta_fail( - [u('f'), u('z')], [u('b'), u('c'), u('g')], [u('w'), u('a')], [u('m'), u('f')], - IllegalParameterError('Users in the remove list cannot be in any other ACL')) + [u("f"), u("z")], + [u("b"), u("c"), u("g")], + [u("w"), u("a")], + [u("m"), u("f")], + IllegalParameterError("Users in the remove list cannot be in any other ACL"), + ) _build_delta_fail( - [u('a'), u('z')], [u('x'), u('c')], [u('w'), u('b')], [u('m'), u('x')], - IllegalParameterError('Users in the remove list cannot be in any other ACL')) + [u("a"), u("z")], + [u("x"), u("c")], + [u("w"), u("b")], + [u("m"), u("x")], + IllegalParameterError("Users in the remove list cannot be in any other ACL"), + ) _build_delta_fail( - [u('a'), u('z')], [u('b'), u('c')], [u('w'), u('y')], [u('y')], - IllegalParameterError('Users in the remove list cannot be in any other ACL')) + [u("a"), u("z")], + [u("b"), u("c")], + [u("w"), u("y")], + [u("y")], + IllegalParameterError("Users in the remove list cannot be in any other ACL"), + ) def _build_delta_fail(admin, write, read, remove, expected): @@ -390,17 +596,17 @@ def _build_delta_fail(admin, write, read, remove, expected): def test_delta_eq(): assert SampleACLDelta() == SampleACLDelta() - assert SampleACLDelta([u('bar')]) == SampleACLDelta([u('bar')]) - assert SampleACLDelta([u('bar')]) != SampleACLDelta([u('baz')]) + assert SampleACLDelta([u("bar")]) == SampleACLDelta([u("bar")]) + assert SampleACLDelta([u("bar")]) != SampleACLDelta([u("baz")]) - assert SampleACLDelta(write=[u('bar')]) == SampleACLDelta(write=[u('bar')]) - assert SampleACLDelta(write=[u('bar')]) != SampleACLDelta(write=[u('baz')]) + assert SampleACLDelta(write=[u("bar")]) == SampleACLDelta(write=[u("bar")]) + assert SampleACLDelta(write=[u("bar")]) != SampleACLDelta(write=[u("baz")]) - assert SampleACLDelta(read=[u('bar')]) == SampleACLDelta(read=[u('bar')]) - assert SampleACLDelta(read=[u('bar')]) != SampleACLDelta(read=[u('baz')]) + assert SampleACLDelta(read=[u("bar")]) == SampleACLDelta(read=[u("bar")]) + assert SampleACLDelta(read=[u("bar")]) != SampleACLDelta(read=[u("baz")]) - assert SampleACLDelta(remove=[u('bar')]) == SampleACLDelta(remove=[u('bar')]) - assert SampleACLDelta(remove=[u('bar')]) != SampleACLDelta(remove=[u('baz')]) + assert SampleACLDelta(remove=[u("bar")]) == SampleACLDelta(remove=[u("bar")]) + assert SampleACLDelta(remove=[u("bar")]) != SampleACLDelta(remove=[u("baz")]) assert SampleACLDelta(public_read=True) == SampleACLDelta(public_read=True) assert SampleACLDelta(public_read=True) != SampleACLDelta(public_read=False) @@ -419,20 +625,36 @@ def test_delta_hash(): assert hash(SampleACLDelta()) == hash(SampleACLDelta()) - assert hash(SampleACLDelta([u('bar')])) == hash(SampleACLDelta([u('bar')])) - assert hash(SampleACLDelta([u('bar')])) != hash(SampleACLDelta([u('baz')])) - - assert hash(SampleACLDelta(write=[u('bar')])) == hash(SampleACLDelta(write=[u('bar')])) - assert hash(SampleACLDelta(write=[u('bar')])) != hash(SampleACLDelta(write=[u('baz')])) - - assert hash(SampleACLDelta(read=[u('bar')])) == hash(SampleACLDelta(read=[u('bar')])) - assert hash(SampleACLDelta(read=[u('bar')])) != hash(SampleACLDelta(read=[u('baz')])) - - assert hash(SampleACLDelta(remove=[u('bar')])) == hash(SampleACLDelta(remove=[u('bar')])) - assert hash(SampleACLDelta(remove=[u('bar')])) != hash(SampleACLDelta(remove=[u('baz')])) - - assert hash(SampleACLDelta(public_read=True)) == hash(SampleACLDelta(public_read=True)) - assert hash(SampleACLDelta(public_read=True)) != hash(SampleACLDelta(public_read=False)) + assert hash(SampleACLDelta([u("bar")])) == hash(SampleACLDelta([u("bar")])) + assert hash(SampleACLDelta([u("bar")])) != hash(SampleACLDelta([u("baz")])) + + assert hash(SampleACLDelta(write=[u("bar")])) == hash( + SampleACLDelta(write=[u("bar")]) + ) + assert hash(SampleACLDelta(write=[u("bar")])) != hash( + SampleACLDelta(write=[u("baz")]) + ) + + assert hash(SampleACLDelta(read=[u("bar")])) == hash( + SampleACLDelta(read=[u("bar")]) + ) + assert hash(SampleACLDelta(read=[u("bar")])) != hash( + SampleACLDelta(read=[u("baz")]) + ) + + assert hash(SampleACLDelta(remove=[u("bar")])) == hash( + SampleACLDelta(remove=[u("bar")]) + ) + assert hash(SampleACLDelta(remove=[u("bar")])) != hash( + SampleACLDelta(remove=[u("baz")]) + ) + + assert hash(SampleACLDelta(public_read=True)) == hash( + SampleACLDelta(public_read=True) + ) + assert hash(SampleACLDelta(public_read=True)) != hash( + SampleACLDelta(public_read=False) + ) assert hash(SampleACLDelta(at_least=True)) == hash(SampleACLDelta(at_least=True)) assert hash(SampleACLDelta(at_least=True)) != hash(SampleACLDelta(at_least=False)) diff --git a/test/core/api_translation_test.py b/test/core/api_translation_test.py index 27b7cebb..c617fe2f 100644 --- a/test/core/api_translation_test.py +++ b/test/core/api_translation_test.py @@ -5,10 +5,16 @@ import json from unittest.mock import create_autospec -from SampleService.core.api_translation import datetime_to_epochmilliseconds, get_id_from_object +from SampleService.core.api_translation import ( + datetime_to_epochmilliseconds, + get_id_from_object, +) from SampleService.core.api_translation import get_version_from_object, sample_to_dict from SampleService.core.api_translation import acls_to_dict, acls_from_dict -from SampleService.core.api_translation import create_sample_params, get_sample_address_from_object +from SampleService.core.api_translation import ( + create_sample_params, + get_sample_address_from_object, +) from SampleService.core.api_translation import ( check_admin, get_static_key_metadata_params, @@ -19,7 +25,7 @@ get_data_unit_id_from_object, get_user_from_object, get_admin_request_from_object, - acl_delta_from_dict + acl_delta_from_dict, ) from SampleService.core.data_link import DataLink from SampleService.core.sample import ( @@ -36,7 +42,7 @@ IllegalParameterError, MissingParameterError, UnauthorizedError, - NoSuchUserError + NoSuchUserError, ) from SampleService.core.acls import AdminPermission from SampleService.core.user_lookup import KBaseUserLookup @@ -47,19 +53,25 @@ def test_get_user_from_object(): - assert get_user_from_object({}, 'user') is None - assert get_user_from_object({'user': None}, 'user') is None - assert get_user_from_object({'user': ' a '}, 'user') == UserID('a') + assert get_user_from_object({}, "user") is None + assert get_user_from_object({"user": None}, "user") is None + assert get_user_from_object({"user": " a "}, "user") == UserID("a") def test_get_user_from_object_fail_bad_args(): - _get_user_from_object_fail(None, 'us', ValueError('params cannot be None')) - _get_user_from_object_fail({'us': 'foo'}, None, MissingParameterError('key')) - _get_user_from_object_fail({'us': []}, 'us', IllegalParameterError( - 'us must be a string if present')) - _get_user_from_object_fail({'us': 'baz\tbaat'}, 'us', IllegalParameterError( - # probably not worth the trouble to change the key name, we'll see - 'userid contains control characters')) + _get_user_from_object_fail(None, "us", ValueError("params cannot be None")) + _get_user_from_object_fail({"us": "foo"}, None, MissingParameterError("key")) + _get_user_from_object_fail( + {"us": []}, "us", IllegalParameterError("us must be a string if present") + ) + _get_user_from_object_fail( + {"us": "baz\tbaat"}, + "us", + IllegalParameterError( + # probably not worth the trouble to change the key name, we'll see + "userid contains control characters" + ), + ) def _get_user_from_object_fail(params, key, expected): @@ -69,28 +81,50 @@ def _get_user_from_object_fail(params, key, expected): def test_get_admin_request_from_object(): - assert get_admin_request_from_object({'user': 'foo'}, 'as_ad', 'user') == (False, None) + assert get_admin_request_from_object({"user": "foo"}, "as_ad", "user") == ( + False, + None, + ) assert get_admin_request_from_object( - {'as_ad': False, 'user': 'a'}, 'as_ad', 'user') == (False, None) + {"as_ad": False, "user": "a"}, "as_ad", "user" + ) == (False, None) assert get_admin_request_from_object( - {'as_ad': [], 'user': 'a'}, 'as_ad', 'user') == (False, None) - assert get_admin_request_from_object({'as_ad': True}, 'as_ad', 'user') == (True, None) + {"as_ad": [], "user": "a"}, "as_ad", "user" + ) == (False, None) + assert get_admin_request_from_object({"as_ad": True}, "as_ad", "user") == ( + True, + None, + ) assert get_admin_request_from_object( - {'as_ad': True, 'user': None}, 'as_ad', 'user') == (True, None) + {"as_ad": True, "user": None}, "as_ad", "user" + ) == (True, None) assert get_admin_request_from_object( - {'as_ad': 3, 'user': 'a'}, 'as_ad', 'user') == (True, UserID('a')) + {"as_ad": 3, "user": "a"}, "as_ad", "user" + ) == (True, UserID("a")) def test_get_admin_request_from_object_fail_bad_args(): - _get_admin_request_from_object_fail(None, '1', '2', ValueError('params cannot be None')) - _get_admin_request_from_object_fail({'a': 'b'}, None, '2', MissingParameterError('as_admin')) - _get_admin_request_from_object_fail({'a': 'b'}, '1', None, MissingParameterError('as_user')) _get_admin_request_from_object_fail( - {'asa': True, 'asu': ['foo']}, 'asa', 'asu', IllegalParameterError( - 'asu must be a string if present')) + None, "1", "2", ValueError("params cannot be None") + ) + _get_admin_request_from_object_fail( + {"a": "b"}, None, "2", MissingParameterError("as_admin") + ) + _get_admin_request_from_object_fail( + {"a": "b"}, "1", None, MissingParameterError("as_user") + ) _get_admin_request_from_object_fail( - {'asa': True, 'asu': 'whe\tee'}, 'asa', 'asu', IllegalParameterError( - 'userid contains control characters')) + {"asa": True, "asu": ["foo"]}, + "asa", + "asu", + IllegalParameterError("asu must be a string if present"), + ) + _get_admin_request_from_object_fail( + {"asa": True, "asu": "whe\tee"}, + "asa", + "asu", + IllegalParameterError("userid contains control characters"), + ) def _get_admin_request_from_object_fail(params, akey, ukey, expected): @@ -100,36 +134,62 @@ def _get_admin_request_from_object_fail(params, akey, ukey, expected): def test_get_id_from_object(): - assert get_id_from_object(None, 'id', False) is None - assert get_id_from_object({}, 'id', False) is None - assert get_id_from_object({'id': None}, 'id', 'foo', False) is None - assert get_id_from_object({'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41e'}, 'id', False) == UUID( - 'f5bd78c3-823e-40b2-9f93-20e78680e41e') + assert get_id_from_object(None, "id", False) is None + assert get_id_from_object({}, "id", False) is None + assert get_id_from_object({"id": None}, "id", "foo", False) is None assert get_id_from_object( - {'lid': 'f5bd78c3-823e-40b2-9f93-20e78680e41e'}, - 'lid', - 'foo', - True) == UUID('f5bd78c3-823e-40b2-9f93-20e78680e41e') + {"id": "f5bd78c3-823e-40b2-9f93-20e78680e41e"}, "id", False + ) == UUID("f5bd78c3-823e-40b2-9f93-20e78680e41e") + assert get_id_from_object( + {"lid": "f5bd78c3-823e-40b2-9f93-20e78680e41e"}, "lid", "foo", True + ) == UUID("f5bd78c3-823e-40b2-9f93-20e78680e41e") def test_get_id_from_object_fail_bad_args(): - _get_id_from_object_fail(None, 'id', None, True, MissingParameterError('id')) - _get_id_from_object_fail(None, 'id', 'thing', True, MissingParameterError('thing')) - _get_id_from_object_fail({}, 'id', None, True, MissingParameterError('id')) - _get_id_from_object_fail({'id': None}, 'id', None, True, MissingParameterError('id')) - _get_id_from_object_fail({'id': None}, 'id', 'foo', True, MissingParameterError('foo')) - _get_id_from_object_fail({'wid': 6}, 'wid', None, True, IllegalParameterError( - 'wid 6 must be a UUID string')) - _get_id_from_object_fail({'id': 6}, 'id', 'whew', True, IllegalParameterError( - 'whew 6 must be a UUID string')) + _get_id_from_object_fail(None, "id", None, True, MissingParameterError("id")) + _get_id_from_object_fail(None, "id", "thing", True, MissingParameterError("thing")) + _get_id_from_object_fail({}, "id", None, True, MissingParameterError("id")) + _get_id_from_object_fail( + {"id": None}, "id", None, True, MissingParameterError("id") + ) + _get_id_from_object_fail( + {"id": None}, "id", "foo", True, MissingParameterError("foo") + ) + _get_id_from_object_fail( + {"wid": 6}, + "wid", + None, + True, + IllegalParameterError("wid 6 must be a UUID string"), + ) + _get_id_from_object_fail( + {"id": 6}, + "id", + "whew", + True, + IllegalParameterError("whew 6 must be a UUID string"), + ) + _get_id_from_object_fail( + {"id": "f5bd78c3-823e-40b2-9f93-20e78680e41"}, + "id", + None, + False, + IllegalParameterError( + "id f5bd78c3-823e-40b2-9f93-20e78680e41 must be a UUID string" + ), + ) _get_id_from_object_fail( - {'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41'}, 'id', None, False, IllegalParameterError( - 'id f5bd78c3-823e-40b2-9f93-20e78680e41 must be a UUID string')) - _get_id_from_object_fail({'id': 'bar'}, 'id', 'whee', False, IllegalParameterError( - 'whee bar must be a UUID string')) + {"id": "bar"}, + "id", + "whee", + False, + IllegalParameterError("whee bar must be a UUID string"), + ) - goodid = 'f5bd78c3-823e-40b2-9f93-20e78680e41e' - _get_id_from_object_fail({'id': goodid}, None, None, True, MissingParameterError('key')) + goodid = "f5bd78c3-823e-40b2-9f93-20e78680e41e" + _get_id_from_object_fail( + {"id": goodid}, None, None, True, MissingParameterError("key") + ) def _get_id_from_object_fail(d, key, name, required, expected): @@ -150,272 +210,423 @@ def test_to_epochmilliseconds(): def test_to_epochmilliseconds_fail_bad_args(): with raises(Exception) as got: datetime_to_epochmilliseconds(None) - assert_exception_correct(got.value, ValueError('d cannot be a value that evaluates to false')) + assert_exception_correct( + got.value, ValueError("d cannot be a value that evaluates to false") + ) def test_create_sample_params_minimal(): - params = {'sample': {'version': 7, # should be ignored - 'save_date': 9, # should be ignored - 'node_tree': [{'id': 'foo', - 'type': 'BioReplicate'}] - }} - expected = Sample([SampleNode('foo')]) + params = { + "sample": { + "version": 7, # should be ignored + "save_date": 9, # should be ignored + "node_tree": [{"id": "foo", "type": "BioReplicate"}], + } + } + expected = Sample([SampleNode("foo")]) assert create_sample_params(params) == (expected, None, None) def test_create_sample_params_maximal(): - params = {'sample': {'version': 7, # should be ignored - 'save_date': 9, # should be ignored - 'id': '706fe9e1-70ef-4feb-bbd9-32295104a119', - 'name': 'myname', - 'node_tree': [{'id': 'foo', - 'type': 'BioReplicate'}, - {'id': 'bar', - 'parent': 'foo', - 'type': 'TechReplicate', - 'meta_controlled': - {'concentration/NO2': - {'species': 'NO2', - 'units': 'ppm', - 'value': 78.91, - 'protocol_id': 782, - 'some_boolean_or_other': True - }, - 'a': {'b': 'c'} - }, - 'meta_user': {'location_name': {'name': 'my_buttocks'}}, - 'source_meta': [ - {'key': 'concentration/NO2', - 'skey': 'conc_nitrous_oxide', - 'svalue': { - 'spec': 'nit ox', - 'ppb': 0.07891, - 'prot+2': 784, - 'is this totally made up': False - } - }, - {'key': 'a', - 'skey': 'vscode', - 'svalue': {'really': 'stinks'} - } - ] - } - ] - }, - 'prior_version': 1} + params = { + "sample": { + "version": 7, # should be ignored + "save_date": 9, # should be ignored + "id": "706fe9e1-70ef-4feb-bbd9-32295104a119", + "name": "myname", + "node_tree": [ + {"id": "foo", "type": "BioReplicate"}, + { + "id": "bar", + "parent": "foo", + "type": "TechReplicate", + "meta_controlled": { + "concentration/NO2": { + "species": "NO2", + "units": "ppm", + "value": 78.91, + "protocol_id": 782, + "some_boolean_or_other": True, + }, + "a": {"b": "c"}, + }, + "meta_user": {"location_name": {"name": "my_buttocks"}}, + "source_meta": [ + { + "key": "concentration/NO2", + "skey": "conc_nitrous_oxide", + "svalue": { + "spec": "nit ox", + "ppb": 0.07891, + "prot+2": 784, + "is this totally made up": False, + }, + }, + {"key": "a", "skey": "vscode", "svalue": {"really": "stinks"}}, + ], + }, + ], + }, + "prior_version": 1, + } assert create_sample_params(params) == ( - Sample([ - SampleNode('foo'), - SampleNode( - 'bar', - SubSampleType.TECHNICAL_REPLICATE, - 'foo', - {'concentration/NO2': - {'species': 'NO2', - 'units': 'ppm', - 'value': 78.91, - 'protocol_id': 782, - 'some_boolean_or_other': True - }, - 'a': {'b': 'c'} - }, - {'location_name': {'name': 'my_buttocks'}}, - [SourceMetadata( - 'concentration/NO2', - 'conc_nitrous_oxide', + Sample( + [ + SampleNode("foo"), + SampleNode( + "bar", + SubSampleType.TECHNICAL_REPLICATE, + "foo", { - 'spec': 'nit ox', - 'ppb': 0.07891, - 'prot+2': 784, - 'is this totally made up': False - } - ), - SourceMetadata('a', 'vscode', {'really': 'stinks'}) - ] - ) + "concentration/NO2": { + "species": "NO2", + "units": "ppm", + "value": 78.91, + "protocol_id": 782, + "some_boolean_or_other": True, + }, + "a": {"b": "c"}, + }, + {"location_name": {"name": "my_buttocks"}}, + [ + SourceMetadata( + "concentration/NO2", + "conc_nitrous_oxide", + { + "spec": "nit ox", + "ppb": 0.07891, + "prot+2": 784, + "is this totally made up": False, + }, + ), + SourceMetadata("a", "vscode", {"really": "stinks"}), + ], + ), ], - 'myname' + "myname", ), - UUID('706fe9e1-70ef-4feb-bbd9-32295104a119'), - 1) + UUID("706fe9e1-70ef-4feb-bbd9-32295104a119"), + 1, + ) def test_create_sample_params_fail_bad_input(): + create_sample_params_fail(None, ValueError("params cannot be None")) create_sample_params_fail( - None, ValueError('params cannot be None')) - create_sample_params_fail( - {}, IllegalParameterError('params must contain sample key that maps to a structure')) + {}, + IllegalParameterError( + "params must contain sample key that maps to a structure" + ), + ) create_sample_params_fail( - {'sample': {}}, - IllegalParameterError('sample node tree must be present and a list')) + {"sample": {}}, + IllegalParameterError("sample node tree must be present and a list"), + ) create_sample_params_fail( - {'sample': {'node_tree': {'foo', 'bar'}}}, - IllegalParameterError('sample node tree must be present and a list')) + {"sample": {"node_tree": {"foo", "bar"}}}, + IllegalParameterError("sample node tree must be present and a list"), + ) create_sample_params_fail( - {'sample': {'node_tree': [], 'name': 6}}, - IllegalParameterError('sample name must be omitted or a string')) + {"sample": {"node_tree": [], "name": 6}}, + IllegalParameterError("sample name must be omitted or a string"), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 'BioReplicate'}, 'foo']}}, - IllegalParameterError('Node at index 1 is not a structure')) + {"sample": {"node_tree": [{"id": "foo", "type": "BioReplicate"}, "foo"]}}, + IllegalParameterError("Node at index 1 is not a structure"), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'type': 'BioReplicate'}, 'foo']}}, - IllegalParameterError('Node at index 0 must have an id key that maps to a string')) + {"sample": {"node_tree": [{"type": "BioReplicate"}, "foo"]}}, + IllegalParameterError( + "Node at index 0 must have an id key that maps to a string" + ), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 'BioReplicate'}, - {'id': None, 'type': 'BioReplicate'}, 'foo']}}, - IllegalParameterError('Node at index 1 must have an id key that maps to a string')) + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate"}, + {"id": None, "type": "BioReplicate"}, + "foo", + ] + } + }, + IllegalParameterError( + "Node at index 1 must have an id key that maps to a string" + ), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 'BioReplicate'}, - {'id': 6, 'type': 'BioReplicate'}, 'foo']}}, - IllegalParameterError('Node at index 1 must have an id key that maps to a string')) + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate"}, + {"id": 6, "type": "BioReplicate"}, + "foo", + ] + } + }, + IllegalParameterError( + "Node at index 1 must have an id key that maps to a string" + ), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 'BioReplicate'}, - {'id': 'foo'}, 'foo']}}, - IllegalParameterError('Node at index 1 has an invalid sample type: None')) + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate"}, + {"id": "foo"}, + "foo", + ] + } + }, + IllegalParameterError("Node at index 1 has an invalid sample type: None"), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 6}, - {'id': 'foo'}, 'foo']}}, - IllegalParameterError('Node at index 0 has an invalid sample type: 6')) + {"sample": {"node_tree": [{"id": "foo", "type": 6}, {"id": "foo"}, "foo"]}}, + IllegalParameterError("Node at index 0 has an invalid sample type: 6"), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 'BioReplicate2'}, - {'id': 'foo'}, 'foo']}}, - IllegalParameterError('Node at index 0 has an invalid sample type: BioReplicate2')) + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate2"}, + {"id": "foo"}, + "foo", + ] + } + }, + IllegalParameterError( + "Node at index 0 has an invalid sample type: BioReplicate2" + ), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'TechReplicate', 'parent': 6}]}}, - IllegalParameterError('Node at index 1 has a parent entry that is not a string')) + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate"}, + {"id": "foo", "type": "TechReplicate", "parent": 6}, + ] + } + }, + IllegalParameterError( + "Node at index 1 has a parent entry that is not a string" + ), + ) create_sample_params_meta_fail(6, "Node at index {}'s {} entry must be a mapping") create_sample_params_meta_fail( - {'foo': {}, 'bar': 'yay'}, - "Node at index {}'s {} entry does not have a dict as a value at key bar") + {"foo": {}, "bar": "yay"}, + "Node at index {}'s {} entry does not have a dict as a value at key bar", + ) create_sample_params_meta_fail( - {'foo': {}, 'bar': {'baz': 1, 'bat': ['yay']}}, - "Node at index {}'s {} entry does not have a primitive type as the value at bar/bat") + {"foo": {}, "bar": {"baz": 1, "bat": ["yay"]}}, + "Node at index {}'s {} entry does not have a primitive type as the value at bar/bat", + ) create_sample_params_meta_fail( - {'foo': {}, None: 'yay'}, - "Node at index {}'s {} entry contains a non-string key") + {"foo": {}, None: "yay"}, + "Node at index {}'s {} entry contains a non-string key", + ) create_sample_params_meta_fail( - {'foo': {None: 'foo'}, 'bar': {'a': 'yay'}}, - "Node at index {}'s {} entry contains a non-string key under key foo") + {"foo": {None: "foo"}, "bar": {"a": "yay"}}, + "Node at index {}'s {} entry contains a non-string key under key foo", + ) - m = {'foo': {'b\nar': 'foo'}, 'bar': {'a': 'yay'}} + m = {"foo": {"b\nar": "foo"}, "bar": {"a": "yay"}} create_sample_params_fail( - {'sample': {'node_tree': [ - {'id': 'foo', 'type': 'BioReplicate', 'meta_controlled': m}]}}, - IllegalParameterError('Error for node at index 0: Controlled metadata value key b\nar ' + - 'associated with metadata key foo has a character at index 1 that ' + - 'is a control character.')) + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate", "meta_controlled": m} + ] + } + }, + IllegalParameterError( + "Error for node at index 0: Controlled metadata value key b\nar " + + "associated with metadata key foo has a character at index 1 that " + + "is a control character." + ), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 'BioReplicate'}, - {'id': 'bar', 'type': 'TechReplicate', 'parent': 'yay'}]}}, - IllegalParameterError('Parent yay of node bar does not appear in node list prior to node.')) + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate"}, + {"id": "bar", "type": "TechReplicate", "parent": "yay"}, + ] + } + }, + IllegalParameterError( + "Parent yay of node bar does not appear in node list prior to node." + ), + ) # the id getting function is tested above so we don't repeat here, just 1 failing test create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 'BioReplicate'}, - {'id': 'bar', 'type': 'TechReplicate', 'parent': 'bar'}], - 'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41'}}, + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate"}, + {"id": "bar", "type": "TechReplicate", "parent": "bar"}, + ], + "id": "f5bd78c3-823e-40b2-9f93-20e78680e41", + } + }, IllegalParameterError( - 'sample.id f5bd78c3-823e-40b2-9f93-20e78680e41 must be a UUID string')) + "sample.id f5bd78c3-823e-40b2-9f93-20e78680e41 must be a UUID string" + ), + ) create_sample_params_fail( - {'sample': {'node_tree': [{'id': 'foo', 'type': 'BioReplicate'}, - {'id': 'bar', 'type': 'TechReplicate', 'parent': 'foo'}]}, - 'prior_version': 'six'}, - IllegalParameterError('prior_version must be an integer if supplied')) + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate"}, + {"id": "bar", "type": "TechReplicate", "parent": "foo"}, + ] + }, + "prior_version": "six", + }, + IllegalParameterError("prior_version must be an integer if supplied"), + ) def create_sample_params_meta_fail(m, expected): create_sample_params_fail( - {'sample': {'node_tree': [ - {'id': 'foo', 'type': 'BioReplicate', 'meta_controlled': m}]}}, - IllegalParameterError(expected.format(0, 'controlled metadata'))) + { + "sample": { + "node_tree": [ + {"id": "foo", "type": "BioReplicate", "meta_controlled": m} + ] + } + }, + IllegalParameterError(expected.format(0, "controlled metadata")), + ) create_sample_params_fail( - {'sample': {'node_tree': [ - {'id': 'bar', 'type': 'BioReplicate'}, - {'id': 'foo', 'type': 'SubSample', 'parent': 'bar', 'meta_user': m}]}}, - IllegalParameterError(expected.format(1, 'user metadata'))) + { + "sample": { + "node_tree": [ + {"id": "bar", "type": "BioReplicate"}, + {"id": "foo", "type": "SubSample", "parent": "bar", "meta_user": m}, + ] + } + }, + IllegalParameterError(expected.format(1, "user metadata")), + ) def test_create_sample_params_source_metadata_fail(): _create_sample_params_source_metadata_fail( - {'a': {'b': 'c'}}, {'a': {'b': 'c'}}, - IllegalParameterError("Node at index 1's source metadata must be a list") + {"a": {"b": "c"}}, + {"a": {"b": "c"}}, + IllegalParameterError("Node at index 1's source metadata must be a list"), ) _create_sample_params_source_metadata_fail( - {'a': {'b': 'c'}}, [{'key': 'a', 'skey': 'b', 'svalue': {'b': 'c'}}, 'foo'], + {"a": {"b": "c"}}, + [{"key": "a", "skey": "b", "svalue": {"b": "c"}}, "foo"], IllegalParameterError( - "Node at index 1's source metadata has an entry at index 1 that is not a dict") + "Node at index 1's source metadata has an entry at index 1 that is not a dict" + ), ) _create_sample_params_source_metadata_fail( - {'a': {'b': 'c'}}, - [{'key': 'a', 'skey': 'b', 'svalue': {'b': 'c'}}, - {'key': 8, 'skey': 'b', 'svalue': {'b': 'c'}}], - IllegalParameterError("Node at index 1's source metadata has an entry at index 1 " + - 'where the required key field is not a string') + {"a": {"b": "c"}}, + [ + {"key": "a", "skey": "b", "svalue": {"b": "c"}}, + {"key": 8, "skey": "b", "svalue": {"b": "c"}}, + ], + IllegalParameterError( + "Node at index 1's source metadata has an entry at index 1 " + + "where the required key field is not a string" + ), ) _create_sample_params_source_metadata_fail( - {'a': {'b': 'c'}}, - [{'key': 'a', 'skey': [], 'svalue': {'b': 'c'}}, - {'key': 'b', 'skey': 'b', 'svalue': {'b': 'c'}}], - IllegalParameterError("Node at index 1's source metadata has an entry at index 0 " + - 'where the required skey field is not a string') + {"a": {"b": "c"}}, + [ + {"key": "a", "skey": [], "svalue": {"b": "c"}}, + {"key": "b", "skey": "b", "svalue": {"b": "c"}}, + ], + IllegalParameterError( + "Node at index 1's source metadata has an entry at index 0 " + + "where the required skey field is not a string" + ), ) _create_sample_params_source_metadata_fail( - {'a': {'b': 'c'}}, - [{'key': 'a', 'skey': 'a', 'svalue': {'b': 'c'}}, - {'key': 'b', 'skey': 'b', 'svalue': ['f']}], - IllegalParameterError("Node at index 1's source metadata has an entry at index 1 " + - 'where the required svalue field is not a mapping') + {"a": {"b": "c"}}, + [ + {"key": "a", "skey": "a", "svalue": {"b": "c"}}, + {"key": "b", "skey": "b", "svalue": ["f"]}, + ], + IllegalParameterError( + "Node at index 1's source metadata has an entry at index 1 " + + "where the required svalue field is not a mapping" + ), ) _create_sample_params_source_metadata_fail( - {'a': {'b': 'c'}}, - [{'key': 'a', 'skey': 'a', 'svalue': {8: 'c'}}, - {'key': 'b', 'skey': 'b', 'svalue': {'b': 'c'}}], - IllegalParameterError("Node at index 1's source metadata has an entry at index 0 " + - 'with a value mapping key that is not a string') + {"a": {"b": "c"}}, + [ + {"key": "a", "skey": "a", "svalue": {8: "c"}}, + {"key": "b", "skey": "b", "svalue": {"b": "c"}}, + ], + IllegalParameterError( + "Node at index 1's source metadata has an entry at index 0 " + + "with a value mapping key that is not a string" + ), ) _create_sample_params_source_metadata_fail( - {'a': {'b': 'c'}}, - [{'key': 'a', 'skey': 'a', 'svalue': {'c': [43]}}, - {'key': 'b', 'skey': 'b', 'svalue': {'b': 'c'}}], + {"a": {"b": "c"}}, + [ + {"key": "a", "skey": "a", "svalue": {"c": [43]}}, + {"key": "b", "skey": "b", "svalue": {"b": "c"}}, + ], IllegalParameterError( - "Node at index 1's source metadata has an entry at index 0 with a value in the " + - 'value mapping under key c that is not a primitive type') + "Node at index 1's source metadata has an entry at index 0 with a value in the " + + "value mapping under key c that is not a primitive type" + ), ) _create_sample_params_source_metadata_fail( - {'a': {'b': 'c'}}, - [{'key': 'a', 'skey': 'a', 'svalue': {'c': 'x'}}, - {'key': 'b', 'skey': 'b', 'svalue': {'b\n': 'c'}}], + {"a": {"b": "c"}}, + [ + {"key": "a", "skey": "a", "svalue": {"c": "x"}}, + {"key": "b", "skey": "b", "svalue": {"b\n": "c"}}, + ], IllegalParameterError( - "Node at index 1's source metadata has an error at index 1: Source metadata value " + - 'key b\n associated with metadata key b has a character at index 1 that is a control ' + - 'character.') + "Node at index 1's source metadata has an error at index 1: Source metadata value " + + "key b\n associated with metadata key b has a character at index 1 that is a control " + + "character." + ), ) _create_sample_params_source_metadata_fail( - {'a': {'b': 'c'}}, - [{'key': 'a', 'skey': 'a', 'svalue': {'c': 'x'}}, - {'key': 'b', 'skey': 'b', 'svalue': {'b': 'c'}}], + {"a": {"b": "c"}}, + [ + {"key": "a", "skey": "a", "svalue": {"c": "x"}}, + {"key": "b", "skey": "b", "svalue": {"b": "c"}}, + ], IllegalParameterError( - 'Error for node at index 1: Source metadata key b does not appear in the ' + - 'controlled metadata') + "Error for node at index 1: Source metadata key b does not appear in the " + + "controlled metadata" + ), ) def _create_sample_params_source_metadata_fail(m, s, expected): create_sample_params_fail( - {'sample': {'node_tree': [ - {'id': 'bar', 'type': 'BioReplicate'}, - {'id': 'foo', - 'type': 'SubSample', - 'parent': 'bar', - 'meta_controlled': m, - 'source_meta': s}]}}, - expected) + { + "sample": { + "node_tree": [ + {"id": "bar", "type": "BioReplicate"}, + { + "id": "foo", + "type": "SubSample", + "parent": "bar", + "meta_controlled": m, + "source_meta": s, + }, + ] + } + }, + expected, + ) def create_sample_params_fail(params, expected): @@ -426,22 +637,28 @@ def create_sample_params_fail(params, expected): def test_get_version_from_object(): assert get_version_from_object({}) is None - assert get_version_from_object({'version': None}) is None - assert get_version_from_object({'version': 3}, True) == 3 - assert get_version_from_object({'version': 1}) == 1 + assert get_version_from_object({"version": None}) is None + assert get_version_from_object({"version": 3}, True) == 3 + assert get_version_from_object({"version": 1}) == 1 def test_get_version_from_object_fail_bad_args(): - get_version_from_object_fail(None, False, ValueError('params cannot be None')) - get_version_from_object_fail({}, True, MissingParameterError('version')) + get_version_from_object_fail(None, False, ValueError("params cannot be None")) + get_version_from_object_fail({}, True, MissingParameterError("version")) get_version_from_object_fail( - {'version': None}, True, MissingParameterError('version')) + {"version": None}, True, MissingParameterError("version") + ) get_version_from_object_fail( - {'version': 'whee'}, False, IllegalParameterError('Illegal version argument: whee')) + {"version": "whee"}, + False, + IllegalParameterError("Illegal version argument: whee"), + ) get_version_from_object_fail( - {'version': 0}, True, IllegalParameterError('Illegal version argument: 0')) + {"version": 0}, True, IllegalParameterError("Illegal version argument: 0") + ) get_version_from_object_fail( - {'version': -3}, False, IllegalParameterError('Illegal version argument: -3')) + {"version": -3}, False, IllegalParameterError("Illegal version argument: -3") + ) def get_version_from_object_fail(params, required, expected): @@ -451,38 +668,59 @@ def get_version_from_object_fail(params, required, expected): def test_get_sample_address_from_object(): - assert get_sample_address_from_object({'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41e'}) == ( - UUID('f5bd78c3-823e-40b2-9f93-20e78680e41e'), None) - assert get_sample_address_from_object({ - 'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41e', - 'version': 1}, version_required=True) == ( - UUID('f5bd78c3-823e-40b2-9f93-20e78680e41e'), 1) + assert get_sample_address_from_object( + {"id": "f5bd78c3-823e-40b2-9f93-20e78680e41e"} + ) == (UUID("f5bd78c3-823e-40b2-9f93-20e78680e41e"), None) + assert ( + get_sample_address_from_object( + {"id": "f5bd78c3-823e-40b2-9f93-20e78680e41e", "version": 1}, + version_required=True, + ) + == (UUID("f5bd78c3-823e-40b2-9f93-20e78680e41e"), 1) + ) def test_get_sample_address_from_object_fail_bad_args(): - get_sample_address_from_object_fail(None, False, MissingParameterError('id')) - get_sample_address_from_object_fail({}, False, MissingParameterError('id')) - get_sample_address_from_object_fail({'id': None}, False, MissingParameterError('id')) - get_sample_address_from_object_fail({'id': 6}, False, IllegalParameterError( - 'id 6 must be a UUID string')) + get_sample_address_from_object_fail(None, False, MissingParameterError("id")) + get_sample_address_from_object_fail({}, False, MissingParameterError("id")) + get_sample_address_from_object_fail( + {"id": None}, False, MissingParameterError("id") + ) get_sample_address_from_object_fail( - {'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41'}, False, + {"id": 6}, False, IllegalParameterError("id 6 must be a UUID string") + ) + get_sample_address_from_object_fail( + {"id": "f5bd78c3-823e-40b2-9f93-20e78680e41"}, + False, IllegalParameterError( - 'id f5bd78c3-823e-40b2-9f93-20e78680e41 must be a UUID string')) - id_ = 'f5bd78c3-823e-40b2-9f93-20e78680e41e' + "id f5bd78c3-823e-40b2-9f93-20e78680e41 must be a UUID string" + ), + ) + id_ = "f5bd78c3-823e-40b2-9f93-20e78680e41e" get_sample_address_from_object_fail( - {'id': id_}, True, MissingParameterError('version')) + {"id": id_}, True, MissingParameterError("version") + ) get_sample_address_from_object_fail( - {'id': id_, 'version': [1]}, False, IllegalParameterError('Illegal version argument: [1]')) + {"id": id_, "version": [1]}, + False, + IllegalParameterError("Illegal version argument: [1]"), + ) get_version_from_object_fail( - {'id': id_, 'version': 'whee'}, + {"id": id_, "version": "whee"}, True, - IllegalParameterError('Illegal version argument: whee')) + IllegalParameterError("Illegal version argument: whee"), + ) get_version_from_object_fail( - {'id': id_, 'version': 0}, True, IllegalParameterError('Illegal version argument: 0')) + {"id": id_, "version": 0}, + True, + IllegalParameterError("Illegal version argument: 0"), + ) get_version_from_object_fail( - {'id': id_, 'version': -3}, True, IllegalParameterError('Illegal version argument: -3')) + {"id": id_, "version": -3}, + True, + IllegalParameterError("Illegal version argument: -3"), + ) def get_sample_address_from_object_fail(params, required, expected): @@ -493,23 +731,29 @@ def get_sample_address_from_object_fail(params, required, expected): def test_sample_to_dict_minimal(): - expected = {'node_tree': [{'id': 'foo', - 'type': 'BioReplicate', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - 'parent': None - }], - 'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41e', - 'user': 'user2', - 'save_date': 87897, - 'name': None, - 'version': None, - } + expected = { + "node_tree": [ + { + "id": "foo", + "type": "BioReplicate", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + "parent": None, + } + ], + "id": "f5bd78c3-823e-40b2-9f93-20e78680e41e", + "user": "user2", + "save_date": 87897, + "name": None, + "version": None, + } - id_ = UUID('f5bd78c3-823e-40b2-9f93-20e78680e41e') + id_ = UUID("f5bd78c3-823e-40b2-9f93-20e78680e41e") - s = sample_to_dict(SavedSample(id_, UserID('user2'), [SampleNode('foo')], dt(87.8971))) + s = sample_to_dict( + SavedSample(id_, UserID("user2"), [SampleNode("foo")], dt(87.8971)) + ) assert s == expected @@ -518,48 +762,60 @@ def test_sample_to_dict_minimal(): def test_sample_to_dict_maximal(): - expected = {'node_tree': [{'id': 'foo', - 'type': 'BioReplicate', - 'meta_controlled': {}, - 'meta_user': {}, - 'source_meta': [], - 'parent': None - }, - {'id': 'bar', - 'type': 'TechReplicate', - 'meta_controlled': {'a': {'b': 'c', 'm': 6.7}, 'b': {'c': 'd'}}, - 'meta_user': {'d': {'e': True}, 'g': {'h': 1}}, - 'source_meta': [ - {'key': 'a', 'skey': 'x', 'svalue': {'v': 2}}, - {'key': 'b', 'skey': 'y', 'svalue': {'z': 3}} - ], - 'parent': 'foo' - }], - 'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41e', - 'user': 'user3', - 'save_date': 87897, - 'name': 'myname', - 'version': 23, - } - - id_ = UUID('f5bd78c3-823e-40b2-9f93-20e78680e41e') + expected = { + "node_tree": [ + { + "id": "foo", + "type": "BioReplicate", + "meta_controlled": {}, + "meta_user": {}, + "source_meta": [], + "parent": None, + }, + { + "id": "bar", + "type": "TechReplicate", + "meta_controlled": {"a": {"b": "c", "m": 6.7}, "b": {"c": "d"}}, + "meta_user": {"d": {"e": True}, "g": {"h": 1}}, + "source_meta": [ + {"key": "a", "skey": "x", "svalue": {"v": 2}}, + {"key": "b", "skey": "y", "svalue": {"z": 3}}, + ], + "parent": "foo", + }, + ], + "id": "f5bd78c3-823e-40b2-9f93-20e78680e41e", + "user": "user3", + "save_date": 87897, + "name": "myname", + "version": 23, + } + + id_ = UUID("f5bd78c3-823e-40b2-9f93-20e78680e41e") s = sample_to_dict( SavedSample( id_, - UserID('user3'), - [SampleNode('foo'), - SampleNode( - 'bar', - SubSampleType.TECHNICAL_REPLICATE, - 'foo', - {'a': {'b': 'c', 'm': 6.7}, 'b': {'c': 'd'}}, - {'d': {'e': True}, 'g': {'h': 1}}, - [SourceMetadata('a', 'x', {'v': 2}), SourceMetadata('b', 'y', {'z': 3})]), - ], + UserID("user3"), + [ + SampleNode("foo"), + SampleNode( + "bar", + SubSampleType.TECHNICAL_REPLICATE, + "foo", + {"a": {"b": "c", "m": 6.7}, "b": {"c": "d"}}, + {"d": {"e": True}, "g": {"h": 1}}, + [ + SourceMetadata("a", "x", {"v": 2}), + SourceMetadata("b", "y", {"z": 3}), + ], + ), + ], dt(87.8971), - 'myname', - 23)) + "myname", + 23, + ) + ) assert s == expected @@ -571,50 +827,56 @@ def test_sample_to_dict_fail(): with raises(Exception) as got: sample_to_dict(None) assert_exception_correct( - got.value, ValueError('sample cannot be a value that evaluates to false')) + got.value, ValueError("sample cannot be a value that evaluates to false") + ) def test_acls_to_dict_minimal(): - assert acls_to_dict(SampleACL(UserID('user'), dt(1))) == { - 'owner': 'user', - 'admin': (), - 'write': (), - 'read': (), - 'public_read': 0 + assert acls_to_dict(SampleACL(UserID("user"), dt(1))) == { + "owner": "user", + "admin": (), + "write": (), + "read": (), + "public_read": 0, } def test_acls_to_dict_maximal(): assert acls_to_dict( SampleACL( - UserID('user'), + UserID("user"), dt(1), - [UserID('foo'), UserID('bar')], - [UserID('baz')], - [UserID('hello'), UserID("I'm"), UserID('a'), UserID('robot')], - True)) == { - 'owner': 'user', - 'admin': ('bar', 'foo'), - 'write': ('baz',), - 'read': ("I'm", 'a', 'hello', 'robot'), - 'public_read': 1 + [UserID("foo"), UserID("bar")], + [UserID("baz")], + [UserID("hello"), UserID("I'm"), UserID("a"), UserID("robot")], + True, + ) + ) == { + "owner": "user", + "admin": ("bar", "foo"), + "write": ("baz",), + "read": ("I'm", "a", "hello", "robot"), + "public_read": 1, } def test_acls_to_dict_remove_service_token(): assert acls_to_dict( SampleACL( - UserID('user'), + UserID("user"), dt(1), - [UserID('foo'), UserID('bar')], - [UserID('baz')], - [UserID('hello'), UserID("I'm"), UserID('a'), UserID('robot')], - True), read_exempt_roles=["I'm", 'a']) == { - 'owner': 'user', - 'admin': ('bar', 'foo'), - 'write': ('baz',), - 'read': ('hello', 'robot'), - 'public_read': 1 + [UserID("foo"), UserID("bar")], + [UserID("baz")], + [UserID("hello"), UserID("I'm"), UserID("a"), UserID("robot")], + True, + ), + read_exempt_roles=["I'm", "a"], + ) == { + "owner": "user", + "admin": ("bar", "foo"), + "write": ("baz",), + "read": ("hello", "robot"), + "public_read": 1, } @@ -622,41 +884,54 @@ def test_acls_to_dict_fail(): with raises(Exception) as got: acls_to_dict(None) assert_exception_correct( - got.value, ValueError('acls cannot be a value that evaluates to false')) + got.value, ValueError("acls cannot be a value that evaluates to false") + ) def test_acls_from_dict(): - assert acls_from_dict({'acls': {}}) == SampleACLOwnerless() - assert acls_from_dict({'acls': {'public_read': 0}}) == SampleACLOwnerless() - assert acls_from_dict({'acls': { - 'read': [], - 'admin': ['whee', 'whoo'], - 'public_read': None}}) == SampleACLOwnerless([UserID('whee'), UserID('whoo')]) - assert acls_from_dict({'acls': { - 'read': ['a', 'b'], - 'write': ['x'], - 'admin': ['whee', 'whoo'], - 'public_read': 1}}) == SampleACLOwnerless( - [UserID('whee'), UserID('whoo')], [UserID('x')], [UserID('a'), UserID('b')], True) + assert acls_from_dict({"acls": {}}) == SampleACLOwnerless() + assert acls_from_dict({"acls": {"public_read": 0}}) == SampleACLOwnerless() + assert acls_from_dict( + {"acls": {"read": [], "admin": ["whee", "whoo"], "public_read": None}} + ) == SampleACLOwnerless([UserID("whee"), UserID("whoo")]) + assert acls_from_dict( + { + "acls": { + "read": ["a", "b"], + "write": ["x"], + "admin": ["whee", "whoo"], + "public_read": 1, + } + } + ) == SampleACLOwnerless( + [UserID("whee"), UserID("whoo")], + [UserID("x")], + [UserID("a"), UserID("b")], + True, + ) def test_acls_from_dict_fail_bad_args(): - _acls_from_dict_fail(None, ValueError('d cannot be a value that evaluates to false')) - _acls_from_dict_fail({}, ValueError('d cannot be a value that evaluates to false')) - m = 'ACLs must be supplied in the acls key and must be a mapping' - _acls_from_dict_fail({'acls': None}, IllegalParameterError(m)) - _acls_from_dict_fail({'acls': 'foo'}, IllegalParameterError(m)) - _acls_from_dict_fail_acl_check('read') - _acls_from_dict_fail_acl_check('write') - _acls_from_dict_fail_acl_check('admin') + _acls_from_dict_fail( + None, ValueError("d cannot be a value that evaluates to false") + ) + _acls_from_dict_fail({}, ValueError("d cannot be a value that evaluates to false")) + m = "ACLs must be supplied in the acls key and must be a mapping" + _acls_from_dict_fail({"acls": None}, IllegalParameterError(m)) + _acls_from_dict_fail({"acls": "foo"}, IllegalParameterError(m)) + _acls_from_dict_fail_acl_check("read") + _acls_from_dict_fail_acl_check("write") + _acls_from_dict_fail_acl_check("admin") def _acls_from_dict_fail_acl_check(acltype): - _acls_from_dict_fail({'acls': {acltype: {}}}, - IllegalParameterError(f'{acltype} ACL must be a list')) _acls_from_dict_fail( - {'acls': {acltype: ['one', 2, 'three']}}, - IllegalParameterError(f'Index 1 of {acltype} ACL does not contain a string')) + {"acls": {acltype: {}}}, IllegalParameterError(f"{acltype} ACL must be a list") + ) + _acls_from_dict_fail( + {"acls": {acltype: ["one", 2, "three"]}}, + IllegalParameterError(f"Index 1 of {acltype} ACL does not contain a string"), + ) def _acls_from_dict_fail(d, expected): @@ -667,47 +942,69 @@ def _acls_from_dict_fail(d, expected): def test_acl_delta_from_dict(): assert acl_delta_from_dict({}) == SampleACLDelta() - assert acl_delta_from_dict({'public_read': 0, 'at_least': 0}) == SampleACLDelta() - assert acl_delta_from_dict({'public_read': 1, 'at_least': None}) == SampleACLDelta( - public_read=True) - assert acl_delta_from_dict({'public_read': -1, 'at_least': 1}) == SampleACLDelta( - public_read=False, at_least=True) - assert acl_delta_from_dict({'public_read': -50, 'at_least': -50}) == SampleACLDelta( - public_read=False, at_least=True) - assert acl_delta_from_dict({ - 'read': [], - 'admin': ['whee', 'whoo'], - 'public_read': None, - 'at_least': 100}) == SampleACLDelta([UserID('whee'), UserID('whoo')], at_least=True) - assert acl_delta_from_dict({ - 'read': ['a', 'b'], - 'write': ['c'], - 'admin': ['whee', 'whoo'], - 'remove': ['e', 'f'], - 'public_read': 100}) == SampleACLDelta( - [UserID('whee'), UserID('whoo')], [UserID('c')], [UserID('a'), UserID('b')], - [UserID('e'), UserID('f')], True) + assert acl_delta_from_dict({"public_read": 0, "at_least": 0}) == SampleACLDelta() + assert acl_delta_from_dict({"public_read": 1, "at_least": None}) == SampleACLDelta( + public_read=True + ) + assert acl_delta_from_dict({"public_read": -1, "at_least": 1}) == SampleACLDelta( + public_read=False, at_least=True + ) + assert acl_delta_from_dict({"public_read": -50, "at_least": -50}) == SampleACLDelta( + public_read=False, at_least=True + ) + assert acl_delta_from_dict( + {"read": [], "admin": ["whee", "whoo"], "public_read": None, "at_least": 100} + ) == SampleACLDelta([UserID("whee"), UserID("whoo")], at_least=True) + assert acl_delta_from_dict( + { + "read": ["a", "b"], + "write": ["c"], + "admin": ["whee", "whoo"], + "remove": ["e", "f"], + "public_read": 100, + } + ) == SampleACLDelta( + [UserID("whee"), UserID("whoo")], + [UserID("c")], + [UserID("a"), UserID("b")], + [UserID("e"), UserID("f")], + True, + ) def test_acl_delta_from_dict_fail_bad_args(): - _acl_delta_from_dict_fail({'public_read': '0'}, IllegalParameterError( - 'public_read must be an integer if present')) - _acl_delta_from_dict_fail({'admin': {}}, IllegalParameterError( - 'admin ACL must be a list')) - _acl_delta_from_dict_fail({'admin': ['foo', 1, '32']}, IllegalParameterError( - 'Index 1 of admin ACL does not contain a string')) - _acl_delta_from_dict_fail({'write': 'foo'}, IllegalParameterError( - 'write ACL must be a list')) - _acl_delta_from_dict_fail({'write': [[], 1, '32']}, IllegalParameterError( - 'Index 0 of write ACL does not contain a string')) - _acl_delta_from_dict_fail({'read': 64.2}, IllegalParameterError( - 'read ACL must be a list')) - _acl_delta_from_dict_fail({'read': ['f', 'z', {}]}, IllegalParameterError( - 'Index 2 of read ACL does not contain a string')) - _acl_delta_from_dict_fail({'remove': (1,)}, IllegalParameterError( - 'remove ACL must be a list')) - _acl_delta_from_dict_fail({'remove': ['f', id, {}]}, IllegalParameterError( - 'Index 1 of remove ACL does not contain a string')) + _acl_delta_from_dict_fail( + {"public_read": "0"}, + IllegalParameterError("public_read must be an integer if present"), + ) + _acl_delta_from_dict_fail( + {"admin": {}}, IllegalParameterError("admin ACL must be a list") + ) + _acl_delta_from_dict_fail( + {"admin": ["foo", 1, "32"]}, + IllegalParameterError("Index 1 of admin ACL does not contain a string"), + ) + _acl_delta_from_dict_fail( + {"write": "foo"}, IllegalParameterError("write ACL must be a list") + ) + _acl_delta_from_dict_fail( + {"write": [[], 1, "32"]}, + IllegalParameterError("Index 0 of write ACL does not contain a string"), + ) + _acl_delta_from_dict_fail( + {"read": 64.2}, IllegalParameterError("read ACL must be a list") + ) + _acl_delta_from_dict_fail( + {"read": ["f", "z", {}]}, + IllegalParameterError("Index 2 of read ACL does not contain a string"), + ) + _acl_delta_from_dict_fail( + {"remove": (1,)}, IllegalParameterError("remove ACL must be a list") + ) + _acl_delta_from_dict_fail( + {"remove": ["f", id, {}]}, + IllegalParameterError("Index 1 of remove ACL does not contain a string"), + ) def _acl_delta_from_dict_fail(d, expected): @@ -719,15 +1016,39 @@ def _acl_delta_from_dict_fail(d, expected): def test_check_admin(): f = AdminPermission.FULL r = AdminPermission.READ - _check_admin(f, f, 'user1', 'somemethod', None, - 'User user1 is running method somemethod with administration permission FULL') - _check_admin(f, f, 'user1', 'somemethod', UserID('otheruser'), - 'User user1 is running method somemethod with administration permission FULL ' + - 'as user otheruser') - _check_admin(f, r, 'someuser', 'a_method', None, - 'User someuser is running method a_method with administration permission FULL') - _check_admin(r, r, 'user2', 'm', None, - 'User user2 is running method m with administration permission READ') + _check_admin( + f, + f, + "user1", + "somemethod", + None, + "User user1 is running method somemethod with administration permission FULL", + ) + _check_admin( + f, + f, + "user1", + "somemethod", + UserID("otheruser"), + "User user1 is running method somemethod with administration permission FULL " + + "as user otheruser", + ) + _check_admin( + f, + r, + "someuser", + "a_method", + None, + "User someuser is running method a_method with administration permission FULL", + ) + _check_admin( + r, + r, + "user2", + "m", + None, + "User user2 is running method m with administration permission READ", + ) def _check_admin(perm, permreq, user, method, as_user, expected_log): @@ -737,10 +1058,14 @@ def _check_admin(perm, permreq, user, method, as_user, expected_log): ul.is_admin.return_value = (perm, user) ul.invalid_users.return_value = [] - assert check_admin( - ul, 'thisisatoken', permreq, method, lambda x: logs.append(x), as_user) is True + assert ( + check_admin( + ul, "thisisatoken", permreq, method, lambda x: logs.append(x), as_user + ) + is True + ) - assert ul.is_admin.call_args_list == [(('thisisatoken',), {})] + assert ul.is_admin.call_args_list == [(("thisisatoken",), {})] if as_user: ul.invalid_users.assert_called_once_with([as_user]) assert logs == [expected_log] @@ -750,10 +1075,19 @@ def test_check_admin_skip(): ul = create_autospec(KBaseUserLookup, spec_set=True, instance=True) logs = [] - ul.is_admin.return_value = (AdminPermission.FULL, 'u') + ul.is_admin.return_value = (AdminPermission.FULL, "u") - assert check_admin(ul, 'thisisatoken', AdminPermission.FULL, 'm', lambda x: logs.append(x), - skip_check=True) is False + assert ( + check_admin( + ul, + "thisisatoken", + AdminPermission.FULL, + "m", + lambda x: logs.append(x), + skip_check=True, + ) + is False + ) assert ul.is_admin.call_args_list == [] assert logs == [] @@ -763,29 +1097,80 @@ def test_check_admin_fail_bad_args(): ul = create_autospec(KBaseUserLookup, spec_set=True, instance=True) p = AdminPermission.FULL - _check_admin_fail(None, 't', p, 'm', lambda _: None, None, ValueError( - 'user_lookup cannot be a value that evaluates to false')) - _check_admin_fail(ul, '', p, 'm', lambda _: None, None, UnauthorizedError( - 'Anonymous users may not act as service administrators.')) - _check_admin_fail(ul, 't', None, 'm', lambda _: None, None, ValueError( - 'perm cannot be a value that evaluates to false')) - _check_admin_fail(ul, 't', p, None, lambda _: None, None, ValueError( - 'method cannot be a value that evaluates to false')) - _check_admin_fail(ul, 't', p, 'm', None, None, ValueError( - 'log_fn cannot be a value that evaluates to false')) + _check_admin_fail( + None, + "t", + p, + "m", + lambda _: None, + None, + ValueError("user_lookup cannot be a value that evaluates to false"), + ) + _check_admin_fail( + ul, + "", + p, + "m", + lambda _: None, + None, + UnauthorizedError("Anonymous users may not act as service administrators."), + ) + _check_admin_fail( + ul, + "t", + None, + "m", + lambda _: None, + None, + ValueError("perm cannot be a value that evaluates to false"), + ) + _check_admin_fail( + ul, + "t", + p, + None, + lambda _: None, + None, + ValueError("method cannot be a value that evaluates to false"), + ) + _check_admin_fail( + ul, + "t", + p, + "m", + None, + None, + ValueError("log_fn cannot be a value that evaluates to false"), + ) def test_check_admin_fail_none_perm(): ul = create_autospec(KBaseUserLookup, spec_set=True, instance=True) - _check_admin_fail(ul, 't', AdminPermission.NONE, 'm', lambda _: None, None, ValueError( - 'what are you doing calling this method with no permission requirement? ' + - 'That totally makes no sense. Get a brain moran')) + _check_admin_fail( + ul, + "t", + AdminPermission.NONE, + "m", + lambda _: None, + None, + ValueError( + "what are you doing calling this method with no permission requirement? " + + "That totally makes no sense. Get a brain moran" + ), + ) def test_check_admin_fail_read_with_impersonate(): ul = create_autospec(KBaseUserLookup, spec_set=True, instance=True) - _check_admin_fail(ul, 't', AdminPermission.READ, 'm', lambda _: None, 'user', ValueError( - 'as_user is supplied, but permission is not FULL')) + _check_admin_fail( + ul, + "t", + AdminPermission.READ, + "m", + lambda _: None, + "user", + ValueError("as_user is supplied, but permission is not FULL"), + ) def test_check_admin_fail_no_admin_perms(): @@ -801,9 +1186,11 @@ def _check_admin_fail_no_admin_perms(permhas, permreq): ul = create_autospec(KBaseUserLookup, spec_set=True, instance=True) log = [] - ul.is_admin.return_value = (permhas, 'user1') - err = 'User user1 does not have the necessary administration privileges to run method m' - _check_admin_fail(ul, 't', permreq, 'm', lambda l: log.append(l), None, UnauthorizedError(err)) + ul.is_admin.return_value = (permhas, "user1") + err = "User user1 does not have the necessary administration privileges to run method m" + _check_admin_fail( + ul, "t", permreq, "m", lambda l: log.append(l), None, UnauthorizedError(err) + ) assert log == [err] @@ -812,15 +1199,21 @@ def test_check_admin_fail_no_such_user(): ul = create_autospec(KBaseUserLookup, spec_set=True, instance=True) log = [] - ul.is_admin.return_value = (AdminPermission.FULL, 'user1') - ul.invalid_users.return_value = [UserID('bruh')] + ul.is_admin.return_value = (AdminPermission.FULL, "user1") + ul.invalid_users.return_value = [UserID("bruh")] _check_admin_fail( - ul, 'token', AdminPermission.FULL, 'a method', lambda x: log.append(x), UserID('bruh'), - NoSuchUserError('bruh')) + ul, + "token", + AdminPermission.FULL, + "a method", + lambda x: log.append(x), + UserID("bruh"), + NoSuchUserError("bruh"), + ) - ul.is_admin.assert_called_once_with('token') - ul.invalid_users.assert_called_once_with([UserID('bruh')]) + ul.is_admin.assert_called_once_with("token") + ul.invalid_users.assert_called_once_with([UserID("bruh")]) assert log == [] @@ -831,27 +1224,52 @@ def _check_admin_fail(ul, token, perm, method, logfn, as_user, expected): def test_get_static_key_metadata_params(): - assert get_static_key_metadata_params({'keys': []}) == ([], False) - assert get_static_key_metadata_params({'keys': ['foo'], 'prefix': None}) == (['foo'], False) - assert get_static_key_metadata_params({'keys': ['bar', 'foo'], 'prefix': 0}) == ( - ['bar', 'foo'], False) - assert get_static_key_metadata_params({'keys': ['bar'], 'prefix': False}) == (['bar'], False) - assert get_static_key_metadata_params({'keys': ['bar'], 'prefix': 1}) == (['bar'], None) - assert get_static_key_metadata_params({'keys': ['bar'], 'prefix': 2}) == (['bar'], True) + assert get_static_key_metadata_params({"keys": []}) == ([], False) + assert get_static_key_metadata_params({"keys": ["foo"], "prefix": None}) == ( + ["foo"], + False, + ) + assert get_static_key_metadata_params({"keys": ["bar", "foo"], "prefix": 0}) == ( + ["bar", "foo"], + False, + ) + assert get_static_key_metadata_params({"keys": ["bar"], "prefix": False}) == ( + ["bar"], + False, + ) + assert get_static_key_metadata_params({"keys": ["bar"], "prefix": 1}) == ( + ["bar"], + None, + ) + assert get_static_key_metadata_params({"keys": ["bar"], "prefix": 2}) == ( + ["bar"], + True, + ) def test_get_static_key_metadata_params_fail_bad_args(): - _get_static_key_metadata_params_fail(None, ValueError('params cannot be None')) - _get_static_key_metadata_params_fail({}, IllegalParameterError('keys must be a list')) - _get_static_key_metadata_params_fail({'keys': 0}, IllegalParameterError('keys must be a list')) - _get_static_key_metadata_params_fail({'keys': ['foo', 0]}, IllegalParameterError( - 'index 1 of keys is not a string')) - _get_static_key_metadata_params_fail({'keys': [], 'prefix': -1}, IllegalParameterError( - 'Unexpected value for prefix: -1')) - _get_static_key_metadata_params_fail({'keys': [], 'prefix': 3}, IllegalParameterError( - 'Unexpected value for prefix: 3')) - _get_static_key_metadata_params_fail({'keys': [], 'prefix': 'foo'}, IllegalParameterError( - 'Unexpected value for prefix: foo')) + _get_static_key_metadata_params_fail(None, ValueError("params cannot be None")) + _get_static_key_metadata_params_fail( + {}, IllegalParameterError("keys must be a list") + ) + _get_static_key_metadata_params_fail( + {"keys": 0}, IllegalParameterError("keys must be a list") + ) + _get_static_key_metadata_params_fail( + {"keys": ["foo", 0]}, IllegalParameterError("index 1 of keys is not a string") + ) + _get_static_key_metadata_params_fail( + {"keys": [], "prefix": -1}, + IllegalParameterError("Unexpected value for prefix: -1"), + ) + _get_static_key_metadata_params_fail( + {"keys": [], "prefix": 3}, + IllegalParameterError("Unexpected value for prefix: 3"), + ) + _get_static_key_metadata_params_fail( + {"keys": [], "prefix": "foo"}, + IllegalParameterError("Unexpected value for prefix: foo"), + ) def _get_static_key_metadata_params_fail(params, expected): @@ -862,18 +1280,19 @@ def _get_static_key_metadata_params_fail(params, expected): def test_create_data_link_params_missing_update_key(): params = { - 'id': '706fe9e1-70ef-4feb-bbd9-32295104a119', - 'version': 78, - 'node': 'mynode', - 'upa': '6/7/29', - 'dataid': 'mydata' + "id": "706fe9e1-70ef-4feb-bbd9-32295104a119", + "version": 78, + "node": "mynode", + "upa": "6/7/29", + "dataid": "mydata", } assert create_data_link_params(params) == ( - DataUnitID(UPA('6/7/29'), 'mydata'), + DataUnitID(UPA("6/7/29"), "mydata"), SampleNodeAddress( - SampleAddress(UUID('706fe9e1-70ef-4feb-bbd9-32295104a119'), 78), 'mynode'), - False + SampleAddress(UUID("706fe9e1-70ef-4feb-bbd9-32295104a119"), 78), "mynode" + ), + False, ) @@ -882,71 +1301,85 @@ def test_create_data_link_params_with_update(): _create_data_link_params_with_update(False, False) # doesn't work with SDK _create_data_link_params_with_update(0, False) _create_data_link_params_with_update([], False) # illegal value theoretically - _create_data_link_params_with_update('', False) # illegal value theoretically + _create_data_link_params_with_update("", False) # illegal value theoretically _create_data_link_params_with_update(1, True) # rest of these are theoretically illegal values _create_data_link_params_with_update(100, True) _create_data_link_params_with_update(-1, True) - _create_data_link_params_with_update('m', True) - _create_data_link_params_with_update({'a': 'b'}, True) + _create_data_link_params_with_update("m", True) + _create_data_link_params_with_update({"a": "b"}, True) def _create_data_link_params_with_update(update, expected): params = { - 'id': '706fe9e1-70ef-4feb-bbd9-32295104a119', - 'version': 1, - 'node': 'm', - 'upa': '1/1/1', - 'update': update + "id": "706fe9e1-70ef-4feb-bbd9-32295104a119", + "version": 1, + "node": "m", + "upa": "1/1/1", + "update": update, } assert create_data_link_params(params) == ( - DataUnitID(UPA('1/1/1')), + DataUnitID(UPA("1/1/1")), SampleNodeAddress( - SampleAddress(UUID('706fe9e1-70ef-4feb-bbd9-32295104a119'), 1), 'm'), - expected + SampleAddress(UUID("706fe9e1-70ef-4feb-bbd9-32295104a119"), 1), "m" + ), + expected, ) def test_create_data_link_params_fail_bad_args(): - id_ = '706fe9e1-70ef-4feb-bbd9-32295104a119' - _create_data_link_params_fail(None, ValueError('params cannot be None')) - _create_data_link_params_fail({}, MissingParameterError('id')) - _create_data_link_params_fail({'id': 6}, IllegalParameterError( - 'id 6 must be a UUID string')) - _create_data_link_params_fail({'id': id_[:-1]}, IllegalParameterError( - 'id 706fe9e1-70ef-4feb-bbd9-32295104a11 must be a UUID string')) - _create_data_link_params_fail({'id': id_}, MissingParameterError('version')) + id_ = "706fe9e1-70ef-4feb-bbd9-32295104a119" + _create_data_link_params_fail(None, ValueError("params cannot be None")) + _create_data_link_params_fail({}, MissingParameterError("id")) _create_data_link_params_fail( - {'id': id_, 'version': 'ver'}, - IllegalParameterError('Illegal version argument: ver')) + {"id": 6}, IllegalParameterError("id 6 must be a UUID string") + ) _create_data_link_params_fail( - {'id': id_, 'version': -1}, - IllegalParameterError('Illegal version argument: -1')) + {"id": id_[:-1]}, + IllegalParameterError( + "id 706fe9e1-70ef-4feb-bbd9-32295104a11 must be a UUID string" + ), + ) + _create_data_link_params_fail({"id": id_}, MissingParameterError("version")) _create_data_link_params_fail( - {'id': id_, 'version': 1}, - MissingParameterError('node')) + {"id": id_, "version": "ver"}, + IllegalParameterError("Illegal version argument: ver"), + ) _create_data_link_params_fail( - {'id': id_, 'version': 1, 'node': {'a': 'b'}}, - IllegalParameterError('node key is not a string as required')) + {"id": id_, "version": -1}, + IllegalParameterError("Illegal version argument: -1"), + ) _create_data_link_params_fail( - {'id': id_, 'version': 1, 'node': 'foo\tbar'}, - IllegalParameterError('node contains control characters')) + {"id": id_, "version": 1}, MissingParameterError("node") + ) _create_data_link_params_fail( - {'id': id_, 'version': 1, 'node': 'm'}, - MissingParameterError('upa')) + {"id": id_, "version": 1, "node": {"a": "b"}}, + IllegalParameterError("node key is not a string as required"), + ) _create_data_link_params_fail( - {'id': id_, 'version': 1, 'node': 'm', 'upa': 3.4}, - IllegalParameterError('upa key is not a string as required')) + {"id": id_, "version": 1, "node": "foo\tbar"}, + IllegalParameterError("node contains control characters"), + ) + _create_data_link_params_fail( + {"id": id_, "version": 1, "node": "m"}, MissingParameterError("upa") + ) + _create_data_link_params_fail( + {"id": id_, "version": 1, "node": "m", "upa": 3.4}, + IllegalParameterError("upa key is not a string as required"), + ) _create_data_link_params_fail( - {'id': id_, 'version': 1, 'node': 'm', 'upa': '1/0/1'}, - IllegalParameterError('1/0/1 is not a valid UPA')) + {"id": id_, "version": 1, "node": "m", "upa": "1/0/1"}, + IllegalParameterError("1/0/1 is not a valid UPA"), + ) _create_data_link_params_fail( - {'id': id_, 'version': 1, 'node': 'm', 'upa': '1/1/1', 'dataid': 6}, - IllegalParameterError('dataid key is not a string as required')) + {"id": id_, "version": 1, "node": "m", "upa": "1/1/1", "dataid": 6}, + IllegalParameterError("dataid key is not a string as required"), + ) _create_data_link_params_fail( - {'id': id_, 'version': 1, 'node': 'm', 'upa': '1/1/1', 'dataid': 'yay\nyo'}, - IllegalParameterError('dataid contains control characters')) + {"id": id_, "version": 1, "node": "m", "upa": "1/1/1", "dataid": "yay\nyo"}, + IllegalParameterError("dataid contains control characters"), + ) def _create_data_link_params_fail(params, expected): @@ -956,23 +1389,30 @@ def _create_data_link_params_fail(params, expected): def test_get_data_unit_id_from_object(): - assert get_data_unit_id_from_object({'upa': '1/1/1'}) == DataUnitID(UPA('1/1/1')) - assert get_data_unit_id_from_object({'upa': '8/3/2'}) == DataUnitID(UPA('8/3/2')) - assert get_data_unit_id_from_object( - {'upa': '8/3/2', 'dataid': 'a'}) == DataUnitID(UPA('8/3/2'), 'a') + assert get_data_unit_id_from_object({"upa": "1/1/1"}) == DataUnitID(UPA("1/1/1")) + assert get_data_unit_id_from_object({"upa": "8/3/2"}) == DataUnitID(UPA("8/3/2")) + assert get_data_unit_id_from_object({"upa": "8/3/2", "dataid": "a"}) == DataUnitID( + UPA("8/3/2"), "a" + ) def test_get_data_unit_id_from_object_fail_bad_args(): - _get_data_unit_id_from_object_fail(None, ValueError('params cannot be None')) - _get_data_unit_id_from_object_fail({}, MissingParameterError('upa')) - _get_data_unit_id_from_object_fail({'upa': '1/0/1'}, IllegalParameterError( - '1/0/1 is not a valid UPA')) - _get_data_unit_id_from_object_fail({'upa': 82}, IllegalParameterError( - 'upa key is not a string as required')) - _get_data_unit_id_from_object_fail({'upa': '1/1/1', 'dataid': []}, IllegalParameterError( - 'dataid key is not a string as required')) - _get_data_unit_id_from_object_fail({'upa': '1/1/1', 'dataid': 'f\t/b'}, IllegalParameterError( - 'dataid contains control characters')) + _get_data_unit_id_from_object_fail(None, ValueError("params cannot be None")) + _get_data_unit_id_from_object_fail({}, MissingParameterError("upa")) + _get_data_unit_id_from_object_fail( + {"upa": "1/0/1"}, IllegalParameterError("1/0/1 is not a valid UPA") + ) + _get_data_unit_id_from_object_fail( + {"upa": 82}, IllegalParameterError("upa key is not a string as required") + ) + _get_data_unit_id_from_object_fail( + {"upa": "1/1/1", "dataid": []}, + IllegalParameterError("dataid key is not a string as required"), + ) + _get_data_unit_id_from_object_fail( + {"upa": "1/1/1", "dataid": "f\t/b"}, + IllegalParameterError("dataid contains control characters"), + ) def _get_data_unit_id_from_object_fail(params, expected): @@ -982,16 +1422,19 @@ def _get_data_unit_id_from_object_fail(params, expected): def test_get_upa_from_object(): - assert get_upa_from_object({'upa': '1/1/1'}) == UPA('1/1/1') - assert get_upa_from_object({'upa': '8/3/2'}) == UPA('8/3/2') + assert get_upa_from_object({"upa": "1/1/1"}) == UPA("1/1/1") + assert get_upa_from_object({"upa": "8/3/2"}) == UPA("8/3/2") def test_get_upa_from_object_fail_bad_args(): - _get_upa_from_object_fail(None, ValueError('params cannot be None')) - _get_upa_from_object_fail({}, MissingParameterError('upa')) - _get_upa_from_object_fail({'upa': '1/0/1'}, IllegalParameterError('1/0/1 is not a valid UPA')) - _get_upa_from_object_fail({'upa': 82}, IllegalParameterError( - 'upa key is not a string as required')) + _get_upa_from_object_fail(None, ValueError("params cannot be None")) + _get_upa_from_object_fail({}, MissingParameterError("upa")) + _get_upa_from_object_fail( + {"upa": "1/0/1"}, IllegalParameterError("1/0/1 is not a valid UPA") + ) + _get_upa_from_object_fail( + {"upa": 82}, IllegalParameterError("upa key is not a string as required") + ) def _get_upa_from_object_fail(params, expected): @@ -1002,24 +1445,34 @@ def _get_upa_from_object_fail(params, expected): def test_get_datetime_from_epochmilliseconds_in_object(): gt = get_datetime_from_epochmilliseconds_in_object - assert gt({}, 'foo') is None - assert gt({'bar': 1}, 'foo') is None - assert gt({'foo': 0}, 'foo') == dt(0) - assert gt({'foo': 1}, 'foo') == dt(0.001) - assert gt({'foo': -1}, 'foo') == dt(-0.001) - assert gt({'bar': 1234877807185}, 'bar') == dt(1234877807.185) - assert gt({'bar': -1234877807185}, 'bar') == dt(-1234877807.185) + assert gt({}, "foo") is None + assert gt({"bar": 1}, "foo") is None + assert gt({"foo": 0}, "foo") == dt(0) + assert gt({"foo": 1}, "foo") == dt(0.001) + assert gt({"foo": -1}, "foo") == dt(-0.001) + assert gt({"bar": 1234877807185}, "bar") == dt(1234877807.185) + assert gt({"bar": -1234877807185}, "bar") == dt(-1234877807.185) # should really test overflow but that's system dependent, no reliable test def test_get_datetime_from_epochmilliseconds_in_object_fail_bad_args(): gt = _get_datetime_from_epochmilliseconds_in_object_fail - gt(None, 'bar', ValueError('params cannot be None')) - gt({'foo': 'a'}, 'foo', IllegalParameterError( - "key 'foo' value of 'a' is not a valid epoch millisecond timestamp")) - gt({'ts': 1.2}, 'ts', IllegalParameterError( - "key 'ts' value of '1.2' is not a valid epoch millisecond timestamp")) + gt(None, "bar", ValueError("params cannot be None")) + gt( + {"foo": "a"}, + "foo", + IllegalParameterError( + "key 'foo' value of 'a' is not a valid epoch millisecond timestamp" + ), + ) + gt( + {"ts": 1.2}, + "ts", + IllegalParameterError( + "key 'ts' value of '1.2' is not a valid epoch millisecond timestamp" + ), + ) def _get_datetime_from_epochmilliseconds_in_object_fail(params, key, expected): @@ -1031,67 +1484,74 @@ def _get_datetime_from_epochmilliseconds_in_object_fail(params, key, expected): def test_links_to_dicts(): links = [ DataLink( - UUID('f5bd78c3-823e-40b2-9f93-20e78680e41e'), - DataUnitID(UPA('1/2/3'), 'foo'), + UUID("f5bd78c3-823e-40b2-9f93-20e78680e41e"), + DataUnitID(UPA("1/2/3"), "foo"), SampleNodeAddress( - SampleAddress(UUID('f5bd78c3-823e-40b2-9f93-20e78680e41f'), 6), 'foo'), + SampleAddress(UUID("f5bd78c3-823e-40b2-9f93-20e78680e41f"), 6), "foo" + ), dt(0.067), - UserID('usera'), + UserID("usera"), dt(89), - UserID('userb') + UserID("userb"), ), DataLink( - UUID('f5bd78c3-823e-40b2-9f93-20e78680e41a'), - DataUnitID(UPA('4/9/10')), + UUID("f5bd78c3-823e-40b2-9f93-20e78680e41a"), + DataUnitID(UPA("4/9/10")), SampleNodeAddress( - SampleAddress(UUID('f5bd78c3-823e-40b2-9f93-20e78680e41b'), 4), 'bar'), + SampleAddress(UUID("f5bd78c3-823e-40b2-9f93-20e78680e41b"), 4), "bar" + ), dt(1), - UserID('userc'), + UserID("userc"), ), ] assert links_to_dicts(links) == [ { - 'linkid': 'f5bd78c3-823e-40b2-9f93-20e78680e41e', - 'upa': '1/2/3', - 'dataid': 'foo', - 'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41f', - 'version': 6, - 'node': 'foo', - 'created': 67, - 'createdby': 'usera', - 'expired': 89000, - 'expiredby': 'userb' - }, + "linkid": "f5bd78c3-823e-40b2-9f93-20e78680e41e", + "upa": "1/2/3", + "dataid": "foo", + "id": "f5bd78c3-823e-40b2-9f93-20e78680e41f", + "version": 6, + "node": "foo", + "created": 67, + "createdby": "usera", + "expired": 89000, + "expiredby": "userb", + }, { - 'linkid': 'f5bd78c3-823e-40b2-9f93-20e78680e41a', - 'upa': '4/9/10', - 'dataid': None, - 'id': 'f5bd78c3-823e-40b2-9f93-20e78680e41b', - 'version': 4, - 'node': 'bar', - 'created': 1000, - 'createdby': 'userc', - 'expired': None, - 'expiredby': None - } + "linkid": "f5bd78c3-823e-40b2-9f93-20e78680e41a", + "upa": "4/9/10", + "dataid": None, + "id": "f5bd78c3-823e-40b2-9f93-20e78680e41b", + "version": 4, + "node": "bar", + "created": 1000, + "createdby": "userc", + "expired": None, + "expiredby": None, + }, ] def test_links_to_dicts_fail_bad_args(): dl = DataLink( - UUID('f5bd78c3-823e-40b2-9f93-20e78680e41e'), - DataUnitID(UPA('1/2/3'), 'foo'), - SampleNodeAddress( - SampleAddress(UUID('f5bd78c3-823e-40b2-9f93-20e78680e41f'), 6), 'foo'), - dt(0.067), - UserID('usera'), - dt(89), - UserID('userb') - ) + UUID("f5bd78c3-823e-40b2-9f93-20e78680e41e"), + DataUnitID(UPA("1/2/3"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("f5bd78c3-823e-40b2-9f93-20e78680e41f"), 6), "foo" + ), + dt(0.067), + UserID("usera"), + dt(89), + UserID("userb"), + ) - _links_to_dicts_fail(None, ValueError('links cannot be None')) - _links_to_dicts_fail([dl, None], ValueError( - 'Index 1 of iterable links cannot be a value that evaluates to false')) + _links_to_dicts_fail(None, ValueError("links cannot be None")) + _links_to_dicts_fail( + [dl, None], + ValueError( + "Index 1 of iterable links cannot be a value that evaluates to false" + ), + ) def _links_to_dicts_fail(links, expected): diff --git a/test/core/arg_checkers_test.py b/test/core/arg_checkers_test.py index 4feeda8c..f772fe5a 100644 --- a/test/core/arg_checkers_test.py +++ b/test/core/arg_checkers_test.py @@ -1,108 +1,125 @@ import datetime from pytest import raises from core.test_utils import assert_exception_correct -from SampleService.core.arg_checkers import check_string, not_falsy, not_falsy_in_iterable +from SampleService.core.arg_checkers import ( + check_string, + not_falsy, + not_falsy_in_iterable, +) from SampleService.core.arg_checkers import check_timestamp from SampleService.core.errors import MissingParameterError, IllegalParameterError -LONG_STRING = 'a' * 100 +LONG_STRING = "a" * 100 def test_falsy_true(): - for t in ['a', 1, True, [1], {'a': 1}, {1}]: - assert not_falsy(t, 'foo') is t + for t in ["a", 1, True, [1], {"a": 1}, {1}]: + assert not_falsy(t, "foo") is t def test_falsy_fail(): - for f in ['', 0, False, [], dict(), {}]: + for f in ["", 0, False, [], dict(), {}]: with raises(Exception) as got: - not_falsy(f, 'my name') + not_falsy(f, "my name") assert_exception_correct( - got.value, ValueError('my name cannot be a value that evaluates to false')) + got.value, ValueError("my name cannot be a value that evaluates to false") + ) def test_falsy_in_iterable_true(): - for t in [[], [1, 'a'], [True], [{'foo'}]]: - assert not_falsy_in_iterable(t, 'foo') is t + for t in [[], [1, "a"], [True], [{"foo"}]]: + assert not_falsy_in_iterable(t, "foo") is t def test_falsy_in_iterable_allow_none(): - assert not_falsy_in_iterable(None, 'yay', allow_none=True) is None + assert not_falsy_in_iterable(None, "yay", allow_none=True) is None def test_falsy_in_iterable_no_iterable(): with raises(Exception) as got: - not_falsy_in_iterable(None, 'whee') - assert_exception_correct(got.value, ValueError('whee cannot be None')) + not_falsy_in_iterable(None, "whee") + assert_exception_correct(got.value, ValueError("whee cannot be None")) def test_falsy_in_iterable_false_insides(): - for item, pos in [[['', 'bar'], 0], - [['foo', 0], 1], - [[True, True, False, True], 2], - [[[]], 0], - [[dict()], 0], - [[{}], 0] - ]: + for item, pos in [ + [["", "bar"], 0], + [["foo", 0], 1], + [[True, True, False, True], 2], + [[[]], 0], + [[dict()], 0], + [[{}], 0], + ]: with raises(Exception) as got: - not_falsy_in_iterable(item, 'my name') - assert_exception_correct(got.value, ValueError( - f'Index {pos} of iterable my name cannot be a value that evaluates to false')) + not_falsy_in_iterable(item, "my name") + assert_exception_correct( + got.value, + ValueError( + f"Index {pos} of iterable my name cannot be a value that evaluates to false" + ), + ) def test_check_string(): - for string, expected in {' foo': 'foo', - ' \t baɷr ': 'baɷr', - 'baᚠz \t ': 'baᚠz', - 'bat': 'bat', - 'a' * 1000: 'a' * 1000}.items(): - assert check_string(string, 'name') == expected + for string, expected in { + " foo": "foo", + " \t baɷr ": "baɷr", + "baᚠz \t ": "baᚠz", + "bat": "bat", + "a" * 1000: "a" * 1000, + }.items(): + assert check_string(string, "name") == expected def test_check_string_bad_max_len(): for max_len in [0, -1, -100]: with raises(Exception) as got: - check_string('str', 'var name', max_len=max_len) - assert_exception_correct(got.value, ValueError('max_len must be > 0 if provided')) + check_string("str", "var name", max_len=max_len) + assert_exception_correct( + got.value, ValueError("max_len must be > 0 if provided") + ) def test_check_string_optional_true(): - for string in [None, ' \t ']: - assert check_string(string, 'name', optional=True) is None + for string in [None, " \t "]: + assert check_string(string, "name", optional=True) is None def test_check_string_optional_false(): - for string in [None, ' \t ']: + for string in [None, " \t "]: with raises(Exception) as got: - check_string(string, 'var name') - assert_exception_correct(got.value, MissingParameterError('var name')) + check_string(string, "var name") + assert_exception_correct(got.value, MissingParameterError("var name")) def test_check_string_control_characters(): - for string in ['foo \b bar', 'foo\u200bbar', 'foo\0bar', 'foo\bbar']: + for string in ["foo \b bar", "foo\u200bbar", "foo\0bar", "foo\bbar"]: with raises(Exception) as got: - check_string(string, 'var name') + check_string(string, "var name") assert_exception_correct( - got.value, IllegalParameterError('var name contains control characters')) + got.value, IllegalParameterError("var name contains control characters") + ) def test_check_string_max_len(): - for string, length in {'123456789': 9, - 'a': 1, - 'a' * 100: 100, - 'a' * 10000: 10000, - 'a' * 10000: 1000000}.items(): - assert check_string(string, 'name', max_len=length) == string + for string, length in { + "123456789": 9, + "a": 1, + "a" * 100: 100, + "a" * 10000: 10000, + "a" * 10000: 1000000, + }.items(): + assert check_string(string, "name", max_len=length) == string def test_check_string_long_fail(): - for string, length in {'123456789': 8, - 'ab': 1, - 'a' * 100: 99}.items(): + for string, length in {"123456789": 8, "ab": 1, "a" * 100: 99}.items(): with raises(Exception) as got: - check_string(string, 'var name', max_len=length) + check_string(string, "var name", max_len=length) assert_exception_correct( - got.value, IllegalParameterError(f'var name exceeds maximum length of {length}')) + got.value, + IllegalParameterError(f"var name exceeds maximum length of {length}"), + ) def _dt(timestamp): @@ -111,13 +128,18 @@ def _dt(timestamp): def test_check_timestamp(): for t in [-1000000, -256, -1, 0, 1, 6, 100, 100000000000]: - assert check_timestamp(_dt(t), 'name') == _dt(t) + assert check_timestamp(_dt(t), "name") == _dt(t) def test_check_timestamp_fail_bad_args(): - _check_timestamp_fail(None, 'ts', ValueError('ts cannot be a value that evaluates to false')) - _check_timestamp_fail(datetime.datetime.now(), 'tymestampz', ValueError( - 'tymestampz cannot be a naive datetime')) + _check_timestamp_fail( + None, "ts", ValueError("ts cannot be a value that evaluates to false") + ) + _check_timestamp_fail( + datetime.datetime.now(), + "tymestampz", + ValueError("tymestampz cannot be a naive datetime"), + ) def _check_timestamp_fail(ts, name, expected): diff --git a/test/core/config_test.py b/test/core/config_test.py index 02968292..01bbc327 100644 --- a/test/core/config_test.py +++ b/test/core/config_test.py @@ -14,7 +14,7 @@ from SampleService.core.errors import IllegalParameterError -@fixture(scope='module') +@fixture(scope="module") def temp_dir(): tempdir = test_utils.get_temp_dir() yield tempdir @@ -24,25 +24,32 @@ def temp_dir(): def _write_validator_config(cfg, temp_dir): - tf = tempfile.mkstemp('.tmp.cfg', 'config_test_', dir=temp_dir) + tf = tempfile.mkstemp(".tmp.cfg", "config_test_", dir=temp_dir) os.close(tf[0]) - with open(tf[1], 'w') as temp: + with open(tf[1], "w") as temp: yaml.dump(cfg, temp) return tf[1] def test_split_value(): - assert split_value({}, 'k') == [] - assert split_value({'k': None}, 'k') == [] - assert split_value({'k': ' '}, 'k') == [] - assert split_value({'k': ' foo '}, 'k') == ['foo'] - assert split_value({'k': 'foo , bar , , baz '}, 'k') == ['foo', 'bar', 'baz'] + assert split_value({}, "k") == [] + assert split_value({"k": None}, "k") == [] + assert split_value({"k": " "}, "k") == [] + assert split_value({"k": " foo "}, "k") == ["foo"] + assert split_value({"k": "foo , bar , , baz "}, "k") == [ + "foo", + "bar", + "baz", + ] def test_split_value_fail(): - _split_value_fail(None, 'k', ValueError('d cannot be None')) - _split_value_fail({'k': 'foo\tbar'}, 'k', IllegalParameterError( - 'config param k contains control characters')) + _split_value_fail(None, "k", ValueError("d cannot be None")) + _split_value_fail( + {"k": "foo\tbar"}, + "k", + IllegalParameterError("config param k contains control characters"), + ) def _split_value_fail(d, k, expected): @@ -53,102 +60,138 @@ def _split_value_fail(d, k, expected): def test_config_get_validators(temp_dir): cfg = { - 'validators': { - 'key1': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'val1' - }], - 'key_metadata': {'a': 'b', 'c': 1.56} - }, - 'key2': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'val2', - 'parameters': {'max-len': 7, 'foo': 'bar'} - }, - {'module': 'core.config_test_vals', - 'callable_builder': 'val2', - 'parameters': {'max-len': 5, 'foo': 'bar'}, - }], - }, - 'key3': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'val1', - 'parameters': {'foo': 'bat'} - }], - 'key_metadata': {'f': None, 'g': 1} - } + "validators": { + "key1": { + "validators": [ + {"module": "core.config_test_vals", "callable_builder": "val1"} + ], + "key_metadata": {"a": "b", "c": 1.56}, + }, + "key2": { + "validators": [ + { + "module": "core.config_test_vals", + "callable_builder": "val2", + "parameters": {"max-len": 7, "foo": "bar"}, + }, + { + "module": "core.config_test_vals", + "callable_builder": "val2", + "parameters": {"max-len": 5, "foo": "bar"}, + }, + ], + }, + "key3": { + "validators": [ + { + "module": "core.config_test_vals", + "callable_builder": "val1", + "parameters": {"foo": "bat"}, + } + ], + "key_metadata": {"f": None, "g": 1}, + }, }, - 'prefix_validators': { - 'key4': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'pval1', - }], - }, + "prefix_validators": { + "key4": { + "validators": [ + { + "module": "core.config_test_vals", + "callable_builder": "pval1", + } + ], + }, # check key3 doesn't interfere with above key3 - 'key3': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'pval2', - 'parameters': {'max-len': 7, 'foo': 'bar'}, - }, - {'module': 'core.config_test_vals', - 'callable_builder': 'pval2', - 'parameters': {'max-len': 5, 'foo': 'bar'} - }], - 'key_metadata': {'h': True, 'i': 1000} - }, - 'key5': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'pval1', - 'parameters': {'foo': 'bat'} - }], - 'key_metadata': {'a': 'f', 'c': 'l'} - } - } + "key3": { + "validators": [ + { + "module": "core.config_test_vals", + "callable_builder": "pval2", + "parameters": {"max-len": 7, "foo": "bar"}, + }, + { + "module": "core.config_test_vals", + "callable_builder": "pval2", + "parameters": {"max-len": 5, "foo": "bar"}, + }, + ], + "key_metadata": {"h": True, "i": 1000}, + }, + "key5": { + "validators": [ + { + "module": "core.config_test_vals", + "callable_builder": "pval1", + "parameters": {"foo": "bat"}, + } + ], + "key_metadata": {"a": "f", "c": "l"}, + }, + }, } tf = _write_validator_config(cfg, temp_dir) - vals = get_validators('file://' + tf) + vals = get_validators("file://" + tf) assert len(vals.keys()) == 3 assert len(vals.prefix_keys()) == 3 # the test validators always fail - assert vals.validator_count('key1') == 1 - assert vals.call_validator('key1', 0, {'a': 'b'}) == "1, key1, {}, {'a': 'b'}" - - assert vals.validator_count('key2') == 2 - assert vals.call_validator( - 'key2', 0, {'a': 'd'}) == "2, key2, {'foo': 'bar', 'max-len': 7}, {'a': 'd'}" - assert vals.call_validator( - 'key2', 1, {'a': 'd'}) == "2, key2, {'foo': 'bar', 'max-len': 5}, {'a': 'd'}" - - assert vals.validator_count('key3') == 1 - assert vals.call_validator('key3', 0, {'a': 'c'}) == "1, key3, {'foo': 'bat'}, {'a': 'c'}" - - assert vals.prefix_validator_count('key4') == 1 - assert vals.call_prefix_validator( - 'key4', 0, 'key4stuff', {'a': 'b'}) == "1, key4, key4stuff, {}, {'a': 'b'}" - - assert vals.prefix_validator_count('key3') == 2 - assert vals.call_prefix_validator( - 'key3', 0, 'key3s', {'a': 'd'} - ) == "2, key3, key3s, {'foo': 'bar', 'max-len': 7}, {'a': 'd'}" - assert vals.call_prefix_validator( - 'key3', 1, 'key3s1', {'a': 'd'} - ) == "2, key3, key3s1, {'foo': 'bar', 'max-len': 5}, {'a': 'd'}" - - assert vals.prefix_validator_count('key5') == 1 - assert vals.call_prefix_validator( - 'key5', 0, 'key5s', {'a': 'c'}) == "1, key5, key5s, {'foo': 'bat'}, {'a': 'c'}" + assert vals.validator_count("key1") == 1 + assert vals.call_validator("key1", 0, {"a": "b"}) == "1, key1, {}, {'a': 'b'}" + + assert vals.validator_count("key2") == 2 + assert ( + vals.call_validator("key2", 0, {"a": "d"}) + == "2, key2, {'foo': 'bar', 'max-len': 7}, {'a': 'd'}" + ) + assert ( + vals.call_validator("key2", 1, {"a": "d"}) + == "2, key2, {'foo': 'bar', 'max-len': 5}, {'a': 'd'}" + ) + + assert vals.validator_count("key3") == 1 + assert ( + vals.call_validator("key3", 0, {"a": "c"}) + == "1, key3, {'foo': 'bat'}, {'a': 'c'}" + ) + + assert vals.prefix_validator_count("key4") == 1 + assert ( + vals.call_prefix_validator("key4", 0, "key4stuff", {"a": "b"}) + == "1, key4, key4stuff, {}, {'a': 'b'}" + ) + + assert vals.prefix_validator_count("key3") == 2 + assert ( + vals.call_prefix_validator("key3", 0, "key3s", {"a": "d"}) + == "2, key3, key3s, {'foo': 'bar', 'max-len': 7}, {'a': 'd'}" + ) + assert ( + vals.call_prefix_validator("key3", 1, "key3s1", {"a": "d"}) + == "2, key3, key3s1, {'foo': 'bar', 'max-len': 5}, {'a': 'd'}" + ) + + assert vals.prefix_validator_count("key5") == 1 + assert ( + vals.call_prefix_validator("key5", 0, "key5s", {"a": "c"}) + == "1, key5, key5s, {'foo': 'bat'}, {'a': 'c'}" + ) # Test metadata - assert vals.key_metadata(['key1', 'key2', 'key3']) == { - 'key1': {'a': 'b', 'c': 1.56}, - 'key2': {}, - 'key3': {'f': None, 'g': 1} + assert vals.key_metadata(["key1", "key2", "key3"]) == { + "key1": {"a": "b", "c": 1.56}, + "key2": {}, + "key3": {"f": None, "g": 1}, } - assert vals.prefix_key_metadata(['key3', 'key4', 'key5']) == { - 'key3': {'h': True, 'i': 1000}, - 'key4': {}, - 'key5': {'a': 'f', 'c': 'l'} + assert vals.prefix_key_metadata(["key3", "key4", "key5"]) == { + "key3": {"h": True, "i": 1000}, + "key4": {}, + "key5": {"a": "f", "c": "l"}, } # noop entry cfg = {} tf = _write_validator_config(cfg, temp_dir) - vals = get_validators('file://' + tf) + vals = get_validators("file://" + tf) assert len(vals.keys()) == 0 assert len(vals.prefix_keys()) == 0 @@ -157,143 +200,247 @@ def test_config_get_validators_fail_bad_file(temp_dir): tf = _write_validator_config({}, temp_dir) os.remove(tf) with raises(Exception) as got: - get_validators('file://' + tf) - assert_exception_correct(got.value, ValueError( - f"Failed to open validator configuration file at file://{tf}: " + - f"[Errno 2] No such file or directory: '{tf}'")) + get_validators("file://" + tf) + assert_exception_correct( + got.value, + ValueError( + f"Failed to open validator configuration file at file://{tf}: " + + f"[Errno 2] No such file or directory: '{tf}'" + ), + ) def test_config_get_validators_fail_bad_yaml(temp_dir): # calling str() on ValidationErrors returns more detailed into about the error - tf = tempfile.mkstemp('.tmp.cfg', 'config_test_bad_yaml', dir=temp_dir) + tf = tempfile.mkstemp(".tmp.cfg", "config_test_bad_yaml", dir=temp_dir) os.close(tf[0]) - with open(tf[1], 'w') as temp: - temp.write('[bad yaml') + with open(tf[1], "w") as temp: + temp.write("[bad yaml") with raises(Exception) as got: - get_validators('file://' + tf[1]) - assert_exception_correct(got.value, ValueError( - f'Failed to open validator configuration file at file://{tf[1]}: while parsing a ' + - 'flow sequence\n in "", line 1, column 1\nexpected \',\' or \']\', ' + - 'but got \'\'\n in "", line 1, column 10' - )) + get_validators("file://" + tf[1]) + assert_exception_correct( + got.value, + ValueError( + f"Failed to open validator configuration file at file://{tf[1]}: while parsing a " + + "flow sequence\n in \"\", line 1, column 1\nexpected ',' or ']', " + + "but got ''\n in \"\", line 1, column 10" + ), + ) def test_config_get_validators_fail_bad_params(temp_dir): - _config_get_validators_fail_bad_params(temp_dir, 'validators') - _config_get_validators_fail_bad_params(temp_dir, 'prefix_validators') + _config_get_validators_fail_bad_params(temp_dir, "validators") + _config_get_validators_fail_bad_params(temp_dir, "prefix_validators") def _config_get_validators_fail_bad_params(temp_dir, key_): # calling str() on ValidationErrors returns more detailed into about the error _config_get_validators_fail( - '', temp_dir, - ValidationError("'' is not of type 'object'")) + "", temp_dir, ValidationError("'' is not of type 'object'") + ) _config_get_validators_fail( - ['foo', 'bar'], temp_dir, - ValidationError("['foo', 'bar'] is not of type 'object'")) + ["foo", "bar"], + temp_dir, + ValidationError("['foo', 'bar'] is not of type 'object'"), + ) _config_get_validators_fail( - {'key': 'y'}, temp_dir, - ValidationError("Additional properties are not allowed ('key' was unexpected)")) + {"key": "y"}, + temp_dir, + ValidationError("Additional properties are not allowed ('key' was unexpected)"), + ) _config_get_validators_fail( - {key_: 'y'}, temp_dir, - ValidationError("'y' is not of type 'object'")) + {key_: "y"}, temp_dir, ValidationError("'y' is not of type 'object'") + ) _config_get_validators_fail( - {key_: {'y': ['foo']}}, temp_dir, - ValidationError("['foo'] is not of type 'object'")) + {key_: {"y": ["foo"]}}, + temp_dir, + ValidationError("['foo'] is not of type 'object'"), + ) _config_get_validators_fail( - {key_: {'y': {'key_metadata': {'a': 'b'}}}}, temp_dir, - ValidationError("'validators' is a required property")) + {key_: {"y": {"key_metadata": {"a": "b"}}}}, + temp_dir, + ValidationError("'validators' is a required property"), + ) _config_get_validators_fail( - {key_: {'y': {'randomkey': {'a': 'b'}}}}, temp_dir, - ValidationError("'validators' is a required property")) + {key_: {"y": {"randomkey": {"a": "b"}}}}, + temp_dir, + ValidationError("'validators' is a required property"), + ) _config_get_validators_fail( - {key_: {'key': {'validators': {}}}}, temp_dir, - ValidationError("{} is not of type 'array'")) + {key_: {"key": {"validators": {}}}}, + temp_dir, + ValidationError("{} is not of type 'array'"), + ) _config_get_validators_fail( - {key_: {'key': {'validators': ['foo']}}}, temp_dir, - ValidationError("'foo' is not of type 'object'")) + {key_: {"key": {"validators": ["foo"]}}}, + temp_dir, + ValidationError("'foo' is not of type 'object'"), + ) _config_get_validators_fail( - {key_: {'key': {'validators': [{'module': 'foo', - 'callable_builder': 'bar'}], - 'key_metadata': []}}}, - temp_dir, ValidationError("[] is not of type 'object'")) + { + key_: { + "key": { + "validators": [{"module": "foo", "callable_builder": "bar"}], + "key_metadata": [], + } + } + }, + temp_dir, + ValidationError("[] is not of type 'object'"), + ) _config_get_validators_fail( - {key_: {'key': {'validators': [{}]}}}, temp_dir, - ValidationError("'module' is a required property")) + {key_: {"key": {"validators": [{}]}}}, + temp_dir, + ValidationError("'module' is a required property"), + ) _config_get_validators_fail( - {key_: {'key': {'validators': [{'module': 'foo'}]}}}, temp_dir, - ValidationError("'callable_builder' is a required property")) + {key_: {"key": {"validators": [{"module": "foo"}]}}}, + temp_dir, + ValidationError("'callable_builder' is a required property"), + ) _config_get_validators_fail( - {key_: {'key': {'validators': [{'module': 'foo', 'callable-builder': 'bar'}]}}}, + {key_: {"key": {"validators": [{"module": "foo", "callable-builder": "bar"}]}}}, temp_dir, ValidationError( - "Additional properties are not allowed ('callable-builder' was unexpected)")) + "Additional properties are not allowed ('callable-builder' was unexpected)" + ), + ) _config_get_validators_fail( - {key_: {'key': {'validators': [{'module': 'foo', - 'callable_builder': 'bar', - 'prefix': 1}]}}}, + { + key_: { + "key": { + "validators": [ + {"module": "foo", "callable_builder": "bar", "prefix": 1} + ] + } + } + }, temp_dir, - ValidationError("Additional properties are not allowed ('prefix' was unexpected)")) + ValidationError( + "Additional properties are not allowed ('prefix' was unexpected)" + ), + ) _config_get_validators_fail( - {key_: {'key': {'validators': [{'module': ['foo'], 'callable_builder': 'bar'}]}}}, + { + key_: { + "key": {"validators": [{"module": ["foo"], "callable_builder": "bar"}]} + } + }, temp_dir, - ValidationError("['foo'] is not of type 'string'")) + ValidationError("['foo'] is not of type 'string'"), + ) _config_get_validators_fail( - {key_: {'key': {'validators': [{'module': 'foo', 'callable_builder': ['bar']}]}}}, + { + key_: { + "key": {"validators": [{"module": "foo", "callable_builder": ["bar"]}]} + } + }, temp_dir, - ValidationError("['bar'] is not of type 'string'")) + ValidationError("['bar'] is not of type 'string'"), + ) _config_get_validators_fail( - {key_: {'key': {'validators': [{'module': 'foo', - 'callable_builder': 'bar', - 'parameters': 'foo'}]}}}, + { + key_: { + "key": { + "validators": [ + { + "module": "foo", + "callable_builder": "bar", + "parameters": "foo", + } + ] + } + } + }, temp_dir, - ValidationError("'foo' is not of type 'object'")) + ValidationError("'foo' is not of type 'object'"), + ) def test_config_get_validators_fail_no_module(temp_dir): _config_get_validators_fail( - {'validators': {'key': {'validators': [{'module': 'no_modules_here', - 'callable_builder': 'foo'}]}}}, + { + "validators": { + "key": { + "validators": [ + {"module": "no_modules_here", "callable_builder": "foo"} + ] + } + } + }, temp_dir, - ModuleNotFoundError("No module named 'no_modules_here'")) + ModuleNotFoundError("No module named 'no_modules_here'"), + ) def test_config_get_validators_fail_no_function(temp_dir): _config_get_validators_fail( - {'validators': {'x': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'foo'}]}}}, + { + "validators": { + "x": { + "validators": [ + {"module": "core.config_test_vals", "callable_builder": "foo"} + ] + } + } + }, temp_dir, - ValueError("Metadata validator callable build #0 failed for key x: " + - "module 'core.config_test_vals' has no attribute 'foo'")) + ValueError( + "Metadata validator callable build #0 failed for key x: " + + "module 'core.config_test_vals' has no attribute 'foo'" + ), + ) def test_config_get_validators_fail_function_exception(temp_dir): _config_get_validators_fail( - {'validators': {'x': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'val1'}, - {'module': 'core.config_test_vals', - 'callable_builder': 'fail_val'}]}}}, + { + "validators": { + "x": { + "validators": [ + {"module": "core.config_test_vals", "callable_builder": "val1"}, + { + "module": "core.config_test_vals", + "callable_builder": "fail_val", + }, + ] + } + } + }, temp_dir, - ValueError("Metadata validator callable build #1 failed for key x: " + - "we've no functions 'ere")) + ValueError( + "Metadata validator callable build #1 failed for key x: " + + "we've no functions 'ere" + ), + ) def test_config_get_prefix_validators_fail_function_exception(temp_dir): _config_get_validators_fail( - {'prefix_validators': {'p': {'validators': [{'module': 'core.config_test_vals', - 'callable_builder': 'val1'}, - {'module': 'core.config_test_vals', - 'callable_builder': 'val1'}, - {'module': 'core.config_test_vals', - 'callable_builder': 'fail_prefix_val'} - ]}}}, + { + "prefix_validators": { + "p": { + "validators": [ + {"module": "core.config_test_vals", "callable_builder": "val1"}, + {"module": "core.config_test_vals", "callable_builder": "val1"}, + { + "module": "core.config_test_vals", + "callable_builder": "fail_prefix_val", + }, + ] + } + } + }, temp_dir, - ValueError('Prefix metadata validator callable build #2 failed for key p: ' + - "we've no prefix functions 'ere")) + ValueError( + "Prefix metadata validator callable build #2 failed for key p: " + + "we've no prefix functions 'ere" + ), + ) def _config_get_validators_fail(cfg, temp_dir, expected): tf = _write_validator_config(cfg, temp_dir) with raises(Exception) as got: - get_validators('file://' + tf) + get_validators("file://" + tf) assert_exception_correct(got.value, expected) diff --git a/test/core/config_test_vals.py b/test/core/config_test_vals.py index 8315ee4a..65bdbe04 100644 --- a/test/core/config_test_vals.py +++ b/test/core/config_test_vals.py @@ -4,13 +4,15 @@ def s(d): def val1(d1): def f(key, d2): - return f'1, {key}, {s(d1)}, {s(d2)}' + return f"1, {key}, {s(d1)}, {s(d2)}" + return f def val2(d1): def f(key, d2): - return f'2, {key}, {s(d1)}, {s(d2)}' + return f"2, {key}, {s(d1)}, {s(d2)}" + return f @@ -20,13 +22,15 @@ def fail_val(d): def pval1(d1): def f(prefix, key, d2): - return f'1, {prefix}, {key}, {s(d1)}, {s(d2)}' + return f"1, {prefix}, {key}, {s(d1)}, {s(d2)}" + return f def pval2(d1): def f(prefix, key, d2): - return f'2, {prefix}, {key}, {s(d1)}, {s(d2)}' + return f"2, {prefix}, {key}, {s(d1)}, {s(d2)}" + return f @@ -35,9 +39,10 @@ def fail_prefix_val(d): def prefix_validator_test_builder(cfg): - arg = cfg['fail_on_arg'] + arg = cfg["fail_on_arg"] def val(prefix, key, args): if arg in args: - return f'{prefix}, {key}, {dict(sorted(args.items()))}' + return f"{prefix}, {key}, {dict(sorted(args.items()))}" + return val diff --git a/test/core/data_link_test.py b/test/core/data_link_test.py index ed52bfee..4db3f2f4 100644 --- a/test/core/data_link_test.py +++ b/test/core/data_link_test.py @@ -15,107 +15,170 @@ def dt(timestamp): def test_init_no_expire(): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') + sid = uuid.UUID("1234567890abcdef1234567890abcdef") dl = DataLink( - uuid.UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('2/3/4')), - SampleNodeAddress(SampleAddress(sid, 5), 'foo'), + uuid.UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("2/3/4")), + SampleNodeAddress(SampleAddress(sid, 5), "foo"), dt(500), - UserID('usera'), - expired_by=UserID('u') # should be ignored + UserID("usera"), + expired_by=UserID("u"), # should be ignored ) - assert dl.id == uuid.UUID('1234567890abcdef1234567890abcdee') - assert dl.duid == DataUnitID(UPA('2/3/4')) - assert dl.sample_node_address == SampleNodeAddress(SampleAddress(sid, 5), 'foo') + assert dl.id == uuid.UUID("1234567890abcdef1234567890abcdee") + assert dl.duid == DataUnitID(UPA("2/3/4")) + assert dl.sample_node_address == SampleNodeAddress(SampleAddress(sid, 5), "foo") assert dl.created == dt(500) - assert dl.created_by == UserID('usera') + assert dl.created_by == UserID("usera") assert dl.expired is None assert dl.expired_by is None - assert str(dl) == ('id=12345678-90ab-cdef-1234-567890abcdee ' + - 'duid=[2/3/4] ' + - 'sample_node_address=[12345678-90ab-cdef-1234-567890abcdef:5:foo] ' + - 'created=500.0 created_by=usera expired=None expired_by=None') + assert str(dl) == ( + "id=12345678-90ab-cdef-1234-567890abcdee " + + "duid=[2/3/4] " + + "sample_node_address=[12345678-90ab-cdef-1234-567890abcdef:5:foo] " + + "created=500.0 created_by=usera expired=None expired_by=None" + ) def test_init_with_expire1(): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') + sid = uuid.UUID("1234567890abcdef1234567890abcdef") dl = DataLink( - uuid.UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('2/6/4'), 'whee'), - SampleNodeAddress(SampleAddress(sid, 7), 'bar'), + uuid.UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("2/6/4"), "whee"), + SampleNodeAddress(SampleAddress(sid, 7), "bar"), dt(400), - UserID('u'), + UserID("u"), dt(800), - UserID('gotdam') + UserID("gotdam"), ) - assert dl.id == uuid.UUID('1234567890abcdef1234567890abcdee') - assert dl.duid == DataUnitID(UPA('2/6/4'), 'whee') - assert dl.sample_node_address == SampleNodeAddress(SampleAddress(sid, 7), 'bar') + assert dl.id == uuid.UUID("1234567890abcdef1234567890abcdee") + assert dl.duid == DataUnitID(UPA("2/6/4"), "whee") + assert dl.sample_node_address == SampleNodeAddress(SampleAddress(sid, 7), "bar") assert dl.created == dt(400) - assert dl.created_by == UserID('u') + assert dl.created_by == UserID("u") assert dl.expired == dt(800) - assert dl.expired_by == UserID('gotdam') - assert str(dl) == ('id=12345678-90ab-cdef-1234-567890abcdee ' + - 'duid=[2/6/4:whee] ' + - 'sample_node_address=[12345678-90ab-cdef-1234-567890abcdef:7:bar] ' + - 'created=400.0 created_by=u expired=800.0 expired_by=gotdam') + assert dl.expired_by == UserID("gotdam") + assert str(dl) == ( + "id=12345678-90ab-cdef-1234-567890abcdee " + + "duid=[2/6/4:whee] " + + "sample_node_address=[12345678-90ab-cdef-1234-567890abcdef:7:bar] " + + "created=400.0 created_by=u expired=800.0 expired_by=gotdam" + ) def test_init_with_expire2(): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') + sid = uuid.UUID("1234567890abcdef1234567890abcdef") dl = DataLink( - uuid.UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('2/6/4'), 'whee'), - SampleNodeAddress(SampleAddress(sid, 7), 'bar'), + uuid.UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("2/6/4"), "whee"), + SampleNodeAddress(SampleAddress(sid, 7), "bar"), dt(400), - UserID('myuserᚥnameisHank'), + UserID("myuserᚥnameisHank"), dt(400), - UserID('yay') + UserID("yay"), ) - assert dl.id == uuid.UUID('1234567890abcdef1234567890abcdee') - assert dl.duid == DataUnitID(UPA('2/6/4'), 'whee') - assert dl.sample_node_address == SampleNodeAddress(SampleAddress(sid, 7), 'bar') + assert dl.id == uuid.UUID("1234567890abcdef1234567890abcdee") + assert dl.duid == DataUnitID(UPA("2/6/4"), "whee") + assert dl.sample_node_address == SampleNodeAddress(SampleAddress(sid, 7), "bar") assert dl.created == dt(400) - assert dl.created_by == UserID('myuserᚥnameisHank') + assert dl.created_by == UserID("myuserᚥnameisHank") assert dl.expired == dt(400) - assert dl.expired_by == UserID('yay') - assert str(dl) == ('id=12345678-90ab-cdef-1234-567890abcdee ' + - 'duid=[2/6/4:whee] ' + - 'sample_node_address=[12345678-90ab-cdef-1234-567890abcdef:7:bar] ' + - 'created=400.0 created_by=myuserᚥnameisHank expired=400.0 expired_by=yay') + assert dl.expired_by == UserID("yay") + assert str(dl) == ( + "id=12345678-90ab-cdef-1234-567890abcdee " + + "duid=[2/6/4:whee] " + + "sample_node_address=[12345678-90ab-cdef-1234-567890abcdef:7:bar] " + + "created=400.0 created_by=myuserᚥnameisHank expired=400.0 expired_by=yay" + ) def test_init_fail(): - lid = uuid.UUID('1234567890abcdef1234567890abcdee') - d = DataUnitID(UPA('1/1/1')) - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - s = SampleNodeAddress(SampleAddress(sid, 1), 'a') + lid = uuid.UUID("1234567890abcdef1234567890abcdee") + d = DataUnitID(UPA("1/1/1")) + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + s = SampleNodeAddress(SampleAddress(sid, 1), "a") t = dt(1) - u = UserID('u') + u = UserID("u") bt = datetime.datetime.now() n = None - _init_fail(None, d, s, t, u, n, n, ValueError('id_ cannot be a value that evaluates to false')) - _init_fail(lid, None, s, t, u, n, n, ValueError( - 'duid cannot be a value that evaluates to false')) - _init_fail(lid, d, None, t, u, n, n, ValueError( - 'sample_node_address cannot be a value that evaluates to false')) - _init_fail(lid, d, s, None, u, n, n, ValueError( - 'created cannot be a value that evaluates to false')) - _init_fail(lid, d, s, bt, u, n, n, ValueError('created cannot be a naive datetime')) - _init_fail(lid, d, s, t, None, n, n, ValueError( - 'created_by cannot be a value that evaluates to false')) - _init_fail(lid, d, s, t, u, bt, u, ValueError('expired cannot be a naive datetime')) - _init_fail(lid, d, s, t, u, t, None, ValueError( - 'expired_by cannot be a value that evaluates to false')) - _init_fail(lid, d, s, dt(100), u, dt(99), u, ValueError( - 'link cannot expire before it is created')) + _init_fail( + None, + d, + s, + t, + u, + n, + n, + ValueError("id_ cannot be a value that evaluates to false"), + ) + _init_fail( + lid, + None, + s, + t, + u, + n, + n, + ValueError("duid cannot be a value that evaluates to false"), + ) + _init_fail( + lid, + d, + None, + t, + u, + n, + n, + ValueError("sample_node_address cannot be a value that evaluates to false"), + ) + _init_fail( + lid, + d, + s, + None, + u, + n, + n, + ValueError("created cannot be a value that evaluates to false"), + ) + _init_fail(lid, d, s, bt, u, n, n, ValueError("created cannot be a naive datetime")) + _init_fail( + lid, + d, + s, + t, + None, + n, + n, + ValueError("created_by cannot be a value that evaluates to false"), + ) + _init_fail(lid, d, s, t, u, bt, u, ValueError("expired cannot be a naive datetime")) + _init_fail( + lid, + d, + s, + t, + u, + t, + None, + ValueError("expired_by cannot be a value that evaluates to false"), + ) + _init_fail( + lid, + d, + s, + dt(100), + u, + dt(99), + u, + ValueError("link cannot expire before it is created"), + ) def _init_fail(lid, duid, sna, cr, cru, ex, eu, expected): @@ -125,52 +188,52 @@ def _init_fail(lid, duid, sna, cr, cru, ex, eu, expected): def test_is_equivalent_link(): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') + sid = uuid.UUID("1234567890abcdef1234567890abcdef") _is_equivalent( - DataUnitID(UPA('2/6/4'), 'whee'), - SampleNodeAddress(SampleAddress(sid, 7), 'bar'), - DataUnitID(UPA('2/6/4'), 'whee'), - SampleNodeAddress(SampleAddress(sid, 7), 'bar'), - True + DataUnitID(UPA("2/6/4"), "whee"), + SampleNodeAddress(SampleAddress(sid, 7), "bar"), + DataUnitID(UPA("2/6/4"), "whee"), + SampleNodeAddress(SampleAddress(sid, 7), "bar"), + True, ) _is_equivalent( - DataUnitID(UPA('2/6/4'), 'wheo'), - SampleNodeAddress(SampleAddress(sid, 7), 'bar'), - DataUnitID(UPA('2/6/4'), 'whee'), - SampleNodeAddress(SampleAddress(sid, 7), 'bar'), - False + DataUnitID(UPA("2/6/4"), "wheo"), + SampleNodeAddress(SampleAddress(sid, 7), "bar"), + DataUnitID(UPA("2/6/4"), "whee"), + SampleNodeAddress(SampleAddress(sid, 7), "bar"), + False, ) _is_equivalent( - DataUnitID(UPA('2/6/4'), 'whee'), - SampleNodeAddress(SampleAddress(sid, 8), 'bar'), - DataUnitID(UPA('2/6/4'), 'whee'), - SampleNodeAddress(SampleAddress(sid, 7), 'bar'), - False + DataUnitID(UPA("2/6/4"), "whee"), + SampleNodeAddress(SampleAddress(sid, 8), "bar"), + DataUnitID(UPA("2/6/4"), "whee"), + SampleNodeAddress(SampleAddress(sid, 7), "bar"), + False, ) def _is_equivalent(duid1, sna1, duid2, sna2, expected): dl1 = DataLink( - uuid.UUID('1234567890abcdef1234567890abcdee'), + uuid.UUID("1234567890abcdef1234567890abcdee"), duid1, sna1, dt(400), - UserID('myuserᚥnameisHank'), + UserID("myuserᚥnameisHank"), dt(400), - UserID('yay') + UserID("yay"), ) dl2 = DataLink( - uuid.UUID('1234567890abcdef1234567890abcdef'), + uuid.UUID("1234567890abcdef1234567890abcdef"), duid2, sna2, dt(500), - UserID('waaaahrrgg'), + UserID("waaaahrrgg"), dt(8900), - UserID('go faster stripes') + UserID("go faster stripes"), ) assert dl1.is_equivalent(dl2) is expected @@ -178,48 +241,53 @@ def _is_equivalent(duid1, sna1, duid2, sna2, expected): def test_is_equivalent_fail(): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') + sid = uuid.UUID("1234567890abcdef1234567890abcdef") dl1 = DataLink( - uuid.UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('2/6/4'), 'whee'), - SampleNodeAddress(SampleAddress(sid, 8), 'bar'), + uuid.UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("2/6/4"), "whee"), + SampleNodeAddress(SampleAddress(sid, 8), "bar"), dt(400), - UserID('myuserᚥnameisHank'), + UserID("myuserᚥnameisHank"), dt(400), - UserID('yay') + UserID("yay"), ) with raises(Exception) as got: dl1.is_equivalent(None) - assert_exception_correct(got.value, ValueError( - 'link cannot be a value that evaluates to false')) + assert_exception_correct( + got.value, ValueError("link cannot be a value that evaluates to false") + ) def test_equals(): - lid1 = uuid.UUID('1234567890abcdef1234567890abcdee') - lid1a = uuid.UUID('1234567890abcdef1234567890abcdee') - lid2 = uuid.UUID('1234567890abcdef1234567890abcdec') - lid2a = uuid.UUID('1234567890abcdef1234567890abcdec') - d1 = DataUnitID(UPA('1/1/1')) - d1a = DataUnitID(UPA('1/1/1')) - d2 = DataUnitID(UPA('1/1/2')) - d2a = DataUnitID(UPA('1/1/2')) - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - s1 = SampleNodeAddress(SampleAddress(sid, 1), 'foo') - s1a = SampleNodeAddress(SampleAddress(sid, 1), 'foo') - s2 = SampleNodeAddress(SampleAddress(sid, 2), 'foo') - s2a = SampleNodeAddress(SampleAddress(sid, 2), 'foo') + lid1 = uuid.UUID("1234567890abcdef1234567890abcdee") + lid1a = uuid.UUID("1234567890abcdef1234567890abcdee") + lid2 = uuid.UUID("1234567890abcdef1234567890abcdec") + lid2a = uuid.UUID("1234567890abcdef1234567890abcdec") + d1 = DataUnitID(UPA("1/1/1")) + d1a = DataUnitID(UPA("1/1/1")) + d2 = DataUnitID(UPA("1/1/2")) + d2a = DataUnitID(UPA("1/1/2")) + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + s1 = SampleNodeAddress(SampleAddress(sid, 1), "foo") + s1a = SampleNodeAddress(SampleAddress(sid, 1), "foo") + s2 = SampleNodeAddress(SampleAddress(sid, 2), "foo") + s2a = SampleNodeAddress(SampleAddress(sid, 2), "foo") t1 = dt(500) t1a = dt(500) t2 = dt(600) t2a = dt(600) - u1 = UserID('u') - u1a = UserID('u') - u2 = UserID('y') - u2a = UserID('y') + u1 = UserID("u") + u1a = UserID("u") + u2 = UserID("y") + u2a = UserID("y") assert DataLink(lid1, d1, s1, t1, u1) == DataLink(lid1a, d1a, s1a, t1a, u1a) - assert DataLink(lid1, d1, s1, t1, u1, None) == DataLink(lid1a, d1a, s1a, t1a, u1a, None) - assert DataLink(lid2, d2, s2, t1, u2, t2, u1) == DataLink(lid2a, d2a, s2a, t1a, u2a, t2a, u1a) + assert DataLink(lid1, d1, s1, t1, u1, None) == DataLink( + lid1a, d1a, s1a, t1a, u1a, None + ) + assert DataLink(lid2, d2, s2, t1, u2, t2, u1) == DataLink( + lid2a, d2a, s2a, t1a, u2a, t2a, u1a + ) assert DataLink(lid1, d1, s1, t1, u1) != (lid1, d1, s1, t1, u1) assert DataLink(lid1, d1, s1, t1, u1, t2, u2) != (lid1, d1, s1, t1, u1, t2, u2) @@ -229,54 +297,78 @@ def test_equals(): assert DataLink(lid1, d1, s1, t1, u1) != DataLink(lid1a, d1a, s2, t1a, u1a) assert DataLink(lid1, d1, s1, t1, u1) != DataLink(lid1a, d1a, s1a, t2, u1a) assert DataLink(lid1, d1, s1, t1, u1) != DataLink(lid1a, d1a, s1a, t1, u2) - assert DataLink(lid1, d1, s1, t1, u1, t2, u2) != DataLink(lid1a, d1a, s1a, t1a, u1a, t1, u2a) - assert DataLink(lid1, d1, s1, t1, u1, t2, u2) != DataLink(lid1a, d1a, s1a, t1a, u1a, t2a, u1) + assert DataLink(lid1, d1, s1, t1, u1, t2, u2) != DataLink( + lid1a, d1a, s1a, t1a, u1a, t1, u2a + ) + assert DataLink(lid1, d1, s1, t1, u1, t2, u2) != DataLink( + lid1a, d1a, s1a, t1a, u1a, t2a, u1 + ) assert DataLink(lid1, d1, s1, t1, u1, t1, u1) != DataLink(lid1a, d1a, s1a, t1a, u1a) - assert DataLink(lid1, d1, s1, t1, u1) != DataLink(lid1a, d1a, s1a, t1a, u1a, t1a, u1a) + assert DataLink(lid1, d1, s1, t1, u1) != DataLink( + lid1a, d1a, s1a, t1a, u1a, t1a, u1a + ) def test_hash(): # hashes will change from instance to instance of the python interpreter, and therefore # tests can't be written that directly test the hash value. See # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - lid1 = uuid.UUID('1234567890abcdef1234567890abcdee') - lid1a = uuid.UUID('1234567890abcdef1234567890abcdee') - lid2 = uuid.UUID('1234567890abcdef1234567890abcdec') - lid2a = uuid.UUID('1234567890abcdef1234567890abcdec') - d1 = DataUnitID(UPA('1/1/1')) - d1a = DataUnitID(UPA('1/1/1')) - d2 = DataUnitID(UPA('1/1/2')) - d2a = DataUnitID(UPA('1/1/2')) - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - s1 = SampleNodeAddress(SampleAddress(id_, 1), 'foo') - s1a = SampleNodeAddress(SampleAddress(id_, 1), 'foo') - s2 = SampleNodeAddress(SampleAddress(id_, 2), 'foo') - s2a = SampleNodeAddress(SampleAddress(id_, 2), 'foo') + lid1 = uuid.UUID("1234567890abcdef1234567890abcdee") + lid1a = uuid.UUID("1234567890abcdef1234567890abcdee") + lid2 = uuid.UUID("1234567890abcdef1234567890abcdec") + lid2a = uuid.UUID("1234567890abcdef1234567890abcdec") + d1 = DataUnitID(UPA("1/1/1")) + d1a = DataUnitID(UPA("1/1/1")) + d2 = DataUnitID(UPA("1/1/2")) + d2a = DataUnitID(UPA("1/1/2")) + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + s1 = SampleNodeAddress(SampleAddress(id_, 1), "foo") + s1a = SampleNodeAddress(SampleAddress(id_, 1), "foo") + s2 = SampleNodeAddress(SampleAddress(id_, 2), "foo") + s2a = SampleNodeAddress(SampleAddress(id_, 2), "foo") t1 = dt(500) t1a = dt(500) t2 = dt(600) t2a = dt(600) - u1 = UserID('u') - u1a = UserID('u') - u2 = UserID('y') - u2a = UserID('y') + u1 = UserID("u") + u1a = UserID("u") + u2 = UserID("y") + u2a = UserID("y") - assert hash(DataLink(lid1, d1, s1, t1, u1)) == hash(DataLink(lid1a, d1a, s1a, t1a, u1a)) + assert hash(DataLink(lid1, d1, s1, t1, u1)) == hash( + DataLink(lid1a, d1a, s1a, t1a, u1a) + ) assert hash(DataLink(lid1, d1, s1, t1, u1, None)) == hash( - DataLink(lid1a, d1a, s1a, t1a, u1a, None)) + DataLink(lid1a, d1a, s1a, t1a, u1a, None) + ) assert hash(DataLink(lid2, d2, s2, t1, u2, t2, u1)) == hash( - DataLink(lid2a, d2a, s2a, t1a, u2a, t2a, u1a)) + DataLink(lid2a, d2a, s2a, t1a, u2a, t2a, u1a) + ) - assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash(DataLink(lid2, d1a, s1a, t1a, u1a)) - assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash(DataLink(lid1a, d2, s1a, t1a, u1a)) - assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash(DataLink(lid1a, d1a, s2, t1a, u1a)) - assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash(DataLink(lid1a, d1a, s1a, t2, u1a)) - assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash(DataLink(lid1a, d1a, s1a, t1a, u2)) + assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash( + DataLink(lid2, d1a, s1a, t1a, u1a) + ) + assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash( + DataLink(lid1a, d2, s1a, t1a, u1a) + ) + assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash( + DataLink(lid1a, d1a, s2, t1a, u1a) + ) + assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash( + DataLink(lid1a, d1a, s1a, t2, u1a) + ) + assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash( + DataLink(lid1a, d1a, s1a, t1a, u2) + ) assert hash(DataLink(lid1, d1, s1, t1, u1, t2, u2)) != hash( - DataLink(lid1a, d1a, s1a, t1a, u1a, t1, u2a)) + DataLink(lid1a, d1a, s1a, t1a, u1a, t1, u2a) + ) assert hash(DataLink(lid1, d1, s1, t1, u1, t2, u2)) != hash( - DataLink(lid1a, d1a, s1a, t1a, u1a, t2a, u1a)) + DataLink(lid1a, d1a, s1a, t1a, u1a, t2a, u1a) + ) assert hash(DataLink(lid1, d1, s1, t1, u1, t1, u2)) != hash( - DataLink(lid1a, d1a, s1a, t1a, u1a)) + DataLink(lid1a, d1a, s1a, t1a, u1a) + ) assert hash(DataLink(lid1, d1, s1, t1, u1)) != hash( - DataLink(lid1a, d1a, s1a, t1a, u1a, t1a, u2a)) + DataLink(lid1a, d1a, s1a, t1a, u1a, t1a, u2a) + ) diff --git a/test/core/errors_test.py b/test/core/errors_test.py index 6cc5d517..cddd0f8c 100644 --- a/test/core/errors_test.py +++ b/test/core/errors_test.py @@ -9,29 +9,28 @@ def test_error_root_no_message(): e = SampleError(ErrorType.UNSUPPORTED_OP) - errstr = 'Sample service error code 100000 Unsupported operation' + errstr = "Sample service error code 100000 Unsupported operation" assert e.error_type == ErrorType.UNSUPPORTED_OP assert e.args == (errstr,) assert e.message is None - for msg in [None, ' \t ', '']: + for msg in [None, " \t ", ""]: e = SampleError(ErrorType.UNSUPPORTED_OP, msg) - errstr = 'Sample service error code 100000 Unsupported operation' + errstr = "Sample service error code 100000 Unsupported operation" assert e.error_type == ErrorType.UNSUPPORTED_OP assert e.args == (errstr,) assert e.message is None def test_error_root_with_message(): - e = SampleError(ErrorType.UNSUPPORTED_OP, ' really important message \t ') - errstr = 'Sample service error code 100000 Unsupported operation: really important message' + e = SampleError(ErrorType.UNSUPPORTED_OP, " really important message \t ") + errstr = "Sample service error code 100000 Unsupported operation: really important message" assert e.error_type == ErrorType.UNSUPPORTED_OP assert e.args == (errstr,) - assert e.message == 'really important message' + assert e.message == "really important message" def test_error_root_no_error_type(): with raises(Exception) as got: SampleError(None) - assert_exception_correct( - got.value, TypeError('error_type cannot be None')) + assert_exception_correct(got.value, TypeError("error_type cannot be None")) diff --git a/test/core/sample_test.py b/test/core/sample_test.py index 4db83528..1f03d0c0 100644 --- a/test/core/sample_test.py +++ b/test/core/sample_test.py @@ -16,74 +16,156 @@ def test_source_metadata_build(): - sm = SourceMetadata('k' * 256, 'f' * 256, { - 'foo': 1, - 'a' * 256: 'whee\twhoo', - 'k': 'b\n' + 'b' * 1022, - 'f': 1.4, - 'g': True}) - - assert sm.key == 'k' * 256 - assert sm.sourcekey == 'f' * 256 + sm = SourceMetadata( + "k" * 256, + "f" * 256, + { + "foo": 1, + "a" * 256: "whee\twhoo", + "k": "b\n" + "b" * 1022, + "f": 1.4, + "g": True, + }, + ) + + assert sm.key == "k" * 256 + assert sm.sourcekey == "f" * 256 assert sm.sourcevalue == { - 'foo': 1, - 'a' * 256: 'whee\twhoo', - 'k': 'b\n' + 'b' * 1022, - 'f': 1.4, - 'g': True} + "foo": 1, + "a" * 256: "whee\twhoo", + "k": "b\n" + "b" * 1022, + "f": 1.4, + "g": True, + } - sm = SourceMetadata('k', 'f', {'x': 'y'}) + sm = SourceMetadata("k", "f", {"x": "y"}) - assert sm.key == 'k' - assert sm.sourcekey == 'f' - assert sm.sourcevalue == {'x': 'y'} + assert sm.key == "k" + assert sm.sourcekey == "f" + assert sm.sourcevalue == {"x": "y"} def test_source_metadata_build_fail(): - _source_metadata_build_fail(None, 's', {}, IllegalParameterError( - 'Controlled metadata keys may not be null or whitespace only')) - _source_metadata_build_fail(' \n \t ', 's', {}, IllegalParameterError( - 'Controlled metadata keys may not be null or whitespace only')) _source_metadata_build_fail( - 'z' * 255 + 'ff', 'skey', {}, - IllegalParameterError(f"Controlled metadata has key starting with {'z' * 255 + 'f'} " + - 'that exceeds maximum length of 256')) + None, + "s", + {}, + IllegalParameterError( + "Controlled metadata keys may not be null or whitespace only" + ), + ) + _source_metadata_build_fail( + " \n \t ", + "s", + {}, + IllegalParameterError( + "Controlled metadata keys may not be null or whitespace only" + ), + ) _source_metadata_build_fail( - '\twhee', 'skey', {}, + "z" * 255 + "ff", + "skey", + {}, IllegalParameterError( - "Controlled metadata key \twhee's character at index 0 is a control character.")) + f"Controlled metadata has key starting with {'z' * 255 + 'f'} " + + "that exceeds maximum length of 256" + ), + ) + _source_metadata_build_fail( + "\twhee", + "skey", + {}, + IllegalParameterError( + "Controlled metadata key \twhee's character at index 0 is a control character." + ), + ) - _source_metadata_build_fail('k', None, {}, IllegalParameterError( - 'Source metadata keys may not be null or whitespace only')) - _source_metadata_build_fail('k', ' \n \t ', {}, IllegalParameterError( - 'Source metadata keys may not be null or whitespace only')) _source_metadata_build_fail( - 'k', 'b' * 255 + 'ff', {}, - IllegalParameterError(f"Source metadata has key starting with {'b' * 255 + 'f'} " + - 'that exceeds maximum length of 256')) + "k", + None, + {}, + IllegalParameterError( + "Source metadata keys may not be null or whitespace only" + ), + ) _source_metadata_build_fail( - 'k', 'thingy\n', {}, + "k", + " \n \t ", + {}, IllegalParameterError( - "Source metadata key thingy\n's character at index 6 is a control character.")) - - _source_metadata_build_fail('k', 'sk', None, IllegalParameterError( - 'Source metadata value associated with metadata key k is null or empty')) - _source_metadata_build_fail('k', 'sk', {}, IllegalParameterError( - 'Source metadata value associated with metadata key k is null or empty')) - _source_metadata_build_fail('k', 'skey', {'a' * 255 + 'ff': 'whee'}, IllegalParameterError( - 'Source metadata has a value key associated with metadata key k starting with ' + - f"{'a' * 255 + 'f'} that exceeds maximum length of 256")) - _source_metadata_build_fail('k2', 'skey', {'\twhee': {}}, IllegalParameterError( - 'Source metadata value key \twhee associated with metadata key k2 has a character at ' + - 'index 0 that is a control character.')) + "Source metadata keys may not be null or whitespace only" + ), + ) _source_metadata_build_fail( - 'somekey', 'skey', {'whee': 'a' * 255 + 'f' * 770}, + "k", + "b" * 255 + "ff", + {}, IllegalParameterError( - 'Source metadata has a value associated with metadata key somekey and value key ' + - f"whee starting with {'a' * 255 + 'f'} that exceeds maximum length of 1024")) - _source_metadata_build_fail('k3', 'skey', {'whee': 'whoop\bbutt'}, IllegalParameterError( - 'Source metadata value associated with metadata key k3 and value key whee has a ' + - 'character at index 5 that is a control character.')) + f"Source metadata has key starting with {'b' * 255 + 'f'} " + + "that exceeds maximum length of 256" + ), + ) + _source_metadata_build_fail( + "k", + "thingy\n", + {}, + IllegalParameterError( + "Source metadata key thingy\n's character at index 6 is a control character." + ), + ) + + _source_metadata_build_fail( + "k", + "sk", + None, + IllegalParameterError( + "Source metadata value associated with metadata key k is null or empty" + ), + ) + _source_metadata_build_fail( + "k", + "sk", + {}, + IllegalParameterError( + "Source metadata value associated with metadata key k is null or empty" + ), + ) + _source_metadata_build_fail( + "k", + "skey", + {"a" * 255 + "ff": "whee"}, + IllegalParameterError( + "Source metadata has a value key associated with metadata key k starting with " + + f"{'a' * 255 + 'f'} that exceeds maximum length of 256" + ), + ) + _source_metadata_build_fail( + "k2", + "skey", + {"\twhee": {}}, + IllegalParameterError( + "Source metadata value key \twhee associated with metadata key k2 has a character at " + + "index 0 that is a control character." + ), + ) + _source_metadata_build_fail( + "somekey", + "skey", + {"whee": "a" * 255 + "f" * 770}, + IllegalParameterError( + "Source metadata has a value associated with metadata key somekey and value key " + + f"whee starting with {'a' * 255 + 'f'} that exceeds maximum length of 1024" + ), + ) + _source_metadata_build_fail( + "k3", + "skey", + {"whee": "whoop\bbutt"}, + IllegalParameterError( + "Source metadata value associated with metadata key k3 and value key whee has a " + + "character at index 5 that is a control character." + ), + ) def _source_metadata_build_fail(key, skey, value, expected): @@ -94,13 +176,13 @@ def _source_metadata_build_fail(key, skey, value, expected): def test_source_metadata_eq(): - assert SourceMetadata('k', 'f', {'x': 'y'}) == SourceMetadata('k', 'f', {'x': 'y'}) - assert SourceMetadata('k', 'f', {'x': 'y'}) != SourceMetadata('k1', 'f', {'x': 'y'}) - assert SourceMetadata('k', 'f', {'x': 'y'}) != SourceMetadata('k', 'f1', {'x': 'y'}) - assert SourceMetadata('k', 'f', {'a': 'b'}) != SourceMetadata('k', 'f', {'a': 'c'}) + assert SourceMetadata("k", "f", {"x": "y"}) == SourceMetadata("k", "f", {"x": "y"}) + assert SourceMetadata("k", "f", {"x": "y"}) != SourceMetadata("k1", "f", {"x": "y"}) + assert SourceMetadata("k", "f", {"x": "y"}) != SourceMetadata("k", "f1", {"x": "y"}) + assert SourceMetadata("k", "f", {"a": "b"}) != SourceMetadata("k", "f", {"a": "c"}) - assert SourceMetadata('k', 'f', {'x': 'y'}) != 'k' - assert {} != SourceMetadata('k', 'f', {'x': 'y'}) + assert SourceMetadata("k", "f", {"x": "y"}) != "k" + assert {} != SourceMetadata("k", "f", {"x": "y"}) def test_source_metadata_hash(): @@ -108,59 +190,79 @@ def test_source_metadata_hash(): # tests can't be written that directly test the hash value. See # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - assert hash(SourceMetadata('k', 'f', {'x': 'y'})) == hash(SourceMetadata('k', 'f', {'x': 'y'})) - assert hash(SourceMetadata('k', 'f', {'x': 'y'})) != hash(SourceMetadata('k1', 'f', {'x': 'y'})) - assert hash(SourceMetadata('k', 'f', {'x': 'y'})) != hash(SourceMetadata('k', 'f1', {'x': 'y'})) - assert hash(SourceMetadata('k', 'f', {'a': 'b'})) != hash(SourceMetadata('k', 'f', {'a': 'c'})) + assert hash(SourceMetadata("k", "f", {"x": "y"})) == hash( + SourceMetadata("k", "f", {"x": "y"}) + ) + assert hash(SourceMetadata("k", "f", {"x": "y"})) != hash( + SourceMetadata("k1", "f", {"x": "y"}) + ) + assert hash(SourceMetadata("k", "f", {"x": "y"})) != hash( + SourceMetadata("k", "f1", {"x": "y"}) + ) + assert hash(SourceMetadata("k", "f", {"a": "b"})) != hash( + SourceMetadata("k", "f", {"a": "c"}) + ) def test_sample_node_build(): - sn = SampleNode('foo', SubSampleType.BIOLOGICAL_REPLICATE) - assert sn.name == 'foo' + sn = SampleNode("foo", SubSampleType.BIOLOGICAL_REPLICATE) + assert sn.name == "foo" assert sn.type == SubSampleType.BIOLOGICAL_REPLICATE assert sn.parent is None assert sn.controlled_metadata == {} assert sn.user_metadata == {} assert sn.source_metadata == () - sn = SampleNode('a' * 256, SubSampleType.TECHNICAL_REPLICATE, 'b' * 256) - assert sn.name == 'a' * 256 + sn = SampleNode("a" * 256, SubSampleType.TECHNICAL_REPLICATE, "b" * 256) + assert sn.name == "a" * 256 assert sn.type == SubSampleType.TECHNICAL_REPLICATE - assert sn.parent == 'b' * 256 + assert sn.parent == "b" * 256 assert sn.controlled_metadata == {} assert sn.user_metadata == {} assert sn.source_metadata == () - sn = SampleNode('a' * 256, SubSampleType.TECHNICAL_REPLICATE, 'b' * 256, - {'a' * 256: {'bar': 'baz', 'bat': 'wh\tee'}, - 'wugga': {'a': 'b' * 1024}, - # tests that having a controlled key doesn't force a source key - 'z': {'u': 'v'}}, - {'a': {'b' * 256: 'fo\no', 'c': 1, 'd': 1.5, 'e': False}}, - [SourceMetadata('a' * 256, 'sk', {'a': 'b'}), - SourceMetadata('wugga', 'sk', {'a': 'b'})]) - assert sn.name == 'a' * 256 + sn = SampleNode( + "a" * 256, + SubSampleType.TECHNICAL_REPLICATE, + "b" * 256, + { + "a" * 256: {"bar": "baz", "bat": "wh\tee"}, + "wugga": {"a": "b" * 1024}, + # tests that having a controlled key doesn't force a source key + "z": {"u": "v"}, + }, + {"a": {"b" * 256: "fo\no", "c": 1, "d": 1.5, "e": False}}, + [ + SourceMetadata("a" * 256, "sk", {"a": "b"}), + SourceMetadata("wugga", "sk", {"a": "b"}), + ], + ) + assert sn.name == "a" * 256 assert sn.type == SubSampleType.TECHNICAL_REPLICATE - assert sn.parent == 'b' * 256 - assert sn.controlled_metadata == {'a' * 256: {'bar': 'baz', 'bat': 'wh\tee'}, - 'wugga': {'a': 'b' * 1024}, - 'z': {'u': 'v'}} - assert sn.user_metadata == {'a': {'b' * 256: 'fo\no', 'c': 1, 'd': 1.5, 'e': False}} - assert sn.source_metadata == (SourceMetadata('a' * 256, 'sk', {'a': 'b'}), - SourceMetadata('wugga', 'sk', {'a': 'b'})) + assert sn.parent == "b" * 256 + assert sn.controlled_metadata == { + "a" * 256: {"bar": "baz", "bat": "wh\tee"}, + "wugga": {"a": "b" * 1024}, + "z": {"u": "v"}, + } + assert sn.user_metadata == {"a": {"b" * 256: "fo\no", "c": 1, "d": 1.5, "e": False}} + assert sn.source_metadata == ( + SourceMetadata("a" * 256, "sk", {"a": "b"}), + SourceMetadata("wugga", "sk", {"a": "b"}), + ) # 100KB when serialized to json - meta = {str(i): {'b': '𐎦' * 25} for i in range(848)} - meta['a'] = {'b': 'c' * 30} + meta = {str(i): {"b": "𐎦" * 25} for i in range(848)} + meta["a"] = {"b": "c" * 30} # Also 100KB when the size calculation routine is run - smeta = [SourceMetadata(str(i), 'sksksk', {'x': '𐎦' * 25}) for i in range(848)] - smeta.append(SourceMetadata('a', 'b' * 35, {'u': 'v'})) + smeta = [SourceMetadata(str(i), "sksksk", {"x": "𐎦" * 25}) for i in range(848)] + smeta.append(SourceMetadata("a", "b" * 35, {"u": "v"})) - sn = SampleNode('a', SubSampleType.SUB_SAMPLE, 'b', meta, meta, smeta) - assert sn.name == 'a' + sn = SampleNode("a", SubSampleType.SUB_SAMPLE, "b", meta, meta, smeta) + assert sn.name == "a" assert sn.type == SubSampleType.SUB_SAMPLE - assert sn.parent == 'b' + assert sn.parent == "b" assert sn.controlled_metadata == meta assert sn.user_metadata == meta assert sn.source_metadata == tuple(smeta) @@ -169,23 +271,51 @@ def test_sample_node_build(): def test_sample_node_build_fail(): # not testing every permutation of failing check_string here, just one test to make sure # it's there - _sample_node_build_fail('', SubSampleType.BIOLOGICAL_REPLICATE, None, - MissingParameterError('subsample name')) - _sample_node_build_fail('a' * 257, SubSampleType.BIOLOGICAL_REPLICATE, None, - IllegalParameterError('subsample name exceeds maximum length of 256')) - _sample_node_build_fail('a', None, None, - ValueError('type cannot be a value that evaluates to false')) - _sample_node_build_fail('a', SubSampleType.TECHNICAL_REPLICATE, 'b' * 257, - IllegalParameterError('parent exceeds maximum length of 256')) _sample_node_build_fail( - 'a', SubSampleType.BIOLOGICAL_REPLICATE, 'badparent', IllegalParameterError( - 'Node a is of type BioReplicate and therefore cannot have a parent')) + "", + SubSampleType.BIOLOGICAL_REPLICATE, + None, + MissingParameterError("subsample name"), + ) + _sample_node_build_fail( + "a" * 257, + SubSampleType.BIOLOGICAL_REPLICATE, + None, + IllegalParameterError("subsample name exceeds maximum length of 256"), + ) _sample_node_build_fail( - 'a', SubSampleType.TECHNICAL_REPLICATE, None, IllegalParameterError( - 'Node a is of type TechReplicate and therefore must have a parent')) + "a", None, None, ValueError("type cannot be a value that evaluates to false") + ) + _sample_node_build_fail( + "a", + SubSampleType.TECHNICAL_REPLICATE, + "b" * 257, + IllegalParameterError("parent exceeds maximum length of 256"), + ) + _sample_node_build_fail( + "a", + SubSampleType.BIOLOGICAL_REPLICATE, + "badparent", + IllegalParameterError( + "Node a is of type BioReplicate and therefore cannot have a parent" + ), + ) _sample_node_build_fail( - 'a', SubSampleType.SUB_SAMPLE, None, IllegalParameterError( - 'Node a is of type SubSample and therefore must have a parent')) + "a", + SubSampleType.TECHNICAL_REPLICATE, + None, + IllegalParameterError( + "Node a is of type TechReplicate and therefore must have a parent" + ), + ) + _sample_node_build_fail( + "a", + SubSampleType.SUB_SAMPLE, + None, + IllegalParameterError( + "Node a is of type SubSample and therefore must have a parent" + ), + ) def _sample_node_build_fail(name, type_, parent, expected): @@ -196,93 +326,114 @@ def _sample_node_build_fail(name, type_, parent, expected): def test_sample_node_build_fail_metadata(): _sample_node_build_fail_metadata( - {None: {}}, - '{} metadata keys may not be null or whitespace only') + {None: {}}, "{} metadata keys may not be null or whitespace only" + ) _sample_node_build_fail_metadata( - {' \t \n ': {}}, - '{} metadata keys may not be null or whitespace only') + {" \t \n ": {}}, "{} metadata keys may not be null or whitespace only" + ) _sample_node_build_fail_metadata( - {'foo': None}, - '{} metadata value associated with metadata key foo is null or empty') + {"foo": None}, + "{} metadata value associated with metadata key foo is null or empty", + ) _sample_node_build_fail_metadata( - {'foo': {}}, - '{} metadata value associated with metadata key foo is null or empty') + {"foo": {}}, + "{} metadata value associated with metadata key foo is null or empty", + ) _sample_node_build_fail_metadata( - {'a' * 255 + 'ff': {}}, - f"{{}} metadata has key starting with {'a' * 255 + 'f'} " + - "that exceeds maximum length of 256") + {"a" * 255 + "ff": {}}, + f"{{}} metadata has key starting with {'a' * 255 + 'f'} " + + "that exceeds maximum length of 256", + ) _sample_node_build_fail_metadata( - {'wh\tee': {}}, - "{} metadata key wh\tee's character at index 2 is a control character.") + {"wh\tee": {}}, + "{} metadata key wh\tee's character at index 2 is a control character.", + ) _sample_node_build_fail_metadata( - {'bat': {'a' * 255 + 'ff': 'whee'}}, - '{} metadata has a value key associated with metadata key bat starting with ' + - f"{'a' * 255 + 'f'} that exceeds maximum length of 256") + {"bat": {"a" * 255 + "ff": "whee"}}, + "{} metadata has a value key associated with metadata key bat starting with " + + f"{'a' * 255 + 'f'} that exceeds maximum length of 256", + ) _sample_node_build_fail_metadata( - {'wugga': {'wh\tee': {}}}, - '{} metadata value key wh\tee associated with metadata key wugga has a character at ' + - 'index 2 that is a control character.') + {"wugga": {"wh\tee": {}}}, + "{} metadata value key wh\tee associated with metadata key wugga has a character at " + + "index 2 that is a control character.", + ) _sample_node_build_fail_metadata( - {'bat': {'whee': 'a' * 255 + 'f' * 770}}, - '{} metadata has a value associated with metadata key bat and value key whee starting ' + - f"with {'a' * 255 + 'f'} that exceeds maximum length of 1024") + {"bat": {"whee": "a" * 255 + "f" * 770}}, + "{} metadata has a value associated with metadata key bat and value key whee starting " + + f"with {'a' * 255 + 'f'} that exceeds maximum length of 1024", + ) _sample_node_build_fail_metadata( - {'bat': {'whee': '\bwhoopbutt'}}, - '{} metadata value associated with metadata key bat and value key whee has a ' + - 'character at index 0 that is a control character.') + {"bat": {"whee": "\bwhoopbutt"}}, + "{} metadata value associated with metadata key bat and value key whee has a " + + "character at index 0 that is a control character.", + ) # 100001B when serialized to json - meta = {str(i): {'b': '𐎦' * 25} for i in range(848)} - meta['a'] = {'b': 'c' * 31} - _sample_node_build_fail_metadata(meta, "{} metadata is larger than maximum of 100000B") + meta = {str(i): {"b": "𐎦" * 25} for i in range(848)} + meta["a"] = {"b": "c" * 31} + _sample_node_build_fail_metadata( + meta, "{} metadata is larger than maximum of 100000B" + ) def _sample_node_build_fail_metadata(meta, expected): with raises(Exception) as got: - SampleNode('n', SubSampleType.BIOLOGICAL_REPLICATE, controlled_metadata=meta) - assert_exception_correct(got.value, IllegalParameterError(expected.format('Controlled'))) + SampleNode("n", SubSampleType.BIOLOGICAL_REPLICATE, controlled_metadata=meta) + assert_exception_correct( + got.value, IllegalParameterError(expected.format("Controlled")) + ) with raises(Exception) as got: - SampleNode('n', SubSampleType.BIOLOGICAL_REPLICATE, user_metadata=meta) - assert_exception_correct(got.value, IllegalParameterError(expected.format('User'))) + SampleNode("n", SubSampleType.BIOLOGICAL_REPLICATE, user_metadata=meta) + assert_exception_correct(got.value, IllegalParameterError(expected.format("User"))) def test_sample_node_build_fail_source_metadata(): _sample_node_build_fail_source_metadata( - [SourceMetadata('k', 'k1', {'a': 'b'}), None], ValueError( - 'Index 1 of iterable source_metadata cannot be a value that evaluates to false')) + [SourceMetadata("k", "k1", {"a": "b"}), None], + ValueError( + "Index 1 of iterable source_metadata cannot be a value that evaluates to false" + ), + ) _sample_node_build_fail_source_metadata( - [SourceMetadata('f', 'k1', {'a': 'b'}), SourceMetadata('k', 'k1', {'c': 'd'})], + [SourceMetadata("f", "k1", {"a": "b"}), SourceMetadata("k", "k1", {"c": "d"})], IllegalParameterError( - 'Source metadata key k does not appear in the controlled metadata'), - cmeta={'f': {'x': 'y'}}) + "Source metadata key k does not appear in the controlled metadata" + ), + cmeta={"f": {"x": "y"}}, + ) _sample_node_build_fail_source_metadata( - [SourceMetadata('k', 'k1', {'a': 'b'}), SourceMetadata('k', 'k2', {'a': 2})], - IllegalParameterError('Duplicate source metadata key: k')) + [SourceMetadata("k", "k1", {"a": "b"}), SourceMetadata("k", "k2", {"a": 2})], + IllegalParameterError("Duplicate source metadata key: k"), + ) # 100001KB when the size calculation routine is run - smeta = [SourceMetadata(str(i), 'sksksk', {'x': '𐎦' * 25}) for i in range(848)] - smeta.append(SourceMetadata('a', 'b' * 36, {'x': 'y'})) - _sample_node_build_fail_source_metadata(smeta, IllegalParameterError( - 'Source metadata is larger than maximum of 100000B')) + smeta = [SourceMetadata(str(i), "sksksk", {"x": "𐎦" * 25}) for i in range(848)] + smeta.append(SourceMetadata("a", "b" * 36, {"x": "y"})) + _sample_node_build_fail_source_metadata( + smeta, + IllegalParameterError("Source metadata is larger than maximum of 100000B"), + ) def _sample_node_build_fail_source_metadata(meta, expected, cmeta=None): if cmeta is None: - cmeta = {sm.key: {'x': 'y'} for sm in meta if sm is not None} + cmeta = {sm.key: {"x": "y"} for sm in meta if sm is not None} with raises(Exception) as got: SampleNode( - 'n', + "n", SubSampleType.BIOLOGICAL_REPLICATE, controlled_metadata=cmeta, - source_metadata=meta) + source_metadata=meta, + ) assert_exception_correct(got.value, expected) @@ -291,59 +442,71 @@ def test_sample_node_eq(): s = SubSampleType.SUB_SAMPLE r = SubSampleType.BIOLOGICAL_REPLICATE - assert SampleNode('foo') == SampleNode('foo') - assert SampleNode('foo') != SampleNode('bar') - - assert SampleNode('foo', r) == SampleNode('foo', r) - assert SampleNode('foo', r) != SampleNode('foo', s, 'baz') - - assert SampleNode('foo', s, 'bar') == SampleNode('foo', s, 'bar') - assert SampleNode('foo', s, 'bar') != SampleNode('foo', t, 'bar') - assert SampleNode('foo', s, 'bar') != SampleNode('foo', s, 'bat') - - assert SampleNode('foo', s, 'bar', {'foo': {'a': 'b'}}) == SampleNode( - 'foo', s, 'bar', {'foo': {'a': 'b'}}) - assert SampleNode('foo', s, 'bar', {'foo': {'a': 'b'}}) != SampleNode( - 'foo', s, 'bar', {'foo': {'a': 'c'}}) - assert SampleNode('foo', s, 'bar', {'foo': {'a': 'b'}}) != SampleNode( - 'foo', s, 'bar', {'foo': {'z': 'b'}}) - assert SampleNode('foo', s, 'bar', {'foo': {'a': 'b'}}) != SampleNode( - 'foo', s, 'bar', {'fo': {'a': 'b'}}) - - assert SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'b'}}) == SampleNode( - 'foo', s, 'bar', user_metadata={'foo': {'a': 'b'}}) - assert SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'b'}}) != SampleNode( - 'foo', s, 'bar', user_metadata={'foo': {'a': 'c'}}) - assert SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'b'}}) != SampleNode( - 'foo', s, 'bar', user_metadata={'foo': {'z': 'b'}}) - assert SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'b'}}) != SampleNode( - 'foo', s, 'bar', user_metadata={'fo': {'a': 'b'}}) + assert SampleNode("foo") == SampleNode("foo") + assert SampleNode("foo") != SampleNode("bar") + + assert SampleNode("foo", r) == SampleNode("foo", r) + assert SampleNode("foo", r) != SampleNode("foo", s, "baz") + + assert SampleNode("foo", s, "bar") == SampleNode("foo", s, "bar") + assert SampleNode("foo", s, "bar") != SampleNode("foo", t, "bar") + assert SampleNode("foo", s, "bar") != SampleNode("foo", s, "bat") + + assert SampleNode("foo", s, "bar", {"foo": {"a": "b"}}) == SampleNode( + "foo", s, "bar", {"foo": {"a": "b"}} + ) + assert SampleNode("foo", s, "bar", {"foo": {"a": "b"}}) != SampleNode( + "foo", s, "bar", {"foo": {"a": "c"}} + ) + assert SampleNode("foo", s, "bar", {"foo": {"a": "b"}}) != SampleNode( + "foo", s, "bar", {"foo": {"z": "b"}} + ) + assert SampleNode("foo", s, "bar", {"foo": {"a": "b"}}) != SampleNode( + "foo", s, "bar", {"fo": {"a": "b"}} + ) + + assert SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "b"}}) == SampleNode( + "foo", s, "bar", user_metadata={"foo": {"a": "b"}} + ) + assert SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "b"}}) != SampleNode( + "foo", s, "bar", user_metadata={"foo": {"a": "c"}} + ) + assert SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "b"}}) != SampleNode( + "foo", s, "bar", user_metadata={"foo": {"z": "b"}} + ) + assert SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "b"}}) != SampleNode( + "foo", s, "bar", user_metadata={"fo": {"a": "b"}} + ) assert SampleNode( - 'foo', + "foo", s, - 'bar', - controlled_metadata={'k': {'a': 'b'}}, - source_metadata=[SourceMetadata('k', 'k', {'c': 'd'})]) == SampleNode( - 'foo', - s, - 'bar', - controlled_metadata={'k': {'a': 'b'}}, - source_metadata=[SourceMetadata('k', 'k', {'c': 'd'})]) + "bar", + controlled_metadata={"k": {"a": "b"}}, + source_metadata=[SourceMetadata("k", "k", {"c": "d"})], + ) == SampleNode( + "foo", + s, + "bar", + controlled_metadata={"k": {"a": "b"}}, + source_metadata=[SourceMetadata("k", "k", {"c": "d"})], + ) assert SampleNode( - 'foo', + "foo", s, - 'bar', - controlled_metadata={'k': {'a': 'b'}}, - source_metadata=[SourceMetadata('k', 'k', {'c': 'd'})]) != SampleNode( - 'foo', - s, - 'bar', - controlled_metadata={'k': {'a': 'b'}}, - source_metadata=[SourceMetadata('k', 'v', {'c': 'd'})]) + "bar", + controlled_metadata={"k": {"a": "b"}}, + source_metadata=[SourceMetadata("k", "k", {"c": "d"})], + ) != SampleNode( + "foo", + s, + "bar", + controlled_metadata={"k": {"a": "b"}}, + source_metadata=[SourceMetadata("k", "v", {"c": "d"})], + ) - assert SampleNode('foo') != 'foo' - assert 'foo' != SampleNode('foo') + assert SampleNode("foo") != "foo" + assert "foo" != SampleNode("foo") def test_sample_node_hash(): @@ -355,56 +518,76 @@ def test_sample_node_hash(): s = SubSampleType.SUB_SAMPLE r = SubSampleType.BIOLOGICAL_REPLICATE - assert hash(SampleNode('foo')) == hash(SampleNode('foo')) - assert hash(SampleNode('bar')) == hash(SampleNode('bar')) - assert hash(SampleNode('foo')) != hash(SampleNode('bar')) - assert hash(SampleNode('foo', r)) == hash(SampleNode('foo', r)) - assert hash(SampleNode('foo', r)) != hash(SampleNode('foo', s, 'baz')) - assert hash(SampleNode('foo', s, 'bar')) == hash(SampleNode('foo', s, 'bar')) - assert hash(SampleNode('foo', t, 'bat')) == hash(SampleNode('foo', t, 'bat')) - assert hash(SampleNode('foo', s, 'bar')) != hash(SampleNode('foo', t, 'bar')) - assert hash(SampleNode('foo', s, 'bar')) != hash(SampleNode('foo', s, 'bat')) - - assert hash(SampleNode('foo', s, 'bar', {'foo': {'a': 'b'}})) == hash(SampleNode( - 'foo', s, 'bar', {'foo': {'a': 'b'}})) - assert hash(SampleNode('foo', s, 'bar', {'foo': {'a': 'b'}})) != hash(SampleNode( - 'foo', s, 'bar', {'foo': {'a': 'c'}})) - assert hash(SampleNode('foo', s, 'bar', {'foo': {'a': 'b'}})) != hash(SampleNode( - 'foo', s, 'bar', {'foo': {'z': 'b'}})) - assert hash(SampleNode('foo', s, 'bar', {'foo': {'a': 'b'}})) != hash(SampleNode( - 'foo', s, 'bar', {'fo': {'a': 'b'}})) - - assert hash(SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'b'}})) == hash( - SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'b'}})) - assert hash(SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'b'}})) != hash( - SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'c'}})) - assert hash(SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'b'}})) != hash( - SampleNode('foo', s, 'bar', user_metadata={'foo': {'z': 'b'}})) - assert hash(SampleNode('foo', s, 'bar', user_metadata={'foo': {'a': 'b'}})) != hash( - SampleNode('foo', s, 'bar', user_metadata={'fo': {'a': 'b'}})) - - assert hash(SampleNode( - 'foo', - s, - 'bar', - controlled_metadata={'k': {'a': 'b'}}, - source_metadata=[SourceMetadata('k', 'k', {'c': 'd'})])) == hash(SampleNode( - 'foo', + assert hash(SampleNode("foo")) == hash(SampleNode("foo")) + assert hash(SampleNode("bar")) == hash(SampleNode("bar")) + assert hash(SampleNode("foo")) != hash(SampleNode("bar")) + assert hash(SampleNode("foo", r)) == hash(SampleNode("foo", r)) + assert hash(SampleNode("foo", r)) != hash(SampleNode("foo", s, "baz")) + assert hash(SampleNode("foo", s, "bar")) == hash(SampleNode("foo", s, "bar")) + assert hash(SampleNode("foo", t, "bat")) == hash(SampleNode("foo", t, "bat")) + assert hash(SampleNode("foo", s, "bar")) != hash(SampleNode("foo", t, "bar")) + assert hash(SampleNode("foo", s, "bar")) != hash(SampleNode("foo", s, "bat")) + + assert hash(SampleNode("foo", s, "bar", {"foo": {"a": "b"}})) == hash( + SampleNode("foo", s, "bar", {"foo": {"a": "b"}}) + ) + assert hash(SampleNode("foo", s, "bar", {"foo": {"a": "b"}})) != hash( + SampleNode("foo", s, "bar", {"foo": {"a": "c"}}) + ) + assert hash(SampleNode("foo", s, "bar", {"foo": {"a": "b"}})) != hash( + SampleNode("foo", s, "bar", {"foo": {"z": "b"}}) + ) + assert hash(SampleNode("foo", s, "bar", {"foo": {"a": "b"}})) != hash( + SampleNode("foo", s, "bar", {"fo": {"a": "b"}}) + ) + + assert hash(SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "b"}})) == hash( + SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "b"}}) + ) + assert hash(SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "b"}})) != hash( + SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "c"}}) + ) + assert hash(SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "b"}})) != hash( + SampleNode("foo", s, "bar", user_metadata={"foo": {"z": "b"}}) + ) + assert hash(SampleNode("foo", s, "bar", user_metadata={"foo": {"a": "b"}})) != hash( + SampleNode("foo", s, "bar", user_metadata={"fo": {"a": "b"}}) + ) + + assert hash( + SampleNode( + "foo", s, - 'bar', - controlled_metadata={'k': {'a': 'b'}}, - source_metadata=[SourceMetadata('k', 'k', {'c': 'd'})])) - assert hash(SampleNode( - 'foo', - s, - 'bar', - controlled_metadata={'k': {'a': 'b'}}, - source_metadata=[SourceMetadata('k', 'k', {'c': 'd'})])) != hash(SampleNode( - 'foo', + "bar", + controlled_metadata={"k": {"a": "b"}}, + source_metadata=[SourceMetadata("k", "k", {"c": "d"})], + ) + ) == hash( + SampleNode( + "foo", + s, + "bar", + controlled_metadata={"k": {"a": "b"}}, + source_metadata=[SourceMetadata("k", "k", {"c": "d"})], + ) + ) + assert hash( + SampleNode( + "foo", s, - 'bar', - controlled_metadata={'k': {'a': 'b'}}, - source_metadata=[SourceMetadata('k', 'v', {'c': 'd'})])) + "bar", + controlled_metadata={"k": {"a": "b"}}, + source_metadata=[SourceMetadata("k", "k", {"c": "d"})], + ) + ) != hash( + SampleNode( + "foo", + s, + "bar", + controlled_metadata={"k": {"a": "b"}}, + source_metadata=[SourceMetadata("k", "v", {"c": "d"})], + ) + ) def dt(timestamp): @@ -412,60 +595,60 @@ def dt(timestamp): def test_sample_build(): - sn = SampleNode('foo') - sn2 = SampleNode('bat') - sn3 = SampleNode('bar', type_=SubSampleType.TECHNICAL_REPLICATE, parent='foo') - sn4 = SampleNode('baz', type_=SubSampleType.SUB_SAMPLE, parent='foo') - sndup = SampleNode('foo') + sn = SampleNode("foo") + sn2 = SampleNode("bat") + sn3 = SampleNode("bar", type_=SubSampleType.TECHNICAL_REPLICATE, parent="foo") + sn4 = SampleNode("baz", type_=SubSampleType.SUB_SAMPLE, parent="foo") + sndup = SampleNode("foo") s = Sample([sn]) assert s.nodes == (sndup,) assert s.name is None - s = Sample([sn, sn2, sn4, sn3], ' \t foo ') + s = Sample([sn, sn2, sn4, sn3], " \t foo ") assert s.nodes == (sndup, sn2, sn4, sn3) - assert s.name == 'foo' + assert s.name == "foo" - s = Sample([sn], 'a' * 256) + s = Sample([sn], "a" * 256) assert s.nodes == (sndup,) - assert s.name == 'a' * 256 + assert s.name == "a" * 256 - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - s = SavedSample(id_, UserID('user'), [sn], dt(6)) - assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef') - assert s.user == UserID('user') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + s = SavedSample(id_, UserID("user"), [sn], dt(6)) + assert s.id == uuid.UUID("1234567890abcdef1234567890abcdef") + assert s.user == UserID("user") assert s.nodes == (sndup,) assert s.savetime == dt(6) assert s.name is None assert s.version is None - s = SavedSample(id_, UserID('user2'), [sn], dt(6), 'foo') - assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef') - assert s.user == UserID('user2') + s = SavedSample(id_, UserID("user2"), [sn], dt(6), "foo") + assert s.id == uuid.UUID("1234567890abcdef1234567890abcdef") + assert s.user == UserID("user2") assert s.nodes == (sndup,) assert s.savetime == dt(6) - assert s.name == 'foo' + assert s.name == "foo" assert s.version is None - s = SavedSample(id_, UserID('user'), [sn], dt(6), 'foo', 1) - assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef') - assert s.user == UserID('user') + s = SavedSample(id_, UserID("user"), [sn], dt(6), "foo", 1) + assert s.id == uuid.UUID("1234567890abcdef1234567890abcdef") + assert s.user == UserID("user") assert s.nodes == (sndup,) assert s.savetime == dt(6) - assert s.name == 'foo' + assert s.name == "foo" assert s.version == 1 - s = SavedSample(id_, UserID('user'), [sn], dt(6), 'foo', 8) - assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef') - assert s.user == UserID('user') + s = SavedSample(id_, UserID("user"), [sn], dt(6), "foo", 8) + assert s.id == uuid.UUID("1234567890abcdef1234567890abcdef") + assert s.user == UserID("user") assert s.nodes == (sndup,) assert s.savetime == dt(6) - assert s.name == 'foo' + assert s.name == "foo" assert s.version == 8 - s = SavedSample(id_, UserID('user'), [sn], dt(6), version=8) - assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef') - assert s.user == UserID('user') + s = SavedSample(id_, UserID("user"), [sn], dt(6), version=8) + assert s.id == uuid.UUID("1234567890abcdef1234567890abcdef") + assert s.user == UserID("user") assert s.nodes == (sndup,) assert s.savetime == dt(6) assert s.name is None @@ -476,56 +659,104 @@ def test_sample_build_fail(): # not testing every permutation of failing check_string here, just one test to make sure # it's there - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - u = UserID('user') - sn = SampleNode('foo') - tn = SampleNode('bar', SubSampleType.TECHNICAL_REPLICATE, 'foo') - sn2 = SampleNode('baz') - dup = SampleNode('foo') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + u = UserID("user") + sn = SampleNode("foo") + tn = SampleNode("bar", SubSampleType.TECHNICAL_REPLICATE, "foo") + sn2 = SampleNode("baz") + dup = SampleNode("foo") d = dt(8) _sample_build_fail( - [sn], 'a' * 257, IllegalParameterError('name exceeds maximum length of 256')) - _sample_build_fail([], None, MissingParameterError('At least one node per sample is required')) + [sn], "a" * 257, IllegalParameterError("name exceeds maximum length of 256") + ) + _sample_build_fail( + [], None, MissingParameterError("At least one node per sample is required") + ) _sample_build_fail( - [tn, sn], 'a', IllegalParameterError('The first node in a sample must be a BioReplicate')) - _sample_build_fail([sn, tn, sn2], 'a', IllegalParameterError( - 'BioReplicates must be the first nodes in the list of sample nodes.')) - _sample_build_fail([sn, sn2, dup], 'a', IllegalParameterError( - 'Duplicate sample node name: foo')) - _sample_build_fail([sn2, tn], 'a', IllegalParameterError( - 'Parent foo of node bar does not appear in node list prior to node.')) - - _sample_with_id_build_fail(None, u, [sn], d, None, None, - ValueError('id_ cannot be a value that evaluates to false')) - _sample_with_id_build_fail(id_, None, [sn], d, None, None, - ValueError('user cannot be a value that evaluates to false')) - _sample_with_id_build_fail(id_, u, [sn], None, None, None, ValueError( - 'savetime cannot be a value that evaluates to false')) - _sample_with_id_build_fail(id_, u, [sn], datetime.datetime.now(), None, None, ValueError( - 'savetime cannot be a naive datetime')) - _sample_with_id_build_fail(id_, u, [sn], d, None, 0, ValueError('version must be > 0')) + [tn, sn], + "a", + IllegalParameterError("The first node in a sample must be a BioReplicate"), + ) + _sample_build_fail( + [sn, tn, sn2], + "a", + IllegalParameterError( + "BioReplicates must be the first nodes in the list of sample nodes." + ), + ) + _sample_build_fail( + [sn, sn2, dup], "a", IllegalParameterError("Duplicate sample node name: foo") + ) + _sample_build_fail( + [sn2, tn], + "a", + IllegalParameterError( + "Parent foo of node bar does not appear in node list prior to node." + ), + ) + + _sample_with_id_build_fail( + None, + u, + [sn], + d, + None, + None, + ValueError("id_ cannot be a value that evaluates to false"), + ) + _sample_with_id_build_fail( + id_, + None, + [sn], + d, + None, + None, + ValueError("user cannot be a value that evaluates to false"), + ) + _sample_with_id_build_fail( + id_, + u, + [sn], + None, + None, + None, + ValueError("savetime cannot be a value that evaluates to false"), + ) + _sample_with_id_build_fail( + id_, + u, + [sn], + datetime.datetime.now(), + None, + None, + ValueError("savetime cannot be a naive datetime"), + ) + _sample_with_id_build_fail( + id_, u, [sn], d, None, 0, ValueError("version must be > 0") + ) def test_sample_build_fail_sample_count(): - nodes = [SampleNode('s' + str(i)) for i in range(10000)] + nodes = [SampleNode("s" + str(i)) for i in range(10000)] s = Sample(nodes) assert s.nodes == tuple(nodes) assert s.name is None - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - s = SavedSample(id_, UserID('u'), nodes, dt(8)) - assert s.id == uuid.UUID('1234567890abcdef1234567890abcdef') - assert s.user == UserID('u') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + s = SavedSample(id_, UserID("u"), nodes, dt(8)) + assert s.id == uuid.UUID("1234567890abcdef1234567890abcdef") + assert s.user == UserID("u") assert s.nodes == tuple(nodes) assert s.savetime == dt(8) assert s.name is None assert s.version is None - nodes.append(SampleNode('s10000')) - _sample_build_fail(nodes, None, IllegalParameterError( - 'At most 10000 nodes are allowed per sample')) + nodes.append(SampleNode("s10000")) + _sample_build_fail( + nodes, None, IllegalParameterError("At most 10000 nodes are allowed per sample") + ) def _sample_build_fail(nodes, name, expected): @@ -533,9 +764,9 @@ def _sample_build_fail(nodes, name, expected): Sample(nodes, name) assert_exception_correct(got.value, expected) - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") with raises(Exception) as got: - SavedSample(id_, UserID('u'), nodes, dt(8), name) + SavedSample(id_, UserID("u"), nodes, dt(8), name) assert_exception_correct(got.value, expected) @@ -547,19 +778,19 @@ def _sample_with_id_build_fail(id_, user, nodes, savetime, name, version, expect def test_sample_eq(): - sn = SampleNode('foo') - sn2 = SampleNode('bar') + sn = SampleNode("foo") + sn2 = SampleNode("bar") - assert Sample([sn], 'yay') == Sample([sn], 'yay') - assert Sample([sn], 'yay') != Sample([sn2], 'yay') - assert Sample([sn], 'yay') != Sample([sn], 'yooo') + assert Sample([sn], "yay") == Sample([sn], "yay") + assert Sample([sn], "yay") != Sample([sn2], "yay") + assert Sample([sn], "yay") != Sample([sn], "yooo") - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdea') + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdea") dt1 = dt(5) dt2 = dt(8) - u = UserID('u') - u2 = UserID('u2') + u = UserID("u") + u2 = UserID("u2") assert SavedSample(id1, u, [sn], dt1) == SavedSample(id1, u, [sn], dt(5)) assert SavedSample(id1, u, [sn], dt1) != SavedSample(id2, u, [sn], dt1) @@ -567,14 +798,22 @@ def test_sample_eq(): assert SavedSample(id1, u, [sn], dt1) != SavedSample(id1, u, [sn2], dt1) assert SavedSample(id1, u, [sn], dt1) != SavedSample(id1, u, [sn], dt2) - assert SavedSample(id1, u, [sn], dt1, 'yay') == SavedSample(id1, u, [sn], dt1, 'yay') - assert SavedSample(id1, u, [sn], dt1, 'yay') != SavedSample(id1, u, [sn], dt1, 'yooo') + assert SavedSample(id1, u, [sn], dt1, "yay") == SavedSample( + id1, u, [sn], dt1, "yay" + ) + assert SavedSample(id1, u, [sn], dt1, "yay") != SavedSample( + id1, u, [sn], dt1, "yooo" + ) - assert SavedSample(id1, u, [sn], dt2, 'yay', 6) == SavedSample(id1, u, [sn], dt2, 'yay', 6) - assert SavedSample(id1, u, [sn], dt1, 'yay', 6) != SavedSample(id1, u, [sn], dt1, 'yay', 7) + assert SavedSample(id1, u, [sn], dt2, "yay", 6) == SavedSample( + id1, u, [sn], dt2, "yay", 6 + ) + assert SavedSample(id1, u, [sn], dt1, "yay", 6) != SavedSample( + id1, u, [sn], dt1, "yay", 7 + ) - assert SavedSample(id1, u, [sn], dt1, 'yay') != Sample([sn], 'yay') - assert Sample([sn], 'yay') != SavedSample(id1, u, [sn], dt1, 'yay') + assert SavedSample(id1, u, [sn], dt1, "yay") != Sample([sn], "yay") + assert Sample([sn], "yay") != SavedSample(id1, u, [sn], dt1, "yay") def test_sample_hash(): @@ -582,63 +821,73 @@ def test_sample_hash(): # tests can't be written that directly test the hash value. See # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - sn = SampleNode('foo') - sn2 = SampleNode('bar') - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdea') + sn = SampleNode("foo") + sn2 = SampleNode("bar") + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdea") dt1 = dt(5) dt2 = dt(8) - u = UserID('u') - u2 = UserID('u2') - - assert hash(Sample([sn], 'yay')) == hash(Sample([sn], 'yay')) - assert hash(Sample([sn], 'foo')) == hash(Sample([sn], 'foo')) - assert hash(Sample([sn], 'yay')) != hash(Sample([sn2], 'yay')) - assert hash(Sample([sn], 'yay')) != hash(Sample([sn], 'yo')) - - assert hash(SavedSample(id1, u, [sn], dt1, 'yay')) == hash(SavedSample( - id1, u, [sn], dt(5), 'yay')) - assert hash(SavedSample(id2, u, [sn], dt1, 'foo')) == hash(SavedSample( - id2, u, [sn], dt1, 'foo')) - assert hash(SavedSample(id1, u, [sn], dt1, 'foo')) != hash(SavedSample( - id2, u, [sn], dt1, 'foo')) - assert hash(SavedSample(id1, u, [sn], dt1, 'foo')) != hash(SavedSample( - id1, u2, [sn], dt1, 'foo')) - assert hash(SavedSample(id2, u, [sn], dt1, 'foo')) != hash(SavedSample( - id2, u, [sn2], dt1, 'foo')) - assert hash(SavedSample(id2, u, [sn], dt1, 'foo')) != hash(SavedSample( - id2, u, [sn], dt2, 'foo')) - assert hash(SavedSample(id2, u, [sn], dt1, 'foo')) != hash(SavedSample( - id2, u, [sn], dt1, 'bar')) - assert hash(SavedSample(id1, u, [sn], dt1, 'foo', 6)) == hash(SavedSample( - id1, u, [sn], dt1, 'foo', 6)) - assert hash(SavedSample(id1, u, [sn], dt1, 'foo', 6)) != hash(SavedSample( - id1, u, [sn], dt1, 'foo', 7)) + u = UserID("u") + u2 = UserID("u2") + + assert hash(Sample([sn], "yay")) == hash(Sample([sn], "yay")) + assert hash(Sample([sn], "foo")) == hash(Sample([sn], "foo")) + assert hash(Sample([sn], "yay")) != hash(Sample([sn2], "yay")) + assert hash(Sample([sn], "yay")) != hash(Sample([sn], "yo")) + + assert hash(SavedSample(id1, u, [sn], dt1, "yay")) == hash( + SavedSample(id1, u, [sn], dt(5), "yay") + ) + assert hash(SavedSample(id2, u, [sn], dt1, "foo")) == hash( + SavedSample(id2, u, [sn], dt1, "foo") + ) + assert hash(SavedSample(id1, u, [sn], dt1, "foo")) != hash( + SavedSample(id2, u, [sn], dt1, "foo") + ) + assert hash(SavedSample(id1, u, [sn], dt1, "foo")) != hash( + SavedSample(id1, u2, [sn], dt1, "foo") + ) + assert hash(SavedSample(id2, u, [sn], dt1, "foo")) != hash( + SavedSample(id2, u, [sn2], dt1, "foo") + ) + assert hash(SavedSample(id2, u, [sn], dt1, "foo")) != hash( + SavedSample(id2, u, [sn], dt2, "foo") + ) + assert hash(SavedSample(id2, u, [sn], dt1, "foo")) != hash( + SavedSample(id2, u, [sn], dt1, "bar") + ) + assert hash(SavedSample(id1, u, [sn], dt1, "foo", 6)) == hash( + SavedSample(id1, u, [sn], dt1, "foo", 6) + ) + assert hash(SavedSample(id1, u, [sn], dt1, "foo", 6)) != hash( + SavedSample(id1, u, [sn], dt1, "foo", 7) + ) def test_sample_address_init(): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdef') + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdef") sa = SampleAddress(id1, 1) assert sa.sampleid == id2 assert sa.version == 1 - assert str(sa) == '12345678-90ab-cdef-1234-567890abcdef:1' + assert str(sa) == "12345678-90ab-cdef-1234-567890abcdef:1" sa = SampleAddress(id1, 394) assert sa.sampleid == id2 assert sa.version == 394 - assert str(sa) == '12345678-90ab-cdef-1234-567890abcdef:394' + assert str(sa) == "12345678-90ab-cdef-1234-567890abcdef:394" def test_sample_address_init_fail(): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") - _sample_address_init_fail(None, 6, ValueError( - 'sampleid cannot be a value that evaluates to false')) - _sample_address_init_fail(id1, None, IllegalParameterError('version must be > 0')) - _sample_address_init_fail(id1, 0, IllegalParameterError('version must be > 0')) - _sample_address_init_fail(id1, -5, IllegalParameterError('version must be > 0')) + _sample_address_init_fail( + None, 6, ValueError("sampleid cannot be a value that evaluates to false") + ) + _sample_address_init_fail(id1, None, IllegalParameterError("version must be > 0")) + _sample_address_init_fail(id1, 0, IllegalParameterError("version must be > 0")) + _sample_address_init_fail(id1, -5, IllegalParameterError("version must be > 0")) def _sample_address_init_fail(sid, ver, expected): @@ -648,10 +897,10 @@ def _sample_address_init_fail(sid, ver, expected): def test_sample_address_equals(): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdef') - idd1 = uuid.UUID('1234567890abcdef1234567890abcded') - idd2 = uuid.UUID('1234567890abcdef1234567890abcded') + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdef") + idd1 = uuid.UUID("1234567890abcdef1234567890abcded") + idd2 = uuid.UUID("1234567890abcdef1234567890abcded") assert SampleAddress(id1, 7) == SampleAddress(id2, 7) assert SampleAddress(idd1, 89) == SampleAddress(idd2, 89) @@ -666,10 +915,10 @@ def test_sample_address_hash(): # hashes will change from instance to instance of the python interpreter, and therefore # tests can't be written that directly test the hash value. See # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdef') - idd1 = uuid.UUID('1234567890abcdef1234567890abcded') - idd2 = uuid.UUID('1234567890abcdef1234567890abcded') + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdef") + idd1 = uuid.UUID("1234567890abcdef1234567890abcded") + idd2 = uuid.UUID("1234567890abcdef1234567890abcded") assert hash(SampleAddress(id1, 7)) == hash(SampleAddress(id2, 7)) assert hash(SampleAddress(idd1, 89)) == hash(SampleAddress(idd2, 89)) @@ -679,33 +928,35 @@ def test_sample_address_hash(): def test_sample_node_address_init(): - id_a1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id_a2 = uuid.UUID('1234567890abcdef1234567890abcdef') - id_b1 = uuid.UUID('1234567890abcdef1234567890abcdee') - id_b2 = uuid.UUID('1234567890abcdef1234567890abcdee') + id_a1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id_a2 = uuid.UUID("1234567890abcdef1234567890abcdef") + id_b1 = uuid.UUID("1234567890abcdef1234567890abcdee") + id_b2 = uuid.UUID("1234567890abcdef1234567890abcdee") - sa = SampleNodeAddress(SampleAddress(id_a1, 5), 'somenode') + sa = SampleNodeAddress(SampleAddress(id_a1, 5), "somenode") assert sa.sampleid == id_a2 assert sa.version == 5 - assert sa.node == 'somenode' - assert str(sa) == '12345678-90ab-cdef-1234-567890abcdef:5:somenode' + assert sa.node == "somenode" + assert str(sa) == "12345678-90ab-cdef-1234-567890abcdef:5:somenode" - sa = SampleNodeAddress(SampleAddress(id_b1, 1), 'e' * 256) + sa = SampleNodeAddress(SampleAddress(id_b1, 1), "e" * 256) assert sa.sampleid == id_b2 assert sa.version == 1 - assert sa.node == 'e' * 256 - assert str(sa) == ('12345678-90ab-cdef-1234-567890abcdee:1:' + 'e' * 256) + assert sa.node == "e" * 256 + assert str(sa) == ("12345678-90ab-cdef-1234-567890abcdee:1:" + "e" * 256) def test_sample_node_address_init_fail(): - sa = SampleAddress(uuid.UUID('1234567890abcdef1234567890abcdef'), 5) + sa = SampleAddress(uuid.UUID("1234567890abcdef1234567890abcdef"), 5) - _sample_node_address_init_fail(None, 'f', ValueError( - 'sample cannot be a value that evaluates to false')) - _sample_node_address_init_fail(sa, None, MissingParameterError('node')) - _sample_node_address_init_fail(sa, ' \t \n ', MissingParameterError('node')) - _sample_node_address_init_fail(sa, '3' * 257, IllegalParameterError( - 'node exceeds maximum length of 256')) + _sample_node_address_init_fail( + None, "f", ValueError("sample cannot be a value that evaluates to false") + ) + _sample_node_address_init_fail(sa, None, MissingParameterError("node")) + _sample_node_address_init_fail(sa, " \t \n ", MissingParameterError("node")) + _sample_node_address_init_fail( + sa, "3" * 257, IllegalParameterError("node exceeds maximum length of 256") + ) def _sample_node_address_init_fail(sample, node, expected): @@ -715,10 +966,10 @@ def _sample_node_address_init_fail(sample, node, expected): def test_sample_node_address_equals(): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdef') - idd1 = uuid.UUID('1234567890abcdef1234567890abcded') - idd2 = uuid.UUID('1234567890abcdef1234567890abcded') + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdef") + idd1 = uuid.UUID("1234567890abcdef1234567890abcded") + idd2 = uuid.UUID("1234567890abcdef1234567890abcded") sa_a1 = SampleAddress(id1, 6) sa_a2 = SampleAddress(id2, 6) @@ -726,24 +977,26 @@ def test_sample_node_address_equals(): sa_b1 = SampleAddress(idd1, 6) sa_b2 = SampleAddress(idd2, 6) - assert SampleNodeAddress(sa_a1, 'n') == SampleNodeAddress(sa_a2, 'n') - assert SampleNodeAddress(sa_b1, 'this is a node') == SampleNodeAddress(sa_b2, 'this is a node') + assert SampleNodeAddress(sa_a1, "n") == SampleNodeAddress(sa_a2, "n") + assert SampleNodeAddress(sa_b1, "this is a node") == SampleNodeAddress( + sa_b2, "this is a node" + ) - assert SampleNodeAddress(sa_a1, 'z') != (sa_a1, 'z') + assert SampleNodeAddress(sa_a1, "z") != (sa_a1, "z") - assert SampleNodeAddress(sa_a1, 'n') != SampleNodeAddress(sa_b1, 'n') - assert SampleNodeAddress(sa_a1, 'n') != SampleNodeAddress(sa_ad, 'n') - assert SampleNodeAddress(sa_a1, 'n') != SampleNodeAddress(sa_a2, 'z') + assert SampleNodeAddress(sa_a1, "n") != SampleNodeAddress(sa_b1, "n") + assert SampleNodeAddress(sa_a1, "n") != SampleNodeAddress(sa_ad, "n") + assert SampleNodeAddress(sa_a1, "n") != SampleNodeAddress(sa_a2, "z") def test_sample_node_address_hash(): # hashes will change from instance to instance of the python interpreter, and therefore # tests can't be written that directly test the hash value. See # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdef') - idd1 = uuid.UUID('1234567890abcdef1234567890abcded') - idd2 = uuid.UUID('1234567890abcdef1234567890abcded') + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdef") + idd1 = uuid.UUID("1234567890abcdef1234567890abcded") + idd2 = uuid.UUID("1234567890abcdef1234567890abcded") sa_a1 = SampleAddress(id1, 6) sa_a2 = SampleAddress(id2, 6) @@ -751,10 +1004,11 @@ def test_sample_node_address_hash(): sa_b1 = SampleAddress(idd1, 6) sa_b2 = SampleAddress(idd2, 6) - assert hash(SampleNodeAddress(sa_a1, 'n')) == hash(SampleNodeAddress(sa_a2, 'n')) - assert hash(SampleNodeAddress(sa_b1, 'this is a node')) == hash( - SampleNodeAddress(sa_b2, 'this is a node')) + assert hash(SampleNodeAddress(sa_a1, "n")) == hash(SampleNodeAddress(sa_a2, "n")) + assert hash(SampleNodeAddress(sa_b1, "this is a node")) == hash( + SampleNodeAddress(sa_b2, "this is a node") + ) - assert hash(SampleNodeAddress(sa_a1, 'n')) != hash(SampleNodeAddress(sa_b1, 'n')) - assert hash(SampleNodeAddress(sa_a1, 'n')) != hash(SampleNodeAddress(sa_ad, 'n')) - assert hash(SampleNodeAddress(sa_a1, 'n')) != hash(SampleNodeAddress(sa_a2, 'z')) + assert hash(SampleNodeAddress(sa_a1, "n")) != hash(SampleNodeAddress(sa_b1, "n")) + assert hash(SampleNodeAddress(sa_a1, "n")) != hash(SampleNodeAddress(sa_ad, "n")) + assert hash(SampleNodeAddress(sa_a1, "n")) != hash(SampleNodeAddress(sa_a2, "z")) diff --git a/test/core/samples_test.py b/test/core/samples_test.py index 69030c75..3e7c73f0 100644 --- a/test/core/samples_test.py +++ b/test/core/samples_test.py @@ -3,7 +3,7 @@ from pytest import raises from uuid import UUID -from unittest.mock import (create_autospec, call) +from unittest.mock import create_autospec, call from SampleService.core.storage.arango_sample_storage import ArangoSampleStorage from SampleService.core.acls import SampleACL, SampleACLOwnerless, SampleACLDelta @@ -13,7 +13,7 @@ UnauthorizedError, NoSuchUserError, MetadataValidationError, - NoSuchLinkError + NoSuchLinkError, ) from SampleService.core.notification import KafkaNotifier from SampleService.core.sample import Sample, SampleNode, SavedSample, SampleAddress @@ -45,20 +45,62 @@ def test_init_fail_bad_args(): lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - ug = lambda: UUID('1234567890abcdef1234567890abcdef') # noqa E731 + ug = lambda: UUID("1234567890abcdef1234567890abcdef") # noqa E731 - _init_fail(None, lu, meta, ws, nw, ug, ValueError( - 'storage cannot be a value that evaluates to false')) - _init_fail(storage, None, meta, ws, nw, ug, ValueError( - 'user_lookup cannot be a value that evaluates to false')) - _init_fail(storage, lu, None, ws, nw, ug, ValueError( - 'metadata_validator cannot be a value that evaluates to false')) - _init_fail(storage, lu, meta, None, nw, ug, ValueError( - 'workspace cannot be a value that evaluates to false')) - _init_fail(storage, lu, meta, ws, None, ug, ValueError( - 'now cannot be a value that evaluates to false')) - _init_fail(storage, lu, meta, ws, nw, None, ValueError( - 'uuid_gen cannot be a value that evaluates to false')) + _init_fail( + None, + lu, + meta, + ws, + nw, + ug, + ValueError("storage cannot be a value that evaluates to false"), + ) + _init_fail( + storage, + None, + meta, + ws, + nw, + ug, + ValueError("user_lookup cannot be a value that evaluates to false"), + ) + _init_fail( + storage, + lu, + None, + ws, + nw, + ug, + ValueError("metadata_validator cannot be a value that evaluates to false"), + ) + _init_fail( + storage, + lu, + meta, + None, + nw, + ug, + ValueError("workspace cannot be a value that evaluates to false"), + ) + _init_fail( + storage, + lu, + meta, + ws, + None, + ug, + ValueError("now cannot be a value that evaluates to false"), + ) + _init_fail( + storage, + lu, + meta, + ws, + nw, + None, + ValueError("uuid_gen cannot be a value that evaluates to false"), + ) def _init_fail(storage, lookup, meta, ws, now, uuid_gen, expected): @@ -70,7 +112,7 @@ def _init_fail(storage, lookup, meta, ws, now, uuid_gen, expected): def test_validate_sample_with_name(): _validate_sample(None) - _validate_sample('foo') + _validate_sample("foo") def _validate_sample(name): @@ -79,25 +121,45 @@ def _validate_sample(name): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) kafka = create_autospec(KafkaNotifier, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, kafka, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - s.validate_sample(Sample([ - SampleNode( - 'foo', - controlled_metadata={'key1': {'val': 'foo'}, 'key2': {'val': 'bar'}}, - user_metadata={'key_not_in_validator_map': {'val': 'yay'}, - 'key1': {'val': 'wrong'}} - ), - SampleNode( - 'foo2', - controlled_metadata={'key3': {'val': 'foo'}, 'key4': {'val': 'bar'}}, - ) - ], name)) + s = Samples( + storage, + lu, + meta, + ws, + kafka, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + s.validate_sample( + Sample( + [ + SampleNode( + "foo", + controlled_metadata={ + "key1": {"val": "foo"}, + "key2": {"val": "bar"}, + }, + user_metadata={ + "key_not_in_validator_map": {"val": "yay"}, + "key1": {"val": "wrong"}, + }, + ), + SampleNode( + "foo2", + controlled_metadata={ + "key3": {"val": "foo"}, + "key4": {"val": "bar"}, + }, + ), + ], + name, + ) + ) def test_save_sample(): _save_sample_with_name(None, True) - _save_sample_with_name('bar', False) + _save_sample_with_name("bar", False) def _save_sample_with_name(name, as_admin): @@ -106,61 +168,98 @@ def _save_sample_with_name(name, as_admin): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) kafka = create_autospec(KafkaNotifier, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, kafka, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - - assert s.save_sample( - Sample([ - SampleNode( - 'foo', - controlled_metadata={'key1': {'val': 'foo'}, 'key2': {'val': 'bar'}}, - user_metadata={'key_not_in_validator_map': {'val': 'yay'}, - 'key1': {'val': 'wrong'}} - ), - SampleNode( - 'foo2', - controlled_metadata={'key3': {'val': 'foo'}, 'key4': {'val': 'bar'}}, - ) - ], - name), - UserID('auser'), - as_admin=as_admin) == (UUID('1234567890abcdef1234567890abcdef'), 1) + s = Samples( + storage, + lu, + meta, + ws, + kafka, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + + assert ( + s.save_sample( + Sample( + [ + SampleNode( + "foo", + controlled_metadata={ + "key1": {"val": "foo"}, + "key2": {"val": "bar"}, + }, + user_metadata={ + "key_not_in_validator_map": {"val": "yay"}, + "key1": {"val": "wrong"}, + }, + ), + SampleNode( + "foo2", + controlled_metadata={ + "key3": {"val": "foo"}, + "key4": {"val": "bar"}, + }, + ), + ], + name, + ), + UserID("auser"), + as_admin=as_admin, + ) + == (UUID("1234567890abcdef1234567890abcdef"), 1) + ) assert storage.save_sample.call_args_list == [ - ((SavedSample(UUID('1234567890abcdef1234567890abcdef'), - UserID('auser'), - [SampleNode( - 'foo', - controlled_metadata={'key1': {'val': 'foo'}, 'key2': {'val': 'bar'}}, - user_metadata={'key_not_in_validator_map': {'val': 'yay'}, - 'key1': {'val': 'wrong'}} - ), - SampleNode( - 'foo2', - controlled_metadata={'key3': {'val': 'foo'}, 'key4': {'val': 'bar'}}, - ) - ], - datetime.datetime.fromtimestamp(6, tz=datetime.timezone.utc), - name - ), # make a tuple - ), {})] + ( + ( + SavedSample( + UUID("1234567890abcdef1234567890abcdef"), + UserID("auser"), + [ + SampleNode( + "foo", + controlled_metadata={ + "key1": {"val": "foo"}, + "key2": {"val": "bar"}, + }, + user_metadata={ + "key_not_in_validator_map": {"val": "yay"}, + "key1": {"val": "wrong"}, + }, + ), + SampleNode( + "foo2", + controlled_metadata={ + "key3": {"val": "foo"}, + "key4": {"val": "bar"}, + }, + ), + ], + datetime.datetime.fromtimestamp(6, tz=datetime.timezone.utc), + name, + ), # make a tuple + ), + {}, + ) + ] call_arg_list = [ - call({'key1': {'val': 'foo'}, 'key2': {'val': 'bar'}}, False), - call({'key3': {'val': 'foo'}, 'key4': {'val': 'bar'}}, False) + call({"key1": {"val": "foo"}, "key2": {"val": "bar"}}, False), + call({"key3": {"val": "foo"}, "key4": {"val": "bar"}}, False), ] meta.validate_metadata.assert_has_calls(call_arg_list) kafka.notify_new_sample_version.assert_called_once_with( - UUID('1234567890abcdef1234567890abcdef'), 1) + UUID("1234567890abcdef1234567890abcdef"), 1 + ) def test_save_sample_version(): - _save_sample_version_per_user(UserID('someuser'), None, None) - _save_sample_version_per_user(UserID('otheruser'), 'sample name', 2) + _save_sample_version_per_user(UserID("someuser"), None, None) + _save_sample_version_per_user(UserID("otheruser"), "sample name", 2) # this one should really fail based on the mock output... but it's a mock so it won't - _save_sample_version_per_user(UserID('anotheruser'), 'ur dad yeah', 1) + _save_sample_version_per_user(UserID("anotheruser"), "ur dad yeah", 1) def _save_sample_version_per_user(user: UserID, name, prior_version): @@ -169,71 +268,104 @@ def _save_sample_version_per_user(user: UserID, name, prior_version): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) kafka = create_autospec(KafkaNotifier, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, kafka, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + kafka, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.')]) + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr.")], + ) storage.save_sample_version.return_value = 3 - assert s.save_sample( - Sample([SampleNode('foo')], name), - user, - UUID('1234567890abcdef1234567890abcdea'), - prior_version) == (UUID('1234567890abcdef1234567890abcdea'), 3) + assert ( + s.save_sample( + Sample([SampleNode("foo")], name), + user, + UUID("1234567890abcdef1234567890abcdea"), + prior_version, + ) + == (UUID("1234567890abcdef1234567890abcdea"), 3) + ) assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcdea'),), {})] + ((UUID("1234567890abcdef1234567890abcdea"),), {}) + ] meta.validate_metadata.assert_has_calls([call({}, False)]) assert storage.save_sample_version.call_args_list == [ - ((SavedSample(UUID('1234567890abcdef1234567890abcdea'), - user, - [SampleNode('foo')], - datetime.datetime.fromtimestamp(6, tz=datetime.timezone.utc), - name - ), - prior_version), {})] + ( + ( + SavedSample( + UUID("1234567890abcdef1234567890abcdea"), + user, + [SampleNode("foo")], + datetime.datetime.fromtimestamp(6, tz=datetime.timezone.utc), + name, + ), + prior_version, + ), + {}, + ) + ] kafka.notify_new_sample_version.assert_called_once_with( - UUID('1234567890abcdef1234567890abcdea'), 3) + UUID("1234567890abcdef1234567890abcdea"), 3 + ) def test_save_sample_version_as_admin(): - ''' + """ Also test that not providing a notifier causes no issues. - ''' + """ storage = create_autospec(ArangoSampleStorage, spec_set=True, instance=True) lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) storage.save_sample_version.return_value = 3 - assert s.save_sample( - Sample([SampleNode('foo')], 'some sample'), - UserID('usera'), - UUID('1234567890abcdef1234567890abcdea'), - as_admin=True) == (UUID('1234567890abcdef1234567890abcdea'), 3) + assert ( + s.save_sample( + Sample([SampleNode("foo")], "some sample"), + UserID("usera"), + UUID("1234567890abcdef1234567890abcdea"), + as_admin=True, + ) + == (UUID("1234567890abcdef1234567890abcdea"), 3) + ) meta.validate_metadata.assert_has_calls([call({}, False)]) storage.save_sample_version.assert_called_once_with( - SavedSample(UUID('1234567890abcdef1234567890abcdea'), - UserID('usera'), - [SampleNode('foo')], - datetime.datetime.fromtimestamp(6, tz=datetime.timezone.utc), - 'some sample' - ), - None) + SavedSample( + UUID("1234567890abcdef1234567890abcdea"), + UserID("usera"), + [SampleNode("foo")], + datetime.datetime.fromtimestamp(6, tz=datetime.timezone.utc), + "some sample", + ), + None, + ) def test_save_sample_fail_bad_args(): @@ -242,18 +374,37 @@ def test_save_sample_fail_bad_args(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) - s = Sample([SampleNode('foo')]) - id_ = UUID('1234567890abcdef1234567890abcdef') - u = UserID('u') + s = Sample([SampleNode("foo")]) + id_ = UUID("1234567890abcdef1234567890abcdef") + u = UserID("u") _save_sample_fail( - samples, None, u, id_, 1, ValueError('sample cannot be a value that evaluates to false')) + samples, + None, + u, + id_, + 1, + ValueError("sample cannot be a value that evaluates to false"), + ) _save_sample_fail( - samples, s, None, id_, 1, ValueError('user cannot be a value that evaluates to false')) + samples, + s, + None, + id_, + 1, + ValueError("user cannot be a value that evaluates to false"), + ) _save_sample_fail( - samples, s, u, id_, 0, IllegalParameterError('Prior version must be > 0')) + samples, s, u, id_, 0, IllegalParameterError("Prior version must be > 0") + ) def test_save_sample_fail_no_metadata_validator(): @@ -262,23 +413,38 @@ def test_save_sample_fail_no_metadata_validator(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - meta.validate_metadata.side_effect = MetadataValidationError('No validator for key3') - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + meta.validate_metadata.side_effect = MetadataValidationError( + "No validator for key3" + ) + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) with raises(Exception) as got: s.save_sample( - Sample([ - SampleNode( - 'foo', - controlled_metadata={'key1': {'val': 'foo'}, 'key3': {'val': 'bar'}}, - user_metadata={'key_not_in_validator_map': {'val': 'yay'}} + Sample( + [ + SampleNode( + "foo", + controlled_metadata={ + "key1": {"val": "foo"}, + "key3": {"val": "bar"}, + }, + user_metadata={"key_not_in_validator_map": {"val": "yay"}}, ) ], - 'foo'), - UserID('auser')) - assert_exception_correct(got.value, MetadataValidationError( - 'Node at index 0: No validator for key3')) + "foo", + ), + UserID("auser"), + ) + assert_exception_correct( + got.value, MetadataValidationError("Node at index 0: No validator for key3") + ) def test_save_sample_fail_metadata_validator_exception(): @@ -287,32 +453,51 @@ def test_save_sample_fail_metadata_validator_exception(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - meta.validate_metadata.side_effect = [None, MetadataValidationError('key2: u suk lol')] - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + meta.validate_metadata.side_effect = [ + None, + MetadataValidationError("key2: u suk lol"), + ] + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) with raises(Exception) as got: s.save_sample( - Sample([ - SampleNode( - 'foo', - controlled_metadata={'key1': {'val': 'foo'}, 'key2': {'val': 'bar'}}, - user_metadata={'key_not_in_validator_map': {'val': 'yay'}} + Sample( + [ + SampleNode( + "foo", + controlled_metadata={ + "key1": {"val": "foo"}, + "key2": {"val": "bar"}, + }, + user_metadata={"key_not_in_validator_map": {"val": "yay"}}, + ), + SampleNode( + "foo2", + controlled_metadata={ + "key1": {"val": "foo"}, + "key2": {"val": "bar"}, + }, ), - SampleNode( - 'foo2', - controlled_metadata={'key1': {'val': 'foo'}, 'key2': {'val': 'bar'}}, - ) ], - 'foo'), - UserID('auser')) - assert_exception_correct(got.value, MetadataValidationError( - 'Node at index 1: key2: u suk lol')) + "foo", + ), + UserID("auser"), + ) + assert_exception_correct( + got.value, MetadataValidationError("Node at index 1: key2: u suk lol") + ) def test_save_sample_fail_unauthorized(): - _save_sample_fail_unauthorized(UserID('x')) - _save_sample_fail_unauthorized(UserID('nouserhere')) + _save_sample_fail_unauthorized(UserID("x")) + _save_sample_fail_unauthorized(UserID("nouserhere")) def _save_sample_fail_unauthorized(user: UserID): @@ -321,26 +506,37 @@ def _save_sample_fail_unauthorized(user: UserID): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')], - True) # public read should not allow saving a new version of a sample + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + True, + ) # public read should not allow saving a new version of a sample with raises(Exception) as got: samples.save_sample( - Sample([SampleNode('foo')]), - user, - UUID('1234567890abcdef1234567890abcdea')) - assert_exception_correct(got.value, UnauthorizedError( - f'User {user} cannot write to sample 12345678-90ab-cdef-1234-567890abcdea')) + Sample([SampleNode("foo")]), user, UUID("1234567890abcdef1234567890abcdea") + ) + assert_exception_correct( + got.value, + UnauthorizedError( + f"User {user} cannot write to sample 12345678-90ab-cdef-1234-567890abcdea" + ), + ) assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcdea'),), {})] + ((UUID("1234567890abcdef1234567890abcdea"),), {}) + ] def _save_sample_fail(samples, sample, user, id_, prior_version, expected): @@ -351,23 +547,23 @@ def _save_sample_fail(samples, sample, user, id_, prior_version, expected): def test_get_sample(): # sample versions other than 4 don't really make sense but the mock doesn't care - _get_sample(UserID('someuser'), None, False) - _get_sample(UserID('otheruser'), 4, False) - _get_sample(UserID('anotheruser'), 2, False) - _get_sample(UserID('x'), None, False) - _get_sample(UserID('notinacl'), None, True) - _get_sample(UserID('notinacl'), None, False, True) # public read + _get_sample(UserID("someuser"), None, False) + _get_sample(UserID("otheruser"), 4, False) + _get_sample(UserID("anotheruser"), 2, False) + _get_sample(UserID("x"), None, False) + _get_sample(UserID("notinacl"), None, True) + _get_sample(UserID("notinacl"), None, False, True) # public read _get_sample(None, None, False, True) # public read & anon def test_get_samples(): # sample versions other than 4 don't really make sense but the mock doesn't care - _get_samples(UserID('someuser'), None, False) - _get_samples(UserID('otheruser'), 4, False) - _get_samples(UserID('anotheruser'), 2, False) - _get_samples(UserID('x'), None, False) - _get_samples(UserID('notinacl'), None, True) - _get_samples(UserID('notinacl'), None, False, True) # public read + _get_samples(UserID("someuser"), None, False) + _get_samples(UserID("otheruser"), 4, False) + _get_samples(UserID("anotheruser"), 2, False) + _get_samples(UserID("x"), None, False) + _get_samples(UserID("notinacl"), None, True) + _get_samples(UserID("notinacl"), None, False, True) # public read _get_samples(None, None, False, True) # public read & anon @@ -377,40 +573,47 @@ def _get_sample(user, version, as_admin, public_read=False): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + storage, lu, meta, ws, uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef") + ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')], - public_read=public_read) + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + public_read=public_read, + ) storage.get_sample.return_value = SavedSample( - UUID('1234567890abcdef1234567890abcdea'), - UserID('anotheruser'), - [SampleNode('foo')], + UUID("1234567890abcdef1234567890abcdea"), + UserID("anotheruser"), + [SampleNode("foo")], datetime.datetime.fromtimestamp(42, tz=datetime.timezone.utc), - 'bar', - 4) + "bar", + 4, + ) assert samples.get_sample( - UUID('1234567890abcdef1234567890abcdea'), user, version, as_admin) == SavedSample( - UUID('1234567890abcdef1234567890abcdea'), - UserID('anotheruser'), - [SampleNode('foo')], - datetime.datetime.fromtimestamp(42, tz=datetime.timezone.utc), - 'bar', - 4) + UUID("1234567890abcdef1234567890abcdea"), user, version, as_admin + ) == SavedSample( + UUID("1234567890abcdef1234567890abcdea"), + UserID("anotheruser"), + [SampleNode("foo")], + datetime.datetime.fromtimestamp(42, tz=datetime.timezone.utc), + "bar", + 4, + ) if not as_admin: assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcdea'),), {})] + ((UUID("1234567890abcdef1234567890abcdea"),), {}) + ] else: assert storage.get_sample_acls.call_args_list == [] assert storage.get_sample.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcdea'), version), {})] + ((UUID("1234567890abcdef1234567890abcdea"), version), {}) + ] def _get_samples(user, version, as_admin, public_read=False): @@ -419,54 +622,66 @@ def _get_samples(user, version, as_admin, public_read=False): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, uuid_gen=lambda: UUID('1234567890abcdef1234567890fbcdef')) + storage, lu, meta, ws, uuid_gen=lambda: UUID("1234567890abcdef1234567890fbcdef") + ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')], - public_read=public_read) + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + public_read=public_read, + ) storage.get_samples.return_value = [ SavedSample( - UUID('1234567890abcdef1234567890fbcdef'), - UserID('anotheruser'), - [SampleNode('foo')], + UUID("1234567890abcdef1234567890fbcdef"), + UserID("anotheruser"), + [SampleNode("foo")], datetime.datetime.fromtimestamp(42, tz=datetime.timezone.utc), - 'bar', - 4 + "bar", + 4, ), SavedSample( - UUID('1234567890abcdef1234567890fbcdeb'), - UserID('anotheruser'), - [SampleNode('fid')], + UUID("1234567890abcdef1234567890fbcdeb"), + UserID("anotheruser"), + [SampleNode("fid")], datetime.datetime.fromtimestamp(42, tz=datetime.timezone.utc), - 'baz', - 4 - ) + "baz", + 4, + ), ] - assert samples.get_samples([ - {'id': UUID('1234567890abcdef1234567890fbcdef'), 'version': version}, - {'id': UUID('1234567890abcdef1234567890fbcdeb'), 'version': version} - ], user, as_admin) == [ + assert samples.get_samples( + [ + {"id": UUID("1234567890abcdef1234567890fbcdef"), "version": version}, + {"id": UUID("1234567890abcdef1234567890fbcdeb"), "version": version}, + ], + user, + as_admin, + ) == [ SavedSample( - UUID('1234567890abcdef1234567890fbcdef'), - UserID('anotheruser'), [SampleNode('foo')], + UUID("1234567890abcdef1234567890fbcdef"), + UserID("anotheruser"), + [SampleNode("foo")], datetime.datetime.fromtimestamp(42, tz=datetime.timezone.utc), - 'bar', 4), + "bar", + 4, + ), SavedSample( - UUID('1234567890abcdef1234567890fbcdeb'), - UserID('anotheruser'), [SampleNode('fid')], + UUID("1234567890abcdef1234567890fbcdeb"), + UserID("anotheruser"), + [SampleNode("fid")], datetime.datetime.fromtimestamp(42, tz=datetime.timezone.utc), - 'baz', 4) + "baz", + 4, + ), ] if not as_admin: assert storage.get_sample_acls.call_args_list == [ - call(UUID('12345678-90ab-cdef-1234-567890fbcdef')), - call(UUID('12345678-90ab-cdef-1234-567890fbcdeb')) + call(UUID("12345678-90ab-cdef-1234-567890fbcdef")), + call(UUID("12345678-90ab-cdef-1234-567890fbcdeb")), ] else: assert storage.get_sample_acls.call_args_list == [] @@ -476,10 +691,18 @@ def _get_samples(user, version, as_admin, public_read=False): # print('-'*80) assert storage.get_samples.call_args_list == [ - call([ - {'id': UUID('12345678-90ab-cdef-1234-567890fbcdef'), 'version': version}, - {'id': UUID('12345678-90ab-cdef-1234-567890fbcdeb'), 'version': version} - ]) + call( + [ + { + "id": UUID("12345678-90ab-cdef-1234-567890fbcdef"), + "version": version, + }, + { + "id": UUID("12345678-90ab-cdef-1234-567890fbcdeb"), + "version": version, + }, + ] + ) ] # assert storage.get_samples.call_args_list == [ @@ -493,19 +716,40 @@ def test_get_sample_fail_bad_args(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcdef') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcdef") - _get_sample_fail(samples, None, UserID('foo'), 1, ValueError( - 'id_ cannot be a value that evaluates to false')) - _get_sample_fail(samples, id_, UserID('a'), 0, IllegalParameterError('Version must be > 0')) + _get_sample_fail( + samples, + None, + UserID("foo"), + 1, + ValueError("id_ cannot be a value that evaluates to false"), + ) + _get_sample_fail( + samples, id_, UserID("a"), 0, IllegalParameterError("Version must be > 0") + ) def test_get_sample_fail_unauthorized(): - _get_sample_fail_unauthorized(UserID('y'), UnauthorizedError( - 'User y cannot read sample 12345678-90ab-cdef-1234-567890abcdef')) - _get_sample_fail_unauthorized(None, UnauthorizedError( - 'Anonymous users cannot read sample 12345678-90ab-cdef-1234-567890abcdef')) + _get_sample_fail_unauthorized( + UserID("y"), + UnauthorizedError( + "User y cannot read sample 12345678-90ab-cdef-1234-567890abcdef" + ), + ) + _get_sample_fail_unauthorized( + None, + UnauthorizedError( + "Anonymous users cannot read sample 12345678-90ab-cdef-1234-567890abcdef" + ), + ) def _get_sample_fail_unauthorized(user, expected): @@ -514,20 +758,29 @@ def _get_sample_fail_unauthorized(user, expected): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]) + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ) _get_sample_fail( - samples, UUID('1234567890abcdef1234567890abcdef'), user, 3, expected) + samples, UUID("1234567890abcdef1234567890abcdef"), user, 3, expected + ) assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcdef'),), {})] + ((UUID("1234567890abcdef1234567890abcdef"),), {}) + ] def _get_sample_fail(samples, id_, user, version, expected): @@ -537,12 +790,12 @@ def _get_sample_fail(samples, id_, user, version, expected): def test_get_sample_acls(): - _get_sample_acls(UserID('someuser'), False) - _get_sample_acls(UserID('otheruser'), False) - _get_sample_acls(UserID('anotheruser'), False) - _get_sample_acls(UserID('x'), False) - _get_sample_acls(UserID('no_rights_here'), True) - _get_sample_acls(UserID('no_rights_here'), False, True) + _get_sample_acls(UserID("someuser"), False) + _get_sample_acls(UserID("otheruser"), False) + _get_sample_acls(UserID("anotheruser"), False) + _get_sample_acls(UserID("x"), False) + _get_sample_acls(UserID("no_rights_here"), True) + _get_sample_acls(UserID("no_rights_here"), False, True) _get_sample_acls(None, False, True) @@ -552,27 +805,36 @@ def _get_sample_acls(user, as_admin, public_read=False): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(78), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')], - public_read=public_read) + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + public_read=public_read, + ) assert samples.get_sample_acls(id_, user, as_admin) == SampleACL( - u('someuser'), + u("someuser"), dt(78), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')], - public_read=public_read) + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + public_read=public_read, + ) assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcde0'),), {})] + ((UUID("1234567890abcdef1234567890abcde0"),), {}) + ] def test_get_sample_acls_fail_bad_args(): @@ -581,17 +843,35 @@ def test_get_sample_acls_fail_bad_args(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) - _get_sample_acls_fail(samples, None, UserID('foo'), ValueError( - 'id_ cannot be a value that evaluates to false')) + _get_sample_acls_fail( + samples, + None, + UserID("foo"), + ValueError("id_ cannot be a value that evaluates to false"), + ) def test_get_sample_acls_fail_unauthorized(): - _get_sample_acls_fail_unauthorized(UserID('y'), UnauthorizedError( - 'User y cannot read sample 12345678-90ab-cdef-1234-567890abcdea')) - _get_sample_acls_fail_unauthorized(None, UnauthorizedError( - 'Anonymous users cannot read sample 12345678-90ab-cdef-1234-567890abcdea')) + _get_sample_acls_fail_unauthorized( + UserID("y"), + UnauthorizedError( + "User y cannot read sample 12345678-90ab-cdef-1234-567890abcdea" + ), + ) + _get_sample_acls_fail_unauthorized( + None, + UnauthorizedError( + "Anonymous users cannot read sample 12345678-90ab-cdef-1234-567890abcdea" + ), + ) def _get_sample_acls_fail_unauthorized(user, expected): @@ -600,20 +880,29 @@ def _get_sample_acls_fail_unauthorized(user, expected): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]) + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ) _get_sample_acls_fail( - samples, UUID('1234567890abcdef1234567890abcdea'), user, expected) + samples, UUID("1234567890abcdef1234567890abcdea"), user, expected + ) assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcdea'),), {})] + ((UUID("1234567890abcdef1234567890abcdea"),), {}) + ] def _get_sample_acls_fail(samples, id_, user, expected): @@ -623,9 +912,9 @@ def _get_sample_acls_fail(samples, id_, user, expected): def test_replace_sample_acls(): - _replace_sample_acls(UserID('someuser'), True, False) - _replace_sample_acls(UserID('otheruser'), False, False) - _replace_sample_acls(UserID('super_admin_man'), False, True) + _replace_sample_acls(UserID("someuser"), True, False) + _replace_sample_acls(UserID("otheruser"), False, False) + _replace_sample_acls(UserID("super_admin_man"), False, True) def _replace_sample_acls(user: UserID, public_read, as_admin): @@ -634,36 +923,55 @@ def _replace_sample_acls(user: UserID, public_read, as_admin): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) kafka = create_autospec(KafkaNotifier, spec_set=True, instance=True) - samples = Samples(storage, lu, meta, ws, kafka, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + samples = Samples( + storage, + lu, + meta, + ws, + kafka, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") lu.invalid_users.return_value = [] storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]) + [u("otheruser"), u("y")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ) - samples.replace_sample_acls(id_, user, SampleACLOwnerless( - [u('x'), u('y')], [u('z'), u('a')], [u('b'), u('c')], public_read), - as_admin=as_admin) + samples.replace_sample_acls( + id_, + user, + SampleACLOwnerless( + [u("x"), u("y")], [u("z"), u("a")], [u("b"), u("c")], public_read + ), + as_admin=as_admin, + ) - lu.invalid_users.assert_called_once_with([u(x) for x in ['x', 'y', 'a', 'z', 'b', 'c']]) + lu.invalid_users.assert_called_once_with( + [u(x) for x in ["x", "y", "a", "z", "b", "c"]] + ) - storage.get_sample_acls.assert_called_once_with(UUID('1234567890abcdef1234567890abcde0')) + storage.get_sample_acls.assert_called_once_with( + UUID("1234567890abcdef1234567890abcde0") + ) storage.replace_sample_acls.assert_called_once_with( - UUID('1234567890abcdef1234567890abcde0'), + UUID("1234567890abcdef1234567890abcde0"), SampleACL( - u('someuser'), + u("someuser"), dt(6), - [u('x'), u('y')], - [u('z'), u('a')], - [u('b'), u('c')], - public_read)) + [u("x"), u("y")], + [u("z"), u("a")], + [u("b"), u("c")], + public_read, + ), + ) kafka.notify_sample_acl_change.assert_called_once_with(id_) @@ -677,37 +985,59 @@ def test_replace_sample_acls_with_owner_change(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") lu.invalid_users.return_value = [] storage.get_sample_acls.side_effect = [ SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]), + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ), SampleACL( - u('someuser2'), dt(1), [u('otheruser'), u('y')],) - ] + u("someuser2"), + dt(1), + [u("otheruser"), u("y")], + ), + ] storage.replace_sample_acls.side_effect = [OwnerChangedError, None] - samples.replace_sample_acls(id_, UserID('otheruser'), SampleACLOwnerless([u('a')])) + samples.replace_sample_acls(id_, UserID("otheruser"), SampleACLOwnerless([u("a")])) - assert lu.invalid_users.call_args_list == [(([u('a')],), {})] + assert lu.invalid_users.call_args_list == [(([u("a")],), {})] assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcde0'),), {}), - ((UUID('1234567890abcdef1234567890abcde0'),), {}) - ] + ((UUID("1234567890abcdef1234567890abcde0"),), {}), + ((UUID("1234567890abcdef1234567890abcde0"),), {}), + ] assert storage.replace_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcde0'), SampleACL(u('someuser'), dt(6), [u('a')])), {}), - ((UUID('1234567890abcdef1234567890abcde0'), SampleACL(u('someuser2'), dt(6), [u('a')])), {}) - ] + ( + ( + UUID("1234567890abcdef1234567890abcde0"), + SampleACL(u("someuser"), dt(6), [u("a")]), + ), + {}, + ), + ( + ( + UUID("1234567890abcdef1234567890abcde0"), + SampleACL(u("someuser2"), dt(6), [u("a")]), + ), + {}, + ), + ] def test_replace_sample_acls_with_owner_change_fail_lost_perms(): @@ -716,39 +1046,58 @@ def test_replace_sample_acls_with_owner_change_fail_lost_perms(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") lu.invalid_users.return_value = [] storage.get_sample_acls.side_effect = [ SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]), + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ), SampleACL( - u('someuser2'), dt(1), [u('otheruser2'), u('y')],) - ] + u("someuser2"), + dt(1), + [u("otheruser2"), u("y")], + ), + ] storage.replace_sample_acls.side_effect = [OwnerChangedError, None] _replace_sample_acls_fail( - samples, id_, UserID('otheruser'), SampleACLOwnerless(write=[u('b')]), - UnauthorizedError(f'User otheruser cannot administrate sample {id_}')) + samples, + id_, + UserID("otheruser"), + SampleACLOwnerless(write=[u("b")]), + UnauthorizedError(f"User otheruser cannot administrate sample {id_}"), + ) - assert lu.invalid_users.call_args_list == [(([u('b')],), {})] + assert lu.invalid_users.call_args_list == [(([u("b")],), {})] assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcde0'),), {}), - ((UUID('1234567890abcdef1234567890abcde0'),), {}) - ] + ((UUID("1234567890abcdef1234567890abcde0"),), {}), + ((UUID("1234567890abcdef1234567890abcde0"),), {}), + ] assert storage.replace_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcde0'), - SampleACL(u('someuser'), dt(6), write=[u('b')])), {}) - ] + ( + ( + UUID("1234567890abcdef1234567890abcde0"), + SampleACL(u("someuser"), dt(6), write=[u("b")]), + ), + {}, + ) + ] def test_replace_sample_acls_with_owner_change_fail_5_times(): @@ -757,31 +1106,47 @@ def test_replace_sample_acls_with_owner_change_fail_5_times(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") lu.invalid_users.return_value = [] storage.get_sample_acls.side_effect = [ - SampleACL(u(f'someuser{x}'), dt(1), [u('otheruser')]) for x in range(5) + SampleACL(u(f"someuser{x}"), dt(1), [u("otheruser")]) for x in range(5) ] storage.replace_sample_acls.side_effect = OwnerChangedError _replace_sample_acls_fail( - samples, id_, UserID('otheruser'), SampleACLOwnerless(read=[u('c')]), - ValueError(f'Failed setting ACLs after 5 attempts for sample {id_}')) + samples, + id_, + UserID("otheruser"), + SampleACLOwnerless(read=[u("c")]), + ValueError(f"Failed setting ACLs after 5 attempts for sample {id_}"), + ) - assert lu.invalid_users.call_args_list == [(([u('c')],), {})] + assert lu.invalid_users.call_args_list == [(([u("c")],), {})] assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcde0'),), {}) for _ in range(5) - ] + ((UUID("1234567890abcdef1234567890abcde0"),), {}) for _ in range(5) + ] assert storage.replace_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcde0'), - SampleACL(u(f'someuser{x}'), dt(6), read=[u('c')]),), {}) for x in range(5) - ] + ( + ( + UUID("1234567890abcdef1234567890abcde0"), + SampleACL(u(f"someuser{x}"), dt(6), read=[u("c")]), + ), + {}, + ) + for x in range(5) + ] def test_replace_sample_acls_fail_bad_input(): @@ -790,16 +1155,37 @@ def test_replace_sample_acls_fail_bad_input(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') - u = UserID('u') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") + u = UserID("u") - _replace_sample_acls_fail(samples, None, u, SampleACLOwnerless(), ValueError( - 'id_ cannot be a value that evaluates to false')) - _replace_sample_acls_fail(samples, id_, None, SampleACLOwnerless(), ValueError( - 'user cannot be a value that evaluates to false')) - _replace_sample_acls_fail(samples, id_, u, None, ValueError( - 'new_acls cannot be a value that evaluates to false')) + _replace_sample_acls_fail( + samples, + None, + u, + SampleACLOwnerless(), + ValueError("id_ cannot be a value that evaluates to false"), + ) + _replace_sample_acls_fail( + samples, + id_, + None, + SampleACLOwnerless(), + ValueError("user cannot be a value that evaluates to false"), + ) + _replace_sample_acls_fail( + samples, + id_, + u, + None, + ValueError("new_acls cannot be a value that evaluates to false"), + ) def test_replace_sample_acls_fail_nonexistent_user_4_users(): @@ -808,21 +1194,28 @@ def test_replace_sample_acls_fail_nonexistent_user_4_users(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") - lu.invalid_users.return_value = [u('whoo'), u('yay'), u('bugga'), u('w')] + lu.invalid_users.return_value = [u("whoo"), u("yay"), u("bugga"), u("w")] acls = SampleACLOwnerless( - [u('x'), u('whoo')], - [u('yay'), u('fwew')], - [u('y'), u('bugga'), u('z'), u('w')]) + [u("x"), u("whoo")], [u("yay"), u("fwew")], [u("y"), u("bugga"), u("z"), u("w")] + ) _replace_sample_acls_fail( - samples, id_, UserID('foo'), acls, NoSuchUserError('whoo, yay, bugga, w')) + samples, id_, UserID("foo"), acls, NoSuchUserError("whoo, yay, bugga, w") + ) lu.invalid_users.assert_called_once_with( - [u('whoo'), u('x'), u('fwew'), u('yay'), u('bugga'), u('w'), u('y'), u('z')]) + [u("whoo"), u("x"), u("fwew"), u("yay"), u("bugga"), u("w"), u("y"), u("z")] + ) def test_replace_sample_acls_fail_nonexistent_user_5_users(): @@ -831,21 +1224,40 @@ def test_replace_sample_acls_fail_nonexistent_user_5_users(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") - lu.invalid_users.return_value = [u('whoo'), u('yay'), u('bugga'), u('w'), u('c')] + lu.invalid_users.return_value = [u("whoo"), u("yay"), u("bugga"), u("w"), u("c")] acls = SampleACLOwnerless( - [u('x'), u('whoo')], - [u('yay'), u('fwew')], - [u('y'), u('bugga'), u('z'), u('w'), u('c')]) + [u("x"), u("whoo")], + [u("yay"), u("fwew")], + [u("y"), u("bugga"), u("z"), u("w"), u("c")], + ) _replace_sample_acls_fail( - samples, id_, UserID('foo'), acls, NoSuchUserError('whoo, yay, bugga, w, c')) + samples, id_, UserID("foo"), acls, NoSuchUserError("whoo, yay, bugga, w, c") + ) lu.invalid_users.assert_called_once_with( - [u('whoo'), u('x'), u('fwew'), u('yay'), u('bugga'), u('c'), u('w'), u('y'), u('z')]) + [ + u("whoo"), + u("x"), + u("fwew"), + u("yay"), + u("bugga"), + u("c"), + u("w"), + u("y"), + u("z"), + ] + ) def test_replace_sample_acls_fail_nonexistent_user_6_users(): @@ -854,22 +1266,48 @@ def test_replace_sample_acls_fail_nonexistent_user_6_users(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') - - lu.invalid_users.return_value = [u('whoo'), u('yay'), u('bugga'), u('w'), u('c'), u('whee')] + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") + + lu.invalid_users.return_value = [ + u("whoo"), + u("yay"), + u("bugga"), + u("w"), + u("c"), + u("whee"), + ] acls = SampleACLOwnerless( - [u('x'), u('whoo')], - [u('yay'), u('fwew')], - [u('y'), u('bugga'), u('z'), u('w'), u('c'), u('whee')]) + [u("x"), u("whoo")], + [u("yay"), u("fwew")], + [u("y"), u("bugga"), u("z"), u("w"), u("c"), u("whee")], + ) _replace_sample_acls_fail( - samples, id_, UserID('foo'), acls, NoSuchUserError('whoo, yay, bugga, w, c')) + samples, id_, UserID("foo"), acls, NoSuchUserError("whoo, yay, bugga, w, c") + ) lu.invalid_users.assert_called_once_with( - [u('whoo'), u('x'), u('fwew'), u('yay'), u('bugga'), u('c'), u('w'), u('whee'), u('y'), - u('z')]) + [ + u("whoo"), + u("x"), + u("fwew"), + u("yay"), + u("bugga"), + u("c"), + u("w"), + u("whee"), + u("y"), + u("z"), + ] + ) def test_replace_sample_acls_fail_invalid_user(): @@ -878,20 +1316,43 @@ def test_replace_sample_acls_fail_invalid_user(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") - lu.invalid_users.side_effect = user_lookup.InvalidUserError('o shit waddup') + lu.invalid_users.side_effect = user_lookup.InvalidUserError("o shit waddup") acls = SampleACLOwnerless( - [u('o shit waddup'), u('whoo')], - [u('yay'), u('fwew')], - [u('y'), u('bugga'), u('z')]) + [u("o shit waddup"), u("whoo")], + [u("yay"), u("fwew")], + [u("y"), u("bugga"), u("z")], + ) - _replace_sample_acls_fail(samples, id_, UserID('foo'), acls, NoSuchUserError('o shit waddup')) + _replace_sample_acls_fail( + samples, id_, UserID("foo"), acls, NoSuchUserError("o shit waddup") + ) assert lu.invalid_users.call_args_list == [ - (([u('o shit waddup'), u('whoo'), u('fwew'), u('yay'), u('bugga'), u('y'), u('z')],), {})] + ( + ( + [ + u("o shit waddup"), + u("whoo"), + u("fwew"), + u("yay"), + u("bugga"), + u("y"), + u("z"), + ], + ), + {}, + ) + ] def test_replace_sample_acls_fail_invalid_token(): @@ -900,27 +1361,40 @@ def test_replace_sample_acls_fail_invalid_token(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") - lu.invalid_users.side_effect = user_lookup.InvalidTokenError('you big dummy') + lu.invalid_users.side_effect = user_lookup.InvalidTokenError("you big dummy") acls = SampleACLOwnerless( - [u('x'), u('whoo')], - [u('yay'), u('fwew')], - [u('y'), u('bugga'), u('z')]) + [u("x"), u("whoo")], [u("yay"), u("fwew")], [u("y"), u("bugga"), u("z")] + ) - _replace_sample_acls_fail(samples, id_, UserID('foo'), acls, ValueError( - 'user lookup token for KBase auth server is invalid, cannot continue')) + _replace_sample_acls_fail( + samples, + id_, + UserID("foo"), + acls, + ValueError( + "user lookup token for KBase auth server is invalid, cannot continue" + ), + ) assert lu.invalid_users.call_args_list == [ - (([u('whoo'), u('x'), u('fwew'), u('yay'), u('bugga'), u('y'), u('z')],), {})] + (([u("whoo"), u("x"), u("fwew"), u("yay"), u("bugga"), u("y"), u("z")],), {}) + ] def test_replace_sample_acls_fail_unauthorized(): - _replace_sample_acls_fail_unauthorized(UserID('anotheruser')) - _replace_sample_acls_fail_unauthorized(UserID('x')) - _replace_sample_acls_fail_unauthorized(UserID('MrsEntity')) + _replace_sample_acls_fail_unauthorized(UserID("anotheruser")) + _replace_sample_acls_fail_unauthorized(UserID("x")) + _replace_sample_acls_fail_unauthorized(UserID("MrsEntity")) def _replace_sample_acls_fail_unauthorized(user: UserID): @@ -929,26 +1403,41 @@ def _replace_sample_acls_fail_unauthorized(user: UserID): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") lu.invalid_users.return_value = [] storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')], - public_read=True) # public read shouldn't grant privs. + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + public_read=True, + ) # public read shouldn't grant privs. - _replace_sample_acls_fail(samples, id_, user, SampleACLOwnerless(), UnauthorizedError( - f'User {user} cannot administrate sample 12345678-90ab-cdef-1234-567890abcde0')) + _replace_sample_acls_fail( + samples, + id_, + user, + SampleACLOwnerless(), + UnauthorizedError( + f"User {user} cannot administrate sample 12345678-90ab-cdef-1234-567890abcde0" + ), + ) assert lu.invalid_users.call_args_list == [(([],), {})] assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcde0'),), {})] + ((UUID("1234567890abcdef1234567890abcde0"),), {}) + ] def _replace_sample_acls_fail(samples, id_, user, acls: SampleACLOwnerless, expected): @@ -958,8 +1447,8 @@ def _replace_sample_acls_fail(samples, id_, user, acls: SampleACLOwnerless, expe def test_update_sample_acls(): - _update_sample_acls(UserID('someuser'), True) - _update_sample_acls(UserID('otheruser'), False) + _update_sample_acls(UserID("someuser"), True) + _update_sample_acls(UserID("otheruser"), False) def _update_sample_acls(user: UserID, public_read): @@ -968,80 +1457,112 @@ def _update_sample_acls(user: UserID, public_read): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) kafka = create_autospec(KafkaNotifier, spec_set=True, instance=True) - samples = Samples(storage, lu, meta, ws, kafka, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + samples = Samples( + storage, + lu, + meta, + ws, + kafka, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") lu.invalid_users.return_value = [] storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]) + [u("otheruser"), u("y")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ) - samples.update_sample_acls(id_, user, SampleACLDelta( - [u('x'), u('y')], [u('z'), u('a')], [u('b'), u('c')], [u('r'), u('q')], - public_read)) + samples.update_sample_acls( + id_, + user, + SampleACLDelta( + [u("x"), u("y")], + [u("z"), u("a")], + [u("b"), u("c")], + [u("r"), u("q")], + public_read, + ), + ) lu.invalid_users.assert_called_once_with( - [u(x) for x in ['x', 'y', 'a', 'z', 'b', 'c', 'q', 'r']]) + [u(x) for x in ["x", "y", "a", "z", "b", "c", "q", "r"]] + ) - storage.get_sample_acls.assert_called_once_with(UUID('1234567890abcdef1234567890abcde0')) + storage.get_sample_acls.assert_called_once_with( + UUID("1234567890abcdef1234567890abcde0") + ) storage.update_sample_acls.assert_called_once_with( - UUID('1234567890abcdef1234567890abcde0'), + UUID("1234567890abcdef1234567890abcde0"), SampleACLDelta( - [u('x'), u('y')], - [u('z'), u('a')], - [u('b'), u('c')], - [u('r'), u('q')], - public_read), - dt(6)) + [u("x"), u("y")], + [u("z"), u("a")], + [u("b"), u("c")], + [u("r"), u("q")], + public_read, + ), + dt(6), + ) kafka.notify_sample_acl_change.assert_called_once_with( - UUID('1234567890abcdef1234567890abcde0')) + UUID("1234567890abcdef1234567890abcde0") + ) def test_update_sample_acls_as_admin_without_notifier(): - ''' + """ Also use None for public read. - ''' + """ storage = create_autospec(ArangoSampleStorage, spec_set=True, instance=True) lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - samples = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + samples = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") lu.invalid_users.return_value = [] storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]) + [u("otheruser"), u("y")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ) - samples.update_sample_acls(id_, UserID('someguy'), SampleACLDelta( - [u('x'), u('y')], [u('z'), u('a')], [u('b'), u('c')], [u('r'), u('q')], - None), - as_admin=True) + samples.update_sample_acls( + id_, + UserID("someguy"), + SampleACLDelta( + [u("x"), u("y")], [u("z"), u("a")], [u("b"), u("c")], [u("r"), u("q")], None + ), + as_admin=True, + ) lu.invalid_users.assert_called_once_with( - [u(x) for x in ['x', 'y', 'a', 'z', 'b', 'c', 'q', 'r']]) + [u(x) for x in ["x", "y", "a", "z", "b", "c", "q", "r"]] + ) storage.update_sample_acls.assert_called_once_with( - UUID('1234567890abcdef1234567890abcde0'), + UUID("1234567890abcdef1234567890abcde0"), SampleACLDelta( - [u('x'), u('y')], - [u('z'), u('a')], - [u('b'), u('c')], - [u('r'), u('q')], - None), - dt(6)) + [u("x"), u("y")], [u("z"), u("a")], [u("b"), u("c")], [u("r"), u("q")], None + ), + dt(6), + ) def test_update_sample_acls_fail_bad_input(): @@ -1050,16 +1571,37 @@ def test_update_sample_acls_fail_bad_input(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') - u = UserID('u') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") + u = UserID("u") - _update_sample_acls_fail(samples, None, u, SampleACLDelta(), ValueError( - 'id_ cannot be a value that evaluates to false')) - _update_sample_acls_fail(samples, id_, None, SampleACLDelta(), ValueError( - 'user cannot be a value that evaluates to false')) - _update_sample_acls_fail(samples, id_, u, None, ValueError( - 'update cannot be a value that evaluates to false')) + _update_sample_acls_fail( + samples, + None, + u, + SampleACLDelta(), + ValueError("id_ cannot be a value that evaluates to false"), + ) + _update_sample_acls_fail( + samples, + id_, + None, + SampleACLDelta(), + ValueError("user cannot be a value that evaluates to false"), + ) + _update_sample_acls_fail( + samples, + id_, + u, + None, + ValueError("update cannot be a value that evaluates to false"), + ) def test_update_sample_acls_fail_nonexistent_user_5_users(): @@ -1068,23 +1610,42 @@ def test_update_sample_acls_fail_nonexistent_user_5_users(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") - lu.invalid_users.return_value = [u('whoo'), u('yay'), u('bugga'), u('w'), u('c')] + lu.invalid_users.return_value = [u("whoo"), u("yay"), u("bugga"), u("w"), u("c")] acls = SampleACLDelta( - [u('x'), u('whoo')], - [u('yay'), u('fwew')], - [u('y'), u('bugga'), u('z'), u('w'), u('c')], - [u('rem')]) + [u("x"), u("whoo")], + [u("yay"), u("fwew")], + [u("y"), u("bugga"), u("z"), u("w"), u("c")], + [u("rem")], + ) _update_sample_acls_fail( - samples, id_, UserID('foo'), acls, NoSuchUserError('whoo, yay, bugga, w, c')) + samples, id_, UserID("foo"), acls, NoSuchUserError("whoo, yay, bugga, w, c") + ) lu.invalid_users.assert_called_once_with( - [u('whoo'), u('x'), u('fwew'), u('yay'), u('bugga'), u('c'), u('w'), u('y'), u('z'), - u('rem')]) + [ + u("whoo"), + u("x"), + u("fwew"), + u("yay"), + u("bugga"), + u("c"), + u("w"), + u("y"), + u("z"), + u("rem"), + ] + ) def test_update_sample_acls_fail_nonexistent_user_6_users(): @@ -1093,23 +1654,50 @@ def test_update_sample_acls_fail_nonexistent_user_6_users(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') - - lu.invalid_users.return_value = [u('whoo'), u('yay'), u('bugga'), u('w'), u('c'), u('whee')] + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") + + lu.invalid_users.return_value = [ + u("whoo"), + u("yay"), + u("bugga"), + u("w"), + u("c"), + u("whee"), + ] acls = SampleACLDelta( - [u('x'), u('whoo')], - [u('yay'), u('fwew')], - [u('y'), u('bugga'), u('z'), u('w'), u('c'), u('whee')], - [u('rem')]) + [u("x"), u("whoo")], + [u("yay"), u("fwew")], + [u("y"), u("bugga"), u("z"), u("w"), u("c"), u("whee")], + [u("rem")], + ) _update_sample_acls_fail( - samples, id_, UserID('foo'), acls, NoSuchUserError('whoo, yay, bugga, w, c')) + samples, id_, UserID("foo"), acls, NoSuchUserError("whoo, yay, bugga, w, c") + ) lu.invalid_users.assert_called_once_with( - [u('whoo'), u('x'), u('fwew'), u('yay'), u('bugga'), u('c'), u('w'), u('whee'), u('y'), - u('z'), u('rem')]) + [ + u("whoo"), + u("x"), + u("fwew"), + u("yay"), + u("bugga"), + u("c"), + u("w"), + u("whee"), + u("y"), + u("z"), + u("rem"), + ] + ) def test_update_sample_acls_fail_invalid_user(): @@ -1118,20 +1706,43 @@ def test_update_sample_acls_fail_invalid_user(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") - lu.invalid_users.side_effect = user_lookup.InvalidUserError('o shit waddup') + lu.invalid_users.side_effect = user_lookup.InvalidUserError("o shit waddup") acls = SampleACLDelta( - [u('o shit waddup'), u('whoo')], - [u('yay'), u('fwew')], - [u('y'), u('bugga'), u('z')]) + [u("o shit waddup"), u("whoo")], + [u("yay"), u("fwew")], + [u("y"), u("bugga"), u("z")], + ) - _update_sample_acls_fail(samples, id_, UserID('foo'), acls, NoSuchUserError('o shit waddup')) + _update_sample_acls_fail( + samples, id_, UserID("foo"), acls, NoSuchUserError("o shit waddup") + ) assert lu.invalid_users.call_args_list == [ - (([u('o shit waddup'), u('whoo'), u('fwew'), u('yay'), u('bugga'), u('y'), u('z')],), {})] + ( + ( + [ + u("o shit waddup"), + u("whoo"), + u("fwew"), + u("yay"), + u("bugga"), + u("y"), + u("z"), + ], + ), + {}, + ) + ] def test_update_sample_acls_fail_invalid_token(): @@ -1140,27 +1751,40 @@ def test_update_sample_acls_fail_invalid_token(): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") - lu.invalid_users.side_effect = user_lookup.InvalidTokenError('you big dummy') + lu.invalid_users.side_effect = user_lookup.InvalidTokenError("you big dummy") acls = SampleACLDelta( - [u('x'), u('whoo')], - [u('yay'), u('fwew')], - [u('y'), u('bugga'), u('z')]) + [u("x"), u("whoo")], [u("yay"), u("fwew")], [u("y"), u("bugga"), u("z")] + ) - _update_sample_acls_fail(samples, id_, UserID('foo'), acls, ValueError( - 'user lookup token for KBase auth server is invalid, cannot continue')) + _update_sample_acls_fail( + samples, + id_, + UserID("foo"), + acls, + ValueError( + "user lookup token for KBase auth server is invalid, cannot continue" + ), + ) assert lu.invalid_users.call_args_list == [ - (([u('whoo'), u('x'), u('fwew'), u('yay'), u('bugga'), u('y'), u('z')],), {})] + (([u("whoo"), u("x"), u("fwew"), u("yay"), u("bugga"), u("y"), u("z")],), {}) + ] def test_update_sample_acls_fail_unauthorized(): - _update_sample_acls_fail_unauthorized(UserID('anotheruser')) - _update_sample_acls_fail_unauthorized(UserID('x')) - _update_sample_acls_fail_unauthorized(UserID('MrsEntity')) + _update_sample_acls_fail_unauthorized(UserID("anotheruser")) + _update_sample_acls_fail_unauthorized(UserID("x")) + _update_sample_acls_fail_unauthorized(UserID("MrsEntity")) def _update_sample_acls_fail_unauthorized(user: UserID): @@ -1169,26 +1793,41 @@ def _update_sample_acls_fail_unauthorized(user: UserID): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) samples = Samples( - storage, lu, meta, ws, now=nw, uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) - id_ = UUID('1234567890abcdef1234567890abcde0') + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) + id_ = UUID("1234567890abcdef1234567890abcde0") lu.invalid_users.return_value = [] storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')], - public_read=True) # public read shouldn't grant privs. + [u("otheruser")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + public_read=True, + ) # public read shouldn't grant privs. - _update_sample_acls_fail(samples, id_, user, SampleACLDelta(), UnauthorizedError( - f'User {user} cannot administrate sample 12345678-90ab-cdef-1234-567890abcde0')) + _update_sample_acls_fail( + samples, + id_, + user, + SampleACLDelta(), + UnauthorizedError( + f"User {user} cannot administrate sample 12345678-90ab-cdef-1234-567890abcde0" + ), + ) assert lu.invalid_users.call_args_list == [(([],), {})] assert storage.get_sample_acls.call_args_list == [ - ((UUID('1234567890abcdef1234567890abcde0'),), {})] + ((UUID("1234567890abcdef1234567890abcde0"),), {}) + ] def _update_sample_acls_fail(samples, id_, user, update, expected): @@ -1202,21 +1841,30 @@ def test_get_key_metadata(): lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) meta.key_metadata.side_effect = [ - {'a': {'c': 'd'}, 'b': {'e': 3}}, - {'a': {'c': 'e'}, 'b': {'e': 4}}, - ] + {"a": {"c": "d"}, "b": {"e": 3}}, + {"a": {"c": "e"}, "b": {"e": 4}}, + ] - assert s.get_key_static_metadata(['a', 'b']) == {'a': {'c': 'd'}, 'b': {'e': 3}} + assert s.get_key_static_metadata(["a", "b"]) == {"a": {"c": "d"}, "b": {"e": 3}} - meta.key_metadata.assert_called_once_with(['a', 'b']) + meta.key_metadata.assert_called_once_with(["a", "b"]) - assert s.get_key_static_metadata(['a', 'b'], prefix=False) == {'a': {'c': 'e'}, 'b': {'e': 4}} + assert s.get_key_static_metadata(["a", "b"], prefix=False) == { + "a": {"c": "e"}, + "b": {"e": 4}, + } - meta.key_metadata.assert_called_with(['a', 'b']) + meta.key_metadata.assert_called_with(["a", "b"]) assert meta.key_metadata.call_count == 2 @@ -1226,23 +1874,33 @@ def test_get_prefix_key_metadata(): lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) meta.prefix_key_metadata.side_effect = [ - {'a': {'c': 'd'}, 'b': {'e': 3}}, - {'a': {'c': 'f'}, 'b': {'e': 5}} - ] + {"a": {"c": "d"}, "b": {"e": 3}}, + {"a": {"c": "f"}, "b": {"e": 5}}, + ] - assert s.get_key_static_metadata(['a', 'b'], prefix=None) == { - 'a': {'c': 'd'}, 'b': {'e': 3}} + assert s.get_key_static_metadata(["a", "b"], prefix=None) == { + "a": {"c": "d"}, + "b": {"e": 3}, + } - meta.prefix_key_metadata.assert_called_once_with(['a', 'b'], exact_match=True) + meta.prefix_key_metadata.assert_called_once_with(["a", "b"], exact_match=True) - assert s.get_key_static_metadata(['a', 'b'], prefix=True) == { - 'a': {'c': 'f'}, 'b': {'e': 5}} + assert s.get_key_static_metadata(["a", "b"], prefix=True) == { + "a": {"c": "f"}, + "b": {"e": 5}, + } - meta.prefix_key_metadata.assert_called_with(['a', 'b'], exact_match=False) + meta.prefix_key_metadata.assert_called_with(["a", "b"], exact_match=False) assert meta.prefix_key_metadata.call_count == 2 @@ -1252,18 +1910,24 @@ def test_get_prefix_key_fail_bad_args(): lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) with raises(Exception) as got: s.get_key_static_metadata(None) - assert_exception_correct(got.value, ValueError('keys cannot be None')) + assert_exception_correct(got.value, ValueError("keys cannot be None")) def test_create_data_link(): - _create_data_link(UserID('someuser')) - _create_data_link(UserID('otheruser')) - _create_data_link(UserID('y')) + _create_data_link(UserID("someuser")) + _create_data_link(UserID("otheruser")) + _create_data_link(UserID("y")) def _create_data_link(user): @@ -1272,91 +1936,131 @@ def _create_data_link(user): meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) kafka = create_autospec(KafkaNotifier, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, kafka, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + kafka, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]) + [u("otheruser"), u("y")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ) assert s.create_data_link( user, - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode') - ) == DataLink( - UUID('1234567890abcdef1234567890abcdef'), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), - dt(6), - user - ) + DataUnitID(UPA("1/1/1")), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), + ) == DataLink( + UUID("1234567890abcdef1234567890abcdef"), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), + dt(6), + user, + ) - storage.get_sample_acls.assert_called_once_with(UUID('1234567890abcdef1234567890abcdee')) + storage.get_sample_acls.assert_called_once_with( + UUID("1234567890abcdef1234567890abcdee") + ) - ws.has_permission.assert_called_once_with(user, WorkspaceAccessType.WRITE, upa=UPA('1/1/1')) + ws.has_permission.assert_called_once_with( + user, WorkspaceAccessType.WRITE, upa=UPA("1/1/1") + ) dl = DataLink( - UUID('1234567890abcdef1234567890abcdef'), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), + UUID("1234567890abcdef1234567890abcdef"), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), dt(6), - user + user, ) storage.create_data_link.assert_called_once_with(dl, update=False) - kafka.notify_new_link.assert_called_once_with(UUID('1234567890abcdef1234567890abcdef')) + kafka.notify_new_link.assert_called_once_with( + UUID("1234567890abcdef1234567890abcdef") + ) def test_create_data_link_with_data_id_and_update(): - ''' + """ Test with a data id in the DUID and update=True. - ''' + """ storage = create_autospec(ArangoSampleStorage, spec_set=True, instance=True) lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) kafka = create_autospec(KafkaNotifier, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, kafka, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + kafka, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) - storage.get_sample_acls.return_value = SampleACL(u('someuser'), dt(1)) + storage.get_sample_acls.return_value = SampleACL(u("someuser"), dt(1)) - storage.create_data_link.return_value = UUID('1234567890abcdef1234567890abcde1') + storage.create_data_link.return_value = UUID("1234567890abcdef1234567890abcde1") assert s.create_data_link( - UserID('someuser'), - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), - update=True - ) == DataLink( - UUID('1234567890abcdef1234567890abcdef'), - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), - dt(6), - UserID('someuser') - ) + UserID("someuser"), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), + update=True, + ) == DataLink( + UUID("1234567890abcdef1234567890abcdef"), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), + dt(6), + UserID("someuser"), + ) - storage.get_sample_acls.assert_called_once_with(UUID('1234567890abcdef1234567890abcdee')) + storage.get_sample_acls.assert_called_once_with( + UUID("1234567890abcdef1234567890abcdee") + ) ws.has_permission.assert_called_once_with( - UserID('someuser'), WorkspaceAccessType.WRITE, upa=UPA('1/1/1')) + UserID("someuser"), WorkspaceAccessType.WRITE, upa=UPA("1/1/1") + ) dl = DataLink( - UUID('1234567890abcdef1234567890abcdef'), - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), + UUID("1234567890abcdef1234567890abcdef"), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), dt(6), - UserID('someuser') + UserID("someuser"), ) storage.create_data_link.assert_called_once_with(dl, update=True) - kafka.notify_new_link.assert_called_once_with(UUID('1234567890abcdef1234567890abcdef')) - kafka.notify_expired_link.assert_called_once_with(UUID('1234567890abcdef1234567890abcde1')) + kafka.notify_new_link.assert_called_once_with( + UUID("1234567890abcdef1234567890abcdef") + ) + kafka.notify_expired_link.assert_called_once_with( + UUID("1234567890abcdef1234567890abcde1") + ) def test_create_data_link_as_admin(): @@ -1367,31 +2071,44 @@ def test_create_data_link_as_admin(): lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) assert s.create_data_link( - UserID('someuser'), - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), - as_admin=True - ) == DataLink( - UUID('1234567890abcdef1234567890abcdef'), - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), - dt(6), - UserID('someuser') - ) + UserID("someuser"), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), + as_admin=True, + ) == DataLink( + UUID("1234567890abcdef1234567890abcdef"), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), + dt(6), + UserID("someuser"), + ) ws.has_permission.assert_called_once_with( - UserID('someuser'), WorkspaceAccessType.NONE, upa=UPA('1/1/1')) + UserID("someuser"), WorkspaceAccessType.NONE, upa=UPA("1/1/1") + ) dl = DataLink( - UUID('1234567890abcdef1234567890abcdef'), - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), + UUID("1234567890abcdef1234567890abcdef"), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), dt(6), - UserID('someuser') + UserID("someuser"), ) storage.create_data_link.assert_called_once_with(dl, update=False) @@ -1402,25 +2119,36 @@ def test_create_data_link_fail_bad_args(): lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) - u = UserID('u') - d = DataUnitID(UPA('1/1/1')) - sna = SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'node') + u = UserID("u") + d = DataUnitID(UPA("1/1/1")) + sna = SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "node" + ) - _create_data_link_fail(s, None, d, sna, ValueError( - 'user cannot be a value that evaluates to false')) - _create_data_link_fail(s, u, None, sna, ValueError( - 'duid cannot be a value that evaluates to false')) - _create_data_link_fail(s, u, d, None, ValueError( - 'sna cannot be a value that evaluates to false')) + _create_data_link_fail( + s, None, d, sna, ValueError("user cannot be a value that evaluates to false") + ) + _create_data_link_fail( + s, u, None, sna, ValueError("duid cannot be a value that evaluates to false") + ) + _create_data_link_fail( + s, u, d, None, ValueError("sna cannot be a value that evaluates to false") + ) def test_create_data_link_fail_no_sample_access(): - _create_data_link_fail_no_sample_access(UserID('writeonly')) - _create_data_link_fail_no_sample_access(UserID('readonly')) - _create_data_link_fail_no_sample_access(UserID('noaccess')) + _create_data_link_fail_no_sample_access(UserID("writeonly")) + _create_data_link_fail_no_sample_access(UserID("readonly")) + _create_data_link_fail_no_sample_access(UserID("noaccess")) def _create_data_link_fail_no_sample_access(user): @@ -1428,26 +2156,39 @@ def _create_data_link_fail_no_sample_access(user): lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('writeonly')], - [u('readonly'), u('x')], - public_read=True) # public read shouldn't grant privs + [u("otheruser"), u("y")], + [u("anotheruser"), u("writeonly")], + [u("readonly"), u("x")], + public_read=True, + ) # public read shouldn't grant privs _create_data_link_fail( s, user, - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), UnauthorizedError( - f'User {user} cannot administrate sample 12345678-90ab-cdef-1234-567890abcdee')) + f"User {user} cannot administrate sample 12345678-90ab-cdef-1234-567890abcdee" + ), + ) - storage.get_sample_acls.assert_called_once_with(UUID('1234567890abcdef1234567890abcdee')) + storage.get_sample_acls.assert_called_once_with( + UUID("1234567890abcdef1234567890abcdee") + ) def test_create_data_link_fail_no_ws_access(): @@ -1455,24 +2196,36 @@ def test_create_data_link_fail_no_ws_access(): lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) - s = Samples(storage, lu, meta, ws, now=nw, - uuid_gen=lambda: UUID('1234567890abcdef1234567890abcdef')) + s = Samples( + storage, + lu, + meta, + ws, + now=nw, + uuid_gen=lambda: UUID("1234567890abcdef1234567890abcdef"), + ) - storage.get_sample_acls.return_value = SampleACL(u('someuser'), dt(1)) + storage.get_sample_acls.return_value = SampleACL(u("someuser"), dt(1)) - ws.has_permission.side_effect = UnauthorizedError('nope. uh uh') + ws.has_permission.side_effect = UnauthorizedError("nope. uh uh") _create_data_link_fail( s, - UserID('someuser'), - DataUnitID(UPA('7/3/2'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), - UnauthorizedError('nope. uh uh')) + UserID("someuser"), + DataUnitID(UPA("7/3/2"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), + UnauthorizedError("nope. uh uh"), + ) - storage.get_sample_acls.assert_called_once_with(UUID('1234567890abcdef1234567890abcdee')) + storage.get_sample_acls.assert_called_once_with( + UUID("1234567890abcdef1234567890abcdee") + ) ws.has_permission.assert_called_once_with( - UserID('someuser'), WorkspaceAccessType.WRITE, upa=UPA('7/3/2')) + UserID("someuser"), WorkspaceAccessType.WRITE, upa=UPA("7/3/2") + ) def _create_data_link_fail(samples, user, duid, sna, expected): @@ -1482,11 +2235,11 @@ def _create_data_link_fail(samples, user, duid, sna, expected): def test_get_links_from_sample(): - _get_links_from_sample(UserID('someuser')) - _get_links_from_sample(UserID('otheruser')) - _get_links_from_sample(UserID('ur mum')) - _get_links_from_sample(UserID('x')) - _get_links_from_sample(UserID('noaccess'), True) + _get_links_from_sample(UserID("someuser")) + _get_links_from_sample(UserID("otheruser")) + _get_links_from_sample(UserID("ur mum")) + _get_links_from_sample(UserID("x")) + _get_links_from_sample(UserID("noaccess"), True) _get_links_from_sample(None, True) @@ -1498,44 +2251,50 @@ def _get_links_from_sample(user, public_read=False): s = Samples(storage, lu, meta, ws, now=nw) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')], - public_read=public_read) + [u("otheruser"), u("y")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + public_read=public_read, + ) ws.get_user_workspaces.return_value = [7, 90, 106] dl1 = DataLink( - UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), + UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), dt(5), - UserID('userb') + UserID("userb"), ) dl2 = DataLink( - UUID('1234567890abcdef1234567890abcdec'), - DataUnitID(UPA('1/2/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode3'), + UUID("1234567890abcdef1234567890abcdec"), + DataUnitID(UPA("1/2/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode3" + ), dt(5), - UserID('usera') + UserID("usera"), ) storage.get_links_from_sample.return_value = [dl1, dl2] assert s.get_links_from_sample( - user, SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3)) == ([dl1, dl2], dt(6)) + user, SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3) + ) == ([dl1, dl2], dt(6)) - storage.get_sample_acls.assert_called_once_with(UUID('1234567890abcdef1234567890abcdee')) + storage.get_sample_acls.assert_called_once_with( + UUID("1234567890abcdef1234567890abcdee") + ) ws.get_user_workspaces.assert_called_once_with(user) storage.get_links_from_sample.assert_called_once_with( - SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), - [7, 90, 106], - dt(6) + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), [7, 90, 106], dt(6) ) @@ -1547,32 +2306,38 @@ def test_get_links_from_sample_as_admin(): s = Samples(storage, lu, meta, ws, now=nw) dl1 = DataLink( - UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), + UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), dt(5), - UserID('userb') + UserID("userb"), ) dl2 = DataLink( - UUID('1234567890abcdef1234567890abcdec'), - DataUnitID(UPA('1/2/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode3'), + UUID("1234567890abcdef1234567890abcdec"), + DataUnitID(UPA("1/2/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode3" + ), dt(5), - UserID('usera') + UserID("usera"), ) storage.get_links_from_sample.return_value = [dl1, dl2] - assert s.get_links_from_sample( - UserID('whateva'), - SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), - as_admin=True) == ([dl1, dl2], dt(6)) + assert ( + s.get_links_from_sample( + UserID("whateva"), + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), + as_admin=True, + ) + == ([dl1, dl2], dt(6)) + ) storage.get_links_from_sample.assert_called_once_with( - SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), - None, - dt(6) + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), None, dt(6) ) @@ -1583,33 +2348,39 @@ def test_get_links_from_sample_with_timestamp(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - storage.get_sample_acls.return_value = SampleACL(u('someuser'), dt(1)) + storage.get_sample_acls.return_value = SampleACL(u("someuser"), dt(1)) ws.get_user_workspaces.return_value = [3] dl1 = DataLink( - UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('1/1/1'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), 'mynode'), + UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("1/1/1"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), "mynode" + ), dt(5), - UserID('userb') + UserID("userb"), ) storage.get_links_from_sample.return_value = [dl1] - assert s.get_links_from_sample( - UserID('someuser'), - SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), - dt(40)) == ([dl1], dt(40)) + assert ( + s.get_links_from_sample( + UserID("someuser"), + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), + dt(40), + ) + == ([dl1], dt(40)) + ) - storage.get_sample_acls.assert_called_once_with(UUID('1234567890abcdef1234567890abcdee')) + storage.get_sample_acls.assert_called_once_with( + UUID("1234567890abcdef1234567890abcdee") + ) - ws.get_user_workspaces.assert_called_once_with(UserID('someuser')) + ws.get_user_workspaces.assert_called_once_with(UserID("someuser")) storage.get_links_from_sample.assert_called_once_with( - SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), - [3], - dt(40) + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), [3], dt(40) ) @@ -1620,21 +2391,31 @@ def test_get_links_from_sample_fail_bad_args(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - u = UserID('u') - sa = SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3) + u = UserID("u") + sa = SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3) bt = datetime.datetime.fromtimestamp(1) - _get_links_from_sample_fail(s, u, None, None, ValueError( - 'sample cannot be a value that evaluates to false')) - _get_links_from_sample_fail(s, u, sa, bt, ValueError( - 'timestamp cannot be a naive datetime')) + _get_links_from_sample_fail( + s, u, None, None, ValueError("sample cannot be a value that evaluates to false") + ) + _get_links_from_sample_fail( + s, u, sa, bt, ValueError("timestamp cannot be a naive datetime") + ) def test_get_links_from_sample_fail_unauthorized(): - _get_links_from_sample_fail_unauthorized(UserID('z'), UnauthorizedError( - 'User z cannot read sample 12345678-90ab-cdef-1234-567890abcdee')) - _get_links_from_sample_fail_unauthorized(None, UnauthorizedError( - 'Anonymous users cannot read sample 12345678-90ab-cdef-1234-567890abcdee')) + _get_links_from_sample_fail_unauthorized( + UserID("z"), + UnauthorizedError( + "User z cannot read sample 12345678-90ab-cdef-1234-567890abcdee" + ), + ) + _get_links_from_sample_fail_unauthorized( + None, + UnauthorizedError( + "Anonymous users cannot read sample 12345678-90ab-cdef-1234-567890abcdee" + ), + ) def _get_links_from_sample_fail_unauthorized(user, expected): @@ -1645,20 +2426,24 @@ def _get_links_from_sample_fail_unauthorized(user, expected): s = Samples(storage, lu, meta, ws, now=nw) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]) + [u("otheruser"), u("y")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ) _get_links_from_sample_fail( s, user, - SampleAddress(UUID('1234567890abcdef1234567890abcdee'), 3), + SampleAddress(UUID("1234567890abcdef1234567890abcdee"), 3), None, - expected) + expected, + ) - storage.get_sample_acls.assert_called_once_with(UUID('1234567890abcdef1234567890abcdee')) + storage.get_sample_acls.assert_called_once_with( + UUID("1234567890abcdef1234567890abcdee") + ) def _get_links_from_sample_fail(samples, user, sample, ts, expected): @@ -1675,29 +2460,34 @@ def test_get_links_from_data(): s = Samples(storage, lu, meta, ws, now=nw) dl1 = DataLink( - UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('2/4/6'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdea'), 3), 'mynode'), + UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("2/4/6"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdea"), 3), "mynode" + ), dt(5), - UserID('userb') + UserID("userb"), ) dl2 = DataLink( - UUID('1234567890abcdef1234567890abcdec'), - DataUnitID(UPA('2/4/6')), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdeb'), 1), 'mynode3'), + UUID("1234567890abcdef1234567890abcdec"), + DataUnitID(UPA("2/4/6")), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdeb"), 1), "mynode3" + ), dt(4), - UserID('usera') + UserID("usera"), ) storage.get_links_from_data.return_value = [dl1, dl2] - assert s.get_links_from_data(UserID('u1'), UPA('2/4/6')) == ([dl1, dl2], dt(6)) + assert s.get_links_from_data(UserID("u1"), UPA("2/4/6")) == ([dl1, dl2], dt(6)) ws.has_permission.assert_called_once_with( - UserID('u1'), WorkspaceAccessType.READ, upa=UPA('2/4/6')) + UserID("u1"), WorkspaceAccessType.READ, upa=UPA("2/4/6") + ) - storage.get_links_from_data.assert_called_once_with(UPA('2/4/6'), dt(6)) + storage.get_links_from_data.assert_called_once_with(UPA("2/4/6"), dt(6)) def test_get_links_from_data_with_timestamp_and_anon_user(): @@ -1708,21 +2498,27 @@ def test_get_links_from_data_with_timestamp_and_anon_user(): s = Samples(storage, lu, meta, ws, now=nw) dl1 = DataLink( - UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('2/4/6'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdea'), 3), 'mynode'), + UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("2/4/6"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdea"), 3), "mynode" + ), dt(5), - UserID('userb') + UserID("userb"), ) storage.get_links_from_data.return_value = [dl1] - assert s.get_links_from_data(None, UPA('2/4/6'), timestamp=dt(700)) == ([dl1], dt(700)) + assert s.get_links_from_data(None, UPA("2/4/6"), timestamp=dt(700)) == ( + [dl1], + dt(700), + ) ws.has_permission.assert_called_once_with( - None, WorkspaceAccessType.READ, upa=UPA('2/4/6')) + None, WorkspaceAccessType.READ, upa=UPA("2/4/6") + ) - storage.get_links_from_data.assert_called_once_with(UPA('2/4/6'), dt(700)) + storage.get_links_from_data.assert_called_once_with(UPA("2/4/6"), dt(700)) def test_get_links_from_data_as_admin(): @@ -1733,29 +2529,37 @@ def test_get_links_from_data_as_admin(): s = Samples(storage, lu, meta, ws, now=nw) dl1 = DataLink( - UUID('1234567890abcdef1234567890abcdee'), - DataUnitID(UPA('2/4/6'), 'foo'), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdea'), 3), 'mynode'), + UUID("1234567890abcdef1234567890abcdee"), + DataUnitID(UPA("2/4/6"), "foo"), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdea"), 3), "mynode" + ), dt(5), - UserID('userb') + UserID("userb"), ) dl2 = DataLink( - UUID('1234567890abcdef1234567890abcdec'), - DataUnitID(UPA('2/4/6')), - SampleNodeAddress(SampleAddress(UUID('1234567890abcdef1234567890abcdeb'), 1), 'mynode3'), + UUID("1234567890abcdef1234567890abcdec"), + DataUnitID(UPA("2/4/6")), + SampleNodeAddress( + SampleAddress(UUID("1234567890abcdef1234567890abcdeb"), 1), "mynode3" + ), dt(4), - UserID('usera') + UserID("usera"), ) storage.get_links_from_data.return_value = [dl1, dl2] - assert s.get_links_from_data(UserID('u1'), UPA('2/4/6'), as_admin=True) == ([dl1, dl2], dt(6)) + assert s.get_links_from_data(UserID("u1"), UPA("2/4/6"), as_admin=True) == ( + [dl1, dl2], + dt(6), + ) ws.has_permission.assert_called_once_with( - UserID('u1'), WorkspaceAccessType.NONE, upa=UPA('2/4/6')) + UserID("u1"), WorkspaceAccessType.NONE, upa=UPA("2/4/6") + ) - storage.get_links_from_data.assert_called_once_with(UPA('2/4/6'), dt(6)) + storage.get_links_from_data.assert_called_once_with(UPA("2/4/6"), dt(6)) def test_get_links_from_data_fail_bad_args(): @@ -1765,18 +2569,20 @@ def test_get_links_from_data_fail_bad_args(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - u = UserID('u') - up = UPA('1/1/1') + u = UserID("u") + up = UPA("1/1/1") bt = datetime.datetime.fromtimestamp(1) - _get_links_from_from_data_fail(s, u, None, None, ValueError( - 'upa cannot be a value that evaluates to false')) - _get_links_from_from_data_fail(s, u, up, bt, ValueError( - 'timestamp cannot be a naive datetime')) + _get_links_from_from_data_fail( + s, u, None, None, ValueError("upa cannot be a value that evaluates to false") + ) + _get_links_from_from_data_fail( + s, u, up, bt, ValueError("timestamp cannot be a naive datetime") + ) def test_get_links_from_data_fail_no_ws_access(): - _get_links_from_data_fail_no_ws_access(UserID('u')) + _get_links_from_data_fail_no_ws_access(UserID("u")) _get_links_from_data_fail_no_ws_access(None) @@ -1787,13 +2593,15 @@ def _get_links_from_data_fail_no_ws_access(user): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - ws.has_permission.side_effect = UnauthorizedError('oh honey') + ws.has_permission.side_effect = UnauthorizedError("oh honey") - _get_links_from_from_data_fail(s, user, UPA('1/1/1'), None, - UnauthorizedError('oh honey')) + _get_links_from_from_data_fail( + s, user, UPA("1/1/1"), None, UnauthorizedError("oh honey") + ) ws.has_permission.assert_called_once_with( - user, WorkspaceAccessType.READ, upa=UPA('1/1/1')) + user, WorkspaceAccessType.READ, upa=UPA("1/1/1") + ) def _get_links_from_from_data_fail(samples, user, upa, ts, expected): @@ -1804,7 +2612,7 @@ def _get_links_from_from_data_fail(samples, user, upa, ts, expected): def test_get_sample_via_data(): _get_sample_via_data(None) - _get_sample_via_data(UserID('someguy')) + _get_sample_via_data(UserID("someguy")) def _get_sample_via_data(user): @@ -1814,31 +2622,23 @@ def _get_sample_via_data(user): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - id_ = UUID('1234567890abcdef1234567890abcdee') + id_ = UUID("1234567890abcdef1234567890abcdee") storage.has_data_link.return_value = True storage.get_sample.return_value = SavedSample( - id_, - UserID('yay'), - [SampleNode('myname')], - dt(84), - version=4 + id_, UserID("yay"), [SampleNode("myname")], dt(84), version=4 ) assert s.get_sample_via_data( - user, UPA('4/5/7'), SampleAddress(id_, 4)) == SavedSample( - id_, - UserID('yay'), - [SampleNode('myname')], - dt(84), - version=4 - ) + user, UPA("4/5/7"), SampleAddress(id_, 4) + ) == SavedSample(id_, UserID("yay"), [SampleNode("myname")], dt(84), version=4) ws.has_permission.assert_called_once_with( - user, WorkspaceAccessType.READ, upa=UPA('4/5/7')) + user, WorkspaceAccessType.READ, upa=UPA("4/5/7") + ) - storage.has_data_link.assert_called_once_with(UPA('4/5/7'), id_) + storage.has_data_link.assert_called_once_with(UPA("4/5/7"), id_) storage.get_sample.assert_called_once_with(id_, 4) @@ -1849,19 +2649,25 @@ def test_get_sample_via_data_fail_bad_args(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - u = UserID('u') - up = UPA('1/1/1') + u = UserID("u") + up = UPA("1/1/1") sa = SampleAddress(uuid.uuid4(), 1) - _get_sample_via_data_fail(s, u, None, sa, ValueError( - 'upa cannot be a value that evaluates to false')) - _get_sample_via_data_fail(s, u, up, None, ValueError( - 'sample_address cannot be a value that evaluates to false')) + _get_sample_via_data_fail( + s, u, None, sa, ValueError("upa cannot be a value that evaluates to false") + ) + _get_sample_via_data_fail( + s, + u, + up, + None, + ValueError("sample_address cannot be a value that evaluates to false"), + ) def test_get_sample_via_data_fail_no_ws_access(): _get_sample_via_data_fail_no_ws_access(None) - _get_sample_via_data_fail_no_ws_access(UserID('u')) + _get_sample_via_data_fail_no_ws_access(UserID("u")) def _get_sample_via_data_fail_no_ws_access(user): @@ -1871,13 +2677,19 @@ def _get_sample_via_data_fail_no_ws_access(user): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - ws.has_permission.side_effect = UnauthorizedError('oh honey boo boo') + ws.has_permission.side_effect = UnauthorizedError("oh honey boo boo") - _get_sample_via_data_fail(s, user, UPA('1/1/1'), SampleAddress(uuid.uuid4(), 5), - UnauthorizedError('oh honey boo boo')) + _get_sample_via_data_fail( + s, + user, + UPA("1/1/1"), + SampleAddress(uuid.uuid4(), 5), + UnauthorizedError("oh honey boo boo"), + ) ws.has_permission.assert_called_once_with( - user, WorkspaceAccessType.READ, upa=UPA('1/1/1')) + user, WorkspaceAccessType.READ, upa=UPA("1/1/1") + ) def test_get_sample_via_data_fail_no_link(): @@ -1887,17 +2699,23 @@ def test_get_sample_via_data_fail_no_link(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - id_ = UUID('1234567890abcdef1234567890abcdee') + id_ = UUID("1234567890abcdef1234567890abcdee") storage.has_data_link.return_value = False - _get_sample_via_data_fail(s, UserID('u'), UPA('4/5/6'), SampleAddress(id_, 3), - NoSuchLinkError(f'There is no link from UPA 4/5/6 to sample {id_}')) + _get_sample_via_data_fail( + s, + UserID("u"), + UPA("4/5/6"), + SampleAddress(id_, 3), + NoSuchLinkError(f"There is no link from UPA 4/5/6 to sample {id_}"), + ) ws.has_permission.assert_called_once_with( - UserID('u'), WorkspaceAccessType.READ, upa=UPA('4/5/6')) + UserID("u"), WorkspaceAccessType.READ, upa=UPA("4/5/6") + ) - storage.has_data_link.assert_called_once_with(UPA('4/5/6'), id_) + storage.has_data_link.assert_called_once_with(UPA("4/5/6"), id_) def _get_sample_via_data_fail(samples, user, upa, sa, expected): @@ -1907,9 +2725,9 @@ def _get_sample_via_data_fail(samples, user, upa, sa, expected): def test_expire_data_link(): - _expire_data_link(UserID('someuser')) - _expire_data_link(UserID('otheruser')) - _expire_data_link(UserID('y')) + _expire_data_link(UserID("someuser")) + _expire_data_link(UserID("otheruser")) + _expire_data_link(UserID("y")) def _expire_data_link(user): @@ -1920,29 +2738,31 @@ def _expire_data_link(user): kafka = create_autospec(KafkaNotifier, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, kafka, now=nw) - sid = UUID('1234567890abcdef1234567890abcdee') - lid = UUID('1234567890abcdef1234567890abcde2') + sid = UUID("1234567890abcdef1234567890abcdee") + lid = UUID("1234567890abcdef1234567890abcde2") storage.get_data_link.return_value = DataLink( lid, - DataUnitID(UPA('6/1/2'), 'foo'), - SampleNodeAddress(SampleAddress(sid, 3), 'node'), + DataUnitID(UPA("6/1/2"), "foo"), + SampleNodeAddress(SampleAddress(sid, 3), "node"), dt(34), - UserID('userc') + UserID("userc"), ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]) + [u("otheruser"), u("y")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ) - s.expire_data_link(user, DataUnitID(UPA('6/1/2'), 'foo')) + s.expire_data_link(user, DataUnitID(UPA("6/1/2"), "foo")) ws.has_permission.assert_called_once_with( - user, WorkspaceAccessType.WRITE, workspace_id=6) + user, WorkspaceAccessType.WRITE, workspace_id=6 + ) - storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA('6/1/2'), 'foo')) + storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA("6/1/2"), "foo")) storage.get_sample_acls.assert_called_once_with(sid) storage.expire_data_link.assert_called_once_with(dt(6), user, id_=lid) @@ -1959,23 +2779,24 @@ def test_expire_data_link_as_admin(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - sid = UUID('1234567890abcdef1234567890abcdee') - lid = UUID('1234567890abcdef1234567890abcde1') + sid = UUID("1234567890abcdef1234567890abcdee") + lid = UUID("1234567890abcdef1234567890abcde1") storage.get_data_link.return_value = DataLink( lid, - DataUnitID(UPA('6/1/2'), 'foo'), - SampleNodeAddress(SampleAddress(sid, 3), 'node'), + DataUnitID(UPA("6/1/2"), "foo"), + SampleNodeAddress(SampleAddress(sid, 3), "node"), dt(34), - UserID('userc') + UserID("userc"), ) - s.expire_data_link(UserID('userf'), DataUnitID(UPA('6/1/2'), 'foo'), as_admin=True) + s.expire_data_link(UserID("userf"), DataUnitID(UPA("6/1/2"), "foo"), as_admin=True) ws.has_permission.assert_called_once_with( - UserID('userf'), WorkspaceAccessType.NONE, workspace_id=6) + UserID("userf"), WorkspaceAccessType.NONE, workspace_id=6 + ) - storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA('6/1/2'), 'foo')) - storage.expire_data_link.assert_called_once_with(dt(6), UserID('userf'), id_=lid) + storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA("6/1/2"), "foo")) + storage.expire_data_link.assert_called_once_with(dt(6), UserID("userf"), id_=lid) def test_expire_data_link_fail_bad_args(): @@ -1985,10 +2806,18 @@ def test_expire_data_link_fail_bad_args(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - _expire_data_link_fail(s, None, DataUnitID(UPA('1/1/1'), 'foo'), ValueError( - 'user cannot be a value that evaluates to false')) - _expire_data_link_fail(s, UserID('u'), None, ValueError( - 'duid cannot be a value that evaluates to false')) + _expire_data_link_fail( + s, + None, + DataUnitID(UPA("1/1/1"), "foo"), + ValueError("user cannot be a value that evaluates to false"), + ) + _expire_data_link_fail( + s, + UserID("u"), + None, + ValueError("duid cannot be a value that evaluates to false"), + ) def test_expire_data_link_fail_no_ws_access(): @@ -1998,13 +2827,18 @@ def test_expire_data_link_fail_no_ws_access(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - ws.has_permission.side_effect = UnauthorizedError('oh honey boo boo foofy foo') + ws.has_permission.side_effect = UnauthorizedError("oh honey boo boo foofy foo") - _expire_data_link_fail(s, UserID('u'), DataUnitID(UPA('1/1/1')), - UnauthorizedError('oh honey boo boo foofy foo')) + _expire_data_link_fail( + s, + UserID("u"), + DataUnitID(UPA("1/1/1")), + UnauthorizedError("oh honey boo boo foofy foo"), + ) ws.has_permission.assert_called_once_with( - UserID('u'), WorkspaceAccessType.WRITE, workspace_id=1) + UserID("u"), WorkspaceAccessType.WRITE, workspace_id=1 + ) def test_expire_data_link_fail_no_link(): @@ -2014,21 +2848,23 @@ def test_expire_data_link_fail_no_link(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - storage.get_data_link.side_effect = NoSuchLinkError('oh lordy') + storage.get_data_link.side_effect = NoSuchLinkError("oh lordy") - _expire_data_link_fail(s, UserID('a'), DataUnitID(UPA('6/1/2'), 'foo'), - NoSuchLinkError('oh lordy')) + _expire_data_link_fail( + s, UserID("a"), DataUnitID(UPA("6/1/2"), "foo"), NoSuchLinkError("oh lordy") + ) ws.has_permission.assert_called_once_with( - UserID('a'), WorkspaceAccessType.WRITE, workspace_id=6) + UserID("a"), WorkspaceAccessType.WRITE, workspace_id=6 + ) - storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA('6/1/2'), 'foo')) + storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA("6/1/2"), "foo")) def test_expire_data_link_fail_no_sample_access(): - _expire_data_link_fail_no_sample_access(UserID('anotheruser')) - _expire_data_link_fail_no_sample_access(UserID('x')) - _expire_data_link_fail_no_sample_access(UserID('z')) + _expire_data_link_fail_no_sample_access(UserID("anotheruser")) + _expire_data_link_fail_no_sample_access(UserID("x")) + _expire_data_link_fail_no_sample_access(UserID("z")) def _expire_data_link_fail_no_sample_access(user): @@ -2038,72 +2874,81 @@ def _expire_data_link_fail_no_sample_access(user): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - sid = UUID('1234567890abcdef1234567890abcdee') + sid = UUID("1234567890abcdef1234567890abcdee") storage.get_data_link.return_value = DataLink( uuid.uuid4(), # unused - DataUnitID(UPA('9/1/2'), 'foo'), - SampleNodeAddress(SampleAddress(sid, 3), 'node'), + DataUnitID(UPA("9/1/2"), "foo"), + SampleNodeAddress(SampleAddress(sid, 3), "node"), dt(34), - UserID('userc') + UserID("userc"), ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')], - public_read=True) # public read shouldn't grant perms + [u("otheruser"), u("y")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + public_read=True, + ) # public read shouldn't grant perms - _expire_data_link_fail(s, user, DataUnitID(UPA('9/1/2'), 'foo'), - UnauthorizedError(f'User {user} cannot administrate sample {sid}')) + _expire_data_link_fail( + s, + user, + DataUnitID(UPA("9/1/2"), "foo"), + UnauthorizedError(f"User {user} cannot administrate sample {sid}"), + ) ws.has_permission.assert_called_once_with( - user, WorkspaceAccessType.WRITE, workspace_id=9) + user, WorkspaceAccessType.WRITE, workspace_id=9 + ) - storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA('9/1/2'), 'foo')) + storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA("9/1/2"), "foo")) storage.get_sample_acls.assert_called_once_with(sid) def test_expire_data_link_fail_no_link_at_storage(): - ''' + """ Tests the improbable case where the link is expired after fetching it from storage but before the expire command is sent from storage. - ''' + """ storage = create_autospec(ArangoSampleStorage, spec_set=True, instance=True) lu = create_autospec(KBaseUserLookup, spec_set=True, instance=True) meta = create_autospec(MetadataValidatorSet, spec_set=True, instance=True) ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - sid = UUID('1234567890abcdef1234567890abcdee') - lid = UUID('1234567890abcdef1234567890abcde1') + sid = UUID("1234567890abcdef1234567890abcdee") + lid = UUID("1234567890abcdef1234567890abcde1") storage.get_data_link.return_value = DataLink( lid, - DataUnitID(UPA('6/1/2')), - SampleNodeAddress(SampleAddress(sid, 3), 'node'), + DataUnitID(UPA("6/1/2")), + SampleNodeAddress(SampleAddress(sid, 3), "node"), dt(34), - UserID('userc') + UserID("userc"), ) storage.get_sample_acls.return_value = SampleACL( - u('someuser'), + u("someuser"), dt(1), - [u('otheruser'), u('y')], - [u('anotheruser'), u('ur mum')], - [u('Fungus J. Pustule Jr.'), u('x')]) + [u("otheruser"), u("y")], + [u("anotheruser"), u("ur mum")], + [u("Fungus J. Pustule Jr."), u("x")], + ) storage.expire_data_link.side_effect = NoSuchLinkError("dang y'all") - _expire_data_link_fail(s, UserID('y'), DataUnitID(UPA('6/1/2')), - NoSuchLinkError("dang y'all")) + _expire_data_link_fail( + s, UserID("y"), DataUnitID(UPA("6/1/2")), NoSuchLinkError("dang y'all") + ) ws.has_permission.assert_called_once_with( - UserID('y'), WorkspaceAccessType.WRITE, workspace_id=6) + UserID("y"), WorkspaceAccessType.WRITE, workspace_id=6 + ) - storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA('6/1/2'))) + storage.get_data_link.assert_called_once_with(duid=DataUnitID(UPA("6/1/2"))) storage.get_sample_acls.assert_called_once_with(sid) - storage.expire_data_link.assert_called_once_with(dt(6), UserID('y'), id_=lid) + storage.expire_data_link.assert_called_once_with(dt(6), UserID("y"), id_=lid) def _expire_data_link_fail(samples, user, duid, expected): @@ -2119,23 +2964,26 @@ def test_get_data_link_admin(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - sid = UUID('1234567890abcdef1234567890abcdee') + sid = UUID("1234567890abcdef1234567890abcdee") storage.get_data_link.return_value = DataLink( - UUID('1234567890abcdef1234567890abcde1'), - DataUnitID(UPA('6/1/2')), - SampleNodeAddress(SampleAddress(sid, 3), 'node'), + UUID("1234567890abcdef1234567890abcde1"), + DataUnitID(UPA("6/1/2")), + SampleNodeAddress(SampleAddress(sid, 3), "node"), dt(34), - UserID('userc') + UserID("userc"), ) - assert s.get_data_link_admin(UUID('1234567890abcdef1234567890abcde1')) == DataLink( - UUID('1234567890abcdef1234567890abcde1'), - DataUnitID(UPA('6/1/2')), - SampleNodeAddress(SampleAddress(sid, 3), 'node'), + assert s.get_data_link_admin(UUID("1234567890abcdef1234567890abcde1")) == DataLink( + UUID("1234567890abcdef1234567890abcde1"), + DataUnitID(UPA("6/1/2")), + SampleNodeAddress(SampleAddress(sid, 3), "node"), dt(34), - UserID('userc')) + UserID("userc"), + ) - storage.get_data_link.assert_called_once_with(UUID('1234567890abcdef1234567890abcde1')) + storage.get_data_link.assert_called_once_with( + UUID("1234567890abcdef1234567890abcde1") + ) def test_get_data_link_fail_bad_args(): @@ -2145,7 +2993,9 @@ def test_get_data_link_fail_bad_args(): ws = create_autospec(WS, spec_set=True, instance=True) s = Samples(storage, lu, meta, ws, now=nw) - _get_data_link_fail(s, None, ValueError('link_id cannot be a value that evaluates to false')) + _get_data_link_fail( + s, None, ValueError("link_id cannot be a value that evaluates to false") + ) def _get_data_link_fail(samples, linkid, expected): diff --git a/test/core/storage/arango_sample_storage_scrubber_test.py b/test/core/storage/arango_sample_storage_scrubber_test.py index c2f75a9b..ab09fa56 100644 --- a/test/core/storage/arango_sample_storage_scrubber_test.py +++ b/test/core/storage/arango_sample_storage_scrubber_test.py @@ -17,9 +17,15 @@ SourceMetadata, ) from SampleService.core.errors import ( - MissingParameterError, NoSuchSampleError, ConcurrencyError, UnauthorizedError, - NoSuchSampleVersionError, DataLinkExistsError, TooManyDataLinksError, NoSuchLinkError, - NoSuchSampleNodeError + MissingParameterError, + NoSuchSampleError, + ConcurrencyError, + UnauthorizedError, + NoSuchSampleVersionError, + DataLinkExistsError, + TooManyDataLinksError, + NoSuchLinkError, + NoSuchSampleNodeError, ) from SampleService.core.storage.arango_sample_storage import ArangoSampleStorage from SampleService.core.storage.errors import SampleStorageError, StorageInitError @@ -27,38 +33,40 @@ from SampleService.core.user import UserID from SampleService.core.workspace import UPA, DataUnitID -TEST_NODE = SampleNode('foo') +TEST_NODE = SampleNode("foo") -TEST_DB_NAME = 'test_sample_service' -TEST_COL_SAMPLE = 'samples' -TEST_COL_VERSION = 'samples_version' -TEST_COL_VER_EDGE = 'ver_to_sample' -TEST_COL_NODES = 'samples_nodes' -TEST_COL_NODE_EDGE = 'node_edges' -TEST_COL_WS_OBJ_VER = 'ws_obj_ver' -TEST_COL_DATA_LINK = 'samples_data_link' -TEST_COL_SCHEMA = 'schema' -TEST_USER = 'user1' -TEST_PWD = 'password1' +TEST_DB_NAME = "test_sample_service" +TEST_COL_SAMPLE = "samples" +TEST_COL_VERSION = "samples_version" +TEST_COL_VER_EDGE = "ver_to_sample" +TEST_COL_NODES = "samples_nodes" +TEST_COL_NODE_EDGE = "node_edges" +TEST_COL_WS_OBJ_VER = "ws_obj_ver" +TEST_COL_DATA_LINK = "samples_data_link" +TEST_COL_SCHEMA = "schema" +TEST_USER = "user1" +TEST_PWD = "password1" -@fixture(scope='module') +@fixture(scope="module") def arango(): arangoexe = test_utils.get_arango_exe() arangojs = test_utils.get_arango_js() tempdir = test_utils.get_temp_dir() arango = ArangoController(arangoexe, arangojs, tempdir) create_test_db(arango) - print('running arango on port {} in dir {}'.format(arango.port, arango.temp_dir)) + print("running arango on port {} in dir {}".format(arango.port, arango.temp_dir)) yield arango del_temp = test_utils.get_delete_temp_files() - print('shutting down arango, delete_temp_files={}'.format(del_temp)) + print("shutting down arango, delete_temp_files={}".format(del_temp)) arango.destroy(del_temp) def create_test_db(arango): systemdb = arango.client.db(verify=True) # default access to _system db - systemdb.create_database(TEST_DB_NAME, [{'username': TEST_USER, 'password': TEST_PWD}]) + systemdb.create_database( + TEST_DB_NAME, [{"username": TEST_USER, "password": TEST_PWD}] + ) return arango.client.db(TEST_DB_NAME, TEST_USER, TEST_PWD) @@ -92,59 +100,80 @@ def samplestorage_method(arango): TEST_COL_NODE_EDGE, TEST_COL_WS_OBJ_VER, TEST_COL_DATA_LINK, - TEST_COL_SCHEMA) + TEST_COL_SCHEMA, + ) + def dt(timestamp): return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + def _create_and_expire_data_link(samplestorage, link, expired, user): samplestorage.create_data_link(link) samplestorage.expire_data_link(expired, user, link.id) + def test_timestamp_seconds_to_milliseconds(samplestorage): - ts1=1614958000000 # milliseconds - ts2=1614958000 # seconds - ts3=1614958 # seconds - ts4=9007199254740.991 # seconds - - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdee') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(ts3), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(id1, UserID('user'), [SampleNode('mynode1')], dt(ts2), 'foo')) == 2 - assert samplestorage.save_sample( - SavedSample(id2, UserID('user'), [SampleNode('mynode2')], dt(ts3), 'foo')) is True - - lid1=uuid.UUID('1234567890abcdef1234567890abcde2') - lid2=uuid.UUID('1234567890abcdef1234567890abcde3') - lid3=uuid.UUID('1234567890abcdef1234567890abcde4') - samplestorage.create_data_link(DataLink( - lid1, - DataUnitID(UPA('42/42/42'), 'dataunit1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(ts2), - UserID('user')) + ts1 = 1614958000000 # milliseconds + ts2 = 1614958000 # seconds + ts3 = 1614958 # seconds + ts4 = 9007199254740.991 # seconds + + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdee") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(ts3), "foo") + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(id1, UserID("user"), [SampleNode("mynode1")], dt(ts2), "foo") + ) + == 2 + ) + assert ( + samplestorage.save_sample( + SavedSample(id2, UserID("user"), [SampleNode("mynode2")], dt(ts3), "foo") + ) + is True ) - samplestorage.create_data_link(DataLink( - lid2, - DataUnitID(UPA('5/89/32'), 'dataunit2'), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode2'), - dt(ts3), - UserID('user')) + lid1 = uuid.UUID("1234567890abcdef1234567890abcde2") + lid2 = uuid.UUID("1234567890abcdef1234567890abcde3") + lid3 = uuid.UUID("1234567890abcdef1234567890abcde4") + samplestorage.create_data_link( + DataLink( + lid1, + DataUnitID(UPA("42/42/42"), "dataunit1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(ts2), + UserID("user"), + ) + ) + + samplestorage.create_data_link( + DataLink( + lid2, + DataUnitID(UPA("5/89/32"), "dataunit2"), + SampleNodeAddress(SampleAddress(id2, 1), "mynode2"), + dt(ts3), + UserID("user"), + ) ) _create_and_expire_data_link( samplestorage, DataLink( lid3, - DataUnitID(UPA('5/89/33'), 'dataunit1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("5/89/33"), "dataunit1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(ts3), - UserID('user')), - dt(ts3+100), - UserID('user') + UserID("user"), + ), + dt(ts3 + 100), + UserID("user"), ) assert samplestorage.get_sample(id1, 1).savetime == dt(ts3) @@ -153,10 +182,12 @@ def test_timestamp_seconds_to_milliseconds(samplestorage): assert samplestorage.get_data_link(lid1).created == dt(ts2) assert samplestorage.get_data_link(lid2).created == dt(ts3) assert samplestorage.get_data_link(lid3).created == dt(ts3) - assert samplestorage.get_data_link(lid3).expired == dt(ts3+100) + assert samplestorage.get_data_link(lid3).expired == dt(ts3 + 100) - threshold=1000000000000 # current timestamp in milliseconds is above 1600000000000 - query=""" + threshold = ( + 1000000000000 # current timestamp in milliseconds is above 1600000000000 + ) + query = """ FOR sample1 IN samples_nodes FILTER sample1.saved < @threshold UPDATE sample1 WITH { saved: ROUND(sample1.saved * 1000) } IN samples_nodes @@ -171,7 +202,7 @@ def test_timestamp_seconds_to_milliseconds(samplestorage): } IN samples_data_link """ - samplestorage._db.aql.execute(query, bind_vars={'threshold': threshold}) + samplestorage._db.aql.execute(query, bind_vars={"threshold": threshold}) assert samplestorage.get_sample(id1, 1).savetime == dt(ts2) assert samplestorage.get_sample(id1, 2).savetime == dt(ts2) @@ -179,9 +210,9 @@ def test_timestamp_seconds_to_milliseconds(samplestorage): assert samplestorage.get_data_link(lid1).created == dt(ts2) assert samplestorage.get_data_link(lid2).created == dt(ts2) assert samplestorage.get_data_link(lid3).created == dt(ts2) - assert samplestorage.get_data_link(lid3).expired == dt((ts3+100) * 1000) + assert samplestorage.get_data_link(lid3).expired == dt((ts3 + 100) * 1000) - samplestorage._db.aql.execute(query, bind_vars={'threshold': threshold}) + samplestorage._db.aql.execute(query, bind_vars={"threshold": threshold}) assert samplestorage.get_sample(id1, 1).savetime == dt(ts2) assert samplestorage.get_sample(id1, 2).savetime == dt(ts2) @@ -189,5 +220,4 @@ def test_timestamp_seconds_to_milliseconds(samplestorage): assert samplestorage.get_data_link(lid1).created == dt(ts2) assert samplestorage.get_data_link(lid2).created == dt(ts2) assert samplestorage.get_data_link(lid3).created == dt(ts2) - assert samplestorage.get_data_link(lid3).expired == dt((ts3+100) * 1000) - + assert samplestorage.get_data_link(lid3).expired == dt((ts3 + 100) * 1000) diff --git a/test/core/storage/arango_sample_storage_test.py b/test/core/storage/arango_sample_storage_test.py index c23f7968..ac6d0c99 100644 --- a/test/core/storage/arango_sample_storage_test.py +++ b/test/core/storage/arango_sample_storage_test.py @@ -17,9 +17,15 @@ SourceMetadata, ) from SampleService.core.errors import ( - MissingParameterError, NoSuchSampleError, ConcurrencyError, UnauthorizedError, - NoSuchSampleVersionError, DataLinkExistsError, TooManyDataLinksError, NoSuchLinkError, - NoSuchSampleNodeError + MissingParameterError, + NoSuchSampleError, + ConcurrencyError, + UnauthorizedError, + NoSuchSampleVersionError, + DataLinkExistsError, + TooManyDataLinksError, + NoSuchLinkError, + NoSuchSampleNodeError, ) from SampleService.core.storage.arango_sample_storage import ArangoSampleStorage from SampleService.core.storage.errors import SampleStorageError, StorageInitError @@ -27,38 +33,40 @@ from SampleService.core.user import UserID from SampleService.core.workspace import UPA, DataUnitID -TEST_NODE = SampleNode('foo') +TEST_NODE = SampleNode("foo") -TEST_DB_NAME = 'test_sample_service' -TEST_COL_SAMPLE = 'samples' -TEST_COL_VERSION = 'versions' -TEST_COL_VER_EDGE = 'ver_to_sample' -TEST_COL_NODES = 'nodes' -TEST_COL_NODE_EDGE = 'node_edges' -TEST_COL_WS_OBJ_VER = 'ws_obj_ver' -TEST_COL_DATA_LINK = 'data_link' -TEST_COL_SCHEMA = 'schema' -TEST_USER = 'user1' -TEST_PWD = 'password1' +TEST_DB_NAME = "test_sample_service" +TEST_COL_SAMPLE = "samples" +TEST_COL_VERSION = "versions" +TEST_COL_VER_EDGE = "ver_to_sample" +TEST_COL_NODES = "nodes" +TEST_COL_NODE_EDGE = "node_edges" +TEST_COL_WS_OBJ_VER = "ws_obj_ver" +TEST_COL_DATA_LINK = "data_link" +TEST_COL_SCHEMA = "schema" +TEST_USER = "user1" +TEST_PWD = "password1" -@fixture(scope='module') +@fixture(scope="module") def arango(): arangoexe = test_utils.get_arango_exe() arangojs = test_utils.get_arango_js() tempdir = test_utils.get_temp_dir() arango = ArangoController(arangoexe, arangojs, tempdir) create_test_db(arango) - print('running arango on port {} in dir {}'.format(arango.port, arango.temp_dir)) + print("running arango on port {} in dir {}".format(arango.port, arango.temp_dir)) yield arango del_temp = test_utils.get_delete_temp_files() - print('shutting down arango, delete_temp_files={}'.format(del_temp)) + print("shutting down arango, delete_temp_files={}".format(del_temp)) arango.destroy(del_temp) def create_test_db(arango): systemdb = arango.client.db(verify=True) # default access to _system db - systemdb.create_database(TEST_DB_NAME, [{'username': TEST_USER, 'password': TEST_PWD}]) + systemdb.create_database( + TEST_DB_NAME, [{"username": TEST_USER, "password": TEST_PWD}] + ) return arango.client.db(TEST_DB_NAME, TEST_USER, TEST_PWD) @@ -92,7 +100,8 @@ def samplestorage_method(arango): TEST_COL_NODE_EDGE, TEST_COL_WS_OBJ_VER, TEST_COL_DATA_LINK, - TEST_COL_SCHEMA) + TEST_COL_SCHEMA, + ) def nw(): @@ -103,9 +112,9 @@ def test_startup_and_check_config_doc(samplestorage): # this is very naughty assert samplestorage._col_schema.count() == 1 cfgdoc = samplestorage._col_schema.find({}).next() - assert cfgdoc['_key'] == 'schema' - assert cfgdoc['schemaver'] == 1 - assert cfgdoc['inupdate'] is False + assert cfgdoc["_key"] == "schema" + assert cfgdoc["schemaver"] == 1 + assert cfgdoc["inupdate"] is False # check startup works with cfg object in place # this is also very naughty @@ -118,20 +127,27 @@ def test_startup_and_check_config_doc(samplestorage): samplestorage._col_node_edge.name, samplestorage._col_ws.name, samplestorage._col_data_link.name, - samplestorage._col_schema.name) + samplestorage._col_schema.name, + ) - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - n = SampleNode('rootyroot') - assert ss.save_sample(SavedSample(id_, UserID('u'), [n], dt(1), 'foo')) is True - assert ss.get_sample(id_) == SavedSample(id_, UserID('u'), [n], dt(1), 'foo', version=1) + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + n = SampleNode("rootyroot") + assert ss.save_sample(SavedSample(id_, UserID("u"), [n], dt(1), "foo")) is True + assert ss.get_sample(id_) == SavedSample( + id_, UserID("u"), [n], dt(1), "foo", version=1 + ) def test_startup_with_extra_config_doc(arango): db = clear_db_and_recreate(arango) - scol = db.collection('schema') - scol.insert_many([{'_key': 'schema', 'schemaver': 1, 'inupdate': False}, - {'schema': 'schema', 'schemaver': 2, 'inupdate': False}]) + scol = db.collection("schema") + scol.insert_many( + [ + {"_key": "schema", "schemaver": 1, "inupdate": False}, + {"schema": "schema", "schemaver": 2, "inupdate": False}, + ] + ) s = TEST_COL_SAMPLE v = TEST_COL_VERSION @@ -142,15 +158,28 @@ def test_startup_with_extra_config_doc(arango): dl = TEST_COL_DATA_LINK sc = TEST_COL_SCHEMA - _fail_startup(db, s, v, ve, n, ne, ws, dl, sc, nw, StorageInitError( - 'Multiple config objects found ' + - 'in the database. This should not happen, something is very wrong.')) + _fail_startup( + db, + s, + v, + ve, + n, + ne, + ws, + dl, + sc, + nw, + StorageInitError( + "Multiple config objects found " + + "in the database. This should not happen, something is very wrong." + ), + ) def test_startup_with_bad_schema_version(arango): db = clear_db_and_recreate(arango) col = db.collection(TEST_COL_SCHEMA) - col.insert({'_key': 'schema', 'schemaver': 4, 'inupdate': False}) + col.insert({"_key": "schema", "schemaver": 4, "inupdate": False}) s = TEST_COL_SAMPLE v = TEST_COL_VERSION @@ -161,14 +190,25 @@ def test_startup_with_bad_schema_version(arango): dl = TEST_COL_DATA_LINK sc = TEST_COL_SCHEMA - _fail_startup(db, s, v, ve, n, ne, ws, dl, sc, nw, StorageInitError( - 'Incompatible database schema. Server is v1, DB is v4')) + _fail_startup( + db, + s, + v, + ve, + n, + ne, + ws, + dl, + sc, + nw, + StorageInitError("Incompatible database schema. Server is v1, DB is v4"), + ) def test_startup_in_update(arango): db = clear_db_and_recreate(arango) col = db.collection(TEST_COL_SCHEMA) - col.insert({'_key': 'schema', 'schemaver': 1, 'inupdate': True}) + col.insert({"_key": "schema", "schemaver": 1, "inupdate": True}) s = TEST_COL_SAMPLE v = TEST_COL_VERSION @@ -179,27 +219,44 @@ def test_startup_in_update(arango): dl = TEST_COL_DATA_LINK sc = TEST_COL_SCHEMA - _fail_startup(db, s, v, ve, n, ne, ws, dl, sc, nw, StorageInitError( - 'The database is in the middle of an update from v1 of the schema. Aborting startup.')) + _fail_startup( + db, + s, + v, + ve, + n, + ne, + ws, + dl, + sc, + nw, + StorageInitError( + "The database is in the middle of an update from v1 of the schema. Aborting startup." + ), + ) def test_startup_with_unupdated_version_and_node_docs(samplestorage): # this test simulates a server coming up after a dirty shutdown, where version and # node doc integer versions have not been updated - n1 = SampleNode('root') - n2 = SampleNode('kid1', SubSampleType.TECHNICAL_REPLICATE, 'root') - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1') - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root') + n1 = SampleNode("root") + n2 = SampleNode("kid1", SubSampleType.TECHNICAL_REPLICATE, "root") + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1") + n4 = SampleNode("kid3", SubSampleType.TECHNICAL_REPLICATE, "root") - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [n1, n2, n3, n4], dt(1), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [n1, n2, n3, n4], dt(1), "foo") + ) + is True + ) # this is very naughty # checked that these modifications actually work by viewing the db contents - samplestorage._col_version.update_match({}, {'ver': -1}) - samplestorage._col_nodes.update_match({'name': 'kid2'}, {'ver': -1}) + samplestorage._col_version.update_match({}, {"ver": -1}) + samplestorage._col_nodes.update_match({"name": "kid2"}, {"ver": -1}) # this is also very naughty ArangoSampleStorage( @@ -211,7 +268,8 @@ def test_startup_with_unupdated_version_and_node_docs(samplestorage): samplestorage._col_node_edge.name, samplestorage._col_ws.name, samplestorage._col_data_link.name, - samplestorage._col_schema.name) + samplestorage._col_schema.name, + ) assert samplestorage._col_version.count() == 1 assert samplestorage._col_ver_edge.count() == 1 @@ -219,10 +277,10 @@ def test_startup_with_unupdated_version_and_node_docs(samplestorage): assert samplestorage._col_node_edge.count() == 4 for v in samplestorage._col_version.all(): - assert v['ver'] == 1 + assert v["ver"] == 1 for v in samplestorage._col_nodes.all(): - assert v['ver'] == 1 + assert v["ver"] == 1 def test_startup_with_unupdated_node_docs(samplestorage): @@ -230,25 +288,35 @@ def test_startup_with_unupdated_node_docs(samplestorage): # node doc integer versions have not been updated # version doc cannot be modified such that ver = -1 or the version check will also correct the # node docs, negating the point of this test - n1 = SampleNode('root') - n2 = SampleNode('kid1', SubSampleType.TECHNICAL_REPLICATE, 'root') - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1') - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root') + n1 = SampleNode("root") + n2 = SampleNode("kid1", SubSampleType.TECHNICAL_REPLICATE, "root") + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1") + n4 = SampleNode("kid3", SubSampleType.TECHNICAL_REPLICATE, "root") - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") - assert samplestorage.save_sample( - SavedSample(id_, UserID('u'), [n1, n2, n3, n4], dt(1), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("u"), [n1, n2, n3, n4], dt(1), "foo") + ) + is True + ) - assert samplestorage.save_sample_version( - SavedSample(id_, UserID('u'), [n1, n2, n3, n4], dt(1), 'bar')) == 2 + assert ( + samplestorage.save_sample_version( + SavedSample(id_, UserID("u"), [n1, n2, n3, n4], dt(1), "bar") + ) + == 2 + ) # this is very naughty sample = samplestorage._col_sample.find({}).next() - uuidver2 = sample['vers'][1] + uuidver2 = sample["vers"][1] # checked that these modifications actually work by viewing the db contents - samplestorage._col_nodes.update_match({'uuidver': uuidver2, 'name': 'kid2'}, {'ver': -1}) + samplestorage._col_nodes.update_match( + {"uuidver": uuidver2, "name": "kid2"}, {"ver": -1} + ) # this is also very naughty ArangoSampleStorage( @@ -260,7 +328,8 @@ def test_startup_with_unupdated_node_docs(samplestorage): samplestorage._col_node_edge.name, samplestorage._col_ws.name, samplestorage._col_data_link.name, - samplestorage._col_schema.name) + samplestorage._col_schema.name, + ) assert samplestorage._col_version.count() == 2 assert samplestorage._col_ver_edge.count() == 2 @@ -268,29 +337,37 @@ def test_startup_with_unupdated_node_docs(samplestorage): assert samplestorage._col_node_edge.count() == 8 for v in samplestorage._col_version.all(): - assert v['ver'] == 2 if v['uuidver'] == uuidver2 else 1 + assert v["ver"] == 2 if v["uuidver"] == uuidver2 else 1 for v in samplestorage._col_nodes.all(): - assert v['ver'] == 2 if v['uuidver'] == uuidver2 else 1 + assert v["ver"] == 2 if v["uuidver"] == uuidver2 else 1 def test_startup_with_no_sample_doc(samplestorage): # this test simulates a server coming up after a dirty shutdown, where version and # node docs were saved but the sample document was not while saving the first version of # a sample - n1 = SampleNode('root') - n2 = SampleNode('kid1', SubSampleType.TECHNICAL_REPLICATE, 'root') - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1') - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root') + n1 = SampleNode("root") + n2 = SampleNode("kid1", SubSampleType.TECHNICAL_REPLICATE, "root") + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1") + n4 = SampleNode("kid3", SubSampleType.TECHNICAL_REPLICATE, "root") - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdea') + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdea") - assert samplestorage.save_sample( - SavedSample(id1, UserID('u'), [n1, n2, n3, n4], dt(1), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("u"), [n1, n2, n3, n4], dt(1), "foo") + ) + is True + ) - assert samplestorage.save_sample( - SavedSample(id2, UserID('u'), [n1, n2, n3, n4], dt(1000), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id2, UserID("u"), [n1, n2, n3, n4], dt(1000), "foo") + ) + is True + ) # this is very naughty assert samplestorage._col_version.count() == 2 @@ -298,11 +375,11 @@ def test_startup_with_no_sample_doc(samplestorage): assert samplestorage._col_nodes.count() == 8 assert samplestorage._col_node_edge.count() == 8 - samplestorage._col_sample.delete({'_key': str(id2)}) + samplestorage._col_sample.delete({"_key": str(id2)}) # if the sample document hasn't been saved, then none of the integer versions for the # sample can have been updated to 1 - samplestorage._col_version.update_match({'id': str(id2)}, {'ver': -1}) - samplestorage._col_nodes.update_match({'id': str(id2)}, {'ver': -1}) + samplestorage._col_version.update_match({"id": str(id2)}, {"ver": -1}) + samplestorage._col_nodes.update_match({"id": str(id2)}, {"ver": -1}) # first test that bringing up the server before the 1hr deletion time limit doesn't change the # db: @@ -317,7 +394,8 @@ def test_startup_with_no_sample_doc(samplestorage): samplestorage._col_ws.name, samplestorage._col_data_link.name, samplestorage._col_schema.name, - now=lambda: datetime.datetime.fromtimestamp(4600, tz=datetime.timezone.utc)) + now=lambda: datetime.datetime.fromtimestamp(4600, tz=datetime.timezone.utc), + ) assert samplestorage._col_version.count() == 2 assert samplestorage._col_ver_edge.count() == 2 @@ -335,7 +413,8 @@ def test_startup_with_no_sample_doc(samplestorage): samplestorage._col_ws.name, samplestorage._col_data_link.name, samplestorage._col_schema.name, - now=lambda: datetime.datetime.fromtimestamp(4601, tz=datetime.timezone.utc)) + now=lambda: datetime.datetime.fromtimestamp(4601, tz=datetime.timezone.utc), + ) assert samplestorage._col_sample.count() == 1 assert samplestorage._col_version.count() == 1 @@ -344,31 +423,39 @@ def test_startup_with_no_sample_doc(samplestorage): assert samplestorage._col_node_edge.count() == 4 sample = samplestorage._col_sample.find({}).next() - assert sample['id'] == str(id1) - uuidver = sample['vers'][0] + assert sample["id"] == str(id1) + uuidver = sample["vers"][0] - assert len(list(samplestorage._col_version.find({'uuidver': uuidver}))) == 1 - assert len(list(samplestorage._col_ver_edge.find({'uuidver': uuidver}))) == 1 - assert len(list(samplestorage._col_nodes.find({'uuidver': uuidver}))) == 4 - assert len(list(samplestorage._col_node_edge.find({'uuidver': uuidver}))) == 4 + assert len(list(samplestorage._col_version.find({"uuidver": uuidver}))) == 1 + assert len(list(samplestorage._col_ver_edge.find({"uuidver": uuidver}))) == 1 + assert len(list(samplestorage._col_nodes.find({"uuidver": uuidver}))) == 4 + assert len(list(samplestorage._col_node_edge.find({"uuidver": uuidver}))) == 4 def test_startup_with_no_version_in_sample_doc(samplestorage): # this test simulates a server coming up after a dirty shutdown, where version and # node docs were saved but the sample document was not updated while saving the second # version of # a sample - n1 = SampleNode('root') - n2 = SampleNode('kid1', SubSampleType.TECHNICAL_REPLICATE, 'root') - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1') - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root') + n1 = SampleNode("root") + n2 = SampleNode("kid1", SubSampleType.TECHNICAL_REPLICATE, "root") + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1") + n4 = SampleNode("kid3", SubSampleType.TECHNICAL_REPLICATE, "root") - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") - assert samplestorage.save_sample( - SavedSample(id1, UserID('u'), [n1, n2, n3, n4], dt(1), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("u"), [n1, n2, n3, n4], dt(1), "foo") + ) + is True + ) - assert samplestorage.save_sample_version( - SavedSample(id1, UserID('u'), [n1, n2, n3, n4], dt(2000), 'foo')) == 2 + assert ( + samplestorage.save_sample_version( + SavedSample(id1, UserID("u"), [n1, n2, n3, n4], dt(2000), "foo") + ) + == 2 + ) # this is very naughty assert samplestorage._col_sample.count() == 1 @@ -378,13 +465,13 @@ def test_startup_with_no_version_in_sample_doc(samplestorage): assert samplestorage._col_node_edge.count() == 8 sample = samplestorage._col_sample.find({}).next() - samplestorage._col_sample.update_match({}, {'vers': sample['vers'][:1]}) - uuidver2 = sample['vers'][1] + samplestorage._col_sample.update_match({}, {"vers": sample["vers"][:1]}) + uuidver2 = sample["vers"][1] # if the sample document hasn't been updated, then none of the integer versions for the # sample can have been updated to 1 - samplestorage._col_version.update_match({'uuidver': uuidver2}, {'ver': -1}) - samplestorage._col_nodes.update_match({'uuidver': uuidver2}, {'ver': -1}) + samplestorage._col_version.update_match({"uuidver": uuidver2}, {"ver": -1}) + samplestorage._col_nodes.update_match({"uuidver": uuidver2}, {"ver": -1}) # first test that bringing up the server before the 1hr deletion time limit doesn't change the # db: @@ -399,7 +486,8 @@ def test_startup_with_no_version_in_sample_doc(samplestorage): samplestorage._col_ws.name, samplestorage._col_data_link.name, samplestorage._col_schema.name, - now=lambda: datetime.datetime.fromtimestamp(5600, tz=datetime.timezone.utc)) + now=lambda: datetime.datetime.fromtimestamp(5600, tz=datetime.timezone.utc), + ) assert samplestorage._col_version.count() == 2 assert samplestorage._col_ver_edge.count() == 2 @@ -417,19 +505,20 @@ def test_startup_with_no_version_in_sample_doc(samplestorage): samplestorage._col_ws.name, samplestorage._col_data_link.name, samplestorage._col_schema.name, - now=lambda: datetime.datetime.fromtimestamp(5601, tz=datetime.timezone.utc)) + now=lambda: datetime.datetime.fromtimestamp(5601, tz=datetime.timezone.utc), + ) assert samplestorage._col_version.count() == 1 assert samplestorage._col_ver_edge.count() == 1 assert samplestorage._col_nodes.count() == 4 assert samplestorage._col_node_edge.count() == 4 - uuidver1 = sample['vers'][0] + uuidver1 = sample["vers"][0] - assert len(list(samplestorage._col_version.find({'uuidver': uuidver1}))) == 1 - assert len(list(samplestorage._col_ver_edge.find({'uuidver': uuidver1}))) == 1 - assert len(list(samplestorage._col_nodes.find({'uuidver': uuidver1}))) == 4 - assert len(list(samplestorage._col_node_edge.find({'uuidver': uuidver1}))) == 4 + assert len(list(samplestorage._col_version.find({"uuidver": uuidver1}))) == 1 + assert len(list(samplestorage._col_ver_edge.find({"uuidver": uuidver1}))) == 1 + assert len(list(samplestorage._col_nodes.find({"uuidver": uuidver1}))) == 4 + assert len(list(samplestorage._col_node_edge.find({"uuidver": uuidver1}))) == 4 def test_fail_startup_bad_args(arango): @@ -448,29 +537,112 @@ def test_fail_startup_bad_args(arango): def nw(): datetime.datetime.fromtimestamp(1, tz=datetime.timezone.utc) - _fail_startup(None, s, v, ve, n, ne, ws, dl, sc, nw, - ValueError('db cannot be a value that evaluates to false')) - _fail_startup(db, '', v, ve, n, ne, ws, dl, sc, nw, MissingParameterError('sample_collection')) - _fail_startup(db, s, '', ve, n, ne, ws, dl, sc, nw, MissingParameterError( - 'version_collection')) - _fail_startup(db, s, v, '', n, ne, ws, dl, sc, nw, MissingParameterError( - 'version_edge_collection')) - _fail_startup(db, s, v, ve, '', ne, ws, dl, sc, nw, MissingParameterError('node_collection')) - _fail_startup(db, s, v, ve, n, '', ws, dl, sc, nw, MissingParameterError( - 'node_edge_collection')) - _fail_startup(db, s, v, ve, n, ne, '', dl, sc, nw, MissingParameterError( - 'workspace_object_version_shadow_collection')) - _fail_startup(db, s, v, ve, n, ne, ws, '', sc, nw, MissingParameterError( - 'data_link_collection')) - _fail_startup(db, s, v, ve, n, ne, ws, dl, '', nw, MissingParameterError('schema_collection')) - _fail_startup(db, s, v, ve, n, ne, ws, dl, sc, None, - ValueError('now cannot be a value that evaluates to false')) + _fail_startup( + None, + s, + v, + ve, + n, + ne, + ws, + dl, + sc, + nw, + ValueError("db cannot be a value that evaluates to false"), + ) + _fail_startup( + db, "", v, ve, n, ne, ws, dl, sc, nw, MissingParameterError("sample_collection") + ) + _fail_startup( + db, + s, + "", + ve, + n, + ne, + ws, + dl, + sc, + nw, + MissingParameterError("version_collection"), + ) + _fail_startup( + db, + s, + v, + "", + n, + ne, + ws, + dl, + sc, + nw, + MissingParameterError("version_edge_collection"), + ) + _fail_startup( + db, s, v, ve, "", ne, ws, dl, sc, nw, MissingParameterError("node_collection") + ) + _fail_startup( + db, + s, + v, + ve, + n, + "", + ws, + dl, + sc, + nw, + MissingParameterError("node_edge_collection"), + ) + _fail_startup( + db, + s, + v, + ve, + n, + ne, + "", + dl, + sc, + nw, + MissingParameterError("workspace_object_version_shadow_collection"), + ) + _fail_startup( + db, + s, + v, + ve, + n, + ne, + ws, + "", + sc, + nw, + MissingParameterError("data_link_collection"), + ) + _fail_startup( + db, s, v, ve, n, ne, ws, dl, "", nw, MissingParameterError("schema_collection") + ) + _fail_startup( + db, + s, + v, + ve, + n, + ne, + ws, + dl, + sc, + None, + ValueError("now cannot be a value that evaluates to false"), + ) def test_fail_startup_incorrect_collection_type(arango): samplestorage_method(arango) db = arango.client.db(TEST_DB_NAME, TEST_USER, TEST_PWD) - db.create_collection('sampleedge', edge=True) + db.create_collection("sampleedge", edge=True) s = TEST_COL_SAMPLE v = TEST_COL_VERSION @@ -484,36 +656,127 @@ def test_fail_startup_incorrect_collection_type(arango): def nw(): datetime.datetime.fromtimestamp(1, tz=datetime.timezone.utc) - _fail_startup(db, 'sampleedge', v, ve, n, ne, ws, dl, sc, nw, StorageInitError( - 'sample collection sampleedge is not a vertex collection')) - _fail_startup(db, s, ve, ve, n, ne, ws, dl, sc, nw, StorageInitError( - 'version collection ver_to_sample is not a vertex collection')) - _fail_startup(db, s, v, v, n, ne, ws, dl, sc, nw, StorageInitError( - 'version edge collection versions is not an edge collection')) - _fail_startup(db, s, v, ve, ne, ne, ws, dl, sc, nw, StorageInitError( - 'node collection node_edges is not a vertex collection')) - _fail_startup(db, s, v, ve, n, n, ws, dl, sc, nw, StorageInitError( - 'node edge collection nodes is not an edge collection')) - _fail_startup(db, s, v, ve, n, ne, dl, dl, sc, nw, StorageInitError( - 'workspace object version shadow collection data_link is not a vertex collection')) - _fail_startup(db, s, v, ve, n, ne, ws, ws, sc, nw, StorageInitError( - 'data link collection ws_obj_ver is not an edge collection')) - _fail_startup(db, s, v, ve, n, ne, ws, dl, ne, nw, StorageInitError( - 'schema collection node_edges is not a vertex collection')) + _fail_startup( + db, + "sampleedge", + v, + ve, + n, + ne, + ws, + dl, + sc, + nw, + StorageInitError("sample collection sampleedge is not a vertex collection"), + ) + _fail_startup( + db, + s, + ve, + ve, + n, + ne, + ws, + dl, + sc, + nw, + StorageInitError("version collection ver_to_sample is not a vertex collection"), + ) + _fail_startup( + db, + s, + v, + v, + n, + ne, + ws, + dl, + sc, + nw, + StorageInitError("version edge collection versions is not an edge collection"), + ) + _fail_startup( + db, + s, + v, + ve, + ne, + ne, + ws, + dl, + sc, + nw, + StorageInitError("node collection node_edges is not a vertex collection"), + ) + _fail_startup( + db, + s, + v, + ve, + n, + n, + ws, + dl, + sc, + nw, + StorageInitError("node edge collection nodes is not an edge collection"), + ) + _fail_startup( + db, + s, + v, + ve, + n, + ne, + dl, + dl, + sc, + nw, + StorageInitError( + "workspace object version shadow collection data_link is not a vertex collection" + ), + ) + _fail_startup( + db, + s, + v, + ve, + n, + ne, + ws, + ws, + sc, + nw, + StorageInitError("data link collection ws_obj_ver is not an edge collection"), + ) + _fail_startup( + db, + s, + v, + ve, + n, + ne, + ws, + dl, + ne, + nw, + StorageInitError("schema collection node_edges is not a vertex collection"), + ) def _fail_startup( - db, - colsample, - colver, - colveredge, - colnode, - colnodeedge, - colws, - coldatalink, - colschema, - now, - expected): + db, + colsample, + colver, + colveredge, + colnode, + colnodeedge, + colws, + coldatalink, + colschema, + now, + expected, +): with raises(Exception) as got: ArangoSampleStorage( @@ -526,7 +789,8 @@ def _fail_startup( colws, coldatalink, colschema, - now=now) + now=now, + ) assert_exception_correct(got.value, expected) @@ -534,78 +798,84 @@ def test_indexes_created(samplestorage): # Shoudn't reach into the internals but only testing # Purpose here is to make tests fail if collections are added so devs are reminded to # set up any necessary indexes and add index tests - cols = sorted([x['name'] for x in samplestorage._db.collections() - if not x['name'].startswith('_')]) + cols = sorted( + [ + x["name"] + for x in samplestorage._db.collections() + if not x["name"].startswith("_") + ] + ) assert cols == [ - 'data_link', - 'node_edges', - 'nodes', - 'samples', - 'schema', - 'ver_to_sample', - 'versions', - 'ws_obj_ver'] + "data_link", + "node_edges", + "nodes", + "samples", + "schema", + "ver_to_sample", + "versions", + "ws_obj_ver", + ] indexes = samplestorage._col_sample.indexes() assert len(indexes) == 1 - assert indexes[0]['fields'] == ['_key'] + assert indexes[0]["fields"] == ["_key"] indexes = samplestorage._col_nodes.indexes() assert len(indexes) == 3 - assert indexes[0]['fields'] == ['_key'] - _check_index(indexes[1], ['uuidver']) - _check_index(indexes[2], ['ver']) + assert indexes[0]["fields"] == ["_key"] + _check_index(indexes[1], ["uuidver"]) + _check_index(indexes[2], ["ver"]) indexes = samplestorage._col_version.indexes() assert len(indexes) == 3 - assert indexes[0]['fields'] == ['_key'] - _check_index(indexes[1], ['uuidver']) - _check_index(indexes[2], ['ver']) + assert indexes[0]["fields"] == ["_key"] + _check_index(indexes[1], ["uuidver"]) + _check_index(indexes[2], ["ver"]) indexes = samplestorage._col_node_edge.indexes() assert len(indexes) == 3 - assert indexes[0]['fields'] == ['_key'] - assert indexes[1]['fields'] == ['_from', '_to'] - _check_index(indexes[2], ['uuidver']) + assert indexes[0]["fields"] == ["_key"] + assert indexes[1]["fields"] == ["_from", "_to"] + _check_index(indexes[2], ["uuidver"]) indexes = samplestorage._col_ver_edge.indexes() assert len(indexes) == 3 - assert indexes[0]['fields'] == ['_key'] - assert indexes[1]['fields'] == ['_from', '_to'] - _check_index(indexes[2], ['uuidver']) + assert indexes[0]["fields"] == ["_key"] + assert indexes[1]["fields"] == ["_from", "_to"] + _check_index(indexes[2], ["uuidver"]) # Don't add indexes here, Relation engine is responsible for setting up indexes # Sample service doesn't use the collection other than verifying it exists indexes = samplestorage._col_ws.indexes() assert len(indexes) == 1 - assert indexes[0]['fields'] == ['_key'] + assert indexes[0]["fields"] == ["_key"] indexes = samplestorage._col_data_link.indexes() assert len(indexes) == 6 - assert indexes[0]['fields'] == ['_key'] - assert indexes[1]['fields'] == ['_from', '_to'] - _check_index(indexes[2], ['id']) - _check_index(indexes[3], ['wsid', 'objid', 'objver']) - _check_index(indexes[4], ['samuuidver']) - _check_index(indexes[5], ['sampleid']) + assert indexes[0]["fields"] == ["_key"] + assert indexes[1]["fields"] == ["_from", "_to"] + _check_index(indexes[2], ["id"]) + _check_index(indexes[3], ["wsid", "objid", "objver"]) + _check_index(indexes[4], ["samuuidver"]) + _check_index(indexes[5], ["sampleid"]) indexes = samplestorage._col_schema.indexes() assert len(indexes) == 1 - assert indexes[0]['fields'] == ['_key'] + assert indexes[0]["fields"] == ["_key"] def _check_index(index, fields): - assert index['fields'] == fields - assert index['deduplicate'] is True - assert index['sparse'] is False - assert index['type'] == 'persistent' - assert index['unique'] is False + assert index["fields"] == fields + assert index["deduplicate"] is True + assert index["sparse"] is False + assert index["type"] == "persistent" + assert index["unique"] is False def test_start_consistency_checker_fail_bad_args(samplestorage): with raises(Exception) as got: samplestorage.start_consistency_checker(interval_sec=0) - assert_exception_correct(got.value, ValueError('interval_sec must be > 0')) + assert_exception_correct(got.value, ValueError("interval_sec must be > 0")) def test_consistency_checker_run(samplestorage): @@ -613,31 +883,48 @@ def test_consistency_checker_run(samplestorage): # The cleaning functionality is tested thoroughly above. # The db could be in an unclean state if a sample server does down mid save and doesn't # come back up. - n1 = SampleNode('root') - n2 = SampleNode('kid1', SubSampleType.TECHNICAL_REPLICATE, 'root') - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1') - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root') + n1 = SampleNode("root") + n2 = SampleNode("kid1", SubSampleType.TECHNICAL_REPLICATE, "root") + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1") + n4 = SampleNode("kid3", SubSampleType.TECHNICAL_REPLICATE, "root") - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") - assert samplestorage.save_sample( - SavedSample(id_, UserID('u'), [n1, n2, n3, n4], dt(1), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("u"), [n1, n2, n3, n4], dt(1), "foo") + ) + is True + ) - assert samplestorage.save_sample_version( - SavedSample(id_, UserID('u'), [n1, n2, n3, n4], dt(1), 'bar')) == 2 + assert ( + samplestorage.save_sample_version( + SavedSample(id_, UserID("u"), [n1, n2, n3, n4], dt(1), "bar") + ) + == 2 + ) # this is very naughty sample = samplestorage._col_sample.find({}).next() - uuidver2 = sample['vers'][1] + uuidver2 = sample["vers"][1] - samplestorage._col_nodes.update_match({'uuidver': uuidver2, 'name': 'kid2'}, {'ver': -1}) + samplestorage._col_nodes.update_match( + {"uuidver": uuidver2, "name": "kid2"}, {"ver": -1} + ) samplestorage.start_consistency_checker(interval_sec=1) - samplestorage.start_consistency_checker(interval_sec=1) # test that running twice does nothing + samplestorage.start_consistency_checker( + interval_sec=1 + ) # test that running twice does nothing time.sleep(0.5) - assert samplestorage._col_nodes.find({'uuidver': uuidver2, 'name': 'kid2'}).next()['ver'] == -1 + assert ( + samplestorage._col_nodes.find({"uuidver": uuidver2, "name": "kid2"}).next()[ + "ver" + ] + == -1 + ) time.sleep(1) @@ -647,25 +934,37 @@ def test_consistency_checker_run(samplestorage): assert samplestorage._col_node_edge.count() == 8 for v in samplestorage._col_version.all(): - assert v['ver'] == 2 if v['uuidver'] == uuidver2 else 1 + assert v["ver"] == 2 if v["uuidver"] == uuidver2 else 1 for v in samplestorage._col_nodes.all(): - assert v['ver'] == 2 if v['uuidver'] == uuidver2 else 1 + assert v["ver"] == 2 if v["uuidver"] == uuidver2 else 1 # test that pausing stops updating samplestorage.stop_consistency_checker() samplestorage.stop_consistency_checker() # test that running twice in a row does nothing - samplestorage._col_nodes.update_match({'uuidver': uuidver2, 'name': 'kid2'}, {'ver': -1}) + samplestorage._col_nodes.update_match( + {"uuidver": uuidver2, "name": "kid2"}, {"ver": -1} + ) time.sleep(1.5) - assert samplestorage._col_nodes.find({'uuidver': uuidver2, 'name': 'kid2'}).next()['ver'] == -1 + assert ( + samplestorage._col_nodes.find({"uuidver": uuidver2, "name": "kid2"}).next()[ + "ver" + ] + == -1 + ) samplestorage.start_consistency_checker(1) time.sleep(1.5) - assert samplestorage._col_nodes.find({'uuidver': uuidver2, 'name': 'kid2'}).next()['ver'] == 2 + assert ( + samplestorage._col_nodes.find({"uuidver": uuidver2, "name": "kid2"}).next()[ + "ver" + ] + == 2 + ) # leaving the checker running can occasionally interfere with other tests, deleting documents # that are in the middle of the save process. Stop the checker and wait until the job must've @@ -679,112 +978,170 @@ def dt(timestamp): def test_save_and_get_sample(samplestorage): - n1 = SampleNode('root') + n1 = SampleNode("root") n2 = SampleNode( - 'kid1', SubSampleType.TECHNICAL_REPLICATE, 'root', - {'a': {'b': 'c', 'd': 'e'}, 'f': {'g': 'h'}}, - {'m': {'n': 'o'}}, - [SourceMetadata('a', 'sk', {'a': 'b'}), SourceMetadata('f', 'sk', {'c': 'd'})]) - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1', {'a': {'b': 'c'}}) - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root', - user_metadata={'f': {'g': 'h'}}) + "kid1", + SubSampleType.TECHNICAL_REPLICATE, + "root", + {"a": {"b": "c", "d": "e"}, "f": {"g": "h"}}, + {"m": {"n": "o"}}, + [SourceMetadata("a", "sk", {"a": "b"}), SourceMetadata("f", "sk", {"c": "d"})], + ) + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1", {"a": {"b": "c"}}) + n4 = SampleNode( + "kid3", + SubSampleType.TECHNICAL_REPLICATE, + "root", + user_metadata={"f": {"g": "h"}}, + ) - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") - assert samplestorage.save_sample( - SavedSample(id_, UserID('auser'), [n1, n2, n3, n4], dt(8), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("auser"), [n1, n2, n3, n4], dt(8), "foo") + ) + is True + ) assert samplestorage.get_sample(id_) == SavedSample( - id_, UserID('auser'), [n1, n2, n3, n4], dt(8), 'foo', 1) + id_, UserID("auser"), [n1, n2, n3, n4], dt(8), "foo", 1 + ) assert samplestorage.get_sample_acls(id_) == SampleACL( - UserID('auser'), dt(8), public_read=False) + UserID("auser"), dt(8), public_read=False + ) + def test_save_and_get_samples(samplestorage): - n1 = SampleNode('root') + n1 = SampleNode("root") n2 = SampleNode( - 'kid1', SubSampleType.TECHNICAL_REPLICATE, 'root', - {'a': {'b': 'c', 'd': 'e'}, 'f': {'g': 'h'}}, - {'m': {'n': 'o'}}, - [SourceMetadata('a', 'sk', {'a': 'b'}), SourceMetadata('f', 'sk', {'c': 'd'})]) - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1', {'a': {'b': 'c'}}) - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root', - user_metadata={'f': {'g': 'h'}}) - - id1_ = uuid.UUID('1234567890abcdef1234567890fbcdef') - id2_ = uuid.UUID('1234567890abcdef1234567890fbcdea') - id3_ = uuid.UUID('1234567890abcdef1234567890fbcdeb') + "kid1", + SubSampleType.TECHNICAL_REPLICATE, + "root", + {"a": {"b": "c", "d": "e"}, "f": {"g": "h"}}, + {"m": {"n": "o"}}, + [SourceMetadata("a", "sk", {"a": "b"}), SourceMetadata("f", "sk", {"c": "d"})], + ) + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1", {"a": {"b": "c"}}) + n4 = SampleNode( + "kid3", + SubSampleType.TECHNICAL_REPLICATE, + "root", + user_metadata={"f": {"g": "h"}}, + ) + + id1_ = uuid.UUID("1234567890abcdef1234567890fbcdef") + id2_ = uuid.UUID("1234567890abcdef1234567890fbcdea") + id3_ = uuid.UUID("1234567890abcdef1234567890fbcdeb") # save three separate samples - assert samplestorage.save_sample( - SavedSample(id1_, UserID('auser'), [n1, n2, n3, n4], dt(8), 'foo')) is True - assert samplestorage.save_sample( - SavedSample(id2_, UserID('auser'), [n1, n2, n3], dt(8), 'bar')) is True - assert samplestorage.save_sample( - SavedSample(id3_, UserID('auser'), [n1, n2, n4], dt(8), 'baz')) is True - - assert samplestorage.get_samples([ - {"id": id1_, "version": 1}, - {"id": id2_, "version": 1}, - {"id": id3_, "version": 1} - ]) == [ - SavedSample(id1_, UserID('auser'), [n1, n2, n3, n4], dt(8), 'foo', 1), - SavedSample(id2_, UserID('auser'), [n1, n2, n3], dt(8), 'bar', 1), - SavedSample(id3_, UserID('auser'), [n1, n2, n4], dt(8), 'baz', 1) + assert ( + samplestorage.save_sample( + SavedSample(id1_, UserID("auser"), [n1, n2, n3, n4], dt(8), "foo") + ) + is True + ) + assert ( + samplestorage.save_sample( + SavedSample(id2_, UserID("auser"), [n1, n2, n3], dt(8), "bar") + ) + is True + ) + assert ( + samplestorage.save_sample( + SavedSample(id3_, UserID("auser"), [n1, n2, n4], dt(8), "baz") + ) + is True + ) + + assert samplestorage.get_samples( + [ + {"id": id1_, "version": 1}, + {"id": id2_, "version": 1}, + {"id": id3_, "version": 1}, + ] + ) == [ + SavedSample(id1_, UserID("auser"), [n1, n2, n3, n4], dt(8), "foo", 1), + SavedSample(id2_, UserID("auser"), [n1, n2, n3], dt(8), "bar", 1), + SavedSample(id3_, UserID("auser"), [n1, n2, n4], dt(8), "baz", 1), ] + def test_save_sample_fail_bad_input(samplestorage): with raises(Exception) as got: samplestorage.save_sample(None) assert_exception_correct( - got.value, ValueError('sample cannot be a value that evaluates to false')) + got.value, ValueError("sample cannot be a value that evaluates to false") + ) def test_save_sample_fail_duplicate(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user1'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user1"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - assert samplestorage.save_sample( - SavedSample(id_, UserID('user1'), [TEST_NODE], dt(1), 'bar')) is False + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user1"), [TEST_NODE], dt(1), "bar") + ) + is False + ) def test_save_sample_fail_duplicate_race_condition(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) # this is a very bad and naughty thing to do - assert samplestorage._save_sample_pt2( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'bar')) is False + assert ( + samplestorage._save_sample_pt2( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "bar") + ) + is False + ) def test_get_sample_with_non_updated_version_doc(samplestorage): # simulates the case where a save failed part way through. The version UUID was added to the # sample doc but the node and version doc updates were not completed - n1 = SampleNode('root') - n2 = SampleNode('kid1', SubSampleType.TECHNICAL_REPLICATE, 'root') - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1') - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root') + n1 = SampleNode("root") + n2 = SampleNode("kid1", SubSampleType.TECHNICAL_REPLICATE, "root") + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1") + n4 = SampleNode("kid3", SubSampleType.TECHNICAL_REPLICATE, "root") - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") - assert samplestorage.save_sample( - SavedSample(id_, UserID('auser'), [n1, n2, n3, n4], dt(1), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("auser"), [n1, n2, n3, n4], dt(1), "foo") + ) + is True + ) # this is very naughty # checked that these modifications actually work by viewing the db contents - samplestorage._col_version.update_match({}, {'ver': -1}) - samplestorage._col_nodes.update_match({'name': 'kid2'}, {'ver': -1}) + samplestorage._col_version.update_match({}, {"ver": -1}) + samplestorage._col_nodes.update_match({"name": "kid2"}, {"ver": -1}) assert samplestorage.get_sample(id_) == SavedSample( - id_, UserID('auser'), [n1, n2, n3, n4], dt(1), 'foo', 1) + id_, UserID("auser"), [n1, n2, n3, n4], dt(1), "foo", 1 + ) for v in samplestorage._col_version.all(): - assert v['ver'] == 1 + assert v["ver"] == 1 for v in samplestorage._col_nodes.all(): - assert v['ver'] == 1 + assert v["ver"] == 1 def test_get_sample_with_non_updated_node_doc(samplestorage): @@ -793,25 +1150,30 @@ def test_get_sample_with_non_updated_node_doc(samplestorage): # the version doc update *must* have been updated for this test to exercise the # node checking logic because a non-updated version doc will cause the nodes to be updated # immediately. - n1 = SampleNode('root') - n2 = SampleNode('kid1', SubSampleType.TECHNICAL_REPLICATE, 'root') - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1') - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root') + n1 = SampleNode("root") + n2 = SampleNode("kid1", SubSampleType.TECHNICAL_REPLICATE, "root") + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1") + n4 = SampleNode("kid3", SubSampleType.TECHNICAL_REPLICATE, "root") - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") - assert samplestorage.save_sample( - SavedSample(id_, UserID('auser'), [n1, n2, n3, n4], dt(1), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("auser"), [n1, n2, n3, n4], dt(1), "foo") + ) + is True + ) # this is very naughty # checked that these modifications actually work by viewing the db contents - samplestorage._col_nodes.update_match({'name': 'kid1'}, {'ver': -1}) + samplestorage._col_nodes.update_match({"name": "kid1"}, {"ver": -1}) assert samplestorage.get_sample(id_) == SavedSample( - id_, UserID('auser'), [n1, n2, n3, n4], dt(1), 'foo', 1) + id_, UserID("auser"), [n1, n2, n3, n4], dt(1), "foo", 1 + ) for v in samplestorage._col_nodes.all(): - assert v['ver'] == 1 + assert v["ver"] == 1 def test_get_sample_with_missing_source_metadata_key(samplestorage, arango): @@ -819,17 +1181,25 @@ def test_get_sample_with_missing_source_metadata_key(samplestorage, arango): Backwards compatibility test. Checks that a missing smeta key in the sample node returns an empty source metadata list. """ - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample(SavedSample( - id1, - UserID('user'), - [SampleNode('mynode', - controlled_metadata={'a': {'c': 'd'}}, - source_metadata=[SourceMetadata('a', 'b', {'x': 'y'})] + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample( + id1, + UserID("user"), + [ + SampleNode( + "mynode", + controlled_metadata={"a": {"c": "d"}}, + source_metadata=[SourceMetadata("a", "b", {"x": "y"})], ) - ], - dt(7), - 'foo')) is True + ], + dt(7), + "foo", + ) + ) + is True + ) arango.client.db(TEST_DB_NAME).aql.execute( """ @@ -838,7 +1208,7 @@ def test_get_sample_with_missing_source_metadata_key(samplestorage, arango): UPDATE n WITH {smeta: null} IN @@col OPTIONS {keepNull: false} """, - bind_vars={'@col': TEST_COL_NODES, 'name': 'mynode'} + bind_vars={"@col": TEST_COL_NODES, "name": "mynode"}, ) cur = arango.client.db(TEST_DB_NAME).aql.execute( @@ -847,196 +1217,294 @@ def test_get_sample_with_missing_source_metadata_key(samplestorage, arango): FILTER n.name == @name RETURN n """, - bind_vars={'@col': TEST_COL_NODES, 'name': 'mynode'} + bind_vars={"@col": TEST_COL_NODES, "name": "mynode"}, ) doc = cur.next() - del doc['_rev'] - del doc['_id'] - del doc['_key'] - del doc['uuidver'] + del doc["_rev"] + del doc["_id"] + del doc["_key"] + del doc["uuidver"] assert doc == { - 'id': str(id1), - 'ver': 1, - 'saved': 7000, - 'name': 'mynode', - 'type': 'BIOLOGICAL_REPLICATE', - 'parent': None, - 'index': 0, - 'cmeta': [{'k': 'c', 'ok': 'a', 'v': 'd'}], - 'ucmeta': [], + "id": str(id1), + "ver": 1, + "saved": 7000, + "name": "mynode", + "type": "BIOLOGICAL_REPLICATE", + "parent": None, + "index": 0, + "cmeta": [{"k": "c", "ok": "a", "v": "d"}], + "ucmeta": [], } assert samplestorage.get_sample(id1) == SavedSample( id1, - UserID('user'), - [SampleNode('mynode', controlled_metadata={'a': {'c': 'd'}})], + UserID("user"), + [SampleNode("mynode", controlled_metadata={"a": {"c": "d"}})], dt(7), - 'foo', - 1) + "foo", + 1, + ) def test_get_sample_fail_bad_input(samplestorage): with raises(Exception) as got: samplestorage.get_sample(None) assert_exception_correct( - got.value, ValueError('id_ cannot be a value that evaluates to false')) + got.value, ValueError("id_ cannot be a value that evaluates to false") + ) def test_get_sample_fail_no_sample(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) with raises(Exception) as got: - samplestorage.get_sample(uuid.UUID('1234567890abcdef1234567890abcdea')) + samplestorage.get_sample(uuid.UUID("1234567890abcdef1234567890abcdea")) assert_exception_correct( - got.value, NoSuchSampleError('12345678-90ab-cdef-1234-567890abcdea')) + got.value, NoSuchSampleError("12345678-90ab-cdef-1234-567890abcdea") + ) def test_get_sample_fail_no_such_version(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) with raises(Exception) as got: - samplestorage.get_sample(uuid.UUID('1234567890abcdef1234567890abcdef'), version=2) + samplestorage.get_sample( + uuid.UUID("1234567890abcdef1234567890abcdef"), version=2 + ) assert_exception_correct( - got.value, NoSuchSampleVersionError('12345678-90ab-cdef-1234-567890abcdef ver 2')) + got.value, + NoSuchSampleVersionError("12345678-90ab-cdef-1234-567890abcdef ver 2"), + ) - assert samplestorage.save_sample_version( - SavedSample(id_, UserID('user2'), [TEST_NODE], dt(1), 'bar')) == 2 + assert ( + samplestorage.save_sample_version( + SavedSample(id_, UserID("user2"), [TEST_NODE], dt(1), "bar") + ) + == 2 + ) assert samplestorage.get_sample(id_) == SavedSample( - id_, UserID('user2'), [TEST_NODE], dt(1), 'bar', 2) + id_, UserID("user2"), [TEST_NODE], dt(1), "bar", 2 + ) with raises(Exception) as got: - samplestorage.get_sample(uuid.UUID('1234567890abcdef1234567890abcdef'), version=3) + samplestorage.get_sample( + uuid.UUID("1234567890abcdef1234567890abcdef"), version=3 + ) assert_exception_correct( - got.value, NoSuchSampleVersionError('12345678-90ab-cdef-1234-567890abcdef ver 3')) + got.value, + NoSuchSampleVersionError("12345678-90ab-cdef-1234-567890abcdef ver 3"), + ) def test_get_sample_fail_no_version_doc_1_version(samplestorage): # This should be impossible in practice unless someone actively deletes records from the db. - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) # this is very naughty - verdoc_filters = {'id': '12345678-90ab-cdef-1234-567890abcdef', 'ver': 1} + verdoc_filters = {"id": "12345678-90ab-cdef-1234-567890abcdef", "ver": 1} verdoc = samplestorage._col_version.find(verdoc_filters).next() samplestorage._col_version.delete_match(verdoc_filters) with raises(Exception) as got: - samplestorage.get_sample(uuid.UUID('1234567890abcdef1234567890abcdef'), version=1) + samplestorage.get_sample( + uuid.UUID("1234567890abcdef1234567890abcdef"), version=1 + ) assert_exception_correct( - got.value, SampleStorageError(f'Corrupt DB: Missing version {verdoc["uuidver"]} ' + - 'for sample 12345678-90ab-cdef-1234-567890abcdef')) + got.value, + SampleStorageError( + f'Corrupt DB: Missing version {verdoc["uuidver"]} ' + + "for sample 12345678-90ab-cdef-1234-567890abcdef" + ), + ) def test_get_sample_fail_no_version_doc_2_versions(samplestorage): # This should be impossible in practice unless someone actively deletes records from the db. - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'bar')) == 2 + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "bar") + ) + == 2 + ) # this is very naughty - verdoc_filters = {'id': '12345678-90ab-cdef-1234-567890abcdef', 'ver': 2} + verdoc_filters = {"id": "12345678-90ab-cdef-1234-567890abcdef", "ver": 2} verdoc = samplestorage._col_version.find(verdoc_filters).next() samplestorage._col_version.delete_match(verdoc_filters) assert samplestorage.get_sample(id_, version=1) == SavedSample( - id_, UserID('user'), [TEST_NODE], dt(1), 'foo', 1) + id_, UserID("user"), [TEST_NODE], dt(1), "foo", 1 + ) with raises(Exception) as got: - samplestorage.get_sample(uuid.UUID('1234567890abcdef1234567890abcdef'), version=2) + samplestorage.get_sample( + uuid.UUID("1234567890abcdef1234567890abcdef"), version=2 + ) assert_exception_correct( - got.value, SampleStorageError(f'Corrupt DB: Missing version {verdoc["uuidver"]} ' + - 'for sample 12345678-90ab-cdef-1234-567890abcdef')) + got.value, + SampleStorageError( + f'Corrupt DB: Missing version {verdoc["uuidver"]} ' + + "for sample 12345678-90ab-cdef-1234-567890abcdef" + ), + ) def test_get_sample_fail_no_node_docs_1_version(samplestorage): # This should be impossible in practice unless someone actively deletes records from the db. - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) # this is very naughty - nodedoc_filters = {'id': '12345678-90ab-cdef-1234-567890abcdef', 'ver': 1} + nodedoc_filters = {"id": "12345678-90ab-cdef-1234-567890abcdef", "ver": 1} nodedoc = samplestorage._col_nodes.find(nodedoc_filters).next() samplestorage._col_nodes.delete_match(nodedoc_filters) with raises(Exception) as got: - samplestorage.get_sample(uuid.UUID('1234567890abcdef1234567890abcdef'), version=1) + samplestorage.get_sample( + uuid.UUID("1234567890abcdef1234567890abcdef"), version=1 + ) assert_exception_correct( - got.value, SampleStorageError( - f'Corrupt DB: Missing nodes for version {nodedoc["uuidver"]} of sample ' + - '12345678-90ab-cdef-1234-567890abcdef')) + got.value, + SampleStorageError( + f'Corrupt DB: Missing nodes for version {nodedoc["uuidver"]} of sample ' + + "12345678-90ab-cdef-1234-567890abcdef" + ), + ) def test_get_sample_fail_no_node_docs_2_versions(samplestorage): # This should be impossible in practice unless someone actively deletes records from the db. - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'bar')) == 2 + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "bar") + ) + == 2 + ) # this is very naughty - nodedoc_filters = {'id': '12345678-90ab-cdef-1234-567890abcdef', 'ver': 2} + nodedoc_filters = {"id": "12345678-90ab-cdef-1234-567890abcdef", "ver": 2} nodedoc = samplestorage._col_nodes.find(nodedoc_filters).next() samplestorage._col_nodes.delete_match(nodedoc_filters) assert samplestorage.get_sample(id_, version=1) == SavedSample( - id_, UserID('user'), [TEST_NODE], dt(1), 'foo', 1) + id_, UserID("user"), [TEST_NODE], dt(1), "foo", 1 + ) with raises(Exception) as got: - samplestorage.get_sample(uuid.UUID('1234567890abcdef1234567890abcdef'), version=2) + samplestorage.get_sample( + uuid.UUID("1234567890abcdef1234567890abcdef"), version=2 + ) assert_exception_correct( - got.value, SampleStorageError( - f'Corrupt DB: Missing nodes for version {nodedoc["uuidver"]} of sample ' + - '12345678-90ab-cdef-1234-567890abcdef')) + got.value, + SampleStorageError( + f'Corrupt DB: Missing nodes for version {nodedoc["uuidver"]} of sample ' + + "12345678-90ab-cdef-1234-567890abcdef" + ), + ) def test_save_and_get_sample_version(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(42), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(42), "foo") + ) + is True + ) - n1 = SampleNode('root') + n1 = SampleNode("root") n2 = SampleNode( - 'kid1', SubSampleType.TECHNICAL_REPLICATE, 'root', - {'a': {'b': 'c', 'd': 'e'}, 'f': {'g': 'h'}}, - {'m': {'n': 'o'}}) - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1', {'a': {'b': 'c'}}) - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root', - user_metadata={'f': {'g': 'h'}}) - - assert samplestorage.save_sample_version( - SavedSample(id_, UserID('user2'), [n1, n2, n3, n4], dt(86), 'bar')) == 2 - assert samplestorage.save_sample_version( - SavedSample(id_, UserID('user3'), [n1], dt(7), 'whiz', version=6)) == 3 + "kid1", + SubSampleType.TECHNICAL_REPLICATE, + "root", + {"a": {"b": "c", "d": "e"}, "f": {"g": "h"}}, + {"m": {"n": "o"}}, + ) + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1", {"a": {"b": "c"}}) + n4 = SampleNode( + "kid3", + SubSampleType.TECHNICAL_REPLICATE, + "root", + user_metadata={"f": {"g": "h"}}, + ) + + assert ( + samplestorage.save_sample_version( + SavedSample(id_, UserID("user2"), [n1, n2, n3, n4], dt(86), "bar") + ) + == 2 + ) + assert ( + samplestorage.save_sample_version( + SavedSample(id_, UserID("user3"), [n1], dt(7), "whiz", version=6) + ) + == 3 + ) assert samplestorage.get_sample(id_, version=1) == SavedSample( - id_, UserID('user'), [TEST_NODE], dt(42), 'foo', 1) + id_, UserID("user"), [TEST_NODE], dt(42), "foo", 1 + ) assert samplestorage.get_sample(id_, version=2) == SavedSample( - id_, UserID('user2'), [n1, n2, n3, n4], dt(86), 'bar', 2) + id_, UserID("user2"), [n1, n2, n3, n4], dt(86), "bar", 2 + ) - expected = SavedSample(id_, UserID('user3'), [n1], dt(7), 'whiz', 3) + expected = SavedSample(id_, UserID("user3"), [n1], dt(7), "whiz", 3) assert samplestorage.get_sample(id_) == expected assert samplestorage.get_sample(id_, version=3) == expected def test_save_sample_version_fail_bad_input(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - s = SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + s = SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") - _save_sample_version_fail(samplestorage, None, None, ValueError( - 'sample cannot be a value that evaluates to false')) - _save_sample_version_fail(samplestorage, s, 0, ValueError( - 'prior_version must be > 0')) + _save_sample_version_fail( + samplestorage, + None, + None, + ValueError("sample cannot be a value that evaluates to false"), + ) + _save_sample_version_fail( + samplestorage, s, 0, ValueError("prior_version must be > 0") + ) def _save_sample_version_fail(samplestorage, sample, prior_version, expected): @@ -1046,67 +1514,103 @@ def _save_sample_version_fail(samplestorage, sample, prior_version, expected): def test_save_sample_version_fail_no_sample(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - id2 = uuid.UUID('1234567890abcdef1234567890abcdea') + id2 = uuid.UUID("1234567890abcdef1234567890abcdea") with raises(Exception) as got: samplestorage.save_sample_version( - SavedSample(id2, UserID('user'), [TEST_NODE], dt(1), 'whiz')) - assert_exception_correct(got.value, NoSuchSampleError('12345678-90ab-cdef-1234-567890abcdea')) + SavedSample(id2, UserID("user"), [TEST_NODE], dt(1), "whiz") + ) + assert_exception_correct( + got.value, NoSuchSampleError("12345678-90ab-cdef-1234-567890abcdea") + ) def test_save_sample_version_fail_prior_version(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(id_, UserID('user'), [SampleNode('bat')], dt(1), 'bar')) == 2 + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(id_, UserID("user"), [SampleNode("bat")], dt(1), "bar") + ) + == 2 + ) with raises(Exception) as got: samplestorage.save_sample_version( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'whiz'), prior_version=1) - assert_exception_correct(got.value, ConcurrencyError( - 'Version required for sample ' + - '12345678-90ab-cdef-1234-567890abcdef is 1, but current version is 2')) + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "whiz"), + prior_version=1, + ) + assert_exception_correct( + got.value, + ConcurrencyError( + "Version required for sample " + + "12345678-90ab-cdef-1234-567890abcdef is 1, but current version is 2" + ), + ) # this is naughty, but need to check race condition with raises(Exception) as got: samplestorage._save_sample_version_pt2( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'whiz'), 1) - assert_exception_correct(got.value, ConcurrencyError( - 'Version required for sample ' + - '12345678-90ab-cdef-1234-567890abcdef is 1, but current version is 2')) + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "whiz"), 1 + ) + assert_exception_correct( + got.value, + ConcurrencyError( + "Version required for sample " + + "12345678-90ab-cdef-1234-567890abcdef is 1, but current version is 2" + ), + ) def test_sample_version_update(samplestorage): # tests that the versions on node and version documents are updated correctly - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [SampleNode('baz')], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [SampleNode("baz")], dt(1), "foo") + ) + is True + ) - assert samplestorage.save_sample_version( - SavedSample(id_, UserID('user'), [SampleNode('bat')], dt(1), 'bar')) == 2 + assert ( + samplestorage.save_sample_version( + SavedSample(id_, UserID("user"), [SampleNode("bat")], dt(1), "bar") + ) + == 2 + ) assert samplestorage.get_sample(id_, version=1) == SavedSample( - id_, UserID('user'), [SampleNode('baz')], dt(1), 'foo', 1) + id_, UserID("user"), [SampleNode("baz")], dt(1), "foo", 1 + ) assert samplestorage.get_sample(id_) == SavedSample( - id_, UserID('user'), [SampleNode('bat')], dt(1), 'bar', 2) + id_, UserID("user"), [SampleNode("bat")], dt(1), "bar", 2 + ) - idstr = '12345678-90ab-cdef-1234-567890abcdef' + idstr = "12345678-90ab-cdef-1234-567890abcdef" vers = set() # this is naughty - for n in samplestorage._col_version.find({'id': idstr}): - vers.add((n['name'], n['ver'])) - assert vers == {('foo', 1), ('bar', 2)} + for n in samplestorage._col_version.find({"id": idstr}): + vers.add((n["name"], n["ver"])) + assert vers == {("foo", 1), ("bar", 2)} nodes = set() # this is naughty - for n in samplestorage._col_nodes.find({'id': idstr}): - nodes.add((n['name'], n['ver'])) - assert nodes == {('baz', 1), ('bat', 2)} + for n in samplestorage._col_nodes.find({"id": idstr}): + nodes.add((n["name"], n["ver"])) + assert nodes == {("baz", 1), ("bat", 2)} def test_get_sample_acls_with_missing_public_read_key(samplestorage, arango): @@ -1114,9 +1618,13 @@ def test_get_sample_acls_with_missing_public_read_key(samplestorage, arango): Backwards compatibility test. Checks that a missing pubread key in the ACLs is registered as false, and that then changing pubread to True works normally. """ - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) arango.client.db(TEST_DB_NAME).aql.execute( """ @@ -1125,7 +1633,7 @@ def test_get_sample_acls_with_missing_public_read_key(samplestorage, arango): UPDATE s WITH {acls: {pubread: null}} IN @@col OPTIONS {keepNull: false} """, - bind_vars={'@col': TEST_COL_SAMPLE, 'id': str(id1)} + bind_vars={"@col": TEST_COL_SAMPLE, "id": str(id1)}, ) cur = arango.client.db(TEST_DB_NAME).aql.execute( @@ -1134,108 +1642,138 @@ def test_get_sample_acls_with_missing_public_read_key(samplestorage, arango): FILTER s.id == @id RETURN s """, - bind_vars={'@col': TEST_COL_SAMPLE, 'id': str(id1)} + bind_vars={"@col": TEST_COL_SAMPLE, "id": str(id1)}, ) - assert cur.next()['acls'] == {'owner': 'user', - 'admin': [], - 'write': [], - 'read': [] - } + assert cur.next()["acls"] == {"owner": "user", "admin": [], "write": [], "read": []} - assert samplestorage.get_sample_acls(id1) == SampleACL(UserID('user'), dt(1)) + assert samplestorage.get_sample_acls(id1) == SampleACL(UserID("user"), dt(1)) - samplestorage.replace_sample_acls(id1, SampleACL(UserID('user'), dt(3), public_read=True)) + samplestorage.replace_sample_acls( + id1, SampleACL(UserID("user"), dt(3), public_read=True) + ) - assert samplestorage.get_sample_acls(id1) == SampleACL(UserID('user'), dt(3), public_read=True) + assert samplestorage.get_sample_acls(id1) == SampleACL( + UserID("user"), dt(3), public_read=True + ) def test_get_sample_acls_fail_bad_input(samplestorage): with raises(Exception) as got: samplestorage.get_sample_acls(None) assert_exception_correct( - got.value, ValueError('id_ cannot be a value that evaluates to false')) + got.value, ValueError("id_ cannot be a value that evaluates to false") + ) def test_get_sample_acls_fail_no_sample(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) with raises(Exception) as got: - samplestorage.get_sample_acls(uuid.UUID('1234567890abcdef1234567890abcdea')) + samplestorage.get_sample_acls(uuid.UUID("1234567890abcdef1234567890abcdea")) assert_exception_correct( - got.value, NoSuchSampleError('12345678-90ab-cdef-1234-567890abcdea')) + got.value, NoSuchSampleError("12345678-90ab-cdef-1234-567890abcdea") + ) def test_replace_sample_acls(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - samplestorage.replace_sample_acls(id_, SampleACL( - UserID('user'), - dt(56), - [UserID('foo'), UserID('bar')], - [UserID('baz'), UserID('bat')], - [UserID('whoo')], - True)) + samplestorage.replace_sample_acls( + id_, + SampleACL( + UserID("user"), + dt(56), + [UserID("foo"), UserID("bar")], + [UserID("baz"), UserID("bat")], + [UserID("whoo")], + True, + ), + ) assert samplestorage.get_sample_acls(id_) == SampleACL( - UserID('user'), + UserID("user"), dt(56), - [UserID('foo'), UserID('bar')], - [UserID('baz'), UserID('bat')], - [UserID('whoo')], - True) + [UserID("foo"), UserID("bar")], + [UserID("baz"), UserID("bat")], + [UserID("whoo")], + True, + ) - samplestorage.replace_sample_acls(id_, SampleACL(UserID('user'), dt(83), write=[UserID('baz')])) + samplestorage.replace_sample_acls( + id_, SampleACL(UserID("user"), dt(83), write=[UserID("baz")]) + ) assert samplestorage.get_sample_acls(id_) == SampleACL( - UserID('user'), dt(83), write=[UserID('baz')]) + UserID("user"), dt(83), write=[UserID("baz")] + ) def test_replace_sample_acls_fail_bad_args(samplestorage): with raises(Exception) as got: - samplestorage.replace_sample_acls(None, SampleACL(UserID('user'), dt(1))) - assert_exception_correct(got.value, ValueError( - 'id_ cannot be a value that evaluates to false')) + samplestorage.replace_sample_acls(None, SampleACL(UserID("user"), dt(1))) + assert_exception_correct( + got.value, ValueError("id_ cannot be a value that evaluates to false") + ) - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") with raises(Exception) as got: samplestorage.replace_sample_acls(id_, None) - assert_exception_correct(got.value, ValueError( - 'acls cannot be a value that evaluates to false')) + assert_exception_correct( + got.value, ValueError("acls cannot be a value that evaluates to false") + ) def test_replace_sample_acls_fail_no_sample(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - id2 = uuid.UUID('1234567890abcdef1234567890abcdea') + id2 = uuid.UUID("1234567890abcdef1234567890abcdea") with raises(Exception) as got: - samplestorage.replace_sample_acls(id2, SampleACL(UserID('user'), dt(1))) + samplestorage.replace_sample_acls(id2, SampleACL(UserID("user"), dt(1))) assert_exception_correct(got.value, NoSuchSampleError(str(id2))) def test_replace_sample_acls_fail_owner_changed(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) # this is naughty samplestorage._db.aql.execute( - ''' + """ FOR s IN @@col UPDATE s WITH {acls: MERGE(s.acls, @acls)} IN @@col RETURN s - ''', - bind_vars={'@col': 'samples', 'acls': {'owner': 'user2'}}) + """, + bind_vars={"@col": "samples", "acls": {"owner": "user2"}}, + ) with raises(Exception) as got: samplestorage.replace_sample_acls( - id_, SampleACL(UserID('user'), dt(1), write=[UserID('foo')])) + id_, SampleACL(UserID("user"), dt(1), write=[UserID("foo")]) + ) assert_exception_correct(got.value, OwnerChangedError()) @@ -1250,35 +1788,44 @@ def test_update_sample_acls_with_at_least_True(samplestorage): def _update_sample_acls(samplestorage, at_least): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True - - samplestorage.update_sample_acls(id_, SampleACLDelta( - [UserID('foo'), UserID('bar1')], - [UserID('baz1'), UserID('bat')], - [UserID('whoo1')], - public_read=True, - at_least=at_least), - dt(101)) + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) + + samplestorage.update_sample_acls( + id_, + SampleACLDelta( + [UserID("foo"), UserID("bar1")], + [UserID("baz1"), UserID("bat")], + [UserID("whoo1")], + public_read=True, + at_least=at_least, + ), + dt(101), + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(101), - [UserID('foo'), UserID('bar1')], - [UserID('baz1'), UserID('bat')], - [UserID('whoo1')], - True) + [UserID("foo"), UserID("bar1")], + [UserID("baz1"), UserID("bat")], + [UserID("whoo1")], + True, + ) def test_update_sample_acls_with_at_least_True_and_owner_in_admin_acl(samplestorage): # owner should be included in any changes with at_least = True. _update_sample_acls_with_owner_in_acl( samplestorage, - [UserID('foo'), UserID('bar1'), UserID('user')], - [UserID('baz1'), UserID('bat')], - [UserID('whoo1')], + [UserID("foo"), UserID("bar1"), UserID("user")], + [UserID("baz1"), UserID("bat")], + [UserID("whoo1")], ) @@ -1286,9 +1833,9 @@ def test_update_sample_acls_with_at_least_True_and_owner_in_write_acl(samplestor # owner should be included in any changes with at_least = True. _update_sample_acls_with_owner_in_acl( samplestorage, - [UserID('foo'), UserID('bar1')], - [UserID('baz1'), UserID('bat'), UserID('user')], - [UserID('whoo1')], + [UserID("foo"), UserID("bar1")], + [UserID("baz1"), UserID("bat"), UserID("user")], + [UserID("whoo1")], ) @@ -1296,28 +1843,36 @@ def test_update_sample_acls_with_at_least_True_and_owner_in_read_acl(samplestora # owner should be included in any changes with at_least = True. _update_sample_acls_with_owner_in_acl( samplestorage, - [UserID('foo'), UserID('bar1')], - [UserID('baz1'), UserID('bat')], - [UserID('whoo1'), UserID('user')], + [UserID("foo"), UserID("bar1")], + [UserID("baz1"), UserID("bat")], + [UserID("whoo1"), UserID("user")], ) def _update_sample_acls_with_owner_in_acl(samplestorage, admin, write, read): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) samplestorage.update_sample_acls( - id_, SampleACLDelta(admin, write, read, public_read=True, at_least=True), dt(101)) + id_, + SampleACLDelta(admin, write, read, public_read=True, at_least=True), + dt(101), + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(101), - [UserID('foo'), UserID('bar1')], - [UserID('baz1'), UserID('bat')], - [UserID('whoo1')], - True) + [UserID("foo"), UserID("bar1")], + [UserID("baz1"), UserID("bat")], + [UserID("whoo1")], + True, + ) def test_update_sample_acls_noop_with_at_least_False(samplestorage): @@ -1329,75 +1884,105 @@ def test_update_sample_acls_noop_with_at_least_True(samplestorage): def _update_sample_acls_noop(samplestorage, at_least): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - samplestorage.replace_sample_acls(id_, SampleACL( - UserID('user'), - dt(56), - [UserID('foo'), UserID('bar')], - [UserID('baz'), UserID('bat')], - [UserID('whoo')], - True)) - - samplestorage.update_sample_acls(id_, SampleACLDelta( - [UserID('foo')], - [UserID('bat')], - [UserID('whoo')], - [UserID('nouser'), UserID('nouser2')], - public_read=True, - at_least=at_least), - dt(103)) + samplestorage.replace_sample_acls( + id_, + SampleACL( + UserID("user"), + dt(56), + [UserID("foo"), UserID("bar")], + [UserID("baz"), UserID("bat")], + [UserID("whoo")], + True, + ), + ) + + samplestorage.update_sample_acls( + id_, + SampleACLDelta( + [UserID("foo")], + [UserID("bat")], + [UserID("whoo")], + [UserID("nouser"), UserID("nouser2")], + public_read=True, + at_least=at_least, + ), + dt(103), + ) res = samplestorage.get_sample_acls(id_) print(res) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(56), - [UserID('foo'), UserID('bar')], - [UserID('baz'), UserID('bat')], - [UserID('whoo')], - True) + [UserID("foo"), UserID("bar")], + [UserID("baz"), UserID("bat")], + [UserID("whoo")], + True, + ) -def test_update_sample_acls_with_remove_and_null_public_and_at_least_False(samplestorage): +def test_update_sample_acls_with_remove_and_null_public_and_at_least_False( + samplestorage, +): _update_sample_acls_with_remove_and_null_public(samplestorage, False) -def test_update_sample_acls_with_remove_and_null_public_and_at_least_True(samplestorage): +def test_update_sample_acls_with_remove_and_null_public_and_at_least_True( + samplestorage, +): _update_sample_acls_with_remove_and_null_public(samplestorage, True) def _update_sample_acls_with_remove_and_null_public(samplestorage, at_least): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - samplestorage.replace_sample_acls(id_, SampleACL( - UserID('user'), - dt(56), - [UserID('foo'), UserID('bar')], - [UserID('baz'), UserID('bat')], - [UserID('whoo')], - True)) - - samplestorage.update_sample_acls(id_, SampleACLDelta( - [UserID('admin')], - [UserID('write'), UserID('write2')], - [UserID('read')], - [UserID('foo'), UserID('bat'), UserID('whoo'), UserID('notauser')], - at_least=at_least), - dt(102)) + samplestorage.replace_sample_acls( + id_, + SampleACL( + UserID("user"), + dt(56), + [UserID("foo"), UserID("bar")], + [UserID("baz"), UserID("bat")], + [UserID("whoo")], + True, + ), + ) + + samplestorage.update_sample_acls( + id_, + SampleACLDelta( + [UserID("admin")], + [UserID("write"), UserID("write2")], + [UserID("read")], + [UserID("foo"), UserID("bat"), UserID("whoo"), UserID("notauser")], + at_least=at_least, + ), + dt(102), + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(102), - [UserID('bar'), UserID('admin')], - [UserID('baz'), UserID('write'), UserID('write2')], - [UserID('read')], - True) + [UserID("bar"), UserID("admin")], + [UserID("baz"), UserID("write"), UserID("write2")], + [UserID("read")], + True, + ) def test_update_sample_acls_with_False_public_and_at_least_False(samplestorage): @@ -1409,177 +1994,221 @@ def test_update_sample_acls_with_False_public_and_at_least_True(samplestorage): def _update_sample_acls_with_false_public(samplestorage, at_least): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - samplestorage.replace_sample_acls(id_, SampleACL( - UserID('user'), - dt(56), - [UserID('foo'), UserID('bar')], - [UserID('baz'), UserID('bat')], - [UserID('whoo')], - True)) + samplestorage.replace_sample_acls( + id_, + SampleACL( + UserID("user"), + dt(56), + [UserID("foo"), UserID("bar")], + [UserID("baz"), UserID("bat")], + [UserID("whoo")], + True, + ), + ) samplestorage.update_sample_acls( - id_, SampleACLDelta(public_read=False, at_least=at_least), dt(89)) + id_, SampleACLDelta(public_read=False, at_least=at_least), dt(89) + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(89), - [UserID('bar'), UserID('foo')], - [UserID('baz'), UserID('bat')], - [UserID('whoo')], - False) + [UserID("bar"), UserID("foo")], + [UserID("baz"), UserID("bat")], + [UserID("whoo")], + False, + ) def test_update_sample_acls_with_existing_users(samplestorage): - ''' + """ Tests that when a user is added to an acl it's removed from any other acls. - ''' - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + """ + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - samplestorage.replace_sample_acls(id_, SampleACL( - UserID('user'), - dt(56), - admin=[UserID('a1'), UserID('a2'), UserID('arem')], - write=[UserID('w1'), UserID('w2'), UserID('wrem')], - read=[UserID('r1'), UserID('r2'), UserID('rrem')])) + samplestorage.replace_sample_acls( + id_, + SampleACL( + UserID("user"), + dt(56), + admin=[UserID("a1"), UserID("a2"), UserID("arem")], + write=[UserID("w1"), UserID("w2"), UserID("wrem")], + read=[UserID("r1"), UserID("r2"), UserID("rrem")], + ), + ) # move user from write -> admin, remove admin - samplestorage.update_sample_acls(id_, SampleACLDelta( - admin=[UserID('w1')], remove=[UserID('arem')]), dt(89)) + samplestorage.update_sample_acls( + id_, SampleACLDelta(admin=[UserID("w1")], remove=[UserID("arem")]), dt(89) + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(89), - [UserID('a1'), UserID('a2'), UserID('w1')], - [UserID('w2'), UserID('wrem')], - [UserID('r1'), UserID('r2'), UserID('rrem')], - False) + [UserID("a1"), UserID("a2"), UserID("w1")], + [UserID("w2"), UserID("wrem")], + [UserID("r1"), UserID("r2"), UserID("rrem")], + False, + ) # move user from read -> admin - samplestorage.update_sample_acls(id_, SampleACLDelta(admin=[UserID('r1')]), dt(90)) + samplestorage.update_sample_acls(id_, SampleACLDelta(admin=[UserID("r1")]), dt(90)) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(90), - [UserID('a1'), UserID('a2'), UserID('w1'), UserID('r1')], - [UserID('w2'), UserID('wrem')], - [UserID('r2'), UserID('rrem')], - False) + [UserID("a1"), UserID("a2"), UserID("w1"), UserID("r1")], + [UserID("w2"), UserID("wrem")], + [UserID("r2"), UserID("rrem")], + False, + ) # move user from write -> read, remove write - samplestorage.update_sample_acls(id_, SampleACLDelta( - read=[UserID('w1')], remove=[UserID('wrem')]), dt(91)) + samplestorage.update_sample_acls( + id_, SampleACLDelta(read=[UserID("w1")], remove=[UserID("wrem")]), dt(91) + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(91), - [UserID('a1'), UserID('a2'), UserID('r1')], - [UserID('w2')], - [UserID('r2'), UserID('w1'), UserID('rrem')], - False) + [UserID("a1"), UserID("a2"), UserID("r1")], + [UserID("w2")], + [UserID("r2"), UserID("w1"), UserID("rrem")], + False, + ) # move user from admin -> read - samplestorage.update_sample_acls(id_, SampleACLDelta(read=[UserID('a1')]), dt(92)) + samplestorage.update_sample_acls(id_, SampleACLDelta(read=[UserID("a1")]), dt(92)) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(92), - [UserID('a2'), UserID('r1')], - [UserID('w2')], - [UserID('r2'), UserID('w1'), UserID('a1'), UserID('rrem')], - False) + [UserID("a2"), UserID("r1")], + [UserID("w2")], + [UserID("r2"), UserID("w1"), UserID("a1"), UserID("rrem")], + False, + ) # move user from admin -> write, remove read - samplestorage.update_sample_acls(id_, SampleACLDelta( - write=[UserID('a2')], remove=[UserID('rrem')]), dt(93)) + samplestorage.update_sample_acls( + id_, SampleACLDelta(write=[UserID("a2")], remove=[UserID("rrem")]), dt(93) + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(93), - [UserID('r1')], - [UserID('w2'), UserID('a2')], - [UserID('r2'), UserID('w1'), UserID('a1')], - False) + [UserID("r1")], + [UserID("w2"), UserID("a2")], + [UserID("r2"), UserID("w1"), UserID("a1")], + False, + ) # move user from read -> write, move user from write -> read, noop on read user - samplestorage.update_sample_acls(id_, SampleACLDelta( - write=[UserID('r2')], read=[UserID('a2'), UserID('a1')]), dt(94)) + samplestorage.update_sample_acls( + id_, + SampleACLDelta(write=[UserID("r2")], read=[UserID("a2"), UserID("a1")]), + dt(94), + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(94), - [UserID('r1')], - [UserID('w2'), UserID('r2')], - [UserID('w1'), UserID('a2'), UserID('a1')], - False) + [UserID("r1")], + [UserID("w2"), UserID("r2")], + [UserID("w1"), UserID("a2"), UserID("a1")], + False, + ) def test_update_sample_acls_with_existing_users_and_at_least_True(samplestorage): - ''' + """ Tests that when a user is added to an acl it's state is unchanged if it's already in a 'better' acl. - ''' - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + """ + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - samplestorage.replace_sample_acls(id_, SampleACL( - UserID('user'), - dt(56), - admin=[UserID('a1'), UserID('a2'), UserID('arem')], - write=[UserID('w1'), UserID('w2'), UserID('wrem')], - read=[UserID('r1'), UserID('r2'), UserID('rrem')])) + samplestorage.replace_sample_acls( + id_, + SampleACL( + UserID("user"), + dt(56), + admin=[UserID("a1"), UserID("a2"), UserID("arem")], + write=[UserID("w1"), UserID("w2"), UserID("wrem")], + read=[UserID("r1"), UserID("r2"), UserID("rrem")], + ), + ) samplestorage.update_sample_acls( id_, SampleACLDelta( - admin=[UserID('a1')], # noop admin->admin - write=[UserID('a2'), UserID('r1')], # noop admin->write, read->write - read=[UserID('r2')], # noop read->read - remove=[UserID('arem')], # remove admin - at_least=True), - dt(89)) + admin=[UserID("a1")], # noop admin->admin + write=[UserID("a2"), UserID("r1")], # noop admin->write, read->write + read=[UserID("r2")], # noop read->read + remove=[UserID("arem")], # remove admin + at_least=True, + ), + dt(89), + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(89), - [UserID('a1'), UserID('a2')], - [UserID('w1'), UserID('w2'), UserID('wrem'), UserID('r1')], - [UserID('r2'), UserID('rrem')], - False) + [UserID("a1"), UserID("a2")], + [UserID("w1"), UserID("w2"), UserID("wrem"), UserID("r1")], + [UserID("r2"), UserID("rrem")], + False, + ) samplestorage.update_sample_acls( id_, SampleACLDelta( - admin=[UserID('r1'), UserID('r2')], # write->admin, read->admin - write=[UserID('w2')], # noop write->write - read=[UserID('a1'), UserID('w1')], # noop admin->read, noop write->read - remove=[UserID('rrem'), UserID('wrem')], # remove read and write + admin=[UserID("r1"), UserID("r2")], # write->admin, read->admin + write=[UserID("w2")], # noop write->write + read=[UserID("a1"), UserID("w1")], # noop admin->read, noop write->read + remove=[UserID("rrem"), UserID("wrem")], # remove read and write at_least=True, - public_read=True), - dt(90)) + public_read=True, + ), + dt(90), + ) res = samplestorage.get_sample_acls(id_) assert res == SampleACL( - UserID('user'), + UserID("user"), dt(90), - [UserID('a1'), UserID('a2'), UserID('r1'), UserID('r2')], - [UserID('w1'), UserID('w2')], + [UserID("a1"), UserID("a2"), UserID("r1"), UserID("r2")], + [UserID("w1"), UserID("w2")], [], - True) + True, + ) def test_update_sample_acls_fail_bad_args(samplestorage): @@ -1588,66 +2217,117 @@ def test_update_sample_acls_fail_bad_args(samplestorage): t = dt(1) _update_sample_acls_fail( - samplestorage, None, s, t, ValueError('id_ cannot be a value that evaluates to false')) + samplestorage, + None, + s, + t, + ValueError("id_ cannot be a value that evaluates to false"), + ) + _update_sample_acls_fail( + samplestorage, + id_, + None, + t, + ValueError("update cannot be a value that evaluates to false"), + ) + _update_sample_acls_fail( + samplestorage, + id_, + s, + None, + ValueError("update_time cannot be a value that evaluates to false"), + ) _update_sample_acls_fail( - samplestorage, id_, None, t, ValueError('update cannot be a value that evaluates to false')) - _update_sample_acls_fail(samplestorage, id_, s, None, ValueError( - 'update_time cannot be a value that evaluates to false')) - _update_sample_acls_fail(samplestorage, id_, s, datetime.datetime.fromtimestamp(1), ValueError( - 'update_time cannot be a naive datetime')) + samplestorage, + id_, + s, + datetime.datetime.fromtimestamp(1), + ValueError("update_time cannot be a naive datetime"), + ) def test_update_sample_acls_fail_no_sample(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) _update_sample_acls_fail( - samplestorage, uuid.UUID('1234567890abcdef1234567890abcde1'), SampleACLDelta(), dt(1), - NoSuchSampleError('12345678-90ab-cdef-1234-567890abcde1')) + samplestorage, + uuid.UUID("1234567890abcdef1234567890abcde1"), + SampleACLDelta(), + dt(1), + NoSuchSampleError("12345678-90ab-cdef-1234-567890abcde1"), + ) _update_sample_acls_fail( samplestorage, - uuid.UUID('1234567890abcdef1234567890abcde1'), + uuid.UUID("1234567890abcdef1234567890abcde1"), SampleACLDelta(at_least=True), dt(1), - NoSuchSampleError('12345678-90ab-cdef-1234-567890abcde1')) + NoSuchSampleError("12345678-90ab-cdef-1234-567890abcde1"), + ) def test_update_sample_acls_fail_alters_owner(samplestorage): - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('us'), [TEST_NODE], dt(1), 'foo')) is True + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("us"), [TEST_NODE], dt(1), "foo") + ) + is True + ) - err = UnauthorizedError('ACLs for the sample owner us may not be modified by a delta update.') + err = UnauthorizedError( + "ACLs for the sample owner us may not be modified by a delta update." + ) t = dt(1) - _update_sample_acls_fail(samplestorage, id_, SampleACLDelta([UserID('us')]), t, err) - _update_sample_acls_fail(samplestorage, id_, SampleACLDelta(write=[UserID('us')]), t, err) - _update_sample_acls_fail(samplestorage, id_, SampleACLDelta(read=[UserID('us')]), t, err) - _update_sample_acls_fail(samplestorage, id_, SampleACLDelta(remove=[UserID('us')]), t, err) + _update_sample_acls_fail(samplestorage, id_, SampleACLDelta([UserID("us")]), t, err) _update_sample_acls_fail( - samplestorage, id_, SampleACLDelta(remove=[UserID('us')], at_least=True), t, err) + samplestorage, id_, SampleACLDelta(write=[UserID("us")]), t, err + ) + _update_sample_acls_fail( + samplestorage, id_, SampleACLDelta(read=[UserID("us")]), t, err + ) + _update_sample_acls_fail( + samplestorage, id_, SampleACLDelta(remove=[UserID("us")]), t, err + ) + _update_sample_acls_fail( + samplestorage, id_, SampleACLDelta(remove=[UserID("us")], at_least=True), t, err + ) def test_update_sample_acls_fail_owner_changed(samplestorage): - ''' + """ This tests a race condition that could occur when the owner of a sample changes after the sample ACLs are pulled from Arango to check against the sample delta to ensure the owner is not altered by the delta. - ''' - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [TEST_NODE], dt(1), 'foo')) is True + """ + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [TEST_NODE], dt(1), "foo") + ) + is True + ) for al in [True, False]: with raises(Exception) as got: samplestorage._update_sample_acls_pt2( - id_, SampleACLDelta([UserID('a')], at_least=al), UserID('user2'), dt(1)) - assert_exception_correct(got.value, OwnerChangedError( - # we don't really ever expect this to happen, but just in case... - 'The sample owner unexpectedly changed during the operation. Please retry. ' + - 'If this error occurs frequently, code changes may be necessary.')) + id_, SampleACLDelta([UserID("a")], at_least=al), UserID("user2"), dt(1) + ) + assert_exception_correct( + got.value, + OwnerChangedError( + # we don't really ever expect this to happen, but just in case... + "The sample owner unexpectedly changed during the operation. Please retry. " + + "If this error occurs frequently, code changes may be necessary." + ), + ) def _update_sample_acls_fail(samplestorage, id_, update, update_time, expected): @@ -1657,630 +2337,772 @@ def _update_sample_acls_fail(samplestorage, id_, update, update_time, expected): def test_create_and_get_data_link(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdee') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(id1, UserID('user'), [SampleNode('mynode1')], dt(2), 'foo')) == 2 - assert samplestorage.save_sample( - SavedSample(id2, UserID('user'), [SampleNode('mynode2')], dt(3), 'foo')) is True - - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde1'), - DataUnitID(UPA('5/89/32')), - SampleNodeAddress(SampleAddress(id1, 2), 'mynode1'), - dt(500), - UserID('usera')) - ) is None + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdee") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(id1, UserID("user"), [SampleNode("mynode1")], dt(2), "foo") + ) + == 2 + ) + assert ( + samplestorage.save_sample( + SavedSample(id2, UserID("user"), [SampleNode("mynode2")], dt(3), "foo") + ) + is True + ) + + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde1"), + DataUnitID(UPA("5/89/32")), + SampleNodeAddress(SampleAddress(id1, 2), "mynode1"), + dt(500), + UserID("usera"), + ) + ) + is None + ) # test different workspace object and different sample version - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde2'), - DataUnitID(UPA('42/42/42'), 'dataunit1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(600), - UserID('userb')) - ) is None + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde2"), + DataUnitID(UPA("42/42/42"), "dataunit1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(600), + UserID("userb"), + ) + ) + is None + ) # test data unit vs just UPA, different sample, and expiration date - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde3'), - DataUnitID(UPA('5/89/32'), 'dataunit2'), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode2'), - dt(700), - UserID('u')) - ) is None + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde3"), + DataUnitID(UPA("5/89/32"), "dataunit2"), + SampleNodeAddress(SampleAddress(id2, 1), "mynode2"), + dt(700), + UserID("u"), + ) + ) + is None + ) # test data units don't collide if they have different names - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde4'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(800), - UserID('userd')) - ) is None + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde4"), + DataUnitID(UPA("5/89/32"), "dataunit1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(800), + UserID("userd"), + ) + ) + is None + ) # this is naughty - verdoc1 = samplestorage._col_version.find({'id': str(id1), 'ver': 1}).next() - verdoc2 = samplestorage._col_version.find({'id': str(id1), 'ver': 2}).next() - verdoc3 = samplestorage._col_version.find({'id': str(id2), 'ver': 1}).next() - nodedoc1 = samplestorage._col_nodes.find({'name': 'mynode'}).next() - nodedoc2 = samplestorage._col_nodes.find({'name': 'mynode1'}).next() - nodedoc3 = samplestorage._col_nodes.find({'name': 'mynode2'}).next() + verdoc1 = samplestorage._col_version.find({"id": str(id1), "ver": 1}).next() + verdoc2 = samplestorage._col_version.find({"id": str(id1), "ver": 2}).next() + verdoc3 = samplestorage._col_version.find({"id": str(id2), "ver": 1}).next() + nodedoc1 = samplestorage._col_nodes.find({"name": "mynode"}).next() + nodedoc2 = samplestorage._col_nodes.find({"name": "mynode1"}).next() + nodedoc3 = samplestorage._col_nodes.find({"name": "mynode2"}).next() assert samplestorage._col_data_link.count() == 4 # check arango documents correct, particularly _* values - link1 = samplestorage._col_data_link.get('5_89_32') + link1 = samplestorage._col_data_link.get("5_89_32") assert link1 == { - '_key': '5_89_32', - '_id': 'data_link/5_89_32', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc2['_id'], - '_rev': link1['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde1', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': None, - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc2['uuidver'], - 'samintver': 2, - 'node': 'mynode1', - 'created': 500000, - 'createby': 'usera', - 'expired': 9007199254740991, - 'expireby': None + "_key": "5_89_32", + "_id": "data_link/5_89_32", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc2["_id"], + "_rev": link1["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde1", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": None, + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc2["uuidver"], + "samintver": 2, + "node": "mynode1", + "created": 500000, + "createby": "usera", + "expired": 9007199254740991, + "expireby": None, } - link2 = samplestorage._col_data_link.get('42_42_42_bc7324de86d54718dd0dc29c55c6d53a') + link2 = samplestorage._col_data_link.get( + "42_42_42_bc7324de86d54718dd0dc29c55c6d53a" + ) assert link2 == { - '_key': '42_42_42_bc7324de86d54718dd0dc29c55c6d53a', - '_id': 'data_link/42_42_42_bc7324de86d54718dd0dc29c55c6d53a', - '_from': 'ws_obj_ver/42:42:42', - '_to': nodedoc1['_id'], - '_rev': link2['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde2', - 'wsid': 42, - 'objid': 42, - 'objver': 42, - 'dataid': 'dataunit1', - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode', - 'created': 600000, - 'createby': 'userb', - 'expired': 9007199254740991, - 'expireby': None + "_key": "42_42_42_bc7324de86d54718dd0dc29c55c6d53a", + "_id": "data_link/42_42_42_bc7324de86d54718dd0dc29c55c6d53a", + "_from": "ws_obj_ver/42:42:42", + "_to": nodedoc1["_id"], + "_rev": link2["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde2", + "wsid": 42, + "objid": 42, + "objver": 42, + "dataid": "dataunit1", + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode", + "created": 600000, + "createby": "userb", + "expired": 9007199254740991, + "expireby": None, } - link3 = samplestorage._col_data_link.get('5_89_32_3735ce9bbe59e7ec245da484772f9524') + link3 = samplestorage._col_data_link.get("5_89_32_3735ce9bbe59e7ec245da484772f9524") assert link3 == { - '_key': '5_89_32_3735ce9bbe59e7ec245da484772f9524', - '_id': 'data_link/5_89_32_3735ce9bbe59e7ec245da484772f9524', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc3['_id'], - '_rev': link3['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde3', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': 'dataunit2', - 'sampleid': '12345678-90ab-cdef-1234-567890abcdee', - 'samuuidver': verdoc3['uuidver'], - 'samintver': 1, - 'node': 'mynode2', - 'created': 700000, - 'createby': 'u', - 'expired': 9007199254740991, - 'expireby': None + "_key": "5_89_32_3735ce9bbe59e7ec245da484772f9524", + "_id": "data_link/5_89_32_3735ce9bbe59e7ec245da484772f9524", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc3["_id"], + "_rev": link3["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde3", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": "dataunit2", + "sampleid": "12345678-90ab-cdef-1234-567890abcdee", + "samuuidver": verdoc3["uuidver"], + "samintver": 1, + "node": "mynode2", + "created": 700000, + "createby": "u", + "expired": 9007199254740991, + "expireby": None, } - link4 = samplestorage._col_data_link.get('5_89_32_bc7324de86d54718dd0dc29c55c6d53a') + link4 = samplestorage._col_data_link.get("5_89_32_bc7324de86d54718dd0dc29c55c6d53a") assert link4 == { - '_key': '5_89_32_bc7324de86d54718dd0dc29c55c6d53a', - '_id': 'data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc1['_id'], - '_rev': link4['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde4', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': 'dataunit1', - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode', - 'created': 800000, - 'createby': 'userd', - 'expired': 9007199254740991, - 'expireby': None + "_key": "5_89_32_bc7324de86d54718dd0dc29c55c6d53a", + "_id": "data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc1["_id"], + "_rev": link4["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde4", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": "dataunit1", + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode", + "created": 800000, + "createby": "userd", + "expired": 9007199254740991, + "expireby": None, } # test get method - dl1 = samplestorage.get_data_link(uuid.UUID('12345678-90ab-cdef-1234-567890abcde1')) + dl1 = samplestorage.get_data_link(uuid.UUID("12345678-90ab-cdef-1234-567890abcde1")) assert dl1 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde1'), - DataUnitID(UPA('5/89/32')), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde1"), + DataUnitID(UPA("5/89/32")), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 2), - 'mynode1'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 2), + "mynode1", + ), dt(500), - UserID('usera') - ) + UserID("usera"), + ) - dl2 = samplestorage.get_data_link(duid=DataUnitID(UPA('42/42/42'), 'dataunit1')) + dl2 = samplestorage.get_data_link(duid=DataUnitID(UPA("42/42/42"), "dataunit1")) assert dl2 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde2'), - DataUnitID(UPA('42/42/42'), 'dataunit1'), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde2"), + DataUnitID(UPA("42/42/42"), "dataunit1"), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), - 'mynode'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode", + ), dt(600), - UserID('userb') - ) + UserID("userb"), + ) - dl3 = samplestorage.get_data_link(duid=DataUnitID(UPA('5/89/32'), 'dataunit2')) + dl3 = samplestorage.get_data_link(duid=DataUnitID(UPA("5/89/32"), "dataunit2")) assert dl3 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde3'), - DataUnitID(UPA('5/89/32'), 'dataunit2'), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde3"), + DataUnitID(UPA("5/89/32"), "dataunit2"), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdee'), 1), - 'mynode2'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdee"), 1), + "mynode2", + ), dt(700), - UserID('u') - ) + UserID("u"), + ) - dl4 = samplestorage.get_data_link(id_=uuid.UUID('12345678-90ab-cdef-1234-567890abcde4')) + dl4 = samplestorage.get_data_link( + id_=uuid.UUID("12345678-90ab-cdef-1234-567890abcde4") + ) assert dl4 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde4'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde4"), + DataUnitID(UPA("5/89/32"), "dataunit1"), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), - 'mynode'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode", + ), dt(800), - UserID('userd') - ) + UserID("userd"), + ) def test_creaate_data_link_with_update_no_extant_link(samplestorage): - ''' + """ Tests the case where an update is requested but is not necessary. - ''' - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample(SavedSample( - id1, UserID('user'), [SampleNode('mynode'), SampleNode('mynode1')], dt(1), 'foo')) is True - - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde1'), - DataUnitID(UPA('5/89/32')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('usera')), - update=True - ) is None - - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde2'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode1'), - dt(550), - UserID('user')), - update=True - ) is None + """ + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample( + id1, + UserID("user"), + [SampleNode("mynode"), SampleNode("mynode1")], + dt(1), + "foo", + ) + ) + is True + ) + + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde1"), + DataUnitID(UPA("5/89/32")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("usera"), + ), + update=True, + ) + is None + ) + + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde2"), + DataUnitID(UPA("5/89/32"), "dataunit1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode1"), + dt(550), + UserID("user"), + ), + update=True, + ) + is None + ) # this is naughty - verdoc1 = samplestorage._col_version.find({'id': str(id1), 'ver': 1}).next() - nodedoc1 = samplestorage._col_nodes.find({'name': 'mynode'}).next() - nodedoc2 = samplestorage._col_nodes.find({'name': 'mynode1'}).next() + verdoc1 = samplestorage._col_version.find({"id": str(id1), "ver": 1}).next() + nodedoc1 = samplestorage._col_nodes.find({"name": "mynode"}).next() + nodedoc2 = samplestorage._col_nodes.find({"name": "mynode1"}).next() assert samplestorage._col_data_link.count() == 2 # check arango documents correct, particularly _* values - link1 = samplestorage._col_data_link.get('5_89_32') + link1 = samplestorage._col_data_link.get("5_89_32") assert link1 == { - '_key': '5_89_32', - '_id': 'data_link/5_89_32', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc1['_id'], - '_rev': link1['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde1', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': None, - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode', - 'created': 500000, - 'createby': 'usera', - 'expired': 9007199254740991, - 'expireby': None + "_key": "5_89_32", + "_id": "data_link/5_89_32", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc1["_id"], + "_rev": link1["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde1", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": None, + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode", + "created": 500000, + "createby": "usera", + "expired": 9007199254740991, + "expireby": None, } - link2 = samplestorage._col_data_link.get('5_89_32_bc7324de86d54718dd0dc29c55c6d53a') + link2 = samplestorage._col_data_link.get("5_89_32_bc7324de86d54718dd0dc29c55c6d53a") assert link2 == { - '_key': '5_89_32_bc7324de86d54718dd0dc29c55c6d53a', - '_id': 'data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc2['_id'], - '_rev': link2['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde2', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': 'dataunit1', - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode1', - 'created': 550000, - 'createby': 'user', - 'expired': 9007199254740991, - 'expireby': None + "_key": "5_89_32_bc7324de86d54718dd0dc29c55c6d53a", + "_id": "data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc2["_id"], + "_rev": link2["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde2", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": "dataunit1", + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode1", + "created": 550000, + "createby": "user", + "expired": 9007199254740991, + "expireby": None, } # test get method - dl1 = samplestorage.get_data_link(uuid.UUID('12345678-90ab-cdef-1234-567890abcde1')) + dl1 = samplestorage.get_data_link(uuid.UUID("12345678-90ab-cdef-1234-567890abcde1")) assert dl1 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde1'), - DataUnitID(UPA('5/89/32')), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde1"), + DataUnitID(UPA("5/89/32")), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), - 'mynode'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode", + ), dt(500), - UserID('usera') - ) + UserID("usera"), + ) - dl2 = samplestorage.get_data_link(uuid.UUID('12345678-90ab-cdef-1234-567890abcde2')) + dl2 = samplestorage.get_data_link(uuid.UUID("12345678-90ab-cdef-1234-567890abcde2")) assert dl2 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde2'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde2"), + DataUnitID(UPA("5/89/32"), "dataunit1"), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), - 'mynode1'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode1", + ), dt(550), - UserID('user') - ) + UserID("user"), + ) def test_create_data_link_with_update_noop(samplestorage): - ''' + """ Tests the case where a link update is requested but an equivalent link already exists. - ''' - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample(SavedSample( - id1, UserID('user'), [SampleNode('mynode'), SampleNode('mynode1')], dt(1), 'foo')) is True - - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde1'), - DataUnitID(UPA('5/89/32')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('usera')) - ) is None + """ + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample( + id1, + UserID("user"), + [SampleNode("mynode"), SampleNode("mynode1")], + dt(1), + "foo", + ) + ) + is True + ) - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde2'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode1'), - dt(550), - UserID('user')) - ) is None + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde1"), + DataUnitID(UPA("5/89/32")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("usera"), + ) + ) + is None + ) + + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde2"), + DataUnitID(UPA("5/89/32"), "dataunit1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode1"), + dt(550), + UserID("user"), + ) + ) + is None + ) # expect noop - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde3'), - DataUnitID(UPA('5/89/32')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(600), - UserID('userb')), - update=True - ) is None + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde3"), + DataUnitID(UPA("5/89/32")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(600), + UserID("userb"), + ), + update=True, + ) + is None + ) # expect noop - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde4'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode1'), - dt(700), - UserID('userc')), - update=True - ) is None + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde4"), + DataUnitID(UPA("5/89/32"), "dataunit1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode1"), + dt(700), + UserID("userc"), + ), + update=True, + ) + is None + ) # this is naughty - verdoc1 = samplestorage._col_version.find({'id': str(id1), 'ver': 1}).next() - nodedoc1 = samplestorage._col_nodes.find({'name': 'mynode'}).next() - nodedoc2 = samplestorage._col_nodes.find({'name': 'mynode1'}).next() + verdoc1 = samplestorage._col_version.find({"id": str(id1), "ver": 1}).next() + nodedoc1 = samplestorage._col_nodes.find({"name": "mynode"}).next() + nodedoc2 = samplestorage._col_nodes.find({"name": "mynode1"}).next() assert samplestorage._col_data_link.count() == 2 # check arango documents correct, particularly _* values - link1 = samplestorage._col_data_link.get('5_89_32') + link1 = samplestorage._col_data_link.get("5_89_32") assert link1 == { - '_key': '5_89_32', - '_id': 'data_link/5_89_32', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc1['_id'], - '_rev': link1['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde1', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': None, - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode', - 'created': 500000, - 'createby': 'usera', - 'expired': 9007199254740991, - 'expireby': None + "_key": "5_89_32", + "_id": "data_link/5_89_32", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc1["_id"], + "_rev": link1["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde1", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": None, + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode", + "created": 500000, + "createby": "usera", + "expired": 9007199254740991, + "expireby": None, } - link2 = samplestorage._col_data_link.get('5_89_32_bc7324de86d54718dd0dc29c55c6d53a') + link2 = samplestorage._col_data_link.get("5_89_32_bc7324de86d54718dd0dc29c55c6d53a") assert link2 == { - '_key': '5_89_32_bc7324de86d54718dd0dc29c55c6d53a', - '_id': 'data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc2['_id'], - '_rev': link2['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde2', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': 'dataunit1', - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode1', - 'created': 550000, - 'createby': 'user', - 'expired': 9007199254740991, - 'expireby': None + "_key": "5_89_32_bc7324de86d54718dd0dc29c55c6d53a", + "_id": "data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc2["_id"], + "_rev": link2["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde2", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": "dataunit1", + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode1", + "created": 550000, + "createby": "user", + "expired": 9007199254740991, + "expireby": None, } # test get method - dl1 = samplestorage.get_data_link(uuid.UUID('12345678-90ab-cdef-1234-567890abcde1')) + dl1 = samplestorage.get_data_link(uuid.UUID("12345678-90ab-cdef-1234-567890abcde1")) assert dl1 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde1'), - DataUnitID(UPA('5/89/32')), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde1"), + DataUnitID(UPA("5/89/32")), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), - 'mynode'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode", + ), dt(500), - UserID('usera') - ) + UserID("usera"), + ) - dl2 = samplestorage.get_data_link(uuid.UUID('12345678-90ab-cdef-1234-567890abcde2')) + dl2 = samplestorage.get_data_link(uuid.UUID("12345678-90ab-cdef-1234-567890abcde2")) assert dl2 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde2'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde2"), + DataUnitID(UPA("5/89/32"), "dataunit1"), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), - 'mynode1'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode1", + ), dt(550), - UserID('user') - ) + UserID("user"), + ) def test_create_data_link_with_update(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample(SavedSample( - id1, UserID('user'), [SampleNode('mynode'), SampleNode('mynode1'), SampleNode('mynode2')], - dt(1), 'foo')) is True - - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde1'), - DataUnitID(UPA('5/89/32')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('usera')) - ) is None + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample( + id1, + UserID("user"), + [SampleNode("mynode"), SampleNode("mynode1"), SampleNode("mynode2")], + dt(1), + "foo", + ) + ) + is True + ) - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde2'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode1'), - dt(550), - UserID('user')) - ) is None + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde1"), + DataUnitID(UPA("5/89/32")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("usera"), + ) + ) + is None + ) - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde3'), - DataUnitID(UPA('5/89/32')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode1'), # update the node - dt(600), - UserID('userb')), - update=True - ) == uuid.UUID('1234567890abcdef1234567890abcde1') - - assert samplestorage.create_data_link(DataLink( - uuid.UUID('1234567890abcdef1234567890abcde4'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode2'), # update the node - dt(700), - UserID('userc')), - update=True - ) == uuid.UUID('1234567890abcdef1234567890abcde2') + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde2"), + DataUnitID(UPA("5/89/32"), "dataunit1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode1"), + dt(550), + UserID("user"), + ) + ) + is None + ) + + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde3"), + DataUnitID(UPA("5/89/32")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode1"), # update the node + dt(600), + UserID("userb"), + ), + update=True, + ) + == uuid.UUID("1234567890abcdef1234567890abcde1") + ) + + assert ( + samplestorage.create_data_link( + DataLink( + uuid.UUID("1234567890abcdef1234567890abcde4"), + DataUnitID(UPA("5/89/32"), "dataunit1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode2"), # update the node + dt(700), + UserID("userc"), + ), + update=True, + ) + == uuid.UUID("1234567890abcdef1234567890abcde2") + ) # this is naughty - verdoc1 = samplestorage._col_version.find({'id': str(id1), 'ver': 1}).next() - nodedoc1 = samplestorage._col_nodes.find({'name': 'mynode'}).next() - nodedoc2 = samplestorage._col_nodes.find({'name': 'mynode1'}).next() - nodedoc3 = samplestorage._col_nodes.find({'name': 'mynode2'}).next() + verdoc1 = samplestorage._col_version.find({"id": str(id1), "ver": 1}).next() + nodedoc1 = samplestorage._col_nodes.find({"name": "mynode"}).next() + nodedoc2 = samplestorage._col_nodes.find({"name": "mynode1"}).next() + nodedoc3 = samplestorage._col_nodes.find({"name": "mynode2"}).next() assert samplestorage._col_data_link.count() == 4 # check arango documents correct, particularly _* values - link1 = samplestorage._col_data_link.get('5_89_32_500.0') + link1 = samplestorage._col_data_link.get("5_89_32_500.0") assert link1 == { - '_key': '5_89_32_500.0', - '_id': 'data_link/5_89_32_500.0', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc1['_id'], - '_rev': link1['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde1', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': None, - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode', - 'created': 500000, - 'createby': 'usera', - 'expired': 599999, - 'expireby': 'userb' + "_key": "5_89_32_500.0", + "_id": "data_link/5_89_32_500.0", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc1["_id"], + "_rev": link1["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde1", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": None, + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode", + "created": 500000, + "createby": "usera", + "expired": 599999, + "expireby": "userb", } - link2 = samplestorage._col_data_link.get('5_89_32_bc7324de86d54718dd0dc29c55c6d53a_550.0') + link2 = samplestorage._col_data_link.get( + "5_89_32_bc7324de86d54718dd0dc29c55c6d53a_550.0" + ) assert link2 == { - '_key': '5_89_32_bc7324de86d54718dd0dc29c55c6d53a_550.0', - '_id': 'data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a_550.0', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc2['_id'], - '_rev': link2['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde2', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': 'dataunit1', - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode1', - 'created': 550000, - 'createby': 'user', - 'expired': 699999, - 'expireby': 'userc' + "_key": "5_89_32_bc7324de86d54718dd0dc29c55c6d53a_550.0", + "_id": "data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a_550.0", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc2["_id"], + "_rev": link2["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde2", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": "dataunit1", + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode1", + "created": 550000, + "createby": "user", + "expired": 699999, + "expireby": "userc", } - link3 = samplestorage._col_data_link.get('5_89_32') + link3 = samplestorage._col_data_link.get("5_89_32") assert link3 == { - '_key': '5_89_32', - '_id': 'data_link/5_89_32', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc2['_id'], - '_rev': link3['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde3', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': None, - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode1', - 'created': 600000, - 'createby': 'userb', - 'expired': 9007199254740991, - 'expireby': None + "_key": "5_89_32", + "_id": "data_link/5_89_32", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc2["_id"], + "_rev": link3["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde3", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": None, + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode1", + "created": 600000, + "createby": "userb", + "expired": 9007199254740991, + "expireby": None, } - link4 = samplestorage._col_data_link.get('5_89_32_bc7324de86d54718dd0dc29c55c6d53a') + link4 = samplestorage._col_data_link.get("5_89_32_bc7324de86d54718dd0dc29c55c6d53a") assert link4 == { - '_key': '5_89_32_bc7324de86d54718dd0dc29c55c6d53a', - '_id': 'data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a', - '_from': 'ws_obj_ver/5:89:32', - '_to': nodedoc3['_id'], - '_rev': link4['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde4', - 'wsid': 5, - 'objid': 89, - 'objver': 32, - 'dataid': 'dataunit1', - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode2', - 'created': 700000, - 'createby': 'userc', - 'expired': 9007199254740991, - 'expireby': None + "_key": "5_89_32_bc7324de86d54718dd0dc29c55c6d53a", + "_id": "data_link/5_89_32_bc7324de86d54718dd0dc29c55c6d53a", + "_from": "ws_obj_ver/5:89:32", + "_to": nodedoc3["_id"], + "_rev": link4["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde4", + "wsid": 5, + "objid": 89, + "objver": 32, + "dataid": "dataunit1", + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode2", + "created": 700000, + "createby": "userc", + "expired": 9007199254740991, + "expireby": None, } # test get method. Expired, so DUID won't work here - dl1 = samplestorage.get_data_link(uuid.UUID('12345678-90ab-cdef-1234-567890abcde1')) + dl1 = samplestorage.get_data_link(uuid.UUID("12345678-90ab-cdef-1234-567890abcde1")) assert dl1 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde1'), - DataUnitID(UPA('5/89/32')), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde1"), + DataUnitID(UPA("5/89/32")), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), 'mynode'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode", + ), dt(500), - UserID('usera'), + UserID("usera"), dt(599.999), - UserID('userb') - ) + UserID("userb"), + ) - dl2 = samplestorage.get_data_link(uuid.UUID('12345678-90ab-cdef-1234-567890abcde2')) + dl2 = samplestorage.get_data_link(uuid.UUID("12345678-90ab-cdef-1234-567890abcde2")) assert dl2 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde2'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde2"), + DataUnitID(UPA("5/89/32"), "dataunit1"), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), 'mynode1'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode1", + ), dt(550), - UserID('user'), + UserID("user"), dt(699.999), - UserID('userc') - ) + UserID("userc"), + ) - dl3 = samplestorage.get_data_link(duid=DataUnitID(UPA('5/89/32'))) + dl3 = samplestorage.get_data_link(duid=DataUnitID(UPA("5/89/32"))) assert dl3 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde3'), - DataUnitID(UPA('5/89/32')), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde3"), + DataUnitID(UPA("5/89/32")), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), 'mynode1'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode1", + ), dt(600), - UserID('userb') - ) + UserID("userb"), + ) - dl3 = samplestorage.get_data_link(uuid.UUID('12345678-90ab-cdef-1234-567890abcde4')) + dl3 = samplestorage.get_data_link(uuid.UUID("12345678-90ab-cdef-1234-567890abcde4")) assert dl3 == DataLink( - uuid.UUID('12345678-90ab-cdef-1234-567890abcde4'), - DataUnitID(UPA('5/89/32'), 'dataunit1'), + uuid.UUID("12345678-90ab-cdef-1234-567890abcde4"), + DataUnitID(UPA("5/89/32"), "dataunit1"), SampleNodeAddress( - SampleAddress(uuid.UUID('12345678-90ab-cdef-1234-567890abcdef'), 1), 'mynode2'), + SampleAddress(uuid.UUID("12345678-90ab-cdef-1234-567890abcdef"), 1), + "mynode2", + ), dt(700), - UserID('userc'), - ) + UserID("userc"), + ) def test_create_data_link_correct_missing_versions(samplestorage): - ''' + """ Checks that the version correction code runs when needed on creating a data link. Since the method is tested extensively in the get_sample tests, we only run one test here to ensure the method is called. This test simulates a server coming up after a dirty shutdown, where version and node doc integer versions have not been updated - ''' - n1 = SampleNode('root') - n2 = SampleNode('kid1', SubSampleType.TECHNICAL_REPLICATE, 'root') - n3 = SampleNode('kid2', SubSampleType.SUB_SAMPLE, 'kid1') - n4 = SampleNode('kid3', SubSampleType.TECHNICAL_REPLICATE, 'root') + """ + n1 = SampleNode("root") + n2 = SampleNode("kid1", SubSampleType.TECHNICAL_REPLICATE, "root") + n3 = SampleNode("kid2", SubSampleType.SUB_SAMPLE, "kid1") + n4 = SampleNode("kid3", SubSampleType.TECHNICAL_REPLICATE, "root") - id_ = uuid.UUID('1234567890abcdef1234567890abcdef') + id_ = uuid.UUID("1234567890abcdef1234567890abcdef") - assert samplestorage.save_sample( - SavedSample(id_, UserID('user'), [n1, n2, n3, n4], dt(1), 'foo')) is True + assert ( + samplestorage.save_sample( + SavedSample(id_, UserID("user"), [n1, n2, n3, n4], dt(1), "foo") + ) + is True + ) # this is very naughty # checked that these modifications actually work by viewing the db contents - samplestorage._col_version.update_match({}, {'ver': -1}) - samplestorage._col_nodes.update_match({'name': 'kid2'}, {'ver': -1}) - - assert samplestorage.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('5/89/32')), - SampleNodeAddress(SampleAddress(id_, 1), 'kid1'), - dt(500), - UserID('user')) - ) is None + samplestorage._col_version.update_match({}, {"ver": -1}) + samplestorage._col_nodes.update_match({"name": "kid2"}, {"ver": -1}) + + assert ( + samplestorage.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("5/89/32")), + SampleNodeAddress(SampleAddress(id_, 1), "kid1"), + dt(500), + UserID("user"), + ) + ) + is None + ) assert samplestorage._col_version.count() == 1 assert samplestorage._col_ver_edge.count() == 1 @@ -2288,237 +3110,320 @@ def test_create_data_link_correct_missing_versions(samplestorage): assert samplestorage._col_node_edge.count() == 4 for v in samplestorage._col_version.all(): - assert v['ver'] == 1 + assert v["ver"] == 1 for v in samplestorage._col_nodes.all(): - assert v['ver'] == 1 + assert v["ver"] == 1 def test_create_data_link_fail_no_link(samplestorage): - _create_data_link_fail(samplestorage, None, ValueError( - 'link cannot be a value that evaluates to false')) + _create_data_link_fail( + samplestorage, + None, + ValueError("link cannot be a value that evaluates to false"), + ) def test_create_data_link_fail_expired(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) _create_data_link_fail( samplestorage, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(-100), - UserID('user'), + UserID("user"), dt(0), - UserID('user')), - ValueError('link cannot be expired') - ) + UserID("user"), + ), + ValueError("link cannot be expired"), + ) def test_create_data_link_fail_no_sample(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdee') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdee") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) _create_data_link_fail( samplestorage, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id2, 1), "mynode"), dt(1), - UserID('user')), - NoSuchSampleError(str(id2)) - ) + UserID("user"), + ), + NoSuchSampleError(str(id2)), + ) def test_create_data_link_fail_no_sample_version(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(id1, UserID('user'), [SampleNode('mynode1')], dt(2), 'foo')) == 2 + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(id1, UserID("user"), [SampleNode("mynode1")], dt(2), "foo") + ) + == 2 + ) _create_data_link_fail( samplestorage, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 3), 'mynode'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 3), "mynode"), dt(1), - UserID('user')), - NoSuchSampleVersionError('12345678-90ab-cdef-1234-567890abcdef ver 3') - ) + UserID("user"), + ), + NoSuchSampleVersionError("12345678-90ab-cdef-1234-567890abcdef ver 3"), + ) def test_create_data_link_fail_no_sample_node(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample(SavedSample( - id1, UserID('user'), [SampleNode('mynode'), SampleNode('mynode1')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample( + id1, + UserID("user"), + [SampleNode("mynode"), SampleNode("mynode1")], + dt(1), + "foo", + ) + ) + is True + ) _create_data_link_fail( samplestorage, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode2'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode2"), dt(1), - UserID('user')), - NoSuchSampleNodeError('12345678-90ab-cdef-1234-567890abcdef ver 1 mynode2') - ) + UserID("user"), + ), + NoSuchSampleNodeError("12345678-90ab-cdef-1234-567890abcdef ver 1 mynode2"), + ) def test_create_data_link_fail_link_exists(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - samplestorage.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('user')) + samplestorage.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("user"), + ) ) - samplestorage.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1'), 'du1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('user')) + samplestorage.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1"), "du1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("user"), + ) ) _create_data_link_fail( samplestorage, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(1), - UserID('user')), - DataLinkExistsError('1/1/1') - ) + UserID("user"), + ), + DataLinkExistsError("1/1/1"), + ) _create_data_link_fail( samplestorage, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1'), 'du1'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), "du1"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(1), - UserID('user')), - DataLinkExistsError('1/1/1:du1') - ) + UserID("user"), + ), + DataLinkExistsError("1/1/1:du1"), + ) def test_create_data_link_fail_too_many_links_from_ws_obj_basic(samplestorage): ss = _samplestorage_with_max_links(samplestorage, 3) - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcde3') - assert ss.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True - assert ss.save_sample( - SavedSample(id2, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcde3") + assert ( + ss.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) + assert ( + ss.save_sample( + SavedSample(id2, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("user"), + ) ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '1'), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1"), "1"), + SampleNodeAddress(SampleAddress(id2, 1), "mynode"), + dt(500), + UserID("user"), + ) ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '2'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1"), "2"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("user"), + ) ) _create_data_link_fail( ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '3'), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), "3"), + SampleNodeAddress(SampleAddress(id2, 1), "mynode"), dt(1), - UserID('user')), - TooManyDataLinksError('More than 3 links from workspace object 1/1/1') - ) + UserID("user"), + ), + TooManyDataLinksError("More than 3 links from workspace object 1/1/1"), + ) def test_create_data_link_fail_too_many_links_from_sample_ver_basic(samplestorage): ss = _samplestorage_with_max_links(samplestorage, 2) - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert ss.save_sample(SavedSample( - id1, UserID('user'), [SampleNode('mynode'), SampleNode('mynode2')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + ss.save_sample( + SavedSample( + id1, + UserID("user"), + [SampleNode("mynode"), SampleNode("mynode2")], + dt(1), + "foo", + ) + ) + is True + ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("user"), + ) ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/2')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode2'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/2")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode2"), + dt(500), + UserID("user"), + ) ) _create_data_link_fail( ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/3')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/3")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(1), - UserID('user')), + UserID("user"), + ), TooManyDataLinksError( - 'More than 2 links from sample 12345678-90ab-cdef-1234-567890abcdef version 1') - ) + "More than 2 links from sample 12345678-90ab-cdef-1234-567890abcdef version 1" + ), + ) def test_create_data_link_fail_too_many_links_from_ws_obj_time_travel(samplestorage): # tests that links that do not co-exist with the new link are not counted against the total. ss = _samplestorage_with_max_links(samplestorage, 3) - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert ss.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True - assert ss.save_sample_version( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) == 2 + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + ss.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) + assert ( + ss.save_sample_version( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + == 2 + ) # completely outside the new sample time range. _create_and_expire_data_link( ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(100), - UserID('user')), + UserID("user"), + ), dt(299), - UserID('user') + UserID("user"), ) # expire matches create @@ -2526,12 +3431,13 @@ def test_create_data_link_fail_too_many_links_from_ws_obj_time_travel(samplestor ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '1'), - SampleNodeAddress(SampleAddress(id1, 2), 'mynode'), + DataUnitID(UPA("1/1/1"), "1"), + SampleNodeAddress(SampleAddress(id1, 2), "mynode"), dt(100), - UserID('user')), + UserID("user"), + ), dt(300), - UserID('user') + UserID("user"), ) # overlaps create @@ -2539,12 +3445,13 @@ def test_create_data_link_fail_too_many_links_from_ws_obj_time_travel(samplestor ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '2'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), "2"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(250), - UserID('user')), + UserID("user"), + ), dt(350), - UserID('user') + UserID("user"), ) # contained inside @@ -2552,45 +3459,54 @@ def test_create_data_link_fail_too_many_links_from_ws_obj_time_travel(samplestor ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '3'), - SampleNodeAddress(SampleAddress(id1, 2), 'mynode'), + DataUnitID(UPA("1/1/1"), "3"), + SampleNodeAddress(SampleAddress(id1, 2), "mynode"), dt(325), - UserID('user')), + UserID("user"), + ), dt(375), - UserID('user') + UserID("user"), ) _create_data_link_fail( ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '8'), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), "8"), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(300), - UserID('user')), - TooManyDataLinksError('More than 3 links from workspace object 1/1/1') - ) + UserID("user"), + ), + TooManyDataLinksError("More than 3 links from workspace object 1/1/1"), + ) -def test_create_data_link_fail_too_many_links_from_sample_ver_time_travel(samplestorage): +def test_create_data_link_fail_too_many_links_from_sample_ver_time_travel( + samplestorage, +): # tests that links that do not co-exist with the new link are not counted against the total. ss = _samplestorage_with_max_links(samplestorage, 3) - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert ss.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + ss.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) # completely outside the new sample time range. _create_and_expire_data_link( ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(100), - UserID('user')), + UserID("user"), + ), dt(299), - UserID('user') + UserID("user"), ) # expire matches create @@ -2598,12 +3514,13 @@ def test_create_data_link_fail_too_many_links_from_sample_ver_time_travel(sample ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/2')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/2")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(100), - UserID('user')), + UserID("user"), + ), dt(300), - UserID('user') + UserID("user"), ) # overlaps create @@ -2611,12 +3528,13 @@ def test_create_data_link_fail_too_many_links_from_sample_ver_time_travel(sample ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/3')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/3")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(250), - UserID('user')), + UserID("user"), + ), dt(350), - UserID('user') + UserID("user"), ) # contained inside @@ -2624,128 +3542,173 @@ def test_create_data_link_fail_too_many_links_from_sample_ver_time_travel(sample ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/4')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/4")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(325), - UserID('user')), + UserID("user"), + ), dt(375), - UserID('user') + UserID("user"), ) _create_data_link_fail( ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/9')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), + DataUnitID(UPA("1/1/9")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), dt(300), - UserID('user')), - TooManyDataLinksError('More than 3 links from sample ' + - '12345678-90ab-cdef-1234-567890abcdef version 1') - ) + UserID("user"), + ), + TooManyDataLinksError( + "More than 3 links from sample " + + "12345678-90ab-cdef-1234-567890abcdef version 1" + ), + ) def test_create_data_link_update_links_from_ws_object_count_limit(samplestorage): - ''' + """ Tests that replacing a link doesn't trigger the link count limit from ws objects when it's at the max - ''' + """ ss = _samplestorage_with_max_links(samplestorage, 2) - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcde3') - assert ss.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True - assert ss.save_sample( - SavedSample(id2, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcde3") + assert ( + ss.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) + assert ( + ss.save_sample( + SavedSample(id2, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("user"), + ) ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '1'), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1"), "1"), + SampleNodeAddress(SampleAddress(id2, 1), "mynode"), + dt(500), + UserID("user"), + ) ) # should pass. Changing the data ID causes a fail - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '1'), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode'), - dt(500), - UserID('user')), - update=True + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1"), "1"), + SampleNodeAddress(SampleAddress(id2, 1), "mynode"), + dt(500), + UserID("user"), + ), + update=True, ) _create_data_link_fail( ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '2'), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), "2"), + SampleNodeAddress(SampleAddress(id2, 1), "mynode"), dt(500), - UserID('user')), - TooManyDataLinksError('More than 2 links from workspace object 1/1/1') - ) + UserID("user"), + ), + TooManyDataLinksError("More than 2 links from workspace object 1/1/1"), + ) def test_create_data_link_update_links_from_sample_ver_count_limit(samplestorage): - ''' + """ Tests updating a link with regard to the limit on number of links to a sample version. Specifically, updating a link to a different node of the same sample version should not cause an error, while violating the limit for a new sample version should cause an error. - ''' + """ ss = _samplestorage_with_max_links(samplestorage, 1) - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdee') - assert samplestorage.save_sample(SavedSample( - id1, UserID('user'), [SampleNode('mynode'), SampleNode('XmynodeX')], dt(1), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(id1, UserID('user'), [SampleNode('mynode1')], dt(2), 'foo')) == 2 - assert samplestorage.save_sample( - SavedSample(id2, UserID('user'), [SampleNode('mynode2')], dt(3), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdee") + assert ( + samplestorage.save_sample( + SavedSample( + id1, + UserID("user"), + [SampleNode("mynode"), SampleNode("XmynodeX")], + dt(1), + "foo", + ) + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(id1, UserID("user"), [SampleNode("mynode1")], dt(2), "foo") + ) + == 2 + ) + assert ( + samplestorage.save_sample( + SavedSample(id2, UserID("user"), [SampleNode("mynode2")], dt(3), "foo") + ) + is True + ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'mynode'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "mynode"), + dt(500), + UserID("user"), + ) ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/2')), - SampleNodeAddress(SampleAddress(id1, 2), 'mynode1'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/2")), + SampleNodeAddress(SampleAddress(id1, 2), "mynode1"), + dt(500), + UserID("user"), + ) ) - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/3')), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode2'), - dt(500), - UserID('user')) + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/3")), + SampleNodeAddress(SampleAddress(id2, 1), "mynode2"), + dt(500), + UserID("user"), + ) ) # Should not trigger an error - replacing a link to the same sample version. - ss.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 1), 'XmynodeX'), - dt(600), - UserID('user')), - update=True + ss.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 1), "XmynodeX"), + dt(600), + UserID("user"), + ), + update=True, ) # fail on version change @@ -2753,28 +3716,32 @@ def test_create_data_link_update_links_from_sample_ver_count_limit(samplestorage ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id1, 2), 'mynode1'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id1, 2), "mynode1"), dt(700), - UserID('user')), + UserID("user"), + ), TooManyDataLinksError( - 'More than 1 links from sample 12345678-90ab-cdef-1234-567890abcdef version 2'), - update=True - ) + "More than 1 links from sample 12345678-90ab-cdef-1234-567890abcdef version 2" + ), + update=True, + ) # fail on sample change _create_data_link_fail( ss, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(id2, 1), 'mynode2'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(id2, 1), "mynode2"), dt(700), - UserID('user')), + UserID("user"), + ), TooManyDataLinksError( - 'More than 1 links from sample 12345678-90ab-cdef-1234-567890abcdee version 1'), - update=True - ) + "More than 1 links from sample 12345678-90ab-cdef-1234-567890abcdee version 1" + ), + update=True, + ) def _create_and_expire_data_link(samplestorage, link, expired, user): @@ -2794,7 +3761,8 @@ def _samplestorage_with_max_links(samplestorage, max_links): samplestorage._col_ws.name, samplestorage._col_data_link.name, samplestorage._col_schema.name, - max_links=max_links) + max_links=max_links, + ) def _create_data_link_fail(samplestorage, link, expected, update=False): @@ -2804,87 +3772,123 @@ def _create_data_link_fail(samplestorage, link, expected, update=False): def test_get_data_link_fail_no_bad_args(samplestorage): - _get_data_link_fail(samplestorage, None, None, ValueError( - 'exactly one of id_ or duid must be provided')) - _get_data_link_fail(samplestorage, uuid.uuid4(), DataUnitID('1/1/1'), ValueError( - 'exactly one of id_ or duid must be provided')) + _get_data_link_fail( + samplestorage, + None, + None, + ValueError("exactly one of id_ or duid must be provided"), + ) + _get_data_link_fail( + samplestorage, + uuid.uuid4(), + DataUnitID("1/1/1"), + ValueError("exactly one of id_ or duid must be provided"), + ) def test_get_data_link_fail_no_link(samplestorage): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid, - DataUnitID(UPA('1/1/1'), 'a'), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(100), - UserID('user')) + lid = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid, + DataUnitID(UPA("1/1/1"), "a"), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(100), + UserID("user"), + ) ) _get_data_link_fail( samplestorage, - uuid.UUID('1234567890abcdef1234567890abcde2'), + uuid.UUID("1234567890abcdef1234567890abcde2"), None, - NoSuchLinkError('12345678-90ab-cdef-1234-567890abcde2')) + NoSuchLinkError("12345678-90ab-cdef-1234-567890abcde2"), + ) - _get_data_link_fail(samplestorage, None, DataUnitID(UPA('1/2/1'), 'a'), - NoSuchLinkError('1/2/1:a')) - _get_data_link_fail(samplestorage, None, DataUnitID(UPA('1/1/1'), 'b'), - NoSuchLinkError('1/1/1:b')) + _get_data_link_fail( + samplestorage, None, DataUnitID(UPA("1/2/1"), "a"), NoSuchLinkError("1/2/1:a") + ) + _get_data_link_fail( + samplestorage, None, DataUnitID(UPA("1/1/1"), "b"), NoSuchLinkError("1/1/1:b") + ) def test_get_data_link_fail_expired_link(samplestorage): - ''' + """ Only fails for DUID based fetch - ''' - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + """ + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid = uuid.UUID('1234567890abcdef1234567890abcde1') + lid = uuid.UUID("1234567890abcdef1234567890abcde1") _create_and_expire_data_link( samplestorage, DataLink( lid, - DataUnitID(UPA('1/1/1'), 'a'), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), "a"), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), dt(100), - UserID('user')), + UserID("user"), + ), dt(600), - UserID('f') + UserID("f"), ) - _get_data_link_fail(samplestorage, None, DataUnitID(UPA('1/1/1'), 'a'), - NoSuchLinkError('1/1/1:a')) + _get_data_link_fail( + samplestorage, None, DataUnitID(UPA("1/1/1"), "a"), NoSuchLinkError("1/1/1:a") + ) def test_get_data_link_fail_too_many_links(samplestorage): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid, - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(100), - UserID('user')) + lid = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid, + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(100), + UserID("user"), + ) ) - samplestorage.create_data_link(DataLink( - lid, - DataUnitID(UPA('1/1/2')), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(100), - UserID('user')) + samplestorage.create_data_link( + DataLink( + lid, + DataUnitID(UPA("1/1/2")), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(100), + UserID("user"), + ) ) _get_data_link_fail( - samplestorage, lid, None, SampleStorageError( - 'More than one data link found for ID 12345678-90ab-cdef-1234-567890abcde1')) + samplestorage, + lid, + None, + SampleStorageError( + "More than one data link found for ID 12345678-90ab-cdef-1234-567890abcde1" + ), + ) def _get_data_link_fail(samplestorage, id_, duid, expected): @@ -2894,147 +3898,165 @@ def _get_data_link_fail(samplestorage, id_, duid, expected): def test_expire_and_get_data_link_via_duid(samplestorage): - _expire_and_get_data_link_via_duid(samplestorage, 600, None, '') + _expire_and_get_data_link_via_duid(samplestorage, 600, None, "") def test_expire_and_get_data_link_via_duid_with_dataid(samplestorage): _expire_and_get_data_link_via_duid( - samplestorage, -100, 'foo', 'acbd18db4cc2f85cedef654fccc4a4d8_') + samplestorage, -100, "foo", "acbd18db4cc2f85cedef654fccc4a4d8_" + ) def _expire_and_get_data_link_via_duid(samplestorage, expired, dataid, expectedmd5): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid, - DataUnitID(UPA('1/1/1'), dataid), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(-100), - UserID('userb')) + lid = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid, + DataUnitID(UPA("1/1/1"), dataid), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(-100), + UserID("userb"), + ) ) # this is naughty - verdoc1 = samplestorage._col_version.find({'id': str(sid), 'ver': 1}).next() - nodedoc1 = samplestorage._col_nodes.find({'name': 'mynode'}).next() + verdoc1 = samplestorage._col_version.find({"id": str(sid), "ver": 1}).next() + nodedoc1 = samplestorage._col_nodes.find({"name": "mynode"}).next() assert samplestorage.expire_data_link( - dt(expired), UserID('yay'), duid=DataUnitID(UPA('1/1/1'), dataid)) == DataLink( - lid, - DataUnitID(UPA('1/1/1'), dataid), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(-100), - UserID('userb'), - dt(expired), - UserID('yay') - ) + dt(expired), UserID("yay"), duid=DataUnitID(UPA("1/1/1"), dataid) + ) == DataLink( + lid, + DataUnitID(UPA("1/1/1"), dataid), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(-100), + UserID("userb"), + dt(expired), + UserID("yay"), + ) assert samplestorage._col_data_link.count() == 1 - link = samplestorage._col_data_link.get(f'1_1_1_{expectedmd5}-100.0') + link = samplestorage._col_data_link.get(f"1_1_1_{expectedmd5}-100.0") assert link == { - '_key': f'1_1_1_{expectedmd5}-100.0', - '_id': f'data_link/1_1_1_{expectedmd5}-100.0', - '_from': 'ws_obj_ver/1:1:1', - '_to': nodedoc1['_id'], - '_rev': link['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde1', - 'wsid': 1, - 'objid': 1, - 'objver': 1, - 'dataid': dataid, - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode', - 'created': -100000, - 'createby': 'userb', - 'expired': expired * 1000, - 'expireby': 'yay' + "_key": f"1_1_1_{expectedmd5}-100.0", + "_id": f"data_link/1_1_1_{expectedmd5}-100.0", + "_from": "ws_obj_ver/1:1:1", + "_to": nodedoc1["_id"], + "_rev": link["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde1", + "wsid": 1, + "objid": 1, + "objver": 1, + "dataid": dataid, + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode", + "created": -100000, + "createby": "userb", + "expired": expired * 1000, + "expireby": "yay", } assert samplestorage.get_data_link(lid) == DataLink( lid, - DataUnitID(UPA('1/1/1'), dataid), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), dataid), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), dt(-100), - UserID('userb'), + UserID("userb"), dt(expired), - UserID('yay') + UserID("yay"), ) def test_expire_and_get_data_link_via_id(samplestorage): - _expire_and_get_data_link_via_id(samplestorage, 1000, None, '') + _expire_and_get_data_link_via_id(samplestorage, 1000, None, "") def test_expire_and_get_data_link_via_id_with_dataid(samplestorage): - _expire_and_get_data_link_via_id(samplestorage, 10, 'foo', 'acbd18db4cc2f85cedef654fccc4a4d8_') + _expire_and_get_data_link_via_id( + samplestorage, 10, "foo", "acbd18db4cc2f85cedef654fccc4a4d8_" + ) def _expire_and_get_data_link_via_id(samplestorage, expired, dataid, expectedmd5): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid, - DataUnitID(UPA('1/1/1'), dataid), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(5), - UserID('usera')) + lid = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid, + DataUnitID(UPA("1/1/1"), dataid), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(5), + UserID("usera"), + ) ) # this is naughty - verdoc1 = samplestorage._col_version.find({'id': str(sid), 'ver': 1}).next() - nodedoc1 = samplestorage._col_nodes.find({'name': 'mynode'}).next() + verdoc1 = samplestorage._col_version.find({"id": str(sid), "ver": 1}).next() + nodedoc1 = samplestorage._col_nodes.find({"name": "mynode"}).next() - assert samplestorage.expire_data_link(dt(expired), UserID('user'), id_=lid) == DataLink( + assert samplestorage.expire_data_link( + dt(expired), UserID("user"), id_=lid + ) == DataLink( lid, - DataUnitID(UPA('1/1/1'), dataid), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), dataid), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), dt(5), - UserID('usera'), + UserID("usera"), dt(expired), - UserID('user') - ) + UserID("user"), + ) assert samplestorage._col_data_link.count() == 1 - link = samplestorage._col_data_link.get(f'1_1_1_{expectedmd5}5.0') + link = samplestorage._col_data_link.get(f"1_1_1_{expectedmd5}5.0") assert link == { - '_key': f'1_1_1_{expectedmd5}5.0', - '_id': f'data_link/1_1_1_{expectedmd5}5.0', - '_from': 'ws_obj_ver/1:1:1', - '_to': nodedoc1['_id'], - '_rev': link['_rev'], # no need to test this - 'id': '12345678-90ab-cdef-1234-567890abcde1', - 'wsid': 1, - 'objid': 1, - 'objver': 1, - 'dataid': dataid, - 'sampleid': '12345678-90ab-cdef-1234-567890abcdef', - 'samuuidver': verdoc1['uuidver'], - 'samintver': 1, - 'node': 'mynode', - 'created': 5000, - 'createby': 'usera', - 'expired': expired * 1000, - 'expireby': 'user' + "_key": f"1_1_1_{expectedmd5}5.0", + "_id": f"data_link/1_1_1_{expectedmd5}5.0", + "_from": "ws_obj_ver/1:1:1", + "_to": nodedoc1["_id"], + "_rev": link["_rev"], # no need to test this + "id": "12345678-90ab-cdef-1234-567890abcde1", + "wsid": 1, + "objid": 1, + "objver": 1, + "dataid": dataid, + "sampleid": "12345678-90ab-cdef-1234-567890abcdef", + "samuuidver": verdoc1["uuidver"], + "samintver": 1, + "node": "mynode", + "created": 5000, + "createby": "usera", + "expired": expired * 1000, + "expireby": "user", } link = samplestorage.get_data_link(lid) expected = DataLink( lid, - DataUnitID(UPA('1/1/1'), dataid), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), dataid), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), dt(5), - UserID('usera'), + UserID("usera"), dt(expired), - UserID('user') + UserID("user"), ) assert link == expected @@ -3043,168 +4065,258 @@ def test_expire_data_link_fail_bad_args(samplestorage): ss = samplestorage e = dt(100) i = uuid.uuid4() - d = DataUnitID('1/1/1') + d = DataUnitID("1/1/1") eb = datetime.datetime.fromtimestamp(400) - u = UserID('u') + u = UserID("u") - _expire_data_link_fail(ss, None, u, i, None, ValueError( - 'expired cannot be a value that evaluates to false')) - _expire_data_link_fail(ss, eb, u, None, d, ValueError('expired cannot be a naive datetime')) - _expire_data_link_fail(ss, e, None, None, d, ValueError( - 'expired_by cannot be a value that evaluates to false')) - _expire_data_link_fail(ss, e, u, i, d, ValueError( - 'exactly one of id_ or duid must be provided')) - _expire_data_link_fail(ss, e, u, None, None, ValueError( - 'exactly one of id_ or duid must be provided')) + _expire_data_link_fail( + ss, + None, + u, + i, + None, + ValueError("expired cannot be a value that evaluates to false"), + ) + _expire_data_link_fail( + ss, eb, u, None, d, ValueError("expired cannot be a naive datetime") + ) + _expire_data_link_fail( + ss, + e, + None, + None, + d, + ValueError("expired_by cannot be a value that evaluates to false"), + ) + _expire_data_link_fail( + ss, e, u, i, d, ValueError("exactly one of id_ or duid must be provided") + ) + _expire_data_link_fail( + ss, e, u, None, None, ValueError("exactly one of id_ or duid must be provided") + ) def test_expire_data_link_fail_no_id(samplestorage): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid, - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(-100), - UserID('user')) + lid = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid, + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(-100), + UserID("user"), + ) ) _expire_data_link_fail( - samplestorage, dt(1), UserID('u'), uuid.UUID('1234567890abcdef1234567890abcde2'), None, - NoSuchLinkError('12345678-90ab-cdef-1234-567890abcde2')) + samplestorage, + dt(1), + UserID("u"), + uuid.UUID("1234567890abcdef1234567890abcde2"), + None, + NoSuchLinkError("12345678-90ab-cdef-1234-567890abcde2"), + ) def test_expire_data_link_fail_with_id_expired(samplestorage): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid, - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(-100), - UserID('user')) + lid = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid, + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(-100), + UserID("user"), + ) ) - samplestorage.expire_data_link(dt(0), UserID('u'), lid) + samplestorage.expire_data_link(dt(0), UserID("u"), lid) _expire_data_link_fail( - samplestorage, dt(1), UserID('u'), lid, None, - NoSuchLinkError('12345678-90ab-cdef-1234-567890abcde1')) + samplestorage, + dt(1), + UserID("u"), + lid, + None, + NoSuchLinkError("12345678-90ab-cdef-1234-567890abcde1"), + ) def test_expire_data_link_fail_with_id_too_many_links(samplestorage): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid, - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(-100), - UserID('usera')) + lid = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid, + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(-100), + UserID("usera"), + ) ) - samplestorage.expire_data_link(dt(-50), UserID('u'), lid) + samplestorage.expire_data_link(dt(-50), UserID("u"), lid) - samplestorage.create_data_link(DataLink( - lid, - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(0), - UserID('usera')) + samplestorage.create_data_link( + DataLink( + lid, + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(0), + UserID("usera"), + ) ) - _expire_data_link_fail(samplestorage, dt(1), UserID('u'), lid, None, SampleStorageError( - 'More than one data link found for ID 12345678-90ab-cdef-1234-567890abcde1')) + _expire_data_link_fail( + samplestorage, + dt(1), + UserID("u"), + lid, + None, + SampleStorageError( + "More than one data link found for ID 12345678-90ab-cdef-1234-567890abcde1" + ), + ) def test_expire_data_link_fail_no_duid(samplestorage): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid1 = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid1, - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(-100), - UserID('usera')) + lid1 = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid1, + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(-100), + UserID("usera"), + ) ) - lid2 = uuid.UUID('1234567890abcdef1234567890abcde2') - samplestorage.create_data_link(DataLink( - lid2, - DataUnitID(UPA('1/1/2'), 'foo'), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(-100), - UserID('usera')) + lid2 = uuid.UUID("1234567890abcdef1234567890abcde2") + samplestorage.create_data_link( + DataLink( + lid2, + DataUnitID(UPA("1/1/2"), "foo"), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(-100), + UserID("usera"), + ) ) _expire_data_link_fail( - samplestorage, dt(1), UserID('u'), None, DataUnitID(UPA('1/2/1')), - NoSuchLinkError('1/2/1')) + samplestorage, + dt(1), + UserID("u"), + None, + DataUnitID(UPA("1/2/1")), + NoSuchLinkError("1/2/1"), + ) _expire_data_link_fail( - samplestorage, dt(1), UserID('u'), None, DataUnitID(UPA('1/1/2'), 'fo'), - NoSuchLinkError('1/1/2:fo')) + samplestorage, + dt(1), + UserID("u"), + None, + DataUnitID(UPA("1/1/2"), "fo"), + NoSuchLinkError("1/1/2:fo"), + ) def test_expire_data_link_fail_expire_before_create_by_id(samplestorage): - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid1 = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid1, - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(100), - UserID('someuser')) + lid1 = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid1, + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(100), + UserID("someuser"), + ) ) - _expire_data_link_fail(samplestorage, dt(99), UserID('u'), lid1, None, ValueError( - 'expired is < link created time: 100000')) + _expire_data_link_fail( + samplestorage, + dt(99), + UserID("u"), + lid1, + None, + ValueError("expired is < link created time: 100000"), + ) def test_expire_data_link_fail_race_condition(samplestorage): - ''' + """ Tests the case where a link is expire after pulling it from the DB in the first part of the method. See notes in the code. - ''' + """ - sid = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(sid, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + sid = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(sid, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) - lid1 = uuid.UUID('1234567890abcdef1234567890abcde1') - samplestorage.create_data_link(DataLink( - lid1, - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid, 1), 'mynode'), - dt(-100), - UserID('usera')) + lid1 = uuid.UUID("1234567890abcdef1234567890abcde1") + samplestorage.create_data_link( + DataLink( + lid1, + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid, 1), "mynode"), + dt(-100), + UserID("usera"), + ) ) # ok, we have the link doc from the db. This is what part 1 of the code does, and then # passes to part 2. - linkdoc = samplestorage._col_data_link.get('1_1_1') + linkdoc = samplestorage._col_data_link.get("1_1_1") # Now we simulate a race condition by expiring that link and calling part 2 of the expire # method. Part 2 should fail without modifying the links collection. - samplestorage.expire_data_link(dt(200), UserID('foo'), lid1) + samplestorage.expire_data_link(dt(200), UserID("foo"), lid1) with raises(Exception) as got: - samplestorage._expire_data_link_pt2(linkdoc, dt(300), UserID('foo'), 'some id') - assert_exception_correct(got.value, NoSuchLinkError('some id')) + samplestorage._expire_data_link_pt2(linkdoc, dt(300), UserID("foo"), "some id") + assert_exception_correct(got.value, NoSuchLinkError("some id")) def _expire_data_link_fail(samplestorage, expire, user, id_, duid, expected): @@ -3214,32 +4326,51 @@ def _expire_data_link_fail(samplestorage, expire, user, id_, duid, expected): def test_get_links_from_sample(samplestorage): - sid1 = uuid.UUID('1234567890abcdef1234567890abcdef') - sid2 = uuid.UUID('1234567890abcdef1234567890abcdee') - assert samplestorage.save_sample(SavedSample( - sid1, UserID('user'), - [SampleNode('mynode'), SampleNode('XmynodeX')], dt(1), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(sid1, UserID('user'), [SampleNode('mynode1')], dt(2), 'foo')) == 2 - assert samplestorage.save_sample( - SavedSample(sid2, UserID('user'), [SampleNode('mynode2')], dt(3), 'foo')) is True + sid1 = uuid.UUID("1234567890abcdef1234567890abcdef") + sid2 = uuid.UUID("1234567890abcdef1234567890abcdee") + assert ( + samplestorage.save_sample( + SavedSample( + sid1, + UserID("user"), + [SampleNode("mynode"), SampleNode("XmynodeX")], + dt(1), + "foo", + ) + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(sid1, UserID("user"), [SampleNode("mynode1")], dt(2), "foo") + ) + == 2 + ) + assert ( + samplestorage.save_sample( + SavedSample(sid2, UserID("user"), [SampleNode("mynode2")], dt(3), "foo") + ) + is True + ) # shouldn't be found, different sample l1 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid2, 1), 'mynode2'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid2, 1), "mynode2"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l1) # shouldn't be found, different version l2 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/2')), - SampleNodeAddress(SampleAddress(sid1, 2), 'mynode1'), + DataUnitID(UPA("1/1/2")), + SampleNodeAddress(SampleAddress(sid1, 2), "mynode1"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l2) # shouldn't be found, expired @@ -3248,54 +4379,60 @@ def test_get_links_from_sample(samplestorage): samplestorage, DataLink( l3id, - DataUnitID(UPA('1/1/3')), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID(UPA("1/1/3")), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(-100), - UserID('usera')), + UserID("usera"), + ), dt(100), - UserID('userb') + UserID("userb"), ) l3 = DataLink( l3id, - DataUnitID(UPA('1/1/3')), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID(UPA("1/1/3")), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(-100), - UserID('usera'), + UserID("usera"), dt(100), - UserID('userb')) + UserID("userb"), + ) # shouldn't be found, not created yet l4 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/4')), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID(UPA("1/1/4")), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(1000), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l4) # shouldn't be found, not in ws list l5 = DataLink( uuid.uuid4(), - DataUnitID(UPA('10/1/1')), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID(UPA("10/1/1")), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l5) l6 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/5')), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID(UPA("1/1/5")), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l6) l7 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/2'), 'whee'), - SampleNodeAddress(SampleAddress(sid1, 1), 'XmynodeX'), + DataUnitID(UPA("1/1/2"), "whee"), + SampleNodeAddress(SampleAddress(sid1, 1), "XmynodeX"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l7) l8id = uuid.uuid4() @@ -3303,123 +4440,95 @@ def test_get_links_from_sample(samplestorage): samplestorage, DataLink( l8id, - DataUnitID(UPA('2/1/6'),), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID( + UPA("2/1/6"), + ), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(-100), - UserID('usera')), + UserID("usera"), + ), dt(800), - UserID('usera') + UserID("usera"), ) l8 = DataLink( l8id, - DataUnitID(UPA('2/1/6')), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID(UPA("2/1/6")), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(-100), - UserID('usera'), + UserID("usera"), dt(800), - UserID('usera')) + UserID("usera"), + ) # test lower bound for expired got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [1, 2, 3], - dt(100.001) + SampleAddress(sid1, 1), [1, 2, 3], dt(100.001) ) assert set(got) == {l6, l7, l8} # order is undefined # test lower expired got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [1, 2, 3], - dt(100) + SampleAddress(sid1, 1), [1, 2, 3], dt(100) ) assert set(got) == {l6, l7, l8, l3} # order is undefined # test upper bound for expired got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [1, 2, 3], - dt(800) + SampleAddress(sid1, 1), [1, 2, 3], dt(800) ) assert set(got) == {l6, l7, l8} # order is undefined # test upper expired got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [1, 2, 3], - dt(800.001) + SampleAddress(sid1, 1), [1, 2, 3], dt(800.001) ) assert set(got) == {l6, l7} # order is undefined # test lower bound for created got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [1, 2, 3], - dt(999.999) + SampleAddress(sid1, 1), [1, 2, 3], dt(999.999) ) assert set(got) == {l6, l7} # order is undefined # test created got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [1, 2, 3], - dt(1000) + SampleAddress(sid1, 1), [1, 2, 3], dt(1000) ) assert set(got) == {l6, l7, l4} # order is undefined # test limiting workspace ids - got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [1], - dt(500) - ) + got = samplestorage.get_links_from_sample(SampleAddress(sid1, 1), [1], dt(500)) assert set(got) == {l6, l7} # order is undefined # no results, no matching wsids got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [4, 5, 6], - dt(500) + SampleAddress(sid1, 1), [4, 5, 6], dt(500) ) assert got == [] # no results, no wsids - got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [], - dt(500) - ) + got = samplestorage.get_links_from_sample(SampleAddress(sid1, 1), [], dt(500)) assert got == [] # test not filtering on WS id - got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - None, - dt(500) - ) + got = samplestorage.get_links_from_sample(SampleAddress(sid1, 1), None, dt(500)) assert set(got) == {l5, l6, l7, l8} # no results, prior to created got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 1), - [1, 2, 3], - dt(-100.001) + SampleAddress(sid1, 1), [1, 2, 3], dt(-100.001) ) assert got == [] # test different sample got = samplestorage.get_links_from_sample( - SampleAddress(sid2, 1), - [1, 2, 3], - dt(500) + SampleAddress(sid2, 1), [1, 2, 3], dt(500) ) assert got == [l1] # test different sample version got = samplestorage.get_links_from_sample( - SampleAddress(sid1, 2), - [1, 2, 3], - dt(500) + SampleAddress(sid1, 2), [1, 2, 3], dt(500) ) assert got == [l2] @@ -3431,78 +4540,127 @@ def test_get_links_from_sample_fail_bad_args(samplestorage): ts = dt(1) tb = datetime.datetime.fromtimestamp(1) - _get_links_from_sample_fail(ss, None, ws, ts, ValueError( - 'sample cannot be a value that evaluates to false')) - _get_links_from_sample_fail(ss, sa, [1, None], ts, ValueError( - 'Index 1 of iterable readable_wsids cannot be a value that evaluates to false')) - _get_links_from_sample_fail(ss, sa, ws, None, ValueError( - 'timestamp cannot be a value that evaluates to false')) - _get_links_from_sample_fail(ss, sa, ws, tb, ValueError( - 'timestamp cannot be a naive datetime')) + _get_links_from_sample_fail( + ss, None, ws, ts, ValueError("sample cannot be a value that evaluates to false") + ) + _get_links_from_sample_fail( + ss, + sa, + [1, None], + ts, + ValueError( + "Index 1 of iterable readable_wsids cannot be a value that evaluates to false" + ), + ) + _get_links_from_sample_fail( + ss, + sa, + ws, + None, + ValueError("timestamp cannot be a value that evaluates to false"), + ) + _get_links_from_sample_fail( + ss, sa, ws, tb, ValueError("timestamp cannot be a naive datetime") + ) def test_get_links_from_sample_fail_no_sample(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - id2 = uuid.UUID('1234567890abcdef1234567890abcdee') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + id2 = uuid.UUID("1234567890abcdef1234567890abcdee") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) _get_links_from_sample_fail( - samplestorage, SampleAddress(id2, 1), [1], dt(1), NoSuchSampleError(str(id2))) + samplestorage, SampleAddress(id2, 1), [1], dt(1), NoSuchSampleError(str(id2)) + ) def test_get_links_from_sample_fail_no_sample_version(samplestorage): - id1 = uuid.UUID('1234567890abcdef1234567890abcdef') - assert samplestorage.save_sample( - SavedSample(id1, UserID('user'), [SampleNode('mynode')], dt(1), 'foo')) is True + id1 = uuid.UUID("1234567890abcdef1234567890abcdef") + assert ( + samplestorage.save_sample( + SavedSample(id1, UserID("user"), [SampleNode("mynode")], dt(1), "foo") + ) + is True + ) _get_links_from_sample_fail( - samplestorage, SampleAddress(id1, 2), [1], dt(1), NoSuchSampleVersionError(f'{id1} ver 2')) + samplestorage, + SampleAddress(id1, 2), + [1], + dt(1), + NoSuchSampleVersionError(f"{id1} ver 2"), + ) def _get_links_from_sample_fail( - samplestorage, sample_address, wsids, timestamp, expected): + samplestorage, sample_address, wsids, timestamp, expected +): with raises(Exception) as got: samplestorage.get_links_from_sample(sample_address, wsids, timestamp) assert_exception_correct(got.value, expected) def test_get_links_from_data(samplestorage): - sid1 = uuid.UUID('1234567890abcdef1234567890abcdef') - sid2 = uuid.UUID('1234567890abcdef1234567890abcdee') - assert samplestorage.save_sample(SavedSample( - sid1, UserID('user'), - [SampleNode('mynode'), SampleNode('XmynodeX')], dt(1), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(sid1, UserID('user'), [SampleNode('mynode1')], dt(2), 'foo')) == 2 - assert samplestorage.save_sample( - SavedSample(sid2, UserID('user'), [SampleNode('mynode2')], dt(3), 'foo')) is True + sid1 = uuid.UUID("1234567890abcdef1234567890abcdef") + sid2 = uuid.UUID("1234567890abcdef1234567890abcdee") + assert ( + samplestorage.save_sample( + SavedSample( + sid1, + UserID("user"), + [SampleNode("mynode"), SampleNode("XmynodeX")], + dt(1), + "foo", + ) + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(sid1, UserID("user"), [SampleNode("mynode1")], dt(2), "foo") + ) + == 2 + ) + assert ( + samplestorage.save_sample( + SavedSample(sid2, UserID("user"), [SampleNode("mynode2")], dt(3), "foo") + ) + is True + ) # shouldn't be found, different ws l1 = DataLink( uuid.uuid4(), - DataUnitID(UPA('10/1/1')), - SampleNodeAddress(SampleAddress(sid2, 1), 'mynode2'), + DataUnitID(UPA("10/1/1")), + SampleNodeAddress(SampleAddress(sid2, 1), "mynode2"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l1) # shouldn't be found, different object l2 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/10/1')), - SampleNodeAddress(SampleAddress(sid1, 2), 'mynode1'), + DataUnitID(UPA("1/10/1")), + SampleNodeAddress(SampleAddress(sid1, 2), "mynode1"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l2) # shouldn't be found, different version l3 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/10')), - SampleNodeAddress(SampleAddress(sid1, 2), 'mynode1'), + DataUnitID(UPA("1/1/10")), + SampleNodeAddress(SampleAddress(sid1, 2), "mynode1"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l3) # shouldn't be found, expired @@ -3511,45 +4669,50 @@ def test_get_links_from_data(samplestorage): samplestorage, DataLink( l4id, - DataUnitID(UPA('1/1/1'), 'expired'), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), "expired"), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(-100), - UserID('usera')), + UserID("usera"), + ), dt(100), - UserID('userb') + UserID("userb"), ) l4 = DataLink( l4id, - DataUnitID(UPA('1/1/1'), 'expired'), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), "expired"), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(-100), - UserID('usera'), + UserID("usera"), dt(100), - UserID('userb')) + UserID("userb"), + ) # shouldn't be found, not created yet l5 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1'), 'not created'), - SampleNodeAddress(SampleAddress(sid1, 1), 'mynode'), + DataUnitID(UPA("1/1/1"), "not created"), + SampleNodeAddress(SampleAddress(sid1, 1), "mynode"), dt(1000), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l5) l6 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid1, 2), 'mynode1'), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid1, 2), "mynode1"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l6) l7 = DataLink( uuid.uuid4(), - DataUnitID(UPA('1/1/1'), '1'), - SampleNodeAddress(SampleAddress(sid1, 1), 'XmynodeX'), + DataUnitID(UPA("1/1/1"), "1"), + SampleNodeAddress(SampleAddress(sid1, 1), "XmynodeX"), dt(-100), - UserID('usera')) + UserID("usera"), + ) samplestorage.create_data_link(l7) l8id = uuid.uuid4() @@ -3557,83 +4720,88 @@ def test_get_links_from_data(samplestorage): samplestorage, DataLink( l8id, - DataUnitID(UPA('1/1/1'), '2'), - SampleNodeAddress(SampleAddress(sid2, 1), 'mynode2'), + DataUnitID(UPA("1/1/1"), "2"), + SampleNodeAddress(SampleAddress(sid2, 1), "mynode2"), dt(-100), - UserID('usera')), + UserID("usera"), + ), dt(800), - UserID('usera') + UserID("usera"), ) l8 = DataLink( l8id, - DataUnitID(UPA('1/1/1'), '2'), - SampleNodeAddress(SampleAddress(sid2, 1), 'mynode2'), + DataUnitID(UPA("1/1/1"), "2"), + SampleNodeAddress(SampleAddress(sid2, 1), "mynode2"), dt(-100), - UserID('usera'), + UserID("usera"), dt(800), - UserID('usera')) + UserID("usera"), + ) # test lower bound for expired - got = samplestorage.get_links_from_data(UPA('1/1/1'), dt(100.001)) + got = samplestorage.get_links_from_data(UPA("1/1/1"), dt(100.001)) assert set(got) == {l6, l7, l8} # order is undefined # test lower expired - got = samplestorage.get_links_from_data(UPA('1/1/1'), dt(100)) + got = samplestorage.get_links_from_data(UPA("1/1/1"), dt(100)) assert set(got) == {l6, l7, l8, l4} # order is undefined # test upper bound for expired - got = samplestorage.get_links_from_data(UPA('1/1/1'), dt(800)) + got = samplestorage.get_links_from_data(UPA("1/1/1"), dt(800)) assert set(got) == {l6, l7, l8} # order is undefined # test upper expired - got = samplestorage.get_links_from_data(UPA('1/1/1'), dt(800.001)) + got = samplestorage.get_links_from_data(UPA("1/1/1"), dt(800.001)) assert set(got) == {l6, l7} # order is undefined # test lower bound for created - got = samplestorage.get_links_from_data(UPA('1/1/1'), dt(999.999)) + got = samplestorage.get_links_from_data(UPA("1/1/1"), dt(999.999)) assert set(got) == {l6, l7} # order is undefined # test created - got = samplestorage.get_links_from_data(UPA('1/1/1'), dt(1000)) + got = samplestorage.get_links_from_data(UPA("1/1/1"), dt(1000)) assert set(got) == {l6, l7, l5} # order is undefined # different wsid - got = samplestorage.get_links_from_data(UPA('10/1/1'), dt(500)) + got = samplestorage.get_links_from_data(UPA("10/1/1"), dt(500)) assert got == [l1] # different objid - got = samplestorage.get_links_from_data(UPA('1/10/1'), dt(500)) + got = samplestorage.get_links_from_data(UPA("1/10/1"), dt(500)) assert got == [l2] # different ver - got = samplestorage.get_links_from_data(UPA('1/1/10'), dt(500)) + got = samplestorage.get_links_from_data(UPA("1/1/10"), dt(500)) assert got == [l3] # no results, no matching objects - got = samplestorage.get_links_from_data(UPA('9/1/1'), dt(500)) + got = samplestorage.get_links_from_data(UPA("9/1/1"), dt(500)) assert got == [] - got = samplestorage.get_links_from_data(UPA('1/9/1'), dt(500)) + got = samplestorage.get_links_from_data(UPA("1/9/1"), dt(500)) assert got == [] - got = samplestorage.get_links_from_data(UPA('1/1/9'), dt(500)) + got = samplestorage.get_links_from_data(UPA("1/1/9"), dt(500)) assert got == [] # no results, prior to created - got = samplestorage.get_links_from_data(UPA('1/1/1'), dt(-100.001)) + got = samplestorage.get_links_from_data(UPA("1/1/1"), dt(-100.001)) assert got == [] def test_get_links_from_data_fail_bad_args(samplestorage): ss = samplestorage - u = UPA('1/1/1') + u = UPA("1/1/1") ts = dt(1) td = datetime.datetime.fromtimestamp(1) - _get_links_from_data_fail(ss, None, ts, ValueError( - 'upa cannot be a value that evaluates to false')) - _get_links_from_data_fail(ss, u, None, ValueError( - 'timestamp cannot be a value that evaluates to false')) - _get_links_from_data_fail(ss, u, td, ValueError( - 'timestamp cannot be a naive datetime')) + _get_links_from_data_fail( + ss, None, ts, ValueError("upa cannot be a value that evaluates to false") + ) + _get_links_from_data_fail( + ss, u, None, ValueError("timestamp cannot be a value that evaluates to false") + ) + _get_links_from_data_fail( + ss, u, td, ValueError("timestamp cannot be a naive datetime") + ) def _get_links_from_data_fail(samplestorage, upa, ts, expected): @@ -3643,58 +4811,92 @@ def _get_links_from_data_fail(samplestorage, upa, ts, expected): def test_has_data_link(samplestorage): - sid1 = uuid.UUID('1234567890abcdef1234567890abcdef') - sid2 = uuid.UUID('1234567890abcdef1234567890abcdee') - sid3 = uuid.UUID('1234567890abcdef1234567890abcded') - assert samplestorage.save_sample(SavedSample( - sid1, UserID('user'), - [SampleNode('mynode'), SampleNode('XmynodeX')], dt(1), 'foo')) is True - assert samplestorage.save_sample_version( - SavedSample(sid1, UserID('user'), [SampleNode('mynode1')], dt(2), 'foo')) == 2 - assert samplestorage.save_sample( - SavedSample(sid2, UserID('user'), [SampleNode('mynode2')], dt(3), 'foo')) is True - assert samplestorage.save_sample( - SavedSample(sid3, UserID('user'), [SampleNode('mynode4')], dt(3), 'foo')) is True - - samplestorage.create_data_link(DataLink( - uuid.uuid4(), - DataUnitID(UPA('1/1/1')), - SampleNodeAddress(SampleAddress(sid2, 1), 'mynode2'), - dt(-100), - UserID('usera'))) + sid1 = uuid.UUID("1234567890abcdef1234567890abcdef") + sid2 = uuid.UUID("1234567890abcdef1234567890abcdee") + sid3 = uuid.UUID("1234567890abcdef1234567890abcded") + assert ( + samplestorage.save_sample( + SavedSample( + sid1, + UserID("user"), + [SampleNode("mynode"), SampleNode("XmynodeX")], + dt(1), + "foo", + ) + ) + is True + ) + assert ( + samplestorage.save_sample_version( + SavedSample(sid1, UserID("user"), [SampleNode("mynode1")], dt(2), "foo") + ) + == 2 + ) + assert ( + samplestorage.save_sample( + SavedSample(sid2, UserID("user"), [SampleNode("mynode2")], dt(3), "foo") + ) + is True + ) + assert ( + samplestorage.save_sample( + SavedSample(sid3, UserID("user"), [SampleNode("mynode4")], dt(3), "foo") + ) + is True + ) + + samplestorage.create_data_link( + DataLink( + uuid.uuid4(), + DataUnitID(UPA("1/1/1")), + SampleNodeAddress(SampleAddress(sid2, 1), "mynode2"), + dt(-100), + UserID("usera"), + ) + ) _create_and_expire_data_link( samplestorage, DataLink( uuid.uuid4(), - DataUnitID(UPA('1/2/1'), '2'), - SampleNodeAddress(SampleAddress(sid1, 2), 'mynode1'), + DataUnitID(UPA("1/2/1"), "2"), + SampleNodeAddress(SampleAddress(sid1, 2), "mynode1"), dt(-100), - UserID('usera')), + UserID("usera"), + ), dt(800), - UserID('usera')) + UserID("usera"), + ) - assert samplestorage.has_data_link(UPA('1/1/1'), sid2) is True + assert samplestorage.has_data_link(UPA("1/1/1"), sid2) is True # sample version & expired state shouldn't matter - assert samplestorage.has_data_link(UPA('1/2/1'), sid1) is True + assert samplestorage.has_data_link(UPA("1/2/1"), sid1) is True # wrong sample - assert samplestorage.has_data_link(UPA('1/2/1'), sid3) is False + assert samplestorage.has_data_link(UPA("1/2/1"), sid3) is False # wrong object id - assert samplestorage.has_data_link(UPA('1/2/1'), sid2) is False + assert samplestorage.has_data_link(UPA("1/2/1"), sid2) is False # wrong version - assert samplestorage.has_data_link(UPA('1/2/2'), sid1) is False + assert samplestorage.has_data_link(UPA("1/2/2"), sid1) is False # wrong wsid - assert samplestorage.has_data_link(UPA('2/2/1'), sid1) is False + assert samplestorage.has_data_link(UPA("2/2/1"), sid1) is False def test_has_data_link_fail_bad_args(samplestorage): - u = UPA('1/1/1') - s = uuid.UUID('1234567890abcdef1234567890abcdef') + u = UPA("1/1/1") + s = uuid.UUID("1234567890abcdef1234567890abcdef") - _has_data_link_fail(samplestorage, None, s, ValueError( - 'upa cannot be a value that evaluates to false')) - _has_data_link_fail(samplestorage, u, None, ValueError( - 'sample cannot be a value that evaluates to false')) + _has_data_link_fail( + samplestorage, + None, + s, + ValueError("upa cannot be a value that evaluates to false"), + ) + _has_data_link_fail( + samplestorage, + u, + None, + ValueError("sample cannot be a value that evaluates to false"), + ) def _has_data_link_fail(samplestorage, upa, sample, expected): diff --git a/test/core/test_utils.py b/test/core/test_utils.py index 1fc19170..dfdefde4 100644 --- a/test/core/test_utils.py +++ b/test/core/test_utils.py @@ -9,18 +9,18 @@ from logging import LogRecord import time -ARANGO_EXE = 'test.arango.exe' -ARANGO_JS = 'test.arango.js' -KAFKA_BIN_DIR = 'test.kafka.bin.dir' -MONGO_EXE = 'test.mongo.exe' -MONGO_USE_WIRED_TIGER = 'test.mongo.wired_tiger' -JARS_DIR = 'test.jars.dir' -TEST_TEMP_DIR = 'test.temp.dir' -KEEP_TEMP_DIR = 'test.temp.dir.keep' +ARANGO_EXE = "test.arango.exe" +ARANGO_JS = "test.arango.js" +KAFKA_BIN_DIR = "test.kafka.bin.dir" +MONGO_EXE = "test.mongo.exe" +MONGO_USE_WIRED_TIGER = "test.mongo.wired_tiger" +JARS_DIR = "test.jars.dir" +TEST_TEMP_DIR = "test.temp.dir" +KEEP_TEMP_DIR = "test.temp.dir.keep" -TEST_CONFIG_FILE_SECTION = 'sampleservicetest' +TEST_CONFIG_FILE_SECTION = "sampleservicetest" -TEST_FILE_LOC_ENV_KEY = 'SAMPLESERV_TEST_FILE' +TEST_FILE_LOC_ENV_KEY = "SAMPLESERV_TEST_FILE" _CONFIG = None @@ -42,7 +42,7 @@ def get_mongo_exe() -> Path: def get_use_wired_tiger() -> bool: - return _get_test_property(MONGO_USE_WIRED_TIGER) == 'true' + return _get_test_property(MONGO_USE_WIRED_TIGER) == "true" def get_jars_dir() -> Path: @@ -54,13 +54,15 @@ def get_temp_dir() -> Path: def get_delete_temp_files() -> bool: - return _get_test_property(KEEP_TEMP_DIR) != 'true' + return _get_test_property(KEEP_TEMP_DIR) != "true" def _get_test_config_file_path() -> Path: p = os.environ.get(TEST_FILE_LOC_ENV_KEY) if not p: - raise TestException("Can't find key {} in environment".format(TEST_FILE_LOC_ENV_KEY)) + raise TestException( + "Can't find key {} in environment".format(TEST_FILE_LOC_ENV_KEY) + ) return Path(p) @@ -71,21 +73,27 @@ def _get_test_property(prop: str) -> str: config = configparser.ConfigParser() config.read(test_cfg) if TEST_CONFIG_FILE_SECTION not in config: - raise TestException('No section {} found in test config file {}' - .format(TEST_CONFIG_FILE_SECTION, test_cfg)) + raise TestException( + "No section {} found in test config file {}".format( + TEST_CONFIG_FILE_SECTION, test_cfg + ) + ) sec = config[TEST_CONFIG_FILE_SECTION] # a section is not a real map and is missing methods _CONFIG = {x: sec[x] for x in sec.keys()} if prop not in _CONFIG: test_cfg = _get_test_config_file_path() - raise TestException('Property {} in section {} of test file {} is missing' - .format(prop, TEST_CONFIG_FILE_SECTION, test_cfg)) + raise TestException( + "Property {} in section {} of test file {} is missing".format( + prop, TEST_CONFIG_FILE_SECTION, test_cfg + ) + ) return _CONFIG[prop] def find_free_port() -> int: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - s.bind(('', 0)) + s.bind(("", 0)) return s.getsockname()[1] @@ -109,7 +117,7 @@ def __init__(self): def format(self, record): self.logs.append(record) - return 'no logs here, no sir' + return "no logs here, no sir" class TestException(Exception): @@ -118,36 +126,40 @@ class TestException(Exception): def create_auth_user(auth_url, username, displayname): ret = requests.post( - auth_url + '/testmode/api/V2/testmodeonly/user', - headers={'accept': 'application/json'}, - json={'user': username, 'display': displayname}) + auth_url + "/testmode/api/V2/testmodeonly/user", + headers={"accept": "application/json"}, + json={"user": username, "display": displayname}, + ) if not ret.ok: ret.raise_for_status() def create_auth_login_token(auth_url, username): ret = requests.post( - auth_url + '/testmode/api/V2/testmodeonly/token', - headers={'accept': 'application/json'}, - json={'user': username, 'type': 'Login'}) + auth_url + "/testmode/api/V2/testmodeonly/token", + headers={"accept": "application/json"}, + json={"user": username, "type": "Login"}, + ) if not ret.ok: ret.raise_for_status() - return ret.json()['token'] + return ret.json()["token"] def create_auth_role(auth_url, role, description): ret = requests.post( - auth_url + '/testmode/api/V2/testmodeonly/customroles', - headers={'accept': 'application/json'}, - json={'id': role, 'desc': description}) + auth_url + "/testmode/api/V2/testmodeonly/customroles", + headers={"accept": "application/json"}, + json={"id": role, "desc": description}, + ) if not ret.ok: ret.raise_for_status() def set_custom_roles(auth_url, user, roles): ret = requests.put( - auth_url + '/testmode/api/V2/testmodeonly/userroles', - headers={'accept': 'application/json'}, - json={'user': user, 'customroles': roles}) + auth_url + "/testmode/api/V2/testmodeonly/userroles", + headers={"accept": "application/json"}, + json={"user": user, "customroles": roles}, + ) if not ret.ok: ret.raise_for_status() diff --git a/test/core/user_test.py b/test/core/user_test.py index 2b60ceaf..1b2f60ee 100644 --- a/test/core/user_test.py +++ b/test/core/user_test.py @@ -7,30 +7,32 @@ def test_init(): - u = UserID('foo') + u = UserID("foo") - assert u.id == 'foo' - assert str(u) == 'foo' + assert u.id == "foo" + assert str(u) == "foo" assert repr(u) == 'UserID("foo")' - u = UserID('u' * 256) + u = UserID("u" * 256) - assert u.id == 'u' * 256 - assert str(u) == 'u' * 256 + assert u.id == "u" * 256 + assert str(u) == "u" * 256 assert repr(u) == f'UserID("{"u" * 256}")' - u = UserID('u⎇a') + u = UserID("u⎇a") - assert u.id == 'u⎇a' - assert str(u) == 'u⎇a' + assert u.id == "u⎇a" + assert str(u) == "u⎇a" assert repr(u) == 'UserID("u⎇a")' def test_init_fail(): - _init_fail(None, MissingParameterError('userid')) - _init_fail(' \t ', MissingParameterError('userid')) - _init_fail('foo \t bar', IllegalParameterError('userid contains control characters')) - _init_fail('u' * 257, IllegalParameterError('userid exceeds maximum length of 256')) + _init_fail(None, MissingParameterError("userid")) + _init_fail(" \t ", MissingParameterError("userid")) + _init_fail( + "foo \t bar", IllegalParameterError("userid contains control characters") + ) + _init_fail("u" * 257, IllegalParameterError("userid exceeds maximum length of 256")) def _init_fail(u, expected): @@ -40,17 +42,17 @@ def _init_fail(u, expected): def test_equals(): - assert UserID('u') == UserID('u') + assert UserID("u") == UserID("u") - assert UserID('u') != 'u' + assert UserID("u") != "u" - assert UserID('u') != UserID('v') + assert UserID("u") != UserID("v") def test_hash(): # string hashes will change from instance to instance of the python interpreter, and therefore # tests can't be written that directly test the hash value. See # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - assert hash(UserID('u')) == hash(UserID('u')) + assert hash(UserID("u")) == hash(UserID("u")) - assert hash(UserID('u')) != hash(UserID('v')) + assert hash(UserID("u")) != hash(UserID("v")) diff --git a/test/core/validator/builtin_test.py b/test/core/validator/builtin_test.py index 6e1d3aff..dbd87963 100644 --- a/test/core/validator/builtin_test.py +++ b/test/core/validator/builtin_test.py @@ -8,14 +8,16 @@ def test_noop(): # not much to test here. n = builtin.noop({}) - assert n('key', {}) is None + assert n("key", {}) is None def test_noop_fail_bad_input(): - _noop_fail_build(None, ValueError('d must be a dict')) - _noop_fail_build([], ValueError('d must be a dict')) - _noop_fail_build({'key4': 'a', 'key86': 'a', 'key23': 'b', 'key6': 'c', 'key3': 'd'}, - ValueError('Unexpected configuration parameter: key23')) + _noop_fail_build(None, ValueError("d must be a dict")) + _noop_fail_build([], ValueError("d must be a dict")) + _noop_fail_build( + {"key4": "a", "key86": "a", "key23": "b", "key6": "c", "key3": "d"}, + ValueError("Unexpected configuration parameter: key23"), + ) def _noop_fail_build(cfg, expected): @@ -25,63 +27,92 @@ def _noop_fail_build(cfg, expected): def test_string_general(): - sl = builtin.string({'max-len': 2}) - assert sl('key', { - 'fo': 'b', - 'e': 'fb', - 'a': True, - 'b': 1111111111, - 'c': 1.23456789}) is None + sl = builtin.string({"max-len": 2}) + assert ( + sl("key", {"fo": "b", "e": "fb", "a": True, "b": 1111111111, "c": 1.23456789}) + is None + ) def test_string_single_keys(): - sl = builtin.string({'keys': 'whee', 'required': False}) - assert sl('key', { - 'fo': 'b', - 'e': 'fb', - 'whee': 'whooooooooooooo', - 'a': True, - 'b': 1111111111, - 'c': 1.23456789}) is None + sl = builtin.string({"keys": "whee", "required": False}) + assert ( + sl( + "key", + { + "fo": "b", + "e": "fb", + "whee": "whooooooooooooo", + "a": True, + "b": 1111111111, + "c": 1.23456789, + }, + ) + is None + ) def test_string_multiple_keys_max_len(): - sl = builtin.string({'keys': ['whee', 'whoo', 'whoa'], 'max-len': 9}) + sl = builtin.string({"keys": ["whee", "whoo", "whoa"], "max-len": 9}) # missing whoo key - assert sl('key', { - 'fo': 'b', - 'e': 'fb', - 'whee': 'whoo', - 'a': True, - 'b': 1111111111, - 'whoa': 'free mind', - 'c': 1.23456789}) is None + assert ( + sl( + "key", + { + "fo": "b", + "e": "fb", + "whee": "whoo", + "a": True, + "b": 1111111111, + "whoa": "free mind", + "c": 1.23456789, + }, + ) + is None + ) def test_string_multiple_keys_required(): - sl = builtin.string({'keys': ['whee', 'whoo', 'whoa'], 'required': True}) - assert sl('key', { - 'fo': 'b', - 'e': 'fb', - 'whee': None, # test that none is allowed as a value, even w/ required - 'a': True, - 'whoo': 'whoopty doo', - 'b': 1111111111, - 'whoa': 'free mind', - 'c': 1.23456789}) is None + sl = builtin.string({"keys": ["whee", "whoo", "whoa"], "required": True}) + assert ( + sl( + "key", + { + "fo": "b", + "e": "fb", + "whee": None, # test that none is allowed as a value, even w/ required + "a": True, + "whoo": "whoopty doo", + "b": 1111111111, + "whoa": "free mind", + "c": 1.23456789, + }, + ) + is None + ) def test_string_fail_bad_constructor_args(): - _string_fail_construct(None, ValueError('d must be a dict')) - _string_fail_construct({}, ValueError( - 'If the keys parameter is not specified, max-len must be specified')) - _string_fail_construct({'keys': 'foo', 'foo': 'bar', 'max-len': 25}, ValueError( - 'Unexpected configuration parameter: foo')) - _string_fail_construct({'max-len': 'shazzbat'}, ValueError('max-len must be an integer')) - _string_fail_construct({'max-len': '0'}, ValueError('max-len must be > 0')) - _string_fail_construct({'keys': 356}, ValueError('keys parameter must be a string or list')) - _string_fail_construct({'keys': ['foo', 'bar', 356, 'baz']}, ValueError( - 'keys parameter contains a non-string entry at index 2')) + _string_fail_construct(None, ValueError("d must be a dict")) + _string_fail_construct( + {}, + ValueError("If the keys parameter is not specified, max-len must be specified"), + ) + _string_fail_construct( + {"keys": "foo", "foo": "bar", "max-len": 25}, + ValueError("Unexpected configuration parameter: foo"), + ) + _string_fail_construct( + {"max-len": "shazzbat"}, ValueError("max-len must be an integer") + ) + _string_fail_construct({"max-len": "0"}, ValueError("max-len must be > 0")) + _string_fail_construct( + {"keys": 356}, ValueError("keys parameter must be a string or list") + ) + _string_fail_construct( + {"keys": ["foo", "bar", 356, "baz"]}, + ValueError("keys parameter contains a non-string entry at index 2"), + ) def _string_fail_construct(d, expected): @@ -92,75 +123,113 @@ def _string_fail_construct(d, expected): def test_string_validate_fail_bad_metadata_values(): _string_validate_fail( - {'max-len': 2}, {'foo': 'ba', 'ba': 'f'}, - 'Metadata contains key longer than max length of 2') + {"max-len": 2}, + {"foo": "ba", "ba": "f"}, + "Metadata contains key longer than max length of 2", + ) _string_validate_fail( - {'max-len': 4}, {'foo': 'ba', 'ba': 'fudge'}, - 'Metadata value at key ba is longer than max length of 4') + {"max-len": 4}, + {"foo": "ba", "ba": "fudge"}, + "Metadata value at key ba is longer than max length of 4", + ) _string_validate_fail( - {'keys': ['foo', 'bar'], 'required': True}, {'foo': 'ba', 'ba': 'fudge'}, - 'Required key bar is missing') + {"keys": ["foo", "bar"], "required": True}, + {"foo": "ba", "ba": "fudge"}, + "Required key bar is missing", + ) _string_validate_fail( - {'keys': 'foo'}, {'foo': ['a'], 'ba': 'fudge'}, - 'Metadata value at key foo is not a string') + {"keys": "foo"}, + {"foo": ["a"], "ba": "fudge"}, + "Metadata value at key foo is not a string", + ) _string_validate_fail( - {'keys': ['foo', 'bar'], 'required': True, 'max-len': 6}, {'foo': 'ba', 'bar': 'fudgera'}, - 'Metadata value at key bar is longer than max length of 6') + {"keys": ["foo", "bar"], "required": True, "max-len": 6}, + {"foo": "ba", "bar": "fudgera"}, + "Metadata value at key bar is longer than max length of 6", + ) def _string_validate_fail(cfg, meta, expected): - assert builtin.string(cfg)('key', meta)['message'] == expected + assert builtin.string(cfg)("key", meta)["message"] == expected def test_enum(): - en = builtin.enum({'allowed-values': ['a', 1, 3.1, True]}) + en = builtin.enum({"allowed-values": ["a", 1, 3.1, True]}) - assert en('key', {'z': 'a', 'w': 1, 'x': 3.1, 'y': True}) is None + assert en("key", {"z": "a", "w": 1, "x": 3.1, "y": True}) is None def test_enum_with_single_key(): - en = builtin.enum({'keys': '4', 'allowed-values': ['b', 2, 3.2, False]}) + en = builtin.enum({"keys": "4", "allowed-values": ["b", 2, 3.2, False]}) - assert en('key', {'4': 2}) is None + assert en("key", {"4": 2}) is None def test_enum_with_keys(): - en = builtin.enum({'keys': ['1', '2', '3', '4'], 'allowed-values': ['b', 2, 3.2, False]}) + en = builtin.enum( + {"keys": ["1", "2", "3", "4"], "allowed-values": ["b", 2, 3.2, False]} + ) - assert en('key', {'1': 'b', '2': 2, '3': 3.2, '4': False}) is None + assert en("key", {"1": "b", "2": 2, "3": 3.2, "4": False}) is None def test_enum_build_fail(): - _enum_build_fail(None, ValueError('d must be a dict')) - _enum_build_fail(['foo'], ValueError('d must be a dict')) - _enum_build_fail({'keys': ['foo']}, ValueError('allowed-values is a required parameter')) - _enum_build_fail({'keys': ['foo'], 'allowed-values': [], 'bar': 'bat'}, - ValueError('Unexpected configuration parameter: bar')) - _enum_build_fail({'allowed-values': {'a': 'b'}}, ValueError( - 'allowed-values parameter must be a list')) - _enum_build_fail({'allowed-values': ['a', True, []]}, ValueError( - 'allowed-values parameter contains a non-primitive type entry at index 2')) - _enum_build_fail({'allowed-values': ['a', True, 1, {}]}, ValueError( - 'allowed-values parameter contains a non-primitive type entry at index 3')) - _enum_build_fail({'keys': {'a': 'b'}, 'allowed-values': [1]}, ValueError( - 'keys parameter must be a string or list')) - _enum_build_fail({'keys': ['a', 1], 'allowed-values': [1]}, ValueError( - 'keys parameter contains a non-string entry at index 1')) + _enum_build_fail(None, ValueError("d must be a dict")) + _enum_build_fail(["foo"], ValueError("d must be a dict")) + _enum_build_fail( + {"keys": ["foo"]}, ValueError("allowed-values is a required parameter") + ) + _enum_build_fail( + {"keys": ["foo"], "allowed-values": [], "bar": "bat"}, + ValueError("Unexpected configuration parameter: bar"), + ) + _enum_build_fail( + {"allowed-values": {"a": "b"}}, + ValueError("allowed-values parameter must be a list"), + ) + _enum_build_fail( + {"allowed-values": ["a", True, []]}, + ValueError( + "allowed-values parameter contains a non-primitive type entry at index 2" + ), + ) + _enum_build_fail( + {"allowed-values": ["a", True, 1, {}]}, + ValueError( + "allowed-values parameter contains a non-primitive type entry at index 3" + ), + ) + _enum_build_fail( + {"keys": {"a": "b"}, "allowed-values": [1]}, + ValueError("keys parameter must be a string or list"), + ) + _enum_build_fail( + {"keys": ["a", 1], "allowed-values": [1]}, + ValueError("keys parameter contains a non-string entry at index 1"), + ) def test_enum_validate_fail(): _enum_validate_fail( - {'allowed-values': ['a', 1]}, {'foo': 'a', 'bar': 1, 'baz': 'whee'}, - 'Metadata value at key baz is not in the allowed list of values') + {"allowed-values": ["a", 1]}, + {"foo": "a", "bar": 1, "baz": "whee"}, + "Metadata value at key baz is not in the allowed list of values", + ) _enum_validate_fail( - {'allowed-values': ['a', 1]}, {'foo': 'a', 'bar': 1, 'bat': None}, - 'Metadata value at key bat is not in the allowed list of values') + {"allowed-values": ["a", 1]}, + {"foo": "a", "bar": 1, "bat": None}, + "Metadata value at key bat is not in the allowed list of values", + ) _enum_validate_fail( - {'allowed-values': ['a', 1], 'keys': ['bat']}, {'foo': 'b', 'bar': 'b', 'bat': 'whoo'}, - 'Metadata value at key bat is not in the allowed list of values') + {"allowed-values": ["a", 1], "keys": ["bat"]}, + {"foo": "b", "bar": "b", "bat": "whoo"}, + "Metadata value at key bat is not in the allowed list of values", + ) _enum_validate_fail( - {'allowed-values': ['a', 1], 'keys': ['bat']}, {'foo': 'b', 'bar': 'b', 'bat': None}, - 'Metadata value at key bat is not in the allowed list of values') + {"allowed-values": ["a", 1], "keys": ["bat"]}, + {"foo": "b", "bar": "b", "bat": None}, + "Metadata value at key bat is not in the allowed list of values", + ) def _enum_build_fail(cfg, expected): @@ -170,56 +239,82 @@ def _enum_build_fail(cfg, expected): def _enum_validate_fail(cfg, meta, expected): - assert builtin.enum(cfg)('key', meta)['message'] == expected + assert builtin.enum(cfg)("key", meta)["message"] == expected def test_units(): - _units_good({'key': 'u', 'units': 'N'}, {'u': 'lb * ft/ s^2'}) - _units_good({'key': 'y', 'units': 'degF'}, {'y': 'K', 'u': 'not a unit'}) - _units_good({'key': 'u', 'units': '(lb * ft^2) / (s^3 * A^2)'}, {'u': 'ohm'}) - _units_good({'key': 'y', 'units': 'cells'}, {'y': "cell"}) - _units_good({'key': 'u', 'units': 'cells / gram'}, {'u': "cells / g"}) - _units_good({'key': 'y', 'units': 'percent'}, {'y': "percent"}) + _units_good({"key": "u", "units": "N"}, {"u": "lb * ft/ s^2"}) + _units_good({"key": "y", "units": "degF"}, {"y": "K", "u": "not a unit"}) + _units_good({"key": "u", "units": "(lb * ft^2) / (s^3 * A^2)"}, {"u": "ohm"}) + _units_good({"key": "y", "units": "cells"}, {"y": "cell"}) + _units_good({"key": "u", "units": "cells / gram"}, {"u": "cells / g"}) + _units_good({"key": "y", "units": "percent"}, {"y": "percent"}) + def _units_good(cfg, meta): - assert builtin.units(cfg)('key', meta) is None + assert builtin.units(cfg)("key", meta) is None def test_units_build_fail(): - _units_build_fail(None, ValueError('d must be a dict')) - _units_build_fail([], ValueError('d must be a dict')) - _units_build_fail({}, ValueError('key is a required parameter')) - _units_build_fail({'key': None}, ValueError('key is a required parameter')) - _units_build_fail({'key': ['foo']}, ValueError('the key parameter must be a string')) - _units_build_fail({'key': 'foo'}, ValueError('units is a required parameter')) - _units_build_fail({'key': 'foo', 'units': None}, ValueError('units is a required parameter')) - _units_build_fail({'key': 'foo', 'units': 'm', 'whee': 'whoo'}, - ValueError('Unexpected configuration parameter: whee')) + _units_build_fail(None, ValueError("d must be a dict")) + _units_build_fail([], ValueError("d must be a dict")) + _units_build_fail({}, ValueError("key is a required parameter")) + _units_build_fail({"key": None}, ValueError("key is a required parameter")) + _units_build_fail( + {"key": ["foo"]}, ValueError("the key parameter must be a string") + ) + _units_build_fail({"key": "foo"}, ValueError("units is a required parameter")) + _units_build_fail( + {"key": "foo", "units": None}, ValueError("units is a required parameter") + ) _units_build_fail( - {'key': 'foo', 'units': ['N']}, ValueError('the units parameter must be a string')) + {"key": "foo", "units": "m", "whee": "whoo"}, + ValueError("Unexpected configuration parameter: whee"), + ) _units_build_fail( - {'key': 'foo', 'units': 'not a unit'}, ValueError( - "unable to parse units 'not a unit': undefined unit: not")) + {"key": "foo", "units": ["N"]}, + ValueError("the units parameter must be a string"), + ) _units_build_fail( - {'key': 'foo', 'units': 'm ^'}, ValueError( - 'unable to parse units \'m ^\': syntax error: missing unary operator "**"')) + {"key": "foo", "units": "not a unit"}, + ValueError("unable to parse units 'not a unit': undefined unit: not"), + ) + _units_build_fail( + {"key": "foo", "units": "m ^"}, + ValueError( + "unable to parse units 'm ^': syntax error: missing unary operator \"**\"" + ), + ) def test_units_validate_fail(): - _units_validate_fail({'key': 'u', 'units': 'm'}, {}, 'metadata value key u is required') - _units_validate_fail({'key': 'u', 'units': 'm'}, {'u': None}, - 'metadata value key u is required') - _units_validate_fail({'key': 'u', 'units': 'm'}, {'u': ['ft']}, - 'metadata value key u must be a string') - _units_validate_fail({'key': 'u', 'units': 'm'}, {'u': 'no_units_here'}, - "unable to parse units 'm' at key u: undefined unit: no_units_here") _units_validate_fail( - {'key': 'u', 'units': 'm'}, {'u': 'ft / '}, - 'unable to parse units \'m\' at key u: syntax error: missing unary operator "/"') + {"key": "u", "units": "m"}, {}, "metadata value key u is required" + ) + _units_validate_fail( + {"key": "u", "units": "m"}, {"u": None}, "metadata value key u is required" + ) + _units_validate_fail( + {"key": "u", "units": "m"}, + {"u": ["ft"]}, + "metadata value key u must be a string", + ) _units_validate_fail( - {'key': 'u', 'units': 'm'}, {'u': 's'}, - "Units at key u, 's', are not equivalent to required units, 'm': Cannot convert " + - "from 'second' ([time]) to 'meter' ([length])") + {"key": "u", "units": "m"}, + {"u": "no_units_here"}, + "unable to parse units 'm' at key u: undefined unit: no_units_here", + ) + _units_validate_fail( + {"key": "u", "units": "m"}, + {"u": "ft / "}, + "unable to parse units 'm' at key u: syntax error: missing unary operator \"/\"", + ) + _units_validate_fail( + {"key": "u", "units": "m"}, + {"u": "s"}, + "Units at key u, 's', are not equivalent to required units, 'm': Cannot convert " + + "from 'second' ([time]) to 'meter' ([length])", + ) def _units_build_fail(cfg, expected): @@ -229,146 +324,236 @@ def _units_build_fail(cfg, expected): def _units_validate_fail(cfg, meta, expected): - assert builtin.units(cfg)('key', meta)['message'] == expected + assert builtin.units(cfg)("key", meta)["message"] == expected def test_number(): - _number_success({}, {'whee': 1, '2': 2.3, '3': 10312.5677, '4': 682}) + _number_success({}, {"whee": 1, "2": 2.3, "3": 10312.5677, "4": 682}) # null type should allow floats - _number_success({'type': None}, {'whee': 1, '2': 2.3, '3': 10312.5677, '4': 682}) + _number_success({"type": None}, {"whee": 1, "2": 2.3, "3": 10312.5677, "4": 682}) # float type should allow floats - _number_success({'type': 'float'}, {'whee': 1, '2': 2.3, '3': 10312.5677, '4': 682}) + _number_success({"type": "float"}, {"whee": 1, "2": 2.3, "3": 10312.5677, "4": 682}) # required should be ignored - _number_success({'required': True}, {'whee': 1, '2': 2.3, '3': 10312.567, '4': 682, '5': None}) + _number_success( + {"required": True}, {"whee": 1, "2": 2.3, "3": 10312.567, "4": 682, "5": None} + ) # test none is ok and ints are ok - _number_success({'type': 'int'}, {'whee': 1, '2': 2, '3': 10312, '4': 682, '5': None}) + _number_success( + {"type": "int"}, {"whee": 1, "2": 2, "3": 10312, "4": 682, "5": None} + ) def test_number_with_ranges(): - _number_success({'gt': 4.6}, {'whee': 4.6000000000000001, '2': 4.7, '3': 10312.5677, '4': 682}) - _number_success({'gt': 4}, {'whee': 5, '2': 6, '3': 10312.5677, '4': 682}) - _number_success({'gte': 4.6}, {'whee': 4.6, '2': 4.7, '3': 10312.5677, '4': 682}) - _number_success({'gte': 4}, {'whee': 4, '2': 4.7, '3': 10312.5677, '4': 682}) - - _number_success({'lt': 10312.5678}, {'whee': 4.600000001, '2': 4.7, '3': 10312.5677, '4': 682}) - _number_success({'lt': 10312}, {'whee': 5, '2': 6, '3': 10311, '4': 682}) - _number_success({'lte': 10312.5678}, {'whee': 4.6, '2': 4.7, '3': 10312.5678, '4': 682}) - _number_success({'lte': 10312}, {'whee': 4, '2': 4.7, '3': 10312, '4': 682}) - - _number_success({'lt': 10312.5678, 'gte': 4}, {'whee': 4, '2': 4.7, '3': 10312.5677, '4': 682}) - _number_success({'lte': 10.1, 'gt': 10}, {'whee': 10.1, '2': 10.0000000000001, '3': 10.01}) + _number_success( + {"gt": 4.6}, {"whee": 4.6000000000000001, "2": 4.7, "3": 10312.5677, "4": 682} + ) + _number_success({"gt": 4}, {"whee": 5, "2": 6, "3": 10312.5677, "4": 682}) + _number_success({"gte": 4.6}, {"whee": 4.6, "2": 4.7, "3": 10312.5677, "4": 682}) + _number_success({"gte": 4}, {"whee": 4, "2": 4.7, "3": 10312.5677, "4": 682}) + + _number_success( + {"lt": 10312.5678}, {"whee": 4.600000001, "2": 4.7, "3": 10312.5677, "4": 682} + ) + _number_success({"lt": 10312}, {"whee": 5, "2": 6, "3": 10311, "4": 682}) + _number_success( + {"lte": 10312.5678}, {"whee": 4.6, "2": 4.7, "3": 10312.5678, "4": 682} + ) + _number_success({"lte": 10312}, {"whee": 4, "2": 4.7, "3": 10312, "4": 682}) + + _number_success( + {"lt": 10312.5678, "gte": 4}, {"whee": 4, "2": 4.7, "3": 10312.5677, "4": 682} + ) + _number_success( + {"lte": 10.1, "gt": 10}, {"whee": 10.1, "2": 10.0000000000001, "3": 10.01} + ) def test_number_with_keys(): - _number_success({'keys': 'whee'}, {'whee': 1, '2': 7.4, '3': True, '4': 'nan'}) + _number_success({"keys": "whee"}, {"whee": 1, "2": 7.4, "3": True, "4": "nan"}) # test keys are not required - _number_success({'keys': ['whee', '2', 'a']}, {'whee': 1, '2': 2.3, '3': True, '4': 'nan'}) - _number_success({'required': True, 'keys': ['whee', '2', '5']}, - {'whee': 64, '2': 18.3, '3': 'foo', '4': False, '5': None}) + _number_success( + {"keys": ["whee", "2", "a"]}, {"whee": 1, "2": 2.3, "3": True, "4": "nan"} + ) + _number_success( + {"required": True, "keys": ["whee", "2", "5"]}, + {"whee": 64, "2": 18.3, "3": "foo", "4": False, "5": None}, + ) # test int only & None is ok - _number_success({'type': 'int', 'keys': ['whee', '2', '5']}, - {'whee': 1, '2': 2, '3': 10312.3, '4': 'bar', '5': None}) + _number_success( + {"type": "int", "keys": ["whee", "2", "5"]}, + {"whee": 1, "2": 2, "3": 10312.3, "4": "bar", "5": None}, + ) def test_number_with_keys_and_ranges(): - _number_success({'gt': 4.6, 'keys': ['whee', '2']}, - {'whee': 4.6000000000000001, '2': 4.7, '3': 0, '4': 4}) - _number_success({'gt': 4, 'keys': ['whee', '2']}, - {'whee': 5, '2': 6, '3': 4, '4': -12}) - _number_success({'gte': 4.6, 'keys': ['whee', '2']}, - {'whee': 4.6, '2': 4.7, '3': 4.5999, '4': -300}) - _number_success({'gte': 4, 'keys': ['whee', '2']}, - {'whee': 4, '2': 4.7, '3': 3, '4': 682}) - - _number_success({'lt': 10312.5678, 'keys': ['whee', '3']}, - {'whee': 4.600000001, '2': 10321.5678, '3': 10312.5677, '4': 100000}) - _number_success({'lt': 10312, 'keys': ['whee', '3']}, - {'whee': 5, '2': 10312, '3': 10311, '4': 777777}) - _number_success({'lte': 10312.5678, 'keys': ['whee', '3']}, - {'whee': 4.6, '2': 10312.5679, '3': 10312.5678, '4': 1000000000}) - _number_success({'lte': 10312, 'keys': ['whee', '3']}, - {'whee': 4, '2': 10313, '3': 10312, '4': 1000000}) - - _number_success({'lt': 10312.5678, 'gte': 4, 'keys': ['whee', '2', '3']}, - {'whee': 4, '2': 4.7, '3': 10312.5677, '4': 3, '5': 10312.5679}) - _number_success({'lte': 10.1, 'gt': 10, 'keys': ['whee', '2', '3']}, - {'whee': 10.1, '2': 10.0000000000001, '3': 10.01, '4': 10.11, '5': 10}) + _number_success( + {"gt": 4.6, "keys": ["whee", "2"]}, + {"whee": 4.6000000000000001, "2": 4.7, "3": 0, "4": 4}, + ) + _number_success( + {"gt": 4, "keys": ["whee", "2"]}, {"whee": 5, "2": 6, "3": 4, "4": -12} + ) + _number_success( + {"gte": 4.6, "keys": ["whee", "2"]}, + {"whee": 4.6, "2": 4.7, "3": 4.5999, "4": -300}, + ) + _number_success( + {"gte": 4, "keys": ["whee", "2"]}, {"whee": 4, "2": 4.7, "3": 3, "4": 682} + ) + + _number_success( + {"lt": 10312.5678, "keys": ["whee", "3"]}, + {"whee": 4.600000001, "2": 10321.5678, "3": 10312.5677, "4": 100000}, + ) + _number_success( + {"lt": 10312, "keys": ["whee", "3"]}, + {"whee": 5, "2": 10312, "3": 10311, "4": 777777}, + ) + _number_success( + {"lte": 10312.5678, "keys": ["whee", "3"]}, + {"whee": 4.6, "2": 10312.5679, "3": 10312.5678, "4": 1000000000}, + ) + _number_success( + {"lte": 10312, "keys": ["whee", "3"]}, + {"whee": 4, "2": 10313, "3": 10312, "4": 1000000}, + ) + + _number_success( + {"lt": 10312.5678, "gte": 4, "keys": ["whee", "2", "3"]}, + {"whee": 4, "2": 4.7, "3": 10312.5677, "4": 3, "5": 10312.5679}, + ) + _number_success( + {"lte": 10.1, "gt": 10, "keys": ["whee", "2", "3"]}, + {"whee": 10.1, "2": 10.0000000000001, "3": 10.01, "4": 10.11, "5": 10}, + ) def _number_success(cfg, params): - assert builtin.number(cfg)('key', params) is None + assert builtin.number(cfg)("key", params) is None def test_number_build_fail(): - _number_build_fail(None, ValueError('d must be a dict')) - _number_build_fail([], ValueError('d must be a dict')) - _number_build_fail({'keys': {'a': 'b'}}, ValueError('keys parameter must be a string or list')) - _number_build_fail({'keys': ['a', 'b', 7]}, ValueError( - 'keys parameter contains a non-string entry at index 2')) - _number_build_fail({'type': 'foo'}, ValueError('Illegal value for type parameter: foo')) - _number_build_fail({'type': 'int', 'keys': ['bar'], 'required': True, - 'gt': 1, 'lte': 4, 'fakekey': 'yay'}, - ValueError('Unexpected configuration parameter: fakekey')) - _number_build_fail({'type': []}, ValueError('Illegal value for type parameter: []')) - - for r in ['gt', 'gte', 'lt', 'lte']: - _number_build_fail({r: 'foo'}, ValueError(f'Value for {r} parameter is not a number')) - _number_build_fail({r: []}, ValueError(f'Value for {r} parameter is not a number')) - - _number_build_fail({'gt': 7, 'gte': 8}, ValueError('Cannot specify both gt and gte')) - _number_build_fail({'lt': 5, 'lte': 89}, ValueError('Cannot specify both lt and lte')) + _number_build_fail(None, ValueError("d must be a dict")) + _number_build_fail([], ValueError("d must be a dict")) + _number_build_fail( + {"keys": {"a": "b"}}, ValueError("keys parameter must be a string or list") + ) + _number_build_fail( + {"keys": ["a", "b", 7]}, + ValueError("keys parameter contains a non-string entry at index 2"), + ) + _number_build_fail( + {"type": "foo"}, ValueError("Illegal value for type parameter: foo") + ) + _number_build_fail( + { + "type": "int", + "keys": ["bar"], + "required": True, + "gt": 1, + "lte": 4, + "fakekey": "yay", + }, + ValueError("Unexpected configuration parameter: fakekey"), + ) + _number_build_fail({"type": []}, ValueError("Illegal value for type parameter: []")) + + for r in ["gt", "gte", "lt", "lte"]: + _number_build_fail( + {r: "foo"}, ValueError(f"Value for {r} parameter is not a number") + ) + _number_build_fail( + {r: []}, ValueError(f"Value for {r} parameter is not a number") + ) + + _number_build_fail( + {"gt": 7, "gte": 8}, ValueError("Cannot specify both gt and gte") + ) + _number_build_fail( + {"lt": 5, "lte": 89}, ValueError("Cannot specify both lt and lte") + ) def test_number_validate_fails(): _number_validate_fail( - {}, {'a': 'foo'}, 'Metadata value at key a is not an accepted number type') + {}, {"a": "foo"}, "Metadata value at key a is not an accepted number type" + ) _number_validate_fail( - {}, {'a': {}}, 'Metadata value at key a is not an accepted number type') + {}, {"a": {}}, "Metadata value at key a is not an accepted number type" + ) _number_validate_fail( - {'type': 'int'}, {'a': 3.1}, 'Metadata value at key a is not an accepted number type') + {"type": "int"}, + {"a": 3.1}, + "Metadata value at key a is not an accepted number type", + ) _number_validate_fail( - {'gt': 7, 'lt': 9}, {'a': 7}, - 'Metadata value at key a is not within the range (7, 9)') + {"gt": 7, "lt": 9}, + {"a": 7}, + "Metadata value at key a is not within the range (7, 9)", + ) _number_validate_fail( - {'gt': 7, 'lt': 9}, {'a': 9}, - 'Metadata value at key a is not within the range (7, 9)') + {"gt": 7, "lt": 9}, + {"a": 9}, + "Metadata value at key a is not within the range (7, 9)", + ) _number_validate_fail( - {'gte': 0, 'lte': 100}, {'a': -0.00000000000000000001}, - 'Metadata value at key a is not within the range [0, 100]') + {"gte": 0, "lte": 100}, + {"a": -0.00000000000000000001}, + "Metadata value at key a is not within the range [0, 100]", + ) _number_validate_fail( - {'gte': 0, 'lte': 100}, {'a': 100.01}, - 'Metadata value at key a is not within the range [0, 100]') + {"gte": 0, "lte": 100}, + {"a": 100.01}, + "Metadata value at key a is not within the range [0, 100]", + ) # 0 is falsy, which is dumb and could cause problems here _number_validate_fail( - {'gte': 0.1, 'lte': 100}, {'a': 0}, - 'Metadata value at key a is not within the range [0.1, 100]') + {"gte": 0.1, "lte": 100}, + {"a": 0}, + "Metadata value at key a is not within the range [0.1, 100]", + ) def test_number_validate_with_keys_fails(): _number_validate_fail( - {'required': True, 'keys': ['a', 'b']}, {'a': 1}, - 'Required key b is missing') + {"required": True, "keys": ["a", "b"]}, {"a": 1}, "Required key b is missing" + ) _number_validate_fail( - {'keys': 'a'}, {'a': True}, 'Metadata value at key a is not an accepted number type') + {"keys": "a"}, + {"a": True}, + "Metadata value at key a is not an accepted number type", + ) _number_validate_fail( - {'type': 'int', 'keys': 'a'}, {'a': 3.1}, - 'Metadata value at key a is not an accepted number type') + {"type": "int", "keys": "a"}, + {"a": 3.1}, + "Metadata value at key a is not an accepted number type", + ) _number_validate_fail( - {'keys': 'a', 'gt': 7, 'lt': 9}, {'a': 7}, - 'Metadata value at key a is not within the range (7, 9)') + {"keys": "a", "gt": 7, "lt": 9}, + {"a": 7}, + "Metadata value at key a is not within the range (7, 9)", + ) _number_validate_fail( - {'keys': 'a', 'gt': 7, 'lt': 9}, {'a': 9}, - 'Metadata value at key a is not within the range (7, 9)') + {"keys": "a", "gt": 7, "lt": 9}, + {"a": 9}, + "Metadata value at key a is not within the range (7, 9)", + ) _number_validate_fail( - {'keys': 'a', 'gte': 0, 'lte': 100}, {'a': -0.00000000000000000001}, - 'Metadata value at key a is not within the range [0, 100]') + {"keys": "a", "gte": 0, "lte": 100}, + {"a": -0.00000000000000000001}, + "Metadata value at key a is not within the range [0, 100]", + ) _number_validate_fail( - {'keys': 'a', 'gte': 0, 'lte': 100}, {'a': 100.01}, - 'Metadata value at key a is not within the range [0, 100]') + {"keys": "a", "gte": 0, "lte": 100}, + {"a": 100.01}, + "Metadata value at key a is not within the range [0, 100]", + ) # 0 is falsy, which is dumb and could cause problems here _number_validate_fail( - {'gte': 0.1, 'lte': 100, 'keys': 'a'}, {'a': 0}, - 'Metadata value at key a is not within the range [0.1, 100]') + {"gte": 0.1, "lte": 100, "keys": "a"}, + {"a": 0}, + "Metadata value at key a is not within the range [0.1, 100]", + ) def _number_build_fail(cfg, expected): @@ -378,50 +563,72 @@ def _number_build_fail(cfg, expected): def _number_validate_fail(cfg, meta, expected): - assert builtin.number(cfg)('key', meta)['message'] == expected + assert builtin.number(cfg)("key", meta)["message"] == expected + def test_ontology_has_ancestor(): - _ontology_has_ancestor_success( - {'ontology': 'envo_ontology', 'ancestor_term':'ENVO:00010483' }, - {'Material': 'ENVO:00002041', 'ENVO:Material': 'ENVO:00002006'}) + _ontology_has_ancestor_success( + {"ontology": "envo_ontology", "ancestor_term": "ENVO:00010483"}, + {"Material": "ENVO:00002041", "ENVO:Material": "ENVO:00002006"}, + ) + def _ontology_has_ancestor_success(cfg, meta): - assert builtin.ontology_has_ancestor(cfg)('key', meta) is None + assert builtin.ontology_has_ancestor(cfg)("key", meta) is None + def test_ontology_has_ancestor_build_fail(): - _ontology_has_ancestor_build_fail(None, ValueError('d must be a dict')) - _ontology_has_ancestor_build_fail({'whee': 'whoo'}, - ValueError('Unexpected configuration parameter: whee')) - _ontology_has_ancestor_build_fail({'ontology': None}, - ValueError('ontology is a required parameter')) - _ontology_has_ancestor_build_fail({'ontology': ['foo']}, - ValueError('ontology must be a string')) - _ontology_has_ancestor_build_fail({'ontology': 'foo', 'ancestor_term': None}, - ValueError('ancestor_term is a required parameter')) - _ontology_has_ancestor_build_fail({'ontology': 'foo', 'ancestor_term': ['foo']}, - ValueError('ancestor_term must be a string')) + _ontology_has_ancestor_build_fail(None, ValueError("d must be a dict")) + _ontology_has_ancestor_build_fail( + {"whee": "whoo"}, ValueError("Unexpected configuration parameter: whee") + ) + _ontology_has_ancestor_build_fail( + {"ontology": None}, ValueError("ontology is a required parameter") + ) + _ontology_has_ancestor_build_fail( + {"ontology": ["foo"]}, ValueError("ontology must be a string") + ) _ontology_has_ancestor_build_fail( - {'ontology': 'foo', 'ancestor_term':'ENVO:00010483'}, - ValueError('ontology foo doesn\'t exist')) + {"ontology": "foo", "ancestor_term": None}, + ValueError("ancestor_term is a required parameter"), + ) _ontology_has_ancestor_build_fail( - {'ontology': 'envo_ontology', 'ancestor_term':'baz' }, - ValueError('ancestor_term baz is not found in envo_ontology')) + {"ontology": "foo", "ancestor_term": ["foo"]}, + ValueError("ancestor_term must be a string"), + ) + _ontology_has_ancestor_build_fail( + {"ontology": "foo", "ancestor_term": "ENVO:00010483"}, + ValueError("ontology foo doesn't exist"), + ) + _ontology_has_ancestor_build_fail( + {"ontology": "envo_ontology", "ancestor_term": "baz"}, + ValueError("ancestor_term baz is not found in envo_ontology"), + ) + def _ontology_has_ancestor_build_fail(cfg, expected): with raises(Exception) as got: builtin.ontology_has_ancestor(cfg) assert_exception_correct(got.value, expected) + def test_ontology_has_ancestor_validate_fail(): _ontology_has_ancestor_validate_fail( - {'ontology': 'envo_ontology', 'ancestor_term':'ENVO:00010483' }, - {'a': None}, 'Metadata value at key a is None') + {"ontology": "envo_ontology", "ancestor_term": "ENVO:00010483"}, + {"a": None}, + "Metadata value at key a is None", + ) _ontology_has_ancestor_validate_fail( - {'ontology': 'envo_ontology', 'ancestor_term':'ENVO:00010483' }, - {'a': 'foo'}, 'Metadata value at key a does not have envo_ontology ancestor term ENVO:00010483') + {"ontology": "envo_ontology", "ancestor_term": "ENVO:00010483"}, + {"a": "foo"}, + "Metadata value at key a does not have envo_ontology ancestor term ENVO:00010483", + ) _ontology_has_ancestor_validate_fail( - {'ontology': 'envo_ontology', 'ancestor_term':'ENVO:00002010' }, - {'a': 'ENVO:00002041'}, 'Metadata value at key a does not have envo_ontology ancestor term ENVO:00002010') + {"ontology": "envo_ontology", "ancestor_term": "ENVO:00002010"}, + {"a": "ENVO:00002041"}, + "Metadata value at key a does not have envo_ontology ancestor term ENVO:00002010", + ) + def _ontology_has_ancestor_validate_fail(cfg, meta, expected): - assert builtin.ontology_has_ancestor(cfg)('key', meta)['message'] == expected + assert builtin.ontology_has_ancestor(cfg)("key", meta)["message"] == expected diff --git a/test/core/validator/metadata_validator_test.py b/test/core/validator/metadata_validator_test.py index 8ff2f989..29a0ac97 100644 --- a/test/core/validator/metadata_validator_test.py +++ b/test/core/validator/metadata_validator_test.py @@ -2,7 +2,10 @@ from pytest import raises from core.test_utils import assert_exception_correct -from SampleService.core.validator.metadata_validator import MetadataValidatorSet, MetadataValidator +from SampleService.core.validator.metadata_validator import ( + MetadataValidatorSet, + MetadataValidator, +) from SampleService.core.errors import MetadataValidationError, IllegalParameterError @@ -15,51 +18,72 @@ def _noop3(_, __, ___): def val1(key, value): - return f'{key} {dict(sorted(value.items()))} 1' + return f"{key} {dict(sorted(value.items()))} 1" def val2(key, value): - return f'{key} {dict(sorted(value.items()))} 2' + return f"{key} {dict(sorted(value.items()))} 2" def test_construct_val_std(): - mv = MetadataValidator('mykey', [val1, val2]) + mv = MetadataValidator("mykey", [val1, val2]) - assert mv.key == 'mykey' + assert mv.key == "mykey" assert mv.metadata == {} assert mv.is_prefix_validator() is False assert len(mv.validators) == 2 assert len(mv.prefix_validators) == 0 - assert mv.validators[0]('foo', {'a': 'b'}) == "foo {'a': 'b'} 1" - assert mv.validators[1]('bar', {'c': 'd'}) == "bar {'c': 'd'} 2" + assert mv.validators[0]("foo", {"a": "b"}) == "foo {'a': 'b'} 1" + assert mv.validators[1]("bar", {"c": "d"}) == "bar {'c': 'd'} 2" def test_construct_val_prefix_and_meta(): mv = MetadataValidator( - 'my other key', prefix_validators=[val2], metadata={'foo': 'bar', 'baz': 'bat'}) + "my other key", prefix_validators=[val2], metadata={"foo": "bar", "baz": "bat"} + ) - assert mv.key == 'my other key' - assert mv.metadata == {'foo': 'bar', 'baz': 'bat'} + assert mv.key == "my other key" + assert mv.metadata == {"foo": "bar", "baz": "bat"} assert mv.is_prefix_validator() is True assert len(mv.validators) == 0 assert len(mv.prefix_validators) == 1 - assert mv.prefix_validators[0]('foo', {'a': 'b'}) == "foo {'a': 'b'} 2" + assert mv.prefix_validators[0]("foo", {"a": "b"}) == "foo {'a': 'b'} 2" def test_construct_val_fail_bad_args(): - _fail_construct_val(None, [val1], None, ValueError( - 'key cannot be a value that evaluates to false')) - _fail_construct_val('', [val1], None, ValueError( - 'key cannot be a value that evaluates to false')) - _fail_construct_val('k', None, None, ValueError( - 'Exactly one of validators or prefix_validators must be supplied and must contain ' + - 'at least one validator')) - _fail_construct_val('k', [], [], ValueError( - 'Exactly one of validators or prefix_validators must be supplied and must contain ' + - 'at least one validator')) - _fail_construct_val('k', [val1], [val2], ValueError( - 'Exactly one of validators or prefix_validators must be supplied and must contain ' + - 'at least one validator')) + _fail_construct_val( + None, [val1], None, ValueError("key cannot be a value that evaluates to false") + ) + _fail_construct_val( + "", [val1], None, ValueError("key cannot be a value that evaluates to false") + ) + _fail_construct_val( + "k", + None, + None, + ValueError( + "Exactly one of validators or prefix_validators must be supplied and must contain " + + "at least one validator" + ), + ) + _fail_construct_val( + "k", + [], + [], + ValueError( + "Exactly one of validators or prefix_validators must be supplied and must contain " + + "at least one validator" + ), + ) + _fail_construct_val( + "k", + [val1], + [val2], + ValueError( + "Exactly one of validators or prefix_validators must be supplied and must contain " + + "at least one validator" + ), + ) def _fail_construct_val(key, validators, prefix_validators, expected): @@ -69,18 +93,22 @@ def _fail_construct_val(key, validators, prefix_validators, expected): def test_construct_set_fail_bad_args(): - _fail_construct_set([ - MetadataValidator('key1', [_noop]), - MetadataValidator('key3', [_noop]), - MetadataValidator('key1', [_noop]), + _fail_construct_set( + [ + MetadataValidator("key1", [_noop]), + MetadataValidator("key3", [_noop]), + MetadataValidator("key1", [_noop]), ], - ValueError('Duplicate validator: key1')) - _fail_construct_set([ - MetadataValidator('key1', prefix_validators=[_noop]), - MetadataValidator('key3', prefix_validators=[_noop]), - MetadataValidator('key1', prefix_validators=[_noop]), + ValueError("Duplicate validator: key1"), + ) + _fail_construct_set( + [ + MetadataValidator("key1", prefix_validators=[_noop]), + MetadataValidator("key3", prefix_validators=[_noop]), + MetadataValidator("key1", prefix_validators=[_noop]), ], - ValueError('Duplicate prefix validator: key1')) + ValueError("Duplicate prefix validator: key1"), + ) def _fail_construct_set(validators, expected): @@ -97,164 +125,227 @@ def test_empty_set(): def test_set_with_validators(): - mv = MetadataValidatorSet([ - # this is vile - MetadataValidator( - 'key1', - [lambda k, v: exec('assert k == "key1"'), - lambda k, v: exec('assert v == {"a": "b"}') - ]), - MetadataValidator('key2', [lambda k, v: exec('assert k == "key2"')]) - ]) - - md = {'key1': {'a': 'b'}, 'key2': {'foo': 'bar'}} + mv = MetadataValidatorSet( + [ + # this is vile + MetadataValidator( + "key1", + [ + lambda k, v: exec('assert k == "key1"'), + lambda k, v: exec('assert v == {"a": "b"}'), + ], + ), + MetadataValidator("key2", [lambda k, v: exec('assert k == "key2"')]), + ] + ) + + md = {"key1": {"a": "b"}, "key2": {"foo": "bar"}} mv.validate_metadata(md) mv.validate_metadata(maps.FrozenMap(md)) - assert mv.keys() == ['key1', 'key2'] + assert mv.keys() == ["key1", "key2"] assert mv.prefix_keys() == [] - assert mv.validator_count('key1') == 2 - assert mv.validator_count('key2') == 1 + assert mv.validator_count("key1") == 2 + assert mv.validator_count("key2") == 1 def test_set_with_prefix_validators(): - mv = MetadataValidatorSet([ - # this is vile - MetadataValidator('pre1', prefix_validators=[ - lambda p, k, v: exec('assert p == "pre1"'), - lambda p, k, v: exec('assert k == "pre1stuff"'), - lambda p, k, v: exec('assert v == {"a": "b"}') - ]), - MetadataValidator('pre2', prefix_validators=[lambda p, k, v: exec('assert p == "pre2"')]) - ]) - - md = {'pre1stuff': {'a': 'b'}, 'pre2thingy': {'foo': 'bar'}} + mv = MetadataValidatorSet( + [ + # this is vile + MetadataValidator( + "pre1", + prefix_validators=[ + lambda p, k, v: exec('assert p == "pre1"'), + lambda p, k, v: exec('assert k == "pre1stuff"'), + lambda p, k, v: exec('assert v == {"a": "b"}'), + ], + ), + MetadataValidator( + "pre2", prefix_validators=[lambda p, k, v: exec('assert p == "pre2"')] + ), + ] + ) + + md = {"pre1stuff": {"a": "b"}, "pre2thingy": {"foo": "bar"}} mv.validate_metadata(md) mv.validate_metadata(maps.FrozenMap(md)) assert mv.keys() == [] - assert mv.prefix_keys() == ['pre1', 'pre2'] - assert mv.prefix_validator_count('pre1') == 3 - assert mv.prefix_validator_count('pre2') == 1 + assert mv.prefix_keys() == ["pre1", "pre2"] + assert mv.prefix_validator_count("pre1") == 3 + assert mv.prefix_validator_count("pre2") == 1 def test_set_with_prefix_validators_and_standard_validator(): - mv = MetadataValidatorSet([ - # this is vile - MetadataValidator('pre1', prefix_validators=[ - lambda p, k, v: exec('assert p == "pre1"'), - lambda p, k, v: exec('assert k == "pre1stuff"'), - lambda p, k, v: exec('assert v == {"a": "b"}') - ]), - MetadataValidator('pre2', prefix_validators=[lambda p, k, v: exec('assert p == "pre2"')]), - # test that non-prefix validator with same name is ok - MetadataValidator('pre2', [lambda k, v: exec('raise ValueError()')])] - ) - - md = {'pre1stuff': {'a': 'b'}, 'pre2thingy': {'foo': 'bar'}} + mv = MetadataValidatorSet( + [ + # this is vile + MetadataValidator( + "pre1", + prefix_validators=[ + lambda p, k, v: exec('assert p == "pre1"'), + lambda p, k, v: exec('assert k == "pre1stuff"'), + lambda p, k, v: exec('assert v == {"a": "b"}'), + ], + ), + MetadataValidator( + "pre2", prefix_validators=[lambda p, k, v: exec('assert p == "pre2"')] + ), + # test that non-prefix validator with same name is ok + MetadataValidator("pre2", [lambda k, v: exec("raise ValueError()")]), + ] + ) + + md = {"pre1stuff": {"a": "b"}, "pre2thingy": {"foo": "bar"}} mv.validate_metadata(md) mv.validate_metadata(maps.FrozenMap(md)) - assert mv.keys() == ['pre2'] - assert mv.prefix_keys() == ['pre1', 'pre2'] - assert mv.prefix_validator_count('pre1') == 3 - assert mv.prefix_validator_count('pre2') == 1 + assert mv.keys() == ["pre2"] + assert mv.prefix_keys() == ["pre1", "pre2"] + assert mv.prefix_validator_count("pre1") == 3 + assert mv.prefix_validator_count("pre2") == 1 def test_set_with_prefix_validators_multiple_matches(): results = [] - mv = MetadataValidatorSet([ - MetadataValidator('somekey', [lambda k, v: results.append((k, v))]), - MetadataValidator( - 'somekeya', - prefix_validators=[lambda p, k, v: exec('raise ValueError("test failed somekeya")')]), - MetadataValidator( - 'somekex', - prefix_validators=[lambda p, k, v: exec('raise ValueError("test failed somekex")')]), - MetadataValidator( - 'somekey', - prefix_validators=[lambda p, k, v: results.append((p, k, v))]), - MetadataValidator( - 'somekez', - prefix_validators=[lambda p, k, v: exec('raise ValueError("test failed somekez")')]), - MetadataValidator('someke', prefix_validators=[lambda p, k, v: results.append((p, k, v))]), - MetadataValidator('s', prefix_validators=[lambda p, k, v: results.append((p, k, v))]), - MetadataValidator( - 't', - prefix_validators=[lambda p, k, v: exec('raise ValueError("test failed t")')]), - ]) - md = {'somekey': {'x', 'y'}} + mv = MetadataValidatorSet( + [ + MetadataValidator("somekey", [lambda k, v: results.append((k, v))]), + MetadataValidator( + "somekeya", + prefix_validators=[ + lambda p, k, v: exec('raise ValueError("test failed somekeya")') + ], + ), + MetadataValidator( + "somekex", + prefix_validators=[ + lambda p, k, v: exec('raise ValueError("test failed somekex")') + ], + ), + MetadataValidator( + "somekey", prefix_validators=[lambda p, k, v: results.append((p, k, v))] + ), + MetadataValidator( + "somekez", + prefix_validators=[ + lambda p, k, v: exec('raise ValueError("test failed somekez")') + ], + ), + MetadataValidator( + "someke", prefix_validators=[lambda p, k, v: results.append((p, k, v))] + ), + MetadataValidator( + "s", prefix_validators=[lambda p, k, v: results.append((p, k, v))] + ), + MetadataValidator( + "t", + prefix_validators=[ + lambda p, k, v: exec('raise ValueError("test failed t")') + ], + ), + ] + ) + md = {"somekey": {"x", "y"}} mv.validate_metadata(md) print(results) assert results == [ - ('somekey', {'x', 'y'}), - ('s', 'somekey', {'x', 'y'}), - ('someke', 'somekey', {'x', 'y'}), - ('somekey', 'somekey', {'x', 'y'}), + ("somekey", {"x", "y"}), + ("s", "somekey", {"x", "y"}), + ("someke", "somekey", {"x", "y"}), + ("somekey", "somekey", {"x", "y"}), ] def test_set_with_key_metadata(): - mv = MetadataValidatorSet([ - MetadataValidator('pre1', prefix_validators=[_noop], metadata={'a': 'b', 'c': 'd'}), - MetadataValidator('pre2', prefix_validators=[_noop]), - MetadataValidator('pre3', prefix_validators=[_noop], metadata={'c': 'd'}), - MetadataValidator('pre1', [_noop]), - MetadataValidator('pre2', [_noop], metadata={'e': 'f', 'h': 'i'}), - MetadataValidator('pre3', [_noop], metadata={'h': 'i'}) - ]) + mv = MetadataValidatorSet( + [ + MetadataValidator( + "pre1", prefix_validators=[_noop], metadata={"a": "b", "c": "d"} + ), + MetadataValidator("pre2", prefix_validators=[_noop]), + MetadataValidator("pre3", prefix_validators=[_noop], metadata={"c": "d"}), + MetadataValidator("pre1", [_noop]), + MetadataValidator("pre2", [_noop], metadata={"e": "f", "h": "i"}), + MetadataValidator("pre3", [_noop], metadata={"h": "i"}), + ] + ) assert mv.key_metadata([]) == {} assert mv.prefix_key_metadata([]) == {} - assert mv.key_metadata(['pre1']) == {'pre1': {}} - assert mv.prefix_key_metadata(['pre1'], exact_match=True) == {'pre1': {'a': 'b', 'c': 'd'}} - - assert mv.key_metadata(['pre1', 'pre3']) == {'pre1': {}, - 'pre3': {'h': 'i'}} - assert mv.prefix_key_metadata(['pre1', 'pre2']) == {'pre1': {'a': 'b', 'c': 'd'}, - 'pre2': {}} - - assert mv.key_metadata(['pre1', 'pre2', 'pre3']) == {'pre1': {}, - 'pre2': {'e': 'f', 'h': 'i'}, - 'pre3': {'h': 'i'}} - assert mv.prefix_key_metadata(['pre1', 'pre2', 'pre3']) == {'pre1': {'a': 'b', 'c': 'd'}, - 'pre2': {}, - 'pre3': {'c': 'd'}} + assert mv.key_metadata(["pre1"]) == {"pre1": {}} + assert mv.prefix_key_metadata(["pre1"], exact_match=True) == { + "pre1": {"a": "b", "c": "d"} + } + + assert mv.key_metadata(["pre1", "pre3"]) == {"pre1": {}, "pre3": {"h": "i"}} + assert mv.prefix_key_metadata(["pre1", "pre2"]) == { + "pre1": {"a": "b", "c": "d"}, + "pre2": {}, + } + + assert mv.key_metadata(["pre1", "pre2", "pre3"]) == { + "pre1": {}, + "pre2": {"e": "f", "h": "i"}, + "pre3": {"h": "i"}, + } + assert mv.prefix_key_metadata(["pre1", "pre2", "pre3"]) == { + "pre1": {"a": "b", "c": "d"}, + "pre2": {}, + "pre3": {"c": "d"}, + } def test_set_with_prefix_match_key_metadata(): - mv = MetadataValidatorSet([ - MetadataValidator('a', prefix_validators=[_noop], metadata={'a': 'b'}), - MetadataValidator('abc', prefix_validators=[_noop], metadata={'c': 'd'}), - MetadataValidator('abcdef', prefix_validators=[_noop], metadata={'f': 'g'}), - MetadataValidator('abcdefhi', prefix_validators=[_noop], metadata={'f': 'g'}), - MetadataValidator('abzhi', prefix_validators=[_noop], metadata={'z': 'w'}), - MetadataValidator('abzhijk', prefix_validators=[_noop], metadata={'q': 'q'}), - MetadataValidator('b', prefix_validators=[_noop], metadata={'bbb': 'bbb'}), - ]) - - assert mv.prefix_key_metadata(['abcdef']) == {'abcdef': {'f': 'g'}} - assert mv.prefix_key_metadata(['abcdef'], exact_match=False) == { - 'a': {'a': 'b'}, 'abc': {'c': 'd'}, 'abcdef': {'f': 'g'}} - assert mv.prefix_key_metadata(['abcdefh'], exact_match=False) == { - 'a': {'a': 'b'}, 'abc': {'c': 'd'}, 'abcdef': {'f': 'g'}} - assert mv.prefix_key_metadata(['abzhij'], exact_match=False) == { - 'a': {'a': 'b'}, 'abzhi': {'z': 'w'}} - assert mv.prefix_key_metadata(['abcdef', 'abzhij'], exact_match=False) == { - 'a': {'a': 'b'}, - 'abc': {'c': 'd'}, - 'abcdef': {'f': 'g'}, - 'abzhi': {'z': 'w'}} + mv = MetadataValidatorSet( + [ + MetadataValidator("a", prefix_validators=[_noop], metadata={"a": "b"}), + MetadataValidator("abc", prefix_validators=[_noop], metadata={"c": "d"}), + MetadataValidator("abcdef", prefix_validators=[_noop], metadata={"f": "g"}), + MetadataValidator( + "abcdefhi", prefix_validators=[_noop], metadata={"f": "g"} + ), + MetadataValidator("abzhi", prefix_validators=[_noop], metadata={"z": "w"}), + MetadataValidator( + "abzhijk", prefix_validators=[_noop], metadata={"q": "q"} + ), + MetadataValidator("b", prefix_validators=[_noop], metadata={"bbb": "bbb"}), + ] + ) + + assert mv.prefix_key_metadata(["abcdef"]) == {"abcdef": {"f": "g"}} + assert mv.prefix_key_metadata(["abcdef"], exact_match=False) == { + "a": {"a": "b"}, + "abc": {"c": "d"}, + "abcdef": {"f": "g"}, + } + assert mv.prefix_key_metadata(["abcdefh"], exact_match=False) == { + "a": {"a": "b"}, + "abc": {"c": "d"}, + "abcdef": {"f": "g"}, + } + assert mv.prefix_key_metadata(["abzhij"], exact_match=False) == { + "a": {"a": "b"}, + "abzhi": {"z": "w"}, + } + assert mv.prefix_key_metadata(["abcdef", "abzhij"], exact_match=False) == { + "a": {"a": "b"}, + "abc": {"c": "d"}, + "abcdef": {"f": "g"}, + "abzhi": {"z": "w"}, + } def test_set_key_metadata_fail_bad_args(): - _key_metadata_fail_([], None, ValueError('keys cannot be None')) + _key_metadata_fail_([], None, ValueError("keys cannot be None")) _key_metadata_fail_( - [MetadataValidator('key1', [_noop]), MetadataValidator('key3', [_noop])], - ['key1', 'key2', 'key3'], - IllegalParameterError('No such metadata key: key2')) + [MetadataValidator("key1", [_noop]), MetadataValidator("key3", [_noop])], + ["key1", "key2", "key3"], + IllegalParameterError("No such metadata key: key2"), + ) def _key_metadata_fail_(vals, keys, expected): @@ -265,12 +356,15 @@ def _key_metadata_fail_(vals, keys, expected): def test_set_prefix_key_metadata_fail_bad_args(): - _prefix_key_metadata_fail_([], None, ValueError('keys cannot be None')) + _prefix_key_metadata_fail_([], None, ValueError("keys cannot be None")) _prefix_key_metadata_fail_( - [MetadataValidator('key1', prefix_validators=[_noop]), - MetadataValidator('key2', prefix_validators=[_noop])], - ['key1', 'key2', 'key3'], - IllegalParameterError('No such prefix metadata key: key3')) + [ + MetadataValidator("key1", prefix_validators=[_noop]), + MetadataValidator("key2", prefix_validators=[_noop]), + ], + ["key1", "key2", "key3"], + IllegalParameterError("No such prefix metadata key: key3"), + ) def _prefix_key_metadata_fail_(vals, keys, expected): @@ -285,49 +379,77 @@ def _prefix_key_metadata_fail_(vals, keys, expected): def test_prefix_key_metadata_fail_prefix_match(): - mv = MetadataValidatorSet([ - MetadataValidator('abcdef', prefix_validators=[_noop], metadata={'f': 'g'}), - MetadataValidator('abcdefhi', prefix_validators=[_noop], metadata={'f': 'g'}) - ]) + mv = MetadataValidatorSet( + [ + MetadataValidator("abcdef", prefix_validators=[_noop], metadata={"f": "g"}), + MetadataValidator( + "abcdefhi", prefix_validators=[_noop], metadata={"f": "g"} + ), + ] + ) with raises(Exception) as got: mv.prefix_key_metadata(None, exact_match=False) - assert_exception_correct(got.value, ValueError('keys cannot be None')) + assert_exception_correct(got.value, ValueError("keys cannot be None")) with raises(Exception) as got: - mv.prefix_key_metadata(['abcde'], exact_match=False) - assert_exception_correct(got.value, IllegalParameterError( - 'No prefix metadata keys matching key abcde')) + mv.prefix_key_metadata(["abcde"], exact_match=False) + assert_exception_correct( + got.value, IllegalParameterError("No prefix metadata keys matching key abcde") + ) def test_set_call_validator(): - mv = MetadataValidatorSet([ - MetadataValidator('key1', [lambda k, v: (k, v, 1), lambda k, v: (k, v, 2)]), - MetadataValidator('key2', [lambda k, v: (k, v, 3)]) - ]) - assert mv.call_validator('key1', 0, {'foo', 'bar'}) == ('key1', {'foo', 'bar'}, 1) - assert mv.call_validator('key1', 1, {'foo', 'bat'}) == ('key1', {'foo', 'bat'}, 2) - assert mv.call_validator('key2', 0, {'foo', 'baz'}) == ('key2', {'foo', 'baz'}, 3) + mv = MetadataValidatorSet( + [ + MetadataValidator("key1", [lambda k, v: (k, v, 1), lambda k, v: (k, v, 2)]), + MetadataValidator("key2", [lambda k, v: (k, v, 3)]), + ] + ) + assert mv.call_validator("key1", 0, {"foo", "bar"}) == ("key1", {"foo", "bar"}, 1) + assert mv.call_validator("key1", 1, {"foo", "bat"}) == ("key1", {"foo", "bat"}, 2) + assert mv.call_validator("key2", 0, {"foo", "baz"}) == ("key2", {"foo", "baz"}, 3) def test_set_call_prefix_validator(): - mv = MetadataValidatorSet([ - MetadataValidator( - 'p1', prefix_validators=[lambda p, k, v: (p, k, v, 1), lambda p, k, v: (p, k, v, 2)]), - MetadataValidator('p2', prefix_validators=[lambda p, k, v: (p, k, v, 3)]) - ]) - assert mv.call_prefix_validator( - 'p1', 0, 'key1', {'foo', 'bar'}) == ('p1', 'key1', {'foo', 'bar'}, 1) - assert mv.call_prefix_validator( - 'p1', 1, 'key11', {'foo', 'bat'}) == ('p1', 'key11', {'foo', 'bat'}, 2) - assert mv.call_prefix_validator( - 'p2', 0, 'key2', {'foo', 'baz'}) == ('p2', 'key2', {'foo', 'baz'}, 3) + mv = MetadataValidatorSet( + [ + MetadataValidator( + "p1", + prefix_validators=[ + lambda p, k, v: (p, k, v, 1), + lambda p, k, v: (p, k, v, 2), + ], + ), + MetadataValidator("p2", prefix_validators=[lambda p, k, v: (p, k, v, 3)]), + ] + ) + assert mv.call_prefix_validator("p1", 0, "key1", {"foo", "bar"}) == ( + "p1", + "key1", + {"foo", "bar"}, + 1, + ) + assert mv.call_prefix_validator("p1", 1, "key11", {"foo", "bat"}) == ( + "p1", + "key11", + {"foo", "bat"}, + 2, + ) + assert mv.call_prefix_validator("p2", 0, "key2", {"foo", "baz"}) == ( + "p2", + "key2", + {"foo", "baz"}, + 3, + ) def test_set_validator_count_fail(): - _validator_count_fail([ - MetadataValidator('key1', [_noop]), MetadataValidator('key2', [_noop])], - 'key3', ValueError('No validators for key key3')) + _validator_count_fail( + [MetadataValidator("key1", [_noop]), MetadataValidator("key2", [_noop])], + "key3", + ValueError("No validators for key key3"), + ) def _validator_count_fail(vals, key, expected): @@ -338,18 +460,30 @@ def _validator_count_fail(vals, key, expected): def test_set_prefix_validator_count_fail(): - _prefix_validator_count_fail([ - MetadataValidator('key1', prefix_validators=[_noop]), - MetadataValidator('key2', prefix_validators=[_noop])], - 'key3', ValueError('No prefix validators for prefix key3')) - _prefix_validator_count_fail([ # exact match required - MetadataValidator('key1', prefix_validators=[_noop]), - MetadataValidator('key', prefix_validators=[_noop])], - 'key3', ValueError('No prefix validators for prefix key3')) - _prefix_validator_count_fail([ - MetadataValidator('key1', prefix_validators=[_noop]), - MetadataValidator('key3', prefix_validators=[_noop])], - 'key', ValueError('No prefix validators for prefix key')) + _prefix_validator_count_fail( + [ + MetadataValidator("key1", prefix_validators=[_noop]), + MetadataValidator("key2", prefix_validators=[_noop]), + ], + "key3", + ValueError("No prefix validators for prefix key3"), + ) + _prefix_validator_count_fail( + [ # exact match required + MetadataValidator("key1", prefix_validators=[_noop]), + MetadataValidator("key", prefix_validators=[_noop]), + ], + "key3", + ValueError("No prefix validators for prefix key3"), + ) + _prefix_validator_count_fail( + [ + MetadataValidator("key1", prefix_validators=[_noop]), + MetadataValidator("key3", prefix_validators=[_noop]), + ], + "key", + ValueError("No prefix validators for prefix key"), + ) def _prefix_validator_count_fail(vals, prefix, expected): @@ -360,14 +494,18 @@ def _prefix_validator_count_fail(vals, prefix, expected): def test_set_call_validator_fail(): - _call_validator_fail([ - MetadataValidator('key1', [_noop]), - MetadataValidator('key2', [_noop])], - 'key3', 0, ValueError('No validators for key key3')) - _call_validator_fail([ - MetadataValidator('key1', [_noop]), - MetadataValidator('key2', [_noop, _noop])], - 'key2', 2, IndexError('Requested validator index 2 for key key2 but maximum index is 1')) + _call_validator_fail( + [MetadataValidator("key1", [_noop]), MetadataValidator("key2", [_noop])], + "key3", + 0, + ValueError("No validators for key key3"), + ) + _call_validator_fail( + [MetadataValidator("key1", [_noop]), MetadataValidator("key2", [_noop, _noop])], + "key2", + 2, + IndexError("Requested validator index 2 for key key2 but maximum index is 1"), + ) def _call_validator_fail(vals, key, index, expected): @@ -378,25 +516,50 @@ def _call_validator_fail(vals, key, index, expected): def test_set_call_prefix_validator_fail(): - _call_prefix_validator_fail([ - MetadataValidator('key1', prefix_validators=[_noop]), - MetadataValidator('key2', prefix_validators=[_noop])], - 'key3', 0, 'key3stuff', ValueError('No prefix validators for prefix key3')) - _call_prefix_validator_fail([ # exact match required - MetadataValidator('key1', prefix_validators=[_noop]), - MetadataValidator('key2', prefix_validators=[_noop]), - MetadataValidator('key', prefix_validators=[_noop])], - 'key3', 0, 'key3stuff', ValueError('No prefix validators for prefix key3')) - _call_prefix_validator_fail([ - MetadataValidator('key1', prefix_validators=[_noop]), - MetadataValidator('key2', prefix_validators=[_noop]), - MetadataValidator('key3', prefix_validators=[_noop])], - 'key', 0, 'key3stuff', ValueError('No prefix validators for prefix key')) - _call_prefix_validator_fail([ - MetadataValidator('key1', prefix_validators=[_noop]), - MetadataValidator('key2', prefix_validators=[_noop, _noop])], - 'key2', 2, 'key2stuff', - IndexError('Requested validator index 2 for prefix key2 but maximum index is 1')) + _call_prefix_validator_fail( + [ + MetadataValidator("key1", prefix_validators=[_noop]), + MetadataValidator("key2", prefix_validators=[_noop]), + ], + "key3", + 0, + "key3stuff", + ValueError("No prefix validators for prefix key3"), + ) + _call_prefix_validator_fail( + [ # exact match required + MetadataValidator("key1", prefix_validators=[_noop]), + MetadataValidator("key2", prefix_validators=[_noop]), + MetadataValidator("key", prefix_validators=[_noop]), + ], + "key3", + 0, + "key3stuff", + ValueError("No prefix validators for prefix key3"), + ) + _call_prefix_validator_fail( + [ + MetadataValidator("key1", prefix_validators=[_noop]), + MetadataValidator("key2", prefix_validators=[_noop]), + MetadataValidator("key3", prefix_validators=[_noop]), + ], + "key", + 0, + "key3stuff", + ValueError("No prefix validators for prefix key"), + ) + _call_prefix_validator_fail( + [ + MetadataValidator("key1", prefix_validators=[_noop]), + MetadataValidator("key2", prefix_validators=[_noop, _noop]), + ], + "key2", + 2, + "key2stuff", + IndexError( + "Requested validator index 2 for prefix key2 but maximum index is 1" + ), + ) def _call_prefix_validator_fail(vals, prefix, index, key, expected): @@ -405,83 +568,128 @@ def _call_prefix_validator_fail(vals, prefix, index, key, expected): mv.call_prefix_validator(prefix, index, key, {}) assert_exception_correct(got.value, expected) + def test_set_validate_metadata_return_errors(): _validate_metadata_errors( - [MetadataValidator('key1', [_noop]), - MetadataValidator('key2', [_noop]), - MetadataValidator('key', prefix_validators=[_noop3])], - {'key1': 'a', 'key2': 'b', 'kex': 'c'}, - 'kex', 'Cannot validate controlled field "kex", no matching validator found') + [ + MetadataValidator("key1", [_noop]), + MetadataValidator("key2", [_noop]), + MetadataValidator("key", prefix_validators=[_noop3]), + ], + {"key1": "a", "key2": "b", "kex": "c"}, + "kex", + 'Cannot validate controlled field "kex", no matching validator found', + ) _validate_metadata_errors( - [MetadataValidator('key1', [_noop]), MetadataValidator('key2', [_noop])], - {'key1': 'a', 'key2': 'b', 'key3': 'c'}, - 'key3', 'Cannot validate controlled field "key3", no matching validator found') + [MetadataValidator("key1", [_noop]), MetadataValidator("key2", [_noop])], + {"key1": "a", "key2": "b", "key3": "c"}, + "key3", + 'Cannot validate controlled field "key3", no matching validator found', + ) _validate_metadata_errors( - [MetadataValidator('keyx', prefix_validators=[_noop3])], - {'keyx1': 'a', 'keyx2': 'b', 'key': 'c'}, - 'key', 'Cannot validate controlled field "key", no matching validator found') + [MetadataValidator("keyx", prefix_validators=[_noop3])], + {"keyx1": "a", "keyx2": "b", "key": "c"}, + "key", + 'Cannot validate controlled field "key", no matching validator found', + ) _validate_metadata_errors( - [MetadataValidator('key1', [_noop]), - MetadataValidator('key2', [_noop]), - MetadataValidator('key3', [_noop, lambda _, __: 'oh poop'])], - {'key1': 'a', 'key2': 'b', 'key3': 'c'}, - 'key3', 'Key key3: oh poop') + [ + MetadataValidator("key1", [_noop]), + MetadataValidator("key2", [_noop]), + MetadataValidator("key3", [_noop, lambda _, __: "oh poop"]), + ], + {"key1": "a", "key2": "b", "key3": "c"}, + "key3", + "Key key3: oh poop", + ) _validate_metadata_errors( - [MetadataValidator('key1', prefix_validators=[_noop3]), - MetadataValidator('key2', prefix_validators=[_noop3]), - MetadataValidator('key3', prefix_validators=[_noop3, lambda _, __, ___: 'oh poop'])], - {'key1stuff': 'a', 'key2': 'b', 'key3yay': 'c'}, - 'key3yay', 'Prefix validator key3, key key3yay: oh poop') + [ + MetadataValidator("key1", prefix_validators=[_noop3]), + MetadataValidator("key2", prefix_validators=[_noop3]), + MetadataValidator( + "key3", prefix_validators=[_noop3, lambda _, __, ___: "oh poop"] + ), + ], + {"key1stuff": "a", "key2": "b", "key3yay": "c"}, + "key3yay", + "Prefix validator key3, key key3yay: oh poop", + ) + def test_set_validate_metadata_return_errors_with_subkey(): _validate_metadata_errors( - [MetadataValidator('key1', [_noop]), - MetadataValidator('key2', [_noop]), - MetadataValidator('key3', [_noop, lambda _, __: {'subkey': 'somekey','message':'oh poop'}])], - {'key1': 'a', 'key2': 'b', 'key3': 'c'}, - 'key3', 'Key key3: oh poop', 'somekey') + [ + MetadataValidator("key1", [_noop]), + MetadataValidator("key2", [_noop]), + MetadataValidator( + "key3", + [_noop, lambda _, __: {"subkey": "somekey", "message": "oh poop"}], + ), + ], + {"key1": "a", "key2": "b", "key3": "c"}, + "key3", + "Key key3: oh poop", + "somekey", + ) -def _validate_metadata_errors(vals, meta, expected_key, expected_dev_message, expected_subkey=None): + +def _validate_metadata_errors( + vals, meta, expected_key, expected_dev_message, expected_subkey=None +): mv = MetadataValidatorSet(vals) # with raises(Exception) as got: errors = mv.validate_metadata(meta, return_error_detail=True) assert len(errors) == 1 - assert str(errors[0]['key']) == str(expected_key) - assert str(errors[0]['dev_message']) == str(expected_dev_message) - assert str(errors[0]['subkey']) == str(expected_subkey) + assert str(errors[0]["key"]) == str(expected_key) + assert str(errors[0]["dev_message"]) == str(expected_dev_message) + assert str(errors[0]["subkey"]) == str(expected_subkey) def test_set_validate_metadata_fail(): _validate_metadata_fail( - [MetadataValidator('key1', [_noop]), MetadataValidator('key2', [_noop])], + [MetadataValidator("key1", [_noop]), MetadataValidator("key2", [_noop])], [], - ValueError('metadata must be a dict')) + ValueError("metadata must be a dict"), + ) _validate_metadata_fail( - [MetadataValidator('key1', [_noop]), - MetadataValidator('key2', [_noop]), - MetadataValidator('key', prefix_validators=[_noop3])], - {'key1': 'a', 'key2': 'b', 'kex': 'c'}, - MetadataValidationError('No validator available for metadata key kex')) + [ + MetadataValidator("key1", [_noop]), + MetadataValidator("key2", [_noop]), + MetadataValidator("key", prefix_validators=[_noop3]), + ], + {"key1": "a", "key2": "b", "kex": "c"}, + MetadataValidationError("No validator available for metadata key kex"), + ) _validate_metadata_fail( - [MetadataValidator('key1', [_noop]), MetadataValidator('key2', [_noop])], - {'key1': 'a', 'key2': 'b', 'key3': 'c'}, - MetadataValidationError('No validator available for metadata key key3')) + [MetadataValidator("key1", [_noop]), MetadataValidator("key2", [_noop])], + {"key1": "a", "key2": "b", "key3": "c"}, + MetadataValidationError("No validator available for metadata key key3"), + ) _validate_metadata_fail( - [MetadataValidator('keyx', prefix_validators=[_noop3])], - {'keyx1': 'a', 'keyx2': 'b', 'key': 'c'}, - MetadataValidationError('No validator available for metadata key key')) + [MetadataValidator("keyx", prefix_validators=[_noop3])], + {"keyx1": "a", "keyx2": "b", "key": "c"}, + MetadataValidationError("No validator available for metadata key key"), + ) _validate_metadata_fail( - [MetadataValidator('key1', [_noop]), - MetadataValidator('key2', [_noop]), - MetadataValidator('key3', [_noop, lambda _, __: 'oh poop'])], - {'key1': 'a', 'key2': 'b', 'key3': 'c'}, - MetadataValidationError('Key key3: oh poop')) + [ + MetadataValidator("key1", [_noop]), + MetadataValidator("key2", [_noop]), + MetadataValidator("key3", [_noop, lambda _, __: "oh poop"]), + ], + {"key1": "a", "key2": "b", "key3": "c"}, + MetadataValidationError("Key key3: oh poop"), + ) _validate_metadata_fail( - [MetadataValidator('key1', prefix_validators=[_noop3]), - MetadataValidator('key2', prefix_validators=[_noop3]), - MetadataValidator('key3', prefix_validators=[_noop3, lambda _, __, ___: 'oh poop'])], - {'key1stuff': 'a', 'key2': 'b', 'key3yay': 'c'}, - MetadataValidationError('Prefix validator key3, key key3yay: oh poop')) + [ + MetadataValidator("key1", prefix_validators=[_noop3]), + MetadataValidator("key2", prefix_validators=[_noop3]), + MetadataValidator( + "key3", prefix_validators=[_noop3, lambda _, __, ___: "oh poop"] + ), + ], + {"key1stuff": "a", "key2": "b", "key3yay": "c"}, + MetadataValidationError("Prefix validator key3, key key3yay: oh poop"), + ) def _validate_metadata_fail(vals, meta, expected): diff --git a/test/core/workspace_test.py b/test/core/workspace_test.py index 3a3fcb69..4e6b8a95 100644 --- a/test/core/workspace_test.py +++ b/test/core/workspace_test.py @@ -16,20 +16,20 @@ def test_upa_init(): - _upa_init_str('1/1/1', 1, 1, 1, '1/1/1') - _upa_init_str('64/790/17895101', 64, 790, 17895101, '64/790/17895101') + _upa_init_str("1/1/1", 1, 1, 1, "1/1/1") + _upa_init_str("64/790/17895101", 64, 790, 17895101, "64/790/17895101") - _upa_init_int(1, 1, 1, 1, 1, 1, '1/1/1') - _upa_init_int(89, 356, 2, 89, 356, 2, '89/356/2') + _upa_init_int(1, 1, 1, 1, 1, 1, "1/1/1") + _upa_init_int(89, 356, 2, 89, 356, 2, "89/356/2") def test_upa_init_all_args(): - u = UPA(upa='6/3/9', wsid=7, objid=4, version=10) + u = UPA(upa="6/3/9", wsid=7, objid=4, version=10) assert u.wsid == 6 assert u.objid == 3 assert u.version == 9 - assert str(u) == '6/3/9' + assert str(u) == "6/3/9" def _upa_init_str(str_, wsid, objid, ver, outstr): @@ -53,32 +53,40 @@ def _upa_init_int(wsid, objid, ver, wside, objide, vere, outstr): def test_upa_init_fail(): with raises(Exception) as got: UPA() - assert_exception_correct(got.value, IllegalParameterError('Illegal workspace ID: None')) - - _upa_init_str_fail('1', IllegalParameterError('1 is not a valid UPA')) - _upa_init_str_fail('1/2', IllegalParameterError('1/2 is not a valid UPA')) - _upa_init_str_fail('1/2/3/5', IllegalParameterError('1/2/3/5 is not a valid UPA')) - _upa_init_str_fail('1/2/3/', IllegalParameterError('1/2/3/ is not a valid UPA')) - _upa_init_str_fail('/1/2/3', IllegalParameterError('/1/2/3 is not a valid UPA')) - _upa_init_str_fail('f/2/3', IllegalParameterError('f/2/3 is not a valid UPA')) - _upa_init_str_fail('1/f/3', IllegalParameterError('1/f/3 is not a valid UPA')) - _upa_init_str_fail('1/2/f', IllegalParameterError('1/2/f is not a valid UPA')) - _upa_init_str_fail('0/2/3', IllegalParameterError('0/2/3 is not a valid UPA')) - _upa_init_str_fail('1/0/3', IllegalParameterError('1/0/3 is not a valid UPA')) - _upa_init_str_fail('1/2/0', IllegalParameterError('1/2/0 is not a valid UPA')) - _upa_init_str_fail('-24/2/3', IllegalParameterError('-24/2/3 is not a valid UPA')) - _upa_init_str_fail('1/-42/3', IllegalParameterError('1/-42/3 is not a valid UPA')) - _upa_init_str_fail('1/2/-10677810', IllegalParameterError('1/2/-10677810 is not a valid UPA')) - - _upa_init_int_fail(None, 2, 3, IllegalParameterError('Illegal workspace ID: None')) - _upa_init_int_fail(1, None, 3, IllegalParameterError('Illegal object ID: None')) - _upa_init_int_fail(1, 2, None, IllegalParameterError('Illegal object version: None')) - _upa_init_int_fail(0, 2, 3, IllegalParameterError('Illegal workspace ID: 0')) - _upa_init_int_fail(1, 0, 3, IllegalParameterError('Illegal object ID: 0')) - _upa_init_int_fail(1, 2, 0, IllegalParameterError('Illegal object version: 0')) - _upa_init_int_fail(-98, 2, 3, IllegalParameterError('Illegal workspace ID: -98')) - _upa_init_int_fail(1, -6, 3, IllegalParameterError('Illegal object ID: -6')) - _upa_init_int_fail(1, 2, -87501, IllegalParameterError('Illegal object version: -87501')) + assert_exception_correct( + got.value, IllegalParameterError("Illegal workspace ID: None") + ) + + _upa_init_str_fail("1", IllegalParameterError("1 is not a valid UPA")) + _upa_init_str_fail("1/2", IllegalParameterError("1/2 is not a valid UPA")) + _upa_init_str_fail("1/2/3/5", IllegalParameterError("1/2/3/5 is not a valid UPA")) + _upa_init_str_fail("1/2/3/", IllegalParameterError("1/2/3/ is not a valid UPA")) + _upa_init_str_fail("/1/2/3", IllegalParameterError("/1/2/3 is not a valid UPA")) + _upa_init_str_fail("f/2/3", IllegalParameterError("f/2/3 is not a valid UPA")) + _upa_init_str_fail("1/f/3", IllegalParameterError("1/f/3 is not a valid UPA")) + _upa_init_str_fail("1/2/f", IllegalParameterError("1/2/f is not a valid UPA")) + _upa_init_str_fail("0/2/3", IllegalParameterError("0/2/3 is not a valid UPA")) + _upa_init_str_fail("1/0/3", IllegalParameterError("1/0/3 is not a valid UPA")) + _upa_init_str_fail("1/2/0", IllegalParameterError("1/2/0 is not a valid UPA")) + _upa_init_str_fail("-24/2/3", IllegalParameterError("-24/2/3 is not a valid UPA")) + _upa_init_str_fail("1/-42/3", IllegalParameterError("1/-42/3 is not a valid UPA")) + _upa_init_str_fail( + "1/2/-10677810", IllegalParameterError("1/2/-10677810 is not a valid UPA") + ) + + _upa_init_int_fail(None, 2, 3, IllegalParameterError("Illegal workspace ID: None")) + _upa_init_int_fail(1, None, 3, IllegalParameterError("Illegal object ID: None")) + _upa_init_int_fail( + 1, 2, None, IllegalParameterError("Illegal object version: None") + ) + _upa_init_int_fail(0, 2, 3, IllegalParameterError("Illegal workspace ID: 0")) + _upa_init_int_fail(1, 0, 3, IllegalParameterError("Illegal object ID: 0")) + _upa_init_int_fail(1, 2, 0, IllegalParameterError("Illegal object version: 0")) + _upa_init_int_fail(-98, 2, 3, IllegalParameterError("Illegal workspace ID: -98")) + _upa_init_int_fail(1, -6, 3, IllegalParameterError("Illegal object ID: -6")) + _upa_init_int_fail( + 1, 2, -87501, IllegalParameterError("Illegal object version: -87501") + ) def _upa_init_str_fail(upa, expected): @@ -94,15 +102,15 @@ def _upa_init_int_fail(wsid, objid, ver, expected): def test_upa_equals(): - assert UPA('1/2/3') == UPA(wsid=1, objid=2, version=3) - assert UPA('1/2/3') == UPA('1/2/3') - assert UPA(wsid=56, objid=90, version=211) == UPA('56/90/211') - assert UPA('56/90/211') == UPA('56/90/211') + assert UPA("1/2/3") == UPA(wsid=1, objid=2, version=3) + assert UPA("1/2/3") == UPA("1/2/3") + assert UPA(wsid=56, objid=90, version=211) == UPA("56/90/211") + assert UPA("56/90/211") == UPA("56/90/211") - assert UPA('1/2/3') != '1/2/3' + assert UPA("1/2/3") != "1/2/3" - assert UPA('1/2/3') != UPA(wsid=2, objid=2, version=3) - assert UPA('1/3/3') != UPA('1/2/3') + assert UPA("1/2/3") != UPA(wsid=2, objid=2, version=3) + assert UPA("1/3/3") != UPA("1/2/3") assert UPA(wsid=1, objid=2, version=3) != UPA(wsid=1, objid=2, version=4) @@ -110,79 +118,89 @@ def test_upa_hash(): # string hashes will change from instance to instance of the python interpreter, and therefore # tests can't be written that directly test the hash value. See # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - assert hash(UPA('1/2/3')) == hash(UPA(wsid=1, objid=2, version=3)) - assert hash(UPA('1/2/3')) == hash(UPA('1/2/3')) - assert hash(UPA(wsid=56, objid=90, version=211)) == hash(UPA('56/90/211')) - assert hash(UPA('56/90/211')) == hash(UPA('56/90/211')) - - assert hash(UPA('1/2/3')) != hash(UPA(wsid=2, objid=2, version=3)) - assert hash(UPA('1/3/3')) != hash(UPA('1/2/3')) - assert hash(UPA(wsid=1, objid=2, version=3)) != hash(UPA(wsid=1, objid=2, version=4)) + assert hash(UPA("1/2/3")) == hash(UPA(wsid=1, objid=2, version=3)) + assert hash(UPA("1/2/3")) == hash(UPA("1/2/3")) + assert hash(UPA(wsid=56, objid=90, version=211)) == hash(UPA("56/90/211")) + assert hash(UPA("56/90/211")) == hash(UPA("56/90/211")) + + assert hash(UPA("1/2/3")) != hash(UPA(wsid=2, objid=2, version=3)) + assert hash(UPA("1/3/3")) != hash(UPA("1/2/3")) + assert hash(UPA(wsid=1, objid=2, version=3)) != hash( + UPA(wsid=1, objid=2, version=4) + ) def test_duid_init(): - duid = DataUnitID(UPA('1/2/3')) - assert duid.upa == UPA('1/2/3') + duid = DataUnitID(UPA("1/2/3")) + assert duid.upa == UPA("1/2/3") assert duid.dataid is None - assert str(duid) == '1/2/3' + assert str(duid) == "1/2/3" - duid = DataUnitID(UPA('1/2/3'), '') - assert duid.upa == UPA('1/2/3') + duid = DataUnitID(UPA("1/2/3"), "") + assert duid.upa == UPA("1/2/3") assert duid.dataid is None - assert str(duid) == '1/2/3' + assert str(duid) == "1/2/3" - duid = DataUnitID(UPA('1/2/3'), 'foo') - assert duid.upa == UPA('1/2/3') - assert duid.dataid == 'foo' - assert str(duid) == '1/2/3:foo' + duid = DataUnitID(UPA("1/2/3"), "foo") + assert duid.upa == UPA("1/2/3") + assert duid.dataid == "foo" + assert str(duid) == "1/2/3:foo" - duid = DataUnitID(UPA('1/2/3'), 'f' * 256) - assert duid.upa == UPA('1/2/3') - assert duid.dataid == 'f' * 256 - assert str(duid) == '1/2/3:' + 'f' * 256 + duid = DataUnitID(UPA("1/2/3"), "f" * 256) + assert duid.upa == UPA("1/2/3") + assert duid.dataid == "f" * 256 + assert str(duid) == "1/2/3:" + "f" * 256 def test_duid_init_fail(): with raises(Exception) as got: DataUnitID(None) - assert_exception_correct(got.value, ValueError( - 'upa cannot be a value that evaluates to false')) + assert_exception_correct( + got.value, ValueError("upa cannot be a value that evaluates to false") + ) with raises(Exception) as got: - DataUnitID(UPA('1/1/1'), 'a' * 257) - assert_exception_correct(got.value, IllegalParameterError( - 'dataid exceeds maximum length of 256')) + DataUnitID(UPA("1/1/1"), "a" * 257) + assert_exception_correct( + got.value, IllegalParameterError("dataid exceeds maximum length of 256") + ) def test_duid_equals(): - assert DataUnitID(UPA('1/1/1')) == DataUnitID(UPA('1/1/1')) - assert DataUnitID(UPA('1/1/1'), 'foo') == DataUnitID(UPA('1/1/1'), 'foo') + assert DataUnitID(UPA("1/1/1")) == DataUnitID(UPA("1/1/1")) + assert DataUnitID(UPA("1/1/1"), "foo") == DataUnitID(UPA("1/1/1"), "foo") - assert DataUnitID(UPA('1/1/1')) != UPA('1/1/1') + assert DataUnitID(UPA("1/1/1")) != UPA("1/1/1") - assert DataUnitID(UPA('1/1/1')) != DataUnitID(UPA('2/1/1')) - assert DataUnitID(UPA('1/1/1'), 'foo') != DataUnitID(UPA('2/1/1'), 'foo') - assert DataUnitID(UPA('1/1/1'), 'foo') != DataUnitID(UPA('1/1/1'), 'fooo') + assert DataUnitID(UPA("1/1/1")) != DataUnitID(UPA("2/1/1")) + assert DataUnitID(UPA("1/1/1"), "foo") != DataUnitID(UPA("2/1/1"), "foo") + assert DataUnitID(UPA("1/1/1"), "foo") != DataUnitID(UPA("1/1/1"), "fooo") def test_duid_hash(): # string hashes will change from instance to instance of the python interpreter, and therefore # tests can't be written that directly test the hash value. See # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - assert hash(DataUnitID(UPA('1/1/1'))) == hash(DataUnitID(UPA('1/1/1'))) - assert hash(DataUnitID(UPA('1/1/1'), 'foo')) == hash(DataUnitID(UPA('1/1/1'), 'foo')) + assert hash(DataUnitID(UPA("1/1/1"))) == hash(DataUnitID(UPA("1/1/1"))) + assert hash(DataUnitID(UPA("1/1/1"), "foo")) == hash( + DataUnitID(UPA("1/1/1"), "foo") + ) - assert hash(DataUnitID(UPA('1/1/1'))) != hash(DataUnitID(UPA('2/1/1'))) - assert hash(DataUnitID(UPA('1/1/1'), 'foo')) != hash(DataUnitID(UPA('2/1/1'), 'foo')) - assert hash(DataUnitID(UPA('1/1/1'), 'foo')) != hash(DataUnitID(UPA('1/1/1'), 'fooo')) + assert hash(DataUnitID(UPA("1/1/1"))) != hash(DataUnitID(UPA("2/1/1"))) + assert hash(DataUnitID(UPA("1/1/1"), "foo")) != hash( + DataUnitID(UPA("2/1/1"), "foo") + ) + assert hash(DataUnitID(UPA("1/1/1"), "foo")) != hash( + DataUnitID(UPA("1/1/1"), "fooo") + ) def test_init_fail(): - _init_fail(None, ValueError('client cannot be a value that evaluates to false')) + _init_fail(None, ValueError("client cannot be a value that evaluates to false")) wsc = create_autospec(Workspace, spec_set=True, instance=True) - wsc.administer.side_effect = ServerError('jsonrpcerror', 24, 'poopoo') - _init_fail(wsc, ServerError('jsonrpcerror', 24, 'poopoo')) + wsc.administer.side_effect = ServerError("jsonrpcerror", 24, "poopoo") + _init_fail(wsc, ServerError("jsonrpcerror", 24, "poopoo")) def _init_fail(wsc, expected): @@ -192,65 +210,107 @@ def _init_fail(wsc, expected): def test_has_permission(): - _has_permission(UserID('a'), None, UPA('42/65/3'), WorkspaceAccessType.READ, 42) - _has_permission(UserID('b'), 24, UPA('67/2/92'), WorkspaceAccessType.READ, 24) - _has_permission(UserID('c'), 1, None, WorkspaceAccessType.READ, 1) - _has_permission(UserID('a'), None, UPA('7/45/789'), WorkspaceAccessType.WRITE, 7) - _has_permission(UserID('c'), None, UPA('1/1/1'), WorkspaceAccessType.WRITE, 1) - _has_permission(UserID('c'), 301, None, WorkspaceAccessType.ADMIN, 301) - _has_permission(UserID('none'), 301, None, WorkspaceAccessType.NONE, 301) - _has_permission(UserID('none'), 301, None, WorkspaceAccessType.READ, 301, public=True) + _has_permission(UserID("a"), None, UPA("42/65/3"), WorkspaceAccessType.READ, 42) + _has_permission(UserID("b"), 24, UPA("67/2/92"), WorkspaceAccessType.READ, 24) + _has_permission(UserID("c"), 1, None, WorkspaceAccessType.READ, 1) + _has_permission(UserID("a"), None, UPA("7/45/789"), WorkspaceAccessType.WRITE, 7) + _has_permission(UserID("c"), None, UPA("1/1/1"), WorkspaceAccessType.WRITE, 1) + _has_permission(UserID("c"), 301, None, WorkspaceAccessType.ADMIN, 301) + _has_permission(UserID("none"), 301, None, WorkspaceAccessType.NONE, 301) + _has_permission( + UserID("none"), 301, None, WorkspaceAccessType.READ, 301, public=True + ) _has_permission(None, 301, None, WorkspaceAccessType.READ, 301, public=True) def test_has_permission_fail_bad_input(): r = WorkspaceAccessType.READ - u = UserID('b') - _has_permission_fail(u, None, None, r, ValueError( - 'Either an UPA or a workpace ID must be supplied')) - _has_permission_fail(u, 0, None, r, IllegalParameterError('0 is not a valid workspace ID')) - _has_permission_fail(u, 1, None, None, ValueError( - 'perm cannot be a value that evaluates to false')) + u = UserID("b") + _has_permission_fail( + u, None, None, r, ValueError("Either an UPA or a workpace ID must be supplied") + ) + _has_permission_fail( + u, 0, None, r, IllegalParameterError("0 is not a valid workspace ID") + ) + _has_permission_fail( + u, 1, None, None, ValueError("perm cannot be a value that evaluates to false") + ) def test_has_permission_fail_unauthorized(): r = WorkspaceAccessType.READ w = WorkspaceAccessType.WRITE a = WorkspaceAccessType.ADMIN - _has_permission_fail(UserID('d'), 1, None, r, UnauthorizedError( - 'User d cannot read workspace 1')) - _has_permission_fail(None, None, UPA('1/1/1'), r, UnauthorizedError( - 'Anonymous users cannot read upa 1/1/1')) - _has_permission_fail(UserID('d'), 34, None, w, UnauthorizedError( - 'User d cannot write to workspace 34'), public=True) - _has_permission_fail(UserID('b'), None, UPA('6/7/8'), w, UnauthorizedError( - 'User b cannot write to upa 6/7/8'), public=True) - _has_permission_fail(UserID('d'), 6, None, a, UnauthorizedError( - 'User d cannot administrate workspace 6'), public=True) - _has_permission_fail(UserID('b'), 74, None, a, UnauthorizedError( - 'User b cannot administrate workspace 74'), public=True) - _has_permission_fail(UserID('a'), None, UPA('890/44/1'), a, UnauthorizedError( - 'User a cannot administrate upa 890/44/1'), public=True) + _has_permission_fail( + UserID("d"), 1, None, r, UnauthorizedError("User d cannot read workspace 1") + ) + _has_permission_fail( + None, + None, + UPA("1/1/1"), + r, + UnauthorizedError("Anonymous users cannot read upa 1/1/1"), + ) + _has_permission_fail( + UserID("d"), + 34, + None, + w, + UnauthorizedError("User d cannot write to workspace 34"), + public=True, + ) + _has_permission_fail( + UserID("b"), + None, + UPA("6/7/8"), + w, + UnauthorizedError("User b cannot write to upa 6/7/8"), + public=True, + ) + _has_permission_fail( + UserID("d"), + 6, + None, + a, + UnauthorizedError("User d cannot administrate workspace 6"), + public=True, + ) + _has_permission_fail( + UserID("b"), + 74, + None, + a, + UnauthorizedError("User b cannot administrate workspace 74"), + public=True, + ) + _has_permission_fail( + UserID("a"), + None, + UPA("890/44/1"), + a, + UnauthorizedError("User a cannot administrate upa 890/44/1"), + public=True, + ) def test_has_permission_fail_on_get_perms_no_workspace(): _has_permission_fail_ws_exception( - ServerError('JSONRPCError', -32500, 'No workspace with id 22 exists'), - NoSuchWorkspaceDataError('No workspace with id 22 exists') + ServerError("JSONRPCError", -32500, "No workspace with id 22 exists"), + NoSuchWorkspaceDataError("No workspace with id 22 exists"), ) def test_has_permission_fail_on_get_perms_deleted_workspace(): _has_permission_fail_ws_exception( - ServerError('JSONRPCError', -32500, 'Workspace 22 is deleted'), - NoSuchWorkspaceDataError('Workspace 22 is deleted') + ServerError("JSONRPCError", -32500, "Workspace 22 is deleted"), + NoSuchWorkspaceDataError("Workspace 22 is deleted"), ) def test_has_permission_fail_on_get_perms_server_error(): _has_permission_fail_ws_exception( - ServerError('JSONRPCError', -32500, "Things is f'up"), - ServerError('JSONRPCError', -32500, "Things is f'up") + ServerError("JSONRPCError", -32500, "Things is f'up"), + ServerError("JSONRPCError", -32500, "Things is f'up"), ) @@ -258,56 +318,68 @@ def test_has_permission_fail_no_object(): wsc = create_autospec(Workspace, spec_set=True, instance=True) ws = WS(wsc) - wsc.administer.assert_called_once_with({'command': 'listModRequests'}) + wsc.administer.assert_called_once_with({"command": "listModRequests"}) wsc.administer.side_effect = [ - {'perms': [{'a': 'w', 'b': 'r', 'c': 'a'}]}, - {'infos': [None]}] + {"perms": [{"a": "w", "b": "r", "c": "a"}]}, + {"infos": [None]}, + ] with raises(Exception) as got: - ws.has_permission(UserID('b'), WorkspaceAccessType.READ, upa=UPA('67/8/90')) - assert_exception_correct(got.value, NoSuchWorkspaceDataError('Object 67/8/90 does not exist')) + ws.has_permission(UserID("b"), WorkspaceAccessType.READ, upa=UPA("67/8/90")) + assert_exception_correct( + got.value, NoSuchWorkspaceDataError("Object 67/8/90 does not exist") + ) def test_has_permission_fail_on_get_info_server_error(): wsc = create_autospec(Workspace, spec_set=True, instance=True) ws = WS(wsc) - wsc.administer.assert_called_once_with({'command': 'listModRequests'}) + wsc.administer.assert_called_once_with({"command": "listModRequests"}) wsc.administer.side_effect = [ - {'perms': [{'a': 'w', 'b': 'r', 'c': 'a'}]}, - ServerError('JSONRPCError', -32500, 'Thanks Obama')] + {"perms": [{"a": "w", "b": "r", "c": "a"}]}, + ServerError("JSONRPCError", -32500, "Thanks Obama"), + ] with raises(Exception) as got: - ws.has_permission(UserID('b'), WorkspaceAccessType.READ, upa=UPA('67/8/90')) - assert_exception_correct(got.value, ServerError('JSONRPCError', -32500, 'Thanks Obama')) + ws.has_permission(UserID("b"), WorkspaceAccessType.READ, upa=UPA("67/8/90")) + assert_exception_correct( + got.value, ServerError("JSONRPCError", -32500, "Thanks Obama") + ) def _has_permission(user, wsid, upa, perm, expected_wsid, public=False): wsc = create_autospec(Workspace, spec_set=True, instance=True) ws = WS(wsc) - wsc.administer.assert_called_once_with({'command': 'listModRequests'}) - retperms = {'perms': [{'a': 'w', 'b': 'r', 'c': 'a'}]} + wsc.administer.assert_called_once_with({"command": "listModRequests"}) + retperms = {"perms": [{"a": "w", "b": "r", "c": "a"}]} if public: - retperms['perms'][0]['*'] = 'r' + retperms["perms"][0]["*"] = "r" if wsid: wsc.administer.return_value = retperms else: - wsc.administer.side_effect = [retperms, {'infos': [['objinfo goes here']]}] + wsc.administer.side_effect = [retperms, {"infos": [["objinfo goes here"]]}] ws.has_permission(user, perm, wsid, upa) - getperms = {'command': 'getPermissionsMass', 'params': {'workspaces': [{'id': expected_wsid}]}} + getperms = { + "command": "getPermissionsMass", + "params": {"workspaces": [{"id": expected_wsid}]}, + } if wsid: wsc.administer.assert_called_with(getperms) else: wsc.administer.assert_any_call(getperms) - wsc.administer.assert_called_with({'command': 'getObjectInfo', - 'params': {'objects': [{'ref': str(upa)}], - 'ignoreErrors': 1}}) + wsc.administer.assert_called_with( + { + "command": "getObjectInfo", + "params": {"objects": [{"ref": str(upa)}], "ignoreErrors": 1}, + } + ) assert wsc.administer.call_count == 2 if wsid else 3 @@ -323,12 +395,12 @@ def _has_permission_fail_ws_exception(ws_exception, expected): ws = WS(wsc) - wsc.administer.assert_called_once_with({'command': 'listModRequests'}) + wsc.administer.assert_called_once_with({"command": "listModRequests"}) wsc.administer.side_effect = ws_exception with raises(Exception) as got: - ws.has_permission('foo', WorkspaceAccessType.READ, 22) + ws.has_permission("foo", WorkspaceAccessType.READ, 22) assert_exception_correct(got.value, expected) @@ -344,15 +416,19 @@ def _get_user_workspaces(workspaces, pub, expected): ws = WS(wsc) - wsc.administer.assert_called_once_with({'command': 'listModRequests'}) + wsc.administer.assert_called_once_with({"command": "listModRequests"}) - wsc.administer.return_value = {'workspaces': workspaces, 'pub': pub} + wsc.administer.return_value = {"workspaces": workspaces, "pub": pub} - assert ws.get_user_workspaces(UserID('usera')) == expected + assert ws.get_user_workspaces(UserID("usera")) == expected - wsc.administer.assert_called_with({'command': 'listWorkspaceIDs', - 'user': 'usera', - 'params': {'perm': 'r', 'excludeGlobal': 0}}) + wsc.administer.assert_called_with( + { + "command": "listWorkspaceIDs", + "user": "usera", + "params": {"perm": "r", "excludeGlobal": 0}, + } + ) assert wsc.administer.call_count == 2 @@ -362,26 +438,26 @@ def test_get_user_workspaces_anonymous(): ws = WS(wsc) - wsc.administer.assert_called_once_with({'command': 'listModRequests'}) + wsc.administer.assert_called_once_with({"command": "listModRequests"}) - wsc.list_workspace_ids.return_value = {'workspaces': [], 'pub': [6, 7]} + wsc.list_workspace_ids.return_value = {"workspaces": [], "pub": [6, 7]} assert ws.get_user_workspaces(None) == [6, 7] - wsc.list_workspace_ids.assert_called_once_with({'onlyGlobal': 1}) + wsc.list_workspace_ids.assert_called_once_with({"onlyGlobal": 1}) def test_get_user_workspaces_fail_invalid_user(): _get_user_workspaces_fail_ws_exception( - ServerError('JSONRPCError', -32500, 'User foo is not a valid user'), - NoSuchUserError('User foo is not a valid user') + ServerError("JSONRPCError", -32500, "User foo is not a valid user"), + NoSuchUserError("User foo is not a valid user"), ) def test_get_user_workspaces_fail_server_error(): _get_user_workspaces_fail_ws_exception( - ServerError('JSONRPCError', -32500, 'aw crapadoodles'), - ServerError('JSONRPCError', -32500, 'aw crapadoodles') + ServerError("JSONRPCError", -32500, "aw crapadoodles"), + ServerError("JSONRPCError", -32500, "aw crapadoodles"), ) @@ -390,19 +466,19 @@ def _get_user_workspaces_fail_ws_exception(ws_exception, expected): ws = WS(wsc) - wsc.administer.assert_called_once_with({'command': 'listModRequests'}) + wsc.administer.assert_called_once_with({"command": "listModRequests"}) wsc.administer.side_effect = ws_exception with raises(Exception) as got: - ws.get_user_workspaces(UserID('foo')) + ws.get_user_workspaces(UserID("foo")) assert_exception_correct(got.value, expected) def test_get_user_workspaces_anonymous_fail_server_error(): _get_user_workspaces_anonymous_fail_ws_exception( - ServerError('JSONRPCError', -32500, 'aw crapadoodles'), - ServerError('JSONRPCError', -32500, 'aw crapadoodles') + ServerError("JSONRPCError", -32500, "aw crapadoodles"), + ServerError("JSONRPCError", -32500, "aw crapadoodles"), ) @@ -411,7 +487,7 @@ def _get_user_workspaces_anonymous_fail_ws_exception(ws_exception, expected): ws = WS(wsc) - wsc.administer.assert_called_once_with({'command': 'listModRequests'}) + wsc.administer.assert_called_once_with({"command": "listModRequests"}) wsc.list_workspace_ids.side_effect = ws_exception diff --git a/test/kafka_controller.py b/test/kafka_controller.py index 1b767e66..7d87e291 100644 --- a/test/kafka_controller.py +++ b/test/kafka_controller.py @@ -25,12 +25,12 @@ class KafkaController: producer - a kafka-python producer pointed at the server. """ - _ZOO_EXE = 'zookeeper-server-start.sh' - _KAFKA_EXE = 'kafka-server-start.sh' - _KAFKA_TOPIC_EXE = 'kafka-topics.sh' + _ZOO_EXE = "zookeeper-server-start.sh" + _KAFKA_EXE = "kafka-server-start.sh" + _KAFKA_TOPIC_EXE = "kafka-topics.sh" def __init__(self, kafka_bin_dir: Path, root_temp_dir: Path) -> None: - ''' + """ Create and start a new Kafka server. An unused port will be selected for the server and for zookeeper. @@ -38,7 +38,7 @@ def __init__(self, kafka_bin_dir: Path, root_temp_dir: Path) -> None: shell scripts. :param root_temp_dir: A temporary directory in which to store Kafka data and log files. The files will be stored inside a child directory that is unique per invocation. - ''' + """ self._bin_dir = Path(os.path.expanduser(kafka_bin_dir)) zookeeperexe = self._bin_dir.joinpath(self._ZOO_EXE) kafkaexe = self._bin_dir.joinpath(self._KAFKA_EXE) @@ -47,99 +47,112 @@ def __init__(self, kafka_bin_dir: Path, root_temp_dir: Path) -> None: self._check_exe(kafkaexe) self._check_exe(topicsexe) if not zookeeperexe or not os.access(zookeeperexe, os.X_OK): - raise TestException('zookeeper executable path {} does not exist or is not executable.' - .format(zookeeperexe)) + raise TestException( + "zookeeper executable path {} does not exist or is not executable.".format( + zookeeperexe + ) + ) if not kafkaexe or not os.access(kafkaexe, os.X_OK): - raise TestException('kafka executable path {} does not exist or is not executable.' - .format(kafkaexe)) + raise TestException( + "kafka executable path {} does not exist or is not executable.".format( + kafkaexe + ) + ) if not root_temp_dir: - raise ValueError('root_temp_dir is None') + raise ValueError("root_temp_dir is None") # make temp dirs root_temp_dir = root_temp_dir.absolute() - self.temp_dir = Path(tempfile.mkdtemp(prefix='KafkaController-', dir=str(root_temp_dir))) - zoo_dir = self.temp_dir.joinpath('zookeeper') + self.temp_dir = Path( + tempfile.mkdtemp(prefix="KafkaController-", dir=str(root_temp_dir)) + ) + zoo_dir = self.temp_dir.joinpath("zookeeper") os.makedirs(zoo_dir, exist_ok=True) - kafka_dir = self.temp_dir.joinpath('kafka') + kafka_dir = self.temp_dir.joinpath("kafka") os.makedirs(kafka_dir, exist_ok=True) self._zooport = find_free_port() - self._zoo_proc, self._zoo_out = self._start_zoo(zookeeperexe, self._zooport, zoo_dir) + self._zoo_proc, self._zoo_out = self._start_zoo( + zookeeperexe, self._zooport, zoo_dir + ) self.port = find_free_port() self._kafka_proc, self._kafka_out = self._start_kafka( - kafkaexe, self.port, self._zooport, kafka_dir) - self.producer = KafkaProducer(bootstrap_servers=[f'localhost:{self.port}']) + kafkaexe, self.port, self._zooport, kafka_dir + ) + self.producer = KafkaProducer(bootstrap_servers=[f"localhost:{self.port}"]) def _check_exe(self, exe): if not exe or not os.access(exe, os.X_OK): - raise TestException(f'executable path {exe} does not exist or is not executable.') + raise TestException( + f"executable path {exe} does not exist or is not executable." + ) def _start_zoo(self, exe, port, zoo_dir): - datadir = zoo_dir.joinpath('data') + datadir = zoo_dir.joinpath("data") os.makedirs(datadir, exist_ok=True) - configfile = zoo_dir.joinpath('zoo.cfg') - with open(configfile, 'w') as c: + configfile = zoo_dir.joinpath("zoo.cfg") + with open(configfile, "w") as c: # this is the default config for zookeeper provided in the kafka tarball - c.write(f'dataDir={datadir}\n') - c.write(f'clientPort={port}\n') - c.write('maxClientCnxns=0\n') - c.write('admin.enableServer=false') + c.write(f"dataDir={datadir}\n") + c.write(f"clientPort={port}\n") + c.write("maxClientCnxns=0\n") + c.write("admin.enableServer=false") command = [str(exe), str(configfile)] - outfile = open(zoo_dir.joinpath('zoo.out'), 'w') + outfile = open(zoo_dir.joinpath("zoo.out"), "w") proc = subprocess.Popen(command, stdout=outfile, stderr=subprocess.STDOUT) time.sleep(1) # wait for server to start up return proc, outfile def _start_kafka(self, exe, port, zooport, kafka_dir): - logdir = kafka_dir.joinpath('logs') - configfile = kafka_dir.joinpath('kafka.cfg') - with open(configfile, 'w') as c: + logdir = kafka_dir.joinpath("logs") + configfile = kafka_dir.joinpath("kafka.cfg") + with open(configfile, "w") as c: # this is the default config for kafka provided in the kafka tarball - c.write('broker.id=0\n') - c.write(f'listeners=PLAINTEXT://localhost:{port}\n') - c.write('num.network.threads=3\n') - c.write('num.io.threads=8\n') - c.write('socket.send.buffer.bytes=102400\n') - c.write('socket.receive.buffer.bytes=102400\n') - c.write('socket.request.max.bytes=104857600\n') - c.write(f'log.dirs={logdir}\n') # this is the data directory - c.write('num.partitions=1\n') - c.write('num.recovery.threads.per.data.dir=1\n') - c.write('offsets.topic.replication.factor=1\n') - c.write('transaction.state.log.replication.factor=1\n') - c.write('transaction.state.log.min.isr=1\n') - c.write('log.retention.hours=168\n') - c.write('log.segment.bytes=1073741824\n') - c.write('log.retention.check.interval.ms=300000\n') - c.write(f'zookeeper.connect=localhost:{zooport}\n') - c.write('zookeeper.connection.timeout.ms=1000\n') - c.write('group.initial.rebalance.delay.ms=0\n') + c.write("broker.id=0\n") + c.write(f"listeners=PLAINTEXT://localhost:{port}\n") + c.write("num.network.threads=3\n") + c.write("num.io.threads=8\n") + c.write("socket.send.buffer.bytes=102400\n") + c.write("socket.receive.buffer.bytes=102400\n") + c.write("socket.request.max.bytes=104857600\n") + c.write(f"log.dirs={logdir}\n") # this is the data directory + c.write("num.partitions=1\n") + c.write("num.recovery.threads.per.data.dir=1\n") + c.write("offsets.topic.replication.factor=1\n") + c.write("transaction.state.log.replication.factor=1\n") + c.write("transaction.state.log.min.isr=1\n") + c.write("log.retention.hours=168\n") + c.write("log.segment.bytes=1073741824\n") + c.write("log.retention.check.interval.ms=300000\n") + c.write(f"zookeeper.connect=localhost:{zooport}\n") + c.write("zookeeper.connection.timeout.ms=1000\n") + c.write("group.initial.rebalance.delay.ms=0\n") # this is additional config - c.write('delete.topic.enable=true\n') + c.write("delete.topic.enable=true\n") command = [str(exe), str(configfile)] - outfile = open(kafka_dir.joinpath('kafka.out'), 'w') + outfile = open(kafka_dir.joinpath("kafka.out"), "w") proc = subprocess.Popen(command, stdout=outfile, stderr=subprocess.STDOUT) for count in range(40): err = None time.sleep(1) # wait for server to start try: - KafkaProducer(bootstrap_servers=[f'localhost:{port}']) + KafkaProducer(bootstrap_servers=[f"localhost:{port}"]) break except NoBrokersAvailable as e: - err = TestException('No Kafka brokers available') + err = TestException("No Kafka brokers available") err.__cause__ = e if err: self._print_kafka_logs() - self._print_logs(outfile, 'Kafka', True) + self._print_logs(outfile, "Kafka", True) raise err self.startup_count = count + 1 return proc, outfile @@ -154,7 +167,13 @@ def clear_topic(self, topic: str): """ exe = self._bin_dir.joinpath(self._KAFKA_TOPIC_EXE) command = [ - str(exe), '--zookeeper', f'localhost:{self._zooport}', '--delete', '--topic', topic] + str(exe), + "--zookeeper", + f"localhost:{self._zooport}", + "--delete", + "--topic", + topic, + ] # just let any exceptions raise subprocess.run(command, capture_output=True, check=True) @@ -164,11 +183,15 @@ def clear_all_topics(self): Note this takes about 2 seconds per topic. """ - cons = KafkaConsumer(bootstrap_servers=[f'localhost:{self.port}'], group_id='foo') + cons = KafkaConsumer( + bootstrap_servers=[f"localhost:{self.port}"], group_id="foo" + ) for topic in cons.topics(): self.clear_topic(topic) - def destroy(self, delete_temp_files: bool = True, dump_logs_to_stdout: bool = False) -> None: + def destroy( + self, delete_temp_files: bool = True, dump_logs_to_stdout: bool = False + ) -> None: """ Shut down the Kafka server. @@ -189,39 +212,40 @@ def destroy(self, delete_temp_files: bool = True, dump_logs_to_stdout: bool = Fa # closes logfile def _print_kafka_logs(self, dump_logs_to_stdout=True): - self._print_logs(self._zoo_out, 'Zookeeper', dump_logs_to_stdout) - self._print_logs(self._kafka_out, 'Kafka', dump_logs_to_stdout) + self._print_logs(self._zoo_out, "Zookeeper", dump_logs_to_stdout) + self._print_logs(self._kafka_out, "Kafka", dump_logs_to_stdout) def _print_logs(self, file_, name, dump_logs_to_stdout): if file_: file_.close() if dump_logs_to_stdout: - print(f'\n{name} logs:') + print(f"\n{name} logs:") with open(file_.name) as f: for line in f: print(line) def main(): - bindir = Path('~/kafka/kafka_2.12-2.5.0/bin/') + bindir = Path("~/kafka/kafka_2.12-2.5.0/bin/") - kc = KafkaController(bindir, Path('./test_temp_can_delete')) - print(f'port: {kc.port}') - print(f'temp_dir: {kc.temp_dir}') - kc.producer.send('mytopic', 'some message'.encode('utf-8')) + kc = KafkaController(bindir, Path("./test_temp_can_delete")) + print(f"port: {kc.port}") + print(f"temp_dir: {kc.temp_dir}") + kc.producer.send("mytopic", "some message".encode("utf-8")) # kc.clear_topic('mytopic') # comment out to test consumer getting message # kc.clear_all_topics() # comment out to test consumer getting message cons = KafkaConsumer( - 'mytopic', - bootstrap_servers=[f'localhost:{kc.port}'], - auto_offset_reset='earliest', - group_id='foo') + "mytopic", + bootstrap_servers=[f"localhost:{kc.port}"], + auto_offset_reset="earliest", + group_id="foo", + ) print(cons.poll(timeout_ms=1000)) - input('press enter to shut down') + input("press enter to shut down") kc.destroy(True) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/test/mongo_controller.py b/test/mongo_controller.py index 13d9a64b..6f1d0bb5 100644 --- a/test/mongo_controller.py +++ b/test/mongo_controller.py @@ -34,8 +34,10 @@ class MongoController: indexes, false otherwise. """ - def __init__(self, mongoexe: Path, root_temp_dir: Path, use_wired_tiger: bool = False) -> None: - ''' + def __init__( + self, mongoexe: Path, root_temp_dir: Path, use_wired_tiger: bool = False + ) -> None: + """ Create and start a new MongoDB database. An unused port will be selected for the server. :param mongoexe: The path to the MongoDB server executable (e.g. mongod) to run. @@ -43,42 +45,57 @@ def __init__(self, mongoexe: Path, root_temp_dir: Path, use_wired_tiger: bool = The files will be stored inside a child directory that is unique per invocation. :param use_wired_tiger: For MongoDB versions > 3.0, specify that the Wired Tiger storage engine should be used. Setting this to true for other versions will cause an error. - ''' + """ if not mongoexe or not os.access(mongoexe, os.X_OK): - raise TestException('mongod executable path {} does not exist or is not executable.' - .format(mongoexe)) + raise TestException( + "mongod executable path {} does not exist or is not executable.".format( + mongoexe + ) + ) if not root_temp_dir: - raise ValueError('root_temp_dir is None') + raise ValueError("root_temp_dir is None") # make temp dirs root_temp_dir = root_temp_dir.absolute() os.makedirs(root_temp_dir, exist_ok=True) - self.temp_dir = Path(tempfile.mkdtemp(prefix='MongoController-', dir=str(root_temp_dir))) - data_dir = self.temp_dir.joinpath('data') + self.temp_dir = Path( + tempfile.mkdtemp(prefix="MongoController-", dir=str(root_temp_dir)) + ) + data_dir = self.temp_dir.joinpath("data") os.makedirs(data_dir) self.port = test_utils.find_free_port() - command = [str(mongoexe), '--port', str(self.port), '--dbpath', str(data_dir), - '--nojournal'] + command = [ + str(mongoexe), + "--port", + str(self.port), + "--dbpath", + str(data_dir), + "--nojournal", + ] if use_wired_tiger: - command.extend(['--storageEngine', 'wiredTiger']) + command.extend(["--storageEngine", "wiredTiger"]) - self._outfile = open(self.temp_dir.joinpath('mongo.log'), 'w') + self._outfile = open(self.temp_dir.joinpath("mongo.log"), "w") - self._proc = subprocess.Popen(command, stdout=self._outfile, stderr=subprocess.STDOUT) + self._proc = subprocess.Popen( + command, stdout=self._outfile, stderr=subprocess.STDOUT + ) time.sleep(1) # wait for server to start up - self.client = MongoClient('localhost', self.port) + self.client = MongoClient("localhost", self.port) # check that the server is up. See # https://api.mongodb.com/python/3.7.0/api/pymongo/mongo_client.html # #pymongo.mongo_client.MongoClient - self.client.admin.command('ismaster') + self.client.admin.command("ismaster") # get some info about the db - self.db_version = self.client.server_info()['version'] + self.db_version = self.client.server_info()["version"] s = semver.VersionInfo.parse - self.index_version = 2 if (s(self.db_version) >= s('3.4.0')) else 1 - self.includes_system_indexes = (s(self.db_version) < s('3.2.0') and not use_wired_tiger) + self.index_version = 2 if (s(self.db_version) >= s("3.4.0")) else 1 + self.includes_system_indexes = ( + s(self.db_version) < s("3.2.0") and not use_wired_tiger + ) def destroy(self, delete_temp_files: bool) -> None: """ @@ -97,19 +114,19 @@ def destroy(self, delete_temp_files: bool) -> None: shutil.rmtree(self.temp_dir) def clear_database(self, db_name, drop_indexes=False): - ''' + """ Remove all data from a database. :param db_name: the name of the db to clear. :param drop_indexes: drop all indexes if true, retain indexes (which will be empty) if false. - ''' + """ if drop_indexes: self.client.drop_database(db_name) else: db = self.client[db_name] for name in db.list_collection_names(): - if not name.startswith('system.'): + if not name.startswith("system."): # don't drop collection since that drops indexes db.get_collection(name).delete_many({}) @@ -119,16 +136,16 @@ def main(): root_temp_dir = test_utils.get_temp_dir() mc = MongoController(mongoexe, root_temp_dir, False) - print('port: ' + str(mc.port)) - print('temp_dir: ' + str(mc.temp_dir)) - print('db_version: ' + mc.db_version) - print('index_version: ' + str(mc.index_version)) - print('includes_system_indexes: ' + str(mc.includes_system_indexes)) - mc.client['foo']['bar'].insert_one({'foo': 'bar'}) - mc.clear_database('foo') - input('press enter to shut down') + print("port: " + str(mc.port)) + print("temp_dir: " + str(mc.temp_dir)) + print("db_version: " + mc.db_version) + print("index_version: " + str(mc.index_version)) + print("includes_system_indexes: " + str(mc.includes_system_indexes)) + mc.client["foo"]["bar"].insert_one({"foo": "bar"}) + mc.clear_database("foo") + input("press enter to shut down") mc.destroy(True) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/test/workspace_controller.py b/test/workspace_controller.py index 84a1b748..b919a66e 100644 --- a/test/workspace_controller.py +++ b/test/workspace_controller.py @@ -1,8 +1,8 @@ -''' +""" Q&D Utility to run a Workspace server for the purposes of testing. Initializes a GridFS backend and does not support handles. -''' +""" import os as _os import shutil as _shutil @@ -21,8 +21,8 @@ from core.test_utils import TestException as _TestException from mongo_controller import MongoController as _MongoController -_WS_CLASS = 'us.kbase.workspace.WorkspaceServer' -_JARS_FILE = _Path(__file__).resolve().parent.joinpath('wsjars') +_WS_CLASS = "us.kbase.workspace.WorkspaceServer" +_JARS_FILE = _Path(__file__).resolve().parent.joinpath("wsjars") class WorkspaceController: @@ -41,14 +41,15 @@ class WorkspaceController: # TODO This code is similar to the auth controller code, DRY it up? def __init__( - self, - jars_dir: _Path, - mongo_controller: _MongoController, - mongo_db: str, - mongo_type_db: str, - auth_url: str, - root_temp_dir: _Path): - ''' + self, + jars_dir: _Path, + mongo_controller: _MongoController, + mongo_db: str, + mongo_type_db: str, + auth_url: str, + root_temp_dir: _Path, + ): + """ Create and start a new Workspace service. An unused port will be selected for the server. :param jars_dir: The path to the lib/jars dir of the KBase Jars repo @@ -59,20 +60,21 @@ def __init__( :param auth_url: The root url of an instance of the KBase auth service. :param root_temp_dir: A temporary directory in which to store Auth data and log files. The files will be stored inside a child directory that is unique per invocation. - ''' + """ if not jars_dir or not _os.access(jars_dir, _os.X_OK): - raise _TestException('jars_dir {} does not exist or is not executable.' - .format(jars_dir)) + raise _TestException( + "jars_dir {} does not exist or is not executable.".format(jars_dir) + ) if not mongo_controller: - raise _TestException('mongo_controller must be provided') + raise _TestException("mongo_controller must be provided") if not mongo_db: - raise _TestException('mongo_db must be provided') + raise _TestException("mongo_db must be provided") if not mongo_type_db: - raise _TestException('mongo_type_db must be provided') + raise _TestException("mongo_type_db must be provided") if not auth_url: - raise _TestException('auth_url must be provided') + raise _TestException("auth_url must be provided") if not root_temp_dir: - raise _TestException('root_temp_dir is None') + raise _TestException("root_temp_dir is None") self._mongo = mongo_controller self._db = mongo_db @@ -83,31 +85,34 @@ def __init__( root_temp_dir = root_temp_dir.absolute() _os.makedirs(root_temp_dir, exist_ok=True) self.temp_dir = _Path( - _tempfile.mkdtemp(prefix='WorkspaceController-', dir=str(root_temp_dir))) - ws_temp_dir = self.temp_dir.joinpath('temp_files') + _tempfile.mkdtemp(prefix="WorkspaceController-", dir=str(root_temp_dir)) + ) + ws_temp_dir = self.temp_dir.joinpath("temp_files") _os.makedirs(ws_temp_dir) configfile = self._create_deploy_cfg( self.temp_dir, ws_temp_dir, - f'localhost:{self._mongo.port}', + f"localhost:{self._mongo.port}", mongo_db, mongo_type_db, - auth_url) + auth_url, + ) newenv = _os.environ.copy() - newenv['KB_DEPLOYMENT_CONFIG'] = configfile + newenv["KB_DEPLOYMENT_CONFIG"] = configfile self.port = _test_utils.find_free_port() - command = ['java', '-classpath', class_path, _WS_CLASS, str(self.port)] + command = ["java", "-classpath", class_path, _WS_CLASS, str(self.port)] - self._wslog = self.temp_dir / 'ws.log' - self._outfile = open(self._wslog, 'w') + self._wslog = self.temp_dir / "ws.log" + self._outfile = open(self._wslog, "w") self._proc = _subprocess.Popen( - command, stdout=self._outfile, stderr=_subprocess.STDOUT, env=newenv) + command, stdout=self._outfile, stderr=_subprocess.STDOUT, env=newenv + ) - ws = _Workspace(f'http://localhost:{self.port}') + ws = _Workspace(f"http://localhost:{self.port}") for count in range(40): err = None _time.sleep(1) # wait for server to start @@ -118,7 +123,7 @@ def __init__( err = _TestException(se.args[0]) err.__cause__ = se if err: - print('Error starting workspace service. Dumping logs and throwing error') + print("Error starting workspace service. Dumping logs and throwing error") self._print_ws_logs() raise err self.startup_count = count + 1 @@ -127,48 +132,43 @@ def _get_class_path(self, jars_dir: _Path): cp = [] with open(_JARS_FILE) as jf: for l in jf: - if l.strip() and not l.startswith('#'): + if l.strip() and not l.startswith("#"): p = jars_dir.joinpath(l.strip()) if not p.is_file(): - raise _TestException(f'Required jar does not exist: {p}') + raise _TestException(f"Required jar does not exist: {p}") cp.append(str(p)) - return ':'.join(cp) + return ":".join(cp) def _create_deploy_cfg( - self, - temp_dir, - ws_temp_dir, - mongo_host, - mongo_db, - mongo_type_db, - auth_url): + self, temp_dir, ws_temp_dir, mongo_host, mongo_db, mongo_type_db, auth_url + ): cp = _ConfigParser() - cp['Workspace'] = { - 'mongodb-host': mongo_host, - 'mongodb-database': mongo_db, - 'mongodb-type-database': mongo_type_db, - 'backend-type': 'GridFS', - 'auth-service-url': auth_url + '/api/legacy/KBase', - 'auth-service-url-allow-insecure': 'true', - 'auth2-service-url': auth_url + '/', # TODO WS should not be necessary - 'temp-dir': str(ws_temp_dir), - 'ignore-handle-service': 'true', - 'auth2-ws-admin-read-only-roles': 'WS_READ_ADMIN', - 'auth2-ws-admin-full-roles': 'WS_FULL_ADMIN' + cp["Workspace"] = { + "mongodb-host": mongo_host, + "mongodb-database": mongo_db, + "mongodb-type-database": mongo_type_db, + "backend-type": "GridFS", + "auth-service-url": auth_url + "/api/legacy/KBase", + "auth-service-url-allow-insecure": "true", + "auth2-service-url": auth_url + "/", # TODO WS should not be necessary + "temp-dir": str(ws_temp_dir), + "ignore-handle-service": "true", + "auth2-ws-admin-read-only-roles": "WS_READ_ADMIN", + "auth2-ws-admin-full-roles": "WS_FULL_ADMIN", } - f = temp_dir / 'test.cfg' - with open(f, 'w') as inifile: + f = temp_dir / "test.cfg" + with open(f, "w") as inifile: cp.write(inifile) return f def clear_db(self): - ''' + """ Remove all data, but not indexes, from the database. Do not remove any installed types. - ''' + """ self._mongo.clear_database(self._db) def destroy(self, delete_temp_files: bool = True, dump_logs_to_stdout: bool = True): - ''' + """ Shut down the server and optionally delete any files generated. :param delete_temp_files: if true, delete all the temporary files generated as part of @@ -176,7 +176,7 @@ def destroy(self, delete_temp_files: bool = True, dump_logs_to_stdout: bool = Tr :param dump_logs_to_stdout: Write the contents of the workspace log file to stdout. This is useful in the context of 3rd party CI services, where the log file is not necessarily accessible. - ''' + """ if self._proc: self._proc.terminate() self._print_ws_logs(dump_logs_to_stdout=dump_logs_to_stdout)