feat: sync proposals, experiments and samples to SciCat from UOS

.env.tests

@@ -15,6 +15,9 @@ SCICAT_USERNAME=user
 SCICAT_PASSWORD=pass
 SCICAT_PROPOSAL_TRIGGERING_STATUSES="SCHEDULING, ALLOCATED"
 
+USER_OFFICE_GRAPHQL_URL=http://localhost:8080
+USER_OFFICE_JWT=some-token
+
 SYNAPSE_SERVER_URL=https://server-scichat
 SYNAPSE_SERVER_NAME=serverName
 SYNAPSE_SERVICE_USER=user

src/index.spec.ts

@@ -6,6 +6,11 @@ jest.mock('express', () => {
 
   return jest.fn(() => express);
 });
+jest.mock('./queue/consumers/scicat/scicatProposal/utils/scicatApi', () => ({
+  scicatApi: {
+    serviceUsername: 'testuser',
+  },
+}));
 
 jest.mock('@user-office-software/duo-logger');
 jest.mock('./middlewares/metrics/metrics', () => jest.fn());

src/index.ts

@@ -34,6 +34,9 @@ async function bootstrap() {
   const enableScicatProposalUpsert = str2Bool(
     process.env.ENABLE_SCICAT_PROPOSAL_UPSERT as string
   );
+  const enableScicatExperimentUpsert = str2Bool(
+    process.env.ENABLE_SCICAT_EXPERIMENT_UPSERT as string
+  );
   const enableScichatRoomCreation = str2Bool(
     process.env.ENABLE_SCICHAT_ROOM_CREATION as string
   );
@@ -55,6 +58,7 @@ async function bootstrap() {
 
   logger.logInfo('Services configuration', {
     SciCat_Proposal_Upsert: enableScicatProposalUpsert,
+    SciCat_Experiment_Upsert: enableScicatExperimentUpsert,
     Scichat_Room_Creation: enableScichatRoomCreation,
     Proposal_Folders_Creation: enableProposalFoldersCreation,
     Nicos_to_Scichat_Messages: enableNicosToScichatMessages,
@@ -75,6 +79,7 @@ async function bootstrap() {
 
   if (
     enableScicatProposalUpsert ||
+    enableScicatExperimentUpsert ||
     enableScichatRoomCreation ||
     enableProposalFoldersCreation ||
     enableMoodleFoldersCreation ||

src/models/Event.ts

@@ -4,4 +4,6 @@ export enum Event {
   PROPOSAL_UPDATED = 'PROPOSAL_UPDATED',
   VISIT_CREATED = 'VISIT_CREATED',
   VISIT_DELETED = 'VISIT_DELETED',
+  EXPERIMENT_CREATED = 'EXPERIMENT_CREATED',
+  EXPERIMENT_UPDATED = 'EXPERIMENT_UPDATED',
 }

src/models/ProposalMessage.ts

@@ -1,11 +1,16 @@
-import { ProposalUser } from './../queue/consumers/scicat/scicatProposal/dto';
+import { ProposalUser } from '../queue/consumers/scicat/scicatProposal/dto';
 
 export type Instrument = {
   id: number;
   shortCode: string;
   allocatedTime: number;
 };
 
+export type Sample = {
+  id: number;
+  title: string;
+};
+
 export interface InstrumentDto {
   _id: string;
   id: string;
@@ -34,16 +39,27 @@ export enum ProposalStatusDefaultShortCodes {
 }
 
 export type ProposalMessageData = {
-  proposalPk: number;
-  shortCode: string;
-  title: string;
   abstract: string;
   callId: number;
-  newStatus?: ProposalStatusDefaultShortCodes;
-  submitted: boolean;
+  instruments?: Instrument[];
   members: ProposalUser[];
-  dataAccessUsers?: ProposalUser[];
-  visitors?: ProposalUser[];
+  dataAccessUsers: ProposalUser[];
+  visitors: ProposalUser[];
+  newStatus?: string;
+  proposalPk: number;
   proposer?: ProposalUser;
-  instruments?: Instrument[];
+  shortCode: string;
+  title: string;
+  submitted: boolean;
+  samples?: Sample[];
+};
+
+export type ExperimentMessageData = {
+  experimentId: string;
+  experimentPk: number;
+  startsAt: Date;
+  endsAt: Date;
+  status: string;
+  proposal?: ProposalMessageData;
+  samples?: Sample[];
 };

src/queue/consumers/scicat/scicatProposal/consumerCallbacks/upsertProposalInScicat.ts

@@ -1,256 +1,52 @@
 import { logger } from '@user-office-software/duo-logger';
 
+import { upsertSamplesInScicat } from './upsertSampleInScicat';
 import {
-  Instrument,
-  InstrumentDto,
+  ExperimentMessageData,
+  ProposalMessageData,
 } from '../../../../../models/ProposalMessage';
-import { ValidProposalMessageData } from '../../../utils/validateProposalMessage';
-import { CreateProposalDto, UpdateProposalDto } from '../dto';
-
-const sciCatBaseUrl = process.env.SCICAT_BASE_URL;
-const sciCatLoginEndpoint = process.env.SCICAT_LOGIN_ENDPOINT || '/Users/login';
-const sciCatUsername = process.env.SCICAT_USERNAME;
-const sciCatPassword = process.env.SCICAT_PASSWORD;
-
-async function request<TResponse>(
-  url: string,
-  config: RequestInit
-): Promise<TResponse> {
-  // NOTE: Node v18 comes with fetch API by default
-  const response = await fetch(url, config);
-
-  if (!response.ok) {
-    return response.text().then((errorDetail) => {
-      throw new Error(errorDetail);
-    });
-  }
-
-  return (await response.json()) as TResponse;
-}
-
-const getSciCatAccessToken = async () => {
-  const loginCredentials = {
-    username: sciCatUsername,
-    password: sciCatPassword,
-  };
-
-  // NOTE: We login every time when there is new message to get the access_token
-  const { access_token: sciCatAccessToken } = await request<{
-    access_token: string;
-  }>(`${sciCatBaseUrl}${sciCatLoginEndpoint}`, {
-    method: 'POST',
-    body: JSON.stringify(loginCredentials),
-    headers: {
-      'Content-Type': 'application/json',
-    },
-  });
-
-  if (!sciCatAccessToken) {
-    throw new Error('No access token found');
-  }
-
-  return sciCatAccessToken;
-};
-
-const getCreateProposalDto = (proposalMessage: ValidProposalMessageData) => {
-  const createProposalDto: CreateProposalDto = {
-    proposalId: proposalMessage.shortCode,
-    title: proposalMessage.title,
-    pi_email: proposalMessage.proposer.email,
-    pi_firstname: proposalMessage.proposer.firstName,
-    pi_lastname: proposalMessage.proposer.lastName,
-    email: proposalMessage.proposer.email,
-    firstname: proposalMessage.proposer.firstName,
-    lastname: proposalMessage.proposer.lastName,
-    abstract: proposalMessage.abstract,
-    ownerGroup: proposalMessage.shortCode,
-    instrumentIds: [],
-    accessGroups: [],
-    startTime: new Date(),
-    endTime: new Date(),
-    MeasurementPeriodList: [],
-    metadata: {},
-  };
-
-  return createProposalDto;
-};
-
-const getUpdateProposalDto = (proposalMessage: ValidProposalMessageData) => {
-  const updateProposalDto: UpdateProposalDto = {
-    title: proposalMessage.title,
-    pi_email: proposalMessage.proposer.email,
-    pi_firstname: proposalMessage.proposer.firstName,
-    pi_lastname: proposalMessage.proposer.lastName,
-    email: proposalMessage.proposer.email,
-    firstname: proposalMessage.proposer.firstName,
-    lastname: proposalMessage.proposer.lastName,
-    abstract: proposalMessage.abstract,
-    ownerGroup: proposalMessage.shortCode,
-    instrumentIds: [],
-    accessGroups: [],
-    startTime: new Date(),
-    endTime: new Date(),
-    MeasurementPeriodList: [],
-    metadata: {},
-  };
-
-  return updateProposalDto;
-};
-
-const createProposal = async (
-  proposalMessage: ValidProposalMessageData,
-  sciCatAccessToken: string
-) => {
-  const url = `${sciCatBaseUrl}/Proposals`;
-  const createProposalDto = getCreateProposalDto(proposalMessage);
-
-  logger.logInfo('POST', { url });
-  logger.logInfo('Proposal data', { proposalData: createProposalDto });
-
-  // RabbitMQ message only provides shortCodes (instrument names).
-  // To persist proposals with proper references, we resolve those shortCodes to
-  // actual Instrument IDs from SciCat and store the instrumentIds in the record.
-  createProposalDto.instrumentIds = await getInstrumentIds(
-    proposalMessage.instruments
-  );
-
-  const createProposalResponse = await request<string>(url, {
-    method: 'POST',
-    body: JSON.stringify(createProposalDto),
-    headers: {
-      'Content-Type': 'application/json',
-      Authorization: `Bearer ${sciCatAccessToken}`,
-    },
-  });
-
-  logger.logInfo('createProposalResponse', { createProposalResponse });
-
-  logger.logInfo('Proposal was created in scicat', {
-    proposalId: createProposalDto.proposalId,
-  });
-};
-
-const updateProposal = async (
-  proposalMessage: ValidProposalMessageData,
-  sciCatAccessToken: string
-) => {
-  const url = `${sciCatBaseUrl}/Proposals/${proposalMessage.shortCode}`;
-  const updateProposalDto = getUpdateProposalDto(proposalMessage);
-
-  // RabbitMQ message only provides shortCodes (instrument names).
-  // To persist proposals with proper references, we resolve those shortCodes to
-  // actual Instrument IDs from SciCat and store the instrumentIds in the record.
-  updateProposalDto.instrumentIds = await getInstrumentIds(
-    proposalMessage.instruments
-  );
-  const updateProposalResponse = await request(url, {
-    method: 'PATCH',
-    body: JSON.stringify(updateProposalDto),
-    headers: {
-      'Content-Type': 'application/json',
-      Authorization: `Bearer ${sciCatAccessToken}`,
-    },
-  });
-
-  logger.logInfo('Patch', { url });
-  logger.logInfo('Proposal data', { proposalData: updateProposalDto });
-  logger.logInfo('updateProposalResponse', { updateProposalResponse });
-
-  logger.logInfo('Proposal was updated in scicat', {
-    proposalId: proposalMessage.shortCode,
-  });
-};
+import {
+  fetchUoExperiment,
+  fetchUoProposal,
+} from '../../../../../services/userOfficeApi/uoApi';
+import { scicatApi } from '../utils/scicatApi';
 
-const checkProposalExists = async (
-  proposalId: string,
-  sciCatAccessToken: string
+export const upsertProposalInScicat = async (
+  proposalMessage: ProposalMessageData
 ) => {
-  const url = `${sciCatBaseUrl}/Proposals/${proposalId}`;
-  const response = await request<string>(url, {
-    headers: {
-      'Content-Type': 'application/json',
-      Authorization: `Bearer ${sciCatAccessToken}`,
-    },
-  }).catch((error) => {
-    try {
-      const parsedError = JSON.parse(error.message);
-      if (parsedError.statusCode === 404) {
-        return false;
-      }
-    } catch (reason) {
-      logger.logError('Error parsing error message', {
-        error,
-        reason,
-      });
-    }
-    throw error;
-  });
+  const proposal = await fetchUoProposal(proposalMessage.proposalPk);
+  const exists = await scicatApi.checkProposalExists(proposal.proposalId);
 
-  if (response) {
-    return true;
+  if (exists) {
+    logger.logInfo('Proposal already exists, updating...', {
+      proposalId: proposal.proposalId,
+    });
+    await scicatApi.updateProposal(proposal);
   } else {
-    return false;
-  }
-};
-
-const getInstrumentIds = async (instruments: Instrument[]) => {
-  const sciCatAccessToken = await getSciCatAccessToken();
-  const instrumentNames = instruments.map((inst) => inst.shortCode);
-
-  const instrumentIds = [];
-
-  for (const name of instrumentNames) {
-    const instrumentNameLowerCase = name.toLowerCase();
-
-    const filterString = JSON.stringify({
-      where: { name: { ilike: instrumentNameLowerCase } },
+    logger.logInfo('Proposal does not exist yet, creating...', {
+      proposalId: proposal.proposalId,
     });
-
-    const url = `${sciCatBaseUrl}/Instruments?filter=${encodeURIComponent(filterString)}`;
-
-    try {
-      const res = await request<InstrumentDto[]>(url, {
-        method: 'GET',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: `Bearer ${sciCatAccessToken}`,
-        },
-      });
-
-      instrumentIds.push(res[0].pid);
-    } catch (error) {
-      logger.logError(`Error fetching instrument ID from scicat for ${name}`, {
-        error,
-      });
-    }
+    await scicatApi.createProposal(proposal);
   }
-
-  return instrumentIds;
 };
 
-const upsertProposalInScicat = async (
-  proposalMessage: ValidProposalMessageData
+export const upsertExperimentInScicat = async (
+  experimentMessage: ExperimentMessageData
 ) => {
-  const sciCatAccessToken = await getSciCatAccessToken();
-
-  const proposalExists = await checkProposalExists(
-    proposalMessage.shortCode,
-    sciCatAccessToken
-  );
+  const experiment = await fetchUoExperiment(experimentMessage.experimentPk);
+  const exists = await scicatApi.checkProposalExists(experiment.experimentId);
 
-  if (proposalExists) {
-    logger.logInfo('Proposal already exists, updating...', {
-      proposalId: proposalMessage.shortCode,
+  if (exists) {
+    logger.logInfo('Experiment already exists, updating...', {
+      proposalId: experiment.experimentId,
     });
-
-    updateProposal(proposalMessage, sciCatAccessToken);
+    await scicatApi.updateExperiment(experiment);
   } else {
-    logger.logInfo('Proposal does not exist yet, creating...', {
-      proposalId: proposalMessage.shortCode,
+    logger.logInfo('Experiment does not exist yet, creating...', {
+      proposalId: experiment.experimentId,
     });
-
-    createProposal(proposalMessage, sciCatAccessToken);
+    await scicatApi.createExperiment(experiment);
   }
-};
 
-export { upsertProposalInScicat };
+  await upsertSamplesInScicat(experiment);
+};