Skip to content

Moved 2PC logic to TwoPhaseCommit plugin. Closes #1233 #1306

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brollb merged 14 commits into from
Nov 14, 2019
Merged

Moved 2PC logic to TwoPhaseCommit plugin. Closes #1233 #1306

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
ee4751a
Moved 2PC logic to TwoPhaseCommit plugin. Closes #1233
brollb Oct 30, 2019
f2b2916
Update CreatedNodes in node caches
brollb Oct 30, 2019
6919578
WIP Add tests for setPointer
brollb Oct 31, 2019
7d31aef
WIP delete nodes using CreatedNode object
brollb Oct 31, 2019
9d5deb7
Added loadChildren to 2PC
brollb Nov 6, 2019
e065333
WIP Updatedd ExecuteJob.Metadata.js
brollb Nov 6, 2019
cbb18a4
WIP added working edits, debug statements, comments
brollb Nov 12, 2019
6110b07
Merge branch 'master' into 1233-refactor-safesave
brollb Nov 14, 2019
c27cabb
WIP Wrap the core instead of adding methods to plugin
brollb Nov 14, 2019
99e191c
WIP updated plugins to use wrapped core
brollb Nov 14, 2019
e9c7263
WIP removed extra console logs, debug comments
brollb Nov 14, 2019
14ac5ed
WIP removed useless comments
brollb Nov 14, 2019
27092ed
WIP Update ExecutePipeline tests
brollb Nov 14, 2019
dfa08c9
WIP Update model editing to use (wrapped) core
brollb Nov 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions src/common/plugin/LocalExecutor.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ define([
.find(cntr => this.isMetaTypeOf(cntr, this.META.Outputs));

const dataNodes = await this.core.loadChildren(outputContainer);
const dataInfo = this.getAttribute(dataNodes[0], 'data');
const dataInfo = this.core.getAttribute(dataNodes[0], 'data');

// Pass the dataInfo to the next nodes
const outputs = (await this.getOutputs(node))
Expand Down Expand Up @@ -75,7 +75,7 @@ define([
});

const saveDir = containers.find(c =>
this.getAttribute(c, 'name').toLowerCase().includes('artifacts')
this.core.getAttribute(c, 'name').toLowerCase().includes('artifacts')
) || containers[0];

return saveDir || this.rootNode; // default to rootNode
Expand All @@ -86,8 +86,8 @@ define([
const artifacts = await this.core.loadChildren(artifactsDir);
const currNameHashPairs = artifacts
.map(node => [
this.getAttribute(node, 'name'),
this.getAttribute(node, 'data')
this.core.getAttribute(node, 'name'),
this.core.getAttribute(node, 'data')
]);
const inputs = await this.getInputs(node);
const ids = inputs.map(i => this.core.getPath(i[2]));
Expand All @@ -99,28 +99,29 @@ define([

// Remove nodes that already exist
const dataNodes = incomingData.filter(dataNode => {
const hash = this.getAttribute(dataNode, 'data');
const hash = this.core.getAttribute(dataNode, 'data');
const name = this.core.getOwnAttribute(node, 'saveName') ||
this.getAttribute(dataNode, 'name');
this.core.getAttribute(dataNode, 'name');

return !(currNameHashPairs
.find(pair => pair[0] === name && pair[1] === hash));
});

const saveDir = `${this.projectId}/artifacts/`;
const storage = await this.getStorageClient();
const createParams = {base: this.META.Data, parent: artifactsDir};
for (let i = dataNodes.length; i--;) {
const artifact = this.createNode('Data', artifactsDir);
const artifact = this.core.createNode(createParams);
const name = this.core.getOwnAttribute(node, 'saveName') ||
this.getAttribute(dataNodes[i], 'name');
this.core.getAttribute(dataNodes[i], 'name');
const createdAt = Date.now();
const originalData = JSON.parse(this.getAttribute(dataNodes[i], 'data'));
const originalData = JSON.parse(this.core.getAttribute(dataNodes[i], 'data'));
const userAsset = await storage.copy(originalData, saveDir + name);

this.setAttribute(artifact, 'data', JSON.stringify(userAsset));
this.setAttribute(artifact, 'name', name);
this.setAttribute(artifact, 'createdAt', createdAt);
this.setPointer(artifact, 'origin', inputs[0][2]);
this.core.setAttribute(artifact, 'data', JSON.stringify(userAsset));
this.core.setAttribute(artifact, 'name', name);
this.core.setAttribute(artifact, 'createdAt', createdAt);
this.core.setPointer(artifact, 'origin', inputs[0][2]);
}

this.logger.info(`Saved ${dataNodes.length} artifacts in ${this.projectId}.`);
Expand Down
8 changes: 4 additions & 4 deletions src/common/plugin/Operation.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ define([
};

OperationOps.prototype.getOutputs = function (node) {
const code = this.getAttribute(node, 'code');
const code = this.core.getAttribute(node, 'code');
let outputNames = [];
if (code) {
const operation = OperationCode.findOperation(code);
Expand All @@ -25,7 +25,7 @@ define([
};

OperationOps.prototype.getInputs = function (node) {
const code = this.getAttribute(node, 'code');
const code = this.core.getAttribute(node, 'code');
let inputNames = [];
if (code) {
const operation = OperationCode.findOperation(code);
Expand All @@ -50,8 +50,8 @@ define([
var bases = outputs.map(node => this.core.getMetaType(node));
// return [[arg1, Type1, node1], [arg2, Type2, node2]]
return outputs.map((node, i) => [
this.getAttribute(node, 'name'),
this.getAttribute(bases[i], 'name'),
this.core.getAttribute(node, 'name'),
this.core.getAttribute(bases[i], 'name'),
node
]);
});
Expand Down
111 changes: 39 additions & 72 deletions src/plugins/ExecuteJob/ExecuteJob.Metadata.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,10 @@ define([
this._markForDeletion = {}; // id -> node
this._oldMetadataByName = {}; // name -> id
this.createdMetadataIds = {};

this.plotLines = {};
this.createIdToMetadataId = {};
};

// I think I should convert these to just a single 'update graph' command
// TODO: Add tests
ExecuteJob.prototype[CONSTANTS.PLOT_UPDATE] = async function (job, state) {
const jobId = this.core.getPath(job);

Expand All @@ -23,86 +22,53 @@ define([
let id = jobId + '/' + state.id;
let graph = this.getExistingMetadataById(job, 'Graph', id);
if (!graph) {
graph = this.createNode('Graph', job);
this.setAttribute(graph, 'id', id);

this.createIdToMetadataId[graph] = id;
graph = this.core.createNode({
base: this.META.Graph,
parent: job
});
this.core.setAttribute(graph, 'id', id);
this._metadata[id] = graph;
}

// Apply whatever updates are needed
// Set the plot title
// Only support a single axes for now
const axes = state.axes[0];
this.setAttribute(graph, 'name', axes.title);
this.setAttribute(graph, 'xlabel', axes.xlabel);
this.setAttribute(graph, 'ylabel', axes.ylabel);
this.core.setAttribute(graph, 'name', axes.title);
this.core.setAttribute(graph, 'xlabel', axes.xlabel);
this.core.setAttribute(graph, 'ylabel', axes.ylabel);
this.logger.info(`Updating graph named ${axes.title}`);

// Delete current line nodes
if (!this.isCreateId(graph)) {
//const children = await this.core.loadChildren(graph);
const childIds = this.core.getChildrenPaths(graph);
childIds.forEach(id => this.deleteNode(id));
}

if (this.plotLines[id]) {
this.plotLines[id].forEach(lineId => {
if (this._metadata[lineId]) {
const nodeId = this.core.getPath(this._metadata[lineId]);
this.deleteNode(nodeId);
} else {
const createId = Object.keys(this.createIdToMetadataId)
.find(createId => this.createIdToMetadataId[createId] === lineId);

if (createId) {
this.deleteNode(createId);
}
}
});
}
this.plotLines[id] = [];
const children = await this.core.loadChildren(graph);
children.forEach(node => this.core.deleteNode(node));

// Update the points for each of the lines
axes.lines.forEach((line, index) => {
let lineId = id + '/' + index;
let node = this.createNode('Line', graph);
this.plotLines[id].push(lineId);
this.createIdToMetadataId[node] = lineId;
let node = this.core.createNode({
base: this.META.Line,
parent: graph,
});

this.setAttribute(node, 'name', line.label || `line ${index+1}`);
this.core.setAttribute(node, 'name', line.label || `line ${index+1}`);
let points = line.points.map(pts => pts.join(',')).join(';');
this.setAttribute(node, 'points', points);
this.core.setAttribute(node, 'points', points);
});
};

ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE_LINE] = function (job, graphId, id) {
var jobId = this.core.getPath(job),
graph = this._metadata[jobId + '/' + graphId],
name = Array.prototype.slice.call(arguments, 3).join(' '),
line;

// Create a 'line' node in the given Graph metadata node
name = name.replace(/\s+$/, '');
line = this.createNode('Line', graph);
this.setAttribute(line, 'name', name);
this._metadata[jobId + '/' + id] = line;
this.createIdToMetadataId[line] = jobId + '/' + id;
};

ExecuteJob.prototype[CONSTANTS.IMAGE.BASIC] =
ExecuteJob.prototype[CONSTANTS.IMAGE.UPDATE] =
ExecuteJob.prototype[CONSTANTS.IMAGE.CREATE] = function (job, hash, imgId) {
var name = Array.prototype.slice.call(arguments, 3).join(' '),
imageNode = this._getImageNode(job, imgId, name);

this.setAttribute(imageNode, 'data', hash);
this.core.setAttribute(imageNode, 'data', hash);
};

ExecuteJob.prototype[CONSTANTS.IMAGE.NAME] = function (job, imgId) {
var name = Array.prototype.slice.call(arguments, 2).join(' '),
imageNode = this._getImageNode(job, imgId, name);

this.setAttribute(imageNode, 'name', name);
this.core.setAttribute(imageNode, 'name', name);
};

ExecuteJob.prototype._getImageNode = function (job, imgId, name) {
Expand All @@ -116,9 +82,11 @@ define([
imageNode = this._getExistingMetadata(jobId, 'Image', name);
if (!imageNode) {
this.logger.info(`Creating image ${id} named ${name}`);
imageNode = this.createNode('Image', job);
this.setAttribute(imageNode, 'name', name);
this.createIdToMetadataId[imageNode] = id;
imageNode = this.core.createNode({
base: this.META.Image,
parent: job,
});
this.core.setAttribute(imageNode, 'name', name);
}
this._metadata[id] = imageNode;
}
Expand Down Expand Up @@ -147,7 +115,7 @@ define([
if (this.isMetaTypeOf(child, this.META.Metadata)) {
id = this.core.getPath(child);
base = this.core.getBase(child);
type = this.getAttribute(base, 'name');
type = this.core.getAttribute(base, 'name');

this._markForDeletion[nodeId][id] = child;
// namespace by metadata type
Expand All @@ -166,7 +134,7 @@ define([
// make the deletion ids relative to the job node
this.logger.debug(`About to delete ${idsToDelete.length}: ${idsToDelete.join(', ')}`);
for (i = idsToDelete.length; i--;) {
this.deleteNode(idsToDelete[i]);
this.core.deleteNode(idsToDelete[i]);
}
});
};
Expand All @@ -182,14 +150,14 @@ define([
this.logger.debug(`About to delete ${nodeIds.length}: ${nodeIds.join(', ')}`);
for (var i = nodeIds.length; i--;) {
const node = this._markForDeletion[nodeId][nodeIds[i]];
this.deleteNode(this.core.getPath(node));
this.core.deleteNode(this.core.getPath(node));
}
delete this.lastAppliedCmd[nodeId];
delete this.createdMetadataIds[nodeId];
delete this._markForDeletion[nodeId];
}

this.delAttribute(job, 'jobInfo');
this.core.delAttribute(job, 'jobInfo');
};

ExecuteJob.prototype.resultMsg = function(msg) {
Expand All @@ -198,21 +166,14 @@ define([
};

ExecuteJob.prototype.getExistingMetadataById = function (job, type, id) {
const createId = Object.keys(this.createIdToMetadataId)
.find(createId => this.createIdToMetadataId[createId] === id);

if (createId) { // on the queue to be created
return createId;
}

if (this._metadata[id]) { // already created
if (this._metadata[id]) {
return this._metadata[id];
}

return this._getExistingMetadata( // exists from prev run
return this._getExistingMetadata( // exists from prev run
this.core.getPath(job),
type,
node => this.getAttribute(node, 'id') === id
node => this.core.getAttribute(node, 'id') === id
);
};

Expand Down Expand Up @@ -271,5 +232,11 @@ define([
};
};

ExecuteJob.prototype.onNodeCreated = async function (tmpId, node) {
const id = this.createIdToMetadataId[tmpId];
delete this.createIdToMetadataId[tmpId];
this._metadata[id] = node;
};

return ExecuteJob;
});
Loading