Skip to content

Commit

Permalink
Remove msi_json* calls in tape rules
Browse files Browse the repository at this point in the history 
[DHS-2808, DHS-3335]
  • Loading branch information
@JonathanMELIUS
JonathanMELIUS committed Oct 11, 2023
1 parent 646a590 commit 6a33bc7
Show file tree
Hide file tree
Showing 4 changed files with 96 additions and 164 deletions.
48 changes: 1 addition & 47 deletions native_irods_ruleset/tapeArchive/prepareTapeArchive.r
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ irule_dummy() {
}

IRULE_prepareTapeArchive(*archColl, *initiator) {

# split the *archColl into *project and *projectCollection
uuChopPath(*archColl, *dir, *projectCollection);
uuChopPath(*dir, *dir2, *project);
Expand Down Expand Up @@ -49,40 +48,6 @@ IRULE_prepareTapeArchive(*archColl, *initiator) {
# Open collection to modify state AVU
openProjectCollection(*project, *projectCollection, *aclChange, 'own');

*dataPerResources = "";
# Query all children resources info
# Create *dataPerResources - json object: coordResourceName -> array [dataPath, ...]
# -> create an empty array for each resources
# Create *rescParentsLocation - key pair value variable: resourceParentId -> resourceHostLocation
# -> to execute a remote call on resource host
# Create *rescParentsName - key pair value variable: resourceParentId -> resourceParentName
# -> msiDataObjTrim need a resource name as input (not the id ...)
foreach (*r in select RESC_PARENT, RESC_LOC where RESC_LOC != 'EMPTY_RESC_HOST'){
*resourceHostLocation = *r.RESC_LOC;
*resourceParentId = *r.RESC_PARENT;

if (*resourceParentId != ""){
# Query the resource parent name by its ID
foreach (*p in select RESC_NAME where RESC_ID = *resourceParentId ){
*coordResourceName = *p.RESC_NAME
*rescParentsName.*resourceParentId = *coordResourceName;
}
# Check if the parent resource was already added
# Parent resource are duplicated in the query's result because of replicated resources
# Duplicate doesn't keep *dataArray empty
msiAddKeyVal(*kvp, '*coordResourceName', '');
msi_json_objops(*dataPerResources, *kvp, "get");
*dataArray = *kvp.*coordResourceName;
if (str(*dataArray) == "<null>"){
# Then add the parent resource in *dataPerResources & *rescParentsLocation
msiString2KeyValPair("", *kvp);
msiAddKeyVal(*kvp, '*coordResourceName', '[]');
msi_json_objops(*dataPerResources, *kvp, "add");

*rescParentsLocation.*resourceParentId = *resourceHostLocation;
}
}
}
# count the number of files that match the criteria
*counter=0;

Expand All @@ -98,17 +63,6 @@ IRULE_prepareTapeArchive(*archColl, *initiator) {
COLL_NAME like '*archColl%'
AND DATA_RESC_NAME != '*archiveResc'
AND DATA_SIZE >= '*minimumSize'){
# Build data full path
*dataPath = *ScanColl.COLL_NAME++"/"++*ScanColl.DATA_NAME;

# Get the coordinating resource name
*resourceParentId = *ScanColl.RESC_PARENT;
*coordResourceName = *rescParentsName.*resourceParentId;

# Add new data path entry to *dataArray in *dataPerResources
msiString2KeyValPair("", *kvp);
msiAddKeyVal(*kvp, '*coordResourceName', '*dataPath');
msi_json_objops(*dataPerResources, *kvp, "add");

*counter=*counter+1;
}
Expand All @@ -118,7 +72,7 @@ IRULE_prepareTapeArchive(*archColl, *initiator) {

# Delay before replication
delay("<PLUSET>1s</PLUSET><INST_NAME>irods_rule_engine_plugin-irods_rule_language-instance</INST_NAME>") {
tapeArchive(*archColl, *initiator, *counter, *rescParentsLocation, *dataPerResources, *rescParentsName);
tapeArchive(*archColl, *initiator, *counter);
}
}

Expand Down
147 changes: 86 additions & 61 deletions native_irods_ruleset/tapeArchive/tapeArchive.r
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
#
# NOT RECOMMENDED to be called with irule, since it is part of a greater workflow and has to be called from within prepareTapeArchive.r rule

tapeArchive(*archColl, *initiator, *counter, *rescParentsLocation, *dataPerResources, *rescParentsName){
tapeArchive(*archColl, *initiator, *counter){
*stateAttrName = "archiveState";

# split the *archColl into *project and *projectCollection
Expand All @@ -15,69 +15,54 @@ tapeArchive(*archColl, *initiator, *counter, *rescParentsLocation, *dataPerResou
# Count how many file have been archived
*isMoved=0;

foreach(*srcResourceId in *rescParentsLocation) {
# Get the coordinating resource name
*coordResourceName = *rescParentsName.*srcResourceId;

# Get the data array in *dataPerResources by the *coordResourceName
msiString2KeyValPair("", *kvp);
msiAddKeyVal(*kvp, '*coordResourceName', '');
msi_json_objops(*dataPerResources, *kvp, "get");
*dataArray = *kvp.*coordResourceName;

# Get the data array size
*size = 0;
msi_json_arrayops(*dataArray , "", "size", *size);

# Get the resource host location
*resourceHostLocation= *rescParentsLocation.*srcResourceId;

if ( *size > 0 ){
for (*index=0; *index < *size; *index = *index + 1) {
# Get the data's path by its index in *dataArray
*dataPath = "";
msi_json_arrayops(*dataArray, *dataPath, "get", *index)
msiWriteRodsLog("DEBUG: *dataPath", 0);

# Update state AVU with progress
# Collection have been opened in prepareTapeArchive
# Otherwise update will failed
*value = "archive-in-progress "++str(*isMoved+1)++"/"++str(*counter);
setCollectionAVU(*archColl,*stateAttrName,*value);

# Calculate data checksum
# We do not pass any options, this way we get the existing checksum, or a new one is calculated.
# If a failure occurs, the replication is stopped, no trimming happens and we can manually verify
# any other replicas
msiDataObjChksum(*dataPath,"",*chksum);

# Replicate data from *coordResourceName to *archiveResc
# Checksum verification is implicit here, because we calculated the checksum already, msiDataObjRepl
# will automatically also include a checksum check on the destination
# 'errorcode()' catches the microservice's error, making it non-fatal, so that the rule continues processing and is able to 'setTapeErrorAVU()'
*moveError = errorcode(msiDataObjRepl(*dataPath, "destRescName=*archiveResc", *moveStatus));
msiWriteRodsLog("DEBUG: moveError *moveError", 0);
if ( *moveError != 0 ) {
setTapeErrorAVU(*archColl, *initiator, *stateAttrName, "error-archive-failed", "Replication of *dataPath from *coordResourceName to *archiveResc FAILED.")
}

# Trim data from *coordResourceName
# 'errorcode()' catches the microservice's error, making it non-fatal, so that the rule continues processing and is able to 'setTapeErrorAVU()'
*trimError = errorcode(msiDataObjTrim(*dataPath, *coordResourceName, "null", "1", "null", *trimStatus));
msiWriteRodsLog("DEBUG: trimError *trimError", 0);
if ( *trimError != 0 ) {
setTapeErrorAVU(*archColl, *initiator, *stateAttrName, "error-archive-failed", "Trim *dataPath from *coordResourceName FAILED.")
}

# Update counter
*isMoved=*isMoved+1;

# Report logs
msiWriteRodsLog("DEBUG: \t\tchksum done *chksum", 0);
msiWriteRodsLog("DEBUG: \t\tReplicate from *coordResourceName to *archiveResc", 0);
msiWriteRodsLog("INFO: surfArchiveScanner found *counter files", 0);

if (*counter > 0) {
getResourcesNames(*rescParentsName);
getDataPathToArchive(*archColl, *archiveResc, *dataPerResources, *rescParentsName);

foreach(*dataPath in *dataPerResources) {
*coordResourceName = *dataPerResources.*dataPath;

# Update state AVU with progress
# Collection have been opened in prepareTapeArchive
# Otherwise update will failed
*value = "archive-in-progress "++str(*isMoved+1)++"/"++str(*counter);
setCollectionAVU(*archColl,*stateAttrName,*value);

# Calculate data checksum
# We do not pass any options, this way we get the existing checksum, or a new one is calculated.
# If a failure occurs, the replication is stopped, no trimming happens and we can manually verify
# any other replicas
msiDataObjChksum(*dataPath,"",*chksum);
msiWriteRodsLog("DEBUG: \t\tchksum done *chksum", 0);

# Replicate data from *coordResourceName to *archiveResc
# Checksum verification is implicit here, because we calculated the checksum already, msiDataObjRepl
# will automatically also include a checksum check on the destination
# 'errorcode()' catches the microservice's error, making it non-fatal, so that the rule continues processing and is able to 'setTapeErrorAVU()'
*moveError = errorcode(msiDataObjRepl(*dataPath, "destRescName=*archiveResc", *moveStatus));
msiWriteRodsLog("DEBUG: \t\t*dataPath -> moveError: *moveError", 0);
if ( *moveError != 0 ) {
setTapeErrorAVU(*archColl, *initiator, *stateAttrName, "error-archive-failed", "Replication of *dataPath from *coordResourceName to *archiveResc FAILED.")
}

# Trim data from *coordResourceName
# 'errorcode()' catches the microservice's error, making it non-fatal, so that the rule continues processing and is able to 'setTapeErrorAVU()'
*trimError = errorcode(msiDataObjTrim(*dataPath, *coordResourceName, "null", "1", "null", *trimStatus));
msiWriteRodsLog("DEBUG: \t\t*dataPath -> trimError: *trimError", 0);
if ( *trimError != 0 ) {
setTapeErrorAVU(*archColl, *initiator, *stateAttrName, "error-archive-failed", "Trim *dataPath from *coordResourceName FAILED.")
}

# Update counter
*isMoved=*isMoved+1;

# Report logs
msiWriteRodsLog("DEBUG: \t\tReplicate from *coordResourceName to *archiveResc", 0);
}
}

# Update state AVU to done
setCollectionAVU(*archColl, *stateAttrName, "archive-done")
msiWriteRodsLog("DEBUG: surfArchiveScanner archived *isMoved files ", 0);
Expand All @@ -97,3 +82,43 @@ tapeArchive(*archColl, *initiator, *counter, *rescParentsLocation, *dataPerResou
# Close collection by making all access read only
closeProjectCollection(*project, *projectCollection);
}

getResourcesNames(*rescParentsName){
foreach (*r in select RESC_PARENT, RESC_LOC where RESC_LOC != 'EMPTY_RESC_HOST'){
*resourceParentId = *r.RESC_PARENT;

if (*resourceParentId != ""){
# Query the resource parent name by its ID
foreach (*p in select RESC_NAME where RESC_ID = *resourceParentId ){
*coordResourceName = *p.RESC_NAME;
*rescParentsName.*resourceParentId = *coordResourceName;
}
}
}
}

getDataPathToArchive(*archColl, *archiveResc, *dataPerResources, *rescParentsName){
# The minimum file size criteria (in bytes)
*minimumSize=262144000;

foreach( *ScanColl in
SELECT
RESC_PARENT,
COLL_NAME,
DATA_NAME
WHERE
COLL_NAME like '*archColl%'
AND DATA_RESC_NAME != '*archiveResc'
AND DATA_SIZE >= '*minimumSize'){
# Build data full path
*dataPath = *ScanColl.COLL_NAME++"/"++*ScanColl.DATA_NAME;

# Get the coordinating resource name
*resourceParentId = *ScanColl.RESC_PARENT;
*coordResourceName = *rescParentsName.*resourceParentId;

*dataPerResources.*dataPath = *coordResourceName

}
msiWriteRodsLog("DEBUG: dataPerResources: *dataPerResources", 0);
}
27 changes: 9 additions & 18 deletions native_irods_ruleset/tapeArchive/tapeUnArchive.r
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# Call with
#
# NOT RECOMMENDED to be called with irule, since it is part of a greater workflow and has to be called from within prepareTapeUnArchive.r rule

tapeUnArchive(*count, *archColl, *initiator){
#Matthew Saum
Expand All @@ -22,14 +25,11 @@ tapeUnArchive(*count, *archColl, *initiator){
getCollectionAVU(*projectPath,"archiveDestinationResource",*archiveResc,"N/A","true");

*minimumSize=262144000; #The minimum file size (in bytes)

# rodsadmin user running the rule
# get this from avu set on archive
getResourceAVU(*archiveResc,"service-account",*aclChange,"N/A","true");

*isMoved=0; #Number of files moved counter
*stateAttrName = "unArchiveState";

msiWriteRodsLog("DEBUG: surfArchiveScanner found *count files", 0);

msiGetObjType(*archColl, *inputType);
if (*inputType like '-d'){
uuChopPath(*archColl, *dir, *dataName);
Expand All @@ -42,15 +42,6 @@ tapeUnArchive(*count, *archColl, *initiator){
*projectResource = *av.META_COLL_ATTR_VALUE;
}

#Find our resource location
*resourceLocation = ""
foreach(*parent in SELECT RESC_ID WHERE RESC_NAME = *projectResource){
*parentID = *parent.RESC_ID;
foreach(*r in SELECT RESC_LOC WHERE RESC_PARENT = *parentID){
*resourceLocation = *r.RESC_LOC;
}
}

if (*inputType like '-d'){
foreach(*ScanColl in
SELECT
Expand All @@ -76,14 +67,14 @@ tapeUnArchive(*count, *archColl, *initiator){
# will automatically also include a checksum check on the destination
# 'errorcode()' catches the microservice's error, making it non-fatal, so that the rule continues processing and is able to 'setTapeErrorAVU()'
*moveError = errorcode(msiDataObjRepl(*ipath, "destRescName=*projectResource", *moveStatus));
msiWriteRodsLog("DEBUG: moveError *moveError", 0);
msiWriteRodsLog("DEBUG: \t\t*ipath -> moveError: *moveError", 0);
if ( *moveError != 0 ) {
setTapeErrorAVU(*projectCollectionPath, *initiator, *stateAttrName, "error-unarchive-failed", "Replication of *ipath from *projectResource to *archiveResc FAILED.")
}

# 'errorcode()' catches the microservice's error, making it non-fatal, so that the rule continues processing and is able to 'setTapeErrorAVU()'
*trimError = errorcode(msiDataObjTrim(*ipath, *archiveResc, "null", "1", "null", *trimStatus));
msiWriteRodsLog("DEBUG: trimError *trimError", 0);
msiWriteRodsLog("DEBUG: \t\t*ipath -> trimError: *trimError", 0);
if ( *trimError != 0 ) {
setTapeErrorAVU(*projectCollectionPath, *initiator, *stateAttrName, "error-unarchive-failed", "Trim *ipath from *projectResource FAILED.")
}
Expand Down Expand Up @@ -122,14 +113,14 @@ tapeUnArchive(*count, *archColl, *initiator){
# will automatically also include a checksum check on the destination
# 'errorcode()' catches the microservice's error, making it non-fatal, so that the rule continues processing and is able to 'setTapeErrorAVU()'
*moveError = errorcode(msiDataObjRepl(*ipath, "destRescName=*projectResource", *moveStatus));
msiWriteRodsLog("DEBUG: moveError *moveError", 0);
msiWriteRodsLog("DEBUG: \t\t*ipath -> moveError: *moveError", 0);
if ( *moveError != 0 ) {
setTapeErrorAVU(*projectCollectionPath, *initiator, *stateAttrName, "error-unarchive-failed", "Replication of *ipath from *projectResource to *archiveResc FAILED.")
}

# 'errorcode()' catches the microservice's error, making it non-fatal, so that the rule continues processing and is able to 'setTapeErrorAVU()'
*trimError = errorcode(msiDataObjTrim(*ipath, *archiveResc, "null", "1", "null", *trimStatus));
msiWriteRodsLog("DEBUG: trimError *trimError", 0);
msiWriteRodsLog("DEBUG: \t\t*ipath -> trimError: *trimError", 0);
if ( *trimError != 0 ) {
setTapeErrorAVU(*projectCollectionPath, *initiator, *stateAttrName, "error-unarchive-failed", "Trim *ipath from *projectResource FAILED.")
}
Expand Down
38 changes: 0 additions & 38 deletions native_irods_ruleset/tapeArchive/tapeUnArchiveFile.r
View file

This file was deleted.

0 comments on commit 6a33bc7

Please sign in to comment.