Merge pull request #7878 from fstagni/cherry-pick-2-7060c5098-integra…

…tion

[sweep:integration] added some tests jobs for certifications

src/DIRAC/tests/Utilities/testJobDefinitions.py

@@ -3,14 +3,13 @@
 
 # pylint: disable=invalid-name
 
+import errno
 import os
 import time
-import errno
 
 from DIRAC import rootPath
-from DIRAC.Interfaces.API.Job import Job
 from DIRAC.Interfaces.API.Dirac import Dirac
-
+from DIRAC.Interfaces.API.Job import Job
 from DIRAC.tests.Utilities.utils import find_all
 
 
@@ -78,6 +77,43 @@ def helloWorldJenkins():
             J.setInputSandbox([find_all("exe-script.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]])
     J.setExecutable("exe-script.py", "", "helloWorld.log")
     J.setDestination("DIRAC.Jenkins.ch")
+
+
+def helloWorld_input():
+    """simple hello world job with input"""
+
+    J = baseToAllJobs("helloWorld_input")
+    try:
+        J.setInputSandbox([find_all("exe-script-with-input.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script-with-input.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox(
+                [find_all("exe-script-with-input.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]]
+            )
+    J.setExecutable("exe-script-with-input.py", "", "helloWorld.log")
+    return endOfAllJobs(J)
+
+
+def helloWorld_input_single():
+    """simple hello world job with input"""
+
+    J = baseToAllJobs("helloWorld_input_single")
+    try:
+        J.setInputSandbox([find_all("exe-script-with-input-single-location.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script-with-input-single-location.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox(
+                [
+                    find_all(
+                        "exe-script-with-input-single-location.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow"
+                    )[0]
+                ]
+            )
+    J.setExecutable("exe-script-with-input-single-location.py", "", "helloWorld.log")
     return endOfAllJobs(J)
 
 
@@ -326,6 +362,28 @@ def parametricJob():
     return endOfAllJobs(J)
 
 
+def parametricJobInputData():
+    """Creates a parametric job with 3 subjobs which are simple hello world jobs, but with input data"""
+
+    J = baseToAllJobs("parametricJobInput")
+    try:
+        J.setInputSandbox([find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox([find_all("exe-script.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]])
+    J.setParameterSequence("args", ["one", "two", "three"])
+    J.setParameterSequence("iargs", [1, 2, 3])
+    J.setParameterSequence(
+        "InputData",
+        ["/dteam/user/f/fstagni/test/1.txt", "/dteam/user/f/fstagni/test/2.txt", "/dteam/user/f/fstagni/test/3.txt"],
+    )
+    J.setInputDataPolicy("download")
+    J.setExecutable("exe-script.py", arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log")
+    return endOfAllJobs(J)
+
+
 def jobWithOutput():
     """Creates a job that uploads an output.
     The output SE is not set here, so it would use the default /Resources/StorageElementGroups/SE-USER

tests/System/exe-script-with-input-single-location.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+"""Script to run Executable application"""
+from os import system
+import sys
+
+# Main
+if __name__ == "__main__":
+    sys.exit(int(system("""cat testInputFileSingleLocation.txt""") / 256))

tests/System/exe-script-with-input.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+"""Script to run Executable application"""
+import sys
+from os import system
+
+# Main
+if __name__ == "__main__":
+    sys.exit(int(system("""cat testInputFile.txt""") / 256))

tests/System/testInputFile.txt

@@ -0,0 +1 @@
+this is just a test

tests/System/testInputFileSingleLocation.txt

@@ -0,0 +1,3 @@
+this is just a test
+
+This file should be at /dteam/user/f/fstagni/test/testInputFileSingleLocation.txt in RAL-SE

tests/System/transformationSystem.sh

@@ -45,7 +45,7 @@ directory=/dteam/diracCertification/Test/INIT/$version/$tdate/$stime
 #selecting a random USER Storage Element
 #SEs=$(dirac-dms-show-se-status |grep USER |grep -v 'Banned\|Degraded\|-2' | awk '{print $1}')
 #get all SEs ending with -SE that are Active
-SEs=$(dirac-dms-show-se-status | grep -e "-SE \|-disk" | grep -v 'RAL\|CESNET-SE\|Banned\|Probing\|Error\|-new' | awk '{print $1}')
+SEs=$(dirac-dms-show-se-status | grep -e "-SE \|-disk" | grep -v 'RAL\|Banned\|Probing\|Error\|-new' | awk '{print $1}')
 
 x=0
 for n in $SEs

tests/System/transformation_replication.sh

@@ -45,7 +45,7 @@ echo "Creating TransformationSystemTest"
 mkdir -p TransformationSystemTest
 directory=/dteam/diracCertification/Test/INIT/$version/$tdate/$stime/replication
 #get all SEs ending with -SE that are Active, randomize, turn into array
-SEs=( $(dirac-dms-show-se-status | grep -e "-SE \|-disk " | grep -v 'RAL\|CESNET\|Banned\|Probing\|Error\|-new' | awk '{print $1}' | sort -R | xargs) )
+SEs=( $(dirac-dms-show-se-status | grep -e "-SE \|-disk " | grep -v 'RAL\|Banned\|Probing\|Error\|-new' | awk '{print $1}' | sort -R | xargs) )
 
 SOURCE_SE=${SEs[0]}
 TARGET_SE=${SEs[1]}

tests/System/unitTestUserJobs.py

@@ -1,6 +1,7 @@
 """ Collection of user jobs for testing purposes
 """
 # pylint: disable=wrong-import-position, invalid-name
+import sys
 import time
 import unittest
 
@@ -22,7 +23,29 @@ class GridSubmissionTestCase(unittest.TestCase):
     """Base class for the Regression test cases"""
 
     def setUp(self):
-        pass
+        result = getProxyInfo()
+        if result["Value"]["group"] not in ["dteam_user", "gridpp_user"]:
+            print("GET A USER GROUP")
+            sys.exit(1)
+
+        res = DataManager().getReplicas(
+            [
+                "/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt",
+                "/dteam/user/f/fstagni/test/testInputFile.txt",
+            ]
+        )
+        if not res["OK"]:
+            print(f"DATAMANAGER.getRepicas failure: {res['Message']}")
+            sys.exit(1)
+        if res["Value"]["Failed"]:
+            print(f"DATAMANAGER.getRepicas failed for something: {res['Value']['Failed']}")
+            sys.exit(1)
+
+        replicas = res["Value"]["Successful"]
+        if list(replicas["/dteam/user/f/fstagni/test/testInputFile.txt"]) != ["RAL-SE", "UKI-LT2-IC-HEP-disk"]:
+            print("/dteam/user/f/fstagni/test/testInputFile.txt locations are not correct")
+        if list(replicas["/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt"]) != ["RAL-SE"]:
+            print("/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt locations are not correct")
 
     def tearDown(self):
         pass
@@ -41,6 +64,14 @@ def test_submit(self):
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
 
+        res = helloWorld_input()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
+        res = helloWorld_input_single()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
         res = helloWorldCERN()
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
@@ -93,6 +124,10 @@ def test_submit(self):
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
 
+        res = parametricJobInputData()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
         res = jobWithOutput()
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
@@ -104,51 +139,11 @@ def test_submit(self):
         print(f"submitted {len(jobsSubmittedList)} jobs: {','.join(str(js) for js in jobsSubmittedList)}")
 
 
-# FIXME: This is also in the extension...? To try!
-# class monitorSuccess( GridSubmissionTestCase ):
-#
-#   def test_monitor( self ):
-#
-#     toRemove = []
-#     fail = False
-#
-#     # we will check every 10 minutes, up to 6 hours
-#     counter = 0
-#     while counter < 36:
-#       jobStatus = self.dirac.status( jobsSubmittedList )
-#       self.assertTrue( jobStatus['OK'] )
-#       for jobID in jobsSubmittedList:
-#         status = jobStatus['Value'][jobID]['Status']
-#         minorStatus = jobStatus['Value'][jobID]['MinorStatus']
-#         if status == 'Done':
-#           self.assertTrue( minorStatus in ['Execution Complete', 'Requests Done'] )
-#           jobsSubmittedList.remove( jobID )
-#           res = self.dirac.getJobOutputLFNs( jobID )
-#           if res['OK']:
-#             lfns = res['Value']
-#             toRemove += lfns
-#         if status in ['Failed', 'Killed', 'Deleted']:
-#           fail = True
-#           jobsSubmittedList.remove( jobID )
-#       if jobsSubmittedList:
-#         time.sleep( 600 )
-#         counter = counter + 1
-#       else:
-#         break
-#
-#     # removing produced files
-#     res = self.dirac.removeFile( toRemove )
-#     self.assertTrue( res['OK'] )
-#
-#     if fail:
-#       self.assertFalse( True )
-
 #############################################################################
 # Test Suite run
 #############################################################################
 
 if __name__ == "__main__":
     suite = unittest.defaultTestLoader.loadTestsFromTestCase(GridSubmissionTestCase)
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(submitSuccess))
-    # suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase( monitorSuccess ) )
     testResult = unittest.TextTestRunner(verbosity=2).run(suite)