DIRAC_TS_Tutorial/submit_ts_step3.py at master · arrabito/DIRAC_TS_Tutorial · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
""" Transformation creating the job building the final mandelbrot image
"""
import json
import os

from DIRAC.Core.Base import Script
Script.parseCommandLine()

import DIRAC
from DIRAC.Interfaces.API.Job import Job
from DIRAC.TransformationSystem.Client.Transformation import Transformation

def submitTS():

  ########################################
  # Modify here with your dirac username
  owner = 'user02'
  ########################################


  ########################################
  # Job description
  ########################################
  job = Job()
  job.setName('build mandelbrot')
  job.setOutputSandbox( ['*log'] )
  job.setType('DataReprocessing')

  ## define the job workflow in 3 steps
  # job step1: setup software
  job.setExecutable('git clone https://github.com/bregeon/mandel4ts.git')
  # job step2: run mandelbrot build image
  job.setExecutable('./mandel4ts/build_merged_img.py')

  outputPath = os.path.join('/vo.france-grilles.fr/user',owner[0],owner,'mandelbrot/images/final')
  outputPattern = 'merged_image.bmp'
  outputSE = 'DIRAC-USER'
  outputMetadata = json.dumps( {"application":"mandelbrot","image_format":"bmp", "image_width":7680, "image_height":4200, "owner":owner} )

  # job step3: upload data and set metadata
  job.setExecutable( './mandel4ts/dirac-add-files.py', arguments = "%s '%s' %s '%s'" % (outputPath, outputPattern, outputSE, outputMetadata ) )

  # job step4: mark input files as done with the FailoverRequest (and a few other things)
  job.setExecutable('/bin/ls -l', modulesList=['Script', 'FailoverRequest'])

  ########################################
  # Transformation definition
  ########################################
  t = Transformation()

  t.setTransformationName( owner+'_step3' )
  t.setType( "DataReprocessing" )
  t.setDescription( "Merge mandelbrot images production" )
  t.setLongDescription( "Merge mandelbrot images production" )
  t.setGroupSize( 3 ) # group input files
  # set the job workflow to the transformation
  t.setBody ( job.workflow.toXML() )

  # define input data by metadata query
  inputMetaquery = {"application":"mandelbrot","image_format":"ascii", "image_width":7680, "image_height":1400, "owner":owner}
  t.setInputMetaQuery(inputMetaquery)

  ########################################
  # Transformation submission
  ########################################
  res = t.addTransformation()

  if not res['OK']:
    print(res['Message'])
    DIRAC.exit( -1 )

  t.setStatus( "Active" )
  t.setAgentType( "Automatic" )

  return res


#########################################################
if __name__ == '__main__':

  try:
    res = submitTS()
    if not res['OK']:
      DIRAC.gLogger.error ( res['Message'] )
      DIRAC.exit( -1 )
  except Exception:
    DIRAC.gLogger.exception()
    DIRAC.exit( -1 )