Remove extra files from TW tarballs (#8718)

* remove extra files from TaskManagerRun.tar.gz

* remove extra files from CMSRunAnalysis.tar.gz

* avoid using ServerUtilities in cmscp.py

cicd/crabtaskworker_pypi/buildTWTarballs.sh

@@ -78,9 +78,9 @@ cp -r "${CRABSERVERDIR}/scripts"/{TweakPSet.py,CMSRunAnalysis.py,task_process} .
 cp "${CRABSERVERDIR}/src/python"/{ServerUtilities.py,RucioUtils.py,CMSGroupMapper.py,RESTInteractions.py} .
 
 echo "Making TaskManagerRun tarball"
-tar zcf "${RUNTIME_WORKDIR}/TaskManagerRun.tar.gz" CRAB3.zip TweakPSet.py CMSRunAnalysis.py task_process ServerUtilities.py RucioUtils.py CMSGroupMapper.py RESTInteractions.py || exit 4
+tar zcf "${RUNTIME_WORKDIR}/TaskManagerRun.tar.gz" CRAB3.zip task_process ServerUtilities.py RucioUtils.py CMSGroupMapper.py RESTInteractions.py || exit 4
 echo "Making CMSRunAnalysis tarball"
-tar zcf "${RUNTIME_WORKDIR}/CMSRunAnalysis.tar.gz" WMCore.zip TweakPSet.py CMSRunAnalysis.py ServerUtilities.py CMSGroupMapper.py RESTInteractions.py || exit 4
+tar zcf "${RUNTIME_WORKDIR}/CMSRunAnalysis.tar.gz" WMCore.zip TweakPSet.py CMSRunAnalysis.py || exit 4
 
 # cleanup.
 # current directory ($RUNTIME_WORKDIR) should only have TaskManagerRun.tar.gz and CMSRunAnalysis.tar.gz

scripts/cmscp.py

@@ -17,7 +17,7 @@
 if os.path.exists("WMCore.zip") and "WMCore.zip" not in sys.path:
     sys.path.append("WMCore.zip")
 
-from ServerUtilities import cmd_exist, parseJobAd
+from CMSRunAnalysis import parseAd
 
 if 'http_proxy' in os.environ and not os.environ['http_proxy'].startswith("http://"):
     os.environ['http_proxy'] = "http://%s" % (os.environ['http_proxy'])
@@ -798,7 +798,7 @@ def update_exit_info(exit_info, exit_code, exit_msg = None, force = False):
         return exit_info
     global G_JOB_AD
     try:
-        G_JOB_AD = parseJobAd(os.environ['_CONDOR_JOB_AD'])
+        G_JOB_AD = parseAd()
     except Exception:
         msg  = "WARNING: Unable to parse job's HTCondor ClassAd."
         msg += "\n%s" % (traceback.format_exc())
@@ -1254,11 +1254,9 @@ def update_exit_info(exit_info, exit_code, exit_msg = None, force = False):
     ## This stageout implementation will be used for all direct stageout
     ## attempts (for the user logs archive file and the user output files).
     direct_stageout_impl = None
-    direct_stageout_command = "srmv2-lcg"
+    direct_stageout_command = "gfal2"
+    print('Will use gfal2 commands for direct stageout.')
     direct_stageout_protocol = "srmv2"
-    if cmd_exist("gfal-copy"):
-        print('Will use gfal2 commands for direct stageout.')
-        direct_stageout_command = "gfal2"
     condition = ('remote' in stageout_policy and condition_stageout)
     if skip['init_direct_stageout_impl']:
         msg  = "WARNING: Internal wrapper flag skip['init_direct_stageout_impl'] is True."

src/python/ServerUtilities.py

@@ -424,33 +424,6 @@ def isFailurePermanent(reason, gridJob=False):
     return False, "", None
 
 
-def parseJobAd(filename):
-    """ Parse the jobad file provided as argument and return a dict representing it
-        SB: why do we have this ? the classAd object returned by classad.parse has
-            the semantic of a dictionary ! Currently it is only used in cmscp.py
-            and in job wrapper we are not sure to have HTCondor available.
-            Note that we also have a parseAd() method inside CMSRunAnalysis.py which should
-            do finely also in cmscp.py
-    """
-    jobAd = {}
-    with open(filename, 'r', encoding='utf-8') as fd:
-        for adline in fd.readlines():
-            info = adline.split(' = ', 1)
-            if len(info) != 2:
-                continue
-            if info[1].startswith('undefined'):
-                val = info[1].strip()
-            elif info[1].startswith('"'):
-                val = info[1].strip().replace('"', '')
-            else:
-                try:
-                    val = int(info[1].strip())
-                except ValueError:
-                    continue
-            jobAd[info[0]] = val
-    return jobAd
-
-
 def mostCommon(lst, default=0):
     """ Return the most common error among the list
     """