Skip to content

fix: switch InputDataAgent from FileCatalogClient to FileCatalog #8757

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
maxnoe wants to merge 1 commit into
base: integration
Choose a base branch
from
+17 −11
Draft

fix: switch InputDataAgent from FileCatalogClient to FileCatalog #8757

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 17 additions & 11 deletions src/DIRAC/TransformationSystem/Agent/InputDataAgent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,18 @@
:dedent: 2
:caption: InputDataAgent options
"""
import time
import datetime

import time
from errno import ENOENT

from DIRAC import S_OK
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
from DIRAC.Core.Base.AgentModule import AgentModule
from DIRAC.Core.Utilities.DErrno import cmpError
from DIRAC.Core.Utilities.TimeUtilities import DiracTime
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations

AGENT_NAME = "Transformation/InputDataAgent"

Expand All @@ -45,7 +45,7 @@ def __init__(self, *args, **kwargs):
self.dateKey = self.am_getOption("DateKey", None)

self.transClient = TransformationClient()
self.metadataClient = FileCatalogClient()
self.metadataClient = FileCatalog()
self.transformationTypes = None
self.multiVO = False

Expand All @@ -70,6 +70,17 @@ def initialize(self):

return S_OK()

def _getMetadataCatalog(self, transDict):
"""Return the configured metadata catalog for the transformation."""
if not self.multiVO:
return self.metadataClient

ownerDN = transDict["AuthorDN"]
ownerGroup = transDict["AuthorGroup"]
ownerVO = getVOForGroup(ownerGroup)
self.log.debug(f"Querying file catalog for VO {ownerVO} as {ownerDN}, {ownerGroup}")
return FileCatalog(vo=ownerVO)

##############################################################################
def execute(self):
"""Main execution method"""
Expand Down Expand Up @@ -123,12 +134,7 @@ def execute(self):
# Perform the query to the metadata catalog
self.log.verbose("Using input data query for transformation", "%d: %s" % (transID, str(inputDataQuery)))
start = time.time()
mdc = self.metadataClient
if self.multiVO:
ownerDN = transDict["AuthorDN"]
ownerGroup = transDict["AuthorGroup"]
self.log.debug(f"Querying file catalog as {ownerDN}, {ownerGroup}")
mdc = FileCatalogClient(useCertificates=True, delegatedDN=ownerDN, delegatedGroup=ownerGroup)
mdc = self._getMetadataCatalog(transDict)
result = mdc.findFilesByMetadata(inputDataQuery)
rtime = time.time() - start
self.log.verbose("Metadata catalog query time", f": {rtime:.2f} seconds.")
Expand Down
Loading