Python lxml.etree.parse() Examples

def package_source_hash(apiurl, project, package, revision=None):
    query = {}
    if revision:
        query['rev'] = revision

    # Will not catch packages that previous had a link, but no longer do.
    if package_source_link_copy(apiurl, project, package):
        query['expand'] = 1

    try:
        url = makeurl(apiurl, ['source', project, package], query)
        root = ETL.parse(http_GET(url)).getroot()
    except HTTPError as e:
        if e.code == 400 or e.code == 404:
            # 400: revision not found, 404: package not found.
            return None

        raise e

    if revision and root.find('error') is not None:
        # OBS returns XML error instead of HTTP 404 if revision not found.
        return None

    from osclib.util import sha1_short
    return sha1_short(root.xpath('entry[@name!="_link"]/@md5')) 

def list():
        """ get list of all collection entries """

        cols_elem = ET.Element('collections')
        for col in Col.objects.all():
            path = os.path.join('data', 'collections', col.name)
            if not os.path.exists(path):
                logging.error('Collection has inconstancy : %s !!' % col.name)
                continue
            files = glob.glob(os.path.join(path, '*'))
            for _file in files:
                payload = ET.parse(_file)
                for child in payload.getroot():
                    if child.tag == 'metadata':
                        cols_elem.append(child)
        return cols_elem 

def ensure_staging_archs(self, project):
        meta = ET.parse(http_GET(self.project_meta_url(project)))
        repository = meta.find('repository[@name="{}"]'.format(self.cmain_repo))

        changed = False
        for arch in self.cstaging_archs:
            if not repository.xpath('./arch[text()="{}"]'.format(arch)):
                elm = ET.SubElement(repository, 'arch')
                elm.text = arch
                changed = True

        if not changed:
            return

        meta = ET.tostring(meta)
        http_PUT(self.project_meta_url(project), data=meta) 

def project_status_requests(self, request_type, filter_function=None):
        requests = []
        for status in self.project_status(None, status=False):
            for request in status.findall(f'{request_type}_requests/request'):
                updated_at = dateutil.parser.parse(request.get('updated'), ignoretz=True)
                updated_delta = datetime.utcnow() - updated_at
                if updated_delta.total_seconds() < 0 * 60:
                    # Allow for dashboard to update caches by not considering
                    # requests whose state has changed in the last 5 minutes.
                    continue

                if filter_function and not filter_function(request, updated_delta):
                    continue

                requests.append(str(request.get('id')))

        return requests 

def ensure_staging_archs(self, project):
        meta = ET.parse(http_GET(self.project_meta_url(project)))
        repository = meta.find('repository[@name="{}"]'.format(self.cmain_repo))

        changed = False
        for arch in self.cstaging_archs:
            if not repository.xpath('./arch[text()="{}"]'.format(arch)):
                elm = ET.SubElement(repository, 'arch')
                elm.text = arch
                changed = True

        if not changed:
            return

        meta = ET.tostring(meta)
        http_PUT(self.project_meta_url(project), data=meta) 

def find_via_stagingapi(self, pkgs):
        """
        Search for all various mutations and return list of SR#s. Use
        and instance of StagingAPI to direct the search, this makes
        sure that the SR# are inside a staging project.
        :param pkgs: mesh of argumets to search for

        This function is only called for its side effect.
        """

        url = self.api.makeurl(['staging', self.api.project, 'staging_projects'], { 'requests': 1 })
        status = ET.parse(self.api.retried_GET(url)).getroot()

        for p in pkgs:
            found = False
            for staging in status.findall('staging_project'):
                for request in staging.findall('staged_requests/request'):
                    if request.get('package') == p or request.get('id') == p:
                        self.srs[int(request.get('id'))] = {'staging': staging.get('name')}
                        found = True
                        break
            if not found:
                raise oscerr.WrongArgs('No SR# found for: {}'.format(p)) 

def project_status_requests(self, request_type, filter_function=None):
        requests = []
        for status in self.project_status(None, status=False):
            for request in status.findall(f'{request_type}_requests/request'):
                updated_at = dateutil.parser.parse(request.get('updated'), ignoretz=True)
                updated_delta = datetime.utcnow() - updated_at
                if updated_delta.total_seconds() < 0 * 60:
                    # Allow for dashboard to update caches by not considering
                    # requests whose state has changed in the last 5 minutes.
                    continue

                if filter_function and not filter_function(request, updated_delta):
                    continue

                requests.append(str(request.get('id')))

        return requests 

def package_list_kind_filtered(apiurl, project, kinds_allowed=['source']):
    query = {
        'view': 'info',
        'nofilename': '1',
    }
    url = makeurl(apiurl, ['source', project], query)
    root = ETL.parse(http_GET(url)).getroot()

    for package in root.xpath('sourceinfo/@package'):
        kind = package_kind(apiurl, project, package)
        if kind not in kinds_allowed:
            continue

        yield package 

def job_history_get(self, project, repository, architecture, package=None, limit=20):
        query = {}
        if package:
            query['package'] = package
        if limit != None and int(limit) > 0:
            query['limit'] = int(limit)
        u = makeurl(self.apiurl, ['build', project, repository, architecture, '_jobhistory'], query)
        return ET.parse(http_GET(u)).getroot()

    # Modified from osc.core.print_buildlog() 

def buildlog_get(self, prj, package, repository, arch, offset=0, strip_time=False, last=False):
        # to protect us against control characters
        all_bytes = bytes.maketrans(b'', b'')
        remove_bytes = all_bytes[:8] + all_bytes[14:32]  # accept tabs and newlines

        path = ['build', prj, repository, arch, package, '_log']
        if offset < 0:
            url = makeurl(self.apiurl, path, {'view': 'entry'})
            root = ET.parse(http_GET(url)).getroot()
            size = root.xpath('entry[@name="_log"]/@size')
            if size:
                offset += int(size[0])

        query = {'nostream': '1', 'start': '%s' % offset}
        if last:
            query['last'] = 1
        log = StringIO()
        while True:
            query['start'] = offset
            start_offset = offset
            u = makeurl(self.apiurl, path, query)
            for data in streamfile(u, bufsize="line"):
                offset += len(data)
                if strip_time:
                    data = buildlog_strip_time(data)
                log.write(decode_it(data.translate(all_bytes, remove_bytes)))
            if start_offset == offset:
                break

        return log.getvalue() 

def project_status(self, staging, status=True, requests=True, reload=False):
        opts = {}
        if requests:
            opts['requests'] = 1
        if status:
            opts['status'] = 1
        paths = ['staging', self.project, 'staging_projects']
        if staging:
            paths.append(staging)
        url = self.makeurl(paths, opts)
        return ET.parse(self.retried_GET(url)).getroot() 

def source_info(self, project, package, rev=None):
        query = {'view': 'info'}
        if rev is not None:
            query['rev'] = rev
        url = makeurl(self.apiurl, ('source', project, package), query=query)
        try:
            return ET.parse(http_GET(url)).getroot()
        except (HTTPError, URLError):
            return None 

def fileinfo_ext_all(apiurl, project, repo, arch, package):
    url = makeurl(apiurl, ['build', project, repo, arch, package])
    binaries = ET.parse(http_GET(url)).getroot()
    for binary in binaries.findall('binary'):
        filename = binary.get('filename')
        if not filename.endswith('.rpm'):
            continue

        yield fileinfo_ext(apiurl, project, repo, arch, package, filename) 

def fileinfo_ext(apiurl, project, repo, arch, package, filename):
    url = makeurl(apiurl,
                  ['build', project, repo, arch, package, filename],
                  {'view': 'fileinfo_ext'})
    return ET.parse(http_GET(url)).getroot() 

def builddepinfo(apiurl, project, repo, arch, order = False):
    query = {}
    if order:
        query['view'] = 'order'
    url = makeurl(apiurl, ['build', project, repo, arch, '_builddepinfo'], query)
    return ETL.parse(http_GET(url)).getroot() 

def entity_email(apiurl, key, entity_type='person', include_name=False):
    url = makeurl(apiurl, [entity_type, key])
    root = ET.parse(http_GET(url)).getroot()

    email = root.find('email')
    if email is None:
        return None
    email = email.text

    realname = root.find('realname')
    if include_name and realname is not None:
        email = '{} <{}>'.format(realname.text, email)

    return email 

def package_kind(apiurl, project, package):
    if package.startswith('00') or package.startswith('_'):
        return 'meta'

    if ':' in package:
        return 'multibuild_subpackage'

    if package.startswith('patchinfo.'):
        return 'patchinfo'

    try:
        url = makeurl(apiurl, ['source', project, package, '_meta'])
        root = ETL.parse(http_GET(url)).getroot()
    except HTTPError as e:
        if e.code == 404:
            return None

        raise e

    if root.find('releasename') is not None and root.find('releasename').text != package:
        return 'maintenance_update'

    # Some multispec subpackages do not have bcntsynctag, so check link.
    link = entity_source_link(apiurl, project, package)
    if link is not None and link.get('cicount') == 'copy':
        kind_target = package_kind(apiurl, project, link.get('package'))
        if kind_target != 'maintenance_update':
            # If a multispec subpackage was updated via a maintenance update the
            # proper link information is lost and it will be considered source.
            return 'multispec_subpackage'

    return 'source' 

def get_ignored_requests(self):
        ignore = {}
        url = self.makeurl(['staging', self.project, 'excluded_requests'])
        root = ET.parse(self.retried_GET(url)).getroot()
        for entry in root.findall('request'):
            ignore[int(entry.get('id'))] = entry.get('description')
        return ignore 

def linked_packages(self, package, project=None):
        if not project:
            project = self.project

        url = self.makeurl(['source', project, package], { 'cmd': 'showlinked' })
        f = http_POST(url)
        root = ET.parse(f).getroot()
        result = []
        for package in root.findall('package'):
            result.append({'project': package.get('project'), 'package': package.get('name')})
        return result 

def find_devel_project_from_adi_frozenlinks(self, prj):
        try:
            url = self.makeurl(['source', prj, '_project', '_frozenlinks'], {'meta': '1'})
            root = ET.parse(http_GET(url)).getroot()
        except HTTPError as e:
            if e.code == 404:
                return None
        meta = self.get_prj_pseudometa(prj)
        # the first package's devel project is good enough
        return devel_project_get(self.apiurl, self.project, meta['requests'][0].get('package'))[0] 

def get_staging_projects(self):
        """
        Get all current running staging projects
        :return list of known staging projects
        """

        result = []
        url = self.makeurl(['staging', self.project, 'staging_projects'])
        status = ET.parse(self.retried_GET(url)).getroot()
        for project in status.findall('staging_project'):
            result.append(project.get('name'))
        return result 

def _get_staged_requests(self):
        """
        Get all requests that are already staged
        :return dict of staged requests with their project and srid
        """

        packages_staged = {}
        url = self.makeurl(['staging', self.project, 'staging_projects'], { 'requests': 1 })
        status = ET.parse(self.retried_GET(url)).getroot()
        for prj in status.findall('staging_project'):
            for req in prj.findall('./staged_requests/request'):
                packages_staged[req.get('package')] = {'prj': prj.get('name'), 'rq_id': req.get('id')}

        return packages_staged 

def mirror(apiurl, project, repository, arch):
    """Call bs_mirrorfull script to mirror packages."""
    directory = os.path.join(CACHEDIR, project, repository, arch)

    if not os.path.exists(directory):
        os.makedirs(directory)

    meta = ETL.parse(http_GET('{}/public/source/{}/_meta'.format(apiurl, project))).getroot()
    repotag = meta.xpath("/project/repository[@name='{}']".format(repository))[0]
    if arch not in repotag.xpath("./arch/text()"):
        # Arch not in this project, skip mirroring
        return directory

    download = repotag.xpath("./download[@arch='{}']".format(arch))
    if download is not None and len(download) > 0:
        if len(download) > 1:
            raise Exception('Multiple download urls unsupported')
        repotype = download[0].get('repotype')
        if repotype != 'rpmmd':
            raise Exception('repotype {} not supported'.format(repotype))
        return mirrorRepomd(directory, download[0].get('url'))

    script = os.path.join(SCRIPT_PATH, '..', 'bs_mirrorfull')
    path = '/'.join((project, repository, arch))
    logger.info('mirroring {}'.format(path))
    url = '{}/public/build/{}'.format(apiurl, path)
    p = subprocess.run(['perl', script, '--nodebug', url, directory])

    if p.returncode:
        raise Exception('failed to mirror {}'.format(path))

    return directory 

def check_leaf_package(self, project, repository, arch, package):
        url = makeurl(self.apiurl, ['build', project, repository, arch, package, '_buildinfo'])
        root = ET.parse(http_GET(url)).getroot()
        subpacks = set()
        for sp in root.findall('subpack'):
            subpacks.add(sp.text)
        build_deps = dict()
        for bd in root.findall('bdep'):
            if bd.get('notmeta') == '1':
                continue
            build_deps[bd.get('name')] = bd.get('version') + '-' + bd.get('release')
        return subpacks, build_deps 

def find_request_project(self, source_project, newcand):
        """
        Look up the source project by its name and return the SR#(s)
        :param source_project: name of the source project
        :param newcand: the review state of staging-group must be new
        """

        query = 'types=submit,delete&states=new,review&project={}&view=collection'.format(self.api.project)
        url = makeurl(self.api.apiurl, ['request'], query)
        f = http_GET(url)
        root = ET.parse(f).getroot()

        ret = None
        for sr in root.findall('request'):
            # ensure staging tool don't picks the processed request again
            if newcand:
                staging_group_states = [review.get('state') for review in sr.findall('review') if review.get('by_group') == self.api.cstaging_group]
                if 'new' not in staging_group_states:
                    continue
            for act in sr.findall('action'):
                src = act.find('source')
                if src is not None and src.get('project') == source_project:
                    request = int(sr.attrib['id'])
                    state = sr.find('state').get('name')
                    self.srs[request] = {'project': self.api.project, 'state': state}
                    ret = True

        return ret 

def find_request_package(self, package):
        """
        Look up the package by its name and return the SR#
        :param package: name of the package
        """

        query = 'types=submit,delete&states=new,review&project={}&view=collection&package={}'
        query = query.format(self.api.project, quote(package))
        url = makeurl(self.api.apiurl, ['request'], query)
        f = http_GET(url)

        root = ET.parse(f).getroot()

        requests = []
        for sr in root.findall('request'):
            # Check the target matches - OBS query is case insensitive, but OBS is not
            rq_target = sr.find('action').find('target')
            if package != rq_target.get('package') or self.api.project != rq_target.get('project'):
                continue

            request = sr.get('id')
            state = sr.find('state').get('name')

            self.srs[int(request)] = {'project': self.api.project, 'state': state}
            requests.append(request)

        if len(requests) > 1:
            msg = 'There are multiple requests for package "{}": {}'
            msg = msg.format(package, ', '.join(requests))
            raise oscerr.WrongArgs(msg)

        request = int(requests[0]) if requests else None
        return request 

def request_state_change(apiurl, request_id, state):
    query = { 'newstate': state, 'cmd': 'changestate' }
    url = makeurl(apiurl, ['request', request_id], query)
    return ETL.parse(http_POST(url)).getroot().get('code') 

def create_change_devel_request(apiurl, source_project, source_package,
                                target_project, target_package=None, message=None):
    if not message:
        message = message_suffix('created')

    request = ETL.Element('request')

    state = ETL.Element('state')
    state.set('name', 'new')
    request.append(state)

    description = ETL.Element('description')
    description.text = message
    request.append(description)

    action = ETL.Element('action')
    action.set('type', 'change_devel')
    request.append(action)

    source = ETL.Element('source')
    source.set('project', source_project)
    source.set('package', source_package)
    action.append(source)

    target = ETL.Element('target')
    target.set('project', target_project)
    target.set('package', target_package)
    action.append(target)

    url = makeurl(apiurl, ['request'], {'cmd': 'create'})
    root = ETL.parse(http_POST(url, data=ETL.tostring(request))).getroot()
    return root.get('id') 

def create_delete_request(apiurl, target_project, target_package=None, message=None):
    if not message:
        message = message_suffix('created')

    request = ETL.Element('request')

    state = ETL.Element('state')
    state.set('name', 'new')
    request.append(state)

    description = ETL.Element('description')
    description.text = message
    request.append(description)

    action = ETL.Element('action')
    action.set('type', 'delete')
    request.append(action)

    target = ETL.Element('target')
    target.set('project', target_project)
    if target_package:
        target.set('package', target_package)
    action.append(target)

    url = makeurl(apiurl, ['request'], {'cmd': 'create'})
    root = ETL.parse(http_POST(url, data=ETL.tostring(request))).getroot()
    return root.get('id')

# Should exist within osc.core like create_submit_request(), but rather it was
# duplicated in osc.commandline. 

def search(apiurl, path, xpath, query={}):
    query['match'] = xpath
    url = makeurl(apiurl, ['search', path], query)
    return ETL.parse(http_GET(url)).getroot()