update gentoo repo handling

Author: furlongm

client/patchman-client

@@ -574,24 +574,52 @@ get_repos() {
     fi
 
     # Gentoo
-    if [ "${os}" == "Gentoo" ] ; then
+    if [[ "${os}" =~ "Gentoo" ]] ; then
         if [ ${verbose} == 1 ] ; then
             echo 'Finding portage repos...'
         fi
-        declare -A repos
-        repos[gentoo]='-1000'
-        repos_conf=$(awk '/\[/{prefix=$0; next} $1{print prefix $0}' /etc/portage/repos.conf/*.conf | grep '^\[')
-        for stanza in ${repos_conf} ; do
-            repo=$(echo ${stanza} | cut -d ']' -f 1 | sed -e 's/\[//')
-            rhs=$(echo ${stanza} | cut -d ']' -f 2 | grep -v '^#')
-            if [[ ${rhs} =~ "priority" ]] ; then
-                priority=$(echo ${rhs} | sed -e 's/^ *priority *= *//')
-                repos[${repo}]+=${priority}
-                unset priority
+        declare -A repo_info
+        repos_output=$(portageq repos_config /)
+        repo_name=""
+        priority=""
+        sync_uri=""
+
+        while IFS= read -r line; do
+            # if the line starts with a section header (e.g., [gentoo], [guru]), it's the repo name
+            if [[ "${line}" =~ ^\[(.*)\] ]]; then
+                # if we already have a repo_name, save the previous entry
+                if [[ -n "${repo_name}" && -n "${sync_uri}" ]]; then
+                    repo_info["${repo_name}"]="${priority},${sync_uri}"
+                fi
+                # else start new repo parsing, resetting vars
+                repo_name="${BASH_REMATCH[1]}"
+                priority=""
+                sync_uri=""
             fi
-        done
-        for r in "${!repos[@]}"; do
-            echo "'gentoo' 'Gentoo Linux ${r} ${host_arch}' '${r}' '${repos[${r}]}'" >> "${tmpfile_rep}"
+
+            # if the line contains "priority", extract the value, 0 if it doesnt exist
+            if [[ "${line}" =~ "priority" ]]; then
+                priority=$(echo "${line}" | cut -d'=' -f2 | xargs)
+            fi
+
+            # if the line contains "sync-uri", extract the value
+            if [[ "${line}" =~ "sync-uri" ]]; then
+                sync_uri=$(echo "${line}" | cut -d'=' -f2 | xargs)
+            fi
+        done <<< "${repos_output}"
+
+        # save the last repository entry if it's available
+        if [[ -n "${repo_name}" && -n "${sync_uri}" ]]; then
+            repo_info["${repo_name}"]="${priority},${sync_uri}"
+        fi
+
+        for repo in "${!repo_info[@]}"; do
+            priority=$(echo ${repo_info[$repo]} | cut -d',' -f1)
+            sync_uri=$(echo ${repo_info[$repo]} | cut -d',' -f2)
+            if [ "${priority}" == "" ] ; then
+                priority=0
+            fi
+            echo "'gentoo' 'Gentoo Linux ${repo} Repo ${host_arch}' '${repo}' '${priority}' '${sync_uri}'" >> "${tmpfile_rep}"
         done
     fi
 

reports/utils.py

@@ -34,10 +34,6 @@
 def process_repos(report, host):
     """ Processes the quoted repos string sent with a report
     """
-    if host.osvariant.name.startswith('Gentoo'):
-        gentoo_repo = Repository.objects.get(repo_id='gentoo')
-        host_repos = HostRepo.objects.filter(host=host)
-        hostrepo, c = host_repos.get_or_create(host=host, repo=gentoo_repo)
     if report.repos:
         repo_ids = []
         host_repos = HostRepo.objects.filter(host=host)
@@ -224,6 +220,7 @@ def process_repo(repo, arch):
         r_type = Repository.GENTOO
         r_id = repo.pop(2)
         r_priority = repo[2]
+        arch = 'any'
 
     if repo[1]:
         r_name = repo[1]
@@ -232,6 +229,8 @@ def process_repo(repo, arch):
 
     unknown = []
     for r_url in repo[3:]:
+        if r_type == Repository.GENTOO and r_url.startswith('rsync'):
+            r_url = 'https://api.gentoo.org/mirrors/distfiles.xml'
         try:
             mirror = Mirror.objects.get(url=r_url.strip('/'))
         except Mirror.DoesNotExist:
@@ -360,19 +359,6 @@ def process_gentoo_package(package, name, category, repo):
     package.category = category
     package.save()
 
-    repo_arch, created = MachineArchitecture.objects.get_or_create(name='any')
-    repo_name = 'Gentoo Linux'
-    gentoo_repo = get_or_create_repo(repo_name, repo_arch, Repository.GENTOO, repo)
-
-    if repo == 'gentoo':
-        url = 'https://api.gentoo.org/mirrors/distfiles.xml'
-    else:
-        # this may not be correct. the urls are hardcoded anyway in repos/utils.py
-        # need to figure out a better way to determine which repo/repo url to use
-        url = 'https://api.gentoo.org/overlays/repositories.xml'
-    mirror, c = Mirror.objects.get_or_create(repo=gentoo_repo, url=url, mirrorlist=True)
-    MirrorPackage.objects.create(mirror=mirror, package=package)
-
 
 def get_arch(arch):
     """ Get or create MachineArchitecture from arch

repos/repo_types/gentoo.py

@@ -16,15 +16,14 @@
 
 import git
 import os
-import re
 import shutil
 import tarfile
 import tempfile
 from defusedxml import ElementTree
 from fnmatch import fnmatch
 from io import BytesIO
+from pathlib import Path
 
-from arch.models import PackageArchitecture
 from packages.models import PackageString
 from packages.utils import find_evr
 from patchman.signals import info_message, warning_message, error_message, pbar_start, pbar_update
@@ -37,13 +36,70 @@ def refresh_gentoo_main_repo(repo):
     """
     mirrors = get_gentoo_mirror_urls()
     add_mirrors_from_urls(repo, mirrors)
+    ts = get_datetime_now()
+    for mirror in repo.mirror_set.filter(mirrorlist=False, refresh=True, enabled=True):
+        if mirror.url == 'https://api.gentoo.org/mirrors/distfiles.xml':
+            mirror.mirrorlist = True
+            mirror.save()
+            continue
+
+        res = get_url(mirror.url + '.md5sum')
+        data = fetch_content(res, 'Fetching Repo checksum')
+        if data is None:
+            mirror.fail()
+            continue
+
+        checksum = data.decode().split()[0]
+        if checksum is None:
+            mirror.fail()
+            continue
+
+        if mirror.packages_checksum == checksum:
+            text = 'Mirror checksum has not changed, not refreshing Package metadata'
+            warning_message.send(sender=None, text=text)
+            continue
+
+        res = get_url(mirror.url)
+        mirror.last_access_ok = response_is_valid(res)
+        if not mirror.last_access_ok:
+            mirror.fail()
+            continue
+
+        data = fetch_content(res, 'Fetching Repo data')
+        if data is None:
+            mirror.fail()
+            continue
+        extracted = extract(data, mirror.url)
+        info_message.send(sender=None, text=f'Found Gentoo Repo - {mirror.url}')
+
+        computed_checksum = get_checksum(data, Checksum.md5)
+        if not mirror_checksum_is_valid(computed_checksum, checksum, mirror, 'package'):
+            mirror.fail()
+            continue
+        else:
+            mirror.packages_checksum = checksum
+
+        packages = extract_gentoo_packages(mirror, extracted)
+        if packages:
+            update_mirror_packages(mirror, packages)
+
+        mirror.timestamp = ts
+        mirror.save()
 
 
 def refresh_gentoo_overlay_repo(repo):
     """ Refresh all mirrors of a Gentoo overlay repo
     """
     mirrors = get_gentoo_overlay_mirrors(repo.repo_id)
     add_mirrors_from_urls(repo, mirrors)
+    ts = get_datetime_now()
+    for mirror in repo.mirror_set.filter(mirrorlist=False, refresh=True, enabled=True):
+        # FIXME: need to check for failure
+        packages = extract_gentoo_overlay_packages(mirror)
+        if packages:
+            update_mirror_packages(mirror, packages)
+        mirror.timestamp = ts
+        mirror.save()
 
 
 def get_gentoo_ebuild_keywords(content):
@@ -82,7 +138,10 @@ def get_gentoo_ebuild_keywords(content):
                 continue
             keywords.add(keyword)
         break
-    return keywords
+    if keywords:
+        return keywords
+    else:
+        return default_keywords
 
 
 def get_gentoo_overlay_mirrors(repo_name):
@@ -157,7 +216,25 @@ def extract_gentoo_ebuilds(data):
         for member in tar.getmembers():
             if member.isfile() and member.name.endswith('ebuild') and not member.name.endswith('skel.ebuild'):
                 file_content = tar.extractfile(member).read()
-                extracted_ebuilds[member.name] = file_content
+                full_path = Path(member.name)
+                ebuild_path = Path(*full_path.parts[1:])
+                extracted_ebuilds[str(ebuild_path)] = file_content
+    return extracted_ebuilds
+
+
+def extract_gentoo_overlay_ebuilds(t):
+    """ Extract ebuilds from a Gentoo overlay tarball
+    """
+    extracted_ebuilds = {}
+    for root, dirs, files in os.walk(t):
+        for name in files:
+            if fnmatch(name, '*.ebuild'):
+                package_name = root.replace(t + '/', '')
+                if len(package_name.split('/')) > 2:
+                    continue
+                with open(os.path.join(root, name), 'rb') as f:
+                    content = f.read()
+                extracted_ebuilds[f'{package_name}/{name}'] = content
     return extracted_ebuilds
 
 
@@ -175,14 +252,14 @@ def extract_gentoo_packages_from_ebuilds(extracted_ebuilds):
         return
 
     packages = set()
-    flen = len(extracted_ebuilds)
-    pbar_start.send(sender=None, ptext=f'Processing {flen} ebuilds', plen=flen)
+    elen = len(extracted_ebuilds)
+    pbar_start.send(sender=None, ptext=f'Processing {elen} ebuilds', plen=elen)
     for i, (path, content) in enumerate(extracted_ebuilds.items()):
         pbar_update.send(sender=None, index=i + 1)
         components = path.split(os.sep)
-        category = components[1]
-        name = components[2]
-        evr = components[3].replace(f'{name}-', '').replace('.ebuild', '')
+        category = components[0]
+        name = components[1]
+        evr = components[2].replace(f'{name}-', '').replace('.ebuild', '')
         epoch, version, release = find_evr(evr)
         arches = get_gentoo_ebuild_keywords(content)
         for arch in arches:
@@ -205,78 +282,19 @@ def extract_gentoo_overlay_packages(mirror):
     """ Extract packages from gentoo overlay repo
     """
     t = tempfile.mkdtemp()
-    git.Repo.clone_from(mirror.url, t, branch='master', depth=1)
+    info_message.send(sender=None, text=f'Extracting Gentoo packages from {mirror.url}')
+    git.Repo.clone_from(mirror.url, t, depth=1)
     packages = set()
-    arch, c = PackageArchitecture.objects.get_or_create(name='any')
-    for root, dirs, files in os.walk(t):
-        for name in files:
-            if fnmatch(name, '*.ebuild'):
-                full_name = root.replace(t + '/', '')
-                p_category, p_name = full_name.split('/')
-                m = re.match(fr'{p_name}-(.*)\.ebuild', name)
-                if m:
-                    p_evr = m.group(1)
-                epoch, version, release = find_evr(p_evr)
-                package = PackageString(
-                    name=p_name.lower(),
-                    epoch=epoch,
-                    version=version,
-                    release=release,
-                    arch=arch,
-                    packagetype='G',
-                    category=p_category,
-                )
-                packages.add(package)
+    extracted_ebuilds = extract_gentoo_overlay_ebuilds(t)
     shutil.rmtree(t)
+    packages = extract_gentoo_packages_from_ebuilds(extracted_ebuilds)
     return packages
 
 
 def refresh_gentoo_repo(repo):
     """ Refresh a Gentoo repo
     """
     if repo.repo_id == 'gentoo':
-        repo_type = 'main'
         refresh_gentoo_main_repo(repo)
     else:
         refresh_gentoo_overlay_repo(repo)
-        repo_type = 'overlay'
-    ts = get_datetime_now()
-    for mirror in repo.mirror_set.filter(mirrorlist=False, refresh=True, enabled=True):
-        res = get_url(mirror.url + '.md5sum')
-        data = fetch_content(res, 'Fetching Repo checksum')
-        if data is None:
-            mirror.fail()
-            continue
-        checksum = data.decode().split()[0]
-        if checksum is None:
-            mirror.fail()
-            continue
-        if mirror.packages_checksum == checksum:
-            text = 'Mirror checksum has not changed, not refreshing Package metadata'
-            warning_message.send(sender=None, text=text)
-            continue
-        res = get_url(mirror.url)
-        mirror.last_access_ok = response_is_valid(res)
-        if mirror.last_access_ok:
-            data = fetch_content(res, 'Fetching Repo data')
-            if data is None:
-                mirror.fail()
-                continue
-            extracted = extract(data, mirror.url)
-            text = f'Found Gentoo Repo - {mirror.url}'
-            info_message.send(sender=None, text=text)
-            computed_checksum = get_checksum(data, Checksum.md5)
-            if not mirror_checksum_is_valid(computed_checksum, checksum, mirror, 'package'):
-                continue
-            else:
-                mirror.packages_checksum = checksum
-            if repo_type == 'main':
-                packages = extract_gentoo_packages(mirror, extracted)
-            elif repo_type == 'overlay':
-                packages = extract_gentoo_overlay_packages(mirror)
-            mirror.timestamp = ts
-            if packages:
-                update_mirror_packages(mirror, packages)
-        else:
-            mirror.fail()
-        mirror.save()

repos/utils.py

@@ -177,7 +177,7 @@ def add_mirrors_from_urls(repo, mirror_urls):
         from repos.models import Mirror
         # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml?
         # that is what metalink urls return now
-        m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').rstrip('repodata/repomd.xml'))
+        m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', ''))
         if c:
             text = f'Added Mirror - {mirror_url}'
             info_message.send(sender=None, text=text)