celery and task updates

Author: furlongm

errata/tasks.py

@@ -14,22 +14,11 @@
 # You should have received a copy of the GNU General Public License
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
-from datetime import timedelta
-
-from django.conf import settings
-
 from celery import shared_task
-from patchman.celery import app
 
 from errata.models import Erratum
 from security.tasks import update_cves, update_cwes
 
-app.conf.beat_schedule = {
-    'update-errata-cves-cwes-every-6-hours': {
-        'task': 'tasks.update_errata',
-        'schedule': timedelta(hours=6),
-    },
-}
 
 @shared_task
 def update_erratum(erratum):

etc/patchman/local_settings.py

@@ -29,19 +29,43 @@
 ALLOWED_HOSTS = ['127.0.0.1', '*']
 
 # Maximum number of mirrors to add or refresh per repo
-MAX_MIRRORS = 5
+MAX_MIRRORS = 3
 
-# Number of days to wait before notifying users that a host has not reported
+# Number of days to wait before raising that a host has not reported
 DAYS_WITHOUT_REPORT = 14
 
 # Whether to run patchman under the gunicorn web server
 RUN_GUNICORN = False
 
-# Enable redis caching for 30 seconds
 CACHES = {
     'default': {
-        'BACKEND': 'django.core.cache.backends.redis.RedisCache',
-        'LOCATION': 'redis://127.0.0.1:6379',
-        'TIMEOUT': 30,
+        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
     }
 }
+
+# Uncomment to enable redis caching for e.g. 30 seconds
+# Note that the UI results may be out of date for this amount of time
+# CACHES = {
+#     'default': {
+#         'BACKEND': 'django.core.cache.backends.redis.RedisCache',
+#         'LOCATION': 'redis://127.0.0.1:6379',
+#         'TIMEOUT': 30,
+#     }
+# }
+
+from datetime import timedelta        # noqa
+from celery.schedules import crontab  # noqa
+CELERY_BEAT_SCHEDULE = {
+    'process_all_unprocessed_reports': {
+        'task': 'reports.tasks.process_reports',
+        'schedule': crontab(minute='*/5'),
+     },
+    'refresh_repos_daily': {
+        'task': 'tasks.refresh_repos',
+        'schedule': crontab(hour=6, minute=00),
+    },
+    'update_errata_cves_cwes_every_12_hours': {
+        'task': 'tasks.update_errata',
+        'schedule': timedelta(hours=12),
+    },
+}

patchman/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2021 Marcus Furlong <furlongm@gmail.com>
+# Copyright 2013-2025 Marcus Furlong <furlongm@gmail.com>
 #
 # This file is part of Patchman.
 #

patchman/settings.py

@@ -4,10 +4,6 @@
 import site
 import sys
 
-#import django
-#from django.utils.encoding import smart_str
-#django.utils.encoding.smart_text = smart_str
-
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 

reports/tasks.py

@@ -15,24 +15,16 @@
 # You should have received a copy of the GNU General Public License
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
-from django.conf import settings
+from celery import shared_task
 
-from reports.models import Report
+from django.db.utils import OperationalError
 
-from celery import shared_task
-from celery.schedules import crontab
-from patchman.celery import app
+from reports.models import Report
 
-app.conf.beat_schedule = {
-    'process-reports': {
-        'task': 'reports.tasks.process_reports',
-        'schedule': crontab(minute='*/5'),
-    },
-}
 
-@shared_task
-def process_report(report_id):
-    report = Report.objects.get(report_id)
+@shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5})
+def process_report(self, report_id):
+    report = Report.objects.get(id=report_id)
     report.process()
 
 

reports/urls.py

@@ -26,5 +26,5 @@
     path('upload/', views.upload),
     path('<int:report_id>/', views.report_detail, name='report_detail'),
     path('<int:report_id>/delete/', views.report_delete, name='report_delete'),
-    path('<int:report_id>/process/', views.report_process, name='report_process'),  # noqa
+    path('<int:report_id>/process/', views.report_process, name='report_process'),
 ]

repos/models.py

@@ -23,8 +23,8 @@
 from packages.models import Package
 from util import has_setting_of_type
 
-from repos.utils import refresh_deb_repo, refresh_rpm_repo, \
-    refresh_arch_repo, refresh_gentoo_repo, update_mirror_packages
+from repos.utils import refresh_deb_repo, refresh_rpm_repo, refresh_arch_repo, refresh_gentoo_repo, \
+     update_mirror_packages
 from patchman.signals import info_message, warning_message, error_message
 
 
@@ -96,8 +96,7 @@ def refresh(self, force=False):
             elif self.repotype == Repository.GENTOO:
                 refresh_gentoo_repo(self)
             else:
-                text = 'Error: unknown repo type for repo '
-                text += f'{self.id}: {self.repotype}'
+                text = 'Error: unknown repo type for repo {self.id}: {self.repotype}'
                 error_message.send(sender=None, text=text)
         else:
             text = 'Repo requires certificate authentication, not updating'

repos/tasks.py

@@ -14,25 +14,16 @@
 # You should have received a copy of the GNU General Public License
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
-from django.conf import settings
-
-from repos.models import Repository, Mirror
-
 from celery import shared_task
-from celery.schedules import crontab
-from patchman.celery import app
 
-app.conf.beat_schedule = {
-    'refresh-repos-every-day': {
-        'task': 'tasks.refresh_repos',
-        'schedule': crontab(hour=6, minute=00),
-    },
-}
+from repos.models import Repository
+
 
 @shared_task
-def refresh_repo(force=False):
+def refresh_repo(repo_id, force=False):
     """ Refresh metadata for a single repo
     """
+    repo = Repository.objects.get(id=repo_id)
     repo.refresh(force)
 
 

repos/urls.py

@@ -24,12 +24,13 @@
 urlpatterns = [
     path('', views.repo_list, name='repo_list'),
     path('<int:repo_id>/', views.repo_detail, name='repo_detail'),
-    path('<int:repo_id>/toggle_enabled/', views.repo_toggle_enabled, name='repo_toggle_enabled'),  # noqa
-    path('<int:repo_id>/toggle_security/', views.repo_toggle_security, name='repo_toggle_security'),  # noqa
+    path('<int:repo_id>/toggle_enabled/', views.repo_toggle_enabled, name='repo_toggle_enabled'),
+    path('<int:repo_id>/toggle_security/', views.repo_toggle_security, name='repo_toggle_security'),
     path('<int:repo_id>/edit/', views.repo_edit, name='repo_edit'),
     path('<int:repo_id>/delete/', views.repo_delete, name='repo_delete'),
+    path('<int:repo_id>/refresh/', views.repo_refresh, name='repo_refresh'),
     path('mirrors/', views.mirror_list, name='mirror_list'),
-    path('mirrors/mirror/<int:mirror_id>/', views.mirror_detail, name='mirror_detail'),  # noqa
-    path('mirrors/mirror/<int:mirror_id>/edit/', views.mirror_edit, name='mirror_edit'),  # noqa
-    path('mirrors/mirror/<int:mirror_id>/delete/', views.mirror_delete, name='mirror_delete'),  # noqa
+    path('mirrors/mirror/<int:mirror_id>/', views.mirror_detail, name='mirror_detail'),
+    path('mirrors/mirror/<int:mirror_id>/edit/', views.mirror_edit, name='mirror_edit'),
+    path('mirrors/mirror/<int:mirror_id>/delete/', views.mirror_delete, name='mirror_delete'),
 ]

repos/utils.py

@@ -45,7 +45,7 @@
     info_message, warning_message, error_message, debug_message
 
 
-def get_or_create_repo(r_name, r_arch, r_type):
+def get_or_create_repo(r_name, r_arch, r_type, r_id=None):
     """ Get or create a Repository object. Returns the object. Returns None if
         it cannot get or create the object.
     """
@@ -64,6 +64,9 @@ def get_or_create_repo(r_name, r_arch, r_type):
     except DatabaseError as e:
         error_message.send(sender=None, text=e)
     if repository:
+        if r_id:
+            repository.repo_id = r_id
+            repository.save()
         return repository
 
 

security/tasks.py

@@ -14,33 +14,33 @@
 # You should have received a copy of the GNU General Public License
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
-from django.conf import settings
+from celery import shared_task
 
 from security.models import CVE, CWE
 
-from celery import shared_task
-from patchman.celery import app
-
 
 @shared_task
 def update_cve(cve):
     """ Task to update a CVE
     """
     cve.update()
 
+
 @shared_task
 def update_cves():
     """ Task to update all CVEs
     """
     for cve in CVE.objects.all():
         update_cve.delay(cve)
 
+
 @shared_task
 def update_cwe(cwe):
     """ Task to update a CWE
     """
     cwe.update()
 
+
 @shared_task
 def update_cwes():
     """ Task to update all CWEa