Implement analyst evaluation mode

etc/db-config.yaml

@@ -118,12 +118,13 @@
             type: int
             default_value: 1
             public: false
-            description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging and is typically used when running as analyst system.
+            description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging. Analyst mode tries to judge only interesting testcases.
             options:
                 1: Lazy
                 2: Full judging
                 3: Only on request
-            regex: /^[123]$/
+                4: Analyst mode
+            regex: /^[1234]$/
             error_message: A value between 1 and 3 is required.
         -   name: judgehost_warning
             type: int

webapp/src/Controller/API/JudgehostController.php

@@ -1083,7 +1083,9 @@ private function addSingleJudgingRun(
                     throw new BadMethodCallException('internal bug: the evaluated result changed during judging');
                 }
 
-                if ($lazyEval !== DOMJudgeService::EVAL_FULL) {
+                if ($lazyEval === DOMJudgeService::EVAL_ANALYST) {
+                    // Explicitly do not update priorities or cancel activated tasks.
+                } elseif ($lazyEval !== DOMJudgeService::EVAL_FULL) {
                     // We don't want to continue on this problem, even if there's spare resources.
                     $this->em->getConnection()->executeStatement(
                         'UPDATE judgetask SET valid=0, priority=:priority'

webapp/src/Controller/Jury/JudgeRemainingTrait.php

@@ -5,6 +5,7 @@
 use App\Entity\JudgeTask;
 use App\Entity\Judging;
 use App\Entity\QueueTask;
+use App\Service\DOMJudgeService;
 
 trait JudgeRemainingTrait
 {
@@ -13,13 +14,18 @@ trait JudgeRemainingTrait
      */
     protected function judgeRemainingJudgings(array $judgings): void
     {
+        $lazyEval = $this->config->get('lazy_eval_results');
         $inProgress = [];
         $alreadyRequested = [];
         $invalidJudgings = [];
         $numRequested = 0;
+
+        // In analyst mode, when explicitly requested judging the remaining tasks is most important.
+        $priority = $lazyEval === DOMJudgeService::EVAL_ANALYST ? JudgeTask::PRIORITY_HIGH : JudgeTask::PRIORITY_LOW;
+
         foreach ($judgings as $judging) {
             $judgingId = $judging->getJudgingid();
-            if ($judging->getResult() === null) {
+            if ($judging->getResult() === null && $lazyEval !== DOMJudgeService::EVAL_ANALYST) {
                 $inProgress[] = $judgingId;
             } elseif ($judging->getJudgeCompletely()) {
                 $alreadyRequested[] = $judgingId;
@@ -40,7 +46,7 @@ protected function judgeRemainingJudgings(array $judgings): void
 
                 $queueTask = new QueueTask();
                 $queueTask->setJudging($judging)
-                    ->setPriority(JudgeTask::PRIORITY_LOW)
+                    ->setPriority($priority)
                     ->setTeam($submission->getTeam())
                     ->setTeamPriority((int)$submission->getSubmittime())
                     ->setStartTime(null);

webapp/src/Controller/Jury/SubmissionController.php

@@ -583,6 +583,7 @@ public function viewAction(
             'version_warnings' => [],
             'isMultiPassProblem' => $submission->getProblem()->isMultipassProblem(),
             'thumbnailSize' => $this->config->get('thumbnail_size'),
+            'isAnalystMode' => $this->config->get('lazy_eval_results') === DOMJudgeService::EVAL_ANALYST,
         ];
 
         if ($selectedJudging === null) {

webapp/src/Service/DOMJudgeService.php

@@ -74,6 +74,7 @@ class DOMJudgeService
     final public const EVAL_LAZY = 1;
     final public const EVAL_FULL = 2;
     final public const EVAL_DEMAND = 3;
+    final public const EVAL_ANALYST = 4;
 
     // Regex external identifiers must adhere to. Note that we are not checking whether it
     // does not start with a dot or dash or ends with a dot. We could but it would make the
@@ -1184,7 +1185,7 @@ public function unblockJudgeTasks(): void
         }
     }
 
-    public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0): void
+    public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0, bool $valid = true): void
     {
         $submission = $judging->getSubmission();
         $problem    = $submission->getContestProblem();
@@ -1197,7 +1198,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas
             return;
         }
 
-        $this->actuallyCreateJudgetasks($priority, $judging, $overshoot);
+        $this->actuallyCreateJudgetasks($priority, $judging, $overshoot, $valid);
 
         $team = $submission->getTeam();
         $result = $this->em->createQueryBuilder()
@@ -1215,7 +1216,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas
 
         // Teams that submit frequently slow down the judge queue but should not be able to starve other teams of their
         // deserved and timely judgement.
-        // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definiition of "recent"
+        // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definition of "recent"
         // includes all submissions that have been placed at a virtual time (including penalty) more recent than 60s
         // ago. This is done in order to avoid punishing teams who submit while their submissions are stuck in the queue
         // for other reasons, for example an internal error for a problem or language.
@@ -1586,19 +1587,20 @@ private function allowJudge(ContestProblem $problem, Submission $submission, Lan
         return !$evalOnDemand;
     }
 
-    private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0): void
+    private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0, bool $valid = true): void
     {
         $submission = $judging->getSubmission();
         $problem    = $submission->getContestProblem();
         // We use a mass insert query, since that is way faster than doing a separate insert for each testcase.
-        // We first insert judgetasks, then select their ID's and finally insert the judging runs.
+        // We first insert judgetasks, then select their IDs and finally insert the judging runs.
 
         // Step 1: Create the template for the judgetasks.
         $compileExecutable = $submission->getLanguage()->getCompileExecutable()->getImmutableExecutable();
         $judgetaskInsertParams = [
             ':type' => JudgeTaskType::JUDGING_RUN,
             ':submitid' => $submission->getSubmitid(),
             ':priority' => $priority,
+            ':valid' => $valid ? 1 : 0,
             ':jobid' => $judging->getJudgingid(),
             ':uuid' => $judging->getUuid(),
             ':compile_script_id' => $compileExecutable->getImmutableExecId(),

webapp/src/Service/ExternalContestSourceService.php

@@ -27,8 +27,10 @@
 use App\Entity\ExternalJudgement;
 use App\Entity\ExternalRun;
 use App\Entity\ExternalSourceWarning;
+use App\Entity\JudgeTask;
 use App\Entity\Language;
 use App\Entity\Problem;
+use App\Entity\QueueTask;
 use App\Entity\Submission;
 use App\Entity\SubmissionSource;
 use App\Entity\Team;
@@ -1777,13 +1779,13 @@ protected function importRun(Event $event, EventData $data): void
         }
 
         // First, load the external run.
+        $persist = false;
         $run     = $this->em
             ->getRepository(ExternalRun::class)
             ->findOneBy([
                             'contest'    => $this->getSourceContest(),
                             'externalid' => $runId,
                         ]);
-        $persist = false;
         if (!$run) {
             $run = new ExternalRun();
             $run
@@ -1858,9 +1860,59 @@ protected function importRun(Event $event, EventData $data): void
         if ($persist) {
             $this->em->persist($run);
         }
+
+        $lazyEval = $this->config->get('lazy_eval_results');
+        if ($lazyEval === DOMJudgeService::EVAL_ANALYST) {
+            // Check if we want to judge this testcase locally to provide useful information for analysts
+            $priority = $this->getAnalystRunPriority($run);
+            if ($priority !== null) {
+                // Make the judgetask valid and assign running priority if no judgehost has picked it up yet.
+                $submission = $externalJudgement->getSubmission();
+                $this->em->createQueryBuilder()
+                        ->update(JudgeTask::class, 'jt')
+                        ->set('jt.valid', true)
+                        ->set('jt.priority', $priority)
+                        ->andWhere('jt.testcase_id = :testcase_id')
+                        ->andWhere('jt.submission = :submission')
+                        ->andWhere('jt.judgehost IS NULL')
+                        ->setParameter('testcase_id', $testcase->getTestcaseid())
+                        ->setParameter('submission', $submission)
+                        ->getQuery()
+                        ->execute();
+
+                $queueTask = new QueueTask();
+                $queueTask->setJudging($submission->getJudgings()->first())
+                    ->setPriority($priority)
+                    ->setTeam($submission->getTeam())
+                    ->setTeamPriority((int)$submission->getSubmittime())
+                    ->setStartTime(null);
+                $this->em->persist($queueTask);
+            }
+        }
+
         $this->em->flush();
     }
 
+    /**
+     * Checks if this run is interesting to judge locally for more analysis results.
+     * @param ExternalRun $run
+     * @return int The judging priority if it should be run locally, null otherwise.
+     */
+    protected function getAnalystRunPriority(ExternalRun $run): int | null {
+        return match ($run->getResult()) {
+            // We will not get any new useful information for TLE testcases, while they take a lot of judgedaemon time.
+            'timelimit' => null,
+            // We often do not get new useful information for judging correct testcases.
+            'correct' => null,
+            // Wrong answers are interesting for the analysts, assign a high priority but below manual judging.
+            'wrong-answer' => JudgeTask::PRIORITY_HIGH + 1,
+            // Compile errors could be interesting to see what went wrong, assign a low priority.
+            'compiler-error' => JudgeTask::PRIORITY_LOW - 1,
+            // Otherwise, judge with normal priority.
+            default => 0,
+        };
+    }
+
     protected function processPendingEvents(string $type, string|int $id): void
     {
         // Process pending events.

webapp/src/Service/SubmissionService.php

@@ -732,8 +732,15 @@ public function submitSolution(
             // This is so that we can use the submitid/judgingid below.
             $this->em->flush();
 
-            $this->dj->maybeCreateJudgeTasks($judging,
-                $source === SubmissionSource::PROBLEM_IMPORT ? JudgeTask::PRIORITY_LOW : JudgeTask::PRIORITY_DEFAULT);
+            $priority = match ($source) {
+                SubmissionSource::PROBLEM_IMPORT => JudgeTask::PRIORITY_LOW,
+                default => JudgeTask::PRIORITY_DEFAULT,
+            };
+            // Create judgetask as invalid when evaluating as analyst.
+            $lazyEval = $this->config->get('lazy_eval_results');
+            // We create invalid judgetasks, and only mark them valid when they are interesting for the analysts.
+            $start_invalid = $lazyEval === DOMJudgeService::EVAL_ANALYST && $source == SubmissionSource::SHADOWING;
+            $this->dj->maybeCreateJudgeTasks($judging, $priority, valid: !$start_invalid);
         }
 
         $this->em->wrapInTransaction(function () use ($contest, $submission) {

webapp/templates/jury/submission.html.twig

@@ -541,7 +541,7 @@
                                 {% if selectedJudging is not null and runsOutstanding %}
                                     {% if selectedJudging.judgeCompletely %}
                                         <i class="fas fa-balance-scale" title="remaining test cases requested to be judged"></i>
-                                    {% elseif selectedJudging.result is not null %}
+                                    {% elseif selectedJudging.result is not null or isAnalystMode %}
                                         <form action="{{ path('jury_submission_request_remaining', {'judgingId': selectedJudging.judgingid}) }}" method="post"
                                               style="display: inline; ">
                                             <button type="submit" class="btn btn-outline-secondary btn-sm" style="padding: 0.1rem 0.5rem; font-size: 0.7em"><i class="fa-solid fa-gavel"></i> judge remaining</button>

webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php

@@ -60,6 +60,7 @@ protected function setUp(): void
             'shadow_mode' => 0,
             'sourcefiles_limit' => 1,
             'sourcesize_limit' => 1024*256,
+            'lazy_eval_results' => 1,
         ];
 
         $this->config = $this->createMock(ConfigurationService::class);