From 0e6ef559dcc334da74e18094c2384ba4e9a15639 Mon Sep 17 00:00:00 2001 From: Kevin Jilissen Date: Sun, 6 Jul 2025 12:44:32 +0200 Subject: [PATCH 1/5] Implement analyst evaluation mode The idea behind this mode is that we are not really shadowing and trust the event feed results. However, we want to judge "interesting" runs locally to get useful information without the judging capacity to judge all testcases due to limited judgehost assignment. We do not consider 'TLE' or 'AC' interesting, as rerunning will not yield much more information. We consider 'WA' very interesting and prioritize the judging, but allow manual judging to overtake the priority. We consider 'CE' somewhat interesting, but downprioritize them a lot. For other verdicts, keep the normal priority. --- etc/db-config.yaml | 5 ++- .../Controller/API/JudgehostController.php | 4 +- webapp/src/Service/DOMJudgeService.php | 12 ++--- .../Service/ExternalContestSourceService.php | 44 ++++++++++++++++++- webapp/src/Service/SubmissionService.php | 11 ++++- .../Integration/QueuetaskIntegrationTest.php | 1 + 6 files changed, 66 insertions(+), 11 deletions(-) diff --git a/etc/db-config.yaml b/etc/db-config.yaml index 665a48a6e8..a741b2a469 100644 --- a/etc/db-config.yaml +++ b/etc/db-config.yaml @@ -118,12 +118,13 @@ type: int default_value: 1 public: false - description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging and is typically used when running as analyst system. + description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging. Analyst mode tries to judge only interesting testcases. options: 1: Lazy 2: Full judging 3: Only on request - regex: /^[123]$/ + 4: Analyst mode + regex: /^[1234]$/ error_message: A value between 1 and 3 is required. - name: judgehost_warning type: int diff --git a/webapp/src/Controller/API/JudgehostController.php b/webapp/src/Controller/API/JudgehostController.php index 1045447fa1..1215b0a9ef 100644 --- a/webapp/src/Controller/API/JudgehostController.php +++ b/webapp/src/Controller/API/JudgehostController.php @@ -1083,7 +1083,9 @@ private function addSingleJudgingRun( throw new BadMethodCallException('internal bug: the evaluated result changed during judging'); } - if ($lazyEval !== DOMJudgeService::EVAL_FULL) { + if ($$lazyEval === DOMJudgeService::EVAL_ANALYST) { + // Explicitly do not update priorities or cancel activated tasks. + } elseif ($lazyEval !== DOMJudgeService::EVAL_FULL) { // We don't want to continue on this problem, even if there's spare resources. $this->em->getConnection()->executeStatement( 'UPDATE judgetask SET valid=0, priority=:priority' diff --git a/webapp/src/Service/DOMJudgeService.php b/webapp/src/Service/DOMJudgeService.php index 5e8a92ca40..1603e6dd0b 100644 --- a/webapp/src/Service/DOMJudgeService.php +++ b/webapp/src/Service/DOMJudgeService.php @@ -74,6 +74,7 @@ class DOMJudgeService final public const EVAL_LAZY = 1; final public const EVAL_FULL = 2; final public const EVAL_DEMAND = 3; + final public const EVAL_ANALYST = 4; // Regex external identifiers must adhere to. Note that we are not checking whether it // does not start with a dot or dash or ends with a dot. We could but it would make the @@ -1184,7 +1185,7 @@ public function unblockJudgeTasks(): void } } - public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0): void + public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0, bool $valid = true): void { $submission = $judging->getSubmission(); $problem = $submission->getContestProblem(); @@ -1197,7 +1198,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas return; } - $this->actuallyCreateJudgetasks($priority, $judging, $overshoot); + $this->actuallyCreateJudgetasks($priority, $judging, $overshoot, $valid); $team = $submission->getTeam(); $result = $this->em->createQueryBuilder() @@ -1215,7 +1216,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas // Teams that submit frequently slow down the judge queue but should not be able to starve other teams of their // deserved and timely judgement. - // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definiition of "recent" + // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definition of "recent" // includes all submissions that have been placed at a virtual time (including penalty) more recent than 60s // ago. This is done in order to avoid punishing teams who submit while their submissions are stuck in the queue // for other reasons, for example an internal error for a problem or language. @@ -1586,12 +1587,12 @@ private function allowJudge(ContestProblem $problem, Submission $submission, Lan return !$evalOnDemand; } - private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0): void + private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0, bool $valid = true): void { $submission = $judging->getSubmission(); $problem = $submission->getContestProblem(); // We use a mass insert query, since that is way faster than doing a separate insert for each testcase. - // We first insert judgetasks, then select their ID's and finally insert the judging runs. + // We first insert judgetasks, then select their IDs and finally insert the judging runs. // Step 1: Create the template for the judgetasks. $compileExecutable = $submission->getLanguage()->getCompileExecutable()->getImmutableExecutable(); @@ -1599,6 +1600,7 @@ private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $ ':type' => JudgeTaskType::JUDGING_RUN, ':submitid' => $submission->getSubmitid(), ':priority' => $priority, + ':valid' => $valid ? 1 : 0, ':jobid' => $judging->getJudgingid(), ':uuid' => $judging->getUuid(), ':compile_script_id' => $compileExecutable->getImmutableExecId(), diff --git a/webapp/src/Service/ExternalContestSourceService.php b/webapp/src/Service/ExternalContestSourceService.php index 7970b6b134..1ef6c9ebf0 100644 --- a/webapp/src/Service/ExternalContestSourceService.php +++ b/webapp/src/Service/ExternalContestSourceService.php @@ -27,6 +27,7 @@ use App\Entity\ExternalJudgement; use App\Entity\ExternalRun; use App\Entity\ExternalSourceWarning; +use App\Entity\JudgeTask; use App\Entity\Language; use App\Entity\Problem; use App\Entity\Submission; @@ -1777,13 +1778,13 @@ protected function importRun(Event $event, EventData $data): void } // First, load the external run. + $persist = false; $run = $this->em ->getRepository(ExternalRun::class) ->findOneBy([ 'contest' => $this->getSourceContest(), 'externalid' => $runId, ]); - $persist = false; if (!$run) { $run = new ExternalRun(); $run @@ -1858,9 +1859,50 @@ protected function importRun(Event $event, EventData $data): void if ($persist) { $this->em->persist($run); } + + $lazyEval = $this->config->get('lazy_eval_results'); + if ($lazyEval === DOMJudgeService::EVAL_ANALYST) { + // Check if we want to judge this testcase locally to provide useful information for analysts + $priority = $this->getAnalystRunPriority($run); + if ($priority !== null) { + // Make the judgetask valid and assign running priority if no judgehost has picked it up yet. + $this->em->createQueryBuilder() + ->update(JudgeTask::class, 'jt') + ->set('jt.valid', true) + ->set('jt.priority', $priority) + ->andWhere('jt.testcase_id = :testcase_id') + ->andWhere('jt.submission = :submission') + ->andWhere('jt.judgehost IS NULL') + ->setParameter('testcase_id', $testcase->getTestcaseid()) + ->setParameter('submission', $externalJudgement->getSubmission()) + ->getQuery() + ->execute(); + } + } + $this->em->flush(); } + /** + * Checks if this run is interesting to judge locally for more analysis results. + * @param ExternalRun $run + * @return int The judging priority if it should be run locally, null otherwise. + */ + protected function getAnalystRunPriority(ExternalRun $run): int | null { + return match ($run->getResult()) { + // We will not get any new useful information for TLE testcases, while they take a lot of judgedaemon time. + 'timelimit' => null, + // We often do not get new useful information for judging correct testcases. + 'correct' => null, + // Wrong answers are interesting for the analysts, assign a high priority but below manual judging. + 'wrong-answer' => -9, + // Compile errors could be interesting to see what went wrong, assign a low priority. + 'compiler-error' => 9, + // Otherwise, judge with normal priority. + default => 0, + }; + } + protected function processPendingEvents(string $type, string|int $id): void { // Process pending events. diff --git a/webapp/src/Service/SubmissionService.php b/webapp/src/Service/SubmissionService.php index 792038885d..7bca8c9460 100644 --- a/webapp/src/Service/SubmissionService.php +++ b/webapp/src/Service/SubmissionService.php @@ -732,8 +732,15 @@ public function submitSolution( // This is so that we can use the submitid/judgingid below. $this->em->flush(); - $this->dj->maybeCreateJudgeTasks($judging, - $source === SubmissionSource::PROBLEM_IMPORT ? JudgeTask::PRIORITY_LOW : JudgeTask::PRIORITY_DEFAULT); + $priority = match ($source) { + SubmissionSource::PROBLEM_IMPORT => JudgeTask::PRIORITY_LOW, + default => JudgeTask::PRIORITY_DEFAULT, + }; + // Create judgetask as invalid when evaluating as analyst. + $lazyEval = $this->config->get('lazy_eval_results'); + // We create invalid judgetasks, and only mark them valid when they are interesting for the analysts. + $start_invalid = $lazyEval === DOMJudgeService::EVAL_ANALYST && $source == SubmissionSource::SHADOWING; + $this->dj->maybeCreateJudgeTasks($judging, $priority, valid: !$start_invalid); } $this->em->wrapInTransaction(function () use ($contest, $submission) { diff --git a/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php b/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php index e5c98344b4..87603c9ee5 100644 --- a/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php +++ b/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php @@ -60,6 +60,7 @@ protected function setUp(): void 'shadow_mode' => 0, 'sourcefiles_limit' => 1, 'sourcesize_limit' => 1024*256, + 'lazy_eval_results' => 1, ]; $this->config = $this->createMock(ConfigurationService::class); From e609d99be0fbe5af5f50cb133d36573ef7733392 Mon Sep 17 00:00:00 2001 From: Kevin Jilissen Date: Tue, 2 Sep 2025 21:55:40 +0400 Subject: [PATCH 2/5] Allow judging the remaining testcases as analyst --- webapp/src/Controller/API/JudgehostController.php | 2 +- webapp/src/Controller/Jury/JudgeRemainingTrait.php | 4 +++- webapp/src/Controller/Jury/SubmissionController.php | 1 + webapp/templates/jury/submission.html.twig | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/webapp/src/Controller/API/JudgehostController.php b/webapp/src/Controller/API/JudgehostController.php index 1215b0a9ef..cd578247f3 100644 --- a/webapp/src/Controller/API/JudgehostController.php +++ b/webapp/src/Controller/API/JudgehostController.php @@ -1083,7 +1083,7 @@ private function addSingleJudgingRun( throw new BadMethodCallException('internal bug: the evaluated result changed during judging'); } - if ($$lazyEval === DOMJudgeService::EVAL_ANALYST) { + if ($lazyEval === DOMJudgeService::EVAL_ANALYST) { // Explicitly do not update priorities or cancel activated tasks. } elseif ($lazyEval !== DOMJudgeService::EVAL_FULL) { // We don't want to continue on this problem, even if there's spare resources. diff --git a/webapp/src/Controller/Jury/JudgeRemainingTrait.php b/webapp/src/Controller/Jury/JudgeRemainingTrait.php index 466a652e26..e68233e553 100644 --- a/webapp/src/Controller/Jury/JudgeRemainingTrait.php +++ b/webapp/src/Controller/Jury/JudgeRemainingTrait.php @@ -5,6 +5,7 @@ use App\Entity\JudgeTask; use App\Entity\Judging; use App\Entity\QueueTask; +use App\Service\DOMJudgeService; trait JudgeRemainingTrait { @@ -13,13 +14,14 @@ trait JudgeRemainingTrait */ protected function judgeRemainingJudgings(array $judgings): void { + $lazyEval = $this->config->get('lazy_eval_results'); $inProgress = []; $alreadyRequested = []; $invalidJudgings = []; $numRequested = 0; foreach ($judgings as $judging) { $judgingId = $judging->getJudgingid(); - if ($judging->getResult() === null) { + if ($judging->getResult() === null && $lazyEval !== DOMJudgeService::EVAL_ANALYST) { $inProgress[] = $judgingId; } elseif ($judging->getJudgeCompletely()) { $alreadyRequested[] = $judgingId; diff --git a/webapp/src/Controller/Jury/SubmissionController.php b/webapp/src/Controller/Jury/SubmissionController.php index 6c0bd7d4cd..0bec479332 100644 --- a/webapp/src/Controller/Jury/SubmissionController.php +++ b/webapp/src/Controller/Jury/SubmissionController.php @@ -583,6 +583,7 @@ public function viewAction( 'version_warnings' => [], 'isMultiPassProblem' => $submission->getProblem()->isMultipassProblem(), 'thumbnailSize' => $this->config->get('thumbnail_size'), + 'isAnalystMode' => $this->config->get('lazy_eval_results') === DOMJudgeService::EVAL_ANALYST, ]; if ($selectedJudging === null) { diff --git a/webapp/templates/jury/submission.html.twig b/webapp/templates/jury/submission.html.twig index 2ea923714a..917d53b555 100644 --- a/webapp/templates/jury/submission.html.twig +++ b/webapp/templates/jury/submission.html.twig @@ -541,7 +541,7 @@ {% if selectedJudging is not null and runsOutstanding %} {% if selectedJudging.judgeCompletely %} - {% elseif selectedJudging.result is not null %} + {% elseif selectedJudging.result is not null or isAnalystMode %}
From ecf6b3e22fc3ade929080f630c03ff15eda6aed3 Mon Sep 17 00:00:00 2001 From: Kevin Jilissen Date: Mon, 1 Sep 2025 17:04:24 +0400 Subject: [PATCH 3/5] Create new QueueTask when enabling JudgeTask When the QueueTask is already processed before the arrival of the upstream verdict, the udpated JudgeTask will never be picked up by a judge daemon. To ensure that the interesting run is actually picked up, create a new QueueTask. Note that, in case of an already existing unprocessed queue task, this should not break anything. --- webapp/src/Service/ExternalContestSourceService.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/webapp/src/Service/ExternalContestSourceService.php b/webapp/src/Service/ExternalContestSourceService.php index 1ef6c9ebf0..ee752fdbbd 100644 --- a/webapp/src/Service/ExternalContestSourceService.php +++ b/webapp/src/Service/ExternalContestSourceService.php @@ -30,6 +30,7 @@ use App\Entity\JudgeTask; use App\Entity\Language; use App\Entity\Problem; +use App\Entity\QueueTask; use App\Entity\Submission; use App\Entity\SubmissionSource; use App\Entity\Team; @@ -1866,6 +1867,7 @@ protected function importRun(Event $event, EventData $data): void $priority = $this->getAnalystRunPriority($run); if ($priority !== null) { // Make the judgetask valid and assign running priority if no judgehost has picked it up yet. + $submission = $externalJudgement->getSubmission(); $this->em->createQueryBuilder() ->update(JudgeTask::class, 'jt') ->set('jt.valid', true) @@ -1874,9 +1876,17 @@ protected function importRun(Event $event, EventData $data): void ->andWhere('jt.submission = :submission') ->andWhere('jt.judgehost IS NULL') ->setParameter('testcase_id', $testcase->getTestcaseid()) - ->setParameter('submission', $externalJudgement->getSubmission()) + ->setParameter('submission', $submission) ->getQuery() ->execute(); + + $queueTask = new QueueTask(); + $queueTask->setJudging($submission->getJudgings()->first()) + ->setPriority($priority) + ->setTeam($submission->getTeam()) + ->setTeamPriority((int)$submission->getSubmittime()) + ->setStartTime(null); + $this->em->persist($queueTask); } } From becce1ab143cb2412d9908e21c87514437d8ee3d Mon Sep 17 00:00:00 2001 From: Kevin Jilissen Date: Wed, 3 Sep 2025 10:25:18 +0200 Subject: [PATCH 4/5] Replace magic priority values with constants Otherwise there is a hidden dependency between the 'actual' constants, and these constants. So instead of `-9` use `JudgeTask::PRIORITY_HIGH + 1` and instead of `9` use `JudgeTask::PRIORITY_LOW - 1`. Suggested-by: Mart Pluijmaekers --- webapp/src/Service/ExternalContestSourceService.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp/src/Service/ExternalContestSourceService.php b/webapp/src/Service/ExternalContestSourceService.php index ee752fdbbd..a42cef4212 100644 --- a/webapp/src/Service/ExternalContestSourceService.php +++ b/webapp/src/Service/ExternalContestSourceService.php @@ -1905,9 +1905,9 @@ protected function getAnalystRunPriority(ExternalRun $run): int | null { // We often do not get new useful information for judging correct testcases. 'correct' => null, // Wrong answers are interesting for the analysts, assign a high priority but below manual judging. - 'wrong-answer' => -9, + 'wrong-answer' => JudgeTask::PRIORITY_HIGH + 1, // Compile errors could be interesting to see what went wrong, assign a low priority. - 'compiler-error' => 9, + 'compiler-error' => JudgeTask::PRIORITY_LOW - 1, // Otherwise, judge with normal priority. default => 0, }; From fc7a6813aae62b5c6629f8f2e540ca06a9bbd272 Mon Sep 17 00:00:00 2001 From: Kevin Jilissen Date: Wed, 3 Sep 2025 11:24:02 +0200 Subject: [PATCH 5/5] Judging the remaining tasks is most important --- webapp/src/Controller/Jury/JudgeRemainingTrait.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/webapp/src/Controller/Jury/JudgeRemainingTrait.php b/webapp/src/Controller/Jury/JudgeRemainingTrait.php index e68233e553..8466616198 100644 --- a/webapp/src/Controller/Jury/JudgeRemainingTrait.php +++ b/webapp/src/Controller/Jury/JudgeRemainingTrait.php @@ -19,6 +19,10 @@ protected function judgeRemainingJudgings(array $judgings): void $alreadyRequested = []; $invalidJudgings = []; $numRequested = 0; + + // In analyst mode, when explicitly requested judging the remaining tasks is most important. + $priority = $lazyEval === DOMJudgeService::EVAL_ANALYST ? JudgeTask::PRIORITY_HIGH : JudgeTask::PRIORITY_LOW; + foreach ($judgings as $judging) { $judgingId = $judging->getJudgingid(); if ($judging->getResult() === null && $lazyEval !== DOMJudgeService::EVAL_ANALYST) { @@ -42,7 +46,7 @@ protected function judgeRemainingJudgings(array $judgings): void $queueTask = new QueueTask(); $queueTask->setJudging($judging) - ->setPriority(JudgeTask::PRIORITY_LOW) + ->setPriority($priority) ->setTeam($submission->getTeam()) ->setTeamPriority((int)$submission->getSubmittime()) ->setStartTime(null);