[COMMIT SLIDER]Preparation for e2e job (openvinotoolkit#23135)

### Details:
 - *item1*
 - *...*

### Tickets:
 - *ticket-id*

Author: yury-intel

src/plugins/intel_cpu/tools/commit_slider/commit_slider.py

@@ -6,6 +6,7 @@
 import shutil
 import sys
 from distutils.dir_util import copy_tree
+from distutils.errors import DistutilsFileError
 from utils.helpers import safeClearDir, getParams
 
 args, cfgData, customCfgPath = getParams()
@@ -21,7 +22,10 @@
     from utils.helpers import checkArgAndGetCommits
 
     commitList = []
-    if args.commitSeq is None:
+    if "commitList" in cfgData["runConfig"] and\
+        "explicitList" in cfgData["runConfig"]["commitList"]:
+            commitList = cfgData["runConfig"]["commitList"]["explicitList"]
+    elif args.commitSeq is None:
         if "getCommitListCmd" in cfgData["runConfig"]["commitList"]:
             commitListCmd = cfgData["runConfig"]["commitList"]
             commitListCmd = commitListCmd["getCommitListCmd"]
@@ -69,7 +73,11 @@
     tempCachePath = cfgData["cachePath"].format(workPath=workPath)
     permCachePath = cfgData["cachePath"].format(workPath=curPath)
     safeClearDir(permCachePath, cfgData)
-    copy_tree(tempCachePath, permCachePath)
+    try:
+        copy_tree(tempCachePath, permCachePath)
+    except DistutilsFileError:
+        # prevent exception raising while cache is empty
+        pass
 
     try:
         shutil.copyfile(

src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py

@@ -140,3 +140,113 @@ def testBmSteppedBreak2(self):
             e.exception.errType,
             BmValidationError.BmValErrType.LOW_LOCAL_GAP
         )
+
+    @skip_commit_slider_devtest
+    def testForsubstitutionRule(self):
+        from utils.helpers import applySubstitutionRules
+        cfg = {
+                "serviceConfig": {
+                    "previousKey": "previousValue"
+                },
+                "wrongDst": "{commitHash1} is unchanged",
+                "dst": {
+                    "complex": {
+                        "path": [
+                            "{commitHash1} is natural number",
+                            "{commitHash2} is natural number",
+                            "{commitHash1} is {commitHash2}"
+                        ]
+                    }
+                },
+                "src": {
+                    "complex": {
+                        "path": {
+                            "one": "1",
+                            "two": "2"
+                        }
+                    }
+                }
+        }
+        rules = [
+            {
+                "name": "testRule1",
+                "enabled": True,
+                "type": "map",
+                "placeholder": "commitHash1",
+                "from": "$.src.complex.path",
+                "to": "$.dst.complex.path"
+            },
+            {
+                "name": "testRule2",
+                "enabled": True,
+                "type": "map",
+                "placeholder": "commitHash2",
+                "from": "$.src.complex.path",
+                "to": "$.dst.complex.path"
+            }
+        ]
+        def applyByRef(cfg: map, rules: list, substitution: str):
+            applySubstitutionRules(cfg, rules, substitution)
+
+        applyByRef(cfg, rules, "one")
+
+        # assert first substitution
+        self.assertEqual(
+            cfg["dst"]["complex"]["path"][0],
+            "1 is natural number"
+        )
+        self.assertEqual(
+            cfg["dst"]["complex"]["path"][1],
+            "1 is natural number"
+        )
+        self.assertEqual(
+            cfg["dst"]["complex"]["path"][2],
+            "1 is 1"
+        )
+        self.assertEqual(
+            cfg["wrongDst"],
+            "{commitHash1} is unchanged"
+        )
+
+        applyByRef(cfg, rules, "two")
+
+        # assert second substitution
+        self.assertEqual(
+            cfg["dst"]["complex"]["path"][0],
+            "2 is natural number"
+        )
+        self.assertEqual(
+            cfg["dst"]["complex"]["path"][1],
+            "2 is natural number"
+        )
+        self.assertEqual(
+            cfg["dst"]["complex"]["path"][2],
+            "2 is 2"
+        )
+        self.assertEqual(
+            cfg["wrongDst"],
+            "{commitHash1} is unchanged"
+        )
+
+    @skip_commit_slider_devtest
+    def testForDeepUpdate(self):
+        from utils.helpers import deepMapUpdate
+        cfg = {
+            "another": {
+                "path": "not updated"
+            },
+            "path": {
+                "to": {
+                    "placeholder": "not updated"
+                }
+            }
+        }
+        cfg = deepMapUpdate(cfg, ["path", "to", "placeholder"], "updated")
+        self.assertEqual(
+            cfg["path"]["to"]["placeholder"],
+            "updated"
+        )
+        self.assertEqual(
+            cfg["another"]["path"],
+            "not updated"
+        )

src/plugins/intel_cpu/tools/commit_slider/tests/test_util.py

@@ -10,6 +10,7 @@
 from os import path
 from test_data import TestData
 from test_data import TestError
+from utils.helpers import formatJSON
 
 sys.path.append('../')
 from utils.helpers import getMeaningfullCommitTail
@@ -55,7 +56,6 @@ def makeRepoContent(td: TestData):
 
     td.repoStructure['files'] = formatJSON(
         td.repoStructure['files'],
-        td,
         lambda content: content.format(
             repoName=td.repoName,
             mainFile=td.mainFile)
@@ -92,22 +92,6 @@ def runCmd(cmd, cwd, verbose=False):
     proc.communicate()
     return output
 
-
-def formatJSON(content, td: TestData, formatLambda):
-    if isinstance(content, dict):
-        for k, value in content.items():
-            content[k] = formatJSON(value, td, formatLambda)
-    elif isinstance(content, list):
-        for id, item in enumerate(content):
-            content[id] = formatJSON(item, td, formatLambda)
-    elif isinstance(content, str):
-        content = formatLambda(content)
-    else:
-        # bool or digit object
-        pass
-    return content
-
-
 def createRepo(td: TestData):
     repoName = td.repoName
     repoPath = td.repoPath
@@ -169,7 +153,7 @@ def getActualCommit(td: TestData):
         raise TestError("Running actual commit before expected.")
 
     # prepare config
-    cfg = formatJSON(td.testCfg, td, td.formatConfig)
+    cfg = formatJSON(td.testCfg, td.formatConfig)
     testCfg = "test_cfg.json"
 
     with open(testCfg, "w+") as customCfg:

src/plugins/intel_cpu/tools/commit_slider/utils/cfg.json

@@ -14,7 +14,8 @@
     "utilMap" : {
         "map_builder" : "printMap",
         "log_parser" : "logParser",
-        "break_validator": "breakValidator"
+        "break_validator": "breakValidator",
+        "e2e_preparator": "getWheelMap"
     },
     "extendBuildCommand" : false,
     "commandList" : [
@@ -77,6 +78,11 @@
     "printCSV" : true,
     "usePrevRunCache" : false,
     "verboseOutput": false,
+    "venvCfg": {
+        "venvEnabled": false,
+        "venvDir": "{workPath}/venv/",
+        "venvName": "tempVenv"
+    },
     "preliminaryCheckCfg": {
         "leftCheck": true,
         "rightCheck": false,
@@ -89,17 +95,40 @@
     "dlbConfig" : {
         "launchedAsJob" : false,
         "toolName" : "{e2e|ac} - specified outside tool, supposed to be downloaded by job",
-        "appPath" : "path, substituted by job"
+        "wheelVersionsMap": {},
+        "commonPath": "",
+        "subPath": "",
+        "appPath" : "path, substituted by job",
+        "appCmd": ""
     },
     "cachedPathConfig": {
         "enabled" : false,
         "scheme" : "optional | mandatory",
         "comment" : "'mandatory' skips lacking hash-appPath pair for given key, 'optional' tries to handle it by building",
         "passCmdList": true,
         "changeAppPath": true,
-        "generateMap": false,
         "commonPath": "",
         "subPath": "",
         "cashMap" : {}
-    }
+    },
+    "subscriptions" : [
+        {
+            "name": "wheelPathsMap",
+            "enabled": false
+        },
+        {
+            "name": "wheelVersionsMap",
+            "enabled": false
+        }
+    ],
+    "substitutionRules": [
+        {
+            "name": "ruleName",
+            "enabled": false,
+            "type": "commit_map | static",
+            "placeholder": "placeholder",
+            "from": "$.json.path.expression.to.cfg.value",
+            "to": "$.json.path.expression.to.commit.map"
+        }
+    ]
 }

src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py

@@ -0,0 +1,45 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+
+
+class CfgManager():
+    def __init__(self, cfg) -> None:
+        self.cfg = cfg
+
+    def applyTemplate(self):
+        if not "template" in self.cfg:
+            return self.cfg
+        logPath = self.cfg["logPath"]
+        tmplName = self.cfg["template"]["name"]
+        fullCfg = {}
+        if tmplName == "bm_simple":
+            fullCfg = self.generatebmSimpleTemplate()
+        else:
+            raise Exception(
+                "Unknown template '{}'".format(tmplName)
+            )
+        fullCfg["logPath"] = logPath
+        return fullCfg
+    
+    def readJsonTmpl(self, tmplFileName: str):
+        tmplFileName = os.path.join(
+            "utils/cfg_samples/", tmplFileName
+        )
+        with open(tmplFileName) as cfgFile:
+            tmplJSON = json.load(cfgFile)
+            return tmplJSON
+
+    def generatebmSimpleTemplate(self):
+        tmpl = self.cfg["template"]
+        tmpJSON = self.readJsonTmpl("bm_perf_for_CI.json")
+        devParam = "perfAppropriateDeviation"
+        if "appCmd" in tmpl:
+            tmpJSON["appCmd"] = tmpl["appCmd"]
+        else:
+            raise("No 'appcmd' in template")
+        if devParam in tmpl:
+            tmpJSON["runConfig"][devParam] = tmpl[devParam]
+        return tmpJSON

src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_perf_for_CI.json

@@ -0,0 +1,10 @@
+{
+    "appCmd":"{appCmd}",
+    "makeCmd":"cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=OFF -DTHREADING=TBB -DENABLE_MKL_DNN=ON -DENABLE_CLDNN=OFF -DENABLE_INTEL_GNA=OFF -DENABLE_INTEL_VPU=OFF -DENABLE_INTEL_MYRIAD=OFF -DENABLE_INTEL_MYRIAD_COMMON=OFF -DENABLE_HDDL=OFF -DENABLE_MODELS=OFF -DENABLE_SAMPLES=ON -DENABLE_TESTS=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_CPU_DEBUG_CAPS=OFF -DENABLE_DEBUG_CAPS=OFF -DENABLE_OV_CORE_BACKEND_UNIT_TESTS=OFF -DENABLE_OPENVINO_DEBUG=OFF  -DCMAKE_CXX_FLAGS=-Wno-deprecated -DCMAKE_C_FLAGS=-Wno-deprecated -DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations -DCMAKE_C_FLAGS=-Wno-deprecated-declarations ..",
+    "runConfig":{
+       "mode":"bmPerf",
+       "traversal":"firstFailedVersion",
+       "perfAppropriateDeviation": 0.05
+    },
+    "extendBuildCommand":true
+ }

src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e.json

@@ -2,10 +2,10 @@
     "appPath" : "/<e2e_path>/e2e/frameworks.ai.openvino.tests/e2e_oss/",
     "appCmd" : "pytest test_dynamism.py <e2e_args>",
     "envVars" : [
-        {"name" : "PYTHONPATH", "val" : "/<ov_path>/bin/intel64/Release/python/"},
-        {"name" : "LD_LIBRARY_PATH", "val" : "/<ov_path>/bin/intel64/Release/"},
-        {"name" : "MO_ROOT", "val" : "/<ov_path>/tools/mo/openvino/tools/"},
-        {"name" : "OPENVINO_ROOT_DIR", "val" : "/<ov_path>/"}
+        {"name" : "PYTHONPATH", "val" : "{gitPath}/bin/intel64/Release/python/"},
+        {"name" : "LD_LIBRARY_PATH", "val" : "{gitPath}/bin/intel64/Release/"},
+        {"name" : "MO_ROOT", "val" : "{gitPath}/tools/mo/openvino/tools/"},
+        {"name" : "OPENVINO_ROOT_DIR", "val" : "{gitPath}/"}
     ],
     "makeCmd" : "cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=ON -DPython3_EXECUTABLE=/usr/bin/python3.8 -DTHREADING=TBB -DENABLE_INTEL_GPU=OFF -DENABLE_SAMPLES=OFF -DENABLE_TESTS=OFF -DENABLE_CPU_DEBUG_CAPS=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_CPU_DEBUG_CAPS=OFF -DENABLE_DEBUG_CAPS=OFF -DENABLE_OPENVINO_DEBUG=OFF -DCMAKE_CXX_FLAGS=-Wno-deprecated -DCMAKE_C_FLAGS=-Wno-deprecated -DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations -DCMAKE_C_FLAGS=-Wno-deprecated-declarations ..",
     "runConfig" : {

src/plugins/intel_cpu/tools/commit_slider/utils/common_mode.py

@@ -3,7 +3,7 @@
 
 from abc import ABC
 import utils.helpers as util
-import utils.map_builder as mapBuilder
+from utils.subscription import SubscriptionManager
 from utils.break_validator import validateBMOutput
 from utils.break_validator import BmValidationError
 import json
@@ -119,6 +119,14 @@ def prepareRun(self, list, cfg):
                         c1=newList[0], c2=newList[-1])
                 )
                 list = newList
+            elif self.traversal.isComparative():
+                raise util.PreliminaryAnalysisError(
+                    "No degradation for reduced interval: \
+                    {i1} and {i2} don't differ".format(
+                        i1=list[0], i2=list[-1]),
+                    util.PreliminaryAnalysisError.\
+                        PreliminaryErrType.NO_DEGRADATION
+                    )
         else:
             self.preliminaryCheck(list, cfg)
         return list
@@ -132,12 +140,9 @@ def normalizeCfg(self, cfg):
         # switch off illegal check
         if not self.traversal.isComparative():
             cfg["checkIfBordersDiffer"] = False
-        cashCfg = cfg["cachedPathConfig"]
-        # build cash map
-        if (cashCfg["enabled"] and cashCfg["generateMap"]):
-            cfg["cachedPathConfig"]["cashMap"] = mapBuilder(
-                cashCfg["commonPath"], cashCfg["subPath"]
-            )
+        # apply necessary subscriptions for cfg
+        subManager = SubscriptionManager(cfg)
+        subManager.apply()
         if "modeName" in cfg["skipMode"]:
             errorHandlingMode = cfg["skipMode"]["modeName"]
             if errorHandlingMode == "skip":