From 24d6c720b1e71feb0d1b765d167c2bffbc263d70 Mon Sep 17 00:00:00 2001 From: rupozzi Date: Tue, 22 Mar 2022 10:02:50 +0100 Subject: [PATCH 1/5] fix: Removing activity monitor from web app --- .../WebApp/handler/MonitoringHandler.py | 257 --- .../classes/ActivityMonitor.js | 1375 ----------------- .../classes/ActivityTreeModel.js | 5 - .../ActivityMonitor/css/ActivityMonitor.css | 3 - 4 files changed, 1640 deletions(-) delete mode 100644 src/WebAppDIRAC/WebApp/handler/MonitoringHandler.py delete mode 100755 src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/classes/ActivityMonitor.js delete mode 100644 src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/classes/ActivityTreeModel.js delete mode 100644 src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/css/ActivityMonitor.css diff --git a/src/WebAppDIRAC/WebApp/handler/MonitoringHandler.py b/src/WebAppDIRAC/WebApp/handler/MonitoringHandler.py deleted file mode 100644 index 99b37929f..000000000 --- a/src/WebAppDIRAC/WebApp/handler/MonitoringHandler.py +++ /dev/null @@ -1,257 +0,0 @@ -import os -import json -import tempfile -import datetime - -from DIRAC import gConfig, S_OK, S_ERROR, gLogger -from DIRAC.Core.DISET.TransferClient import TransferClient -from DIRAC.Core.Utilities import Time, List, DictCache -from DIRAC.Core.Utilities.Plotting.FileCoding import extractRequestFromFileId, codeRequestInFileId -from DIRAC.MonitoringSystem.Client.MonitoringClient import MonitoringClient - -from WebAppDIRAC.Lib.WebHandler import _WebHandler as WebHandler, FileResponse - - -class MonitoringHandler(WebHandler): - - DEFAULT_AUTHORIZATION = "authenticated" - __keysCache = DictCache.DictCache() - - def __getUniqueKeyValues(self, typeName): - cacheKey = (self.getUserName(), self.getUserGroup(), self.getUserSetup(), typeName) - if not (data := MonitoringHandler.__keysCache.get(cacheKey)): - retVal = MonitoringClient().listUniqueKeyValues(typeName) - if "rpcStub" in retVal: - del retVal["rpcStub"] - if not retVal["OK"]: - return retVal - - # Site ordering based on TierLevel / alpha - if "Site" in retVal["Value"]: - siteLevel = {} - for siteName in retVal["Value"]["Site"]: - sitePrefix = siteName.split(".")[0].strip() - level = gConfig.getValue("/Resources/Sites/%s/%s/MoUTierLevel" % (sitePrefix, siteName), 10) - if level not in siteLevel: - siteLevel[level] = [] - siteLevel[level].append(siteName) - orderedSites = [] - for level in sorted(siteLevel): - orderedSites.extend(sorted(siteLevel[level])) - retVal["Value"]["Site"] = orderedSites - data = retVal - MonitoringHandler.__keysCache.add(cacheKey, 300, data) - return data - - def web_getSelectionData(self, monitoringType): - callback = {} - # Get unique key values - if not (retVal := self.__getUniqueKeyValues(monitoringType))["OK"]: - return {"success": "false", "result": "", "error": retVal["Message"]} - - records = {} - for record in retVal["Value"]: # may have more than 1000 of records. - # do not show all of them in the web portal - length = len(retVal["Value"][record]) - if length > 10000: - records[record] = retVal["Value"][record][length - 5000 :] - message = f"The {monitoringType} monitoring type contains too many rows: {record} -> {length}" - message += " Note: Only 5000 rows are returned!" - gLogger.warn(message) - else: - records[record] = retVal["Value"][record] - callback["selectionValues"] = records - - # Cache for plotsList? - if not (data := MonitoringHandler.__keysCache.get(f"reportsList:{monitoringType}")): - if not (retVal := MonitoringClient().listReports(monitoringType))["OK"]: - return {"success": "false", "result": "", "error": retVal["Message"]} - data = retVal["Value"] - MonitoringHandler.__keysCache.add(f"reportsList:{monitoringType}", 300, data) - callback["plotsList"] = data - return {"success": "true", "result": callback} - - def __parseFormParams(self): - pD = {} - extraParams = {} - pinDates = False - - for name in self.request.arguments: - pD[name] = self.get_argument(name) - - # Personalized title? - if "plotTitle" in pD: - extraParams["plotTitle"] = pD["plotTitle"] - del pD["plotTitle"] - # Pin dates? - if "pinDates" in pD: - pinDates = pD["pinDates"] - del pD["pinDates"] - pinDates = pinDates.lower() in ("yes", "y", "true", "1") - # Get plotname - if "grouping" not in pD: - return S_ERROR("Missing grouping!") - grouping = pD["grouping"] - # Get plotname - if "typeName" not in pD: - return S_ERROR("Missing type name!") - typeName = pD["typeName"] - del pD["typeName"] - # Get plotname - if "plotName" not in pD: - return S_ERROR("Missing plot name!") - reportName = pD["plotName"] - del pD["plotName"] - # Get times - if "timeSelector" not in pD: - return S_ERROR("Missing time span!") - # Find the proper time! - pD["timeSelector"] = int(pD["timeSelector"]) - if pD["timeSelector"] > 0: - end = Time.dateTime() - start = end - datetime.timedelta(seconds=pD["timeSelector"]) - if not pinDates: - extraParams["lastSeconds"] = pD["timeSelector"] - else: - if "endTime" not in pD: - end = False - else: - end = Time.fromString(pD["endTime"]) - del pD["endTime"] - if "startTime" not in pD: - return S_ERROR("Missing starTime!") - else: - start = Time.fromString(pD["startTime"]) - del pD["startTime"] - del pD["timeSelector"] - - for k in pD: - if k.find("ex_") == 0: - extraParams[k[3:]] = pD[k] - - # Selection data - data = {} - - # Listify the rest - for selName in pD: - if selName == "grouping": - pD[selName] = [pD[selName]] - else: - try: - pD[selName] = json.loads(pD[selName]) - except ValueError: - pD[selName] = List.fromChar(pD[selName], ",") - - # If json parse value as string, listify it - if isinstance(pD[selName], str): - pD[selName] = List.fromChar(pD[selName], ",") - - # Convert 'value*' to list of values that starts with 'value' - fullList = [] - for value in pD[selName]: - if value.endswith("*"): - if not data: - retVal = self.__getUniqueKeyValues(typeName) - if not retVal["OK"]: - return retVal - data = retVal["Value"] - for v in data[selName]: - if v.startswith(value[:-1]): - fullList.append(v) - else: - fullList.append(value) - - pD[selName] = fullList - - return S_OK((typeName, reportName, start, end, pD, grouping, extraParams)) - - def web_generatePlot(self): - if (retVal := self.__queryForPlot())["OK"]: - return {"success": "true", "data": retVal["Value"]["plot"]} - return {"success": "false", "errors": retVal["Message"]} - - def __queryForPlot(self): - res = self.__parseFormParams() - return MonitoringClient().generateDelayedPlot(*res["Value"]) if res["OK"] else res - - def web_getPlotImg(self, fileName=None): - """Get plot image""" - if not fileName: - return {"success": "false", "error": "Maybe you forgot the file?"} - # Prevent directory traversal - plotImageFile = os.path.normpath("/" + fileName).lstrip("/") - - transferClient = TransferClient("Monitoring/Monitoring") - tempFile = tempfile.TemporaryFile() - retVal = transferClient.receiveFile(tempFile, plotImageFile) - if not retVal["OK"]: - return {"success": "false", "error": retVal["Message"]} - tempFile.seek(0) - data = tempFile.read() - return FileResponse(data, plotImageFile, ext="png") - - def web_getPlotImgFromCache(self, fileName=None): - """Get plot image from cache.""" - if not fileName: - return {"success": "false", "error": "Maybe you forgot the file?"} - - retVal = extractRequestFromFileId(fileName) - if not retVal["OK"]: - return {"success": "false", "error": retVal["Message"]} - fields = retVal["Value"] - if "extraArgs" in fields: # in order to get the plot from the cache we have to clean the extraArgs... - plotTitle = "" - if "plotTitle" in fields["extraArgs"]: - plotTitle = fields["extraArgs"]["plotTitle"] - fields["extraArgs"] = {} - fields["extraArgs"]["plotTitle"] = plotTitle - else: - fields["extraArgs"] = {} - - retVal = codeRequestInFileId(fields) - if not retVal["OK"]: - return {"success": "false", "error": retVal["Message"]} - plotImageFile = retVal["Value"]["plot"] - - transferClient = TransferClient("Monitoring/Monitoring") - tempFile = tempfile.TemporaryFile() - if not (retVal := transferClient.receiveFile(tempFile, plotImageFile))["OK"]: - return {"success": "false", "error": retVal["Message"]} - tempFile.seek(0) - data = tempFile.read() - return FileResponse(data, plotImageFile, ext="png", cache=False) - - def web_getCsvPlotData(self): - if not (retVal := self.__parseFormParams())["OK"]: - return {"success": "false", "error": retVal["Message"]} - params = retVal["Value"] - if not (retVal := MonitoringClient().getReport(*params))["OK"]: - return {"success": "false", "error": retVal["Message"]} - rawData = retVal["Value"] - groupKeys = sorted(rawData["data"]) - if "granularity" in rawData: - granularity = rawData["granularity"] - data = rawData["data"] - tS = int(Time.toEpoch(params[2])) - timeStart = tS - tS % granularity - strData = "epoch,%s\n" % ",".join(groupKeys) - for timeSlot in range(timeStart, int(Time.toEpoch(params[3])), granularity): - lineData = [str(timeSlot)] - for key in groupKeys: - if timeSlot in data[key]: - lineData.append(str(data[key][timeSlot])) - else: - lineData.append("") - strData += "%s\n" % ",".join(lineData) - else: - strData = "%s\n" % ",".join(groupKeys) - strData += ",".join([str(rawData["data"][k]) for k in groupKeys]) - return FileResponse(strData, str(params), ext="csv", cache=False) - - def web_getPlotData(self): - if not (retVal := self.__parseFormParams())["OK"]: - return {"success": "false", "error": retVal["Message"]} - params = retVal["Value"] - if not (retVal := MonitoringClient().getReport(*params))["OK"]: - return {"success": "false", "error": retVal["Message"]} - return retVal["Value"]["data"] diff --git a/src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/classes/ActivityMonitor.js b/src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/classes/ActivityMonitor.js deleted file mode 100755 index d03e4cca0..000000000 --- a/src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/classes/ActivityMonitor.js +++ /dev/null @@ -1,1375 +0,0 @@ -Ext.define("DIRAC.ActivityMonitor.classes.ActivityMonitor", { - extend: "Ext.dirac.core.Module", - requires: [ - "Ext.util.*", - "Ext.panel.Panel", - "Ext.form.field.Text", - "Ext.button.Button", - "Ext.menu.Menu", - "Ext.form.field.ComboBox", - "Ext.layout.*", - "Ext.form.field.Date", - "Ext.form.field.TextArea", - "Ext.form.field.Checkbox", - "Ext.form.FieldSet", - "Ext.dirac.utils.DiracMultiSelect", - "Ext.toolbar.Toolbar", - "Ext.data.Record", - "Ext.Array", - "Ext.data.TreeStore", - "Ext.ux.form.MultiSelect", - "DIRAC.ActivityMonitor.classes.ActivityTreeModel", - ], - - expansionState: {}, - initComponent: function () { - var me = this; - me.launcher.title = "Activity Monitor"; - - Ext.apply(me, { - layout: "border", - bodyBorder: false, - defaults: { - collapsible: true, - split: true, - }, - items: [], - header: false, - }); - - me.callParent(arguments); - }, - - buildUI: function () { - var me = this; - - var oToolb = Ext.create("Ext.toolbar.Toolbar", { - dock: "top", - border: false, - layout: { - pack: "center", - }, - items: [ - { - xtype: "button", - text: "Proxy Monitor", - handler: function () { - me.mainPanel.getLayout().setActiveItem(0); - }, - toggleGroup: me.id + "-ids-submodule", - allowDepress: false, - }, - { - xtype: "button", - text: "Plot Creator", - handler: function () { - me.mainPanel.getLayout().setActiveItem(1); - }, - toggleGroup: me.id + "-ids-submodule", - allowDepress: false, - }, - { - xtype: "button", - text: "Plot Viewer", - handler: function () { - me.mainPanel.getLayout().setActiveItem(2); - }, - toggleGroup: me.id + "-ids-submodule", - allowDepress: false, - }, - { - xtype: "button", - text: "System overview plots", - handler: function () { - me.mainPanel.getLayout().setActiveItem(3); - }, - toggleGroup: me.id + "-ids-submodule", - allowDepress: false, - }, - ], - }); - - oToolb.items.getAt(0).toggle(); - - me.mainPanel = new Ext.create("Ext.panel.Panel", { - floatable: false, - layout: "card", - region: "center", - header: false, - border: false, - dockedItems: [oToolb], - }); - - me.__buildActivityMonitor(); - me.__buildPlotManagement(); - me.__buildPlotViewer(); - me.__buildSystemViewer(); - - me.add([me.mainPanel]); - }, - - __buildActivityMonitor: function () { - var me = this; - - me.activityMonitorDataStore = new Ext.data.JsonStore({ - proxy: { - type: "ajax", - url: GLOBAL.BASE_URL + "ActivityMonitor/getActivityData", - reader: { - keepRawData: true, - type: "json", - rootProperty: "result", - }, - timeout: 1800000, - }, - autoLoad: true, - fields: [ - { - name: "sources_id", - type: "int", - }, - "sources_site", - "sources_componentType", - "sources_componentLocation", - "sources_componentName", - { - name: "activities_id", - type: "int", - }, - "activities_name", - "activities_category", - "activities_unit", - "activities_type", - "activities_description", - { - name: "activities_bucketLength", - type: "int", - }, - "activities_filename", - { - name: "activities_lastUpdate", - type: "float", - }, - ], - remoteSort: true, - pageSize: 100, - listeners: { - load: function (oStore, records, successful, eOpts) { - var bResponseOK = oStore.proxy.getReader().rawData["success"] == "true"; - - if (!bResponseOK) { - GLOBAL.APP.CF.alert(oStore.proxy.getReader().rawData["error"], "info"); - - if (parseInt(oStore.proxy.getReader().rawData["total"], 10) == 0) { - me.dataStore.removeAll(); - } - } - }, - }, - }); - - var oDeleteSelectedActivities = new Ext.Button({ - text: "Delete", - iconCls: "dirac-icon-delete", - handler: function () { - var oElems = Ext.query("#" + me.activityMonitorPanel.id + " input.checkrow"); - - var oItems = []; - - for (var i = 0; i < oElems.length; i++) if (oElems[i].checked) oItems.push(oElems[i].value); - - if (oItems.length < 1) { - GLOBAL.APP.CF.alert("No jobs were selected", "error"); - return; - } - - Ext.Ajax.request({ - url: GLOBAL.BASE_URL + "ActivityMonitor/deleteActivities", - params: { - ids: oItems.join(","), - }, - scope: me, - success: function (response) { - me.activityMonitorDataStore.store.load(); - }, - }); - }, - tooltip: "Click to delete all selected proxies", - }); - - me.activityMonitorToolbar = Ext.create("Ext.toolbar.Paging", { - store: me.activityMonitorDataStore, - displayInfo: true, - displayMsg: "Displaying topics {0} - {1} of {2}", - items: [oDeleteSelectedActivities, "->"], - prependButtons: true, - emptyMsg: "No topics to display", - layout: { - overflowHandler: "Scroller", - }, - }); - - var sGridId = Ext.id(); - - var sCheckboxDefinition = ""; - sCheckboxDefinition += '' - ); - }, - renderLastUpdate: function (value, metadata, record, rowIndex, colIndex, store) { - var lastUpdated = record.data.activities_lastUpdate; - var timeLimit = 86400 * 30; - if (lastUpdated > timeLimit) lastUpdated = timeLimit; - var green = parseInt((200 * (timeLimit - lastUpdated)) / timeLimit); - var red = parseInt((200 * lastUpdated) / timeLimit); - return '' + lastUpdated + ""; - }, - bbar: me.activityMonitorToolbar, - }); - - me.htmlDescriptionVariables = ""; - me.htmlDescriptionVariables += ""; - me.htmlDescriptionVariables += ""; - me.htmlDescriptionVariables += ""; - me.htmlDescriptionVariables += ""; - me.htmlDescriptionVariables += ""; - - var oVariables = [ - ["$DESCRIPTION", "Description of the activity"], - ["$SITE", "Site for the originating component"], - ["$COMPONENTTYPE", "Type of component to generate the activity (for example: service, agent, web)"], - ["$COMPONENTNAME", "Full name of component"], - ["$COMPONENTLOCATION", " Location of component"], - ["$UNIT", "Activity unit"], - ["$CATEGORY", "Activity category"], - ]; - - for (var i = 0; i < oVariables.length; i++) { - me.htmlDescriptionVariables += ""; - me.htmlDescriptionVariables += ""; - me.htmlDescriptionVariables += ""; - me.htmlDescriptionVariables += ""; - } - - me.htmlDescriptionVariables += "
"; - me.htmlDescriptionVariables += "Variable Name"; - me.htmlDescriptionVariables += ""; - me.htmlDescriptionVariables += "Description"; - me.htmlDescriptionVariables += "
"; - me.htmlDescriptionVariables += oVariables[i][0]; - me.htmlDescriptionVariables += ""; - me.htmlDescriptionVariables += oVariables[i][1]; - me.htmlDescriptionVariables += "
"; - - me.mainPanel.add([me.activityMonitorPanel]); - }, - - __buildPlotManagement: function () { - var me = this; - - me.plotManagementMainPanel = new Ext.create("Ext.panel.Panel", { - floatable: false, - layout: "border", - header: false, - border: false, - items: [], - defaults: { - collapsible: true, - split: true, - }, - }); - - var sGridId = Ext.id(); - - var sCheckboxDefinition = ""; - sCheckboxDefinition += ''; - }, - dockedItems: [ - { - dock: "top", - xtype: "toolbar", - border: false, - items: [ - "->", - { - xtype: "button", - iconCls: "dirac-icon-delete", - text: "Delete", - handler: function () {}, - }, - ], - tooltip: "Delete selected views", - }, - ], - }); - - me.plotManagementListPanel.refreshData(); - - // ------------------------------------------------------------------------------------- - - var oTopPanel = new Ext.create("Ext.panel.Panel", { - region: "center", - floatable: false, - layout: "border", - header: false, - defaults: { - collapsible: true, - split: true, - }, - }); - - me.viewDefinitionData = {}; - me.viewDefinitionDataForServer = {}; - me.viewDefinitionDataForServerVariable = []; - - me.plotManagementFieldCreator = new Ext.create("Ext.panel.Panel", { - region: "west", - floatable: false, - layout: "anchor", - header: false, - width: 350, - bodyPadding: 5, - dockedItems: [ - { - dock: "top", - xtype: "toolbar", - border: false, - layout: { - pack: "center", - }, - items: [ - "->", - { - xtype: "button", - iconCls: "dirac-icon-plus", - text: "Add to view definition", - handler: function () { - var sTypeValue = me.restrictByFieldCreator.getValue(); - var oSelectedValues = me.valuesFieldCreator.getValue(); - - if (sTypeValue == null) { - GLOBAL.APP.CF.alert("No data type selected !", "warning"); - return; - } - - if (me.variableFieldCreator.getValue() || (!me.variableFieldCreator.getValue() && oSelectedValues.length > 0)) { - var sTypeText = me.restrictByFieldCreator.findRecordByValue(sTypeValue).get("text"); - - me.viewDefinitionData[sTypeText] = { - value: sTypeValue, - checked: me.variableFieldCreator.getValue(), - }; - - if (me.variableFieldCreator.getValue()) me.viewDefinitionDataForServerVariable.push(sTypeValue); - else me.viewDefinitionDataForServer[sTypeValue] = oSelectedValues; - - oNodeData = { - text: sTypeText, - leaf: false, - }; - var oRoot = me.plotManagementViewTreeStore.getRootNode(); - - var oTypeNode = oRoot.createNode(oNodeData); - oRoot.appendChild(oTypeNode); - - if (me.variableFieldCreator.getValue()) { - var oNewNode = oTypeNode.createNode({ - text: "variable value", - leaf: true, - }); - oTypeNode.appendChild(oNewNode); - } else { - for (var i = 0; i < oSelectedValues.length; i++) { - var oNewNode = oTypeNode.createNode({ - text: oSelectedValues[i], - leaf: true, - }); - oTypeNode.appendChild(oNewNode); - } - } - - oRoot.expand(); - oTypeNode.expand(); - - // we have to remove the selected option - - me.restrictByFieldCreator.store.remove(me.restrictByFieldCreator.findRecordByValue(sTypeValue)); - - // we have to set the value to null - me.restrictByFieldCreator.setValue(null); - me.valuesFieldCreator.store.removeAll(); - me.valuesFieldCreator.reset(); - } else { - GLOBAL.APP.CF.alert("No values have been selected !", "warning"); - } - }, - }, - ], - }, - ], - }); - - me.restrictByFieldCreator = Ext.create("Ext.form.field.ComboBox", { - fieldLabel: "Restrict by", - queryMode: "local", - labelAlign: "left", - displayField: "text", - valueField: "value", - anchor: "100%", - editable: false, - store: new Ext.data.ArrayStore({ - fields: ["value", "text"], - data: [ - ["sources.site", "Site"], - ["sources.componentType", "Component type"], - ["sources.componentLocation", "Component location"], - ["sources.componentName", "Component name"], - ["activities.description", "Activity"], - ["activities.category", "Activity category"], - ], - }), - value: null, - listeners: { - change: function (oComp, sNewValue, sOldValue, eOpts) { - if (sNewValue != null && Ext.util.Format.trim(sNewValue) != "") { - Ext.Ajax.request({ - url: GLOBAL.BASE_URL + "ActivityMonitor/queryFieldValue", - params: { - queryField: sNewValue, - selectedFields: Ext.JSON.encode(me.viewDefinitionDataForServer), - }, - scope: me, - success: function (response) { - var response = Ext.JSON.decode(response.responseText); - - if (response["success"] == "true") { - me.valuesFieldCreator.store.removeAll(); - - oData = response.result; - - for (var i = 0; i < oData.length; i++) { - me.valuesFieldCreator.store.add({ - value: Ext.util.Format.trim(oData[i][0]), - }); - } - - me.valuesFieldCreator.reset(); - } else { - GLOBAL.APP.CF.alert(response["error"], "warning"); - } - }, - }); - } - }, - }, - }); - - me.valuesFieldCreator = new Ext.ux.form.MultiSelect({ - fieldLabel: "Having values", - queryMode: "local", - labelAlign: "left", - displayField: "value", - valueField: "value", - anchor: "100%", - height: 150, - store: new Ext.data.ArrayStore({ - fields: ["value"], - data: [], - }), - }); - - me.variableFieldCreator = Ext.create("Ext.form.field.Checkbox", { - fieldLabel: "Variable field", - labelAlign: "left", - }); - - me.plotManagementFieldCreator.add([me.restrictByFieldCreator, me.valuesFieldCreator, me.variableFieldCreator]); - - // ------------------------------------------------------------------------------------- - - me.plotManagementViewOptions = new Ext.create("Ext.panel.Panel", { - region: "east", - floatable: false, - layout: "anchor", - header: false, - width: 350, - bodyPadding: 5, - dockedItems: [ - { - dock: "top", - xtype: "toolbar", - border: false, - layout: { - pack: "center", - }, - items: [ - "->", - { - xtype: "button", - text: "Test View", - handler: function () { - me.oprTestView(); - }, - }, - ], - }, - ], - }); - - me.checkGroupByViewOptions = new Ext.create("Ext.form.CheckboxGroup", { - columns: 1, - vertical: true, - items: [ - { - boxLabel: "Site", - name: "rb", - inputValue: "sources.site", - }, - { - boxLabel: "Component type", - name: "rb", - inputValue: "sources.componentType", - checked: true, - }, - { - boxLabel: "Component location", - name: "rb", - inputValue: "sources.componentLocation", - }, - { - boxLabel: "Component name", - name: "rb", - inputValue: "sources.componentName", - }, - { - boxLabel: "Activity", - name: "rb", - inputValue: "activities.description", - }, - { - boxLabel: "Activity category", - name: "rb", - inputValue: "activities.category", - }, - ], - }); - - me.txtActivityLabelViewOptions = Ext.create("Ext.form.field.Text", { - fieldLabel: "Activities label", - labelAlign: "left", - value: "$DESCRIPTION", - anchor: "100%", - }); - - me.txtViewNameViewOptions = Ext.create("Ext.form.field.Text", { - fieldLabel: "View name", - labelAlign: "left", - anchor: "100%", - }); - - me.txtStackActivityViewOptions = Ext.create("Ext.form.field.Checkbox", { - fieldLabel: "Stack activities", - labelAlign: "left", - }); - - me.plotManagementViewOptions.add([ - { - xtype: "fieldset", - columnWidth: 0.5, - title: "Group plots by", - collapsible: true, - items: [me.checkGroupByViewOptions], - }, - me.txtActivityLabelViewOptions, - me.txtStackActivityViewOptions, - me.txtViewNameViewOptions, - ]); - - // ------------------------------------------------------------------------------------- - - me.contextTreeMenu = new Ext.menu.Menu({ - width: 150, - items: [ - { - text: "Delete", - iconCls: "dirac-icon-delete", - listeners: { - click: function () { - var oNode = me.contextTreeMenu.node; - - if (oNode.isLeaf()) { - oNode = oNode.parentNode; - } - - var sType = oNode.data.text; - - me.restrictByFieldCreator.store.add({ - value: me.viewDefinitionData[sType].value, - text: sType, - }); - - if (me.viewDefinitionData[sType].value in me.viewDefinitionDataForServer) { - delete me.viewDefinitionDataForServer[me.viewDefinitionData[sType].value]; - } else { - Ext.Array.remove(me.viewDefinitionDataForServerVariable, me.viewDefinitionData[sType].value); - } - - delete me.viewDefinitionData[sType]; - - var oParentNode = oNode.parentNode; - oParentNode.removeChild(oNode); - }, - }, - }, - ], - moduleObject: me, - }); - - me.plotManagementViewTreeStore = Ext.create("Ext.data.TreeStore", { - proxy: { - type: "localstorage", - id: me.id + "treeStoreLocalStorage", - }, - root: { - text: "View Definition", - }, - }); - - me.plotManagementViewTree = new Ext.create("Ext.tree.Panel", { - region: "center", - store: me.plotManagementViewTreeStore, - header: false, - listeners: { - beforeitemcontextmenu: function (oView, oNode, item, index, e, eOpts) { - e.preventDefault(); - if (oNode.data.id != "root") { - me.contextTreeMenu.node = oNode; - me.contextTreeMenu.showAt(e.getXY()); - } - }, - }, - }); - - // ------------------------------------------------------------------------------------- - oTopPanel.add([me.plotManagementFieldCreator, me.plotManagementViewOptions, me.plotManagementViewTree]); - me.plotManagementMainPanel.add([me.plotManagementListPanel, oTopPanel]); - - me.mainPanel.add(me.plotManagementMainPanel); - }, - - __buildPlotViewer: function () { - var me = this; - - me.plotViewerMainPanel = new Ext.create("Ext.panel.Panel", { - floatable: false, - layout: "border", - header: false, - border: false, - defaults: { - collapsible: true, - split: true, - }, - }); - - Ext.Ajax.request({ - url: GLOBAL.BASE_URL + "ActivityMonitor/getStaticPlotViews", - scope: me, - success: function (response) { - var me = this; - var response = Ext.JSON.decode(response.responseText); - - me.plotViewerListPanel.reconfigure( - new Ext.data.ArrayStore({ - fields: ["id", "name", "variable_fields"], - data: response.result, - }), - undefined - ); - }, - }); - - me.plotViewerListPanel = Ext.create("Ext.grid.Panel", { - region: "west", - width: 250, - store: null, - header: false, - stateful: true, - stateId: "ActivityViewList", - viewConfig: { - stripeRows: true, - enableTextSelection: true, - }, - columns: [ - { - header: "Click on a view to plot it", - sortable: true, - dataIndex: "name", - flex: 1, - }, - ], - listeners: { - cellclick: function (oTable, td, cellIndex, record, tr, rowIndex, e, eOpts) { - me.plotViewerResultPanel.removeAll(); - - var oNewPlots = me.__buildPlotView(record.get("name"), ""); - - me.plotViewerResultPanel.add(oNewPlots); - }, - }, - }); - - me.plotViewerResultPanel = new Ext.create("Ext.panel.Panel", { - floatable: false, - region: "center", - layout: "border", - header: false, - border: false, - items: [], - }); - - me.plotViewerMainPanel.add([me.plotViewerListPanel, me.plotViewerResultPanel]); - me.mainPanel.add([me.plotViewerMainPanel]); - }, - - __buildPlotView: function (sViewId, sVariableData) { - var me = this; - - var oMainPanel = new Ext.create("Ext.panel.Panel", { - floatable: false, - region: "center", - layout: "border", - header: false, - items: [], - viewId: sViewId, - }); - - var oLeftPanel = new Ext.create("Ext.panel.Panel", { - title: "Activity view options", - floatable: false, - region: "west", - layout: "anchor", - width: 300, - bodyPadding: 10, - items: [], - }); - - var oRightPanel = new Ext.create("Ext.panel.Panel", { - floatable: false, - region: "center", - header: false, - autoScroll: true, - layout: { - type: "table", - columns: 1, - }, - }); - - var oCalenFrom = new Ext.create("Ext.form.field.Date", { - width: 100, - format: "Y-m-d", - fieldLabel: "From", - labelAlign: "left", - hidden: true, - anchor: "100%", - }); - - var oCalenTo = new Ext.create("Ext.form.field.Date", { - width: 100, - format: "Y-m-d", - fieldLabel: "To", - labelAlign: "left", - hidden: true, - anchor: "100%", - }); - - var oTimeSpan = new Ext.create("Ext.form.field.ComboBox", { - labelAlign: "left", - fieldLabel: "Time Span", - editable: false, - store: new Ext.data.ArrayStore({ - fields: ["value", "text"], - data: [ - [3600, "Last Hour"], - [86400, "Last Day"], - [604800, "Last Week"], - [2592000, "Last Month"], - [-1, "Manual Selection"], - ], - }), - displayField: "text", - valueField: "value", - anchor: "100%", - value: 86400, - listeners: { - change: function (field, newValue, oldValue, eOpts) { - oCalenFrom.hide(); - oCalenTo.hide(); - - switch (newValue) { - case -1: - oCalenFrom.show(); - oCalenTo.show(); - break; - } - }, - }, - }); - - var oPlotSize = new Ext.create("Ext.form.field.ComboBox", { - labelAlign: "left", - fieldLabel: "Plot Size", - editable: false, - store: new Ext.data.ArrayStore({ - fields: ["value", "text"], - data: [ - [0, "Small"], - [1, "Medium"], - [2, "Big"], - [3, "Very Big"], - ], - }), - displayField: "text", - valueField: "value", - anchor: "100%", - value: 1, - }); - - oLeftPanel.add([oTimeSpan, oCalenFrom, oCalenTo, oPlotSize]); - - var oToolbar = new Ext.create("Ext.toolbar.Toolbar", { - dock: "bottom", - border: false, - layout: { - pack: "center", - }, - items: [], - }); - - var oSubmitBtn = new Ext.Button({ - text: "Submit", - margin: 3, - iconCls: "dirac-icon-submit", - handler: function () { - oRightPanel.setLoading(true); - var oParams = {}; - var bValid = true; - - oParams["id"] = sViewId; - - if (oTimeSpan.getValue() == -1) { - if (oCalenFrom.getValue() == null || Ext.Date.format(oCalenFrom.getValue(), "Y-m-d") == "") { - GLOBAL.APP.CF.alert("Select a from date", "warning"); - bValid = false; - } - - if (oCalenTo.getValue() == null || Ext.Date.format(oCalenTo.getValue(), "Y-m-d") == "") { - GLOBAL.APP.CF.alert("Select a to date", "warning"); - bValid = false; - } - - oParams["timespan"] = -1; - oParams["fromDate"] = Ext.Date.format(oCalenFrom.getValue(), "Y-m-d"); - oParams["toDate"] = Ext.Date.format(oCalenTo.getValue(), "Y-m-d"); - } else { - oParams["timespan"] = oTimeSpan.getValue(); - } - - if (sVariableData) { - oParams["varData"] = Ext.encode(sVariableData); - } - - oParams["size"] = oPlotSize.getValue(); - - if (bValid) { - Ext.Ajax.request({ - url: GLOBAL.BASE_URL + "ActivityMonitor/plotView", - params: oParams, - scope: me, - success: function (response) { - oRightPanel.setLoading(false); - var me = this; - var response = Ext.JSON.decode(response.responseText); - - if (response.success == "true") { - var plotsList = response.data; - if (plotsList.length) { - oRightPanel.removeAll(); - - for (var i = 0; i < plotsList.length; i++) { - var oNewImage = Ext.create("Ext.Img", { - region: "center", - src: GLOBAL.BASE_URL + "ActivityMonitor/getPlotImg?file=" + plotsList[i], - }); - - oRightPanel.add(oNewImage); - } - } - } else { - GLOBAL.APP.CF.alert(response.error, "warning"); - } - }, - }); - } - }, - }); - - var oResetBtn = new Ext.Button({ - text: "Reset", - margin: 3, - iconCls: "dirac-icon-reset", - handler: function () {}, - }); - - oToolbar.add([oSubmitBtn, oResetBtn]); - - oLeftPanel.addDocked([oToolbar]); - - oMainPanel.add([oLeftPanel, oRightPanel]); - - return oMainPanel; - }, - - generateViewRequest: function () { - var me = this; - - if (me.viewDefinitionDataForServer.length == 0) { - GLOBAL.APP.CF.alert("Select at least one non variable restriction", "warning"); - return; - } - - var oGrouping = me.checkGroupByViewOptions.getValue().rb; - - var sActDesc = me.txtActivityLabelViewOptions.getValue(); - if (!sActDesc) { - GLOBAL.APP.CF.alert("Write the activities description", "warning"); - return; - } - - var bStack = false; - if (me.txtStackActivityViewOptions.getValue()) var bStack = true; - - return Ext.JSON.encode({ - groupBy: oGrouping, - definition: me.viewDefinitionDataForServer, - variable: me.viewDefinitionDataForServerVariable, - stacked: bStack, - label: sActDesc, - }); - }, - - oprTestView: function () { - var me = this; - - var sViewRequest = me.generateViewRequest(); - - if (!sViewRequest) return; - - var sViewName = Ext.util.Format.trim(me.txtViewNameViewOptions.getValue()); - - if (sViewName.length == 0) { - GLOBAL.APP.CF.alert("The name of the view is missing.", "warning"); - return; - } - - me.getContainer().body.mask("Wait ..."); - Ext.Ajax.request({ - url: GLOBAL.BASE_URL + "ActivityMonitor/tryView", - success: function (response) { - var response = Ext.JSON.decode(response.responseText); - - if (response.success == "true") { - var plotsList = response.images; - if (plotsList.length) { - var oWindow = me.getContainer().createChildWindow(sViewName, false, 700, 500); - - var oPlotContainer = Ext.create("Ext.panel.Panel", { - floatable: false, - region: "center", - header: false, - autoScroll: true, - layout: { - type: "table", - columns: 1, - }, - dockedItems: [ - { - xtype: "toolbar", - dock: "top", - items: [ - { - xtype: "button", - text: "Save view", - handler: function () { - var sViewRequest = me.generateViewRequest(); - - if (!sViewRequest) return; - - var sViewName = Ext.util.Format.trim(me.txtViewNameViewOptions.getValue()); - - if (sViewName.length == 0) { - GLOBAL.APP.CF.alert("The name of the view is missing.", "warning"); - return; - } - - me.getContainer().body.mask("Wait ..."); - Ext.Ajax.request({ - url: GLOBAL.BASE_URL + "ActivityMonitor/saveView", - success: function (response) { - var response = Ext.JSON.decode(response.responseText); - - if (response.success == "true") { - me.plotManagementListPanel.store.load(); - } else { - GLOBAL.APP.CF.alert("Error: " + response.error, "warning"); - } - - me.getContainer().body.unmask(); - }, - failure: function (oRequest) { - GLOBAL.APP.CF.showAjaxErrorMessage(response); - me.getContainer().body.unmask(); - }, - params: { - plotRequest: sViewRequest, - viewName: sViewName, - }, - }); - }, - }, - ], - }, - ], - }); - - for (var i = 0; i < plotsList.length; i++) { - var oNewImage = Ext.create("Ext.Img", { - src: GLOBAL.BASE_URL + "ActivityMonitor/getPlotImg?file=" + plotsList[i], - }); - - oPlotContainer.add(oNewImage); - } - - oWindow.add(oPlotContainer); - - oWindow.show(); - } - } else { - GLOBAL.APP.CF.alert(response.error, "warning"); - } - me.getContainer().body.unmask(); - }, - failure: function (oRequest) { - GLOBAL.APP.CF.alert("Error: " + oRequest.statusText, "warning"); - me.getContainer().body.unmask(); - }, - params: { - plotRequest: sViewRequest, - timeLength: "day", - viewName: sViewName, - }, - }); - }, - __buildSystemViewer: function () { - var me = this; - - me.systemPlotViewerMainPanel = new Ext.create("Ext.panel.Panel", { - floatable: false, - layout: "border", - header: false, - border: false, - defaults: { - collapsible: true, - split: true, - }, - }); - - me.systemPlotsTreeStore = Ext.create("Ext.data.TreeStore", { - model: "DIRAC.ActivityMonitor.classes.ActivityTreeModel", - fields: ["text", "component"], - scope: me, - proxy: { - type: "ajax", - url: GLOBAL.BASE_URL + "ActivityMonitor/getDynamicPlotViews", - }, - root: { - text: "/", - id: "/Systems", - expanded: true, - }, - folderSort: true, - sorters: [ - { - property: "text", - direction: "ASC", - }, - ], - listeners: { - collapse: function (oNode, eOpts) { - me.__oprUnsetPathAsExpanded(me.__getNodePath(oNode), true); - }, - expand: function (oNode, eOpts) { - me.__oprPathAsExpanded(me.__getNodePath(oNode), true); - }, - scope: me, - }, - }); - - me.systemPlotViewerTreePanel = Ext.create("Ext.tree.TreePanel", { - region: "west", - width: 250, - store: me.systemPlotsTreeStore, - listeners: { - itemclick: function (record, item, index, e, eOpts) { - console.log(record); - if (item.get("component") != "") { - me.systemPlotViewerResultPanel.removeAll(); - var varData = { - "sources.componentName": item.get("component"), - }; - var oNewPlots = me.__buildPlotView("Dynamic component view", varData); - - me.systemPlotViewerResultPanel.add(oNewPlots); - } - }, - }, - }); - - me.systemPlotViewerResultPanel = new Ext.create("Ext.panel.Panel", { - floatable: false, - region: "center", - layout: "border", - header: false, - border: false, - items: [], - }); - - me.systemPlotViewerMainPanel.add([me.systemPlotViewerTreePanel, me.systemPlotViewerResultPanel]); - me.mainPanel.add([me.systemPlotViewerMainPanel]); - }, - __oprUnsetPathAsExpanded: function (sPath) { - var me = this; - var oParts = sPath.split("/"); - - // The first element is always empty - var oTemp = me.expansionState; - var oStartIndex = 0; - - if (sPath == "/") oStartIndex = 1; - - for (var i = oStartIndex; i < oParts.length; i++) { - if (oParts[i] in oTemp) { - if (i == oParts.length - 1) { - delete oTemp[oParts[i]]; - } else { - oTemp = oTemp[oParts[i]]; - } - } - } - }, - - __oprPathAsExpanded: function (sPath, bInsertIntoStructure) { - var me = this; - var oParts = sPath.split("/"); - - // The first element is always empty - var oTemp = me.expansionState; - var oFound = true; - - var oStartIndex = 0; - - if (sPath == "/") oStartIndex = 1; - - for (var i = oStartIndex; i < oParts.length; i++) { - if (oParts[i] in oTemp) { - oTemp = oTemp[oParts[i]]; - } else { - oFound = false; - - if (bInsertIntoStructure) { - oTemp[oParts[i]] = {}; - } - - break; - } - } - - return oFound; - }, - __getNodePath: function (oNode) { - var sPath = ""; - var oCopyRefNode = oNode; - while (oCopyRefNode) { - if (oCopyRefNode.data.id) sPath = "/" + oCopyRefNode.data.id + sPath; - else break; - oCopyRefNode = oCopyRefNode.parentNode; - } - if (!sPath) return "/"; - return sPath; - }, -}); diff --git a/src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/classes/ActivityTreeModel.js b/src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/classes/ActivityTreeModel.js deleted file mode 100644 index b824f1e5d..000000000 --- a/src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/classes/ActivityTreeModel.js +++ /dev/null @@ -1,5 +0,0 @@ -Ext.define("DIRAC.ActivityMonitor.classes.ActivityTreeModel", { - extend: "Ext.data.Model", - fields: ["text", "component"], - alias: "widget.activityTreemodel", -}); diff --git a/src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/css/ActivityMonitor.css b/src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/css/ActivityMonitor.css deleted file mode 100644 index 0cd02ef76..000000000 --- a/src/WebAppDIRAC/WebApp/static/DIRAC/ActivityMonitor/css/ActivityMonitor.css +++ /dev/null @@ -1,3 +0,0 @@ -.am-main-check-box { - margin: 3px 0px 3px 0px !important; -} From 789aace582689136a8e81920112df97f74d90be4 Mon Sep 17 00:00:00 2001 From: rupozzi Date: Wed, 23 Mar 2022 09:31:11 +0100 Subject: [PATCH 2/5] fix: Undoing previous commit --- .../WebApp/handler/MonitoringHandler.py | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 src/WebAppDIRAC/WebApp/handler/MonitoringHandler.py diff --git a/src/WebAppDIRAC/WebApp/handler/MonitoringHandler.py b/src/WebAppDIRAC/WebApp/handler/MonitoringHandler.py new file mode 100644 index 000000000..99b37929f --- /dev/null +++ b/src/WebAppDIRAC/WebApp/handler/MonitoringHandler.py @@ -0,0 +1,257 @@ +import os +import json +import tempfile +import datetime + +from DIRAC import gConfig, S_OK, S_ERROR, gLogger +from DIRAC.Core.DISET.TransferClient import TransferClient +from DIRAC.Core.Utilities import Time, List, DictCache +from DIRAC.Core.Utilities.Plotting.FileCoding import extractRequestFromFileId, codeRequestInFileId +from DIRAC.MonitoringSystem.Client.MonitoringClient import MonitoringClient + +from WebAppDIRAC.Lib.WebHandler import _WebHandler as WebHandler, FileResponse + + +class MonitoringHandler(WebHandler): + + DEFAULT_AUTHORIZATION = "authenticated" + __keysCache = DictCache.DictCache() + + def __getUniqueKeyValues(self, typeName): + cacheKey = (self.getUserName(), self.getUserGroup(), self.getUserSetup(), typeName) + if not (data := MonitoringHandler.__keysCache.get(cacheKey)): + retVal = MonitoringClient().listUniqueKeyValues(typeName) + if "rpcStub" in retVal: + del retVal["rpcStub"] + if not retVal["OK"]: + return retVal + + # Site ordering based on TierLevel / alpha + if "Site" in retVal["Value"]: + siteLevel = {} + for siteName in retVal["Value"]["Site"]: + sitePrefix = siteName.split(".")[0].strip() + level = gConfig.getValue("/Resources/Sites/%s/%s/MoUTierLevel" % (sitePrefix, siteName), 10) + if level not in siteLevel: + siteLevel[level] = [] + siteLevel[level].append(siteName) + orderedSites = [] + for level in sorted(siteLevel): + orderedSites.extend(sorted(siteLevel[level])) + retVal["Value"]["Site"] = orderedSites + data = retVal + MonitoringHandler.__keysCache.add(cacheKey, 300, data) + return data + + def web_getSelectionData(self, monitoringType): + callback = {} + # Get unique key values + if not (retVal := self.__getUniqueKeyValues(monitoringType))["OK"]: + return {"success": "false", "result": "", "error": retVal["Message"]} + + records = {} + for record in retVal["Value"]: # may have more than 1000 of records. + # do not show all of them in the web portal + length = len(retVal["Value"][record]) + if length > 10000: + records[record] = retVal["Value"][record][length - 5000 :] + message = f"The {monitoringType} monitoring type contains too many rows: {record} -> {length}" + message += " Note: Only 5000 rows are returned!" + gLogger.warn(message) + else: + records[record] = retVal["Value"][record] + callback["selectionValues"] = records + + # Cache for plotsList? + if not (data := MonitoringHandler.__keysCache.get(f"reportsList:{monitoringType}")): + if not (retVal := MonitoringClient().listReports(monitoringType))["OK"]: + return {"success": "false", "result": "", "error": retVal["Message"]} + data = retVal["Value"] + MonitoringHandler.__keysCache.add(f"reportsList:{monitoringType}", 300, data) + callback["plotsList"] = data + return {"success": "true", "result": callback} + + def __parseFormParams(self): + pD = {} + extraParams = {} + pinDates = False + + for name in self.request.arguments: + pD[name] = self.get_argument(name) + + # Personalized title? + if "plotTitle" in pD: + extraParams["plotTitle"] = pD["plotTitle"] + del pD["plotTitle"] + # Pin dates? + if "pinDates" in pD: + pinDates = pD["pinDates"] + del pD["pinDates"] + pinDates = pinDates.lower() in ("yes", "y", "true", "1") + # Get plotname + if "grouping" not in pD: + return S_ERROR("Missing grouping!") + grouping = pD["grouping"] + # Get plotname + if "typeName" not in pD: + return S_ERROR("Missing type name!") + typeName = pD["typeName"] + del pD["typeName"] + # Get plotname + if "plotName" not in pD: + return S_ERROR("Missing plot name!") + reportName = pD["plotName"] + del pD["plotName"] + # Get times + if "timeSelector" not in pD: + return S_ERROR("Missing time span!") + # Find the proper time! + pD["timeSelector"] = int(pD["timeSelector"]) + if pD["timeSelector"] > 0: + end = Time.dateTime() + start = end - datetime.timedelta(seconds=pD["timeSelector"]) + if not pinDates: + extraParams["lastSeconds"] = pD["timeSelector"] + else: + if "endTime" not in pD: + end = False + else: + end = Time.fromString(pD["endTime"]) + del pD["endTime"] + if "startTime" not in pD: + return S_ERROR("Missing starTime!") + else: + start = Time.fromString(pD["startTime"]) + del pD["startTime"] + del pD["timeSelector"] + + for k in pD: + if k.find("ex_") == 0: + extraParams[k[3:]] = pD[k] + + # Selection data + data = {} + + # Listify the rest + for selName in pD: + if selName == "grouping": + pD[selName] = [pD[selName]] + else: + try: + pD[selName] = json.loads(pD[selName]) + except ValueError: + pD[selName] = List.fromChar(pD[selName], ",") + + # If json parse value as string, listify it + if isinstance(pD[selName], str): + pD[selName] = List.fromChar(pD[selName], ",") + + # Convert 'value*' to list of values that starts with 'value' + fullList = [] + for value in pD[selName]: + if value.endswith("*"): + if not data: + retVal = self.__getUniqueKeyValues(typeName) + if not retVal["OK"]: + return retVal + data = retVal["Value"] + for v in data[selName]: + if v.startswith(value[:-1]): + fullList.append(v) + else: + fullList.append(value) + + pD[selName] = fullList + + return S_OK((typeName, reportName, start, end, pD, grouping, extraParams)) + + def web_generatePlot(self): + if (retVal := self.__queryForPlot())["OK"]: + return {"success": "true", "data": retVal["Value"]["plot"]} + return {"success": "false", "errors": retVal["Message"]} + + def __queryForPlot(self): + res = self.__parseFormParams() + return MonitoringClient().generateDelayedPlot(*res["Value"]) if res["OK"] else res + + def web_getPlotImg(self, fileName=None): + """Get plot image""" + if not fileName: + return {"success": "false", "error": "Maybe you forgot the file?"} + # Prevent directory traversal + plotImageFile = os.path.normpath("/" + fileName).lstrip("/") + + transferClient = TransferClient("Monitoring/Monitoring") + tempFile = tempfile.TemporaryFile() + retVal = transferClient.receiveFile(tempFile, plotImageFile) + if not retVal["OK"]: + return {"success": "false", "error": retVal["Message"]} + tempFile.seek(0) + data = tempFile.read() + return FileResponse(data, plotImageFile, ext="png") + + def web_getPlotImgFromCache(self, fileName=None): + """Get plot image from cache.""" + if not fileName: + return {"success": "false", "error": "Maybe you forgot the file?"} + + retVal = extractRequestFromFileId(fileName) + if not retVal["OK"]: + return {"success": "false", "error": retVal["Message"]} + fields = retVal["Value"] + if "extraArgs" in fields: # in order to get the plot from the cache we have to clean the extraArgs... + plotTitle = "" + if "plotTitle" in fields["extraArgs"]: + plotTitle = fields["extraArgs"]["plotTitle"] + fields["extraArgs"] = {} + fields["extraArgs"]["plotTitle"] = plotTitle + else: + fields["extraArgs"] = {} + + retVal = codeRequestInFileId(fields) + if not retVal["OK"]: + return {"success": "false", "error": retVal["Message"]} + plotImageFile = retVal["Value"]["plot"] + + transferClient = TransferClient("Monitoring/Monitoring") + tempFile = tempfile.TemporaryFile() + if not (retVal := transferClient.receiveFile(tempFile, plotImageFile))["OK"]: + return {"success": "false", "error": retVal["Message"]} + tempFile.seek(0) + data = tempFile.read() + return FileResponse(data, plotImageFile, ext="png", cache=False) + + def web_getCsvPlotData(self): + if not (retVal := self.__parseFormParams())["OK"]: + return {"success": "false", "error": retVal["Message"]} + params = retVal["Value"] + if not (retVal := MonitoringClient().getReport(*params))["OK"]: + return {"success": "false", "error": retVal["Message"]} + rawData = retVal["Value"] + groupKeys = sorted(rawData["data"]) + if "granularity" in rawData: + granularity = rawData["granularity"] + data = rawData["data"] + tS = int(Time.toEpoch(params[2])) + timeStart = tS - tS % granularity + strData = "epoch,%s\n" % ",".join(groupKeys) + for timeSlot in range(timeStart, int(Time.toEpoch(params[3])), granularity): + lineData = [str(timeSlot)] + for key in groupKeys: + if timeSlot in data[key]: + lineData.append(str(data[key][timeSlot])) + else: + lineData.append("") + strData += "%s\n" % ",".join(lineData) + else: + strData = "%s\n" % ",".join(groupKeys) + strData += ",".join([str(rawData["data"][k]) for k in groupKeys]) + return FileResponse(strData, str(params), ext="csv", cache=False) + + def web_getPlotData(self): + if not (retVal := self.__parseFormParams())["OK"]: + return {"success": "false", "error": retVal["Message"]} + params = retVal["Value"] + if not (retVal := MonitoringClient().getReport(*params))["OK"]: + return {"success": "false", "error": retVal["Message"]} + return retVal["Value"]["data"] From 72c9712b8a1322563991bb0b5f4ba59f27b76cd5 Mon Sep 17 00:00:00 2001 From: rupozzi Date: Fri, 25 Mar 2022 15:13:53 +0100 Subject: [PATCH 3/5] fix: Fixing wrong usage of gMonitor --- src/WebAppDIRAC/WebApp/handler/SystemAdministrationHandler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/WebAppDIRAC/WebApp/handler/SystemAdministrationHandler.py b/src/WebAppDIRAC/WebApp/handler/SystemAdministrationHandler.py index b53dd3b92..cca3883b5 100644 --- a/src/WebAppDIRAC/WebApp/handler/SystemAdministrationHandler.py +++ b/src/WebAppDIRAC/WebApp/handler/SystemAdministrationHandler.py @@ -6,10 +6,10 @@ from DIRAC import gConfig, gLogger from DIRAC.Core.Utilities.List import uniqueElements -from DIRAC.FrameworkSystem.Client.MonitoringClient import gMonitor from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import SystemAdministratorClient from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient +from DIRAC.FrameworkSystem.DB.ComponentMonitoringDB import ComponentMonitoringDB from WebAppDIRAC.Lib.WebHandler import WebHandler, asyncGen @@ -699,7 +699,7 @@ def web_ComponentLocation(self): condDict = {"Setup": _setup} gLogger.debug("condDict" + str(condDict)) - retVal = gMonitor.getComponentsStatus(condDict) + retVal = ComponentMonitoringDB.getComponentsStatus(condDict) today = datetime.datetime.today() if retVal["OK"]: From 810b76005762645f1bcfe2d070f5f7cf446fd416 Mon Sep 17 00:00:00 2001 From: rupozzi Date: Wed, 30 Mar 2022 13:39:07 +0200 Subject: [PATCH 4/5] fix: removing getActivitiesContent --- src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py b/src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py index 58fa8f83d..fd18e647d 100644 --- a/src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py +++ b/src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py @@ -23,9 +23,6 @@ def web_getActivityData( if sortField and sortDirection: sort = [(sortField.replace("_", "."), sortDirection)] - if not (result := MonitoringClient().getActivitiesContents({}, sort, start, limit))["OK"]: - return {"success": "false", "result": [], "total": -1, "error": result["Message"]} - svcData = result["Value"] callback = {"success": "true", "total": svcData["TotalRecords"], "result": []} now = Time.toEpoch() From 1f395d3096ae747d57e46791590a3b6703528563 Mon Sep 17 00:00:00 2001 From: rupozzi Date: Wed, 30 Mar 2022 13:51:49 +0200 Subject: [PATCH 5/5] fix: Undoing previous error --- src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py b/src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py index fd18e647d..58fa8f83d 100644 --- a/src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py +++ b/src/WebAppDIRAC/WebApp/handler/ActivityMonitorHandler.py @@ -23,6 +23,9 @@ def web_getActivityData( if sortField and sortDirection: sort = [(sortField.replace("_", "."), sortDirection)] + if not (result := MonitoringClient().getActivitiesContents({}, sort, start, limit))["OK"]: + return {"success": "false", "result": [], "total": -1, "error": result["Message"]} + svcData = result["Value"] callback = {"success": "true", "total": svcData["TotalRecords"], "result": []} now = Time.toEpoch()