From 4ad573a81cd58f4d7917db60e05024626203a3c2 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 19 Oct 2021 14:54:02 +0200 Subject: [PATCH] refactor: Run pre-commit --- .eslintignore | 2 - .eslintrc | 52 +- .github/CONTRIBUTING.md | 14 +- .github/workflows/codestyle.yml | 40 +- .github/workflows/deployment.yml | 2 +- .prettierignore | 2 +- .prettierrc | 2 +- release.notes | 2 +- src/WebAppDIRAC/Core/App.py | 273 ++-- src/WebAppDIRAC/Core/CoreHandler.py | 41 +- src/WebAppDIRAC/Core/HandlerMgr.py | 274 ++-- src/WebAppDIRAC/Core/StaticHandler.py | 31 +- src/WebAppDIRAC/Core/TemplateLoader.py | 37 +- src/WebAppDIRAC/Lib/Conf.py | 392 ++--- src/WebAppDIRAC/Lib/SessionData.py | 337 ++-- src/WebAppDIRAC/Lib/WebHandler.py | 641 ++++---- .../WebApp/handler/AccountingHandler.py | 456 +++--- .../WebApp/handler/ActivityMonitorHandler.py | 526 +++--- .../handler/ApplicationWizardHandler.py | 2 +- .../WebApp/handler/ComponentHistoryHandler.py | 257 +-- .../handler/ConfigurationManagerHandler.py | 1221 +++++++------- .../WebApp/handler/DowntimesHandler.py | 156 +- .../WebApp/handler/ExampleAppHandler.py | 31 +- .../WebApp/handler/FileCatalogHandler.py | 833 +++++----- .../WebApp/handler/JobLaunchpadHandler.py | 470 +++--- .../WebApp/handler/JobMonitorHandler.py | 1093 ++++++------- .../WebApp/handler/JobSummaryHandler.py | 793 +++++----- .../WebApp/handler/MonitoringHandler.py | 509 +++--- .../WebApp/handler/NotepadHandler.py | 8 +- .../WebApp/handler/PilotMonitorHandler.py | 607 +++---- .../WebApp/handler/PilotSummaryHandler.py | 344 ++-- .../WebApp/handler/ProxyManagerHandler.py | 357 ++--- .../WebApp/handler/ProxyUploadHandler.py | 343 ++-- .../handler/PublicStateManagerHandler.py | 15 +- .../WebApp/handler/RegistryManagerHandler.py | 663 ++++---- .../WebApp/handler/RequestMonitorHandler.py | 397 ++--- .../WebApp/handler/ResourceSummaryHandler.py | 797 +++++----- src/WebAppDIRAC/WebApp/handler/RootHandler.py | 321 ++-- .../WebApp/handler/SiteSummaryHandler.py | 588 ++++--- .../WebApp/handler/SpaceOccupancyHandler.py | 108 +- .../handler/SystemAdministrationHandler.py | 1403 +++++++++-------- .../WebApp/handler/TokenManagerHandler.py | 108 +- .../handler/TransformationMonitorHandler.py | 975 ++++++------ src/WebAppDIRAC/WebApp/handler/UPHandler.py | 424 +++-- .../WebApp/handler/VMDiracHandler.py | 386 ++--- .../DIRAC/Accounting/classes/Accounting.js | 68 +- .../classes/ActivityMonitor.js | 414 ++--- .../classes/ActivityTreeModel.js | 2 +- .../ActivityMonitor/css/ActivityMonitor.css | 4 +- .../classes/ApplicationWizard.js | 66 +- .../DIRAC/ApplicationWizard/classes/Image.js | 4 +- .../ApplicationWizard/classes/Presenter.js | 82 +- .../classes/ComponentHistory.js | 34 +- .../classes/ConfigurationManager.js | 336 ++-- .../classes/HistoryGridPanel.js | 66 +- .../css/ConfigurationManager.css | 48 +- .../DIRAC/Downtimes/classes/Downtimes.js | 76 +- .../DIRAC/ExampleApp/classes/ExampleApp.js | 42 +- .../DIRAC/FileCatalog/classes/FileCatalog.js | 365 +++-- .../DIRAC/FileCatalog/css/FileCatalog.css | 89 +- .../JobLaunchpad/classes/JobLaunchpad.js | 144 +- .../DIRAC/JobMonitor/classes/JobMonitor.js | 548 +++---- .../DIRAC/JobMonitor/css/JobMonitor.css | 24 +- .../DIRAC/JobSummary/classes/JobSummary.js | 188 +-- .../static/DIRAC/Notepad/classes/Notepad.js | 16 +- .../PilotMonitor/classes/PilotMonitor.js | 342 ++-- .../DIRAC/PilotMonitor/css/PilotMonitor.css | 26 +- .../PilotSummary/classes/PilotSummary.js | 199 ++- .../ProxyManager/classes/ProxyManager.js | 94 +- .../DIRAC/ProxyUpload/classes/ProxyUpload.js | 44 +- .../PublicStateManager/classes/MenuGrid.js | 46 +- .../PublicStateManager/classes/MenuModel.js | 16 +- .../classes/PublicStateManager.js | 24 +- .../classes/RegistryManager.js | 450 +++--- .../RegistryManager/css/RegistryManager.css | 8 +- .../RequestMonitor/classes/RequestMonitor.js | 120 +- .../ResourceSummary/classes/OverviewPanel.js | 200 +-- .../classes/ResourceSummary.js | 250 +-- .../ResourceSummary/classes/TreeModel.js | 2 +- .../SiteSummary/classes/OverviewPanel.js | 234 +-- .../DIRAC/SiteSummary/classes/SiteSummary.js | 212 +-- .../SpaceOccupancy/classes/SpaceOccupancy.js | 88 +- .../classes/SystemAdministration.js | 708 ++++----- .../css/SystemAdministration.css | 4 +- .../TokenManager/classes/TokenManager.js | 86 +- .../classes/TransformationMonitor.js | 575 ++++--- .../static/DIRAC/VMDirac/classes/VMDirac.js | 252 +-- .../static/DIRAC/VMDirac/css/VMDirac.css | 2 +- .../WebApp/static/core/css/css.css | 365 +++-- .../WebApp/static/core/css/iconset.css | 274 ++-- .../WebApp/static/core/css/tabtheme.css | 68 +- .../WebApp/static/core/img/icons/fam/SILK.txt | 2 +- .../WebApp/static/core/js/core/App.js | 60 +- .../WebApp/static/core/js/core/AppView.js | 6 +- .../static/core/js/core/CommonFunctions.js | 78 +- .../WebApp/static/core/js/core/Container.js | 63 +- .../WebApp/static/core/js/core/Module.js | 18 +- .../static/core/js/core/StateManagement.js | 143 +- .../WebApp/static/core/js/core/Stateful.js | 8 +- .../static/core/js/core/TransformationData.js | 12 +- .../static/core/js/utils/DiracAjaxProxy.js | 8 +- .../js/utils/DiracApplicationContextMenu.js | 26 +- .../static/core/js/utils/DiracArrayStore.js | 14 +- .../static/core/js/utils/DiracBaseSelector.js | 108 +- .../static/core/js/utils/DiracBoxSelect.js | 8 +- .../static/core/js/utils/DiracFileLoad.js | 14 +- .../static/core/js/utils/DiracGridPanel.js | 103 +- .../static/core/js/utils/DiracIdListButton.js | 34 +- .../static/core/js/utils/DiracJsonStore.js | 12 +- .../static/core/js/utils/DiracMultiSelect.js | 24 +- .../static/core/js/utils/DiracNumericField.js | 10 +- .../core/js/utils/DiracPageSizeCombo.js | 4 +- .../core/js/utils/DiracPagingToolbar.js | 62 +- .../static/core/js/utils/DiracRowExpander.js | 10 +- .../static/core/js/utils/DiracTextField.js | 8 +- .../core/js/utils/DiracTimeSearchPanel.js | 70 +- .../static/core/js/utils/DiracToolButton.js | 38 +- .../core/js/utils/FileSaver/FileSaver.js | 2 +- .../WebApp/static/core/js/utils/GridPanel.js | 42 +- .../WebApp/static/core/js/utils/Image.js | 4 +- .../static/core/js/utils/Notification.js | 62 +- .../static/core/js/utils/PanelDragDrop.js | 34 +- .../WebApp/static/core/js/utils/PlotView.js | 174 +- .../WebApp/static/core/js/utils/Presenter.js | 150 +- .../WebApp/static/core/js/utils/Printer.js | 18 +- .../WebApp/static/core/js/utils/Tabtheme.js | 156 +- .../static/core/js/utils/WelcomeWindow.js | 6 +- .../core/js/utils/canvg-1.3/MIT-LICENSE.txt | 2 +- .../core/js/utils/canvg-1.3/StackBlur.js | 240 +-- .../static/core/js/utils/canvg-1.3/canvg.js | 966 ++++++------ .../core/js/utils/canvg-1.3/rgbcolor.js | 1 - .../core/js/utils/css/DiracBoxSelect.css | 84 +- .../static/core/js/utils/css/Notification.css | 22 +- .../static/core/js/utils/css/PlotView.css | 255 ++- .../WebApp/static/core/js/utils/css/print.css | 174 +- .../static/core/js/views/tabs/ContextMenu.js | 134 +- .../core/js/views/tabs/DesktopSettings.js | 40 +- .../WebApp/static/core/js/views/tabs/Image.js | 12 +- .../core/js/views/tabs/LeftContainer.js | 16 +- .../WebApp/static/core/js/views/tabs/Main.js | 383 ++--- .../static/core/js/views/tabs/MenuTabs.js | 32 +- .../WebApp/static/core/js/views/tabs/Panel.js | 56 +- .../core/js/views/tabs/PresenterView.js | 54 +- .../core/js/views/tabs/RightContainer.js | 122 +- .../static/core/js/views/tabs/SelPanel.js | 72 +- .../core/js/views/tabs/SettingsPanel.js | 84 +- .../core/js/views/tabs/StateManagement.js | 241 ++- .../core/js/views/tabs/StateManagerMenu.js | 48 +- .../static/core/js/views/tabs/TabPanel.js | 88 +- .../core/js/views/tabs/TabScrollerMenu.js | 36 +- .../core/js/views/tabs/TreeMenuModel.js | 2 +- .../static/core/js/views/tabs/Wallpaper.js | 16 +- .../ConfigurationManager/diffConfig.tpl | 1 - src/WebAppDIRAC/WebApp/template/root.tpl | 12 +- src/WebAppDIRAC/WebApp/web.cfg | 12 +- src/WebAppDIRAC/__init__.py | 58 +- src/WebAppDIRAC/scripts/dirac_webapp_run.py | 236 +-- src/WebAppDIRAC/scripts/tornado-start-web.py | 72 +- tests/.pylintrc3k | 2 +- 159 files changed, 14375 insertions(+), 14427 deletions(-) diff --git a/.eslintignore b/.eslintignore index 65bc03d03..0ac967ae4 100644 --- a/.eslintignore +++ b/.eslintignore @@ -8,5 +8,3 @@ WebApp/static/core/js/views/tabs/TabPanel.js WebApp/static/core/js/utils/DiracGridPanel.js WebAppDIRAC/WebApp/static/core/js/utils/Tabtheme.js WebApp/static/core/js/utils/Tabtheme.js - - diff --git a/.eslintrc b/.eslintrc index eb665e9c6..a5216d25e 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,30 +1,28 @@ { - "parserOptions": { - "ecmaVersion": 6, - }, - "plugins": [ - "@sencha/extjs" - ], - "extends": [ - // "eslint:recommended", - "plugin:@sencha/extjs/recommended" - ], - "settings": { - "extjs": { - // "version": "4" - "fromVersion": "4.2.6", - "toVersion": "6.6.0" - } - }, - "globals": { - "Ext": true, - "SimpleTasks": true, - "SimpleTasksSettings": true - }, - "env": { - "browser": true - }, - "rules": { - // "@sencha/extjs/no-removed-method-call": 2 + "parserOptions": { + "ecmaVersion": 6 + }, + "plugins": ["@sencha/extjs"], + "extends": [ + // "eslint:recommended", + "plugin:@sencha/extjs/recommended" + ], + "settings": { + "extjs": { + // "version": "4" + "fromVersion": "4.2.6", + "toVersion": "6.6.0" } + }, + "globals": { + "Ext": true, + "SimpleTasks": true, + "SimpleTasksSettings": true + }, + "env": { + "browser": true + }, + "rules": { + // "@sencha/extjs/no-removed-method-call": 2 + } } diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 024a075e6..c76806c99 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,7 +1,5 @@ - # Contribution Guidelines for DIRAC - ## Pull Requests When making a `Pull Request` please explain what and why things were @@ -22,11 +20,13 @@ If you notice an issue, consider first creating an issue and then refering to it in your pull-request and commit messages with `#[issue-id]`. ## Coding Conventions - For the python code: - * You should follow the [DIRAC Coding Conventions](https://dirac.readthedocs.io/en/latest/DeveloperGuide/CodingConvention/index.html). - * Your code should not introduce any new pylint warnings, and fix as many existing warnings as possible. - * Use autopep8 + +For the python code: + +- You should follow the [DIRAC Coding Conventions](https://dirac.readthedocs.io/en/latest/DeveloperGuide/CodingConvention/index.html). +- Your code should not introduce any new pylint warnings, and fix as many existing warnings as possible. +- Use autopep8 ## Git workflow - The DIRAC Development Model is described in the [documentation](https://dirac.readthedocs.io/en/latest/DeveloperGuide/DevelopmentModel/index.html) with detailed instructions on the git workflow listed [here](https://dirac.readthedocs.io/en/latest/DeveloperGuide/DevelopmentModel/ContributingCode/index.html). For additional help on the git(hub) workflow please see this [tutorial](https://github.com/andresailer/tutorial#working-updating-pushing). +The DIRAC Development Model is described in the [documentation](https://dirac.readthedocs.io/en/latest/DeveloperGuide/DevelopmentModel/index.html) with detailed instructions on the git workflow listed [here](https://dirac.readthedocs.io/en/latest/DeveloperGuide/DevelopmentModel/ContributingCode/index.html). For additional help on the git(hub) workflow please see this [tutorial](https://github.com/andresailer/tutorial#working-updating-pushing). diff --git a/.github/workflows/codestyle.yml b/.github/workflows/codestyle.yml index 1f97c181e..ad41d0245 100644 --- a/.github/workflows/codestyle.yml +++ b/.github/workflows/codestyle.yml @@ -7,14 +7,14 @@ jobs: runs-on: ubuntu-latest if: github.event_name != 'push' || github.repository == 'DIRACGrid/WebAppDIRAC' steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.9' - - name: Install pre-commit - run: pip install pre-commit - - name: Run pre-commit - run: pre-commit run --all-files --show-diff-on-failure + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.9" + - name: Install pre-commit + run: pip install pre-commit + - name: Run pre-commit + run: pre-commit run --all-files --show-diff-on-failure pyLint: runs-on: ubuntu-latest @@ -25,17 +25,17 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@v2 - - uses: actions/checkout@v2 - with: - repository: DIRACGrid/DIRAC - path: .DIRAC - ref: integration + - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + with: + repository: DIRACGrid/DIRAC + path: .DIRAC + ref: integration - - name: Prepare environment with Micromamba - uses: mamba-org/provision-with-micromamba@main - with: - environment-file: .DIRAC/environment.yml + - name: Prepare environment with Micromamba + uses: mamba-org/provision-with-micromamba@main + with: + environment-file: .DIRAC/environment.yml - - name: Run tests - run: pylint -E src/WebAppDIRAC + - name: Run tests + run: pylint -E src/WebAppDIRAC diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 1a9904bc5..8c99e8466 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -20,7 +20,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v2 with: - python-version: '3.9' + python-version: "3.9" - name: Install dependencies run: pip install build readme_renderer diraccfg packaging requests - name: Validate README for PyPI diff --git a/.prettierignore b/.prettierignore index 9c88ed8d2..89663c023 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1,2 @@ src/WebAppDIRAC/WebApp/static/core/js/utils/canvg-1.3 -src/WebAppDIRAC/WebApp/static/core/js/utils/FileSaver \ No newline at end of file +src/WebAppDIRAC/WebApp/static/core/js/utils/FileSaver diff --git a/.prettierrc b/.prettierrc index 78cdf226b..50ffd400a 100644 --- a/.prettierrc +++ b/.prettierrc @@ -2,4 +2,4 @@ "useTabs": false, "printWidth": 150, "singleQuote": false -} \ No newline at end of file +} diff --git a/release.notes b/release.notes index 4631bc3f7..e369eb2a0 100644 --- a/release.notes +++ b/release.notes @@ -145,4 +145,4 @@ CHANGE: (#408) add diraccfg to the requirements [v4r0p32] -FIX: (#405) PilotMonitor showJobs feature now provide all the jobs bound to a pilot-job reference \ No newline at end of file +FIX: (#405) PilotMonitor showJobs feature now provide all the jobs bound to a pilot-job reference diff --git a/src/WebAppDIRAC/Core/App.py b/src/WebAppDIRAC/Core/App.py index 66426bcdb..a1e6770ce 100644 --- a/src/WebAppDIRAC/Core/App.py +++ b/src/WebAppDIRAC/Core/App.py @@ -26,138 +26,147 @@ class App(object): - - def __init__(self, handlersLoc='WebApp.handler'): - self.__handlerMgr = HandlerMgr(handlersLoc, Conf.rootURL()) - self.__servers = {} - self.log = gLogger.getSubLogger("Web") - - def _logRequest(self, handler): - status = handler.get_status() - if status < 400: - logm = self.log.notice - elif status < 500: - logm = self.log.warn - else: - logm = self.log.error - request_time = 1000.0 * handler.request.request_time() - logm("%d %s %.2fms" % (status, handler._request_summary(), request_time)) - - def __reloadAppCB(self): - gLogger.notice("\n !!!!!! Reloading web app...\n") - - def stopChildProcesses(self, sig, frame): - """ - It is used to properly stop tornado when more than one process is used. - In principle this is doing the job of runsv.... - :param int sig: the signal sent to the process - :param object frame: execution frame which contains the child processes - """ - # tornado.ioloop.IOLoop.instance().add_timeout(time.time()+5, sys.exit) - for child in frame.f_locals.get('children', []): - gLogger.info("Stopping child processes: %d" % child) - os.kill(child, signal.SIGTERM) - # tornado.ioloop.IOLoop.instance().stop() - # gLogger.info('exit success') - sys.exit(0) - - def getAppToDict(self, port=None): - """ Load Web portals + def __init__(self, handlersLoc="WebApp.handler"): + self.__handlerMgr = HandlerMgr(handlersLoc, Conf.rootURL()) + self.__servers = {} + self.log = gLogger.getSubLogger("Web") + + def _logRequest(self, handler): + status = handler.get_status() + if status < 400: + logm = self.log.notice + elif status < 500: + logm = self.log.warn + else: + logm = self.log.error + request_time = 1000.0 * handler.request.request_time() + logm("%d %s %.2fms" % (status, handler._request_summary(), request_time)) + + def __reloadAppCB(self): + gLogger.notice("\n !!!!!! Reloading web app...\n") + + def stopChildProcesses(self, sig, frame): + """ + It is used to properly stop tornado when more than one process is used. + In principle this is doing the job of runsv.... + :param int sig: the signal sent to the process + :param object frame: execution frame which contains the child processes + """ + # tornado.ioloop.IOLoop.instance().add_timeout(time.time()+5, sys.exit) + for child in frame.f_locals.get("children", []): + gLogger.info("Stopping child processes: %d" % child) + os.kill(child, signal.SIGTERM) + # tornado.ioloop.IOLoop.instance().stop() + # gLogger.info('exit success') + sys.exit(0) + + def getAppToDict(self, port=None): + """Load Web portals :return: S_OK(dict)/S_ERROR() - """ - app = {'port': port or Conf.HTTPSPort()} - # Calculating routes - result = self.__handlerMgr.getRoutes() - if not result['OK']: - return result - app['routes'] = result['Value'] - # Initialize the session data - SessionData.setHandlers(self.__handlerMgr.getHandlers()['Value']) - # Create the app - tLoader = TemplateLoader(self.__handlerMgr.getPaths("template")) - app['settings'] = dict(debug=Conf.devMode(), template_loader=tLoader, - cookie_secret=str(Conf.cookieSecret()), - log_function=self._logRequest) - return S_OK(app) - - def bootstrap(self): - """ - Configure and create web app - """ - self.log.always("\n ====== Starting DIRAC web app ====== \n") - - # Calculating routes - result = self.__handlerMgr.getRoutes() - if not result['OK']: - return result - routes = result['Value'] - # Initialize the session data - SessionData.setHandlers(self.__handlerMgr.getHandlers()['Value']) - # Create the app - tLoader = TemplateLoader(self.__handlerMgr.getPaths("template")) - kw = dict(debug=Conf.devMode(), template_loader=tLoader, cookie_secret=str(Conf.cookieSecret()), - log_function=self._logRequest, autoreload=Conf.numProcesses() < 2) - - # please do no move this lines. The lines must be before the fork_processes - signal.signal(signal.SIGTERM, self.stopChildProcesses) - signal.signal(signal.SIGINT, self.stopChildProcesses) - - # Check processes if we're under a load balancert - if Conf.balancer() and Conf.numProcesses() not in (0, 1): - tornado.process.fork_processes(Conf.numProcesses(), max_restarts=0) - kw['debug'] = False - # Debug mode? - if kw['debug']: - self.log.info("Configuring in developer mode...") - # Configure tornado app - self.__app = tornado.web.Application(routes, **kw) - port = Conf.HTTPPort() - self.log.notice("Configuring HTTP on port %s" % port) - # Create the web servers - srv = tornado.httpserver.HTTPServer(self.__app, xheaders=True) - srv.listen(port) - self.__servers[('http', port)] = srv - - Conf.generateRevokedCertsFile() # it is used by nginx.... - - if Conf.HTTPS(): - self.log.notice("Configuring HTTPS on port %s" % Conf.HTTPSPort()) - sslops = dict(certfile=Conf.HTTPSCert(), - keyfile=Conf.HTTPSKey(), - cert_reqs=ssl.CERT_OPTIONAL, - ca_certs=Conf.generateCAFile(), - ssl_version=ssl.PROTOCOL_TLSv1_2) - - sslprotocol = str(Conf.SSLProtocol()) - aviableProtocols = [i for i in dir(ssl) if i.find('PROTOCOL') == 0] - if sslprotocol and sslprotocol != "": - if (sslprotocol in aviableProtocols): - sslops['ssl_version'] = getattr(ssl, sslprotocol) + """ + app = {"port": port or Conf.HTTPSPort()} + # Calculating routes + result = self.__handlerMgr.getRoutes() + if not result["OK"]: + return result + app["routes"] = result["Value"] + # Initialize the session data + SessionData.setHandlers(self.__handlerMgr.getHandlers()["Value"]) + # Create the app + tLoader = TemplateLoader(self.__handlerMgr.getPaths("template")) + app["settings"] = dict( + debug=Conf.devMode(), + template_loader=tLoader, + cookie_secret=str(Conf.cookieSecret()), + log_function=self._logRequest, + ) + return S_OK(app) + + def bootstrap(self): + """ + Configure and create web app + """ + self.log.always("\n ====== Starting DIRAC web app ====== \n") + + # Calculating routes + result = self.__handlerMgr.getRoutes() + if not result["OK"]: + return result + routes = result["Value"] + # Initialize the session data + SessionData.setHandlers(self.__handlerMgr.getHandlers()["Value"]) + # Create the app + tLoader = TemplateLoader(self.__handlerMgr.getPaths("template")) + kw = dict( + debug=Conf.devMode(), + template_loader=tLoader, + cookie_secret=str(Conf.cookieSecret()), + log_function=self._logRequest, + autoreload=Conf.numProcesses() < 2, + ) + + # please do no move this lines. The lines must be before the fork_processes + signal.signal(signal.SIGTERM, self.stopChildProcesses) + signal.signal(signal.SIGINT, self.stopChildProcesses) + + # Check processes if we're under a load balancert + if Conf.balancer() and Conf.numProcesses() not in (0, 1): + tornado.process.fork_processes(Conf.numProcesses(), max_restarts=0) + kw["debug"] = False + # Debug mode? + if kw["debug"]: + self.log.info("Configuring in developer mode...") + # Configure tornado app + self.__app = tornado.web.Application(routes, **kw) + port = Conf.HTTPPort() + self.log.notice("Configuring HTTP on port %s" % port) + # Create the web servers + srv = tornado.httpserver.HTTPServer(self.__app, xheaders=True) + srv.listen(port) + self.__servers[("http", port)] = srv + + Conf.generateRevokedCertsFile() # it is used by nginx.... + + if Conf.HTTPS(): + self.log.notice("Configuring HTTPS on port %s" % Conf.HTTPSPort()) + sslops = dict( + certfile=Conf.HTTPSCert(), + keyfile=Conf.HTTPSKey(), + cert_reqs=ssl.CERT_OPTIONAL, + ca_certs=Conf.generateCAFile(), + ssl_version=ssl.PROTOCOL_TLSv1_2, + ) + + sslprotocol = str(Conf.SSLProtocol()) + aviableProtocols = [i for i in dir(ssl) if i.find("PROTOCOL") == 0] + if sslprotocol and sslprotocol != "": + if sslprotocol in aviableProtocols: + sslops["ssl_version"] = getattr(ssl, sslprotocol) + else: + message = "%s protocol is not provided." % sslprotocol + message += "The following protocols are provided: %s" % str(aviableProtocols) + gLogger.warn(message) + + self.log.debug(" - %s" % "\n - ".join(["%s = %s" % (k, sslops[k]) for k in sslops])) + srv = tornado.httpserver.HTTPServer(self.__app, ssl_options=sslops, xheaders=True) + port = Conf.HTTPSPort() + srv.listen(port) + self.__servers[("https", port)] = srv else: - message = "%s protocol is not provided." % sslprotocol - message += "The following protocols are provided: %s" % str(aviableProtocols) - gLogger.warn(message) - - self.log.debug(" - %s" % "\n - ".join(["%s = %s" % (k, sslops[k]) for k in sslops])) - srv = tornado.httpserver.HTTPServer(self.__app, ssl_options=sslops, xheaders=True) - port = Conf.HTTPSPort() - srv.listen(port) - self.__servers[('https', port)] = srv - else: - # when NGINX is used then the Conf.HTTPS return False, it means tornado - # does not have to be configured using 443 port - Conf.generateCAFile() # if we use Nginx we have to generate the cas as well... - return result - - def run(self): - """ - Start web servers - """ - bu = Conf.rootURL().strip("/") - urls = [] - for proto, port in self.__servers: - urls.append("%s://0.0.0.0:%s/%s/" % (proto, port, bu)) - self.log.always("Listening on %s" % " and ".join(urls)) - tornado.autoreload.add_reload_hook(self.__reloadAppCB) - tornado.ioloop.IOLoop.instance().start() + # when NGINX is used then the Conf.HTTPS return False, it means tornado + # does not have to be configured using 443 port + Conf.generateCAFile() # if we use Nginx we have to generate the cas as well... + return result + + def run(self): + """ + Start web servers + """ + bu = Conf.rootURL().strip("/") + urls = [] + for proto, port in self.__servers: + urls.append("%s://0.0.0.0:%s/%s/" % (proto, port, bu)) + self.log.always("Listening on %s" % " and ".join(urls)) + tornado.autoreload.add_reload_hook(self.__reloadAppCB) + tornado.ioloop.IOLoop.instance().start() diff --git a/src/WebAppDIRAC/Core/CoreHandler.py b/src/WebAppDIRAC/Core/CoreHandler.py index c85132bf7..5f1345d68 100644 --- a/src/WebAppDIRAC/Core/CoreHandler.py +++ b/src/WebAppDIRAC/Core/CoreHandler.py @@ -11,25 +11,24 @@ class CoreHandler(tornado.web.RequestHandler): + def initialize(self, action): + self.__action = action - def initialize(self, action): - self.__action = action - - def get(self, setup, group, route): - if self.__action == "addSlash": - o = urlparse(self.request.uri) - proto = self.request.protocol - if 'X-Scheme' in self.request.headers: - proto = self.request.headers['X-Scheme'] - nurl = "%s://%s%s/" % (proto, self.request.host, o.path) - if o.query: - nurl = "%s?%s" % (nurl, o.query) - self.redirect(nurl, permanent=True) - elif self.__action == "sendToRoot": - dest = "/" - rootURL = Conf.rootURL() - if rootURL: - dest += "%s/" % rootURL.strip("/") - if setup and group: - dest += "s:%s/g:%s/" % (setup, group) - self.redirect(dest) + def get(self, setup, group, route): + if self.__action == "addSlash": + o = urlparse(self.request.uri) + proto = self.request.protocol + if "X-Scheme" in self.request.headers: + proto = self.request.headers["X-Scheme"] + nurl = "%s://%s%s/" % (proto, self.request.host, o.path) + if o.query: + nurl = "%s?%s" % (nurl, o.query) + self.redirect(nurl, permanent=True) + elif self.__action == "sendToRoot": + dest = "/" + rootURL = Conf.rootURL() + if rootURL: + dest += "%s/" % rootURL.strip("/") + if setup and group: + dest += "s:%s/g:%s/" % (setup, group) + self.redirect(dest) diff --git a/src/WebAppDIRAC/Core/HandlerMgr.py b/src/WebAppDIRAC/Core/HandlerMgr.py index 1738ce189..6276f63ef 100644 --- a/src/WebAppDIRAC/Core/HandlerMgr.py +++ b/src/WebAppDIRAC/Core/HandlerMgr.py @@ -28,151 +28,157 @@ @six.add_metaclass(DIRACSingleton) class HandlerMgr(object): - - def __init__(self, handlersLocation, baseURL="/"): - """ Constructor + def __init__(self, handlersLocation, baseURL="/"): + """Constructor :param str handlersLocation: handlers location :param str baseURL: base URL - """ - self.__baseURL = baseURL.strip("/") - self.__handlersLocation = handlersLocation - self.__routes = [] - self.__handlers = [] - self.__setupGroupRE = r"(?:/s:([\w-]*)/g:([\w.-]*))?" - self.__shySetupGroupRE = r"(?:/s:(?:[\w-]*)/g:(?:[\w.-]*))?" - self.log = gLogger.getSubLogger("Routing") - - def getPaths(self, dirName): - """ Get lists of paths for all installed and enabled extensions + """ + self.__baseURL = baseURL.strip("/") + self.__handlersLocation = handlersLocation + self.__routes = [] + self.__handlers = [] + self.__setupGroupRE = r"(?:/s:([\w-]*)/g:([\w.-]*))?" + self.__shySetupGroupRE = r"(?:/s:(?:[\w-]*)/g:(?:[\w.-]*))?" + self.log = gLogger.getSubLogger("Routing") + + def getPaths(self, dirName): + """Get lists of paths for all installed and enabled extensions :param str dirName: path to handlers :return: list - """ - pathList = [] - for extName in extensionsByPriority(): - if six.PY3: - metadata = getExtensionMetadata(extName) - pathList.extend(map(str, metadata.get("web_resources", {}).get(dirName, []))) - else: - try: - modFile, modPath, desc = imp.find_module(extName) - # to match in the real root path to enabling module web extensions (static, templates...) - realModPath = os.path.realpath(modPath) - except ImportError: - continue - staticPath = os.path.join(realModPath, "WebApp", dirName) - if os.path.isdir(staticPath): - pathList.append(staticPath) - return pathList - - def __calculateRoutes(self): - """ Load all handlers and generate the routes + """ + pathList = [] + for extName in extensionsByPriority(): + if six.PY3: + metadata = getExtensionMetadata(extName) + pathList.extend(map(str, metadata.get("web_resources", {}).get(dirName, []))) + else: + try: + modFile, modPath, desc = imp.find_module(extName) + # to match in the real root path to enabling module web extensions (static, templates...) + realModPath = os.path.realpath(modPath) + except ImportError: + continue + staticPath = os.path.join(realModPath, "WebApp", dirName) + if os.path.isdir(staticPath): + pathList.append(staticPath) + return pathList + + def __calculateRoutes(self): + """Load all handlers and generate the routes :return: S_OK()/S_ERROR() - """ - ol = ObjectLoader() - handlerList = [] - self.log.debug("Add handles from: %s", self.__handlersLocation) - for parentClass in [WebHandler, _WebHandler]: - result = ol.getObjects(self.__handlersLocation, parentClass=parentClass, recurse=True, continueOnError=True) - if not result['OK']: - return result - handlerList += list(result['Value'].items()) - self.__handlers = collections.OrderedDict(handlerList) - - # ['/opt/dirac/pro/WebAppExt/WebApp/static', ...] - staticPaths = self.getPaths("static") - self.log.verbose("Static paths found:\n - %s" % "\n - ".join(staticPaths)) - self.__routes = [] - - # Add some standard paths for static files - statDirectories = Conf.getStaticDirs() - self.log.info("The following static directories are used:%s" % str(statDirectories)) - for stdir in statDirectories: - pattern = '/%s/(.*)' % stdir - self.__routes.append((pattern, StaticHandler, dict(pathList=['%s/webRoot/www/%s' % (rootPath, stdir)]))) - self.log.debug(" - Static route: %s" % pattern) - - for pattern in ((r"/static/(.*)", r"/(favicon\.ico)", r"/(robots\.txt)")): - pattern = r"%s%s" % (self.__shySetupGroupRE, pattern) - if self.__baseURL: - pattern = "/%s%s" % (self.__baseURL, pattern) - self.__routes.append((pattern, StaticHandler, dict(pathList=staticPaths))) - self.log.debug(" - Static route: %s" % pattern) - for hn in self.__handlers: - self.log.info("Found handler %s" % hn) - handler = self.__handlers[hn] - # CHeck it has AUTH_PROPS - if isinstance(handler.AUTH_PROPS, type(None)): - return S_ERROR("Handler %s does not have AUTH_PROPS defined. Fix it!" % hn) - # Get the root for the handler - if handler.LOCATION: - handlerRoute = handler.LOCATION.strip("/") and "/%s" % handler.LOCATION.strip("/") or '' - else: - handlerRoute = hn[len(re.sub(r".[A-z]+$", "", hn)):].replace(".", "/").replace("Handler", "") - # Add the setup group RE before - baseRoute = self.__setupGroupRE - # IF theres a base url like /DIRAC add it - baseRoute = "/%s%s" % (self.__baseURL or '', baseRoute) - # Set properly the LOCATION after calculating where it is with helpers to add group and setup later - handler.LOCATION = handlerRoute - handler.PATH_RE = re.compile("%s(%s/[A-z]+|.)" % (baseRoute, handlerRoute)) - handler.URLSCHEMA = "/%s%%(setup)s%%(group)s%%(location)s/%%(action)s" % (self.__baseURL) - if issubclass(handler, WebSocketHandler): - handler.PATH_RE = re.compile("%s(%s)" % (baseRoute, handlerRoute)) - route = "%s(%s)" % (baseRoute, handlerRoute) - self.__routes.append((route, handler)) - self.log.verbose(" - WebSocket %s -> %s" % (handlerRoute, hn)) - self.log.debug(" * %s" % route) - continue - # Look for methods that are exported - for mName, mObj in inspect.getmembers(handler): - if inspect.isroutine(mObj) and mName.find(handler.METHOD_PREFIX) == 0: - self.log.debug(' Find %s method' % mName) - methodName = mName[len(handler.METHOD_PREFIX):] - args = getattr(handler, 'path_%s' % methodName, []) - if mName == "web_index" and handler.__name__ == 'RootHandler': - # Index methods have the bare url - self.log.verbose(" - Route %s -> %s.web_index" % (handlerRoute, hn)) - route = "%s(%s/)" % (baseRoute, handlerRoute) - self.__routes.append((route, handler)) - self.__routes.append(("%s(%s)" % (baseRoute, handlerRoute), CoreHandler, dict(action='addSlash'))) - else: - # Normal methods get the method appended without web_ - self.log.verbose(" - Route %s/%s -> %s.%s" % (handlerRoute, mName[4:], hn, mName)) - route = "%s(%s%s)" % (baseRoute, handlerRoute, '' if methodName == 'index' else ('/%s' % methodName)) - # Use request path as options/values, for ex. ../method/