Expose metrics of engine startup permit status

Author: Linyuan QIN 秦林园

docs/monitor/metrics.md

@@ -65,6 +65,9 @@ These metrics include:
 | `kyuubi.engine.timeout`                          |                                        | counter   | 1.2.0  | <div style='width: 150pt;word-wrap: break-word;white-space: normal'>  cumulative timeout engines</div>                                                                                                                                                                                     |
 | `kyuubi.engine.failed`                           | `${user}`                              | counter   | 1.2.0  | <div style='width: 150pt;word-wrap: break-word;white-space: normal'>  cumulative explicitly failed engine count for a `${user}`</div>                                                                                                                                                      |
 | `kyuubi.engine.failed`                           | `${errorType}`                         | counter   | 1.2.0  | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> cumulative explicitly failed engine count for a particular `${errorType}`, e.g. `ClassNotFoundException`</div>                                                                                                        |
+| `kyuubi.engine.startup.permit.limit.total`       |                                        | meter     | 1.10.1 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> concurrently startup engines permit limit </div>                                                                                                                                                                      |
+| `kyuubi.engine.startup.permit.available`         |                                        | gauge     | 1.10.1 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> available permits of concurrently startup engines </div>                                                                                                                                                              |
+| `kyuubi.engine.startup.permit.waiting`           |                                        | gauge     | 1.10.1 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> startup engines that waiting to acquire permit </div>                                                                                                                                                                 |
 | `kyuubi.backend_service.open_session`            |                                        | timer     | 1.5.0  | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> kyuubi backend service `openSession` method execution time and rate </div>                                                                                                                                            |
 | `kyuubi.backend_service.close_session`           |                                        | timer     | 1.5.0  | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> kyuubi backend service `closeSession` method execution time and rate </div>                                                                                                                                           |
 | `kyuubi.backend_service.get_info`                |                                        | timer     | 1.5.0  | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> kyuubi backend service `getInfo` method execution time and rate </div>                                                                                                                                                |

kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala

@@ -60,6 +60,11 @@ object MetricsConstants {
   final val ENGINE_TIMEOUT: String = ENGINE + "timeout"
   final val ENGINE_TOTAL: String = ENGINE + "total"
 
+  final private val ENGINE_STARTUP_PERMIT: String = ENGINE + "startup.permit."
+  final val ENGINE_STARTUP_PERMIT_LIMIT: String = ENGINE_STARTUP_PERMIT + "limit"
+  final val ENGINE_STARTUP_PERMIT_AVAILABLE: String = ENGINE_STARTUP_PERMIT + "available"
+  final val ENGINE_STARTUP_PERMIT_WAITING: String = ENGINE_STARTUP_PERMIT + "waiting"
+
   final private val OPERATION = KYUUBI + "operation."
   final val OPERATION_OPEN: String = OPERATION + "opened"
   final val OPERATION_FAIL: String = OPERATION + "failed"

kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala

@@ -300,6 +300,14 @@ class KyuubiSessionManager private (name: String) extends SessionManager(name) {
       ms.registerGauge(EXEC_POOL_ALIVE, getExecPoolSize, 0)
       ms.registerGauge(EXEC_POOL_ACTIVE, getActiveCount, 0)
       ms.registerGauge(EXEC_POOL_WORK_QUEUE_SIZE, getWorkQueueSize, 0)
+      this.engineStartupProcessSemaphore.foreach { semaphore =>
+        ms.markMeter(ENGINE_STARTUP_PERMIT_LIMIT, semaphore.availablePermits)
+        ms.registerGauge(
+          ENGINE_STARTUP_PERMIT_AVAILABLE,
+          semaphore.availablePermits,
+          semaphore.availablePermits)
+        ms.registerGauge(ENGINE_STARTUP_PERMIT_WAITING, semaphore.getQueueLength, 0)
+      }
     }
     super.start()
     startEngineAliveChecker()