gpu_fdinfo: read GPU throttling status on Xe/i915

Author: retrixe

src/gpu_fdinfo.cpp

@@ -285,6 +285,11 @@ int GPU_fdinfo::get_gpu_load()
     return result;
 }
 
+std::vector<std::string> intel_throttle_power = {"reason_pl1", "reason_pl2"};
+std::vector<std::string> intel_throttle_current = {"reason_pl4", "reason_vr_tdc"};
+std::vector<std::string> intel_throttle_temp = {
+    "reason_prochot", "reason_ratl", "reason_thermal", "reason_vr_thermalert"};
+
 void GPU_fdinfo::find_i915_gt_dir()
 {
     std::string device = "/sys/bus/pci/devices/" + pci_dev + "/drm";
@@ -299,20 +304,24 @@ void GPU_fdinfo::find_i915_gt_dir()
         }
     }
 
-    device += "/gt_act_freq_mhz";
-
-    if (!fs::exists(device)) {
-        SPDLOG_WARN(
-            "Intel gt file ({}) not found. GPU clock will not be available.",
-            device
-        );
-        return;
-    }
-
-    gpu_clock_stream.open(device);
+    auto gpu_clock_path = device + "/gt_act_freq_mhz";
+    gpu_clock_stream.open(gpu_clock_path);
 
     if (!gpu_clock_stream.good())
         SPDLOG_WARN("Intel i915 gt dir: failed to open {}", device);
+
+    // Assuming gt0 since all recent GPUs have the RCS engine on gt0, and latest GPUs need Xe anyway
+    auto throttle_folder = device + "/gt/gt0/throttle_";
+    auto throttle_status_path = throttle_folder + "reason_status";
+
+    throttle_status_stream.open(throttle_status_path);
+    if (!throttle_status_stream.good()) {
+       SPDLOG_WARN("Intel i915 gt dir: failed to open {}", throttle_status_path);
+    } else {
+        load_xe_i915_throttle_reasons(throttle_folder, intel_throttle_power, throttle_power_streams);
+        load_xe_i915_throttle_reasons(throttle_folder, intel_throttle_current, throttle_current_streams);
+        load_xe_i915_throttle_reasons(throttle_folder, intel_throttle_temp, throttle_temp_streams);
+    }
 }
 
 void GPU_fdinfo::find_xe_gt_dir()
@@ -351,19 +360,47 @@ void GPU_fdinfo::find_xe_gt_dir()
     }
 
     if (!has_rcs) {
-        SPDLOG_WARN(
-            "rcs not found inside \"{}\". GPU clock will not be available.",
-            device
-        );
+        SPDLOG_WARN("rcs not found inside \"{}\". GPU clock will not be available.", device);
         return;
     }
 
-    device += "/freq0/act_freq";
-
-    gpu_clock_stream.open(device);
+    auto gpu_clock_path = device + "/freq0/act_freq";
+    gpu_clock_stream.open(gpu_clock_path);
 
     if (!gpu_clock_stream.good())
-        SPDLOG_WARN("Intel xe gt dir: failed to open {}", device);
+        SPDLOG_WARN("Intel xe gt dir: failed to open {}", gpu_clock_path);
+
+    auto throttle_folder = device + "/freq0/throttle/";
+    auto throttle_status_path = throttle_folder + "status";
+
+    throttle_status_stream.open(throttle_status_path);
+    if (!throttle_status_stream.good()) {
+       SPDLOG_WARN("Intel xe gt dir: failed to open {}", throttle_status_path);
+    } else {
+        load_xe_i915_throttle_reasons(throttle_folder, intel_throttle_power, throttle_power_streams);
+        load_xe_i915_throttle_reasons(throttle_folder, intel_throttle_current, throttle_current_streams);
+        load_xe_i915_throttle_reasons(throttle_folder, intel_throttle_temp, throttle_temp_streams);
+    }
+}
+
+void GPU_fdinfo::load_xe_i915_throttle_reasons(
+    std::string throttle_folder,
+    std::vector<std::string> throttle_reasons,
+    std::vector<std::ifstream>& throttle_reason_streams
+) {
+    for (const auto& throttle_reason : throttle_reasons) {
+        std::string throttle_path = throttle_folder + throttle_reason;
+        if (!fs::exists(throttle_path)) {
+            SPDLOG_WARN("Intel xe/i915 gt dir: Throttle file {} not found", throttle_path);
+            continue;
+        }
+        auto throttle_stream = std::ifstream(throttle_path);
+        if (!throttle_stream.good()) {
+            SPDLOG_WARN("Intel xe/i915 gt dir: failed to open {}", throttle_path);
+            continue;
+        }
+        throttle_reason_streams.push_back(std::move(throttle_stream));
+    }
 }
 
 int GPU_fdinfo::get_gpu_clock()
@@ -383,6 +420,41 @@ int GPU_fdinfo::get_gpu_clock()
     return std::stoi(clock_str);
 }
 
+bool GPU_fdinfo::check_throttle_reasons(std::vector<std::ifstream>& throttle_reason_streams)
+{
+    for (auto& throttle_reason_stream : throttle_reason_streams) {
+        std::string throttle_reason_str;
+        throttle_reason_stream.seekg(0);
+        std::getline(throttle_reason_stream, throttle_reason_str);
+
+        if (throttle_reason_str == "1")
+            return true;
+    }
+
+    return false;
+}
+
+GPU_throttle_status GPU_fdinfo::get_throttling_status()
+{
+    if (!throttle_status_stream.is_open())
+        return GPU_throttle_status::NONE;
+
+    std::string throttle_status_str;
+    throttle_status_stream.seekg(0);
+    std::getline(throttle_status_stream, throttle_status_str);
+
+    if (throttle_status_str != "1")
+        return GPU_throttle_status::NONE;
+    else if (check_throttle_reasons(throttle_power_streams))
+        return GPU_throttle_status::POWER;
+    else if (check_throttle_reasons(throttle_current_streams))
+        return GPU_throttle_status::CURRENT;
+    else if (check_throttle_reasons(throttle_temp_streams))
+        return GPU_throttle_status::TEMP;
+
+    return GPU_throttle_status::OTHER;
+}
+
 void GPU_fdinfo::main_thread()
 {
     while (!stop_thread) {
@@ -396,6 +468,11 @@ void GPU_fdinfo::main_thread()
         metrics.memoryUsed = get_memory_used();
         metrics.powerUsage = get_power_usage();
         metrics.CoreClock = get_gpu_clock();
+        auto throttling = get_throttling_status();
+        metrics.is_power_throttled = throttling == GPU_throttle_status::POWER;
+        metrics.is_current_throttled = throttling == GPU_throttle_status::CURRENT;
+        metrics.is_temp_throttled = throttling == GPU_throttle_status::TEMP;
+        metrics.is_other_throttled = throttling == GPU_throttle_status::OTHER;
         metrics.temp = hwmon_sensors["temp"].val / 1000;
         metrics.fan_speed = hwmon_sensors["fan_speed"].val;
         metrics.voltage = hwmon_sensors["voltage"].val;

src/gpu_fdinfo.h

@@ -28,6 +28,14 @@ struct hwmon_sensor {
     uint64_t val = 0;
 };
 
+enum class GPU_throttle_status {
+    NONE,
+    POWER,
+    CURRENT,
+    TEMP,
+    OTHER
+};
+
 class GPU_fdinfo {
 private:
     bool init = false;
@@ -81,6 +89,17 @@ class GPU_fdinfo {
     void find_xe_gt_dir();
     int get_gpu_clock();
 
+    std::ifstream throttle_status_stream;
+    std::vector<std::ifstream> throttle_power_streams;
+    std::vector<std::ifstream> throttle_current_streams;
+    std::vector<std::ifstream> throttle_temp_streams;
+    void load_xe_i915_throttle_reasons(
+        std::string throttle_folder,
+        std::vector<std::string> throttle_reasons,
+        std::vector<std::ifstream> &throttle_reason_streams);
+    bool check_throttle_reasons(std::vector<std::ifstream> &throttle_reason_streams);
+    GPU_throttle_status get_throttling_status();
+
 public:
     GPU_fdinfo(const std::string module, const std::string pci_dev)
         : module(module)