Update speculative_decoding_lm.cpp

openvinotoolkit · Jan 17, 2025 · c440e08 · c440e08
1 parent 075245e
commit c440e08
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/samples/cpp/text_generation/speculative_decoding_lm.cpp b/samples/cpp/text_generation/speculative_decoding_lm.cpp
@@ -15,6 +15,8 @@ int main(int argc, char* argv[]) try {
     // Speculative decoding generation parameters like `num_assistant_tokens` and `assistant_confidence_threshold` are mutually excluded
     // add parameter to enable speculative decoding to generate `num_assistant_tokens` candidates by draft_model per iteration
     config.num_assistant_tokens = 5;
+    // add parameter to enable speculative decoding to generate candidates by draft_model while candidate probability is higher than `assistant_confidence_threshold`
+    // config.assistant_confidence_threshold = 0.4;
 
     std::string main_model_path = argv[1];
     std::string draft_model_path = argv[2];