fix: quantization

zeroday0619 · zeroday0619 · commit d2f21b31bc36 · 2024-05-07T00:28:38.000+09:00
diff --git a/HakaseCore/llm/llama3.py b/HakaseCore/llm/llama3.py
@@ -66,7 +66,7 @@ def generate_text(self, instruction: str) -> str:
         prompt = self.pipe.tokenizer.apply_chat_template(
             self.prompt, tokenize=False, add_generation_prompt=True
         )
-        outputs = self.pipe(
+        outputs = self.pipe.model.generate(
             prompt,
             do_sample=True,
             temperature=0.4,

Original file line number	Diff line number	Diff line change
`@@ -66,7 +66,7 @@ def generate_text(self, instruction: str) -> str:`
`66`	`66`	`prompt = self.pipe.tokenizer.apply_chat_template(`
`67`	`67`	`self.prompt, tokenize=False, add_generation_prompt=True`
`68`	`68`	`)`
`69`		`- outputs = self.pipe(`
	`69`	`+ outputs = self.pipe.model.generate(`
`70`	`70`	`prompt,`
`71`	`71`	`do_sample=True,`
`72`	`72`	`temperature=0.4,`