Upload handler.py

handler.py

@@ -0,0 +1,36 @@
+# handler.py
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import PeftModel
+
+# Model path in the repo
+BASE_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
+ADAPTER_PATH = "."
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        print("Loading tokenizer and model...")
+        self.tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
+        base_model = AutoModelForCausalLM.from_pretrained(
+            BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
+        )
+        self.model = PeftModel.from_pretrained(base_model, ADAPTER_PATH)
+        self.model = self.model.merge_and_unload()
+        self.model.eval()
+        print("Model loaded successfully.")
+
+    def __call__(self, data):
+        prompt = data.get("inputs", "")
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        with torch.no_grad():
+            outputs = self.model.generate(
+                **inputs,
+                max_new_tokens=512,
+                temperature=0.7,
+                top_p=0.9,
+                do_sample=True,
+                pad_token_id=self.tokenizer.eos_token_id,
+                eos_token_id=self.tokenizer.eos_token_id,
+            )
+        text = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return {"generated_text": text}