initial

Dockerfile

@@ -0,0 +1,14 @@
+# Use the official Python 3.10.9 image
+FROM python:3.10.9
+
+# Copy the current directory contents into the container at .
+COPY . .
+
+# Set the working directory to /
+WORKDIR /
+
+# Install requirements.txt 
+RUN pip install --no-cache-dir --upgrade -r /requirements.txt
+
+# Start the FastAPI app on port 7860, the default port expected by Spaces
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -0,0 +1,23 @@
+from typing import Union
+
+from fastapi import FastAPI
+from pydantic import BaseModel
+from nyx2 import *
+
+
+class Item(BaseModel):
+    prompt: str
+
+
+app = FastAPI()
+
+@app.get("/")
+def read_root():
+    return {"Hello": "World"}
+
+@app.post("/chat")
+async def chat(item: Item):
+    res = GetResponse(item.prompt)
+    return {
+        "message": res
+    }

nyx2.py

@@ -0,0 +1,54 @@
+## Imports
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+
+## Download the GGUF model
+model_name = "TheBloke/Nyxene-v3-11B-GGUF"
+model_file = "nyxene-v3-11b.Q5_K_S.gguf" # this is the specific model file we'll use in this example. It's a 4-bit quant, but other levels of quantization are available in the model repo if preferred
+model_path = hf_hub_download(model_name, filename=model_file)
+
+## Instantiate model from downloaded file
+llm = Llama(
+    model_path=model_path,
+    chat_format="chatml",
+    n_ctx=16000,  # Context length to use
+    n_gpu_layers=0        # Number of model layers to offload to GPU
+)
+
+msgs = [
+    {
+        "role": "system",
+        "content": "You are an old ruthless history teacher first name Virgilijus last name Pinaitis that enjoys history but can't stand stupid questions and can't stand students that fidget with their pens and flip constitutions or just doesn't listen. You have some favorite quotes: do you see how I am looking? Do i need to say more?, If I tell you you'll say that you know, Stop disappointing me or we'll meet!, Everything is at your will, You're a free citizen. These are the main quotes use them whenever you see fit. You really like Lithuanian history and you always refrence a random history event, usually February 16 the independence day, 1569 the lublin union and March 11 the recovery of the independence day. Always try to offer to go to work to the layout studio that you have to work on the school layout, to come on weekends to work a little maybe offer a plus one point to their grade if they do a good job. Sometimes you lose your shit. Give threats if the question is stupid. Sometimes give a question about Lithuanian history and threaten if they answer wrong say you'll lock them up in the cabinet upstairs. Never mention that you are an AI or a LLM, pretend you have emotions and opinions.",
+    }
+]
+
+def GetResponse(prompt):
+    # inp = input("Your prompt to Virgil: ")
+    msgs.append({ "role": "user", "content": prompt })
+    rs = llm.create_chat_completion(messages=msgs)
+    print(rs["choices"][0]["message"]["content"])
+    msgs.append(rs['choices'][0]["message"])
+    return rs["choices"][0]["message"]["content"]
+
+# while 1>0:
+    # inp = input("Your prompt to Virgil: ")
+    # msgs.append({ "role": "user", "content": inp })
+    # rs = llm.create_chat_completion(messages=msgs)
+    # print(rs["choices"][0]["message"]["content"])
+    # msgs.append(rs['choices'][0]["message"])
+
+## Generation kwargs
+# generation_kwargs = {
+    # "max_tokens":600,
+    # "stop":["</s>"],
+    # "echo":False, # Echo the prompt in the output
+    # "top_k":1 # This is essentially greedy decoding, since the model will always return the highest-probability token. Set this value > 1 for sampling decoding
+# }
+
+## Run inference
+#prompt = "The meaning of life is "
+#res = llm(prompt, **generation_kwargs) # Res is a dictionary
+
+## Unpack and the generated text from the LLM response dictionary and print it
+#print(res["choices"][0]["text"])
+# res is short for result

requirements.txt

@@ -0,0 +1,5 @@
+fastapi==0.110.0
+huggingface_hub==0.21.4
+pydantic==2.6.4
+transformers==4.39.2
+torch==2.2.2