muryshev commited on
Commit
36d1bec
1 Parent(s): 7eaf4dc
Files changed (4) hide show
  1. .gitignore +3 -0
  2. Dockerfile +32 -0
  3. app.py +200 -0
  4. llm_backend.py +152 -0
.gitignore ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ .env/
2
+ data/
3
+ .vscode/
Dockerfile ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04"
2
+ FROM nvidia/cuda:${CUDA_IMAGE}
3
+
4
+ # We need to set the host to 0.0.0.0 to allow outside access
5
+ ENV HOST 0.0.0.0
6
+ RUN useradd -m -u 1000 user
7
+ WORKDIR /home/user/app
8
+ COPY --link --chown=1000 ./ /home/user/app
9
+
10
+ RUN apt-get update && apt-get upgrade -y \
11
+ && apt-get install -y git git-lfs build-essential \
12
+ python3 python3-pip gcc wget \
13
+ ocl-icd-opencl-dev opencl-headers clinfo \
14
+ libclblast-dev libopenblas-dev \
15
+ && mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
16
+
17
+ # setting build related env vars
18
+ ENV CUDA_DOCKER_ARCH=all
19
+ ENV LLAMA_CUBLAS=1
20
+
21
+ # Install depencencies
22
+ RUN python3 -m pip install --no-cache-dir --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context huggingface-hub==0.14.1 flask apscheduler
23
+
24
+ # Install llama-cpp-python (build with cuda)
25
+ RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install --no-cache-dir llama-cpp-python
26
+
27
+ RUN git config --global user.email "[email protected]"
28
+ RUN git config --global user.name "Andrew Matenkov"
29
+
30
+ EXPOSE 7860
31
+ # Run the server
32
+ CMD python3 -m app
app.py ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from flask import Flask, request, Response
2
+ import logging
3
+ from llama_cpp import Llama
4
+ import threading
5
+ from huggingface_hub import snapshot_download#, Repository
6
+ import huggingface_hub
7
+ import gc
8
+ import os.path
9
+ import xml.etree.ElementTree as ET
10
+ from apscheduler.schedulers.background import BackgroundScheduler
11
+ from datetime import datetime, timedelta
12
+ from llm_backend import LlmBackend
13
+ import json
14
+
15
+ llm = LlmBackend()
16
+ _lock = threading.Lock()
17
+
18
+ SYSTEM_PROMPT = os.environ.get('SYSTEM_PROMPT') or "Ты — русскоязычный автоматический ассистент. Ты максимально точно и отвечаешь на запросы пользователя, используя русский язык."
19
+
20
+ CONTEXT_SIZE = os.environ.get('CONTEXT_SIZE') or 500
21
+ ENABLE_GPU = os.environ.get('ENABLE_GPU') or False
22
+ GPU_LAYERS = os.environ.get('GPU_LAYERS') or 0
23
+ N_GQA = os.environ.get('N_GQA') or None #must be set to 8 for 70b models
24
+ CHAT_FORMAT = os.environ.get('CHAT_FORMAT') or 'llama-2'
25
+
26
+ # Create a lock object
27
+ lock = threading.Lock()
28
+
29
+ app = Flask(__name__)
30
+ # Configure Flask logging
31
+ app.logger.setLevel(logging.DEBUG)
32
+
33
+ # Variable to store the last request time
34
+ last_request_time = datetime.now()
35
+
36
+ # Initialize the model when the application starts
37
+ #model_path = "../models/model-q4_K.gguf" # Replace with the actual model path
38
+ #model_name = "model/ggml-model-q4_K.gguf"
39
+
40
+ #repo_name = "IlyaGusev/saiga2_13b_gguf"
41
+ #model_name = "model-q4_K.gguf"
42
+
43
+ #epo_name = "IlyaGusev/saiga2_70b_gguf"
44
+ #model_name = "ggml-model-q4_1.gguf"
45
+
46
+ repo_name = "IlyaGusev/saiga2_7b_gguf"
47
+ model_name = "model-q4_K.gguf"
48
+ local_dir = '.'
49
+
50
+ if os.path.isdir('/data'):
51
+ app.logger.info('Persistent storage enabled')
52
+
53
+ model = None
54
+
55
+ MODEL_PATH = snapshot_download(repo_id=repo_name, allow_patterns=model_name) + '/' + model_name
56
+ app.logger.info('Model path: ' + MODEL_PATH)
57
+
58
+ DATASET_REPO_URL = "https://huggingface.co/datasets/muryshev/saiga-chat"
59
+ DATA_FILENAME = "data-saiga-cuda-release.xml"
60
+ DATA_FILE = os.path.join("dataset", DATA_FILENAME)
61
+
62
+ HF_TOKEN = os.environ.get("HF_TOKEN")
63
+ app.logger.info("hfh: "+huggingface_hub.__version__)
64
+
65
+ # repo = Repository(
66
+ # local_dir="dataset", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN
67
+ # )
68
+
69
+
70
+
71
+ # def log(req: str = '', resp: str = ''):
72
+ # if req or resp:
73
+ # element = ET.Element("row", {"time": str(datetime.now()) })
74
+ # req_element = ET.SubElement(element, "request")
75
+ # req_element.text = req
76
+ # resp_element = ET.SubElement(element, "response")
77
+ # resp_element.text = resp
78
+
79
+ # with open(DATA_FILE, "ab+") as xml_file:
80
+ # xml_file.write(ET.tostring(element, encoding="utf-8"))
81
+
82
+ # commit_url = repo.push_to_hub()
83
+ # app.logger.info(commit_url)
84
+
85
+ def generate_tokens(model, generator):
86
+ global stop_generation
87
+ app.logger.info('generate_tokens started')
88
+ with lock:
89
+ try:
90
+ for token in generator:
91
+ if token == model.token_eos() or stop_generation:
92
+ stop_generation = False
93
+ app.logger.info('End generating')
94
+ yield b'' # End of chunk
95
+ break
96
+
97
+ token_str = model.detokenize([token])#.decode("utf-8", errors="ignore")
98
+ yield token_str
99
+ except Exception as e:
100
+ app.logger.info('generator exception')
101
+ app.logger.info(e)
102
+ yield b'' # End of chunk
103
+
104
+ @app.route('/change_context_size', methods=['GET'])
105
+ def handler_change_context_size():
106
+ global stop_generation, model
107
+ stop_generation = True
108
+
109
+ new_size = int(request.args.get('size', CONTEXT_SIZE))
110
+ init_model(new_size, ENABLE_GPU, GPU_LAYERS)
111
+
112
+ return Response('Size changed', content_type='text/plain')
113
+
114
+ @app.route('/stop_generation', methods=['GET'])
115
+ def handler_stop_generation():
116
+ global stop_generation
117
+ stop_generation = True
118
+ return Response('Stopped', content_type='text/plain')
119
+
120
+ @app.route('/', methods=['GET', 'PUT', 'DELETE', 'PATCH'])
121
+ def generate_unknown_response():
122
+ app.logger.info('unknown method: '+request.method)
123
+ try:
124
+ request_payload = request.get_json()
125
+ app.logger.info('payload: '+request.get_json())
126
+ except Exception as e:
127
+ app.logger.info('payload empty')
128
+
129
+ return Response('What do you want?', content_type='text/plain')
130
+
131
+ response_tokens = bytearray()
132
+ def generate_and_log_tokens(user_request, generator):
133
+ global response_tokens, last_request_time
134
+ for token in llm.generate_tokens(generator):
135
+ if token == b'': # or (max_new_tokens is not None and i >= max_new_tokens):
136
+ last_request_time = datetime.now()
137
+ # log(json.dumps(user_request), response_tokens.decode("utf-8", errors="ignore"))
138
+ response_tokens = bytearray()
139
+ break
140
+ response_tokens.extend(token)
141
+ yield token
142
+
143
+ @app.route('/', methods=['POST'])
144
+ def generate_response():
145
+
146
+ app.logger.info('generate_response')
147
+ with _lock:
148
+ if not llm.is_model_loaded():
149
+ app.logger.info('model loading')
150
+ init_model()
151
+
152
+ data = request.get_json()
153
+ app.logger.info(data)
154
+ messages = data.get("messages", [])
155
+ preprompt = data.get("preprompt", "")
156
+ parameters = data.get("parameters", {})
157
+
158
+ # Extract parameters from the request
159
+ p = {
160
+ 'temperature': parameters.get("temperature", 0.01),
161
+ 'truncate': parameters.get("truncate", 1000),
162
+ 'max_new_tokens': parameters.get("max_new_tokens", 1024),
163
+ 'top_p': parameters.get("top_p", 0.85),
164
+ 'repetition_penalty': parameters.get("repetition_penalty", 1.2),
165
+ 'top_k': parameters.get("top_k", 30),
166
+ 'return_full_text': parameters.get("return_full_text", False)
167
+ }
168
+
169
+ generator = llm.create_chat_generator_for_saiga(messages=messages, parameters=p)
170
+ app.logger.info('Generator created')
171
+
172
+
173
+
174
+
175
+ # Use Response to stream tokens
176
+ return Response(generate_and_log_tokens(user_request='1', generator=generator), content_type='text/plain', status=200, direct_passthrough=True)
177
+
178
+ def init_model():
179
+ llm.load_model(model_path=MODEL_PATH, context_size=CONTEXT_SIZE, enable_gpu=ENABLE_GPU, gpu_layer_number=GPU_LAYERS, n_gqa=N_GQA)
180
+
181
+ # Function to check if no requests were made in the last 5 minutes
182
+ def check_last_request_time():
183
+ global last_request_time
184
+ current_time = datetime.now()
185
+ if (current_time - last_request_time).total_seconds() > 300: # 5 minutes in seconds
186
+ # Perform the action (e.g., set a variable)
187
+ llm.unload_model()
188
+ app.logger.info(f"Model unloaded at {current_time}")
189
+ else:
190
+ app.logger.info(f"No action needed at {current_time}")
191
+
192
+
193
+ if __name__ == "__main__":
194
+ scheduler = BackgroundScheduler()
195
+ scheduler.add_job(check_last_request_time, trigger='interval', minutes=1)
196
+ scheduler.start()
197
+
198
+ init_model()
199
+
200
+ app.run(host="0.0.0.0", port=7860, debug=True, threaded=True)
llm_backend.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from llama_cpp import Llama
2
+ import gc
3
+ import threading
4
+
5
+ class LlmBackend:
6
+
7
+ SYSTEM_PROMPT = "Ты — русскоязычный автоматический ассистент. Ты максимально точно и отвечаешь на запросы пользователя, используя русский язык."
8
+ SYSTEM_TOKEN = 1788
9
+ USER_TOKEN = 1404
10
+ BOT_TOKEN = 9225
11
+ LINEBREAK_TOKEN = 13
12
+
13
+ ROLE_TOKENS = {
14
+ "user": USER_TOKEN,
15
+ "bot": BOT_TOKEN,
16
+ "system": SYSTEM_TOKEN
17
+ }
18
+
19
+ _instance = None
20
+ _model = None
21
+ _lock = threading.Lock()
22
+
23
+ def __new__(cls):
24
+ if cls._instance is None:
25
+ cls._instance = super(LlmBackend, cls).__new__(cls)
26
+ return cls._instance
27
+
28
+
29
+ def is_model_loaded(self):
30
+ return self._model is not None
31
+
32
+ def load_model(self, model_path, context_size=2000, enable_gpu=True, gpu_layer_number=35, n_gqa=8, chat_format='llama-2'):
33
+
34
+ if self._model is not None:
35
+ self.unload_model()
36
+
37
+ with self._lock:
38
+ if enable_gpu:
39
+ self._model = Llama(
40
+ model_path=model_path,
41
+ chat_format=chat_format,
42
+ n_ctx=context_size,
43
+ n_parts=1,
44
+ #n_batch=100,
45
+ logits_all=True,
46
+ #n_threads=12,
47
+ verbose=True,
48
+ n_gpu_layers=gpu_layer_number,
49
+ n_gqa=n_gqa #must be set for 70b models
50
+ )
51
+ return self._model
52
+ else:
53
+ self._model = Llama(
54
+ model_path=model_path,
55
+ chat_format=chat_format,
56
+ n_ctx=context_size,
57
+ n_parts=1,
58
+ #n_batch=100,
59
+ logits_all=True,
60
+ #n_threads=12,
61
+ verbose=True,
62
+ n_gqa=n_gqa #must be set for 70b models
63
+ )
64
+ return self._model
65
+
66
+ def set_system_prompt(self, prompt):
67
+ with self._lock:
68
+ self.SYSTEM_PROMPT = prompt
69
+
70
+ def unload_model(self):
71
+ with self._lock:
72
+ if self._model is not None:
73
+ self._model.llama_free_model()
74
+ del self._model
75
+
76
+ def generate_tokens(self, generator):
77
+ print('generate_tokens called')
78
+ with self._lock:
79
+ print('generate_tokens started')
80
+ try:
81
+ for token in generator:
82
+ if token == self._model.token_eos():
83
+ print('End generating')
84
+ yield b'' # End of chunk
85
+ break
86
+
87
+ token_str = self._model.detokenize([token])#.decode("utf-8", errors="ignore")
88
+ yield token_str
89
+ except Exception as e:
90
+ print('generator exception')
91
+ print(e)
92
+ yield b'' # End of chunk
93
+
94
+ def create_chat_completion(self, messages, stream=True):
95
+ print('create_chat_completion called')
96
+ with self._lock:
97
+ print('create_chat_completion started')
98
+ try:
99
+ return self._model.create_chat_completion(messages=messages, stream=stream)
100
+ except Exception as e:
101
+ print('create_chat_completion exception')
102
+ print(e)
103
+ return None
104
+
105
+
106
+ def get_message_tokens(self, role, content):
107
+ message_tokens = self._model.tokenize(content.encode("utf-8"))
108
+ message_tokens.insert(1, self.ROLE_TOKENS[role])
109
+ message_tokens.insert(2, self.LINEBREAK_TOKEN)
110
+ message_tokens.append(self._model.token_eos())
111
+ return message_tokens
112
+
113
+ def get_system_tokens(self):
114
+ system_message = {
115
+ "role": "system",
116
+ "content": self.SYSTEM_PROMPT
117
+ }
118
+ return self.get_message_tokens(self._model, **system_message)
119
+
120
+ def create_chat_generator_for_saiga(self, messages, parameters):
121
+ print('create_chat_completion called')
122
+ with self._lock:
123
+ tokens = self.get_system_tokens()
124
+ for message in messages:
125
+ message_tokens = self.get_message_tokens(role=message.get("from"), content=message.get("content", ""))
126
+ tokens.extend(message_tokens)
127
+
128
+ tokens.extend([self._model.token_bos(), self.BOT_TOKEN, self.LINEBREAK_TOKEN])
129
+ generator = self._model.generate(
130
+ tokens,
131
+ top_k=parameters['top_k'],
132
+ top_p=parameters['top_p'],
133
+ temp=parameters['temperature'],
134
+ repeat_penalty=parameters['repetition_penalty']
135
+ )
136
+ return generator
137
+
138
+ def generate_tokens(self, generator):
139
+ with self._lock:
140
+ try:
141
+ for token in generator:
142
+ if token == self._model.token_eos():
143
+ yield b'' # End of chunk
144
+ break
145
+
146
+ token_str = self._model.detokenize([token])#.decode("utf-8", errors="ignore")
147
+ yield token_str
148
+ except Exception as e:
149
+ yield b'' # End of chunk
150
+
151
+
152
+