update embedding model

load_viewer_data.py

@@ -17,10 +17,10 @@ from utils import get_chroma_client
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
-EMBEDDING_MODEL_NAME = "davanstrien/dataset-viewer-descriptions-processed-st"
-EMBEDDING_MODEL_REVISION = "d09abf1227ac41c6955eb9dd53c21771b0984ade"
+EMBEDDING_MODEL_NAME = "davanstrien/query-to-dataset-viewer-descriptions"
+EMBEDDING_MODEL_REVISION = "07c71d97861a73695f0c53cd6b4b32980007d908"
 INFERENCE_MODEL_URL = (
-    "https://bm143rfir2on1bkw.us-east-1.aws.endpoints.huggingface.cloud"
+    "https://ecg0by60w2vo9j8h.us-east-1.aws.endpoints.huggingface.cloud"
 )
 
 
@@ -61,7 +61,7 @@ def embed_and_upsert_datasets(
     dataset_rows_and_ids: list[dict[str, str]],
     collection: chromadb.Collection,
     inference_client: InferenceClient,
-    batch_size: int = 10,
+    batch_size: int = 100,
 ):
     logger.info(
         f"Embedding and upserting {len(dataset_rows_and_ids)} datasets for viewer data"