Anna Sun commited on
Commit
967ca9c
1 Parent(s): 2bd3674
.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ .vscode/settings.json
2
+ __pycache__/
.vscode/settings.json DELETED
@@ -1,26 +0,0 @@
1
- {
2
- "[python]": {
3
- "editor.defaultFormatter": "ms-python.python"
4
- },
5
- "python.formatting.provider": "none",
6
- "workbench.colorCustomizations": {
7
- "activityBar.activeBackground": "#fbed80",
8
- "activityBar.background": "#fbed80",
9
- "activityBar.foreground": "#15202b",
10
- "activityBar.inactiveForeground": "#15202b99",
11
- "activityBarBadge.background": "#06b9a5",
12
- "activityBarBadge.foreground": "#15202b",
13
- "commandCenter.border": "#15202b99",
14
- "sash.hoverBorder": "#fbed80",
15
- "statusBar.background": "#f9e64f",
16
- "statusBar.foreground": "#15202b",
17
- "statusBarItem.hoverBackground": "#f7df1e",
18
- "statusBarItem.remoteBackground": "#f9e64f",
19
- "statusBarItem.remoteForeground": "#15202b",
20
- "titleBar.activeBackground": "#f9e64f",
21
- "titleBar.activeForeground": "#15202b",
22
- "titleBar.inactiveBackground": "#f9e64f99",
23
- "titleBar.inactiveForeground": "#15202b99"
24
- },
25
- "peacock.remoteColor": "#f9e64f"
26
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
__pycache__/app.cpython-310.pyc DELETED
Binary file (3.65 kB)
 
__pycache__/app.cpython-38.pyc DELETED
Binary file (2.47 kB)
 
__pycache__/lang_list.cpython-310.pyc DELETED
Binary file (4.03 kB)
 
__pycache__/m4t_app.cpython-310.pyc DELETED
Binary file (8.44 kB)
 
__pycache__/sample_wav.cpython-310.pyc DELETED
Binary file (136 kB)
 
__pycache__/simuleval_transcoder.cpython-310.pyc DELETED
Binary file (14.5 kB)
 
__pycache__/simuleval_transcoder.cpython-38.pyc DELETED
Binary file (13.6 kB)
 
__pycache__/test_pipeline.cpython-310.pyc DELETED
Binary file (2.56 kB)
 
seamless_server/.gitignore ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ __pycache__/
2
+ src/__pycache__/
3
+ debug/
4
+ .vscode/
seamless_server/README.md ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Seamless Streaming Server
2
+
3
+ ## Setting up the AWS server
4
+
5
+ - Clone working AMI with things set up: https://us-west-2.console.aws.amazon.com/ec2/home?region=us-west-2#ImageDetails:imageId=ami-013fc3fb27faa03e5
6
+ - This has the conda environment, repository, model etc set up so minimal install was required
7
+
8
+ - Open port tunnelling via ssh so you can connect to the AWS server via localhost (gets around some security issues): e.g. `ssh -L 8000:localhost:8000 -L 3000:localhost:3000 -L 5173:localhost:5173 -i ~/.ssh/mduppes_devserver.pem [email protected]`
9
+
10
+ ## Setting up the dev environment
11
+
12
+ Clone the repo
13
+
14
+ `cd seamless-experiences/seamless_vc/seamless_server`
15
+
16
+ If running for the first time, create conda environment from the environment.yaml `conda env create -f environment.yml`
17
+ (or if you are on Mac OS, replace `environment.yml` with `environment_mac.yml`)
18
+
19
+ In each new terminal you use you will need to activate the conda environment:
20
+ `conda activate smlss_server`
21
+
22
+ Install dependencies with pip:
23
+ `pip install -r requirements.txt`
24
+
25
+ Install Meta-specific dependencies with the dedicated install script:
26
+ `./scripts/update_meta_dependencies.sh`
27
+
28
+ If needed, download and extract the latest model using the dedicated script:
29
+ `./scripts/download_latest_models.sh`
30
+
31
+ ## Setting up the docker environment
32
+
33
+ Alternatively we can build a docker image (for ease of deployment).
34
+
35
+ 1. Make sure github keys are loaded into ssh-agent. Copy over the keys into ~/.ssh/ then run `eval $(ssh-agent)`, then `ssh-add` to load the keys.
36
+ 2. Run `docker-compose build` if you just want to build the image. If you want to also run the server: `docker compose up --build`.
37
+ 3. Optionally to push the built docker image to ECR so it could be pulled into deployments: `./deploy.sh`
38
+
39
+ NOTE: Add your models to the ./models directory.
40
+
41
+ # Running the v2 ("pubsub") server
42
+
43
+ The pubsub server can be loaded with docker above or run locally with uvicorn below.
44
+ Run the server in dev mode:
45
+
46
+ ```
47
+ uvicorn app_pubsub:app --reload
48
+ ```
49
+
50
+ Run the server in prod mode:
51
+
52
+ ```
53
+ uvicorn app_pubsub:app --host 0.0.0.0
54
+ ```
55
+
56
+ To enable additional logging from uvicorn pass `--log-level debug` or `--log-level trace`.
57
+
58
+ ## Running the frontend
59
+
60
+ - For the v2 pubsub server, the frontend lives in the streaming-react-app root directory (not seamless_ui).
61
+
62
+ ## Updating the conda environment after changes to environment.yml
63
+
64
+ Run this command to install/remove packages to match the current environment.yml file:
65
+
66
+ `conda env update --prefix ./env --file environment.yml --prune`
67
+
68
+ See: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html?highlight=sharing#updating-an-environment
69
+
70
+ ## Debuging
71
+
72
+ If you enable "Server Debug Flag" when starting streaming from the client, this enables extensive debug logging and it saves audio files in /debug folder. test_no_silence.wav contains data with silence chunks removed.
seamless_server/models/s2s_m4t_expr-emma_v0.3/vad_s2st_sc_24khz_main.yaml ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ agent_class: seamless_communication.streaming.agents.mma_m4t_s2st.SeamlessS2STJointVADAgent
2
+ # checkpoint: checkpoint_best.pt
3
+ monotonic_decoder_model_name: seamless_streaming_monotonic_decoder
4
+ unity_model_name: seamless_streaming_unity
5
+ sentencepiece_model: spm_256k_nllb100.model
6
+
7
+ task: s2st
8
+ tgt_lang: "eng"
9
+ min_unit_chunk_size: 50
10
+ decision_threshold: 0.7
11
+ no_early_stop: True
12
+ block_ngrams: True
13
+ vocoder_name: vocoder_pretssel
14
+ wav2vec_yaml: wav2vec.yaml
15
+ # min_starting_wait: 12
16
+ # min_starting_wait_w2vbert: 192
17
+
18
+ config_yaml: cfg_fbank_u2t.yaml
19
+ vocoder_sample_rate: 24000
20
+ upstream_idx: 1
21
+ detokenize_only: True
22
+ device: cuda:0
23
+ max_len_a: 0
24
+ max_len_b: 1000
seamless_server/package.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "seamless_server",
3
+ "version": "1.0.0",
4
+ "scripts": {
5
+ "dev": "npm run start:debug -- --reload",
6
+ "start": "uvicorn app_pubsub:app --host 0.0.0.0",
7
+ "start:debug": "npm run start -- --log-level debug",
8
+ "start:trace": "npm run start -- --log-level trace",
9
+ "start:prod": "npm run build:client && npm run start",
10
+ "build:client": "cd ../streaming-react-app/ && npm run build"
11
+ },
12
+ "author": ""
13
+ }
seamless_server/requirements.txt ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fairseq2
2
+ # seamless_communication
3
+ Flask==2.1.3
4
+ Flask_Sockets==0.2.1
5
+ g2p_en==2.1.0
6
+ gevent==22.10.2
7
+ gevent_websocket==0.10.1
8
+ librosa==0.9.2
9
+ numpy==1.24.4
10
+ openai_whisper==20230124
11
+ protobuf==4.24.2
12
+ psola==0.0.1
13
+ pydub==0.25.1
14
+ silero==0.4.1
15
+ simuleval==1.1.1
16
+ soundfile==0.11.0
17
+ stable_ts==1.4.0
18
+ torch # specific torch version depends on fairseq2 installation
19
+ Werkzeug==2.0.3
20
+ whisper==1.1.10
21
+ colorlog==6.7.0
22
+ python-socketio==5.9.0
23
+ uvicorn[standard]==0.23.2
24
+ parallel-wavegan==0.5.5
25
+ python-jose[cryptography]==3.3.0
seamless_server/src/simuleval_agent_directory.py CHANGED
@@ -123,7 +123,7 @@ class SimulevalAgentDirectory:
123
  self.add_agent(agent_info)
124
  else:
125
  s2s_m4t_expr_agent = self.build_agent_if_available(
126
- SimulevalAgentDirectory.s2s_m4t_emma_agent,
127
  config_name="vad_s2st_sc_24khz_main.yaml",
128
  )
129
 
@@ -131,12 +131,12 @@ class SimulevalAgentDirectory:
131
  self.add_agent(
132
  AgentWithInfo(
133
  agent=s2s_m4t_expr_agent,
134
- name=SimulevalAgentDirectory.s2s_m4t_emma_agent,
135
  modalities=["s2t", "s2s"],
136
  source_langs=M4T_P0_LANGS,
137
  target_langs=["eng", "spa", "fra", "deu", "ita", "cmn"],
138
  dynamic_params=["expressive"],
139
- description="ES-EN expressive model that supports S2S and S2T",
140
  )
141
  )
142
 
 
123
  self.add_agent(agent_info)
124
  else:
125
  s2s_m4t_expr_agent = self.build_agent_if_available(
126
+ SimulevalAgentDirectory.s2s_m4t_expr_emma_agent,
127
  config_name="vad_s2st_sc_24khz_main.yaml",
128
  )
129
 
 
131
  self.add_agent(
132
  AgentWithInfo(
133
  agent=s2s_m4t_expr_agent,
134
+ name=SimulevalAgentDirectory.s2s_m4t_expr_emma_agent,
135
  modalities=["s2t", "s2s"],
136
  source_langs=M4T_P0_LANGS,
137
  target_langs=["eng", "spa", "fra", "deu", "ita", "cmn"],
138
  dynamic_params=["expressive"],
139
+ description="multilingual expressive model that supports S2S and S2T",
140
  )
141
  )
142