Zaesar Matthijs commited on
Commit
a18eb4d
0 Parent(s):

Duplicate from Matthijs/speecht5-tts-demo

Browse files

Co-authored-by: Mathijs Hollemans <[email protected]>

.gitattributes ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tflite filter=lfs diff=lfs merge=lfs -text
29
+ *.tgz filter=lfs diff=lfs merge=lfs -text
30
+ *.wasm filter=lfs diff=lfs merge=lfs -text
31
+ *.xz filter=lfs diff=lfs merge=lfs -text
32
+ *.zip filter=lfs diff=lfs merge=lfs -text
33
+ *.zst filter=lfs diff=lfs merge=lfs -text
34
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ *.pyc
2
+ __pycache__/
3
+ .DS_Store
4
+
README.md ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: SpeechT5 Speech Synthesis Demo
3
+ emoji: 👩‍🎤
4
+ colorFrom: yellow
5
+ colorTo: blue
6
+ sdk: gradio
7
+ sdk_version: 3.17.0
8
+ app_file: app.py
9
+ pinned: false
10
+ license: apache-2.0
11
+ duplicated_from: Matthijs/speecht5-tts-demo
12
+ ---
13
+
14
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import librosa
3
+ import numpy as np
4
+ import torch
5
+
6
+ from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
7
+
8
+
9
+ checkpoint = "microsoft/speecht5_tts"
10
+ processor = SpeechT5Processor.from_pretrained(checkpoint)
11
+ model = SpeechT5ForTextToSpeech.from_pretrained(checkpoint)
12
+ vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
13
+
14
+
15
+ speaker_embeddings = {
16
+ "BDL": "spkemb/cmu_us_bdl_arctic-wav-arctic_a0009.npy",
17
+ "CLB": "spkemb/cmu_us_clb_arctic-wav-arctic_a0144.npy",
18
+ "KSP": "spkemb/cmu_us_ksp_arctic-wav-arctic_b0087.npy",
19
+ "RMS": "spkemb/cmu_us_rms_arctic-wav-arctic_b0353.npy",
20
+ "SLT": "spkemb/cmu_us_slt_arctic-wav-arctic_a0508.npy",
21
+ }
22
+
23
+
24
+ def predict(text, speaker):
25
+ if len(text.strip()) == 0:
26
+ return (16000, np.zeros(0).astype(np.int16))
27
+
28
+ inputs = processor(text=text, return_tensors="pt")
29
+
30
+ # limit input length
31
+ input_ids = inputs["input_ids"]
32
+ input_ids = input_ids[..., :model.config.max_text_positions]
33
+
34
+ if speaker == "Surprise Me!":
35
+ # load one of the provided speaker embeddings at random
36
+ idx = np.random.randint(len(speaker_embeddings))
37
+ key = list(speaker_embeddings.keys())[idx]
38
+ speaker_embedding = np.load(speaker_embeddings[key])
39
+
40
+ # randomly shuffle the elements
41
+ np.random.shuffle(speaker_embedding)
42
+
43
+ # randomly flip half the values
44
+ x = (np.random.rand(512) >= 0.5) * 1.0
45
+ x[x == 0] = -1.0
46
+ speaker_embedding *= x
47
+
48
+ #speaker_embedding = np.random.rand(512).astype(np.float32) * 0.3 - 0.15
49
+ else:
50
+ speaker_embedding = np.load(speaker_embeddings[speaker[:3]])
51
+
52
+ speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0)
53
+
54
+ speech = model.generate_speech(input_ids, speaker_embedding, vocoder=vocoder)
55
+
56
+ speech = (speech.numpy() * 32767).astype(np.int16)
57
+ return (16000, speech)
58
+
59
+
60
+ title = "SpeechT5: Speech Synthesis"
61
+
62
+ description = """
63
+ The <b>SpeechT5</b> model is pre-trained on text as well as speech inputs, with targets that are also a mix of text and speech.
64
+ By pre-training on text and speech at the same time, it learns unified representations for both, resulting in improved modeling capabilities.
65
+
66
+ SpeechT5 can be fine-tuned for different speech tasks. This space demonstrates the <b>text-to-speech</b> (TTS) checkpoint for the English language.
67
+
68
+ See also the <a href="https://huggingface.co/spaces/Matthijs/speecht5-asr-demo">speech recognition (ASR) demo</a>
69
+ and the <a href="https://huggingface.co/spaces/Matthijs/speecht5-vc-demo">voice conversion demo</a>.
70
+
71
+ <b>How to use:</b> Enter some English text and choose a speaker. The output is a mel spectrogram, which is converted to a mono 16 kHz waveform by the
72
+ HiFi-GAN vocoder. Because the model always applies random dropout, each attempt will give slightly different results.
73
+ The <em>Surprise Me!</em> option creates a completely randomized speaker.
74
+ """
75
+
76
+ article = """
77
+ <div style='margin:20px auto;'>
78
+
79
+ <p>References: <a href="https://arxiv.org/abs/2110.07205">SpeechT5 paper</a> |
80
+ <a href="https://github.com/microsoft/SpeechT5/">original GitHub</a> |
81
+ <a href="https://huggingface.co/mechanicalsea/speecht5-tts">original weights</a></p>
82
+
83
+ <pre>
84
+ @article{Ao2021SpeechT5,
85
+ title = {SpeechT5: Unified-Modal Encoder-Decoder Pre-training for Spoken Language Processing},
86
+ author = {Junyi Ao and Rui Wang and Long Zhou and Chengyi Wang and Shuo Ren and Yu Wu and Shujie Liu and Tom Ko and Qing Li and Yu Zhang and Zhihua Wei and Yao Qian and Jinyu Li and Furu Wei},
87
+ eprint={2110.07205},
88
+ archivePrefix={arXiv},
89
+ primaryClass={eess.AS},
90
+ year={2021}
91
+ }
92
+ </pre>
93
+
94
+ <p>Speaker embeddings were generated from <a href="http://www.festvox.org/cmu_arctic/">CMU ARCTIC</a> using <a href="https://huggingface.co/mechanicalsea/speecht5-vc/blob/main/manifest/utils/prep_cmu_arctic_spkemb.py">this script</a>.</p>
95
+
96
+ </div>
97
+ """
98
+
99
+ examples = [
100
+ ["It is not in the stars to hold our destiny but in ourselves.", "BDL (male)"],
101
+ ["The octopus and Oliver went to the opera in October.", "CLB (female)"],
102
+ ["She sells seashells by the seashore. I saw a kitten eating chicken in the kitchen.", "RMS (male)"],
103
+ ["Brisk brave brigadiers brandished broad bright blades, blunderbusses, and bludgeons—balancing them badly.", "SLT (female)"],
104
+ ["A synonym for cinnamon is a cinnamon synonym.", "BDL (male)"],
105
+ ["How much wood would a woodchuck chuck if a woodchuck could chuck wood? He would chuck, he would, as much as he could, and chuck as much wood as a woodchuck would if a woodchuck could chuck wood.", "CLB (female)"],
106
+ ]
107
+
108
+ gr.Interface(
109
+ fn=predict,
110
+ inputs=[
111
+ gr.Text(label="Input Text"),
112
+ gr.Radio(label="Speaker", choices=[
113
+ "BDL (male)",
114
+ "CLB (female)",
115
+ "KSP (male)",
116
+ "RMS (male)",
117
+ "SLT (female)",
118
+ "Surprise Me!"
119
+ ],
120
+ value="BDL (male)"),
121
+ ],
122
+ outputs=[
123
+ gr.Audio(label="Generated Speech", type="numpy"),
124
+ ],
125
+ title=title,
126
+ description=description,
127
+ article=article,
128
+ examples=examples,
129
+ ).launch()
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ git+https://github.com/huggingface/transformers.git
2
+ torch
3
+ torchaudio
4
+ soundfile
5
+ librosa
6
+ samplerate
7
+ resampy
8
+ sentencepiece
spkemb/cmu_us_awb_arctic-wav-arctic_a0002.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5db7a684ab490f21cec1628e00d461a184e369fe4eafb1ee441a796faf4ab6ae
3
+ size 2176
spkemb/cmu_us_bdl_arctic-wav-arctic_a0009.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:215326eae3a428af8934c385fbe043b36c72849ca17d1d013adeb189e6bd6962
3
+ size 2176
spkemb/cmu_us_clb_arctic-wav-arctic_a0144.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf67b36c47edfb1851466a1dff081b436bc6809b5ebc12811d9df0c0d0f28d0e
3
+ size 2176
spkemb/cmu_us_ksp_arctic-wav-arctic_b0087.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6c5c2a38c2e400179019c560a74c4322f4ee13beda22ee601807545edee283e
3
+ size 2176
spkemb/cmu_us_rms_arctic-wav-arctic_b0353.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a49dac3e9c3a71a4dbca4c364233c7915ae6e0cb71b2ceaed97296231b95cb50
3
+ size 2176
spkemb/cmu_us_slt_arctic-wav-arctic_a0508.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f71ffadda3f3a4de079740a0b34963824dc644d9d5442283bd0a2b0d4f44ff0b
3
+ size 2176