app.py

import spaces
import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.models.speecht5.number_normalizer import EnglishNumberNormalizer
from string import punctuation
import re
from parler_tts import ParlerTTSForConditionalGeneration
from transformers import AutoTokenizer, AutoFeatureExtractor, set_seed

# Device setup
device = "cuda:0" if torch.cuda.is_available() else "cpu"

# SmolLM setup
checkpoint = "HuggingFaceTB/SmolLM-360M"
smol_tokenizer = AutoTokenizer.from_pretrained(checkpoint)
smol_model = AutoModelForCausalLM.from_pretrained(checkpoint, device_map="auto", torch_dtype=torch.bfloat16)

# Original model setup
repo_id = "ylacombe/p-m-e"
model = ParlerTTSForConditionalGeneration.from_pretrained(repo_id).to(device)
text_tokenizer = AutoTokenizer.from_pretrained(repo_id)
description_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-large")
feature_extractor = AutoFeatureExtractor.from_pretrained(repo_id)

SAMPLE_RATE = feature_extractor.sampling_rate
SEED = 42

default_text = "La voix humaine est un instrument de musique au-dessus de tous les autres."
default_description = "a woman with a slightly low-pitched voice speaks slowly in a clear and close-sounding environment, but her delivery is quite monotone."
examples = [
    [
        "La voix humaine est un instrument de musique au-dessus de tous les autres.",
        "a woman with a slightly low-pitched voice speaks slowly in a clear and close-sounding environment, but her delivery is quite monotone.",
        True,
        None,
    ],
    [
        "The human voice is nature's most perfect instrument.",
        "A woman with a slightly low-pitched voice speaks slowly in a very distant-sounding environment with a clean audio quality, delivering her message in a very monotone manner.",
        True,
        None,
    ],
]

number_normalizer = EnglishNumberNormalizer()

def format_description(raw_description, do_format=True):
    if not do_format:
        return raw_description
        
    prompt = f"""Format this voice description to match exactly:
"a [gender] with a [pitch] voice speaks [speed] in a [environment], [delivery style]"
Where:
- gender: man/woman
- pitch: slightly low-pitched/moderate pitch/high-pitched
- speed: slowly/moderately/quickly
- environment: close-sounding and clear/distant-sounding and noisy
- delivery style: with monotone delivery/with animated delivery

Description to format: {raw_description}
Formatted description:"""
    
    inputs = smol_tokenizer.encode(prompt, return_tensors="pt").to(device)
    outputs = smol_model.generate(
        inputs,
        max_length=200,
        num_return_sequences=1,
        temperature=0.7,
        do_sample=True,
        pad_token_id=smol_tokenizer.eos_token_id
    )
    formatted = smol_tokenizer.decode(outputs[0], skip_special_tokens=True)
    return formatted.split("Formatted description:")[-1].strip()

def preprocess(text):
    text = number_normalizer(text).strip()
    text = text.replace("-", " ")
    if text[-1] not in punctuation:
        text = f"{text}."
    
    abbreviations_pattern = r'\b[A-Z][A-Z\.]+\b'
    
    def separate_abb(chunk):
        chunk = chunk.replace(".","")
        return " ".join(chunk)
    
    abbreviations = re.findall(abbreviations_pattern, text)
    for abv in abbreviations:
        if abv in text:
            text = text.replace(abv, separate_abb(abv))
    return text

@spaces.GPU
def gen_tts(text, description, do_format=True):
    formatted_desc = format_description(description, do_format)
    inputs = description_tokenizer(formatted_desc.strip(), return_tensors="pt").to(device)
    prompt = text_tokenizer(preprocess(text), return_tensors="pt").to(device)

    set_seed(SEED)
    generation = model.generate(
        input_ids=inputs.input_ids, 
        prompt_input_ids=prompt.input_ids, 
        attention_mask=inputs.attention_mask, 
        prompt_attention_mask=prompt.attention_mask, 
        do_sample=True, 
        temperature=1.0
    )
    audio_arr = generation.cpu().numpy().squeeze()
    return formatted_desc, (SAMPLE_RATE, audio_arr)

css = """
        #share-btn-container {
            display: flex;
            padding-left: 0.5rem !important;
            padding-right: 0.5rem !important;
            background-color: #000000;
            justify-content: center;
            align-items: center;
            border-radius: 9999px !important; 
            width: 13rem;
            margin-top: 10px;
            margin-left: auto;
            flex: unset !important;
        }
        #share-btn {
            all: initial;
            color: #ffffff;
            font-weight: 600;
            cursor: pointer;
            font-family: 'IBM Plex Sans', sans-serif;
            margin-left: 0.5rem !important;
            padding-top: 0.25rem !important;
            padding-bottom: 0.25rem !important;
            right:0;
        }
        #share-btn * {
            all: unset !important;
        }
        #share-btn-container div:nth-child(-n+2){
            width: auto !important;
            min-height: 0px !important;
        }
        #share-btn-container .wrap {
            display: none !important;
        }
"""

with gr.Blocks(css=css) as block:
    gr.HTML(
        """
            <div style="text-align: center; max-width: 700px; margin: 0 auto;">
              <div style="display: inline-flex; align-items: center; gap: 0.8rem; font-size: 1.75rem;">
                <h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;">
                  Multi Parler-TTS 🗣️
                </h1>
              </div>
            </div>
        """
    )
    gr.HTML(
        """<p><a href="https://github.com/huggingface/parler-tts">Parler-TTS</a> is a training and inference library for
high-fidelity text-to-speech (TTS) models.</p> 
<p>This multilingual model supports French, Spanish, Italian, Portuguese, Polish, German, Dutch, and English. It generates high-quality speech with features that can be controlled using a simple text prompt.</p>
<p>By default, Parler-TTS generates 🎲 random voice characteristics. To ensure 🎯 <b>speaker consistency</b> across generations, try to use consistent descriptions in your prompts.</p>"""
    )
    
    with gr.Row():
        with gr.Column():
            input_text = gr.Textbox(
                label="Input Text", 
                lines=2, 
                value=default_text
            )
            raw_description = gr.Textbox(
                label="Voice Description", 
                lines=2, 
                value=default_description
            )
            do_format = gr.Checkbox(
                label="Reformat description using SmolLM", 
                value=True
            )
            formatted_description = gr.Textbox(
                label="Used Description", 
                lines=2
            )
            generate_button = gr.Button("Generate Audio", variant="primary")
        with gr.Column():
            audio_out = gr.Audio(label="Parler-TTS generation", type="numpy")

    generate_button.click(
        fn=gen_tts,
        inputs=[input_text, raw_description, do_format],
        outputs=[formatted_description, audio_out]
    )

    gr.Examples(
        examples=examples,
        fn=gen_tts,
        inputs=[input_text, raw_description, do_format],
        outputs=[formatted_description, audio_out],
        cache_examples=True
    )

    gr.HTML(
        """<p>Tips for ensuring good generation:
        <ul>
            <li>Include the term "very clear audio" to generate the highest quality audio, and "very noisy audio" for high levels of background noise</li>
            <li>Punctuation can be used to control the prosody of the generations</li>
            <li>The remaining speech features (gender, speaking rate, pitch and reverberation) can be controlled directly through the prompt</li>
        </ul>
        </p>"""
    )

block.queue()
block.launch(share=True)