Normalize audio

app.py

@@ -6,12 +6,24 @@ import torchaudio
 import time
 from transformers import pipeline
 from speechbrain.inference.classifiers import EncoderClassifier
+from transformers import WhisperProcessor, WhisperForConditionalGeneration
 
-transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-tiny")
+# Whisperモデルとプロセッサのロード
+model_name = "openai/whisper-tiny"
+processor = WhisperProcessor.from_pretrained(model_name)
+model = WhisperForConditionalGeneration.from_pretrained(model_name)
+# デバイスの設定（GPUが利用可能な場合はGPUを使用）
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model.to(device)
+
+
+# speechbrainの言語分類モデルのロード
 language_id = EncoderClassifier.from_hparams(source="speechbrain/lang-id-voxlingua107-ecapa")
 
+# アプリケーションの状態を保持する変数
 data = []
 current_chunk = []
+
 index_to_lang = {
     0: 'Abkhazian', 1: 'Afrikaans', 2: 'Amharic', 3: 'Arabic', 4: 'Assamese',
     5: 'Azerbaijani', 6: 'Bashkir', 7: 'Belarusian', 8: 'Bulgarian', 9: 'Bengali',
@@ -40,6 +52,15 @@ lang_index_JA_EN = {
     'ja': 45,
     'en': 20,
 }
+SAMPLING_RATE = 16000
+CHUNK_DURATION = 5 # 5秒ごとのチャンク
+
+
+def normalize_audio(audio):
+    # 音量の正規化（最大振幅が1になるようにスケーリング）
+    audio = audio / np.max(np.abs(audio))
+    return audio
+
 
 def resample_audio(audio, orig_sr, target_sr=16000):
     if orig_sr != target_sr:
@@ -50,20 +71,21 @@ def resample_audio(audio, orig_sr, target_sr=16000):
     return audio
 
 
-SAMPLING_RATE = 16000
-CHUNK_DURATION = 5  # 5秒ごとのチャンク
-
 def process_audio(audio):
     global data, current_chunk
     print("Process_audio")
     print(audio)
     sr, audio_data = audio
 
-    print(audio_data.shape)
+
+    print(audio_data.shape, audio_data.dtype)
     # 一番最初にSampling rateを揃えておく
     audio_data = resample_audio(audio_data, sr, target_sr=SAMPLING_RATE)
     audio_sec = 0
 
+    # 音量の正規化
+    audio_data = normalize_audio(audio_data)
+
     # 新しいデータを現在のチャンクに追加
     current_chunk.append(audio_data)
     total_chunk = np.concatenate(current_chunk)
@@ -87,9 +109,11 @@ def process_audio(audio):
         top3_indices = torch.topk(lang_guess[0], 3, dim=1, largest=True).indices[0]
         top3_languages = [index_to_lang[idx.item()] for idx in top3_indices]
 
-        # transcript
-        transcript = transcriber(chunk)
-        print(transcript)
+        input_features = processor(chunk, sampling_rate=SAMPLING_RATE, return_tensors="pt").input_features.to(device)
+        predicted_ids = model.generate(input_features)
+        transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)[0]
+        # transcript = transcribe_audio(chunk, SAMPLING_RATE)
+        print(transcription)
 
         data.append({
             # "Time": pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S'),
@@ -98,7 +122,7 @@ def process_audio(audio):
             "Volume": volume_norm,
             "Japanese_English": f"{ja_en} ({ja_prob:.2f}, {en_prob:.2f})",
             "Language": top3_languages,
-            "Text": transcript['text'],
+            "Text": transcription,
         })
 
         df = pd.DataFrame(data)