pip install librosa torch transformers soundfile

import librosa
import numpy as np

# ===================================================================
# 1. 定义核心参数 - 理解这些是理解对齐的关键
# ===================================================================
SR = 22050              # Sample Rate (采样率): 每秒的样本点数
HOP_LENGTH = 512        # Hop Length (帧移): STFT窗口每次滑动的样本点数
N_MFCC = 13             # Number of MFCCs: 要计算的MFCC系数数量

# ===================================================================
# 2. 加载音频并提取MFCC
# ===================================================================
# 使用librosa内置的音频样本，方便直接运行
y, sr_native = librosa.load(librosa.ex('trumpet'), duration=5)
# 确保音频采样率与我们定义的SR一致
if sr_native != SR:
    y = librosa.resample(y, orig_sr=sr_native, target_sr=SR)

# 提取MFCC特征
mfccs = librosa.feature.mfcc(y=y, sr=SR, n_mfcc=N_MFCC, hop_length=HOP_LENGTH)

# ===================================================================
# 3. 理解MFCC输出的数据含义
# ===================================================================
# mfccs.shape 的形式是 (n_mfcc, T)
# - n_mfcc 是我们指定的系数数量 (行)
# - T 是计算出的总时间帧数 (列)
T = mfccs.shape[1]
print(f"MFCC 矩阵形状: {mfccs.shape}")
print(f"- {mfccs.shape[0]} 行: 每个MFCC系数 (代表音色)")
print(f"- {mfccs.shape[1]} 列: 每个时间帧 (代表时间)")

# 每一列 (一个时间帧) 的含义:
# - mfccs[0, j]: 第j帧的能量/响度信息
# - mfccs[1:, j]: 第j帧的音色信息 (频谱包络的形状)

# ===================================================================
# 4. 时间对齐: 将音频帧与25fps视频帧同步
# ===================================================================
VIDEO_FPS = 25

# 关键点1: 计算每个音频帧之间的时间间隔 (秒)
time_per_audio_frame = HOP_LENGTH / SR
print(f"\n每个音频特征帧代表的时间长度: {time_per_audio_frame:.4f} 秒")

# 关键点2: 计算任意一个音频帧j的中心时间戳
# 例如，计算第100个音频帧的时间
audio_frame_index = 100
audio_time = audio_frame_index * time_per_audio_frame
print(f"音频第 {audio_frame_index} 帧的中心时间点: {audio_time:.3f} 秒")

# 关键点3: 对于给定的视频帧k，找到对应的音频帧j
# 假设我们想知道第50个视频帧，应该匹配哪个音频特征？
video_frame_index = 50
video_time = video_frame_index / VIDEO_FPS

# 公式: j = round(k * sr / (fps * hop_length))
corresponding_audio_frame = int(round(video_time / time_per_audio_frame))

print(f"\n--- 对齐示例 ---")
print(f"视频第 {video_frame_index} 帧的时间点是: {video_time:.3f} 秒")
print(f"通过计算，它最接近的音频特征是第 {corresponding_audio_frame} 帧。")

# 验证一下
corresponding_audio_time = corresponding_audio_frame * time_per_audio_frame
print(f"(验证: 音频第 {corresponding_audio_frame} 帧的时间点是 {corresponding_audio_time:.3f} 秒，两者非常接近)")

import torch
import librosa
from transformers import HubertModel, Wav2Vec2FeatureExtractor

# 1. 加载预训练模型和特征提取器
model_name = "facebook/hubert-large-ls960-ft"
feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_name)
model = HubertModel.from_pretrained(model_name)

# 2. 加载并预处理音频
# 确保音频采样率为16kHz，与模型预训练时一致
audio_path = librosa.ex('trumpet')
y, sr = librosa.load(audio_path, sr=16000)

# 特征提取器将波形转换为模型所需的输入格式
inputs = feature_extractor(y, sampling_rate=sr, return_tensors="pt")

# 3. 模型推理
with torch.no_grad():
    outputs = model(**inputs)

# 4. 获取特征
# last_hidden_state 包含了模型最后一层输出的序列特征
# 这就是Hubert提取出的高质量音频特征
last_hidden_state = outputs.last_hidden_state

print(f'Input waveform shape: {y.shape}')
print(f'Feature extractor output shape: {inputs.input_values.shape}')
print(f'Hubert last hidden state shape: {last_hidden_state.shape}') # (批次大小, 序列长度, 隐藏层维度)

import torch
import librosa
from transformers import WavLMModel, AutoFeatureExtractor

# 1. 加载预训练模型和特征提取器
# WavLM有多种版本，Base+ 适用于多种任务
model_name = "microsoft/wavlm-base-plus"
feature_extractor = AutoFeatureExtractor.from_pretrained(model_name)
model = WavLMModel.from_pretrained(model_name)

# 2. 加载并预处理音频 (同样需要16kHz)
y, sr = librosa.load(librosa.ex('trumpet'), sr=16000)

inputs = feature_extractor(y, sampling_rate=sr, return_tensors="pt")

# 3. 模型推理
with torch.no_grad():
    outputs = model(**inputs)

# 4. 获取特征
# WavLM的输出同样可以作为通用音频特征
last_hidden_state = outputs.last_hidden_state

# WavLM的另一个强大之处在于可以提取用于声纹识别的x-vector
# xvector = outputs.xvector # (批次大小, xvector维度)

print(f'WavLM last hidden state shape: {last_hidden_state.shape}')

import torch
import librosa
from transformers import WhisperProcessor, WhisperForConditionalGeneration

# 1. 加载预训练模型和处理器
# 处理器包含了特征提取器和tokenizer
model_name = "openai/whisper-tiny"
processor = WhisperProcessor.from_pretrained(model_name)
model = WhisperForConditionalGeneration.from_pretrained(model_name)

# 2. 加载并预处理音频 (需要16kHz)
# 使用一个更适合语音识别的例子
y, sr = librosa.load(librosa.ex('libri1'), sr=16000)

inputs = processor(y, sampling_rate=sr, return_tensors="pt")

# 3. 模型推理：生成转录结果的token ID

# 4. 解码：将token ID转换为文字
transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)

print(f'Audio Transcription: {transcription[0]}')