基于Huggingface的定制音频数据情感识别

trainer.train()

python -m venv audio_env
source activate audio_env/bin/activate

pip install datasets

dataset_config = {
  "LOADING_SCRIPT_FILES": os.path.join(PROJECT_ROOT, "crema.py"),
  "CONFIG_NAME": "clean",
  "DATA_DIR": os.path.join(PROJECT_ROOT, "data/archive.zip"),
  "CACHE_DIR": os.path.join(PROJECT_ROOT, "cache_crema"),
}
ds = load_dataset(
  dataset_config["LOADING_SCRIPT_FILES"],
  dataset_config["CONFIG_NAME"],
  data_dir=dataset_config["DATA_DIR"],
  cache_dir=dataset_config["CACHE_DIR"]
)
print(ds)
********* OUTPUT ********DatasetDict({
    train: Dataset({
        features: ['file', 'label'],
        num_rows: 7442
    })
})

import os
import logging
import librosa

import wandb
import numpy as np

from datasets import DatasetDict, load_dataset, load_metric
from transformers import (
    HubertForSequenceClassification,
    PretrainedConfig,
    Trainer,
    TrainingArguments,
    Wav2Vec2FeatureExtractor,
)
from utils import collator

logging.basicConfig(
    format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO
)

USER = "vsher"
WANDB_PROJECT = "audio-classifier"
wandb.init(entity=USER, project=WANDB_PROJECT)

>>> from transformers import ViTFeatureExtractor
>>> vit_extractor = ViTFeatureExtractor()
>>> print(vit_extractor)

ViTFeatureExtractor {
“do_normalize”: true,
“do_resize”: true,
“feature_extractor_type”: “ViTFeatureExtractor”,
“image_mean”: [0.5, 0.5, 0.5],
“image_std”: [0.5, 0.5, 0.5],
“resample”: 2,
“size”: 224
}

from transformers import Wav2Vec2FeatureExtractor
feature_extractor = Wav2Vec2FeatureExtractor()
print(feature_extractor)
**** OUTPUT ****
Wav2Vec2FeatureExtractor {
   "do_normalize": true,
   "feature_extractor_type": "Wav2Vec2FeatureExtractor",
   "feature_size": 1,
   "padding_side": "right",
   "padding_value": 0.0,
   "return_attention_mask": false,
   "sampling_rate": 16000 
}

from transformers import Wav2Vec2FeatureExtractor
feature_extractor = Wav2Vec2FeatureExtractor(
          sampling_rate=24000,
          truncation=True
)
print(feature_extractor)
**** OUTPUT ****
Wav2Vec2FeatureExtractor {
   "do_normalize": true,
   "feature_extractor_type": "Wav2Vec2FeatureExtractor",
   "feature_size": 1,
   "padding_side": "right",
   "padding_value": 0.0,
   "return_attention_mask": false,
   "sampling_rate": 24000,
   "truncation": true
}

from transformers import Wav2Vec2FeatureExtractor
model = "facebook/hubert-base-ls960"
feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model)
print(feature_extractor)
*** OUTPUT ***
Wav2Vec2FeatureExtractor {
   "do_normalize": true,
   "feature_extractor_type": "Wav2Vec2FeatureExtractor",
   "feature_size": 1,
   "padding_side": "right",
   "padding_value": 0,
   "return_attention_mask": false,
   "sampling_rate": 16000
 }

model_id = "facebook/hubert-base-ls960"
feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_id)
audio_file = "dummy1.wav"
audio_array = librosa.load(audio_file, sr=16000, mono=False)[0]
input = feature_extractor(
       raw_speech=audio_array,
       sampling_rate=16000,
       padding=True,
      return_tensors="pt"
)
print(input)
print(input.shape)
print(audio_array.shape)
***** OUTPUT ******
{'input_values': tensor([[-0.0003, -0.0003, -0.0003,  ...,  0.0006, -0.0003, -0.0003]])}
torch.Size([1, 36409])
(36409,)

audio_file_1 = "dummy1.wav"
audio_file_2 = "dummy2.wav"
audio_array_1 = librosa.load(audio_file_1, sr=16000, mono=False)[0]
audio_array_2 = librosa.load(audio_file_2, sr=16000, mono=False)[0]
input_with_one_audio = feature_extractor(
       audio_array_1,
       sampling_rate=16000,
       padding=True,
      return_tensors="pt"
)
input_with_two_audio = feature_extractor(
       [audio_array_1, audio_array_2],
       sampling_rate=16000,
       padding=True,
      return_tensors="pt"
)
print(input_with_one_audio.input_values.shape)
print(input_with_two_audios.input_values.shape)
***** OUTPUT ****
torch.Size([1, 36409])
torch.Size([2, 37371])

bare_model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")
last_hidden_state = bare_model(input.input_values).last_hidden_state
print(last_hidden_state.shape)
*** OUTPUT ***
torch.Size([1, 113, 1024]) # the hidden size i.e. 113 can vary depending on audio

model_path = ""facebook/hubert-large-ls960-ft""
hubert_model = HubertForSequenceClassification.from_pretrained(model_path)
hubert_model_config = hubert_model.config
print("Num of labels:", hubert_model_config.num_labels)
**** OUTPUT ****
Num of labels: 2

NUM_LABELS = 6
model_id = "facebook/hubert-base-ls960"

config = PretrainedConfig.from_pretrained(model_id, num_labels=NUM_LABELS)
hubert_model = HubertForSequenceClassification.from_pretrained(
    model_id,
    config=config,  # because we need to update num_labels as per our dataset
    ignore_mismatched_sizes=True,  # to avoid classifier size mismatch from from_pretrained.
)

#首先冻结所有神经网络层
for param in hubert_model.parameters():
    param.requires_grad = False
    
#冻结两个编码器层 
layers_freeze_num = 2
n_layers = (
    4 + layers_freeze_num * 16
)  # 4是指投影器和分类器的权重和偏差。
for name, param in list(hubert_model.named_parameters())[-n_layers:]:
    param.requires_grad = True

dataset_config = {
    "LOADING_SCRIPT_FILES": os.path.join(PROJECT_ROOT, "src/data/crema.py"),
    "CONFIG_NAME": "clean",
    "DATA_FILES": os.path.join(PROJECT_ROOT,"data/archive.zip"),
    "CACHE_DIR": os.path.join(PROJECT_ROOT, "cache_crema")
}

ds = load_dataset(
    dataset_config["LOADING_SCRIPT_FILES"],
    dataset_config["CONFIG_NAME"],
    cache_dir=dataset_config["CACHE_DIR"],
    data_files=dataset_config["DATA_FILES"]
)

# 将原始音频转换为阵列
ds = ds.map(
        lambda x: {
            "array": librosa.load(x["file"], sr=16000, mono=False)[0]
        },
        num_proc=2,
    )

print(ds)
***** OUTPUT *****
DatasetDict({
    train: Dataset({
        features: ['file', 'label', 'array'],
        num_rows: 7442
    })
})

#将数据集处理为训练模型所需的格式

INPUT_FIELD = "input_values"
LABEL_FIELD = "labels"

def prepare_dataset(batch, feature_extractor):
    audio_arr = batch["array"]
    input = feature_extractor(
        audio_arr, sampling_rate=16000, padding=True, return_tensors="pt"
    )

    batch[INPUT_FIELD] = input.input_values[0]
    batch[LABEL_FIELD] = batch[
        "label"
    ]  #colname必须是标签，因为Trainer 类默认会查找它

    return batch

# 针对所有样本使用特征抽取器来预处理数据
ds = ds.map(
    prepare_dataset,
    fn_kwargs={"feature_extractor": feature_extractor},
    # num_proc=2,
)
logging.info("Finished extracting features from audio arrays.")

# LABEL TO ID
ds = ds.class_encode_column("label")

# 引入了训练－测试－值分割

# 90%用于训练，10%用于测试+验证
train_testvalid = ds["train"].train_test_split(shuffle=True, test_size=0.1)
# 把用于测试+验证的10%进一步对半分割：一半用于测试，一半用于校验
test_valid = train_testvalid['test'].train_test_split(test_size=0.5)
# 如果您想拥有一个DatasetDict的话，请收集所有人的信息
ds = DatasetDict({
    'train': train_testvalid['train'],
    'test': test_valid['test'],
    'val': test_valid['train']})

trainer_config = {
  "OUTPUT_DIR": "results",
  "TRAIN_EPOCHS": 3,
  "TRAIN_BATCH_SIZE": 8,
  "EVAL_BATCH_SIZE": 8,
  "GRADIENT_ACCUMULATION_STEPS": 4,
  "WARMUP_STEPS": 500,
  "DECAY": 0.01,
  "LOGGING_STEPS": 10,
  "MODEL_DIR": "models/test-hubert-model",
  "SAVE_STEPS": 100
}

# 使用Trainer类进行微调
training_args = TrainingArguments(
    output_dir=trainer_config["OUTPUT_DIR"],  # output directory
    gradient_accumulation_steps=trainer_config[
        "GRADIENT_ACCUMULATION_STEPS"
    ],  # 在运行优化步骤之前累积梯度
    num_train_epochs=trainer_config[
        "TRAIN_EPOCHS"
    ],  #总训练迭代次数
    per_device_train_batch_size=trainer_config[
        "TRAIN_BATCH_SIZE"
    ],  # 训练期间每个设备的批量大小
    per_device_eval_batch_size=trainer_config[
        "EVAL_BATCH_SIZE"
    ],  #用于评估的批量大小
    warmup_steps=trainer_config[
        "WARMUP_STEPS"
    ],  # 学习率调度器的预热步骤数
    save_steps=trainer_config["SAVE_STEPS"], # 每100步保存检查点
    weight_decay=trainer_config["DECAY"],  #权重衰减强度
    logging_steps=trainer_config["LOGGING_STEPS"],
    evaluation_strategy="epoch",  # 在每个迭代末尾输出指标值
    report_to="wandb",  # 启动指向W&B的日志功能
)

#开始训练
trainer = Trainer(
    model=hubert_model,  # 实例：对应于将训练的转换器模型
    args=training_args,  # 训练参数，如下所定义的
    data_collator=data_collator,
    train_dataset=ds["train"],  # 训练数据集
    eval_dataset=ds["val"],  # 评估数据集
    compute_metrics=compute_metrics,
)

# 定义数据收集器以便动态填充训练批次
data_collator = DataCollatorCTCWithPadding(
            processor=feature_extractor,
            padding=True
)

from dataclasses import dataclass
from typing import Dict, List, Optional, Union

import torch
from transformers import Wav2Vec2Processor

INPUT_FIELD = "input_values"
LABEL_FIELD = "labels"


@dataclass
class DataCollatorCTCWithPadding:
    processor: Wav2Vec2Processor
    padding: Union[bool, str] = True
    max_length: Optional[int] = None
    max_length_labels: Optional[int] = None
    pad_to_multiple_of: Optional[int] = None
    pad_to_multiple_of_labels: Optional[int] = None

    def __call__(
        self, examples: List[Dict[str, Union[List[int], torch.Tensor]]]
    ) -> Dict[str, torch.Tensor]:

        input_features = [
            {INPUT_FIELD: example[INPUT_FIELD]} for example in examples
        ]  # 样本基本上对应于row0, row1,等等...
        labels = [example[LABEL_FIELD] for example in examples]

        batch = self.processor.pad(
            input_features,
            padding=self.padding,
            max_length=self.max_length,
            pad_to_multiple_of=self.pad_to_multiple_of,
            return_tensors="pt",
        )
        batch[LABEL_FIELD] = torch.tensor(labels)

        return batch

from datasets import load_metric

def compute_metrics(eval_pred):
    # 定义评估指标
    compute_accuracy_metric = load_metric("accuracy")
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return compute_accuracy_metric.compute(predictions=predictions, references=labels)

trainer = Trainer(model=..., args=...,...)
output = trainer.predict(ds["test"])
print(output)
**** 输出*****
PredictionOutput(
predictions=array([
       [ 0.0331, -0.0193, -0.98767, 0.0229, 0.01693, -0.0745],
       [-0.0445,  0.0020, 0.13196, 0.2219, 0.94693, -0.0614],
        .
        .
        .
], dtype=float32),
label_ids=array([0, 5, ......]),
metrics={'test_loss': 1.780486822128296, 'test_accuracy': 0.0, 'test_runtime': 1.6074, 'test_samples_per_second': 1.244, 'test_steps_per_second': 0.622}
)

trainer.train()

# 从检查点恢复培训
# trainer.train("results/checkpoint-2000")

#测试结果
test_results = trainer.predict(ds["test"])
logging.info("Test Set Result: {}".format(test_results.metrics))
wandb.log({"test_accuracy": test_results.metrics["test_accuracy"]})

trainer.save_model(os.path.join(PROJECT_ROOT, trainer_config["MODEL_DIR"]))

#把训练后的模型数据日志保存到wandb
wandb.save(
    os.path.join(PROJECT_ROOT, trainer_config["MODEL_DIR"], "*"),
    base_path=os.path.dirname(trainer_config["MODEL_DIR"]),
    policy="end",
)

# 提前停止训练
trainer = Trainer(
  callbacks=[EarlyStoppingCallback(early_stopping_patience = 10)]
)