多模态 | 多模态中单模态中提取特征方法（代码）_多模态特征提取

在多模态任务中，有一种方法时在单模态中先各自提取各模态的特征，然后进行融合，本文主要实现各模态特征的提取。

（Torch框架）

视觉特征提取

在图像（Image）中提取特征（如果是视频的话，需要将视频分帧）

#/usr/bin/env python
"""Script to extract ResNet features from video frames."""
import argparse
from typing import Any, Tuple
 
import h5py
from overrides import overrides
import torch
import torch.nn
import torch.utils.data
import torchvision
from tqdm import tqdm
 
from c3d import C3D
from i3d import I3D
from dataset import SarcasmDataset
 
# noinspection PyUnresolvedReferences
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 
def pretrained_resnet152() -> torch.nn.Module:
    resnet152 = torchvision.models.resnet152(pretrained=True)
    resnet152.eval()
    for param in resnet152.parameters():
        param.requires_grad = False
    return resnet152
 
 
def pretrained_c3d() -> torch.nn.Module:
    c3d = C3D(pretrained=True)
    c3d.eval()
    for param in c3d.parameters():
        param.requires_grad = False
    return c3d
 
 
def pretrained_i3d() -> torch.nn.Module:
    i3d = I3D(pretrained=True)
    i3d.eval()
    for param in i3d.parameters():
        param.requires_grad = False
    return i3d
 
 
def save_resnet_features() -> None:
    transforms = torchvision.transforms.Compose([
        torchvision.transforms.Resize(256),
        torchvision.transforms.CenterCrop(224),
        torchvision.transforms.ToTensor(),
        torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])
    dataset = SarcasmDataset(transform=transforms)
 
    resnet = pretrained_resnet152().to(DEVICE)
 
    class Identity(torch.nn.Module):
        @overrides
        def forward(self, input_: torch.Tensor) -> torch.Tensor:
            return input_
 
    resnet.fc = Identity()  # Trick to avoid computing the fc1000 layer, as we don't need it here.
 
    with h5py.File(SarcasmDataset.features_file_path("resnet", "res5c"), "w") as res5c_features_file, \
            h5py.File(SarcasmDataset.features_file_path("resnet", "pool5"), "w") as pool5_features_file:
 
        for video_id in dataset.video_ids:
            video_frame_count = dataset.frame_count_by_video_id[video_id]
            res5c_features_file.create_dataset(video_id, shape=(video_frame_count, 2048, 7, 7))
            pool5_features_file.create_dataset(video_id, shape=(video_frame_count, 2048))
 
        res5c_output = None
 
        def avg_pool_hook(_module: torch.nn.Module, input_: Tuple[torch.Tensor], _output: Any) -> None:
            nonlocal res5c_output
            res5c_output = input_[0]
 
        resnet.avgpool.register_forward_hook(avg_pool_hook)
 
        total_frame_count = sum(dataset.frame_count_by_video_id[video_id] for video_id in dataset.video_ids)
        with tqdm(total=total_frame_count, desc="Extracting ResNet features") as progress_bar:
            for instance in torch.utils.data.DataLoader(dataset):
                video_id = instance["id"][0]
                frames = instance["frames"][0].to(DEVICE)
 
                batch_size = 32
                for start_index in range(0, len(frames), batch_size):
                    end_index = min(start_index + batch_size, len(frames))
                    frame_ids_range = range(start_index, end_index)
                    frame_batch = frames[frame_ids_range]
 
                    avg_pool_value = resnet(frame_batch)
 
                    res5c_features_file[video_id][frame_ids_range] = res5c_output.cpu()  # noqa
                    pool5_features_file[video_id][frame_ids_range] = avg_pool_value.cpu()
 
                    progress_bar.update(len(frame_ids_range))
 
 
def save_c3d_features() -> None:
    transforms = torchvision.transforms.Compose([
        torchvision.transforms.Resize(128),
        torchvision.transforms.CenterCrop(112),
        torchvision.transforms.ToTensor(),
        torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])
    dataset = SarcasmDataset(transform=transforms)
 
    c3d = pretrained_c3d().to(DEVICE)
 
    with h5py.File(SarcasmDataset.features_file_path("c3d", "fc7"), "w") as fc7_features_file:
        for video_id in dataset.video_ids:
            video_frame_count = dataset.frame_count_by_video_id[video_id]
            feature_count = video_frame_count - 16 + 1
            fc7_features_file.create_dataset(video_id, shape=(feature_count, 4096))
 
        for instance in tqdm(torch.utils.data.DataLoader(dataset), desc="Extracting C3D features"):
            video_id = instance["id"][0]  # noqa
            video_frame_count = dataset.frame_count_by_video_id[video_id]
            feature_count = video_frame_count - 16 + 1
            frames = instance["frames"][0].to(DEVICE)
            frames = frames.unsqueeze(0)  # Add batch dimension
            frames = frames.transpose(1, 2)  # C3D expects (B, C, T, H, W)
 
            for i in range(feature_count):
                output = c3d.extract_features(frames[:, :, i:i + 16, :, :]).squeeze()
                fc7_features_file[video_id][i, :] = output.cpu().data.numpy()
 
 
def save_i3d_features() -> None:
    transforms = torchvision.transforms.Compose([
        torchvision.transforms.Resize(256),
        torchvision.transforms.CenterCrop(224),
        torchvision.transforms.ToTensor(),
    ])
    dataset = SarcasmDataset(transform=transforms)
 
    i3d = pretrained_i3d().to(DEVICE)
 
    with h5py.File(SarcasmDataset.features_file_path("i3d", "avg_pool"), "w") as avg_pool_features_file:
        for video_id in dataset.video_ids:
            video_frame_count = dataset.frame_count_by_video_id[video_id]
            feature_count = video_frame_count - 16 + 1
            avg_pool_features_file.create_dataset(video_id, shape=(feature_count, 1024))
 
        for instance in tqdm(torch.utils.data.DataLoader(dataset), desc="Extracting I3D features"):
            video_id = instance["id"][0]  # noqa
            video_frame_count = dataset.frame_count_by_video_id[video_id]
            feature_count = video_frame_count - 16 + 1
            frames = instance["frames"][0].to(DEVICE)
            frames = frames.unsqueeze(0)  # Add batch dimension
            frames = frames.transpose(1, 2)  # I3D expects (B, C, T, H, W)
 
            for i in range(feature_count):
                output = i3d.extract_features(frames[:, :, i:i + 16, :, :]).squeeze()
                avg_pool_features_file[video_id][i, :] = output.cpu().data.numpy()
 
 
def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Extract video features.")
    parser.add_argument("network", choices=["resnet", "c3d", "i3d"])
    return parser.parse_args()
 
 
def main() -> None:
    args = parse_args()
    if args.network == "resnet":
        save_resnet_features()
    elif args.network == "c3d":
        save_c3d_features()
    elif args.network == "i3d":
        save_i3d_features()
    else:
        raise ValueError(f"Network type not supported: {args.network}")
 
 
if __name__ == "__main__":
    main()

代码提取特征并将它们保存到大型 H5 文件中。运行命令

python extract_features.py resnet

语音特征提取

#!/usr/bin/env python
import os
import pickle
 
import librosa
import numpy as np
from tqdm.auto import tqdm
 
AUDIOS_FOLDER = "data/audios/utterances_final"
AUDIO_FEATURES_PATH = "data/audio_features.p"
 
 
def get_librosa_features(path: str) -> np.ndarray:
    y, sr = librosa.load(path)
 
    hop_length = 512  # Set the hop length; at 22050 Hz, 512 samples ~= 23ms
 
    # Remove vocals first
    D = librosa.stft(y, hop_length=hop_length)
    S_full, phase = librosa.magphase(D)
 
    S_filter = librosa.decompose.nn_filter(S_full, aggregate=np.median, metric="cosine",
                                           width=int(librosa.time_to_frames(0.2, sr=sr)))
 
    S_filter = np.minimum(S_full, S_filter)
 
    margin_i, margin_v = 2, 4
    power = 2
    mask_v = librosa.util.softmask(S_full - S_filter, margin_v * S_filter, power=power)
    S_foreground = mask_v * S_full
 
    # Recreate vocal_removal y
    new_D = S_foreground * phase
    y = librosa.istft(new_D)
 
    mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=13)  # Compute MFCC features from the raw signal
    mfcc_delta = librosa.feature.delta(mfcc)  # And the first-order differences (delta features)
 
    S = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=128, fmax=8000)
    S_delta = librosa.feature.delta(S)
 
    spectral_centroid = librosa.feature.spectral_centroid(S=S_full)
 
    audio_feature = np.vstack((mfcc, mfcc_delta, S, S_delta, spectral_centroid))  # combine features
 
    # binning data
    jump = int(audio_feature.shape[1] / 10)
    return librosa.util.sync(audio_feature, range(1, audio_feature.shape[1], jump))
 
 
def save_audio_features() -> None:
    audio_feature = {}
    for filename in tqdm(os.listdir(AUDIOS_FOLDER), desc="Computing the audio features"):
        id_ = filename.rsplit(".", maxsplit=1)[0]
        audio_feature[id_] = get_librosa_features(os.path.join(AUDIOS_FOLDER, filename))
        print(audio_feature[id_].shape)
 
    with open(AUDIO_FEATURES_PATH, "wb") as file:
        pickle.dump(audio_feature, file, protocol=2)
 
 
def get_audio_duration() -> None:
    filenames = os.listdir(AUDIOS_FOLDER)
    print(sum(librosa.core.get_duration(filename=os.path.join(AUDIOS_FOLDER, filename))
              for filename in tqdm(filenames, desc="Computing the average duration of the audios")) / len(filenames))
 
 
def main() -> None:
    get_audio_duration()
 
    # save_audio_features()
    #
    # with open(AUDIO_FEATURES_PATH, "rb") as file:
    #     pickle.load(file)
 
 
if __name__ == "__main__":
    main()

文本特征提取（BERT）

详情请参考【2】

# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Extract pre-computed feature vectors from BERT."""
 
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
 
import codecs
import collections
import json
import re
 
import modeling
import tokenization
import tensorflow as tf
 
flags = tf.flags
 
FLAGS = flags.FLAGS
 
flags.DEFINE_string("input_file", None, "")
 
flags.DEFINE_string("output_file", None, "")
 
flags.DEFINE_string("layers", "-1,-2,-3,-4", "")
 
flags.DEFINE_string(
    "bert_config_file", None,
    "The config json file corresponding to the pre-trained BERT model. "
    "This specifies the model architecture.")
 
flags.DEFINE_integer(
    "max_seq_length", 128,
    "The maximum total input sequence length after WordPiece tokenization. "
    "Sequences longer than this will be truncated, and sequences shorter "
    "than this will be padded.")
 
flags.DEFINE_string(
    "init_checkpoint", None,
    "Initial checkpoint (usually from a pre-trained BERT model).")
 
flags.DEFINE_string("vocab_file", None,
                    "The vocabulary file that the BERT model was trained on.")
 
flags.DEFINE_bool(
    "do_lower_case", True,
    "Whether to lower case the input text. Should be True for uncased "
    "models and False for cased models.")
 
flags.DEFINE_integer("batch_size", 32, "Batch size for predictions.")
 
flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.")
 
flags.DEFINE_string("master", None,
                    "If using a TPU, the address of the master.")
 
flags.DEFINE_integer(
    "num_tpu_cores", 8,
    "Only used if `use_tpu` is True. Total number of TPU cores to use.")
 
flags.DEFINE_bool(
    "use_one_hot_embeddings", False,
    "If True, tf.one_hot will be used for embedding lookups, otherwise "
    "tf.nn.embedding_lookup will be used. On TPUs, this should be True "
    "since it is much faster.")
 
 
class InputExample(object):
 
  def __init__(self, unique_id, text_a, text_b):
    self.unique_id = unique_id
    self.text_a = text_a
    self.text_b = text_b
 
 
class InputFeatures(object):
  """A single set of features of data."""
 
  def __init__(self, unique_id, tokens, input_ids, input_mask, input_type_ids):
    self.unique_id = unique_id
    self.tokens = tokens
    self.input_ids = input_ids
    self.input_mask = input_mask
    self.input_type_ids = input_type_ids
 
 
def input_fn_builder(features, seq_length):
  """Creates an `input_fn` closure to be passed to TPUEstimator."""
 
  all_unique_ids = []
  all_input_ids = []
  all_input_mask = []
  all_input_type_ids = []
 
  for feature in features:
    all_unique_ids.append(feature.unique_id)
    all_input_ids.append(feature.input_ids)
    all_input_mask.append(feature.input_mask)
    all_input_type_ids.append(feature.input_type_ids)
 
  def input_fn(params):
    """The actual input function."""
    batch_size = params["batch_size"]
 
    num_examples = len(features)
 
    # This is for demo purposes and does NOT scale to large data sets. We do
    # not use Dataset.from_generator() because that uses tf.py_func which is
    # not TPU compatible. The right way to load data is with TFRecordReader.
    d = tf.data.Dataset.from_tensor_slices({
        "unique_ids":
            tf.constant(all_unique_ids, shape=[num_examples], dtype=tf.int32),
        "input_ids":
            tf.constant(
                all_input_ids, shape=[num_examples, seq_length],
                dtype=tf.int32),
        "input_mask":
            tf.constant(
                all_input_mask,
                shape=[num_examples, seq_length],
                dtype=tf.int32),
        "input_type_ids":
            tf.constant(
                all_input_type_ids,
                shape=[num_examples, seq_length],
                dtype=tf.int32),
    })
 
    d = d.batch(batch_size=batch_size, drop_remainder=False)
    return d
 
  return input_fn
 
 
def model_fn_builder(bert_config, init_checkpoint, layer_indexes, use_tpu,
                     use_one_hot_embeddings):
  """Returns `model_fn` closure for TPUEstimator."""
 
  def model_fn(features, labels, mode, params):  # pylint: disable=unused-argument
    """The `model_fn` for TPUEstimator."""
 
    unique_ids = features["unique_ids"]
    input_ids = features["input_ids"]
    input_mask = features["input_mask"]
    input_type_ids = features["input_type_ids"]
 
    model = modeling.BertModel(
        config=bert_config,
        is_training=False,
        input_ids=input_ids,
        input_mask=input_mask,
        token_type_ids=input_type_ids,
        use_one_hot_embeddings=use_one_hot_embeddings)
 
    if mode != tf.estimator.ModeKeys.PREDICT:
      raise ValueError("Only PREDICT modes are supported: %s" % (mode))
 
    tvars = tf.trainable_variables()
    scaffold_fn = None
    (assignment_map,
     initialized_variable_names) = modeling.get_assignment_map_from_checkpoint(
         tvars, init_checkpoint)
    if use_tpu:
 
      def tpu_scaffold():
        tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
        return tf.train.Scaffold()
 
      scaffold_fn = tpu_scaffold
    else:
      tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
 
    tf.logging.info("**** Trainable Variables ****")
    for var in tvars:
      init_string = ""
      if var.name in initialized_variable_names:
        init_string = ", *INIT_FROM_CKPT*"
      tf.logging.info("  name = %s, shape = %s%s", var.name, var.shape,
                      init_string)
 
    all_layers = model.get_all_encoder_layers()
 
    predictions = {
        "unique_id": unique_ids,
    }
 
    for (i, layer_index) in enumerate(layer_indexes):
      predictions["layer_output_%d" % i] = all_layers[layer_index]
 
    output_spec = tf.contrib.tpu.TPUEstimatorSpec(
        mode=mode, predictions=predictions, scaffold_fn=scaffold_fn)
    return output_spec
 
  return model_fn
 
 
def convert_examples_to_features(examples, seq_length, tokenizer):
  """Loads a data file into a list of `InputBatch`s."""
 
  features = []
  for (ex_index, example) in enumerate(examples):
    tokens_a = tokenizer.tokenize(example.text_a)
 
    tokens_b = None
    if example.text_b:
      tokens_b = tokenizer.tokenize(example.text_b)
 
    if tokens_b:
      # Modifies `tokens_a` and `tokens_b` in place so that the total
      # length is less than the specified length.
      # Account for [CLS], [SEP], [SEP] with "- 3"
      _truncate_seq_pair(tokens_a, tokens_b, seq_length - 3)
    else:
      # Account for [CLS] and [SEP] with "- 2"
      if len(tokens_a) > seq_length - 2:
        tokens_a = tokens_a[0:(seq_length - 2)]
 
    # The convention in BERT is:
    # (a) For sequence pairs:
    #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
    #  type_ids: 0     0  0    0    0     0       0 0     1  1  1  1   1 1
    # (b) For single sequences:
    #  tokens:   [CLS] the dog is hairy . [SEP]
    #  type_ids: 0     0   0   0  0     0 0
    #
    # Where "type_ids" are used to indicate whether this is the first
    # sequence or the second sequence. The embedding vectors for `type=0` and
    # `type=1` were learned during pre-training and are added to the wordpiece
    # embedding vector (and position vector). This is not *strictly* necessary
    # since the [SEP] token unambiguously separates the sequences, but it makes
    # it easier for the model to learn the concept of sequences.
    #
    # For classification tasks, the first vector (corresponding to [CLS]) is
    # used as as the "sentence vector". Note that this only makes sense because
    # the entire model is fine-tuned.
    tokens = []
    input_type_ids = []
    tokens.append("[CLS]")
    input_type_ids.append(0)
    for token in tokens_a:
      tokens.append(token)
      input_type_ids.append(0)
    tokens.append("[SEP]")
    input_type_ids.append(0)
 
    if tokens_b:
      for token in tokens_b:
        tokens.append(token)
        input_type_ids.append(1)
      tokens.append("[SEP]")
      input_type_ids.append(1)
 
    input_ids = tokenizer.convert_tokens_to_ids(tokens)
 
    # The mask has 1 for real tokens and 0 for padding tokens. Only real
    # tokens are attended to.
    input_mask = [1] * len(input_ids)
 
    # Zero-pad up to the sequence length.
    while len(input_ids) < seq_length:
      input_ids.append(0)
      input_mask.append(0)
      input_type_ids.append(0)
 
    assert len(input_ids) == seq_length
    assert len(input_mask) == seq_length
    assert len(input_type_ids) == seq_length
 
    if ex_index < 5:
      tf.logging.info("*** Example ***")
      tf.logging.info("unique_id: %s" % (example.unique_id))
      tf.logging.info("tokens: %s" % " ".join(
          [tokenization.printable_text(x) for x in tokens]))
      tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
      tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
      tf.logging.info(
          "input_type_ids: %s" % " ".join([str(x) for x in input_type_ids]))
 
    features.append(
        InputFeatures(
            unique_id=example.unique_id,
            tokens=tokens,
            input_ids=input_ids,
            input_mask=input_mask,
            input_type_ids=input_type_ids))
  return features
 
 
def _truncate_seq_pair(tokens_a, tokens_b, max_length):
  """Truncates a sequence pair in place to the maximum length."""
 
  # This is a simple heuristic which will always truncate the longer sequence
  # one token at a time. This makes more sense than truncating an equal percent
  # of tokens from each, since if one sequence is very short then each token
  # that's truncated likely contains more information than a longer sequence.
  while True:
    total_length = len(tokens_a) + len(tokens_b)
    if total_length <= max_length:
      break
    if len(tokens_a) > len(tokens_b):
      tokens_a.pop()
    else:
      tokens_b.pop()
 
 
def read_examples(input_file):
  """Read a list of `InputExample`s from an input file."""
  examples = []
  unique_id = 0
  with tf.gfile.GFile(input_file, "r") as reader:
    while True:
      line = tokenization.convert_to_unicode(reader.readline())
      if not line:
        break
      line = line.strip()
      text_a = None
      text_b = None
      m = re.match(r"^(.*) \|\|\| (.*)$", line)
      if m is None:
        text_a = line
      else:
        text_a = m.group(1)
        text_b = m.group(2)
      examples.append(
          InputExample(unique_id=unique_id, text_a=text_a, text_b=text_b))
      unique_id += 1
  return examples
 
 
def main(_):
  tf.logging.set_verbosity(tf.logging.INFO)
 
  layer_indexes = [int(x) for x in FLAGS.layers.split(",")]
 
  bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
 
  tokenizer = tokenization.FullTokenizer(
      vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)
 
  is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
  run_config = tf.contrib.tpu.RunConfig(
      master=FLAGS.master,
      tpu_config=tf.contrib.tpu.TPUConfig(
          num_shards=FLAGS.num_tpu_cores,
          per_host_input_for_training=is_per_host))
 
  examples = read_examples(FLAGS.input_file)
 
  features = convert_examples_to_features(
      examples=examples, seq_length=FLAGS.max_seq_length, tokenizer=tokenizer)
 
  unique_id_to_feature = {}
  for feature in features:
    unique_id_to_feature[feature.unique_id] = feature
 
  model_fn = model_fn_builder(
      bert_config=bert_config,
      init_checkpoint=FLAGS.init_checkpoint,
      layer_indexes=layer_indexes,
      use_tpu=FLAGS.use_tpu,
      use_one_hot_embeddings=FLAGS.use_one_hot_embeddings)
 
  # If TPU is not available, this will fall back to normal Estimator on CPU
  # or GPU.
  estimator = tf.contrib.tpu.TPUEstimator(
      use_tpu=FLAGS.use_tpu,
      model_fn=model_fn,
      config=run_config,
      predict_batch_size=FLAGS.batch_size)
 
  input_fn = input_fn_builder(
      features=features, seq_length=FLAGS.max_seq_length)
 
  with codecs.getwriter("utf-8")(tf.gfile.Open(FLAGS.output_file,
                                               "w")) as writer:
    for result in estimator.predict(input_fn, yield_single_examples=True):
      unique_id = int(result["unique_id"])
      feature = unique_id_to_feature[unique_id]
      output_json = collections.OrderedDict()
      output_json["linex_index"] = unique_id
      all_features = []
      for (i, token) in enumerate(feature.tokens):
        all_layers = []
        for (j, layer_index) in enumerate(layer_indexes):
          layer_output = result["layer_output_%d" % j]
          layers = collections.OrderedDict()
          layers["index"] = layer_index
          layers["values"] = [
              round(float(x), 6) for x in layer_output[i:(i + 1)].flat
          ]
          all_layers.append(layers)
        features = collections.OrderedDict()
        features["token"] = token
        features["layers"] = all_layers
        all_features.append(features)
      output_json["features"] = all_features
      writer.write(json.dumps(output_json) + "\n")
 
 
if __name__ == "__main__":
  flags.mark_flag_as_required("input_file")
  flags.mark_flag_as_required("vocab_file")
  flags.mark_flag_as_required("bert_config_file")
  flags.mark_flag_as_required("init_checkpoint")
  flags.mark_flag_as_required("output_file")
  tf.app.run()

参考文献

【1】GitHub - soujanyaporia/MUStARD: Multimodal Sarcasm Detection Dataset

【2】GitHub - google-research/bert at d66a146741588fb208450bde15aa7db143baaa69