Unity3D平台实现全景实时RTMP|RTSP流渲染_unity音视频采集渲染

好多开发者的使用场景，需要在Windows特别是Android平台实现Unity3D的全景实时视频渲染，本文以Windows平台为例，简单介绍下具体实现：

如果是RTSP或RTMP流数据，实际上难点，主要在于拉取RTSP或RTMP流，解析解码，然后把解码后的YUV数据，回调到Unity层，Unity创建个Sphere，创建个材质球(Material)，并把材质球挂在到Sphere即可。

本文以Windows推送端采集全景视频，编码推送到RTMP服务器，播放端拉流回调数据并在Unity渲染为例（左侧是Unity播放端，滑动鼠标，可以实现全景内容切换）：

废话不多说，大概流程如下：

本文以调用我们写的RTSP、RTMP直播播放模块为例，首先是初始化模块，然后设置拉流的参数信息：

public void Play(int sel)
{
    if (videoctrl[sel].is_running)
    {
        Debug.Log("已经在播放..");
        return;
    }
 
    lock (videoctrl[sel].frame_lock_)
    {
        videoctrl[sel].cur_video_frame_ = null;
    }
 
    OpenPlayer(sel);
 
    if (videoctrl[sel].player_handle_ == IntPtr.Zero)
        return;
 
    //设置播放URL
    NTSmartPlayerSDK.NT_SP_SetURL(videoctrl[sel].player_handle_, videoctrl[sel].videoUrl);
 
    /* ++ 播放前参数配置可加在此处 ++ */
 
    int play_buffer_time_ = 0;
    NTSmartPlayerSDK.NT_SP_SetBuffer(videoctrl[sel].player_handle_, play_buffer_time_);                 //设置buffer time
 
    int is_using_tcp = 0;        //TCP模式
 
    NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(videoctrl[sel].player_handle_, is_using_tcp);
 
    int timeout = 10;
    NTSmartPlayerSDK.NT_SP_SetRtspTimeout(videoctrl[sel].player_handle_, timeout);
 
    int is_auto_switch_tcp_udp = 1;
    NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(videoctrl[sel].player_handle_, is_auto_switch_tcp_udp);
 
    Boolean is_mute_ = false;
    NTSmartPlayerSDK.NT_SP_SetMute(videoctrl[sel].player_handle_, is_mute_ ? 1 : 0);                    //是否启动播放的时候静音
 
    int is_fast_startup = 1;
    NTSmartPlayerSDK.NT_SP_SetFastStartup(videoctrl[sel].player_handle_, is_fast_startup);              //设置快速启动模式
 
    Boolean is_low_latency_ = false;
    NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(videoctrl[sel].player_handle_, is_low_latency_ ? 1 : 0);    //设置是否启用低延迟模式
 
    //设置旋转角度(设置0， 90， 180， 270度有效，其他值无效)
    int rotate_degrees = 0;
    NTSmartPlayerSDK.NT_SP_SetRotation(videoctrl[sel].player_handle_, rotate_degrees);
 
int volume = 100;
NTSmartPlayerSDK.NT_SP_SetAudioVolume(videoctrl[sel].player_handle_, volume);   //设置播放音量, 范围是[0, 100], 0是静音，100是最大音量, 默认是100
 
 
    // 设置上传下载报速度
    int is_report = 0;
    int report_interval = 1;
    NTSmartPlayerSDK.NT_SP_SetReportDownloadSpeed(videoctrl[sel].player_handle_, is_report, report_interval);
    /* -- 播放前参数配置可加在此处 -- */
 
    //video frame callback (YUV/RGB)
    videoctrl[sel].video_frame_call_back_ = new SP_SDKVideoFrameCallBack(NT_SP_SetVideoFrameCallBack);
    NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(videoctrl[sel].player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, window_handle_, videoctrl[sel].video_frame_call_back_);
 
    UInt32 flag = NTSmartPlayerSDK.NT_SP_StartPlay(videoctrl[sel].player_handle_);
 
    if (flag == DANIULIVE_RETURN_OK)
    {
        videoctrl[sel].is_need_get_frame_ = true;
        Debug.Log("播放成功");
    }
    else
    {
        videoctrl[sel].is_need_get_frame_ = false;
        Debug.LogError("播放失败");
    }
 
    videoctrl[sel].is_running = true;
}

具体OpenPlayer()实现：

private void OpenPlayer(int sel)
{
    window_handle_ = IntPtr.Zero;
 
    if (videoctrl[sel].player_handle_ == IntPtr.Zero)
    {
        videoctrl[sel].player_handle_ = new IntPtr();
        UInt32 ret_open = NTSmartPlayerSDK.NT_SP_Open(out videoctrl[sel].player_handle_, window_handle_, 0, IntPtr.Zero);
        if (ret_open != 0)
        {
            videoctrl[sel].player_handle_ = IntPtr.Zero;
            Debug.LogError("调用NT_SP_Open失败..");
            return;
        }
    }
 
    videoctrl[sel].event_call_back_ = new SP_SDKEventCallBack(NT_SP_SDKEventCallBack);
    NTSmartPlayerSDK.NT_SP_SetEventCallBack(videoctrl[sel].player_handle_, window_handle_, videoctrl[sel].event_call_back_);
 
    videoctrl[sel].sdk_video_frame_call_back_ = new VideoControl.SetVideoFrameCallBack(SDKVideoFrameCallBack);
    videoctrl[sel].sdk_event_call_back_ = new VideoControl.SetEventCallBack(SDKEventCallBack);
}

数据回调到Unity层：

private void SDKVideoFrameCallBack(UInt32 status, IntPtr frame, int sel)
{
    //这里拿到回调frame，进行相关操作
    NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));
 
    VideoFrame  u3d_frame = new VideoFrame();
 
    u3d_frame.width_  = video_frame.width_;
    u3d_frame.height_ = video_frame.height_;
 
    u3d_frame.timestamp_ = (UInt64)video_frame.timestamp_;
 
    int d_y_stride = video_frame.width_;
    int d_u_stride = (video_frame.width_ + 1) / 2;
    int d_v_stride = d_u_stride;
 
    int d_y_size = d_y_stride * video_frame.height_;
    int d_u_size = d_u_stride * ((video_frame.height_ + 1) / 2);
    int d_v_size = d_u_size;
 
    int u_v_height = ((u3d_frame.height_ + 1) / 2);
 
    u3d_frame.y_stride_ = d_y_stride;
    u3d_frame.u_stride_ = d_u_stride;
    u3d_frame.v_stride_ = d_v_stride;
 
    u3d_frame.y_data_ = new byte[d_y_size];
    u3d_frame.u_data_ = new byte[d_u_size];
    u3d_frame.v_data_ = new byte[d_v_size];
 
 
    CopyFramePlane(u3d_frame.y_data_, d_y_stride,
        video_frame.plane0_, video_frame.stride0_, u3d_frame.height_);
 
    CopyFramePlane(u3d_frame.u_data_, d_u_stride,
       video_frame.plane1_, video_frame.stride1_, u_v_height);
 
    CopyFramePlane(u3d_frame.v_data_, d_v_stride,
       video_frame.plane2_, video_frame.stride2_, u_v_height);
 
    lock (videoctrl[sel].frame_lock_ )
    {
        videoctrl[sel].cur_video_frame_ = u3d_frame;
        //Debug.LogError("sel: " + sel + " w:" + u3d_frame.width_ + "h:" + u3d_frame.height_);
    }
}

Unity刷新Texture：

private void UpdateYUVTexture(VideoFrame video_frame, int sel)
{
    if (video_frame.y_data_ == null || video_frame.u_data_ == null || video_frame.v_data_ == null)
    {
        Debug.Log("video frame with null..");
        return;
    }
 
    if (videoctrl[sel].yTexture_ != null)
    {
        videoctrl[sel].yTexture_.LoadRawTextureData(video_frame.y_data_);
        videoctrl[sel].yTexture_.Apply();
    }
 
    if (videoctrl[sel].uTexture_ != null)
    {
        videoctrl[sel].uTexture_.LoadRawTextureData(video_frame.u_data_);
        videoctrl[sel].uTexture_.Apply();
    }
 
    if (videoctrl[sel].vTexture_ != null)
    {
        videoctrl[sel].vTexture_.LoadRawTextureData(video_frame.v_data_);
        videoctrl[sel].vTexture_.Apply();
    }
}

全景播放，还需要考虑到鼠标或屏幕滑动，这块实现比较多，放个通用的代码参考：

void Update () {
      if (Input.GetMouseButton(0))
      {
          if (PreMouseLPos.x <= 0)
          {
              PreMouseLPos = new Vector3(Input.mousePosition.x, Input.mousePosition.y, 0.0f);
          }
          else
          {
              Vector3 CurMouseLPos = new Vector3(Input.mousePosition.x, Input.mousePosition.y, 0.0f);
              Vector3 offset = CurMouseLPos - PreMouseLPos;
 
              Quaternion tt = Quaternion.Euler(offset);
              float rotationX = transform.localEulerAngles.y - tt.x * 20;
              rotationY += tt.y * 20;
              rotationY = Mathf.Clamp(rotationY, minimumY, maximumY);
              transform.localEulerAngles = new Vector3(rotationY, rotationX, 0);
 
              PreMouseLPos = CurMouseLPos;
          }
      }
      else
      {
          PreMouseLPos = new Vector3(0.0f, 0.0f, 0.0f);
      }
}

感兴趣的开发者，可尝试看看。