Unity3D下如何实现跨平台低延迟的RTMP、RTSP播放

技术背景

好多开发者，希望我们能探讨下Unity平台RTMP或RTSP直播流数据播放和录制相关的模块，实际上，这块流程我们已经聊过多次，无非就是通过原生的RTMP或者RTSP模块，先从协议层拉取到数据，并解包解码，回调YUV或RGB数据，然后，在Unity创建响应的shader，获取图像数据填充纹理即可，说起来流程很简单，但是每个环节，如果做到极致体验，都非常难。简单来说，多一次拷贝，都会增大性能瓶颈或延迟。

目前，Unity3D下，我们覆盖了以下常用的模块：

Windows平台RTMP直播推送模块（采集Unity窗体、摄像头或屏幕）;
Windows平台轻量级RTSP服务模块（采集Unity窗体、摄像头或屏幕）;
Windows平台RTMP|RTSP直播播放模块；
Linux平台RTMP直播推送模块（采集Unity窗体、Unity声音），也可扩展轻量级RTSP服务模块;
Linux平台RTMP|RTSP直播播放模块；
Android平台RTMP直播推送模块（采集Unity窗体、麦克风或Unity声音）；
Android平台轻量级RTSP服务模块（采集Unity窗体、麦克风或Unity声音）；
Android平台RTMP|RTSP直播播放模块；
iOS平台RTMP|RTSP直播播放模块。

下图系Linux平台RTMP播放图，可以看到，延迟非常低。

技术实现

本文以Android平台RTMP、RTSP播放模块为例，介绍下Unity相关接口设置和逻辑处理：

开始播放

    public void Play()
    {
        if (is_running)
        {
            Debug.Log("已经在播放。。");   
            return;
        }

        //获取输入框的url
        string videoUrl = input_url_.text.Trim();

        OpenPlayer();

        if ( player_handle_ == 0 )
            return;

        NT_U3D_Set_Game_Object(player_handle_, game_object_);

        /* ++ 播放前参数配置可加在此处 ++ */
        int is_using_tcp = 0;        //TCP/UDP模式设置
        NT_U3D_SetRTSPTcpMode(player_handle_, is_using_tcp);

        int is_report = 0;
        int report_interval = 1;
        NT_U3D_SetReportDownloadSpeed(player_handle_, is_report, report_interval);  //下载速度回调

        NT_U3D_SetBuffer(player_handle_, play_buffer_time_);                        //设置buffer time

        NT_U3D_SetPlayerLowLatencyMode(player_handle_, is_low_latency_ ? 1 : 0);    //设置是否启用低延迟模式

        NT_U3D_SetMute(player_handle_, is_mute_ ? 1 : 0);                           //是否启动播放的时候静音

        NT_U3D_SetAudioVolume(player_handle_, cur_audio_volume_);                   //设置播放音量

        NT_U3D_SetVideoDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0);          //设置H.264软硬解模式

        NT_U3D_SetVideoHevcDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0);          //设置H.265软硬解模式

        int is_fast_startup = 1;
        NT_U3D_SetFastStartup(player_handle_, is_fast_startup);                     //设置快速启动模式

        int rtsp_timeout = 10;
        NT_U3D_SetRTSPTimeout(player_handle_, rtsp_timeout);                        //设置RTSP超时时间

        int is_auto_switch_tcp_udp = 1;
        NT_U3D_SetRTSPAutoSwitchTcpUdp(player_handle_, is_auto_switch_tcp_udp);    //设置TCP/UDP模式自动切换

        int is_audiotrack = 1;
        NT_U3D_SetAudioOutputType(player_handle_, is_audiotrack);                   //设置音频输出模式: if 0: 自动选择; if with 1: audiotrack模式

        NT_U3D_SetUrl(player_handle_, videoUrl);
        /* -- 播放前参数配置可加在此处 -- */

        int flag = NT_U3D_StartPlay(player_handle_);

        if (flag  == DANIULIVE_RETURN_OK)
        {
            is_need_get_frame_ = true;
            Debug.Log("播放成功");
        }
        else
        {
            is_need_get_frame_ = false;
            Debug.LogError("播放失败");
        }

        is_running = true;  
    }

这里调用了OpenPlayer()设计如下：

    private void OpenPlayer()
    {
        if ( java_obj_cur_activity_ == null )
        {
            Debug.LogError("getApplicationContext is null");
            return;
        }

        player_handle_ = NT_U3D_Open();

        if (player_handle_ != 0)
            Debug.Log("open success");
        else
            Debug.LogError("open fail");
    }

停止播放：

    private void ClosePlayer()
    {
        is_need_get_frame_ = false;
        is_need_init_texture_ = false;
 
        int flag = NT_U3D_StopPlay(player_handle_);
        if (flag == DANIULIVE_RETURN_OK)
        {
            Debug.Log("停止成功");
        }
        else
        {
            Debug.LogError("停止失败");
        }

        flag = NT_U3D_Close(player_handle_);
        if (flag == DANIULIVE_RETURN_OK)
        {
            Debug.Log("关闭成功");
        }
        else
        {
            Debug.LogError("关闭失败");
        }

        player_handle_ = 0;

        NT_U3D_UnInit();

        is_running = false;
        video_width_ = 0;
        video_height_ = 0;
    }

Event状态回调处理：

    /// <summary>
    /// android 传递过来 code
    /// </summary>
    /// <param name="event_message"></param>
    public void onNTSmartEvent(string event_message)
    {
        if (null == event_message || event_message.Length < 1)
            return;

        string[] strs = event_message.Split(',');
        if (null == strs || strs.Length < 6)
            return;

       string player_handle =strs[0];
       string code = strs[1];
       string param1 = strs[2];
       string param2 = strs[3];
       string param3 = strs[4];
       string param4 = strs[5];
        
       Debug.Log("[onNTSmartEvent] code: 0x" + Convert.ToString(Convert.ToInt32(code), 16));

       String player_event = "";

        switch (Convert.ToInt32(code))
        {
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STARTED:
                player_event = "开始..";
                break;
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING:
                player_event = "连接中..";
                break;
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED:
                player_event = "连接失败..";
                break;
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED:
                player_event = "连接成功..";
                break;
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED:
                player_event = "连接断开..";
                break;
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP:
                player_event = "停止播放..";
                break;
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO:
                player_event = "分辨率信息: width: " + Convert.ToInt32(param1) + ", height: " + Convert.ToInt32(param2);
                break;
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED:
                player_event = "收不到媒体数据，可能是url错误..";
                break;
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL:
                player_event = "切换播放URL..";
                break;

            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE:
                player_event = "快照: " + param1 + " 路径：" + param3;

                if (Convert.ToInt32(param1) == 0)
                {
                    player_event = "截取快照成功..";
                }
                else
                {
                    player_event = "截取快照失败..";
                }
                break;

            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE:
                player_event = "[record]开始一个新的录像文件 : " + param3;
                break;
            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED:
                player_event = "[record]已生成一个录像文件 : " + param3;
                break;

            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING:
                player_event = "Start_Buffering..";
                break;

            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING:
                player_event = "Buffering: " + Convert.ToInt32(param1);
                break;

            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING:
                player_event = "Stop_Buffering..";
                break;

            case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED:
                player_event = "download_speed:" + param1 + "Byte/s" + ", "
                        + (Convert.ToInt32(param1) * 8 / 1000) + "kbps" + ", " + (Convert.ToInt32(param1) / 1024)
                        + "KB/s";
                break;
        }

        Debug.Log(player_event);

        player_event = null;
        strs = null;
    }

如果想扩展录像，实际上，我们也针对播放端录像做了接口的封装设计，整体接口设计如下：

    /// <summary>
    /// SmartPlayer Unity Interface
    /// Author: daniusdk.com
    /// </summary>

    /// <summary>
    /// Init
    /// </summary>
    public int NT_U3D_Init()
    {
        return player_obj_.Call<int>("Init", java_obj_cur_activity_);
    }


    /// <summary>
    /// 开始
    /// 返回播放句柄
    /// </summary>
    public long NT_U3D_Open()
    {
        return player_obj_.Call<long>("Open");
    }

    /// <summary>
    /// Register Game Object,用于消息传递
    /// </summary>
    public int NT_U3D_Set_Game_Object(long handle, string gameObjectName)
    {
        return player_obj_.Call<int>("SetGameObject", handle, gameObjectName);
    }

    /// <summary>
    /// 设置H.264解码方式 false 软件解码 true 硬件解码 默认为false
    /// </summary>
    /// <param name="isHwDecoder"></param>
    public int NT_U3D_SetVideoDecoderMode(long handle, int isHwDecoder)
    {
        return player_obj_.Call<int>("SetPlayerVideoHWDecoder", handle, isHwDecoder);
    }

    /// <summary>
    /// 设置H.265 解码方式 false 软件解码 true 硬件解码 默认为false
    /// </summary>
    /// <param name="isHevcHwDecoder"></param>
    public int NT_U3D_SetVideoHevcDecoderMode(long handle, int isHevcHwDecoder)
    {
        return player_obj_.Call<int>("SetPlayerVideoHevcHWDecoder", handle, isHevcHwDecoder);
    }

    /// <summary>
    /// 设置音频输出模式: if 0: 自动选择; if with 1: audiotrack模式
    /// </summary>
    /// <param name="use_audiotrack"></param>
    public int NT_U3D_SetAudioOutputType(long handle, int use_audiotrack)
    {
        return player_obj_.Call<int>("SetAudioOutputType", handle, use_audiotrack);
    }

    /// <summary>
    /// 设置播放端缓存大小, 默认200毫秒
    /// </summary>
    /// <param name="buffer"></param>
    public int NT_U3D_SetBuffer(long handle, int buffer)
    {
        return player_obj_.Call<int>("SetBuffer", handle, buffer);
    }

    /// <summary>
    /// 接口可实时调用：设置是否实时静音，1：静音; 0: 取消静音
    /// </summary>
    /// <param name="is_mute"></param>
    public int NT_U3D_SetMute(long handle, int is_mute)
    {
        return player_obj_.Call<int>("SetMute", handle, is_mute);
    }

    /// <summary>
    /// 接口可实时调用：设置播放音量，范围是[0, 100], 0是静音，100是最大音量, 默认是100
    /// </summary>
    /// <param name="audio_volume"></param>
    public int NT_U3D_SetAudioVolume(long handle, int audio_volume)
    {
        return player_obj_.Call<int>("SetAudioVolume", handle, audio_volume);
    }

    /// <summary>
    /// 设置RTSP TCP模式, 1: TCP; 0: UDP
    /// </summary>
    /// <param name="is_using_tcp"></param>
    public int NT_U3D_SetRTSPTcpMode(long handle, int is_using_tcp)
    {
        return player_obj_.Call<int>("SetRTSPTcpMode", handle, is_using_tcp);
    }

    /// <summary>
    /// 设置RTSP超时时间, timeout单位为秒，必须大于0
    /// </summary>
    /// <param name="timeout"></param>
    public int NT_U3D_SetRTSPTimeout(long handle, int timeout)
    {
        return player_obj_.Call<int>("SetRTSPTimeout", handle, timeout);
    }

    /// <summary>
    /// 设置RTSP TCP/UDP自动切换
    /// NOTE: 对于RTSP来说，有些可能支持rtp over udp方式，有些可能支持使用rtp over tcp方式.
    /// 为了方便使用，有些场景下可以开启自动尝试切换开关, 打开后如果udp无法播放，sdk会自动尝试tcp, 如果tcp方式播放不了,sdk会自动尝试udp.
    /// </summary>
    /// <param name="timeout"></param>
    /// timeout：如果设置1的话, sdk将在tcp和udp之间尝试切换播放，如果设置为0，则不尝试切换.
    public int NT_U3D_SetRTSPAutoSwitchTcpUdp(long handle, int is_auto_switch_tcp_udp)
    {
        return player_obj_.Call<int>("SetRTSPAutoSwitchTcpUdp", handle, is_auto_switch_tcp_udp);
    }

    /// <summary>
    /// 设置快速启动该模式，
    /// </summary>
    /// <param name="is_fast_startup"></param>
    public int NT_U3D_SetFastStartup(long handle, int is_fast_startup)
    {
        return player_obj_.Call<int>("SetFastStartup", handle, is_fast_startup);
    }

    /// <summary>
    /// 设置超低延迟模式 false不开启 true开启 默认false
    /// </summary>
    /// <param name="mode"></param>
    public int NT_U3D_SetPlayerLowLatencyMode(long handle, int mode)
    {
        return player_obj_.Call<int>("SetPlayerLowLatencyMode", handle, mode);
    }

    /// <summary>
    /// 设置视频垂直反转
    /// is_flip： 0: 不反转, 1: 反转
    /// </summary>
    /// <param name="is_flip"></param>
    public int NT_U3D_SetFlipVertical(long handle, int is_flip)
    {
        return player_obj_.Call<int>("SetFlipVertical", handle, is_flip);
    }

    /// <summary>
    /// 设置视频水平反转
    /// is_flip： 0: 不反转, 1: 反转
    /// </summary>
    /// <param name="is_flip"></param>
    public int NT_U3D_SetFlipHorizontal(long handle, int is_flip)
    {
        return player_obj_.Call<int>("SetFlipHorizontal", handle, is_flip);
    }

    /// <summary>
    /// 设置顺时针旋转, 注意除了0度之外， 其他角度都会额外消耗性能
    /// degress： 当前支持 0度，90度, 180度, 270度 旋转
    /// </summary>
    /// <param name="degress"></param>
    public int NT_U3D_SetRotation(long handle, int degress)
    {
        return player_obj_.Call<int>("SetRotation", handle, degress);
    }

    /// <summary>
    /// 设置是否回调下载速度
    /// is_report: if 1: 上报下载速度, 0: 不上报.
    /// report_interval: 上报间隔，以秒为单位，>0.
    /// </summary>
    /// <param name="is_report"></param>
    /// <param name="report_interval"></param>
    public int NT_U3D_SetReportDownloadSpeed(long handle, int is_report, int report_interval)
    {
        return player_obj_.Call<int>("SetReportDownloadSpeed", handle, is_report, report_interval);
    }

    /// <summary>
    /// 设置是否需要在播放或录像过程中快照
    /// </summary>
    /// <param name="is_save_image"></param>
    public int NT_U3D_SetSaveImageFlag(long handle, int is_save_image)
    {
        return player_obj_.Call<int>("SetSaveImageFlag", handle, is_save_image);
    }

    /// <summary>
    /// 播放或录像过程中快照
    /// </summary>
    /// <param name="imageName"></param>
    public int NT_U3D_SaveCurImage(long handle, string imageName)
    {
        return player_obj_.Call<int>("SaveCurImage", handle, imageName);
    }

    /// <summary>
    /// 播放或录像过程中，快速切换url
    /// </summary>
    /// <param name="uri"></param>
    public int NT_U3D_SwitchPlaybackUrl(long handle, string uri)
    {
        return player_obj_.Call<int>("SwitchPlaybackUrl", handle, uri);
    }

    /// <summary>
    /// 创建录像存储路径
    /// </summary>
    /// <param name="path"></param>
    public int NT_U3D_CreateFileDirectory(string path)
    {
        return player_obj_.Call<int>("CreateFileDirectory", path);
    }

    /// <summary>
    /// 设置录像存储路径
    /// </summary>
    /// <param name="path"></param>
    public int NT_U3D_SetRecorderDirectory(long handle, string path)
    {
        return player_obj_.Call<int>("SetRecorderDirectory", handle, path);
    }

    /// <summary>
    /// 设置单个录像文件大小
    /// </summary>
    /// <param name="size"></param>
    public int NT_U3D_SetRecorderFileMaxSize(long handle, int size)
    {
        return player_obj_.Call<int>("SetRecorderFileMaxSize", handle, size);
    }

    /// <summary>
    /// 设置录像时音频转AAC编码的开关
    /// aac比较通用，sdk增加其他音频编码(比如speex, pcmu, pcma等)转aac的功能.
    /// 注意: 转码会增加性能消耗
    /// </summary>
    /// <param name="is_transcode"></param>
    /// is_transcode：设置为1的话，如果音频编码不是aac，则转成aac，如果是aac，则不做转换. 设置为0的话，则不做任何转换. 默认是0.
    public int NT_U3D_SetRecorderAudioTranscodeAAC(long handle, int is_transcode)
    {
        return player_obj_.Call<int>("SetRecorderAudioTranscodeAAC", handle, is_transcode);
    }

    /// <summary>
    /// 设置播放路径
    /// </summary>
    public int NT_U3D_SetUrl(long handle, string url)
    {
        return player_obj_.Call<int>("SetUrl", handle, url);
    }

    /// <summary>
    /// 开始播放
    /// </summary>
    public int NT_U3D_StartPlay(long handle)
    {
        return player_obj_.Call<int>("StartPlay", handle);
    }

    /// <summary>
    /// 获取YUV数据
    /// </summary>
    public AndroidJavaObject NT_U3D_GetVideoFrame(long handle)
    {
        return player_obj_.Call<AndroidJavaObject>("GetVideoFrame", handle);
    }

    /// <summary>
    /// 停止播放
    /// </summary>
    public int NT_U3D_StopPlay(long handle)
    {
        return player_obj_.Call<int>("StopPlay", handle);
    }

    /// <summary>
    /// 开始录像
    /// </summary>
    public int NT_U3D_StartRecorder(long handle)
    {
        return player_obj_.Call<int>("StartRecorder", handle);
    }

    /// <summary>
    /// 停止录像
    /// </summary>
    public int NT_U3D_StopRecorder(long handle)
    {
        return player_obj_.Call<int>("StopRecorder", handle);
    }

    /// <summary>
    /// 关闭播放
    /// </summary>
    public int NT_U3D_Close(long handle)
    {
        return player_obj_.Call<int>("Close", handle);
    }

    /// <summary>
    /// UnInit Player
    /// </summary>
    public int NT_U3D_UnInit()
    {
        return DANIULIVE_RETURN_OK;
    }

总结

Unity下实现RTMP或RTSP无论是播放还是录像，甚至快照，说难不难，但是做好真的比较难，特别是移动端，Unity和原生层交互的时候，数据交互效率相对较低，需要尽可能减少拷贝。录像的话，还需要考虑硬件性能瓶颈。此外，还需要逻辑分离，确保播放和录像相互不影响，以上是抛砖引玉，感兴趣的开发者，可以自行参考实现，如果需要单独和我交流的，可以相互交流。