WebrtcNode, publish-sdp offer 流程(1)

1. AmqpClient - New message received

sdp offer 的消息

AmqpClient - RpcServer New message received {
  method: 'onTransportSignaling',
  args: [
    'aa230ce0863e42baa8bae5c14e91e809',
    {
      sdp: 'v=0\r\n' +
        'o=- 2367615733001925388 2 IN IP4 127.0.0.1\r\n' +
        's=-\r\n' +
        't=0 0\r\n' +
        'a=group:BUNDLE 0 1\r\n' +
        'a=msid-semantic: WMS MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa\r\n' +
        'm=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 102 0 8 106 105 13 110 112 113 126\r\n' +
        'c=IN IP4 0.0.0.0\r\n' +
        'a=rtcp:9 IN IP4 0.0.0.0\r\n' +
        'a=ice-ufrag:YH25\r\n' +
        'a=ice-pwd:l47tWnjyoR1kn3nR0JWjuotU\r\n' +
        'a=ice-options:trickle\r\n' +
        'a=fingerprint:sha-256 65:8A:AE:75:0E:6B:50:51:A4:93:A4:01:76:43:3D:83:26:31:E7:75:B8:8A:DC:D4:13:78:14:4E:4D:0B:93:3E\r\n' +
        'a=setup:actpass\r\n' +
        'a=mid:0\r\n' +
        'a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n' +
        'a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n' +
        'a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n' +
        'a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n' +
        'a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n' +
        'a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n' +
        'a=sendonly\r\n' +
        'a=msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa AudioTrack-abd86f8a-1f74-4c73-b6fa-c20f5b175a1d\r\n' +
        'a=rtcp-mux\r\n' +
        'a=rtpmap:111 opus/48000/2\r\n' +
        'a=rtcp-fb:111 transport-cc\r\n' +
        'a=fmtp:111 minptime=10;useinbandfec=1\r\n' +
        'a=rtpmap:103 ISAC/16000\r\n' +
        'a=rtpmap:104 ISAC/32000\r\n' +
        'a=rtpmap:9 G722/8000\r\n' +
        'a=rtpmap:102 ILBC/8000\r\n' +
        'a=rtpmap:0 PCMU/8000\r\n' +
        'a=rtpmap:8 PCMA/8000\r\n' +
        'a=rtpmap:106 CN/32000\r\n' +
        'a=rtpmap:105 CN/16000\r\n' +
        'a=rtpmap:13 CN/8000\r\n' +
        'a=rtpmap:110 telephone-event/48000\r\n' +
        'a=rtpmap:112 telephone-event/32000\r\n' +
        'a=rtpmap:113 telephone-event/16000\r\n' +
        'a=rtpmap:126 telephone-event/8000\r\n' +
        'a=ssrc:3582341671 cname:JTBK6RkjYuislL5i\r\n' +
        'a=ssrc:3582341671 msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa AudioTrack-abd86f8a-1f74-4c73-b6fa-c20f5b175a1d\r\n' +
        'a=ssrc:3582341671 mslabel:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa\r\n' +
        'a=ssrc:3582341671 label:AudioTrack-abd86f8a-1f74-4c73-b6fa-c20f5b175a1d\r\n' +
        'm=video 9 UDP/TLS/RTP/SAVPF 123 114 115 116 119\r\n' +
        'c=IN IP4 0.0.0.0\r\n' +
        'a=rtcp:9 IN IP4 0.0.0.0\r\n' +
        'a=ice-ufrag:YH25\r\n' +
        'a=ice-pwd:l47tWnjyoR1kn3nR0JWjuotU\r\n' +
        'a=ice-options:trickle\r\n' +
        'a=fingerprint:sha-256 65:8A:AE:75:0E:6B:50:51:A4:93:A4:01:76:43:3D:83:26:31:E7:75:B8:8A:DC:D4:13:78:14:4E:4D:0B:93:3E\r\n' +
        'a=setup:actpass\r\n' +
        'a=mid:1\r\n' +
        'a=extmap:14 urn:ietf:params:rtp-hdrext:toffset\r\n' +
        'a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n' +
        'a=extmap:13 urn:3gpp:video-orientation\r\n' +
        'a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n' +
        'a=extmap:12 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\n' +
        'a=extmap:11 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\r\n' +
        'a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\r\n' +
        'a=extmap:8 http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07\r\n' +
        'a=extmap:9 http://www.webrtc.org/experiments/rtp-hdrext/color-space\r\n' +
        'a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n' +
        'a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n' +
        'a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n' +
        'a=sendonly\r\n' +
        'a=msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n' +
        'a=rtcp-mux\r\n' +
        'a=rtcp-rsize\r\n' +
        'a=rtpmap:123 AV1/90000\r\n' +
        'a=rtcp-fb:123 goog-remb\r\n' +
        'a=rtcp-fb:123 transport-cc\r\n' +
        'a=rtcp-fb:123 ccm fir\r\n' +
        'a=rtcp-fb:123 nack\r\n' +
        'a=rtcp-fb:123 nack pli\r\n' +
        'a=rtpmap:114 red/90000\r\n' +
        'a=rtpmap:115 rtx/90000\r\n' +
        'a=fmtp:115 apt=114\r\n' +
        'a=rtpmap:116 ulpfec/90000\r\n' +
        'a=rtpmap:119 rtx/90000\r\n' +
        'a=fmtp:119 apt=123\r\n' +
        'a=ssrc-group:FID 4143402920 1935417042\r\n' +
        'a=ssrc:4143402920 cname:JTBK6RkjYuislL5i\r\n' +
        'a=ssrc:4143402920 msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n' +
        'a=ssrc:4143402920 mslabel:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa\r\n' +
        'a=ssrc:4143402920 label:VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n' +
        'a=ssrc:1935417042 cname:JTBK6RkjYuislL5i\r\n' +
        'a=ssrc:1935417042 msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n' +
        'a=ssrc:1935417042 mslabel:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa\r\n' +
        'a=ssrc:1935417042 label:VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n',
      type: 'offer'
    }
  ],
  corrID: 14,
  replyTo: 'amq.gen-pj6CYWZ4AOsOkYmlAePDEA'
}

2. WebrtcNode - onTransportSignaling

dist/webrtc_agent/webrtc/index.js

// connectionId 应该就是operationId
that.onTransportSignaling = function (connectionId, msg, callback) {
        log.debug('onTransportSignaling, connection id:', connectionId, 'msg:', msg);
        // 通过 operationId 从mappingTransports 获取 transportId，
        // 接着通过transportId从peerConnections 获取WrtcConnection
        // 参考小结 2.1
        var conn = getWebRTCConnection(connectionId);
        // WrtcConnection conn
        if (conn) {
            // 参考小结 2.2
            conn.onSignalling(msg, connectionId);
            callback('callback', 'ok');
        } else {
          callback('callback', 'error', 'Connection does NOT exist:' + connectionId);
        }
    };

2.1 WebrtcNode - getWebRTCConnection——返回WrtcConnection

dist/webrtc_agent/webrtc/index.js

    // 返回的是WrtcConnection
    var getWebRTCConnection = function (operationId) {
        // 在WebrtcNode-createWebRTCConnection 创建
        var transportId = mappingTransports.get(operationId);
        if (peerConnections.has(transportId)) {
            return peerConnections.get(transportId);
        }
        return null;
    };

2.2 WrtcConnection.onSignalling

dist/webrtc_agent/webrtc/wrtcConnection.js

处理 offer sdp。

  that.onSignalling = function (msg, operationId) {
    var processSignalling = function () {
  // 这里type=’offer‘
      if (msg.type === 'offer') {
        log.debug('on offer:', msg.sdp);
        processOffer(msg.sdp);
      } else if (msg.type === 'candidate') {
      ...
      } else if (msg.type === 'removed-candidates') {
      ...
      }
    };
    // wrtc 是 Connection
    if (wrtc) {
      processSignalling();
    } else {
      // should not reach here
      log.error('wrtc is not ready');
    }
  };

2.3 ==========WrtcConnection.processOffer

dist/webrtc_agent/webrtc/wrtcConnection.js

这里的 remoteSdp 是null，

 const processOffer = function (sdp) {
 // 第一进来，remoteSdp 是null
    if (!remoteSdp) {
      // First offer
      remoteSdp = new SdpInfo(sdp);
      // Check mid， 获取到mid 数组
      const mids = remoteSdp.mids();
      for (const mid of mids) {
         // 设置finalformat
        processOfferMedia(mid);
      }
      // 创建answer sdp
      localSdp = remoteSdp.answer();

      // Setup transport
      let opId = null;
      for (const mid of mids) {
        if (remoteSdp.getMediaPort(mid) !== 0) {
          opId = setupTransport(mid);
        }
      }
      if (opId) {
        on_track({
          type: 'tracks-complete',
          operationId: opId
        });
      }

    } else {
     ...
    }
  };

  var remoteSdp = null;
  var localSdp = null;

2.3.1 SdpInfo.SdpInfo

dist/webrtc_agent/webrtc/sdpInfo.js

2.3.2 WrtcConnection.processOfferMedia——设置finalformat

WrtcConnection.addTrackOperation, 创建了元素operationMap

设置finalFormat

 const processOfferMedia = function (mid) {
    ...

    // Determine media format in offer
    if (remoteSdp.mediaType(mid) === 'audio') {
      const audioPreference = operationMap.get(mid).formatPreference;
      const audioFormat = remoteSdp.filterAudioPayload(mid, audioPreference);
      operationMap.get(mid).finalFormat = audioFormat;
    } else if (remoteSdp.mediaType(mid) === 'video') {
      const videoPreference = operationMap.get(mid).formatPreference;
      const videoFormat = remoteSdp.filterVideoPayload(mid, videoPreference);
      operationMap.get(mid).finalFormat = videoFormat;
    }
  };

operationMap 的元素就是WrtcConnection.addTrackOperation中添加的，这里是给finalFormat赋值

// mid => { operationId, sdpDirection, type, formatPreference, rids, enabled, finalFormat }
  var operationMap = new Map();

dist/webrtc_agent/webrtc/wrtcConnection.js，在WebrtcNode-publish流程.md 的addTrackOperation 小节有详细说明。

2.3.3 SdpInfo.answer——localSdp

通过offer到sdp，进行对应的修改，作为answer的sdp。赋值给localSdp中，在WrtcConnection.setupTransport用到。

 answer() {
    const answer = new SdpInfo(this.toString());
    answer.obj.origin = {
      username: '-',
      sessionId: '0',
      sessionVersion: 0,
      netType: 'IN',
      ipVer: 4,
      address: '127.0.0.1'
    };
    answer.obj.media.forEach(mediaInfo => {
      mediaInfo.port = 1;
      mediaInfo.rtcp = {
        port: 1,
        netType: 'IN',
        ipVer: 4,
        address: '0.0.0.0'
      };
      if (mediaInfo.setup === 'active') {
        mediaInfo.setup = 'passive';
      } else {
        mediaInfo.setup = 'active';
      }

      delete mediaInfo.iceOptions;  
      delete mediaInfo.rtcpRsize;
      if (mediaInfo.direction === 'recvonly') {
        mediaInfo.direction = 'sendonly';
      } else if (mediaInfo.direction === 'sendonly') {
        delete mediaInfo.msid;
        delete mediaInfo.ssrcGroups;
        delete mediaInfo.ssrcs;
        mediaInfo.direction = 'recvonly';
      }

      if (mediaInfo.ext && Array.isArray(mediaInfo.ext)) {
        const extMappings = [
          'urn:ietf:params:rtp-hdrext:ssrc-audio-level',
          // 'http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01',
          'urn:ietf:params:rtp-hdrext:sdes:mid',
          'urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id',
          'urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id',
          'urn:ietf:params:rtp-hdrext:toffset',
          'http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time',
          // 'urn:3gpp:video-orientation',
          // 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay',
        ];
        if (mediaInfo.type === 'video') {
          extMappings.push(
            'http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01'
          );
        }
        mediaInfo.ext = mediaInfo.ext.filter((e) => {
          return extMappings.includes(e.uri);
        });
      }

      if (mediaInfo.rids && Array.isArray(mediaInfo.rids)) {
        // Reverse rids direction
        mediaInfo.rids.forEach(r => {
          r.direction = (r.direction === 'send') ? 'recv' : 'send';
        });
      }
      if (mediaInfo.simulcast) {
        // Reverse simulcast direction
        if (mediaInfo.simulcast.dir1 === 'send') {
          mediaInfo.simulcast.dir1 = 'recv';
        } else {
          mediaInfo.simulcast.dir1 = 'send';
        }
      }
    });

    return answer;
  }

SdpInfo.getMediaPort

SdpInfo.mediaType

operationMap——存放的是track相关的内容

dist/webrtc_agent/webrtc/wrtcConnection.js

在WebrtcNode-publish流程.md 的addTrackOperation 小节有详细说明

  // mid => { operationId, sdpDirection, type, formatPreference, rids, enabled, finalFormat }
  var operationMap = new Map();

2.3.4 WrtcConnection.setupTransport——创建WrtcStream

dist/webrtc_agent/webrtc/wrtcConnection.js

小节 3

2.3.5 on_track——这是创建WrtcConnection 传入的callback,tracks-complete

在WebrtcNode-publish流程.md 的 3.WebrtcNode - createWebRTCConnection——返回WrtcConnection 中传入了

function onTrack(trackInfo) {
            handleTrackInfo(transportId, trackInfo, controller);
        }

调用的代码

        on_track({
          type: 'tracks-complete',
          operationId: opId
        });

详细的看3.3， 3.4 小节

3. WrtcConnection.setupTransport——创建WrtcStream

主要是创建MediaStream。

dist/webrtc_agent/webrtc/wrtcConnection.js

const setupTransport = function (mid) {
    let rids = remoteSdp.rids(mid);
    const opSettings = operationMap.get(mid);
    const direction = (opSettings.sdpDirection === 'sendonly') ? 'in' : 'out';
    const simSsrcs = remoteSdp.getLegacySimulcast(mid);
    const trackSettings = remoteSdp.getMediaSettings(mid);
    const mediaType = remoteSdp.mediaType(mid);

    trackSettings.owner = owner;
    trackSettings.enableBWE = that.enableBWE;
    if (opSettings.finalFormat) {
      trackSettings[mediaType].format = opSettings.finalFormat;
      if (opSettings.finalFormat.codec === 'vp9' && simSsrcs) {
        // Legacy simulcast for VP9 SVC
        rids = null;
        trackSettings['video'].scalabilityMode = opSettings.scalabilityMode;
      }
    }

    if (rids) {
        ...
    } else {
        // 走这里的代码
      // No simulcast
      if (!trackMap.has(mid)) {
        // 1. Connection wrtc
        trackMap.set(mid, new WrtcStream(mid, wrtc, direction, trackSettings));
        // Set ssrc in local sdp for out direction
        // direct 是in
        if (direction === 'out') {
          ...
        }
        // 2. Connection wrtc
        // remoteSdp 是在processOffer 创建的对象
        wrtc.setRemoteSdp(remoteSdp.singleMediaSdp(mid).toString(), mid);
        // 3. Notify new track， 详细见小节 3.3， 3.4
        on_track({
          type: 'track-added',
          track: trackMap.get(mid),
          operationId: opSettings.operationId,
          mid: mid
        });
      } else {
        log.warn(`Conflict trackId ${mid} for ${wrtcId}`);
      }
    }

    return opSettings.operationId;
  };

SdpInfo.rids

trackMap 属性——存放了WrtcStream

  // composedId => WrtcStream
  var trackMap = new Map();

composeId

  const composeId = function (mid, rid) {
    return mid + ':' + rid;
  };

3.1 new WrtcStream

小节 4
创建了MediaStream

3.2 Connection.setRemoteSdp

// streamId = mid
setRemoteSdp(sdp, streamId) {
    // webRtcConnection wrtc
    this.wrtc.setRemoteSdp(sdp, streamId || this.id);
  }

3.2.1 NAN_METHOD(WebRtcConnection::setRemoteSdp)

NAN_METHOD(WebRtcConnection::setRemoteSdp) {
  WebRtcConnection* obj = Nan::ObjectWrap::Unwrap<WebRtcConnection>(info.Holder());
  std::shared_ptr<erizo::WebRtcConnection> me = obj->me;
  if (!me) {
    return;
  }

  std::string sdp = getString(info[0]);
  std::string stream_id = getString(info[1]);

  bool r = me->setRemoteSdp(sdp, stream_id);

  info.GetReturnValue().Set(Nan::New(r));
}

3.2.2 WebRtcConnection::setRemoteSdp

bool WebRtcConnection::setRemoteSdpInfo(std::shared_ptr<SdpInfo> sdp, std::string stream_id) {
  asyncTask([sdp, stream_id] (std::shared_ptr<WebRtcConnection> connection) {
    ELOG_DEBUG("%s message: setting remote SDPInfo", connection->toLog());

    if (!connection->sending_) {
      return;
    }

    connection->remote_sdp_ = sdp;
    // mid 
    connection->processRemoteSdp(stream_id);
  });
  return true;
}

??? WebRtcConnection::processRemoteSdp

source/agent/webrtc/rtcConn/erizo/src/erizo/WebRtcConnection.cpp

SdpInfo.singleMediaSdp

dist-debug/webrtc_agent/webrtc/sdpInfo.js

  singleMediaSdp(mid) {
    const sdp = new SdpInfo(this.toString());
    sdp.obj.media = sdp.obj.media.filter(m => m.mid.toString() === mid);
    sdp.setBundleMids([mid]);
    return sdp;
  }

3.3 on__track——track add

dist/webrtc_agent/webrtc/wrtcConnection.js

通知到WebrtcNode（/dist/webrtc_agent/webrtc/index.js）的createWebRTCConnection 中注册的callback, 回调一个对象，

        // Notify new track
        on_track({
          type: 'track-added',
          track: trackMap.get(mid), // WrtcStream
          operationId: opSettings.operationId,
          mid: mid
        })

3.4 ======WebrtcNode.handleTrackInfo

dist/webrtc_agent/webrtc/index.js

var connection = new WrtcConnection({
            connectionId: transportId,
            threadPool: threadPool,
            ioThreadPool: ioThreadPool,
            network_interfaces: global.config.webrtc.network_interfaces,
            owner,
        }, function onTransportStatus(status) {
            notifyTransportStatus(controller, transportId, status);
        }, function onTrack(trackInfo) {
            handleTrackInfo(transportId, trackInfo, controller);
        });

// trackInfo 就是on__track 回调回来
var handleTrackInfo = function (transportId, trackInfo, controller) {
        var publicTrackId;
        var updateInfo;
        if (trackInfo.type === 'track-added') {
            // Generate public track ID
            const track = trackInfo.track; // WrtcStream
            publicTrackId = transportId + '-' + track.id;
            if (mediaTracks.has(publicTrackId)) {
                log.error('Conflict public track id:', publicTrackId, transportId, track.id);
                return;
            }
            mediaTracks.set(publicTrackId, track);
            mappingPublicId.get(transportId).set(track.id, publicTrackId);
            if (track.direction === 'in') {
                const trackSource = track.sender();
                router.addLocalSource(publicTrackId, 'webrtc', trackSource)
                .catch(e => log.warn('Unexpected error during track add:', e));
            } else {
                router.addLocalDestination(publicTrackId, 'webrtc', track)
                .catch(e => log.warn('Unexpected error during track add:', e));
            }

            // Bind media-update handler
            track.on('media-update', (jsonUpdate) => {
                log.debug('notifyMediaUpdate:', publicTrackId, jsonUpdate);
                notifyMediaUpdate(controller, publicTrackId, track.direction, JSON.parse(jsonUpdate));
            });
            // Notify controller
            const mediaType = track.format('audio') ? 'audio' : 'video';
            updateInfo = {
                type: 'track-added',
                trackId: publicTrackId,
                mediaType: track.format('audio') ? 'audio' : 'video',
                mediaFormat: track.format(mediaType),
                direction: track.direction,
                operationId: trackInfo.operationId,
                mid: trackInfo.mid,
                rid: trackInfo.rid,
                active: true,
            };
            log.debug('notifyTrackUpdate', controller, publicTrackId, updateInfo);
            notifyTrackUpdate(controller, transportId, updateInfo);

        } else if (trackInfo.type === 'track-removed') {
            publicTrackId = mappingPublicId.get(transportId).get(trackInfo.trackId);
            if (!mediaTracks.has(publicTrackId)) {
                log.error('Non-exist public track id:', publicTrackId, transportId, trackInfo.trackId);
                return;
            }
            log.debug('track removed:', publicTrackId);
            router.removeConnection(publicTrackId)
            .then(ok => {
                mediaTracks.get(publicTrackId).close();
                mediaTracks.delete(publicTrackId);
                mappingPublicId.get(transportId).delete(trackInfo.trackId);
            })
            .catch(e => log.warn('Unexpected error during track remove:', e));

            // Notify controller
            updateInfo = {
                type: 'track-removed',
                trackId: publicTrackId,
            };
            notifyTrackUpdate(controller, transportId, updateInfo);

        } else if (trackInfo.type === 'tracks-complete') {
            updateInfo = {
                type: 'tracks-complete',
                operationId: trackInfo.operationId
            };
            notifyTrackUpdate(controller, transportId, updateInfo);
        }
    };