wfs.js之h264转码mp4分析

准备源文件

下载源文件

git clone https://github.com/ChihChengYang/wfs.js.git

编译后得到wfs.js这个文件

调用

在demo/index.html中，前端对wfs.js进行了调用

var video1 = document.getElementById("video1"),
wfs = new Wfs();    
wfs.attachMedia(video1,'ch1');

对应wfs.js中的function Wfs() 函数

key: 'attachMedia',
    value: function attachMedia(media) {
      //arg1 通道
      var channelName = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 'chX';
      //arg2 媒体格式
      var mediaType = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 'H264Raw';
      //arg3 未定义
      var websocketName = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : 'play2';
      // 'H264Raw' 'FMp4'    
      this.mediaType = mediaType;
      this.media = media;
      //MEDIA_ATTACHING 事件处理
      this.trigger(_events2.default.MEDIA_ATTACHING, { media: media, channelName: channelName, mediaType: mediaType, websocketName: websocketName });

MEDIA_ATTACHING 事件的处理

MEDIA_ATTACHING: 'wfsMediaAttaching',

在var BufferController = function (_EventHandler)中

    key: 'onMediaAttaching',
    value: function onMediaAttaching(data) {
      var media = this.media = data.media;
      this.mediaType = data.mediaType;
      this.websocketName = data.websocketName;
      this.channelName = data.channelName;
      if (media) {
        // setup the media source
        var ms = this.mediaSource = new MediaSource();
        //Media Source listeners
        //绑定ms的监听事件
        //打开时候onMediaSourceOpen
        this.onmso = this.onMediaSourceOpen.bind(this);
        this.onmse = this.onMediaSourceEnded.bind(this);
        this.onmsc = this.onMediaSourceClose.bind(this);
        ms.addEventListener('sourceopen', this.onmso);
        ms.addEventListener('sourceended', this.onmse);
        ms.addEventListener('sourceclose', this.onmsc);
        // link video and media Source
        //连接video和刚刚创建的ms media Source
        media.src = URL.createObjectURL(ms);
      }
    }

ms的onMediaSourceOpen

在ms被打开的时候。调用 onMediaSourceOpen

key: 'onMediaSourceOpen',
    value: function onMediaSourceOpen() {
      var mediaSource = this.mediaSource;
      if (mediaSource) {
        // once received, don't listen anymore to sourceopen event
        mediaSource.removeEventListener('sourceopen', this.onmso);
      }

      if (this.mediaType === 'FMp4') {
        this.checkPendingTracks();
      }

      //调用MEDIA_ATTACHED事件处理
      this.wfs.trigger(_events2.default.MEDIA_ATTACHED, { media: this.media, channelName: this.channelName, mediaType: this.mediaType, websocketName: this.websocketName });

MEDIA_ATTACHED事件处理

MEDIA_ATTACHED: 'wfsMediaAttached',

    key: 'onMediaAttached',
    value: function onMediaAttached(data) {
      if (data.websocketName != undefined) {
        //创建WebSocket连接
        var client = new WebSocket('ws://' + window.location.host + '/' + data.websocketName);
        //调用attachWebsocket方法
        this.wfs.attachWebsocket(client, data.channelName);
      } else {
        console.log('websocketName ERROE!!!');
      }
    }

    key: 'attachWebsocket',
    value: function attachWebsocket(websocket, channelName) {
      //调用WEBSOCKET_ATTACHING事件处理
      this.trigger(_events2.default.WEBSOCKET_ATTACHING, { websocket: websocket, mediaType: this.mediaType, channelName: channelName });
    }

WEBSOCKET_ATTACHING事件处理

WEBSOCKET_ATTACHING: 'wfsWebsocketAttaching',

    key: 'onWebsocketAttaching',
    value: function onWebsocketAttaching(data) {
      this.mediaType = data.mediaType;
      this.channelName = data.channelName;
      if (data.websocket instanceof WebSocket) {
        this.client = data.websocket;
        //websocket 的 onopen事件绑定绑定initSocketClient
        this.client.onopen = this.initSocketClient.bind(this);
        this.client.onclose = function (e) {
          console.log('Websocket Disconnected!');
        };
      }
    }

    key: 'initSocketClient',
    value: function initSocketClient(client) {
      this.client.binaryType = 'arraybuffer';
      //websocket 的 onmessage 事件绑定绑定receiveSocketMessage
      this.client.onmessage = this.receiveSocketMessage.bind(this);
      //WEBSOCKET_MESSAGE_SENDING事件处理
      this.wfs.trigger(_events2.default.WEBSOCKET_MESSAGE_SENDING, { commandType: "open", channelName: this.channelName, commandValue: "NA" });
      console.log('Websocket Open!');
    }

//WEBSOCKET_MESSAGE_SENDING事件处理

WEBSOCKET_MESSAGE_SENDING: 'wfsWebsocketMessageSending'

    key: 'onWebsocketMessageSending',
    value: function onWebsocketMessageSending(event) {
      this.client.send(JSON.stringify({ t: event.commandType, c: event.channelName, v: event.commandValue }));
    }

这样把信息通过json发送出去。

接收信息的处理

上面websocket 的 onmessage 事件绑定绑定receiveSocketMessage， 当websocket接收到信息时，调用receiveSocketMessage

    key: 'receiveSocketMessage',
    value: function receiveSocketMessage(event) {
      if (document['hidden']) return;
      this.buf = new Uint8Array(event.data);
      var copy = new Uint8Array(this.buf);

      if (this.mediaType === 'FMp4') {
        this.wfs.trigger(_events2.default.WEBSOCKET_ATTACHED, { payload: copy });
      }
      if (this.mediaType === 'H264Raw') {
        //H264_DATA_PARSING事件处理
        this.wfs.trigger(_events2.default.H264_DATA_PARSING, { data: copy });
      }
    }

H264_DATA_PARSING事件处理

H264_DATA_PARSING: 'wfsH264DataParsing',

        key: 'onH264DataParsing',
        value: function onH264DataParsing(event) {
            //读取数据
            this._read(event.data);
            var $this = this;
            //对于每个nal
            this.nals.forEach(function (nal) {
                //H264_DATA_PARSED事件处理
                $this.wfs.trigger(_events2.default.H264_DATA_PARSED, {
                    data: nal
                });
            });
        }

H264_DATA_PARSED事件处理

H264_DATA_PARSED: 'wfsH264DataParsed',

    key: 'onH264DataParsed',
    value: function onH264DataParsed(event) {
      //分析AVCTrack
      this._parseAVCTrack(event.data);
      if (this.browserType === 1 || this._avcTrack.samples.length >= 20) {
        // Firefox
        this.remuxer.pushVideo(0, this.sn, this._avcTrack, this.timeOffset, this.contiguous);
        this.sn += 1;
      }
    }

数据经过_parseAVCTrack处理后，调用remuxer.pushVideo方法

    key: 'pushVideo',
    value: function pushVideo(level, sn, videoTrack, timeOffset, contiguous) {
      this.level = level;
      this.sn = sn;
      var videoData = void 0;
      // generate Init Segment if needed
      if (!this.ISGenerated) {
        this.generateVideoIS(videoTrack, timeOffset);
      }
      if (this.ISGenerated) {
        if (videoTrack.samples.length) {
          this.remuxVideo_2(videoTrack, timeOffset, contiguous);
        }
      }
    }

最后调用remuxVideo_2方法

    key: 'remuxVideo_2',
    value: function remuxVideo_2(track, timeOffset, contiguous, audioTrackLength) {
      var offset = 8,
          pesTimeScale = this.PES_TIMESCALE,
          pes2mp4ScaleFactor = this.PES2MP4SCALEFACTOR,
          mp4SampleDuration,
          mdat,
          moof,
          firstPTS,
          firstDTS,
          nextDTS,
          inputSamples = track.samples,
          outputSamples = [];

      /* concatenate the video data and construct the mdat in place
        (need 8 more bytes to fill length and mpdat type) */
      mdat = new Uint8Array(track.len + 4 * track.nbNalu + 8);
      var view = new DataView(mdat.buffer);
      view.setUint32(0, mdat.byteLength);
      mdat.set(_mp4Generator2.default.types.mdat, 4);
      var sampleDuration = 0;
      var ptsnorm = void 0,
          dtsnorm = void 0,
          mp4Sample = void 0,
          lastDTS = void 0;

      for (var i = 0; i < inputSamples.length; i++) {
        var avcSample = inputSamples[i],
            mp4SampleLength = 0,
            compositionTimeOffset = void 0;
        // convert NALU bitstream to MP4 format (prepend NALU with size field)
        while (avcSample.units.units.length) {
          var unit = avcSample.units.units.shift();
          view.setUint32(offset, unit.data.byteLength);
          offset += 4;
          mdat.set(unit.data, offset);
          offset += unit.data.byteLength;
          mp4SampleLength += 4 + unit.data.byteLength;
        }

        var pts = avcSample.pts - this._initPTS;
        var dts = avcSample.dts - this._initDTS;
        dts = Math.min(pts, dts);

        if (lastDTS !== undefined) {
          ptsnorm = this._PTSNormalize(pts, lastDTS);
          dtsnorm = this._PTSNormalize(dts, lastDTS);
          sampleDuration = dtsnorm - lastDTS;
          if (sampleDuration <= 0) {
            _logger.logger.log('invalid sample duration at PTS/DTS: ' + avcSample.pts + '/' + avcSample.dts + '|dts norm: ' + dtsnorm + '|lastDTS: ' + lastDTS + ':' + sampleDuration);
            sampleDuration = 1;
          }
        } else {
          var nextAvcDts = this.nextAvcDts,
              delta;
          ptsnorm = this._PTSNormalize(pts, nextAvcDts);
          dtsnorm = this._PTSNormalize(dts, nextAvcDts);
          if (nextAvcDts) {
            delta = Math.round(dtsnorm - nextAvcDts);
            if ( /*contiguous ||*/Math.abs(delta) < 600) {
              if (delta) {
                if (delta > 1) {
                  _logger.logger.log('AVC:' + delta + ' ms hole between fragments detected,filling it');
                } else if (delta < -1) {
                  _logger.logger.log('AVC:' + -delta + ' ms overlapping between fragments detected');
                }
                dtsnorm = nextAvcDts;
                ptsnorm = Math.max(ptsnorm - delta, dtsnorm);
                _logger.logger.log('Video/PTS/DTS adjusted: ' + ptsnorm + '/' + dtsnorm + ',delta:' + delta);
              }
            }
          }
          this.firstPTS = Math.max(0, ptsnorm);
          this.firstDTS = Math.max(0, dtsnorm);
          sampleDuration = 0.03;
        }

        outputSamples.push({
          size: mp4SampleLength,
          duration: this.H264_TIMEBASE,
          cts: 0,
          flags: {
            isLeading: 0,
            isDependedOn: 0,
            hasRedundancy: 0,
            degradPrio: 0,
            dependsOn: avcSample.key ? 2 : 1,
            isNonSync: avcSample.key ? 0 : 1
          }
        });
        lastDTS = dtsnorm;
      }

      var lastSampleDuration = 0;
      if (outputSamples.length >= 2) {
        lastSampleDuration = outputSamples[outputSamples.length - 2].duration;
        outputSamples[0].duration = lastSampleDuration;
      }
      this.nextAvcDts = dtsnorm + lastSampleDuration;
      var dropped = track.dropped;
      track.len = 0;
      track.nbNalu = 0;
      track.dropped = 0;
      if (outputSamples.length && navigator.userAgent.toLowerCase().indexOf('chrome') > -1) {
        var flags = outputSamples[0].flags;
        flags.dependsOn = 2;
        flags.isNonSync = 0;
      }
      track.samples = outputSamples;
      moof = _mp4Generator2.default.moof(track.sequenceNumber++, dtsnorm, track);
      track.samples = [];

      var data = {
        id: this.id,
        level: this.level,
        sn: this.sn,
        data1: moof,
        data2: mdat,
        startPTS: ptsnorm,
        endPTS: ptsnorm,
        startDTS: dtsnorm,
        endDTS: dtsnorm,
        type: 'video',
        nb: outputSamples.length,
        dropped: dropped
      };

      this.observer.trigger(_events2.default.FRAG_PARSING_DATA, data);
      return data;
    }

在处理完数据后，进行FRAG_PARSING_DATA事件处理

  FRAG_PARSING_DATA: 'wfsFragParsingData',

    key: 'onFragParsingData',
    value: function onFragParsingData(data) {
      var _this2 = this;

      if (data.type === 'video') {}

      [data.data1, data.data2].forEach(function (buffer) {
        if (buffer) {
          _this2.pendingAppending++;
          _this2.wfs.trigger(_events2.default.BUFFER_APPENDING, { type: data.type, data: buffer, parent: 'main' });
        }
      });
    }

BUFFER_APPENDING事件处理

BUFFER_APPENDING: 'wfsBufferAppending',

    key: 'onBufferAppending',
    value: function onBufferAppending(data) {
      if (!this.segments) {
        this.segments = [data];
      } else {
        this.segments.push(data);
      }
      this.doAppending();
    }

    key: 'doAppending',
    value: function doAppending() {

      var wfs = this.wfs,
          sourceBuffer = this.sourceBuffer,
          segments = this.segments;
      if (Object.keys(sourceBuffer).length) {

        if (this.media.error) {
          this.segments = [];
          console.log('trying to append although a media error occured, flush segment and abort');
          return;
        }
        if (this.appending) {
          return;
        }

        if (segments && segments.length) {
          var segment = segments.shift();
          try {
            if (sourceBuffer[segment.type]) {
              this.parent = segment.parent;
              sourceBuffer[segment.type].appendBuffer(segment.data);
              this.appendError = 0;
              this.appended++;
              this.appending = true;
            } else {}
          } catch (err) {
            // in case any error occured while appending, put back segment in segments table 
            segments.unshift(segment);
            var event = { type: ErrorTypes.MEDIA_ERROR };
            if (err.code !== 22) {
              if (this.appendError) {
                this.appendError++;
              } else {
                this.appendError = 1;
              }
              event.details = ErrorDetails.BUFFER_APPEND_ERROR;
              event.frag = this.fragCurrent;
              if (this.appendError > wfs.config.appendErrorMaxRetry) {
                segments = [];
                event.fatal = true;
                return;
              } else {
                event.fatal = false;
              }
            } else {
              this.segments = [];
              event.details = ErrorDetails.BUFFER_FULL_ERROR;
              return;
            }
          }
        }
      }
    }

通过sourceBuffer[segment.type].appendBuffer(segment.data);把数据发送到video去。