androidandroid-mediacodechevc

Hevc video stream Media Codec Half of video is broken when receive 2 I frame consecutively


I'm trying to decode packages from Hevc Stream raw, then I receive 2 I Frames in a row because NaluType returns value 19 twice. Then half of the video is broken. I'm using Media Codec. How can I split the packages the hevc live video to render video without broken image? I've concatenated 2 I Frame but half of the video still broken.

private static final byte[] NALUStartCode = {0x00,0x00,0x00,0x01};

private boolean gotVPS = false;
private boolean gotPPS = false;
private boolean gotSPS = false;

private byte[] vspSet;
int temp;
private byte[] idrSet;

public DataStruct(byte[] data, long mTimeNum, boolean isParams) {
        this.mVideoData = data;
        this.mTimeNum = mTimeNum;
        this.isParams = isParams;
    }

private void splitRtpPackage(DataStruct hevcStruct) {
        if (hevcStruct == null) {
            return;
        }
        byte[] totalBytes = hevcStruct.getVideoData();
        int payloadSize = totalBytes.length;
        if (payloadSize > 8) {
            byte[] hevcRaw = Arrays.copyOfRange(totalBytes, 12, totalBytes.length);
            long timestamp = bytesToInt(Arrays.copyOfRange(totalBytes, 2, 10));

            int check = totalBytes[1];
            if (check != 2) {
                stopSplitter();
                return;
            }

            // extract the bit 2nd -> 6th from byte 1st
            int nalType = (hevcRaw[0] >> 1) & 0x3f;
            if ((9 < nalType && nalType < 16) || (21 < nalType && nalType < 32) || nalType > 40) {
                return;
            }
            switch (nalType) {
                case 32: //vps
                    this.vspSet = null;
                    this.vspSet = concatenateByteArrays(NALUStartCode, hevcRaw);
                    this.gotVPS = true;
                    this.gotSPS = false;
                    this.gotPPS = false;
                    break;
                case 33:   //sps
                    if (this.gotVPS) {
                        this.vspSet = concatenateByteArrays(this.vspSet, concatenateByteArrays(NALUStartCode, hevcRaw));
                        this.gotSPS = true;
                    }
                    break;
                case 34:  //pps
                    if (this.gotSPS) {
                        this.vspSet = concatenateByteArrays(this.vspSet, concatenateByteArrays(NALUStartCode, hevcRaw));
                        this.gotPPS = true;
                        this.gotVPS = false;
                        this.gotSPS = false;
                        onHevcPackageReceived(new DataStruct(this.vspSet, timestamp, true));
                    }
                    break;

                case 19:    //I Frame
                case 20:
                    if (this.gotPPS) {
                        if (this.temp != 19 && this.temp != 20) {
                            this.idrSet = null;
                            this.idrSet = concatenateByteArrays(NALUStartCode, hevcRaw);
                        } else {
                            this.idrSet = concatenateByteArrays(this.idrSet, concatenateByteArrays(NALUStartCode, hevcRaw)); 
// concatened 2 I Frame
                        }
                    }
                    break;

                case 0:   // P Frame
                case 1:
                    if (this.gotPPS) {
                        if (this.temp == 19 || this.temp == 20) {
                            onHevcPackageReceived(new DataStruct(this.idrSet, timestamp, false));
                        } else {
                            onHevcPackageReceived(new DataStruct(concatenateByteArrays(NALUStartCode, hevcRaw), timestamp, false));
                        }
                    }
                    break;

            default:
                if (this.gotPPS) {
                    onHevcPackageReceived(new DataStruct(concatenateByteArrays(NALUStartCode, hevcRaw), timestamp, false));
                }
                break;
            }
            this.temp = nalType;
        }
    }

    private void onHevcPackageReceived(DataStruct dataStruct) {
        this.Player.getDataQueue().enqueue(dataStruct); // enqueue to list to decode
    }

    private static byte[] concatenateByteArrays(byte[] a, byte[] b) {
        byte[] result = new byte[(a.length + b.length)];
        System.arraycopy(a, 0, result, 0, a.length);
        System.arraycopy(b, 0, result, a.length, b.length);
        return result;
    }


    private int bytesToInt(byte[] range) {
        return ByteBuffer.wrap(range).getInt();
    }

Solution

  • I've already solved problem by concatening P-frame equal to concatened I-Frame then sent it to decoder:

          private int countIdr;
          private int countPframe;
          switch{....
                case 19:
                case 20:
                        if(this.gotPPS){
                            if(this.nalTemp != 19 && this.nalTemp !=20){
                                this.countIdr = 1;
                                this.idrSet = null;
                                this.idrSet = concatenateByteArrays(NALUStartCode, hevcRaw);
                            }else {
                                this.idrSet = concatenateByteArrays(this.idrSet, concatenateByteArrays(NALUStartCode, hevcRaw));
                                this.countIdr++;
                            }
                        }
                        break;
                case 1:
                case 0:
                    if (this.gotPPS) {
                        if (this.nalTemp == 19 || this.nalTemp == 20){
                            onHevcPackageReceived(new DataStruct(this.idrSet, timestamp, false));
                            this.countPframe = 1;
                            if(this.countIdr == 1){
                                onHevcPackageReceived(new DataStruct(concatenateByteArrays(NALUStartCode, hevcRaw), timestamp, false));
                            }else if(this.countIdr > 1){
                                this.pSet = null;
                                this.pSet = concatenateByteArrays(NALUStartCode, hevcRaw);
                            }
                        } else {
                            if(this.countIdr == 1){
                                onHevcPackageReceived(new DataStruct(concatenateByteArrays(NALUStartCode, hevcRaw), timestamp, false));
                            }else if(this.countIdr > 1){
                                if(this.countPframe > 0){
                                    if(this.countPframe < this.countIdr && this.pSet != null){
                                        this.pSet = concatenateByteArrays(this.pSet, concatenateByteArrays(NALUStartCode, hevcRaw));
                                        this.countPframe++;
                                        if(this.countPframe == this.countIdr){
                                            onHevcPackageReceived(new DataStruct(this.pSet, timestamp, false));
                                            this.countPframe -= this.countIdr;
                                        }
                                    }
                                }else if(this.countPframe == 0){
                                    this.pSet = null;
                                    this.pSet = concatenateByteArrays(NALUStartCode, hevcRaw);
                                    this.countPframe++;
                                }
                            }
                        }
                    }
                    break;
            this.nalTemp = nalType;