c++videoencodingh.264

Add SEI message after every frame

I need to write a simple encoder that will add an SEI unit after each encoded frame.

I looked at the BSAnalyzer and it looks like what I need

enter image description here

SEI comes after each frame. But when I play a file in ffmpeg I get errors:

[NULL @ 000001cd901df1c0] SEI type 5 size 179843 truncated at 179776
[h264 @ 000001cd96eae780] SEI type 5 size 179843 truncated at 179774
[NULL @ 000001cd901df1c0] SEI type 5 size 179843 truncated at 179776
[h264 @ 000001cd970789c0] SEI type 5 size 179843 truncated at 179774
...and more

My code of write SEI:

static const uint8_t START_MARKER[4] = {0x0, 0x0, 0x0, 0x1};
static const uint8_t UUID[16] = {
    0x81, 0x39, 0xe9, 0xbd, 0xa6, 0x09, 0x48, 0xb7,
    0x76, 0x2c, 0xc8, 0x20, 0xd9, 0x68, 0xcc, 0xcf};

static const uint8_t SEI_TYPE[1] = {0x06};
static const uint8_t PAYLOAD_TYPE[1] = {0x05};

int calc_sei_size(size_t user_data_size)
{
    int new_buffer_size = 0;
    new_buffer_size += sizeof(START_MARKER); // NAL marker
    new_buffer_size += sizeof(SEI_TYPE);     //+NAL type = 0x06
    new_buffer_size += sizeof(PAYLOAD_TYPE); //+PayloadType = 0x05

    int payload_size = user_data_size + sizeof(UUID); //user_data + uuid
    while (payload_size > 0xFF)
    {
        payload_size -= 0xFF;
        new_buffer_size++; // PayloadSize how FF while is more 255
    }
    new_buffer_size += 0xFF - payload_size; //+PayloadSize diff
    new_buffer_size += sizeof(UUID);        //+Uuid
    new_buffer_size += user_data_size;      //+userData
    return new_buffer_size;
}

void sei_write(uint8_t *user_data, size_t user_data_size)
{
    int newSize = calc_sei_size(user_data_size);
    uint8_t *buffer = (uint8_t *)calloc(sizeof(uint8_t), newSize);

    int offset = 0;
    memcpy(&buffer[offset], START_MARKER, sizeof(START_MARKER)); // copy start marker
    offset += sizeof(START_MARKER);
    memcpy(&buffer[offset], SEI_TYPE, sizeof(SEI_TYPE)); // copy SEI_TYPE
    offset += sizeof(SEI_TYPE);
    memcpy(&buffer[offset], PAYLOAD_TYPE, sizeof(PAYLOAD_TYPE)); // copy SEI_TYPE
    offset += sizeof(PAYLOAD_TYPE);

    int payload_size = user_data_size + sizeof(UUID);
    while (payload_size > 0xFF)
    {
        payload_size -= 0xFF;
        buffer[offset++] = 0xFF;
    }
    buffer[offset++] = 0xFF - payload_size;
    memcpy(&buffer[offset], UUID, sizeof(UUID)); // copy UUID
    offset += sizeof(UUID);
    memcpy(&buffer[offset], user_data, user_data_size); // copy user_data

    DoSomeWithBuffer(buffer, newSize)
}
Solution

  • Okay, I think I figured out what the problem was. There are several features in packaging user_date in the SEI format:

    1. If the user data contains the following bytes in a row:
    00 00 00
00 00 01
00 00 02
00 00 03

    this can be perceived as NAL markers, so it is necessary to insert 03 after the 2nd byte 00, i.e. convert them to this form:

    00 00 03 00
00 00 03 01
00 00 03 02
00 00 03 03

    this transformation is called rbsp https://docs.rs/h264-reader/latest/h264_reader/rbsp/

    1. Since in the first paragraph we changed the user_data and size, you should in any case write down the payloadsize of the original size, not the size after the RBSP conversion

    2. The last byte should not be 00; it is recommended to use some endbyte as an end of data marker. I insert 0x80 at the end of the data

    3. PayloadSize actually means LengthSizeMinusOne, and since I always insert an endbyte (0x80) at the end, I do not take into account the length of this byte https://membrane.stream/learn/h264/3

    4. If the last byte is PayloadSize = FF then we must add 00, otherwise decoders will perceive the next byte as still payodsize, and there will be a user date

    As a result, I wrote the following code:

    
static const uint8_t START_MARKER[4] = {0x0, 0x0, 0x0, 0x1};
static const uint8_t UUID[16] = {
    0x81, 0x39, 0xe9, 0xbd, 0xa6, 0x09, 0x48, 0xb7,
    0x76, 0x2c, 0xc8, 0x20, 0xd9, 0x68, 0xcc, 0xcf};

static const uint8_t SEI_TYPE[1] = {0x06};
static const uint8_t PAYLOAD_TYPE[1] = {0x05};

int calc_sei_size(size_t user_data_size) {
  int new_buffer_size = 0;
  new_buffer_size += sizeof(START_MARKER); // NAL marker
  new_buffer_size += sizeof(SEI_TYPE);     //+NAL type = 0x06
  new_buffer_size += sizeof(PAYLOAD_TYPE); //+PayloadType = 0x05

  int payload_size = user_data_size + sizeof(UUID); // user_data + uuid
  while (payload_size > 0xFF) {
    payload_size -= 0xFF;
    new_buffer_size++; // PayloadSize how FF while is more 255
  }
  if (payload_size == 0xFF)
    new_buffer_size+= 1;

  new_buffer_size += 0xFF - payload_size; //+PayloadSize diff
  new_buffer_size += sizeof(UUID);        //+Uuid
  new_buffer_size += user_data_size;      //+userData
  return new_buffer_size;
}

void sei_write(uint8_t *user_data, size_t user_data_size) {
  // allocate a larger buffer because the data may be enlarged after rbsp conversion
  int new_size = calc_sei_size(user_data_size) * 2;
  uint8_t *buffer = (uint8_t *)calloc(sizeof(uint8_t), new_size);

  int offset = 0;
  memcpy(&buffer[offset], START_MARKER, sizeof(START_MARKER)); // copy start marker
  offset += sizeof(START_MARKER);
  memcpy(&buffer[offset], SEI_TYPE, sizeof(SEI_TYPE)); // copy SEI_TYPE
  offset += sizeof(SEI_TYPE);
  memcpy(&buffer[offset], PAYLOAD_TYPE, sizeof(PAYLOAD_TYPE)); // copy SEI_TYPE
  offset += sizeof(PAYLOAD_TYPE);

  // write payload size
  int payload_size = user_data_size + sizeof(UUID);
  while (payload_size > 0xFF) {
    payload_size -= 0xFF;
    buffer[offset++] = 0xFF;
  }
  buffer[offset++] = payload_size; // Correctly write the remaining size
  if (payload_size == 0xFF)        // if last byte is FF add 00
    buffer[offset++] = 0x00;

  buffer[offset++] = 0xFF - payload_size;
  memcpy(&buffer[offset], UUID, sizeof(UUID)); // copy UUID
  offset += sizeof(UUID);

  // convert to rbsp
  uint8_t *new_buffer = NULL;
  int new_buffer_size = 0;
  rbsp_encode(user_data, user_data_size, new_buffer, &new_buffer_size);

  memcpy(&buffer[offset], new_buffer, new_buffer_size); // copy user_data
  offset += new_buffer_size;
  buffer[offset++] = 0x80; // add EndByte

  write_buffer(buffer, offset);
  free(new_buffer);
  free(buffer);
}

void rbsp_encode(uint8_t *data, int data_size, uint8_t *&new_buffer, int *new_size) {
  // 0x00 00 00 =>  0x00 00 03 00
  // 0x00 00 01 =>  0x00 00 03 01
  // 0x00 00 02 =>  0x00 00 03 02
  // 0x00 00 03 =>  0x00 00 03 03
  new_buffer = (uint8_t *)calloc(sizeof(uint8_t), data_size * 2);
  int offset = 0;
  *new_size = 0;
  int value = *new_size;

  while (offset < data_size) {
    if (offset + 2 < data_size &&
        data[offset] == 0 && data[offset + 1] == 0 &&
        (data[offset + 2] == 0 || data[offset + 2] == 1 || data[offset + 2] == 2 || data[offset + 2] == 3)) {
      new_buffer[ *new_size += 1] = data[offset++];
      new_buffer[ *new_size += 1] = data[offset++];
      new_buffer[ *new_size += 1] = 0x03;
    } else {
      new_buffer[ *new_size += 1] = data[offset++];
    }
  }
}

//just for sample
void write_buffer(uint8_t *buffer_data, size_t buffer_size) {
  FILE *ptrFile = fopen("C:\\Temp\\data_conv", "wb");
  fwrite(buffer_data, 1, buffer_size, ptrFile); // записать в файл содержимое буфера
  fclose(ptrFile);
}