c++kotlinffmpegkotlin-multiplatformbytedeco-javacv

FFMPEG C++/Kotlin Encode data from ByteArray (ByteBuffer) to subtitles and mux it with video


I need to mux some byte data encoded to base64 String to mpeg-ts container. Can't find any solution to somehow encode this data to AVPacket. Using of "avcodec_encode_subtitle" supposes to have AVSubtitle with proper data. But how can I create/allocate AVSubtitle with my data?

Upd. The only variant I managed to create (in Kotlin, as my app is KMP) is that, but it gives error "Invalid data found when processing input" when encoding

val data = AVSubtitleRect()
data.type(SUBTITLE_ASS)
data.ass(BytePointer(buffer))
val subtitle = AVSubtitle()
val pointerPointer = PointerPointer<AVSubtitleRect>(1)
pointerPointer.put(data)
subtitle.rects(pointerPointer)
subtitle.pts(packet.pts())

val bufferSize = 1024 * 1024
val encodedBuffer = BytePointer(av_malloc(bufferSize.toLong()))

val result = avcodec_encode_subtitle(subtitleContext, encodedBuffer, bufferSize, subtitle)

if (result >= 0) {
    val outSubtitlePacket = AVPacket()
    outSubtitlePacket.apply {
        data(encodedBuffer)
        outSubtitlePacket.size(result)
        outSubtitlePacket.stream_index(subtitleStreamIndex)
        duration(packet.duration())
        dts(packet.dts())
        pts(packet.pts())
        pos(packet.pos())
    }

    av_interleaved_write_frame(avOutputCtx, outSubtitlePacket)

    av_packet_unref(outSubtitlePacket)
    av_free(encodedBuffer)
}

C equivalent

// Allocate the subtitle rect
AVSubtitleRect *data = av_mallocz(sizeof(AVSubtitleRect));
if (!data) {
    fprintf(stderr, "Could not allocate AVSubtitleRect.\n");
    return AVERROR(ENOMEM);
}
data->type = SUBTITLE_ASS;
data->ass = av_strdup(buffer);
if (!data->ass) {
    fprintf(stderr, "Could not allocate ASS buffer.\n");
    av_free(data);
    return AVERROR(ENOMEM);
}

// Allocate the subtitle
AVSubtitle subtitle;
memset(&subtitle, 0, sizeof(subtitle));
subtitle.rects = av_mallocz(sizeof(*subtitle.rects));
if (!subtitle.rects) {
    fprintf(stderr, "Could not allocate AVSubtitle rects.\n");
    av_free(data->ass);
    av_free(data);
    return AVERROR(ENOMEM);
}
subtitle.rects[0] = data;
subtitle.num_rects = 1;
subtitle.pts = packet->pts;

// Allocate buffer for encoded subtitle
int bufferSize = 1024 * 1024;
uint8_t *encodedBuffer = av_malloc(bufferSize);
if (!encodedBuffer) {
    fprintf(stderr, "Could not allocate buffer for encoded subtitle.\n");
    av_free(subtitle.rects);
    av_free(data->ass);
    av_free(data);
    return AVERROR(ENOMEM);
}

// Encode the subtitle
int result = avcodec_encode_subtitle(subtitleContext, encodedBuffer, bufferSize, &subtitle);
if (result >= 0) {
    // Create the output subtitle packet
    AVPacket outSubtitlePacket;
    av_init_packet(&outSubtitlePacket);
    outSubtitlePacket.data = encodedBuffer;
    outSubtitlePacket.size = result;
    outSubtitlePacket.stream_index = subtitleStreamIndex;
    outSubtitlePacket.duration = packet->duration;
    outSubtitlePacket.dts = packet->dts;
    outSubtitlePacket.pts = packet->pts;
    outSubtitlePacket.pos = packet->pos;

    // Write the subtitle packet
    result = av_interleaved_write_frame(avOutputCtx, &outSubtitlePacket);
    if (result < 0) {
        fprintf(stderr, "Error while writing subtitle packet: %s\n", av_err2str(result));
    }

    av_packet_unref(&outSubtitlePacket);
    av_free(encodedBuffer);
} else {
    fprintf(stderr, "Failed to encode subtitles. Reason: %s\n", av_err2str(result));
    av_free(encodedBuffer);
}

av_free(subtitle.rects);
av_free(data->ass);
av_free(data);

Solution

  • I've managed to do this. But i get access violation exception when calling "av_interleaved_write_frame"

    if (!subtitleCodec)
        subtitleCodec = avcodec_find_encoder(AV_CODEC_ID_ASS);
    
    if (!subtitleStream)
        subtitleStream = avformat_new_stream(avOutputCtxRec, subtitleCodec);
    if (!subtitleStream) {
        std::cout << "Could not allocate subtitle stream" << std::endl;
        return;
    }
    
    if (!subtitleCtx) {
        subtitleCtx = avcodec_alloc_context3(subtitleCodec);
        subtitleCtx->codec_id = AV_CODEC_ID_ASS;
        subtitleCtx->codec_type = AVMEDIA_TYPE_SUBTITLE;
        subtitleCtx->width = width;
        subtitleCtx->height = height;
        subtitleCtx->framerate = av_make_q(1000, 1);
        subtitleCtx->time_base = av_make_q(1, 1000);
    }
    
    subtitleStream->id = 1;
    subtitleStream->codecpar->codec_id = AV_CODEC_ID_ASS;
    subtitleStream->codecpar->codec_type = AVMEDIA_TYPE_SUBTITLE;
    subtitleStream->codecpar->width = width;
    subtitleStream->codecpar->height = height;
    subtitleStream->codecpar->framerate = av_make_q(1000, 1);
    subtitleStream->time_base = av_make_q(1, 1000);
    subtitleStream->avg_frame_rate = av_make_q(1000, 1);
    
    if (avOutputCtxRec->oformat->flags & AVFMT_GLOBALHEADER)
        subtitleCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    
    if (avcodec_open2(subtitleCtx, subtitleCodec, NULL) < 0) {
        std::cout << "Could not open subtitle codec" << std::endl;
        return;
    }
    
    if (avcodec_parameters_from_context(subtitleStream->codecpar, subtitleCtx) < 0) {
        std::cout << "Could not copy codec parameters" << std::endl;
        return;
    }
    
    AVSubtitle subtitle;
    memset(&subtitle, 0, sizeof(AVSubtitle));
    
    subtitle.num_rects = 1;
    subtitle.rects = (AVSubtitleRect**)av_mallocz(sizeof(AVSubtitleRect*));
    subtitle.rects[0] = (AVSubtitleRect*)av_mallocz(sizeof(AVSubtitleRect));
    
    AVSubtitleRect* rect = subtitle.rects[0];
    rect->type = SUBTITLE_ASS;
    rect->ass = av_strdup(text.c_str());
    
    subtitle.start_display_time = 0;
    subtitle.end_display_time = 0;
    
    auto now = std::chrono::high_resolution_clock::now();
    auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
    if (subtitleTimeStamp == 0)
        subtitleTimeStamp = millis;
    subtitle.pts = millis - subtitleTimeStamp;
    
    int buf_size = 1024 * 1024;
    uint8_t* buf = (uint8_t*)av_malloc(buf_size);
    if (!buf) {
        av_freep(&subtitle.rects[0]->ass);
        av_freep(&subtitle.rects[0]);
        av_freep(&subtitle.rects);
        std::cout << "Could not allocate buffer" << std::endl;
        return;
    }
    
    int encoded_size = avcodec_encode_subtitle(subtitleCtx, buf, buf_size, &subtitle);
    if (encoded_size < 0) {
        av_free(buf);
        av_freep(&subtitle.rects[0]->ass);
        av_freep(&subtitle.rects[0]);
        av_freep(&subtitle.rects);
        std::cout << "Failed to encode subtitle" << std::endl;
        return;
    }
    
    AVPacket* pkt = av_packet_alloc();
    if (av_packet_from_data(pkt, buf, encoded_size) < 0) {
        av_free(buf);
        av_packet_free(&pkt);
        std::cout << "Error setting packet data" << std::endl;
        return;
    }
    pkt->stream_index = subtitleStream->index;
    pkt->pts = av_rescale_q(subtitle.pts, subtitleCtx->time_base, subtitleStream->time_base);
    pkt->dts = pkt->pts;
    pkt->duration = 0;
    
    if (av_interleaved_write_frame(avOutputCtxRec, pkt) < 0) {
        std::cout << "Error writing subtitle packet" << std::endl;
    }
    
    av_packet_unref(pkt);
    av_packet_free(&pkt);
    av_freep(&subtitle.rects[0]->ass);
    av_freep(&subtitle.rects[0]);
    av_freep(&subtitle.rects);
    std::cout << "Subtitle written" << std::endl;