I'm trying to send the encoded h264 frames I have over a network as I get them, currently I'm only streaming the nal units I get from each frame as it is encoded, is this the correct approach?
I wrote a receiver application on a different computer to get the nals and wrote them all to a file sequentially, when played with vlc I didn't get any video and instead just got a screeching noise. I'm not sure exactly where the problem would lie here. I have included the result of the FFmpeg -I command on the file created.
Encoder and sender code
//Udp initialisation
struct sockaddr_in broadcastAddr;
int sock;
int yes = 1;
int addr_len;
int count;
fd_set readfd;
char buffer[1024];
int i;
sock = socket(AF_INET, SOCK_DGRAM,0);
if(sock < 0){
std::cout << "Failed to initialise socket!" << std::endl;
}
int ret = setsockopt(sock, SOL_SOCKET, SO_BROADCAST, (char*)&yes, sizeof(yes));
if(ret < 0){
std::cout << "setsockopt error!" <<std::endl;
}
addr_len = sizeof(struct sockaddr_in); // the size of the address
memset((void*)&broadcastAddr,0,addr_len); //0 out the address bits
broadcastAddr.sin_family = AF_INET;
broadcastAddr.sin_addr.s_addr = INADDR_BROADCAST;
broadcastAddr.sin_port = PORT;
//Set the encoder parameters
x264_param_t param;
x264_param_default_preset(¶m,"veryfast","zerolatency");
param.i_threads = 1;
param.i_width = camera.getWidth();
param.i_height = camera.getHeight();
param.i_fps_num = 30;
param.i_fps_den = 1;
// Intra refres:
param.i_keyint_max = 30;
param.b_intra_refresh = 1;
//Rate control:
param.rc.i_rc_method = X264_RC_CRF;
param.rc.f_rf_constant = 25;
param.rc.f_rf_constant_max = 35;
//For streaming:
param.b_repeat_headers = 1;
param.b_annexb = 1;
x264_param_apply_profile(¶m, "baseline");
x264_t *encoder = x264_encoder_open(¶m); //H.264 encoder object
x264_picture_t pic_in, pic_out;
x264_picture_alloc(&pic_in, X264_CSP_I420,camera.getWidth(), camera.getHeight());
//Network abstraction layer units for broadcast
x264_nal_t *nals;
int i_nals;
while(true){
//If there is valid data in the processing queue
if(!encoderQueue.empty()){
//File the x264 input data structure with the file data
fillImage(encoderQueue.front(),camera.getWidth(),camera.getHeight(),&pic_in);
//Encode and send
int frame_size = x264_encoder_encode(encoder, &nals, &i_nals, &pic_in, &pic_out);
if (frame_size >= 0) {
//The frame is ready to be sent over UDP!
for(int i = 0; i < i_nals; i++){
ret = sendto(sock, &nals[0].p_payload, frame_size,0,(struct sockaddr*)&broadcastAddr,addr_len);
if(ret > 0){
std::cout << "Streamed frame nal unit " << i << std::endl;
} else{
std::cout << "Failed to stream nal unit " << i << std::endl;
}
}
}
else{
std::cout<<"Failed to encode h264 frame!" << std::endl;
}
//Finsihed with the current frame, pop it off the queue and remove any nals to do with it
encoderQueue.pop();
frame_size = 0;
nals = nullptr;
i_nals = 0;
}
}
Receiver application
#include <iostream>
#include <x264.h>
#include <Network/Network.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <queue>
#define BUFFER_LEN 10000
#define PORT_NO 3879
int main(int argc, const char * argv[]) {
FILE *file; //File to write the h264 nals too
//Declare the address memory space
struct sockaddr_in sockAddr , bcAddr;
socklen_t bcAddr_len = sizeof(&bcAddr); //Store the length of the broadcast address structure
//0 out the assigned memory
memset(&sockAddr, 0, sizeof(sockAddr));
memset(&bcAddr, 0 ,sizeof(bcAddr));
//Set the address parameters to look for incoming IpV4/UDP data
sockAddr.sin_family = AF_INET;
sockAddr.sin_port = htons(PORT_NO);
sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
bcAddr.sin_family = AF_INET;
bcAddr.sin_port = PORT_NO;
inet_aton("255.255.255.255",&bcAddr.sin_addr);
//Initialise a udp socket to read broadcast bytes
int soc = socket(AF_INET, SOCK_DGRAM,0);
//Check socket init
if(soc < 0){
std::cout << "Failed to initialise UDP socket!" << std::endl;
return -1;
}
//Bind the address details to the socket, check for errors
if(bind(soc, (struct sockaddr*)&sockAddr, sizeof(sockAddr)) < 0){
std::cout << "Failed to bind address structure to socket!" << std::endl;
return -2;
}
file = fopen("stream.h264","wb"); // Open the file for writing
unsigned char buffer[BUFFER_LEN];
while(true){
memset(&buffer, 0, sizeof(unsigned char) * BUFFER_LEN);
int recv_len = recvfrom(soc, buffer, BUFFER_LEN, 0, (struct sockaddr *)&bcAddr, &bcAddr_len);
std::cout<< "Received " << recv_len << "bytes on broadcast address" << std::endl;
fwrite(&buffer, sizeof(unsigned char), recv_len, file);
}
return 0;
}
FFMPEG -I output
Any help would be greatly appreciated.
There is no “correct” or “incorrect” approach, as long as the receiver and sender are in agreement with the protocol. In this example, VLC is looking for some sort of container or formating. Most commonly, a transport stream.
If you plan on reading the h264 stream and write it to disk, or pipe it to something expecting an annex b stream. What you are doing is correct. If you expect common players to read the stream directly, use a transport stream.