From cb6ecf862b8803e6f2d323af23374bebaf422e79 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 12 Jun 2015 20:05:23 +0200 Subject: [PATCH 01/17] bmdcapture: Refactor Video --- bmdcapture.cpp | 173 +++++++++++++++++++++++++------------------------ 1 file changed, 90 insertions(+), 83 deletions(-) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index 557f771..a8292a9 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -373,92 +373,117 @@ void write_data_packet(char *data, int size, int64_t pts) } +void write_video_packet(IDeckLinkVideoInputFrame *videoFrame, + int64_t pts, int64_t duration) +{ + AVPacket pkt; + AVCodecContext *c; + void *frameBytes; + time_t cur_time; + + av_init_packet(&pkt); + c = video_st->codec; + if (g_verbose && frameCount % 25 == 0) { + unsigned long long qsize = avpacket_queue_size(&queue); + fprintf(stderr, + "Frame received (#%lu) - Valid (%liB) - QSize %f\n", + frameCount, + videoFrame->GetRowBytes() * videoFrame->GetHeight(), + (double)qsize / 1024 / 1024); + } + + videoFrame->GetBytes(&frameBytes); + + if (videoFrame->GetFlags() & bmdFrameHasNoInputSource) { + if (pix_fmt == AV_PIX_FMT_UYVY422 && draw_bars) { + unsigned bars[8] = { + 0xEA80EA80, 0xD292D210, 0xA910A9A5, 0x90229035, + 0x6ADD6ACA, 0x51EF515A, 0x286D28EF, 0x10801080 }; + int width = videoFrame->GetWidth(); + int height = videoFrame->GetHeight(); + unsigned *p = (unsigned *)frameBytes; + + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x += 2) + *p++ = bars[(x * 8) / width]; + } + } + if (!no_video) { + time(&cur_time); + fprintf(stderr,"%s " + "Frame received (#%lu) - No input signal detected " + "- Frames dropped %u - Total dropped %u\n", + ctime(&cur_time), + frameCount, ++dropped, ++totaldropped); + } + no_video = 1; + } else { + if (no_video) { + time(&cur_time); + fprintf(stderr, "%s " + "Frame received (#%lu) - Input returned " + "- Frames dropped %u - Total dropped %u\n", + ctime(&cur_time), + frameCount, ++dropped, ++totaldropped); + } + no_video = 0; + } + + pkt.dts = pkt.pts = pts; + + pkt.duration = duration; + //To be made sure it still applies + pkt.flags |= AV_PKT_FLAG_KEY; + pkt.stream_index = video_st->index; + pkt.data = (uint8_t *)frameBytes; + pkt.size = videoFrame->GetRowBytes() * + videoFrame->GetHeight(); + //fprintf(stderr,"Video Frame size %d ts %d\n", pkt.size, pkt.pts); + c->frame_number++; + avpacket_queue_put(&queue, &pkt); +} + + HRESULT DeckLinkCaptureDelegate::VideoInputFrameArrived( IDeckLinkVideoInputFrame *videoFrame, IDeckLinkAudioInputPacket *audioFrame) { - void *frameBytes; void *audioFrameBytes; BMDTimeValue frameTime; BMDTimeValue frameDuration; - time_t cur_time; frameCount++; // Handle Video Frame if (videoFrame) { - AVPacket pkt; - AVCodecContext *c; - av_init_packet(&pkt); - c = video_st->codec; - if (g_verbose && frameCount % 25 == 0) { - unsigned long long qsize = avpacket_queue_size(&queue); - fprintf(stderr, - "Frame received (#%lu) - Valid (%liB) - QSize %f\n", - frameCount, - videoFrame->GetRowBytes() * videoFrame->GetHeight(), - (double)qsize / 1024 / 1024); - } - - videoFrame->GetBytes(&frameBytes); + int64_t pts; videoFrame->GetStreamTime(&frameTime, &frameDuration, video_st->time_base.den); - if (videoFrame->GetFlags() & bmdFrameHasNoInputSource) { - if (pix_fmt == AV_PIX_FMT_UYVY422 && draw_bars) { - unsigned bars[8] = { - 0xEA80EA80, 0xD292D210, 0xA910A9A5, 0x90229035, - 0x6ADD6ACA, 0x51EF515A, 0x286D28EF, 0x10801080 }; - int width = videoFrame->GetWidth(); - int height = videoFrame->GetHeight(); - unsigned *p = (unsigned *)frameBytes; - - for (int y = 0; y < height; y++) { - for (int x = 0; x < width; x += 2) - *p++ = bars[(x * 8) / width]; - } - } - if (!no_video) { - time(&cur_time); - fprintf(stderr,"%s " - "Frame received (#%lu) - No input signal detected " - "- Frames dropped %u - Total dropped %u\n", - ctime(&cur_time), - frameCount, ++dropped, ++totaldropped); - } - no_video = 1; - } else { - if (no_video) { - time(&cur_time); - fprintf(stderr, "%s " - "Frame received (#%lu) - Input returned " - "- Frames dropped %u - Total dropped %u\n", - ctime(&cur_time), - frameCount, ++dropped, ++totaldropped); - } - no_video = 0; - } - - pkt.pts = frameTime / video_st->time_base.num; + pts = frameTime / video_st->time_base.num; if (initial_video_pts == AV_NOPTS_VALUE) { - initial_video_pts = pkt.pts; + initial_video_pts = pts; } - pkt.pts -= initial_video_pts; - pkt.dts = pkt.pts; + pts -= initial_video_pts; - pkt.duration = frameDuration; - //To be made sure it still applies - pkt.flags |= AV_PKT_FLAG_KEY; - pkt.stream_index = video_st->index; - pkt.data = (uint8_t *)frameBytes; - pkt.size = videoFrame->GetRowBytes() * - videoFrame->GetHeight(); - //fprintf(stderr,"Video Frame size %d ts %d\n", pkt.size, pkt.pts); - c->frame_number++; - avpacket_queue_put(&queue, &pkt); + write_video_packet(videoFrame, pts, frameDuration); + if (serial_fd > 0) { + char line[8] = {0}; + int count = read(serial_fd, line, 7); + if (count > 0) + fprintf(stderr, "read %d bytes: %s \n", count, line); + else line[0] = ' '; + write_data_packet(line, 7, pts); + } + if (wallclock) { + int64_t t = av_gettime(); + char line[20]; + snprintf(line, sizeof(line), "%ld", t); + write_data_packet(line, strlen(line), pts); + } } // Handle Audio Frame @@ -497,24 +522,6 @@ HRESULT DeckLinkCaptureDelegate::VideoInputFrameArrived( avpacket_queue_put(&queue, &pkt); } - if (serial_fd > 0) { - char line[8] = {0}; - int count = read(serial_fd, line, 7); - if (count > 0) - fprintf(stderr, "read %d bytes: %s \n", count, line); - else line[0] = ' '; - write_data_packet(line, 7, - frameTime / video_st->time_base.num - - initial_video_pts); - } - if (wallclock) { - int64_t t = av_gettime(); - char line[20]; - snprintf(line, sizeof(line), "%ld", t); - write_data_packet(line, strlen(line), - frameTime / video_st->time_base.num - - initial_video_pts); - } return S_OK; } From f6bf1ffe5a0fcaeb3c4f08f51b3a97aacc6abc49 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 12 Jun 2015 20:09:31 +0200 Subject: [PATCH 02/17] bmdcapture: Refactor Audio --- bmdcapture.cpp | 72 ++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index a8292a9..73fb296 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -372,6 +372,37 @@ void write_data_packet(char *data, int size, int64_t pts) avpacket_queue_put(&queue, &pkt); } +void write_audio_packet(IDeckLinkAudioInputPacket *audioFrame) +{ + AVCodecContext *c; + AVPacket pkt; + BMDTimeValue audio_pts; + void *audioFrameBytes; + + av_init_packet(&pkt); + + c = audio_st->codec; + //hack among hacks + pkt.size = audioFrame->GetSampleFrameCount() * + g_audioChannels * (g_audioSampleDepth / 8); + audioFrame->GetBytes(&audioFrameBytes); + audioFrame->GetPacketTime(&audio_pts, audio_st->time_base.den); + pkt.pts = audio_pts / audio_st->time_base.num; + + if (initial_audio_pts == AV_NOPTS_VALUE) { + initial_audio_pts = pkt.pts; + } + + pkt.pts -= initial_audio_pts; + pkt.dts = pkt.pts; + + pkt.flags |= AV_PKT_FLAG_KEY; + pkt.stream_index = audio_st->index; + pkt.data = (uint8_t *)audioFrameBytes; + c->frame_number++; + + avpacket_queue_put(&queue, &pkt); +} void write_video_packet(IDeckLinkVideoInputFrame *videoFrame, int64_t pts, int64_t duration) @@ -447,14 +478,13 @@ void write_video_packet(IDeckLinkVideoInputFrame *videoFrame, HRESULT DeckLinkCaptureDelegate::VideoInputFrameArrived( IDeckLinkVideoInputFrame *videoFrame, IDeckLinkAudioInputPacket *audioFrame) { - void *audioFrameBytes; - BMDTimeValue frameTime; - BMDTimeValue frameDuration; frameCount++; // Handle Video Frame if (videoFrame) { + BMDTimeValue frameTime; + BMDTimeValue frameDuration; int64_t pts; videoFrame->GetStreamTime(&frameTime, &frameDuration, video_st->time_base.den); @@ -487,40 +517,8 @@ HRESULT DeckLinkCaptureDelegate::VideoInputFrameArrived( } // Handle Audio Frame - if (audioFrame) { - AVCodecContext *c; - AVPacket pkt; - BMDTimeValue audio_pts; - av_init_packet(&pkt); - - c = audio_st->codec; - //hack among hacks - pkt.size = audioFrame->GetSampleFrameCount() * - g_audioChannels * (g_audioSampleDepth / 8); - audioFrame->GetBytes(&audioFrameBytes); - audioFrame->GetPacketTime(&audio_pts, audio_st->time_base.den); - pkt.pts = audio_pts / audio_st->time_base.num; - - if (initial_audio_pts == AV_NOPTS_VALUE) { - initial_audio_pts = pkt.pts; - } - - pkt.pts -= initial_audio_pts; - pkt.dts = pkt.pts; - - //fprintf(stderr,"Audio Frame size %d ts %d\n", pkt.size, pkt.pts); - pkt.flags |= AV_PKT_FLAG_KEY; - pkt.stream_index = audio_st->index; - pkt.data = (uint8_t *)audioFrameBytes; - //pkt.size= avcodec_encode_audio(c, audio_outbuf, audio_outbuf_size, samples); - c->frame_number++; - //write(audioOutputFile, audioFrameBytes, audioFrame->GetSampleFrameCount() * g_audioChannels * (g_audioSampleDepth / 8)); -/* if (av_interleaved_write_frame(oc, &pkt) != 0) { - * fprintf(stderr, "Error while writing audio frame\n"); - * exit(1); - * } */ - avpacket_queue_put(&queue, &pkt); - } + if (audioFrame) + write_audio_packet(audioFrame); return S_OK; From decf2ef73e027b7ba238c067ee2de0b0bc7ab353 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 12 Jun 2015 20:16:46 +0200 Subject: [PATCH 03/17] bmdcapture: Fix all the clang warnings --- bmdcapture.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index 73fb296..23fd4e1 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -19,6 +19,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#define __STDC_FORMAT_MACROS +#include + #include #include #include @@ -511,7 +514,7 @@ HRESULT DeckLinkCaptureDelegate::VideoInputFrameArrived( if (wallclock) { int64_t t = av_gettime(); char line[20]; - snprintf(line, sizeof(line), "%ld", t); + snprintf(line, sizeof(line), "%" PRId64, t); write_data_packet(line, strlen(line), pts); } } @@ -628,7 +631,7 @@ static void *push_packet(void *ctx) while (avpacket_queue_get(&queue, &pkt, 1)) { av_interleaved_write_frame(s, &pkt); - if (g_maxFrames > 0 && frameCount >= g_maxFrames || + if ((g_maxFrames > 0 && frameCount >= g_maxFrames) || avpacket_queue_size(&queue) > g_memoryLimit) { pthread_cond_signal(&sleepCond); } @@ -786,9 +789,9 @@ int main(int argc, char *argv[]) } if (serial_fd > 0 && wallclock) { - fprintf(stderr, - "Wallclock and serial are not supported together\n", - "Please disable either\n"); + fprintf(stderr, "%s", + "Wallclock and serial are not supported together\n" + "Please disable either.\n"); exit(1); } @@ -900,7 +903,6 @@ int main(int argc, char *argv[]) usage(0); } - selectedDisplayMode = -1; while (displayModeIterator->Next(&displayMode) == S_OK) { if (g_videoModeIndex == displayModeCount) { selectedDisplayMode = displayMode->GetDisplayMode(); @@ -910,11 +912,6 @@ int main(int argc, char *argv[]) displayMode->Release(); } - if (selectedDisplayMode < 0) { - fprintf(stderr, "Invalid mode %d specified\n", g_videoModeIndex); - goto bail; - } - result = deckLinkInput->EnableVideoInput(selectedDisplayMode, pix, 0); if (result != S_OK) { fprintf(stderr, From f75b3e6219ad34fb7cc7578aea7edcf0a7bce461 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 12 Jun 2015 21:58:45 +0200 Subject: [PATCH 04/17] wip: VANC support --- bmdcapture.cpp | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index 23fd4e1..9192647 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -361,6 +361,32 @@ int64_t initial_audio_pts = AV_NOPTS_VALUE; static int no_video = 0; +#define CC_LINE 9 + + +// FIXME fail properly. +void vanc_as_side_data(AVPacket *pkt, + IDeckLinkVideoInputFrame *frame) +{ + IDeckLinkVideoFrameAncillary *ancillary; + void *buf; + uint8_t *vanc; + int len = frame->GetRowBytes(); + int ret = 0; + + ret = frame->GetAncillaryData(&ancillary); + if (ret < 0) + return; + + ret = ancillary->GetBufferForVerticalBlankingLine(CC_LINE, &buf); + if (ret < 0) + return; + + vanc = av_packet_new_side_data(pkt, AV_PKT_DATA_VANC, len); + + memcpy(vanc, buf, len); +} + void write_data_packet(char *data, int size, int64_t pts) { AVPacket pkt; @@ -474,6 +500,10 @@ void write_video_packet(IDeckLinkVideoInputFrame *videoFrame, videoFrame->GetHeight(); //fprintf(stderr,"Video Frame size %d ts %d\n", pkt.size, pkt.pts); c->frame_number++; + + if (pix_fmt == AV_PIX_FMT_YUV422P10) + vanc_as_side_data(&pkt, videoFrame); + avpacket_queue_put(&queue, &pkt); } From 667f67d7741ad03572e7cd549856a2c55f8faded Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 12 Jun 2015 23:51:33 +0200 Subject: [PATCH 05/17] bmdcapture: Update to the current api usage --- bmdcapture.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index 9192647..daccbe0 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -259,8 +259,8 @@ static AVStream *add_video_stream(AVFormatContext *oc, enum AVCodecID codec_id) * timebase should be 1/framerate and timestamp increments should be * identically 1.*/ displayMode->GetFrameRate(&frameRateDuration, &frameRateScale); - c->time_base.den = frameRateScale; - c->time_base.num = frameRateDuration; + st->time_base.den = frameRateScale; + st->time_base.num = frameRateDuration; c->pix_fmt = pix_fmt; if (codec_id == AV_CODEC_ID_V210 || codec_id == AV_CODEC_ID_R210) @@ -305,8 +305,8 @@ static AVStream *add_data_stream(AVFormatContext *oc, enum AVCodecID codec_id) c->codec_type = AVMEDIA_TYPE_DATA; displayMode->GetFrameRate(&frameRateDuration, &frameRateScale); - c->time_base.den = frameRateScale; - c->time_base.num = frameRateDuration; + st->time_base.den = frameRateScale; + st->time_base.num = frameRateDuration; // some formats want stream headers to be separate if(oc->oformat->flags & AVFMT_GLOBALHEADER) From 947d3c655b1c00f0651bfe5e6036315758fc74a4 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sat, 13 Jun 2015 01:17:42 +0200 Subject: [PATCH 06/17] wip: Free the ancillary data --- bmdcapture.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index daccbe0..41e0e6e 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -385,6 +385,7 @@ void vanc_as_side_data(AVPacket *pkt, vanc = av_packet_new_side_data(pkt, AV_PKT_DATA_VANC, len); memcpy(vanc, buf, len); + ancillary->Release(); } void write_data_packet(char *data, int size, int64_t pts) From 9b39b20011952feb8d3b21f64f69817ff3ff3627 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 14 Jun 2015 01:24:13 +0200 Subject: [PATCH 07/17] bmdplay: Add VANC support --- bmdcapture.cpp | 3 --- bmdplay.cpp | 25 +++++++++++++++++++++++-- modes.h | 2 ++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index 41e0e6e..ca85e80 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -361,9 +361,6 @@ int64_t initial_audio_pts = AV_NOPTS_VALUE; static int no_video = 0; -#define CC_LINE 9 - - // FIXME fail properly. void vanc_as_side_data(AVPacket *pkt, IDeckLinkVideoInputFrame *frame) diff --git a/bmdplay.cpp b/bmdplay.cpp index e0894f5..c9a058e 100644 --- a/bmdplay.cpp +++ b/bmdplay.cpp @@ -657,6 +657,12 @@ void Player::ScheduleNextFrame(bool prerolling) { AVPacket pkt; AVPicture picture; + void *frame; + int got_picture; + IDeckLinkVideoFrameAncillary *ancillary; + int side_data_size; + uint8_t *side_data; + if (serial_fd > 0 && packet_queue_get(&dataqueue, &pkt, 0)) { if (pkt.data[0] != ' '){ @@ -676,8 +682,14 @@ void Player::ScheduleNextFrame(bool prerolling) pix, bmdFrameFlagDefault, &videoFrame); - void *frame; - int got_picture; + + side_data = av_packet_get_side_data(&pkt, AV_PKT_DATA_VANC, + &side_data_size); + + if (pix_fmt == AV_PIX_FMT_YUV422P10 && side_data) + m_deckLinkOutput->CreateAncillaryData(pix, &ancillary); + + videoFrame->GetBytes(&frame); avcodec_decode_video2(video_st->codec, avframe, &got_picture, &pkt); @@ -688,6 +700,15 @@ void Player::ScheduleNextFrame(bool prerolling) sws_scale(sws, avframe->data, avframe->linesize, 0, avframe->height, picture.data, picture.linesize); + if (pix_fmt == AV_PIX_FMT_YUV422P10 && side_data) { + void *buf; + ancillary->GetBufferForVerticalBlankingLine(CC_LINE, &buf); + + memcpy(buf, side_data, side_data_size); + + videoFrame->SetAncillaryData(ancillary); + } + if (m_deckLinkOutput->ScheduleVideoFrame(videoFrame, pkt.pts * video_st->time_base.num, diff --git a/modes.h b/modes.h index c05663f..cc35422 100644 --- a/modes.h +++ b/modes.h @@ -24,6 +24,8 @@ #include "DeckLinkAPI.h" +#define CC_LINE 9 + void print_input_modes(IDeckLink *deckLink); void print_output_modes(IDeckLink *deckLink); From 2d7458f8b713596d307e64811ee09c425d22ee75 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 14 Jun 2015 14:56:00 +0200 Subject: [PATCH 08/17] bmdplay: Verbose flag --- bmdplay.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bmdplay.cpp b/bmdplay.cpp index c9a058e..17963b6 100644 --- a/bmdplay.cpp +++ b/bmdplay.cpp @@ -57,6 +57,7 @@ static BMDPixelFormat pix = bmdFormat8BitYUV; static int buffer = 2000 * 1000; static int serial_fd = -1; +static int verbose = 0; const unsigned long kAudioWaterlevel = 48000 / 4; /* small */ @@ -302,6 +303,7 @@ int usage(int status) fprintf( stderr, + " -v Verbose reporting\n" " -f Filename raw video will be written to\n" " -C Card number to be used\n" " -b Milliseconds of pre-buffering before playback (default = 2000 ms)\n" @@ -325,7 +327,7 @@ int main(int argc, char *argv[]) int camera = 0; char *filename = NULL; - while ((ch = getopt(argc, argv, "?hs:f:a:m:n:F:C:O:b:p:S:")) != -1) { + while ((ch = getopt(argc, argv, "?hs:f:a:m:n:F:C:O:b:p:S:v")) != -1) { switch (ch) { case 'p': switch (atoi(optarg)) { @@ -362,6 +364,9 @@ int main(int argc, char *argv[]) case 'S': serial_fd = open(optarg, O_RDWR | O_NONBLOCK); break; + case 'v': + verbose = 1; + break; case '?': case 'h': return usage(0); @@ -703,6 +708,13 @@ void Player::ScheduleNextFrame(bool prerolling) if (pix_fmt == AV_PIX_FMT_YUV422P10 && side_data) { void *buf; ancillary->GetBufferForVerticalBlankingLine(CC_LINE, &buf); + if (verbose) + av_log(NULL, AV_LOG_INFO|AV_LOG_C(132), + "VANC 0x%02x 0x%02x 0x%02x 0x%02x\n", + side_data[0], + side_data[1], + side_data[2], + side_data[3]); memcpy(buf, side_data, side_data_size); From 07c55264468044dd4574e536ce368b4ed5038bba Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 14 Jun 2015 16:49:25 +0200 Subject: [PATCH 09/17] wip: Fix VANC support --- bmdplay.cpp | 68 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/bmdplay.cpp b/bmdplay.cpp index 17963b6..3e5fff6 100644 --- a/bmdplay.cpp +++ b/bmdplay.cpp @@ -618,7 +618,7 @@ void Player::StartRunning(int videomode) // Set the video output mode if (m_deckLinkOutput->EnableVideoOutput(videoDisplayMode->GetDisplayMode(), - bmdVideoOutputFlagDefault) != + bmdVideoOutputVANC) != S_OK) { fprintf(stderr, "Failed to enable video output\n"); return; @@ -663,12 +663,11 @@ void Player::ScheduleNextFrame(bool prerolling) AVPacket pkt; AVPicture picture; void *frame; - int got_picture; + int got_picture = 0; IDeckLinkVideoFrameAncillary *ancillary; - int side_data_size; + int side_data_size, linesize; uint8_t *side_data; - if (serial_fd > 0 && packet_queue_get(&dataqueue, &pkt, 0)) { if (pkt.data[0] != ' '){ fprintf(stderr,"written %.*s \n", pkt.size, pkt.data); @@ -680,30 +679,31 @@ void Player::ScheduleNextFrame(bool prerolling) if (packet_queue_get(&videoqueue, &pkt, 0) < 0) return; - IDeckLinkMutableVideoFrame *videoFrame; - m_deckLinkOutput->CreateVideoFrame(m_frameWidth, - m_frameHeight, - m_frameWidth * 2, - pix, - bmdFrameFlagDefault, - &videoFrame); - - side_data = av_packet_get_side_data(&pkt, AV_PKT_DATA_VANC, - &side_data_size); + if (pix == bmdFormat8BitYUV) + linesize = m_frameWidth * 2; + else // v210 + linesize = m_frameWidth * 8 / 3; - if (pix_fmt == AV_PIX_FMT_YUV422P10 && side_data) - m_deckLinkOutput->CreateAncillaryData(pix, &ancillary); + IDeckLinkMutableVideoFrame *videoFrame; + if (m_deckLinkOutput->CreateVideoFrame(m_frameWidth, + m_frameHeight, + linesize, + pix, + bmdFrameFlagDefault, + &videoFrame) != S_OK) + av_log(NULL, AV_LOG_ERROR, "Cannot get frame\n"); videoFrame->GetBytes(&frame); - avcodec_decode_video2(video_st->codec, avframe, &got_picture, &pkt); - if (got_picture) { - avpicture_fill(&picture, (uint8_t *)frame, pix_fmt, - m_frameWidth, m_frameHeight); + if (pix == bmdFormat10BitYUV) { + side_data = av_packet_get_side_data(&pkt, AV_PKT_DATA_VANC, + &side_data_size); + + if (side_data) + m_deckLinkOutput->CreateAncillaryData(pix, &ancillary); - sws_scale(sws, avframe->data, avframe->linesize, 0, avframe->height, - picture.data, picture.linesize); + memcpy(frame, pkt.data, pkt.size); // v210 data as-is if (pix_fmt == AV_PIX_FMT_YUV422P10 && side_data) { void *buf; @@ -720,16 +720,28 @@ void Player::ScheduleNextFrame(bool prerolling) videoFrame->SetAncillaryData(ancillary); } + got_picture = 1; + } else { + avcodec_decode_video2(video_st->codec, avframe, &got_picture, &pkt); + if (got_picture) { + avpicture_fill(&picture, (uint8_t *)frame, pix_fmt, + m_frameWidth, m_frameHeight); + + sws_scale(sws, avframe->data, avframe->linesize, 0, avframe->height, + picture.data, picture.linesize); + } + } + if (got_picture) { if (m_deckLinkOutput->ScheduleVideoFrame(videoFrame, - pkt.pts * - video_st->time_base.num, - pkt.duration * - video_st->time_base.num, - video_st->time_base.den) != - S_OK) + pkt.pts * + video_st->time_base.num, + pkt.duration * + video_st->time_base.num, + video_st->time_base.den) != S_OK) fprintf(stderr, "Error scheduling frame\n"); } + videoFrame->Release(); av_free_packet(&pkt); } From d6ed855d944e4655e9e0cc3b1431e03fbe6cde77 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 14 Jun 2015 17:13:57 +0200 Subject: [PATCH 10/17] bmdplay: Simplify --- bmdplay.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bmdplay.cpp b/bmdplay.cpp index 3e5fff6..f7e3f25 100644 --- a/bmdplay.cpp +++ b/bmdplay.cpp @@ -700,13 +700,12 @@ void Player::ScheduleNextFrame(bool prerolling) side_data = av_packet_get_side_data(&pkt, AV_PKT_DATA_VANC, &side_data_size); - if (side_data) - m_deckLinkOutput->CreateAncillaryData(pix, &ancillary); - memcpy(frame, pkt.data, pkt.size); // v210 data as-is - if (pix_fmt == AV_PIX_FMT_YUV422P10 && side_data) { + if (side_data) { void *buf; + m_deckLinkOutput->CreateAncillaryData(pix, &ancillary); + ancillary->GetBufferForVerticalBlankingLine(CC_LINE, &buf); if (verbose) av_log(NULL, AV_LOG_INFO|AV_LOG_C(132), From ca60060edef5b4580d51e8ede7e2b2915cca1143 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 14 Jun 2015 17:14:11 +0200 Subject: [PATCH 11/17] bmdplay: Exit on write samples failure --- bmdplay.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bmdplay.cpp b/bmdplay.cpp index f7e3f25..bbfa0cd 100644 --- a/bmdplay.cpp +++ b/bmdplay.cpp @@ -773,8 +773,10 @@ void Player::WriteNextAudioSamples() samples, pkt.pts + off, audio_st->time_base.den / audio_st->time_base.num, - &samplesWritten) != S_OK) + &samplesWritten) != S_OK) { fprintf(stderr, "error writing audio sample\n"); + break; + } samples -= samplesWritten; off += samplesWritten; } while (samples > 0); From 5d23407329e36841ad22312f0029090634275a99 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 23 Jun 2015 10:26:38 +0300 Subject: [PATCH 12/17] wip: Explicitly release the ancillary data --- bmdplay.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bmdplay.cpp b/bmdplay.cpp index bbfa0cd..04fd25f 100644 --- a/bmdplay.cpp +++ b/bmdplay.cpp @@ -664,7 +664,7 @@ void Player::ScheduleNextFrame(bool prerolling) AVPicture picture; void *frame; int got_picture = 0; - IDeckLinkVideoFrameAncillary *ancillary; + IDeckLinkVideoFrameAncillary *ancillary = NULL; int side_data_size, linesize; uint8_t *side_data; @@ -741,6 +741,8 @@ void Player::ScheduleNextFrame(bool prerolling) fprintf(stderr, "Error scheduling frame\n"); } + if (ancillary) + ancillary->Release(); videoFrame->Release(); av_free_packet(&pkt); } From ba44f814390715d1e6dd3f2e75fa33168a61c92c Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 23 Jun 2015 18:32:15 +0300 Subject: [PATCH 13/17] wip: side-data wallclock --- bmdcapture.cpp | 28 +++++++++++++++++++++------- bmdplay.cpp | 15 +++++++++++++-- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index ca85e80..1bd8933 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -36,6 +36,7 @@ #include "modes.h" extern "C" { #include "libavformat/avformat.h" +#include "libavutil/intreadwrite.h" #include "libavutil/time.h" } @@ -57,7 +58,7 @@ const char *g_videoOutputFile = NULL; const char *g_audioOutputFile = NULL; static int g_maxFrames = -1; static int serial_fd = -1; -static int wallclock = 0; +static int wallclock = -1; static int draw_bars = 1; bool g_verbose = false; unsigned long long g_memoryLimit = 1024 * 1024 * 1024; // 1GByte(>50 sec) @@ -361,6 +362,14 @@ int64_t initial_audio_pts = AV_NOPTS_VALUE; static int no_video = 0; +void wallclock_as_side_data(AVPacket *pkt) +{ + uint8_t *wallclock = av_packet_new_side_data(pkt, AV_PKT_DATA_WALLCLOCK, sizeof(int64_t)); + int64_t time = av_gettime(); + + AV_WB64(wallclock, time); +} + // FIXME fail properly. void vanc_as_side_data(AVPacket *pkt, IDeckLinkVideoInputFrame *frame) @@ -502,6 +511,9 @@ void write_video_packet(IDeckLinkVideoInputFrame *videoFrame, if (pix_fmt == AV_PIX_FMT_YUV422P10) vanc_as_side_data(&pkt, videoFrame); + if (wallclock == 0) + wallclock_as_side_data(&pkt); + avpacket_queue_put(&queue, &pkt); } @@ -539,7 +551,7 @@ HRESULT DeckLinkCaptureDelegate::VideoInputFrameArrived( write_data_packet(line, 7, pts); } - if (wallclock) { + if (wallclock == 1) { int64_t t = av_gettime(); char line[20]; snprintf(line, sizeof(line), "%" PRId64, t); @@ -638,7 +650,9 @@ int usage(int status) " 5: Optical SDI\n" " 6: S-Video\n" " -o AVFormat options\n" - " -w Embed a wallclock stream\n" + " -w Embed a wallclock stream\n" + " 0: as side-data\n" + " 1: as data-stream\n" " -d When the source is offline draw a black frame or color bars\n" " 0: black frame\n" " 1: color bars\n" @@ -704,7 +718,7 @@ int main(int argc, char *argv[]) } // Parse command line options - while ((ch = getopt(argc, argv, "?hvc:s:f:a:m:n:p:M:F:C:A:V:o:w")) != -1) { + while ((ch = getopt(argc, argv, "?hvc:s:f:a:m:n:p:M:F:C:A:V:o:w:")) != -1) { switch (ch) { case 'v': g_verbose = true; @@ -805,7 +819,7 @@ int main(int argc, char *argv[]) } break; case 'w': - wallclock = true; + wallclock = atoi(optarg); break; case 'd': draw_bars = atoi(optarg); @@ -816,7 +830,7 @@ int main(int argc, char *argv[]) } } - if (serial_fd > 0 && wallclock) { + if (serial_fd > 0 && wallclock == 1) { fprintf(stderr, "%s", "Wallclock and serial are not supported together\n" "Please disable either.\n"); @@ -981,7 +995,7 @@ int main(int argc, char *argv[]) video_st = add_video_stream(oc, fmt->video_codec); audio_st = add_audio_stream(oc, fmt->audio_codec); - if (serial_fd > 0 || wallclock) + if (serial_fd > 0 || wallclock == 1) data_st = add_data_stream(oc, AV_CODEC_ID_TEXT); if (!(fmt->flags & AVFMT_NOFILE)) { diff --git a/bmdplay.cpp b/bmdplay.cpp index 04fd25f..bceb270 100644 --- a/bmdplay.cpp +++ b/bmdplay.cpp @@ -35,7 +35,9 @@ extern "C" { #include #include +#include #include +#include #include "libswscale/swscale.h" } #include "compat.h" @@ -696,6 +698,15 @@ void Player::ScheduleNextFrame(bool prerolling) videoFrame->GetBytes(&frame); + side_data = av_packet_get_side_data(&pkt, AV_PKT_DATA_WALLCLOCK, + &side_data_size); + if (side_data && side_data_size == sizeof(int64_t)) { + int64_t t = AV_RB64(side_data); + av_log(NULL, AV_LOG_INFO|AV_LOG_C(124), + "Time delta %" PRId64 "\n", + av_gettime() - t); + } + if (pix == bmdFormat10BitYUV) { side_data = av_packet_get_side_data(&pkt, AV_PKT_DATA_VANC, &side_data_size); @@ -707,14 +718,14 @@ void Player::ScheduleNextFrame(bool prerolling) m_deckLinkOutput->CreateAncillaryData(pix, &ancillary); ancillary->GetBufferForVerticalBlankingLine(CC_LINE, &buf); - if (verbose) +/* if (verbose) av_log(NULL, AV_LOG_INFO|AV_LOG_C(132), "VANC 0x%02x 0x%02x 0x%02x 0x%02x\n", side_data[0], side_data[1], side_data[2], side_data[3]); - +*/ memcpy(buf, side_data, side_data_size); videoFrame->SetAncillaryData(ancillary); From 9ffa6fd4c8af06d3e8fa397f8db701269ed60777 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Wed, 24 Jun 2015 01:13:48 +0300 Subject: [PATCH 14/17] build: Add PRId64 and friends --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f19408d..0682539 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ PKG_DEPS = libavcodec libavformat libswscale libavutil CXXFLAGS = $(ECXXFLAGS) LDFLAGS = $(ELDFLAGS) -CXXFLAGS += `pkg-config --cflags $(PKG_DEPS)` -D__STDC_CONSTANT_MACROS +CXXFLAGS += `pkg-config --cflags $(PKG_DEPS)` -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS LDFLAGS += `pkg-config --libs $(PKG_DEPS)` CXXFLAGS+= -Wno-multichar -I $(SDK_PATH) -fno-rtti -g From 92ad53e47ea8b18d8bb5c2d6573d4e8445852e94 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 31 Jul 2015 23:07:42 +0200 Subject: [PATCH 15/17] bmdcapture: Drop the stdint macro It is passed as compiler flag now. --- bmdcapture.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index 1bd8933..86ebbad 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -19,7 +19,6 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -#define __STDC_FORMAT_MACROS #include #include From b9f107911f2ec70b16d93eeed23db99bc8c2c571 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 31 Jul 2015 23:12:54 +0200 Subject: [PATCH 16/17] bmdplay: Cleanup the code Drop all the warnings from clang. --- bmdplay.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bmdplay.cpp b/bmdplay.cpp index bceb270..c11a799 100644 --- a/bmdplay.cpp +++ b/bmdplay.cpp @@ -140,7 +140,7 @@ static int packet_queue_put(PacketQueue *q, AVPacket *pkt) q->nb_packets++; if (q->nb_packets > 5000) fprintf(stderr, - "%ld storing %p, %s - is the input faster than realtime?\n", + "%"PRId64" storing %p, %s - is the input faster than realtime?\n", q->nb_packets, q, q == &videoqueue ? "videoqueue" : "audioqueue"); @@ -167,7 +167,7 @@ static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block) q->last_pkt = NULL; q->nb_packets--; if (q->nb_packets > 5000) - fprintf(stderr, "pulling %ld from %p %s\n", + fprintf(stderr, "pulling %"PRId64" from %p %s\n", q->nb_packets, q, q == &videoqueue ? "videoqueue" : "audioqueue"); @@ -242,6 +242,9 @@ void *fill_queues(void *unused) case AVMEDIA_TYPE_DATA: packet_queue_put(&dataqueue, &pkt); break; + default: + av_packet_unref(&pkt); + break; } } return NULL; @@ -433,7 +436,8 @@ int main(int argc, char *argv[]) avformat_close_input(&ic); - fprintf(stderr, "video %ld audio %ld", videoqueue.nb_packets, + fprintf(stderr, "video %"PRId64" audio %"PRId64"\n", + videoqueue.nb_packets, audioqueue.nb_packets); return ret; @@ -479,7 +483,7 @@ bool Player::Init(int videomode, int connection, int camera) case 32: break; default: - fprintf(stderr, "%dbit audio not supported use 16bit or 32bit\n", + fprintf(stderr, "%lubit audio not supported use 16bit or 32bit\n", m_audioSampleDepth); } @@ -533,7 +537,7 @@ bool Player::Init(int videomode, int connection, int camera) m_deckLinkOutput->SetScheduledFrameCompletionCallback(this); m_deckLinkOutput->SetAudioCallback(this); - avframe = avcodec_alloc_frame(); + avframe = av_frame_alloc(); packet_queue_init(&audioqueue); packet_queue_init(&videoqueue); From 4233f649ef79117009329fcd8f0771f5a65c449e Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 31 Jul 2015 23:25:51 +0200 Subject: [PATCH 17/17] Add support for Serial side data --- bmdcapture.cpp | 23 +++++++++++++++++++++-- bmdplay.cpp | 20 ++++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/bmdcapture.cpp b/bmdcapture.cpp index 86ebbad..69e5130 100644 --- a/bmdcapture.cpp +++ b/bmdcapture.cpp @@ -58,6 +58,7 @@ const char *g_audioOutputFile = NULL; static int g_maxFrames = -1; static int serial_fd = -1; static int wallclock = -1; +static int serial = 2; // Pending proper options static int draw_bars = 1; bool g_verbose = false; unsigned long long g_memoryLimit = 1024 * 1024 * 1024; // 1GByte(>50 sec) @@ -369,6 +370,22 @@ void wallclock_as_side_data(AVPacket *pkt) AV_WB64(wallclock, time); } +#define SERIAL_SIZE 7 +void serial_as_side_data(AVPacket *pkt) +{ + uint8_t *line = av_packet_new_side_data(pkt, AV_PKT_DATA_SERIAL, SERIAL_SIZE + 1); + int count; + + if (!line) + return; + + count = read(serial_fd, line, 7); + if (count > 0) + fprintf(stderr, "read %d bytes: %s \n", count, line); + else + line[0] = ' '; +} + // FIXME fail properly. void vanc_as_side_data(AVPacket *pkt, IDeckLinkVideoInputFrame *frame) @@ -512,6 +529,8 @@ void write_video_packet(IDeckLinkVideoInputFrame *videoFrame, if (wallclock == 0) wallclock_as_side_data(&pkt); + if (serial_fd > 0 && serial == 2) + serial_as_side_data(&pkt); avpacket_queue_put(&queue, &pkt); } @@ -541,7 +560,7 @@ HRESULT DeckLinkCaptureDelegate::VideoInputFrameArrived( write_video_packet(videoFrame, pts, frameDuration); - if (serial_fd > 0) { + if (serial_fd > 0 && serial == 1) { char line[8] = {0}; int count = read(serial_fd, line, 7); if (count > 0) @@ -829,7 +848,7 @@ int main(int argc, char *argv[]) } } - if (serial_fd > 0 && wallclock == 1) { + if (serial_fd > 0 && wallclock == 1 && serial == 1) { fprintf(stderr, "%s", "Wallclock and serial are not supported together\n" "Please disable either.\n"); diff --git a/bmdplay.cpp b/bmdplay.cpp index c11a799..81d0bf6 100644 --- a/bmdplay.cpp +++ b/bmdplay.cpp @@ -58,6 +58,7 @@ static enum PixelFormat pix_fmt = PIX_FMT_UYVY422; static BMDPixelFormat pix = bmdFormat8BitYUV; static int buffer = 2000 * 1000; +static int serial = 2; // for testing static int serial_fd = -1; static int verbose = 0; @@ -674,10 +675,12 @@ void Player::ScheduleNextFrame(bool prerolling) int side_data_size, linesize; uint8_t *side_data; - if (serial_fd > 0 && packet_queue_get(&dataqueue, &pkt, 0)) { - if (pkt.data[0] != ' '){ - fprintf(stderr,"written %.*s \n", pkt.size, pkt.data); - write(serial_fd, pkt.data, pkt.size); + if (packet_queue_get(&dataqueue, &pkt, 0)) { + if (serial_fd > 0 && serial == 1) { + if (pkt.data[0] != ' '){ + fprintf(stderr,"written %.*s \n", pkt.size, pkt.data); + write(serial_fd, pkt.data, pkt.size); + } } av_free_packet(&pkt); } @@ -711,6 +714,15 @@ void Player::ScheduleNextFrame(bool prerolling) av_gettime() - t); } + side_data = av_packet_get_side_data(&pkt, AV_PKT_DATA_SERIAL, + &side_data_size); + if (side_data) { + if (side_data[0] != ' '){ + fprintf(stderr,"written %.*s \n", side_data_size, side_data); + write(serial_fd, side_data, side_data_size); + } + } + if (pix == bmdFormat10BitYUV) { side_data = av_packet_get_side_data(&pkt, AV_PKT_DATA_VANC, &side_data_size);