Discussion:
[vlc-devel] [PATCH 0/4] Asynchronous decoder Queue functions and MediaCodec
Thomas Guillem
9 years ago
Permalink
* About Decoder:

Last time I proposed an asynchronous decoder patch (see
https://mailman.videolan.org/pipermail/vlc-devel/2015-September/104159.html),
there were few issues on the core:

The 2/ issue about changing fmt_in is fixed: fmt_in won't change during the
decoder module life.

The 1/ issue about fmt_out being the format of following buffer allocations, or
the format of following output buffers is still not fixed. This need to be
fixed but I don't think it's related to decoder_QueueVideo or MediaCodec.
Indeed, Contrary to avcodec-MT, MediaCodec won't keep a decoder picture
internally and will return it directly via decoder_QueueVideo (or was returned
via pf_decode_video before). Therefore, in MediaCodec case, if fmt_out changes,
only the vout will own the pictures that will be reseted.

There is also an issue with input_DecoderFrameNext, FrameNext won't work
properly when using decoder_QueueVideo. input_DecoderFrameNext need to be
rewritten completely to handle asynchronous decoder and input not feed (since
decoder is paused).

decoder_QueueVideo and decoder_QueueAudio will be used by MediaCodec (and
lately by omx and mmal?), but I don't know if decoder_QueueSub is very
useful. I don't know if there will be an asynchronous subtitle decoder one day.

* About MediaCodec:

The fourth commit improves performance on few devices (it depends on the number
of input/output buffers allocated by the OMX Codec) and allows to remove the
following hack in android:
http://git.videolan.org/gitweb.cgi/vlc-ports/android.git/?p=vlc-ports/android.git;a=blob;f=libvlc/src/org/videolan/libvlc/Media.java;h=90983af51a6979748b26b8419ffc0af2b7c40abe;hb=HEAD#l620

Thomas Guillem (4):
decoder: add decoder_QueueVideo
decoder: add decoder_QueueAudio
decoder: add decoder_QueueSub
mediacodec: add an output thread, remove polling

include/vlc_codec.h | 56 +++++
modules/codec/omxil/mediacodec.c | 526 ++++++++++++++++++++++++---------------
src/input/decoder.c | 239 ++++++++++++------
3 files changed, 541 insertions(+), 280 deletions(-)
--
2.1.4
Thomas Guillem
9 years ago
Permalink
This function allow asynchronous decoders to queue a picture to the video
output. Decoders that use this function should return NULL in pf_decode_video
callback.
---
include/vlc_codec.h | 20 ++++++++++++
src/input/decoder.c | 90 ++++++++++++++++++++++++++++++++++++-----------------
2 files changed, 81 insertions(+), 29 deletions(-)

diff --git a/include/vlc_codec.h b/include/vlc_codec.h
index 81090ba..b5170f7 100644
--- a/include/vlc_codec.h
+++ b/include/vlc_codec.h
@@ -121,6 +121,9 @@ struct decoder_t
* XXX use decoder_GetDisplayRate */
int (*pf_get_display_rate)( decoder_t * );

+ /* XXX use decoder_QueueVideo */
+ int (*pf_queue_video)( decoder_t *, picture_t * );
+
/* Private structure for the owner of the decoder */
decoder_owner_sys_t *p_owner;

@@ -239,6 +242,23 @@ static inline picture_t *decoder_NewPicture( decoder_t *dec )
}

/**
+ * This function queues a picture to the video output.
+ *
+ * \note
+ * The caller doesn't own the picture anymore after this call (even in case of
+ * error).
+ * FIXME: input_DecoderFrameNext won't work if a module use this function.
+ *
+ * \return 0 if the picture is queued, -1 on error
+ */
+static inline int decoder_QueueVideo( decoder_t *dec, picture_t *p_pic )
+{
+ if( !dec->pf_queue_video )
+ return -1;
+ return dec->pf_queue_video( dec, p_pic );
+}
+
+/**
* This function notifies the audio output pipeline of a new audio output
* format (fmt_out.audio). If there is currently no audio output or if the
* audio output format has changed, a new audio output will be set up.
diff --git a/src/input/decoder.c b/src/input/decoder.c
index c4fb6a1..9b04df3 100644
--- a/src/input/decoder.c
+++ b/src/input/decoder.c
@@ -888,45 +888,40 @@ static void DecoderPlayVideo( decoder_t *p_dec, picture_t *p_picture,
*pi_lost_sum += i_tmp_lost;
}

-static void DecoderDecodeVideo( decoder_t *p_dec, block_t *p_block )
+static int DecoderPreparePlayVideo( decoder_t *p_dec, picture_t *p_pic )
{
decoder_owner_sys_t *p_owner = p_dec->p_owner;
- picture_t *p_pic;
- int i_lost = 0;
- int i_decoded = 0;
- int i_displayed = 0;
+ vout_thread_t *p_vout = p_owner->p_vout;

- while( (p_pic = p_dec->pf_decode_video( p_dec, &p_block )) )
+ if( p_owner->i_preroll_end > VLC_TS_INVALID && p_pic->date < p_owner->i_preroll_end )
{
- vout_thread_t *p_vout = p_owner->p_vout;
-
- i_decoded++;
-
- if( p_owner->i_preroll_end > VLC_TS_INVALID && p_pic->date < p_owner->i_preroll_end )
- {
- picture_Release( p_pic );
- continue;
- }
+ picture_Release( p_pic );
+ return -1;
+ }

- if( p_owner->i_preroll_end > VLC_TS_INVALID )
- {
- msg_Dbg( p_dec, "End of video preroll" );
- if( p_vout )
- vout_Flush( p_vout, VLC_TS_INVALID+1 );
- /* */
- p_owner->i_preroll_end = VLC_TS_INVALID;
- }
+ if( p_owner->i_preroll_end > VLC_TS_INVALID )
+ {
+ msg_Dbg( p_dec, "End of video preroll" );
+ if( p_vout )
+ vout_Flush( p_vout, VLC_TS_INVALID+1 );
+ /* */
+ p_owner->i_preroll_end = VLC_TS_INVALID;
+ }

- if( p_dec->pf_get_cc &&
- ( !p_owner->p_packetizer || !p_owner->p_packetizer->pf_get_cc ) )
- DecoderGetCc( p_dec, p_dec );
+ if( p_dec->pf_get_cc &&
+ ( !p_owner->p_packetizer || !p_owner->p_packetizer->pf_get_cc ) )
+ DecoderGetCc( p_dec, p_dec );

- DecoderPlayVideo( p_dec, p_pic, &i_displayed, &i_lost );
- }
+ return 0;
+}

- /* Update ugly stat */
+static void DecoderUpdateStatVideo( decoder_t *p_dec, int i_decoded,
+ int i_lost, int i_displayed )
+{
+ decoder_owner_sys_t *p_owner = p_dec->p_owner;
input_thread_t *p_input = p_owner->p_input;

+ /* Update ugly stat */
if( p_input != NULL && (i_decoded > 0 || i_lost > 0 || i_displayed > 0) )
{
vlc_mutex_lock( &p_input->p->counters.counters_lock );
@@ -938,6 +933,42 @@ static void DecoderDecodeVideo( decoder_t *p_dec, block_t *p_block )
}
}

+static int DecoderQueueVideo( decoder_t *p_dec, picture_t *p_pic )
+{
+ assert( p_pic );
+ int i_lost = 0;
+ int i_displayed = 0;
+
+ if( DecoderPreparePlayVideo( p_dec, p_pic ) != 0 )
+ return -1;
+
+ DecoderPlayVideo( p_dec, p_pic, &i_displayed, &i_lost );
+
+ DecoderUpdateStatVideo( p_dec, 1, i_lost, i_displayed );
+
+ return 0;
+}
+
+static void DecoderDecodeVideo( decoder_t *p_dec, block_t *p_block )
+{
+ picture_t *p_pic;
+ int i_lost = 0;
+ int i_decoded = 0;
+ int i_displayed = 0;
+
+ while( (p_pic = p_dec->pf_decode_video( p_dec, &p_block )) )
+ {
+ i_decoded++;
+
+ if( DecoderPreparePlayVideo( p_dec, p_pic ) != 0 )
+ continue;
+
+ DecoderPlayVideo( p_dec, p_pic, &i_displayed, &i_lost );
+ }
+
+ DecoderUpdateStatVideo( p_dec, i_decoded, i_lost, i_displayed );
+}
+
/* This function process a video block
*/
static void DecoderProcessVideo( decoder_t *p_dec, block_t *p_block )
@@ -1509,6 +1540,7 @@ static decoder_t * CreateDecoder( vlc_object_t *p_parent,
p_dec->pf_get_attachments = DecoderGetInputAttachments;
p_dec->pf_get_display_date = DecoderGetDisplayDate;
p_dec->pf_get_display_rate = DecoderGetDisplayRate;
+ p_dec->pf_queue_video = DecoderQueueVideo;

/* Load a packetizer module if the input is not already packetized */
if( p_sout == NULL && !fmt->b_packetized )
--
2.1.4
Rémi Denis-Courmont
9 years ago
Permalink
...
That does not seem to match the \note just above.
...
I don´t follow how access to i_preroll_end is protected.
...
This seems to break stats (not that I care much about them).
...
--
Rémi Denis-Courmont
http://www.remlab.net/
Thomas Guillem
9 years ago
Permalink
...
That's what happen when you write a comment after coding
...
It was not, see next patch in the ML
...
Ah yes, good catch
...
Thanks for the review
Post by Rémi Denis-Courmont
--
Rémi Denis-Courmont
http://www.remlab.net/
_______________________________________________
vlc-devel mailing list
https://mailman.videolan.org/listinfo/vlc-devel
Thomas Guillem
9 years ago
Permalink
This function allow asynchronous decoders to queue an audio block to the audio
output. Decoders that use this function should return NULL in pf_decode_audio
callback.
---
include/vlc_codec.h | 18 ++++++++++++
src/input/decoder.c | 83 +++++++++++++++++++++++++++++++++++++----------------
2 files changed, 76 insertions(+), 25 deletions(-)

diff --git a/include/vlc_codec.h b/include/vlc_codec.h
index b5170f7..96fd1e9 100644
--- a/include/vlc_codec.h
+++ b/include/vlc_codec.h
@@ -123,6 +123,8 @@ struct decoder_t

/* XXX use decoder_QueueVideo */
int (*pf_queue_video)( decoder_t *, picture_t * );
+ /* XXX use decoder_QueueAudio */
+ int (*pf_queue_audio)( decoder_t *, block_t * );

/* Private structure for the owner of the decoder */
decoder_owner_sys_t *p_owner;
@@ -259,6 +261,22 @@ static inline int decoder_QueueVideo( decoder_t *dec, picture_t *p_pic )
}

/**
+ * This function queues an audio block to the audio output.
+ *
+ * \note
+ * The caller doesn't own the audio block anymore after this call (even in case
+ * of error).
+ *
+ * \return 0 if the block is queued, -1 on error
+ */
+static inline int decoder_QueueAudio( decoder_t *dec, block_t *p_aout_buf )
+{
+ if( !dec->pf_queue_audio )
+ return -1;
+ return dec->pf_queue_audio( dec, p_aout_buf );
+}
+
+/**
* This function notifies the audio output pipeline of a new audio output
* format (fmt_out.audio). If there is currently no audio output or if the
* audio output format has changed, a new audio output will be set up.
diff --git a/src/input/decoder.c b/src/input/decoder.c
index 9b04df3..ec4fefe 100644
--- a/src/input/decoder.c
+++ b/src/input/decoder.c
@@ -1069,40 +1069,36 @@ static void DecoderPlayAudio( decoder_t *p_dec, block_t *p_audio,
*pi_lost_sum += aout_DecGetResetLost( p_aout );
}

-static void DecoderDecodeAudio( decoder_t *p_dec, block_t *p_block )
+static int DecoderPreparePlayAudio( decoder_t *p_dec, block_t *p_aout_buf )
{
decoder_owner_sys_t *p_owner = p_dec->p_owner;
- block_t *p_aout_buf;
- int i_decoded = 0;
- int i_lost = 0;
- int i_played = 0;

- while( (p_aout_buf = p_dec->pf_decode_audio( p_dec, &p_block )) )
+ if( p_owner->i_preroll_end > VLC_TS_INVALID &&
+ p_aout_buf->i_pts < p_owner->i_preroll_end )
{
- i_decoded++;
+ block_Release( p_aout_buf );
+ return -1;
+ }

- if( p_owner->i_preroll_end > VLC_TS_INVALID &&
- p_aout_buf->i_pts < p_owner->i_preroll_end )
- {
- block_Release( p_aout_buf );
- continue;
- }
+ if( p_owner->i_preroll_end > VLC_TS_INVALID )
+ {
+ msg_Dbg( p_dec, "End of audio preroll" );
+ if( p_owner->p_aout )
+ aout_DecFlush( p_owner->p_aout, false );
+ /* */
+ p_owner->i_preroll_end = VLC_TS_INVALID;
+ }

- if( p_owner->i_preroll_end > VLC_TS_INVALID )
- {
- msg_Dbg( p_dec, "End of audio preroll" );
- if( p_owner->p_aout )
- aout_DecFlush( p_owner->p_aout, false );
- /* */
- p_owner->i_preroll_end = VLC_TS_INVALID;
- }
+ return 0;
+}

- DecoderPlayAudio( p_dec, p_aout_buf, &i_played, &i_lost );
- }
+static void DecoderUpdateStatAudio( decoder_t *p_dec, int i_decoded,
+ int i_lost, int i_played )
+{
+ decoder_owner_sys_t *p_owner = p_dec->p_owner;
+ input_thread_t *p_input = p_owner->p_input;

/* Update ugly stat */
- input_thread_t *p_input = p_owner->p_input;
-
if( p_input != NULL && (i_decoded > 0 || i_lost > 0 || i_played > 0) )
{
vlc_mutex_lock( &p_input->p->counters.counters_lock);
@@ -1113,6 +1109,42 @@ static void DecoderDecodeAudio( decoder_t *p_dec, block_t *p_block )
}
}

+static int DecoderQueueAudio( decoder_t *p_dec, block_t *p_aout_buf )
+{
+ assert( p_aout_buf );
+ int i_lost = 0;
+ int i_played = 0;
+
+ if( DecoderPreparePlayAudio( p_dec, p_aout_buf ) != 0 )
+ return -1;
+
+ DecoderPlayAudio( p_dec, p_aout_buf, &i_played, &i_lost );
+
+ DecoderUpdateStatAudio( p_dec, 1, i_lost, i_played );
+
+ return 0;
+}
+
+static void DecoderDecodeAudio( decoder_t *p_dec, block_t *p_block )
+{
+ block_t *p_aout_buf;
+ int i_decoded = 0;
+ int i_lost = 0;
+ int i_played = 0;
+
+ while( (p_aout_buf = p_dec->pf_decode_audio( p_dec, &p_block )) )
+ {
+ i_decoded++;
+
+ if( DecoderPreparePlayAudio( p_dec, p_aout_buf ) != 0 )
+ continue;
+
+ DecoderPlayAudio( p_dec, p_aout_buf, &i_played, &i_lost );
+ }
+
+ DecoderUpdateStatAudio( p_dec, i_decoded, i_lost, i_played );
+}
+
/* This function process a audio block
*/
static void DecoderProcessAudio( decoder_t *p_dec, block_t *p_block )
@@ -1541,6 +1573,7 @@ static decoder_t * CreateDecoder( vlc_object_t *p_parent,
p_dec->pf_get_display_date = DecoderGetDisplayDate;
p_dec->pf_get_display_rate = DecoderGetDisplayRate;
p_dec->pf_queue_video = DecoderQueueVideo;
+ p_dec->pf_queue_audio = DecoderQueueAudio;

/* Load a packetizer module if the input is not already packetized */
if( p_sout == NULL && !fmt->b_packetized )
--
2.1.4
Rémi Denis-Courmont
9 years ago
Permalink
Same as previous.
--
Rémi Denis-Courmont
http://www.remlab.net/
Thomas Guillem
9 years ago
Permalink
This function allow asynchronous decoders to queue a subtitle to the video
output. Decoders that use this function should return NULL in pf_decode_sub
callback.
---
include/vlc_codec.h | 18 +++++++++++++++
src/input/decoder.c | 66 ++++++++++++++++++++++++++++++-----------------------
2 files changed, 55 insertions(+), 29 deletions(-)

diff --git a/include/vlc_codec.h b/include/vlc_codec.h
index 96fd1e9..dbb9957 100644
--- a/include/vlc_codec.h
+++ b/include/vlc_codec.h
@@ -125,6 +125,8 @@ struct decoder_t
int (*pf_queue_video)( decoder_t *, picture_t * );
/* XXX use decoder_QueueAudio */
int (*pf_queue_audio)( decoder_t *, block_t * );
+ /* XXX use decoder_QueueSub */
+ int (*pf_queue_sub)( decoder_t *, subpicture_t *);

/* Private structure for the owner of the decoder */
decoder_owner_sys_t *p_owner;
@@ -277,6 +279,22 @@ static inline int decoder_QueueAudio( decoder_t *dec, block_t *p_aout_buf )
}

/**
+ * This function queues a subtitle to the video output.
+ *
+ * \note
+ * The caller doesn't own the subtitle anymore after this call (even in case of
+ * error).
+ *
+ * \return 0 if the subtitle is queued, -1 on error
+ */
+static inline int decoder_QueueSub( decoder_t *dec, subpicture_t *p_spu )
+{
+ if( !dec->pf_queue_sub )
+ return -1;
+ return dec->pf_queue_sub( dec, p_spu );
+}
+
+/**
* This function notifies the audio output pipeline of a new audio output
* format (fmt_out.audio). If there is currently no audio output or if the
* audio output format has changed, a new audio output will be set up.
diff --git a/src/input/decoder.c b/src/input/decoder.c
index ec4fefe..394fb3d 100644
--- a/src/input/decoder.c
+++ b/src/input/decoder.c
@@ -1229,47 +1229,54 @@ static void DecoderPlaySpu( decoder_t *p_dec, subpicture_t *p_subpic )
vout_PutSubpicture( p_vout, p_subpic );
}

-/* This function process a subtitle block
- */
-static void DecoderProcessSpu( decoder_t *p_dec, block_t *p_block )
+static int DecoderQueueSpu( decoder_t *p_dec, subpicture_t *p_spu )
{
+ assert( p_spu );
decoder_owner_sys_t *p_owner = p_dec->p_owner;
-
input_thread_t *p_input = p_owner->p_input;
- vout_thread_t *p_vout;
- subpicture_t *p_spu;

- while( (p_spu = p_dec->pf_decode_sub( p_dec, p_block ? &p_block : NULL ) ) )
+ if( p_input != NULL )
{
- if( p_input != NULL )
- {
- vlc_mutex_lock( &p_input->p->counters.counters_lock );
- stats_Update( p_input->p->counters.p_decoded_sub, 1, NULL );
- vlc_mutex_unlock( &p_input->p->counters.counters_lock );
- }
+ vlc_mutex_lock( &p_input->p->counters.counters_lock );
+ stats_Update( p_input->p->counters.p_decoded_sub, 1, NULL );
+ vlc_mutex_unlock( &p_input->p->counters.counters_lock );
+ }

- p_vout = input_resource_HoldVout( p_owner->p_resource );
- if( p_vout && p_owner->p_spu_vout == p_vout )
+ int i_ret = -1;
+ vout_thread_t *p_vout = input_resource_HoldVout( p_owner->p_resource );
+ if( p_vout && p_owner->p_spu_vout == p_vout )
+ {
+ /* Preroll does not work very well with subtitle */
+ if( p_spu->i_start > VLC_TS_INVALID &&
+ p_spu->i_start < p_owner->i_preroll_end &&
+ ( p_spu->i_stop <= VLC_TS_INVALID || p_spu->i_stop < p_owner->i_preroll_end ) )
{
- /* Preroll does not work very well with subtitle */
- if( p_spu->i_start > VLC_TS_INVALID &&
- p_spu->i_start < p_owner->i_preroll_end &&
- ( p_spu->i_stop <= VLC_TS_INVALID || p_spu->i_stop < p_owner->i_preroll_end ) )
- {
- subpicture_Delete( p_spu );
- }
- else
- {
- DecoderPlaySpu( p_dec, p_spu );
- }
+ subpicture_Delete( p_spu );
}
else
{
- subpicture_Delete( p_spu );
+ DecoderPlaySpu( p_dec, p_spu );
+ i_ret = 0;
}
- if( p_vout )
- vlc_object_release( p_vout );
}
+ else
+ {
+ subpicture_Delete( p_spu );
+ }
+ if( p_vout )
+ vlc_object_release( p_vout );
+ return i_ret;
+}
+
+/* This function process a subtitle block
+ */
+static void DecoderProcessSpu( decoder_t *p_dec, block_t *p_block )
+{
+ decoder_owner_sys_t *p_owner = p_dec->p_owner;
+ subpicture_t *p_spu;
+
+ while( (p_spu = p_dec->pf_decode_sub( p_dec, p_block ? &p_block : NULL ) ) )
+ DecoderQueueSpu( p_dec, p_spu );
}

/**
@@ -1574,6 +1581,7 @@ static decoder_t * CreateDecoder( vlc_object_t *p_parent,
p_dec->pf_get_display_rate = DecoderGetDisplayRate;
p_dec->pf_queue_video = DecoderQueueVideo;
p_dec->pf_queue_audio = DecoderQueueAudio;
+ p_dec->pf_queue_sub = DecoderQueueSpu;

/* Load a packetizer module if the input is not already packetized */
if( p_sout == NULL && !fmt->b_packetized )
--
2.1.4
Rémi Denis-Courmont
9 years ago
Permalink
Same as 1.
--
Rémi Denis-Courmont
http://www.remlab.net/
Thomas Guillem
9 years ago
Permalink
This commit improve MediaCodec performances and remove polling every 10 msecs
when DecodeCommon is waiting for an input or an output buffer.

One new thread is used for output: once an input buffer is queued, this thread
will wait indefinitely for a MediaCodec output buffer (this can be cancelled by
a flush). This buffer will be queued to the video or audio output via
decoder_QueueVideo or decoder_QueueAudio.

This thread, and pf_decode call are locked by the same mutex. Only
mc->dequeue_in/mc->dequeue_out are not locked since these functions can block.

Fixes #15079
---
modules/codec/omxil/mediacodec.c | 526 ++++++++++++++++++++++++---------------
1 file changed, 329 insertions(+), 197 deletions(-)

diff --git a/modules/codec/omxil/mediacodec.c b/modules/codec/omxil/mediacodec.c
index 4507b65..0566f5d 100644
--- a/modules/codec/omxil/mediacodec.c
+++ b/modules/codec/omxil/mediacodec.c
@@ -83,7 +83,7 @@ struct decoder_sys_t
{
mc_api *api;

- /* Codec Specific Data buffer: sent in PutInput after a start or a flush
+ /* Codec Specific Data buffer: sent in DecodeCommon after a start or a flush
* with the BUFFER_FLAG_CODEC_CONFIG flag.*/
block_t **pp_csd;
size_t i_csd_count;
@@ -92,9 +92,6 @@ struct decoder_sys_t
bool b_update_format;
bool b_has_format;

- bool decoded;
- bool error_state;
- bool b_new_block;
int64_t i_preroll_end;
int i_quirks;

@@ -103,6 +100,17 @@ struct decoder_sys_t
dec_on_flush_cb pf_on_flush;
dec_process_output_cb pf_process_output;

+ vlc_mutex_t lock;
+ vlc_cond_t cond;
+ vlc_cond_t dec_cond;
+ vlc_thread_t out_thread;
+ bool b_out_thread_running;
+ bool b_flush_out;
+ bool b_output_ready;
+ bool b_mc_running;
+ bool b_error;
+ bool b_error_signaled; // TODO remove
+
union
{
struct
@@ -145,6 +153,10 @@ static void Audio_OnFlush(decoder_t *);
static int Audio_ProcessOutput(decoder_t *, mc_api_out *, picture_t **, block_t **);
static block_t *DecodeAudio(decoder_t *, block_t **);

+static int DecodeFlush(decoder_t *);
+static void StopMediaCodec(decoder_t *);
+static void *OutThread(void *);
+
static void InvalidateAllPictures(decoder_t *);
static void RemoveInflightPictures(decoder_t *);

@@ -179,7 +191,6 @@ vlc_module_begin ()
add_shortcut( "mediacodec_jni" )
vlc_module_end ()

-
static void CSDFree(decoder_t *p_dec)
{
decoder_sys_t *p_sys = p_dec->p_sys;
@@ -194,7 +205,7 @@ static void CSDFree(decoder_t *p_dec)
p_sys->i_csd_count = 0;
}

-/* Create the p_sys->p_csd that will be sent via PutInput */
+/* Create the p_sys->p_csd that will be sent from DecodeCommon */
static int CSDDup(decoder_t *p_dec, const struct csd *p_csd, size_t i_count)
{
decoder_sys_t *p_sys = p_dec->p_sys;
@@ -468,6 +479,7 @@ static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
decoder_sys_t *p_sys;
mc_api *api;
const char *mime = NULL;
+ bool b_late_opening = false;

/* Video or Audio if "mediacodec-audio" bool is true */
if (p_dec->fmt_in.i_cat != VIDEO_ES && (p_dec->fmt_in.i_cat != AUDIO_ES
@@ -557,9 +569,13 @@ static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
p_dec->fmt_out.video = p_dec->fmt_in.video;
p_dec->fmt_out.audio = p_dec->fmt_in.audio;
- p_sys->b_new_block = true;
p_sys->api->psz_mime = mime;

+ vlc_mutex_init(&p_sys->lock);
+ vlc_cond_init(&p_sys->cond);
+ vlc_cond_init(&p_sys->dec_cond);
+ vlc_cond_init(&p_sys->dec_cond);
+
if (p_dec->fmt_in.i_cat == VIDEO_ES)
{
p_sys->pf_on_new_block = Video_OnNewBlock;
@@ -594,7 +610,7 @@ static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
{
msg_Warn(p_dec, "waiting for a valid video size for codec %4.4s",
(const char *)&p_dec->fmt_in.i_codec);
- return VLC_SUCCESS;
+ b_late_opening = true;
}
}
else
@@ -617,7 +633,7 @@ static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
&& !p_sys->u.audio.i_channels)
{
msg_Warn(p_dec, "waiting for valid channel count");
- return VLC_SUCCESS;
+ b_late_opening = true;
}
}
if ((p_sys->i_quirks & OMXCODEC_QUIRKS_NEED_CSD)
@@ -630,13 +646,31 @@ static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
msg_Warn(p_dec, "late opening with MPEG4 not handled"); /* TODO */
goto bailout;
}
- return VLC_SUCCESS;
+ b_late_opening = true;
}

- if (StartMediaCodec(p_dec) == VLC_SUCCESS)
- return VLC_SUCCESS;
- else
+ vlc_mutex_lock(&p_sys->lock);
+
+ if (!b_late_opening && StartMediaCodec(p_dec) != VLC_SUCCESS)
+ {
msg_Err(p_dec, "StartMediaCodec failed");
+ vlc_mutex_unlock(&p_sys->lock);
+ goto bailout;
+ }
+
+ p_sys->b_out_thread_running = true;
+ if (vlc_clone(&p_sys->out_thread, OutThread, p_dec,
+ VLC_THREAD_PRIORITY_LOW))
+ {
+ msg_Err(p_dec, "vlc_clone failed");
+ p_sys->b_out_thread_running = false;
+ vlc_mutex_unlock(&p_sys->lock);
+ goto bailout;
+ }
+ vlc_mutex_unlock(&p_sys->lock);
+
+ return VLC_SUCCESS;
+
bailout:
CloseDecoder(p_this);
return VLC_EGENERIC;
@@ -662,6 +696,21 @@ static void CloseDecoder(vlc_object_t *p_this)

if (!p_sys)
return;
+ vlc_mutex_lock(&p_sys->lock);
+ if (p_sys->b_out_thread_running)
+ {
+ p_sys->b_out_thread_running = false;
+ DecodeFlush(p_dec);
+ vlc_cond_broadcast(&p_sys->cond);
+ vlc_mutex_unlock(&p_sys->lock);
+ vlc_join(p_sys->out_thread, NULL);
+ }
+ else
+ vlc_mutex_unlock(&p_sys->lock);
+
+ vlc_mutex_destroy(&p_sys->lock);
+ vlc_cond_destroy(&p_sys->cond);
+ vlc_cond_destroy(&p_sys->dec_cond);

StopMediaCodec(p_dec);

@@ -731,8 +780,8 @@ static int Video_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
picture_t **pp_out_pic, block_t **pp_out_block)
{
decoder_sys_t *p_sys = p_dec->p_sys;
-
- assert(pp_out_pic && !pp_out_block);
+ (void) pp_out_block;
+ assert(pp_out_pic);

if (p_out->type == MC_OUT_TYPE_BUF)
{
@@ -880,8 +929,8 @@ static int Audio_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
picture_t **pp_out_pic, block_t **pp_out_block)
{
decoder_sys_t *p_sys = p_dec->p_sys;
-
- assert(!pp_out_pic && pp_out_block);
+ (void) pp_out_pic;
+ assert(pp_out_block);

if (p_out->type == MC_OUT_TYPE_BUF)
{
@@ -1011,227 +1060,316 @@ static void HEVCProcessBlock(decoder_t *p_dec, block_t *p_block,
static int DecodeFlush(decoder_t *p_dec)
{
decoder_sys_t *p_sys = p_dec->p_sys;
+ bool b_was_running = p_sys->b_mc_running;

- if (p_sys->decoded || p_sys->i_csd_send > 0)
- {
- p_sys->pf_on_flush(p_dec);
+ p_sys->b_mc_running = false;
+ p_sys->b_flush_out = true;
+ p_sys->i_preroll_end = 0;
+ p_sys->b_output_ready = false;
+ /* Resend CODEC_CONFIG buffer after a flush */
+ p_sys->i_csd_send = 0;
+
+ p_sys->pf_on_flush(p_dec);
+
+ if (b_was_running && p_sys->api->flush(p_sys->api) != VLC_SUCCESS)
+ return VLC_EGENERIC;
+
+ vlc_cond_broadcast(&p_sys->cond);
+
+ while (!p_sys->b_error && p_sys->b_flush_out)
+ vlc_cond_wait(&p_sys->dec_cond, &p_sys->lock);
+
+ if (p_sys->b_error)
+ return VLC_EGENERIC;

- p_sys->i_preroll_end = 0;
- if (p_sys->api->flush(p_sys->api) != VLC_SUCCESS)
- return VLC_EGENERIC;
- /* resend CODEC_CONFIG buffer after a flush */
- p_sys->i_csd_send = 0;
- }
- p_sys->decoded = false;
return VLC_SUCCESS;
}

-static int GetAndProcessOutput(decoder_t *p_dec, picture_t **pp_out_pic,
- block_t **pp_out_block, mtime_t i_timeout)
+static void *OutThread(void *data)
{
+ decoder_t *p_dec = data;
decoder_sys_t *p_sys = p_dec->p_sys;
- struct mc_api_out out;
- int i_index, i_ret;
-
- i_index = p_sys->api->dequeue_out(p_sys->api, i_timeout);
- if (i_index >= 0 || i_index == MC_API_INFO_OUTPUT_FORMAT_CHANGED
- || i_index == MC_API_INFO_OUTPUT_BUFFERS_CHANGED)
- i_ret = p_sys->api->get_out(p_sys->api, i_index, &out);
- else if (i_index == MC_API_INFO_TRYAGAIN)
- i_ret = 0;
- else
- i_ret = -1;

- if (i_ret != 1)
- return i_ret;
+ for (;;)
+ {
+ int i_index;
+
+ vlc_mutex_lock(&p_sys->lock);
+
+ /* Wait for output ready */
+ while (!p_sys->b_error && p_sys->b_out_thread_running
+ && !p_sys->b_flush_out && !p_sys->b_output_ready)
+ vlc_cond_wait(&p_sys->cond, &p_sys->lock);
+
+ if (p_sys->b_flush_out)
+ {
+ /* Acknowledge flushed state */
+ p_sys->b_flush_out = false;
+ vlc_cond_broadcast(&p_sys->dec_cond);
+ goto next;
+ }
+
+ /* Check if thread is not stopped */
+ if (!p_sys->b_out_thread_running || p_sys->b_error)
+ goto end;
+
+ vlc_mutex_unlock(&p_sys->lock);
+ /* Wait for an output buffer. This function returns when a new output
+ * is available or if output is flushed. */
+ i_index = p_sys->api->dequeue_out(p_sys->api, -1);
+ vlc_mutex_lock(&p_sys->lock);
+
+ /* Ignore dequeue_out errors caused by flush */
+ if (p_sys->b_flush_out)
+ {
+ /* If i_index >= 0, Release it. There is no way to know if i_index
+ * is owned by us, so don't check the error. */
+ if (i_index >= 0)
+ p_sys->api->release_out(p_sys->api, i_index, false);
+
+ /* Parse output format/buffers even when we are flushing */
+ if (i_index != MC_API_INFO_OUTPUT_FORMAT_CHANGED
+ && i_index != MC_API_INFO_OUTPUT_BUFFERS_CHANGED)
+ goto next;
+ }
+
+ if (i_index >= 0 || i_index == MC_API_INFO_OUTPUT_FORMAT_CHANGED
+ || i_index == MC_API_INFO_OUTPUT_BUFFERS_CHANGED)
+ {
+ struct mc_api_out out;
+ int i_ret = p_sys->api->get_out(p_sys->api, i_index, &out);
+
+ if (i_ret == 1)
+ {
+ picture_t *p_pic = NULL;
+ block_t *p_block = NULL;
+
+ if (p_sys->pf_process_output(p_dec, &out, &p_pic,
+ &p_block) == -1)
+ {
+ msg_Err(p_dec, "pf_process_output failed");
+ goto end;
+ }
+ if (p_pic)
+ decoder_QueueVideo(p_dec, p_pic);
+ else if (p_block)
+ decoder_QueueAudio(p_dec, p_block);
+ } else if (i_ret != 0)
+ {
+ msg_Err(p_dec, "get_out failed");
+ goto end;
+ }
+ }
+ else
+ {
+ msg_Err(p_dec, "dequeue_out failed");
+ goto end;
+ }
+next:
+ vlc_mutex_unlock(&p_sys->lock);
+ continue;
+end:
+ msg_Warn(p_dec, "OutThread stopped");
+ p_sys->b_error = true;
+ vlc_cond_signal(&p_sys->dec_cond);
+ vlc_mutex_unlock(&p_sys->lock);
+ break;
+ }
+
+ return NULL;
+}

- return p_sys->pf_process_output(p_dec, &out, pp_out_pic,
- pp_out_block);
+static block_t *GetNextBlock(decoder_sys_t *p_sys, block_t *p_block)
+{
+ if (p_sys->i_csd_send < p_sys->i_csd_count)
+ return p_sys->pp_csd[p_sys->i_csd_send++];
+ else
+ return p_block;
}

/**
* DecodeCommon called from DecodeVideo or DecodeAudio.
- * It returns -1 in case of error, 0 otherwise. The output buffer is returned
- * in pp_out_pic for Video, and pp_out_block for Audio.
+ * It returns -1 in case of error, 0 otherwise.
*/
-static int DecodeCommon(decoder_t *p_dec, block_t **pp_block,
- picture_t **pp_out_pic, block_t **pp_out_block)
+static int DecodeCommon(decoder_t *p_dec, block_t **pp_block)
{
decoder_sys_t *p_sys = p_dec->p_sys;
- block_t *p_block = pp_block ? *pp_block : NULL;
- unsigned int i_attempts = 0;
- jlong timeout = 0;
- int i_output_ret = 0;
- int i_input_ret = 0;
- bool b_abort = false;
- bool b_error = false;
- bool b_new_block = p_block ? p_sys->b_new_block : false;
-
- if (p_sys->error_state)
- goto endclean;
-
- if (b_new_block)
- {
- int i_ret, i_flags = 0;
+ int i_flags = 0;
+ int i_ret;
+ bool b_dequeue_timeout = false;
+ block_t *p_block;

- p_sys->b_new_block = false;
+ if (!pp_block || !*pp_block)
+ return 0;

- if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED))
- {
- if (DecodeFlush(p_dec) != VLC_SUCCESS)
- b_error = true;
- goto endclean;
- }
+ vlc_mutex_lock(&p_sys->lock);
+
+ if (p_sys->b_error)
+ goto end;

- i_ret = p_sys->pf_on_new_block(p_dec, p_block, &i_flags);
- if (i_ret != 1)
+ p_block = *pp_block;
+
+ /* Flush */
+ if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|
+ BLOCK_FLAG_CORRUPTED))
+ {
+ if (DecodeFlush(p_dec) != VLC_SUCCESS)
{
- if (i_ret == -1)
- b_error = true;
- goto endclean;
+ msg_Err(p_dec, "DecodeFlush failed");
+ p_sys->b_error = true;
}
- if (i_flags & NEWBLOCK_FLAG_FLUSH)
+ goto end;
+ }
+
+ /* Parse input block */
+ if ((i_ret = p_sys->pf_on_new_block(p_dec, p_block, &i_flags)) == 1)
+ {
+ if (i_flags & (NEWBLOCK_FLAG_FLUSH|NEWBLOCK_FLAG_RESTART))
{
+ msg_Warn(p_dec, "Flushing from DecodeCommon");
+
+ /* Flush before restart to unblock OutThread */
if (DecodeFlush(p_dec) != VLC_SUCCESS)
{
- b_error = true;
- goto endclean;
+ p_sys->b_error = true;
+ goto end;
}
- }

- if (i_flags & NEWBLOCK_FLAG_RESTART)
- {
- StopMediaCodec(p_dec);
- if (StartMediaCodec(p_dec) != VLC_SUCCESS)
+ if (i_flags & NEWBLOCK_FLAG_RESTART)
{
- b_error = true;
- goto endclean;
+ msg_Warn(p_dec, "Restarting from DecodeCommon");
+ StopMediaCodec(p_dec);
+ if (StartMediaCodec(p_dec) != VLC_SUCCESS)
+ {
+ msg_Err(p_dec, "StartMediaCodec failed");
+ p_sys->b_error = true;
+ goto end;
+ }
}
}
}
+ else
+ {
+ if (i_ret != 0)
+ {
+ p_sys->b_error = true;
+ msg_Err(p_dec, "pf_on_new_block failed");
+ }
+ goto end;
+ }
+
+ /* Abort if MediaCodec is not yet started */
if (!p_sys->api->b_started)
- goto endclean;
+ goto end;

- do
+ /* Queue CSD blocks and input blocks */
+ while ((p_block = GetNextBlock(p_sys, *pp_block)))
{
- if ((p_sys->i_csd_send < p_sys->i_csd_count || p_block)
- && i_input_ret == 0)
+ int i_index;
+
+ vlc_mutex_unlock(&p_sys->lock);
+ /* Wait for an input buffer. This function returns when a new input
+ * buffer is available or after 1sec of timeout. */
+ i_index = p_sys->api->dequeue_in(p_sys->api,
+ p_sys->api->b_direct_rendering ?
+ INT64_C(1000000) : -1);
+ vlc_mutex_lock(&p_sys->lock);
+
+ if (p_sys->b_error)
+ goto end;
+
+ if (i_index >= 0)
{
- int i_index = p_sys->api->dequeue_in(p_sys->api, timeout);
+ bool b_config = p_block && (p_block->i_flags & BLOCK_FLAG_CSD);
+ mtime_t i_ts = 0;
+ p_sys->b_mc_running = true;

- if (i_index >= 0)
+ if (!b_config)
{
- block_t *p_in_block;
- mtime_t i_ts;
+ i_ts = p_block->i_pts;
+ if (!i_ts && p_block->i_dts)
+ i_ts = p_block->i_dts;
+ }

- if (p_sys->i_csd_send < p_sys->i_csd_count)
- {
- p_in_block = p_sys->pp_csd[p_sys->i_csd_send];
- i_ts = 0;
- }
- else
- {
- p_in_block = p_block;
- i_ts = p_block->i_pts;
- if (!i_ts && p_block->i_dts)
- i_ts = p_block->i_dts;
- }
- i_input_ret = p_sys->api->queue_in(p_sys->api, i_index,
- p_in_block->p_buffer,
- p_in_block->i_buffer, i_ts,
- p_in_block->i_flags & BLOCK_FLAG_CSD) == 0 ? 1 : -1;
- if (i_input_ret == 1)
+ if (p_sys->api->queue_in(p_sys->api, i_index,
+ p_block->p_buffer, p_block->i_buffer, i_ts,
+ b_config) == 0)
+ {
+ if (!b_config)
{
- if (p_sys->i_csd_send < p_sys->i_csd_count)
- {
- p_sys->i_csd_send++;
- i_input_ret = 0;
- }
- else
- {
- p_sys->decoded = true;
- if (p_block->i_flags & BLOCK_FLAG_PREROLL )
- p_sys->i_preroll_end = i_ts;
- }
+ if (p_block->i_flags & BLOCK_FLAG_PREROLL )
+ p_sys->i_preroll_end = i_ts;
+
+ /* One input buffer is queued, unblock OutThread that will
+ * fetch output buffers */
+ p_sys->b_output_ready = true;
+ vlc_cond_broadcast(&p_sys->cond);
+
+ assert(p_block == *pp_block),
+ block_Release(p_block);
+ *pp_block = NULL;
}
+ b_dequeue_timeout = false;
+ } else
+ {
+ msg_Err(p_dec, "queue_in failed");
+ p_sys->b_error = true;
+ goto end;
}
- else if (i_index == MC_API_INFO_TRYAGAIN)
- i_input_ret = 0;
- else
- i_input_ret = -1;
-
- /* No need to try output if no input buffer is decoded */
- if (!p_sys->decoded)
- continue;
}
-
- if (i_input_ret != -1 && p_sys->decoded && i_output_ret == 0)
+ else if (i_index == MC_API_INFO_TRYAGAIN)
{
- i_output_ret = GetAndProcessOutput(p_dec, pp_out_pic, pp_out_block,
- timeout);
-
- if (i_output_ret == 0 && i_input_ret == 0)
+ /* HACK: When direct rendering is enabled, there is a possible
+ * deadlock between the Decoder and the Vout. It happens when the
+ * Vout is paused and when the Decoder is flushing. In that case,
+ * the Vout won't release any output buffers, therefore MediaCodec
+ * won't dequeue any input buffers. To work around this issue,
+ * release all output buffers if DecodeCommon is waiting more than
+ * 1sec for a new input buffer. */
+ if (!b_dequeue_timeout)
{
- if (++i_attempts == 20)
- {
- /* HACK: When direct rendering is enabled, there is a
- * possible deadlock between the Decoder and the Vout. It
- * happens when the Vout is paused and when the Decoder is
- * flushing. In that case, the Vout won't release any
- * output buffers, therefore MediaCodec won't dequeue any
- * input buffers. To work around this issue, release all
- * output buffers if DecodeCommon is waiting more than 400
- * msec for a new input buffer. */
- msg_Warn(p_dec, "Decoder stuck: invalidate all buffers");
- InvalidateAllPictures(p_dec);
- }
- if (!p_sys->b_has_format && ++i_attempts > 100)
- {
- /* No output and no format, thereforce mediacodec didn't
- * produce any output or events yet. Don't wait
- * indefinitely and abort after 2seconds (100 * 2 * 10ms)
- * without any data. Indeed, MediaCodec can fail without
- * throwing any exception or error returns... */
- msg_Err(p_dec, "No output/input for %lld us, abort",
- i_attempts * timeout);
- b_error = true;
- break;
- }
+ msg_Warn(p_dec, "Decoder stuck: invalidate all buffers");
+ InvalidateAllPictures(p_dec);
+ b_dequeue_timeout = true;
+ continue;
+ }
+ else
+ {
+ msg_Err(p_dec, "dequeue_in timeout: no input available for 2secs");
+ p_sys->b_error = true;
+ goto end;
}
}
- timeout = 10 * 1000; // 10 ms
- /* loop until either the input or the output are processed (i_input_ret
- * or i_output_ret == 1 ) or caused an error (i_input_ret or
- * i_output_ret == -1 )*/
- } while (p_block && i_input_ret == 0 && i_output_ret == 0 && !b_abort);
-
- if (i_input_ret == -1 || i_output_ret == -1)
- {
- msg_Err(p_dec, "%s failed",
- i_input_ret == -1 ? "PutInput" : "GetOutput");
- b_error = true;
+ else
+ {
+ msg_Err(p_dec, "dequeue_in failed");
+ p_sys->b_error = true;
+ goto end;
+ }
}

-endclean:
-
- /* If pf_decode returns NULL, we'll get a new p_block from the next
- * pf_decode call. Therefore we need to release the current one even if we
- * couldn't process it (it happens in case or error or if MediaCodec is
- * still not opened). We also must release the current p_block if we were
- * able to process it. */
- if (p_block && (i_output_ret != 1 || i_input_ret != 0))
+end:
+ if (pp_block && *pp_block)
{
- block_Release(p_block);
+ block_Release(*pp_block);
*pp_block = NULL;
- p_sys->b_new_block = true;
}
- if (b_error && !p_sys->error_state) {
- /* Signal the error to the Java. */
- jni_EventHardwareAccelerationError();
- p_sys->error_state = true;
+ if (p_sys->b_error)
+ {
+ if (!p_sys->b_error_signaled) {
+ /* Signal the error to the Java. */
+ jni_EventHardwareAccelerationError();
+ p_sys->b_error_signaled = true;
+ vlc_cond_broadcast(&p_sys->cond);
+ }
+ vlc_mutex_unlock(&p_sys->lock);
+ return -1;
+ }
+ else
+ {
+ vlc_mutex_unlock(&p_sys->lock);
+ return 0;
}
-
- return b_error ? -1 : 0;
}

static int Video_OnNewBlock(decoder_t *p_dec, block_t *p_block, int *p_flags)
@@ -1298,11 +1436,8 @@ static void Video_OnFlush(decoder_t *p_dec)

static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
{
- picture_t *p_out = NULL;
-
- if (DecodeCommon(p_dec, pp_block, &p_out, NULL))
- return NULL;
- return p_out;
+ DecodeCommon(p_dec, pp_block);
+ return NULL;
}

static int Audio_OnNewBlock(decoder_t *p_dec, block_t *p_block, int *p_flags)
@@ -1346,9 +1481,6 @@ static void Audio_OnFlush(decoder_t *p_dec)

static block_t *DecodeAudio(decoder_t *p_dec, block_t **pp_block)
{
- block_t *p_out = NULL;
-
- if (DecodeCommon(p_dec, pp_block, NULL, &p_out))
- return NULL;
- return p_out;
+ DecodeCommon(p_dec, pp_block);
+ return NULL;
}
--
2.1.4
Loading...