Implement an FfmpegVideoRenderer

libraries/common/src/main/java/androidx/media3/common/MimeTypes.java

@@ -47,6 +47,7 @@ public final class MimeTypes {
   public static final String VIDEO_H264 = BASE_TYPE_VIDEO + "/avc";
   @UnstableApi public static final String VIDEO_APV = BASE_TYPE_VIDEO + "/apv";
   public static final String VIDEO_H265 = BASE_TYPE_VIDEO + "/hevc";
+  public static final String VIDEO_ProRes = BASE_TYPE_VIDEO + "/prores";
   @UnstableApi public static final String VIDEO_VP8 = BASE_TYPE_VIDEO + "/x-vnd.on2.vp8";
   @UnstableApi public static final String VIDEO_VP9 = BASE_TYPE_VIDEO + "/x-vnd.on2.vp9";
   public static final String VIDEO_AV1 = BASE_TYPE_VIDEO + "/av01";

libraries/container/src/main/java/androidx/media3/container/Mp4Box.java

@@ -482,6 +482,25 @@ public abstract class Mp4Box {
   @SuppressWarnings({"ConstantCaseForConstants", "IdentifierName"})
   public static final int TYPE_free = 0x66726565;
 
+  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_ap4x = 0x61703478;
+
+  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_ap4h = 0x61703468;
+
+  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_apch = 0x61706368;
+
+  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_apcn = 0x6170636e;
+
+  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_apcs = 0x61706373;
+
+  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_apco = 0x6170636f;
+
+
   public final int type;
 
   // private to only allow sub-classing from within this file.

libraries/decoder_ffmpeg/README.md

@@ -1,7 +1,7 @@
 # FFmpeg decoder module
 
-The FFmpeg module provides `FfmpegAudioRenderer`, which uses FFmpeg for decoding
-and can render audio encoded in a variety of formats.
+The FFmpeg module provides `FfmpegAudioRenderer` and `ExperimentalFfmpegVideoRenderer`, which uses FFmpeg for decoding
+and can render audio & video encoded in a variety of formats.
 
 ## License note
 
@@ -65,7 +65,7 @@ FFMPEG_PATH="$(pwd)"
   details of the available decoders, and which formats they support.
 
 ```
-ENABLED_DECODERS=(vorbis opus flac)
+ENABLED_DECODERS=(vorbis opus flac h264 hevc)
 ```
 
 *   Add a link to the FFmpeg source code in the FFmpeg module `jni` directory.
@@ -85,6 +85,34 @@ cd "${FFMPEG_MODULE_PATH}/jni" && \
   "${FFMPEG_MODULE_PATH}" "${NDK_PATH}" "${HOST_PLATFORM}" "${ANDROID_ABI}" "${ENABLED_DECODERS[@]}"
 ```
 
+
+Attempt to Rotate ``AVPixelFormat::AV_PIX_FMT_YUV420P`` & Copy the Pixels to ``ANativeWindow`` Buffer. The `libyuv` is also required. 
+
+* Fetch `libyuv` and checkout an appropriate branch:
+
+```
+cd "<preferred location for libyuv>" && \
+git clone https://chromium.googlesource.com/libyuv/libyuv && \
+YUV_PATH="$(pwd)"
+```
+
+*   Add a link to the `libyuv` source code in the `libyuv` module `jni` directory.
+
+```
+cd "${FFMPEG_MODULE_PATH}/jni" && \
+ln -s "$YUV_PATH" libyuv
+```
+
+* Execute `build_yuv.sh` to build libyuv for `armeabi-v7a`, `arm64-v8a`,
+  `x86` and `x86_64`. The script can be edited if you need to build for
+  different architectures:
+
+```
+cd "${FFMPEG_MODULE_PATH}/jni" && \
+./build_yuv.sh \
+  "${FFMPEG_MODULE_PATH}" "${NDK_PATH}" "${ANDROID_ABI}"
+```
+
 ## Build instructions (Windows)
 
 We do not provide support for building this module on Windows, however it should

libraries/decoder_ffmpeg/build.gradle

@@ -19,7 +19,7 @@ android {
 
 // Configure the native build only if ffmpeg is present to avoid gradle sync
 // failures if ffmpeg hasn't been built according to the README instructions.
-if (project.file('src/main/jni/ffmpeg').exists()) {
+if (project.file('src/main/jni/ffmpeg').exists() && project.file('src/main/jni/libyuv').exists()) {
     android.externalNativeBuild.cmake.path = 'src/main/jni/CMakeLists.txt'
     // LINT.IfChange
     // Should match cmake_minimum_required.
@@ -32,6 +32,7 @@ dependencies {
     // TODO(b/203752526): Remove this dependency.
     implementation project(modulePrefix + 'lib-exoplayer')
     implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion
+    implementation project(modulePrefix + 'lib-common')
     compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion
     compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion
     testImplementation project(modulePrefix + 'test-utils')

libraries/decoder_ffmpeg/proguard-rules.txt

@@ -9,3 +9,7 @@
 -keep, includedescriptorclasses class androidx.media3.decoder.ffmpeg.FfmpegAudioDecoder {
   private java.nio.ByteBuffer growOutputBuffer(androidx.media3.decoder.SimpleDecoderOutputBuffer, int);
 }
+
+-keep class androidx.media3.decoder.VideoDecoderOutputBuffer {
+  *;
+}

libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/ExperimentalFfmpegVideoDecoder.java

@@ -0,0 +1,257 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.decoder.ffmpeg;
+
+import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
+
+import android.view.Surface;
+
+import androidx.annotation.Nullable;
+import androidx.annotation.VisibleForTesting;
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.util.Assertions;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.common.util.Util;
+import androidx.media3.decoder.DecoderInputBuffer;
+import androidx.media3.decoder.SimpleDecoder;
+import androidx.media3.decoder.VideoDecoderOutputBuffer;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/**
+ * Ffmpeg Video decoder.
+ */
+@VisibleForTesting(otherwise = PACKAGE_PRIVATE)
+@UnstableApi
+/* package */ final class ExperimentalFfmpegVideoDecoder
+    extends SimpleDecoder<DecoderInputBuffer, VideoDecoderOutputBuffer, FfmpegDecoderException> {
+
+  private static final String TAG = "FfmpegVideoDecoder";
+
+  // LINT.IfChange
+  private static final int VIDEO_DECODER_SUCCESS = 0;
+  private static final int VIDEO_DECODER_ERROR_INVALID_DATA = -1;
+  private static final int VIDEO_DECODER_ERROR_OTHER = -2;
+  private static final int VIDEO_DECODER_ERROR_READ_FRAME = -3;
+  // LINT.ThenChange(../../../../../../../jni/ffmpeg_jni.cc)
+
+  private final String codecName;
+  private long nativeContext;
+  @Nullable
+  private final byte[] extraData;
+  @C.VideoOutputMode
+  private volatile int outputMode;
+
+  private int degree = 0;
+
+  /**
+   * Creates a Ffmpeg video Decoder.
+   *
+   * @param numInputBuffers        Number of input buffers.
+   * @param numOutputBuffers       Number of output buffers.
+   * @param initialInputBufferSize The initial size of each input buffer, in bytes.
+   * @param threads                Number of threads libffmpeg will use to decode.
+   * @throws FfmpegDecoderException Thrown if an exception occurs when initializing the decoder.
+   */
+  public ExperimentalFfmpegVideoDecoder(
+      int numInputBuffers, int numOutputBuffers, int initialInputBufferSize, int threads,
+      Format format)
+      throws FfmpegDecoderException {
+    super(
+        new DecoderInputBuffer[numInputBuffers],
+        new VideoDecoderOutputBuffer[numOutputBuffers]);
+    if (!FfmpegLibrary.isAvailable()) {
+      throw new FfmpegDecoderException("Failed to load decoder native library.");
+    }
+    codecName = Assertions.checkNotNull(FfmpegLibrary.getCodecName(format.sampleMimeType));
+    extraData = getExtraData(format.sampleMimeType, format.initializationData);
+    degree = format.rotationDegrees;
+    nativeContext = ffmpegInitialize(codecName, extraData, threads, degree, format.width, format.height);
+    if (nativeContext == 0) {
+      throw new FfmpegDecoderException("Failed to initialize decoder.");
+    }
+    setInitialInputBufferSize(initialInputBufferSize);
+  }
+
+  /**
+   * Returns FFmpeg-compatible codec-specific initialization data ("extra data"), or {@code null} if
+   * not required.
+   */
+  @Nullable
+  private static byte[] getExtraData(String mimeType, List<byte[]> initializationData) {
+    int size = 0;
+    for (int i = 0; i < initializationData.size(); i++) {
+      size += initializationData.get(i).length;
+    }
+    if (size > 0) {
+      byte[] extra = new byte[size];
+      ByteBuffer wrapper = ByteBuffer.wrap(extra);
+      for (int i = 0; i < initializationData.size(); i++) {
+        wrapper.put(initializationData.get(i));
+      }
+      return extra;
+    }
+    return null;
+  }
+
+  @Override
+  public String getName() {
+    return "ffmpeg" + FfmpegLibrary.getVersion() + "-" + codecName;
+  }
+
+  /**
+   * Sets the output mode for frames rendered by the decoder.
+   *
+   * @param outputMode The output mode.
+   */
+  public void setOutputMode(@C.VideoOutputMode int outputMode) {
+    this.outputMode = outputMode;
+  }
+
+  @Override
+  protected DecoderInputBuffer createInputBuffer() {
+    return new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
+  }
+
+  @Override
+  protected VideoDecoderOutputBuffer createOutputBuffer() {
+    return new VideoDecoderOutputBuffer(this::releaseOutputBuffer);
+  }
+
+  @Override
+  @Nullable
+  protected FfmpegDecoderException decode(
+      DecoderInputBuffer inputBuffer, VideoDecoderOutputBuffer outputBuffer, boolean reset) {
+    if (reset) {
+
+      nativeContext = ffmpegReset(nativeContext);
+      if (nativeContext == 0) {
+        return new FfmpegDecoderException("Error resetting (see logcat).");
+      }
+    }
+
+    // send packet
+    ByteBuffer inputData = Util.castNonNull(inputBuffer.data);
+    int inputSize = inputData.limit();
+    // enqueue origin data
+    int sendPacketResult = ffmpegSendPacket(nativeContext, inputData, inputSize,
+        inputBuffer.timeUs);
+
+    if (sendPacketResult == VIDEO_DECODER_ERROR_INVALID_DATA) {
+      outputBuffer.shouldBeSkipped = true;
+      return null;
+    } else if (sendPacketResult == VIDEO_DECODER_ERROR_READ_FRAME) {
+      // need read frame
+    } else if (sendPacketResult == VIDEO_DECODER_ERROR_OTHER) {
+      return new FfmpegDecoderException("ffmpegDecode error: (see logcat)");
+    }
+
+    // receive frame
+    boolean decodeOnly = !isAtLeastOutputStartTimeUs(inputBuffer.timeUs);
+    // We need to dequeue the decoded frame from the decoder even when the input data is
+    // decode-only.
+    if (!decodeOnly) {
+      outputBuffer.init(inputBuffer.timeUs, outputMode, null);
+    }
+    int getFrameResult = ffmpegReceiveFrame(nativeContext, outputMode, outputBuffer, decodeOnly);
+    if (getFrameResult == VIDEO_DECODER_ERROR_OTHER) {
+      return new FfmpegDecoderException("ffmpegDecode error: (see logcat)");
+    }
+
+    if (getFrameResult == VIDEO_DECODER_ERROR_INVALID_DATA) {
+      outputBuffer.shouldBeSkipped = true;
+    }
+
+    if (!decodeOnly) {
+      outputBuffer.format = inputBuffer.format;
+    }
+
+    return null;
+  }
+
+  @Override
+  protected FfmpegDecoderException createUnexpectedDecodeException(Throwable error) {
+    return new FfmpegDecoderException("Unexpected decode error", error);
+  }
+
+  @Override
+  public void release() {
+    super.release();
+    ffmpegRelease(nativeContext);
+    nativeContext = 0;
+  }
+
+  /**
+   * Renders output buffer to the given surface. Must only be called when in {@link
+   * C#VIDEO_OUTPUT_MODE_SURFACE_YUV} mode.
+   *
+   * @param outputBuffer Output buffer.
+   * @param surface      Output surface.
+   * @throws FfmpegDecoderException Thrown if called with invalid output mode or frame rendering
+   *                                fails.
+   */
+  public void renderToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface)
+      throws FfmpegDecoderException {
+    if (outputBuffer.mode != C.VIDEO_OUTPUT_MODE_SURFACE_YUV) {
+      throw new FfmpegDecoderException("Invalid output mode.");
+    }
+    int rst = ffmpegRenderFrame(nativeContext, surface, outputBuffer, outputBuffer.width,
+        outputBuffer.height);
+    if (rst == VIDEO_DECODER_ERROR_OTHER) {
+      throw new FfmpegDecoderException(
+          "Buffer render error: ");
+    }
+  }
+
+  private native long ffmpegInitialize(String codecName, @Nullable byte[] extraData, int threads,
+                                       int degree, int width, int height);
+
+  private native long ffmpegReset(long context);
+
+  private native void ffmpegRelease(long context);
+
+  private native int ffmpegRenderFrame(
+      long context, Surface surface, VideoDecoderOutputBuffer outputBuffer,
+      int displayedWidth,
+      int displayedHeight);
+
+  /**
+   * Decodes the encoded data passed.
+   *
+   * @param context     Decoder context.
+   * @param encodedData Encoded data.
+   * @param length      Length of the data buffer.
+   * @return {@link #VIDEO_DECODER_SUCCESS} if successful, {@link #VIDEO_DECODER_ERROR_OTHER} if an
+   * error occurred.
+   */
+  private native int ffmpegSendPacket(long context, ByteBuffer encodedData, int length,
+      long inputTime);
+
+  /**
+   * Gets the decoded frame.
+   *
+   * @param context      Decoder context.
+   * @param outputBuffer Output buffer for the decoded frame.
+   * @return {@link #VIDEO_DECODER_SUCCESS} if successful, {@link #VIDEO_DECODER_ERROR_INVALID_DATA}
+   * if successful but the frame is decode-only, {@link #VIDEO_DECODER_ERROR_OTHER} if an error
+   * occurred.
+   */
+  private native int ffmpegReceiveFrame(
+      long context, int outputMode, VideoDecoderOutputBuffer outputBuffer, boolean decodeOnly);
+
+}