我正在为一个支持流媒体和录制视频的应用程序编写一个类。简而言之，当手机在流媒体和录制时，音频保存在 PCM 文件中，视频保存在 mp4 文件中，使用 MediaRecorder。我的目标是，当录制完成时，使用 MediaMuxer 并将两个输入组合成一个新的组合 .mp4 文件。
我尝试使用 MediaMuxer 对音频进行编码并使用 MediaExtractor 提取视频。原始视频和音频文件都完好无损，输出文件包含正确的音频，但视频似乎已损坏，就好像跳帧一样。
这是我目前正在使用的代码：
public class StreamRecordingMuxer {
private static final String TAG = StreamRecordingMuxer.class.getSimpleName();
private static final String COMPRESSED_AUDIO_FILE_MIME_TYPE = "audio/mp4a-latm";
private static final int CODEC_TIMEOUT = 5000;
private int bitrate;
private int sampleRate;
private int channelCount;
// Audio state
private MediaFormat audioFormat;
private MediaCodec mediaCodec;
private MediaMuxer mediaMuxer;
private ByteBuffer[] codecInputBuffers;
private ByteBuffer[] codecOutputBuffers;
private MediaCodec.BufferInfo audioBufferInfo;
private String outputPath;
private int audioTrackId;
private int totalBytesRead;
private double presentationTimeUs;
// Video state
private int videoTrackId;
private MediaExtractor videoExtractor;
private MediaFormat videoFormat;
private String videoPath;
private int videoTrackIndex;
private int frameMaxInputSize;
private int rotationDegrees;
public StreamRecordingMuxer(final int bitrate, final int sampleRate, int channelCount) {
this.bitrate = bitrate;
this.sampleRate = sampleRate;
this.channelCount = channelCount;
}
public void setOutputPath(final String outputPath) {
this.outputPath = outputPath;
}
public void setVideoPath(String videoPath) {
this.videoPath = videoPath;
}
public void prepare() {
if (outputPath == null) {
throw new IllegalStateException("The output path must be set first!");
}
try {
audioFormat = MediaFormat.createAudioFormat(COMPRESSED_AUDIO_FILE_MIME_TYPE, sampleRate, channelCount);
audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
if (videoPath != null) {
videoExtractor = new MediaExtractor();
videoExtractor.setDataSource(videoPath);
videoFormat = findVideoFormat(videoExtractor);
}
mediaCodec = MediaCodec.createEncoderByType(COMPRESSED_AUDIO_FILE_MIME_TYPE);
mediaCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mediaCodec.start();
codecInputBuffers = mediaCodec.getInputBuffers();
codecOutputBuffers = mediaCodec.getOutputBuffers();
audioBufferInfo = new MediaCodec.BufferInfo();
mediaMuxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
if (videoPath != null) {
videoTrackId = mediaMuxer.addTrack(videoFormat);
mediaMuxer.setOrientationHint(rotationDegrees);
}
totalBytesRead = 0;
presentationTimeUs = 0;
} catch (IOException e) {
Log.e(TAG, "Exception while initializing StreamRecordingMuxer", e);
}
}
public void stop() {
Log.d(TAG, "Stopping StreamRecordingMuxer");
handleEndOfStream();
mediaCodec.stop();
mediaCodec.release();
mediaMuxer.stop();
mediaMuxer.release();
if (videoExtractor != null) {
videoExtractor.release();
}
}
private void handleEndOfStream() {
int inputBufferIndex = mediaCodec.dequeueInputBuffer(CODEC_TIMEOUT);
mediaCodec.queueInputBuffer(inputBufferIndex, 0, 0, (long) presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
writeAudioOutputs();
}
private MediaFormat findVideoFormat(MediaExtractor extractor) {
MediaFormat videoFormat;
int videoTrackCount = extractor.getTrackCount();
for (int i = 0; i < videoTrackCount; i++) {
videoFormat = extractor.getTrackFormat(i);
Log.d(TAG, "Video Format " + videoFormat.toString());
String mimeType = videoFormat.getString(MediaFormat.KEY_MIME);
if (mimeType.startsWith("video/")) {
videoTrackIndex = i;
frameMaxInputSize = videoFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE);
rotationDegrees = videoFormat.getInteger(MediaFormat.KEY_ROTATION);
// frameRate = videoFormat.getInteger(MediaFormat.KEY_FRAME_RATE);
// videoDuration = videoFormat.getLong(MediaFormat.KEY_DURATION);
return videoFormat;
}
}
return null;
}
private void writeVideoToMuxer() {
ByteBuffer buffer = ByteBuffer.allocate(frameMaxInputSize);
MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();
videoExtractor.unselectTrack(videoTrackIndex);
videoExtractor.selectTrack(videoTrackIndex);
while (true) {
buffer.clear();
int sampleSize = videoExtractor.readSampleData(buffer, 0);
if (sampleSize < 0) {
videoExtractor.unselectTrack(videoTrackIndex);
break;
}
videoBufferInfo.size = sampleSize;
videoBufferInfo.presentationTimeUs = videoExtractor.getSampleTime();
videoBufferInfo.flags = videoExtractor.getSampleFlags();
mediaMuxer.writeSampleData(videoTrackId, buffer, videoBufferInfo);
videoExtractor.advance();
}
}
private void encodeAudioPCM(InputStream is) throws IOException {
byte[] tempBuffer = new byte[2 * sampleRate];
boolean hasMoreData = true;
boolean stop = false;
while (!stop) {
int inputBufferIndex = 0;
int currentBatchRead = 0;
while (inputBufferIndex != -1 && hasMoreData && currentBatchRead <= 50 * sampleRate) {
inputBufferIndex = mediaCodec.dequeueInputBuffer(CODEC_TIMEOUT);
if (inputBufferIndex >= 0) {
ByteBuffer buffer = codecInputBuffers[inputBufferIndex];
buffer.clear();
int bytesRead = is.read(tempBuffer, 0, buffer.limit());
if (bytesRead == -1) {
mediaCodec.queueInputBuffer(inputBufferIndex, 0, 0, (long) presentationTimeUs, 0);
hasMoreData = false;
stop = true;
} else {
totalBytesRead += bytesRead;
currentBatchRead += bytesRead;
buffer.put(tempBuffer, 0, bytesRead);
mediaCodec.queueInputBuffer(inputBufferIndex, 0, bytesRead, (long) presentationTimeUs, 0);
presentationTimeUs = 1000000L * (totalBytesRead / 2) / sampleRate;
}
}
}
writeAudioOutputs();
}
is.close();
}
public void start(InputStream inputStream) throws IOException {
Log.d(TAG, "Starting encoding of InputStream");
encodeAudioPCM(inputStream);
Log.d(TAG, "Finished encoding of InputStream");
if (videoPath != null) {
writeVideoToMuxer();
}
}
private void writeAudioOutputs() {
int outputBufferIndex = 0;
while (outputBufferIndex != MediaCodec.INFO_TRY_AGAIN_LATER) {
outputBufferIndex = mediaCodec.dequeueOutputBuffer(audioBufferInfo, CODEC_TIMEOUT);
if (outputBufferIndex >= 0) {
ByteBuffer encodedData = codecOutputBuffers[outputBufferIndex];
encodedData.position(audioBufferInfo.offset);
encodedData.limit(audioBufferInfo.offset + audioBufferInfo.size);
if ((audioBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0 && audioBufferInfo.size != 0) {
mediaCodec.releaseOutputBuffer(outputBufferIndex, false);
} else {
mediaMuxer.writeSampleData(audioTrackId, codecOutputBuffers[outputBufferIndex], audioBufferInfo);
mediaCodec.releaseOutputBuffer(outputBufferIndex, false);
}
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
audioFormat = mediaCodec.getOutputFormat();
audioTrackId = mediaMuxer.addTrack(audioFormat);
mediaMuxer.start();
}
}
}
}