Concatenate multiple mp4 audio files using android´s MediaMuxer

Question

I am trying to concatenate multiple mp4 audio files (each containing only one audio track, all recorded with the same MediaRecorder and the same parameters) into one using the following function:

@TargetApi(Build.VERSION_CODES.JELLY_BEAN_MR2)
public static boolean concatenateFiles(File dst, File... sources) {
    if ((sources == null) || (sources.length == 0)) {
        return false;
    }

    boolean result;
    MediaExtractor extractor = null;
    MediaMuxer muxer = null;
    try {
        // Set up MediaMuxer for the destination.
        muxer = new MediaMuxer(dst.getPath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);

        // Copy the samples from MediaExtractor to MediaMuxer.
        boolean sawEOS = false;
        int bufferSize = MAX_SAMPLE_SIZE;
        int frameCount = 0;
        int offset = 100;

        ByteBuffer dstBuf = ByteBuffer.allocate(bufferSize);
        BufferInfo bufferInfo = new BufferInfo();

        long timeOffsetUs = 0;
        int dstTrackIndex = -1;

        for (int fileIndex = 0; fileIndex < sources.length; fileIndex++) {
            int numberOfSamplesInSource = getNumberOfSamples(sources[fileIndex]);
            if (VERBOSE) {
                Log.d(TAG, String.format("Source file: %s", sources[fileIndex].getPath()));
            }

            // Set up MediaExtractor to read from the source.
            extractor = new MediaExtractor();
            extractor.setDataSource(sources[fileIndex].getPath());

            // Set up the tracks.
            SparseIntArray indexMap = new SparseIntArray(extractor.getTrackCount());
            for (int i = 0; i < extractor.getTrackCount(); i++) {
                extractor.selectTrack(i);
                MediaFormat format = extractor.getTrackFormat(i);
                if (dstTrackIndex < 0) {
                    dstTrackIndex = muxer.addTrack(format);
                    muxer.start();
                }
                indexMap.put(i, dstTrackIndex);
            }

            long lastPresentationTimeUs = 0;
            int currentSample = 0;

            while (!sawEOS) {
                bufferInfo.offset = offset;
                bufferInfo.size = extractor.readSampleData(dstBuf, offset);

                if (bufferInfo.size < 0) {
                    sawEOS = true;
                    bufferInfo.size = 0;
                    timeOffsetUs += (lastPresentationTimeUs + APPEND_DELAY);
                }
                else {
                    lastPresentationTimeUs = extractor.getSampleTime();
                    bufferInfo.presentationTimeUs = extractor.getSampleTime() + timeOffsetUs;
                    bufferInfo.flags = extractor.getSampleFlags();
                    int trackIndex = extractor.getSampleTrackIndex();

                    if ((currentSample < numberOfSamplesInSource) || (fileIndex == sources.length - 1)) {
                        muxer.writeSampleData(indexMap.get(trackIndex), dstBuf, bufferInfo);
                    }
                    extractor.advance();

                    frameCount++;
                    currentSample++;
                    if (VERBOSE) {
                        Log.d(TAG, "Frame (" + frameCount + ") " +
                                "PresentationTimeUs:" + bufferInfo.presentationTimeUs +
                                " Flags:" + bufferInfo.flags +
                                " TrackIndex:" + trackIndex +
                                " Size(KB) " + bufferInfo.size / 1024);
                    }
                }
            }
            extractor.release();
            extractor = null;
        }

        result = true;
    }
    catch (IOException e) {
        result = false;
    }
    finally {
        if (extractor != null) {
            extractor.release();
        }
        if (muxer != null) {
            muxer.stop();
            muxer.release();
        }
    }
    return result;
}

@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
public static int getNumberOfSamples(File src) {
    MediaExtractor extractor = new MediaExtractor();
    int result;
    try {
        extractor.setDataSource(src.getPath());
        extractor.selectTrack(0);

        result = 0;
        while (extractor.advance()) {
            result ++;
        }
    }
    catch(IOException e) {
        result = -1;
    }
    finally {
        extractor.release();
    }
    return result;
}

The code compiles and runs, but when playing the resulting file, I hear only the contents of the first file. I do not see what I am doing wrong.

However, after Marlon pointed me to that direction, there is something strange about the messages I am getting from the MediaMuxer. Here they are:

05-04 15:30:01.869: D/MediaMuxerTest(5455): Source file: /storage/emulated/0/Android/data/de.absprojects.catalogizer/files/copy.mp4
05-04 15:30:01.889: D/QCUtils(5455): extended extractor not needed, return default
05-04 15:30:01.889: I/MPEG4Writer(5455): limits: 2147483647/0 bytes/us, bit rate: -1 bps and the estimated moov size 3072 bytes
05-04 15:30:01.889: I/MPEG4Writer(5455): setStartTimestampUs: 0
05-04 15:30:01.889: I/MPEG4Writer(5455): Earliest track starting time: 0
05-04 15:30:01.889: D/MediaMuxerTest(5455): Frame (1) PresentationTimeUs:0 Flags:1 TrackIndex:0 Size(KB) 0
05-04 15:30:01.889: D/MediaMuxerTest(5455): Frame (2) PresentationTimeUs:23219 Flags:1 TrackIndex:0 Size(KB) 0
05-04 15:30:01.889: D/MediaMuxerTest(5455): Frame (3) PresentationTimeUs:46439 Flags:1 TrackIndex:0 Size(KB) 0
[...]
05-04 15:30:01.959: D/MediaMuxerTest(5455): Frame (117) PresentationTimeUs:2693401 Flags:1 TrackIndex:0 Size(KB) 0
05-04 15:30:01.959: D/MediaMuxerTest(5455): Frame (118) PresentationTimeUs:2716621 Flags:1 TrackIndex:0 Size(KB) 0
05-04 15:30:01.959: D/MediaMuxerTest(5455): Frame (119) PresentationTimeUs:2739841 Flags:1 TrackIndex:0 Size(KB) 0
05-04 15:30:01.959: D/MediaMuxerTest(5455): Frame (120) PresentationTimeUs:2763061 Flags:1 TrackIndex:0 Size(KB) 0
05-04 15:30:01.979: D/QCUtils(5455): extended extractor not needed, return default
05-04 15:30:01.979: D/MediaMuxerTest(5455): Source file: /storage/emulated/0/Android/data/de.absprojects.catalogizer/files/temp.mp4
05-04 15:30:01.979: I/MPEG4Writer(5455): Received total/0-length (120/0) buffers and encoded 120 frames. - audio
05-04 15:30:01.979: I/MPEG4Writer(5455): Audio track drift time: 0 us
05-04 15:30:01.979: D/MPEG4Writer(5455): Setting Audio track to done
05-04 15:30:01.979: D/MPEG4Writer(5455): Stopping Audio track
05-04 15:30:01.979: D/MPEG4Writer(5455): Stopping Audio track source
05-04 15:30:01.979: D/MPEG4Writer(5455): Audio track stopped
05-04 15:30:01.979: D/MPEG4Writer(5455): Stopping writer thread
05-04 15:30:01.979: D/MPEG4Writer(5455): 0 chunks are written in the last batch
05-04 15:30:01.979: D/MPEG4Writer(5455): Writer thread stopped
05-04 15:30:01.979: D/MPEG4Writer(5455): Stopping Audio track
05-04 15:30:01.979: E/MPEG4Writer(5455): Stop() called but track is not started
05-04 15:30:01.999: D/QCUtils(5455): extended extractor not needed, return default
05-04 15:30:01.999: D/copyOriginalFile()(5455): 120 samples in original file
05-04 15:30:02.009: D/QCUtils(5455): extended extractor not needed, return default
05-04 15:30:02.019: D/copyOriginalFile()(5455): 120 samples in copied file
05-04 15:30:02.019: W/MediaRecorder(5455): mediarecorder went away with unhandled events
05-04 15:30:02.099: I/dalvikvm(5455): Jit: resizing JitTable from 4096 to 8192

It seems that after copying data from the first file, MPEG4Writer (why not MediaMuxer?) stops the track and does not write further data. How can I prevent that? Do I have to manipulate the headers directly, and if so, how?

Any help would be appreciated.

Best regards,

Christian

Stonemason

Which value did you choose for MAX_SAMPLE_SIZE ? (I'm trying to use your code for something else) Thx ! – Czarra

Answer 1

Formally you can't join 2 encoded audio tracks: each track could be encoded with different parameters which are stored in headers. For sure if both files were created by the same encoder\muxer, same encoding parameters and both headers are equal it can work, but it is rather strict limitation. As far as i see you set audio format (it contains headers) to audio track in muxer to format from 1st file. So if 2nd file audio format differs it can cause different sorts of errors resulting in not correct second file audio.

Please try to put one source file twice to dst file, as first and second. If it works - than the problem is in headers. If not - then somewhere else, i think.

Thynne

Thank you for your help. Yes, all source files have been recorded with the same parameters. Calling my function with twice the same file does not solve the problem - I still hear only the first copy. – Physique

what about timestamps sent to muxer? are they constantly increasing? do you see any warnings\errors from muxer in logcat? – Thynne

Dear Marlon, thanks again for your help. There is something strange about the log messages from the muxer. It seems that it stops encoding of its own account after the first file (see edited question), but I do not see why. – Physique

were you able to debug the code? are you sure that no exception etc. happens and execution does not jump to finally{} instead of remuxing of 2nd file? what line call causes muxer stop? – Thynne

Thank you for making me work properly. There were two reasons for my problem: 1) Calling setDataSource() multiple times on a MediaExtractor does not work, it throws an IOException. I had to release() the Extractor and initialize it again for each source file. 2) I initialized the sawEOS variable outside the for() loop which iterates over the source files - therefore the second file was never read because sawEOS was already true. Moving sawEOS´s initialization inside the for() loop solved the problem - now everything works fine. Thank you again! – Physique

it can worth to try INDE media pack, it is able to join media files on android: software.intel.com/en-us/articles/… – Thynne

@ChristianReck-Würges Can you post the updated code ? There's so few resources about MediaMuxer out there... – Czarra

Answer 2

I am looking to do the same, and thinking about it more, it can't work. Wished it did, because I need it too. It's like trying to push two bottles together and expecting them to become one bigger bottle. You need to take the... beer? from each (decode audio from each file) and then pour it in a new bottle (encode audio again, feeding from the second one when the first is done)... Once the bottle is capped, you cannot add more beer in it

Hiphuggers

Answer 3

This code works if the two video files have the same video resolution, video codec, fps, audio sample rate and audio codec.

private const val MAX_SAMPLE_SIZE = 256 * 1024

fun concatenateFiles(dst: File, sources: ArrayList<File>): Boolean {

    println("---------------------")
    println("concatenateFiles")
    println("---------------------")

    if (sources.isEmpty()) {

        return false

    }

    var result : Boolean
    var muxer : MediaMuxer? = null

    try {

        // Set up MediaMuxer for the destination.

        muxer = MediaMuxer(dst.path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)

        // Copy the samples from MediaExtractor to MediaMuxer.

        var videoFormat : MediaFormat? = null
        var audioFormat : MediaFormat? = null

        var idx = 0

        var muxerStarted : Boolean = false

        var videoTrackIndex = -1
        var audioTrackIndex = -1

        var totalDuration = 0

        for (file in sources) {

            println("-------------------")
            println("file: $idx")
            println("-------------------")

            // new

            // MediaMetadataRetriever

            val m = MediaMetadataRetriever()
            m.setDataSource(file.absolutePath)

            var trackDuration : Int = 0

            try {

                trackDuration = m.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)!!.toInt()

            } catch (e: java.lang.Exception) {

                // error

            }

            // extractorVideo

            var extractorVideo = MediaExtractor()

            extractorVideo.setDataSource(file.path)

            val tracks = extractorVideo.trackCount

            for (i in 0 until tracks) {

                val mf = extractorVideo.getTrackFormat(i)

                val mime = mf.getString(MediaFormat.KEY_MIME)

                println("mime: $mime")

                if (mime!!.startsWith("video/")) {

                    extractorVideo.selectTrack(i)
                    videoFormat = extractorVideo.getTrackFormat(i)

                    break

                }

            }

            // extractorAudio

            var extractorAudio = MediaExtractor()

            extractorAudio.setDataSource(file.path)

            for (i in 0 until tracks) {

                val mf = extractorAudio.getTrackFormat(i)

                val mime = mf.getString(MediaFormat.KEY_MIME)

                if (mime!!.startsWith("audio/")) {

                    extractorAudio.selectTrack(i)
                    audioFormat = extractorAudio.getTrackFormat(i)

                    break

                }

            }

            // audioTracks

            val audioTracks = extractorAudio.trackCount

            println("audioTracks: $audioTracks")

            // videoTrackIndex

            if (videoTrackIndex == -1) {

                videoTrackIndex = muxer.addTrack(videoFormat!!)

            }

            // audioTrackIndex

            if (audioTrackIndex == -1) {

                audioTrackIndex = muxer.addTrack(audioFormat!!)

            }

            var sawEOS = false
            var sawAudioEOS = false
            val bufferSize = MAX_SAMPLE_SIZE
            val dstBuf = ByteBuffer.allocate(bufferSize)
            val offset = 0
            val bufferInfo = BufferInfo()

            // start muxer

            println("muxer.start()")

            if (!muxerStarted) {

                muxer.start()

                muxerStarted = true

            }

            // write video

            println("write video")

            while (!sawEOS) {

                bufferInfo.offset = offset
                bufferInfo.size = extractorVideo.readSampleData(dstBuf, offset)

                if (bufferInfo.size < 0) {

                    //println("videoBufferInfo.size < 0")

                    sawEOS = true
                    bufferInfo.size = 0

                } else {

                    bufferInfo.presentationTimeUs = extractorVideo.sampleTime + totalDuration
                    bufferInfo.flags = MediaCodec.BUFFER_FLAG_KEY_FRAME
                    muxer.writeSampleData(videoTrackIndex, dstBuf, bufferInfo)
                    extractorVideo.advance()

                }

            }

            // write audio

            println("write audio")

            val audioBuf = ByteBuffer.allocate(bufferSize)

            while (!sawAudioEOS) {

                bufferInfo.offset = offset
                bufferInfo.size = extractorAudio.readSampleData(audioBuf, offset)

                if (bufferInfo.size < 0) {

                    //println("audioBufferInfo.size < 0")

                    sawAudioEOS = true
                    bufferInfo.size = 0

                } else {

                    bufferInfo.presentationTimeUs = extractorAudio.sampleTime + totalDuration
                    bufferInfo.flags = MediaCodec.BUFFER_FLAG_KEY_FRAME
                    muxer.writeSampleData(audioTrackIndex, audioBuf, bufferInfo)
                    extractorAudio.advance()

                }

            }

            extractorVideo.release()
            extractorAudio.release()

            // should match

            totalDuration += (trackDuration * 1_000)

            if (VERBOSE) {
                println("PresentationTimeUs:" + bufferInfo.presentationTimeUs)
                println("totalDuration: $totalDuration")
            }

            // increment file index

            idx += 1

        }

        result = true

    } catch (e: IOException) {

        result = false

    } finally {

        if (muxer != null) {
            muxer.stop()
            muxer.release()
        }

    }

    return result

}

Plasticity