Bit-Based BinaryWriter in C#

Question

I'm working on a bit-based B/W/Greyscale Pre-Compiled font format, and was having issues with either reading or writing the format, (I've not been able to determine where the issue was. (I do have a B/W bit-based version working, but an Aliased font doesn't look too good, as you can imagine, especially when working with a 320x200 pixel screen) ) but decided that just using a BinaryWriter would be much easier than writing to a bool[] when I pulled the image data.

The basic format of a pixel in the file looks like this:

1 - White Pixel (Shortest, as this would be most of the pixels)

00 - Black Pixel (No reason to write 10-bits for a pure black pixel, which there are a reasonable number of)

01 - Greyscale Pixel, and is followed by 1 byte describing the shade of the pixel

Now, everything is fine and dandy with writing the required info, as that's all full bytes, but the default .Net 4.0 BinaryWriter writes a Boolean value as a full byte, and as you can imagine, that negates the use of a bit-based format. So I was wondering, is there a BinaryWriter, (and BinaryReader) implementation out there that's bit-based

Edit: I ended up creating my own. (See the answer for the code.)

Forswear

No, you'll have to create your own. – Stubstad

@Orvid: consider adding your code as an answer to your thread, as it is a full-blown solution. At S-O we look at the code inside a question as the problem-code, not the solution code. You are allowed to answer your own questions. – Tap

Note that you override ReadBoolean and don't read a boolean but a bit. I assume that is on purpose, but because a boolean is one byte large in memory, it may seem odd and unexpected to users of your class, esp. since (this is the MSDN description)[msdn.microsoft.com/en-us/library/…: "Reads a Boolean value from the current stream and advances the current position of the stream by one byte.". In other words, you change the contract. Note that Jon Skeet suggested wrapping the class instead. – Tap

@Abel: I hadn't thought about contracts when I wrote the code, and was mostly being lazy, because I didn't want to type all of the method names and arguments out, although now that I think of it, I probably could have gone into my local copy of the Mono sources, and copied the method names from there, then removed the insides of the methods. But since it's already done, I'll leave it as-is. Also, I added the code as an answer to my question instead of in the question. – Forswear

Answer 1

I ended up writing my own, so here they are.

The BinaryWriter (I've only overridden the ones that I needed)

private class BinaryWriter : System.IO.BinaryWriter
{
    private bool[] curByte = new bool[8];
    private byte curBitIndx = 0;
    private System.Collections.BitArray ba;

    public BinaryWriter(Stream s) : base(s) { }

    public override void Flush()
    {
        base.Write(ConvertToByte(curByte));
        base.Flush();
    }

    public override void Write(bool value)
    {
        curByte[curBitIndx] = value;
        curBitIndx++;

        if (curBitIndx == 8)
        {
            base.Write(ConvertToByte(curByte));
            this.curBitIndx = 0;
            this.curByte = new bool[8];
        }
    }

    public override void Write(byte value)
    {
        ba = new BitArray(new byte[] { value });
        for (byte i = 0; i < 8; i++)
        {
            this.Write(ba[i]);
        }
        ba = null;
    }

    public override void Write(byte[] buffer)
    {
        for (int i = 0; i < buffer.Length; i++)
        {
            this.Write((byte)buffer[i]);
        }
    }

    public override void Write(uint value)
    {
        ba = new BitArray(BitConverter.GetBytes(value));
        for (byte i = 0; i < 32; i++)
        {
            this.Write(ba[i]);
        }
        ba = null;
    }

    public override void Write(ulong value)
    {
        ba = new BitArray(BitConverter.GetBytes(value));
        for (byte i = 0; i < 64; i++)
        {
            this.Write(ba[i]);
        }
        ba = null;
    }

    public override void Write(ushort value)
    {
        ba = new BitArray(BitConverter.GetBytes(value));
        for (byte i = 0; i < 16; i++)
        {
            this.Write(ba[i]);
        }
        ba = null;
    }

    private static byte ConvertToByte(bool[] bools)
    {
        byte b = 0;

        byte bitIndex = 0;
        for (int i = 0; i < 8; i++)
        {
            if (bools[i])
            {
                b |= (byte)(((byte)1) << bitIndex);
            }
            bitIndex++;
        }

        return b;
    }
}

And, the BinaryReader, once again, I've only overridden the methods that I needed.

private class BinaryReader : System.IO.BinaryReader
{
    private bool[] curByte = new bool[8];
    private byte curBitIndx = 0;
    private BitArray ba;

    public BinaryReader(Stream s) : base(s)
    {
        ba = new BitArray(new byte[] { base.ReadByte() });
        ba.CopyTo(curByte, 0);
        ba = null;
    }

    public override bool ReadBoolean()
    {
        if (curBitIndx == 8)
        {
            ba = new BitArray(new byte[] { base.ReadByte() });
            ba.CopyTo(curByte, 0);
            ba = null;
            this.curBitIndx = 0;
        }

        bool b = curByte[curBitIndx];
        curBitIndx++;
        return b;
    }

    public override byte ReadByte()
    {
        bool[] bar = new bool[8];
        byte i;
        for (i = 0; i < 8; i++)
        {
            bar[i] = this.ReadBoolean();
        }

        byte b = 0;
        byte bitIndex = 0;
        for (i = 0; i < 8; i++)
        {
            if (bar[i])
            {
                b |= (byte)(((byte)1) << bitIndex);
            }
            bitIndex++;
        }
        return b;
    }

    public override byte[] ReadBytes(int count)
    {
        byte[] bytes = new byte[count];
        for (int i = 0; i < count; i++)
        {
            bytes[i] = this.ReadByte();
        }
        return bytes;
    }

    public override ushort ReadUInt16()
    {
        byte[] bytes = ReadBytes(2);
        return BitConverter.ToUInt16(bytes, 0);
    }

    public override uint ReadUInt32()
    {
        byte[] bytes = ReadBytes(4);
        return BitConverter.ToUInt32(bytes, 0);
    }

    public override ulong ReadUInt64()
    {
        byte[] bytes = ReadBytes(8);
        return BitConverter.ToUInt64(bytes, 0);
    }
}

Forswear

Answer 2

I don't believe there's anything in the framework for this, no. Basically you'd need to write a class to wrap a BinaryWriter (or just a stream) and "the byte written so far" and the number of bits written. When the number of bits gets to 8, write the byte to the underlying stream and clear.

EDIT: the OP posted a possible and working implementation of the above suggestion below.

Decasyllable

Wrapping a BinaryWriter or subclassing a stream is probably good. The OP took the advice the other way and subclassed BinaryWriter. While that seems good at first, it changes the contract for BinaryRead.ReadBoolean and the like: it doesn't read a boolean, it reads a bit, which changes the contract. – Tap

Answer 3

If you keep your data in a byte array (bools are of no use and take too much space, if you use them for bits, they take up a byte in memory) or in an array of a particular struct that fits your dataformat.

Once you have an internal memory representation, you don't need a bit-based binary writer anymore. You can simply write the data to a BinaryWriter and you're done with it.

...but the default .Net 4.0 BinaryWriter writes a Boolean value as a full byte, and as you can imagine, that negates the use of a bit-based format....

The reason for this is: the bool is, by definition, of 1 byte size in C#. The BinaryWriter simply writes what you give it.

Tap

Answer 4

I found myself in need of this as well, so I built upon OP and filled in all the read/writes (except char & string since those are a bit special).

I also made a quick unit test try it out. For streams containing only boolean (or other custom sub-byte value types) it's obviously 87.5% cheaper, and for a random mixed stream containing 75% boolean values, it was about 33% cheaper. So could be useful for some scenarios.

Here are the both classes in case anyone else needs them, use at your own risk:

/// <summary>
/// A binary writer that packs data into bits, to preserve space when using many bit/boolean values. Up to about 87.5% cheaper for streams that only contains boolean values.
/// By: [email protected], based on posters classes in this post: https://mcmap.net/q/1136067/-bit-based-binarywriter-in-c
/// </summary>
public class BinaryBitWriter : BinaryWriter
{
    public byte BitPosition { get; private set; } = 0;
    private bool[] curByte = new bool[8];
    private System.Collections.BitArray ba;

    public BinaryBitWriter(Stream s) : base(s) { }

    public override void Flush()
    {
        flushBitBuffer();
        base.Flush();
    }

    public override void Write(byte[] buffer, int index, int count)
    {
        for (int i = index; i < index + count; i++)
            Write((byte)buffer[i]);
    }
    public override void Write(byte value)
    {
        ba = new BitArray(new byte[] { value });
        for (byte i = 0; i < 8; i++)
            Write(ba[i]);
    }
    public override void Write(bool value)
    {
        curByte[BitPosition] = value;
        BitPosition++;

        if (BitPosition == 8)
            flushBitBuffer();
    }
    public override void Write(char[] chars, int index, int count)
    {
        for (int i = index; i < index + count; i++)
            Write(chars[i]);
    }
    public override void Write(string value)
    {
        // write strings as normal for now, so flush the bits first
        flushBitBuffer();
        base.Write(value);
    }
    public override void Write(decimal value)
    {
        var ints = decimal.GetBits(value);
        for (int i = 0; i < ints.Length; i++)
            Write(ints[i]);
    }
    public override void Write(float value) => Write(BitConverter.GetBytes(value));
    public override void Write(ulong value) => Write(BitConverter.GetBytes(value));
    public override void Write(long value) => Write(BitConverter.GetBytes(value));
    public override void Write(uint value) => Write(BitConverter.GetBytes(value));
    public override void Write(int value) => Write(BitConverter.GetBytes(value));
    public override void Write(ushort value) => Write(BitConverter.GetBytes(value));
    public override void Write(short value) => Write(BitConverter.GetBytes(value));
    public override void Write(double value) => Write(BitConverter.GetBytes(value));
    public override void Write(char[] value) => Write(value, 0, value.Length);
    public override void Write(char value)
    {
        // write strings as normal for now, so flush the bits first
        flushBitBuffer();
        base.Write(value);
        //var b = BitConverter.GetBytes(value);
        //Write(b);
    }
    public override void Write(byte[] buffer) => Write(buffer, 0, buffer.Length);
    public override void Write(sbyte value) => Write((byte)value);

    void flushBitBuffer()
    {
        if (BitPosition == 0) // Nothing to flush
            return;

        base.Write(ConvertToByte(curByte));
        BitPosition = 0;
        curByte = new bool[8];
    }

    private static byte ConvertToByte(bool[] bools)
    {
        byte b = 0;

        byte bitIndex = 0;
        for (int i = 0; i < 8; i++)
        {
            if (bools[i])
                b |= (byte)(((byte)1) << bitIndex);
            bitIndex++;
        }

        return b;
    }
}

public class BinaryBitReader : BinaryReader
{
    public byte BitPosition { get; private set; } = 8;
    private bool[] curByte = new bool[8];

    public BinaryBitReader(Stream s) : base(s)
    {

    }

    public override bool ReadBoolean()
    {
        if (BitPosition == 8)
        {
            var ba = new BitArray(new byte[] { base.ReadByte() });
            ba.CopyTo(curByte, 0);
            BitPosition = 0;
        }

        bool b = curByte[BitPosition];
        BitPosition++;
        return b;
    }

    public override byte ReadByte()
    {
        bool[] bar = new bool[8];
        byte i;
        for (i = 0; i < 8; i++)
        {
            bar[i] = this.ReadBoolean();
        }

        byte b = 0;
        byte bitIndex = 0;
        for (i = 0; i < 8; i++)
        {
            if (bar[i])
            {
                b |= (byte)(((byte)1) << bitIndex);
            }
            bitIndex++;
        }
        return b;
    }

    public override byte[] ReadBytes(int count)
    {
        byte[] bytes = new byte[count];
        for (int i = 0; i < count; i++)
        {
            bytes[i] = this.ReadByte();
        }
        return bytes;
    }


    //public override int Read() => BitConverter.ToUInt64(ReadBytes(8), 0);
    public override int Read(byte[] buffer, int index, int count)
    {
        for (int i = index; i < index + count; i++)
            buffer[i] = ReadByte();
        return count; // we can return this here, it will die at the above row if anything is off
    }
    public override int Read(char[] buffer, int index, int count)
    {
        for (int i = index; i < index + count; i++)
            buffer[i] = ReadChar();
        return count; // we can return this here, it will die at the above row if anything is off
    }
    public override char ReadChar()
    {
        BitPosition = 8;
        return base.ReadChar();
        //BitConverter.ToChar(ReadBytes(2), 0);
    }
    public override char[] ReadChars(int count)
    {
        var chars = new char[count];
        Read(chars, 0, count);
        return chars;
    }
    public override decimal ReadDecimal()
    {
        int[] ints = new int[4];
        for (int i = 0; i < ints.Length; i++)
            ints[i] = ReadInt32();
        return new decimal(ints);
    }
    public override double ReadDouble() => BitConverter.ToDouble(ReadBytes(8), 0);
    public override short ReadInt16() => BitConverter.ToInt16(ReadBytes(2), 0);
    public override int ReadInt32() => BitConverter.ToInt32(ReadBytes(4), 0);
    public override long ReadInt64() => BitConverter.ToInt64(ReadBytes(8), 0);
    public override sbyte ReadSByte() => (sbyte)ReadByte();
    public override float ReadSingle() => BitConverter.ToSingle(ReadBytes(4), 0);
    public override string ReadString()
    {
        BitPosition = 8; // Make sure we read a new byte when we start reading the string
        return base.ReadString();
    }
    public override ushort ReadUInt16() => BitConverter.ToUInt16(ReadBytes(2), 0);
    public override uint ReadUInt32() => BitConverter.ToUInt32(ReadBytes(4), 0);
    public override ulong ReadUInt64() => BitConverter.ToUInt64(ReadBytes(8), 0);
}

And the unit tests:

public static bool UnitTest()
{
    const int testPairs = 512;

    var bitstream = new MemoryStream();
    var bitwriter = new BinaryBitWriter(bitstream);
    var bitreader = new BinaryBitReader(bitstream);

    byte[] bytes = new byte[] { 1, 2, 3, 4, 255 };
    byte Byte = 128;
    bool Bool = true;
    char[] chars = new char[] { 'a', 'b', 'c' };
    string str = "hello";
    var Float = 2.5f;
    ulong Ulong = 12345678901234567890;
    long Long = 1122334455667788;
    uint Uint = 1234567890;
    int Int = 999998888;
    ushort UShort = 12345;
    short Short = 4321;
    double Double = 9.9;
    char Char = 'A';
    sbyte Sbyte = -128;
    decimal Decimal = 10000.00001m;

    List<BBTest> pairs = new List<BBTest>();

    // Make pairs of write and read tests
    pairs.Add(new BBTest(Bool, (w) => w.Write(Bool), (r) => { if (r.ReadBoolean() != Bool) throw new Exception(); }));
    pairs.Add(new BBTest(bytes, (w) => w.Write(bytes, 0, 5), (r) => { if (arrayCompare(r.ReadBytes(5), bytes)) throw new Exception(); }));
    pairs.Add(new BBTest(Byte, (w) => w.Write(Byte), (r) => { if (r.ReadByte() != Byte) throw new Exception(); }));
    pairs.Add(new BBTest(chars, (w) => w.Write(chars, 0, 3), (r) => { if (arrayCompare(r.ReadChars(3), chars)) throw new Exception(); })); /////////////
    pairs.Add(new BBTest(str, (w) => w.Write(str), (r) => { string s; if ((s = r.ReadString()) != str) throw new Exception(); }));
    pairs.Add(new BBTest(Decimal, (w) => w.Write(Decimal), (r) => { if (r.ReadDecimal() != Decimal) throw new Exception(); }));
    pairs.Add(new BBTest(Float, (w) => w.Write(Float), (r) => { if (r.ReadSingle() != Float) throw new Exception(); }));
    pairs.Add(new BBTest(Ulong, (w) => w.Write(Ulong), (r) => { if (r.ReadUInt64() != Ulong) throw new Exception(); }));
    pairs.Add(new BBTest(Long, (w) => w.Write(Long), (r) => { if (r.ReadInt64() != Long) throw new Exception(); }));
    pairs.Add(new BBTest(Uint, (w) => w.Write(Uint), (r) => { if (r.ReadUInt32() != Uint) throw new Exception(); }));
    pairs.Add(new BBTest(Int, (w) => w.Write(Int), (r) => { if (r.ReadInt32() != Int) throw new Exception(); }));
    pairs.Add(new BBTest(UShort, (w) => w.Write(UShort), (r) => { if (r.ReadUInt16() != UShort) throw new Exception(); }));
    pairs.Add(new BBTest(Short, (w) => w.Write(Short), (r) => { if (r.ReadInt16() != Short) throw new Exception(); }));
    pairs.Add(new BBTest(Double, (w) => w.Write(Double), (r) => { if (r.ReadDouble() != Double) throw new Exception(); }));
    pairs.Add(new BBTest(Char, (w) => w.Write(Char), (r) => { if (r.ReadChar() != Char) throw new Exception(); })); ///////////////
    pairs.Add(new BBTest(bytes, (w) => w.Write(bytes), (r) => { if (arrayCompare(r.ReadBytes(5), bytes)) throw new Exception(); }));
    pairs.Add(new BBTest(Sbyte, (w) => w.Write(Sbyte), (r) => { if (r.ReadSByte() != Sbyte) throw new Exception(); }));

    // Now add all tests, and then a bunch of randomized tests, to make sure we test lots of combinations incase there is some offsetting error
    List<BBTest> test = new List<BBTest>();
    test.AddRange(pairs);
    var rnd = new Random();

    for (int i = 0; i < testPairs - test.Count; i++)
    {
        if (rnd.NextDouble() < 0.75)
            test.Add(pairs[0]);
        else
            test.Add(pairs[rnd.Next(pairs.Count)]);
    }

    // now write all the tests
    for (int i = 0; i < test.Count; i++)
        test[i].Writer(bitwriter);
    bitwriter.Flush();

    // now reset the stream and test to see that they are the same
    bitstream.Position = 0;
    for (int i = 0; i < test.Count; i++)
        test[i].ReadTest(bitreader);

    // As comparison, lets write the same stuff to a normal binarywriter and compare sized
    var binstream = new MemoryStream();
    var binwriter = new BinaryWriter(binstream);
    for (int i = 0; i < test.Count; i++)
        test[i].Writer(binwriter);
    binwriter.Flush();

    var saved = 1 - bitstream.Length / (float)binstream.Length;
    var result = $"BinaryBitWriter was {(saved * 100).ToString("0.00")}% cheaper than a normal BinaryWriter with random data";

    bool arrayCompare(IEnumerable a, IEnumerable b)
    {
        var B = b.GetEnumerator();
        B.MoveNext();
        foreach (var item in a)
        {
            if (item != B.Current)
                return false;
            B.MoveNext();
        }
        return true;
    }

    return true;
}
delegate void writer(BinaryWriter w);
delegate void reader(BinaryReader r);
class BBTest
{
    public object Object;
    public writer Writer;
    public reader ReadTest;
    public BBTest(object obj, writer w, reader r) { Object = obj; Writer = w; ReadTest = r; }
    public override string ToString() => Object.ToString();
}

Jerold