C# Signed & Unsigned Integral to Big Endian Byte Array, and vice versa using Bitwise way with "best" performance

Asked 28/11, 2017 at 10:27 Answered 1/12, 2017 at 3:29

2nd edit:

I think my original test script has an issue, the 10000000 times loop is, in fact, dealing with the same memory location of an array, which makes the unsafe version (provided by Marc here) much faster than bitwise version. I wrote another test script, I noticed that bitwise and unsafe offers almost the same performance.

loop for 10,000,000 times

Bitwise: 4218484; UnsafeRaw: 4101719

0.0284673328426447545529081831 (~2% Difference)

here is the code:

unsafe class UnsafeRaw
{
    public static short ToInt16BigEndian(byte* buf)
    {
        return (short)ToUInt16BigEndian(buf);
    }

    public static int ToInt32BigEndian(byte* buf)
    {
        return (int)ToUInt32BigEndian(buf);
    }

    public static long ToInt64BigEndian(byte* buf)
    {
        return (long)ToUInt64BigEndian(buf);
    }

    public static ushort ToUInt16BigEndian(byte* buf)
    {
        return (ushort)((*buf++ << 8) | *buf);
    }

    public static uint ToUInt32BigEndian(byte* buf)
    {
        return unchecked((uint)((*buf++ << 24) | (*buf++ << 16) | (*buf++ << 8) | *buf));
    }

    public static ulong ToUInt64BigEndian(byte* buf)
    {
        unchecked
        {
            var x = (uint)((*buf++ << 24) | (*buf++ << 16) | (*buf++ << 8) | *buf++);
            var y = (uint)((*buf++ << 24) | (*buf++ << 16) | (*buf++ << 8) | *buf);
            return ((ulong)x << 32) | y;
        }
    }
}

class Bitwise
{
    public static short ToInt16BigEndian(byte[] buffer, int beginIndex)
    {
        return unchecked((short)(buffer[beginIndex] << 8 | buffer[beginIndex + 1]));
    }

    public static int ToInt32BigEndian(byte[] buffer, int beginIndex)
    {
        return unchecked(buffer[beginIndex] << 24 | buffer[beginIndex + 1] << 16 | buffer[beginIndex + 2] << 8 | buffer[beginIndex + 3]);
    }

    public static long ToInt64BigEndian(byte[] buffer, int beginIndex)
    {
        return unchecked((long)buffer[beginIndex] << 56 | (long)buffer[beginIndex + 1] << 48 | (long)buffer[beginIndex + 2] << 40 | (long)buffer[beginIndex + 3] << 32 | (long)buffer[beginIndex + 4] << 24 | (long)buffer[beginIndex + 5] << 16 | (long)buffer[beginIndex + 6] << 8 | buffer[beginIndex + 7]);
    }

    public static ushort ToUInt16BigEndian(byte[] buffer, int beginIndex)
    {
        return unchecked((ushort)ToInt16BigEndian(buffer, beginIndex));
    }

    public static uint ToUInt32BigEndian(byte[] buffer, int beginIndex)
    {
        return unchecked((uint)ToInt32BigEndian(buffer, beginIndex));
    }

    public static ulong ToUInt64BigEndian(byte[] buffer, int beginIndex)
    {
        return unchecked((ulong)ToInt64BigEndian(buffer, beginIndex));
    }
}

class BufferTest
{
    static long LongRandom(long min, long max, Random rand)
    {
        long result = rand.Next((Int32)(min >> 32), (Int32)(max >> 32));
        result = result << 32;
        result = result | (long)rand.Next((Int32)min, (Int32)max);
        return result;
    }

    public static void Main()
    {
        const int times = 10000000;
        const int index = 100;

        Random r = new Random();

        Stopwatch sw1 = new Stopwatch();

        Console.WriteLine($"loop for {times:##,###} times");

        Thread.Sleep(1000);

        for (int j = 0; j < times; j++)
        {
            short a = (short)r.Next(short.MinValue, short.MaxValue);
            int b = r.Next(int.MinValue, int.MaxValue);
            long c = LongRandom(int.MinValue, int.MaxValue, r);
            ushort d = (ushort)r.Next(ushort.MinValue, ushort.MaxValue);
            uint e = (uint)r.Next(int.MinValue, int.MaxValue);
            ulong f = (ulong)LongRandom(int.MinValue, int.MaxValue, r);

            var arr1 = BitConverter.GetBytes(a);
            var arr2 = BitConverter.GetBytes(b);
            var arr3 = BitConverter.GetBytes(c);
            var arr4 = BitConverter.GetBytes(d);
            var arr5 = BitConverter.GetBytes(e);
            var arr6 = BitConverter.GetBytes(f);

            Array.Reverse(arr1);
            Array.Reverse(arr2);
            Array.Reverse(arr3);
            Array.Reverse(arr4);
            Array.Reverse(arr5);
            Array.Reverse(arr6);

            var largerArr1 = new byte[1024];
            var largerArr2 = new byte[1024];
            var largerArr3 = new byte[1024];
            var largerArr4 = new byte[1024];
            var largerArr5 = new byte[1024];
            var largerArr6 = new byte[1024];

            Array.Copy(arr1, 0, largerArr1, index, arr1.Length);
            Array.Copy(arr2, 0, largerArr2, index, arr2.Length);
            Array.Copy(arr3, 0, largerArr3, index, arr3.Length);
            Array.Copy(arr4, 0, largerArr4, index, arr4.Length);
            Array.Copy(arr5, 0, largerArr5, index, arr5.Length);
            Array.Copy(arr6, 0, largerArr6, index, arr6.Length);

            sw1.Start();
            var n1 = Bitwise.ToInt16BigEndian(largerArr1, index);
            var n2 = Bitwise.ToInt32BigEndian(largerArr2, index);
            var n3 = Bitwise.ToInt64BigEndian(largerArr3, index);
            var n4 = Bitwise.ToUInt16BigEndian(largerArr4, index);
            var n5 = Bitwise.ToUInt32BigEndian(largerArr5, index);
            var n6 = Bitwise.ToUInt64BigEndian(largerArr6, index);
            sw1.Stop();

            //Console.WriteLine(n1 == a);
            //Console.WriteLine(n2 == b);
            //Console.WriteLine(n3 == c);
            //Console.WriteLine(n4 == d);
            //Console.WriteLine(n5 == e);
            //Console.WriteLine(n6 == f);
        }

        Stopwatch sw2 = new Stopwatch();

        for (int j = 0; j < times; j++)
        {
            short a = (short)r.Next(short.MinValue, short.MaxValue);
            int b = r.Next(int.MinValue, int.MaxValue);
            long c = LongRandom(int.MinValue, int.MaxValue, r);
            ushort d = (ushort)r.Next(ushort.MinValue, ushort.MaxValue);
            uint e = (uint)r.Next(int.MinValue, int.MaxValue);
            ulong f = (ulong)LongRandom(int.MinValue, int.MaxValue, r);

            var arr1 = BitConverter.GetBytes(a);
            var arr2 = BitConverter.GetBytes(b);
            var arr3 = BitConverter.GetBytes(c);
            var arr4 = BitConverter.GetBytes(d);
            var arr5 = BitConverter.GetBytes(e);
            var arr6 = BitConverter.GetBytes(f);

            Array.Reverse(arr1);
            Array.Reverse(arr2);
            Array.Reverse(arr3);
            Array.Reverse(arr4);
            Array.Reverse(arr5);
            Array.Reverse(arr6);

            var largerArr1 = new byte[1024];
            var largerArr2 = new byte[1024];
            var largerArr3 = new byte[1024];
            var largerArr4 = new byte[1024];
            var largerArr5 = new byte[1024];
            var largerArr6 = new byte[1024];

            Array.Copy(arr1, 0, largerArr1, index, arr1.Length);
            Array.Copy(arr2, 0, largerArr2, index, arr2.Length);
            Array.Copy(arr3, 0, largerArr3, index, arr3.Length);
            Array.Copy(arr4, 0, largerArr4, index, arr4.Length);
            Array.Copy(arr5, 0, largerArr5, index, arr5.Length);
            Array.Copy(arr6, 0, largerArr6, index, arr6.Length);

            sw2.Start();
            unsafe
            {
                fixed (byte* p1 = &largerArr1[index], p2 = &largerArr2[index], p3 = &largerArr3[index], p4 = &largerArr4[index], p5 = &largerArr5[index], p6 = &largerArr6[index])
                {
                    var u1 = UnsafeRaw.ToInt16BigEndian(p1);
                    var u2 = UnsafeRaw.ToInt32BigEndian(p2);
                    var u3 = UnsafeRaw.ToInt64BigEndian(p3);
                    var u4 = UnsafeRaw.ToUInt16BigEndian(p4);
                    var u5 = UnsafeRaw.ToUInt32BigEndian(p5);
                    var u6 = UnsafeRaw.ToUInt64BigEndian(p6);

                    //Console.WriteLine(u1 == a);
                    //Console.WriteLine(u2 == b);
                    //Console.WriteLine(u3 == c);
                    //Console.WriteLine(u4 == d);
                    //Console.WriteLine(u5 == e);
                    //Console.WriteLine(u6 == f);
                }
            }
            sw2.Stop();
        }

        Console.WriteLine($"Bitwise: {sw1.ElapsedTicks}; UnsafeRaw: {sw2.ElapsedTicks}");

        Console.WriteLine((decimal)sw1.ElapsedTicks / sw2.ElapsedTicks - 1);

        Console.ReadKey();
    }
}

1st edit:

I tried to implement using fixed and stackalloc, compare with bitwise. It seems that bitwise is faster than unsafe approach in my test code.

Here is the measurement:

after loops of 10000000 times with stopwatch frequency 3515622

unsafe fixed - 2239790 ticks

bitwise - 672159 ticks

unsafe stackalloc - 1624166 ticks

Is there anything I did wrong? I thought unsafe will be faster than bitwise.

Here is the code:

class Bitwise
{
    public static short ToInt16BigEndian(byte[] buf, int i)
    {
        return (short)((buf[i] << 8) | buf[i + 1]);
    }

    public static int ToInt32BigEndian(byte[] buf, int i)
    {
        return (buf[i] << 24) | (buf[i + 1] << 16) | (buf[i + 2] << 8) | buf[i + 3];
    }

    public static long ToInt64BigEndian(byte[] buf, int i)
    {
        return (buf[i] << 56) | (buf[i + 1] << 48) | (buf[i + 2] << 40) | buf[i + 3] << 32 | (buf[i + 4] << 24) | (buf[i + 5] << 16) | (buf[i + 6] << 8) | buf[i + 7];
    }

    public static ushort ToUInt16BigEndian(byte[] buf, int i)
    {
        ushort value = 0;

        for (var j = 0; j < 2; j++)
        {
            value = (ushort)unchecked((value << 8) | buf[j + i]);
        }

        return value;
    }

    public static uint ToUInt32BigEndian(byte[] buf, int i)
    {
        uint value = 0;

        for (var j = 0; j < 4; j++)
        {
            value = unchecked((value << 8) | buf[j + i]);
        }

        return value;
    }

    public static ulong ToUInt64BigEndian(byte[] buf, int i)
    {
        ulong value = 0;

        for (var j = 0; j < 8; j++)
        {
            value = unchecked((value << 8) | buf[i + j]);
        }

        return value;
    }
}

class Unsafe
{
    public static short ToInt16BigEndian(byte[] buf, int i)
    {
        byte[] arr = new byte[2];

        arr[0] = buf[i + 1];
        arr[1] = buf[i];

        unsafe
        {
            fixed (byte* ptr = arr)
            {
                return *(short*)ptr;
            }
        }
    }

    public static int ToInt32BigEndian(byte[] buf, int i)
    {
        byte[] arr = new byte[4];

        arr[0] = buf[i + 3];
        arr[1] = buf[i + 2];
        arr[2] = buf[i + 1];
        arr[3] = buf[i];

        unsafe
        {
            fixed (byte* ptr = arr)
            {
                return *(int*)ptr;
            }
        }
    }

    public static long ToInt64BigEndian(byte[] buf, int i)
    {
        byte[] arr = new byte[8];

        arr[0] = buf[i + 7];
        arr[1] = buf[i + 6];
        arr[2] = buf[i + 5];
        arr[3] = buf[i + 6];
        arr[4] = buf[i + 3];
        arr[5] = buf[i + 2];
        arr[6] = buf[i + 1];
        arr[7] = buf[i];

        unsafe
        {
            fixed (byte* ptr = arr)
            {
                return *(long*)ptr;
            }
        }
    }

    public static ushort ToUInt16BigEndian(byte[] buf, int i)
    {
        byte[] arr = new byte[2];

        arr[0] = buf[i + 1];
        arr[1] = buf[i];

        unsafe
        {
            fixed (byte* ptr = arr)
            {
                return *(ushort*)ptr;
            }
        }
    }

    public static uint ToUInt32BigEndian(byte[] buf, int i)
    {
        byte[] arr = new byte[4];

        arr[0] = buf[i + 3];
        arr[1] = buf[i + 2];
        arr[2] = buf[i + 1];
        arr[3] = buf[i];

        unsafe
        {
            fixed (byte* ptr = arr)
            {
                return *(uint*)ptr;
            }
        }
    }

    public static ulong ToUInt64BigEndian(byte[] buf, int i)
    {
        byte[] arr = new byte[8];

        arr[0] = buf[i + 7];
        arr[1] = buf[i + 6];
        arr[2] = buf[i + 5];
        arr[3] = buf[i + 6];
        arr[4] = buf[i + 3];
        arr[5] = buf[i + 2];
        arr[6] = buf[i + 1];
        arr[7] = buf[i];

        unsafe
        {
            fixed (byte* ptr = arr)
            {
                return *(ulong*)ptr;
            }
        }
    }
}

class UnsafeAlloc
{
    public static short ToInt16BigEndian(byte[] buf, int i)
    {
        unsafe
        {
            const int length = sizeof(short);
            byte* arr = stackalloc byte[length];
            byte* p = arr;

            for (int j = length - 1; j >= 0; j--)
            {
                *p = buf[i + j];
                p++;
            }

            return *(short*)arr;
        }
    }

    public static int ToInt32BigEndian(byte[] buf, int i)
    {
        unsafe
        {
            const int length = sizeof(int);
            byte* arr = stackalloc byte[length];
            byte* p = arr;

            for (int j = length - 1; j >= 0; j--)
            {
                *p = buf[i + j];
                p++;
            }

            return *(int*)arr;
        }
    }

    public static long ToInt64BigEndian(byte[] buf, int i)
    {
        unsafe
        {
            const int length = sizeof(long);
            byte* arr = stackalloc byte[length];
            byte* p = arr;

            for (int j = length - 1; j >= 0; j--)
            {
                *p = buf[i + j];
                p++;
            }

            return *(long*)arr;
        }
    }

    public static ushort ToUInt16BigEndian(byte[] buf, int i)
    {
        unsafe
        {
            const int length = sizeof(ushort);
            byte* arr = stackalloc byte[length];
            byte* p = arr;

            for (int j = length - 1; j >= 0; j--)
            {
                *p = buf[i + j];
                p++;
            }

            return *(ushort*)arr;
        }
    }

    public static uint ToUInt32BigEndian(byte[] buf, int i)
    {
        unsafe
        {
            const int length = sizeof(uint);
            byte* arr = stackalloc byte[length];
            byte* p = arr;

            for (int j = length - 1; j >= 0; j--)
            {
                *p = buf[i + j];
                p++;
            }

            return *(uint*)arr;
        }
    }

    public static ulong ToUInt64BigEndian(byte[] buf, int i)
    {
        unsafe
        {
            const int length = sizeof(ulong);
            byte* arr = stackalloc byte[length];
            byte* p = arr;

            for (int j = length - 1; j >= 0; j--)
            {
                *p = buf[i + j];
                p++;
            }

            return *(ulong*)arr;
        }
    }
}

class Program
{
    static void Main()
    {
        short a = short.MinValue + short.MaxValue / 2;
        int b = int.MinValue + int.MaxValue / 2;
        long c = long.MinValue + long.MaxValue / 2;
        ushort d = ushort.MaxValue / 2;
        uint e = uint.MaxValue / 2;
        ulong f = ulong.MaxValue / 2;

        Console.WriteLine(a);
        Console.WriteLine(b);
        Console.WriteLine(c);
        Console.WriteLine(d);
        Console.WriteLine(e);
        Console.WriteLine(f);
        Console.WriteLine();

        var arr1 = BitConverter.GetBytes(a);
        var arr2 = BitConverter.GetBytes(b);
        var arr3 = BitConverter.GetBytes(c);
        var arr4 = BitConverter.GetBytes(d);
        var arr5 = BitConverter.GetBytes(e);
        var arr6 = BitConverter.GetBytes(f);

        Array.Reverse(arr1);
        Array.Reverse(arr2);
        Array.Reverse(arr3);
        Array.Reverse(arr4);
        Array.Reverse(arr5);
        Array.Reverse(arr6);

        Console.WriteLine(Unsafe.ToInt16BigEndian(arr1, 0));
        Console.WriteLine(Unsafe.ToInt32BigEndian(arr2, 0));
        Console.WriteLine(Unsafe.ToInt64BigEndian(arr3, 0));
        Console.WriteLine(Unsafe.ToUInt16BigEndian(arr4, 0));
        Console.WriteLine(Unsafe.ToUInt32BigEndian(arr5, 0));
        Console.WriteLine(Unsafe.ToUInt64BigEndian(arr6, 0));
        Console.WriteLine();

        Console.WriteLine(Bitwise.ToInt16BigEndian(arr1, 0));
        Console.WriteLine(Bitwise.ToInt32BigEndian(arr2, 0));
        Console.WriteLine(Bitwise.ToInt64BigEndian(arr3, 0));
        Console.WriteLine(Bitwise.ToUInt16BigEndian(arr4, 0));
        Console.WriteLine(Bitwise.ToUInt32BigEndian(arr5, 0));
        Console.WriteLine(Bitwise.ToUInt64BigEndian(arr6, 0));
        Console.WriteLine();

        Console.WriteLine(UnsafeAlloc.ToInt16BigEndian(arr1, 0));
        Console.WriteLine(UnsafeAlloc.ToInt32BigEndian(arr2, 0));
        Console.WriteLine(UnsafeAlloc.ToInt64BigEndian(arr3, 0));
        Console.WriteLine(UnsafeAlloc.ToUInt16BigEndian(arr4, 0));
        Console.WriteLine(UnsafeAlloc.ToUInt32BigEndian(arr5, 0));
        Console.WriteLine(UnsafeAlloc.ToUInt64BigEndian(arr6, 0));
        Console.WriteLine();

        Stopwatch sw = new Stopwatch();
        sw.Start();

        int times = 10000000;

        var t0 = sw.ElapsedTicks;
        for (int i = 0; i < times; i++)
        {
            Unsafe.ToInt16BigEndian(arr1, 0);
            Unsafe.ToInt32BigEndian(arr2, 0);
            Unsafe.ToInt64BigEndian(arr3, 0);
            Unsafe.ToUInt16BigEndian(arr4, 0);
            Unsafe.ToUInt32BigEndian(arr5, 0);
            Unsafe.ToUInt64BigEndian(arr6, 0);
        }
        var t1 = sw.ElapsedTicks;

        var t2 = sw.ElapsedTicks;
        for (int i = 0; i < times; i++)
        {
            Bitwise.ToInt16BigEndian(arr1, 0);
            Bitwise.ToInt32BigEndian(arr2, 0);
            Bitwise.ToInt64BigEndian(arr3, 0);
            Bitwise.ToUInt16BigEndian(arr4, 0);
            Bitwise.ToUInt32BigEndian(arr5, 0);
            Bitwise.ToUInt64BigEndian(arr6, 0);
        }
        var t3 = sw.ElapsedTicks;

        var t4 = sw.ElapsedTicks;
        for (int i = 0; i < times; i++)
        {
            UnsafeAlloc.ToInt16BigEndian(arr1, 0);
            UnsafeAlloc.ToInt32BigEndian(arr2, 0);
            UnsafeAlloc.ToInt64BigEndian(arr3, 0);
            UnsafeAlloc.ToUInt16BigEndian(arr4, 0);
            UnsafeAlloc.ToUInt32BigEndian(arr5, 0);
            UnsafeAlloc.ToUInt64BigEndian(arr6, 0);
        }
        var t5 = sw.ElapsedTicks;

        Console.WriteLine($"{t1 - t0} {t3 - t2} {t5 - t4}");

        Console.ReadKey();
    }

    public static string ByteArrayToString(byte[] ba)
    {
        return string.Concat(ba.Select(b => Convert.ToString(b, 2).PadLeft(8, '0')));
    }
}

Original:

I'm confused about the bitwise way to convert c# data type short, int, long and ushort, uint, ulong to a byte array, and vice versa.

Performance is really really important to me.

I know there is a slow way of doing all these using BitConverter and Array.Reverse, but the performance is terrible.

I know there are basically two more approaches other than BitConverter, one is bitwise, and the other is unsafe.

After research on StackOverflow like:

Efficient way to read big-endian data in C#

Bitwise endian swap for various types

In C#, convert ulong[64] to byte[512] faster?

Fast string to byte[] conversion

I tried and tested bitwise method first, combining all these small pieces into one whole picture.

15555
43425534
54354444354
432
234234
34324432234

15555
43425534
-1480130482 // wrong
432
234234
34324432234

I'm even more confused about all these bit shifting, here is my testing code:

class Program
{
    static void Main()
    {
        short a = 15555;
        int b = 43425534;
        long c = 54354444354;
        ushort d = 432;
        uint e = 234234;
        ulong f = 34324432234;

        Console.WriteLine(a);
        Console.WriteLine(b);
        Console.WriteLine(c);
        Console.WriteLine(d);
        Console.WriteLine(e);
        Console.WriteLine(f);

        var arr1 = BitConverter.GetBytes(a);
        var arr2 = BitConverter.GetBytes(b);
        var arr3 = BitConverter.GetBytes(c);
        var arr4 = BitConverter.GetBytes(d);
        var arr5 = BitConverter.GetBytes(e);
        var arr6 = BitConverter.GetBytes(f);

        //Console.WriteLine(ByteArrayToString(arr1));
        //Console.WriteLine(ByteArrayToString(arr2));
        //Console.WriteLine(ByteArrayToString(arr3));
        //Console.WriteLine(ByteArrayToString(arr4));
        //Console.WriteLine(ByteArrayToString(arr5));
        //Console.WriteLine(ByteArrayToString(arr6));

        Array.Reverse(arr1);
        Array.Reverse(arr2);
        Array.Reverse(arr3);
        Array.Reverse(arr4);
        Array.Reverse(arr5);
        Array.Reverse(arr6);

        //Console.WriteLine(ByteArrayToString(arr1));
        //Console.WriteLine(ByteArrayToString(arr2));
        //Console.WriteLine(ByteArrayToString(arr3));
        //Console.WriteLine(ByteArrayToString(arr4));
        //Console.WriteLine(ByteArrayToString(arr5));
        //Console.WriteLine(ByteArrayToString(arr6));

        Console.WriteLine(ToInt16BigEndian(arr1, 0));
        Console.WriteLine(ToInt32BigEndian(arr2, 0));
        Console.WriteLine(ToInt64BigEndian(arr3, 0));
        Console.WriteLine(ToUInt16BigEndian(arr4, 0));
        Console.WriteLine(ToUInt32BigEndian(arr5, 0));
        Console.WriteLine(ToUInt64BigEndian(arr6, 0));

        Console.ReadKey();
    }

    public static string ByteArrayToString(byte[] ba)
    {
        return string.Concat(ba.Select(b => Convert.ToString(b, 2).PadLeft(8, '0')));
    }

    public static short ToInt16BigEndian(byte[] buf, int i)
    {
        return (short)((buf[i] << 8) | buf[i + 1]);
    }

    public static int ToInt32BigEndian(byte[] buf, int i)
    {
        return (buf[i] << 24) | (buf[i + 1] << 16) | (buf[i + 2] << 8) | buf[i + 3];
    }

    public static long ToInt64BigEndian(byte[] buf, int i)
    {
        return (buf[i] << 56) | (buf[i + 1] << 48) | (buf[i + 2] << 40) | (buf[i + 3] << 32) | (buf[i + 4] << 24) | (buf[i + 5] << 16) | (buf[i + 6] << 8) | buf[i + 7];
    }

    public static ushort ToUInt16BigEndian(byte[] buf, int i)
    {
        ushort value = 0;

        for (var j = 0; j < 2; j++)
        {
            value = (ushort)unchecked((value << 8) | buf[j + i]);
        }

        return value;
    }

    public static uint ToUInt32BigEndian(byte[] buf, int i)
    {
        uint value = 0;

        for (var j = 0; j < 4; j++)
        {
            value = unchecked((value << 8) | buf[j + i]);
        }

        return value;
    }

    public static ulong ToUInt64BigEndian(byte[] buf, int i)
    {
        ulong value = 0;

        for (var j = 0; j < 8; j++)
        {
            value = unchecked((value << 8) | buf[i + j]);
        }

        return value;
    }
}

I'm asking a solution that provides best performance, with consistent code style and cleanness of the code.

Church answered 28/11, 2017 at 10:27 Comment(4)

So what is slow exactly? Converting to byte array or back to number (or both)? – Osmen 28/11, 2017 at 10:47

@Osmen hi, the most common approach if performance is not that crucial is using bitconverter. The slowness will occur on both directions, these are the proven fact and I tested on my local machine as well, it is much slower than bitwise method. – Church 28/11, 2017 at 10:53

But how much slower is it compared to your performance requirements? Blindly trying to optimize without knowing when you've met your goals means that you could spend months trying to eke out better and better performance and ignore the fact that you got "good enough" after one days work and you've neglected to complete the rest of the product. – Skied 28/11, 2017 at 11:0

@Skied Yes. I totally agree with you. I had never used bitwise and unsafe to do such things. Bitconverter is always my first choice. But now this code will be used in financial related stuff, milliseconds or etc means more $$$, that is why I want to tune it at least beat our current implementation. – Church 28/11, 2017 at 11:6

I'm asking a solution that provides best performance, with consistent code style and cleanness of the code.

Nope; pick any two. You can't have all three. For example, if you're after the best performance, then you may have to compromise on some of those other things.

In the future, Span<T> (Span<byte>) will be pretty useful for this scenario - with several IO APIs gaining support for Span<T> and Memory<T> - but for now your best bet is probably unsafe code using a stackalloc byte* (or using fixed on a byte[]) and writing to that directly, using either shifting or masking with offsets in the "other endian" case.

Bonds answered 28/11, 2017 at 10:40 Comment(7)

Marc, thanks for answering my question. I tested with your suggested approach, it seems that bitwise method is faster than unsafe, since I'm not familiar with unsafe code, could you please help to see is there anything that I did wrong, the implementation and/or testing code. – Church 29/11, 2017 at 3:17

@Church see enclosed; key numbers after the change: Unsafe: 2149214 Bitwise: 741436 UnsafeRaw: 411032 (with UnsafeRaw being what I had in mind) - gist.github.com/mgravell/0379c974268d145be7c009615bc33ab4 ; note I haven't actually used stackalloc since you already have the arrays - I've gone for fixed instead. If you didn't already have arrays to pin with fixed, then stackalloc is a useful way to get a small buffer as a pointer. This will all be much prettier in C# vNext with Span<T> – Bonds 29/11, 2017 at 8:11

@Church note I also looked at a "thunked" version (smashing the pointer into the native type then reversing the bytes in-place in chunks), but it didn't improve the perf – Bonds 29/11, 2017 at 8:34

Tested on my local machine, UnsafeRaw about 20% faster than my bitwise version, but 80% faster with your testing. – Church 30/11, 2017 at 3:28

Marc, I suspect the performance is the same. Because my original test script is not accurate, my loop was dealing with the same byte array, that is why it performs faster. – Church 30/11, 2017 at 9:53

@Church it will also depend heavily on what runtime you target - I'm using .net core in my console rig, which has some awesome JIT improvements – Bonds 30/11, 2017 at 11:9

I see. Currently we are running on 4.5.1 – Church 30/11, 2017 at 11:10

Implemented and tested with my latest script, you can still further tune it, but the performance difference between Bitwise and Unsafe is really small.

Environment:

Intel Core i7-7700 CPU @3.6GHz
8.00 GB
64-bit
Windows 10 Pro
.NET FRAMEWORK 4.5.1

Here is my result:

Loop for 10 x 10,000,000 times

READ VALUE FROM BUFFER ANY CPU

Bitwise: 27270845; UnsafeRaw: 26828068;
UnsafeRaw is 1.65% faster than Bitwise

READ VALUE FROM BUFFER X64

Bitwise: 27210757; UnsafeRaw: 26847482;
UnsafeRaw is 1.35% faster than Bitwise

WRITE VALUE TO BUFFER ANY CPU

Bitwise: 26364519; UnsafeRaw: 26258470;
UnsafeRaw is 0.40% faster than Bitwise

WRITE VALUE TO BUFFER X64

Bitwise: 25728215; UnsafeRaw: 25733755;
UnsafeRaw is -0.02% faster than Bitwise

Paste my code here if anyone wants to do their own test.

Bitwise, read value from buffer

public static short ToInt16BigEndian(this byte[] buffer, int beginIndex)
{
    return unchecked((short)(buffer[beginIndex] << 8 | buffer[beginIndex + 1]));
}

public static int ToInt32BigEndian(this byte[] buffer, int beginIndex)
{
    return unchecked(buffer[beginIndex] << 24 | buffer[beginIndex + 1] << 16 | buffer[beginIndex + 2] << 8 | buffer[beginIndex + 3]);
}

public static long ToInt64BigEndian(this byte[] buffer, int beginIndex)
{
    return unchecked((long)buffer[beginIndex] << 56 | (long)buffer[beginIndex + 1] << 48 | (long)buffer[beginIndex + 2] << 40 | (long)buffer[beginIndex + 3] << 32 | (long)buffer[beginIndex + 4] << 24 | (long)buffer[beginIndex + 5] << 16 | (long)buffer[beginIndex + 6] << 8 | buffer[beginIndex + 7]);
}

public static ushort ToUInt16BigEndian(this byte[] buffer, int beginIndex)
{
    return unchecked((ushort)ToInt16BigEndian(buffer, beginIndex));
}

public static uint ToUInt32BigEndian(this byte[] buffer, int beginIndex)
{
    return unchecked((uint)ToInt32BigEndian(buffer, beginIndex));
}

public static ulong ToUInt64BigEndian(this byte[] buffer, int beginIndex)
{
    return unchecked((ulong)ToInt64BigEndian(buffer, beginIndex));
}

Bitwise, write value to buffer

public static void SetBuffer(this ushort value, byte[] arr, int beginIndex)
{
    arr[beginIndex] = (byte)(value >> 8);
    arr[beginIndex + 1] = (byte)value;
}

public static void SetBuffer(this uint value, byte[] arr, int beginIndex)
{
    arr[beginIndex] = (byte)(value >> 24);
    arr[beginIndex + 1] = (byte)(value >> 16);
    arr[beginIndex + 2] = (byte)(value >> 8);
    arr[beginIndex + 3] = (byte)value;
}

public static void SetBuffer(this int value, byte[] arr, int beginIndex)
{
    arr[beginIndex] = (byte)(value >> 24);
    arr[beginIndex + 1] = (byte)(value >> 16);
    arr[beginIndex + 2] = (byte)(value >> 8);
    arr[beginIndex + 3] = (byte)value;
}

public static void SetBuffer(this short value, byte[] arr, int beginIndex)
{
    arr[beginIndex] = (byte)(value >> 8);
    arr[beginIndex + 1] = (byte)value;
}

public static void SetBuffer(this ulong value, byte[] arr, int beginIndex)
{
    arr[beginIndex] = (byte)(value >> 56);
    arr[beginIndex + 1] = (byte)(value >> 48);
    arr[beginIndex + 2] = (byte)(value >> 40);
    arr[beginIndex + 3] = (byte)(value >> 32);
    arr[beginIndex + 4] = (byte)(value >> 24);
    arr[beginIndex + 5] = (byte)(value >> 16);
    arr[beginIndex + 6] = (byte)(value >> 8);
    arr[beginIndex + 7] = (byte)value;
}

public static void SetBuffer(this long value, byte[] arr, int beginIndex)
{
    arr[beginIndex] = (byte)(value >> 56);
    arr[beginIndex + 1] = (byte)(value >> 48);
    arr[beginIndex + 2] = (byte)(value >> 40);
    arr[beginIndex + 3] = (byte)(value >> 32);
    arr[beginIndex + 4] = (byte)(value >> 24);
    arr[beginIndex + 5] = (byte)(value >> 16);
    arr[beginIndex + 6] = (byte)(value >> 8);
    arr[beginIndex + 7] = (byte)value;
}

Unsafe, read value from buffer

public static short ToInt16BigEndian(byte* buf)
{
    return (short)ToUInt16BigEndian(buf);
}

public static int ToInt32BigEndian(byte* buf)
{
    return (int)ToUInt32BigEndian(buf);
}

public static long ToInt64BigEndian(byte* buf)
{
    return (long)ToUInt64BigEndian(buf);
}

public static ushort ToUInt16BigEndian(byte* buf)
{
    return (ushort)((*buf++ << 8) | *buf);
}

public static uint ToUInt32BigEndian(byte* buf)
{
    return unchecked((uint)((*buf++ << 24) | (*buf++ << 16) | (*buf++ << 8) | *buf));
}

public static ulong ToUInt64BigEndian(byte* buf)
{
    unchecked
    {
        var x = (uint)((*buf++ << 24) | (*buf++ << 16) | (*buf++ << 8) | *buf++);
        var y = (uint)((*buf++ << 24) | (*buf++ << 16) | (*buf++ << 8) | *buf);
        return ((ulong)x << 32) | y;
    }
}

Unsafe, write value to buffer

public static void SetBuffer(byte* arr, ushort value, int beginIndex)
{
    *(arr + beginIndex++) = (byte)(value >> 8);
    *(arr + beginIndex) = (byte)value;
}

public static void SetBuffer(byte* arr, uint value, int beginIndex)
{
    *(arr + beginIndex++) = (byte)(value >> 24);
    *(arr + beginIndex++) = (byte)(value >> 16);
    *(arr + beginIndex++) = (byte)(value >> 8);
    *(arr + beginIndex) = (byte)value;
}

public static void SetBuffer(byte* arr, ulong value, int beginIndex)
{
    *(arr + beginIndex++) = (byte)(value >> 56);
    *(arr + beginIndex++) = (byte)(value >> 48);
    *(arr + beginIndex++) = (byte)(value >> 40);
    *(arr + beginIndex++) = (byte)(value >> 32);
    *(arr + beginIndex++) = (byte)(value >> 24);
    *(arr + beginIndex++) = (byte)(value >> 16);
    *(arr + beginIndex++) = (byte)(value >> 8);
    *(arr + beginIndex) = (byte)value;
}

public static void SetBuffer(byte* arr, short value, int beginIndex)
{
    *(arr + beginIndex++) = (byte)(value >> 8);
    *(arr + beginIndex) = (byte)value;
}

public static void SetBuffer(byte* arr, int value, int beginIndex)
{
    *(arr + beginIndex++) = (byte)(value >> 24);
    *(arr + beginIndex++) = (byte)(value >> 16);
    *(arr + beginIndex++) = (byte)(value >> 8);
    *(arr + beginIndex) = (byte)value;
}

public static void SetBuffer(byte* arr, long value, int beginIndex)
{
    *(arr + beginIndex++) = (byte)(value >> 56);
    *(arr + beginIndex++) = (byte)(value >> 48);
    *(arr + beginIndex++) = (byte)(value >> 40);
    *(arr + beginIndex++) = (byte)(value >> 32);
    *(arr + beginIndex++) = (byte)(value >> 24);
    *(arr + beginIndex++) = (byte)(value >> 16);
    *(arr + beginIndex++) = (byte)(value >> 8);
    *(arr + beginIndex) = (byte)value;
}

Church answered 1/12, 2017 at 3:29 Comment(0)

Hot tags

Godot Unity Godot Help Programming Godot 4.X GUI GDScript 3D 2D Physics CSharp Godot 3.X VR XR Projects C++

Recommended topics

Hot tags