C# Quickest Way to Get Average Colors of Screen

Question

I'm currently working on creating an Ambilight for my computer monitor with C#, an arduino, and an Ikea Dioder. Currently the hardware portion runs flawlessly; however, I'm having a problem with detecting the average color of a section of screen.

I have two issues with the implementations that I'm using:

1. Performance - Both of these algorithms add a somewhat noticeable stutter to the screen. Nothing showstopping, but it's annoying while watching video.

2. No Fullscreen Game Support - When a game is in fullscreen mode both of these methods just return white.

   public class DirectxColorProvider : IColorProvider
   {
   
       private static Device d;
       private static Collection<long> colorPoints;
   
       public DirectxColorProvider()
       {
           PresentParameters present_params = new PresentParameters();
           if (d == null)
           {
               d = new Device(new Direct3D(), 0, DeviceType.Hardware, IntPtr.Zero, CreateFlags.SoftwareVertexProcessing, present_params);
           }
           if (colorPoints == null)
           {
               colorPoints = GetColorPoints();
           }
       }
   
       public byte[] GetColors()
       {
           var color = new byte[4];
   
           using (var screen = this.CaptureScreen())
           {
               DataRectangle dr = screen.LockRectangle(LockFlags.None);
               using (var gs = dr.Data)
               {
                   color = avcs(gs, colorPoints);
               }
           }
   
           return color;
       }
   
       private Surface CaptureScreen()
       {
           Surface s = Surface.CreateOffscreenPlain(d, Screen.PrimaryScreen.Bounds.Width, Screen.PrimaryScreen.Bounds.Height, Format.A8R8G8B8, Pool.Scratch);
           d.GetFrontBufferData(0, s);
           return s;
       }
   
       private static byte[] avcs(DataStream gs, Collection<long> positions)
       {
           byte[] bu = new byte[4];
           int r = 0;
           int g = 0;
           int b = 0;
           int i = 0;
   
           foreach (long pos in positions)
           {
               gs.Position = pos;
               gs.Read(bu, 0, 4);
               r += bu[2];
               g += bu[1];
               b += bu[0];
               i++;
           }
   
           byte[] result = new byte[3];
           result[0] = (byte)(r / i);
           result[1] = (byte)(g / i);
           result[2] = (byte)(b / i);
   
           return result;
       }
   
       private Collection<long> GetColorPoints()
       {
           const long offset = 20;
           const long Bpp = 4;
   
           var box = GetBox();
   
           var colorPoints = new Collection<long>();
           for (var x = box.X; x < (box.X + box.Length); x += offset)
           {
               for (var y = box.Y; y < (box.Y + box.Height); y += offset)
               {
                   long pos = (y * Screen.PrimaryScreen.Bounds.Width + x) * Bpp;
                   colorPoints.Add(pos);
               }
           }
   
           return colorPoints;
       }
   
       private ScreenBox GetBox()
       {
           var box = new ScreenBox();
   
           int m = 8;
   
           box.X = (Screen.PrimaryScreen.Bounds.Width - m) / 3;
           box.Y = (Screen.PrimaryScreen.Bounds.Height - m) / 3;
   
           box.Length = box.X * 2;
           box.Height = box.Y * 2;
   
           return box;
       }
   
       private class ScreenBox
       {
           public long X { get; set; }
           public long Y { get; set; }
           public long Length { get; set; }
           public long Height { get; set; }
       }
   
   }
   

You can find the file for the directX implmentation here.

public class GDIColorProvider : Form, IColorProvider
{
    private static Rectangle box;
    private readonly IColorHelper _colorHelper;

    public GDIColorProvider()
    {
        _colorHelper = new ColorHelper();
        box = _colorHelper.GetCenterBox();
    }

    public byte[] GetColors()
    {
        var colors = new byte[3];

        IntPtr hDesk = GetDesktopWindow();
        IntPtr hSrce = GetDC(IntPtr.Zero);
        IntPtr hDest = CreateCompatibleDC(hSrce);
        IntPtr hBmp = CreateCompatibleBitmap(hSrce, box.Width, box.Height);
        IntPtr hOldBmp = SelectObject(hDest, hBmp);
        bool b = BitBlt(hDest, box.X, box.Y, (box.Width - box.X), (box.Height - box.Y), hSrce, 0, 0, CopyPixelOperation.SourceCopy);
        using(var bmp = Bitmap.FromHbitmap(hBmp))
        {
            colors = _colorHelper.AverageColors(bmp);
        }

        SelectObject(hDest, hOldBmp);
        DeleteObject(hBmp);
        DeleteDC(hDest);
        ReleaseDC(hDesk, hSrce);

        return colors;
    }

    // P/Invoke declarations
    [DllImport("gdi32.dll")]
    static extern bool BitBlt(IntPtr hdcDest, int xDest, int yDest, int
    wDest, int hDest, IntPtr hdcSource, int xSrc, int ySrc, CopyPixelOperation rop);
    [DllImport("user32.dll")]
    static extern bool ReleaseDC(IntPtr hWnd, IntPtr hDc);
    [DllImport("gdi32.dll")]
    static extern IntPtr DeleteDC(IntPtr hDc);
    [DllImport("gdi32.dll")]
    static extern IntPtr DeleteObject(IntPtr hDc);
    [DllImport("gdi32.dll")]
    static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight);
    [DllImport("gdi32.dll")]
    static extern IntPtr CreateCompatibleDC(IntPtr hdc);
    [DllImport("gdi32.dll")]
    static extern IntPtr SelectObject(IntPtr hdc, IntPtr bmp);
    [DllImport("user32.dll")]
    private static extern IntPtr GetDesktopWindow();
    [DllImport("user32.dll")]
    private static extern IntPtr GetWindowDC(IntPtr ptr);
    [DllImport("user32.dll")]
    private static extern IntPtr GetDC(IntPtr ptr);
}

You Can Find the File for the GDI implementation Here.

The Full Codebase Can be Found Here.

Fascista

For the stuttering, have you profiled to see what takes so long? You might see better speed by reading every, say, 1000th pixel (still a sample of ~2,000 points from a 1080p screen; you could choose an even larger value) and only doing that every 10th frame. – Repurchase

For the DirectX option, 90% of the time executing the method d.GetFrontBufferData(0, s);. For the GDI option, 70% of the time is spent on BitBlt(hDest, box.X, box.Y, (box.Width - box.X), (box.Height - box.Y), hSrce, 0, 0, CopyPixelOperation.SourceCopy); – Fascista

Also regarding the timing; I have the methods running in a timer on a winforms app that is ticking everyone 25ms. I've upped it to every 100ms without a noticeable improvement. – Fascista

GetFrontBufferData's documentation says "The method is slow by design, and therefore should not be used in a performance-critical path.". Have you tried GetBackBuffer? – Repurchase

Instead of getting one large box, how much time would it take to get many (e.g. 200) small (e.g. 1 pixel) boxes with BitBlt? – Repurchase

Actually, probably better to use GetPixel if you want to try that approach. – Repurchase

You aren't going to be able to easily get the color on the screen for fullscreen applications such as games or some video players. They draw directly to the screen and they don't always expose the ability to read from their buffers. Even when they do you have to explicitly write code specific to DirectX/OpenGL/etc. – Industrialism

For desktop use including windowed applications etc., you should be able to simply use Graphics.CopyFromScreen in C#. If you copy an entire screen it's going to take some time... maybe about 100ms +- 50ms. If you're only doing this like 5 times per second it shouldn't be too bad performance-wise. What kind of lag are you experiencing? Using other applications in Windows while this is running? – Industrialism

If you want to improve the performance you should call Graphics.CopyFromScreen multiple times to read small sections of the screen. You would get exponentially better performance if you only read about 10-20% of the screen spread across lots of little evenly distributed rectangles. Though you would lose some accuracy. – Industrialism

I'm going to test out all of these options tonight with benchmarks. I'll be reporting back either tonight or tomorrow the results of those tests in the form of an answer to see which one is the fastest. – Fascista

How about using a shader to write the screen output to a texture and then using the maximum mip map level to generate a one pixel average and then sampling this back to the CPU from a staging resource? Using mip maps is an approach used in HDR techniques to derive the average luminosity – Crossquestion

Answer 1

Updated Answer

The problem of slow screen capture performance most likely is caused by BitBlt() doing a pixel conversion when the pixel formats of source and destination don't match. From the docs:

If the color formats of the source and destination device contexts do not match, the BitBlt function converts the source color format to match the destination format.

This is what caused slow performance in my code, especially in higher resolutions.

The default pixel format seems to be PixelFormat.Format32bppArgb, so that's what you should use for the buffer:

var screen = new Bitmap(bounds.Width, bounds.Height, PixelFormat.Format32bppArgb);
var gfx = Graphics.FromImage(screen);
gfx.CopyFromScreen(bounds.Location, new Point(0, 0), bounds.Size);

The next source for slow performance is Bitmap.GetPixel() which does boundary checks. Never use it when analyzing every pixel. Instead lock the bitmap data and get a pointer to it:

public unsafe Color GetAverageColor(Bitmap image, int sampleStep = 1) {
    var data = image.LockBits(
        new Rectangle(Point.Empty, Image.Size),
        ImageLockMode.ReadOnly,
        PixelFormat.Format32bppArgb);

    var row = (int*)data.Scan0.ToPointer();
    var (sumR, sumG, sumB) = (0L, 0L, 0L);
    var stride = data.Stride / sizeof(int) * sampleStep;

    for (var y = 0; y < data.Height; y += sampleStep) {
        for (var x = 0; x < data.Width; x += sampleStep) {
            var argb = row[x];
            sumR += (argb & 0x00FF0000) >> 16;
            sumG += (argb & 0x0000FF00) >> 8;
            sumB += argb & 0x000000FF;
        }
        row += stride;
    }

    image.UnlockBits(data);

    var numSamples = data.Width / sampleStep * data.Height / sampleStep;
    var avgR = sumR / numSamples;
    var avgG = sumG / numSamples;
    var avgB = sumB / numSamples;
    return Color.FromArgb((int)avgR, (int)avgG, (int)avgB);
}

This should get you well below 10 ms, depending on the screen size. In case it is still too slow you can increase the sampleStep parameter of GetAverageColor().

Original Answer

I recently did the same thing and came up with something that worked surprisingly good.

The trick is to create an additional bitmap that is 1x1 pixels in size and set a good interpolation mode on its graphics context (bilinear or bicubic, but not nearest neighbor).

Then you draw your captured bitmap into that 1x1 bitmap exploiting the interpolation and retrieve that pixel to get the average color.

I'm doing that at a rate of ~30 fps. When the screen shows a GPU rendering (e.g. watching YouTube full screen with enabled hardware acceleration in Chrome) there is no visible stuttering or anything. In fact, CPU utilization of the application is way below 10%. However, if I turn off Chrome's hardware acceleration then there is definitely some slight stuttering noticeable if you watch close enough.

Here are the relevant parts of the code:

using var screen = new Bitmap(width, height);
using var screenGfx = Graphics.FromImage(screen);

using var avg = new Bitmap(1, 1);
using var avgGfx = Graphics.FromImage(avg);
avgGfx.InterpolationMode = InterpolationMode.HighQualityBicubic;

while (true) {
    screenGfx.CopyFromScreen(left, top, 0, 0, screen.Size);
    avgGfx.DrawImage(screen, 0, 0, avg.Width, avg.Height);
    var color = avg.GetPixel(0, 0);
    var bright = (int)Math.Round(Math.Clamp(color.GetBrightness() * 100, 1, 100));
    // set color and brightness on your device
    // wait 1000/fps milliseconds
}

Note that this works for GPU renderings, because System.Drawing.Common uses GDI+ nowadays. However, it does not work when the content is DRM protected. So it won't work with Netflix for example :(

I published the code on GitHub. Even though I abandoned the project due to Netflix' DRM protection it might help someone else.

Tann