C#: Downloading a URL with timeout

Question

What's the best way to do it in .NET? I always forget what I need to Dispose() (or wrap with using).

EDIT: after a long time using WebRequest, I found out about customizing WebClient. Much better.

Adriel

Answer 1

Following Thomas Levesque's comment here, there's a simpler and more generic solution.

We create a WebClient subclass with timeout support, and we get all of WebClient's goodness.

public class WebClientWithTimeout : WebClient
{
  private readonly int timeoutMilliseconds;
  public WebClientWithTimeout(int timeoutMilliseconds)
  {
    this.timeoutMilliseconds = timeoutMilliseconds;
  }

  protected override WebRequest GetWebRequest(Uri address)
  {
    var result = base.GetWebRequest(address);
    result.Timeout = timeoutMilliseconds;
    return result;
  }
}

Sample usage:

public string GetRequest(Uri uri, int timeoutMilliseconds)
{
  using (var client = new WebClientWithTimeout(timeoutMilliseconds))
  {
    return client.DownloadString();
  }
}

Adriel

Answer 2

Syncronous Way:

var request = HttpWebRequest.Create("http://www.contoso.com");
request.Timeout = 50000;
using (var response = request.GetResponse())
{
    //your code here
}

You can also have the asynchronous way:

using System;
using System.Net;
using System.IO;
using System.Text;
using System.Threading;


public class RequestState
{
  // This class stores the State of the request.
  const int BUFFER_SIZE = 1024;
  public StringBuilder requestData;
  public byte[] BufferRead;
  public HttpWebRequest request;
  public HttpWebResponse response;
  public Stream streamResponse;
  public RequestState()
  {
    BufferRead = new byte[BUFFER_SIZE];
    requestData = new StringBuilder("");
    request = null;
    streamResponse = null;
  }
}

class HttpWebRequest_BeginGetResponse
{
    public static ManualResetEvent allDone = new ManualResetEvent(false);
    const int BUFFER_SIZE = 1024;
    const int DefaultTimeout = 2 * 60 * 1000; // 2 minutes timeout

    // Abort the request if the timer fires.
    private static void TimeoutCallback(object state, bool timedOut)
    {
        if (timedOut)
        {
            HttpWebRequest request = state as HttpWebRequest;
            if (request != null)
            {
                request.Abort();
            }
        }
    }

    static void Main()
    {

        try
        {
            // Create a HttpWebrequest object to the desired URL. 
            HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create("http://www.contoso.com");
            myHttpWebRequest.ReadWriteTimeout = DefaultTimeout;


            // Create an instance of the RequestState and assign the previous myHttpWebRequest
            // object to its request field.  
            RequestState myRequestState = new RequestState();
            myRequestState.request = myHttpWebRequest;


            // Start the asynchronous request.
            IAsyncResult result =
              (IAsyncResult)myHttpWebRequest.BeginGetResponse(new AsyncCallback(RespCallback), myRequestState);

            // this line implements the timeout, if there is a timeout, the callback fires and the request becomes aborted
            ThreadPool.RegisterWaitForSingleObject(result.AsyncWaitHandle, new WaitOrTimerCallback(TimeoutCallback), myHttpWebRequest, DefaultTimeout, true);

            // The response came in the allowed time. The work processing will happen in the 
            // callback function.
            allDone.WaitOne();

            // Release the HttpWebResponse resource.
            myRequestState.response.Close();
        }
        catch (WebException e)
        {
            Console.WriteLine("\nMain Exception raised!");
            Console.WriteLine("\nMessage:{0}", e.Message);
            Console.WriteLine("\nStatus:{0}", e.Status);
            Console.WriteLine("Press any key to continue..........");
        }
        catch (Exception e)
        {
            Console.WriteLine("\nMain Exception raised!");
            Console.WriteLine("Source :{0} ", e.Source);
            Console.WriteLine("Message :{0} ", e.Message);
            Console.WriteLine("Press any key to continue..........");
            Console.Read();
        }
    }
    private static void RespCallback(IAsyncResult asynchronousResult)
    {
        try
        {
            // State of request is asynchronous.
            RequestState myRequestState = (RequestState)asynchronousResult.AsyncState;
            HttpWebRequest myHttpWebRequest = myRequestState.request;
            myRequestState.response = (HttpWebResponse)myHttpWebRequest.EndGetResponse(asynchronousResult);

            // Read the response into a Stream object.
            Stream responseStream = myRequestState.response.GetResponseStream();
            myRequestState.streamResponse = responseStream;

            // Begin the Reading of the contents of the HTML page and print it to the console.
            IAsyncResult asynchronousInputRead = responseStream.BeginRead(myRequestState.BufferRead, 0, BUFFER_SIZE, new AsyncCallback(ReadCallBack), myRequestState);
            return;
        }
        catch (WebException e)
        {
            Console.WriteLine("\nRespCallback Exception raised!");
            Console.WriteLine("\nMessage:{0}", e.Message);
            Console.WriteLine("\nStatus:{0}", e.Status);
        }
        allDone.Set();
    }
    private static void ReadCallBack(IAsyncResult asyncResult)
    {
        try
        {

            RequestState myRequestState = (RequestState)asyncResult.AsyncState;
            Stream responseStream = myRequestState.streamResponse;
            int read = responseStream.EndRead(asyncResult);
            // Read the HTML page and then print it to the console.
            if (read > 0)
            {
                myRequestState.requestData.Append(Encoding.ASCII.GetString(myRequestState.BufferRead, 0, read));
                IAsyncResult asynchronousResult = responseStream.BeginRead(myRequestState.BufferRead, 0, BUFFER_SIZE, new AsyncCallback(ReadCallBack), myRequestState);
                return;
            }
            else
            {
                Console.WriteLine("\nThe contents of the Html page are : ");
                if (myRequestState.requestData.Length > 1)
                {
                    string stringContent;
                    stringContent = myRequestState.requestData.ToString();
                    Console.WriteLine(stringContent);
                }
                Console.WriteLine("Press any key to continue..........");
                Console.ReadLine();

                responseStream.Close();
            }

        }
        catch (WebException e)
        {
            Console.WriteLine("\nReadCallBack Exception raised!");
            Console.WriteLine("\nMessage:{0}", e.Message);
            Console.WriteLine("\nStatus:{0}", e.Status);
        }
        allDone.Set();

    }
}

Rapine

Answer 3

Here's what I use, it seems to work, but I don't know if it's the best way:

public string GetRequest(Uri uri, int timeoutMilliseconds)
{
    var request = System.Net.WebRequest.Create(uri);
    request.Timeout = timeoutMilliseconds;
    using (var response = request.GetResponse())
    using (var stream = response.GetResponseStream())
    using (var reader = new System.IO.StreamReader(stream))
    {
        return reader.ReadToEnd();
    }
}

Adriel

Not bad for simple url that doesn't require to send POST variable. You should tag that you use Framework 3.5... – Dorotea

I use only asynchronous requests for production code. In my answer you can find a giant asynchronous code I cloned from MSDN and added the line myHttpWebRequest.ReadWriteTimeout = DefaultTimeout; – Rapine

@Vernicht - thanks! I was looking for dead-simple blocking (synchronous) download, but the async code example is welcome! – Adriel

Answer 4

Run System.Net.WebClient in a seperate thread, set a timer to kill it after your maximum time.

Caution

You don't need a thread. If you want to do it asynchronously, You can use WebRequest's BeginGetResponse and EndGetResponse methods: msdn.microsoft.com/en-us/library/86wf6409.aspx – Adriel