Read lines from file async using WINAPI ReadFile

Question

I need to read several lines from file simultaneously, i.e. asynchronously. Lines in file are of the same size.

For instance, I need to read the second and the fourth lines of file to separate variables or to an array.

I'm more used to c#'s async/await and all these OVERLAPPED things are a bit difficult for understanding to me.

Using msdn examples, I achieved this, it just reads the data from the file as a one string (is that even asynchronous reading?):

BOOL ReadFromFileAsync(PCTSTR path)
{
    HANDLE hFile = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING,
        FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING, NULL);

    if (hFile == INVALID_HANDLE_VALUE)
    {
        _tprintf_s(TEXT("INVALID_HANDLE_VALUE\n"));
        return FALSE;
    }

    BOOL bResult;
    BYTE bReadBuf[2048];

    OVERLAPPED oRead = { 0 };
    oRead.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
    bResult = ReadFile(hFile, bReadBuf, _countof(bReadBuf), NULL, &oRead);

    if (!bResult && GetLastError() != ERROR_IO_PENDING)
    {
        _tprintf_s(TEXT("ERROR io pending"));
        CloseHandle(hFile);
        return FALSE;
    }

    // perform some stuff asynchronously
    _tprintf_s(TEXT("HEY\n"));

    HANDLE hEvents[2];
    hEvents[0] = oRead.hEvent;
    hEvents[1] = oRead.hEvent;
    DWORD dwWaitRes = WaitForMultipleObjects(_countof(hEvents), hEvents, FALSE, INFINITE);

    switch (dwWaitRes - WAIT_OBJECT_0)
    {
        case 0: // reading finished
            _tprintf_s(TEXT("String that was read from file: "));

            for (int i = 0; i < oRead.InternalHigh; ++i)
                _tprintf_s(TEXT("%c"), bReadBuf[i]);

            _tprintf_s(TEXT("\n"));
            break;

        default:
            _tprintf_s(TEXT("Nooo"));
    }

    CloseHandle(hFile);

    return TRUE;
}

Could you help me with reading two lines from file asynchronously?

Should I use SetFilePointer for that to move through the lines?

Brescia

Using overlapped I/O is only beneficial if you have something else to do while the async read is busy. You don't have anything useful to do, you call WaitForMultipleObjects() right after starting the read. So just don't use it. – Otalgia

@HansPassant: the code above does say: "perform some stuff asynchronously", so Denis just did not show what is being done while the reads are in progress... – Cooe

Answer 1

When you open a file with the FILE_FLAG_OVERLAPPED flag and then use an OVERLAPPED structure with ReadFile(), use the OVERLAPPED.Offset and OVERLAPPED.OffsetHigh fields to specify the byte offset where reading should start from. Also, you must use a separate OVERLAPPED instance for each ReadFile() if you run them simultaneously. This is clearly stated in the documentation:

If hFile is opened with FILE_FLAG_OVERLAPPED, the lpOverlapped parameter must point to a valid and unique OVERLAPPED structure, otherwise the function can incorrectly report that the read operation is complete.

For an hFile that supports byte offsets, if you use this parameter you must specify a byte offset at which to start reading from the file or device. This offset is specified by setting the Offset and OffsetHigh members of the OVERLAPPED structure. For an hFile that does not support byte offsets, Offset and OffsetHigh are ignored.

As your lines are the same length, you can easily calculate the offsets of the second and fourth lines and then issue two asynchronous ReadFile() calls for those offsets, and then wait for the two operations to complete as needed.

On a side note, you really should not be using FILE_FLAG_NO_BUFFERING unless you REALLY know what you are doing:

There are strict requirements for successfully working with files opened with CreateFile using the FILE_FLAG_NO_BUFFERING flag, for details see File Buffering.

Try something more like this instead:

#include <vector>

BOOL ReadFromFileAsync(PCTSTR path)
{
    BOOL bResult = FALSE;

    HANDLE hFile = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED /*| FILE_FLAG_NO_BUFFERING*/, NULL);    
    if (hFile == INVALID_HANDLE_VALUE)
    {
        _tprintf_s(TEXT("Error opening file: %s\n"), path);
        return FALSE;
    }

    DWORD dwLineSize = ...; // size of each line, in bytes
    std::vector<BYTE> bSecondLineBuf(dwLineSize);
    std::vector<BYTE> bFourthLineBuf(dwLineSize);

    OVERLAPPED oReadSecondLine = { 0 };
    OVERLAPPED oReadFourthLine = { 0 };

    oReadSecondLine.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
    if (!oReadSecondLine.hEvent)
    {
        _tprintf_s(TEXT("Error creating I/O event for reading second line\n"));
        goto done;
    }
    oReadSecondLine.Offset = ...; // offset of second line

    oReadFourthLine.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
    if (!oReadFourthLine.hEvent)
    {
        _tprintf_s(TEXT("Error creating I/O event for reading fourth line\n"));
        goto done;
    }
    oReadFourthLine.Offset = ...; // offset of fourth line

    if (!ReadFile(hFile, &bSecondLineBuf[0], dwLineSize, NULL, &oReadSecondLine))
    {
        if (GetLastError() != ERROR_IO_PENDING)
        {
            _tprintf_s(TEXT("Error starting I/O to read second line\n"));
            goto done;
        }
    }

    if (!ReadFile(hFile, &bFourthLineBuf[0], dwLineSize, NULL, &oReadFourthLine))
    {
        if (GetLastError() != ERROR_IO_PENDING)
        {
            _tprintf_s(TEXT("Error starting I/O to read fourth line\n"));
            CancelIo(hFile);
            goto done;
        }
    }

    // perform some stuff asynchronously
    _tprintf_s(TEXT("HEY\n"));

    HANDLE hEvents[2];
    hEvents[0] = oReadSecondLine.hEvent;
    hEvents[1] = oReadFourthLine.hEvent;

    DWORD dwWaitRes = WaitForMultipleObjects(_countof(hEvents), hEvents, TRUE, INFINITE);
    if (dwWaitRes == WAIT_FAILED)
    {
        _tprintf_s(TEXT("Error waiting for I/O to finish\n"));
        CancelIo(hFile);
        goto done;
    }    

    _tprintf_s(TEXT("Strings that were read from file: "));

    for (int i = 0; i < oReadSecondLine.InternalHigh; ++i)
        _tprintf_s(TEXT("%c"), (TCHAR) &bSecondLineBuf[i]);    
    _tprintf_s(TEXT("\n"));

    for (int i = 0; i < oReadFourthLine.InternalHigh; ++i)
        _tprintf_s(TEXT("%c"), (TCHAR) &bFourthLineBuf[i]);    
    _tprintf_s(TEXT("\n"));

done:
    if (oReadSecondLine.hEvent) CloseHandle(oReadSecondLine.hEvent);
    if (oReadFourthLine.hEvent) CloseHandle(oReadFourthLine.hEvent);
    CloseHandle(hFile);

    return bResult;
}

Alternatively:

#include <vector>

BOOL ReadFromFileAsync(PCTSTR path)
{
    BOOL bResult = FALSE;

    HANDLE hFile = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED /*| FILE_FLAG_NO_BUFFERING*/, NULL);    
    if (hFile == INVALID_HANDLE_VALUE)
    {
        _tprintf_s(TEXT("Error opening file: %s\n"), path);
        return FALSE;
    }

    DWORD dwLineSize = ...; // size of each line, in bytes
    std::vector<BYTE> bSecondLineBuf(dwLineSize);
    std::vector<BYTE> bFourthLineBuf(dwLineSize);

    OVERLAPPED oReadSecondLine = { 0 };
    OVERLAPPED oReadFourthLine = { 0 };

    oReadSecondLine.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
    if (!oReadSecondLine.hEvent)
    {
        _tprintf_s(TEXT("Error creating I/O event for reading second line\n"));
        goto done;
    }
    oReadSecondLine.Offset = ...; // offset of second line

    oReadFourthLine.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
    if (!oReadFourthLine.hEvent)
    {
        _tprintf_s(TEXT("Error creating I/O event for reading fourth line\n"));
        goto done;
    }
    oReadFourthLine.Offset = ...; // offset of fourth line

    if (!ReadFile(hFile, &bSecondLineBuf[0], dwLineSize, NULL, &oReadSecondLine))
    {
        if (GetLastError() != ERROR_IO_PENDING)
        {
            _tprintf_s(TEXT("Error starting I/O to read second line\n"));
            goto done;
        }
    }

    if (!ReadFile(hFile, &bFourthLineBuf[0], dwLineSize, NULL, &oReadFourthLine))
    {
        if (GetLastError() != ERROR_IO_PENDING)
        {
            _tprintf_s(TEXT("Error starting I/O to read fourth line\n"));
            CancelIo(hFile);
            goto done;
        }
    }

    // perform some stuff asynchronously
    _tprintf_s(TEXT("HEY\n"));

    HANDLE hEvents[2];
    hEvents[0] = oReadSecondLine.hEvent;
    hEvents[1] = oReadFourthLine.hEvent;

    OVERLAPPED* pOverlappeds[2];
    pOverlappeds[0] = &oReadSecondLine;
    pOverlappeds[1] = &oReadFourthLine;

    BYTE* pBufs[2];
    pBufs[0] = &bSecondLineBuf[0];
    pBufs[1] = &bFourthLineBuf[0];

    DWORD dwNumReads = _countof(hEvents);

    do
    {
        DWORD dwWaitRes = WaitForMultipleObjects(dwNumReads, hEvents, FALSE, INFINITE);
        if (dwWaitRes == WAIT_FAILED)
        {
            _tprintf_s(TEXT("Error waiting for I/O to finish\n"));
            CancelIo(hFile);
            goto done;
        }    

        if ((dwWaitRes >= WAIT_OBJECT_0) && (dwWaitRes < (WAIT_OBJECT_0+dwNumReads)))
        {
            DWORD dwIndex = dwWaitRes - WAIT_OBJECT_0;

            _tprintf_s(TEXT("String that was read from file: "));

            for (int i = 0; i < pOverlappeds[dwIndex]->InternalHigh; ++i)
                _tprintf_s(TEXT("%c"), (TCHAR) pBufs[dwIndex][i]);    
            _tprintf_s(TEXT("\n"));

           --dwNumReads;
           if (dwNumReads == 0)
               break;

           if (dwIndex == 0)
           {
               hEvents[0] = hEvents[1];
               pOverlappeds[0] = pOverlappeds[1];
               pBufs[0] = pBufs[1];
           }
        }
    }
    while (true);

done:
    if (oReadSecondLine.hEvent) CloseHandle(oReadSecondLine.hEvent);
    if (oReadFourthLine.hEvent) CloseHandle(oReadFourthLine.hEvent);
    CloseHandle(hFile);

    return bResult;
}

Cooe