[core] switch to async I/O for image writing

* Combined with the increase in buffer size from previous commits, this
  should help us get close to a device's maximum write speed.
* Also add async write support to winio.h
* Also increase the buffer size for bad blocks check operations

res/appstore/Package.appxmanifest

@@ -11,7 +11,7 @@
   <Identity
     Name="19453.net.Rufus"
     Publisher="CN=7AC86D13-3E5A-491A-ADD5-80095C212740"
-    Version="3.14.1775.0" />
+    Version="3.14.1776.0" />
 
   <Properties>
     <DisplayName>Rufus</DisplayName>

src/format.c

@@ -37,6 +37,7 @@
 #include "missing.h"
 #include "resource.h"
 #include "settings.h"
+#include "winio.h"
 #include "msapi_utf8.h"
 #include "localization.h"
 
@@ -52,6 +53,9 @@
 #include "bled/bled.h"
 #include "../res/grub/grub_version.h"
 
+/* Numbers of buffer used for asynchronous DD reads */
+#define NUM_BUFFERS 2
+
 /*
  * Globals
  */
@@ -1489,13 +1493,13 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 	BOOL s, ret = FALSE;
 	LARGE_INTEGER li;
 	HANDLE hSourceImage = INVALID_HANDLE_VALUE;
-	DWORD i, read_size, write_size, comp_size, buf_size;
+	DWORD i, read_size[NUM_BUFFERS], write_size, comp_size, buf_size;
 	uint64_t wb, target_size = bZeroDrive ? SelectedDrive.DiskSize : img_report.image_size;
 	uint64_t cur_value, last_value = UINT64_MAX;
 	int64_t bled_ret;
 	uint8_t* buffer = NULL;
 	uint32_t zero_data, *cmp_buffer = NULL;
-	int throttle_fast_zeroing = 0;
+	int throttle_fast_zeroing = 0, read_bufnum = 0, proc_bufnum = 1;
 
 	if (SelectedDrive.SectorSize < 512) {
 		uprintf("Unexpected sector size (%d) - Aborting", SelectedDrive.SectorSize);
@@ -1533,7 +1537,7 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 			assert((uintptr_t)cmp_buffer % SelectedDrive.SectorSize == 0);
 		}
 
-		read_size = buf_size;
+		read_size[0] = buf_size;
 		for (wb = 0, write_size = 0; wb < target_size; wb += write_size) {
 			UpdateProgressWithInfo(OP_FORMAT, fast_zeroing ? MSG_306 : MSG_286, wb, target_size);
 			cur_value = (wb * min(80, target_size)) / target_size;
@@ -1542,13 +1546,13 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 				uprintfs("+");
 			}
 			// Don't overflow our projected size (mostly for VHDs)
-			if (wb + read_size > target_size) {
+			if (wb + read_size[0] > target_size) {
-				read_size = (DWORD)(target_size - wb);
+				read_size[0] = (DWORD)(target_size - wb);
 			}
 
 			// WriteFile fails unless the size is a multiple of sector size
-			if (read_size % SelectedDrive.SectorSize != 0)
+			if (read_size[0] % SelectedDrive.SectorSize != 0)
-				read_size = ((read_size + SelectedDrive.SectorSize - 1) / SelectedDrive.SectorSize) * SelectedDrive.SectorSize;
+				read_size[0] = ((read_size[0] + SelectedDrive.SectorSize - 1) / SelectedDrive.SectorSize) * SelectedDrive.SectorSize;
 
 			// Fast-zeroing: Depending on your hardware, reading from flash may be much faster than writing, so
 			// we might speed things up by skipping empty blocks, or skipping the write if the data is the same.
@@ -1561,8 +1565,8 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 				CHECK_FOR_USER_CANCEL;
 
 				// Read block and compare against the block that needs to be written
-				s = ReadFile(hPhysicalDrive, cmp_buffer, read_size, &comp_size, NULL);
+				s = ReadFile(hPhysicalDrive, cmp_buffer, read_size[0], &comp_size, NULL);
-				if ((!s) || (comp_size != read_size)) {
+				if ((!s) || (comp_size != read_size[0])) {
 					uprintf("Read error: Could not read data for fast zeroing comparison - %s", WindowsErrorString());
 					goto out;
 				}
@@ -1572,10 +1576,10 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 				// Check all bits are the same
 				if ((zero_data == 0) || (zero_data == 0xffffffff)) {
 					// Compare the rest of the block against the first element
-					for (i = 1; (i < read_size / sizeof(uint32_t)) && (cmp_buffer[i] == zero_data); i++);
+					for (i = 1; (i < read_size[0] / sizeof(uint32_t)) && (cmp_buffer[i] == zero_data); i++);
-					if (i >= read_size / sizeof(uint32_t)) {
+					if (i >= read_size[0] / sizeof(uint32_t)) {
 						// Block is empty, skip write
-						write_size = read_size;
+						write_size = read_size[0];
 						continue;
 					}
 				}
@@ -1592,11 +1596,11 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 
 			for (i = 1; i <= WRITE_RETRIES; i++) {
 				CHECK_FOR_USER_CANCEL;
-				s = WriteFile(hPhysicalDrive, buffer, read_size, &write_size, NULL);
+				s = WriteFile(hPhysicalDrive, buffer, read_size[0], &write_size, NULL);
-				if ((s) && (write_size == read_size))
+				if ((s) && (write_size == read_size[0]))
 					break;
 				if (s)
-					uprintf("Write error: Wrote %d bytes, expected %d bytes", write_size, read_size);
+					uprintf("Write error: Wrote %d bytes, expected %d bytes", write_size, read_size[0]);
 				else
 					uprintf("Write error at sector %lld: %s", wb / SelectedDrive.SectorSize, WindowsErrorString());
 				if (i < WRITE_RETRIES) {
@@ -1652,9 +1656,9 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 			goto out;
 		}
 	} else {
-		hSourceImage = CreateFileU(image_path, GENERIC_READ, FILE_SHARE_READ, NULL,
+		hSourceImage = CreateFileAsync(image_path, GENERIC_READ, FILE_SHARE_READ,
-			OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
+			OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN);
-		if (hSourceImage == INVALID_HANDLE_VALUE) {
+		if (hSourceImage == NULL) {
 			uprintf("Could not open image '%s': %s", image_path, WindowsErrorString());
 			FormatStatus = ERROR_SEVERITY_ERROR | FAC(FACILITY_STORAGE) | ERROR_OPEN_FAILED;
 			goto out;
@@ -1662,7 +1666,7 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 
 		// Our buffer size must be a multiple of the sector size and *ALIGNED* to the sector size
 		buf_size = ((DD_BUFFER_SIZE + SelectedDrive.SectorSize - 1) / SelectedDrive.SectorSize) * SelectedDrive.SectorSize;
-		buffer = (uint8_t*)_mm_malloc(buf_size, SelectedDrive.SectorSize);
+		buffer = (uint8_t*)_mm_malloc(buf_size * NUM_BUFFERS, SelectedDrive.SectorSize);
 		if (buffer == NULL) {
 			FormatStatus = ERROR_SEVERITY_ERROR | FAC(FACILITY_STORAGE) | ERROR_NOT_ENOUGH_MEMORY;
 			uprintf("Could not allocate disk write buffer");
@@ -1670,41 +1674,51 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 		}
 		assert((uintptr_t)buffer % SelectedDrive.SectorSize == 0);
 
-		// Don't bother trying for something clever, using double buffering overlapped and whatnot:
+		// Start the initial read
-		// With Windows' default optimizations, sync read + sync write for sequential operations
+		ReadFileAsync(hSourceImage, &buffer[read_bufnum * buf_size], buf_size);
-		// will be as fast, if not faster, than whatever async scheme you can come up with.
+
-		read_size = buf_size;
+		read_size[proc_bufnum] = 1;	// To avoid early loop exit
-		for (wb = 0, write_size = 0; wb < target_size; wb += write_size) {
+		for (wb = 0; read_size[proc_bufnum] != 0; wb += read_size[proc_bufnum]) {
+			// 0. Update the progress
 			UpdateProgressWithInfo(OP_FORMAT, MSG_261, wb, target_size);
 			cur_value = (wb * min(80, target_size)) / target_size;
 			if (cur_value != last_value) {
 				last_value = cur_value;
 				uprintfs("+");
 			}
-			s = ReadFile(hSourceImage, buffer, buf_size, &read_size, NULL);
+
-			if (!s) {
+			// 1. Wait for the current read operation to complete (and update the read size)
-				FormatStatus = ERROR_SEVERITY_ERROR | FAC(FACILITY_STORAGE) | ERROR_READ_FAULT;
+			if ((!WaitFileAsync(hSourceImage, DRIVE_ACCESS_TIMEOUT)) ||
+				(!GetSizeAsync(hSourceImage, &read_size[read_bufnum]))) {
 				uprintf("Read error: %s", WindowsErrorString());
+				FormatStatus = ERROR_SEVERITY_ERROR | FAC(FACILITY_STORAGE) | ERROR_READ_FAULT;
 				goto out;
 			}
-			if (read_size == 0)
-				break;
-			// Don't overflow our projected size (mostly for VHDs)
-			if (wb + read_size > target_size) {
-				read_size = (DWORD)(target_size - wb);
-			}
 
-			// WriteFile fails unless the size is a multiple of sector size
+			// 2. Update the read size
-			if (read_size % SelectedDrive.SectorSize != 0)
+			// 2a) Don't overflow our projected size (mostly for VHDs)
-				read_size = ((read_size + SelectedDrive.SectorSize - 1) / SelectedDrive.SectorSize) * SelectedDrive.SectorSize;
+			if (wb + read_size[read_bufnum] > target_size)
+				read_size[read_bufnum] = (DWORD)(target_size - wb);
+			// 2b) WriteFile fails unless the size is a multiple of sector size
+			if (read_size[read_bufnum] % SelectedDrive.SectorSize != 0)
+				read_size[read_bufnum] = ((read_size[read_bufnum] + SelectedDrive.SectorSize - 1) /
+					SelectedDrive.SectorSize) * SelectedDrive.SectorSize;
 
+			// 3. Switch to the next reading buffer
+			proc_bufnum = read_bufnum;
+			read_bufnum = (read_bufnum + 1) % NUM_BUFFERS;
+
+			// 3. Launch the next asynchronous read operation
+			ReadFileAsync(hSourceImage, &buffer[read_bufnum * buf_size], buf_size);
+
+			// 4. Synchronously write the current data buffer
 			for (i = 1; i <= WRITE_RETRIES; i++) {
 				CHECK_FOR_USER_CANCEL;
-				s = WriteFile(hPhysicalDrive, buffer, read_size, &write_size, NULL);
+				s = WriteFile(hPhysicalDrive, &buffer[proc_bufnum * buf_size], read_size[proc_bufnum], &write_size, NULL);
-				if ((s) && (write_size == read_size))
+				if ((s) && (write_size == read_size[proc_bufnum]))
 					break;
 				if (s)
-					uprintf("Write error: Wrote %d bytes, expected %d bytes", write_size, read_size);
+					uprintf("Write error: Wrote %d bytes, expected %d bytes", write_size, read_size[proc_bufnum]);
 				else
 					uprintf("Write error at sector %lld: %s", wb / SelectedDrive.SectorSize, WindowsErrorString());
 				if (i < WRITE_RETRIES) {
@@ -1729,7 +1743,10 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive)
 	RefreshDriveLayout(hPhysicalDrive);
 	ret = TRUE;
 out:
+	if (img_report.compression_type != BLED_COMPRESSION_NONE)
 		safe_closehandle(hSourceImage);
+	else
+		CloseFileAsync(hSourceImage);
 	safe_mm_free(buffer);
 	safe_mm_free(cmp_buffer);
 	return ret;

src/rufus.h

@@ -98,7 +98,7 @@
 #define BADBLOCK_PATTERN_SLC        {0x00, 0xff, 0x55, 0xaa}
 #define BADCLOCK_PATTERN_MLC        {0x00, 0xff, 0x33, 0xcc}
 #define BADBLOCK_PATTERN_TLC        {0x00, 0xff, 0x1c71c7, 0xe38e38}
-#define BADBLOCK_BLOCK_SIZE         (128 * 1024)
+#define BADBLOCK_BLOCK_SIZE         (512 * 1024)
 #define LARGE_FAT32_SIZE            (32 * 1073741824LL)	// Size at which we need to use fat32format
 #define UDF_FORMAT_SPEED            3.1f		// Speed estimate at which we expect UDF drives to be formatted (GB/s)
 #define UDF_FORMAT_WARN             20			// Duration (in seconds) above which we warn about long UDF formatting times

src/rufus.rc

@@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL
 IDD_DIALOG DIALOGEX 12, 12, 232, 326
 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU
 EXSTYLE WS_EX_ACCEPTFILES
-CAPTION "Rufus 3.14.1775"
+CAPTION "Rufus 3.14.1776"
 FONT 9, "Segoe UI Symbol", 400, 0, 0x0
 BEGIN
     LTEXT           "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP
@@ -395,8 +395,8 @@ END
 //
 
 VS_VERSION_INFO VERSIONINFO
- FILEVERSION 3,14,1775,0
+ FILEVERSION 3,14,1776,0
- PRODUCTVERSION 3,14,1775,0
+ PRODUCTVERSION 3,14,1776,0
  FILEFLAGSMASK 0x3fL
 #ifdef _DEBUG
  FILEFLAGS 0x1L
@@ -414,13 +414,13 @@ BEGIN
             VALUE "Comments", "https://rufus.ie"
             VALUE "CompanyName", "Akeo Consulting"
             VALUE "FileDescription", "Rufus"
-            VALUE "FileVersion", "3.14.1775"
+            VALUE "FileVersion", "3.14.1776"
             VALUE "InternalName", "Rufus"
             VALUE "LegalCopyright", "© 2011-2021 Pete Batard (GPL v3)"
             VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html"
             VALUE "OriginalFilename", "rufus-3.14.exe"
             VALUE "ProductName", "Rufus"
-            VALUE "ProductVersion", "3.14.1775"
+            VALUE "ProductVersion", "3.14.1776"
         END
     END
     BLOCK "VarFileInfo"

src/winio.h

@@ -25,7 +25,7 @@
 
 // https://docs.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-overlapped
 // See Microsoft? It's not THAT hard to define an OVERLAPPED struct in a manner that
-// doesn't qualify as an example of "Crimes against humanity" for the Geneva convention.
+// doesn't qualify as an example of "Crimes against humanity" in the Geneva convention.
 typedef struct {
 	ULONG_PTR                           Internal[2];
 	ULONG64                             Offset;
@@ -57,7 +57,7 @@ typedef struct {
 /// <param name="dwCreationDisposition">Action to take on a file or device that exists or does not exist</param>
 /// <param name="dwFlagsAndAttributes">The file or device attributes and flags</param>
 /// <returns>Non NULL on success</returns>
-static __inline VOID* CreateFileAsync(LPCSTR lpFileName, DWORD dwDesiredAccess,
+static __inline HANDLE CreateFileAsync(LPCSTR lpFileName, DWORD dwDesiredAccess,
 	DWORD dwShareMode, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes)
 {
 	ASYNC_FD* fd = calloc(sizeof(ASYNC_FD), 1);
@@ -79,77 +79,97 @@ static __inline VOID* CreateFileAsync(LPCSTR lpFileName, DWORD dwDesiredAccess,
 /// <summary>
 /// Close a previously opened asynchronous file
 /// </summary>
-/// <param name="fd">The file descriptor</param>
+/// <param name="h">An async handle, created by a call to CreateFileAsync()</param>
-static __inline VOID CloseFileAsync(VOID* fd)
+static __inline VOID CloseFileAsync(HANDLE h)
 {
-	ASYNC_FD* _fd = (ASYNC_FD*)fd;
+	ASYNC_FD* fd = (ASYNC_FD*)h;
-	if (_fd == NULL)
+	if (fd == NULL || fd == INVALID_HANDLE_VALUE)
 		return;
-	CloseHandle(_fd->hFile);
+	CloseHandle(fd->hFile);
-	CloseHandle(_fd->Overlapped.hEvent);
+	CloseHandle(fd->Overlapped.hEvent);
-	free(_fd);
+	free(fd);
 }
 
 /// <summary>
 /// Initiate a read operation for asynchronous I/O.
 /// </summary>
-/// <param name="fd">The file descriptor</param>
+/// <param name="h">An async handle, created by a call to CreateFileAsync()</param>
 /// <param name="lpBuffer">The buffer that receives the data</param>
 /// <param name="nNumberOfBytesToRead">Number of bytes requested</param>
 /// <returns>TRUE on success, FALSE on error</returns>
-static __inline BOOL ReadFileAsync(VOID* fd, LPVOID lpBuffer, DWORD nNumberOfBytesToRead)
+static __inline BOOL ReadFileAsync(HANDLE h, LPVOID lpBuffer, DWORD nNumberOfBytesToRead)
 {
-	ASYNC_FD* _fd = (ASYNC_FD*)fd;
+	ASYNC_FD* fd = (ASYNC_FD*)h;
-	_fd->Overlapped.bOffsetUpdated = FALSE;
+	fd->Overlapped.bOffsetUpdated = FALSE;
-	if (!ReadFile(_fd->hFile, lpBuffer, nNumberOfBytesToRead, NULL,
+	if (!ReadFile(fd->hFile, lpBuffer, nNumberOfBytesToRead, NULL,
-		(OVERLAPPED*)&_fd->Overlapped))
+		(OVERLAPPED*)&fd->Overlapped))
 		// TODO: Is it possible to get ERROR_HANDLE_EOF here?
-		_fd->iStatus = (GetLastError() == ERROR_IO_PENDING) ? -1 : 0;
+		fd->iStatus = (GetLastError() == ERROR_IO_PENDING) ? -1 : 0;
 	else
-		_fd->iStatus = 1;
+		fd->iStatus = 1;
-	return (_fd->iStatus != 0);
+	return (fd->iStatus != 0);
+}
+
+/// <summary>
+/// Initiate a write operation for asynchronous I/O.
+/// </summary>
+/// <param name="h">An async handle, created by a call to CreateFileAsync()</param>
+/// <param name="lpBuffer">The buffer that contains the data</param>
+/// <param name="nNumberOfBytesToWrite">Number of bytes to write</param>
+/// <returns>TRUE on success, FALSE on error</returns>
+static __inline BOOL WriteFileAsync(HANDLE h, LPVOID lpBuffer, DWORD nNumberOfBytesToWrite)
+{
+	ASYNC_FD* fd = (ASYNC_FD*)h;
+	fd->Overlapped.bOffsetUpdated = FALSE;
+	if (!WriteFile(fd->hFile, lpBuffer, nNumberOfBytesToWrite, NULL,
+		(OVERLAPPED*)&fd->Overlapped))
+		// TODO: Is it possible to get ERROR_HANDLE_EOF here?
+		fd->iStatus = (GetLastError() == ERROR_IO_PENDING) ? -1 : 0;
+	else
+		fd->iStatus = 1;
+	return (fd->iStatus != 0);
 }
 
 /// <summary>
 /// Wait for an asynchronous operation to complete, with timeout.
 /// This function also succeeds if the I/O already completed synchronously.
 /// </summary>
-/// <param name="fd">The file descriptor</param>
+/// <param name="h">An async handle, created by a call to CreateFileAsync()</param>
 /// <param name="dwTimeout">A timeout value, in ms</param>
 /// <returns>TRUE on success, FALSE on error</returns>
-static __inline BOOL WaitFileAsync(VOID* fd, DWORD dwTimeout)
+static __inline BOOL WaitFileAsync(HANDLE h, DWORD dwTimeout)
 {
-	ASYNC_FD* _fd = (ASYNC_FD*)fd;
+	ASYNC_FD* fd = (ASYNC_FD*)h;
-	if (_fd->iStatus > 0)	// Read completed synchronously
+	if (fd->iStatus > 0)	// Read completed synchronously
 		return TRUE;
-	return (WaitForSingleObject(_fd->Overlapped.hEvent, dwTimeout) == WAIT_OBJECT_0);
+	return (WaitForSingleObject(fd->Overlapped.hEvent, dwTimeout) == WAIT_OBJECT_0);
 }
 
 /// <summary>
-/// Return the number of bytes read and keep track/update the current offset
+/// Return the number of bytes read or written and keep track/update the
-/// for an asynchronous read operation.
+/// current offset for an asynchronous read operation.
 /// </summary>
-/// <param name="fd">The file descriptor</param>
+/// <param name="h">An async handle, created by a call to CreateFileAsync()</param>
-/// <param name="lpNumberOfBytesRead">A pointer that receives the number of bytes read.</param>
+/// <param name="lpNumberOfBytes">A pointer that receives the number of bytes transferred.</param>
 /// <returns>TRUE on success, FALSE on error</returns>
-static __inline BOOL GetSizeAsync(VOID* fd, LPDWORD lpNumberOfBytesRead)
+static __inline BOOL GetSizeAsync(HANDLE h, LPDWORD lpNumberOfBytes)
 {
-	ASYNC_FD* _fd = (ASYNC_FD*)fd;
+	ASYNC_FD* fd = (ASYNC_FD*)h;
-	// Previous call to ReadFileAsync() failed
+	// Previous call to [Read/Write]FileAsync() failed
-	if (_fd->iStatus == 0) {
+	if (fd->iStatus == 0) {
-		*lpNumberOfBytesRead = 0;
+		*lpNumberOfBytes = 0;
 		return FALSE;
 	}
 	// Detect if we already read the size and updated the offset
-	if (_fd->Overlapped.bOffsetUpdated) {
+	if (fd->Overlapped.bOffsetUpdated) {
 		SetLastError(ERROR_NO_MORE_ITEMS);
 		return FALSE;
 	}
 	// TODO: Use a timeout and call GetOverlappedResultEx() on Windows 8 and later
-	if (!GetOverlappedResult(_fd->hFile, (OVERLAPPED*)&_fd->Overlapped,
+	if (!GetOverlappedResult(fd->hFile, (OVERLAPPED*)&fd->Overlapped,
-		lpNumberOfBytesRead, (_fd->iStatus < 0)))
+		lpNumberOfBytes, (fd->iStatus < 0)))
 		return (GetLastError() == ERROR_HANDLE_EOF);
-	_fd->Overlapped.Offset += *lpNumberOfBytesRead;
+	fd->Overlapped.Offset += *lpNumberOfBytes;
-	_fd->Overlapped.bOffsetUpdated = TRUE;
+	fd->Overlapped.bOffsetUpdated = TRUE;
 	return TRUE;
 }