[cmp] add support for Ventoy Sparse Image (.vtsi)

.vs/bled.vcxproj

@@ -46,6 +46,7 @@
     <ClCompile Include="..\src\bled\decompress_unlzma.c" />
     <ClCompile Include="..\src\bled\decompress_unxz.c" />
     <ClCompile Include="..\src\bled\decompress_unzip.c" />
+    <ClCompile Include="..\src\bled\decompress_vtsi.c" />
     <ClCompile Include="..\src\bled\filter_accept_all.c" />
     <ClCompile Include="..\src\bled\filter_accept_list.c" />
     <ClCompile Include="..\src\bled\filter_accept_reject_list.c" />

.vs/bled.vcxproj.filters

@@ -84,6 +84,9 @@
     <ClCompile Include="..\src\bled\decompress_unzip.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\src\bled\decompress_vtsi.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\src\bled\bb_archive.h">

res/appstore/Package.appxmanifest

@@ -11,7 +11,7 @@
   <Identity
     Name="19453.net.Rufus"
     Publisher="CN=7AC86D13-3E5A-491A-ADD5-80095C212740"
-    Version="3.14.1799.0" />
+    Version="3.14.1800.0" />
 
   <Properties>
     <DisplayName>Rufus</DisplayName>

src/bled/Makefile.am

@@ -2,7 +2,7 @@ noinst_LIBRARIES = libbled.a
 
 libbled_a_SOURCES = bled.c crc32.c data_align.c data_extract_all.c data_skip.c decompress_bunzip2.c \
   decompress_gunzip.c decompress_uncompress.c decompress_unlzma.c decompress_unxz.c decompress_unzip.c \
-  filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c find_list_entry.c \
+  decompress_vtsi.c filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c find_list_entry.c \
   header_list.c header_skip.c header_verbose_list.c init_handle.c open_transformer.c \
   seek_by_jump.c seek_by_read.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c
 libbled_a_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/.. -Wno-undef -Wno-strict-aliasing

src/bled/Makefile.in

@@ -103,6 +103,7 @@ am_libbled_a_OBJECTS = libbled_a-bled.$(OBJEXT) \
 	libbled_a-decompress_unlzma.$(OBJEXT) \
 	libbled_a-decompress_unxz.$(OBJEXT) \
 	libbled_a-decompress_unzip.$(OBJEXT) \
+	libbled_a-decompress_vtsi.$(OBJEXT) \
 	libbled_a-filter_accept_all.$(OBJEXT) \
 	libbled_a-filter_accept_list.$(OBJEXT) \
 	libbled_a-filter_accept_reject_list.$(OBJEXT) \
@@ -267,7 +268,7 @@ top_srcdir = @top_srcdir@
 noinst_LIBRARIES = libbled.a
 libbled_a_SOURCES = bled.c crc32.c data_align.c data_extract_all.c data_skip.c decompress_bunzip2.c \
   decompress_gunzip.c decompress_uncompress.c decompress_unlzma.c decompress_unxz.c decompress_unzip.c \
-  filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c find_list_entry.c \
+  decompress_vtsi.c filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c find_list_entry.c \
   header_list.c header_skip.c header_verbose_list.c init_handle.c open_transformer.c \
   seek_by_jump.c seek_by_read.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c
 
@@ -393,6 +394,12 @@ libbled_a-decompress_unzip.o: decompress_unzip.c
 libbled_a-decompress_unzip.obj: decompress_unzip.c
 	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-decompress_unzip.obj `if test -f 'decompress_unzip.c'; then $(CYGPATH_W) 'decompress_unzip.c'; else $(CYGPATH_W) '$(srcdir)/decompress_unzip.c'; fi`
 
+libbled_a-decompress_vtsi.o: decompress_vtsi.c
+	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-decompress_vtsi.o `test -f 'decompress_vtsi.c' || echo '$(srcdir)/'`decompress_vtsi.c
+
+libbled_a-decompress_vtsi.obj: decompress_vtsi.c
+	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-decompress_vtsi.obj `if test -f 'decompress_vtsi.c'; then $(CYGPATH_W) 'decompress_vtsi.c'; else $(CYGPATH_W) '$(srcdir)/decompress_vtsi.c'; fi`
+
 libbled_a-filter_accept_all.o: filter_accept_all.c
 	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-filter_accept_all.o `test -f 'filter_accept_all.c' || echo '$(srcdir)/'`filter_accept_all.c
 

src/bled/bb_archive.h

@@ -266,6 +266,7 @@ IF_DESKTOP(long long) int unpack_gz_stream(transformer_state_t *xstate) FAST_FUN
 IF_DESKTOP(long long) int unpack_bz2_stream(transformer_state_t *xstate) FAST_FUNC;
 IF_DESKTOP(long long) int unpack_lzma_stream(transformer_state_t *xstate) FAST_FUNC;
 IF_DESKTOP(long long) int unpack_xz_stream(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_vtsi_stream(transformer_state_t *xstate) FAST_FUNC;
 
 char* append_ext(char *filename, const char *expected_ext) FAST_FUNC;
 int bbunpack(char **argv,

src/bled/bled.c

@@ -46,7 +46,8 @@ unpacker_t unpacker[BLED_COMPRESSION_MAX] = {
 	unpack_lzma_stream,
 	unpack_bz2_stream,
 	unpack_xz_stream,
-	unpack_none
+	unpack_none,
+	unpack_vtsi_stream,
 };
 
 /* Uncompress file 'src', compressed using 'type', to file 'dst' */

src/bled/bled.h

@@ -30,6 +30,7 @@ typedef enum {
 	BLED_COMPRESSION_BZIP2,		// .bz2
 	BLED_COMPRESSION_XZ,		// .xz
 	BLED_COMPRESSION_7ZIP,		// .7z
+	BLED_COMPRESSION_VTSI,		// .vtsi
 	BLED_COMPRESSION_MAX
 } bled_compression_type;
 

src/bled/decompress_vtsi.c

@@ -0,0 +1,189 @@
+/******************************************************************************
+ * decompress_vtsi.c
+ *
+ * Copyright (c) 2021, longpanda <admin@ventoy.net>
+ *
+ * Licensed under GPLv2 or later, see file LICENSE in this source tree.
+ *
+ */
+
+#include "libbb.h"
+#include "bb_archive.h"
+
+/*
+ * Structure of a Ventoy Sparse Image (VTSI) file:
+ *
+ +---------------------------------
+ + sector 0 ~ sector N-1
+ +     data area
+ +---------------------------------
+ + sector N ~ sector M - 1
+ +     segment[0]
+ +     segment[1]
+ +     segment[2]
+ +      .....
+ +     (may be some align data (segment data aligned with 512) ...)
+ +---------------------------------
+ + sector M
+ +     footer
+ +---------------------------------
+ *
+ * All the integers are in little endian
+ * The sector size is fixed 512 for ventoy image file.
+ *
+ */
+
+#define VTSI_MAGIC 0x0000594F544E4556ULL	// "VENTOY\0\0"
+
+#pragma pack(1)
+
+typedef struct {
+	uint64_t disk_start_sector;
+	uint64_t sector_num;
+	uint64_t data_offset;
+} VTSI_SEGMENT;
+
+typedef struct {
+	uint64_t magic;
+	uint16_t version_major;
+	uint16_t version_minor;
+	uint64_t disk_size;
+	uint32_t disk_signature;
+	uint32_t foot_chksum;
+
+	uint32_t segment_num;
+	uint32_t segment_chksum;
+	uint64_t segment_offset;
+
+	uint8_t  reserved[512 - 44];
+} VTSI_FOOTER;
+
+#pragma pack()
+
+extern int __static_assert__[sizeof(VTSI_FOOTER) == 512 ? 1 : -1];
+
+#define MAX_READ_BUF	(8 * 1024 * 1024)
+
+static int check_vtsi_footer(VTSI_FOOTER* footer)
+{
+	int valid = 0;
+	uint32_t i, oldsum, calcsum;
+
+	if (footer->magic != VTSI_MAGIC)
+		bb_error_msg_and_err("invalid vtsi magic 0x%llX", footer->magic);
+
+	/* check footer checksum */
+	oldsum = footer->foot_chksum;
+	footer->foot_chksum = 0;
+	for (i = 0, calcsum = 0; i < sizeof(VTSI_FOOTER); i++)
+		calcsum += *((uint8_t*)footer + i);
+	calcsum = ~calcsum;
+
+	if (calcsum != oldsum)
+		bb_error_msg_and_err("invalid vtsi footer chksum 0x%X 0x%X", calcsum, oldsum);
+
+	/* check version (if the major doesn't match, we consider that a breaking change occurred) */
+	if (footer->version_major != 1)
+		bb_error_msg_and_err("unsupported vtsi version %d.%d", footer->version_major, footer->version_minor);
+
+	valid = 1;
+err:
+	return valid;
+}
+
+static int check_vtsi_segment(VTSI_FOOTER* footer, VTSI_SEGMENT* segment)
+{
+	int valid = 0;
+	uint32_t i, oldsum, calcsum;
+
+	/* check segment checksum */
+	oldsum = footer->segment_chksum;
+
+	for (i = 0, calcsum = 0; i < sizeof(VTSI_SEGMENT) * footer->segment_num; i++)
+		calcsum += *((uint8_t*)segment + i);
+	calcsum = ~calcsum;
+
+	if (calcsum != oldsum)
+		bb_error_msg_and_err("invalid vtsi segment chksum 0x%X 0x%X", calcsum, oldsum);
+
+	valid = 1;
+err:
+	return valid;
+}
+
+IF_DESKTOP(long long) int FAST_FUNC unpack_vtsi_stream(transformer_state_t* xstate)
+{
+	IF_DESKTOP(long long) int n = -EFAULT;
+	long long tot = 0;
+	off_t src_size;
+	int src_fd = 0;
+	size_t wsize = 0;
+	ssize_t retval = 0;
+	uint64_t seg = 0;
+	uint64_t datalen = 0;
+	uint64_t phy_offset = 0;
+	size_t max_buflen = MAX_READ_BUF;
+	uint8_t* buf = NULL;
+	VTSI_SEGMENT* segment = NULL;
+	VTSI_SEGMENT* cur_seg = NULL;
+	VTSI_FOOTER footer;
+
+	if (xstate->dst_dir)
+		bb_error_msg_and_err("decompress to dir is not supported");
+
+	src_fd = xstate->src_fd;
+	src_size = lseek(src_fd, 0, SEEK_END);
+	lseek(src_fd, src_size - sizeof(VTSI_FOOTER), SEEK_SET);
+
+	safe_read(src_fd, &footer, sizeof(footer));
+	if (!check_vtsi_footer(&footer))
+		goto err;
+
+	if (xstate->mem_output_size_max == 512)
+		max_buflen = 1024;
+
+	segment = xmalloc(footer.segment_num * sizeof(VTSI_SEGMENT) + max_buflen);
+	if (!segment)
+		bb_error_msg_and_err("Failed to alloc segment buffer %u", footer.segment_num);
+
+	buf = (uint8_t*)segment + footer.segment_num * sizeof(VTSI_SEGMENT);
+
+	lseek(src_fd, footer.segment_offset, SEEK_SET);
+	safe_read(src_fd, segment, footer.segment_num * sizeof(VTSI_SEGMENT));
+
+	if (!check_vtsi_segment(&footer, segment))
+		goto err;
+
+	/* read data */
+	lseek(src_fd, 0, SEEK_SET);
+	for (seg = 0; seg < footer.segment_num; seg++) {
+		cur_seg = segment + seg;
+		datalen = cur_seg->sector_num * 512;
+		phy_offset = cur_seg->disk_start_sector * 512;
+
+		if (xstate->mem_output_size_max == 0 && xstate->dst_fd >= 0)
+			lseek(xstate->dst_fd, phy_offset, SEEK_SET);
+
+		while (datalen > 0) {
+			wsize = MIN((size_t)datalen, max_buflen);
+			safe_read(src_fd, buf, (unsigned int)wsize);
+
+			retval = transformer_write(xstate, buf, wsize);
+			if (retval != (ssize_t)wsize) {
+				n = (retval == -ENOSPC) ? xstate->mem_output_size_max : -1;
+				goto err;
+			}
+
+			tot += retval;
+			datalen -= wsize;
+		}
+	}
+
+	n = tot;
+
+err:
+	if (segment)
+		free(segment);
+
+	return n;
+}

src/format.c

@@ -1961,22 +1961,22 @@ DWORD WINAPI FormatThread(void* param)
 	if ((boot_type == BT_IMAGE) && write_as_image) {
 		WriteDrive(hPhysicalDrive, FALSE);
 
-	// Trying to mount accessible partitions after writing an image leads to the
+		// Trying to mount accessible partitions after writing an image leads to the
-	// creation of the infamous 'System Volume Information' folder on ESPs, which
+		// creation of the infamous 'System Volume Information' folder on ESPs, which
-	// in turn leads to checksum errors for Ubuntu's boot/grub/efi.img (that maps
+		// in turn leads to checksum errors for Ubuntu's boot/grub/efi.img (that maps
-	// to the Ubuntu ESP). So we no longer call on the code below...
+		// to the Ubuntu ESP). So we only call the code below for Ventoy's vtsi images.
-#if 0
+		if (img_report.compression_type == BLED_COMPRESSION_VTSI) {
-		// If the image contains a partition we might be able to access, try to re-mount it
+			// If the image contains a partition we might be able to access, try to re-mount it
-		safe_unlockclose(hPhysicalDrive);
+			safe_unlockclose(hPhysicalDrive);
-		safe_unlockclose(hLogicalVolume);
+			safe_unlockclose(hLogicalVolume);
-		Sleep(200);
+			Sleep(200);
-		WaitForLogical(DriveIndex, 0);
+			WaitForLogical(DriveIndex, 0);
-		if (GetDrivePartitionData(SelectedDrive.DeviceNumber, fs_name, sizeof(fs_name), TRUE)) {
+			if (GetDrivePartitionData(SelectedDrive.DeviceNumber, fs_name, sizeof(fs_name), TRUE)) {
-			volume_name = GetLogicalName(DriveIndex, 0, TRUE, TRUE);
+				volume_name = GetLogicalName(DriveIndex, 0, TRUE, TRUE);
-			if ((volume_name != NULL) && (MountVolume(drive_name, volume_name)))
+				if ((volume_name != NULL) && (MountVolume(drive_name, volume_name)))
-				uprintf("Remounted %s as %C:", volume_name, drive_name[0]);
+					uprintf("Remounted %s as %C:", volume_name, drive_name[0]);
+			}
 		}
-#endif
 		goto out;
 	}
 

src/rufus.c

@@ -2469,7 +2469,7 @@ static INT_PTR CALLBACK MainCallback(HWND hDlg, UINT message, WPARAM wParam, LPA
 				} else {
 					char* old_image_path = image_path;
 					// If declared globaly, lmprintf(MSG_036) would be called on each message...
-					EXT_DECL(img_ext, NULL, __VA_GROUP__("*.iso;*.img;*.vhd;*.usb;*.bz2;*.bzip2;*.gz;*.lzma;*.xz;*.Z;*.zip;*.wim;*.esd"),
+					EXT_DECL(img_ext, NULL, __VA_GROUP__("*.iso;*.img;*.vhd;*.usb;*.bz2;*.bzip2;*.gz;*.lzma;*.xz;*.Z;*.zip;*.wim;*.esd;*.vtsi"),
 						__VA_GROUP__(lmprintf(MSG_036)));
 					image_path = FileDialog(FALSE, NULL, &img_ext, 0);
 					if (image_path == NULL) {

src/rufus.rc

@@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL
 IDD_DIALOG DIALOGEX 12, 12, 232, 326
 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU
 EXSTYLE WS_EX_ACCEPTFILES
-CAPTION "Rufus 3.14.1799"
+CAPTION "Rufus 3.14.1800"
 FONT 9, "Segoe UI Symbol", 400, 0, 0x0
 BEGIN
     LTEXT           "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP
@@ -395,8 +395,8 @@ END
 //
 
 VS_VERSION_INFO VERSIONINFO
- FILEVERSION 3,14,1799,0
+ FILEVERSION 3,14,1800,0
- PRODUCTVERSION 3,14,1799,0
+ PRODUCTVERSION 3,14,1800,0
  FILEFLAGSMASK 0x3fL
 #ifdef _DEBUG
  FILEFLAGS 0x1L
@@ -414,13 +414,13 @@ BEGIN
             VALUE "Comments", "https://rufus.ie"
             VALUE "CompanyName", "Akeo Consulting"
             VALUE "FileDescription", "Rufus"
-            VALUE "FileVersion", "3.14.1799"
+            VALUE "FileVersion", "3.14.1800"
             VALUE "InternalName", "Rufus"
             VALUE "LegalCopyright", "© 2011-2021 Pete Batard (GPL v3)"
             VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html"
             VALUE "OriginalFilename", "rufus-3.14.exe"
             VALUE "ProductName", "Rufus"
-            VALUE "ProductVersion", "3.14.1799"
+            VALUE "ProductVersion", "3.14.1800"
         END
     END
     BLOCK "VarFileInfo"

src/vhd.c

@@ -237,6 +237,7 @@ static comp_assoc file_assoc[] = {
 	{ ".lzma", BLED_COMPRESSION_LZMA },
 	{ ".bz2", BLED_COMPRESSION_BZIP2 },
 	{ ".xz", BLED_COMPRESSION_XZ },
+	{ ".vtsi", BLED_COMPRESSION_VTSI },
 };
 
 // For now we consider that an image that matches a known extension is bootable