MDEV-27835 innochecksum -S crashes for encrypted .ibd tablespace

As main() invokes parse_page() when -S or -D are set, it can be a case
when parse_page() is invoked when -D filename is not set, that is why
any attempt to write to page dump file must be done only if the file
name is set with -D.

The bug is caused by 2ef7a5a1
(MDEV-13443).

extra/innochecksum.cc

@@ -886,7 +886,7 @@ parse_page(
 
 			is_leaf = (!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL)));
 
-			if (page_type_dump) {
+			if (file) {
 				fprintf(file, "#::" UINT32PF "\t\t|\t\tIndex page\t\t\t|"
 					"\tindex id=%llu,", cur_page_num, id);
 
@@ -939,7 +939,7 @@ parse_page(
 				index.total_data_bytes += data_bytes;
 				index.pages_in_size_range[size_range_id] ++;
 			}
-		} else {
+		} else if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tEncrypted Index page\t\t\t|"
 				"\tkey_version " UINT32PF ",%s\n", cur_page_num, key_version, str);
 		}
@@ -950,20 +950,20 @@ parse_page(
 		page_type.n_fil_page_undo_log++;
 		undo_page_type = mach_read_from_2(page +
 				     TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE);
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tUndo log page\t\t\t|",
 				cur_page_num);
 		}
 		if (undo_page_type == TRX_UNDO_INSERT) {
 			page_type.n_undo_insert++;
-			if (page_type_dump) {
+			if (file) {
 				fprintf(file, "\t%s",
 					"Insert Undo log page");
 			}
 
 		} else if (undo_page_type == TRX_UNDO_UPDATE) {
 			page_type.n_undo_update++;
-			if (page_type_dump) {
+			if (file) {
 				fprintf(file, "\t%s",
 					"Update undo log page");
 			}
@@ -974,7 +974,7 @@ parse_page(
 		switch (undo_page_type) {
 			case TRX_UNDO_ACTIVE:
 				page_type.n_undo_state_active++;
-				if (page_type_dump) {
+				if (file) {
 					fprintf(file, ", %s", "Undo log of "
 						"an active transaction");
 				}
@@ -982,7 +982,7 @@ parse_page(
 
 			case TRX_UNDO_CACHED:
 				page_type.n_undo_state_cached++;
-				if (page_type_dump) {
+				if (file) {
 					fprintf(file, ", %s", "Page is "
 						"cached for quick reuse");
 				}
@@ -990,7 +990,7 @@ parse_page(
 
 			case TRX_UNDO_TO_FREE:
 				page_type.n_undo_state_to_free++;
-				if (page_type_dump) {
+				if (file) {
 					fprintf(file, ", %s", "Insert undo "
 						"segment that can be freed");
 				}
@@ -998,7 +998,7 @@ parse_page(
 
 			case TRX_UNDO_TO_PURGE:
 				page_type.n_undo_state_to_purge++;
-				if (page_type_dump) {
+				if (file) {
 					fprintf(file, ", %s", "Will be "
 						"freed in purge when all undo"
 					"data in it is removed");
@@ -1007,7 +1007,7 @@ parse_page(
 
 			case TRX_UNDO_PREPARED:
 				page_type.n_undo_state_prepared++;
-				if (page_type_dump) {
+				if (file) {
 					fprintf(file, ", %s", "Undo log of "
 						"an prepared transaction");
 				}
@@ -1017,14 +1017,14 @@ parse_page(
 				page_type.n_undo_state_other++;
 				break;
 		}
-		if(page_type_dump) {
+		if(file) {
 			fprintf(file, ", %s\n", str);
 		}
 		break;
 
 	case FIL_PAGE_INODE:
 		page_type.n_fil_page_inode++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tInode page\t\t\t|"
 				"\t%s\n",cur_page_num, str);
 		}
@@ -1032,7 +1032,7 @@ parse_page(
 
 	case FIL_PAGE_IBUF_FREE_LIST:
 		page_type.n_fil_page_ibuf_free_list++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tInsert buffer free list"
 				" page\t|\t%s\n", cur_page_num, str);
 		}
@@ -1040,7 +1040,7 @@ parse_page(
 
 	case FIL_PAGE_TYPE_ALLOCATED:
 		page_type.n_fil_page_type_allocated++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tFreshly allocated "
 				"page\t\t|\t%s\n", cur_page_num, str);
 		}
@@ -1048,7 +1048,7 @@ parse_page(
 
 	case FIL_PAGE_IBUF_BITMAP:
 		page_type.n_fil_page_ibuf_bitmap++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tInsert Buffer "
 				"Bitmap\t\t|\t%s\n", cur_page_num, str);
 		}
@@ -1056,7 +1056,7 @@ parse_page(
 
 	case FIL_PAGE_TYPE_SYS:
 		page_type.n_fil_page_type_sys++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tSystem page\t\t\t|"
 				"\t%s\n", cur_page_num, str);
 		}
@@ -1064,7 +1064,7 @@ parse_page(
 
 	case FIL_PAGE_TYPE_TRX_SYS:
 		page_type.n_fil_page_type_trx_sys++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tTransaction system "
 				"page\t\t|\t%s\n", cur_page_num, str);
 		}
@@ -1072,7 +1072,7 @@ parse_page(
 
 	case FIL_PAGE_TYPE_FSP_HDR:
 		page_type.n_fil_page_type_fsp_hdr++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tFile Space "
 				"Header\t\t|\t%s\n", cur_page_num, str);
 		}
@@ -1080,7 +1080,7 @@ parse_page(
 
 	case FIL_PAGE_TYPE_XDES:
 		page_type.n_fil_page_type_xdes++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tExtent descriptor "
 				"page\t\t|\t%s\n", cur_page_num, str);
 		}
@@ -1088,7 +1088,7 @@ parse_page(
 
 	case FIL_PAGE_TYPE_BLOB:
 		page_type.n_fil_page_type_blob++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tBLOB page\t\t\t|\t%s\n",
 				cur_page_num, str);
 		}
@@ -1096,7 +1096,7 @@ parse_page(
 
 	case FIL_PAGE_TYPE_ZBLOB:
 		page_type.n_fil_page_type_zblob++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tCompressed BLOB "
 				"page\t\t|\t%s\n", cur_page_num, str);
 		}
@@ -1104,7 +1104,7 @@ parse_page(
 
 	case FIL_PAGE_TYPE_ZBLOB2:
 		page_type.n_fil_page_type_zblob2++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tSubsequent Compressed "
 				"BLOB page\t|\t%s\n", cur_page_num, str);
 		}
@@ -1112,7 +1112,7 @@ parse_page(
 
 	case FIL_PAGE_PAGE_COMPRESSED:
 		page_type.n_fil_page_type_page_compressed++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tPage compressed "
 				"page\t|\t%s\n", cur_page_num, str);
 		}
@@ -1120,7 +1120,7 @@ parse_page(
 
 	case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
 		page_type.n_fil_page_type_page_compressed_encrypted++;
-		if (page_type_dump) {
+		if (file) {
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tPage compressed encrypted "
 				"page\t|\t%s\n", cur_page_num, str);
 		}

mysql-test/suite/encryption/r/innochecksum.result

@@ -14,7 +14,7 @@ CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
 # to redo apply the pages of t1.ibd at the time of recovery.
 # We want SQL to initiate the first access to t1.ibd.
 # Wait until disconnected.
-# Run innochecksum on t1
+# Run innochecksum on t1, check -S does not cause crash for encrypted file
 # Run innochecksum on t2
 # Run innochecksum on t3
 # Run innochecksum on t4

mysql-test/suite/encryption/t/innochecksum.test

@@ -68,9 +68,9 @@ shutdown_server;
 --echo # Wait until disconnected.
 --source include/wait_until_disconnected.inc
 
---echo # Run innochecksum on t1
 -- disable_result_log
---exec $INNOCHECKSUM $t1_IBD
+--echo # Run innochecksum on t1, check -S does not cause crash for encrypted file
+--exec $INNOCHECKSUM -S $t1_IBD
 
 --echo # Run innochecksum on t2