To understand how ZFS creates a bookmark, we can trace the code path from zfs bookmark all the way down to dsl_bookmark_add which actually adds the bookmark node to the tree.
This is the bookmark structure physically written to disk:
zfs/include/sys/dsl_bookmark.h
/*
* On disk zap object.
*/
typedef struct zfs_bookmark_phys {
uint64_t zbm_guid; /* guid of bookmarked dataset */
uint64_t zbm_creation_txg; /* birth transaction group */
uint64_t zbm_creation_time; /* bookmark creation time */
/* fields used for redacted send / recv */
uint64_t zbm_redaction_obj; /* redaction list object */
uint64_t zbm_flags; /* ZBM_FLAG_* */
/* fields used for bookmark written size */
uint64_t zbm_referenced_bytes_refd;
uint64_t zbm_compressed_bytes_refd;
uint64_t zbm_uncompressed_bytes_refd;
uint64_t zbm_referenced_freed_before_next_snap;
uint64_t zbm_compressed_freed_before_next_snap;
uint64_t zbm_uncompressed_freed_before_next_snap;
/* fields used for raw sends */
uint64_t zbm_ivset_guid;
} zfs_bookmark_phys_t;
#define BOOKMARK_PHYS_SIZE_V1 (3 * sizeof (uint64_t))
#define BOOKMARK_PHYS_SIZE_V2 (12 * sizeof (uint64_t))
Only the first 3 fields are required for v1 bookmarks, while v2 bookmarks contain all 12 fields. dsl_bookmark_node_add only writes a v2 bookmark if one of the 9 v2 fields are non-zero, so we can leave them all zero to write a v1 bookmark.
After a few iterations, I had a patch which hijacks the normal zfs bookmark pool/dataset#src pool/dataset#dst code path to create a bookmark with arbitrary data when the source bookmark name is missing.
sam@zfshax:~/zfs $ git --no-pager diff
sam@zfshax:~/zfs $ git --no-pager diff
diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c
index 2d81ef31c..73b5d7e70 100644
--- a/cmd/zfs/zfs_main.c
+++ b/cmd/zfs/zfs_main.c
@@ -7892,12 +7892,15 @@ zfs_do_bookmark(int argc, char **argv)
default: abort();
}
+// Skip testing for #missing because it does not exist.
+if (strstr(source, "#missing") == NULL) {
/* test the source exists */
zfs_handle_t *zhp;
zhp = zfs_open(g_zfs, source, source_type);
if (zhp == NULL)
goto usage;
zfs_close(zhp);
+}
nvl = fnvlist_alloc();
fnvlist_add_string(nvl, bookname, source);
diff --git a/module/zfs/dsl_bookmark.c b/module/zfs/dsl_bookmark.c
index 861dd9239..fae882f45 100644
--- a/module/zfs/dsl_bookmark.c
+++ b/module/zfs/dsl_bookmark.c
@@ -263,7 +263,12 @@ dsl_bookmark_create_check_impl(dsl_pool_t *dp,
* Source must exists and be an earlier point in newbm_ds's
* timeline (newbm_ds's origin may be a snap of source's ds)
*/
+// Skip looking up #missing because it does not exist.
+if (strstr(source, "#missing") == NULL) {
error = dsl_bookmark_lookup(dp, source, newbm_ds, &source_phys);
+} else {
+ error = 0;
+}
switch (error) {
case 0:
break; /* happy path */
@@ -545,12 +550,34 @@ dsl_bookmark_create_sync_impl_book(
* because the redaction object might be too large
*/
+// Skip looking up #missing because it does not exist.
+if (strstr(source_name, "#missing") == NULL) {
VERIFY0(dsl_bookmark_lookup_impl(bmark_fs_source, source_shortname,
&source_phys));
+}
dsl_bookmark_node_t *new_dbn = dsl_bookmark_node_alloc(new_shortname);
+// Skip copying from #missing because it does not exist.
+if (strstr(source_name, "#missing") == NULL) {
memcpy(&new_dbn->dbn_phys, &source_phys, sizeof (source_phys));
new_dbn->dbn_phys.zbm_redaction_obj = 0;
+} else {
+ // Manually set the bookmark parameters.
+ new_dbn->dbn_phys = (zfs_bookmark_phys_t){
+ .zbm_guid = 4964628655505655411,
+ .zbm_creation_txg = 12,
+ .zbm_creation_time = 1756699200,
+ .zbm_redaction_obj = 0,
+ .zbm_flags = 0,
+ .zbm_referenced_bytes_refd = 0,
+ .zbm_compressed_bytes_refd = 0,
+ .zbm_uncompressed_bytes_refd = 0,
+ .zbm_referenced_freed_before_next_snap = 0,
+ .zbm_compressed_freed_before_next_snap = 0,
+ .zbm_uncompressed_freed_before_next_snap = 0,
+ .zbm_ivset_guid = 0,
+ };
+}
/* update feature counters */
if (new_dbn->dbn_phys.zbm_flags & ZBM_FLAG_HAS_FBN) {
To test, we recompile ZFS, reload the kernel module, and reimport the pools.
sam@zfshax:~/zfs $ gmake -s -j$(sysctl -n hw.ncpu)
sam@zfshax:~/zfs $ sudo gmake install && sudo ldconfig
sam@zfshax:~/zfs $ sudo zpool export src && sudo zpool export dst
sam@zfshax:~/zfs $ sudo ./scripts/zfs.sh -r
sam@zfshax:~/zfs $ sudo zpool import src -d / && sudo zpool import dst -d /
Then, we create the bookmark ex nihilo using the magic bookmark name missing.
sam@zfshax:~/zfs $ sudo zfs bookmark src/encryptionroot#missing src/encryptionroot#111
sam@zfshax:~/zfs $ sudo zdb src/encryptionroot#111
#111: {guid: 44e5e7755d23c673 creation_txg: 12 creation_time: 1756699200 redaction_obj: 0}Success! We can now use the bookmark to generate an incremental send stream containing the new hex wrapping key parameters.
sam@zfshax:~/zfs $ sudo zfs send --raw -i src/encryptionroot#111 src/encryptionroot@222 | zstreamdump
BEGIN record
hdrtype = 1
features = 1420004
magic = 2f5bacbac
creation_time = 68d3d93f
type = 2
flags = 0xc
toguid = 3f99b9e92cc0aca7
fromguid = 44e5e7755d23c673
toname = src/encryptionroot@222
payloadlen = 1028
nvlist version: 0
crypt_keydata = (embedded nvlist)
nvlist version: 0
DSL_CRYPTO_SUITE = 0x8
DSL_CRYPTO_GUID = 0x6196311f2622e30
DSL_CRYPTO_VERSION = 0x1
DSL_CRYPTO_MASTER_KEY_1 = 0x6c 0x55 0x13 0x78 0x8c 0x2d 0x42 0xb5 0x9e 0x33 0x2 0x7e 0x73 0x3a 0x46 0x20 0xd2 0xf7 0x23 0x7d 0x7c 0x5d 0x5f 0x76 0x63 0x90 0xd2 0x43 0x6a 0xdd 0x63 0x2b
DSL_CRYPTO_HMAC_KEY_1 = 0x85 0xd1 0xf3 0xba 0xed 0xec 0x6 0x28 0x36 0xd6 0x60 0x28 0x8d 0x2f 0x6f 0x14 0xc9 0x2b 0x6f 0xf4 0x19 0x23 0x2d 0xf 0x3d 0xe 0xc4 0x88 0x4 0x6d 0xca 0xb5 0x2d 0x4d 0x8 0x75 0x17 0x1c 0xe3 0xe7 0xe6 0x23 0x7 0x53 0x94 0xba 0xc7 0x4b 0xf5 0xde 0x8c 0x29 0xa3 0x27 0xdf 0x82 0x64 0x9d 0x92 0xb4 0xc1 0x26 0x5b 0x32
DSL_CRYPTO_IV = 0xdf 0x52 0x77 0xe8 0xf 0xfd 0xc2 0x42 0x66 0x88 0xb9 0xf0
DSL_CRYPTO_MAC = 0x54 0x54 0x15 0xa4 0x21 0x55 0x6b 0x4e 0x99 0xe7 0xf 0xef 0x9f 0x90 0x42 0x54
portable_mac = 0x3a 0xd6 0x30 0xc4 0x6a 0x2d 0x60 0x24 0x95 0xfc 0x99 0xbb 0xfa 0x10 0xa0 0x6b 0xc6 0x1 0xdd 0x1d 0x9 0xcd 0xa8 0x19 0xdf 0x57 0xb9 0x90 0x4f 0x2e 0x33 0xc1
keyformat = 0x2
pbkdf2iters = 0x0
pbkdf2salt = 0x0
mdn_checksum = 0x0
mdn_compress = 0x0
mdn_nlevels = 0x6
mdn_blksz = 0x4000
mdn_indblkshift = 0x11
mdn_nblkptr = 0x3
mdn_maxblkid = 0x4
to_ivset_guid = 0x957edeaa7123a7
from_ivset_guid = 0x0
(end crypt_keydata)
END checksum = 14046201258/62f53166ccc36/14023a70758c3195/1e906f4670783cd
SUMMARY:
Total DRR_BEGIN records = 1 (1028 bytes)
Total DRR_END records = 1 (0 bytes)
Total DRR_OBJECT records = 7 (960 bytes)
Total DRR_FREEOBJECTS records = 2 (0 bytes)
Total DRR_WRITE records = 1 (512 bytes)
Total DRR_WRITE_BYREF records = 0 (0 bytes)
Total DRR_WRITE_EMBEDDED records = 0 (0 bytes)
Total DRR_FREE records = 12 (0 bytes)
Total DRR_SPILL records = 0 (0 bytes)
Total records = 26
Total payload size = 2500 (0x9c4)
Total header overhead = 8112 (0x1fb0)
Total stream length = 10612 (0x2974)But we can’t receive the send stream.
sam@zfshax:~ $ sudo zfs send --raw -i src/encryptionroot#111 src/encryptionroot@222 | sudo zfs recv -F dst/encryptionroot
cannot receive incremental stream: IV set guid missing. See errata 4 at https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-ER.About the Author
