Coverage for bzfs/bzfs.py: 99%

1#!/usr/bin/env python3 

2# 

3# Copyright 2024 Wolfgang Hoschek AT mac DOT com 

4# 

5# Licensed under the Apache License, Version 2.0 (the "License"); 

6# you may not use this file except in compliance with the License. 

7# You may obtain a copy of the License at 

8# 

9# http://www.apache.org/licenses/LICENSE-2.0 

10# 

11# Unless required by applicable law or agreed to in writing, software 

12# distributed under the License is distributed on an "AS IS" BASIS, 

13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

14# See the License for the specific language governing permissions and 

15# limitations under the License. 

16 

17# /// script 

18# requires-python = ">=3.7" 

19# dependencies = [] 

20# /// 

21 

22""" 

23* The codebase starts with docs, definition of input data and associated argument parsing into a "Params" class. 

24* All CLI option/parameter values are reachable from the "Params" class. 

25* Control flow starts in main(), far below ..., which kicks off a "Job". 

26* A Job runs one or more "tasks" via run_tasks(), each task replicating a separate dataset tree. 

27* The core replication algorithm is in run_task() and especially in replicate_dataset(). 

28* The filter algorithms that apply include/exclude policies are in filter_datasets() and filter_snapshots(). 

29* The --create-src-snapshots-* and --delete-* and --compare-* algorithms also start in run_task(). 

30* Consider using an IDE/editor that can open multiple windows for the same file, such as PyCharm or Sublime Text, etc. 

31* README.md is mostly auto-generated from the ArgumentParser help texts as the source of "truth", via update_readme.py. 

32Simply run that script whenever you change or add ArgumentParser help text. 

33""" 

34 

35import argparse 

36import ast 

37import bisect 

38import calendar 

39import collections 

40import concurrent 

41import copy 

42import fcntl 

43import glob 

44import hashlib 

45import heapq 

46import inspect 

47import itertools 

48import json 

49import logging 

50import logging.config 

51import logging.handlers 

52import operator 

53import os 

54import platform 

55import pprint 

56import pwd 

57import random 

58import re 

59import selectors 

60import shlex 

61import shutil 

62import signal 

63import socket 

64import stat 

65import subprocess 

66import sys 

67import tempfile 

68import threading 

69import time 

70import traceback 

71from argparse import Namespace 

72from collections import defaultdict, deque, Counter, namedtuple 

73from concurrent.futures import ThreadPoolExecutor, Future, FIRST_COMPLETED 

74from contextlib import redirect_stderr 

75from dataclasses import dataclass, field, fields 

76from datetime import datetime, timedelta, timezone, tzinfo 

77from itertools import groupby 

78from logging import Logger 

79from math import ceil 

80from pathlib import Path 

81from subprocess import CalledProcessError, TimeoutExpired 

82from typing import Deque, Dict, Iterable, Iterator, List, Sequence, Set, Tuple 

83from typing import Any, Callable, Generator, Generic, Optional 

84from typing import ItemsView, TextIO, TypeVar, Union 

85 

86__version__ = "1.10.0-dev" 

87prog_name = "bzfs" 

88prog_author = "Wolfgang Hoschek" 

89die_status = 3 

90still_running_status = 4 

91min_python_version = (3, 7) 

92if sys.version_info < min_python_version: 

93 print(f"ERROR: {prog_name} requires Python version >= {'.'.join(map(str, min_python_version))}!") 

94 sys.exit(die_status) 

95exclude_dataset_regexes_default = r"(.*/)?[Tt][Ee]?[Mm][Pp][-_]?[0-9]*" # skip tmp datasets by default 

96create_src_snapshots_prefix_dflt = prog_name + "_" 

97create_src_snapshots_suffix_dflt = "_adhoc" 

98disable_prg = "-" 

99env_var_prefix = prog_name + "_" 

100pv_file_thread_separator = "_" 

101dummy_dataset = "dummy" 

102zfs_version_is_at_least_2_1_0 = "zfs>=2.1.0" 

103zfs_version_is_at_least_2_2_0 = "zfs>=2.2.0" 

104zfs_recv_groups = {"zfs_recv_o": "-o", "zfs_recv_x": "-x", "zfs_set": ""} 

105snapshot_regex_filter_names = {"include_snapshot_regex", "exclude_snapshot_regex"} 

106snapshot_regex_filter_name = "snapshot_regex" 

107snapshot_filters_var = "snapshot_filters_var" 

108cmp_choices_items = ["src", "dst", "all"] 

109inject_dst_pipe_fail_kbytes = 400 

110unixtime_infinity_secs = 2**64 # billions of years in the future and to be extra safe, larger than the largest ZFS GUID 

111year_with_four_digits_regex = re.compile(r"[1-9][0-9][0-9][0-9]") # regex for empty target shall not match non-empty target 

112log_stderr = (logging.INFO + logging.WARN) // 2 # custom log level is halfway in between 

113log_stdout = (log_stderr + logging.INFO) // 2 # custom log level is halfway in between 

114log_debug = logging.DEBUG 

115log_trace = logging.DEBUG // 2 # custom log level is halfway in between 

116SHARED = "shared" 

117DEDICATED = "dedicated" 

118DONT_SKIP_DATASET = "" 

119DEVNULL = subprocess.DEVNULL 

120PIPE = subprocess.PIPE 

121 

122 

123def argument_parser() -> argparse.ArgumentParser: 

124 create_src_snapshots_plan_example1 = str({"test": {"": {"adhoc": 1}}}).replace(" ", "") 

125 create_src_snapshots_plan_example2 = str({"prod": {"us-west-1": {"hourly": 36, "daily": 31}}}).replace(" ", "") 

126 delete_dst_snapshots_except_plan_example1 = str( 

127 { 

128 "prod": { 

129 "onsite": { 

130 "secondly": 40, 

131 "minutely": 40, 

132 "hourly": 36, 

133 "daily": 31, 

134 "weekly": 12, 

135 "monthly": 18, 

136 "yearly": 5, 

137 } 

138 } 

139 } 

140 ).replace(" ", "") 

141 

142 # fmt: off 

143 parser = argparse.ArgumentParser( 

144 prog=prog_name, 

145 allow_abbrev=False, 

146 description=f""" 

147*{prog_name} is a backup command line tool that reliably replicates ZFS snapshots from a (local or remote) 

148source ZFS dataset (ZFS filesystem or ZFS volume) and its descendant datasets to a (local or remote) 

149destination ZFS dataset to make the destination dataset a recursively synchronized copy of the source dataset, 

150using zfs send/receive/rollback/destroy and ssh tunnel as directed. For example, {prog_name} can be used to 

151incrementally replicate all ZFS snapshots since the most recent common snapshot from source to destination, 

152in order to help protect against data loss or ransomware.* 

153 

154When run for the first time, {prog_name} replicates the dataset and all its snapshots from the source to the 

155destination. On subsequent runs, {prog_name} transfers only the data that has changed since the previous run, 

156i.e. it incrementally replicates to the destination all intermediate snapshots that have been created on 

157the source since the last run. Source ZFS snapshots older than the most recent common snapshot found on the 

158destination are auto-skipped. 

159 

160Unless {prog_name} is explicitly told to create snapshots on the source, it treats the source as read-only, 

161thus the source remains unmodified. With the --dryrun flag, {prog_name} also treats the destination as read-only. 

162In normal operation, {prog_name} treats the destination as append-only. Optional CLI flags are available to 

163delete destination snapshots and destination datasets as directed, for example to make the destination 

164identical to the source if the two have somehow diverged in unforeseen ways. This easily enables 

165(re)synchronizing the backup from the production state, as well as restoring the production state from 

166backup. 

167 

168In the spirit of rsync, {prog_name} supports a variety of powerful include/exclude filters that can be combined to 

169select which datasets, snapshots and properties to create, replicate, delete or compare. 

170 

171Typically, a `cron` job on the source host runs `{prog_name}` periodically to create new snapshots and prune outdated 

172snapshots on the source, whereas another `cron` job on the destination host runs `{prog_name}` periodically to prune 

173outdated destination snapshots. Yet another `cron` job runs `{prog_name}` periodically to replicate the recently created 

174snapshots from the source to the destination. The frequency of these periodic activities is typically every N milliseconds,  

175every second, minute, hour, day, week, month and/or year (or multiples thereof). 

176 

177All {prog_name} functions including snapshot creation, replication, deletion, comparison, etc. happily work with any 

178snapshots in any format, even created or managed by third party ZFS snapshot management tools, including manual  

179zfs snapshot/destroy. All functions can also be used independently. That is, if you wish you can use {prog_name} just  

180for creating snapshots, or just for replicating, or just for deleting/pruning, or just for comparing snapshot lists. 

181 

182The source 'pushes to' the destination whereas the destination 'pulls from' the source. {prog_name} is installed 

183and executed on the 'initiator' host which can be either the host that contains the source dataset (push mode), 

184or the destination dataset (pull mode), or both datasets (local mode, no network required, no ssh required), 

185or any third-party (even non-ZFS OSX) host as long as that host is able to SSH (via standard 'ssh' OpenSSH CLI) into 

186both the source and destination host (pull-push mode). In pull-push mode the source 'zfs send's the data stream 

187to the initiator which immediately pipes the stream (without storing anything locally) to the destination 

188host that 'zfs receive's it. Pull-push mode means that {prog_name} need not be installed or executed on either 

189source or destination host. Only the underlying 'zfs' CLI must be installed on both source and destination host. 

190{prog_name} can run as root or non-root user, in the latter case via a) sudo or b) when granted corresponding 

191ZFS permissions by administrators via 'zfs allow' delegation mechanism. 

192 

193{prog_name} is written in Python and continously runs a wide set of unit tests and integration tests to ensure 

194coverage and compatibility with old and new versions of ZFS on Linux, FreeBSD and Solaris, on all Python 

195versions >= 3.7 (including latest stable which is currently python-3.13). 

196 

197{prog_name} is a stand-alone program with zero required dependencies, consisting of a single file, akin to a 

198stand-alone shell script or binary executable. It is designed to be able to run in restricted barebones server 

199environments. No external Python packages are required; indeed no Python package management at all is required. 

200You can just copy the file wherever you like, for example into /usr/local/bin or similar, and simply run it like 

201any stand-alone shell script or binary executable. 

202 

203{prog_name} automatically replicates the snapshots of multiple datasets in parallel for best performance. 

204Similarly, it quickly deletes (or compares) snapshots of multiple datasets in parallel. Atomic snapshots can be  

205created as frequently as every N milliseconds. 

206 

207Optionally, {prog_name} applies bandwidth rate-limiting and progress monitoring (via 'pv' CLI) during 'zfs 

208send/receive' data transfers. When run across the network, {prog_name} also transparently inserts lightweight 

209data compression (via 'zstd -1' CLI) and efficient data buffering (via 'mbuffer' CLI) into the pipeline 

210between network endpoints during 'zfs send/receive' network transfers. If one of these utilities is not 

211installed this is auto-detected, and the operation continues reliably without the corresponding auxiliary 

212feature. 

213 

214# Periodic Jobs with bzfs_jobrunner 

215 

216The software also ships with the [bzfs_jobrunner](README_bzfs_jobrunner.md) companion program, which is a convenience 

217wrapper around `{prog_name}` that simplifies periodic ZFS snapshot creation, replication, and pruning, across source host  

218and multiple destination hosts, using a single shared [jobconfig](bzfs_tests/bzfs_job_example.py) script. 

219 

220# Quickstart 

221 

222* Create adhoc atomic snapshots without a schedule: 

223 

224```$ {prog_name} tank1/foo/bar dummy --recursive --skip-replication --create-src-snapshots  

225--create-src-snapshots-plan "{create_src_snapshots_plan_example1}"``` 

226 

227```$ zfs list -t snapshot tank1/foo/bar 

228 

229tank1/foo/bar@test_2024-11-06_08:30:05_adhoc 

230``` 

231 

232* Create periodic atomic snapshots on a schedule, every hour and every day, by launching this from a periodic `cron` job: 

233 

234```$ {prog_name} tank1/foo/bar dummy --recursive --skip-replication --create-src-snapshots  

235--create-src-snapshots-plan "{create_src_snapshots_plan_example2}"``` 

236 

237```$ zfs list -t snapshot tank1/foo/bar 

238 

239tank1/foo/bar@prod_us-west-1_2024-11-06_08:30:05_daily 

240 

241tank1/foo/bar@prod_us-west-1_2024-11-06_08:30:05_hourly 

242``` 

243 

244Note: A periodic snapshot is created if it is due per the schedule indicated by its suffix (e.g. `_daily` or `_hourly`  

245or `_minutely` or `_2secondly` or `_100millisecondly`), or if the --create-src-snapshots-even-if-not-due flag is specified, 

246or if the most recent scheduled snapshot is somehow missing. In the latter case {prog_name} immediately creates a snapshot  

247(named with the current time, not backdated to the missed time), and then resumes the original schedule. If the suffix is  

248`_adhoc` or not a known period then a snapshot is considered non-periodic and is thus created immediately regardless of the  

249creation time of any existing snapshot. 

250 

251* Replication example in local mode (no network, no ssh), to replicate ZFS dataset tank1/foo/bar to tank2/boo/bar: 

252 

253```$ {prog_name} tank1/foo/bar tank2/boo/bar``` 

254 

255```$ zfs list -t snapshot tank1/foo/bar 

256 

257tank1/foo/bar@prod_us-west-1_2024-11-06_08:30:05_daily 

258 

259tank1/foo/bar@prod_us-west-1_2024-11-06_08:30:05_hourly``` 

260 

261```$ zfs list -t snapshot tank2/boo/bar 

262 

263tank2/boo/bar@prod_us-west-1_2024-11-06_08:30:05_daily 

264 

265tank2/boo/bar@prod_us-west-1_2024-11-06_08:30:05_hourly``` 

266 

267* Same example in pull mode: 

268 

269```$ {prog_name} root@host1.example.com:tank1/foo/bar tank2/boo/bar``` 

270 

271* Same example in push mode: 

272 

273```$ {prog_name} tank1/foo/bar root@host2.example.com:tank2/boo/bar``` 

274 

275* Same example in pull-push mode: 

276 

277```$ {prog_name} root@host1:tank1/foo/bar root@host2:tank2/boo/bar``` 

278 

279* Example in local mode (no network, no ssh) to recursively replicate ZFS dataset tank1/foo/bar and its descendant 

280datasets to tank2/boo/bar: 

281 

282```$ {prog_name} tank1/foo/bar tank2/boo/bar --recursive``` 

283 

284```$ zfs list -t snapshot -r tank1/foo/bar 

285 

286tank1/foo/bar@prod_us-west-1_2024-11-06_08:30:05_daily 

287 

288tank1/foo/bar@prod_us-west-1_2024-11-06_08:30:05_hourly 

289 

290tank1/foo/bar/baz@prod_us-west-1_2024-11-06_08:40:00_daily 

291 

292tank1/foo/bar/baz@prod_us-west-1_2024-11-06_08:40:00_hourly``` 

293 

294```$ zfs list -t snapshot -r tank2/boo/bar 

295 

296tank2/boo/bar@prod_us-west-1_2024-11-06_08:30:05_daily 

297 

298tank2/boo/bar@prod_us-west-1_2024-11-06_08:30:05_hourly 

299 

300tank2/boo/bar/baz@prod_us-west-1_2024-11-06_08:40:00_daily 

301 

302tank2/boo/bar/baz@prod_us-west-1_2024-11-06_08:40:00_hourly``` 

303 

304* Example that makes destination identical to source even if the two have drastically diverged: 

305 

306```$ {prog_name} tank1/foo/bar tank2/boo/bar --recursive --force --delete-dst-datasets --delete-dst-snapshots``` 

307 

308* Replicate all daily snapshots created during the last 7 days, and at the same time ensure that the latest 7 daily 

309snapshots (per dataset) are replicated regardless of creation time: 

310 

311```$ {prog_name} tank1/foo/bar tank2/boo/bar --recursive --include-snapshot-regex '.*_daily' 

312--include-snapshot-times-and-ranks '7 days ago..anytime' 'latest 7'``` 

313 

314Note: The example above compares the specified times against the standard ZFS 'creation' time property of the snapshots  

315(which is a UTC Unix time in integer seconds), rather than against a timestamp that may be part of the snapshot name. 

316 

317* Delete all daily snapshots older than 7 days, but ensure that the latest 7 daily snapshots (per dataset) are retained 

318regardless of creation time: 

319 

320```$ {prog_name} {dummy_dataset} tank2/boo/bar --dryrun --recursive --skip-replication --delete-dst-snapshots 

321--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks notime 'all except latest 7' 

322--include-snapshot-times-and-ranks 'anytime..7 days ago'``` 

323 

324Note: This also prints how many GB of disk space in total would be freed if the command were to be run for real without  

325the --dryrun flag. 

326 

327* Delete all daily snapshots older than 7 days, but ensure that the latest 7 daily snapshots (per dataset) are retained 

328regardless of creation time. Additionally, only delete a snapshot if no corresponding snapshot or bookmark exists in 

329the source dataset (same as above except replace the 'dummy' source with 'tank1/foo/bar'): 

330 

331```$ {prog_name} tank1/foo/bar tank2/boo/bar --dryrun --recursive --skip-replication --delete-dst-snapshots 

332--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks notime 'all except latest 7' 

333--include-snapshot-times-and-ranks '7 days ago..anytime'``` 

334 

335* Delete all daily snapshots older than 7 days, but ensure that the latest 7 daily snapshots (per dataset) are retained 

336regardless of creation time. Additionally, only delete a snapshot if no corresponding snapshot exists in the source 

337dataset (same as above except append 'no-crosscheck'): 

338 

339```$ {prog_name} tank1/foo/bar tank2/boo/bar --dryrun --recursive --skip-replication --delete-dst-snapshots 

340--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks notime 'all except latest 7' 

341--include-snapshot-times-and-ranks 'anytime..7 days ago' --delete-dst-snapshots-no-crosscheck``` 

342 

343* Delete all daily bookmarks older than 90 days, but retain the latest 200 daily bookmarks (per dataset) regardless 

344of creation time: 

345 

346```$ {prog_name} {dummy_dataset} tank1/foo/bar --dryrun --recursive --skip-replication --delete-dst-snapshots=bookmarks 

347--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks notime 'all except latest 200' 

348--include-snapshot-times-and-ranks 'anytime..90 days ago'``` 

349 

350* Delete all tmp datasets within tank2/boo/bar: 

351 

352```$ {prog_name} {dummy_dataset} tank2/boo/bar --dryrun --recursive --skip-replication --delete-dst-datasets 

353--include-dataset-regex '(.*/)?tmp.*' --exclude-dataset-regex '!.*'``` 

354 

355* Retain all secondly snapshots that were created less than 40 seconds ago, and ensure that the latest 40  

356secondly snapshots (per dataset) are retained regardless of creation time. Same for 40 minutely snapshots, 36 hourly  

357snapshots, 31 daily snapshots, 12 weekly snapshots, 18 monthly snapshots, and 5 yearly snapshots: 

358 

359```$ {prog_name} {dummy_dataset} tank2/boo/bar --dryrun --recursive --skip-replication --delete-dst-snapshots  

360--delete-dst-snapshots-except 

361--include-snapshot-regex '.*_secondly' --include-snapshot-times-and-ranks '40 seconds ago..anytime' 'latest 40' 

362--new-snapshot-filter-group  

363--include-snapshot-regex '.*_minutely' --include-snapshot-times-and-ranks '40 minutes ago..anytime' 'latest 40' 

364--new-snapshot-filter-group  

365--include-snapshot-regex '.*_hourly' --include-snapshot-times-and-ranks '36 hours ago..anytime' 'latest 36' 

366--new-snapshot-filter-group  

367--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks '31 days ago..anytime' 'latest 31' 

368--new-snapshot-filter-group  

369--include-snapshot-regex '.*_weekly' --include-snapshot-times-and-ranks '12 weeks ago..anytime' 'latest 12' 

370--new-snapshot-filter-group  

371--include-snapshot-regex '.*_monthly' --include-snapshot-times-and-ranks '18 months ago..anytime' 'latest 18' 

372--new-snapshot-filter-group  

373--include-snapshot-regex '.*_yearly' --include-snapshot-times-and-ranks '5 years ago..anytime' 'latest 5'``` 

374 

375For convenience, the lengthy command line above can be expressed in a more concise way, like so: 

376 

377```$ {prog_name} {dummy_dataset} tank2/boo/bar --dryrun --recursive --skip-replication --delete-dst-snapshots  

378--delete-dst-snapshots-except-plan "{delete_dst_snapshots_except_plan_example1}"``` 

379 

380* Compare source and destination dataset trees recursively, for example to check if all recently taken snapshots have  

381been successfully replicated by a periodic job. List snapshots only contained in src (tagged with 'src'),  

382only contained in dst (tagged with 'dst'), and contained in both src and dst (tagged with 'all'), restricted to hourly  

383and daily snapshots taken within the last 7 days, excluding the last 4 hours (to allow for some slack/stragglers),  

384excluding temporary datasets: 

385 

386```$ {prog_name} tank1/foo/bar tank2/boo/bar --skip-replication --compare-snapshot-lists=src+dst+all --recursive 

387--include-snapshot-regex '.*_(hourly|daily)' --include-snapshot-times-and-ranks '7 days ago..4 hours ago'  

388--exclude-dataset-regex '(.*/)?tmp.*'``` 

389 

390If the resulting TSV output file contains zero lines starting with the prefix 'src' and zero lines starting with the  

391prefix 'dst' then no source snapshots are missing on the destination, and no destination snapshots are missing  

392on the source, indicating that the periodic replication and pruning jobs perform as expected. The TSV output is sorted  

393by dataset, and by ZFS creation time within each dataset - the first and last line prefixed with 'all' contains the  

394metadata of the oldest and latest common snapshot, respectively. The --compare-snapshot-lists option also directly  

395logs various summary stats, such as the metadata of the latest common snapshot, latest snapshots and oldest snapshots,  

396as well as the time diff between the latest common snapshot and latest snapshot only in src (and only in dst), as well  

397as how many src snapshots and how many GB of data are missing on dst, etc. 

398 

399* Example with further options: 

400 

401```$ {prog_name} tank1/foo/bar root@host2.example.com:tank2/boo/bar --recursive 

402--exclude-snapshot-regex '.*_(secondly|minutely)' --exclude-snapshot-regex 'test_.*' 

403--include-snapshot-times-and-ranks '7 days ago..anytime' 'latest 7' --exclude-dataset /tank1/foo/bar/temporary 

404--exclude-dataset /tank1/foo/bar/baz/trash --exclude-dataset-regex '(.*/)?private' 

405--exclude-dataset-regex '(.*/)?[Tt][Ee]?[Mm][Pp][-_]?[0-9]*' --ssh-dst-private-key /root/.ssh/id_rsa``` 

406""", formatter_class=argparse.RawTextHelpFormatter) 

407 

408 parser.add_argument( 

409 "root_dataset_pairs", nargs="+", action=DatasetPairsAction, metavar="SRC_DATASET DST_DATASET", 

410 help="SRC_DATASET: " 

411 "Source ZFS dataset (and its descendants) that will be replicated. Can be a ZFS filesystem or ZFS volume. " 

412 "Format is [[user@]host:]dataset. The host name can also be an IPv4 address (or an IPv6 address where " 

413 "each ':' colon character must be replaced with a '|' pipe character for disambiguation). If the " 

414 "host name is '-', the dataset will be on the local host, and the corresponding SSH leg will be omitted. " 

415 "The same is true if the host is omitted and the dataset does not contain a ':' colon at the same time. " 

416 "Local dataset examples: `tank1/foo/bar`, `tank1`, `-:tank1/foo/bar:baz:boo` " 

417 "Remote dataset examples: `host:tank1/foo/bar`, `host.example.com:tank1/foo/bar`, " 

418 "`root@host:tank`, `root@host.example.com:tank1/foo/bar`, `user@127.0.0.1:tank1/foo/bar:baz:boo`, " 

419 "`user@||1:tank1/foo/bar:baz:boo`. " 

420 "The first component of the ZFS dataset name is the ZFS pool name, here `tank1`. " 

421 "If the option starts with a `+` prefix then dataset names are read from the UTF-8 text file given " 

422 "after the `+` prefix, with each line in the file containing a SRC_DATASET and a DST_DATASET, " 

423 "separated by a tab character. Example: `+root_dataset_names.txt`, `+/path/to/root_dataset_names.txt`\n\n" 

424 "DST_DATASET: " 

425 "Destination ZFS dataset for replication and deletion. Has same naming format as SRC_DATASET. During " 

426 "replication, destination datasets that do not yet exist are created as necessary, along with their " 

427 "parent and ancestors.\n\n" 

428 f"*Performance Note:* {prog_name} automatically replicates multiple datasets in parallel. It replicates " 

429 "snapshots in parallel across datasets and serially within a dataset. All child datasets of a dataset " 

430 "may be processed in parallel. For consistency, processing of a dataset only starts after processing of " 

431 "all its ancestor datasets has completed. Further, when a thread is ready to start processing another " 

432 "dataset, it chooses the next dataset wrt. case-sensitive sort order from the datasets that are " 

433 "currently available for start of processing. Initially, only the roots of the selected dataset subtrees " 

434 "are available for start of processing. The degree of parallelism is configurable with the --threads " 

435 "option (see below).\n\n") 

436 parser.add_argument( 

437 "--recursive", "-r", action="store_true", 

438 help="During snapshot creation, replication, deletion and comparison, also consider descendant datasets, i.e. " 

439 "datasets within the dataset tree, including children, and children of children, etc.\n\n") 

440 parser.add_argument( 

441 "--include-dataset", action=FileOrLiteralAction, nargs="+", default=[], metavar="DATASET", 

442 help="During snapshot creation, replication, deletion and comparison, select any ZFS dataset (and its descendants) " 

443 "that is contained within SRC_DATASET (DST_DATASET in case of deletion) if its dataset name is one of the " 

444 "given include dataset names but none of the exclude dataset names. If a dataset is excluded its descendants " 

445 "are automatically excluded too, and this decision is never reconsidered even for the descendants because " 

446 "exclude takes precedence over include.\n\n" 

447 "A dataset name is absolute if the specified dataset is prefixed by `/`, e.g. `/tank/baz/tmp`. " 

448 "Otherwise the dataset name is relative wrt. source and destination, e.g. `baz/tmp` if the source " 

449 "is `tank`.\n\n" 

450 "This option is automatically translated to an --include-dataset-regex (see below) and can be " 

451 "specified multiple times.\n\n" 

452 "If the option starts with a `+` prefix then dataset names are read from the newline-separated " 

453 "UTF-8 text file given after the `+` prefix, one dataset per line inside of the text file.\n\n" 

454 "Examples: `/tank/baz/tmp` (absolute), `baz/tmp` (relative), " 

455 "`+dataset_names.txt`, `+/path/to/dataset_names.txt`\n\n") 

456 parser.add_argument( 

457 "--exclude-dataset", action=FileOrLiteralAction, nargs="+", default=[], metavar="DATASET", 

458 help="Same syntax as --include-dataset (see above) except that the option is automatically translated to an " 

459 "--exclude-dataset-regex (see below).\n\n") 

460 parser.add_argument( 

461 "--include-dataset-regex", action=FileOrLiteralAction, nargs="+", default=[], metavar="REGEX", 

462 help="During snapshot creation, replication (and deletion) and comparison, select any ZFS dataset (and its " 

463 "descendants) that is contained within SRC_DATASET (DST_DATASET in case of deletion) if its relative dataset " 

464 "path (e.g. `baz/tmp`) wrt. SRC_DATASET (DST_DATASET in case of deletion) matches at least one of the given " 

465 "include regular expressions but none of the exclude regular expressions. " 

466 "If a dataset is excluded its descendants are automatically excluded too, and this decision is never " 

467 "reconsidered even for the descendants because exclude takes precedence over include.\n\n" 

468 "This option can be specified multiple times. " 

469 "A leading `!` character indicates logical negation, i.e. the regex matches if the regex with the " 

470 "leading `!` character removed does not match.\n\n" 

471 "If the option starts with a `+` prefix then regex names are read from the newline-separated " 

472 "UTF-8 text file given after the `+` prefix, one regex per line inside of the text file.\n\n" 

473 "Default: `.*` (include all datasets).\n\n" 

474 "Examples: `baz/tmp`, `(.*/)?doc[^/]*/(private|confidential).*`, `!public`, " 

475 "`+dataset_regexes.txt`, `+/path/to/dataset_regexes.txt`\n\n") 

476 parser.add_argument( 

477 "--exclude-dataset-regex", action=FileOrLiteralAction, nargs="+", default=[], metavar="REGEX", 

478 help="Same syntax as --include-dataset-regex (see above) except that the default is " 

479 f"`{exclude_dataset_regexes_default}` (exclude tmp datasets). Example: `!.*` (exclude no dataset)\n\n") 

480 parser.add_argument( 

481 "--exclude-dataset-property", default=None, action=NonEmptyStringAction, metavar="STRING", 

482 help="The name of a ZFS dataset user property (optional). If this option is specified, the effective value " 

483 "(potentially inherited) of that user property is read via 'zfs list' for each selected source dataset " 

484 "to determine whether the dataset will be included or excluded, as follows:\n\n" 

485 "a) Value is 'true' or '-' or empty string or the property is missing: Include the dataset.\n\n" 

486 "b) Value is 'false': Exclude the dataset and its descendants.\n\n" 

487 "c) Value is a comma-separated list of host names (no spaces, for example: " 

488 "'store001,store002'): Include the dataset if the host name of " 

489 f"the host executing {prog_name} is contained in the list, otherwise exclude the dataset and its " 

490 "descendants.\n\n" 

491 "If a dataset is excluded its descendants are automatically excluded too, and the property values of the " 

492 "descendants are ignored because exclude takes precedence over include.\n\n" 

493 "Examples: 'syncoid:sync', 'com.example.eng.project.x:backup'\n\n" 

494 "*Note:* The use of --exclude-dataset-property is discouraged for most use cases. It is more flexible, " 

495 "more powerful, *and* more efficient to instead use a combination of --include/exclude-dataset-regex " 

496 "and/or --include/exclude-dataset to achieve the same or better outcome.\n\n") 

497 parser.add_argument( 

498 "--include-snapshot-regex", action=FileOrLiteralAction, nargs="+", default=[], metavar="REGEX", 

499 help="During replication, deletion and comparison, select any source ZFS snapshot that has a name (i.e. the part " 

500 "after the '@') that matches at least one of the given include regular expressions but none of the " 

501 "exclude regular expressions. If a snapshot is excluded this decision is never reconsidered because " 

502 "exclude takes precedence over include.\n\n" 

503 "This option can be specified multiple times. " 

504 "A leading `!` character indicates logical negation, i.e. the regex matches if the regex with the " 

505 "leading `!` character removed does not match.\n\n" 

506 "Default: `.*` (include all snapshots). " 

507 "Examples: `test_.*`, `!prod_.*`, `.*_(hourly|frequent)`, `!.*_(weekly|daily)`\n\n" 

508 "*Note:* All --include/exclude-snapshot-* CLI option groups are combined into a mini filter pipeline. " 

509 "A filter pipeline is executed in the order given on the command line, left to right. For example if " 

510 "--include-snapshot-times-and-ranks (see below) is specified on the command line before " 

511 "--include/exclude-snapshot-regex, then --include-snapshot-times-and-ranks will be applied before " 

512 "--include/exclude-snapshot-regex. The pipeline results would not always be the same if the order were " 

513 "reversed. Order matters.\n\n" 

514 "*Note:* During replication, bookmarks are always retained aka selected in order to help find common " 

515 "snapshots between source and destination.\n\n") 

516 parser.add_argument( 

517 "--exclude-snapshot-regex", action=FileOrLiteralAction, nargs="+", default=[], metavar="REGEX", 

518 help="Same syntax as --include-snapshot-regex (see above) except that the default is to exclude no " 

519 "snapshots.\n\n") 

520 parser.add_argument( 

521 "--include-snapshot-times-and-ranks", action=TimeRangeAndRankRangeAction, nargs="+", default=[], 

522 metavar=("TIMERANGE", "RANKRANGE"), 

523 help="This option takes as input parameters a time range filter and an optional rank range filter. It " 

524 "separately computes the results for each filter and selects the UNION of both results. " 

525 "To instead use a pure rank range filter (no UNION), or a pure time range filter (no UNION), simply " 

526 "use 'notime' aka '0..0' to indicate an empty time range, or omit the rank range, respectively. " 

527 "This option can be specified multiple times.\n\n" 

528 "<b>*Replication Example (UNION):* </b>\n\n" 

529 "Specify to replicate all daily snapshots created during the last 7 days, " 

530 "and at the same time ensure that the latest 7 daily snapshots (per dataset) are replicated regardless " 

531 "of creation time, like so: " 

532 "`--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks '7 days ago..anytime' 'latest 7'`\n\n" 

533 "<b>*Deletion Example (no UNION):* </b>\n\n" 

534 "Specify to delete all daily snapshots older than 7 days, but ensure that the " 

535 "latest 7 daily snapshots (per dataset) are retained regardless of creation time, like so: " 

536 "`--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks notime 'all except latest 7' " 

537 "--include-snapshot-times-and-ranks 'anytime..7 days ago'`" 

538 "\n\n" 

539 "This helps to safely cope with irregular scenarios where no snapshots were created or received within " 

540 "the last 7 days, or where more than 7 daily snapshots were created within the last 7 days. It can also " 

541 "help to avoid accidental pruning of the last snapshot that source and destination have in common.\n\n" 

542 "" 

543 "<b>*TIMERANGE:* </b>\n\n" 

544 "The ZFS 'creation' time of a snapshot (and bookmark) must fall into this time range in order for the " 

545 "snapshot to be included. The time range consists of a 'start' time, followed by a '..' separator, " 

546 "followed by an 'end' time. For example '2024-01-01..2024-04-01', or 'anytime..anytime' aka `*..*` aka all " 

547 "times, or 'notime' aka '0..0' aka empty time range. Only snapshots (and bookmarks) in the half-open time " 

548 "range [start, end) are included; other snapshots (and bookmarks) are excluded. If a snapshot is excluded " 

549 "this decision is never reconsidered because exclude takes precedence over include. Each of the two specified " 

550 "times can take any of the following forms:\n\n" 

551 "* a) `anytime` aka `*` wildcard; represents negative or positive infinity.\n\n" 

552 "* b) a non-negative integer representing a UTC Unix time in seconds. Example: 1728109805\n\n" 

553 "* c) an ISO 8601 datetime string with or without timezone. Examples: '2024-10-05', " 

554 "'2024-10-05T14:48:55', '2024-10-05T14:48:55+02', '2024-10-05T14:48:55-04:30'. If the datetime string " 

555 "does not contain time zone info then it is assumed to be in the local time zone. Timezone string support " 

556 "requires Python >= 3.11.\n\n" 

557 "* d) a duration that indicates how long ago from the current time, using the following syntax: " 

558 "a non-negative integer, followed by an optional space, followed by a duration unit that is " 

559 "*one* of 'seconds', 'secs', 'minutes', 'mins', 'hours', 'days', 'weeks', 'months', 'years', " 

560 "followed by an optional space, followed by the word 'ago'. " 

561 "Examples: '0secs ago', '40 mins ago', '36hours ago', '90days ago', '12weeksago'.\n\n" 

562 "* Note: This option compares the specified time against the standard ZFS 'creation' time property of the " 

563 "snapshot (which is a UTC Unix time in integer seconds), rather than against a timestamp that may be " 

564 "part of the snapshot name. You can list the ZFS creation time of snapshots and bookmarks as follows: " 

565 "`zfs list -t snapshot,bookmark -o name,creation -s creation -d 1 $SRC_DATASET` (optionally add " 

566 "the -p flag to display UTC Unix time in integer seconds).\n\n" 

567 "*Note:* During replication, bookmarks are always retained aka selected in order to help find common " 

568 "snapshots between source and destination.\n\n" 

569 "" 

570 "<b>*RANKRANGE:* </b>\n\n" 

571 "Specifies to include the N (or N%%) oldest snapshots or latest snapshots, and exclude all other " 

572 "snapshots (default: include no snapshots). Snapshots are sorted by creation time (actually, by the " 

573 "'createtxg' ZFS property, which serves the same purpose but is more precise). The rank position of a " 

574 "snapshot is the zero-based integer position of the snapshot within that sorted list. A rank consists of the " 

575 "optional words 'all except' (followed by an optional space), followed by the word 'oldest' or 'latest', " 

576 "followed by a non-negative integer, followed by an optional '%%' percent sign. A rank range consists of a " 

577 "lower rank, followed by a '..' separator, followed by a higher rank. " 

578 "If the optional lower rank is missing it is assumed to be 0. Examples:\n\n" 

579 "* 'oldest 10%%' aka 'oldest 0..oldest 10%%' (include the oldest 10%% of all snapshots)\n\n" 

580 "* 'latest 10%%' aka 'latest 0..latest 10%%' (include the latest 10%% of all snapshots)\n\n" 

581 "* 'all except latest 10%%' aka 'oldest 90%%' aka 'oldest 0..oldest 90%%' (include all snapshots except the " 

582 "latest 10%% of all snapshots)\n\n" 

583 "* 'oldest 90' aka 'oldest 0..oldest 90' (include the oldest 90 snapshots)\n\n" 

584 "* 'latest 90' aka 'latest 0..latest 90' (include the latest 90 snapshots)\n\n" 

585 "* 'all except oldest 90' aka 'oldest 90..oldest 100%%' (include all snapshots except the oldest 90 snapshots)" 

586 "\n\n" 

587 "* 'all except latest 90' aka 'latest 90..latest 100%%' (include all snapshots except the latest 90 snapshots)" 

588 "\n\n" 

589 "* 'latest 1' aka 'latest 0..latest 1' (include the latest snapshot)\n\n" 

590 "* 'all except latest 1' aka 'latest 1..latest 100%%' (include all snapshots except the latest snapshot)\n\n" 

591 "* 'oldest 2' aka 'oldest 0..oldest 2' (include the oldest 2 snapshots)\n\n" 

592 "* 'all except oldest 2' aka 'oldest 2..oldest 100%%' (include all snapshots except the oldest 2 snapshots)\n\n" 

593 "* 'oldest 100%%' aka 'oldest 0..oldest 100%%' (include all snapshots)\n\n" 

594 "* 'oldest 0%%' aka 'oldest 0..oldest 0%%' (include no snapshots)\n\n" 

595 "* 'oldest 0' aka 'oldest 0..oldest 0' (include no snapshots)\n\n" 

596 "*Note:* Percentage calculations are not based on the number of snapshots " 

597 "contained in the dataset on disk, but rather based on the number of snapshots arriving at the filter. " 

598 "For example, if only two daily snapshots arrive at the filter because a prior filter excludes hourly " 

599 "snapshots, then 'latest 10' will only include these two daily snapshots, and 'latest 50%%' will only " 

600 "include one of these two daily snapshots.\n\n" 

601 "*Note:* During replication, bookmarks are always retained aka selected in order to help find common " 

602 "snapshots between source and destination. Bookmarks do not count towards N or N%% wrt. rank.\n\n" 

603 "*Note:* If a snapshot is excluded this decision is never reconsidered because exclude takes precedence " 

604 "over include.\n\n") 

605 

606 def format_dict(dictionary): 

607 return f'"{dictionary}"' 

608 

609 src_snapshot_plan_example = { 

610 "prod": { 

611 "onsite": {"secondly": 40, "minutely": 40, "hourly": 36, "daily": 31, "weekly": 12, "monthly": 18, "yearly": 5}, 

612 "us-west-1": {"secondly": 0, "minutely": 0, "hourly": 36, "daily": 31, "weekly": 12, "monthly": 18, "yearly": 5}, 

613 "eu-west-1": {"secondly": 0, "minutely": 0, "hourly": 36, "daily": 31, "weekly": 12, "monthly": 18, "yearly": 5}, 

614 }, 

615 "test": { 

616 "offsite": {"12hourly": 42, "weekly": 12}, 

617 "onsite": {"100millisecondly": 42}, 

618 }, 

619 } 

620 parser.add_argument( 

621 "--include-snapshot-plan", action=IncludeSnapshotPlanAction, default=None, 

622 metavar="DICT_STRING", 

623 help="Replication periods to be used if replicating snapshots within the selected destination datasets. " 

624 "Has the same format as --create-src-snapshots-plan and --delete-dst-snapshots-except-plan (see below). " 

625 "Snapshots that do not match a period will not be replicated. To avoid unexpected surprises, make sure to " 

626 "carefully specify ALL snapshot names and periods that shall be replicated, in combination with --dryrun.\n\n" 

627 f"Example: `{format_dict(src_snapshot_plan_example)}`. This example will, for the organization 'prod' and the " 

628 "intended logical target 'onsite', replicate secondly snapshots that were created less than 40 seconds ago, " 

629 "yet replicate the latest 40 secondly snapshots regardless of creation time. Analog for the latest 40 minutely " 

630 "snapshots, latest 36 hourly snapshots, etc. " 

631 "Note: A zero within a period (e.g. 'hourly': 0) indicates that no snapshots shall be replicated for the given " 

632 "period.\n\n" 

633 "Note: --include-snapshot-plan is a convenience option that auto-generates a series of the following other " 

634 "options: --new-snapshot-filter-group, --include-snapshot-regex, --include-snapshot-times-and-ranks\n\n") 

635 parser.add_argument( 

636 "--new-snapshot-filter-group", action=NewSnapshotFilterGroupAction, nargs=0, 

637 help="Starts a new snapshot filter group containing separate --{include|exclude}-snapshot-* filter options. The " 

638 "program separately computes the results for each filter group and selects the UNION of all results. " 

639 "This option can be specified multiple times and serves as a separator between groups. Example:\n\n" 

640 "Delete all minutely snapshots older than 40 minutes, but ensure that the latest 40 minutely snapshots (per " 

641 "dataset) are retained regardless of creation time. Additionally, delete all hourly snapshots older than 36 " 

642 "hours, but ensure that the latest 36 hourly snapshots (per dataset) are retained regardless of creation time. " 

643 "Additionally, delete all daily snapshots older than 31 days, but ensure that the latest 31 daily snapshots " 

644 "(per dataset) are retained regardless of creation time: " 

645 f"`{prog_name} {dummy_dataset} tank2/boo/bar --dryrun --recursive --skip-replication --delete-dst-snapshots " 

646 "--include-snapshot-regex '.*_minutely' --include-snapshot-times-and-ranks notime 'all except latest 40' " 

647 "--include-snapshot-times-and-ranks 'anytime..40 minutes ago' " 

648 "--new-snapshot-filter-group " 

649 "--include-snapshot-regex '.*_hourly' --include-snapshot-times-and-ranks notime 'all except latest 36' " 

650 "--include-snapshot-times-and-ranks 'anytime..36 hours ago' " 

651 "--new-snapshot-filter-group " 

652 "--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks notime 'all except latest 31' " 

653 "--include-snapshot-times-and-ranks 'anytime..31 days ago'`\n\n") 

654 parser.add_argument( 

655 "--create-src-snapshots", action="store_true", 

656 help="Do nothing if the --create-src-snapshots flag is missing. Otherwise, before the replication step (see below), " 

657 "atomically create new snapshots of the source datasets selected via --{include|exclude}-dataset* policy. " 

658 "The names of the snapshots can be configured via --create-src-snapshots-* suboptions (see below). " 

659 "To create snapshots only, without any other processing such as replication, etc, consider using this flag " 

660 "together with the --skip-replication flag.\n\n" 

661 "A periodic snapshot is created if it is due per the schedule indicated by --create-src-snapshots-plan " 

662 "(for example '_daily' or '_hourly' or _'10minutely' or '_2secondly' or '_100millisecondly'), or if the " 

663 "--create-src-snapshots-even-if-not-due flag is specified, or if the most recent scheduled snapshot " 

664 f"is somehow missing. In the latter case {prog_name} immediately creates a snapshot (tagged with the current " 

665 "time, not backdated to the missed time), and then resumes the original schedule.\n\n" 

666 "If the snapshot suffix is '_adhoc' or not a known period then a snapshot is considered " 

667 "non-periodic and is thus created immediately regardless of the creation time of any existing snapshot.\n\n" 

668 "The implementation attempts to fit as many datasets as possible into a single (atomic) 'zfs snapshot' command " 

669 "line, using case-sensitive sort order, and using 'zfs snapshot -r' to the extent that this is compatible " 

670 "with the actual results of the schedule and the actual results of the --{include|exclude}-dataset* pruning " 

671 "policy. The snapshots of all datasets that fit " 

672 "within the same single 'zfs snapshot' CLI invocation will be taken within the same ZFS transaction group, and " 

673 "correspondingly have identical 'createtxg' ZFS property (but not necessarily identical 'creation' ZFS time " 

674 "property as ZFS actually provides no such guarantee), and thus be consistent. Dataset names that can't fit " 

675 "into a single command line are spread over multiple command line invocations, respecting the limits that the " 

676 "operating system places on the maximum length of a single command line, per `getconf ARG_MAX`.\n\n" 

677 f"Note: All {prog_name} functions including snapshot creation, replication, deletion, comparison, etc. happily " 

678 "work with any snapshots in any format, even created or managed by third party ZFS snapshot management tools, " 

679 "including manual zfs snapshot/destroy.\n\n") 

680 parser.add_argument( 

681 "--create-src-snapshots-plan", default=None, type=str, metavar="DICT_STRING", 

682 help="Creation periods that specify a schedule for when new snapshots shall be created on src within the selected " 

683 "datasets. Has the same format as --delete-dst-snapshots-except-plan.\n\n" 

684 f"Example: `{format_dict(src_snapshot_plan_example)}`. This example will, for the organization 'prod' and " 

685 "the intended logical target 'onsite', create 'secondly' snapshots every second, 'minutely' snapshots every " 

686 "minute, hourly snapshots every hour, and so on. " 

687 "It will also create snapshots for the targets 'us-west-1' and 'eu-west-1' within the 'prod' organization. " 

688 "In addition, it will create snapshots every 12 hours and every week for the 'test' organization, " 

689 "and name them as being intended for the 'offsite' replication target. Analog for snapshots that are taken " 

690 "every 100 milliseconds within the 'test' organization.\n\n" 

691 "The example creates ZFS snapshots with names like " 

692 "`prod_onsite_<timestamp>_secondly`, `prod_onsite_<timestamp>_minutely`, " 

693 "`prod_us-west-1_<timestamp>_hourly`, `prod_us-west-1_<timestamp>_daily`, " 

694 "`prod_eu-west-1_<timestamp>_hourly`, `prod_eu-west-1_<timestamp>_daily`, " 

695 "`test_offsite_<timestamp>_12hourly`, `test_offsite_<timestamp>_weekly`, and so on.\n\n" 

696 "Note: A period name that is missing indicates that no snapshots shall be created for the given period.\n\n" 

697 "The period name can contain an optional positive integer immediately preceding the time period unit, for " 

698 "example `_2secondly` or `_10minutely` or `_100millisecondly` to indicate that snapshots are taken every 2 " 

699 "seconds, or every 10 minutes, or every 100 milliseconds, respectively.\n\n") 

700 

701 def argparser_escape(text: str) -> str: 

702 return text.replace('%', '%%') 

703 

704 create_src_snapshots_timeformat_dflt = "%Y-%m-%d_%H:%M:%S" 

705 parser.add_argument( 

706 "--create-src-snapshots-timeformat", default=create_src_snapshots_timeformat_dflt, metavar="STRFTIME_SPEC", 

707 help=f"Default is `{argparser_escape(create_src_snapshots_timeformat_dflt)}`. For the strftime format, see " 

708 "https://docs.python.org/3.11/library/datetime.html#strftime-strptime-behavior. " 

709 f"Examples: `{argparser_escape('%Y-%m-%d_%H:%M:%S.%f')}` (adds microsecond resolution), " 

710 f"`{argparser_escape('%Y-%m-%d_%H:%M:%S%z')}` (adds timezone offset), " 

711 f"`{argparser_escape('%Y-%m-%dT%H-%M-%S')}` (no colons).\n\n" 

712 "The name of the snapshot created on the src is `$org_$target_strftime(--create-src-snapshots-time*)_$period`. " 

713 "Example: `tank/foo@prod_us-west-1_2024-09-03_12:26:15_daily`\n\n") 

714 parser.add_argument( 

715 "--create-src-snapshots-timezone", default="", type=str, metavar="TZ_SPEC", 

716 help=f"Default is the local timezone of the system running {prog_name}. When creating a new snapshot on the source, " 

717 "fetch the current time in the specified timezone, and feed that time, and the value of " 

718 "--create-src-snapshots-timeformat, into the standard strftime() function to generate the timestamp portion " 

719 "of the snapshot name. The TZ_SPEC input parameter is of the form 'UTC' or '+HHMM' or '-HHMM' for fixed UTC " 

720 "offsets, or an IANA TZ identifier for auto-adjustment to daylight savings time, or the empty string to use " 

721 "the local timezone, for example '', 'UTC', '+0000', '+0530', '-0400', 'America/Los_Angeles', 'Europe/Vienna'. " 

722 "For a list of valid IANA TZ identifiers see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List" 

723 "\n\nTo change the timezone not only for snapshot name creation, but in all respects for the entire program, " 

724 "use the standard 'TZ' Unix environment variable, like so: `export TZ=UTC`.\n\n") 

725 parser.add_argument( 

726 "--create-src-snapshots-even-if-not-due", action="store_true", 

727 help="Take snapshots immediately regardless of the creation time of any existing snapshot, even if snapshots " 

728 "are periodic and not actually due per the schedule.\n\n") 

729 parser.add_argument( 

730 "--create-src-snapshots-enable-snapshots-changed-cache", action="store_true", 

731 help="Maintain a local cache of recent snapshot creation times, running " 

732 "'zfs list -t filesystem,volume -o snapshots_changed' instead of 'zfs list -t snapshot' to determine if a new " 

733 "snapshot shall be created on the src. This flag improves performance for high-frequency snapshotting use " 

734 "cases. Only relevant if --create-src-snapshots-even-if-not-due is not specified.\n\n") 

735 zfs_send_program_opts_default = "--props --raw --compressed" 

736 parser.add_argument( 

737 "--zfs-send-program-opts", type=str, default=zfs_send_program_opts_default, metavar="STRING", 

738 help="Parameters to fine-tune 'zfs send' behaviour (optional); will be passed into 'zfs send' CLI. " 

739 "The value is split on runs of one or more whitespace characters. " 

740 f"Default is '{zfs_send_program_opts_default}'. To run `zfs send` without options, specify the empty " 

741 "string: `--zfs-send-program-opts=''`. " 

742 "See https://openzfs.github.io/openzfs-docs/man/master/8/zfs-send.8.html " 

743 "and https://github.com/openzfs/zfs/issues/13024\n\n") 

744 zfs_recv_program_opts_default = "-u" 

745 parser.add_argument( 

746 "--zfs-recv-program-opts", type=str, default=zfs_recv_program_opts_default, metavar="STRING", 

747 help="Parameters to fine-tune 'zfs receive' behaviour (optional); will be passed into 'zfs receive' CLI. " 

748 "The value is split on runs of one or more whitespace characters. " 

749 f"Default is '{zfs_recv_program_opts_default}'. To run `zfs receive` without options, specify the empty " 

750 "string: `--zfs-recv-program-opts=''`. " 

751 "Example: '-u -o canmount=noauto -o readonly=on -x keylocation -x keyformat -x encryption'. " 

752 "See https://openzfs.github.io/openzfs-docs/man/master/8/zfs-receive.8.html " 

753 "and https://openzfs.github.io/openzfs-docs/man/master/7/zfsprops.7.html\n\n") 

754 parser.add_argument( 

755 "--zfs-recv-program-opt", action="append", default=[], metavar="STRING", 

756 help="Parameter to fine-tune 'zfs receive' behaviour (optional); will be passed into 'zfs receive' CLI. " 

757 "The value can contain spaces and is not split. This option can be specified multiple times. Example: `" 

758 "--zfs-recv-program-opt=-o " 

759 "--zfs-recv-program-opt='org.zfsbootmenu:commandline=ro debug zswap.enabled=1'`\n\n") 

760 parser.add_argument( 

761 "--force-rollback-to-latest-snapshot", action="store_true", 

762 help="Before replication, rollback the destination dataset to its most recent destination snapshot (if there " 

763 "is one), via 'zfs rollback', just in case the destination dataset was modified since its most recent " 

764 "snapshot. This is much less invasive than the other --force* options (see below).\n\n") 

765 parser.add_argument( 

766 "--force-rollback-to-latest-common-snapshot", action="store_true", 

767 help="Before replication, delete destination ZFS snapshots that are more recent than the most recent common " 

768 "snapshot selected on the source ('conflicting snapshots'), via 'zfs rollback'. Do no rollback if no common " 

769 "snapshot is selected.\n\n") 

770 parser.add_argument( 

771 "--force", action="store_true", 

772 help="Same as --force-rollback-to-latest-common-snapshot (see above), except that additionally, if no common " 

773 "snapshot is selected, then delete all destination snapshots before starting replication, and proceed " 

774 "without aborting. Without the --force* flags, the destination dataset is treated as append-only, hence " 

775 "no destination snapshot that already exists is deleted, and instead the operation is aborted with an " 

776 "error when encountering a conflicting snapshot.\n\n" 

777 "Analogy: --force-rollback-to-latest-snapshot is a tiny hammer, whereas " 

778 "--force-rollback-to-latest-common-snapshot is a medium sized hammer, --force is a large hammer, and " 

779 "--force-destroy-dependents is a very large hammer. " 

780 "Consider using the smallest hammer that can fix the problem. No hammer is ever used by default.\n\n") 

781 parser.add_argument( 

782 "--force-destroy-dependents", action="store_true", 

783 help="On destination, --force and --force-rollback-to-latest-common-snapshot and --delete-* will add the " 

784 "'-R' flag to their use of 'zfs rollback' and 'zfs destroy', causing them to delete dependents such as " 

785 "clones and bookmarks. This can be very destructive and is rarely advisable.\n\n") 

786 parser.add_argument( 

787 "--force-hard", action="store_true", # deprecated; was renamed to --force-destroy-dependents 

788 help=argparse.SUPPRESS) 

789 parser.add_argument( 

790 "--force-unmount", action="store_true", 

791 help="On destination, --force and --force-rollback-to-latest-common-snapshot will add the '-f' flag to their " 

792 "use of 'zfs rollback' and 'zfs destroy'.\n\n") 

793 parser.add_argument( 

794 "--force-once", "--f1", action="store_true", 

795 help="Use the --force option or --force-rollback-to-latest-common-snapshot option at most once to resolve a " 

796 "conflict, then abort with an error on any subsequent conflict. This helps to interactively resolve " 

797 "conflicts, one conflict at a time.\n\n") 

798 parser.add_argument( 

799 "--skip-parent", action="store_true", 

800 help="During replication and deletion, skip processing of the SRC_DATASET and DST_DATASET and only process " 

801 "their descendant datasets, i.e. children, and children of children, etc (with --recursive). No dataset " 

802 "is processed unless --recursive is also specified. " 

803 f"Analogy: `{prog_name} --recursive --skip-parent src dst` is akin to Unix `cp -r src/* dst/` whereas " 

804 f" `{prog_name} --recursive --skip-parent --skip-replication --delete-dst-datasets dummy dst` is akin to " 

805 f"Unix `rm -r dst/*`\n\n") 

806 parser.add_argument( 

807 "--skip-missing-snapshots", choices=["fail", "dataset", "continue"], default="dataset", nargs="?", 

808 help="During replication, handle source datasets that select no snapshots (and no relevant bookmarks) " 

809 "as follows:\n\n" 

810 "a) 'fail': Abort with an error.\n\n" 

811 "b) 'dataset' (default): Skip the source dataset with a warning. Skip descendant datasets if " 

812 "--recursive and destination dataset does not exist. Otherwise skip to the next dataset.\n\n" 

813 "c) 'continue': Skip nothing. If destination snapshots exist, delete them (with --force) or abort " 

814 "with an error (without --force). If there is no such abort, continue processing with the next dataset. " 

815 "Eventually create empty destination dataset and ancestors if they do not yet exist and source dataset " 

816 "has at least one descendant that selects at least one snapshot.\n\n") 

817 retries_default = 2 

818 parser.add_argument( 

819 "--retries", type=int, min=0, default=retries_default, action=CheckRange, metavar="INT", 

820 help="The maximum number of times a retryable replication or deletion step shall be retried if it fails, for " 

821 f"example because of network hiccups (default: {retries_default}, min: 0). " 

822 "Also consider this option if a periodic pruning script may simultaneously delete a dataset or " 

823 f"snapshot or bookmark while {prog_name} is running and attempting to access it.\n\n") 

824 retry_min_sleep_secs_default = 0.125 

825 parser.add_argument( 

826 "--retry-min-sleep-secs", type=float, min=0, default=retry_min_sleep_secs_default, 

827 action=CheckRange, metavar="FLOAT", 

828 help=f"The minimum duration to sleep between retries (default: {retry_min_sleep_secs_default}).\n\n") 

829 retry_max_sleep_secs_default = 5 * 60 

830 parser.add_argument( 

831 "--retry-max-sleep-secs", type=float, min=0, default=retry_max_sleep_secs_default, 

832 action=CheckRange, metavar="FLOAT", 

833 help="The maximum duration to sleep between retries initially starts with --retry-min-sleep-secs (see above), " 

834 "and doubles on each retry, up to the final maximum of --retry-max-sleep-secs " 

835 f"(default: {retry_max_sleep_secs_default}). On each retry a random sleep time in the " 

836 "[--retry-min-sleep-secs, current max] range is picked. The timer resets after each operation.\n\n") 

837 retry_max_elapsed_secs_default = 60 * 60 

838 parser.add_argument( 

839 "--retry-max-elapsed-secs", type=float, min=0, default=retry_max_elapsed_secs_default, 

840 action=CheckRange, metavar="FLOAT", 

841 help="A single operation (e.g. 'zfs send/receive' of the current dataset, or deletion of a list of snapshots " 

842 "within the current dataset) will not be retried (or not retried anymore) once this much time has elapsed " 

843 f"since the initial start of the operation, including retries (default: {retry_max_elapsed_secs_default})." 

844 " The timer resets after each operation completes or retries exhaust, such that subsequently failing " 

845 "operations can again be retried.\n\n") 

846 parser.add_argument( 

847 "--skip-on-error", choices=["fail", "tree", "dataset"], default="dataset", 

848 help="During replication and deletion, if an error is not retryable, or --retries has been exhausted, " 

849 "or --skip-missing-snapshots raises an error, proceed as follows:\n\n" 

850 "a) 'fail': Abort the program with an error. This mode is ideal for testing, clear " 

851 "error reporting, and situations where consistency trumps availability.\n\n" 

852 "b) 'tree': Log the error, skip the dataset tree rooted at the dataset for which the error " 

853 "occurred, and continue processing the next (sibling) dataset tree. " 

854 "Example: Assume datasets tank/user1/foo and tank/user2/bar and an error occurs while processing " 

855 "tank/user1. In this case processing skips tank/user1/foo and proceeds with tank/user2.\n\n" 

856 "c) 'dataset' (default): Same as 'tree' except if the destination dataset already exists, skip to " 

857 "the next dataset instead. " 

858 "Example: Assume datasets tank/user1/foo and tank/user2/bar and an error occurs while " 

859 "processing tank/user1. In this case processing skips tank/user1 and proceeds with tank/user1/foo " 

860 "if the destination already contains tank/user1. Otherwise processing continues with tank/user2. " 

861 "This mode is for production use cases that require timely forward progress even in the presence of " 

862 "partial failures. For example, assume the job is to backup the home directories or virtual machines " 

863 "of thousands of users across an organization. Even if replication of some of the datasets for some " 

864 "users fails due too conflicts, busy datasets, etc, the replication job will continue for the " 

865 "remaining datasets and the remaining users.\n\n") 

866 parser.add_argument( 

867 "--skip-replication", action="store_true", 

868 help="Skip replication step (see above) and proceed to the optional --delete-dst-datasets step " 

869 "immediately (see below).\n\n") 

870 parser.add_argument( 

871 "--delete-dst-datasets", action="store_true", 

872 help="Do nothing if the --delete-dst-datasets option is missing. Otherwise, after successful replication " 

873 "step, if any, delete existing destination datasets that are selected via --{include|exclude}-dataset* " 

874 "policy yet do not exist within SRC_DATASET (which can be an empty dataset, such as the hardcoded virtual " 

875 f"dataset named '{dummy_dataset}'!). Do not recurse without --recursive. With --recursive, never delete " 

876 "non-selected dataset subtrees or their ancestors.\n\n" 

877 "For example, if the destination contains datasets h1,h2,h3,d1 whereas source only contains h3, " 

878 "and the include/exclude policy selects h1,h2,h3,d1, then delete datasets h1,h2,d1 on " 

879 "the destination to make it 'the same'. On the other hand, if the include/exclude policy " 

880 "only selects h1,h2,h3 then only delete datasets h1,h2 on the destination to make it 'the same'.\n\n" 

881 "Example to delete all tmp datasets within tank2/boo/bar: " 

882 f"`{prog_name} {dummy_dataset} tank2/boo/bar --dryrun --skip-replication --recursive " 

883 "--delete-dst-datasets --include-dataset-regex '(.*/)?tmp.*' --exclude-dataset-regex '!.*'`\n\n") 

884 parser.add_argument( 

885 "--delete-dst-snapshots", choices=["snapshots", "bookmarks"], default=None, const="snapshots", 

886 nargs="?", 

887 help="Do nothing if the --delete-dst-snapshots option is missing. Otherwise, after successful " 

888 "replication, and successful --delete-dst-datasets step, if any, delete existing destination snapshots " 

889 "whose GUID does not exist within the source dataset (which can be an empty dummy dataset!) if the " 

890 "destination snapshots are selected by the --include/exclude-snapshot-* policy, and the destination " 

891 "dataset is selected via --{include|exclude}-dataset* policy. Does not recurse without --recursive.\n\n" 

892 "For example, if the destination dataset contains snapshots h1,h2,h3,d1 (h=hourly, d=daily) whereas " 

893 "the source dataset only contains snapshot h3, and the include/exclude policy selects " 

894 "h1,h2,h3,d1, then delete snapshots h1,h2,d1 on the destination dataset to make it 'the same'. " 

895 "On the other hand, if the include/exclude policy only selects snapshots h1,h2,h3 then only " 

896 "delete snapshots h1,h2 on the destination dataset to make it 'the same'.\n\n" 

897 "*Note:* To delete snapshots regardless, consider using --delete-dst-snapshots in combination with a " 

898 f"source that is an empty dataset, such as the hardcoded virtual dataset named '{dummy_dataset}', like so:" 

899 f" `{prog_name} {dummy_dataset} tank2/boo/bar --dryrun --skip-replication --delete-dst-snapshots " 

900 "--include-snapshot-regex '.*_daily' --recursive`\n\n" 

901 "*Note:* Use --delete-dst-snapshots=bookmarks to delete bookmarks instead of snapshots, in which " 

902 "case no snapshots are selected and the --{include|exclude}-snapshot-* filter options treat bookmarks as " 

903 "snapshots wrt. selecting.\n\n" 

904 "*Performance Note:* --delete-dst-snapshots operates on multiple datasets in parallel (and serially " 

905 f"within a dataset), using the same dataset order as {prog_name} replication. " 

906 "The degree of parallelism is configurable with the --threads option (see below).\n\n") 

907 parser.add_argument( 

908 "--delete-dst-snapshots-no-crosscheck", action="store_true", 

909 help="This flag indicates that --delete-dst-snapshots=snapshots shall check the source dataset only for " 

910 "a snapshot with the same GUID, and ignore whether a bookmark with the same GUID is present in the " 

911 "source dataset. Similarly, it also indicates that --delete-dst-snapshots=bookmarks shall check the " 

912 "source dataset only for a bookmark with the same GUID, and ignore whether a snapshot with the same GUID " 

913 "is present in the source dataset.\n\n") 

914 parser.add_argument( 

915 "--delete-dst-snapshots-except", action="store_true", 

916 help="This flag indicates that the --include/exclude-snapshot-* options shall have inverted semantics for the " 

917 "--delete-dst-snapshots option, thus deleting all snapshots except for the selected snapshots (within the " 

918 "specified datasets), instead of deleting all selected snapshots (within the specified datasets). In other" 

919 " words, this flag enables to specify which snapshots to retain instead of which snapshots to delete.\n\n") 

920 parser.add_argument( 

921 "--delete-dst-snapshots-except-plan", action=DeleteDstSnapshotsExceptPlanAction, default=None, 

922 metavar="DICT_STRING", 

923 help="Retention periods to be used if pruning snapshots or bookmarks within the selected destination datasets via " 

924 "--delete-dst-snapshots. Has the same format as --create-src-snapshots-plan. " 

925 "Snapshots (--delete-dst-snapshots=snapshots) or bookmarks (with --delete-dst-snapshots=bookmarks) that " 

926 "do not match a period will be deleted. To avoid unexpected surprises, make sure to carefully specify ALL " 

927 "snapshot names and periods that shall be retained, in combination with --dryrun.\n\n" 

928 f"Example: `{format_dict(src_snapshot_plan_example)}`. This example will, for the organization 'prod' and " 

929 "the intended logical target 'onsite', retain secondly snapshots that were created less than 40 seconds ago, " 

930 "yet retain the latest 40 secondly snapshots regardless of creation time. Analog for the latest 40 minutely " 

931 "snapshots, latest 36 hourly snapshots, etc. " 

932 "It will also retain snapshots for the targets 'us-west-1' and 'eu-west-1' within the 'prod' organization. " 

933 "In addition, within the 'test' organization, it will retain snapshots that are created every 12 hours and " 

934 "every week as specified, and name them as being intended for the 'offsite' replication target. Analog for " 

935 "snapshots that are taken every 100 milliseconds within the 'test' organization. " 

936 "All other snapshots within the selected datasets will be deleted - you've been warned!\n\n" 

937 "The example scans the selected ZFS datasets for snapshots with names like " 

938 "`prod_onsite_<timestamp>_secondly`, `prod_onsite_<timestamp>_minutely`, " 

939 "`prod_us-west-1_<timestamp>_hourly`, `prod_us-west-1_<timestamp>_daily`, " 

940 "`prod_eu-west-1_<timestamp>_hourly`, `prod_eu-west-1_<timestamp>_daily`, " 

941 "`test_offsite_<timestamp>_12hourly`, `test_offsite_<timestamp>_weekly`, and so on, and deletes all snapshots " 

942 "that do not match a retention rule.\n\n" 

943 "Note: A zero within a period (e.g. 'hourly': 0) indicates that no snapshots shall be retained for the given " 

944 "period.\n\n" 

945 "Note: --delete-dst-snapshots-except-plan is a convenience option that auto-generates a series of the " 

946 "following other options: --delete-dst-snapshots-except, " 

947 "--new-snapshot-filter-group, --include-snapshot-regex, --include-snapshot-times-and-ranks\n\n") 

948 parser.add_argument( 

949 "--delete-empty-dst-datasets", choices=["snapshots", "snapshots+bookmarks"], default=None, 

950 const="snapshots+bookmarks", nargs="?", 

951 help="Do nothing if the --delete-empty-dst-datasets option is missing or --recursive is missing. Otherwise, " 

952 "after successful replication " 

953 "step and successful --delete-dst-datasets and successful --delete-dst-snapshots steps, if any, " 

954 "delete any selected destination dataset that has no snapshot and no bookmark if all descendants of " 

955 "that destination dataset are also selected and do not have a snapshot or bookmark either " 

956 "(again, only if the existing destination dataset is selected via --{include|exclude}-dataset* policy). " 

957 "Never delete non-selected dataset subtrees or their ancestors.\n\n" 

958 "For example, if the destination contains datasets h1,d1, and the include/exclude policy " 

959 "selects h1,d1, then check if h1,d1 can be deleted. " 

960 "On the other hand, if the include/exclude policy only selects h1 then only check if h1 " 

961 "can be deleted.\n\n" 

962 "*Note:* Use --delete-empty-dst-datasets=snapshots to delete snapshot-less datasets even if they still " 

963 "contain bookmarks.\n\n") 

964 cmp_choices_dflt = "+".join(cmp_choices_items) 

965 cmp_choices: List[str] = [] 

966 for i in range(0, len(cmp_choices_items)): 

967 cmp_choices += map(lambda item: "+".join(item), itertools.combinations(cmp_choices_items, i + 1)) 

968 parser.add_argument( 

969 "--compare-snapshot-lists", choices=cmp_choices, default=None, const=cmp_choices_dflt, nargs="?", 

970 help="Do nothing if the --compare-snapshot-lists option is missing. Otherwise, after successful replication " 

971 "step and successful --delete-dst-datasets, --delete-dst-snapshots steps and --delete-empty-dst-datasets " 

972 "steps, if any, proceed as follows:\n\n" 

973 "Compare source and destination dataset trees recursively wrt. snapshots, for example to check if all " 

974 "recently taken snapshots have been successfully replicated by a periodic job.\n\n" 

975 "Example: List snapshots only contained in source (tagged with 'src'), only contained in destination " 

976 "(tagged with 'dst'), and contained in both source and destination (tagged with 'all'), restricted to " 

977 "hourly and daily snapshots taken within the last 7 days, excluding the last 4 hours (to allow for some " 

978 "slack/stragglers), excluding temporary datasets: " 

979 f"`{prog_name} tank1/foo/bar tank2/boo/bar --skip-replication " 

980 "--compare-snapshot-lists=src+dst+all --recursive --include-snapshot-regex '.*_(hourly|daily)' " 

981 "--include-snapshot-times-and-ranks '7 days ago..4 hours ago' --exclude-dataset-regex 'tmp.*'`\n\n" 

982 "This outputs a TSV file containing the following columns:\n\n" 

983 "`location creation_iso createtxg rel_name guid root_dataset rel_dataset name creation written`\n\n" 

984 "Example output row:\n\n" 

985 "`src 2024-11-06_08:30:05 17435050 /foo@test_2024-11-06_08:30:05_daily 2406491805272097867 tank1/src " 

986 "/foo tank1/src/foo@test_2024-10-06_08:30:04_daily 1730878205 24576`\n\n" 

987 "If the TSV output file contains zero lines starting with the prefix 'src' and zero lines starting with " 

988 "the prefix 'dst' then no source snapshots are missing on the destination, and no destination " 

989 "snapshots are missing on the source, indicating that the periodic replication and pruning jobs perform " 

990 "as expected. The TSV output is sorted by rel_dataset, and by ZFS creation time within each rel_dataset " 

991 "- the first and last line prefixed with 'all' contains the metadata of the oldest and latest common " 

992 "snapshot, respectively. Third party tools can use this info for post-processing, for example using " 

993 "custom scripts using 'csplit' or duckdb analytics queries.\n\n" 

994 "The --compare-snapshot-lists option also directly logs various summary stats, such as the metadata of " 

995 "the latest common snapshot, latest snapshots and oldest snapshots, as well as the time diff between the " 

996 "latest common snapshot and latest snapshot only in src (and only in dst), as well as how many src " 

997 "snapshots and how many GB of data are missing on dst, etc.\n\n" 

998 "*Note*: Consider omitting the 'all' flag to reduce noise and instead focus on missing snapshots only, " 

999 "like so: --compare-snapshot-lists=src+dst \n\n" 

1000 "*Note*: The source can also be an empty dataset, such as the hardcoded virtual dataset named " 

1001 f"'{dummy_dataset}'.\n\n" 

1002 "*Note*: --compare-snapshot-lists is typically *much* faster than standard 'zfs list -t snapshot' CLI " 

1003 "usage because the former issues requests with a higher degree of parallelism than the latter. The " 

1004 "degree is configurable with the --threads option (see below).\n\n") 

1005 parser.add_argument( 

1006 "--dryrun", "-n", choices=["recv", "send"], default=None, const="send", nargs="?", 

1007 help="Do a dry run (aka 'no-op') to print what operations would happen if the command were to be executed " 

1008 "for real (optional). This option treats both the ZFS source and destination as read-only. " 

1009 "Accepts an optional argument for fine tuning that is handled as follows:\n\n" 

1010 "a) 'recv': Send snapshot data via 'zfs send' to the destination host and receive it there via " 

1011 "'zfs receive -n', which discards the received data there.\n\n" 

1012 "b) 'send': Do not execute 'zfs send' and do not execute 'zfs receive'. This is a less 'realistic' form " 

1013 "of dry run, but much faster, especially for large snapshots and slow networks/disks, as no snapshot is " 

1014 "actually transferred between source and destination. This is the default when specifying --dryrun.\n\n" 

1015 "Examples: --dryrun, --dryrun=send, --dryrun=recv\n\n") 

1016 parser.add_argument( 

1017 "--verbose", "-v", action="count", default=0, 

1018 help="Print verbose information. This option can be specified multiple times to increase the level of " 

1019 "verbosity. To print what ZFS/SSH operation exactly is happening (or would happen), add the `-v -v` " 

1020 "flag, maybe along with --dryrun. All ZFS and SSH commands (even with --dryrun) are logged such that " 

1021 "they can be inspected, copy-and-pasted into a terminal shell and run manually to help anticipate or " 

1022 "diagnose issues. ERROR, WARN, INFO, DEBUG, TRACE output lines are identified by [E], [W], [I], [D], [T] " 

1023 "prefixes, respectively.\n\n") 

1024 parser.add_argument( 

1025 "--quiet", "-q", action="store_true", 

1026 help="Suppress non-error, info, debug, and trace output.\n\n") 

1027 parser.add_argument( 

1028 "--no-privilege-elevation", "-p", action="store_true", 

1029 help="Do not attempt to run state changing ZFS operations 'zfs create/rollback/destroy/send/receive/snapshot' as " 

1030 "root (via 'sudo -u root' elevation granted by administrators appending the following to /etc/sudoers: " 

1031 "`<NON_ROOT_USER_NAME> ALL=NOPASSWD:/path/to/zfs`\n\n" 

1032 "Instead, the --no-privilege-elevation flag is for non-root users that have been granted corresponding "