Coverage for bzfs_main/argparse_cli.py: 100%

1# Copyright 2024 Wolfgang Hoschek AT mac DOT com 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# 

15"""Documentation, definition of input data and ArgumentParser used by the 'bzfs' CLI.""" 

16 

17from __future__ import ( 

18 annotations, 

19) 

20import argparse 

21import dataclasses 

22import itertools 

23from typing import ( 

24 Final, 

25) 

26 

27from bzfs_main.argparse_actions import ( 

28 CheckPercentRange, 

29 DatasetPairsAction, 

30 DeleteDstSnapshotsExceptPlanAction, 

31 FileOrLiteralAction, 

32 IncludeSnapshotPlanAction, 

33 NewSnapshotFilterGroupAction, 

34 NonEmptyStringAction, 

35 SafeDirectoryNameAction, 

36 SafeFileNameAction, 

37 SSHConfigFileNameAction, 

38 TimeRangeAndRankRangeAction, 

39) 

40from bzfs_main.detect import ( 

41 DISABLE_PRG, 

42 DUMMY_DATASET, 

43) 

44from bzfs_main.period_anchors import ( 

45 PeriodAnchors, 

46) 

47from bzfs_main.util.check_range import ( 

48 CheckRange, 

49) 

50from bzfs_main.util.utils import ( 

51 ENV_VAR_PREFIX, 

52 PROG_NAME, 

53 format_dict, 

54) 

55 

56# constants: 

57__version__: Final[str] = "1.26.0.dev0" 

58PROG_AUTHOR: Final[str] = "Wolfgang Hoschek" 

59EXCLUDE_DATASET_REGEXES_DEFAULT: Final[str] = r"(?:.*/)?[Tt][Ee]?[Mm][Pp][-_]?[0-9]*" # skip tmp datasets by default 

60LOG_DIR_DEFAULT: Final[str] = PROG_NAME + "-logs" 

61SKIP_ON_ERROR_DEFAULT: Final[str] = "dataset" 

62CMP_CHOICES_ITEMS: Final[tuple[str, str, str]] = ("src", "dst", "all") 

63WRITTEN_THRESHOLD: Final[str] = "written_threshold" 

64ZFS_RECV_O: Final[str] = "zfs_recv_o" 

65ZFS_RECV_X: Final[str] = "zfs_recv_x" 

66ZFS_RECV_GROUPS: Final[dict[str, str]] = {ZFS_RECV_O: "-o", ZFS_RECV_X: "-x", "zfs_set": ""} 

67ZFS_RECV_O_INCLUDE_REGEX_DEFAULT: Final[str] = "|".join( 

68 [ 

69 "aclinherit", 

70 "aclmode", 

71 "acltype", 

72 "atime", 

73 "checksum", 

74 "compression", 

75 "copies", 

76 "logbias", 

77 "primarycache", 

78 "recordsize", 

79 "redundant_metadata", 

80 "relatime", 

81 "secondarycache", 

82 "snapdir", 

83 "sync", 

84 "xattr", 

85 ] 

86) 

87 

88 

89def argument_parser() -> argparse.ArgumentParser: 

90 """Returns the CLI parser used by bzfs.""" 

91 create_src_snapshots_plan_example1: str = str({"test": {"": {"adhoc": 1}}}).replace(" ", "") 

92 create_src_snapshots_plan_example2: str = str({"prod": {"us-west": {"minutely": 40, "hourly": 36, "daily": 31}}}) 

93 create_src_snapshots_plan_example2 = create_src_snapshots_plan_example2.replace(" ", "") 

94 delete_dst_snapshots_except_plan_example1: str = str( 

95 { 

96 "prod": { 

97 "us-west": { 

98 "secondly": 40, 

99 "minutely": 40, 

100 "hourly": 36, 

101 "daily": 31, 

102 "weekly": 12, 

103 "monthly": 18, 

104 "yearly": 5, 

105 } 

106 } 

107 } 

108 ).replace(" ", "") 

109 monitor_snapshots_example: str = str( 

110 { 

111 "prod": { 

112 "us-west": { 

113 "minutely": {"latest": {"warning": "30 seconds", "critical": "300 seconds"}}, 

114 "hourly": {"latest": {"warning": "30 minutes", "critical": "300 minutes"}}, 

115 "daily": {"latest": {"warning": "4 hours", "critical": "8 hours"}}, 

116 }, 

117 }, 

118 } 

119 ).replace(" ", "") 

120 

121 # fmt: off 

122 parser: argparse.ArgumentParser = argparse.ArgumentParser( 

123 prog=PROG_NAME, 

124 allow_abbrev=False, 

125 formatter_class=argparse.RawTextHelpFormatter, 

126 description=rf""" 

127On the first run, {PROG_NAME} replicates the source dataset and all its snapshots to the destination. 

128On subsequent runs, it sends only changes since the previous run by incrementally replicating snapshots 

129created on the source after that run. Source snapshots older than the most recent common snapshot 

130on the destination are skipped automatically. 

131 

132Unless {PROG_NAME} is told to create snapshots or bookmarks on the source, it treats the source as read-only. With 

133`--dryrun`, it also treats the destination as read-only. In normal operation, the destination is 

134append-only. Optional flags can delete destination snapshots and datasets if you want to manage storage 

135space consumption, reconcile divergence, restore production from backup, or resync backup from production. 

136 

137{PROG_NAME} supports include/exclude filters that can be combined to choose which datasets, 

138snapshots, and properties to create, replicate, delete, or compare. 

139 

140A common setup uses scheduled `cron` jobs: one to create and prune source snapshots, one to prune 

141destination snapshots, and one to replicate recently created snapshots from source to destination. 

142Schedules can run every N milliseconds, seconds, minutes, hours, days, weeks, months, or years. 

143 

144Snapshot creation, replication, pruning, monitoring, and comparison work with snapshots in any naming 

145format, including snapshots created by third-party tools or by manual zfs snapshot commands. 

146These functions can also be used independently. 

147 

148The source can push to the destination, and the destination can pull from the source. {PROG_NAME} 

149runs on the initiator host, which can be the source host (push mode), destination host (pull mode), 

150same host (local mode, no network, no ssh), or a third-party host that can SSH 

151into source and destination (pull-push mode). In pull-push mode, the source `zfs send` stream is 

152relayed by the initiator to the destination `zfs receive`, without storing anything locally. 

153For bulk data transfers, remote-to-remote mode (`--r2r=pull` or `--r2r=push`) can instead transfer the 

154stream directly between source and destination to avoid making the initiator a bandwidth bottleneck. In 

155this mode, {PROG_NAME} does not need to be installed on source or destination; only the `zfs` CLI is 

156required there. {PROG_NAME} can run as root or as a non-root user via sudo or delegated `zfs allow` 

157permissions. 

158 

159{PROG_NAME} is written in Python and continuously tested with unit and integration tests on old and 

160new ZFS versions, on multiple Linux and FreeBSD versions, and on all Python versions >= 3.9 (including 

161latest stable, currently python-3.15). 

162 

163{PROG_NAME} is a stand-alone program with zero required dependencies. It is meant to run in restricted 

164barebones server environments. No external Python packages are required; indeed no Python package 

165management at all is required. You can symlink the program wherever you like, such as /usr/local/bin, 

166and run it like a shell script or binary executable. 

167 

168{PROG_NAME} replicates snapshots for multiple datasets in parallel. It also deletes (or monitors or 

169compares) snapshots of multiple datasets in parallel. Atomic snapshots can be created as often as 

170every N milliseconds. 

171 

172Replication progress (e.g. throughput and ETA) is shown aggregated across parallel replication tasks. 

173Example console status line: 

174 

175`2025-01-17 01:23:04 [I] zfs sent 41.7 GiB 0:00:46 [963 MiB/s] [907 MiB/s] 80% ETA 0:00:04 ETA 01:23:08` 

176 

177{PROG_NAME} uses streaming algorithms to process millions of datasets with low memory usage and low latency. 

178It handles replication policies with multiple sources and multiple destinations per source. 

179 

180Optionally, {PROG_NAME} applies bandwidth rate limiting and progress monitoring (via `pv`) during 

181`zfs send/receive` transfers. Over the network, it can insert lightweight compression (via `zstd`) 

182and buffering (via `mbuffer`) between endpoints. If one of these tools is not installed, {PROG_NAME} 

183auto-detects that and continues without that auxiliary feature. 

184 

185# Periodic Jobs with bzfs_jobrunner 

186 

187The project also ships with [bzfs_jobrunner](README_bzfs_jobrunner.md), a companion program that wraps 

188`{PROG_NAME}` for periodic snapshot creation, replication, pruning, and monitoring across N source hosts 

189and M destination hosts, using one shared fleet-wide [jobconfig](bzfs_testbed/bzfs_job_testbed.py) 

190script. 

191 

192Typical use cases include geo-replicated backup where each destination host is in a different region 

193and receives replicas from the same set of source hosts, low-latency replication from a primary to a 

194secondary or to M read replicas, and backups to removable drives. 

195 

196# Quickstart 

197 

198* Create adhoc atomic snapshots without a schedule: 

199 

200``` 

201$ {PROG_NAME} tank1/foo/bar dummy --recursive --skip-replication --create-src-snapshots \ 

202--create-src-snapshots-plan "{create_src_snapshots_plan_example1}" 

203``` 

204 

205``` 

206$ zfs list -t snapshot tank1/foo/bar 

207tank1/foo/bar@test_2024-11-06_08:30:05_adhoc 

208``` 

209 

210* Create periodic atomic snapshots on a schedule, every minute, every hour and every day, by launching this from a periodic `cron` job: 

211 

212``` 

213$ {PROG_NAME} tank1/foo/bar dummy --recursive --skip-replication --create-src-snapshots \ 

214--create-src-snapshots-plan \ 

215"{create_src_snapshots_plan_example2}" 

216``` 

217 

218``` 

219$ zfs list -t snapshot tank1/foo/bar 

220tank1/foo/bar@prod_us-west_2024-11-06_08:30:05_daily 

221tank1/foo/bar@prod_us-west_2024-11-06_08:30:05_hourly 

222tank1/foo/bar@prod_us-west_2024-11-06_08:30:05_minutely 

223``` 

224 

225Note: A periodic snapshot is created if it is due per the schedule indicated by its suffix (e.g. `_daily` or `_hourly` 

226or `_minutely` or `_2secondly` or `_100millisecondly`), or if the --create-src-snapshots-even-if-not-due flag is specified, 

227or if the most recent scheduled snapshot is somehow missing. In the latter case {PROG_NAME} immediately creates a snapshot 

228(named with the current time, not backdated to the missed time), and then resumes the original schedule. If the suffix is 

229`_adhoc` or not a known period then a snapshot is considered non-periodic and is thus created immediately regardless of the 

230creation time of any existing snapshot. 

231 

232* Replication example in local mode (no network, no ssh), to replicate ZFS dataset tank1/foo/bar to tank2/boo/bar: 

233 

234``` 

235$ {PROG_NAME} tank1/foo/bar tank2/boo/bar 

236``` 

237 

238``` 

239$ zfs list -t snapshot tank1/foo/bar 

240tank1/foo/bar@prod_us-west_2024-11-06_08:30:05_daily 

241tank1/foo/bar@prod_us-west_2024-11-06_08:30:05_hourly 

242tank1/foo/bar@prod_us-west_2024-11-06_08:30:05_minutely 

243``` 

244 

245``` 

246$ zfs list -t snapshot tank2/boo/bar 

247tank2/boo/bar@prod_us-west_2024-11-06_08:30:05_daily 

248tank2/boo/bar@prod_us-west_2024-11-06_08:30:05_hourly 

249tank2/boo/bar@prod_us-west_2024-11-06_08:30:05_minutely 

250``` 

251 

252* Same example in pull mode: 

253 

254``` 

255$ {PROG_NAME} root@host1.example.com:tank1/foo/bar tank2/boo/bar 

256``` 

257 

258* Same example in push mode: 

259 

260``` 

261$ {PROG_NAME} tank1/foo/bar root@host2.example.com:tank2/boo/bar 

262``` 

263 

264* Same example in pull-push mode: 

265 

266``` 

267$ {PROG_NAME} root@host1:tank1/foo/bar root@host2:tank2/boo/bar 

268``` 

269 

270* Same example with direct remote-to-remote transfer via `--r2r=pull`: 

271 

272``` 

273$ {PROG_NAME} --r2r=pull root@host1:tank1/foo/bar root@host2:tank2/boo/bar 

274``` 

275 

276* Same example with direct remote-to-remote transfer via `--r2r=push`: 

277 

278``` 

279$ {PROG_NAME} --r2r=push root@host1:tank1/foo/bar root@host2:tank2/boo/bar 

280``` 

281 

282* Example in local mode (no network, no ssh) to recursively replicate ZFS dataset tank1/foo/bar and its descendant 

283datasets to tank2/boo/bar: 

284 

285``` 

286$ {PROG_NAME} --recursive tank1/foo/bar tank2/boo/bar 

287``` 

288 

289``` 

290$ zfs list -t snapshot -r tank1/foo/bar 

291tank1/foo/bar@prod_us-west_2024-11-06_08:30:05_daily 

292tank1/foo/bar@prod_us-west_2024-11-06_08:30:05_hourly 

293tank1/foo/bar@prod_us-west_2024-11-06_08:30:05_minutely 

294tank1/foo/bar/baz@prod_us-west_2024-11-06_08:40:00_daily 

295tank1/foo/bar/baz@prod_us-west_2024-11-06_08:40:00_hourly 

296tank1/foo/bar/baz@prod_us-west_2024-11-06_08:40:00_minutely 

297``` 

298 

299``` 

300$ zfs list -t snapshot -r tank2/boo/bar 

301tank2/boo/bar@prod_us-west_2024-11-06_08:30:05_daily 

302tank2/boo/bar@prod_us-west_2024-11-06_08:30:05_hourly 

303tank2/boo/bar@prod_us-west_2024-11-06_08:30:05_minutely 

304tank2/boo/bar/baz@prod_us-west_2024-11-06_08:40:00_daily 

305tank2/boo/bar/baz@prod_us-west_2024-11-06_08:40:00_hourly 

306tank2/boo/bar/baz@prod_us-west_2024-11-06_08:40:00_minutely 

307``` 

308 

309* Replicate all daily snapshots created during the last 31 days, and at the same time ensure that the latest 31 daily 

310snapshots (per dataset) are replicated regardless of creation time. Same for 40 minutely snapshots, and 36 hourly 

311snapshots: 

312 

313``` 

314$ {PROG_NAME} tank1/foo/bar tank2/boo/bar --recursive --include-snapshot-plan \ 

315"{create_src_snapshots_plan_example2}" 

316``` 

317 

318Note: The example above compares the specified times against the standard ZFS 'creation' time property of the snapshots 

319(which is a UTC Unix time in integer seconds), rather than against a timestamp that may be part of the snapshot name. 

320 

321* Retain all secondly snapshots that were created less than 40 seconds ago, and ensure that the latest 40 

322secondly snapshots (per dataset) are retained regardless of creation time. Same for 40 minutely snapshots, 36 hourly 

323snapshots, 31 daily snapshots, 12 weekly snapshots, 18 monthly snapshots, and 5 yearly snapshots: 

324 

325``` 

326$ {PROG_NAME} {DUMMY_DATASET} tank2/boo/bar --dryrun --recursive --skip-replication \ 

327--delete-dst-snapshots --delete-dst-snapshots-except-plan \ 

328"{delete_dst_snapshots_except_plan_example1}" 

329``` 

330 

331Note: This also prints how many GB of disk space in total would be freed if the command were to be run for real without 

332the --dryrun flag. 

333 

334* Compare source and destination dataset trees recursively, for example to check if all recently taken snapshots have 

335been successfully replicated by a periodic job. List snapshots only contained in src (tagged with 'src'), 

336only contained in dst (tagged with 'dst'), and contained in both src and dst (tagged with 'all'), restricted to hourly 

337and daily snapshots taken within the last 7 days, excluding the last 4 hours (to allow for some slack/stragglers), 

338excluding temporary datasets: 

339 

340``` 

341$ {PROG_NAME} tank1/foo/bar tank2/boo/bar --skip-replication --compare-snapshot-lists \ 

342--recursive --include-snapshot-regex '.*_(hourly|daily)' \ 

343--include-snapshot-times-and-ranks '7 days ago..4 hours ago' --exclude-dataset-regex \ 

344'(.*/)?tmp.*' 

345``` 

346 

347If the resulting TSV output file contains zero lines starting with the prefix 'src' and zero lines starting with the 

348prefix 'dst' then no source snapshots are missing on the destination, and no destination snapshots are missing 

349on the source, indicating that the periodic replication and pruning jobs perform as expected. The TSV output is sorted 

350by dataset, and by ZFS creation time within each dataset - the first and last line prefixed with 'all' contains the 

351metadata of the oldest and latest common snapshot, respectively. The --compare-snapshot-lists option also directly 

352logs [various summary stats](https://github.com/whoschek/bzfs/blob/main/bzfs_docs/compare-snapshot-lists-example.log), 

353such as the metadata of the latest common snapshot, latest snapshots and oldest snapshots, as well as the time diff 

354between the latest common snapshot and latest snapshot only in src (and only in dst), as well as how many src snapshots 

355and how many GB of data are missing on dst, etc. 

356 

357* Alert the user if the ZFS 'creation' time property of the latest source or destination snapshot for any specified 

358snapshot name pattern within the selected datasets is too old wrt. the specified age limit. The purpose is to check if 

359snapshots are successfully created and replicated on schedule. 

360Process exit code is 0, 1, 2 on OK, WARNING, CRITICAL, respectively. 

361The example alerts the user if the *latest* source or destination snapshot named `prod_us-west_<timestamp>_hourly` is 

362more than 30 minutes late (i.e. more than 30+60=90 minutes old) [warning], or more than 300 minutes late (i.e. more 

363than 300+60=360 minutes old) [critical]. Analog for minutely and daily snapshots: 

364 

365``` 

366$ {PROG_NAME} tank1/foo/bar tank2/boo/bar --recursive --skip-replication -v --monitor-snapshots \ 

367"{monitor_snapshots_example}" 

368``` 

369 

370* Example that makes destination identical to source even if the two have drastically diverged: 

371 

372``` 

373$ {PROG_NAME} tank1/foo/bar tank2/boo/bar --dryrun --recursive --force --delete-dst-datasets \ 

374--delete-dst-snapshots 

375``` 

376 

377""") 

378 

379 parser.add_argument( 

380 "--no-argument-file", action="store_true", 

381 # help="Disable support for reading the names of datasets and snapshots from a file.\n\n") 

382 help=argparse.SUPPRESS) 

383 parser.add_argument( 

384 "root_dataset_pairs", nargs="+", action=DatasetPairsAction, metavar="SRC_DATASET DST_DATASET", 

385 help="SRC_DATASET: " 

386 "Source ZFS dataset (and its descendants) that will be replicated. Can be a ZFS filesystem or ZFS volume. " 

387 "Format is [[user@]host:]dataset. The host name can also be an IPv4 address (or an IPv6 address where " 

388 "each ':' colon character must be replaced with a '|' pipe character for disambiguation). If the " 

389 "host name is '-', the dataset will be on the local host, and the corresponding SSH leg will be omitted. " 

390 "The same is true if the host is omitted and the dataset does not contain a ':' colon at the same time. " 

391 "Local dataset examples: `tank1/foo/bar`, `tank1`, `-:tank1/foo/bar:baz:boo` " 

392 "Remote dataset examples: `host:tank1/foo/bar`, `host.example.com:tank1/foo/bar`, " 

393 "`root@host:tank`, `root@host.example.com:tank1/foo/bar`, `user@127.0.0.1:tank1/foo/bar:baz:boo`, " 

394 "`user@||1:tank1/foo/bar:baz:boo`. " 

395 "The first component of the ZFS dataset name is the ZFS pool name, here `tank1`. " 

396 "If the option starts with a `+` prefix then dataset names are read from the UTF-8 text file given " 

397 "after the `+` prefix, with each line in the file containing a SRC_DATASET and a DST_DATASET, " 

398 "separated by a tab character. The basename must contain the substring 'bzfs_argument_file'. " 

399 "Example: `+root_dataset_names_bzfs_argument_file.txt`, " 

400 "`+/path/to/root_dataset_names_bzfs_argument_file.txt`\n\n" 

401 "DST_DATASET: " 

402 "Destination ZFS dataset for replication and deletion. Has same naming format as SRC_DATASET. During " 

403 "replication, destination datasets that do not yet exist are created as necessary, along with their " 

404 "parent and ancestors.\n\n" 

405 f"*Performance Note:* {PROG_NAME} automatically replicates multiple datasets in parallel. It replicates " 

406 "snapshots in parallel across datasets and serially within a dataset. All child datasets of a dataset " 

407 "may be processed in parallel. For consistency, processing of a dataset only starts after processing of " 

408 "all its ancestor datasets has completed. Further, when a thread is ready to start processing another " 

409 "dataset, it chooses the next dataset wrt. lexicographical sort order from the datasets that are " 

410 "currently available for start of processing. Initially, only the roots of the selected dataset subtrees " 

411 "are available for start of processing. The degree of parallelism is configurable with the --threads " 

412 "option (see below).\n\n") 

413 parser.add_argument( 

414 "--recursive", "-r", action="store_true", 

415 help="During snapshot creation, replication, deletion and comparison, also consider descendant datasets, i.e. " 

416 "datasets within the dataset tree, including children, and children of children, etc.\n\n") 

417 parser.add_argument( 

418 "--include-dataset", action=FileOrLiteralAction, nargs="+", default=[], metavar="DATASET", 

419 help="During snapshot creation, replication, deletion and comparison, select any ZFS dataset (and its descendants) " 

420 "that is contained within SRC_DATASET (DST_DATASET in case of deletion) if its dataset name is one of the " 

421 "given include dataset names but none of the exclude dataset names. If a dataset is excluded its descendants " 

422 "are automatically excluded too, and this decision is never reconsidered even for the descendants because " 

423 "exclude takes precedence over include.\n\n" 

424 "A dataset name is absolute if the specified dataset is prefixed by `/`, e.g. `/tank/baz/tmp`. " 

425 "Otherwise the dataset name is relative wrt. source and destination, e.g. `baz/tmp` if the source " 

426 "is `tank`.\n\n" 

427 "This option is automatically translated to an --include-dataset-regex (see below) and can be " 

428 "specified multiple times.\n\n" 

429 "If the option starts with a `+` prefix then dataset names are read from the newline-separated " 

430 "UTF-8 text file given after the `+` prefix, one dataset per line inside of the text file. The basename " 

431 "must contain the substring 'bzfs_argument_file'.\n\n" 

432 "Examples: `/tank/baz/tmp` (absolute), `baz/tmp` (relative), " 

433 "`+dataset_names_bzfs_argument_file.txt`, `+/path/to/dataset_names_bzfs_argument_file.txt`\n\n") 

434 parser.add_argument( 

435 "--exclude-dataset", action=FileOrLiteralAction, nargs="+", default=[], metavar="DATASET", 

436 help="Same syntax as --include-dataset (see above) except that the option is automatically translated to an " 

437 "--exclude-dataset-regex (see below).\n\n") 

438 parser.add_argument( 

439 "--include-dataset-regex", action=FileOrLiteralAction, nargs="+", default=[], metavar="REGEX", 

440 help="During snapshot creation, replication (and deletion) and comparison, select any ZFS dataset (and its " 

441 "descendants) that is contained within SRC_DATASET (DST_DATASET in case of deletion) if its relative dataset " 

442 "path (e.g. `baz/tmp`) wrt. SRC_DATASET (DST_DATASET in case of deletion) matches at least one of the given " 

443 "include regular expressions but none of the exclude regular expressions. " 

444 "If a dataset is excluded its descendants are automatically excluded too, and this decision is never " 

445 "reconsidered even for the descendants because exclude takes precedence over include.\n\n" 

446 "This option can be specified multiple times. " 

447 "A leading `!` character indicates logical negation, i.e. the regex matches if the regex with the " 

448 "leading `!` character removed does not match.\n\n" 

449 "If the option starts with a `+` prefix then regex names are read from the newline-separated " 

450 "UTF-8 text file given after the `+` prefix, one regex per line inside of the text file. The basename " 

451 "must contain the substring 'bzfs_argument_file'.\n\n" 

452 "Default: `.*` (include all datasets).\n\n" 

453 "Examples: `baz/tmp`, `(.*/)?doc[^/]*/(private|confidential).*`, `!public`, " 

454 "`+dataset_regexes_bzfs_argument_file.txt`, `+/path/to/dataset_regexes_bzfs_argument_file.txt`\n\n") 

455 parser.add_argument( 

456 "--exclude-dataset-regex", action=FileOrLiteralAction, nargs="+", default=[], metavar="REGEX", 

457 help="Same syntax as --include-dataset-regex (see above) except that the default is " 

458 f"`{EXCLUDE_DATASET_REGEXES_DEFAULT}` (exclude tmp datasets). Example: `!.*` (exclude no dataset)\n\n") 

459 parser.add_argument( 

460 "--exclude-dataset-property", default=None, action=NonEmptyStringAction, metavar="STRING", 

461 help="The name of a ZFS dataset user property (optional). If this option is specified, the effective value " 

462 "(potentially inherited) of that user property is read via 'zfs list' for each selected source dataset " 

463 "to determine whether the dataset will be included or excluded, as follows:\n\n" 

464 "a) Value is 'true' or '-' or empty string or the property is missing: Include the dataset.\n\n" 

465 "b) Value is 'false': Exclude the dataset and its descendants.\n\n" 

466 "c) Value is a comma-separated list of host names (no spaces, for example: " 

467 "'store001,store002'): Include the dataset if the host name of " 

468 f"the host executing {PROG_NAME} is contained in the list, otherwise exclude the dataset and its " 

469 "descendants.\n\n" 

470 "If a dataset is excluded its descendants are automatically excluded too, and the property values of the " 

471 "descendants are ignored because exclude takes precedence over include.\n\n" 

472 "Examples: 'syncoid:sync', 'com.example.eng.project.x:backup'\n\n" 

473 "*Note:* The use of --exclude-dataset-property is discouraged for most use cases. It is more flexible, " 

474 "more powerful, *and* more efficient to instead use a combination of --include/exclude-dataset-regex " 

475 "and/or --include/exclude-dataset to achieve the same or better outcome.\n\n") 

476 parser.add_argument( 

477 "--include-snapshot-regex", action=FileOrLiteralAction, nargs="+", default=[], metavar="REGEX", 

478 help="During replication, deletion and comparison, select any source ZFS snapshot that has a name (i.e. the part " 

479 "after the '@') that matches at least one of the given include regular expressions but none of the " 

480 "exclude regular expressions. If a snapshot is excluded this decision is never reconsidered because " 

481 "exclude takes precedence over include.\n\n" 

482 "This option can be specified multiple times. " 

483 "A leading `!` character indicates logical negation, i.e. the regex matches if the regex with the " 

484 "leading `!` character removed does not match.\n\n" 

485 "Default: `.*` (include all snapshots). " 

486 "Examples: `test_.*`, `!prod_.*`, `.*_(hourly|frequent)`, `!.*_(weekly|daily)`\n\n" 

487 "*Note:* All --include/exclude-snapshot-* CLI option groups are combined into a mini filter pipeline. " 

488 "A filter pipeline is executed in the order given on the command line, left to right. For example if " 

489 "--include-snapshot-times-and-ranks (see below) is specified on the command line before " 

490 "--include/exclude-snapshot-regex, then --include-snapshot-times-and-ranks will be applied before " 

491 "--include/exclude-snapshot-regex. The pipeline results would not always be the same if the order were " 

492 "reversed. Order matters.\n\n" 

493 "*Note:* During replication, bookmarks are always retained aka selected in order to help find common " 

494 "snapshots between source and destination.\n\n") 

495 parser.add_argument( 

496 "--exclude-snapshot-regex", action=FileOrLiteralAction, nargs="+", default=[], metavar="REGEX", 

497 help="Same syntax as --include-snapshot-regex (see above) except that the default is to exclude no " 

498 "snapshots.\n\n") 

499 parser.add_argument( 

500 "--include-snapshot-times-and-ranks", action=TimeRangeAndRankRangeAction, nargs="+", default=[], 

501 metavar=("TIMERANGE", "RANKRANGE"), 

502 help="This option takes as input parameters a time range filter and an optional rank range filter. It " 

503 "separately computes the results for each filter and selects the UNION of both results. " 

504 "To instead use a pure rank range filter (no UNION), or a pure time range filter (no UNION), simply " 

505 "use 'notime' aka '0..0' to indicate an empty time range, or omit the rank range, respectively. " 

506 "This option can be specified multiple times.\n\n" 

507 "<b>*Replication Example (UNION):* </b>\n\n" 

508 "Specify to replicate all daily snapshots created during the last 7 days, " 

509 "and at the same time ensure that the latest 7 daily snapshots (per dataset) are replicated regardless " 

510 "of creation time, like so: " 

511 "`--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks '7 days ago..anytime' 'latest 7'`\n\n" 

512 "<b>*Deletion Example (no UNION):* </b>\n\n" 

513 "Specify to delete all daily snapshots older than 7 days, but ensure that the " 

514 "latest 7 daily snapshots (per dataset) are retained regardless of creation time, like so: " 

515 "`--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks notime 'all except latest 7' " 

516 "--include-snapshot-times-and-ranks 'anytime..7 days ago'`" 

517 "\n\n" 

518 "This helps to safely cope with irregular scenarios where no snapshots were created or received within " 

519 "the last 7 days, or where more than 7 daily snapshots were created within the last 7 days. It can also " 

520 "help to avoid accidental pruning of the last snapshot that source and destination have in common.\n\n" 

521 "" 

522 "<b>*TIMERANGE:* </b>\n\n" 

523 "The ZFS 'creation' time of a snapshot (and bookmark) must fall into this time range in order for the " 

524 "snapshot to be included. The time range consists of a 'start' time, followed by a '..' separator, " 

525 "followed by an 'end' time. For example '2024-01-01..2024-04-01', or 'anytime..anytime' aka `*..*` aka all " 

526 "times, or 'notime' aka '0..0' aka empty time range. Only snapshots (and bookmarks) in the half-open time " 

527 "range [start, end) are included; other snapshots (and bookmarks) are excluded. If a snapshot is excluded " 

528 "this decision is never reconsidered because exclude takes precedence over include. Each of the two specified " 

529 "times can take any of the following forms:\n\n" 

530 "* a) `anytime` aka `*` wildcard; represents negative or positive infinity.\n\n" 

531 "* b) a non-negative integer representing a UTC Unix time in seconds. Example: 1728109805\n\n" 

532 "* c) an ISO 8601 datetime string with or without timezone. Examples: '2024-10-05', " 

533 "'2024-10-05T14:48:55', '2024-10-05T14:48:55+02', '2024-10-05T14:48:55-04:30'. If the datetime string " 

534 "does not contain time zone info then it is assumed to be in the local time zone. Timezone string support " 

535 "requires Python ≥ 3.11.\n\n" 

536 "* d) a duration that indicates how long ago from the current time, using the following syntax: " 

537 "a non-negative integer, followed by an optional space, followed by a duration unit that is " 

538 "*one* of 'seconds', 'secs', 'minutes', 'mins', 'hours', 'days', 'weeks', 'months', 'years', " 

539 "followed by an optional space, followed by the word 'ago'. " 

540 "Examples: '0secs ago', '40 mins ago', '36hours ago', '90days ago', '12weeksago'.\n\n" 

541 "* Note: This option compares the specified time against the standard ZFS 'creation' time property of the " 

542 "snapshot (which is a UTC Unix time in integer seconds), rather than against a timestamp that may be " 

543 "part of the snapshot name. You can list the ZFS creation time of snapshots and bookmarks as follows: " 

544 "`zfs list -t snapshot,bookmark -o name,creation -s creation -d 1 $SRC_DATASET` (optionally add " 

545 "the -p flag to display UTC Unix time in integer seconds).\n\n" 

546 "*Note:* During replication, bookmarks are always retained aka selected in order to help find common " 

547 "snapshots between source and destination.\n\n" 

548 "" 

549 "<b>*RANKRANGE:* </b>\n\n" 

550 "Specifies to include the N (or N%%) oldest snapshots or latest snapshots, and exclude all other " 

551 "snapshots (default: include no snapshots). Snapshots are sorted by creation time (actually, by the " 

552 "'createtxg' ZFS property, which serves the same purpose but is more precise). The rank position of a " 

553 "snapshot is the zero-based integer position of the snapshot within that sorted list. A rank consists of the " 

554 "optional words 'all except' (followed by an optional space), followed by the word 'oldest' or 'latest', " 

555 "followed by a non-negative integer, followed by an optional '%%' percent sign. A rank range consists of a " 

556 "lower rank, followed by a '..' separator, followed by a higher rank. " 

557 "If the optional lower rank is missing it is assumed to be 0. Examples:\n\n" 

558 "* 'oldest 10%%' aka 'oldest 0..oldest 10%%' (include the oldest 10%% of all snapshots)\n\n" 

559 "* 'latest 10%%' aka 'latest 0..latest 10%%' (include the latest 10%% of all snapshots)\n\n" 

560 "* 'all except latest 10%%' aka 'oldest 90%%' aka 'oldest 0..oldest 90%%' (include all snapshots except the " 

561 "latest 10%% of all snapshots)\n\n" 

562 "* 'oldest 90' aka 'oldest 0..oldest 90' (include the oldest 90 snapshots)\n\n" 

563 "* 'latest 90' aka 'latest 0..latest 90' (include the latest 90 snapshots)\n\n" 

564 "* 'all except oldest 90' aka 'oldest 90..oldest 100%%' (include all snapshots except the oldest 90 snapshots)" 

565 "\n\n" 

566 "* 'all except latest 90' aka 'latest 90..latest 100%%' (include all snapshots except the latest 90 snapshots)" 

567 "\n\n" 

568 "* 'latest 1' aka 'latest 0..latest 1' (include the latest snapshot)\n\n" 

569 "* 'all except latest 1' aka 'latest 1..latest 100%%' (include all snapshots except the latest snapshot)\n\n" 

570 "* 'oldest 2' aka 'oldest 0..oldest 2' (include the oldest 2 snapshots)\n\n" 

571 "* 'all except oldest 2' aka 'oldest 2..oldest 100%%' (include all snapshots except the oldest 2 snapshots)\n\n" 

572 "* 'oldest 100%%' aka 'oldest 0..oldest 100%%' (include all snapshots)\n\n" 

573 "* 'oldest 0%%' aka 'oldest 0..oldest 0%%' (include no snapshots)\n\n" 

574 "* 'oldest 0' aka 'oldest 0..oldest 0' (include no snapshots)\n\n" 

575 "*Note for multiple RANKRANGEs:* `--include-snapshot-times-and-ranks TIMERANGE RANKRANGE1 RANKRANGE2` is " 

576 "equivalent to `--include-snapshot-times-and-ranks TIMERANGE RANKRANGE1 --include-snapshot-times-and-ranks " 

577 "TIMERANGE RANKRANGE2`.\n\n" 

578 "*Note:* Percentage calculations are not based on the number of snapshots " 

579 "contained in the dataset on disk, but rather based on the number of snapshots arriving at the filter. " 

580 "For example, if only two daily snapshots arrive at the filter because a prior filter excludes hourly " 

581 "snapshots, then 'latest 10' will only include these two daily snapshots, and 'latest 50%%' will only " 

582 "include one of these two daily snapshots.\n\n" 

583 "*Note:* During replication, bookmarks are always retained aka selected in order to help find common " 

584 "snapshots between source and destination. Bookmarks do not count towards N or N%% wrt. rank.\n\n" 

585 "*Note:* If a snapshot is excluded this decision is never reconsidered because exclude takes precedence " 

586 "over include.\n\n") 

587 

588 src_snapshot_plan_example = { 

589 "prod": { 

590 "onsite": {"secondly": 40, "minutely": 40, "hourly": 36, "daily": 31, "weekly": 12, "monthly": 18, "yearly": 5}, 

591 "us-west": {"secondly": 0, "minutely": 0, "hourly": 36, "daily": 31, "weekly": 12, "monthly": 18, "yearly": 5}, 

592 "eu-west": {"secondly": 0, "minutely": 0, "hourly": 36, "daily": 31, "weekly": 12, "monthly": 18, "yearly": 5}, 

593 }, 

594 "test": { 

595 "offsite": {"12hourly": 42, "weekly": 12}, 

596 "onsite": {"100millisecondly": 42}, 

597 }, 

598 } 

599 src_snapshot_plan_attributes_example = { 

600 "prod": { 

601 "onsite": {"secondly": {WRITTEN_THRESHOLD: 1}}, 

602 }, 

603 } 

604 parser.add_argument( 

605 "--include-snapshot-plan", action=IncludeSnapshotPlanAction, default=None, metavar="DICT_STRING", 

606 help="Replication periods to be used if replicating snapshots within the selected destination datasets. " 

607 "Has the same format as --create-src-snapshots-plan and --delete-dst-snapshots-except-plan (see below). " 

608 "Snapshots that do not match a period will not be replicated. To avoid unexpected surprises, make sure to " 

609 "carefully specify ALL snapshot names and periods that shall be replicated, in combination with --dryrun.\n\n" 

610 f"Example: `{format_dict(src_snapshot_plan_example)}`. This example will, for the organization 'prod' and the " 

611 "intended logical target 'onsite', replicate secondly snapshots that were created less than 40 seconds ago, " 

612 "yet replicate the latest 40 secondly snapshots regardless of creation time. Analog for the latest 40 minutely " 

613 "snapshots, latest 36 hourly snapshots, etc. " 

614 "Note: A zero within a period (e.g. 'hourly': 0) indicates that no snapshots shall be replicated for the given " 

615 "period.\n\n" 

616 "Note: --include-snapshot-plan is a convenience option that auto-generates a series of the following other " 

617 "options: --new-snapshot-filter-group, --include-snapshot-regex, --include-snapshot-times-and-ranks\n\n") 

618 parser.add_argument( 

619 "--new-snapshot-filter-group", action=NewSnapshotFilterGroupAction, nargs=0, 

620 help="Starts a new snapshot filter group containing separate --{include|exclude}-snapshot-* filter options. The " 

621 "program separately computes the results for each filter group and selects the UNION of all results. " 

622 "This option can be specified multiple times and serves as a separator between groups. Example:\n\n" 

623 "Delete all minutely snapshots older than 40 minutes, but ensure that the latest 40 minutely snapshots (per " 

624 "dataset) are retained regardless of creation time. Additionally, delete all hourly snapshots older than 36 " 

625 "hours, but ensure that the latest 36 hourly snapshots (per dataset) are retained regardless of creation time. " 

626 "Additionally, delete all daily snapshots older than 31 days, but ensure that the latest 31 daily snapshots " 

627 "(per dataset) are retained regardless of creation time: " 

628 f"`{PROG_NAME} {DUMMY_DATASET} tank2/boo/bar --dryrun --recursive --skip-replication --delete-dst-snapshots " 

629 "--include-snapshot-regex '.*_minutely' --include-snapshot-times-and-ranks notime 'all except latest 40' " 

630 "--include-snapshot-times-and-ranks 'anytime..40 minutes ago' " 

631 "--new-snapshot-filter-group " 

632 "--include-snapshot-regex '.*_hourly' --include-snapshot-times-and-ranks notime 'all except latest 36' " 

633 "--include-snapshot-times-and-ranks 'anytime..36 hours ago' " 

634 "--new-snapshot-filter-group " 

635 "--include-snapshot-regex '.*_daily' --include-snapshot-times-and-ranks notime 'all except latest 31' " 

636 "--include-snapshot-times-and-ranks 'anytime..31 days ago'`\n\n") 

637 parser.add_argument( 

638 "--create-src-snapshots", action="store_true", 

639 help="Do nothing if the --create-src-snapshots flag is missing. Otherwise, before the replication step (see below), " 

640 "atomically create new snapshots of the source datasets selected via --{include|exclude}-dataset* policy. " 

641 "The names of the snapshots can be configured via --create-src-snapshots-* suboptions (see below). " 

642 "To create snapshots only, without any other processing such as replication, etc, consider using this flag " 

643 "together with the --skip-replication flag.\n\n" 

644 "A periodic snapshot is created if it is due per the schedule indicated by --create-src-snapshots-plan " 

645 "(for example '_daily' or '_hourly' or _'10minutely' or '_2secondly' or '_100millisecondly'), or if the " 

646 "--create-src-snapshots-even-if-not-due flag is specified, or if the most recent scheduled snapshot " 

647 f"is somehow missing. In the latter case {PROG_NAME} immediately creates a snapshot (tagged with the current " 

648 "time, not backdated to the missed time), and then resumes the original schedule.\n\n" 

649 "If the snapshot suffix is '_adhoc' or not a known period then a snapshot is considered " 

650 "non-periodic and is thus created immediately regardless of the creation time of any existing snapshot.\n\n" 

651 "The implementation attempts to fit as many datasets as possible into a single (atomic) 'zfs snapshot' command " 

652 "line, using lexicographical sort order, and using 'zfs snapshot -r' to the extent that this is compatible " 

653 "with the actual results of the schedule and the actual results of the --{include|exclude}-dataset* pruning " 

654 "policy. The snapshots of all datasets that fit " 

655 "within the same single 'zfs snapshot' CLI invocation will be taken within the same ZFS transaction group, and " 

656 "correspondingly have identical 'createtxg' ZFS property (but not necessarily identical 'creation' ZFS time " 

657 "property as ZFS actually provides no such guarantee), and thus be consistent. Dataset names that can't fit " 

658 "into a single command line are spread over multiple command line invocations, respecting the limits that the " 

659 "operating system places on the maximum length of a single command line, per `getconf ARG_MAX`.\n\n" 

660 f"Note: All {PROG_NAME} functions including snapshot creation, replication, deletion, monitoring, comparison, " 

661 "etc. happily work with any snapshots in any format, even created or managed by third party ZFS snapshot " 

662 "management tools, including manual zfs snapshot/destroy.\n\n") 

663 parser.add_argument( 

664 "--create-src-snapshots-plan", default=None, type=str, metavar="DICT_STRING", 

665 help="Creation periods that specify a schedule for when new snapshots shall be created on src within the selected " 

666 "datasets. Has the same format as --delete-dst-snapshots-except-plan.\n\n" 

667 f"Example: `{format_dict(src_snapshot_plan_example)}`. This example will, for the organization 'prod' and " 

668 "the intended logical target 'onsite', create 'secondly' snapshots every second, 'minutely' snapshots every " 

669 "minute, hourly snapshots every hour, and so on. " 

670 "It will also create snapshots for the targets 'us-west' and 'eu-west' within the 'prod' organization. " 

671 "In addition, it will create snapshots every 12 hours and every week for the 'test' organization, " 

672 "and name them as being intended for the 'offsite' replication target. Analog for snapshots that are taken " 

673 "every 100 milliseconds within the 'test' organization.\n\n" 

674 "The example creates ZFS snapshots with names like " 

675 "`prod_onsite_<timestamp>_secondly`, `prod_onsite_<timestamp>_minutely`, " 

676 "`prod_us-west_<timestamp>_hourly`, `prod_us-west_<timestamp>_daily`, " 

677 "`prod_eu-west_<timestamp>_hourly`, `prod_eu-west_<timestamp>_daily`, " 

678 "`test_offsite_<timestamp>_12hourly`, `test_offsite_<timestamp>_weekly`, and so on.\n\n" 

679 "Note: A period name that is missing indicates that no snapshots shall be created for the given period.\n\n" 

680 "The period name can contain an optional positive integer immediately preceding the time period unit, for " 

681 "example `_2secondly` or `_10minutely` or `_100millisecondly` to indicate that snapshots are taken every 2 " 

682 "seconds, or every 10 minutes, or every 100 milliseconds, respectively.\n\n") 

683 parser.add_argument( 

684 "--create-src-snapshots-plan-attributes", default=None, type=str, metavar="DICT_STRING", 

685 help=f"Default is `{format_dict({})}`. Optional additional parameters that specify how to create snapshots on a " 

686 "per-period basis. Has the same format as --create-src-snapshots-plan except nested values are dicts instead " 

687 "of integers.\n\n" 

688 f"Example: `{format_dict(src_snapshot_plan_attributes_example)}`. This example will, for the organization " 

689 "'prod' and the intended logical target 'onsite', create a 'secondly' snapshot every second if ZFS has written " 

690 "at least 1 byte to the dataset since its latest snapshot per the standard 'written' ZFS dataset property. " 

691 f"The default for each period is `{format_dict({WRITTEN_THRESHOLD: 0})}`, i.e. take a snapshot even if nothing " 

692 "has been written. See https://openzfs.github.io/openzfs-docs/man/master/7/zfsprops.7.html#written\n\n") 

693 

694 def argparser_escape(text: str) -> str: 

695 return text.replace("%", "%%") 

696 

697 parser.add_argument( 

698 "--create-src-snapshots-timeformat", default="%Y-%m-%d_%H:%M:%S", metavar="STRFTIME_SPEC", 

699 help="Default is `%(default)s`. For the strftime format, see " 

700 "https://docs.python.org/3.11/library/datetime.html#strftime-strptime-behavior. " 

701 f"Examples: `{argparser_escape('%Y-%m-%d_%H:%M:%S.%f')}` (adds microsecond resolution), " 

702 f"`{argparser_escape('%Y-%m-%d_%H:%M:%S%z')}` (adds timezone offset), " 

703 f"`{argparser_escape('%Y-%m-%dT%H-%M-%S')}` (no colons).\n\n" 

704 "The name of the snapshot created on the src is `$org_$target_strftime(--create-src-snapshots-time*)_$period`. " 

705 "Example: `tank/foo@prod_us-west_2024-09-03_12:26:15_daily`\n\n") 

706 parser.add_argument( 

707 "--create-src-snapshots-timezone", default="", type=str, metavar="TZ_SPEC", 

708 help=f"Default is the local timezone of the system running {PROG_NAME}. When creating a new snapshot on the source, " 

709 "fetch the current time in the specified timezone, and feed that time, and the value of " 

710 "--create-src-snapshots-timeformat, into the standard strftime() function to generate the timestamp portion " 

711 "of the snapshot name. The TZ_SPEC input parameter is of the form 'UTC' or '+HHMM' or '-HHMM' for fixed UTC " 

712 "offsets, or an IANA TZ identifier for auto-adjustment to daylight savings time, or the empty string to use " 

713 "the local timezone, for example '', 'UTC', '+0000', '+0530', '-0400', 'America/Los_Angeles', 'Europe/Vienna'. " 

714 "For a list of valid IANA TZ identifiers see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List" 

715 "\n\nTo change the timezone not only for snapshot name creation, but in all respects for the entire program, " 

716 "use the standard 'TZ' Unix environment variable, like so: `export TZ=UTC`.\n\n") 

717 parser.add_argument( 

718 "--create-src-snapshots-even-if-not-due", action="store_true", 

719 help="Take snapshots immediately regardless of the creation time of any existing snapshot, even if snapshots " 

720 "are periodic and not actually due per the schedule and nothing has been written.\n\n") 

721 parser.add_argument( 

722 "--zfs-send-program-opts", type=str, default="--raw --compressed", metavar="STRING", 

723 help="Parameters to fine-tune 'zfs send' behaviour (optional); will be passed into 'zfs send' CLI. " 

724 "The value is split on runs of one or more whitespace characters. " 

725 "Default is '%(default)s'. To run `zfs send` without options, specify the empty " 

726 "string: `--zfs-send-program-opts=''`. " 

727 "See https://openzfs.github.io/openzfs-docs/man/master/8/zfs-send.8.html " 

728 "and https://github.com/openzfs/zfs/issues/13024\n\n" 

729 "If these options request a different effective raw or non-raw mode from the one recorded in an existing ZFS " 

730 f"receive resume token, {PROG_NAME} aborts the token (clears the incomplete receive state) and then proceeds " 

731 "as usual. ZFS may be unable to change the mode for an existing destination dataset. In that case, restore the " 

732 "previous send options.\n\n") 

733 parser.add_argument( 

734 "--zfs-recv-program-opts", type=str, default="-u", metavar="STRING", 

735 help="Parameters to fine-tune 'zfs receive' behaviour (optional); will be passed into 'zfs receive' CLI. " 

736 "The value is split on runs of one or more whitespace characters. " 

737 "Default is '%(default)s'. To run `zfs receive` without options, specify the empty " 

738 "string: `--zfs-recv-program-opts=''`. " 

739 "Example: '-u -o canmount=noauto -o readonly=on -x keylocation -x keyformat -x encryption'. " 

740 "See https://openzfs.github.io/openzfs-docs/man/master/8/zfs-receive.8.html " 

741 "and https://openzfs.github.io/openzfs-docs/man/master/7/zfsprops.7.html\n\n") 

742 parser.add_argument( 

743 "--zfs-recv-program-opt", action="append", default=[], metavar="STRING", 

744 help="Parameter to fine-tune 'zfs receive' behaviour (optional); will be passed into 'zfs receive' CLI. " 

745 "The value can contain spaces and is not split. This option can be specified multiple times. Example: `" 

746 "--zfs-recv-program-opt=-o " 

747 "--zfs-recv-program-opt='org.zfsbootmenu:commandline=ro debug zswap.enabled=1'`\n\n") 

748 parser.add_argument( 

749 "--preserve-properties", nargs="+", default=[], metavar="STRING", 

750 help="On replication, preserve the current value of ZFS properties with the given names on the destination " 

751 "datasets. The destination ignores the property value it zfs receive's from the source if the property name " 

752 "matches one of the given blacklist values. This prevents a compromised or untrusted source from overwriting " 

753 "security-critical properties on the destination. The default is to preserve none, i.e. an empty blacklist.\n\n" 

754 "Example blacklist that protects against dangerous overwrites: " 

755 "mountpoint overlay sharenfs sharesmb exec setuid devices encryption keyformat keylocation\n\n" 

756 "See https://openzfs.github.io/openzfs-docs/man/master/7/zfsprops.7.html and " 

757 "https://openzfs.github.io/openzfs-docs/man/master/8/zfs-receive.8.html#x\n\n" 

758 "Note: --preserve-properties uses the 'zfs recv -x' option and thus requires either OpenZFS ≥ 2.2.0 " 

759 "(see https://github.com/openzfs/zfs/commit/b0269cd8ced242e66afc4fa856d62be29bb5a4ff), or that " 

760 "'zfs send --props' is not used.\n\n") 

761 parser.add_argument( 

762 "--force-rollback-to-latest-snapshot", action="store_true", 

763 help="Before replication, rollback the destination dataset to its most recent destination snapshot (if there " 

764 "is one), via 'zfs rollback', just in case the destination dataset was modified since its most recent " 

765 "snapshot. This is much less invasive than the other --force* options (see below).\n\n") 

766 parser.add_argument( 

767 "--force-rollback-to-latest-common-snapshot", action="store_true", 

768 help="Before replication, delete destination ZFS snapshots that are more recent than the most recent common " 

769 "snapshot ('conflicting snapshots'), via 'zfs rollback'. Do no rollback if no common snapshot exists.\n\n") 

770 parser.add_argument( 

771 "--force", action="store_true", 

772 help="Same as --force-rollback-to-latest-common-snapshot (see above), except that additionally, if no common " 

773 "snapshot exists, then delete all destination snapshots before starting replication, and proceed " 

774 "without aborting. Without the --force* flags, the destination dataset is treated as append-only, hence " 

775 "no destination snapshot that already exists is deleted, and instead the operation is aborted with an " 

776 "error when encountering a conflicting snapshot.\n\n" 

777 "Analogy: --force-rollback-to-latest-snapshot is a tiny hammer, whereas " 

778 "--force-rollback-to-latest-common-snapshot is a medium sized hammer, --force is a large hammer, and " 

779 "--force-destroy-dependents is a very large hammer. " 

780 "Consider using the smallest hammer that can fix the problem. No hammer is ever used by default.\n\n") 

781 parser.add_argument( 

782 "--force-destroy-dependents", action="store_true", 

783 help="On destination, --force and --force-rollback-to-latest-common-snapshot and --delete-* will add the " 

784 "'-R' flag to their use of 'zfs rollback' and 'zfs destroy', causing them to delete dependents such as " 

785 "clones and bookmarks. This can be very destructive and is rarely advisable.\n\n") 

786 parser.add_argument( 

787 "--force-unmount", action="store_true", 

788 help="On destination, --force and --force-rollback-to-latest-common-snapshot will add the '-f' flag to their " 

789 "use of 'zfs rollback' and 'zfs destroy'.\n\n") 

790 parser.add_argument( 

791 "--force-once", "--f1", action="store_true", 

792 help="Use the --force option or --force-rollback-to-latest-common-snapshot option at most once to resolve a " 

793 "conflict, then abort with an error on any subsequent conflict. This helps to interactively resolve " 

794 "conflicts, one conflict at a time.\n\n") 

795 parser.add_argument( 

796 "--skip-parent", action="store_true", 

797 help="During replication and deletion, skip processing of the SRC_DATASET and DST_DATASET and only process " 

798 "their descendant datasets, i.e. children, and children of children, etc (with --recursive). No dataset " 

799 "is processed unless --recursive is also specified. " 

800 f"Analogy: `{PROG_NAME} --recursive --skip-parent src dst` is akin to Unix `cp -r src/* dst/` whereas " 

801 f" `{PROG_NAME} --recursive --skip-parent --skip-replication --delete-dst-datasets dummy dst` is akin to " 

802 "Unix `rm -r dst/*`\n\n") 

803 parser.add_argument( 

804 "--skip-missing-snapshots", choices=["fail", "dataset", "continue"], default="dataset", nargs="?", 

805 help="During replication, handle source datasets that select no snapshots (and no relevant bookmarks) " 

806 "as follows:\n\n" 

807 "a) 'fail': Abort with an error.\n\n" 

808 "b) 'dataset' (default): Skip the source dataset with a warning. Skip descendant datasets if " 

809 "--recursive and destination dataset does not exist or is inconsistent. Otherwise skip to the next dataset.\n\n" 

810 "c) 'continue': Continue processing the next dataset, including descendants. If an inconsistent " 

811 "destination dataset prevents processing selected descendants, fail with an error; --skip-on-error " 

812 "determines whether processing continues elsewhere. " 

813 "Eventually create empty destination dataset and ancestors if they do not yet exist and source dataset " 

814 "has at least one descendant that selects at least one snapshot.\n\n") 

815 parser.add_argument( 

816 "--retries", dest="max_retries", type=int, min=0, default=2, action=CheckRange, metavar="INT", 

817 help="The maximum number of times a retryable replication or deletion step shall be retried if it fails, for " 

818 "example because of network hiccups (default: %(default)s, min: %(min)s). " 

819 "Also consider this option if a periodic pruning script may simultaneously delete a dataset or " 

820 f"snapshot or bookmark while {PROG_NAME} is running and attempting to access it.\n\n") 

821 parser.add_argument( 

822 "--retry-min-sleep-secs", type=float, min=0, default=0, action=CheckRange, metavar="FLOAT", 

823 help="The minimum duration to sleep between retries (default: %(default)s).\n\n") 

824 parser.add_argument( 

825 "--retry-initial-max-sleep-secs", type=float, min=0, default=0.125, action=CheckRange, metavar="FLOAT", 

826 help="The initial maximum duration to sleep between retries (default: %(default)s).\n\n") 

827 parser.add_argument( 

828 "--retry-max-sleep-secs", type=float, min=0, default=5 * 60, action=CheckRange, metavar="FLOAT", 

829 help="The maximum duration to sleep between retries initially starts with --retry-initial-max-sleep-secs " 

830 "(see above), and doubles on each retry, up to the final maximum of --retry-max-sleep-secs " 

831 "(default: %(default)s). On each retry a random sleep time in the [--retry-min-sleep-secs, current max] range " 

832 "is picked. In a nutshell: retry-min-sleep-secs ≤ retry-initial-max-sleep-secs ≤ retry-max-sleep-secs. " 

833 "The timer resets after each operation.\n\n") 

834 parser.add_argument( 

835 "--retry-max-elapsed-secs", type=float, min=0, default=60 * 60, action=CheckRange, metavar="FLOAT", 

836 help="A single operation (e.g. 'zfs send/receive' of the current dataset, or deletion of a list of snapshots " 

837 "within the current dataset) will not be retried (or not retried anymore) once this much time has elapsed " 

838 "since the initial start of the operation, including retries (default: %(default)s). " 

839 "The timer resets after each operation completes or retries exhaust, such that subsequently failing " 

840 "operations can again be retried.\n\n") 

841 parser.add_argument( 

842 "--skip-on-error", choices=["fail", "tree", "dataset"], default=SKIP_ON_ERROR_DEFAULT, 

843 help="During replication and deletion, if an error is not retryable, or --retries has been exhausted, " 

844 "or --skip-missing-snapshots raises an error, proceed as follows:\n\n" 

845 "a) 'fail': Abort the program with an error. This mode is ideal for testing, clear " 

846 "error reporting, and situations where consistency trumps availability.\n\n" 

847 "b) 'tree': Log the error, skip the dataset tree rooted at the dataset for which the error " 

848 "occurred, and continue processing the next (sibling) dataset tree. " 

849 "Example: Assume datasets tank/user1/foo and tank/user2/bar and an error occurs while processing " 

850 "tank/user1. In this case processing skips tank/user1/foo and proceeds with tank/user2.\n\n" 

851 "c) 'dataset' (default): Same as 'tree' except if the destination dataset already exists and is not " 

852 "inconsistent, skip to the next dataset instead.\n\n" 

853 "Example: Assume datasets tank/user1/foo and tank/user2/bar and an error occurs while " 

854 "processing tank/user1. In this case processing skips tank/user1 and proceeds with tank/user1/foo " 

855 "if the destination already contains tank/user1 and that dataset is not inconsistent. Otherwise " 

856 "processing continues with tank/user2. " 

857 "This mode is for production use cases that require timely forward progress even in the presence of " 

858 "partial failures. For example, assume the job is to backup the home directories or virtual machines " 

859 "of thousands of users across an organization. Even if replication of some of the datasets for some " 

860 "users fails due too conflicts, busy datasets, etc, the replication job will continue for the " 

861 "remaining datasets and the remaining users.\n\n") 

862 parser.add_argument( 

863 "--skip-replication", action="store_true", 

864 help="Skip replication step (see above) and proceed to the optional --delete-dst-datasets step " 

865 "immediately (see below).\n\n") 

866 parser.add_argument( 

867 "--delete-dst-datasets", action="store_true", 

868 help="Do nothing if the --delete-dst-datasets option is missing. Otherwise, after successful replication " 

869 "step, if any, delete existing destination datasets that are selected via --{include|exclude}-dataset* " 

870 "policy yet do not exist within SRC_DATASET (which can be an empty dataset, such as the hardcoded virtual " 

871 f"dataset named '{DUMMY_DATASET}'!). Do not recurse without --recursive. With --recursive, never delete " 

872 "non-selected dataset subtrees or their ancestors.\n\n" 

873 "For example, if the destination contains datasets h1,h2,h3,d1 whereas source only contains h3, " 

874 "and the include/exclude policy selects h1,h2,h3,d1, then delete datasets h1,h2,d1 on " 

875 "the destination to make it 'the same'. On the other hand, if the include/exclude policy " 

876 "only selects h1,h2,h3 then only delete datasets h1,h2 on the destination to make it 'the same'.\n\n" 

877 "Example to delete all tmp datasets within tank2/boo/bar: " 

878 f"`{PROG_NAME} {DUMMY_DATASET} tank2/boo/bar --dryrun --skip-replication --recursive " 

879 "--delete-dst-datasets --include-dataset-regex '(.*/)?tmp.*' --exclude-dataset-regex '!.*'`\n\n") 

880 parser.add_argument( 

881 "--delete-dst-snapshots", choices=["snapshots", "bookmarks"], default=None, const="snapshots", nargs="?", 

882 help="Do nothing if the --delete-dst-snapshots option is missing. Otherwise, after successful " 

883 "replication, and successful --delete-dst-datasets step, if any, delete existing destination snapshots " 

884 "whose GUID does not exist within the source dataset (which can be an empty dummy dataset!) if the " 

885 "destination snapshots are selected by the --include/exclude-snapshot-* policy, and the destination " 

886 "dataset is selected via --{include|exclude}-dataset* policy. Does not recurse without --recursive.\n\n" 

887 "For example, if the destination dataset contains snapshots h1,h2,h3,d1 (h=hourly, d=daily) whereas " 

888 "the source dataset only contains snapshot h3, and the include/exclude policy selects " 

889 "h1,h2,h3,d1, then delete snapshots h1,h2,d1 on the destination dataset to make it 'the same'. " 

890 "On the other hand, if the include/exclude policy only selects snapshots h1,h2,h3 then only " 

891 "delete snapshots h1,h2 on the destination dataset to make it 'the same'.\n\n" 

892 "*Note:* To delete snapshots regardless, consider using --delete-dst-snapshots in combination with a " 

893 f"source that is an empty dataset, such as the hardcoded virtual dataset named '{DUMMY_DATASET}', like so:" 

894 f" `{PROG_NAME} {DUMMY_DATASET} tank2/boo/bar --dryrun --skip-replication --delete-dst-snapshots " 

895 "--include-snapshot-regex '.*_daily' --recursive`\n\n" 

896 "*Note:* Use --delete-dst-snapshots=bookmarks to delete non-temporary bookmarks instead of snapshots, in " 

897 "which case no snapshots are selected and the --{include|exclude}-snapshot-* filter options treat bookmarks " 

898 "as snapshots wrt. selecting.\n\n" 

899 "*Note:* Does not attempt to delete snapshots that carry a `zfs hold`; instead auto-skips them without " 

900 "failing.\n\n" 

901 "*Performance Note:* --delete-dst-snapshots operates on multiple datasets in parallel (and serially " 

902 f"within a dataset), using the same dataset order as {PROG_NAME} replication. " 

903 "The degree of parallelism is configurable with the --threads option (see below).\n\n") 

904 parser.add_argument( 

905 "--force-delete-dst-tmp-bookmarks", action="store_true", 

906 help=argparse.SUPPRESS) 

907 parser.add_argument( 

908 "--delete-dst-snapshots-no-crosscheck", action="store_true", 

909 help="This flag indicates that --delete-dst-snapshots=snapshots shall check the source dataset only for " 

910 "a snapshot with the same GUID, and ignore whether a bookmark with the same GUID is present in the " 

911 "source dataset. Similarly, it also indicates that --delete-dst-snapshots=bookmarks shall check the source " 

912 "dataset only for a bookmark with the same GUID, and ignore whether a snapshot with the same GUID is present " 

913 "in the source dataset.\n\n") 

914 parser.add_argument( 

915 "--delete-dst-snapshots-except", action="store_true", 

916 help="This flag indicates that the --include/exclude-snapshot-* options shall have inverted semantics for the " 

917 "--delete-dst-snapshots option, thus deleting all snapshots except for the selected snapshots (within the " 

918 "specified datasets), instead of deleting all selected snapshots (within the specified datasets). In other " 

919 "words, this flag enables to specify which snapshots to retain instead of which snapshots to delete.\n\n" 

920 "*Synchronization vs. Backup*: When a real (non-dummy) source dataset is specified in combination with " 

921 "--delete-dst-snapshots-except, then any destination snapshot retained by the rules above is actually only " 

922 "retained if it also exists in the source dataset - __all other destination snapshots are deleted__. This is " 

923 "great for synchronization use cases but should __NEVER BE USED FOR LONG-TERM ARCHIVAL__. Long-term archival " 

924 "use cases should instead specify the `dummy` source dataset as they require an independent retention policy " 

925 "that is not tied to the current contents of the source dataset.\n\n") 

926 parser.add_argument( 

927 "--delete-dst-snapshots-except-plan", action=DeleteDstSnapshotsExceptPlanAction, default=None, metavar="DICT_STRING", 

928 help="Retention periods to be used if pruning snapshots or bookmarks within the selected destination datasets via " 

929 "--delete-dst-snapshots. Has the same format as --create-src-snapshots-plan. " 

930 "Snapshots (--delete-dst-snapshots=snapshots) or bookmarks (with --delete-dst-snapshots=bookmarks) that " 

931 "do not match a period will be deleted. To avoid unexpected surprises, make sure to carefully specify ALL " 

932 "snapshot names and periods that shall be retained, in combination with --dryrun.\n\n" 

933 f"Example: `{format_dict(src_snapshot_plan_example)}`. This example will, for the organization 'prod' and " 

934 "the intended logical target 'onsite', retain secondly snapshots that were created less than 40 seconds ago, " 

935 "yet retain the latest 40 secondly snapshots regardless of creation time. Analog for the latest 40 minutely " 

936 "snapshots, latest 36 hourly snapshots, etc. " 

937 "It will also retain snapshots for the targets 'us-west' and 'eu-west' within the 'prod' organization. " 

938 "In addition, within the 'test' organization, it will retain snapshots that are created every 12 hours and " 

939 "every week as specified, and name them as being intended for the 'offsite' replication target. Analog for " 

940 "snapshots that are taken every 100 milliseconds within the 'test' organization. " 

941 "All other snapshots within the selected datasets will be deleted - you've been warned!\n\n" 

942 "The example scans the selected ZFS datasets for snapshots with names like " 

943 "`prod_onsite_<timestamp>_secondly`, `prod_onsite_<timestamp>_minutely`, " 

944 "`prod_us-west_<timestamp>_hourly`, `prod_us-west_<timestamp>_daily`, " 

945 "`prod_eu-west_<timestamp>_hourly`, `prod_eu-west_<timestamp>_daily`, " 

946 "`test_offsite_<timestamp>_12hourly`, `test_offsite_<timestamp>_weekly`, and so on, and deletes all snapshots " 

947 "that do not match a retention rule.\n\n" 

948 "Note: A zero within a period (e.g. 'hourly': 0) indicates that no snapshots shall be retained for the given " 

949 "period.\n\n" 

950 "Note: --delete-dst-snapshots-except-plan is a convenience option that auto-generates a series of the " 

951 "following other options: --delete-dst-snapshots-except, " 

952 "--new-snapshot-filter-group, --include-snapshot-regex, --include-snapshot-times-and-ranks\n\n") 

953 parser.add_argument( 

954 "--delete-empty-dst-datasets", choices=["snapshots", "snapshots+bookmarks"], default=None, 

955 const="snapshots+bookmarks", nargs="?", 

956 help="Do nothing if the --delete-empty-dst-datasets option is missing or --recursive is missing. Otherwise, " 

957 "after successful replication " 

958 "step and successful --delete-dst-datasets and successful --delete-dst-snapshots steps, if any, " 

959 "delete any selected destination dataset that has no snapshot and no bookmark if all descendants of " 

960 "that destination dataset are also selected and do not have a snapshot or bookmark either " 

961 "(again, only if the existing destination dataset is selected via --{include|exclude}-dataset* policy). " 

962 "Never delete non-selected dataset subtrees or their ancestors.\n\n" 

963 "For example, if the destination contains datasets h1,d1, and the include/exclude policy " 

964 "selects h1,d1, then check if h1,d1 can be deleted. " 

965 "On the other hand, if the include/exclude policy only selects h1 then only check if h1 can be deleted.\n\n" 

966 "*Note:* Use --delete-empty-dst-datasets=snapshots to delete snapshot-less datasets even if they still " 

967 "contain bookmarks.\n\n") 

968 monitor_snapshot_plan_example = { 

969 "prod": { 

970 "onsite": { 

971 "100millisecondly": {"latest": {"warning": "300 milliseconds", "critical": "2 seconds"}}, 

972 "secondly": {"latest": {"warning": "2 seconds", "critical": "14 seconds"}}, 

973 "minutely": {"latest": {"warning": "30 seconds", "critical": "300 seconds"}}, 

974 "hourly": {"latest": {"warning": "30 minutes", "critical": "300 minutes"}}, 

975 "daily": {"latest": {"warning": "4 hours", "critical": "8 hours"}}, 

976 "weekly": {"latest": {"warning": "2 days", "critical": "8 days"}}, 

977 "monthly": {"latest": {"warning": "2 days", "critical": "8 days"}}, 

978 "yearly": {"latest": {"warning": "5 days", "critical": "14 days"}}, 

979 "10minutely": {"latest": {"warning": "0 minutes", "critical": "0 minutes"}}, 

980 }, 

981 "": { 

982 "daily": {"latest": {"warning": "4 hours", "critical": "8 hours"}}, 

983 }, 

984 }, 

985 } 

986 monitor_snapshots_output_example: str = ( 

987 "`--monitor_snapshots: OK. Latest snapshot for tank/foo@prod_<timestamp>_daily is 4.18h old: @prod_2025-01-10_08:30:05_daily`\n\n" 

988 "`--monitor_snapshots: OK. Latest snapshot for tank/bar@prod_<timestamp>_daily is 4.18h old: @prod_2025-01-10_08:30:05_daily`\n\n" 

989 "`--monitor_snapshots: Latest snapshot for tank/baz@prod_<timestamp>_daily is 1.2d old but should be at most 1.1d old: @prod_2025-01-09_08:30:05_daily`\n\n" 

990 " ...\n\n" 

991 "`ERROR: Exiting bzfs with status code 2. Cause: --monitor_snapshots: Latest snapshot for tank/baz@prod_<timestamp>_daily is 1.2d old but should be at most 1.1d old: @prod_2025-01-09_08:30:05_daily`\n\n" 

992 ) 

993 parser.add_argument( 

994 "--monitor-snapshots", default="{}", type=str, metavar="DICT_STRING", 

995 help="Do nothing if the --monitor-snapshots flag is missing. Otherwise, after all other steps, " 

996 "alert the user if the ZFS 'creation' time property of the latest snapshot for any specified snapshot name " 

997 "pattern within the selected datasets is too old wrt. the specified age limit. The purpose is to check if " 

998 "snapshots are successfully taken on schedule, successfully replicated on schedule, and successfully pruned on " 

999 "schedule. Process exit code is 0, 1, 2 on OK, WARNING, CRITICAL, respectively. " 

1000 f"Example DICT_STRING: `{format_dict(monitor_snapshot_plan_example)}`. " 

1001 "This example alerts the user if the latest src or dst snapshot named `prod_onsite_<timestamp>_hourly` is more " 

1002 "than 30 minutes late (i.e. more than 30+60=90 minutes old) [warning] or more than 300 minutes late (i.e. more " 

1003 "than 300+60=360 minutes old) [critical]. " 

1004 "Analog for the latest snapshot named `prod_<timestamp>_daily`, and so on.\n\n" 

1005 "Note: A duration that is missing or zero (e.g. '0 minutes') indicates that no snapshots shall be checked for " 

1006 "the given snapshot name pattern.\n\n" 

1007 f"Example output with `--verbose`:\n\n{monitor_snapshots_output_example}\n\n") 

1008 parser.add_argument( 

1009 "--monitor-snapshots-dont-warn", action="store_true", 

1010 help="Log a message for monitoring warnings but nonetheless exit with zero exit code.\n\n") 

1011 parser.add_argument( 

1012 "--monitor-snapshots-dont-crit", action="store_true", 

1013 help="Log a message for monitoring criticals but nonetheless exit with zero exit code.\n\n") 

1014 parser.add_argument( 

1015 "--monitor-snapshots-no-latest-check", action="store_true", 

1016 # help="Disable monitoring check of latest snapshot.\n\n") 

1017 help=argparse.SUPPRESS) 

1018 parser.add_argument( 

1019 "--monitor-snapshots-no-oldest-check", action="store_true", 

1020 # help="Disable monitoring check of oldest snapshot.\n\n") 

1021 help=argparse.SUPPRESS)