master
text 537 lines 19.4 KB
Raw
1 #!/usr/bin/env python3
2 # group: rw
3 #
4 # Test bitmap-sync backups (incremental, differential, and partials)
5 #
6 # Copyright (c) 2019 John Snow for Red Hat, Inc.
7 #
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the GNU General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #
21 # owner=jsnow@redhat.com
22
23 import math
24 import os
25
26 import iotests
27 from iotests import log, qemu_img
28
29 SIZE = 64 * 1024 * 1024
30 GRANULARITY = 64 * 1024
31
32
33 class Pattern:
34 def __init__(self, byte, offset, size=GRANULARITY):
35 self.byte = byte
36 self.offset = offset
37 self.size = size
38
39 def bits(self, granularity):
40 lower = self.offset // granularity
41 upper = (self.offset + self.size - 1) // granularity
42 return set(range(lower, upper + 1))
43
44
45 class PatternGroup:
46 """Grouping of Pattern objects. Initialize with an iterable of Patterns."""
47 def __init__(self, patterns):
48 self.patterns = patterns
49
50 def bits(self, granularity):
51 """Calculate the unique bits dirtied by this pattern grouping"""
52 res = set()
53 for pattern in self.patterns:
54 res |= pattern.bits(granularity)
55 return res
56
57
58 GROUPS = [
59 PatternGroup([
60 # Batch 0: 4 clusters
61 Pattern('0x49', 0x0000000),
62 Pattern('0x6c', 0x0100000), # 1M
63 Pattern('0x6f', 0x2000000), # 32M
64 Pattern('0x76', 0x3ff0000)]), # 64M - 64K
65 PatternGroup([
66 # Batch 1: 6 clusters (3 new)
67 Pattern('0x65', 0x0000000), # Full overwrite
68 Pattern('0x77', 0x00f8000), # Partial-left (1M-32K)
69 Pattern('0x72', 0x2008000), # Partial-right (32M+32K)
70 Pattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K)
71 PatternGroup([
72 # Batch 2: 7 clusters (3 new)
73 Pattern('0x74', 0x0010000), # Adjacent-right
74 Pattern('0x69', 0x00e8000), # Partial-left (1M-96K)
75 Pattern('0x6e', 0x2018000), # Partial-right (32M+96K)
76 Pattern('0x67', 0x3fe0000,
77 2*GRANULARITY)]), # Overwrite [(64M-128K)-64M)
78 PatternGroup([
79 # Batch 3: 8 clusters (5 new)
80 # Carefully chosen such that nothing re-dirties the one cluster
81 # that copies out successfully before failure in Group #1.
82 Pattern('0xaa', 0x0010000,
83 3*GRANULARITY), # Overwrite and 2x Adjacent-right
84 Pattern('0xbb', 0x00d8000), # Partial-left (1M-160K)
85 Pattern('0xcc', 0x2028000), # Partial-right (32M+160K)
86 Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right
87 ]
88
89
90 class EmulatedBitmap:
91 def __init__(self, granularity=GRANULARITY):
92 self._bits = set()
93 self.granularity = granularity
94
95 def dirty_bits(self, bits):
96 self._bits |= set(bits)
97
98 def dirty_group(self, n):
99 self.dirty_bits(GROUPS[n].bits(self.granularity))
100
101 def clear(self):
102 self._bits = set()
103
104 def clear_bits(self, bits):
105 self._bits -= set(bits)
106
107 def clear_bit(self, bit):
108 self.clear_bits({bit})
109
110 def clear_group(self, n):
111 self.clear_bits(GROUPS[n].bits(self.granularity))
112
113 @property
114 def first_bit(self):
115 return sorted(self.bits)[0]
116
117 @property
118 def bits(self):
119 return self._bits
120
121 @property
122 def count(self):
123 return len(self.bits)
124
125 def compare(self, qmp_bitmap):
126 """
127 Print a nice human-readable message checking that a bitmap as reported
128 by the QMP interface has as many bits set as we expect it to.
129 """
130
131 name = qmp_bitmap.get('name', '(anonymous)')
132 log("= Checking Bitmap {:s} =".format(name))
133
134 want = self.count
135 have = qmp_bitmap['count'] // qmp_bitmap['granularity']
136
137 log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
138 want, have, "OK!" if want == have else "ERROR!"))
139 log('')
140
141
142 class Drive:
143 """Represents, vaguely, a drive attached to a VM.
144 Includes format, graph, and device information."""
145
146 def __init__(self, path, vm=None):
147 self.path = path
148 self.vm = vm
149 self.fmt = None
150 self.size = None
151 self.node = None
152
153 def img_create(self, fmt, size):
154 self.fmt = fmt
155 self.size = size
156 iotests.qemu_img_create('-f', self.fmt, self.path, str(self.size))
157
158 def create_target(self, name, fmt, size):
159 basename = os.path.basename(self.path)
160 file_node_name = "file_{}".format(basename)
161 vm = self.vm
162
163 log(vm.cmd('blockdev-create', job_id='bdc-file-job',
164 options={
165 'driver': 'file',
166 'filename': self.path,
167 'size': 0,
168 }))
169 vm.run_job('bdc-file-job')
170 log(vm.cmd('blockdev-add', driver='file',
171 node_name=file_node_name, filename=self.path))
172
173 log(vm.cmd('blockdev-create', job_id='bdc-fmt-job',
174 options={
175 'driver': fmt,
176 'file': file_node_name,
177 'size': size,
178 }))
179 vm.run_job('bdc-fmt-job')
180 log(vm.cmd('blockdev-add', driver=fmt,
181 node_name=name,
182 file=file_node_name))
183 self.fmt = fmt
184 self.size = size
185 self.node = name
186
187 def blockdev_backup(vm, device, target, sync, **kwargs):
188 # Strip any arguments explicitly nulled by the caller:
189 kwargs = {key: val for key, val in kwargs.items() if val is not None}
190 result = vm.qmp_log('blockdev-backup',
191 device=device,
192 target=target,
193 sync=sync,
194 filter_node_name='backup-top',
195 x_perf={'max-workers': 1},
196 **kwargs)
197 return result
198
199 def blockdev_backup_mktarget(drive, target_id, filepath, sync, **kwargs):
200 target_drive = Drive(filepath, vm=drive.vm)
201 target_drive.create_target(target_id, drive.fmt, drive.size)
202 blockdev_backup(drive.vm, drive.node, target_id, sync, **kwargs)
203
204 def reference_backup(drive, n, filepath):
205 log("--- Reference Backup #{:d} ---\n".format(n))
206 target_id = "ref_target_{:d}".format(n)
207 job_id = "ref_backup_{:d}".format(n)
208 blockdev_backup_mktarget(drive, target_id, filepath, "full",
209 job_id=job_id)
210 drive.vm.run_job(job_id, auto_dismiss=True)
211 log('')
212
213 def backup(drive, n, filepath, sync, **kwargs):
214 log("--- Test Backup #{:d} ---\n".format(n))
215 target_id = "backup_target_{:d}".format(n)
216 job_id = "backup_{:d}".format(n)
217 kwargs.setdefault('auto-finalize', False)
218 blockdev_backup_mktarget(drive, target_id, filepath, sync,
219 job_id=job_id, **kwargs)
220 return job_id
221
222 def perform_writes(drive, n, filter_node_name=None):
223 log("--- Write #{:d} ---\n".format(n))
224 for pattern in GROUPS[n].patterns:
225 cmd = "write -P{:s} 0x{:07x} 0x{:x}".format(
226 pattern.byte,
227 pattern.offset,
228 pattern.size)
229 log(cmd)
230 log(drive.vm.hmp_qemu_io(filter_node_name or drive.node, cmd))
231 bitmaps = drive.vm.query_bitmaps()
232 log({'bitmaps': bitmaps}, indent=2)
233 log('')
234 return bitmaps
235
236
237 def compare_images(image, reference, baseimg=None, expected_match=True):
238 """
239 Print a nice human-readable message comparing these images.
240 """
241 expected_ret = 0 if expected_match else 1
242 if baseimg:
243 qemu_img("rebase", "-u", "-b", baseimg, '-F', iotests.imgfmt, image)
244
245 sub = qemu_img("compare", image, reference, check=False)
246
247 log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format(
248 image, reference,
249 "Identical" if sub.returncode == 0 else "Mismatch",
250 "OK!" if sub.returncode == expected_ret else "ERROR!"),
251 filters=[iotests.filter_testfiles])
252
253 def test_bitmap_sync(bsync_mode, msync_mode='bitmap', failure=None):
254 """
255 Test bitmap backup routines.
256
257 :param bsync_mode: Is the Bitmap Sync mode, and can be any of:
258 - on-success: This is the "incremental" style mode. Bitmaps are
259 synchronized to what was copied out only on success.
260 (Partial images must be discarded.)
261 - never: This is the "differential" style mode.
262 Bitmaps are never synchronized.
263 - always: This is a "best effort" style mode.
264 Bitmaps are always synchronized, regardless of failure.
265 (Partial images must be kept.)
266
267 :param msync_mode: The mirror sync mode to use for the first backup.
268 Can be any one of:
269 - bitmap: Backups based on bitmap manifest.
270 - full: Full backups.
271 - top: Full backups of the top layer only.
272
273 :param failure: Is the (optional) failure mode, and can be any of:
274 - None: No failure. Test the normative path. Default.
275 - simulated: Cancel the job right before it completes.
276 This also tests writes "during" the job.
277 - intermediate: This tests a job that fails mid-process and produces
278 an incomplete backup. Testing limitations prevent
279 testing competing writes.
280 """
281 with iotests.FilePath(
282 'img', 'bsync1', 'bsync2', 'fbackup0', 'fbackup1', 'fbackup2') as \
283 (img_path, bsync1, bsync2, fbackup0, fbackup1, fbackup2), \
284 iotests.VM() as vm:
285
286 mode = "Mode {:s}; Bitmap Sync {:s}".format(msync_mode, bsync_mode)
287 preposition = "with" if failure else "without"
288 cond = "{:s} {:s}".format(preposition,
289 "{:s} failure".format(failure) if failure
290 else "failure")
291 log("\n=== {:s} {:s} ===\n".format(mode, cond))
292
293 log('--- Preparing image & VM ---\n')
294 drive0 = Drive(img_path, vm=vm)
295 drive0.img_create(iotests.imgfmt, SIZE)
296 vm.add_device("{},id=scsi0".format('virtio-scsi'))
297 vm.launch()
298
299 file_config = {
300 'driver': 'file',
301 'filename': drive0.path
302 }
303
304 if failure == 'intermediate':
305 file_config = {
306 'driver': 'blkdebug',
307 'image': file_config,
308 'set-state': [{
309 'event': 'flush_to_disk',
310 'state': 1,
311 'new_state': 2
312 }, {
313 'event': 'flush_to_disk',
314 'state': 2,
315 'new_state': 3
316 }, {
317 'event': "read_aio",
318 'state': 3,
319 'new_state': 4
320 }],
321 'inject-error': [{
322 'event': 'read_aio',
323 'errno': 5,
324 'state': 4,
325 'immediately': False,
326 'once': True
327 }]
328 }
329
330 drive0.node = 'drive0'
331 vm.qmp_log('blockdev-add',
332 filters=[iotests.filter_qmp_testfiles],
333 node_name=drive0.node,
334 driver=drive0.fmt,
335 file=file_config)
336 log('')
337
338 # 0 - Writes and Reference Backup
339 perform_writes(drive0, 0)
340 reference_backup(drive0, 0, fbackup0)
341 log('--- Add Bitmap ---\n')
342 vm.qmp_log("block-dirty-bitmap-add", node=drive0.node,
343 name="bitmap0", granularity=GRANULARITY)
344 log('')
345 ebitmap = EmulatedBitmap()
346
347 # 1 - Writes and Reference Backup
348 bitmaps = perform_writes(drive0, 1)
349 ebitmap.dirty_group(1)
350 bitmap = vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)
351 ebitmap.compare(bitmap)
352 reference_backup(drive0, 1, fbackup1)
353
354 # 1 - Test Backup (w/ Optional induced failure)
355 if failure == 'intermediate':
356 # Activate blkdebug induced failure for second-to-next read
357 log(vm.hmp_qemu_io(drive0.node, 'flush'))
358 log('')
359 job = backup(drive0, 1, bsync1, msync_mode,
360 bitmap="bitmap0", bitmap_mode=bsync_mode)
361
362 def _callback():
363 """Issue writes while the job is open to test bitmap divergence."""
364 # Note: when `failure` is 'intermediate', this isn't called.
365 log('')
366 bitmaps = perform_writes(drive0, 2, filter_node_name='backup-top')
367 # Named bitmap (static, should be unchanged)
368 ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0',
369 bitmaps=bitmaps))
370 # Anonymous bitmap (dynamic, shows new writes)
371 anonymous = EmulatedBitmap()
372 anonymous.dirty_group(2)
373 anonymous.compare(vm.get_bitmap(drive0.node, '', recording=True,
374 bitmaps=bitmaps))
375
376 # Simulate the order in which this will happen:
377 # group 1 gets cleared first, then group two gets written.
378 if ((bsync_mode == 'on-success' and not failure) or
379 (bsync_mode == 'always')):
380 ebitmap.clear()
381 ebitmap.dirty_group(2)
382
383 vm.run_job(job, auto_dismiss=True, auto_finalize=False,
384 pre_finalize=_callback,
385 cancel=(failure == 'simulated'))
386 bitmaps = vm.query_bitmaps()
387 log({'bitmaps': bitmaps}, indent=2)
388 log('')
389
390 if bsync_mode == 'always' and failure == 'intermediate':
391 # TOP treats anything allocated as dirty, expect to see:
392 if msync_mode == 'top':
393 ebitmap.dirty_group(0)
394
395 # We manage to copy one sector (one bit) before the error.
396 ebitmap.clear_bit(ebitmap.first_bit)
397
398 # Full returns all bits set except what was copied/skipped
399 if msync_mode == 'full':
400 fail_bit = ebitmap.first_bit
401 ebitmap.clear()
402 ebitmap.dirty_bits(range(fail_bit, SIZE // GRANULARITY))
403
404 ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
405
406 # 2 - Writes and Reference Backup
407 bitmaps = perform_writes(drive0, 3)
408 ebitmap.dirty_group(3)
409 ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
410 reference_backup(drive0, 2, fbackup2)
411
412 # 2 - Bitmap Backup (In failure modes, this is a recovery.)
413 job = backup(drive0, 2, bsync2, "bitmap",
414 bitmap="bitmap0", bitmap_mode=bsync_mode)
415 vm.run_job(job, auto_dismiss=True, auto_finalize=False)
416 bitmaps = vm.query_bitmaps()
417 log({'bitmaps': bitmaps}, indent=2)
418 log('')
419 if bsync_mode != 'never':
420 ebitmap.clear()
421 ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
422
423 log('--- Cleanup ---\n')
424 vm.qmp_log("block-dirty-bitmap-remove",
425 node=drive0.node, name="bitmap0")
426 bitmaps = vm.query_bitmaps()
427 log({'bitmaps': bitmaps}, indent=2)
428 vm.shutdown()
429 log('')
430
431 log('--- Verification ---\n')
432 # 'simulated' failures will actually all pass here because we canceled
433 # while "pending". This is actually undefined behavior,
434 # don't rely on this to be true!
435 compare_images(bsync1, fbackup1, baseimg=fbackup0,
436 expected_match=failure != 'intermediate')
437 if not failure or bsync_mode == 'always':
438 # Always keep the last backup on success or when using 'always'
439 base = bsync1
440 else:
441 base = fbackup0
442 compare_images(bsync2, fbackup2, baseimg=base)
443 compare_images(img_path, fbackup2)
444 log('')
445
446 def test_backup_api():
447 """
448 Test malformed and prohibited invocations of the backup API.
449 """
450 with iotests.FilePath('img', 'bsync1') as (img_path, backup_path), \
451 iotests.VM() as vm:
452
453 log("\n=== API failure tests ===\n")
454 log('--- Preparing image & VM ---\n')
455 drive0 = Drive(img_path, vm=vm)
456 drive0.img_create(iotests.imgfmt, SIZE)
457 vm.add_device("{},id=scsi0".format('virtio-scsi'))
458 vm.launch()
459
460 file_config = {
461 'driver': 'file',
462 'filename': drive0.path
463 }
464
465 drive0.node = 'drive0'
466 vm.qmp_log('blockdev-add',
467 filters=[iotests.filter_qmp_testfiles],
468 node_name=drive0.node,
469 driver=drive0.fmt,
470 file=file_config)
471 log('')
472
473 target0 = Drive(backup_path, vm=vm)
474 target0.create_target("backup_target", drive0.fmt, drive0.size)
475 log('')
476
477 vm.qmp_log("block-dirty-bitmap-add", node=drive0.node,
478 name="bitmap0", granularity=GRANULARITY)
479 log('')
480
481 log('-- Testing invalid QMP commands --\n')
482
483 error_cases = {
484 'incremental': {
485 None: ['on-success', 'always', 'never', None],
486 'bitmap404': ['on-success', 'always', 'never', None],
487 'bitmap0': ['always', 'never']
488 },
489 'bitmap': {
490 None: ['on-success', 'always', 'never', None],
491 'bitmap404': ['on-success', 'always', 'never', None],
492 'bitmap0': [None],
493 },
494 'full': {
495 None: ['on-success', 'always', 'never'],
496 'bitmap404': ['on-success', 'always', 'never', None],
497 'bitmap0': ['never', None],
498 },
499 'top': {
500 None: ['on-success', 'always', 'never'],
501 'bitmap404': ['on-success', 'always', 'never', None],
502 'bitmap0': ['never', None],
503 },
504 'none': {
505 None: ['on-success', 'always', 'never'],
506 'bitmap404': ['on-success', 'always', 'never', None],
507 'bitmap0': ['on-success', 'always', 'never', None],
508 }
509 }
510
511 # Dicts, as always, are not stably-ordered prior to 3.7, so use tuples:
512 for sync_mode in ('incremental', 'bitmap', 'full', 'top', 'none'):
513 log("-- Sync mode {:s} tests --\n".format(sync_mode))
514 for bitmap in (None, 'bitmap404', 'bitmap0'):
515 for policy in error_cases[sync_mode][bitmap]:
516 blockdev_backup(drive0.vm, drive0.node, "backup_target",
517 sync_mode, job_id='api_job',
518 bitmap=bitmap, bitmap_mode=policy)
519 log('')
520
521
522 def main():
523 for bsync_mode in ("never", "on-success", "always"):
524 for failure in ("simulated", "intermediate", None):
525 test_bitmap_sync(bsync_mode, "bitmap", failure)
526
527 for sync_mode in ('full', 'top'):
528 for bsync_mode in ('on-success', 'always'):
529 for failure in ('simulated', 'intermediate', None):
530 test_bitmap_sync(bsync_mode, sync_mode, failure)
531
532 test_backup_api()
533
534 if __name__ == '__main__':
535 iotests.script_main(main, supported_fmts=['qcow2'],
536 supported_protocols=['file'],
537 require_hmp=True)