| 1 | #!/usr/bin/env python3 |
| 2 | # group: throttle |
| 3 | # |
| 4 | # Tests for IO throttling |
| 5 | # |
| 6 | # Copyright (C) 2015 Red Hat, Inc. |
| 7 | # Copyright (C) 2015-2016 Igalia, S.L. |
| 8 | # |
| 9 | # This program is free software; you can redistribute it and/or modify |
| 10 | # it under the terms of the GNU General Public License as published by |
| 11 | # the Free Software Foundation; either version 2 of the License, or |
| 12 | # (at your option) any later version. |
| 13 | # |
| 14 | # This program is distributed in the hope that it will be useful, |
| 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 17 | # GNU General Public License for more details. |
| 18 | # |
| 19 | # You should have received a copy of the GNU General Public License |
| 20 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 21 | # |
| 22 | |
| 23 | import iotests |
| 24 | |
| 25 | nsec_per_sec = 1000000000 |
| 26 | |
| 27 | class ThrottleTestCase(iotests.QMPTestCase): |
| 28 | test_driver = "null-aio" |
| 29 | max_drives = 3 |
| 30 | |
| 31 | def blockstats(self, device): |
| 32 | result = self.vm.qmp("query-blockstats") |
| 33 | for r in result['return']: |
| 34 | if r['device'] == device: |
| 35 | stat = r['stats'] |
| 36 | return stat['rd_bytes'], stat['rd_operations'], stat['wr_bytes'], stat['wr_operations'] |
| 37 | raise Exception("Device not found for blockstats: %s" % device) |
| 38 | |
| 39 | def required_drivers(self): |
| 40 | return [self.test_driver] |
| 41 | |
| 42 | @iotests.skip_if_unsupported(required_drivers) |
| 43 | def setUp(self): |
| 44 | self.vm = iotests.VM() |
| 45 | for i in range(0, self.max_drives): |
| 46 | self.vm.add_drive(self.test_driver + "://", "file.read-zeroes=on") |
| 47 | self.vm.launch() |
| 48 | |
| 49 | def tearDown(self): |
| 50 | self.vm.shutdown() |
| 51 | |
| 52 | def configure_throttle(self, ndrives, params): |
| 53 | params['group'] = 'test' |
| 54 | |
| 55 | # Set the I/O throttling parameters to all drives |
| 56 | for i in range(0, ndrives): |
| 57 | params['device'] = 'drive%d' % i |
| 58 | self.vm.cmd("block_set_io_throttle", conv_keys=False, **params) |
| 59 | |
| 60 | def do_test_throttle(self, ndrives, seconds, params, first_drive = 0): |
| 61 | def check_limit(limit, num): |
| 62 | # IO throttling algorithm is discrete, allow 10% error so the test |
| 63 | # is more robust |
| 64 | return limit == 0 or \ |
| 65 | (num < seconds * limit * 1.1 / ndrives |
| 66 | and num > seconds * limit * 0.9 / ndrives) |
| 67 | |
| 68 | # Set vm clock to a known value |
| 69 | ns = seconds * nsec_per_sec |
| 70 | self.vm.qtest("clock_step %d" % ns) |
| 71 | |
| 72 | # Submit enough requests so the throttling mechanism kicks |
| 73 | # in. The throttled requests won't be executed until we |
| 74 | # advance the virtual clock. |
| 75 | rq_size = 512 |
| 76 | rd_nr = max(params['bps'] // rq_size // 2, |
| 77 | params['bps_rd'] // rq_size, |
| 78 | params['iops'] // 2, |
| 79 | params['iops_rd']) |
| 80 | rd_nr *= seconds * 2 |
| 81 | rd_nr //= ndrives |
| 82 | wr_nr = max(params['bps'] // rq_size // 2, |
| 83 | params['bps_wr'] // rq_size, |
| 84 | params['iops'] // 2, |
| 85 | params['iops_wr']) |
| 86 | wr_nr *= seconds * 2 |
| 87 | wr_nr //= ndrives |
| 88 | |
| 89 | # Send I/O requests to all drives |
| 90 | for i in range(rd_nr): |
| 91 | for drive in range(0, ndrives): |
| 92 | idx = first_drive + drive |
| 93 | self.vm.hmp_qemu_io("drive%d" % idx, "aio_read %d %d" % |
| 94 | (i * rq_size, rq_size)) |
| 95 | |
| 96 | for i in range(wr_nr): |
| 97 | for drive in range(0, ndrives): |
| 98 | idx = first_drive + drive |
| 99 | self.vm.hmp_qemu_io("drive%d" % idx, "aio_write %d %d" % |
| 100 | (i * rq_size, rq_size)) |
| 101 | |
| 102 | # We'll store the I/O stats for each drive in these arrays |
| 103 | start_rd_bytes = [0] * ndrives |
| 104 | start_rd_iops = [0] * ndrives |
| 105 | start_wr_bytes = [0] * ndrives |
| 106 | start_wr_iops = [0] * ndrives |
| 107 | end_rd_bytes = [0] * ndrives |
| 108 | end_rd_iops = [0] * ndrives |
| 109 | end_wr_bytes = [0] * ndrives |
| 110 | end_wr_iops = [0] * ndrives |
| 111 | |
| 112 | # Read the stats before advancing the clock |
| 113 | for i in range(0, ndrives): |
| 114 | idx = first_drive + i |
| 115 | start_rd_bytes[i], start_rd_iops[i], start_wr_bytes[i], \ |
| 116 | start_wr_iops[i] = self.blockstats('drive%d' % idx) |
| 117 | |
| 118 | self.vm.qtest("clock_step %d" % ns) |
| 119 | |
| 120 | # Read the stats after advancing the clock |
| 121 | for i in range(0, ndrives): |
| 122 | idx = first_drive + i |
| 123 | end_rd_bytes[i], end_rd_iops[i], end_wr_bytes[i], \ |
| 124 | end_wr_iops[i] = self.blockstats('drive%d' % idx) |
| 125 | |
| 126 | # Check that the I/O is within the limits and evenly distributed |
| 127 | for i in range(0, ndrives): |
| 128 | rd_bytes = end_rd_bytes[i] - start_rd_bytes[i] |
| 129 | rd_iops = end_rd_iops[i] - start_rd_iops[i] |
| 130 | wr_bytes = end_wr_bytes[i] - start_wr_bytes[i] |
| 131 | wr_iops = end_wr_iops[i] - start_wr_iops[i] |
| 132 | |
| 133 | self.assertTrue(check_limit(params['bps'], rd_bytes + wr_bytes)) |
| 134 | self.assertTrue(check_limit(params['bps_rd'], rd_bytes)) |
| 135 | self.assertTrue(check_limit(params['bps_wr'], wr_bytes)) |
| 136 | self.assertTrue(check_limit(params['iops'], rd_iops + wr_iops)) |
| 137 | self.assertTrue(check_limit(params['iops_rd'], rd_iops)) |
| 138 | self.assertTrue(check_limit(params['iops_wr'], wr_iops)) |
| 139 | |
| 140 | # Allow remaining requests to finish. We submitted twice as many to |
| 141 | # ensure the throttle limit is reached. |
| 142 | self.vm.qtest("clock_step %d" % ns) |
| 143 | |
| 144 | # Connect N drives to a VM and test I/O in all of them |
| 145 | def test_all(self): |
| 146 | params = {"bps": 4096, |
| 147 | "bps_rd": 4096, |
| 148 | "bps_wr": 4096, |
| 149 | "iops": 10, |
| 150 | "iops_rd": 10, |
| 151 | "iops_wr": 10, |
| 152 | } |
| 153 | # Repeat the test with different numbers of drives |
| 154 | for ndrives in range(1, self.max_drives + 1): |
| 155 | # Pick each out of all possible params and test |
| 156 | for tk in params: |
| 157 | limits = dict([(k, 0) for k in params]) |
| 158 | limits[tk] = params[tk] * ndrives |
| 159 | self.configure_throttle(ndrives, limits) |
| 160 | self.do_test_throttle(ndrives, 5, limits) |
| 161 | |
| 162 | # Connect N drives to a VM and test I/O in just one of them a time |
| 163 | def test_one(self): |
| 164 | params = {"bps": 4096, |
| 165 | "bps_rd": 4096, |
| 166 | "bps_wr": 4096, |
| 167 | "iops": 10, |
| 168 | "iops_rd": 10, |
| 169 | "iops_wr": 10, |
| 170 | } |
| 171 | # Repeat the test for each one of the drives |
| 172 | for drive in range(0, self.max_drives): |
| 173 | # Pick each out of all possible params and test |
| 174 | for tk in params: |
| 175 | limits = dict([(k, 0) for k in params]) |
| 176 | limits[tk] = params[tk] * self.max_drives |
| 177 | self.configure_throttle(self.max_drives, limits) |
| 178 | self.do_test_throttle(1, 5, limits, drive) |
| 179 | |
| 180 | def test_burst(self): |
| 181 | params = {"bps": 4096, |
| 182 | "bps_rd": 4096, |
| 183 | "bps_wr": 4096, |
| 184 | "iops": 10, |
| 185 | "iops_rd": 10, |
| 186 | "iops_wr": 10, |
| 187 | } |
| 188 | ndrives = 1 |
| 189 | # Pick each out of all possible params and test |
| 190 | for tk in params: |
| 191 | rate = params[tk] * ndrives |
| 192 | burst_rate = rate * 7 |
| 193 | burst_length = 4 |
| 194 | |
| 195 | # Configure the throttling settings |
| 196 | settings = dict([(k, 0) for k in params]) |
| 197 | settings[tk] = rate |
| 198 | settings['%s_max' % tk] = burst_rate |
| 199 | settings['%s_max_length' % tk] = burst_length |
| 200 | self.configure_throttle(ndrives, settings) |
| 201 | |
| 202 | # Wait for the bucket to empty so we can do bursts |
| 203 | wait_ns = nsec_per_sec * burst_length * burst_rate // rate |
| 204 | self.vm.qtest("clock_step %d" % wait_ns) |
| 205 | |
| 206 | # Test I/O at the max burst rate |
| 207 | limits = dict([(k, 0) for k in params]) |
| 208 | limits[tk] = burst_rate |
| 209 | self.do_test_throttle(ndrives, burst_length, limits) |
| 210 | |
| 211 | # Now test I/O at the normal rate |
| 212 | limits[tk] = rate |
| 213 | self.do_test_throttle(ndrives, 5, limits) |
| 214 | |
| 215 | # Test that removing a drive from a throttle group should not |
| 216 | # affect the remaining members of the group. |
| 217 | # https://bugzilla.redhat.com/show_bug.cgi?id=1535914 |
| 218 | def test_remove_group_member(self): |
| 219 | # Create a throttle group with two drives |
| 220 | # and set a 4 KB/s read limit. |
| 221 | params = {"bps": 0, |
| 222 | "bps_rd": 4096, |
| 223 | "bps_wr": 0, |
| 224 | "iops": 0, |
| 225 | "iops_rd": 0, |
| 226 | "iops_wr": 0 } |
| 227 | self.configure_throttle(2, params) |
| 228 | |
| 229 | # Read 4KB from drive0. This is performed immediately. |
| 230 | self.vm.hmp_qemu_io("drive0", "aio_read 0 4096") |
| 231 | |
| 232 | # Read 2KB. The I/O limit has been exceeded so this |
| 233 | # request is throttled and a timer is set to wake it up. |
| 234 | self.vm.hmp_qemu_io("drive0", "aio_read 0 2048") |
| 235 | |
| 236 | # Read 2KB again. We're still over the I/O limit so this is |
| 237 | # request is also throttled, but no new timer is set since |
| 238 | # there's already one. |
| 239 | self.vm.hmp_qemu_io("drive0", "aio_read 0 2048") |
| 240 | |
| 241 | # Read from drive1. This request is also throttled, and no |
| 242 | # timer is set in drive1 because there's already one in |
| 243 | # drive0. |
| 244 | self.vm.hmp_qemu_io("drive1", "aio_read 0 4096") |
| 245 | |
| 246 | # At this point only the first 4KB have been read from drive0. |
| 247 | # The other requests are throttled. |
| 248 | self.assertEqual(self.blockstats('drive0')[0], 4096) |
| 249 | self.assertEqual(self.blockstats('drive1')[0], 0) |
| 250 | |
| 251 | # Remove drive0 from the throttle group and disable its I/O limits. |
| 252 | # drive1 remains in the group with a throttled request. |
| 253 | params['bps_rd'] = 0 |
| 254 | params['device'] = 'drive0' |
| 255 | self.vm.cmd("block_set_io_throttle", conv_keys=False, **params) |
| 256 | |
| 257 | # Removing the I/O limits from drive0 drains its two pending requests. |
| 258 | # The read request in drive1 is still throttled. |
| 259 | self.assertEqual(self.blockstats('drive0')[0], 8192) |
| 260 | self.assertEqual(self.blockstats('drive1')[0], 0) |
| 261 | |
| 262 | # Advance the clock 5 seconds. This completes the request in drive1 |
| 263 | self.vm.qtest("clock_step %d" % (5 * nsec_per_sec)) |
| 264 | |
| 265 | # Now all requests have been processed. |
| 266 | self.assertEqual(self.blockstats('drive0')[0], 8192) |
| 267 | self.assertEqual(self.blockstats('drive1')[0], 4096) |
| 268 | |
| 269 | class ThrottleTestCoroutine(ThrottleTestCase): |
| 270 | test_driver = "null-co" |
| 271 | |
| 272 | class ThrottleTestGroupNames(iotests.QMPTestCase): |
| 273 | max_drives = 3 |
| 274 | |
| 275 | def setUp(self): |
| 276 | self.vm = iotests.VM() |
| 277 | for i in range(0, self.max_drives): |
| 278 | self.vm.add_drive("null-co://", |
| 279 | "throttling.iops-total=100,file.read-zeroes=on") |
| 280 | self.vm.launch() |
| 281 | |
| 282 | def tearDown(self): |
| 283 | self.vm.shutdown() |
| 284 | |
| 285 | def set_io_throttle(self, device, params): |
| 286 | params["device"] = device |
| 287 | self.vm.cmd("block_set_io_throttle", conv_keys=False, **params) |
| 288 | |
| 289 | def verify_name(self, device, name): |
| 290 | result = self.vm.qmp("query-block") |
| 291 | for r in result["return"]: |
| 292 | if r["device"] == device: |
| 293 | info = r["inserted"] |
| 294 | if name: |
| 295 | self.assertEqual(info["group"], name) |
| 296 | else: |
| 297 | self.assertFalse('group' in info) |
| 298 | return |
| 299 | |
| 300 | raise Exception("No group information found for '%s'" % device) |
| 301 | |
| 302 | def test_group_naming(self): |
| 303 | params = {"bps": 0, |
| 304 | "bps_rd": 0, |
| 305 | "bps_wr": 0, |
| 306 | "iops": 0, |
| 307 | "iops_rd": 0, |
| 308 | "iops_wr": 0} |
| 309 | |
| 310 | # Check the drives added using the command line. |
| 311 | # The default throttling group name is the device name. |
| 312 | for i in range(self.max_drives): |
| 313 | devname = "drive%d" % i |
| 314 | self.verify_name(devname, devname) |
| 315 | |
| 316 | # Clear throttling settings => the group name is gone. |
| 317 | for i in range(self.max_drives): |
| 318 | devname = "drive%d" % i |
| 319 | self.set_io_throttle(devname, params) |
| 320 | self.verify_name(devname, None) |
| 321 | |
| 322 | # Set throttling settings using block_set_io_throttle and |
| 323 | # check the default group names. |
| 324 | params["iops"] = 10 |
| 325 | for i in range(self.max_drives): |
| 326 | devname = "drive%d" % i |
| 327 | self.set_io_throttle(devname, params) |
| 328 | self.verify_name(devname, devname) |
| 329 | |
| 330 | # Set a custom group name for each device |
| 331 | for i in range(3): |
| 332 | devname = "drive%d" % i |
| 333 | groupname = "group%d" % i |
| 334 | params['group'] = groupname |
| 335 | self.set_io_throttle(devname, params) |
| 336 | self.verify_name(devname, groupname) |
| 337 | |
| 338 | # Put drive0 in group1 and check that all other devices remain |
| 339 | # unchanged |
| 340 | params['group'] = 'group1' |
| 341 | self.set_io_throttle('drive0', params) |
| 342 | self.verify_name('drive0', 'group1') |
| 343 | for i in range(1, self.max_drives): |
| 344 | devname = "drive%d" % i |
| 345 | groupname = "group%d" % i |
| 346 | self.verify_name(devname, groupname) |
| 347 | |
| 348 | # Put drive0 in group2 and check that all other devices remain |
| 349 | # unchanged |
| 350 | params['group'] = 'group2' |
| 351 | self.set_io_throttle('drive0', params) |
| 352 | self.verify_name('drive0', 'group2') |
| 353 | for i in range(1, self.max_drives): |
| 354 | devname = "drive%d" % i |
| 355 | groupname = "group%d" % i |
| 356 | self.verify_name(devname, groupname) |
| 357 | |
| 358 | # Clear throttling settings from drive0 check that all other |
| 359 | # devices remain unchanged |
| 360 | params["iops"] = 0 |
| 361 | self.set_io_throttle('drive0', params) |
| 362 | self.verify_name('drive0', None) |
| 363 | for i in range(1, self.max_drives): |
| 364 | devname = "drive%d" % i |
| 365 | groupname = "group%d" % i |
| 366 | self.verify_name(devname, groupname) |
| 367 | |
| 368 | class ThrottleTestRemovableMedia(iotests.QMPTestCase): |
| 369 | def setUp(self): |
| 370 | self.vm = iotests.VM() |
| 371 | self.vm.add_device("{},id=virtio-scsi".format('virtio-scsi')) |
| 372 | self.vm.launch() |
| 373 | |
| 374 | def tearDown(self): |
| 375 | self.vm.shutdown() |
| 376 | |
| 377 | def test_removable_media(self): |
| 378 | # Add a couple of dummy nodes named cd0 and cd1 |
| 379 | self.vm.cmd("blockdev-add", driver="null-co", |
| 380 | read_zeroes=True, node_name="cd0") |
| 381 | self.vm.cmd("blockdev-add", driver="null-co", |
| 382 | read_zeroes=True, node_name="cd1") |
| 383 | |
| 384 | # Attach a CD drive with cd0 inserted |
| 385 | self.vm.cmd("device_add", driver="scsi-cd", |
| 386 | id="dev0", drive="cd0") |
| 387 | |
| 388 | # Set I/O limits |
| 389 | args = { "id": "dev0", "iops": 100, "iops_rd": 0, "iops_wr": 0, |
| 390 | "bps": 50, "bps_rd": 0, "bps_wr": 0 } |
| 391 | self.vm.cmd("block_set_io_throttle", conv_keys=False, **args) |
| 392 | |
| 393 | # Check that the I/O limits have been set |
| 394 | result = self.vm.qmp("query-block") |
| 395 | self.assert_qmp(result, 'return[0]/inserted/iops', 100) |
| 396 | self.assert_qmp(result, 'return[0]/inserted/bps', 50) |
| 397 | |
| 398 | # Now eject cd0 and insert cd1 |
| 399 | self.vm.cmd("blockdev-open-tray", id='dev0') |
| 400 | self.vm.cmd("blockdev-remove-medium", id='dev0') |
| 401 | self.vm.cmd("blockdev-insert-medium", id='dev0', node_name='cd1') |
| 402 | |
| 403 | # Check that the I/O limits are still the same |
| 404 | result = self.vm.qmp("query-block") |
| 405 | self.assert_qmp(result, 'return[0]/inserted/iops', 100) |
| 406 | self.assert_qmp(result, 'return[0]/inserted/bps', 50) |
| 407 | |
| 408 | # Eject cd1 |
| 409 | self.vm.cmd("blockdev-remove-medium", id='dev0') |
| 410 | |
| 411 | # Check that we can't set limits if the device has no medium |
| 412 | result = self.vm.qmp("block_set_io_throttle", conv_keys=False, **args) |
| 413 | self.assert_qmp(result, 'error/class', 'GenericError') |
| 414 | |
| 415 | # Remove the CD drive |
| 416 | self.vm.cmd("device_del", id='dev0') |
| 417 | |
| 418 | |
| 419 | if __name__ == '__main__': |
| 420 | if 'null-co' not in iotests.supported_formats(): |
| 421 | iotests.notrun('null-co driver support missing') |
| 422 | iotests.main(supported_fmts=["raw"], |
| 423 | require_hmp=True) |