Use physical utilisation for cached images

Since glance images are downloaded and snapshotted before they are used.
Only a small proportion of its VDI will be in use and it will never grow.
So the real disk usage for VDIs varies with their capability to expand.
Disks connected to VMs continue to use the virtual utilisation as they
are able to expand.

Change-Id: Ie7aae58a47e6651af5b609fb9abc13ab5d61e4df
Closes-bug: 1612057
(cherry picked from commit e1ddae83901735d9bffd9d736f3c033c8986041f)

diff --git a/nova/tests/unit/virt/xenapi/test_xenapi.py b/nova/tests/unit/virt/xenapi/test_xenapi.py
index 4a027e1ea60b707e7b1e604edc1b6846afc844db..08c703d0464ba0a504e2c6c19c72ee692b75ac68 100644 (file)
--- a/nova/tests/unit/virt/xenapi/test_xenapi.py
+++ b/nova/tests/unit/virt/xenapi/test_xenapi.py
@@ -2088,7 +2088,7 @@ class XenAPIHostTestCase(stubs.XenAPITestBase):
         stats = self.conn.host_state.get_host_stats(False)
         # Values from fake.create_local_srs (ext SR)
         self.assertEqual(stats['disk_total'], 40000)
-        self.assertEqual(stats['disk_used'], 20000)
+        self.assertEqual(stats['disk_used'], 0)
         # Values from fake._plugin_xenhost_host_data
         self.assertEqual(stats['host_memory_total'], 10)
         self.assertEqual(stats['host_memory_overhead'], 20)
@@ -2212,6 +2212,7 @@ class XenAPIHostTestCase(stubs.XenAPITestBase):
         self.mox.StubOutWithMock(vm_utils, 'scan_default_sr')
         self.mox.StubOutWithMock(vm_utils, 'list_vms')
         self.mox.StubOutWithMock(self.conn._session, 'call_xenapi')
+        self.mox.StubOutWithMock(host.HostState, 'get_disk_used')
         data = {'disk_total': 0,
                 'disk_used': 0,
                 'disk_available': 0,
@@ -2232,6 +2233,7 @@ class XenAPIHostTestCase(stubs.XenAPITestBase):
             vm_utils.list_vms(self.conn._session).AndReturn([])
             self.conn._session.call_xenapi('SR.get_record', "ref").AndReturn(
                 sr_rec)
+            host.HostState.get_disk_used("ref").AndReturn((0, 0))
             if i == 2:
                 # On the third call (the second below) change the hostname
                 data = dict(data, host_hostname='bar')
@@ -2243,6 +2245,72 @@ class XenAPIHostTestCase(stubs.XenAPITestBase):
         self.assertEqual('foo', stats['hypervisor_hostname'])
 
 
+@mock.patch.object(host.HostState, 'update_status')
+class XenAPIHostStateTestCase(stubs.XenAPITestBaseNoDB):
+
+    def _test_get_disk_used(self, vdis, attached_vbds):
+        session = mock.MagicMock()
+        host_state = host.HostState(session)
+
+        sr_ref = 'sr_ref'
+
+        session.SR.get_VDIs.return_value = vdis.keys()
+        session.VDI.get_virtual_size.side_effect = \
+            lambda vdi_ref: vdis[vdi_ref]['virtual_size']
+        session.VDI.get_physical_utilisation.side_effect = \
+            lambda vdi_ref: vdis[vdi_ref]['physical_utilisation']
+        session.VDI.get_VBDs.side_effect = \
+            lambda vdi_ref: vdis[vdi_ref]['VBDs']
+        session.VBD.get_currently_attached.side_effect = \
+            lambda vbd_ref: vbd_ref in attached_vbds
+
+        disk_used = host_state.get_disk_used(sr_ref)
+        session.SR.get_VDIs.assert_called_once_with(sr_ref)
+        return disk_used
+
+    def test_get_disk_used_virtual(self, mock_update_status):
+        # Both VDIs are attached
+        attached_vbds = ['vbd_1', 'vbd_2']
+        vdis = {
+            'vdi_1': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_1']},
+            'vdi_2': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_2']}
+        }
+        disk_used = self._test_get_disk_used(vdis, attached_vbds)
+        self.assertEqual((200, 2), disk_used)
+
+    def test_get_disk_used_physical(self, mock_update_status):
+        # Neither VDIs are attached
+        attached_vbds = []
+        vdis = {
+            'vdi_1': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_1']},
+            'vdi_2': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_2']}
+        }
+        disk_used = self._test_get_disk_used(vdis, attached_vbds)
+        self.assertEqual((2, 2), disk_used)
+
+    def test_get_disk_used_both(self, mock_update_status):
+        # One VDI is attached
+        attached_vbds = ['vbd_1']
+        vdis = {
+            'vdi_1': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_1']},
+            'vdi_2': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_2']}
+        }
+        disk_used = self._test_get_disk_used(vdis, attached_vbds)
+        self.assertEqual((101, 2), disk_used)
+
+
 class ToSupportedInstancesTestCase(test.NoDBTestCase):
     def test_default_return_value(self):
         self.assertEqual([],

diff --git a/nova/virt/xenapi/host.py b/nova/virt/xenapi/host.py
index 7ff5e9b5116dcb00c12094171c1e8871966cf216..85d6f56a53598a0791d90f32854dacbb74506325 100644 (file)
--- a/nova/virt/xenapi/host.py
+++ b/nova/virt/xenapi/host.py
@@ -225,6 +225,44 @@ class HostState(object):
             self.update_status()
         return self._stats
 
+    def get_disk_used(self, sr_ref):
+        """Since glance images are downloaded and snapshotted before they are
+        used, only a small proportion of its VDI will be in use and it will
+        never grow.  We only need to count the virtual size for disks that
+        are attached to a VM - every other disk can count physical.
+        """
+
+        def _vdi_attached(vdi_ref):
+            try:
+                vbds = self._session.VDI.get_VBDs(vdi_ref)
+                for vbd in vbds:
+                    if self._session.VBD.get_currently_attached(vbd):
+                        return True
+            except self._session.XenAPI.Failure:
+                # VDI or VBD may no longer exist - in which case, it's
+                # not attached
+                pass
+            return False
+
+        allocated = 0
+        physical_used = 0
+
+        all_vdis = self._session.SR.get_VDIs(sr_ref)
+        for vdi_ref in all_vdis:
+            try:
+                vdi_physical = \
+                    int(self._session.VDI.get_physical_utilisation(vdi_ref))
+                if _vdi_attached(vdi_ref):
+                    allocated += \
+                        int(self._session.VDI.get_virtual_size(vdi_ref))
+                else:
+                    allocated += vdi_physical
+                physical_used += vdi_physical
+            except (ValueError, self._session.XenAPI.Failure):
+                LOG.exception(_LE('Unable to get size for vdi %s'), vdi_ref)
+
+        return (allocated, physical_used)
+
     def update_status(self):
         """Since under Xenserver, a compute node runs on a given host,
         we can get host status information using xenapi.
@@ -235,10 +273,10 @@ class HostState(object):
             sr_ref = vm_utils.scan_default_sr(self._session)
             sr_rec = self._session.SR.get_record(sr_ref)
             total = int(sr_rec["physical_size"])
-            used = int(sr_rec["physical_utilisation"])
+            (allocated, used) = self.get_disk_used(sr_ref)
             data["disk_total"] = total
             data["disk_used"] = used
-            data["disk_allocated"] = int(sr_rec["virtual_allocation"])
+            data["disk_allocated"] = allocated
             data["disk_available"] = total - used
             data["supported_instances"] = to_supported_instances(
                 data.get("host_capabilities")