ia64/xen-unstable

changeset 12122:1b923f4a788a

[XEND] Beginnings of the Xend Storage Repository implementation.

Currently it is a basic repository that just creates images on demand
and wraps them as VDIs. Rudimentary support for querying the available
space using 'df'. Creates images via 'qcow-create' after creating an
empty image.

Signed-off-by: Alastair Tse <atse@xensource.com>
author Alastair Tse <atse@xensource.com>
date Thu Oct 12 18:01:32 2006 +0100 (2006-10-12)
parents 255925ae4127
children 58521d4b7c7b
files tools/python/xen/xend/XendNode.py tools/python/xen/xend/XendStorageRepository.py tools/python/xen/xend/XendVDI.py
line diff
     1.1 --- a/tools/python/xen/xend/XendNode.py	Thu Oct 12 18:00:01 2006 +0100
     1.2 +++ b/tools/python/xen/xend/XendNode.py	Thu Oct 12 18:01:32 2006 +0100
     1.3 @@ -21,6 +21,7 @@ import socket
     1.4  import xen.lowlevel.xc
     1.5  from xen.xend import uuid
     1.6  from xen.xend.XendError import XendError
     1.7 +from xen.xend.XendStorageRepository import XendStorageRepository
     1.8  
     1.9  class XendNode:
    1.10      """XendNode - Represents a Domain 0 Host."""
    1.11 @@ -31,6 +32,7 @@ class XendNode:
    1.12          self.cpus = {}
    1.13          self.name = socket.gethostname()
    1.14          self.desc = ""
    1.15 +        self.sr = XendStorageRepository()
    1.16          
    1.17          physinfo = self.physinfo_dict()
    1.18          cpu_count = physinfo['nr_cpus']
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/tools/python/xen/xend/XendStorageRepository.py	Thu Oct 12 18:01:32 2006 +0100
     2.3 @@ -0,0 +1,327 @@
     2.4 +#!/usr/bin/python
     2.5 +#============================================================================
     2.6 +# This library is free software; you can redistribute it and/or
     2.7 +# modify it under the terms of version 2.1 of the GNU Lesser General Public
     2.8 +# License as published by the Free Software Foundation.
     2.9 +#
    2.10 +# This library is distributed in the hope that it will be useful,
    2.11 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
    2.12 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    2.13 +# Lesser General Public License for more details.
    2.14 +#
    2.15 +# You should have received a copy of the GNU Lesser General Public
    2.16 +# License along with this library; if not, write to the Free Software
    2.17 +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    2.18 +#============================================================================
    2.19 +# Copyright (C) 2006 XenSource Ltd.
    2.20 +#============================================================================
    2.21 +#
    2.22 +# The default QCOW Xen API Storage Repository
    2.23 +#
    2.24 +
    2.25 +import os
    2.26 +import commands
    2.27 +import threading
    2.28 +
    2.29 +from xen.xend import uuid
    2.30 +from xen.xend.XendError import XendError
    2.31 +from xen.xend.XendVDI import *
    2.32 +
    2.33 +XEND_STORAGE_MAX_IGNORE = -1
    2.34 +XEND_STORAGE_DIR = "/var/lib/xend/storage/"
    2.35 +XEND_STORAGE_QCOW_FILENAME = "%s.qcow"
    2.36 +XEND_STORAGE_IMG_FILENAME = "%s.img"
    2.37 +DF_COMMAND = "df -kl"
    2.38 +QCOW_CREATE_COMMAND = "/usr/sbin/qcow-create %d %s %s"
    2.39 +
    2.40 +KB = 1024
    2.41 +MB = 1024 *1024
    2.42 +
    2.43 +class DeviceInvalidError(Exception):
    2.44 +    pass
    2.45 +
    2.46 +class XendStorageRepository:
    2.47 +    """A simple file backed QCOW Storage Repository.
    2.48 +
    2.49 +    This class exposes the interface to create VDI's via the
    2.50 +    Xen API. The backend is a file-backed QCOW format that is stored
    2.51 +    in XEND_STORAGE_DIR or any that is specified in the constructor.
    2.52 +
    2.53 +    The actual images are created in the format <uuid>.img and <uuid>.qcow.
    2.54 +    """
    2.55 +    
    2.56 +    def __init__(self, storage_dir = XEND_STORAGE_DIR,
    2.57 +                 storage_max = XEND_STORAGE_MAX_IGNORE):
    2.58 +        """
    2.59 +        @keyword storage_dir: Where the images will be stored.
    2.60 +        @type    storage_dir: string
    2.61 +        @keyword storage_max: Maximum disk space to use in KB.
    2.62 +        @type    storage_max: int
    2.63 +
    2.64 +        @ivar    storage_free: storage space free for this repository
    2.65 +        @ivar    images: mapping of all the images.
    2.66 +        @type    images: dictionary by image uuid.
    2.67 +        @ivar    lock:   lock to provide thread safety.
    2.68 +        """
    2.69 +        
    2.70 +        self.storage_dir = storage_dir
    2.71 +        self.storage_max = storage_max
    2.72 +        self.storage_free = 0
    2.73 +        self.images = {}
    2.74 +
    2.75 +        # XenAPI Parameters
    2.76 +        self.uuid = self._sr_uuid()
    2.77 +        self.type = "qcow-file"
    2.78 +        self.location = self.storage_dir
    2.79 +        self.name_label = "Local"
    2.80 +        self.name_description = "Xend Storage Repository"
    2.81 +
    2.82 +        self.lock = threading.RLock()
    2.83 +        self._refresh()        
    2.84 +
    2.85 +    def _sr_uuid(self):
    2.86 +        uuid_file = os.path.join(XEND_STORAGE_DIR, 'uuid')
    2.87 +        try:
    2.88 +            if os.path.exists(uuid_file):
    2.89 +                return open(uuid_file, 'r').read().strip()
    2.90 +            else:
    2.91 +                new_uuid = uuid.createString()
    2.92 +                open(uuid_file, 'w').write(new_uuid + '\n')
    2.93 +                return new_uuid
    2.94 +        except IOError:
    2.95 +            # TODO: log warning
    2.96 +            pass
    2.97 +
    2.98 +        return uuid.createString()
    2.99 +
   2.100 +    def _refresh(self):
   2.101 +        """Internal function that refreshes the state of the disk and
   2.102 +        updates the list of images available.
   2.103 +        """
   2.104 +        self.lock.acquire()
   2.105 +        try:
   2.106 +            if not os.path.exists(XEND_STORAGE_DIR):
   2.107 +                os.makedirs(XEND_STORAGE_DIR)
   2.108 +                os.chmod(XEND_STORAGE_DIR, 0700)
   2.109 +
   2.110 +            # scan the directory and populate self.images
   2.111 +            total_used = 0
   2.112 +            seen_images = []
   2.113 +            for filename in os.listdir(XEND_STORAGE_DIR):
   2.114 +                if filename[-5:] == XEND_STORAGE_QCOW_FILENAME[-5:]:
   2.115 +                    image_uuid = filename[:-5]
   2.116 +                    seen_images.append(image_uuid)
   2.117 +                    if image_uuid not in self.images:
   2.118 +                        image_file = XEND_STORAGE_IMG_FILENAME % image_uuid
   2.119 +                        qcow_file = XEND_STORAGE_QCOW_FILENAME % image_uuid
   2.120 +                        image_path = os.path.join(XEND_STORAGE_DIR,
   2.121 +                                                  image_file)
   2.122 +                        qcow_path = os.path.join(XEND_STORAGE_DIR, qcow_file)
   2.123 +                        image_size_kb = (os.stat(image_path).st_size)/1024
   2.124 +
   2.125 +                        vdi = XendQCOWVDI(image_uuid, self.uuid,
   2.126 +                                          qcow_path, image_path,
   2.127 +                                          image_size_kb, image_size_kb)
   2.128 +                        self.images[image_uuid] = vdi
   2.129 +                        total_used += image_size_kb
   2.130 +
   2.131 +            # remove images that aren't valid
   2.132 +            for image_uuid in self.images.keys():
   2.133 +                if image_uuid not in seen_images:
   2.134 +                    try:
   2.135 +                        os.unlink(self.images[image_uuid].qcow_path)
   2.136 +                        os.unlink(self.images[image_uuid].image_path)
   2.137 +                    except OSError:
   2.138 +                        pass
   2.139 +                    del self.images[image_uuid]
   2.140 +
   2.141 +            # update free storage if we have to track that
   2.142 +            if self.storage_max != XEND_STORAGE_MAX_IGNORE:
   2.143 +                self.storage_free = self.storage_max - total_used
   2.144 +            else:
   2.145 +                self.storage_free = self._get_free_space()
   2.146 +                        
   2.147 +        finally:
   2.148 +            self.lock.release()
   2.149 +
   2.150 +    def _get_df(self):
   2.151 +        """Returns the output of 'df' in a dictionary where the keys
   2.152 +        are the Linux device numbers, and the values are it's corresponding
   2.153 +        free space in KB.
   2.154 +
   2.155 +        @rtype: dictionary
   2.156 +        """
   2.157 +        df = commands.getoutput(DF_COMMAND)
   2.158 +        devnum_free = {}
   2.159 +        for line in df.split('\n')[1:]:
   2.160 +            words = line.split()
   2.161 +            mount_point = words[-1]
   2.162 +            dev_no = os.stat(mount_point).st_dev
   2.163 +            free_blks = int(words[3])
   2.164 +            devnum_free[dev_no] = free_blks
   2.165 +        return devnum_free
   2.166 +
   2.167 +    def _get_free_space(self):
   2.168 +        """Returns the amount of free space in KB available in the storage
   2.169 +        partition. Note that this may not be used if the storage repository
   2.170 +        is initialised with a maximum size in storage_max.
   2.171 +
   2.172 +        @rtype: int
   2.173 +        """
   2.174 +        df = self._get_df()
   2.175 +        devnum = os.stat(self.storage_dir).st_dev
   2.176 +        if df.has_key(devnum):
   2.177 +            return df[devnum]
   2.178 +        raise DeviceInvalidError("Device not found for storage path: %s" %
   2.179 +                                 self.storage_dir)
   2.180 +
   2.181 +    def _has_space_available_for(self, size_kb):
   2.182 +        """Returns whether there is enough space for an image in the
   2.183 +        partition which the storage_dir resides on.
   2.184 +
   2.185 +        @rtype: bool
   2.186 +        """
   2.187 +        if self.storage_max != -1:
   2.188 +            return self.storage_free
   2.189 +        
   2.190 +        kb_free = self._get_free_space()
   2.191 +        try:
   2.192 +            if size_kb < kb_free:
   2.193 +                return True
   2.194 +        except DeviceInvalidError:
   2.195 +            pass
   2.196 +        return False
   2.197 +
   2.198 +    def create_image(self, desired_size_kb):
   2.199 +        """Create an image and return its assigned UUID.
   2.200 +
   2.201 +        @param desired_size_kb: Desired image size in KB.
   2.202 +        @type  desired_size_kb: int
   2.203 +        @rtype: string
   2.204 +        @return: uuid
   2.205 +
   2.206 +        @raises XendError: If an error occurs.
   2.207 +        """
   2.208 +        self.lock.acquire()
   2.209 +        try:
   2.210 +            if not self._has_space_available_for(desired_size_kb):
   2.211 +                raise XendError("Not enough space")
   2.212 +
   2.213 +            image_uuid = uuid.createString()
   2.214 +            # create file based image
   2.215 +            image_path = os.path.join(XEND_STORAGE_DIR,
   2.216 +                                      XEND_STORAGE_IMG_FILENAME % image_uuid)
   2.217 +            block = '\x00' * 1024
   2.218 +            img = open(image_path, 'w')
   2.219 +            for i in range(desired_size_kb):
   2.220 +                img.write(block)
   2.221 +            img.close()
   2.222 +            
   2.223 +            # TODO: create qcow image
   2.224 +            qcow_path = os.path.join(XEND_STORAGE_DIR,
   2.225 +                                     XEND_STORAGE_QCOW_FILENAME % image_uuid)
   2.226 +            cmd = QCOW_CREATE_COMMAND % (desired_size_kb/1024,
   2.227 +                                         qcow_path, image_path)
   2.228 +
   2.229 +            rc, output = commands.getstatusoutput(cmd)
   2.230 +            if rc != 0:
   2.231 +                # cleanup the image file
   2.232 +                os.unlink(image_path)
   2.233 +                raise XendError("Failed to create QCOW Image: %s" % output)
   2.234 +
   2.235 +            self._refresh()
   2.236 +            return image_uuid
   2.237 +        finally:
   2.238 +            self.lock.release()
   2.239 +        
   2.240 +    def destroy_image(self, image_uuid):
   2.241 +        """Destroy an image that is managed by this storage repository.
   2.242 +
   2.243 +        @param image_uuid: Image UUID
   2.244 +        @type  image_uuid: String
   2.245 +        @rtype: String
   2.246 +        """
   2.247 +        self.lock.acquire()
   2.248 +        try:
   2.249 +            if image_uuid in self.images:
   2.250 +                # TODO: check if it is being used?
   2.251 +                qcow_path = self.images[image_uuid].qcow_path
   2.252 +                image_path = self.images[image_uuid].image_path
   2.253 +                try:
   2.254 +                    os.unlink(qcow_path)
   2.255 +                    os.unlink(image_path)
   2.256 +                except OSError:
   2.257 +                    # TODO: log warning
   2.258 +                    pass
   2.259 +                del self.images[image_uuid]
   2.260 +                return True
   2.261 +        finally:
   2.262 +            self.lock.release()
   2.263 +        
   2.264 +        return False
   2.265 +
   2.266 +    def list_images(self):
   2.267 +        """ List all the available images by UUID.
   2.268 +
   2.269 +        @rtype: list of strings.
   2.270 +        @return: list of UUIDs
   2.271 +        """
   2.272 +        self.lock.acquire()
   2.273 +        try:
   2.274 +            return self.images.keys()
   2.275 +        finally:
   2.276 +            self.lock.release()
   2.277 +
   2.278 +    def free_space_kb(self):
   2.279 +        """Returns the amount of available space in KB.
   2.280 +        @rtype: int
   2.281 +        """
   2.282 +        self.lock.acquire()
   2.283 +        try:
   2.284 +            return self.storage_free
   2.285 +        finally:
   2.286 +            self.lock.release()
   2.287 +            
   2.288 +    def total_space_kb(self):
   2.289 +        """Returns the total usable space of the storage repo in KB.
   2.290 +        @rtype: int
   2.291 +        """
   2.292 +        self.lock.acquire()
   2.293 +        try:
   2.294 +            if self.storage_max != XEND_STORAGE_MAX_IGNORE:
   2.295 +                return self.storage_max
   2.296 +            else:
   2.297 +                return self.free_space_kb() + self.used_space_kb()
   2.298 +        finally:
   2.299 +            self.lock.release()
   2.300 +            
   2.301 +    def used_space_kb(self):
   2.302 +        """Returns the total amount of space used by this storage repository.
   2.303 +        @rtype: int
   2.304 +        """
   2.305 +        self.lock.acquire()
   2.306 +        try:
   2.307 +            total_used = 0
   2.308 +            for val in self.images.values():
   2.309 +                total_used += val.get_physical_utilisation()
   2.310 +            return total_used
   2.311 +        finally:
   2.312 +            self.lock.release()
   2.313 +
   2.314 +    def used_space_bytes(self):
   2.315 +        return self.used_space_kb() * KB
   2.316 +    def free_space_bytes(self):
   2.317 +        return self.free_space_kb() * KB
   2.318 +    def total_space_bytes(self):
   2.319 +        return self.total_space_kb() * KB
   2.320 +
   2.321 +
   2.322 +# remove everything below this line!!
   2.323 +if __name__ == "__main__":
   2.324 +    xsr = XendStorageRepository()
   2.325 +    print 'Free Space: %d MB' % (xsr.free_space_kb()/1024)
   2.326 +    print "Create Image:",
   2.327 +    print xsr.create_image(10 * 1024)
   2.328 +    print 'Delete all images:'
   2.329 +    for image_uuid in xsr.list_images():
   2.330 +        xsr.destroy_image(image_uuid)
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/tools/python/xen/xend/XendVDI.py	Thu Oct 12 18:01:32 2006 +0100
     3.3 @@ -0,0 +1,44 @@
     3.4 +#!/usr/bin/python
     3.5 +#============================================================================
     3.6 +# This library is free software; you can redistribute it and/or
     3.7 +# modify it under the terms of version 2.1 of the GNU Lesser General Public
     3.8 +# License as published by the Free Software Foundation.
     3.9 +#
    3.10 +# This library is distributed in the hope that it will be useful,
    3.11 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
    3.12 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    3.13 +# Lesser General Public License for more details.
    3.14 +#
    3.15 +# You should have received a copy of the GNU Lesser General Public
    3.16 +# License along with this library; if not, write to the Free Software
    3.17 +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    3.18 +#============================================================================
    3.19 +# Copyright (C) 2006 XenSource Ltd.
    3.20 +#============================================================================
    3.21 +#
    3.22 +# Representation of a Xen API VDI
    3.23 +#
    3.24 +
    3.25 +KB = 1024
    3.26 +MB = 1024 * 1024
    3.27 +
    3.28 +class XendVDI:
    3.29 +    def __init__(self, uuid, sr_uuid):
    3.30 +        self.uuid = uuid
    3.31 +        self.sr_uuid = sr_uuid
    3.32 +
    3.33 +class XendQCOWVDI(XendVDI):
    3.34 +    vdi_type = "system"
    3.35 +
    3.36 +    def __init__(self, uuid, sr_uuid, qcow_path, image_path, vsize, psize):
    3.37 +        XendVDI.__init__(self, uuid, sr_uuid)
    3.38 +        self.qcow_path = qcow_path
    3.39 +        self.image_path = image_path
    3.40 +        self.vsize = vsize
    3.41 +        self.psize = psize
    3.42 +
    3.43 +    def get_physical_utilisation(self):
    3.44 +        return self.psize * KB
    3.45 +
    3.46 +    def get_virtual_size(self):
    3.47 +        return self.vsize * KB