From e3c24bf6cbc27cbf83e34c3a5202d3141c990260 Mon Sep 17 00:00:00 2001 From: David Scott Date: Wed, 26 Jan 2011 17:39:06 +0000 Subject: [PATCH] Add the notion of a database schema. Signed-off-by: David Scott --- ocaml/database/database_server_main.ml | 91 +++++++++++++++++++++++++- ocaml/database/database_test.ml | 56 ---------------- ocaml/database/schema.ml | 80 ++++------------------ ocaml/idl/datamodel_schema.ml | 83 +++++++++++++++++++++++ ocaml/xapi/OMakefile | 1 + ocaml/xapi/pool_db_backup.ml | 4 +- ocaml/xapi/redo_log_usage.ml | 2 +- ocaml/xapi/xapi.ml | 2 +- ocaml/xapi/xapi_test_common.ml | 2 +- 9 files changed, 192 insertions(+), 129 deletions(-) create mode 100644 ocaml/idl/datamodel_schema.ml diff --git a/ocaml/database/database_server_main.ml b/ocaml/database/database_server_main.ml index 10d4007f..c9776e95 100644 --- a/ocaml/database/database_server_main.ml +++ b/ocaml/database/database_server_main.ml @@ -32,6 +32,95 @@ let remote_database_access_handler_v2 req bio = module Local_tests = Database_test.Tests(Db_cache_impl) +let schema = + let _ref = { + Schema.Column.name = Db_names.ref; + persistent = true; + empty = ""; + default = None; + issetref = false; + } in + let uuid = { + Schema.Column.name = Db_names.uuid; + persistent = true; + empty = ""; + default = None; + issetref = false; + } in + let name_label = { + Schema.Column.name = Db_names.name_label; + persistent = true; + empty = ""; + default = None; + issetref = false; + } in + let name_description = { + Schema.Column.name = "name__description"; + persistent = true; + empty = ""; + default = None; + issetref = false; + } in + let vbds = { + Schema.Column.name = "VBDs"; + persistent = false; + empty = "()"; + default = Some("()"); + issetref = true; + } in + let other_config = { + Schema.Column.name = "other_config"; + persistent = false; + empty = "()"; + default = Some("()"); + issetref = false; + } in + let pp = { + Schema.Column.name = "protection_policy"; + persistent = true; + empty = ""; + default = Some("OpaqueRef:NULL"); + issetref = false; + } in + let tags = { + Schema.Column.name = "tags"; + persistent = true; + empty = ""; + default = Some("()"); + issetref = false; + } in + let vm = { + Schema.Column.name = "VM"; + persistent = true; + empty = ""; + default = None; + issetref = false; + } in + + let vm_table = { + Schema.Table.name = "VM"; + columns = [ _ref; uuid; name_label; vbds; pp; name_description; tags; other_config ]; + persistent = true; + } in + let vbd_table = { + Schema.Table.name = "VBD"; + columns = [ _ref; uuid; vm ]; + persistent = true; + } in + let database = { + Schema.Database.tables = [ vm_table; vbd_table ]; + } in + let one_to_many = Schema.StringMap.add "VBD" [ "VM", "VM", "VBDs" ] (Schema.StringMap.empty) in + { + + Schema.major_vsn = 1; + minor_vsn = 1; + database = database; + (** indexed by table name, a list of (this field, foreign table, foreign field) *) + one_to_many = one_to_many; + many_to_many = Schema.StringMap.empty; + } + let _ = let listen_path = ref "./database" in let self_test = ref false in @@ -55,7 +144,7 @@ let _ = let db = Parse_db_conf.make db_filename in Db_conn_store.initialise_db_connections [ db ]; let t = Db_backend.make () in - Db_cache_impl.make t [ db ] (Schema.of_datamodel ()); + Db_cache_impl.make t [ db ] schema; Db_cache_impl.sync [ db ] (Db_ref.get_database t); Unixext.unlink_safe !listen_path; diff --git a/ocaml/database/database_test.ml b/ocaml/database/database_test.ml index d030602a..0e07d4e1 100644 --- a/ocaml/database/database_test.ml +++ b/ocaml/database/database_test.ml @@ -20,67 +20,11 @@ module Tests = functor(Client: Db_interface.DB_ACCESS) -> struct let make_vm r uuid = [ -(* "ref", r; *) "uuid", uuid; - "memory__static_max", "0"; - "memory__overhead", "0"; - "PV__ramdisk", ""; - "is_control_domain", "false"; - "actions__after_crash", "restart"; - "resident_on", "OpaqueRef:NULL"; - "snapshot_info", "()"; - "PCI_bus", ""; - "PV__args", ""; - "last_boot_CPU_flags", "()"; - "memory__target", "536870912"; - "is_a_template", "true"; - "user_version", "1"; - "HVM__shadow_multiplier", "1"; - "affinity", "OpaqueRef:NULL"; "name__description", ""; - "PV__legacy_args", ""; - "parent", "OpaqueRef:NULL"; - "snapshot_metadata", ""; - "memory__dynamic_max", "0"; - "ha_always_run", "false"; "other_config", "()"; - "PV__bootloader_args" ,""; - "VCPUs__at_startup", "1"; - "bios_strings", "()"; - "actions__after_shutdown", "destroy"; - "blocked_operations", "()"; "tags", "()"; - "PV__kernel", ""; "name__label", name; - "is_a_snapshot", "false"; - "VCPUs__params", "()"; - "VCPUs__max", "1"; - "allowed_operations", "()"; -(* "protection_policy", "OpaqueRef:NULL"; *) (* test of default *) - "memory__static_min", "268435456"; - "domid", "-1"; - "power_state", "Halted"; - "HVM__boot_policy", ""; - "ha_restart_priority", ""; - "suspend_VDI", "OpaqueRef:NULL"; - "HVM__boot_params", "()"; - "PV__bootloader", "eliloader"; - "transportable_snapshot_id", ""; - "snapshot_of", "OpaqueRef:NULL"; - "guest_metrics", "OpaqueRef:NULL"; - "platform", "()"; - "scheduled_to_be_resident_on", "OpaqueRef:NULL"; - "is_snapshot_from_vmpp", "false"; - "current_operations", "()"; - "recommendations", ""; - "last_booted_record", ""; - "blobs", "()"; - "domarch", ""; - "memory__dynamic_min", "0"; - "metrics", "OpaqueRef:NULL"; - "actions__after_reboot", "restart"; - "xenstore_data", "()"; - "snapshot_time", "19700101T00:00:00Z" ] let make_vbd vm r uuid = [ diff --git a/ocaml/database/schema.ml b/ocaml/database/schema.ml index 2fb2ee8b..53fab1e8 100644 --- a/ocaml/database/schema.ml +++ b/ocaml/database/schema.ml @@ -1,3 +1,16 @@ +(* + * Copyright (C) 2010-2011 Citrix Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + *) module Column = struct type t = { @@ -85,70 +98,3 @@ let many_to_many tblname schema = StringMap.find tblname schema.many_to_many with Not_found -> [] -(* This code could live higher up the stack *) -let of_datamodel () = - let rec flatten_fields fs acc = - match fs with - [] -> acc - | (Datamodel_types.Field f)::fs -> flatten_fields fs (f::acc) - | (Datamodel_types.Namespace (_,internal_fs))::fs -> flatten_fields fs (flatten_fields internal_fs acc) in - let column obj f = - let issetref = match f.Datamodel_types.ty with - | Datamodel_types.Set (Datamodel_types.Ref _) -> true - | _ -> false in - let is_many_to_many f = - let api = Datamodel.all_api in - let this = obj.Datamodel_types.name, f.Datamodel_types.field_name in - Datamodel_utils.Relations.is_in_relation api this && - (Datamodel_utils.Relations.classify api (this,(Datamodel_utils.Relations.other_end_of api this)) = (`Many, `Many)) in - { - Column.name = Escaping.escape_id f.Datamodel_types.full_name; - (* NB we always regenerate one-to-many Set(Ref _) fields *) - persistent = f.Datamodel_types.field_persist && (is_many_to_many f || not issetref); - empty = Datamodel_values.gen_empty_db_val f.Datamodel_types.ty; - (* NB Set(Ref _) fields aren't allowed to have a default value specified so we hardcode one here *) - default = - if issetref - then Some (SExpr.string_of (SExpr.Node [])) - else Opt.map Datamodel_values.to_db_string f.Datamodel_types.default_value ; - issetref = issetref; - } in - - (* We store the reference in two places for no good reason still: *) - let _ref = { - Column.name = Db_names.ref; - persistent = true; - empty = ""; - default = None; - issetref = false; - } in - - let table obj = { - Table.name = Escaping.escape_obj obj.Datamodel_types.name; - columns = _ref :: (List.map (column obj) (flatten_fields obj.Datamodel_types.contents [])); - persistent = obj.Datamodel_types.persist = Datamodel_types.PersistEverything; - } in - let is_one_to_many x = - match Datamodel_utils.Relations.classify Datamodel.all_api x with - | `One, `Many | `Many, `One -> true - | _ -> false in - let is_many_to_many x = - match Datamodel_utils.Relations.classify Datamodel.all_api x with - | `Many, `Many -> true - | _ -> false in - let add_relation p t (((one_tbl, one_fld), (many_tbl, many_fld)) as r) = - let l = if StringMap.mem one_tbl t then StringMap.find one_tbl t else [] in - if p r - then StringMap.add one_tbl ((one_fld, many_tbl, many_fld) :: l) t - else t in - - let database api = { - Database.tables = List.map table (Dm_api.objects_of_api api) - } in - { - major_vsn = Datamodel.schema_major_vsn; - minor_vsn = Datamodel.schema_minor_vsn; - database = database Datamodel.all_api; - one_to_many = List.fold_left (add_relation is_one_to_many) StringMap.empty (Dm_api.relations_of_api Datamodel.all_api); - many_to_many = List.fold_left (add_relation is_many_to_many) StringMap.empty (Dm_api.relations_of_api Datamodel.all_api); - } diff --git a/ocaml/idl/datamodel_schema.ml b/ocaml/idl/datamodel_schema.ml new file mode 100644 index 00000000..1daaa2bb --- /dev/null +++ b/ocaml/idl/datamodel_schema.ml @@ -0,0 +1,83 @@ +(* + * Copyright (C) 2010-2011 Citrix Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + *) + +open Schema + +(* This code could live higher up the stack *) +let of_datamodel () = + let rec flatten_fields fs acc = + match fs with + [] -> acc + | (Datamodel_types.Field f)::fs -> flatten_fields fs (f::acc) + | (Datamodel_types.Namespace (_,internal_fs))::fs -> flatten_fields fs (flatten_fields internal_fs acc) in + let column obj f = + let issetref = match f.Datamodel_types.ty with + | Datamodel_types.Set (Datamodel_types.Ref _) -> true + | _ -> false in + let is_many_to_many f = + let api = Datamodel.all_api in + let this = obj.Datamodel_types.name, f.Datamodel_types.field_name in + Datamodel_utils.Relations.is_in_relation api this && + (Datamodel_utils.Relations.classify api (this,(Datamodel_utils.Relations.other_end_of api this)) = (`Many, `Many)) in + { + Column.name = Escaping.escape_id f.Datamodel_types.full_name; + (* NB we always regenerate one-to-many Set(Ref _) fields *) + persistent = f.Datamodel_types.field_persist && (is_many_to_many f || not issetref); + empty = Datamodel_values.gen_empty_db_val f.Datamodel_types.ty; + (* NB Set(Ref _) fields aren't allowed to have a default value specified so we hardcode one here *) + default = + if issetref + then Some (SExpr.string_of (SExpr.Node [])) + else Opt.map Datamodel_values.to_db_string f.Datamodel_types.default_value ; + issetref = issetref; + } in + + (* We store the reference in two places for no good reason still: *) + let _ref = { + Column.name = Db_names.ref; + persistent = true; + empty = ""; + default = None; + issetref = false; + } in + + let table obj = { + Table.name = Escaping.escape_obj obj.Datamodel_types.name; + columns = _ref :: (List.map (column obj) (flatten_fields obj.Datamodel_types.contents [])); + persistent = obj.Datamodel_types.persist = Datamodel_types.PersistEverything; + } in + let is_one_to_many x = + match Datamodel_utils.Relations.classify Datamodel.all_api x with + | `One, `Many | `Many, `One -> true + | _ -> false in + let is_many_to_many x = + match Datamodel_utils.Relations.classify Datamodel.all_api x with + | `Many, `Many -> true + | _ -> false in + let add_relation p t (((one_tbl, one_fld), (many_tbl, many_fld)) as r) = + let l = if StringMap.mem one_tbl t then StringMap.find one_tbl t else [] in + if p r + then StringMap.add one_tbl ((one_fld, many_tbl, many_fld) :: l) t + else t in + + let database api = { + Database.tables = List.map table (Dm_api.objects_of_api api) + } in + { + major_vsn = Datamodel.schema_major_vsn; + minor_vsn = Datamodel.schema_minor_vsn; + database = database Datamodel.all_api; + one_to_many = List.fold_left (add_relation is_one_to_many) StringMap.empty (Dm_api.relations_of_api Datamodel.all_api); + many_to_many = List.fold_left (add_relation is_many_to_many) StringMap.empty (Dm_api.relations_of_api Datamodel.all_api); + } diff --git a/ocaml/xapi/OMakefile b/ocaml/xapi/OMakefile index 763128b2..2299c509 100644 --- a/ocaml/xapi/OMakefile +++ b/ocaml/xapi/OMakefile @@ -221,6 +221,7 @@ XAPI_MODULES = $(COMMON) \ xapi_upgrade \ xapi_blob \ xapi_message \ + ../idl/datamodel_schema \ debug_populate \ ds \ xapi_remotecmd \ diff --git a/ocaml/xapi/pool_db_backup.ml b/ocaml/xapi/pool_db_backup.ml index 9e5c5678..f7f12f4e 100644 --- a/ocaml/xapi/pool_db_backup.ml +++ b/ocaml/xapi/pool_db_backup.ml @@ -151,7 +151,7 @@ let prepare_database_for_restore ~old_context ~new_context = (** Restore all of our state from an XML backup. This includes the pool config, token etc *) let restore_from_xml __context dry_run (xml_filename: string) = debug "attempting to restore database from %s" xml_filename; - let db = Db_upgrade.generic_database_upgrade (Db_xml.From.file (Schema.of_datamodel ()) xml_filename) in + let db = Db_upgrade.generic_database_upgrade (Db_xml.From.file (Datamodel_schema.of_datamodel ()) xml_filename) in version_check db; let db_ref = Db_ref.in_memory (ref (ref db)) in @@ -220,7 +220,7 @@ let http_fetch_db ~master_address ~pool_secret = (* no content length since it's streaming *) let _, _ = Xmlrpcclient.http_rpc_fd fd headers "" in let inchan = Unix.in_channel_of_descr fd in (* never read from fd again! *) - let db = Db_xml.From.channel (Schema.of_datamodel ()) inchan in + let db = Db_xml.From.channel (Datamodel_schema.of_datamodel ()) inchan in version_check db; db ) diff --git a/ocaml/xapi/redo_log_usage.ml b/ocaml/xapi/redo_log_usage.ml index 91e2e0ee..b8ab8ef1 100644 --- a/ocaml/xapi/redo_log_usage.ml +++ b/ocaml/xapi/redo_log_usage.ml @@ -45,7 +45,7 @@ let read_from_redo_log staging_path = (* Read from the file into the cache *) let conn = Parse_db_conf.make temp_file in (* ideally, the reading from the file would also respect the latest_response_time *) - let db = Backend_xml.populate (Schema.of_datamodel ()) conn in + let db = Backend_xml.populate (Datamodel_schema.of_datamodel ()) conn in let t = Db_backend.make () in Db_ref.update_database t (fun _ -> db); diff --git a/ocaml/xapi/xapi.ml b/ocaml/xapi/xapi.ml index dfd539c7..a8791804 100644 --- a/ocaml/xapi/xapi.ml +++ b/ocaml/xapi/xapi.ml @@ -68,7 +68,7 @@ open Db_cache_types Also this function depends on being able to call API functions through the external interface. *) let start_database_engine () = - let schema = Schema.of_datamodel () in + let schema = Datamodel_schema.of_datamodel () in (* If the temporary restore file is present then we must populate from that *) let connections = diff --git a/ocaml/xapi/xapi_test_common.ml b/ocaml/xapi/xapi_test_common.ml index fb87a2c9..a785889d 100644 --- a/ocaml/xapi/xapi_test_common.ml +++ b/ocaml/xapi/xapi_test_common.ml @@ -14,7 +14,7 @@ (** Make a simple in-memory database containing a single host and dom0 VM record. *) let make_test_database () = - let db = Db_upgrade.generic_database_upgrade (Db_cache_types.Database.make (Schema.of_datamodel ())) in + let db = Db_upgrade.generic_database_upgrade (Db_cache_types.Database.make (Datamodel_schema.of_datamodel ())) in let db_ref = Db_ref.in_memory (ref (ref db)) in let __context = Context.make ~database:db_ref "upgrade_vm_memory_for_dmc" in -- 2.39.5