# HG changeset patch # User David Scott <[email protected]> # Date 1282565147 -3600 # Node ID 23fa063db91d4eae0179fb29179002a85a75cf31 # Parent acfa0e8405cb12be60262655c8f21d97284e1a3b CP-1883: Allow raw VDI import to receive an export-like chunked encoding.
The raw VDI import HTTP handler currently assumes the whole disk is being uploaded at once. Instead we add a 'chunked' mode which allows arbitrary-sized disk blocks to be selectively uploaded. Signed-off-by: David Scott <[email protected]> diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/OMakefile --- a/ocaml/xapi/OMakefile Mon Aug 23 13:03:21 2010 +0100 +++ b/ocaml/xapi/OMakefile Mon Aug 23 13:05:47 2010 +0100 @@ -73,6 +73,7 @@ console \ xen_helpers \ importexport \ + sparse_encoding \ create_storage \ create_networks \ xapi_fist \ diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/import_raw_vdi.ml --- a/ocaml/xapi/import_raw_vdi.ml Mon Aug 23 13:03:21 2010 +0100 +++ b/ocaml/xapi/import_raw_vdi.ml Mon Aug 23 13:05:47 2010 +0100 @@ -20,9 +20,13 @@ open Http open Importexport +open Sparse_encoding open Unixext open Pervasiveext +let receive_chunks (s: Unix.file_descr) (fd: Unix.file_descr) = + Chunk.fold (fun () -> Chunk.write fd) () s + let vdi_of_req ~__context (req: request) = let vdi = if List.mem_assoc "vdi" req.Http.query @@ -37,6 +41,7 @@ Xapi_http.with_context "Importing raw VDI" req s (fun __context -> let vdi = vdi_of_req ~__context req in + let chunked = List.mem_assoc "chunked" req.Http.query in try match req.transfer_encoding, req.content_length with | Some "chunked", _ -> @@ -56,7 +61,9 @@ finally (fun () -> try - Unixext.copy_file ~limit:len s fd; + if chunked + then receive_chunks s fd + else ignore(Unixext.copy_file ~limit:len s fd); Unixext.fsync fd with Unix.Unix_error(Unix.EIO, _, _) -> raise (Api_errors.Server_error (Api_errors.vdi_io_error, ["Device I/O errors"])) diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/importexport.ml --- a/ocaml/xapi/importexport.ml Mon Aug 23 13:03:21 2010 +0100 +++ b/ocaml/xapi/importexport.ml Mon Aug 23 13:05:47 2010 +0100 @@ -212,3 +212,4 @@ Helpers.log_exn_continue "executing cleanup action" (action __context rpc) session_id) x ) ) + diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/sparse_encoding.ml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/ocaml/xapi/sparse_encoding.ml Mon Aug 23 13:05:47 2010 +0100 @@ -0,0 +1,122 @@ +(* + * Copyright (C) 2010 Citrix Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + *) +(** Utility functions for reading and writing disk blocks to/from a network stream. + * @group Import and Export + *) + +module Unmarshal = struct + let int64 (s, offset) = + let (<<) a b = Int64.shift_left a b + and (||) a b = Int64.logor a b in + let a = Int64.of_int (int_of_char (s.[offset + 0])) + and b = Int64.of_int (int_of_char (s.[offset + 1])) + and c = Int64.of_int (int_of_char (s.[offset + 2])) + and d = Int64.of_int (int_of_char (s.[offset + 3])) + and e = Int64.of_int (int_of_char (s.[offset + 4])) + and f = Int64.of_int (int_of_char (s.[offset + 5])) + and g = Int64.of_int (int_of_char (s.[offset + 6])) + and h = Int64.of_int (int_of_char (s.[offset + 7])) in + (a << 0) || (b << 8) || (c << 16) || (d << 24) || (e << 32) || (f << 40) || (g << 48) || (h << 56), + (s, offset + 8) + let int32 (s, offset) = + let (<<) a b = Int32.shift_left a b + and (||) a b = Int32.logor a b in + let a = Int32.of_int (int_of_char (s.[offset + 0])) + and b = Int32.of_int (int_of_char (s.[offset + 1])) + and c = Int32.of_int (int_of_char (s.[offset + 2])) + and d = Int32.of_int (int_of_char (s.[offset + 3])) in + (a << 0) || (b << 8) || (c << 16) || (d << 24), (s, offset + 4) +end + +module Marshal = struct + let int64 x = + let (>>) a b = Int64.shift_right_logical a b + and (&&) a b = Int64.logand a b in + let a = (x >> 0) && 0xffL + and b = (x >> 8) && 0xffL + and c = (x >> 16) && 0xffL + and d = (x >> 24) && 0xffL + and e = (x >> 32) && 0xffL + and f = (x >> 40) && 0xffL + and g = (x >> 48) && 0xffL + and h = (x >> 56) && 0xffL in + let result = String.make 8 '\000' in + result.[0] <- char_of_int (Int64.to_int a); + result.[1] <- char_of_int (Int64.to_int b); + result.[2] <- char_of_int (Int64.to_int c); + result.[3] <- char_of_int (Int64.to_int d); + result.[4] <- char_of_int (Int64.to_int e); + result.[5] <- char_of_int (Int64.to_int f); + result.[6] <- char_of_int (Int64.to_int g); + result.[7] <- char_of_int (Int64.to_int h); + result + let int32 x = + let (>>) a b = Int32.shift_right_logical a b + and (&&) a b = Int32.logand a b in + let a = (x >> 0) && 0xffl + and b = (x >> 8) && 0xffl + and c = (x >> 16) && 0xffl + and d = (x >> 24) && 0xffl in + let result = String.make 4 '\000' in + result.[0] <- char_of_int (Int32.to_int a); + result.[1] <- char_of_int (Int32.to_int b); + result.[2] <- char_of_int (Int32.to_int c); + result.[3] <- char_of_int (Int32.to_int d); + result + +end + +module Chunk = struct + (** Represents an single block of data to write *) + type t = { + start: int64; + data: string; + } + + let really_write fd offset buf off len = + ignore(Unix.LargeFile.lseek fd offset Unix.SEEK_SET); + let n = Unix.write fd buf off len in + if n < len + then failwith "Short write: attempted to write %d bytes at %Ld, only wrote %d" len offset n + + (** Writes a single block of data to the output device *) + let write fd x = really_write fd x.start x.data 0 (String.length x.data) + + (** Reads a type t from a file descriptor *) + let unmarshal fd = + let buf = String.make 12 '\000' in + Unixext.really_read fd buf 0 (String.length buf); + let stream = (buf, 0) in + let start, stream = Unmarshal.int64 stream in + let len, stream = Unmarshal.int32 stream in + let payload = String.make (Int32.to_int len) '\000' in + Unixext.really_read fd payload 0 (String.length payload); + { start = start; data = payload } + + (** Writes a type t from a file descriptor *) + let marshal fd x = + let start' = Marshal.int64 x.start in + let len' = Marshal.int32 (Int32.of_int (String.length x.data)) in + really_write fd 0L start' 0 (String.length start'); + really_write fd 8L len' 0 (String.length len'); + really_write fd 12L x.data 0 (String.length x.data) + + (** Fold [f] across all ts unmarshalled from [fd] *) + let rec fold f init fd = + let x = unmarshal fd in + if x.data = "" + then init + else fold f (f init x) fd +end + ocaml/xapi/OMakefile | 1 + ocaml/xapi/import_raw_vdi.ml | 9 ++- ocaml/xapi/importexport.ml | 1 + ocaml/xapi/sparse_encoding.ml | 122 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 1 deletions(-)
# HG changeset patch # User David Scott <[email protected]> # Date 1282565147 -3600 # Node ID 23fa063db91d4eae0179fb29179002a85a75cf31 # Parent acfa0e8405cb12be60262655c8f21d97284e1a3b CP-1883: Allow raw VDI import to receive an export-like chunked encoding. The raw VDI import HTTP handler currently assumes the whole disk is being uploaded at once. Instead we add a 'chunked' mode which allows arbitrary-sized disk blocks to be selectively uploaded. Signed-off-by: David Scott <[email protected]> diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/OMakefile --- a/ocaml/xapi/OMakefile Mon Aug 23 13:03:21 2010 +0100 +++ b/ocaml/xapi/OMakefile Mon Aug 23 13:05:47 2010 +0100 @@ -73,6 +73,7 @@ console \ xen_helpers \ importexport \ + sparse_encoding \ create_storage \ create_networks \ xapi_fist \ diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/import_raw_vdi.ml --- a/ocaml/xapi/import_raw_vdi.ml Mon Aug 23 13:03:21 2010 +0100 +++ b/ocaml/xapi/import_raw_vdi.ml Mon Aug 23 13:05:47 2010 +0100 @@ -20,9 +20,13 @@ open Http open Importexport +open Sparse_encoding open Unixext open Pervasiveext +let receive_chunks (s: Unix.file_descr) (fd: Unix.file_descr) = + Chunk.fold (fun () -> Chunk.write fd) () s + let vdi_of_req ~__context (req: request) = let vdi = if List.mem_assoc "vdi" req.Http.query @@ -37,6 +41,7 @@ Xapi_http.with_context "Importing raw VDI" req s (fun __context -> let vdi = vdi_of_req ~__context req in + let chunked = List.mem_assoc "chunked" req.Http.query in try match req.transfer_encoding, req.content_length with | Some "chunked", _ -> @@ -56,7 +61,9 @@ finally (fun () -> try - Unixext.copy_file ~limit:len s fd; + if chunked + then receive_chunks s fd + else ignore(Unixext.copy_file ~limit:len s fd); Unixext.fsync fd with Unix.Unix_error(Unix.EIO, _, _) -> raise (Api_errors.Server_error (Api_errors.vdi_io_error, ["Device I/O errors"])) diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/importexport.ml --- a/ocaml/xapi/importexport.ml Mon Aug 23 13:03:21 2010 +0100 +++ b/ocaml/xapi/importexport.ml Mon Aug 23 13:05:47 2010 +0100 @@ -212,3 +212,4 @@ Helpers.log_exn_continue "executing cleanup action" (action __context rpc) session_id) x ) ) + diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/sparse_encoding.ml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/ocaml/xapi/sparse_encoding.ml Mon Aug 23 13:05:47 2010 +0100 @@ -0,0 +1,122 @@ +(* + * Copyright (C) 2010 Citrix Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + *) +(** Utility functions for reading and writing disk blocks to/from a network stream. + * @group Import and Export + *) + +module Unmarshal = struct + let int64 (s, offset) = + let (<<) a b = Int64.shift_left a b + and (||) a b = Int64.logor a b in + let a = Int64.of_int (int_of_char (s.[offset + 0])) + and b = Int64.of_int (int_of_char (s.[offset + 1])) + and c = Int64.of_int (int_of_char (s.[offset + 2])) + and d = Int64.of_int (int_of_char (s.[offset + 3])) + and e = Int64.of_int (int_of_char (s.[offset + 4])) + and f = Int64.of_int (int_of_char (s.[offset + 5])) + and g = Int64.of_int (int_of_char (s.[offset + 6])) + and h = Int64.of_int (int_of_char (s.[offset + 7])) in + (a << 0) || (b << 8) || (c << 16) || (d << 24) || (e << 32) || (f << 40) || (g << 48) || (h << 56), + (s, offset + 8) + let int32 (s, offset) = + let (<<) a b = Int32.shift_left a b + and (||) a b = Int32.logor a b in + let a = Int32.of_int (int_of_char (s.[offset + 0])) + and b = Int32.of_int (int_of_char (s.[offset + 1])) + and c = Int32.of_int (int_of_char (s.[offset + 2])) + and d = Int32.of_int (int_of_char (s.[offset + 3])) in + (a << 0) || (b << 8) || (c << 16) || (d << 24), (s, offset + 4) +end + +module Marshal = struct + let int64 x = + let (>>) a b = Int64.shift_right_logical a b + and (&&) a b = Int64.logand a b in + let a = (x >> 0) && 0xffL + and b = (x >> 8) && 0xffL + and c = (x >> 16) && 0xffL + and d = (x >> 24) && 0xffL + and e = (x >> 32) && 0xffL + and f = (x >> 40) && 0xffL + and g = (x >> 48) && 0xffL + and h = (x >> 56) && 0xffL in + let result = String.make 8 '\000' in + result.[0] <- char_of_int (Int64.to_int a); + result.[1] <- char_of_int (Int64.to_int b); + result.[2] <- char_of_int (Int64.to_int c); + result.[3] <- char_of_int (Int64.to_int d); + result.[4] <- char_of_int (Int64.to_int e); + result.[5] <- char_of_int (Int64.to_int f); + result.[6] <- char_of_int (Int64.to_int g); + result.[7] <- char_of_int (Int64.to_int h); + result + let int32 x = + let (>>) a b = Int32.shift_right_logical a b + and (&&) a b = Int32.logand a b in + let a = (x >> 0) && 0xffl + and b = (x >> 8) && 0xffl + and c = (x >> 16) && 0xffl + and d = (x >> 24) && 0xffl in + let result = String.make 4 '\000' in + result.[0] <- char_of_int (Int32.to_int a); + result.[1] <- char_of_int (Int32.to_int b); + result.[2] <- char_of_int (Int32.to_int c); + result.[3] <- char_of_int (Int32.to_int d); + result + +end + +module Chunk = struct + (** Represents an single block of data to write *) + type t = { + start: int64; + data: string; + } + + let really_write fd offset buf off len = + ignore(Unix.LargeFile.lseek fd offset Unix.SEEK_SET); + let n = Unix.write fd buf off len in + if n < len + then failwith "Short write: attempted to write %d bytes at %Ld, only wrote %d" len offset n + + (** Writes a single block of data to the output device *) + let write fd x = really_write fd x.start x.data 0 (String.length x.data) + + (** Reads a type t from a file descriptor *) + let unmarshal fd = + let buf = String.make 12 '\000' in + Unixext.really_read fd buf 0 (String.length buf); + let stream = (buf, 0) in + let start, stream = Unmarshal.int64 stream in + let len, stream = Unmarshal.int32 stream in + let payload = String.make (Int32.to_int len) '\000' in + Unixext.really_read fd payload 0 (String.length payload); + { start = start; data = payload } + + (** Writes a type t from a file descriptor *) + let marshal fd x = + let start' = Marshal.int64 x.start in + let len' = Marshal.int32 (Int32.of_int (String.length x.data)) in + really_write fd 0L start' 0 (String.length start'); + really_write fd 8L len' 0 (String.length len'); + really_write fd 12L x.data 0 (String.length x.data) + + (** Fold [f] across all ts unmarshalled from [fd] *) + let rec fold f init fd = + let x = unmarshal fd in + if x.data = "" + then init + else fold f (f init x) fd +end +
_______________________________________________ xen-api mailing list [email protected] http://lists.xensource.com/mailman/listinfo/xen-api
