From: Prashanth Mundkur Date: Thu, 14 May 2009 06:33:41 +0000 (-0700) Subject: add substring parsing api to avoid unnecessary string copies X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=d8ee15849ce673d5755830c949f502e62e264adf;p=xenclient%2Ftoolstack.git add substring parsing api to avoid unnecessary string copies --- diff --git a/gen/rpc/gen_rpc.ml b/gen/rpc/gen_rpc.ml index b2dabb3..fbbec08 100644 --- a/gen/rpc/gen_rpc.ml +++ b/gen/rpc/gen_rpc.ml @@ -56,10 +56,10 @@ let parse_file f = let state = ref (Json_parse.init_parse_state ()) in while String.length !input > 0 do match Json_parse.parse !state !input with - | Json_parse.Json_value (v, rem) -> + | Json_parse.Json_value (v, consumed) -> rpc_decls := (!count, v) :: !rpc_decls; incr count; - input := rem; + input := String.sub !input consumed ((String.length !input) - consumed); state := Json_parse.init_parse_state () | Json_parse.Json_parse_incomplete st -> input := ""; diff --git a/libs/http/http.ml b/libs/http/http.ml index 49eef14..5528a10 100644 --- a/libs/http/http.ml +++ b/libs/http/http.ml @@ -440,7 +440,7 @@ module Request_header = struct } type parse_result = - | Result of t * (* remainder *) string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state let get_parse_result state = @@ -452,18 +452,21 @@ module Request_header = struct } | _ -> None - let parse state str = - let len = String.length str in + let parse_substring state str ofs len = let i = ref 0 in - while get_parse_result state = None && !i < len do + let iend = ofs + len in + while get_parse_result state = None && !i < iend do parse_char state str.[!i]; incr i; state.num_bytes_parsed <- state.num_bytes_parsed + 1 done; match get_parse_result state with - | Some v -> Result (v, (String.sub str !i (len - !i))) + | Some v -> Result (v, !i - ofs) | None -> Parse_incomplete state + let parse state str = + parse_substring state str 0 (String.length str) + let serialize buf req = Buffer.add_string buf (string_of_meth req.meth); Buffer.add_string buf " "; @@ -645,7 +648,7 @@ module Response_header = struct } type parse_result = - | Result of t * (* remainder *) string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state let get_parse_result state = @@ -657,18 +660,21 @@ module Response_header = struct } | _ -> None - let parse state str = - let len = String.length str in + let parse_substring state str ofs len = let i = ref 0 in - while get_parse_result state = None && !i < len do + let iend = ofs + len in + while get_parse_result state = None && !i < iend do parse_char state str.[!i]; incr i; state.num_bytes_parsed <- state.num_bytes_parsed + 1 done; match get_parse_result state with - | Some v -> Result (v, (String.sub str !i (len - !i))) + | Some v -> Result (v, !i - ofs) | None -> Parse_incomplete state + let parse state str = + parse_substring state str 0 (String.length str) + let serialize buf resp = if resp.version <> HTTP09 then begin Buffer.add_string buf (string_of_version resp.version); @@ -935,7 +941,7 @@ module Payload = struct } type parse_result = - | Result of t * (* remainder *) string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state let get_parse_result state = @@ -945,18 +951,21 @@ module Payload = struct } | _ -> None - let parse state str = - let len = String.length str in + let parse_substring state str ofs len = let i = ref 0 in - while get_parse_result state = None && !i < len do + let iend = ofs + len in + while get_parse_result state = None && !i < iend do parse_char state str.[!i]; incr i; state.num_bytes_parsed <- Int64.succ state.num_bytes_parsed done; match get_parse_result state with - | Some v -> Result (v, (String.sub str !i (len - !i))) + | Some v -> Result (v, !i - ofs) | None -> Parse_incomplete state + let parse state str = + parse_substring state str 0 (String.length str) + let connection_closed state = if state.content_length = Connection_close then state.cursor <- Done @@ -1026,27 +1035,28 @@ module Request = struct } type parse_result = - | Result of t * (* remainder *) string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state | Error of string - let rec parse state str = + let rec parse_substring state str ofs len = match state.cursor with | In_request_header rs -> - (match Request_header.parse rs str with - | Request_header.Result (v, rem) -> + (match Request_header.parse_substring rs str ofs len with + | Request_header.Result (v, consumed) -> state.s_request <- Some v; state.num_bytes_parsed <- Int64.of_int (Request_header.num_bytes_parsed rs); (match Payload.init_from_request v with | Payload.No_payload -> state.cursor <- Done; - Result ({ request = v; payload = None }, rem) + Result ({ request = v; payload = None }, consumed) | Payload.Error s -> state.cursor <- Done; Error s | Payload.Payload ps -> state.cursor <- In_payload ps; - parse state rem (* recurse on remaining input *) + (* recurse on remaining input *) + parse_substring state str (ofs + consumed) (len - consumed) ) | Request_header.Parse_incomplete rs -> state.cursor <- In_request_header rs; @@ -1054,16 +1064,19 @@ module Request = struct ) | In_payload ps -> (match Payload.parse ps str with - | Payload.Result (p, rem) -> + | Payload.Result (p, consumed) -> state.num_bytes_parsed <- Int64.add state.num_bytes_parsed (Payload.num_bytes_parsed ps); state.cursor <- Done; - Result ({ request = optval state.s_request; payload = Some p }, rem) + Result ({ request = optval state.s_request; payload = Some p }, consumed) | Payload.Parse_incomplete ps -> state.cursor <- In_payload ps; Parse_incomplete state ) | Done -> raise_error (Internal_error "parse called on finished request!") + let parse state str = + parse_substring state str 0 (String.length str) + let connection_closed state = match state.cursor with | In_request_header _ -> () @@ -1122,27 +1135,28 @@ module Response = struct } type parse_result = - | Result of t * (* remainder *) string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state | Error of string - let rec parse state str = + let rec parse_substring state str ofs len = match state.cursor with | In_response_header rs -> (match Response_header.parse rs str with - | Response_header.Result (v, rem) -> + | Response_header.Result (v, consumed) -> state.s_response <- Some v; state.num_bytes_parsed <- Int64.of_int (Response_header.num_bytes_parsed rs); (match Payload.init_from_response v with | Payload.No_payload -> state.cursor <- Done; - Result ({ response = v; payload = None }, rem) + Result ({ response = v; payload = None }, consumed) | Payload.Error s -> state.cursor <- Done; Error s | Payload.Payload ps -> state.cursor <- In_payload ps; - parse state rem (* recurse on remaining input *) + (* recurse on remaining input *) + parse_substring state str (ofs + consumed) (len - consumed) ) | Response_header.Parse_incomplete rs -> state.cursor <- In_response_header rs; @@ -1150,16 +1164,19 @@ module Response = struct ) | In_payload ps -> (match Payload.parse ps str with - | Payload.Result (p, rem) -> + | Payload.Result (p, consumed) -> state.num_bytes_parsed <- Int64.add state.num_bytes_parsed (Payload.num_bytes_parsed ps); state.cursor <- Done; - Result ({ response = optval state.s_response; payload = Some p }, rem) + Result ({ response = optval state.s_response; payload = Some p }, consumed) | Payload.Parse_incomplete ps -> state.cursor <- In_payload ps; Parse_incomplete state ) | Done -> raise_error (Internal_error "parse called on finished response!") + let parse state str = + parse_substring state str 0 (String.length str) + let connection_closed state = match state.cursor with | In_response_header _ -> () diff --git a/libs/http/http.mli b/libs/http/http.mli index 14c3265..4ffdf56 100644 --- a/libs/http/http.mli +++ b/libs/http/http.mli @@ -36,9 +36,10 @@ module Request_header : sig headers : header_fields; } type parse_result = - | Result of t * (* remainder *) string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state val parse : state -> string -> parse_result + val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result type error exception Http_error of error @@ -60,9 +61,10 @@ module Response_header : sig headers : header_fields; } type parse_result = - | Result of t * (* remainder *) string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state val parse : state -> string -> parse_result + val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result type error exception Http_error of error @@ -95,9 +97,10 @@ module Payload : sig trailers : header_fields } type parse_result = - | Result of t * (* remainder *) string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state val parse : state -> string -> parse_result + val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result val get_parse_result : state -> t option val connection_closed : state -> unit @@ -122,10 +125,11 @@ module Request : sig payload : Payload.t option } type parse_result = - | Result of t * string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state | Error of string val parse : state -> string -> parse_result + val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result val connection_closed : state -> unit @@ -147,10 +151,11 @@ module Response : sig payload : Payload.t option } type parse_result = - | Result of t * string + | Result of t * (* number of consumed bytes *) int | Parse_incomplete of state | Error of string val parse : state -> string -> parse_result + val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result val connection_closed : state -> unit diff --git a/libs/json/json_parse.ml b/libs/json/json_parse.ml index 6ed150d..723d2bf 100644 --- a/libs/json/json_parse.ml +++ b/libs/json/json_parse.ml @@ -432,13 +432,13 @@ let rec parse_char s c = type parse_result = - | Json_value of Json.t * (* remainder *) string + | Json_value of Json.t * (* number of consumed bytes *) int | Json_parse_incomplete of parse_state -let call_parser state str = - let len = String.length str in - let i = ref 0 in - while get_parse_result state = None && !i < len do +let parse_substring state str ofs len = + let i = ref ofs in + let iend = ofs + len in + while get_parse_result state = None && !i < iend do parse_char state str.[!i]; (* This is here instead of inside parse_char since parse_char makes (tail-)recursive calls without @@ -449,11 +449,11 @@ let call_parser state str = incr i done; match get_parse_result state with - | Some v -> Json_value (v, (String.sub str !i (len - !i))) + | Some v -> Json_value (v, !i - ofs) | None -> Json_parse_incomplete state let parse state str = - call_parser state str + parse_substring state str 0 (String.length str) (* This is really only required for numbers, since they are only terminated by whitespace, but end-of-file or end-of-connection diff --git a/libs/json/json_parse.mli b/libs/json/json_parse.mli index 1c6eebb..b2b90cb 100644 --- a/libs/json/json_parse.mli +++ b/libs/json/json_parse.mli @@ -18,10 +18,11 @@ type parse_state val init_parse_state: unit -> parse_state type parse_result = - | Json_value of Json.t * (* remainder *) string + | Json_value of Json.t * (* number of consumed bytes *) int | Json_parse_incomplete of parse_state val parse: parse_state -> string -> parse_result +val parse_substring: parse_state -> string -> (* offset *) int -> (* length *) int -> parse_result val finish_parse: parse_state -> Json.t option diff --git a/libs/json/parser_tests/test_parser.ml b/libs/json/parser_tests/test_parser.ml index bdda612..f84b02f 100644 --- a/libs/json/parser_tests/test_parser.ml +++ b/libs/json/parser_tests/test_parser.ml @@ -41,10 +41,10 @@ let parse_file f = let state = ref (Json_parse.init_parse_state ()) in while String.length !input > 0 do match Json_parse.parse !state !input with - | Json_parse.Json_value (v, rem) -> + | Json_parse.Json_value (v, consumed) -> if !do_print then Printf.printf "%s\n" (Json.to_string v); - input := rem; + input := String.sub !input consumed ((String.length !input) - consumed); state := Json_parse.init_parse_state () | Json_parse.Json_parse_incomplete st -> input := "";