add substring parsing api to avoid unnecessary string copies

diff --git a/gen/rpc/gen_rpc.ml b/gen/rpc/gen_rpc.ml
index b2dabb3301463497f8ca77cb8f04c07042d25bd7..fbbec087d7b53905c10831210d4a59eb5b45ddfe 100644 (file)
--- a/gen/rpc/gen_rpc.ml
+++ b/gen/rpc/gen_rpc.ml
@@ -56,10 +56,10 @@ let parse_file f =
        let state = ref (Json_parse.init_parse_state ()) in
        while String.length !input > 0 do
                match Json_parse.parse !state !input with
-               | Json_parse.Json_value (v, rem) ->
+               | Json_parse.Json_value (v, consumed) ->
                        rpc_decls := (!count, v) :: !rpc_decls;
                        incr count;
-                       input := rem;
+                       input := String.sub !input consumed ((String.length !input) - consumed);
                        state := Json_parse.init_parse_state ()
                | Json_parse.Json_parse_incomplete st ->
                        input := "";

diff --git a/libs/http/http.ml b/libs/http/http.ml
index 49eef1488544d3c1509e89636ea6c03dd3d89a35..5528a10b9e583bf07aa71236112fa97b53e705ec 100644 (file)
--- a/libs/http/http.ml
+++ b/libs/http/http.ml
@@ -440,7 +440,7 @@ module Request_header = struct
        }
 
        type parse_result =
-               | Result of t * (* remainder *) string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
 
        let get_parse_result state =
@@ -452,18 +452,21 @@ module Request_header = struct
                               }
                | _ -> None
 
-       let parse state str =
-               let len = String.length str in
+       let parse_substring state str ofs len =
                let i = ref 0 in
-               while get_parse_result state = None && !i < len do
+               let iend = ofs + len in
+               while get_parse_result state = None && !i < iend do
                        parse_char state str.[!i];
                        incr i;
                        state.num_bytes_parsed <- state.num_bytes_parsed + 1
                done;
                match get_parse_result state with
-               | Some v -> Result (v, (String.sub str !i (len - !i)))
+               | Some v -> Result (v, !i - ofs)
                | None -> Parse_incomplete state
 
+       let parse state str =
+               parse_substring state str 0 (String.length str)
+
        let serialize buf req =
                Buffer.add_string buf (string_of_meth req.meth);
                Buffer.add_string buf " ";
@@ -645,7 +648,7 @@ module Response_header = struct
        }
 
        type parse_result =
-               | Result of t * (* remainder *) string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
 
        let get_parse_result state =
@@ -657,18 +660,21 @@ module Response_header = struct
                               }
                | _    -> None
 
-       let parse state str =
-               let len = String.length str in
+       let parse_substring state str ofs len =
                let i = ref 0 in
-               while get_parse_result state = None && !i < len do
+               let iend = ofs + len in
+               while get_parse_result state = None && !i < iend do
                        parse_char state str.[!i];
                        incr i;
                        state.num_bytes_parsed <- state.num_bytes_parsed + 1
                done;
                match get_parse_result state with
-               | Some v -> Result (v, (String.sub str !i (len - !i)))
+               | Some v -> Result (v, !i - ofs)
                | None -> Parse_incomplete state
 
+       let parse state str =
+               parse_substring state str 0 (String.length str)
+
        let serialize buf resp =
                if resp.version <> HTTP09 then begin
                        Buffer.add_string buf (string_of_version resp.version);
@@ -935,7 +941,7 @@ module Payload = struct
        }
 
        type parse_result =
-               | Result of t * (* remainder *) string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
 
        let get_parse_result state =
@@ -945,18 +951,21 @@ module Payload = struct
                               }
                | _    -> None
 
-       let parse state str =
-               let len = String.length str in
+       let parse_substring state str ofs len =
                let i = ref 0 in
-               while get_parse_result state = None && !i < len do
+               let iend = ofs + len in
+               while get_parse_result state = None && !i < iend do
                        parse_char state str.[!i];
                        incr i;
                        state.num_bytes_parsed <- Int64.succ state.num_bytes_parsed
                done;
                match get_parse_result state with
-               | Some v -> Result (v, (String.sub str !i (len - !i)))
+               | Some v -> Result (v, !i - ofs)
                | None -> Parse_incomplete state
 
+       let parse state str =
+               parse_substring state str 0 (String.length str)
+
        let connection_closed state =
                if state.content_length = Connection_close then
                        state.cursor <- Done
@@ -1026,27 +1035,28 @@ module Request = struct
        }
 
        type parse_result =
-               | Result of t * (* remainder *) string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
                | Error of string
 
-       let rec parse state str =
+       let rec parse_substring state str ofs len =
                match state.cursor with
                | In_request_header rs ->
-                       (match Request_header.parse rs str with
-                        | Request_header.Result (v, rem) ->
+                       (match Request_header.parse_substring rs str ofs len with
+                        | Request_header.Result (v, consumed) ->
                                state.s_request <- Some v;
                                state.num_bytes_parsed <- Int64.of_int (Request_header.num_bytes_parsed rs);
                                (match Payload.init_from_request v with
                                 | Payload.No_payload ->
                                        state.cursor <- Done;
-                                       Result ({ request = v; payload = None }, rem)
+                                       Result ({ request = v; payload = None }, consumed)
                                 | Payload.Error s ->
                                        state.cursor <- Done;
                                        Error s
                                 | Payload.Payload ps ->
                                        state.cursor <- In_payload ps;
-                                       parse state rem (* recurse on remaining input *)
+                                       (* recurse on remaining input *)
+                                       parse_substring state str (ofs + consumed) (len - consumed)
                                )
                         | Request_header.Parse_incomplete rs ->
                                state.cursor <- In_request_header rs;
@@ -1054,16 +1064,19 @@ module Request = struct
                        )
                | In_payload ps ->
                        (match Payload.parse ps str with
-                        | Payload.Result (p, rem) ->
+                        | Payload.Result (p, consumed) ->
                                state.num_bytes_parsed <- Int64.add state.num_bytes_parsed (Payload.num_bytes_parsed ps);
                                state.cursor <- Done;
-                               Result ({ request = optval state.s_request; payload = Some p }, rem)
+                               Result ({ request = optval state.s_request; payload = Some p }, consumed)
                         | Payload.Parse_incomplete ps ->
                                state.cursor <- In_payload ps;
                                Parse_incomplete state
                        )
                | Done -> raise_error (Internal_error "parse called on finished request!")
 
+       let parse state str =
+               parse_substring state str 0 (String.length str)
+
        let connection_closed state =
                match state.cursor with
                | In_request_header _ -> ()
@@ -1122,27 +1135,28 @@ module Response = struct
        }
 
        type parse_result =
-               | Result of t * (* remainder *) string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
                | Error of string
 
-       let rec parse state str =
+       let rec parse_substring state str ofs len =
                match state.cursor with
                | In_response_header rs ->
                        (match Response_header.parse rs str with
-                        | Response_header.Result (v, rem) ->
+                        | Response_header.Result (v, consumed) ->
                                state.s_response <- Some v;
                                state.num_bytes_parsed <- Int64.of_int (Response_header.num_bytes_parsed rs);
                                (match Payload.init_from_response v with
                                 | Payload.No_payload ->
                                        state.cursor <- Done;
-                                       Result ({ response = v; payload = None }, rem)
+                                       Result ({ response = v; payload = None }, consumed)
                                 | Payload.Error s ->
                                        state.cursor <- Done;
                                        Error s
                                 | Payload.Payload ps ->
                                        state.cursor <- In_payload ps;
-                                       parse state rem (* recurse on remaining input *)
+                                       (* recurse on remaining input *)
+                                       parse_substring state str (ofs + consumed) (len - consumed)
                                )
                         | Response_header.Parse_incomplete rs ->
                                state.cursor <- In_response_header rs;
@@ -1150,16 +1164,19 @@ module Response = struct
                        )
                | In_payload ps ->
                        (match Payload.parse ps str with
-                        | Payload.Result (p, rem) ->
+                        | Payload.Result (p, consumed) ->
                                state.num_bytes_parsed <- Int64.add state.num_bytes_parsed (Payload.num_bytes_parsed ps);
                                state.cursor <- Done;
-                               Result ({ response = optval state.s_response; payload = Some p }, rem)
+                               Result ({ response = optval state.s_response; payload = Some p }, consumed)
                         | Payload.Parse_incomplete ps ->
                                state.cursor <- In_payload ps;
                                Parse_incomplete state
                        )
                | Done -> raise_error (Internal_error "parse called on finished response!")
 
+       let parse state str =
+               parse_substring state str 0 (String.length str)
+
        let connection_closed state =
                match state.cursor with
                | In_response_header _ -> ()

diff --git a/libs/http/http.mli b/libs/http/http.mli
index 14c32655998ea7e6f4404f9223016fc8500dffeb..4ffdf564e0ab812af8a7977351d607a03f8090a3 100644 (file)
--- a/libs/http/http.mli
+++ b/libs/http/http.mli
@@ -36,9 +36,10 @@ module Request_header : sig
                headers : header_fields;
        }
        type parse_result =
-               | Result of t * (* remainder *) string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
        val parse : state -> string -> parse_result
+       val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result
 
        type error
        exception Http_error of error
@@ -60,9 +61,10 @@ module Response_header : sig
                headers : header_fields;
        }
        type parse_result =
-               | Result of t * (* remainder *) string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
        val parse : state -> string -> parse_result
+       val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result
 
        type error
        exception Http_error of error
@@ -95,9 +97,10 @@ module Payload : sig
                trailers : header_fields
        }
        type parse_result =
-               | Result of t * (* remainder *) string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
        val parse : state -> string -> parse_result
+       val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result
        val get_parse_result : state -> t option
 
        val connection_closed : state -> unit
@@ -122,10 +125,11 @@ module Request : sig
                payload : Payload.t option
        }
        type parse_result =
-               | Result of t * string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
                | Error of string
        val parse : state -> string -> parse_result
+       val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result
 
        val connection_closed : state -> unit
 
@@ -147,10 +151,11 @@ module Response : sig
                payload : Payload.t option
        }
        type parse_result =
-               | Result of t * string
+               | Result of t * (* number of consumed bytes *) int
                | Parse_incomplete of state
                | Error of string
        val parse : state -> string -> parse_result
+       val parse_substring : state -> string -> (* offset *) int -> (* len *) int -> parse_result
 
        val connection_closed : state -> unit
 

diff --git a/libs/json/json_parse.ml b/libs/json/json_parse.ml
index 6ed150de6b17beaa947be3d83cccf5c5537cb962..723d2bf068fdf4728de67708b42b168180a4396f 100644 (file)
--- a/libs/json/json_parse.ml
+++ b/libs/json/json_parse.ml
@@ -432,13 +432,13 @@ let rec parse_char s c =
 
 
 type parse_result =
-  | Json_value of Json.t * (* remainder *) string
+  | Json_value of Json.t * (* number of consumed bytes *) int
   | Json_parse_incomplete of parse_state
 
-let call_parser state str =
-       let len = String.length str in
-       let i = ref 0 in
-       while get_parse_result state = None && !i < len do
+let parse_substring state str ofs len =
+       let i = ref ofs in
+       let iend = ofs + len in
+       while get_parse_result state = None && !i < iend do
                parse_char state str.[!i];
                (* This is here instead of inside parse_char since
                   parse_char makes (tail-)recursive calls without
@@ -449,11 +449,11 @@ let call_parser state str =
                incr i
        done;
        match get_parse_result state with
-       | Some v -> Json_value (v, (String.sub str !i (len - !i)))
+       | Some v -> Json_value (v, !i - ofs)
        | None -> Json_parse_incomplete state
 
 let parse state str =
-       call_parser state str
+       parse_substring state str 0 (String.length str)
 
 (* This is really only required for numbers, since they are only
    terminated by whitespace, but end-of-file or end-of-connection

diff --git a/libs/json/json_parse.mli b/libs/json/json_parse.mli
index 1c6eebbf68de330a146e6d02a027fd1f1816519a..b2b90cbb9ca51f09ba0caebae51ce153ccfde969 100644 (file)
--- a/libs/json/json_parse.mli
+++ b/libs/json/json_parse.mli
@@ -18,10 +18,11 @@ type parse_state
 val init_parse_state: unit -> parse_state
 
 type parse_result =
-  | Json_value of Json.t * (* remainder *) string
+  | Json_value of Json.t * (* number of consumed bytes *) int
   | Json_parse_incomplete of parse_state
 
 val parse: parse_state -> string -> parse_result
+val parse_substring: parse_state -> string -> (* offset *) int -> (* length *) int -> parse_result
 
 val finish_parse: parse_state -> Json.t option
 

diff --git a/libs/json/parser_tests/test_parser.ml b/libs/json/parser_tests/test_parser.ml
index bdda6123102823299657d80db625ea54f8bd5148..f84b02f50649de9c7a52682ce0de9f5638b80a92 100644 (file)
--- a/libs/json/parser_tests/test_parser.ml
+++ b/libs/json/parser_tests/test_parser.ml
@@ -41,10 +41,10 @@ let parse_file f =
        let state = ref (Json_parse.init_parse_state ()) in
        while String.length !input > 0 do
                match Json_parse.parse !state !input with
-               | Json_parse.Json_value (v, rem) ->
+               | Json_parse.Json_value (v, consumed) ->
                        if !do_print then
                                Printf.printf "%s\n" (Json.to_string v);
-                       input := rem;
+                       input := String.sub !input consumed ((String.length !input) - consumed);
                        state := Json_parse.init_parse_state ()
                | Json_parse.Json_parse_incomplete st ->
                        input := "";