aboutsummaryrefslogtreecommitdiff
path: root/lib/http
diff options
context:
space:
mode:
authorShulhan <ms@kilabit.info>2024-01-19 03:39:50 +0700
committerShulhan <ms@kilabit.info>2024-01-24 00:07:18 +0700
commiteb2b4dbdeceb663b1ce29bc08cd5446ecf4c1f07 (patch)
treecfeb884e2890d8f74c516eaad9e0bc7a8bbdce59 /lib/http
parentdbfc1583d77d300f8da8e2d3714fa1592dde09a0 (diff)
downloadpakakeh.go-eb2b4dbdeceb663b1ce29bc08cd5446ecf4c1f07.tar.xz
lib/http: refactoring Range request, limit content served by server
When server receive, GET /big Range: bytes=0- and the requested resources is quite larger, where writing all content of file result in i/o timeout, it is best practice [1][2] if the server write only partial content and let the client continue with the subsequent Range request. In the above case, the server should response with, HTTP/1.1 206 Partial content Content-Range: bytes 0-<limit>/<size> Content-Length: <limit> Where limit is maximum packet that is reasonable [3] for most of the client. In this server we choose 8MB as limit. [1]: https://stackoverflow.com/questions/63614008/how-best-to-respond-to-an-open-http-range-request [2]: https://bugzilla.mozilla.org/show_bug.cgi?id=570755 [3]: https://docs.aws.amazon.com/whitepapers/latest/s3-optimizing-performance-best-practices/use-byte-range-fetches.html
Diffstat (limited to 'lib/http')
-rw-r--r--lib/http/http.go29
-rw-r--r--lib/http/range.go174
-rw-r--r--lib/http/range_example_test.go50
-rw-r--r--lib/http/range_position.go125
-rw-r--r--lib/http/range_position_example_test.go24
-rw-r--r--lib/http/range_position_test.go15
-rw-r--r--lib/http/range_test.go2
-rw-r--r--lib/http/server.go80
-rw-r--r--lib/http/server_test.go181
-rw-r--r--lib/http/testdata/server/range/fail_416_test.txt1
-rw-r--r--lib/http/testdata/server/range/multipart_test.txt4
-rw-r--r--lib/http/testdata/server/range_big_test.txt14
12 files changed, 508 insertions, 191 deletions
diff --git a/lib/http/http.go b/lib/http/http.go
index eb1b8799..d762e720 100644
--- a/lib/http/http.go
+++ b/lib/http/http.go
@@ -156,6 +156,32 @@
//
// {"code":<HTTP_STATUS_CODE>,"message":<err.Error()>}
//
+// # Range request
+//
+// The standard http package provide [http.ServeContent] function that
+// support serving resources with Range request, except that it sometime it
+// has an issue.
+//
+// When server receive,
+//
+// GET /big
+// Range: bytes=0-
+//
+// and the requested resources is quite larger, where writing all content of
+// file result in i/o timeout, it is [best] [practice] if the server
+// write only partial content and let the client continue with the
+// subsequent Range request.
+//
+// In the above case, the server should response with,
+//
+// HTTP/1.1 206 Partial content
+// Content-Range: bytes 0-<limit>/<size>
+// Content-Length: <limit>
+//
+// Where limit is maximum packet that is [reasonable] for most of the
+// client.
+// In this server we choose 8MB as limit, see [DefRangeLimit].
+//
// # Summary
//
// The pseudocode below illustrate how Endpoint, Callback, and
@@ -202,6 +228,9 @@
// "/y" are ambiguous because one is dynamic path using key binding "x" and
// the last one is static path to "y".
//
+// [best]: https://stackoverflow.com/questions/63614008/how-best-to-respond-to-an-open-http-range-request
+// [practice]: https://bugzilla.mozilla.org/show_bug.cgi?id=570755
+// [reasonable]: https://docs.aws.amazon.com/whitepapers/latest/s3-optimizing-performance-best-practices/use-byte-range-fetches.html
// [HTTP Range]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
// [Server-Sent Events]: https://html.spec.whatwg.org/multipage/server-sent-events.html
package http
diff --git a/lib/http/range.go b/lib/http/range.go
index e13cc048..5cdebf30 100644
--- a/lib/http/range.go
+++ b/lib/http/range.go
@@ -11,10 +11,14 @@ import (
libstrings "github.com/shuLhan/share/lib/strings"
)
+// DefRangeLimit limit of content served by server when Range request
+// without end, in example "0-".
+const DefRangeLimit = 8388608
+
// Range define the unit and list of start-end positions for resource.
type Range struct {
unit string
- positions []RangePosition
+ positions []*RangePosition
}
// NewRange create new Range with specified unit.
@@ -29,7 +33,7 @@ func NewRange(unit string) (r *Range) {
return r
}
-// ParseMultipartRange parse multipart/byteranges response body.
+// ParseMultipartRange parse "multipart/byteranges" response body.
// Each Content-Range position and body part in the multipart will be stored
// under RangePosition.
func ParseMultipartRange(body io.Reader, boundary string) (r *Range, err error) {
@@ -65,7 +69,7 @@ func ParseMultipartRange(body io.Reader, boundary string) (r *Range, err error)
return nil, fmt.Errorf(`%s: on ReadAll part: %s`, logp, err)
}
- r.positions = append(r.positions, *pos)
+ r.positions = append(r.positions, pos)
}
return r, nil
}
@@ -102,9 +106,6 @@ func ParseRange(v string) (r Range) {
r.unit = strings.ToLower(tok)
- var (
- start, end int64
- )
par.SetDelimiters(`-,`)
for delim != 0 {
tok, delim = par.ReadNoSpace()
@@ -119,29 +120,36 @@ func ParseRange(v string) (r Range) {
if delim == '-' {
// Probably "-last".
tok, delim = par.ReadNoSpace()
- if delim != 0 && delim != ',' {
+ if delim == '-' {
// Invalid "-start-" or "-start-end".
skipPosition(par, delim)
continue
}
- start, err = strconv.ParseInt(tok, 10, 64)
+ var end int64
+ end, err = strconv.ParseInt(tok, 10, 64)
if err != nil {
- skipPosition(par, delim)
+ continue
+ }
+ if end == 0 {
+ // Invalid range "-0".
continue
}
- r.Add(-1*start, 0)
- skipPosition(par, delim)
+ r.Add(nil, &end)
continue
}
}
- if delim == ',' || delim == 0 {
- // Invalid range "start,..." or "start$".
+ if delim == ',' {
+ // Invalid range "start,".
continue
}
-
- // delim == '-'
+ if delim == 0 {
+ // Invalid range with "start" only.
+ break
+ }
+ // delim is '-'.
+ var start int64
start, err = strconv.ParseInt(tok, 10, 64)
if err != nil {
skipPosition(par, delim)
@@ -155,22 +163,18 @@ func ParseRange(v string) (r Range) {
continue
}
if len(tok) == 0 {
- if start == 0 {
- // Invalid range, "0-" equal to whole body.
- continue
- }
-
- // Range "start-".
- end = 0
+ // Range is "start-".
+ r.Add(&start, nil)
} else {
- // Range "start-end".
+ // Range is "start-end".
+ var end int64
end, err = strconv.ParseInt(tok, 10, 64)
if err != nil {
skipPosition(par, delim)
continue
}
+ r.Add(&start, &end)
}
- r.Add(start, end)
}
return r
@@ -178,7 +182,7 @@ func ParseRange(v string) (r Range) {
// skipPosition Ignore any string until ','.
func skipPosition(par *libstrings.Parser, delim rune) {
- for delim != ',' && delim != 0 {
+ for delim == '-' {
_, delim = par.Read()
}
}
@@ -189,61 +193,89 @@ func skipPosition(par *libstrings.Parser, delim rune) {
// zero.
// For example,
//
-// - [0,0] is valid and equal to first byte (but unusual)
-// - [0,9] is valid and equal to the first 10 bytes.
-// - [10,0] is valid and equal to the bytes from offset 10 until the end.
-// - [-10,0] is valid and equal to the last 10 bytes.
-// - [10,1] or [0,-10] or [-10,10] is not valid position.
+// - [0,+x] is valid, from offset 0 until x+1.
+// - [0,0] is valid and equal to first byte (but unusual).
+// - [+x,+y] is valid iff x <= y.
+// - [+x,-y] is invalid.
+// - [-x,+y] is invalid.
+//
+// The start or end can be nil, but not both.
+// For example,
+//
+// - [nil,+x] is valid, equal to "-x" or the last x bytes.
+// - [nil,0] is invalid.
+// - [nil,-x] is invalid.
+// - [x,nil] is valid, equal to "x-" or from offset x until end of file.
+// - [-x,nil] is invalid.
//
-// The new position will be added and return true if only if it does not
-// overlap with existing list.
-func (r *Range) Add(start, end int64) bool {
- if end != 0 && end < start {
- // [10,1] or [0,-10]
+// The new position will be added and return true iff it does not overlap
+// with existing list.
+func (r *Range) Add(start, end *int64) bool {
+ if start == nil && end == nil {
return false
}
- if start < 0 && end != 0 {
- // [-10,10]
- return false
+ if start == nil {
+ if *end <= 0 {
+ return false
+ }
+ } else if end == nil {
+ if *start < 0 {
+ return false
+ }
+ } else {
+ if *start < 0 || *end < 0 || *end < *start {
+ return false
+ }
}
- var pos RangePosition
- for _, pos = range r.positions {
- if pos.Start < 0 {
- if start < 0 {
- // Given pos:[-10,0], adding another negative
- // start like -20 or -5 will always cause
- // overlap.
- return false
- }
- } else if pos.Start == 0 {
- if start >= 0 && start <= pos.End {
- // pos:[0,+y], start<y.
- return false
- }
- } else {
- if pos.End == 0 {
- // pos:[+x,0] already accept until the end.
- return false
- }
- if start >= 0 && start <= pos.End {
- // pos:[+x,+y], start<y.
- return false
- }
+ var lastpos *RangePosition
+
+ if len(r.positions) == 0 {
+ goto ok
+ }
+
+ lastpos = r.positions[len(r.positions)-1]
+ if lastpos.end == nil {
+ return false
+ }
+ if lastpos.start == nil {
+ if start == nil {
+ // last=[nil,+b] vs. pos=[nil,+y]
+ // The pos will always overlap with previous.
+ return false
+ }
+ if end == nil {
+ // last=[nil,+b] vs. pos=[+x,nil]
+ // The pos will always overlap with previous.
+ return false
}
+ goto ok
+ }
+ if start == nil {
+ // [+a,+b] vs. [nil,+y]
+ goto ok
+ }
+ if end == nil {
+ // [+a,+b] vs. [+x,nil]
+ if *lastpos.end >= *start {
+ return false
+ }
+ }
+ if *lastpos.end >= *start {
+ return false
}
- pos = RangePosition{
- Start: start,
- End: end,
+ok:
+ var pos = &RangePosition{}
+ if start != nil {
+ pos.start = new(int64)
+ *pos.start = *start
}
- if start < 0 {
- pos.Length = start * -1
- } else if start >= 0 && end >= 0 {
- pos.Length = (end - start) + 1
+ if end != nil {
+ pos.end = new(int64)
+ *pos.end = *end
}
r.positions = append(r.positions, pos)
-
return true
}
@@ -253,7 +285,7 @@ func (r *Range) IsEmpty() bool {
}
// Positions return the list of range position.
-func (r *Range) Positions() []RangePosition {
+func (r *Range) Positions() []*RangePosition {
return r.positions
}
@@ -266,7 +298,7 @@ func (r *Range) String() string {
var (
sb strings.Builder
- pos RangePosition
+ pos *RangePosition
x int
)
diff --git a/lib/http/range_example_test.go b/lib/http/range_example_test.go
index 0f5fc54e..3a220175 100644
--- a/lib/http/range_example_test.go
+++ b/lib/http/range_example_test.go
@@ -45,7 +45,7 @@ Part 2
log.Fatal(err)
}
- var pos libhttp.RangePosition
+ var pos *libhttp.RangePosition
for _, pos = range r.Positions() {
fmt.Printf("%s: %s\n", pos.String(), pos.Content())
}
@@ -82,7 +82,6 @@ func ExampleParseRange() {
r = libhttp.ParseRange(`bytes=0-9,10-19,-20`)
fmt.Println(r.String())
- // The 0- is invalid because its equal to whole content.
r = libhttp.ParseRange(`bytes=0-`)
fmt.Println(r.String())
@@ -100,27 +99,38 @@ func ExampleParseRange() {
// bytes=10-20
// bytes=-20
// bytes=0-9,10-19,-20
- //
+ // bytes=0-
// bytes=0-9,10-19,-20
}
+func ptrInt64(v int64) *int64 { return &v }
+
func ExampleRange_Add() {
- var r = libhttp.NewRange(``)
+ var listpos = []struct {
+ start *int64
+ end *int64
+ }{
+ {ptrInt64(0), ptrInt64(9)}, // OK.
+ {ptrInt64(0), ptrInt64(5)}, // Overlap with [0,9].
+ {ptrInt64(9), ptrInt64(19)}, // Overlap with [0,9].
+
+ {ptrInt64(10), ptrInt64(19)}, // OK.
+ {ptrInt64(19), ptrInt64(20)}, // Overlap with [10,19].
+ {ptrInt64(20), ptrInt64(19)}, // End less than start.
- fmt.Println(r.Add(0, 9), r.String()) // OK.
- fmt.Println(r.Add(0, 5), r.String()) // Overlap with [0,9].
- fmt.Println(r.Add(9, 19), r.String()) // Overlap with [0,9].
+ {nil, ptrInt64(10)}, // OK.
+ {nil, ptrInt64(20)}, // Overlap with [nil,10].
- fmt.Println(r.Add(10, 19), r.String()) // OK.
- fmt.Println(r.Add(19, 20), r.String()) // Overlap with [10,19].
- fmt.Println(r.Add(-10, 19), r.String()) // Invalid end.
+ {ptrInt64(20), nil}, // Overlap with [nil,10].
+ {ptrInt64(30), ptrInt64(40)}, // Overlap with [20,nil].
+ {ptrInt64(30), nil}, // Overlap with [20,nil].
+ }
- fmt.Println(r.Add(-10, 0), r.String()) // OK.
- fmt.Println(r.Add(20, 19), r.String()) // Invalid end.
+ var r = libhttp.NewRange(``)
- fmt.Println(r.Add(20, 0), r.String()) // OK.
- fmt.Println(r.Add(-5, 0), r.String()) // Overlap with [-10,0].
- fmt.Println(r.Add(30, 0), r.String()) // Overlap with [20,0].
+ for _, pos := range listpos {
+ fmt.Println(r.Add(pos.start, pos.end), r.String())
+ }
// Output:
// true bytes=0-9
@@ -131,16 +141,16 @@ func ExampleRange_Add() {
// false bytes=0-9,10-19
// true bytes=0-9,10-19,-10
// false bytes=0-9,10-19,-10
- // true bytes=0-9,10-19,-10,20-
- // false bytes=0-9,10-19,-10,20-
- // false bytes=0-9,10-19,-10,20-
+ // false bytes=0-9,10-19,-10
+ // true bytes=0-9,10-19,-10,30-40
+ // false bytes=0-9,10-19,-10,30-40
}
func ExampleRange_Positions() {
var r = libhttp.NewRange(``)
fmt.Println(r.Positions()) // Empty positions.
- r.Add(10, 20)
+ r.Add(ptrInt64(10), ptrInt64(20))
fmt.Println(r.Positions())
// Output:
// []
@@ -152,7 +162,7 @@ func ExampleRange_String() {
fmt.Println(r.String()) // Empty range will return empty string.
- r.Add(0, 9)
+ r.Add(ptrInt64(0), ptrInt64(9))
fmt.Println(r.String())
// Output:
//
diff --git a/lib/http/range_position.go b/lib/http/range_position.go
index 19d9ed08..a6cc8dc0 100644
--- a/lib/http/range_position.go
+++ b/lib/http/range_position.go
@@ -12,21 +12,27 @@ import (
type RangePosition struct {
unit string
- content []byte
+ start *int64
+ end *int64
- Start int64
- End int64
+ // length of resources.
+ // A nil length, or "*", indicated an unknown size.
+ length *int64
- // Length of zero means read until the end.
- Length int64
+ content []byte
}
-// ParseContentRange parse Content-Range value, the following format,
+// ParseContentRange parse the HTTP header "Content-Range" value, as
+// response from server, with the following format,
//
-// unit SP position "/" size
-// SP = " "
-// position = start "-" end / start "-" / "-" last
-// start, end, last, size = 1*DIGIT
+// Content-Range = unit SP valid-range / invalid-range
+// SP = " "
+// valid-range = position "/" size
+// invalid-range = "*" "/" size
+// position = start "-" end
+// size = 1*DIGIT / "*"
+// start = 1*DIGIT
+// end = 1*DIGIT
//
// It will return nil if the v is invalid.
func ParseContentRange(v string) (pos *RangePosition) {
@@ -50,62 +56,70 @@ func ParseContentRange(v string) (pos *RangePosition) {
p.SetDelimiters(`-/`)
tok, delim = p.ReadNoSpace()
- if len(tok) == 0 && delim == '-' {
- // Probably "-last".
- tok, delim = p.ReadNoSpace()
+ if len(tok) == 0 {
+ return nil
+ }
+ if tok == `*` {
if delim != '/' {
return nil
}
-
- pos.Length, err = strconv.ParseInt(tok, 10, 64)
- if err != nil {
+ tok, delim = p.ReadNoSpace()
+ if delim != 0 {
return nil
}
-
- pos.Start = -1 * pos.Length
- } else {
- if delim != '-' || delim == 0 {
- return nil
+ if tok == `*` {
+ // "*/*": invalid range requested with unknown size.
+ pos = &RangePosition{}
+ return pos
}
- pos.Start, err = strconv.ParseInt(tok, 10, 64)
- if err != nil {
- return nil
- }
+ pos = &RangePosition{}
+ goto parselength
+ }
+ if delim != '-' {
+ return nil
+ }
- tok, delim = p.ReadNoSpace()
- if delim != '/' {
- return nil
- }
+ pos = &RangePosition{
+ start: new(int64),
+ end: new(int64),
+ length: new(int64),
+ }
- if len(tok) != 0 {
- // Case of "start-end/size".
- pos.End, err = strconv.ParseInt(tok, 10, 64)
- if err != nil {
- return nil
- }
- pos.Length = (pos.End - pos.Start) + 1
- }
+ *pos.start, err = strconv.ParseInt(tok, 10, 64)
+ if err != nil {
+ return nil
}
- // The size.
tok, delim = p.ReadNoSpace()
- if delim != 0 {
+ if delim != '/' {
+ return nil
+ }
+ *pos.end, err = strconv.ParseInt(tok, 10, 64)
+ if err != nil {
+ return nil
+ }
+ if *pos.end < *pos.start {
return nil
}
- if tok != "*" {
- var size int64
- size, err = strconv.ParseInt(tok, 10, 64)
- if err != nil {
- return nil
- }
- if pos.End == 0 {
- // Case of "start-/size".
- pos.Length = (size - pos.Start)
- }
+ tok, delim = p.ReadNoSpace()
+ if delim != 0 {
+ return nil
+ }
+ if tok == `*` {
+ // "x-y/*"
+ return pos
}
+parselength:
+ *pos.length, err = strconv.ParseInt(tok, 10, 64)
+ if err != nil {
+ return nil
+ }
+ if *pos.length < 0 {
+ return nil
+ }
return pos
}
@@ -126,11 +140,14 @@ func (pos RangePosition) ContentRange(unit string, size int64) (v string) {
}
func (pos RangePosition) String() string {
- if pos.Start < 0 {
- return fmt.Sprintf(`%d`, pos.Start)
+ if pos.start == nil {
+ if pos.end == nil {
+ return `*`
+ }
+ return fmt.Sprintf(`-%d`, *pos.end)
}
- if pos.Start > 0 && pos.End == 0 {
- return fmt.Sprintf(`%d-`, pos.Start)
+ if pos.end == nil {
+ return fmt.Sprintf(`%d-`, *pos.start)
}
- return fmt.Sprintf(`%d-%d`, pos.Start, pos.End)
+ return fmt.Sprintf(`%d-%d`, *pos.start, *pos.end)
}
diff --git a/lib/http/range_position_example_test.go b/lib/http/range_position_example_test.go
index 927a6627..1b333565 100644
--- a/lib/http/range_position_example_test.go
+++ b/lib/http/range_position_example_test.go
@@ -7,35 +7,19 @@ import (
)
func ExampleParseContentRange() {
- fmt.Println(libhttp.ParseContentRange(`bytes 10-/20`)) // OK
+ fmt.Println(libhttp.ParseContentRange(`bytes 10-/20`)) // Invalid, missing end.
fmt.Println(libhttp.ParseContentRange(`bytes 10-19/20`)) // OK
- fmt.Println(libhttp.ParseContentRange(`bytes -10/20`)) // OK
+ fmt.Println(libhttp.ParseContentRange(`bytes -10/20`)) // Invalid, missing start.
fmt.Println(libhttp.ParseContentRange(`10-20/20`)) // Invalid, missing unit.
fmt.Println(libhttp.ParseContentRange(`bytes 10-`)) // Invalid, missing "/size".
fmt.Println(libhttp.ParseContentRange(`bytes -10/x`)) // Invalid, invalid "size".
fmt.Println(libhttp.ParseContentRange(`bytes`)) // Invalid, missing position.
// Output:
- // 10-
+ // <nil>
// 10-19
- // -10
// <nil>
// <nil>
// <nil>
// <nil>
-}
-
-func ExampleRangePosition_ContentRange() {
- var (
- unit = libhttp.AcceptRangesBytes
- pos = libhttp.RangePosition{
- Start: 10,
- End: 20,
- }
- )
-
- fmt.Println(pos.ContentRange(unit, 512))
- fmt.Println(pos.ContentRange(unit, 0))
- // Output:
- // bytes 10-20/512
- // bytes 10-20/*
+ // <nil>
}
diff --git a/lib/http/range_position_test.go b/lib/http/range_position_test.go
index 1147b875..738f2de4 100644
--- a/lib/http/range_position_test.go
+++ b/lib/http/range_position_test.go
@@ -37,3 +37,18 @@ func TestParseContentRange(t *testing.T) {
test.Assert(t, c.v, c.exp, got)
}
}
+
+func ptrInt64(v int64) *int64 { return &v }
+
+func TestRangePositionContentRange(t *testing.T) {
+ var (
+ unit = AcceptRangesBytes
+ pos = RangePosition{
+ start: ptrInt64(10),
+ end: ptrInt64(20),
+ }
+ )
+
+ test.Assert(t, ``, `bytes 10-20/512`, pos.ContentRange(unit, 512))
+ test.Assert(t, ``, `bytes 10-20/*`, pos.ContentRange(unit, 0))
+}
diff --git a/lib/http/range_test.go b/lib/http/range_test.go
index 62da743e..10be878c 100644
--- a/lib/http/range_test.go
+++ b/lib/http/range_test.go
@@ -26,7 +26,7 @@ func TestParseMultipartRange(t *testing.T) {
tdata *test.Data
reader *bytes.Reader
r *Range
- pos RangePosition
+ pos *RangePosition
vbyte []byte
got strings.Builder
)
diff --git a/lib/http/server.go b/lib/http/server.go
index 55674038..a17c736c 100644
--- a/lib/http/server.go
+++ b/lib/http/server.go
@@ -784,7 +784,7 @@ func (srv *Server) handlePut(res http.ResponseWriter, req *http.Request) {
// - 416 StatusRequestedRangeNotSatisfiable, if the request Range start
// position is greater than resource size.
//
-// [HTTP Range]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
+// [HTTP Range]: https://datatracker.ietf.org/doc/html/rfc7233
// [RFC7233 S-3.1]: https://datatracker.ietf.org/doc/html/rfc7233#section-3.1
func HandleRange(res http.ResponseWriter, req *http.Request, bodyReader io.ReadSeeker, contentType string) {
var (
@@ -823,46 +823,76 @@ func handleRange(res http.ResponseWriter, req *http.Request, bodyReader io.ReadS
}
var (
- size int64
- err error
+ size int64
+ nread int64
+ err error
)
+
size, err = bodyReader.Seek(0, io.SeekEnd)
if err != nil {
// An error here assume that the size is unknown ('*').
log.Printf(`%s: seek body size: %s`, logp, err)
+ res.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
+ return
}
var (
- listPos = r.Positions()
- listBody = make([][]byte, 0, len(listPos))
+ header = res.Header()
+ listBody = make([][]byte, 0, len(r.positions))
- pos RangePosition
+ pos *RangePosition
)
- for _, pos = range listPos {
- if pos.Start < 0 {
- _, err = bodyReader.Seek(pos.Start, io.SeekEnd)
- } else {
- _, err = bodyReader.Seek(pos.Start, io.SeekStart)
+ for _, pos = range r.positions {
+ // Refill the position if its nil, for response later,
+ // calculate the number of bytes to read, and move the file
+ // position for read.
+ if pos.start == nil {
+ pos.start = new(int64)
+ if *pos.end > size {
+ *pos.start = 0
+ } else {
+ *pos.start = size - *pos.end
+ }
+ *pos.end = size - 1
+ } else if pos.end == nil {
+ if *pos.start > size {
+ // rfc7233#section-4.4
+ // the first-byte-pos of all of the
+ // byte-range-spec values were greater than
+ // the current length of the selected
+ // representation.
+ pos.start = nil
+ header.Set(HeaderContentRange, pos.ContentRange(r.unit, size))
+ res.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
+ return
+ }
+ pos.end = new(int64)
+ *pos.end = size - 1
}
+
+ _, err = bodyReader.Seek(*pos.start, io.SeekStart)
if err != nil {
- log.Printf(`%s: seek %d: %s`, logp, pos.Start, err)
+ log.Printf(`%s: seek %s: %s`, logp, pos, err)
res.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return
}
+ nread = (*pos.end - *pos.start) + 1
+
+ if nread > DefRangeLimit {
+ nread = DefRangeLimit
+ *pos.end = *pos.start + nread
+ }
+
var (
- body []byte
+ body = make([]byte, nread)
n int
)
- if pos.Length > 0 {
- body = make([]byte, pos.Length)
- } else {
- body = make([]byte, size)
- }
n, err = bodyReader.Read(body)
if n == 0 || err != nil {
- log.Printf(`%s: seek %d, size %d: %s`, logp, pos.Start, size, err)
+ log.Printf(`%s: range %s/%d: %s`, logp, pos, size, err)
+ header.Set(HeaderContentRange, pos.ContentRange(r.unit, size))
res.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return
}
@@ -870,11 +900,15 @@ func handleRange(res http.ResponseWriter, req *http.Request, bodyReader io.ReadS
listBody = append(listBody, body)
}
- var header = res.Header()
-
if len(listBody) == 1 {
- pos = listPos[0]
+ var (
+ body = listBody[0]
+ nbody = strconv.FormatInt(int64(len(body)), 10)
+ )
+ pos = r.positions[0]
+ header.Set(HeaderContentLength, nbody)
header.Set(HeaderContentRange, pos.ContentRange(r.unit, size))
+ header.Set(HeaderContentType, contentType)
res.WriteHeader(http.StatusPartialContent)
_, err = res.Write(listBody[0])
if err != nil {
@@ -890,7 +924,7 @@ func handleRange(res http.ResponseWriter, req *http.Request, bodyReader io.ReadS
x int
)
- for x, pos = range listPos {
+ for x, pos = range r.positions {
fmt.Fprintf(&bb, "--%s\r\n", boundary)
fmt.Fprintf(&bb, "%s: %s\r\n", HeaderContentType, contentType)
fmt.Fprintf(&bb, "%s: %s\r\n\r\n", HeaderContentRange, pos.ContentRange(r.unit, size))
diff --git a/lib/http/server_test.go b/lib/http/server_test.go
index 664f57e9..8a2ef81e 100644
--- a/lib/http/server_test.go
+++ b/lib/http/server_test.go
@@ -9,12 +9,18 @@ import (
"errors"
"fmt"
"io"
+ "log"
"mime"
"net/http"
+ "os"
+ "path/filepath"
"strings"
"testing"
+ "time"
liberrors "github.com/shuLhan/share/lib/errors"
+ "github.com/shuLhan/share/lib/memfs"
+ libnet "github.com/shuLhan/share/lib/net"
"github.com/shuLhan/share/lib/test"
)
@@ -1008,3 +1014,178 @@ func TestServer_handleRange_HEAD(t *testing.T) {
)
test.Assert(t, tag, string(exp), got)
}
+
+// Test HTTP Range request on big file using Range.
+//
+// When server receive,
+//
+// GET /big
+// Range: bytes=0-
+//
+// and the requested resources is quite larger, where writing all content of
+// file result in i/o timeout, it is best practice [1][2] if the server
+// write only partial content and let the client continue with the
+// subsequent Range request.
+//
+// In above case the server should response with,
+//
+// HTTP/1.1 206 Partial content
+// Content-Range: bytes 0-<limit>/<size>
+// Content-Length: <limit>
+//
+// Where limit is maximum packet that is reasonable [3] for most of the
+// client.
+//
+// [1]: https://stackoverflow.com/questions/63614008/how-best-to-respond-to-an-open-http-range-request
+// [2]: https://bugzilla.mozilla.org/show_bug.cgi?id=570755
+// [3]: https://docs.aws.amazon.com/whitepapers/latest/s3-optimizing-performance-best-practices/use-byte-range-fetches.html
+func TestServerHandleRangeBig(t *testing.T) {
+ var (
+ pathBig = `/big`
+ tempDir = t.TempDir()
+ filepathBig = filepath.Join(tempDir, pathBig)
+ bigSize = 10485760 // 10MB
+ )
+
+ createBigFile(t, filepathBig, int64(bigSize))
+
+ var (
+ serverAddress = `127.0.0.1:22672`
+ srv *Server
+ )
+
+ srv = runServerFS(t, serverAddress, tempDir)
+ defer srv.Stop(100 * time.Millisecond)
+
+ var (
+ tdata *test.Data
+ err error
+ )
+
+ tdata, err = test.LoadData(`testdata/server/range_big_test.txt`)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var (
+ clOpts = &ClientOptions{
+ ServerUrl: `http://` + serverAddress,
+ }
+ cl *Client
+ )
+
+ cl = NewClient(clOpts)
+
+ var (
+ tag = `HEAD /big`
+ skipHeaders = []string{HeaderDate}
+
+ httpRes *http.Response
+ gotResp string
+ resBody []byte
+ )
+
+ httpRes, resBody, err = cl.Head(pathBig, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ gotResp = dumpHTTPResponse(httpRes, skipHeaders)
+
+ test.Assert(t, tag, string(tdata.Output[tag]), gotResp)
+ test.Assert(t, tag+`- response body size`, 0, len(resBody))
+
+ var (
+ headers = http.Header{}
+ )
+
+ headers.Set(HeaderRange, `bytes=0-`)
+
+ httpRes, resBody, err = cl.Get(pathBig, headers, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ gotResp = dumpHTTPResponse(httpRes, skipHeaders)
+ tag = `GET /big:Range=0-`
+ test.Assert(t, tag, string(tdata.Output[tag]), gotResp)
+ test.Assert(t, tag+`- response body size`, DefRangeLimit, len(resBody))
+}
+
+func createBigFile(t *testing.T, path string, size int64) {
+ var (
+ fbig *os.File
+ err error
+ )
+
+ fbig, err = os.Create(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = fbig.Truncate(size)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = fbig.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func runServerFS(t *testing.T, address, dir string) (srv *Server) {
+ var (
+ mfsOpts = &memfs.Options{
+ Root: dir,
+ MaxFileSize: -1,
+ }
+
+ mfs *memfs.MemFS
+ err error
+ )
+
+ mfs, err = memfs.New(mfsOpts)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Set the file modification time for predictable result.
+ var (
+ pathBigModTime = time.Date(2024, 1, 1, 1, 1, 1, 0, time.UTC)
+ nodeBig *memfs.Node
+ )
+
+ nodeBig, err = mfs.Get(`/big`)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nodeBig.SetModTime(pathBigModTime)
+
+ var (
+ srvOpts = &ServerOptions{
+ Memfs: mfs,
+ Address: address,
+ }
+ )
+
+ srv, err = NewServer(srvOpts)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ go func() {
+ var err2 = srv.Start()
+ if err2 != nil {
+ log.Fatal(err2)
+ }
+ }()
+
+ err = libnet.WaitAlive(`tcp`, address, 1*time.Second)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return srv
+}
diff --git a/lib/http/testdata/server/range/fail_416_test.txt b/lib/http/testdata/server/range/fail_416_test.txt
index 88b54d14..bac7bc87 100644
--- a/lib/http/testdata/server/range/fail_416_test.txt
+++ b/lib/http/testdata/server/range/fail_416_test.txt
@@ -4,6 +4,7 @@ bytes=50-
<<< http_headers
HTTP/1.1 416 Requested Range Not Satisfiable
Content-Length: 0
+Content-Range: bytes */40
Content-Type: text/html; charset=utf-8
<<< http_body
diff --git a/lib/http/testdata/server/range/multipart_test.txt b/lib/http/testdata/server/range/multipart_test.txt
index 0c1608ab..bb90ca95 100644
--- a/lib/http/testdata/server/range/multipart_test.txt
+++ b/lib/http/testdata/server/range/multipart_test.txt
@@ -3,7 +3,7 @@ bytes=0-5,10-15,-10
<<< http_headers
HTTP/1.1 206 Partial Content
-Content-Length: 325
+Content-Length: 327
Content-Type: multipart/byteranges; boundary=1b4df158039f7cce
<<< http_body
@@ -19,7 +19,7 @@ Content-Range: bytes 10-15/40
y>Hell
--1b4df158039f7cce
Content-Type: text/html; charset=utf-8
-Content-Range: bytes -10/40
+Content-Range: bytes 30-39/40
y></html>
diff --git a/lib/http/testdata/server/range_big_test.txt b/lib/http/testdata/server/range_big_test.txt
new file mode 100644
index 00000000..8d47b707
--- /dev/null
+++ b/lib/http/testdata/server/range_big_test.txt
@@ -0,0 +1,14 @@
+
+<<< HEAD /big
+HTTP/1.1 200 OK
+Accept-Ranges: bytes
+Content-Length: 10485760
+Content-Type: application/octet-stream
+Etag: 1704070861
+
+<<< GET /big:Range=0-
+HTTP/1.1 206 Partial Content
+Content-Length: 8388608
+Content-Range: bytes 0-8388608/10485760
+Content-Type: application/octet-stream
+Etag: 1704070861