Bound read_minified_file streaming by page budget

internal/tools/line_reader.go

@@ -2,10 +2,112 @@ package tools
 
 import (
 	"bufio"
+	"bytes"
 	"io"
 )
 
+// readRawLine reads one line (including the trailing '\n' when present).
+// On EOF with a non-empty unterminated buffer it returns that buffer with
+// ended=false and err=nil. On EOF with an empty buffer it returns io.EOF.
 func readRawLine(reader *bufio.Reader) ([]byte, bool, error) {
+	line, ended, _, _, _, err := readRawLineLimited(reader, 0)
+	return line, ended, err
+}
+
+// readRawLineLimited is like readRawLine but, when maxKeep > 0, retains at most
+// maxKeep bytes of the line (including a trailing newline only if it still fits).
+// Further bytes until the next newline are discarded so a multi-megabyte
+// minified line cannot force a multi-megabyte allocation. clipped is true when
+// any trailing content was discarded.
+func readRawLineLimited(reader *bufio.Reader, maxKeep int) (line []byte, ended bool, clipped, containsNUL bool, bytesScanned int, err error) {
+	if maxKeep <= 0 {
+		line, ended, clipped, err := readRawLineUnlimited(reader)
+		return line, ended, clipped, bytes.IndexByte(line, 0) >= 0, len(line), err
+	}
+	var kept []byte
+	for {
+		fragment, readErr := reader.ReadSlice('\n')
+		bytesScanned += len(fragment)
+		containsNUL = containsNUL || bytes.IndexByte(fragment, 0) >= 0
+		if len(fragment) > 0 {
+			room := maxKeep - len(kept)
+			if room <= 0 {
+				if fragment[len(fragment)-1] == '\n' {
+					// Once maxKeep is full, a fragment containing only the
+					// line break means no line content was discarded.
+					if normalized, onlyLineBreak := trimDiscardedLineBreak(kept, fragment); onlyLineBreak {
+						return normalized, true, false, containsNUL, bytesScanned, nil
+					}
+					return kept, true, true, containsNUL, bytesScanned, nil
+				}
+				if readErr != nil && readErr != bufio.ErrBufferFull && readErr != io.EOF {
+					return kept, false, true, containsNUL, bytesScanned, readErr
+				}
+				discardedNUL, discardedBytes, dErr := discardThroughNewline(reader)
+				bytesScanned += discardedBytes
+				if dErr != nil && dErr != io.EOF {
+					return kept, false, true, containsNUL || discardedNUL, bytesScanned, dErr
+				}
+				return kept, dErr == nil, true, containsNUL || discardedNUL, bytesScanned, nil
+			}
+			if len(fragment) <= room {
+				if kept != nil || readErr == bufio.ErrBufferFull {
+					kept = append(kept, fragment...)
+				} else {
+					kept = fragment
+				}
+			} else {
+				kept = append(kept, fragment[:room]...)
+				rest := fragment[room:]
+				// Discarding only the line break is not content clipping.
+				if normalized, onlyLineBreak := trimDiscardedLineBreak(kept, rest); onlyLineBreak {
+					return normalized, true, false, containsNUL, bytesScanned, nil
+				}
+				if fragment[len(fragment)-1] == '\n' {
+					// Non-newline content past maxKeep was discarded; line ended.
+					return kept, true, true, containsNUL, bytesScanned, nil
+				}
+				if readErr != nil && readErr != bufio.ErrBufferFull && readErr != io.EOF {
+					return kept, false, true, containsNUL, bytesScanned, readErr
+				}
+				discardedTailNUL, discardedBytes, dErr := discardThroughNewline(reader)
+				bytesScanned += discardedBytes
+				if dErr != nil && dErr != io.EOF {
+					return kept, false, true, containsNUL || discardedTailNUL, bytesScanned, dErr
+				}
+				return kept, dErr == nil, true, containsNUL || discardedTailNUL, bytesScanned, nil
+			}
+		}
+		switch readErr {
+		case nil:
+			return kept, true, false, containsNUL, bytesScanned, nil
+		case bufio.ErrBufferFull:
+			continue
+		case io.EOF:
+			if len(kept) > 0 {
+				return kept, false, false, containsNUL, bytesScanned, nil
+			}
+			return nil, false, false, containsNUL, bytesScanned, io.EOF
+		default:
+			return nil, false, false, containsNUL, bytesScanned, readErr
+		}
+	}
+}
+
+func trimDiscardedLineBreak(kept, rest []byte) ([]byte, bool) {
+	if len(rest) == 2 && rest[0] == '\r' && rest[1] == '\n' {
+		return kept, true
+	}
+	if len(rest) != 1 || rest[0] != '\n' {
+		return kept, false
+	}
+	if len(kept) > 0 && kept[len(kept)-1] == '\r' {
+		kept = kept[:len(kept)-1]
+	}
+	return kept, true
+}
+
+func readRawLineUnlimited(reader *bufio.Reader) ([]byte, bool, bool, error) {
 	var line []byte
 	for {
 		fragment, err := reader.ReadSlice('\n')
@@ -18,16 +120,36 @@ func readRawLine(reader *bufio.Reader) ([]byte, bool, error) {
 		}
 		switch err {
 		case nil:
-			return line, true, nil
+			return line, true, false, nil
 		case bufio.ErrBufferFull:
 			continue
 		case io.EOF:
 			if len(line) > 0 {
-				return line, false, nil
+				return line, false, false, nil
 			}
-			return nil, false, io.EOF
+			return nil, false, false, io.EOF
+		default:
+			return nil, false, false, err
+		}
+	}
+}
+
+// discardThroughNewline drops input until a newline or EOF and reports whether
+// any discarded fragment contained a NUL byte.
+func discardThroughNewline(reader *bufio.Reader) (bool, int, error) {
+	containsNUL := false
+	bytesScanned := 0
+	for {
+		fragment, err := reader.ReadSlice('\n')
+		bytesScanned += len(fragment)
+		containsNUL = containsNUL || bytes.IndexByte(fragment, 0) >= 0
+		switch err {
+		case nil:
+			return containsNUL, bytesScanned, nil
+		case bufio.ErrBufferFull:
+			continue
 		default:
-			return nil, false, err
+			return containsNUL, bytesScanned, err
 		}
 	}
 }

internal/tools/line_reader_test.go

@@ -0,0 +1,85 @@
+package tools
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+)
+
+func TestReadRawLineLimitedCRLFAtLimit(t *testing.T) {
+	for _, test := range []struct {
+		name    string
+		maxKeep int
+		want    string
+		clipped bool
+	}{
+		{name: "content plus CRLF", maxKeep: 10, want: "abcdefghij", clipped: false},
+		{name: "content plus CRLF over limit", maxKeep: 11, want: "abcdefghij", clipped: false},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			line, ended, clipped, containsNUL, _, err := readRawLineLimited(bufio.NewReader(strings.NewReader("abcdefghij\r\n")), test.maxKeep)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(line) != test.want || !ended || clipped != test.clipped || containsNUL {
+				t.Fatalf("line=%q ended=%v clipped=%v containsNUL=%v", line, ended, clipped, containsNUL)
+			}
+		})
+	}
+}
+
+func TestReadRawLineLimitedPropagatesFullLineError(t *testing.T) {
+	wantErr := errors.New("full line failed")
+	source := &sequenceReader{steps: []readStep{
+		{data: []byte(strings.Repeat("x", 16)), err: bufio.ErrBufferFull},
+		{data: []byte("tail"), err: wantErr},
+	}}
+	reader := bufio.NewReader(source)
+	_, _, _, _, _, err := readRawLineLimited(reader, 16)
+	if !errors.Is(err, wantErr) {
+		t.Fatalf("err=%v want %v", err, wantErr)
+	}
+	if source.reads != 2 {
+		t.Fatalf("reads=%d want 2", source.reads)
+	}
+}
+
+func TestReadRawLineLimitedPropagatesOverflowError(t *testing.T) {
+	wantErr := errors.New("read failed")
+	source := &sequenceReader{steps: []readStep{{data: []byte(strings.Repeat("x", 17)), err: wantErr}}}
+	reader := bufio.NewReader(source)
+	_, _, _, _, _, err := readRawLineLimited(reader, 16)
+	if !errors.Is(err, wantErr) {
+		t.Fatalf("err=%v want %v", err, wantErr)
+	}
+	if source.reads != 1 {
+		t.Fatalf("reads=%d want 1", source.reads)
+	}
+}
+
+type readStep struct {
+	data []byte
+	err  error
+}
+
+type sequenceReader struct {
+	steps []readStep
+	reads int
+}
+
+func (reader *sequenceReader) Read(buffer []byte) (int, error) {
+	reader.reads++
+	if len(reader.steps) == 0 {
+		return 0, io.EOF
+	}
+	step := reader.steps[0]
+	n := copy(buffer, step.data)
+	if n < len(step.data) {
+		reader.steps[0].data = step.data[n:]
+	} else {
+		reader.steps = reader.steps[1:]
+	}
+	return n, step.err
+}

internal/tools/read_file.go

@@ -286,7 +286,7 @@ type readFileStats struct {
 }
 
 func scanReadFileStats(path string) (readFileStats, error) {
-	file, err := os.Open(path)
+	file, err := openReadableRegularFile(path)
 	if err != nil {
 		return readFileStats{}, err
 	}
@@ -321,7 +321,7 @@ func renderReadFileBytes(path, relativePath string, total, requestedStart, limit
 	if requestedStart >= total {
 		return okResult(fmt.Sprintf("File: %s\n(byte_offset %d is past the end of the file, which has %d bytes)", relativePath, requestedStart, total)), 0, 0
 	}
-	file, err := os.Open(path)
+	file, err := openReadableRegularFile(path)
 	if err != nil {
 		return errorResult("Error reading file " + relativePath + ": " + err.Error()), 0, 0
 	}
@@ -359,7 +359,7 @@ func renderReadFileBytes(path, relativePath string, total, requestedStart, limit
 }
 
 func appendReadFileRange(output *outputBudgetBuilder, path string, startLine int, selectedLines int) error {
-	file, err := os.Open(path)
+	file, err := openReadableRegularFile(path)
 	if err != nil {
 		return err
 	}