From d413a8fcacc81b6f7039371408034c9c2fc6c15f Mon Sep 17 00:00:00 2001 From: coldWater <254244460@qq.com> Date: Thu, 14 Mar 2024 10:51:55 +0800 Subject: [PATCH] Refactor markup/csv: don't read all to memory (#29760) (cherry picked from commit e79a807a8461a73bd66146d816f635b66e198c89) --- modules/markup/csv/csv.go | 63 ++++++++++++++++++++++++++-------- modules/markup/csv/csv_test.go | 10 ++++++ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 12458e954a..570c4f4704 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -77,29 +77,62 @@ func writeField(w io.Writer, element, class, field string) error { } // Render implements markup.Renderer -func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { +func (r Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { tmpBlock := bufio.NewWriter(output) + maxSize := setting.UI.CSV.MaxFileSize - // FIXME: don't read all to memory - rawBytes, err := io.ReadAll(input) + if maxSize == 0 { + return r.tableRender(ctx, input, tmpBlock) + } + + rawBytes, err := io.ReadAll(io.LimitReader(input, maxSize+1)) if err != nil { return err } - if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) { - if _, err := tmpBlock.WriteString("
"); err != nil {
-			return err
-		}
-		if _, err := tmpBlock.WriteString(html.EscapeString(string(rawBytes))); err != nil {
-			return err
-		}
-		if _, err := tmpBlock.WriteString("
"); err != nil { - return err - } - return tmpBlock.Flush() + if int64(len(rawBytes)) <= maxSize { + return r.tableRender(ctx, bytes.NewReader(rawBytes), tmpBlock) + } + return r.fallbackRender(io.MultiReader(bytes.NewReader(rawBytes), input), tmpBlock) +} + +func (Renderer) fallbackRender(input io.Reader, tmpBlock *bufio.Writer) error { + _, err := tmpBlock.WriteString("
")
+	if err != nil {
+		return err
 	}
 
-	rd, err := csv.CreateReaderAndDetermineDelimiter(ctx, bytes.NewReader(rawBytes))
+	scan := bufio.NewScanner(input)
+	scan.Split(bufio.ScanRunes)
+	for scan.Scan() {
+		switch scan.Text() {
+		case `&`:
+			_, err = tmpBlock.WriteString("&")
+		case `'`:
+			_, err = tmpBlock.WriteString("'") // "'" is shorter than "'" and apos was not in HTML until HTML5.
+		case `<`:
+			_, err = tmpBlock.WriteString("<")
+		case `>`:
+			_, err = tmpBlock.WriteString(">")
+		case `"`:
+			_, err = tmpBlock.WriteString(""") // """ is shorter than """.
+		default:
+			_, err = tmpBlock.Write(scan.Bytes())
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err = tmpBlock.WriteString("
") + if err != nil { + return err + } + return tmpBlock.Flush() +} + +func (Renderer) tableRender(ctx *markup.RenderContext, input io.Reader, tmpBlock *bufio.Writer) error { + rd, err := csv.CreateReaderAndDetermineDelimiter(ctx, input) if err != nil { return err } diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index 8c07184b21..3d12be477c 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -4,6 +4,8 @@ package markup import ( + "bufio" + "bytes" "strings" "testing" @@ -29,4 +31,12 @@ func TestRenderCSV(t *testing.T) { assert.NoError(t, err) assert.EqualValues(t, v, buf.String()) } + + t.Run("fallbackRender", func(t *testing.T) { + var buf bytes.Buffer + err := render.fallbackRender(strings.NewReader("1,\n2,"), bufio.NewWriter(&buf)) + assert.NoError(t, err) + want := "
1,<a>\n2,<b>
" + assert.Equal(t, want, buf.String()) + }) }