fix

Author: wxiaoguang

modules/charset/charset.go

@@ -5,12 +5,10 @@ package charset
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"strings"
 	"unicode/utf8"
 
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 
@@ -23,118 +21,98 @@ import (
 var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
 
 type ConvertOpts struct {
-	KeepBOM bool
+	KeepBOM           bool
+	ErrorReplacement  []byte
+	ErrorReturnOrigin bool
 }
 
 // ToUTF8WithFallbackReader detects the encoding of content and converts to UTF-8 reader if possible
 func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
-	buf := make([]byte, 2048)
+	buf := make([]byte, 16*1024)
 	n, err := util.ReadAtMost(rd, buf)
 	if err != nil {
-		return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
+		// read error occurs, don't do any processing
+		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
 	}
 
-	charsetLabel, err := DetectEncoding(buf[:n])
-	if err != nil || charsetLabel == "UTF-8" {
-		return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
+	charsetLabel, _ := DetectEncoding(buf[:n])
+	if charsetLabel == "UTF-8" {
+		// is utf-8, try to remove BOM and read it as-is
+		return io.MultiReader(bytes.NewReader(maybeRemoveBOM(buf[:n], opts)), rd)
 	}
 
 	encoding, _ := charset.Lookup(charsetLabel)
 	if encoding == nil {
+		// unknown charset, don't do any processing
 		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
 	}
 
+	// convert from charset to utf-8
 	return transform.NewReader(
-		io.MultiReader(
-			bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)),
-			rd,
-		),
+		io.MultiReader(bytes.NewReader(buf[:n]), rd),
 		encoding.NewDecoder(),
 	)
 }
 
-// ToUTF8 converts content to UTF8 encoding
-func ToUTF8(content []byte, opts ConvertOpts) (string, error) {
-	charsetLabel, err := DetectEncoding(content)
-	if err != nil {
-		return "", err
-	} else if charsetLabel == "UTF-8" {
-		return string(MaybeRemoveBOM(content, opts)), nil
-	}
-
-	encoding, _ := charset.Lookup(charsetLabel)
-	if encoding == nil {
-		return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
-	}
-
-	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose much data.
-	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
-	if err != nil {
-		result = append(result, content[n:]...)
-	}
-
-	result = MaybeRemoveBOM(result, opts)
-
-	return string(result), err
-}
-
 // ToUTF8WithFallback detects the encoding of content and converts to UTF-8 if possible
 func ToUTF8WithFallback(content []byte, opts ConvertOpts) []byte {
 	bs, _ := io.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content), opts))
 	return bs
 }
 
-// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
-func ToUTF8DropErrors(content []byte, opts ConvertOpts) []byte {
-	charsetLabel, err := DetectEncoding(content)
-	if err != nil || charsetLabel == "UTF-8" {
-		return MaybeRemoveBOM(content, opts)
+func ToUTF8DropErrors(content []byte) []byte {
+	return ToUTF8(content, ConvertOpts{ErrorReplacement: []byte{' '}})
+}
+
+func ToUTF8(content []byte, opts ConvertOpts) []byte {
+	charsetLabel, _ := DetectEncoding(content)
+	if charsetLabel == "UTF-8" {
+		return maybeRemoveBOM(content, opts)
 	}
 
 	encoding, _ := charset.Lookup(charsetLabel)
 	if encoding == nil {
+		setting.PanicInDevOrTesting("unknown detected charset %q, it shouldn't happen", charsetLabel)
 		return content
 	}
 
-	// We ignore any non-decodable parts from the file.
-	// Some parts might be lost
 	var decoded []byte
 	decoder := encoding.NewDecoder()
 	idx := 0
-	for {
+	for idx < len(content) {
 		result, n, err := transform.Bytes(decoder, content[idx:])
 		decoded = append(decoded, result...)
 		if err == nil {
 			break
 		}
-		decoded = append(decoded, ' ')
-		idx = idx + n + 1
-		if idx >= len(content) {
-			break
+		if opts.ErrorReturnOrigin {
+			return content
 		}
+		if opts.ErrorReplacement == nil {
+			decoded = append(decoded, content[idx+n])
+		} else {
+			decoded = append(decoded, opts.ErrorReplacement...)
+		}
+		idx += n + 1
 	}
-
-	return MaybeRemoveBOM(decoded, opts)
+	return maybeRemoveBOM(decoded, opts)
 }
 
-// MaybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
-func MaybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
+// maybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
+func maybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
 	if opts.KeepBOM {
 		return content
 	}
-	if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
-		return content[3:]
-	}
-	return content
+	return bytes.TrimPrefix(content, UTF8BOM)
 }
 
 // DetectEncoding detect the encoding of content
-func DetectEncoding(content []byte) (string, error) {
+// it always returns a detected or guessed "encoding" string, no matter error happens or not
+func DetectEncoding(content []byte) (encoding string, _ error) {
 	// First we check if the content represents valid utf8 content excepting a truncated character at the end.
 
 	// Now we could decode all the runes in turn but this is not necessarily the cheapest thing to do
-	// instead we walk backwards from the end to trim off a the incomplete character
+	// instead we walk backwards from the end to trim off the incomplete character
 	toValidate := content
 	end := len(toValidate) - 1
 
@@ -150,8 +128,8 @@ func DetectEncoding(content []byte) (string, error) {
 		// Incomplete 3 byte extension e.g. 💩 <f0><9f><92><a9> which has been truncated to <f0><9f><92>
 		toValidate = toValidate[:end-2]
 	}
+
 	if utf8.Valid(toValidate) {
-		log.Debug("Detected encoding: utf-8 (fast)")
 		return "UTF-8", nil
 	}
 
@@ -171,14 +149,10 @@ func DetectEncoding(content []byte) (string, error) {
 		detectContent = content
 	}
 
-	// Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie break
+	// Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie-break
 	results, err := textDetector.DetectAll(detectContent)
 	if err != nil {
-		if err == chardet.NotDetectedError && len(setting.Repository.AnsiCharset) > 0 {
-			log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
-			return setting.Repository.AnsiCharset, nil
-		}
-		return "", err
+		return util.IfZero(setting.Repository.AnsiCharset, "UTF-8"), err
 	}
 
 	topConfidence := results[0].Confidence
@@ -202,10 +176,8 @@ func DetectEncoding(content []byte) (string, error) {
 
 	// FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument
 	if topResult.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
-		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
 		return setting.Repository.AnsiCharset, err
 	}
 
-	log.Debug("Detected encoding: %s", topResult.Charset)
-	return topResult.Charset, err
+	return topResult.Charset, nil
 }

modules/charset/charset_test.go

@@ -31,10 +31,10 @@ func resetDefaultCharsetsOrder() {
 }
 
 func TestMaybeRemoveBOM(t *testing.T) {
-	res := MaybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	res := maybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
 	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
 
-	res = MaybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	res = maybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
 	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
 }
 
@@ -45,63 +45,54 @@ func TestToUTF8(t *testing.T) {
 	// locale, so some conversions might behave differently. For that reason, we don't
 	// depend on particular conversions but in expected behaviors.
 
-	res, err := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
-	assert.NoError(t, err)
-	assert.Equal(t, "ABC", res)
+	res := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
+	assert.Equal(t, "ABC", string(res))
 
 	// "áéíóú"
-	res, err = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
-	assert.NoError(t, err)
+	res = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
 	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
 
 	// "áéíóú"
-	res, err = ToUTF8([]byte{
+	res = ToUTF8([]byte{
 		0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
 		0xc3, 0xba,
 	}, ConvertOpts{})
-	assert.NoError(t, err)
 	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
 
-	res, err = ToUTF8([]byte{
+	res = ToUTF8([]byte{
 		0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
 		0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
 	}, ConvertOpts{})
-	assert.NoError(t, err)
 	stringMustStartWith(t, "Hola,", res)
 	stringMustEndWith(t, "AAA.", res)
 
-	res, err = ToUTF8([]byte{
+	res = ToUTF8([]byte{
 		0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
 		0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
 	}, ConvertOpts{})
-	assert.NoError(t, err)
 	stringMustStartWith(t, "Hola,", res)
 	stringMustEndWith(t, "AAA.", res)
 
-	res, err = ToUTF8([]byte{
+	res = ToUTF8([]byte{
 		0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
 		0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
 	}, ConvertOpts{})
-	assert.NoError(t, err)
 	stringMustStartWith(t, "Hola,", res)
 	stringMustEndWith(t, "AAA.", res)
 
 	// Japanese (Shift-JIS)
 	// 日属秘ぞしちゅ。
-	res, err = ToUTF8([]byte{
+	res = ToUTF8([]byte{
 		0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
 		0xBF, 0x82, 0xE3, 0x81, 0x42,
 	}, ConvertOpts{})
-	assert.NoError(t, err)
 	assert.Equal(t, []byte{
 		0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
 		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
-	},
-		[]byte(res))
+	}, res)
 
-	res, err = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
-	assert.NoError(t, err)
-	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
+	res = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
 }
 
 func TestToUTF8WithFallback(t *testing.T) {
@@ -153,43 +144,44 @@ func TestToUTF8WithFallback(t *testing.T) {
 
 func TestToUTF8DropErrors(t *testing.T) {
 	resetDefaultCharsetsOrder()
+
 	// "ABC"
-	res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
+	res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43})
 	assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
 
 	// "áéíóú"
-	res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
 	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
 
 	// UTF8 BOM + "áéíóú"
-	res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
 	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
 
 	// "Hola, así cómo ños"
-	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}, ConvertOpts{})
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
 	assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8])
 	assert.Equal(t, []byte{0x73}, res[len(res)-1:])
 
 	// "Hola, así cómo "
 	minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
 
-	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{})
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
 	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
 	assert.Equal(t, minmatch, res[0:len(minmatch)])
 
-	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{})
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
 	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
 	assert.Equal(t, minmatch, res[0:len(minmatch)])
 
 	// Japanese (Shift-JIS)
 	// "日属秘ぞしちゅ。"
-	res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{})
+	res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
 	assert.Equal(t, []byte{
 		0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
 		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
 	}, res)
 
-	res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
+	res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00})
 	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
 }
 
@@ -231,12 +223,12 @@ func TestDetectEncoding(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func stringMustStartWith(t *testing.T, expected, value string) {
-	assert.Equal(t, expected, value[:len(expected)])
+func stringMustStartWith(t *testing.T, expected string, value []byte) {
+	assert.Equal(t, expected, string(value[:len(expected)]))
 }
 
-func stringMustEndWith(t *testing.T, expected, value string) {
-	assert.Equal(t, expected, value[len(value)-len(expected):])
+func stringMustEndWith(t *testing.T, expected string, value []byte) {
+	assert.Equal(t, expected, string(value[len(value)-len(expected):]))
 }
 
 func TestToUTF8WithFallbackReader(t *testing.T) {

modules/httplib/serve.go

@@ -19,7 +19,6 @@ import (
 	charsetModule "code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/container"
 	"code.gitea.io/gitea/modules/httpcache"
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/typesniffer"
 	"code.gitea.io/gitea/modules/util"
@@ -109,11 +108,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byt
 	}
 
 	if isPlain {
-		charset, err := charsetModule.DetectEncoding(mineBuf)
-		if err != nil {
-			log.Error("Detect raw file %s charset failed: %v, using by default utf-8", opts.Filename, err)
-			charset = "utf-8"
-		}
+		charset, _ := charsetModule.DetectEncoding(mineBuf)
 		opts.ContentTypeCharset = strings.ToLower(charset)
 	}
 

modules/indexer/code/bleve/bleve.go

@@ -203,7 +203,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro
 		RepoID:    repo.ID,
 		CommitID:  commitSha,
 		Filename:  update.Filename,
-		Content:   string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})),
+		Content:   string(charset.ToUTF8DropErrors(fileContents)),
 		Language:  analyze.GetCodeLanguage(update.Filename, fileContents),
 		UpdatedAt: time.Now().UTC(),
 	})