From 4993ab1a767bea5645a65b5639cc00bb107fdfd3 Mon Sep 17 00:00:00 2001 From: Unknwon Date: Thu, 31 Dec 2015 22:13:47 -0500 Subject: [PATCH] #2185 fall back to use custom chardet lib --- .gopmfile | 1 + README.md | 2 +- gogs.go | 2 +- models/git_diff.go | 4 ++-- modules/base/tool.go | 20 +++++++++++--------- modules/template/template.go | 6 ++++-- templates/.VERSION | 2 +- templates/base/footer.tmpl | 2 +- 8 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.gopmfile b/.gopmfile index e249d2bd..ddd76170 100644 --- a/.gopmfile +++ b/.gopmfile @@ -16,6 +16,7 @@ github.com/go-macaron/toolbox = commit:ab30a81 github.com/go-sql-driver/mysql = commit:d512f20 github.com/go-xorm/core = commit:acb6f00 github.com/go-xorm/xorm = commit:a8fba4d +github.com/gogits/chardet = commit:2404f77725 github.com/gogits/git-module = commit:5cd57b9 github.com/gogits/go-gogs-client = commit:78460e9 github.com/issue9/identicon = commit:f8c0d2c diff --git a/README.md b/README.md index ef080422..59c07c24 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Gogs - Go Git Service [![Build Status](https://travis-ci.org/gogits/gogs.svg?bra ![](https://github.com/gogits/gogs/blob/master/public/img/gogs-large-resize.png?raw=true) -##### Current version: 0.8.14 +##### Current version: 0.8.15 | Web | UI | Preview | |:-------------:|:-------:|:-------:| diff --git a/gogs.go b/gogs.go index 5323cfa4..5ad60cf5 100644 --- a/gogs.go +++ b/gogs.go @@ -17,7 +17,7 @@ import ( "github.com/gogits/gogs/modules/setting" ) -const APP_VER = "0.8.14.1230" +const APP_VER = "0.8.15.1231" func init() { runtime.GOMAXPROCS(runtime.NumCPU()) diff --git a/models/git_diff.go b/models/git_diff.go index 1b2fb5f2..22075ef7 100644 --- a/models/git_diff.go +++ b/models/git_diff.go @@ -246,8 +246,8 @@ func ParsePatch(maxlines int, reader io.Reader) (*Diff, error) { buf.WriteString("\n") } } - charsetLabel := base.DetectEncoding(buf.Bytes()) - if charsetLabel != "UTF-8" { + charsetLabel, err := base.DetectEncoding(buf.Bytes()) + if charsetLabel != "UTF-8" && err == nil { encoding, _ := charset.Lookup(charsetLabel) if encoding != nil { d := encoding.NewDecoder() diff --git a/modules/base/tool.go b/modules/base/tool.go index 5927dad0..255c34ef 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -23,7 +23,8 @@ import ( "github.com/Unknwon/com" "github.com/Unknwon/i18n" "github.com/microcosm-cc/bluemonday" - "golang.org/x/net/html/charset" + + "github.com/gogits/chardet" "github.com/gogits/gogs/modules/avatar" "github.com/gogits/gogs/modules/log" @@ -53,19 +54,20 @@ func ShortSha(sha1 string) string { return sha1 } -func DetectEncoding(content []byte) string { - if utf8.Valid(content[:1024]) { +func DetectEncoding(content []byte) (string, error) { + if utf8.Valid(content) { log.Debug("Detected encoding: utf-8 (fast)") - return "utf-8" + return "UTF-8", nil } - _, name, certain := charset.DetermineEncoding(content, "") - if name != "utf-8" && len(setting.Repository.AnsiCharset) > 0 { + result, err := chardet.NewTextDetector().DetectBest(content) + if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 { log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset) - return setting.Repository.AnsiCharset + return setting.Repository.AnsiCharset, err } - log.Debug("Detected encoding: %s (%v)", name, certain) - return name + + log.Debug("Detected encoding: %s", result.Charset) + return result.Charset, err } func BasicAuthDecode(encoded string) (string, string, error) { diff --git a/modules/template/template.go b/modules/template/template.go index 6c070b70..6099fcc9 100644 --- a/modules/template/template.go +++ b/modules/template/template.go @@ -130,8 +130,10 @@ func Sha1(str string) string { } func ToUtf8WithErr(content []byte) (error, string) { - charsetLabel := base.DetectEncoding(content) - if charsetLabel == "utf-8" { + charsetLabel, err := base.DetectEncoding(content) + if err != nil { + return err, "" + } else if charsetLabel == "UTF-8" { return nil, string(content) } diff --git a/templates/.VERSION b/templates/.VERSION index b5c863bb..cfcdb254 100644 --- a/templates/.VERSION +++ b/templates/.VERSION @@ -1 +1 @@ -0.8.14.1230 \ No newline at end of file +0.8.15.1231 \ No newline at end of file diff --git a/templates/base/footer.tmpl b/templates/base/footer.tmpl index 9bbaf7c7..5f2e8da0 100644 --- a/templates/base/footer.tmpl +++ b/templates/base/footer.tmpl @@ -7,7 +7,7 @@