Add precompressed brotli/gzip assets

This uses webpack to emit .gz and .br files for each js/css file into
the public directory. The benefit of this is that our assets are now
always optimally compressed without any runtime performance impact.

The backend will dynamically emit compressed public assets based on
the client's content-encoding support and there is no inteference with
the existing gzip option (which does slow runtime compression).

Overall, this will increase bindata size and memory consumption by
around 20 MiB while saving around 25-50% of transfer size for every
affected asset.

Author: silverwind

Makefile

@@ -27,7 +27,7 @@ COMMA := ,
 
 XGO_VERSION := go-1.14.x
 MIN_GO_VERSION := 001012000
-MIN_NODE_VERSION := 010013000
+MIN_NODE_VERSION := 010016000
 
 DOCKER_IMAGE ?= gitea/gitea
 DOCKER_TAG ?= latest
@@ -196,7 +196,7 @@ node-check:
 	$(eval NODE_VERSION := $(shell printf "%03d%03d%03d" $(shell node -v | cut -c2- | tr '.' ' ');))
 	$(eval NPM_MISSING := $(shell hash npm > /dev/null 2>&1 || echo 1))
 	@if [ "$(NODE_VERSION)" -lt "$(MIN_NODE_VERSION)" -o "$(NPM_MISSING)" = "1" ]; then \
-		echo "Gitea requires Node.js 10 or greater and npm to build. You can get it at https://nodejs.org/en/download/"; \
+		echo "Gitea requires Node.js 10.16 or greater and npm to build. You can get it at https://nodejs.org/en/download/"; \
 		exit 1; \
 	fi
 

README.md

@@ -41,7 +41,7 @@ or if sqlite support is required:
 The `build` target is split into two sub-targets:
 
 - `make backend` which requires [Go 1.12](https://golang.org/dl/) or greater.
-- `make frontend` which requires [Node.js 10.13](https://nodejs.org/en/download/) or greater.
+- `make frontend` which requires [Node.js 10.16](https://nodejs.org/en/download/) or greater.
 
 If pre-built frontend files are present it is possible to only build the backend:
 

custom/conf/app.example.ini

@@ -318,7 +318,7 @@ KEY_FILE = https/key.pem
 STATIC_ROOT_PATH =
 ; Default path for App data
 APP_DATA_PATH = data
-; Application level GZIP support
+; Enable gzip compression for runtime-generated content
 ENABLE_GZIP = false
 ; Application profiling (memory and cpu)
 ; For "web" command it listens on localhost:6060

docs/config.yaml

@@ -21,7 +21,7 @@ params:
   version: 1.12.2
   minGoVersion: 1.12
   goVersion: 1.14
-  minNodeVersion: 10.13
+  minNodeVersion: 10.16
 
 outputs:
   home:

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -203,7 +203,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `KEY_FILE`: **https/key.pem**: Key file path used for HTTPS. From 1.11 paths are relative to `CUSTOM_PATH`.
 - `STATIC_ROOT_PATH`: **./**: Upper level of template and static files path.
 - `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars.
-- `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
+- `ENABLE_GZIP`: **false**: Enable gzip compression for runtime-generated content.
 - `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].
 - `LFS_START_SERVER`: **false**: Enables git-lfs support.
 - `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files.

modules/public/public.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -89,6 +90,8 @@ func (opts *Options) staticHandler(dir string) macaron.Handler {
 	}
 }
 
+var commaRe = regexp.MustCompile(`,\s*`)
+
 func (opts *Options) handle(ctx *macaron.Context, log *log.Logger, opt *Options) bool {
 	if ctx.Req.Method != "GET" && ctx.Req.Method != "HEAD" {
 		return false
@@ -106,7 +109,32 @@ func (opts *Options) handle(ctx *macaron.Context, log *log.Logger, opt *Options)
 		}
 	}
 
-	f, err := opt.FileSystem.Open(file)
+	var f http.File
+	split := commaRe.Split(ctx.Req.Header.Get("Accept-Encoding"), -1)
+	encodings := make(map[string]bool)
+	for i := range split {
+		encodings[split[i]] = true
+	}
+
+	var err error
+	if encodings["br"] {
+		f, err = opt.FileSystem.Open(file + ".br")
+		if err != nil {
+			f, err = opt.FileSystem.Open(file)
+		} else {
+			ctx.Resp.Header().Set("Content-Encoding", "br")
+		}
+	} else if encodings["gzip"] {
+		f, err = opt.FileSystem.Open(file + ".gz")
+		if err != nil {
+			f, err = opt.FileSystem.Open(file)
+		} else {
+			ctx.Resp.Header().Set("Content-Encoding", "gzip")
+		}
+	} else {
+		f, err = opt.FileSystem.Open(file)
+	}
+
 	if err != nil {
 		// 404 requests to any known entries in `public`
 		if path.Base(opts.Directory) == "public" {

package-lock.json

@@ -2,7 +2,7 @@
   "license": "MIT",
   "private": true,
   "engines": {
-    "node": ">= 10.13.0"
+    "node": ">= 10.16.0"
   },
   "dependencies": {
     "@babel/core": "7.10.5",
@@ -14,6 +14,7 @@
     "@primer/octicons": "10.0.0",
     "babel-loader": "8.1.0",
     "clipboard": "2.0.6",
+    "compression-webpack-plugin": "4.0.0",
     "core-js": "3.6.5",
     "css-loader": "4.0.0",
     "cssnano-webpack-plugin": "1.0.3",

package.json

@@ -130,9 +130,6 @@ func NewMacaron() *macaron.Macaron {
 		setupAccessLogger(m)
 	}
 	m.Use(macaron.Recovery())
-	if setting.EnableGzip {
-		m.Use(gzip.Middleware())
-	}
 	if setting.Protocol == setting.FCGI || setting.Protocol == setting.FCGIUnix {
 		m.SetURLPrefix(setting.AppSubURL)
 	}
@@ -149,6 +146,9 @@ func NewMacaron() *macaron.Macaron {
 			ExpiresAfter: setting.StaticCacheTime,
 		},
 	))
+	if setting.EnableGzip {
+		m.Use(gzip.Middleware())
+	}
 	m.Use(public.StaticHandler(
 		setting.AvatarUploadPath,
 		&public.Options{

routers/routes/routes.go

@@ -1,5 +1,7 @@
 const fastGlob = require('fast-glob');
 const wrapAnsi = require('wrap-ansi');
+const {constants} = require('zlib');
+const CompressionPlugin = require('compression-webpack-plugin');
 const CssNanoPlugin = require('cssnano-webpack-plugin');
 const FixStyleOnlyEntriesPlugin = require('webpack-fix-style-only-entries');
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
@@ -37,6 +39,75 @@ const filterCssImport = (url, ...args) => {
   return true;
 };
 
+const plugins = [
+  new VueLoaderPlugin(),
+  // avoid generating useless js output files for css--only chunks
+  new FixStyleOnlyEntriesPlugin({
+    extensions: ['less', 'scss', 'css'],
+    silent: true,
+  }),
+  new MiniCssExtractPlugin({
+    filename: 'css/[name].css',
+    chunkFilename: 'css/[name].css',
+  }),
+  new SourceMapDevToolPlugin({
+    filename: '[file].map',
+    include: [
+      'js/index.js',
+      'css/index.css',
+    ],
+  }),
+  new MonacoWebpackPlugin({
+    filename: 'js/monaco-[name].worker.js',
+  }),
+  new LicenseWebpackPlugin({
+    outputFilename: 'js/licenses.txt',
+    perChunkOutput: false,
+    addBanner: false,
+    skipChildCompilers: true,
+    modulesDirectories: [
+      resolve(__dirname, 'node_modules'),
+    ],
+    renderLicenses: (modules) => {
+      const line = '-'.repeat(80);
+      return modules.map((module) => {
+        const {name, version} = module.packageJson;
+        const {licenseId, licenseText} = module;
+        const body = wrapAnsi(licenseText || '', 80);
+        return `${line}\n${name}@${version} - ${licenseId}\n${line}\n${body}`;
+      }).join('\n');
+    },
+    stats: {
+      warnings: false,
+      errors: true,
+    },
+  }),
+];
+
+if (isProduction) {
+  plugins.push(
+    new CompressionPlugin({
+      filename: '[path].gz',
+      algorithm: 'gzip',
+      test: /\.(js|css)$/,
+      compressionOptions: {
+        level: constants.Z_BEST_COMPRESSION,
+      },
+      threshold: 10240,
+    }),
+    new CompressionPlugin({
+      filename: '[path].br',
+      algorithm: 'brotliCompress',
+      test: /\.(js|css)$/,
+      compressionOptions: {
+        [constants.BROTLI_PARAM_QUALITY]: constants.BROTLI_MAX_QUALITY,
+      },
+      threshold: 10240,
+    }),
+  );
+}
+
+
 module.exports = {
   mode: isProduction ? 'production' : 'development',
   entry: {
@@ -255,50 +326,7 @@ module.exports = {
       },
     ],
   },
-  plugins: [
-    new VueLoaderPlugin(),
-    // avoid generating useless js output files for css--only chunks
-    new FixStyleOnlyEntriesPlugin({
-      extensions: ['less', 'scss', 'css'],
-      silent: true,
-    }),
-    new MiniCssExtractPlugin({
-      filename: 'css/[name].css',
-      chunkFilename: 'css/[name].css',
-    }),
-    new SourceMapDevToolPlugin({
-      filename: '[file].map',
-      include: [
-        'js/index.js',
-        'css/index.css',
-      ],
-    }),
-    new MonacoWebpackPlugin({
-      filename: 'js/monaco-[name].worker.js',
-    }),
-    new LicenseWebpackPlugin({
-      outputFilename: 'js/licenses.txt',
-      perChunkOutput: false,
-      addBanner: false,
-      skipChildCompilers: true,
-      modulesDirectories: [
-        resolve(__dirname, 'node_modules'),
-      ],
-      renderLicenses: (modules) => {
-        const line = '-'.repeat(80);
-        return modules.map((module) => {
-          const {name, version} = module.packageJson;
-          const {licenseId, licenseText} = module;
-          const body = wrapAnsi(licenseText || '', 80);
-          return `${line}\n${name}@${version} - ${licenseId}\n${line}\n${body}`;
-        }).join('\n');
-      },
-      stats: {
-        warnings: false,
-        errors: true,
-      },
-    }),
-  ],
+  plugins,
   performance: {
     hints: false,
     maxEntrypointSize: Infinity,
@@ -321,6 +349,7 @@ module.exports = {
       // exclude monaco's language chunks in stats output for brevity
       // https://github.com/microsoft/monaco-editor-webpack-plugin/issues/113
       /^js\/[0-9]+\.js$/,
+      /\.(gz|br)$/,
     ],
   },
 };