Proxy container

.github/workflows/proxy.yml

@@ -0,0 +1,138 @@
+name: openresty
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+    paths:
+      - ".github/workflows/openresty.yml"
+      - "config/openresty/**"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4 # Update to latest version
+
+      # Cache npm dependencies
+      - name: Cache npm packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-npm-
+
+      # Cache luarocks
+      - name: Cache luarocks
+        uses: actions/cache@v4
+        with:
+          path: ~/.luarocks
+          key: ${{ runner.os }}-luarocks-${{ hashFiles('**/*.rockspec') }}
+          restore-keys: |
+            ${{ runner.os }}-luarocks-
+
+      - name: Install PCRE library
+        run: sudo apt-get install -y libpcre3 libpcre3-dev
+
+      # Debug: Show directory structure before cleanup
+      - name: Debug - Show initial directory structure
+        run: |
+          pwd
+          ls -la
+          ls -la ~/.openresty || echo "No ~/.openresty directory"
+          ls -la /usr/local/openresty || echo "No /usr/local/openresty directory"
+
+      # Clear previous installation
+      - name: Clean previous installation
+        run: |
+          sudo rm -rf ~/.openresty /usr/local/openresty
+          sudo mkdir -p /usr/local/openresty
+          sudo chown -R $USER:$USER /usr/local/openresty
+
+      # Install OpenResty without using cache
+      - name: Install OpenResty
+        uses: leafo/gh-actions-openresty@v1
+        with:
+          openrestyVersion: "1.21.4.2"
+          configureFlags: "--with-pcre-jit --with-ipv6 --with-http_v2_module"
+          buildCache: false
+
+      # Debug: Verify installation
+      - name: Debug - Verify OpenResty installation
+        run: |
+          echo "Directory structure after installation:"
+          ls -la /usr/local/openresty
+          echo "OpenResty version:"
+          /usr/local/openresty/bin/openresty -v || true
+          echo "LuaJIT directory contents:"
+          ls -la /usr/local/openresty/luajit || true
+
+      - uses: leafo/gh-actions-luarocks@v4
+        with:
+          withLuaPath: "/usr/local/openresty/luajit/"
+
+      - name: Install auto-ssl
+        run: |
+          which luarocks
+          sudo /home/runner/work/Blot/Blot/.luarocks/bin/luarocks install lua-resty-auto-ssl
+          sudo /home/runner/work/Blot/Blot/.luarocks/bin/luarocks list
+          luarocks show lua-resty-auto-ssl
+
+      - name: Set up Redis
+        uses: shogo82148/actions-setup-redis@v1
+        with:
+          redis-version: 6.2.6
+
+      - name: Set up Node
+        uses: actions/setup-node@v4 # Update to latest version
+        with:
+          node-version: 21.7.3
+          cache: 'npm' # Enable built-in npm caching
+
+      - name: Install Dependencies
+        run: npm ci # Use ci instead of install for more reliable builds
+
+      - name: Create required directories
+        run: |
+          sudo mkdir -p /var/instance-ssd/logs
+          mkdir -p /home/runner/work/Blot/Blot/config/openresty/data/latest
+
+      - name: Build openresty config
+        env:
+          NODE_PATH: /home/runner/work/Blot/Blot/app
+          NODE_SERVER_IP: 127.0.0.1
+          LUA_PACKAGE_PATH: /home/runner/work/Blot/Blot/.luarocks/share/lua/5.1/?.lua;/home/runner/work/Blot/Blot/config/openresty/conf/?.lua
+          REDIS_IP: 127.0.0.1
+          OPENRESTY_USER: runner
+          DISABLE_HTTP2: true
+          OPENRESTY_CACHE_DIRECTORY: /home/runner/work/Blot/Blot/config/openresty/data/latest/cache
+          OPENRESTY_CONFIG_DIRECTORY: /home/runner/work/Blot/Blot/config/openresty/data
+          SSL_CERTIFICATE: /home/runner/work/Blot/Blot/config/openresty/data/latest/selfsigned.crt
+          SSL_CERTIFICATE_KEY: /home/runner/work/Blot/Blot/config/openresty/data/latest/selfsigned.key
+        run: |
+          node config/openresty/build-config.js --skip-confirmation
+          cat config/openresty/data/latest/openresty.conf
+
+      - name: Generate self-signed SSL certs
+        run: |
+          openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
+            -keyout /home/runner/work/Blot/Blot/config/openresty/data/latest/selfsigned.key \
+            -out /home/runner/work/Blot/Blot/config/openresty/data/latest/selfsigned.crt \
+            -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=localhost"
+
+      - name: Run openresty
+        run: sudo /usr/local/openresty/bin/openresty -c /home/runner/work/Blot/Blot/config/openresty/data/latest/openresty.conf
+
+      - name: Verify openresty is running
+        run: |
+          curl -f localhost:80 || (sudo cat /var/instance-ssd/logs/* && exit 1)
+
+      - name: Run openresty tests
+        env:
+          NODE_PATH: /home/runner/work/Blot/Blot/app
+          BLOT_OPENRESTY_TEST_USER: runner
+          BLOT_OPENRESTY_TEST_GROUP: runner
+        run: node scripts/tests config/openresty

proxy/Dockerfile

@@ -0,0 +1,44 @@
+FROM openresty/openresty:alpine-fat
+
+# Install dependencies with specific versions
+RUN apk add --no-cache bash openssl python3 py3-pip git && \
+    apk add --no-cache py3-cryptography py3-openssl
+
+# Install acme-nginx and create wrapper script
+RUN pip3 install --no-cache-dir acme-nginx pyOpenSSL cryptography --break-system-packages && \
+    echo '#!/usr/bin/env python3' > /usr/local/bin/acme-nginx && \
+    echo 'from acme_nginx.client import main' >> /usr/local/bin/acme-nginx && \
+    echo 'if __name__ == "__main__":' >> /usr/local/bin/acme-nginx && \
+    echo '    main()' >> /usr/local/bin/acme-nginx && \
+    chmod +x /usr/local/bin/acme-nginx
+
+# Verify acme-nginx installation
+RUN acme-nginx --help
+
+# Create ec2-user with a fixed UID (1000)
+RUN addgroup -S ec2-user && adduser -S -u 1000 -G ec2-user ec2-user
+
+# Install lua-resty-auto-ssl using pre-installed LuaRocks
+RUN luarocks install lua-resty-auto-ssl
+
+# Create required directories for lua-resty-auto-ssl
+RUN mkdir -p /etc/resty-auto-ssl/storage \
+    /etc/resty-auto-ssl/letsencrypt/conf \
+    /etc/openresty/lua /etc/ssl/private
+
+# Set correct permissions
+RUN chmod 755 /etc/resty-auto-ssl
+
+# Copy pre-built configuration files
+COPY data/openresty.conf /usr/local/openresty/nginx/conf/nginx.conf
+COPY data/cacher.lua /etc/openresty/lua/cacher.lua
+
+# Copy entrypoint script
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Expose ports
+EXPOSE 80 443
+
+# Set entrypoint
+ENTRYPOINT ["/entrypoint.sh"]

proxy/build/index.js

@@ -0,0 +1,195 @@
+const mustache = require("mustache");
+const config = require("config");
+const fs = require("fs-extra");
+const child_process = require("child_process");
+
+function loadEnvFile() {
+  const envPath = require('path').join(__dirname, "..", "..", ".env");
+  try {
+    const envContent = fs.readFileSync(envPath, "utf8");
+    const envVars = envContent
+      .split("\n")
+      .filter((line) => line.trim() && !line.startsWith("#"))
+      .reduce((vars, line) => {
+        const [key, ...valueParts] = line.split("=");
+        const value = valueParts.join("=").trim();
+        if (key && value) {
+          vars[key.trim()] = value.replace(/^["']|["']$/g, "");
+        }
+        return vars;
+      }, {});
+
+    Object.assign(process.env, envVars);
+  } catch (error) {
+    console.error("Error reading .env file:", error);
+  }
+}
+
+loadEnvFile();
+
+const NETDATA_USER = process.env.NETDATA_USER;
+const NETDATA_PASSWORD = process.env.NETDATA_PASSWORD;
+const NETDATA_PORT = process.env.NETDATA_PORT;
+
+const NODE_SERVER_IP = process.env.NODE_SERVER_IP;
+const REDIS_IP = process.env.REDIS_IP;
+
+if (!NODE_SERVER_IP) throw new Error("NODE_SERVER_IP not set");
+if (!REDIS_IP) throw new Error("REDIS_IP not set");
+
+const OUTPUT = __dirname + "/data/latest";
+const PREVIOUS = OUTPUT + "-previous-" + Date.now();
+const CONFIG_DIRECTORY = __dirname + "/conf";
+
+const template = fs.readFileSync(`${CONFIG_DIRECTORY}/server.conf`, "utf8");
+const partials = {};
+
+// remote config directory on the ec2 instance to which we will copy the config files
+const config_directory =
+  process.env.OPENRESTY_CONFIG_DIRECTORY || "/home/ec2-user/openresty";
+
+// max file size for icloud uploads and webhooks bodies
+// nginx requires 'M' instead of 'MB' but unfortunately
+// node rawbody parser requires 'MB' instead of 'M'
+// so this maps '25MB' to '25M' for nginx
+const iCloud_max_body_size = `${config.icloud.maxFileSize / 1000000}M`;
+const webhooks_client_max_body_size = `${
+  config.webhooks.client_max_body_size / 1000000
+}M`;
+
+const locals = {
+  host: "blot.im",
+  blot_directory: config.blot_directory,
+  disable_http2: process.env.DISABLE_HTTP2,
+  node_ip: NODE_SERVER_IP,
+  node_port: "8088",
+
+  // The maximum size of icloud uploads
+  iCloud_max_body_size,
+
+  // The maximum size of webhooks bodies forwarded to the node server
+  webhooks_client_max_body_size,
+
+  // used in production by the node application container running inside docker
+  // to communicate with the openresty cache purge endpoint on localhost
+  openresty_instance_private_ip: process.env.OPENRESTY_INSTANCE_PRIVATE_IP,
+
+  // used in production if we run multiple openresty instances at the same time
+  server_label: process.env.SERVER_LABEL || "us",
+
+  config_directory,
+  redis: { host: REDIS_IP },
+
+  // used only by the ci test runner since this path changes on github actions
+  lua_package_path: process.env.LUA_PACKAGE_PATH,
+  user: process.env.OPENRESTY_USER || "ec2-user",
+  log_directory:
+    process.env.OPENRESTY_LOG_DIRECTORY || "/var/instance-ssd/logs",
+
+  // if you change the cache directory, you must also update the
+  // script mount-instance-store.sh
+  cache_directory:
+    process.env.OPENRESTY_CACHE_DIRECTORY || "/var/instance-ssd/cache",
+  ssl_certificate:
+    process.env.SSL_CERTIFICATE || "/etc/ssl/private/letsencrypt-domain.pem",
+  ssl_certificate_key:
+    process.env.SSL_CERTIFICATE_KEY ||
+    "/etc/ssl/private/letsencrypt-domain.key",
+
+  NETDATA_PASSWORD,
+  NETDATA_USER,
+  NETDATA_PORT,
+};
+
+// move the previous contents of the data directory to a backup
+// so we can compare the new contents with the old
+if (fs.existsSync(OUTPUT)) fs.moveSync(OUTPUT, PREVIOUS, { overwrite: true });
+
+fs.emptyDirSync(OUTPUT);
+
+fs.copySync(`${__dirname}/html`, `${OUTPUT}/html`);
+
+fs.readdirSync(CONFIG_DIRECTORY).forEach((file) => {
+  // copy lua files to data directory so they are available to nginx
+  if (file.endsWith(".lua")) {
+    fs.copySync(CONFIG_DIRECTORY + "/" + file, OUTPUT + "/" + file);
+  }
+
+  if (!file.endsWith(".conf")) return;
+
+  partials[file] = fs.readFileSync(CONFIG_DIRECTORY + "/" + file, "utf8");
+});
+
+const warning = `
+
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# !!!!!!!!!!!   WARNING                                   !!!!!!!!!!!
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+# Do not edit this file directly
+
+# This file was generated by ../build.js
+# Please update the source files in ./conf and run ../build.js
+
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# !!!!!!!!!!!   WARNING                                   !!!!!!!!!!!
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+`;
+
+const result = mustache.render(template, locals, partials);
+
+fs.outputFileSync(OUTPUT + "/openresty.conf", warning + result);
+
+// used by the proxy-tests ci action on github
+if (process.argv.includes("--skip-confirmation")) {
+  console.log("Build complete");
+  return process.exit(0);
+}
+
+// compare the new contents with the old
+const diff = child_process.spawnSync(
+  "/opt/homebrew/bin/diff",
+  ["--color", "-r", PREVIOUS, OUTPUT],
+  { stdio: "inherit" }
+);
+
+if (diff.error) {
+  console.error(diff.error);
+} else {
+  // ask the user to confirm the changes
+  // if y, exit with success
+  // if n, restore the previous contents to the OUTPUT directory
+  // and remove the PREVIOUS directory
+  // if anything else, ask the user to confirm again
+
+  const readline = require("readline");
+
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  const question = () => {
+    rl.question("Do you want to keep these changes? [y/n] ", (answer) => {
+      if (answer === "y") {
+        console.log("Changes kept. Build complete");
+        rl.close();
+      } else if (answer === "n") {
+        console.log("Changes discarded");
+        fs.removeSync(OUTPUT);
+        fs.moveSync(PREVIOUS, OUTPUT);
+        fs.removeSync(PREVIOUS);
+        console.log("Done");
+        rl.close();
+        // exit with failure
+        process.exit(1);
+      } else {
+        console.log("Please answer 'y' or 'n'");
+        question();
+      }
+    });
+  };
+
+  question();
+}