Version 1.0

Author: krisjan

README.md

@@ -0,0 +1 @@
+Link to related article: https://stacktonic.com/article/create-a-user-profile-and-recommender-service-using-big-query-redis-and-gtm-server

gcf-cloudstorage-to-redis/index.js

@@ -0,0 +1,76 @@
+/**
+ * Author    Krisjan Oldekamp / Stacktonic.com
+ * Email     [email protected]
+ * Article   https://stacktonic.com/article/create-a-fast-user-profile-and-recommender-service-using-big-query-redis-and-gtm-server
+ * Based on  https://futurice.com/blog/bigquery-to-memorystore
+ */
+
+'use strict';
+
+const Redis = require('ioredis');
+const {Storage} = require('@google-cloud/storage');
+const split = require('split');
+
+const REDISHOST = process.env.REDISHOST || 'localhost';
+const REDISPORT = process.env.REDISPORT || 6379;
+const EXPIRATION = process.env.EXPIRATION || 259200;  // Expiration of records in seconds
+const FILE_PATH_OUTBOUND = process.env.FILE_PATH_OUTBOUND || 'outbound'; 
+const FILE_PATH_PROCESSED = process.env.FILE_PATH_PROCESSED || 'processed'; 
+const FILE_PREFIX = process.env.FILE_PREFIX || ''; 
+
+const redisClient = new Redis({
+  host: REDISHOST,
+  port: REDISPORT,
+});
+
+/**
+ * Triggered from a change to a Cloud Storage bucket.
+ *
+ * @param {!Object} event Event payload.
+ * @param {!Object} context Metadata for the event.
+ */
+exports.loadCloudStorageToRedis = async(info, context) => {
+  
+  const path = info.name.split('/')
+  const fileName = path[path.length-1]
+
+  if (info.metageneration === '1' && info.name.startsWith(FILE_PATH_OUTBOUND) && fileName.startsWith(FILE_PREFIX)) {
+
+    console.log(`New file upload: gs://${info.bucket}/${info.name}`)
+
+    const storage = new Storage()
+    const bucket = storage.bucket(info.bucket);
+    const file = bucket.file(info.name);
+
+    let keysWritten = 0;
+
+    try {
+      
+      // Read file and send to Redis
+      file.createReadStream()
+        .on('error', error => reject(error))
+        .on('response', (response) => {
+          // connection to GCS opened
+        }).pipe(split())
+        .on('data', function (record) {
+          if (!record || record === "") return;
+          keysWritten++;
+          const data = JSON.parse(record);
+          redisClient.set(data.key, record, 'EX', EXPIRATION);
+        })
+        .on('end', () => {
+          console.log(`Successfully written ${keysWritten} keys to Memcache Redis.`);
+
+          // Move file to processed folder
+          bucket.file(info.name).move(FILE_PATH_PROCESSED + '/' + fileName)
+          console.log(`File moved to: ${info.bucket}/${FILE_PATH_PROCESSED}/${fileName}`)
+
+          resolve();
+        })
+        .on('error', error => reject(error));
+    
+    } catch(e) {
+      console.log(`Error importing ${fileName} to Redis: ${e}`)
+    }
+  }
+};

gcf-cloudstorage-to-redis/package.json

@@ -0,0 +1,9 @@
+{
+    "name": "gcf-cloudstorage-to-redis",
+    "version": "1.0.0",
+    "dependencies": {
+        "@google-cloud/storage": "^5.14.5",
+        "ioredis": "^4.27.10",
+        "split": "^1.0.1"
+    }
+}

gcf-cloudstorage-to-redis/user_store_test_file.json

@@ -0,0 +1,4 @@
+{"key": "123", "audience_1": "1", "audience_2": "1", "ab_test_1": "3" }
+{"key": "234", "audience_1": "0", "audience_2": "1", "ab_test_2": "2" }
+{"key": "456", "audience_1": "1", "audience_2": "0", "ab_test_3": "1" }
+{"key": "777", "audience_1": "1", "audience_2": "0", "ab_test_4": "3" }

gcf-http-serve-from-redis/index.js

@@ -0,0 +1,56 @@
+/**
+ * Author    Krisjan Oldekamp / Stacktonic.com
+ * Email     [email protected]
+ * Article   https://stacktonic.com/article/
+ */
+
+'use strict'
+
+const Redis = require('ioredis');
+
+const REDISHOST = process.env.REDISHOST || 'localhost';
+const REDISPORT = process.env.REDISPORT || 6379;
+const QUERY_REDIS_KEY = process.env.QUERY_REDIS_KEY || 'id';
+const QUERY_SECRET_KEY = process.env.QUERY_SECRET_KEY || 'secret';
+const SECRET = process.env.SECRET || '';
+
+const redisClient = new Redis({
+  host: REDISHOST,
+  port: REDISPORT,
+});
+
+/**
+ * HTTP function 
+ * 
+ * @param {Object} req Cloud Function request context.
+ * @param {Object} res Cloud Function response context.
+ */
+exports.httpServeFromRedis = (req, res) => {
+
+    res.set('Access-Control-Allow-Origin', "*");
+    res.set('Access-Control-Allow-Methods', 'GET');
+
+    if (SECRET == '' || (req.query[QUERY_SECRET_KEY] == SECRET)) {
+        if (typeof req.query[QUERY_REDIS_KEY] != 'undefined') {
+            const key = req.query[QUERY_REDIS_KEY];
+            
+            redisClient.get(key, function (err, result) {
+                if (err) {
+                    console.error(err);
+                    res.status(400).json({ error: 'Error' });
+                } else {
+                    if (result !== null) {
+                        const data = JSON.parse(result);
+                        res.status(200).json(data);
+                    } else {
+                        res.status(400).json({ error: 'Key not found' });
+                    }
+                }
+            });
+        } else {
+            res.status(400).json( { error: 'No key specified' } );
+        }
+    } else {
+        res.status(401).json( { error: 'Unauthorized' } );
+    }
+};

gcf-http-serve-from-redis/package.json

@@ -0,0 +1,7 @@
+{
+    "name": "gcf-http-serve-from-redis",
+    "version": "1.0.0",
+    "dependencies": {
+        "ioredis": "^4.27.10"
+    }
+}