Private key filename

CMakeLists.txt

@@ -21,6 +21,7 @@
     src/gsheets_requests.cpp
     src/gsheets_read.cpp
     src/gsheets_utils.cpp
+    src/gsheets_get_token.cpp
 )
 
 build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})

docs/pages/index.md

@@ -39,6 +39,15 @@ CREATE SECRET (
     PROVIDER access_token, 
     TOKEN '<your_token>'
 );
+
+-- OR create a non-expiring JSON secret with your Google API private key 
+-- (This enables use in non-interactive workflows like data pipelines)
+-- (see "Getting a Google API Access Private Key" below)
+CREATE SECRET (
+    TYPE gsheet, 
+    PROVIDER private_key, 
+    FILENAME '<path_to_JSON_file_with_private_key>'
+);
 ```
 
 ### Read
@@ -103,6 +112,23 @@ To connect DuckDB to Google Sheets via an access token, you’ll need to create
 
 This token will periodically expire - you can re-run the above command again to generate a new one.
 
+## Getting a Google API Access Private Key
+
+Follow steps 1-9 above to get a JSON file with your private key inside.
+
+Include the path to the file as the `FILENAME` parameter when creating a secret.
+The recommendation is to use an absolute path, not a relative one, and to store it in the `~/.duckdb` folder.
+You will need to be able to access this file while querying GSheets (its content are not persisted. Later versions of this extension may enable that.)
+Ex: `CREATE SECRET (TYPE gsheet, PROVIDER private_key, FILENAME '<path_to_JSON_file_with_private_key>');`
+
+You can skip steps 10, 11, and 12 since this extension will convert from your JSON file to a token on your behalf!
+
+Follow steps 13 and 14.
+
+This private key by default will not expire. Use caution with it. 
+
+This will also require an additional API request for every Google Sheets call, so it will take a small bit of extra time and you may want to use a token directly if you hit a rate limit of any kind.
+
 ## Limitations / Known Issues
 
 - Google Sheets has a limit of 1,000,000 cells per spreadsheet.

src/gsheets_auth.cpp

@@ -6,6 +6,9 @@
 #include "duckdb/main/extension_util.hpp"
 #include <fstream>
 #include <cstdlib>
+#include <json.hpp>
+
+using json = nlohmann::json;
 
 namespace duckdb
 {
@@ -78,6 +81,22 @@ namespace duckdb
         return std::move(result);
     }
 
+    // TODO: Maybe this should be a KeyValueSecret
+    static unique_ptr<BaseSecret> CreateGsheetSecretFromPrivateKey(ClientContext &context, CreateSecretInput &input) {
+        auto scope = input.scope;
+
+        auto result = make_uniq<KeyValueSecret>(scope, input.type, input.provider, input.name);
+
+        // Manage specific secret option
+        CopySecret("filename", input, *result);
+
+        // Redact sensible keys
+        RedactCommonKeys(*result);
+        result->redact_keys.insert("filename");
+
+        return std::move(result);
+    }
+
     void CreateGsheetSecretFunctions::Register(DatabaseInstance &instance)
     {
         string type = "gsheet";
@@ -100,6 +119,12 @@ namespace duckdb
         oauth_function.named_parameters["use_oauth"] = LogicalType::BOOLEAN;
         RegisterCommonSecretParameters(oauth_function);
         ExtensionUtil::RegisterFunction(instance, oauth_function);
+
+        // Register the private key secret provider
+        CreateSecretFunction private_key_function = {type, "private_key", CreateGsheetSecretFromPrivateKey};
+        private_key_function.named_parameters["filename"] = LogicalType::VARCHAR;
+        RegisterCommonSecretParameters(private_key_function);
+        ExtensionUtil::RegisterFunction(instance, private_key_function);
     }
 
     std::string InitiateOAuthFlow()

src/gsheets_copy.cpp

@@ -7,6 +7,9 @@
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/main/secret/secret_manager.hpp"
 #include <json.hpp>
+#include <regex>
+#include "gsheets_get_token.hpp"
+#include <iostream>
 
 using json = nlohmann::json;
 
@@ -49,12 +52,38 @@ namespace duckdb
             throw InvalidInputException("Invalid secret format for 'gsheet' secret");
         }
 
-        Value token_value;
-        if (!kv_secret->TryGetValue("token", token_value)) {
-            throw InvalidInputException("'token' not found in 'gsheet' secret");
-        }
+        std::string token;
+
+        if (secret.GetProvider() == "private_key") {
+            // If using a private key, retrieve the private key from the secret, but convert it 
+            // into a token before use. This is an extra request per Google Sheet read or copy.
+            // The secret is the JSON file that is extracted from Google as per the README
+
+
+            Value filename_value;
+            if (!kv_secret->TryGetValue("filename", filename_value)) {
+                throw InvalidInputException("'filename' not found in 'gsheet' secret");
+            }
+            std::string filename_string = filename_value.ToString();
+
+            std::string token_plus_metadata = get_token(filename_string);
+
+            json token_json = parseJson(token_plus_metadata);
+
+            json failure_string = "failure";
+            if (token_json["status"] == failure_string) {
+                throw InvalidInputException("Conversion from private key to token failed. Check email, key format in JSON file (-----BEGIN PRIVATE KEY-----\\n ... -----END PRIVATE KEY-----\\n), and expiration date.");
+            }
+
+            token = token_json["access_token"].get<std::string>();
+        } else {
+            Value token_value;
+            if (!kv_secret->TryGetValue("token", token_value)) {
+                throw InvalidInputException("'token' not found in 'gsheet' secret");
+            }
 
-        std::string token = token_value.ToString();
+            token = token_value.ToString();
+        }
         std::string spreadsheet_id = extract_spreadsheet_id(file_path);
         std::string sheet_id = extract_sheet_id(file_path);
         std::string sheet_name = "Sheet1";

src/gsheets_get_token.cpp

@@ -0,0 +1,142 @@
+// Taken with modifications from https://gist.github.com/niuk/6365b819a86a7e0b92d82328fcf87da5
+#include <stdio.h>  
+#include <string.h>
+#include <stdlib.h>
+
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+
+#define CPPHTTPLIB_OPENSSL_SUPPORT
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <httplib.hpp>
+
+#include <json.hpp>
+using json = nlohmann::json;
+namespace duckdb
+{
+
+    char get_base64_char(char byte) {
+        if (byte < 26) {
+            return 'A' + byte;
+        } else if (byte < 52) {
+            return 'a' + byte - 26;
+        } else if (byte < 62) {
+            return '0' + byte - 52;
+        } else if (byte == 62) {
+            return '-';
+        } else if (byte == 63) {
+            return '_';
+        } else {
+            fprintf(stderr, "BAD BYTE: %02x\n", byte);
+            exit(1);
+            return 0;
+        }
+    }
+
+    // To execute C, please define "int main()"
+    void base64encode(char *output, const char *input, size_t input_length) {
+        size_t input_index = 0;
+        size_t output_index = 0;
+        for (; input_index < input_length; ++output_index) {
+            switch (output_index % 4) {
+            case 0:
+                output[output_index] = get_base64_char((0xfc & input[input_index]) >> 2);
+                break;
+            case 1:
+                output[output_index] = get_base64_char(((0x03 & input[input_index]) << 4) | ((0xf0 & input[input_index + 1]) >> 4));
+                ++input_index;
+                break;
+            case 2:
+                output[output_index] = get_base64_char(((0x0f & input[input_index]) << 2) | ((0xc0 & input[input_index + 1]) >> 6));
+                ++input_index;
+                break;
+            case 3:
+                output[output_index] = get_base64_char(0x3f & input[input_index]);
+                ++input_index;
+                break;
+            default:
+                exit(1);
+            }
+        }
+
+        output[output_index] = '\0';
+    }
+
+    std::string get_token(const std::string& filename) {
+        const char *header = "{\"alg\":\"RS256\",\"typ\":\"JWT\"}";
+
+        /* Create jwt claim set */
+        json jwt_claim_set;
+        std::time_t t = std::time(NULL);
+        std::ifstream ifs(filename);
+        json credentials_file = json::parse(ifs);
+        std::string email = credentials_file["client_email"].get<std::string>();
+
+        jwt_claim_set["iss"] = email; /* service account email address */
+        jwt_claim_set["scope"] = "https://www.googleapis.com/auth/spreadsheets" /* scope of requested access token */;
+        jwt_claim_set["aud"] = "https://accounts.google.com/o/oauth2/token"; /* intended target of the assertion for an access token */
+        jwt_claim_set["iat"] = std::to_string(t); /* issued time */
+        // Since we get a new token on every request (and max request time is 3 minutes), 
+        // set the limit to 10 minutes. (Max that Google allows is 1 hour)
+        jwt_claim_set["exp"] = std::to_string(t+600); /* expire time*/
+
+        char header_64[1024];
+        base64encode(header_64, header, strlen(header));
+
+        char claim_set_64[1024];
+        base64encode(claim_set_64, jwt_claim_set.dump().c_str(), strlen(jwt_claim_set.dump().c_str()));
+
+        char input[1024];
+        int input_length = sprintf(input, "%s.%s", header_64, claim_set_64);
+
+        unsigned char *digest = SHA256((const unsigned char *)input, input_length, NULL);
+        char digest_str[1024];
+        for (int i = 0; i < SHA256_DIGEST_LENGTH; ++i) {
+            sprintf(digest_str + i * 2, "%02x", digest[i]);
+        }
+
+        digest_str[SHA256_DIGEST_LENGTH * 2] = '\0';
+
+        std::string private_key_string = credentials_file["private_key"].get<std::string>();
+
+        BIO* bio = BIO_new(BIO_s_mem());
+        const void * private_key_pointer = private_key_string.c_str();
+        int private_key_length = std::strlen(private_key_string.c_str());
+        BIO_write(bio, private_key_pointer, private_key_length);
+        EVP_PKEY* evp_key = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL);
+        RSA* rsa = EVP_PKEY_get1_RSA(evp_key);
+
+        if (rsa != NULL) {
+            unsigned char sigret[4096] = {};
+            unsigned int siglen;
+            if (RSA_sign(NID_sha256, digest, SHA256_DIGEST_LENGTH, sigret, &siglen, rsa)) {
+                if (RSA_verify(NID_sha256, digest, SHA256_DIGEST_LENGTH, sigret, siglen, rsa)) {
+                    char signature_64[1024];
+                    base64encode(signature_64, (const char *)sigret, siglen);
+
+                    char jwt[1024];
+                    sprintf(jwt, "%s.%s", input, signature_64);
+
+                    duckdb_httplib_openssl::Client cli("https://oauth2.googleapis.com");
+                    duckdb_httplib_openssl::Params params;
+                    params.emplace("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer");
+                    params.emplace("assertion", jwt);
+                    auto result = cli.Post("/token", params);
+                    return (result -> body);
+                } else {
+                    printf("Could not verify RSA signature.");
+                }
+            } else {
+                unsigned long err = ERR_get_error();
+                printf("RSA_sign failed: %lu, %s\n", err, ERR_error_string(err, NULL));
+            }
+
+            RSA_free(rsa);
+        }
+        std::string failure_message = "{\"status\":\"failed\"}";
+
+        return failure_message;
+    }
+}

src/gsheets_read.cpp

@@ -5,6 +5,9 @@
 #include "duckdb/main/secret/secret_manager.hpp"
 #include "gsheets_requests.hpp"
 #include <json.hpp>
+#include <regex>
+#include "gsheets_get_token.hpp"
+#include <iostream>
 
 namespace duckdb {
 
@@ -129,13 +132,38 @@ unique_ptr<FunctionData> ReadSheetBind(ClientContext &context, TableFunctionBind
         throw InvalidInputException("Invalid secret format for 'gsheet' secret");
     }
 
-    Value token_value;
-    if (!kv_secret->TryGetValue("token", token_value)) {
-        throw InvalidInputException("'token' not found in 'gsheet' secret");
-    }
+    std::string token;
+
+    if (secret.GetProvider() == "private_key") {
+        // If using a private key, retrieve the private key from the secret, but convert it 
+        // into a token before use. This is an extra request per Google Sheet read or copy.
+        // The secret is the JSON file that is extracted from Google as per the README
+
 
-    std::string token = token_value.ToString();
+        Value filename_value;
+        if (!kv_secret->TryGetValue("filename", filename_value)) {
+            throw InvalidInputException("'filename' not found in 'gsheet' secret");
+        }
+        std::string filename_string = filename_value.ToString();
+
+        std::string token_plus_metadata = get_token(filename_string);
+
+        json token_json = parseJson(token_plus_metadata);
+
+        json failure_string = "failure";
+        if (token_json["status"] == failure_string) {
+            throw InvalidInputException("Conversion from private key to token failed. Check email, key format in JSON file (-----BEGIN PRIVATE KEY-----\\n ... -----END PRIVATE KEY-----\\n), and expiration date.");
+        }
 
+        token = token_json["access_token"].get<std::string>();
+    } else {
+        Value token_value;
+        if (!kv_secret->TryGetValue("token", token_value)) {
+            throw InvalidInputException("'token' not found in 'gsheet' secret");
+        }
+
+        token = token_value.ToString();
+    }
     // Get sheet name from URL
     std::string sheet_id = extract_sheet_id(sheet_input);
     sheet_name = get_sheet_name_from_id(spreadsheet_id, sheet_id, token);

src/include/gsheets_get_token.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <string>
+#include <stdlib.h>
+
+namespace duckdb {
+
+    char get_base64_char(char byte);
+
+    void base64encode(char *output, const char *input, size_t input_length) ;
+
+    std::string get_token(const std::string& filename) ;
+
+}

third_party/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 yhirose
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+