Get endpoint_url from s3 env values

Author: Korel

pipelines/lightweight-components/mobile-price-classifications.ipynb

@@ -86,9 +86,13 @@
     "# Upload dataset to MinIO\n",
     "aws_access_key_id = os.getenv(\"AWS_ACCESS_KEY_ID\", \"\")\n",
     "aws_secret_access_key = os.getenv(\"AWS_SECRET_ACCESS_KEY\", \"\")\n",
+    "s3_endpoint = os.getenv(\"S3_ENDPOINT\", \"minio.minio\")\n",
+    "s3_use_https = os.getenv(\"S3_USE_HTTPS\", \"0\") == \"1\"\n",
+    "s3_verify_ssl = os.getenv(\"S3_VERIFY_SSL\", \"0\") == \"1\"\n",
+    "s3_endpoint = f\"http{'s' if s3_use_https else ''}://{s3_endpoint}\"\n",
     "if not aws_access_key_id or not aws_secret_access_key:\n",
     "    raise ValueError(\"AWS credentials not found in environment variables.\")\n",
-    "s3 = s3fs.S3FileSystem(endpoint_url=\"http://minio.minio\", key=aws_access_key_id, secret=aws_secret_access_key)\n",
+    "s3 = s3fs.S3FileSystem(endpoint_url=s3_endpoint, key=aws_access_key_id, secret=aws_secret_access_key, use_ssl=s3_verify_ssl)\n",
     "s3_dataset_path = f\"{s3_bucket}/mobile-price-classification\"\n",
     "s3.put(f\"{dataset_path}/train.csv\", f\"{s3_dataset_path}/train.csv\")\n",
     "s3.put(f\"{dataset_path}/train.csv\", f\"{s3_dataset_path}/test.csv\")\n",