Skip to content

Commit 105dd46

Browse files
gerrod3gerrod3
authored
Merge pull request #945 from gerrod3/update-storage-settings-main
Update Azure and S3 storage settings for main
2 parents ea3bdd4 + 60d9a1a commit 105dd46

File tree

3 files changed

+31
-129
lines changed

3 files changed

+31
-129
lines changed

‎.ci/ansible/settings.py.j2‎

Lines changed: 0 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -26,127 +26,3 @@ API_ROOT = {{ api_root | repr }}
2626
{% endfor %}
2727
{% endif %}
2828

29-
{# =======================================
30-
Macros for legacy and new storage settings
31-
========================================== -#}
32-
33-
{%- macro s3_settings(legacy) -%}
34-
{%- if legacy %}
35-
DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
36-
AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
37-
AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
38-
AWS_S3_REGION_NAME = "eu-central-1"
39-
AWS_S3_ADDRESSING_STYLE = "path"
40-
AWS_S3_SIGNATURE_VERSION = "s3v4"
41-
AWS_STORAGE_BUCKET_NAME = "pulp3"
42-
AWS_S3_ENDPOINT_URL = "http://minio:9000"
43-
AWS_DEFAULT_ACL = "@none None"
44-
{%- else %}
45-
STORAGES = {
46-
"default": {
47-
"BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
48-
"OPTIONS": {
49-
"access_key": "{{ minio_access_key }}",
50-
"secret_key": "{{ minio_secret_key }}",
51-
"region_name": "eu-central-1",
52-
"addressing_style": "path",
53-
"signature_version": "s3v4",
54-
"bucket_name": "pulp3",
55-
"endpoint_url": "http://minio:9000",
56-
"default_acl": "@none None",
57-
},
58-
},
59-
"staticfiles": {
60-
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
61-
},
62-
}
63-
{%- endif %}
64-
{%- endmacro -%}
65-
66-
{%- macro azure_settings(legacy) -%}
67-
{%- if legacy %}
68-
DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
69-
AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
70-
AZURE_ACCOUNT_NAME = "devstoreaccount1"
71-
AZURE_CONTAINER = "pulp-test"
72-
AZURE_LOCATION = "pulp3"
73-
AZURE_OVERWRITE_FILES = True
74-
AZURE_URL_EXPIRATION_SECS = 120
75-
AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
76-
{%- else %}
77-
STORAGES = {
78-
"default": {
79-
"BACKEND": "storages.backends.azure_storage.AzureStorage",
80-
"OPTIONS": {
81-
"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
82-
"account_name": "devstoreaccount1",
83-
"location": "pulp3",
84-
"azure_container": "pulp-test",
85-
"overwrite_files": True,
86-
"expiration_secs": 120,
87-
"connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;",
88-
},
89-
},
90-
"staticfiles": {
91-
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
92-
},
93-
}
94-
{%- endif %}
95-
{%- endmacro -%}
96-
97-
{%- macro gcp_settings(legacy) -%}
98-
{%- if legacy %}
99-
DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
100-
GS_BUCKET_NAME = "gcppulp"
101-
GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
102-
{%- else %}
103-
STORAGES = {
104-
"default": {
105-
"BACKEND": "storages.backends.gcloud.GoogleCloudStorage",
106-
"OPTIONS": {
107-
"bucket_name": "gcppulp",
108-
"custom_endpoint": "http://ci-gcp:4443",
109-
},
110-
},
111-
"staticfiles": {
112-
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
113-
},
114-
}
115-
{%- endif %}
116-
{%- endmacro -%}
117-
118-
{#- ==========================================
119-
Render according to test_storages_compat_layer
120-
==============================================
121-
122-
Case 1) test_storages_compat_layer is unset
123-
- All storages render the legacy setting
124-
- Branches using pulpcore <3.70 must leave this key unset (use legacy)
125-
126-
Case 2) test_storages_compat_layer is True
127-
- To tests both work, only one setting uses the new storage setting
128-
- Branches using pulpcore >=3.70,<3.85 must set this key to True (test both)
129-
130-
Case 3) test_storages_compat_layer is False
131-
- All storages render the new setting
132-
- Branches using pulpcore >=3.85 must set this key to False (use new)
133-
-#}
134-
135-
{% if s3_test | default(false) or azure_test | default(false) or gcp_test | default(false)%}
136-
MEDIA_ROOT=""
137-
{% endif %}
138-
{%- if test_storages_compat_layer is not defined -%}
139-
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=True) }}{%- endif -%}
140-
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
141-
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
142-
{%- else -%}
143-
{%- if test_storages_compat_layer is true -%}
144-
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
145-
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
146-
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
147-
{%- elif test_storages_compat_layer is false -%}
148-
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
149-
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=False) }}{%- endif -%}
150-
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=False) }}{%- endif -%}
151-
{%- endif -%}
152-
{%- endif -%}

‎.github/workflows/scripts/install.sh‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,8 @@ if [ "$TEST" = "s3" ]; then
9595
sed -i -e '$a s3_test: true\
9696
minio_access_key: "'$MINIO_ACCESS_KEY'"\
9797
minio_secret_key: "'$MINIO_SECRET_KEY'"\
98-
pulp_scenario_settings: {"domain_enabled": true}\
98+
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true}\
9999
pulp_scenario_env: {}\
100-
test_storages_compat_layer: false\
101100
' vars/main.yaml
102101
export PULP_API_ROOT="/rerouted/djnd/"
103102
fi
@@ -110,9 +109,8 @@ if [ "$TEST" = "azure" ]; then
110109
- ./azurite:/etc/pulp\
111110
command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml
112111
sed -i -e '$a azure_test: true\
113-
pulp_scenario_settings: {"content_origin": null, "domain_enabled": true}\
112+
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "content_origin": null, "domain_enabled": true}\
114113
pulp_scenario_env: {}\
115-
test_storages_compat_layer: false\
116114
' vars/main.yaml
117115
fi
118116

‎template_config.yml‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,39 @@ pulp_settings:
4747
orphan_protection_time: 0
4848
pypi_api_hostname: https://pulp:443
4949
pulp_settings_azure:
50+
MEDIA_ROOT: ''
51+
STORAGES:
52+
default:
53+
BACKEND: storages.backends.azure_storage.AzureStorage
54+
OPTIONS:
55+
account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
56+
account_name: devstoreaccount1
57+
azure_container: pulp-test
58+
connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;
59+
expiration_secs: 120
60+
location: pulp3
61+
overwrite_files: true
62+
staticfiles:
63+
BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
5064
content_origin: null
5165
domain_enabled: true
5266
pulp_settings_gcp: null
5367
pulp_settings_s3:
68+
MEDIA_ROOT: ''
69+
STORAGES:
70+
default:
71+
BACKEND: storages.backends.s3boto3.S3Boto3Storage
72+
OPTIONS:
73+
access_key: AKIAIT2Z5TDYPX3ARJBA
74+
addressing_style: path
75+
bucket_name: pulp3
76+
default_acl: '@none'
77+
endpoint_url: http://minio:9000
78+
region_name: eu-central-1
79+
secret_key: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS
80+
signature_version: s3v4
81+
staticfiles:
82+
BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
5483
domain_enabled: true
5584
pydocstyle: true
5685
release_email: [email protected]
@@ -73,6 +102,5 @@ test_lowerbounds: true
73102
test_performance: false
74103
test_reroute: true
75104
test_s3: true
76-
test_storages_compat_layer: false
77105
use_issue_template: true
78106

0 commit comments

Comments
 (0)