Merge pull request #2591 from project-koku/rc-2021.01.18

Rc 2021.01.18

Release approved by dev lead and QE

Author: Red-HAP

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 
 # text editor temp files
 *.swp
+*.swo
 
 # C extensions
 *.so

Pipfile

@@ -69,6 +69,7 @@ model-bakery = ">=1.1"
 koku-nise = ">=2.2"
 pre-commit = ">=2.0"
 responses = ">=0.10"
+matplotlib = ">=3.3"
 
 [requires]
 python_version = "3.8"

Pipfile.lock

@@ -65,9 +65,8 @@ ln -s -f /hive-config/hive-exec-log4j2.properties $HIVE_HOME/conf/hive-exec-log4
 
 export HADOOP_LOG_DIR="${HADOOP_HOME}/logs"
 # Set garbage collection settings
-#export GC_SETTINGS="-XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${HADOOP_LOG_DIR}/heap_dump.bin -XX:+ExitOnOutOfMemoryError -XX:ErrorFile=${HADOOP_LOG_DIR}/java_error%p.log"
-
-export VM_OPTIONS="$VM_OPTIONS -XX:+UseContainerSupport -XX:ErrorFile=${HADOOP_LOG_DIR}/java_error%p.log"
+export GC_SETTINGS="-XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${HADOOP_LOG_DIR}/heap_dump.bin -XX:+ExitOnOutOfMemoryError -XX:ErrorFile=${HADOOP_LOG_DIR}/java_error%p.log"
+export VM_OPTIONS="$VM_OPTIONS -XX:+UseContainerSupport"
 
 if [ -n "$JVM_INITIAL_RAM_PERCENTAGE" ]; then
 VM_OPTIONS="$VM_OPTIONS -XX:InitialRAMPercentage=$JVM_INITIAL_RAM_PERCENTAGE"
@@ -83,7 +82,7 @@ fi
 export JMX_OPTIONS="-javaagent:/opt/jmx_exporter/jmx_exporter.jar=8082:/opt/jmx_exporter/config/config.yml -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=8081 -Dcom.sun.management.jmxremote.rmi.port=8081 -Djava.rmi.server.hostname=127.0.0.1"
 
 # Set garbage collection logs
-export GC_SETTINGS="${GC_SETTINGS} -verbose:gc -Xlog:gc*:${HADOOP_LOG_DIR}/gc.log"
+export GC_SETTINGS="${GC_SETTINGS} -verbose:gc"
 
 export HIVE_LOGLEVEL="${HIVE_LOGLEVEL:-INFO}"
 export HADOOP_OPTS="${HADOOP_OPTS} ${VM_OPTIONS} ${GC_SETTINGS} ${JMX_OPTIONS}"

deploy/metastore/hive-scripts/entrypoint.sh

@@ -51,7 +51,7 @@ fi
 
 # add UID to /etc/passwd if missing
 if ! whoami &> /dev/null; then
-    if [ -w /etc/passwd ]; then
+    if test -w /etc/passwd || stat -c "%a" /etc/passwd | grep -qE '.[267].'; then
         echo "Adding user ${USER_NAME:-presto} with current UID $(id -u) to /etc/passwd"
         # Remove existing entry with user first.
         # cannot use sed -i because we do not have permission to write new

deploy/presto/presto-common-config/entrypoint.sh

@@ -268,6 +268,11 @@ services:
         - './testing/local_providers/aws_local_4:/tmp/local_bucket_4'
         - './testing/local_providers/aws_local_5:/tmp/local_bucket_5'
         - './testing/local_providers/insights_local:/var/tmp/masu/insights_local'
+        - './testing/local_providers/gcp_local/:/tmp/gcp_local_bucket'
+        - './testing/local_providers/gcp_local_0/:/tmp/gcp_local_bucket_0'
+        - './testing/local_providers/gcp_local_1/:/tmp/gcp_local_bucket_1'
+        - './testing/local_providers/gcp_local_2/:/tmp/gcp_local_bucket_2'
+        - './testing/local_providers/gcp_local_3/:/tmp/gcp_local_bucket_3'
       links:
           - koku-rabbit
       depends_on:

docker-compose.yml

@@ -1,104 +1,105 @@
--i https://pypi.python.org/simple
+-i http://localhost:3141/root/pypi/+simple/
+--trusted-host localhost:3141
 adal==1.2.5
-amqp==2.6.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-asgiref==3.3.1; python_version >= '3.5'
+amqp==2.6.1
+asgiref==3.3.1
 azure-common==1.1.26
-azure-core==1.9.0
+azure-core==1.10.0
 azure-mgmt-core==1.2.2
 azure-mgmt-costmanagement==0.2.0
 azure-mgmt-resource==15.0.0
 azure-mgmt-storage==11.2.0
-azure-storage-blob==12.6.0
+azure-storage-blob==12.7.0
 beautifulsoup4==4.9.3
 billiard==3.6.3.0
-boto3==1.16.35
-botocore==1.19.35
+boto3==1.16.55
+botocore==1.19.55
 bs4==0.0.1
 cachetools==4.2.0
 celery-prometheus-exporter==1.7.0
 celery==4.4.7
 certifi==2020.12.5
 cffi==1.14.4
-chardet==3.0.4
+chardet==4.0.0
 ciso8601==2.1.3
-click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+click==7.1.2
 confluent-kafka==1.5.0
-cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+cryptography==3.3.1
 django-cors-headers==3.6.0
 django-environ==0.4.5
 django-extensions==3.1.0
 django-filter==2.4.0
 django-prometheus==2.1.0
 django-redis==4.12.1
 django-tenant-schemas==1.10.0
-django==3.1.4
+django==3.1.5
 djangorestframework-csv==2.1.0
 djangorestframework==3.12.2
 flake8==3.8.4
-google-api-core[grpc]==1.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-api-core[grpc]==1.25.0
 google-api-python-client==1.12.8
 google-auth-httplib2==0.0.4
 google-auth==1.24.0
-google-cloud-bigquery==2.6.1
-google-cloud-core==1.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-google-crc32c==1.0.0; python_version >= '3.5'
-google-resumable-media==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-googleapis-common-protos==1.52.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-grpcio==1.34.0
+google-cloud-bigquery==2.6.2
+google-cloud-core==1.5.0
+google-crc32c==1.1.1; python_version >= '3.5'
+google-resumable-media==1.2.0
+googleapis-common-protos==1.52.0
+grpcio==1.34.1
 gunicorn==20.0.4
 httplib2==0.18.1
-idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-importlib-metadata==3.3.0
+idna==2.10
+importlib-metadata==3.4.0
 isodate==0.6.0
-jinja2==2.11.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+jinja2==2.11.2
 jinjasql==0.1.8
-jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+jmespath==0.10.0
 kafka-python==2.0.2
 kombu==4.6.11
-markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+markupsafe==1.1.1
 mccabe==0.6.1
 msrest==0.6.19
 msrestazure==0.6.4
-numpy==1.19.4; python_version >= '3.6'
-oauthlib==3.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-ordered-set==4.0.2; python_version >= '3.5'
-packaging==20.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pandas==1.1.5
+numpy==1.19.5
+oauthlib==3.1.0
+ordered-set==4.0.2
+packaging==20.8
+pandas==1.2.0
 patsy==0.5.1
 pint==0.16.1
 presto-python-client==0.7.0
 prometheus-client==0.9.0
-proto-plus==1.13.0; python_version >= '3.6'
+proto-plus==1.13.0
 protobuf==3.14.0
-psutil==5.7.3
-psycopg2-binary==2.8.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+psutil==5.8.0
+psycopg2-binary==2.8.6
 pyarrow==2.0.0
 pyasn1-modules==0.2.8
 pyasn1==0.4.8
-pycodestyle==2.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pyflakes==2.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pyjwt==1.7.1
-pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pycodestyle==2.6.0
+pycparser==2.20
+pyflakes==2.2.0
+pyjwt==2.0.0
+pyparsing==2.4.7
 python-dateutil==2.8.1
-pytz==2020.4
+pytz==2020.5
 querystring-parser==1.2.4
-redis==3.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+redis==3.5.3
 requests-oauthlib==1.3.0
-requests==2.25.0
-rsa==4.6; python_version >= '3.6'
-s3transfer==0.3.3
-scipy==1.5.4; python_version >= '3.6'
+requests==2.25.1
+rsa==4.7; python_version >= '3.6'
+s3transfer==0.3.4
+scipy==1.6.0
 sentry-sdk==0.19.5
-six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+six==1.15.0
 soupsieve==2.1; python_version >= '3.0'
-sqlparse==0.4.1; python_version >= '3.5'
+sqlparse==0.4.1
 statsmodels==0.12.1
 ujson==4.0.1
 unicodecsv==0.14.1
-uritemplate==3.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+uritemplate==3.0.1
 urllib3==1.26.2; python_version != '3.4'
-vine==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-watchtower==1.0.0
+vine==1.3.0
+watchtower==1.0.5
 whitenoise==5.2.0
-zipp==3.4.0; python_version >= '3.6'
+zipp==3.4.0

docs/rtd_requirements.txt

@@ -45,6 +45,12 @@ Example DEVELOPMENT_IDENTITY object::
                     "aws.account": {
                         "read": ["1234567890AB", "234567890AB1"]
                     }
+                    "gcp.account": {
+                        "read": ["*"]
+                    }
+                    "gcp.project": {
+                        "read": ["*"]
+                    }
                     "azure.subscription_guid": {
                         "read": ["*"]
                     }