Add tests

.gitignore

@@ -354,5 +354,9 @@ Teradata/output/
 Teradata/log/
 Teradata/bin/execute_on_mac.sh
 
+# Tests files
+Tests/Teradata/extracted_code/
+Tests/Teradata/scripts/Teradata_Extraction
+
 # Autogenerated file
 *.DS_Store

Teradata/bin/create_ddls.sh

@@ -4,26 +4,8 @@
 #Version 20230811: Added command to copy the scripts from scripts_template.
 
 
-##### Modify the connection information
-connection_string="dbc,dbc"
-
-##### Modify the condition for the databases and/or objects to include.  
-##### You can change the operator 'LIKE ANY' to 'IN' or '=' 
-##### Use uppercase names.
-include_databases="(UPPER(T1.DATABASENAME) LIKE ANY ('%'))"
-
-##### Modify the condition for the databases to exclude.  
-##### Do not use the LIKE ANY in this condition if already used in the previous condition for include_databases
-##### Use uppercase names.
-exclude_databases="(UPPER(T1.DATABASENAME) NOT IN ('SYS_CALENDAR','ALL','CONSOLE','CRASHDUMPS','DBC','DBCMANAGER','DBCMNGR','DEFAULT','EXTERNAL_AP','EXTUSER','LOCKLOGSHREDDER','PDCRADM','PDCRDATA','PDCRINFO','PUBLIC','SQLJ','SYSADMIN','SYSBAR','SYSJDBC','SYSLIB','SYSSPATIAL','SYSTEMFE','SYSUDTLIB','SYSUIF','TD_SERVER_DB','TD_SYSFNLIB','TD_SYSFNLIB','TD_SYSGPL','TD_SYSXML','TDMAPS', 'TDPUSER','TDQCD','TDSTATS','TDWM','VIEWPOINT','PDCRSTG'))"
-
-##### Modify the condition to include specific object names (tables/views/procedures.  
-##### You can change the operator 'LIKE ANY' to 'IN' or '=' 
-##### Use uppercase names.
-include_objects="(UPPER(T1.TABLENAME) LIKE ANY ('%'))"
-
-###### Constant ddl_leng, max limit in dictionary table is 12500.
-ddl_leng_max_limit_dic=12400
+##### Import configuration variables.
+. ./parameters.sh
 
 ##### Creates directory for output and log files.
 mkdir -p ../log

Teradata/bin/parameters.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#Version 20231214: Script created
+
+##### Modify the connection information
+connection_string="dbc,dbc"
+
+##### Modify the condition for the databases and/or objects to include.  
+##### You can change the operator 'LIKE ANY' to 'IN' or '=' 
+##### Use uppercase names.
+include_databases="(UPPER(T1.DATABASENAME) LIKE ANY ('%'))"
+
+##### Modify the condition for the databases to exclude.  
+##### Do not use the LIKE ANY in this condition if already used in the previous condition for include_databases
+##### Use uppercase names.
+exclude_databases="(UPPER(T1.DATABASENAME) NOT IN ('SYS_CALENDAR','ALL','CONSOLE','CRASHDUMPS','DBC','DBCMANAGER','DBCMNGR','DEFAULT','EXTERNAL_AP','EXTUSER','LOCKLOGSHREDDER','PDCRADM','PDCRDATA','PDCRINFO','PUBLIC','SQLJ','SYSADMIN','SYSBAR','SYSJDBC','SYSLIB','SYSSPATIAL','SYSTEMFE','SYSUDTLIB','SYSUIF','TD_SERVER_DB','TD_SYSFNLIB','TD_SYSFNLIB','TD_SYSGPL','TD_SYSXML','TDMAPS', 'TDPUSER','TDQCD','TDSTATS','TDWM','VIEWPOINT','PDCRSTG'))"
+
+##### Modify the condition to include specific object names (tables/views/procedures.  
+##### You can change the operator 'LIKE ANY' to 'IN' or '=' 
+##### Use uppercase names.
+include_objects="(UPPER(T1.TABLENAME) LIKE ANY ('%'))"
+
+###### Constant ddl_leng, max limit in dictionary table is 12500.
+ddl_leng_max_limit_dic=12400

Tests/Teradata/License.txt

@@ -0,0 +1 @@
+Copyright 2021 Snowflake, Inc.Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the �Software�), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.THE SOFTWARE IS PROVIDED �AS IS�, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Tests/Teradata/README.md

@@ -0,0 +1,3 @@
+## Test files for Teradata
+
+TODO

Tests/Teradata/database_summary/__init__.py

@@ -0,0 +1 @@
+

Tests/Teradata/database_summary/database_source_code_summarizer.py

@@ -0,0 +1,50 @@
+import os
+import re
+from .database_source_code_summary import DatabaseSourceCodeSummary
+from .top_level_object_type import TopLevelObjectType
+
+def sumarize_database_source_code(path: str) -> DatabaseSourceCodeSummary:
+    database_summary = DatabaseSourceCodeSummary()
+
+    for dirpath, dirnames, filenames in os.walk(path):  
+        for filename in filenames: 
+            if not re.match(r'.*\.sql$', filename, flags=re.IGNORECASE):
+                continue
+            file_path = os.path.join(dirpath, filename) 
+            database_summary.add_sql_file(file_path)
+            sql_statements = read_sql_statements_from_file(file_path)
+            analyze_sql_statements(sql_statements, database_summary)
+    return database_summary
+
+def analyze_sql_statements(sql_statements: "list[str]", database_summary: DatabaseSourceCodeSummary):
+    for sql_statement in sql_statements:
+        type = get_sql_statement_type(sql_statement)
+        database_summary.get_top_level_object_to_int_map()[type] += 1
+
+def get_sql_statement_type(sql_statement: str) -> TopLevelObjectType:
+    if is_statement_of_type(sql_statement, TopLevelObjectType.PROCEDURE.name):
+        return TopLevelObjectType.PROCEDURE
+    elif is_statement_of_type(sql_statement, TopLevelObjectType.TRIGGER.name):
+        return TopLevelObjectType.TRIGGER
+    elif is_statement_of_type(sql_statement, TopLevelObjectType.TABLE.name):
+        return TopLevelObjectType.TABLE
+    elif is_statement_of_type(sql_statement, TopLevelObjectType.DATABASE.name):
+        return TopLevelObjectType.DATABASE
+    elif is_statement_of_type(sql_statement, TopLevelObjectType.VIEW.name):
+        return TopLevelObjectType.VIEW
+    else:
+        return TopLevelObjectType.UNDEFINED_TYPE
+
+def is_statement_of_type(statement: str, type_name: str) -> bool:
+    regex = r'^(?:/\*.*\*/|)\s*CREATE(?:\s*\w*\s*){0,2}' + type_name+ r'\s'
+    result = re.search(regex, statement, flags=re.IGNORECASE|re.MULTILINE)
+    return result
+
+def read_sql_statements_from_file(file_path: str) ->"list[str]":
+    with open(file_path) as my_file:
+        sql_statements = my_file.read().split(';')
+        return sql_statements
+
+
+
+

Tests/Teradata/database_summary/database_source_code_summary.py

@@ -0,0 +1,19 @@
+from .top_level_object_type import TopLevelObjectType
+class DatabaseSourceCodeSummary():
+    def __init__(self):
+        self._file_paths = []
+        self._top_level_object_to_int_map = {}
+        for top_level_object_type in TopLevelObjectType:
+            self._top_level_object_to_int_map[top_level_object_type] = 0
+
+    def get_count_of_files(self) -> int:
+        return len(self._file_paths)
+
+    def add_sql_file(self, file_path: str) -> None:
+        self._file_paths+=[file_path]
+
+    def get_top_level_object_to_int_map(self) -> "dict[TopLevelObjectType, int]":
+        return self._top_level_object_to_int_map
+
+
+

Tests/Teradata/database_summary/top_level_object_type.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+class TopLevelObjectType(Enum):
+    TABLE = 1
+    PROCEDURE = 2
+    VIEW = 3
+    DATABASE = 4
+    TRIGGER = 5
+    UNDEFINED_TYPE = 6

Tests/Teradata/scripts/config.sh

@@ -0,0 +1,3 @@
+#
+vm_connection="[email protected]"
+logon_command="dbc,dbc"

Tests/Teradata/scripts/execute_deploy_database_script.sh

@@ -0,0 +1,26 @@
+#
+#####Version 20230810: Script created
+
+#####Constants
+MESSAGE='\033[0;32m' # Green
+ERROR='\033[0;31m' # Red
+NC='\033[0m' # No Color
+
+#####Parameters
+source_code_folder_name="$1"
+if [ ! "$source_code_folder_name" ] || [ ! -d "../source_code/$source_code_folder_name/" ] ; then
+   echo "${ERROR}Invalid parameter '$source_code_folder_name', options are [$(ls ../source_code)]${NC}"
+   exit 1
+fi
+
+#####Import config variables
+source config.sh
+
+#####Commands
+echo "${MESSAGE}Sending the database source code to the Virual Machine...${NC}"
+rsync -r ../source_code/$source_code_folder_name $vm_connection:/root/
+
+echo "${MESSAGE}Executing scripts in the Virtual Machine...${NC}"
+ssh $vm_connection "cd /root/$source_code_folder_name && bash deploy_database.sh $logon_command"
+
+

Tests/Teradata/scripts/execute_drop_database_script.sh

@@ -0,0 +1,21 @@
+#
+#####Version 20230810: Script created
+
+#####Constants
+MESSAGE='\033[0;32m' # Green
+ERROR='\033[0;31m' # Red
+NC='\033[0m' # No Color
+
+#####Parameters
+source_code_folder_name="$1"
+if [ ! "$source_code_folder_name" ] || [ ! -d "../source_code/$source_code_folder_name/" ] ; then
+   echo "${ERROR}Invalid parameter '$source_code_folder_name', options are [$(ls ../source_code)]${NC}"
+   exit 1
+fi
+
+#####Import config variables
+source config.sh
+
+#####Commands
+echo "${MESSAGE}Executing scripts in the Virtual Machine...${NC}"
+ssh $vm_connection "cd /root/$source_code_folder_name && bash drop_database.sh $logon_command"

Tests/Teradata/scripts/execute_extract_database_script.sh

@@ -0,0 +1,41 @@
+#
+#Version 20230810: Script created
+
+#####Constants
+MESSAGE='\033[0;32m' # Green
+ERROR='\033[0;31m' # Red
+NC='\033[0m' # No Color
+folder_name="Teradata_Extraction"
+
+#####Parameters
+source_code_folder_name="$1"
+if [ ! "$source_code_folder_name" ] || [ ! -d "../source_code/$source_code_folder_name/" ] ; then
+   echo "${ERROR}Invalid parameter '$source_code_folder_name', options are [$(ls ../source_code)]${NC}"
+   exit 1
+fi
+
+#####Import config variables
+source config.sh
+
+##### Commands
+echo "${MESSAGE}Copying Teradata Script...${NC}"
+cp -fr ../../../Teradata $folder_name
+cp ../source_code/$source_code_folder_name/extraction_parameters.sh $folder_name/bin/parameters.sh
+sed -i '' "s/connection_string_value/${logon_command}/g" $folder_name/bin/parameters.sh
+mkdir -p ../extracted_code/
+
+echo "${MESSAGE}Removing previous execution output...${NC}"
+rm -r $folder_name/output
+rm -r $folder_name/log
+rm -r ../extracted_code/$source_code_folder_name
+
+echo "${MESSAGE}Sending Teradata scripts to the Virual Machine...${NC}"
+scp -r $folder_name $vm_connection:/root/$folder_name
+rm -r $folder_name
+
+echo "${MESSAGE}Executing scripts in the Virtual Machine...${NC}"
+ssh $vm_connection "cd /root/$folder_name/bin && bash create_ddls.sh"
+
+echo "${MESSAGE}Retrieving the output folder and removing the sent files...${NC}"
+scp -r -OT $vm_connection:"/root/$folder_name/output /root/$folder_name/log" ../extracted_code/$source_code_folder_name
+ssh -q $vm_connection rm -r /root/$folder_name 

Tests/Teradata/scripts/execute_scripts.sh

@@ -0,0 +1,19 @@
+#
+#####Constants
+STEP_MESSAGE='\033[0;34m' # Green
+ERROR='\033[0;31m' # Red
+NC='\033[0m' # No Color
+
+#####Parameters
+source_code_folder_name="$1"
+if [ ! "$source_code_folder_name" ] || [ ! -d "../source_code/$source_code_folder_name/" ] ; then
+   echo "${ERROR}Invalid parameter '$source_code_folder_name', options are [$(ls ../source_code)]${NC}"
+   exit 1
+fi
+
+echo "${STEP_MESSAGE}Step 1/3 Deplying database...${NC}"
+source execute_deploy_database_script.sh $source_code_folder_name
+echo "${STEP_MESSAGE}Step 2/3 Extracting database...${NC}"
+source execute_extract_database_script.sh $source_code_folder_name
+echo "${STEP_MESSAGE}Step 3/3 Removing database...${NC}"
+source execute_drop_database_script.sh $source_code_folder_name

Tests/Teradata/scripts/ssh_automatic_login_configuration.sh

@@ -0,0 +1,4 @@
+#
+source config.sh
+ssh-keygen -t rsa -b 2048
+ssh-copy-id $vm_connection

Tests/Teradata/source_code/demo_database/database_code/DDL_Databases.sql

@@ -0,0 +1,2 @@
+--
+CREATE DATABASE SC_EXAMPLE_DEMO FROM DBC AS PERM = 100000000;

Tests/Teradata/source_code/demo_database/database_code/DDL_Tables.sql

@@ -0,0 +1,63 @@
+CREATE TABLE Employee (
+        EmployeeID INT NOT NULL,
+        FirstName VARCHAR(50),
+        LastName VARCHAR(50),
+        Department VARCHAR(50),
+        Email VARCHAR(100),
+        Salary number,
+        PRIMARY KEY (EmployeeID)
+);
+
+CREATE TABLE salary_log(
+    type_user VARCHAR(50),
+    id INT,
+    old_salary number,
+    new_salary number
+);
+
+CREATE TABLE expandOnTable
+(
+    id INTEGER,
+    pd PERIOD ( TIMESTAMP)
+);
+
+CREATE TABLE project
+(
+    emp_id       INTEGER,
+    project_name VARCHAR(20),
+    dept_id      INTEGER,
+    duration     PERIOD( DATE)
+);
+
+CREATE TABLE MessageStorage
+(
+    MessageID TIMESTAMP(0),
+    Message1 VARCHAR(100),
+    Message2 VARCHAR(100)
+);
+
+CREATE TABLE account_balance
+(
+    account_id INTEGER NOT NULL,
+    month_id   INTEGER,
+    balance    INTEGER
+) UNIQUE PRIMARY INDEX (account_id, month_id);
+
+
+CREATE TABLE vEmployee
+(
+    PersonID  INT,
+    LastName  VARCHAR(255),
+    FirstName VARCHAR(255)
+);
+
+CREATE TABLE ResTable
+(
+    Column1 VARCHAR(255)
+);
+
+CREATE TABLE EMPLOYEE_JOB_PERIODS (
+    FIRST_NAME VARCHAR(100),
+    LAST_NAME VARCHAR(100),
+    JOB_DURATION PERIOD(DATE)
+);

Tests/Teradata/source_code/demo_database/database_code/DDL_Trigger.sql

@@ -0,0 +1,8 @@
+
+CREATE TRIGGER RaiseTrig
+  AFTER UPDATE OF salary ON employee
+  REFERENCING OLD AS OldRow NEW AS NewRow
+  FOR EACH ROW
+    WHEN ((NewRow.salary - OldRow.salary) / OldRow.salary >.10)
+    INSERT INTO salary_log
+    VALUES ('USER', NewRow.EmployeeID, OldRow.salary, NewRow.salary);

Tests/Teradata/source_code/demo_database/database_code/DDL_Views.sql

@@ -0,0 +1,14 @@
+CREATE VIEW EMPLOYEE_JOB_DURATION_COMPARISONS
+AS
+LOCKING ROW FOR ACCESS
+    SELECT 'OVERLAP' FUNC, FIRST_NAME, LAST_NAME
+        FROM EMPLOYEE_JOB_PERIODS
+        WHERE JOB_DURATION OVERLAPS PERIOD(DATE '2009-01-01', DATE '2010-09-24')
+    UNION ALL
+    SELECT 'LDIFF' FUNC, FIRST_NAME, LAST_NAME
+        FROM EMPLOYEE_JOB_PERIODS
+        WHERE INTERVAL(JOB_DURATION LDIFF PERIOD(DATE '2009-01-01', DATE '2010-09-24')) MONTH > 3
+    UNION ALL
+    SELECT 'RDIFF' FUNC, FIRST_NAME, LAST_NAME
+        FROM EMPLOYEE_JOB_PERIODS
+        WHERE JOB_DURATION RDIFF PERIOD(DATE '2009-01-01', DATE '2010-09-24') IS NOT NULL;

Tests/Teradata/source_code/demo_database/deploy_database.sh

@@ -0,0 +1,21 @@
+#
+#####Constants
+ERROR='\033[0;31m' # Red
+NC='\033[0m' # No Color
+
+#####Parameters
+logon_command="$1"
+if [ ! "$logon_command" ];then
+   echo "${ERROR}Logon command not provided${NC}"
+   exit 1
+fi
+
+bteq << EOF
+.logon $logon_command;
+  .RUN FILE ./database_code/DDL_Databases.sql
+  DATABASE SC_EXAMPLE_DEMO;
+  .RUN FILE ./database_code/DDL_SF_Schemas.sql
+  .RUN FILE ./database_code/DDL_Tables.sql
+  .RUN FILE ./database_code/DDL_Trigger.sql
+  .RUN FILE ./database_code/DDL_Views.sql
+EOF