Skip to content

Commit b679d34

Browse files
amrudesh1amrudesh1
authored
Fixed issues related to path and duplication
Fixed issues related to path and duplication
2 parents b465ec2 + d6c472f commit b679d34

File tree

10 files changed

+231
-80
lines changed

10 files changed

+231
-80
lines changed

Diff for: ‎ .dockerignore.

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Ignore files and directories
2+
.history

Diff for: ‎Dockerfile

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ FROM golang:buster AS builder
66
ARG JDK_VERSION=11
77

88
RUN apt-get update && \
9+
apt-get install ca-certificates-java openjdk-${JDK_VERSION}-jre-headless -y && \
910
apt-get install -y --no-install-recommends openjdk-${JDK_VERSION}-jdk && \
1011
apt-get install aapt -y && \
1112
apt-get clean && \

Diff for: ‎apk/analysis.go

+39-29
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,13 @@ limitations under the License.
1616

1717
import (
1818
"encoding/json"
19-
"io/ioutil"
19+
"fmt"
2020
database "morf/db"
2121
"morf/models"
22+
"morf/utils"
2223
util "morf/utils"
2324
"net/http"
24-
"os"
25+
"path/filepath"
2526

2627
log "github.com/sirupsen/logrus"
2728

@@ -31,6 +32,9 @@ import (
3132
)
3233

3334
func StartCliExtraction(apkPath string, db *gorm.DB, is_db_req bool) {
35+
var fileName string
36+
37+
fs := utils.GetAppFS()
3438
if is_db_req {
3539
apkFound, json_data := util.CheckDuplicateInDB(db, apkPath)
3640
if apkFound {
@@ -40,35 +44,47 @@ func StartCliExtraction(apkPath string, db *gorm.DB, is_db_req bool) {
4044
}
4145
packageModel := ExtractPackageData(apkPath)
4246
metadata := StartMetaDataCollection(apkPath)
43-
scanner_data := StartSecScan("temp/input/" + apkPath)
47+
48+
fmt.Println("Metadata: Completed")
49+
50+
if apkPath[0] == '/' {
51+
fileName = filepath.Base(apkPath)
52+
} else {
53+
fileName = apkPath
54+
}
55+
56+
scanner_data := StartSecScan(utils.GetInputDir() + fileName)
57+
scanner_data = utils.SanitizeSecrets(scanner_data)
4458
secret_data, secret_error := json.Marshal(scanner_data)
4559

4660
if secret_error != nil {
4761
log.Error(secret_error)
4862
}
4963

50-
secret := util.CreateSecretModel(apkPath, packageModel, metadata, scanner_data, secret_data)
64+
secret := util.CreateSecretModel(fileName, packageModel, metadata, scanner_data, secret_data)
65+
5166
if is_db_req {
5267
database.InsertSecrets(secret, db)
5368
}
69+
5470
json_data, json_error := json.MarshalIndent(secret, "", " ")
5571

5672
if json_error != nil {
5773
log.Error(json_error)
5874
}
5975

60-
_, err_ := os.Stat(vip.GetString("backup_path"))
61-
62-
if os.IsNotExist(err_) {
63-
os.Mkdir(vip.GetString("backup_path"), 0755)
76+
//Check if backup folder exists
77+
if !util.CheckBackUpDirExists(fs) {
78+
util.CreateBackUpDir(fs)
6479
}
6580

66-
err := ioutil.WriteFile(vip.GetString("backup_path")+"/"+apkPath+"_"+secret.APKVersion+".json", json_data, 0644)
67-
if err != nil {
68-
log.Error(err)
69-
}
81+
util.WriteToFile(fs, vip.GetString("backup_path")+fileName+"_"+secret.APKVersion+".json", string(json_data))
82+
util.WriteToFile(fs, vip.GetString("backup_path")+fileName+"_"+"Secrets_"+secret.APKVersion+".json", string(secret_data))
7083

71-
log.Info("APK Data saved to: " + vip.GetString("backup_path") + "/" + apkPath + "_" + secret.APKVersion + ".json")
84+
util.WriteToFile(fs, "results"+"/"+fileName+"_"+secret.APKVersion+".json", string(json_data))
85+
util.WriteToFile(fs, "results"+"/"+fileName+"_"+"Secrets_"+secret.APKVersion+".json", string(secret_data))
86+
87+
log.Info("APK Data saved to: " + vip.GetString("backup_path") + "/" + fileName + "_" + secret.APKVersion + ".json")
7288
}
7389

7490
func StartJiraProcess(jiramodel models.JiraModel, db *gorm.DB, c *gin.Context) {
@@ -92,7 +108,7 @@ func StartJiraProcess(jiramodel models.JiraModel, db *gorm.DB, c *gin.Context) {
92108

93109
packageModel := ExtractPackageData(apk_path)
94110
metadata := StartMetaDataCollection(apk_path)
95-
scanner_data := StartSecScan("temp/input/" + apk_path)
111+
scanner_data := StartSecScan(utils.GetInputDir() + apk_path)
96112
secret_data, secret_error := json.Marshal(scanner_data)
97113

98114
if secret_error != nil {
@@ -106,6 +122,7 @@ func StartJiraProcess(jiramodel models.JiraModel, db *gorm.DB, c *gin.Context) {
106122
}
107123

108124
func StartExtractProcess(apkPath string, db *gorm.DB, c *gin.Context, isSlack bool, slackData models.SlackData) {
125+
fs := utils.GetAppFS()
109126

110127
apkFound, json_data := util.CheckDuplicateInDB(db, apkPath)
111128
if apkFound {
@@ -124,15 +141,14 @@ func StartExtractProcess(apkPath string, db *gorm.DB, c *gin.Context, isSlack bo
124141

125142
packageModel := ExtractPackageData(apkPath)
126143
metadata := StartMetaDataCollection(apkPath)
127-
scanner_data := StartSecScan("temp/input/" + apkPath)
144+
scanner_data := StartSecScan(utils.GetInputDir() + apkPath)
128145
secret_data, secret_error := json.Marshal(scanner_data)
129146

130147
if secret_error != nil {
131148
log.Error(secret_error)
132149
}
133150

134151
secret := util.CreateSecretModel(apkPath, packageModel, metadata, scanner_data, secret_data)
135-
136152
database.InsertSecrets(secret, db)
137153

138154
json_data, json_error := json.MarshalIndent(secret, "", " ")
@@ -143,22 +159,16 @@ func StartExtractProcess(apkPath string, db *gorm.DB, c *gin.Context, isSlack bo
143159
}
144160

145161
//Check if backup folder exists
146-
_, err_ := os.Stat(vip.GetString("backup_path"))
147-
148-
if os.IsNotExist(err_) {
149-
os.Mkdir(vip.GetString("backup_path"), 0755)
162+
//Check if backup folder exists
163+
if !util.CheckBackUpDirExists(fs) {
164+
util.CreateBackUpDir(fs)
150165
}
151166

152-
// Check if file exists
167+
util.WriteToFile(fs, vip.GetString("backup_path")+apkPath+"_"+secret.APKVersion+".json", string(json_data))
168+
util.WriteToFile(fs, vip.GetString("backup_path")+apkPath+"_"+"Secrets_"+secret.APKVersion+".json", string(secret_data))
153169

154-
//Move the APK Data to backup folder
155-
backupPath := vip.GetString("backup_path") + apkPath + "_" + secret.APKVersion + ".json"
156-
log.Println("Backup Path: ", backupPath)
157-
err := ioutil.WriteFile(backupPath, json_data, 0644)
158-
159-
if err != nil {
160-
log.Error(err)
161-
}
170+
util.WriteToFile(fs, "results"+"/"+apkPath+"_"+secret.APKVersion+".json", string(json_data))
171+
util.WriteToFile(fs, "results"+"/"+apkPath+"_"+"Secrets_"+secret.APKVersion+".json", string(secret_data))
162172

163173
if !isSlack {
164174
c.JSON(http.StatusOK, gin.H{

Diff for: ‎apk/metadata.go

+18-15
Original file line numberDiff line numberDiff line change
@@ -20,38 +20,41 @@ import (
2020
"io"
2121
"log"
2222
"morf/models"
23+
"morf/utils"
2324
"os"
2425
"os/exec"
2526
"path/filepath"
2627
"strings"
28+
29+
alf "github.com/spf13/afero"
2730
)
2831

2932
func StartMetaDataCollection(apkPath string) models.MetaDataModel {
3033
// Check if temp directory exist and If yes delete it and create a new one
3134

32-
if _, err := os.Stat("temp"); err == nil {
35+
fs := alf.NewOsFs()
36+
37+
if utils.CheckifmorftmpDirExists(fs) {
3338
fmt.Println("Deleting the temp directory")
34-
os.RemoveAll("temp")
39+
utils.DeleteTmpDir(fs)
3540
fmt.Println("Creating a new temp directory")
36-
os.Mkdir("temp", 0777)
41+
utils.CreateMorfDirintmp(fs)
3742
} else {
3843
fmt.Println("Creating a new temp directory")
39-
os.Mkdir("temp", 0777)
44+
utils.CreateMorfDirintmp(fs)
4045
}
4146

42-
if _, err := os.Stat("temp/input"); os.IsNotExist(err) {
43-
os.Mkdir("temp/input", 0755)
44-
}
45-
if _, err := os.Stat("temp/output"); os.IsNotExist(err) {
46-
os.Mkdir("temp/output", 0755)
47+
// Create input and output directory
48+
if _, err := os.Stat(utils.GetInputDir()); os.IsNotExist(err) {
49+
utils.CreateInputOutputDir(fs)
4750
}
4851

49-
os.Rename(apkPath, "temp/input/"+apkPath)
50-
apkPath = "temp/input/" + apkPath
52+
// Move APK to input directory
53+
54+
apkPath = utils.CopyApktoInputDir(fs, apkPath)
5155
fmt.Println("Starting metadata collection for " + apkPath)
5256

53-
metadata_success, metadata_error := exec.Command("java", "-cp", "tools/apkanalyzer.jar", "sk.styk.martin.bakalarka.execute.Main", "-analyze", "--in", "temp/input/", "--out", "temp/output").Output()
54-
fmt.Println(metadata_success)
57+
metadata_success, metadata_error := exec.Command("java", "-cp", "tools/apkanalyzer.jar", "sk.styk.martin.bakalarka.execute.Main", "-analyze", "--in", utils.GetInputDir(), "--out", utils.GetOutputDir()).Output()
5558

5659
if metadata_error != nil {
5760
fmt.Println("Error while decompiling the APK file")
@@ -65,8 +68,8 @@ func StartMetaDataCollection(apkPath string) models.MetaDataModel {
6568
fmt.Println(file_path)
6669

6770
// Make file readable
68-
os.Chmod("temp/output/"+strings.Replace(file_name, ".apk", ".json", -1), 0777)
69-
return startFileParser("temp/output/" + strings.Replace(file_name, ".apk", ".json", -1))
71+
os.Chmod(utils.GetOutputDir()+strings.Replace(file_name, ".apk", ".json", -1), 0777)
72+
return startFileParser(utils.GetOutputDir() + strings.Replace(file_name, ".apk", ".json", -1))
7073
}
7174

7275
return models.MetaDataModel{}

Diff for: ‎apk/scanner.go

+15-30
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package apk
1919

2020
import (
2121
"fmt"
22-
"io/ioutil"
2322
"morf/models"
2423
"morf/utils"
2524
"os"
@@ -60,56 +59,43 @@ func CheckAPK(apkPath string) {
6059

6160
func StartSecScan(apkPath string) []models.SecretModel {
6261
//Decompile the sources of the APK file
63-
counter := 0
6462

63+
counter := 0
6564
log.Println("Decompiling the APK file for sources")
66-
source_decompile, source_error := exec.Command("java", "-jar", "tools/apktool.jar", "d", "-r", apkPath, "-o", "temp/output/apk/source").Output()
67-
68-
if source_error != nil {
69-
log.Println("Error while decompiling the APK file")
70-
log.Fatal(source_error)
71-
}
65+
fmt.Println(apkPath)
66+
source_decompile, source_error := exec.Command("java", "-jar", "tools/apktool.jar", "d", "-r", apkPath, "-o", utils.GetSourceDir()).Output()
67+
utils.HandleError(source_error, "Error while decompiling the APK file", true)
7268

7369
if source_decompile != nil {
7470
log.Println("Decompiling the APK file for sources successful")
7571
counter++
7672
}
7773

7874
//Decompile the resources of the APK file
79-
80-
res_decompile, res_error := utils.ExecuteCommand("java", []string{"-jar", "tools/apktool.jar", "d", "-s", apkPath, "-o", "temp/output/apk/appreso"}, false, true)
81-
82-
if res_error != nil {
83-
log.Println("Error while decompiling the resources of the APK file")
84-
log.Error(res_error)
85-
}
75+
res_decompile, res_error := utils.ExecuteCommand("java", []string{"-jar", "tools/apktool.jar", "d", "-s", apkPath, "-o", utils.GetResDir()}, false, true)
76+
utils.HandleError(res_error, "Error while decompiling the APK file", true)
8677

8778
if res_decompile != nil {
8879
log.Println("Decompiling the APK file for resources successful")
8980
counter++
9081
}
91-
files_path := "temp/output/apk/"
82+
9283
if counter == 2 {
9384
log.Println("Decompiling the APK file successful")
94-
return StartScan(files_path)
85+
return StartScan(utils.GetFilesDir())
9586
}
87+
9688
return nil
9789
}
9890

99-
func readPatternFile(patternFilePath string) ([]byte, error) {
100-
patternFile, err := os.OpenFile(patternFilePath, os.O_RDONLY, 0666)
101-
defer patternFile.Close()
102-
utils.HandleError(err, "Error opening pattern file:", true)
91+
func readPatternFile(patternFilePath string) []byte {
10392

104-
yamlFile, err := ioutil.ReadAll(patternFile)
105-
utils.HandleError(err, "Error reading pattern file:", true)
106-
107-
return yamlFile, err
93+
yamlFile := utils.ReadFile(utils.GetAppFS(), patternFilePath)
94+
return yamlFile
10895
}
10996

11097
func StartScan(apkPath string) []models.SecretModel {
111-
files, err := ioutil.ReadDir("patterns")
112-
utils.HandleError(err, "Error reading directory:", true)
98+
files := utils.ReadDir(utils.GetAppFS(), "patterns")
11399

114100
var wg sync.WaitGroup
115101
resultsChan := make(chan models.SecretModel, 100)
@@ -122,9 +108,9 @@ func StartScan(apkPath string) []models.SecretModel {
122108
wg.Add(1)
123109
go func(file os.FileInfo) {
124110
defer wg.Done()
125-
yamlFile, err := readPatternFile("patterns/" + file.Name())
111+
yamlFile := readPatternFile("patterns/" + file.Name())
126112
// Make sure file name is ending with .yml or .yaml
127-
113+
err := error(nil)
128114
if err != nil {
129115
fmt.Println(err)
130116
}
@@ -147,7 +133,6 @@ func StartScan(apkPath string) []models.SecretModel {
147133

148134
for _, pattern := range secretPatterns.Patterns {
149135
pat := pattern.Pattern.Regex
150-
fmt.Println(pat)
151136
stdout, err := utils.ExecuteCommand("rg", []string{"-n", "-e", fmt.Sprintf("\"%s\"", pat), "--multiline", apkPath}, true, false)
152137

153138
utils.HandleError(err, "Error running ripgrep:", true)

Diff for: ‎cmd/cli.go

+3
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ limitations under the License.
1515
*/package cmd
1616

1717
import (
18+
"fmt"
1819
"morf/apk"
1920
"morf/db"
2021

@@ -65,6 +66,8 @@ func add(cmd *cob.Command, args []string) {
6566
return
6667
}
6768
}
69+
fmt.Println("IS DB REQ", is_db_req)
70+
// Check if APK path is absolute or relative
6871

6972
apk.StartCliExtraction(apkPath, db.DB, is_db_req)
7073
}

Diff for: ‎cmd/root.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ distributed under the License is distributed on an "AS IS" BASIS,
1212
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1313
See the License for the specific language governing permissions and
1414
limitations under the License.
15-
*//*
15+
*/ /*
1616
Copyright © 2023 NAME HERE <EMAIL ADDRESS>
1717
*/
1818
package cmd
@@ -72,7 +72,6 @@ func runMORF(cmd *cob.Command, args []string) error {
7272
func init() {
7373

7474
vip.SetDefault("port", 8080)
75-
vip.SetDefault("tempPath", "/temp")
7675
vip.SetDefault("backup_path", "backup/")
7776

7877
MorfCmd.AddCommand(cliCmd)

Diff for: ‎log/infoLog.log

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
16:40:13,951 INFO ApkProcessingTask:51 - [CRED-4.0.9-RC0-qa.apk] Started processing of file CRED-4.0.9-RC0-qa.apk
2-
16:40:13,955 TRACE FileInfoProcessor:56 - [CRED-4.0.9-RC0-qa.apk] Started processing of file info
3-
16:40:13,957 TRACE FileInfoProcessor:64 - [CRED-4.0.9-RC0-qa.apk] Finished processing of file info
4-
16:40:13,963 TRACE CertificateProcessor:94 - [CRED-4.0.9-RC0-qa.apk] Started processing of certificate
1+
12:11:19,293 INFO ApkProcessingTask:51 - [cred-4.2.3-rc0-release.apk] Started processing of file cred-4.2.3-rc0-release.apk
2+
12:11:19,295 TRACE FileInfoProcessor:56 - [cred-4.2.3-rc0-release.apk] Started processing of file info
3+
12:11:19,297 TRACE FileInfoProcessor:64 - [cred-4.2.3-rc0-release.apk] Finished processing of file info
4+
12:11:19,306 TRACE CertificateProcessor:94 - [cred-4.2.3-rc0-release.apk] Started processing of certificate
55

0 commit comments

Comments
 (0)