Skip to content

Commit 06c1d63

Browse files
author
EC2 Default User
committed
docker-compose works
1 parent a5c732c commit 06c1d63

20 files changed

+49
-50
lines changed

‎Dockerfile

+7-5
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
FROM ubuntu:16.04
1+
FROM ubuntu:20.04
22

33
# Update OS Packages, Install OS Dependencies (Do this in one line to ensure Update always happens)
44
RUN apt-get update && \
5-
apt-get install -y git libsqlite3-dev python2.7 python-pip npm libffi-dev libssl-dev mysql-client \
6-
libmysqlclient-dev python-dev libpython-dev git yara=3.4.0+dfsg-2build1 apt-transport-https ca-certificates curl \
5+
DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true apt-get install -y git libsqlite3-dev python3.8 python3-pip npm libffi-dev libssl-dev mysql-client \
6+
libmysqlclient-dev python3-dev libpython3-dev git yara apt-transport-https ca-certificates curl \
77
software-properties-common libpcre3 libpcre3-dev
88

99
# Setup UWSGI Installation
@@ -15,10 +15,12 @@ WORKDIR /opt/threatkb
1515
COPY package.json .bowerrc bower.json Gruntfile.js requirements.txt ./
1616

1717
# Install Python Dependencies
18-
RUN pip install --upgrade pip & pip install virtualenv && pip install -r requirements.txt
18+
RUN /usr/bin/pip3 install --upgrade pip & /usr/bin/pip3 install virtualenv
19+
RUN /usr/local/bin/virtualenv -p /usr/bin/python3.8 env
20+
RUN env/bin/pip3 install -r requirements.txt
1921

2022
# Install Node Dependencies
21-
RUN npm install -g bower && ln -s /usr/bin/nodejs /usr/bin/node && bower install --allow-root
23+
RUN npm install -g bower && bower install --allow-root
2224

2325
# Add Package Files
2426
COPY . /opt/threatkb

‎app/models/activity_log.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ class ActivityLog(db.Model):
1515

1616
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
1717
activity_type = db.Column(db.String(256))
18-
activity_text = db.Column(db.String(65000))
18+
activity_text = db.Column(db.TEXT())
1919
activity_date = db.Column(db.DateTime(timezone=True))
2020

2121
entity_type = db.Column(db.Integer(), index=True, nullable=False)

‎app/models/comments.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ class Comments(db.Model):
1010
date_created = db.Column(db.DateTime(timezone=True), default=db.func.current_timestamp())
1111
date_modified = db.Column(db.DateTime(timezone=True), default=db.func.current_timestamp(),
1212
onupdate=db.func.current_timestamp())
13-
comment = db.Column(db.String(65000))
13+
comment = db.Column(db.TEXT())
1414
entity_type = db.Column(db.Integer(), index=True, nullable=False)
1515
entity_id = db.Column(db.Integer(), index=True, nullable=False)
1616

‎app/models/files.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ class Files(db.Model):
1010
date_modified = db.Column(db.DateTime(timezone=True),
1111
default=db.func.current_timestamp(),
1212
onupdate=db.func.current_timestamp())
13-
filename = db.Column(db.String(65000))
13+
filename = db.Column(db.TEXT())
1414
content_type = db.Column(db.String(100))
1515
entity_type = db.Column(db.Integer(), index=True, nullable=True)
1616
entity_id = db.Column(db.Integer(), index=True, nullable=True)

‎app/models/metadata.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ class MetadataMapping(db.Model):
134134
__tablename__ = "metadata_mapping"
135135

136136
id = db.Column(db.Integer, primary_key=True)
137-
value = db.Column(db.String(60000), nullable=True)
137+
value = db.Column(db.TEXT(), nullable=True)
138138

139139
date_created = db.Column(db.DateTime(timezone=True), default=db.func.current_timestamp())
140140
date_modified = db.Column(db.DateTime(timezone=True), default=db.func.current_timestamp(),

‎app/models/scripts.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ class Scripts(db.Model):
1111
name = db.Column(db.String(128), nullable=False)
1212
description = db.Column(db.String(128), nullable=True)
1313
interpreter = db.Column(db.String(64), nullable=False)
14-
code = db.Column(db.String(60000), nullable=True)
14+
code = db.Column(db.TEXT(), nullable=True)
1515
match_regex = db.Column(db.String(4096), nullable=True)
1616
date_created = db.Column(db.DateTime(timezone=True), default=db.func.current_timestamp())
1717
date_modified = db.Column(db.DateTime(timezone=True), default=db.func.current_timestamp(),

‎app/models/tasks.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ class Tasks(db.Model):
1616
state = db.Column(db.String(32), index=True)
1717
active = db.Column(db.Boolean, nullable=False, default=True)
1818
title = db.Column(db.String(256), index=True)
19-
description = db.Column(db.String(2048), index=True)
19+
description = db.Column(db.TEXT(), index=True)
2020
final_artifact = db.Column(db.String(4096))
2121

2222
created_user_id = db.Column(db.Integer, db.ForeignKey('kb_users.id'), nullable=False)

‎app/models/yara_rule.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@ class Yara_rule(db.Model):
2929
revision = db.Column(db.Integer(), default=1)
3030
name = db.Column(db.String(128), index=True)
3131
category = db.Column(db.String(32), index=True)
32-
condition = db.Column(db.String(2048), index=True)
33-
strings = db.Column(db.String(30000), index=True)
34-
imports = db.Column(db.String(512))
32+
condition = db.Column(db.TEXT(), index=True)
33+
strings = db.Column(db.TEXT(), index=True)
34+
imports = db.Column(db.String(2048))
3535
description = db.Column(db.TEXT(), index=True)
3636
references = db.Column(db.TEXT(), index=True)
3737
active = db.Column(db.Boolean, nullable=False, default=True, index=True)

‎docker-compose.yml

+15-9
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,18 @@ version: '3'
22

33
services:
44
db:
5-
image: mysql:5.7.20
5+
image: mysql:8.0.29
6+
command: mysqld --default-authentication-plugin=mysql_native_password --skip-mysqlx
67
volumes:
78
- db_data:/var/lib/docker/mysql
89
restart: always
910
ports:
10-
- 3307:3306
11+
- 3306:3306
1112
environment:
12-
MYSQL_ROOT_PASSWORD: P@sswordsRC00l
13+
MYSQL_ROOT_PASSWORD: b5vXcqzUtn4suyS
1314
MYSQL_DATABASE: threatkb
1415
MYSQL_USER: threatkb
15-
MYSQL_PASSWORD: P@sswordsRC00l
16+
MYSQL_PASSWORD: b5vXcqzUtn4suyS
1617
redis:
1718
depends_on:
1819
- db
@@ -29,23 +30,28 @@ services:
2930
restart: always
3031
environment:
3132
THREATKB_USER: [email protected]
32-
THREATKB_PASS: P@sswordsRC00l
33-
MYSQL_ROOT_PASSWORD: P@sswordsRC00l
33+
THREATKB_PASS: b5vXcqzUtn4suyS
34+
MYSQL_ROOT_PASSWORD: b5vXcqzUtn4suyS
3435
SQL_DATABASE: threatkb
3536
SQL_USERNAME: threatkb
36-
SQL_PASSWORD: P@sswordsRC00l
37+
SQL_PASSWORD: b5vXcqzUtn4suyS
3738
SQL_PORT: 3306
3839
SQL_HOST: db
3940
SQL_PROTOCOL: mysql
4041
LISTEN_ON: 0.0.0.0
4142
LISTEN_PORT: 5000
43+
REDIS_CACHE_URL: redis://localhost:6379/0
4244
threatkb_agent:
4345
depends_on:
4446
- db
4547
image: threatkb:latest
4648
restart: always
4749
environment:
48-
- RUN_AGENT=1
49-
- C_FORCE_ROOT=true
50+
RUN_AGENT: 1
51+
C_FORCE_ROOT: true
52+
REDIS_ACCEPT_CONTENT: '["json"]'
53+
REDIS_RESULT_SERIALIZER: json
54+
REDIS_TASK_SERIALIZER: json
55+
REDIS_BROKER_URL: redis://redis:6379/0
5056
volumes:
5157
db_data:

‎docker-entrypoint.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,16 @@ cd /opt/threatkb
88
./wait-for-it.sh db:3306 -- echo "db is up"
99

1010
if [[ ! -z "${RUN_AGENT}" ]]; then
11-
/usr/local/bin/celery -A app.celery worker --uid uwsgi --loglevel=info
11+
env/bin/celery -A app.celery worker --uid uwsgi --loglevel=info
1212
else
1313
find . -name "*.pyc" -exec rm -f {} \;
14-
/usr/local/bin/python manage.py db upgrade
14+
env/bin/python3.8 manage.py db upgrade
1515

1616
num_users=`echo "select count(*) from kb_users;" | mysql -u ${SQL_USERNAME} -p"${SQL_PASSWORD}" -h ${SQL_HOST} ${SQL_DATABASE} | sed 's/[^0-9]//g'`
1717
if [ $num_users -lt 1 ]; then
18-
PASSWORD=`/usr/local/bin/python hash_pass.py ${THREATKB_PASS}`
18+
PASSWORD=`env/bin/python3.8 hash_pass.py ${THREATKB_PASS}`
1919
mysql -u ${SQL_USERNAME} -p"${SQL_PASSWORD}" ${SQL_DATABASE} -h ${SQL_HOST} -e "insert into kb_users (email,password,admin,active) values (\"${THREATKB_USER}\", \"${PASSWORD}\", 1, 1);"
2020
fi
2121
#env/bin/python run.py --listen-on ${LISTEN_ON} --listen-port ${LISTEN_PORT}
22-
/usr/local/bin/uwsgi --yaml /etc/uwsgi.yaml --http "${LISTEN_ON:-0.0.0.0}:${LISTEN_PORT:-5000}" --py-autoreload 1
22+
env/bin/python3.8 run.py --listen-on "${LISTEN_ON:-0.0.0.0}" --listen-port "${LISTEN_PORT:-5000}"
2323
fi

‎hash_pass.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@
66
app = Flask(__name__, static_url_path="")
77
b = Bcrypt(app)
88

9-
print((b.generate_password_hash(sys.argv[1])))
9+
print((b.generate_password_hash(sys.argv[1]).decode()))

‎migrations/versions/2221286081e8_initial_migration.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -105,16 +105,16 @@ def upgrade():
105105
sa.Column('test_status', sa.String(length=16), nullable=True),
106106
sa.Column('confidence', sa.Integer(), nullable=True),
107107
sa.Column('severity', sa.Integer(), nullable=True),
108-
sa.Column('description', sa.String(length=4096), nullable=True),
108+
sa.Column('description', sa.TEXT(), nullable=True),
109109
sa.Column('category', sa.String(length=32), nullable=True),
110110
sa.Column('file_type', sa.String(length=32), nullable=True),
111111
sa.Column('subcategory1', sa.String(length=32), nullable=True),
112112
sa.Column('subcategory2', sa.String(length=32), nullable=True),
113113
sa.Column('subcategory3', sa.String(length=32), nullable=True),
114114
sa.Column('reference_link', sa.String(length=2048), nullable=True),
115115
sa.Column('reference_text', sa.String(length=2048), nullable=True),
116-
sa.Column('condition', sa.String(length=2048), nullable=True),
117-
sa.Column('strings', sa.String(length=30000), nullable=True),
116+
sa.Column('condition', sa.TEXT(), nullable=True),
117+
sa.Column('strings', sa.TEXT(), nullable=True),
118118
sa.Column('created_user_id', sa.Integer(), nullable=False),
119119
sa.Column('modified_user_id', sa.Integer(), nullable=False),
120120
sa.ForeignKeyConstraint(['created_user_id'], ['kb_users.id'], ),
@@ -127,7 +127,7 @@ def upgrade():
127127
sa.Column('id', sa.Integer(), nullable=False),
128128
sa.Column('date_created', sa.DateTime(timezone=True), nullable=True),
129129
sa.Column('date_modified', sa.DateTime(timezone=True), nullable=True),
130-
sa.Column('comment', sa.String(length=65000), nullable=True),
130+
sa.Column('comment', sa.TEXT(), nullable=True),
131131
sa.Column('entity_type', sa.Integer(), nullable=False),
132132
sa.Column('entity_id', sa.Integer(), nullable=False),
133133
sa.Column('user_id', sa.Integer(), nullable=False),

‎migrations/versions/5ccd18d62305_add_metadata_and_metadata_mapping.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ def upgrade():
3434
)
3535
op.create_table('metadata_mapping',
3636
sa.Column('id', sa.Integer(), nullable=False),
37-
sa.Column('value', sa.String(length=60000), nullable=True),
37+
sa.Column('value', sa.TEXT(), nullable=True),
3838
sa.Column('date_created', sa.DateTime(timezone=True), nullable=True),
3939
sa.Column('date_modified', sa.DateTime(timezone=True), nullable=True),
4040
sa.Column('metadata_id', sa.Integer(), nullable=False),

‎migrations/versions/650b0ad88d25_add_indexes.py

+2-9
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,7 @@
1717

1818

1919
def upgrade():
20-
op.execute("""CREATE INDEX ifx_yara_rules_description ON yara_rules (`description`(1000));""")
21-
op.execute("""CREATE INDEX ifx_yara_rules_references ON yara_rules (`references`(1000));""")
22-
op.execute("""CREATE INDEX ifx_yara_rules_condition ON yara_rules (`condition`(1000));""")
23-
op.execute("""CREATE INDEX ifx_yara_rules_strings ON yara_rules (`strings`(1000));""")
24-
20+
pass
2521

2622
def downgrade():
27-
op.execute("""DROP INDEX ifx_yara_rules_description ON yara_rules;""")
28-
op.execute("""DROP INDEX ifx_yara_rules_references ON yara_rules;""")
29-
op.execute("""DROP INDEX ifx_yara_rules_condition ON yara_rules;""")
30-
op.execute("""DROP INDEX ifx_yara_rules_strings ON yara_rules;""")
23+
pass

‎migrations/versions/8c3bbaef0f9c_activity_log.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def upgrade():
2121
op.create_table('activity_log',
2222
sa.Column('id', sa.Integer(), nullable=False),
2323
sa.Column('activity_type', sa.String(length=256), nullable=True),
24-
sa.Column('activity_text', sa.String(length=65000), nullable=True),
24+
sa.Column('activity_text', sa.TEXT(), nullable=True),
2525
sa.Column('activity_date', sa.DateTime(timezone=True), nullable=True),
2626
sa.Column('entity_type', sa.Integer(), nullable=False),
2727
sa.Column('entity_id', sa.Integer(), nullable=False),

‎migrations/versions/8d2c133477a2_tasks_model.py

+1-3
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ def upgrade():
2323
sa.Column('date_modified', sa.DateTime(timezone=True), nullable=True),
2424
sa.Column('state', sa.String(length=32), nullable=True),
2525
sa.Column('title', sa.String(length=256), nullable=True),
26-
sa.Column('description', sa.String(length=2048), nullable=True),
26+
sa.Column('description', sa.TEXT(), nullable=True),
2727
sa.Column('final_artifact', sa.String(length=4096), nullable=True),
2828
sa.Column('created_user_id', sa.Integer(), nullable=False),
2929
sa.Column('modified_user_id', sa.Integer(), nullable=False),
@@ -33,7 +33,6 @@ def upgrade():
3333
sa.ForeignKeyConstraint(['owner_user_id'], ['kb_users.id'], ),
3434
sa.PrimaryKeyConstraint('id')
3535
)
36-
op.create_index('ix_tasks_description', 'tasks', ['description'], unique=False)
3736
op.create_index('ix_tasks_state', 'tasks', ['state'], unique=False)
3837
op.create_index('ix_tasks_title', 'tasks', ['title'], unique=False)
3938
# ### end Alembic commands ###
@@ -43,6 +42,5 @@ def downgrade():
4342
# ### commands auto generated by Alembic - please adjust! ###
4443
op.drop_index('ix_tasks_title', table_name='tasks')
4544
op.drop_index('ix_tasks_state', table_name='tasks')
46-
op.drop_index('ix_tasks_description', table_name='tasks')
4745
op.drop_table('tasks')
4846
# ### end Alembic commands ###

‎migrations/versions/b67c53c89680_add_imports_column_to_yara_rules.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
def upgrade():
2020
# ### commands auto generated by Alembic - please adjust! ###
21-
op.add_column('yara_rules', sa.Column('imports', sa.String(length=512), nullable=True))
21+
op.add_column('yara_rules', sa.Column('imports', sa.String(length=2048), nullable=True))
2222
# ### end Alembic commands ###
2323

2424

‎migrations/versions/bf3d4a7f2f15_scripts_table.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ def upgrade():
2222
sa.Column('name', sa.String(length=128), nullable=False),
2323
sa.Column('description', sa.String(length=128), nullable=True),
2424
sa.Column('interpreter', sa.String(length=64), nullable=True),
25-
sa.Column('code', sa.String(length=60000), nullable=True),
25+
sa.Column('code', sa.TEXT(), nullable=True),
2626
sa.Column('match_regex', sa.String(length=4096), nullable=True),
2727
sa.Column('date_created', sa.DateTime(timezone=True), nullable=True),
2828
sa.Column('date_modified', sa.DateTime(timezone=True), nullable=True),

‎migrations/versions/cb108888af0b_new_files_table.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ def upgrade():
2222
sa.Column('id', sa.Integer(), nullable=False),
2323
sa.Column('date_created', sa.DateTime(timezone=True), nullable=True),
2424
sa.Column('date_modified', sa.DateTime(timezone=True), nullable=True),
25-
sa.Column('filename', sa.String(length=65000), nullable=True),
25+
sa.Column('filename', sa.TEXT(), nullable=True),
2626
sa.Column('content_type', sa.String(length=100), nullable=True),
2727
sa.Column('entity_type', sa.Integer(), nullable=False),
2828
sa.Column('entity_id', sa.Integer(), nullable=False),

‎requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ Flask-Bcrypt==1.0.1
3535
Flask-Caching==1.10.1
3636
Flask-Login==0.5.0
3737
Flask-Mail==0.9.1
38-
Flask-Migrate==3.1.0
38+
Flask-Migrate==2.6.0
3939
Flask-OpenID==1.3.0
4040
Flask-Script==2.0.6
4141
flask-selfdoc==1.3.0

0 commit comments

Comments
 (0)