upload files

Author: xros

.gitignore

@@ -0,0 +1,68 @@
+# Ignore .directory created by file manager dolphine
+.directory
+# For csv.gz data
+*.csv.gz
+# For aria2 downloading tempearary files
+*.aria2
+# For url(s) which was captured by pandas from HTML pages
+*.url
+# For ruby tool "dap" packages
+# Ignore rvm files
+.ruby-version
+.ruby-gemset
+
+# Ignore geoip data file
+data/geoip.dat
+
+data/
+env_python27_nervey/
+*.swp
+*.swo
+*.swq
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*~
+.idea/
+.idea/*
+# C extensions
+*.so
+# Distribution / packaging
+.Python
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+# Translations
+*.mo
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+# Rope
+.ropeproject
+# Django stuff:
+*.log
+*.pot
+# Sphinx documentation
+docs/_build/

README.md

@@ -0,0 +1,6 @@
+json-py-es
+==========
+* To import raw JSON data files to ElasticSearch using Python
+
+> Using ElasticSearch Bulk API to import this data, sometimes ES only recognizes data in its request style. Now let `json-py-es` glue them all.
+

export_data.sh

@@ -0,0 +1,2 @@
+import elasticsearch
+es = elasticsearch.Elasticsearch

json-py-es.py

@@ -0,0 +1,4 @@
+{"index":{"_index":"serverdata", "_type":"smtp"}}
+{"ip":"23.91.113.228","timestamp":"2015-10-16T14:00:13-04:00","data":{"banner":"220 poppy.arvixe.com ESMTP\r\n","ehlo":"250-poppy.arvixe.com\r\n250-SIZE 20480000\r\n250 AUTH LOGIN\r\n","starttls":"502 STARTTLS NOT ALLOWED.\r\n"},"error":"Bad return code for STARTTLS","error_component":"starttls"}
+{"index":{"_index":"serverdata", "_type":"smtp"}}
+{"ip":"204.3.67.57","timestamp":"2015-10-16T14:00:13-04:00","data":{"banner":"220 center-transforming-relationships.com ESMTP \r\n","ehlo":"250-center-transforming-relationships.com \r\n250-PIPELINING\r\n250-SIZE 36700160\r\n250 8BITMIME\r\n","starttls":"502 unimplemented (#5.5.1)\r\n"},"error":"Bad return code for STARTTLS","error_component":"starttls"}

new_data.json

@@ -0,0 +1,10 @@
+{"ip":"74.63.161.124","timestamp":"2015-10-16T14:00:13-04:00","data":{},"error":"EOF","error_component":"banner"}
+{"ip":"65.36.165.102","timestamp":"2015-10-16T14:00:13-04:00","data":{"banner":"220 MORS.ad.safesecureweb.com Microsoft ESMTP MAIL Service, Version: 6.0.3790.4675 ready at  Fri, 16 Oct 2015 13:57:02 -0400 \r\n","ehlo":"250-MORS.ad.safesecureweb.com Hello [141.212.122.128]\r\n250-TURN\r\n250-SIZE 2097152\r\n250-ETRN\r\n250-PIPELINING\r\n250-DSN\r\n250-ENHANCEDSTATUSCODES\r\n250-8bitmime\r\n250-BINARYMIME\r\n250-CHUNKING\r\n250-VRFY\r\n250 OK\r\n","starttls":"554 5.7.3 Unable to initialize security subsystem\r\n"},"error":"Bad return code for STARTTLS","error_component":"starttls"}
+{"ip":"23.91.113.228","timestamp":"2015-10-16T14:00:13-04:00","data":{"banner":"220 poppy.arvixe.com ESMTP\r\n","ehlo":"250-poppy.arvixe.com\r\n250-SIZE 20480000\r\n250 AUTH LOGIN\r\n","starttls":"502 STARTTLS NOT ALLOWED.\r\n"},"error":"Bad return code for STARTTLS","error_component":"starttls"}
+{"ip":"204.3.67.57","timestamp":"2015-10-16T14:00:13-04:00","data":{"banner":"220 center-transforming-relationships.com ESMTP \r\n","ehlo":"250-center-transforming-relationships.com \r\n250-PIPELINING\r\n250-SIZE 36700160\r\n250 8BITMIME\r\n","starttls":"502 unimplemented (#5.5.1)\r\n"},"error":"Bad return code for STARTTLS","error_component":"starttls"}
+{"ip":"45.43.231.247","timestamp":"2015-10-16T14:00:13-04:00","data":{},"error":"read tcp 45.43.231.247:25: connection reset by peer","error_component":"banner"}
+{"ip":"204.73.44.226","timestamp":"2015-10-16T14:00:13-04:00","data":{"banner":"220 Augusoft Internal Email Server At Your Service\r\n","ehlo":"250-mrsshepherd.com [141.212.122.128], this server offers 4 extensions\r\n250-AUTH LOGIN\r\n250-SIZE 20480000\r\n250-HELP\r\n250 AUTH=LOGIN\r\n","starttls":"503 Bad sequence of commands\r\n"},"error":"Bad return code for STARTTLS","error_component":"starttls"}
+{"ip":"200.46.210.154","timestamp":"2015-10-16T14:00:13-04:00","data":{},"error":"read tcp 200.46.210.154:25: connection reset by peer","error_component":"banner"}
+{"ip":"50.28.80.160","timestamp":"2015-10-16T14:00:13-04:00","data":{"banner":"220-raptor.ws9000.net ESMTP Exim 4.85 #2 Fri, 16 Oct 2015 14:00:13 -0400 \r\n220-We do not authorize the use of this system to transport unsolicited, \r\n220 and/or bulk e-mail.\r\n","ehlo":"250-raptor.ws9000.net Hello researchscan383.eecs.umich.edu [141.212.122.128]\r\n250-SIZE 52428800\r\n250-8BITMIME\r\n250-PIPELINING\r\n250-AUTH PLAIN LOGIN\r\n250-STARTTLS\r\n250 HELP\r\n","starttls":"220 TLS go ahead\r\n","tls":{"client_hello":{"random":"tF27lvgudpkDoEsorUp0saOIgVPfjc7ZbJFCmAZxbHQ="}}},"error":"remote error: handshake failure","error_component":"starttls"}
+{"ip":"50.93.209.78","timestamp":"2015-10-16T14:00:13-04:00","data":{"banner":"220-charybdis.restechservices.net ESMTP Exim 4.85 #2 Fri, 16 Oct 2015 13:00:12 -0500 \r\n220-We do not authorize the use of this system to transport unsolicited, \r\n220 and/or bulk e-mail.\r\n","ehlo":"250-charybdis.restechservices.net Hello researchscan383.eecs.umich.edu [141.212.122.128]\r\n250-SIZE 52428800\r\n250-8BITMIME\r\n250-PIPELINING\r\n250-AUTH PLAIN LOGIN\r\n250-STARTTLS\r\n250 HELP\r\n","starttls":"220 TLS go ahead\r\n","tls":{"client_hello":{"random":"FT0jMqzr1KkF/eurB1K2xdgfNLw0K8+9P6HwMtrSaWw="}}},"error":"remote error: handshake failure","error_component":"starttls"}
+{"ip":"66.96.180.160","timestamp":"2015-10-16T14:00:13-04:00","data":{"banner":"220 ESMTP Fri, 16 Oct 2015 14:00:13 -0400: UCE strictly prohibited\r\n","ehlo":"250-bosauthsmtp10.yourhostingaccount.com Hello eecs.umich.edu [141.212.122.128]\r\n250-SIZE 34603008\r\n250-8BITMIME\r\n250-PIPELINING\r\n250-AUTH PLAIN LOGIN\r\n250-STARTTLS\r\n250 HELP\r\n","starttls":"220 TLS go ahead\r\n","tls":{"client_hello":{"random":"SoAhgklxKMZefAJywXW8FjEg+815O8NpsTwx2sQxvtQ="}}},"error":"remote error: handshake failure","error_component":"starttls"}

raw_data.json

@@ -0,0 +1 @@
+elasticsearch