Skip to content

Commit 2a892fa

Browse files
tmuth-splunktmuth-splunk
committed
Added build process
1 parent 8c20393 commit 2a892fa

File tree

6 files changed

+145
-3
lines changed

6 files changed

+145
-3
lines changed

‎SPLUNK_HOME-etc-apps/tmuth-data-load/local/props.conf

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,12 @@
11
[access_combined_buttercup]
22
LINE_BREAKER = ([\r\n]+)
33
NO_BINARY_CHECK = true
4-
#REPORT-access = access-extractions
54
SHOULD_LINEMERGE = false
65
TIME_PREFIX=\[
76
TIME_FORMAT=%d/%b/%Y:%H:%M:%S
87
MAX_TIMESTAMP_LOOKAHEAD=50
98
TRUNCATE=999999
109
category = Web
11-
description = National Center for Supercomputing Applications (NCSA) combined format HTTP web server logs (can be generated by apache or other web servers)
12-
#maxDist = 28
1310
TRANSFORMS-buttercup=buttercup-games
1411
#SEDCMD-mask_ip = s/^(\d{1,3})\.(\d{1,3})\.\d{1,3}\.\d{1,3}/\1.\2.x.x/
1512
#TRANSFORMS-buttercup=find-ip-addresses,find-referrer,buttercup-games

‎build/build.sh

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/sh
2+
BUILD_NUM_FILE="build_number"
3+
increment_version ()
4+
{
5+
declare -a part=( ${1//\./ } )
6+
declare new
7+
declare -i carry=1
8+
9+
for (( CNTR=${#part[@]}-1; CNTR>=0; CNTR-=1 )); do
10+
len=${#part[CNTR]}
11+
new=$((part[CNTR]+carry))
12+
[ ${#new} -gt $len ] && carry=1 || carry=0
13+
[ $CNTR -gt 0 ] && part[CNTR]=${new: -len} || part[CNTR]=${new}
14+
done
15+
new="${part[*]}"
16+
echo "${new// /.}"
17+
}
18+
OLD_VERSION=`cat $BUILD_NUM_FILE`
19+
20+
NEW_VERSION=`increment_version "$OLD_VERSION"`
21+
echo "$NEW_VERSION"
22+
echo "$NEW_VERSION" > $BUILD_NUM_FILE
23+
# Replace the version number in the script
24+
sed -i -E "/VERSION=/s/=.*/=$NEW_VERSION/" ../load-splunk-data.sh
25+
26+
# Copy local changes to props.conf etc into the repo
27+
DATA_LOAD_APP=$SPLUNK_HOME/etc/apps/tmuth-data-load/local
28+
REPO_APP=../SPLUNK_HOME-etc-apps/tmuth-data-load/local
29+
cp $DATA_LOAD_APP/props.conf $REPO_APP/
30+
cp $DATA_LOAD_APP/transforms.conf $REPO_APP/
31+
cp $DATA_LOAD_APP/fields.conf $REPO_APP/
32+
33+
git tag -a $NEW_VERSION -m "new release"
34+
git push origin $NEW_VERSION

‎build/build_number

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1.0.1

‎load-splunk-data.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
#!/bin/bash
22

3+
VERSION=1.0.1
4+
35
function show_help {
46
echo "
57
8+
VERSION: ${VERSION}
9+
610
##################################################################
711
This script is meant to streamline the process of getting files into Splunk.
812
The goal is to:

‎column-test.sh renamed to ‎misc/column-test.sh

File renamed without changes.

‎misc/search-test.sh

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#!/bin/bash
2+
3+
SPLUNK_HOST=localhost:8089
4+
SPLUNK_USERNAME=admin
5+
SPLUNK_PASS=welcome1
6+
7+
AUTH_TOKEN="eyJraWQiOiJzcGx1bmsuc2VjcmV0IiwiYWxnIjoiSFM1MTIiLCJ2ZXIiOiJ2MiIsInR0eXAiOiJzdGF0aWMifQ.eyJpc3MiOiJhZG1pbiBmcm9tIHRtdXRoLW1icC1lNDFlOSIsInN1YiI6ImFkbWluIiwiYXVkIjoiUkVTVCBDYWxscyIsImlkcCI6IlNwbHVuayIsImp0aSI6IjA1MzU4ZjYzZjFjZjVkNzdhMjcwMzg1NzM0NDFmMTZiNTgxZWIxMGY1M2RlNzYyNWE4ZThkMTBhM2Y3YzY2OTMiLCJpYXQiOjE2MzEyODYyNzYsImV4cCI6MTY5NDM1ODI3NiwibmJyIjoxNjMxMjg2Mjc2fQ.RQdjHDkozi59yfn-Dh7BhaUmTuWrIT2y5textF7G3t3FTMk5npCUC2GzORyH4PdiKZUXvNb17UGAELOe6Nvifw"
8+
9+
10+
11+
function print_section {
12+
#printf "\n****** ${1} ************************************************\n"
13+
printf "\n****** %-.80s \n" "${1} ************************************************************************************************"
14+
if [[ "$1" = *"_END" ]];then
15+
printf "\n\n"
16+
fi
17+
}
18+
19+
print_section "ONESHOT_BEGIN"
20+
print_section "ONESHOT_END"
21+
22+
23+
# exit 0
24+
25+
# URL escape codes used to pass special characters. DO NOT CHANGE!
26+
dqt="%22"
27+
pct="%25"
28+
29+
function splunk_search_polling {
30+
SEARCH_LEVEL=${2:-verbose}
31+
curl_opts_common=( -s -k -d adhoc_search_level=${SEARCH_LEVEL} )
32+
33+
AUTH_OPTION=()
34+
if [ ! -z "$AUTH_TOKEN" ]; then
35+
AUTH_OPTION+=( -H "Authorization: Bearer ${AUTH_TOKEN}" )
36+
#echo "Using Token Auth"
37+
else
38+
AUTH_OPTION+=( -u ${SPLUNK_USERNAME}:${SPLUNK_PASS} )
39+
#echo "Using User:Pass Auth"
40+
fi
41+
42+
curl_opts=( "${curl_opts_common[@]}" "${AUTH_OPTION[@]}" -d search="${1}" )
43+
#echo ${curl_opts[@]}
44+
45+
RESULTS=`curl "${curl_opts[@]}" -X POST https://${SPLUNK_HOST}/services/search/jobs`
46+
47+
SID=`echo $RESULTS | sed -e 's,.*<sid>\([^<]*\)<\/sid>.*,\1,g' `
48+
printf "\n"
49+
echo "SID: $SID"
50+
51+
SEARCH_STATUS=""
52+
counter=30 # will check for status=DONE this many times, every ${wait_seconds}
53+
wait_seconds=1
54+
while [ $counter -gt 0 ]
55+
do
56+
curl_opts=( "${curl_opts_common[@]}" "${AUTH_OPTION[@]}" )
57+
OUTPUT=`curl "${curl_opts[@]}" -X POST https://${SPLUNK_HOST}/services/search/jobs/${SID}`
58+
#echo "$OUTPUT"
59+
STATUS=`echo $OUTPUT | sed -e 's,.*<s:key name=\"dispatchState\">\([^<]*\)<\/s\:key>.*,\1,g' `
60+
#echo "$STATUS"
61+
if [[ "$STATUS" = "DONE" ]]; then
62+
SEARCH_STATUS="DONE"
63+
#echo "Leaving status loop"
64+
break 1
65+
fi
66+
counter=$(( $counter - 1 ))
67+
sleep ${wait_seconds}
68+
done
69+
70+
if [[ "$SEARCH_STATUS" = "DONE" ]]; then
71+
curl_opts=( "${curl_opts_common[@]}" "${AUTH_OPTION[@]}" -d output_mode=csv )
72+
SEARCH_RESULTS=`curl "${curl_opts[@]}" -X GET https://${SPLUNK_HOST}/services/search/jobs/${SID}/results`
73+
echo "$SEARCH_RESULTS" | sed 's/,/ ,/g' | column -t -s, \
74+
| awk 'NR == 1 {print $0;print $0}; NR > 1 {print $0}' \
75+
| sed '2 s/[^[:space:]]/-/g'
76+
fi
77+
}
78+
79+
80+
81+
function splunk_search_oneshot {
82+
#echo ${1}
83+
printf "\n\n"
84+
echo "splunk_search_oneshot - adhoc_search_level: ${2}"
85+
curl -s -k -u ${SPLUNK_USERNAME}:${SPLUNK_PASS} \
86+
-X POST https://${SPLUNK_HOST}/services/search/jobs/ \
87+
-d search="${1}" \
88+
-d exec_mode=oneshot -d output_mode=csv -d adhoc_search_level=${2}
89+
}
90+
91+
92+
SEARCH_STRING="search index=_internal sourcetype=splunkd component=Metrics earliest=-5m | table _time, group, name | head 5 "
93+
94+
#splunk_search_polling "${SEARCH_STRING}" "verbose"
95+
#splunk_search_polling "${SEARCH_STRING}" "fast"
96+
97+
#splunk_search_oneshot "${SEARCH_STRING}" "verbose"
98+
#splunk_search_oneshot "${SEARCH_STRING}" "fast"
99+
100+
INDEX=main
101+
SEARCH_STRING=" | walklex index=${dqt}${INDEX}${dqt} type=field | search NOT field=${dqt} *${dqt} | where NOT LIKE(field,${dqt}date_${pct}${dqt}) | stats sum(distinct_values) by field "
102+
#SEARCH_STRING=" search | walklex index=$INDEX earliest=1 "
103+
#splunk_search_polling "${SEARCH_STRING}" "verbose"
104+
105+
splunk_search_polling "search index=main | stats count"
106+

0 commit comments

Comments
 (0)