Skip to content

Commit 4e8c924

Browse files
author
jiashiwen
committed
firest commit
1 parent 14fcb32 commit 4e8c924

20 files changed

+1453
-1
lines changed

‎.gitignore

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/bin
2+
/.gradle
3+
/.settings
4+
/build
5+
settings.gradle
6+
.project
7+
.classpath

‎Dockerfile

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM java:8u92-jre-alpine
2+
RUN mkdir -p /elastictransfor
3+
COPY ./build/libs/elastictransfor-1.0.jar /elastictransfor/
4+
WORKDIR /elastictransfor
5+
ENTRYPOINT ["java","-jar", "/elastictransfor/elastictransfor-1.0.jar"]

‎README.md

+224-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,225 @@
11
# elastictransfor
2-
elasticsearch index transfor,reindex tool
2+
3+
4+
## Purpose
5+
6+
This programe is a command line tools for copy elasticsearch index.
7+
8+
# License
9+
10+
Apache License 2.0
11+
12+
13+
## How Install
14+
15+
16+
```
17+
git clone https://github.com/jiashiwen/elastictransfor
18+
```
19+
```
20+
cd elastictransfor
21+
```
22+
```
23+
gradle clean build
24+
```
25+
26+
you should see 'build/libs/elastictransfor-1.0.jar‘
27+
28+
29+
30+
31+
## Usage
32+
33+
### command:
34+
```
35+
java -jar build/libs/elastictransfor-1.0.jar --help
36+
```
37+
### out put:
38+
```
39+
Usage: java -jar elastictransfor.jar [options]
40+
Options:
41+
--dsl
42+
elasticsearch query dsl for Preform a partial transfor based on search
43+
results.you must make content of this variable between '',just like
44+
'{"query":{"term":{"word.primitive":{"value":"keywork"}}}}'
45+
--help
46+
47+
Default: false
48+
--script_file
49+
execute script file write by json
50+
--source_cluster
51+
Source elasticsearch cluster name,default is 'elasticsearch
52+
Default: elasticsearch
53+
--source_host
54+
Source elasticsearch cluster one of master ip address,default is
55+
'127.0.0.1'.
56+
Default: 127.0.0.1
57+
--source_index
58+
Source index name
59+
--source_port
60+
Source port
61+
Default: 9300
62+
--target_cluster
63+
Target elasticsearch cluster name,default is 'elasticsearch
64+
Default: elasticsearch
65+
--target_host
66+
Target elasticsearch cluster one of master ip address,default is
67+
'127.0.0.1'.
68+
Default: 127.0.0.1
69+
--target_index
70+
Target index name
71+
--target_port
72+
Target port
73+
Default: 9300
74+
--type
75+
Transfor type value is [data,meta,force] and default value is 'meta'.If
76+
value is 'metadata' try to create a new empty target index as
77+
source;'data' copy source index documents to target index; 'force'
78+
delete target index if exists and copy source index to target index.
79+
Default: meta
80+
81+
```
82+
83+
## Example
84+
### command
85+
```
86+
java -jar build/libs/elastictransfor-1.0.jar --source_cluster sourceclustername \
87+
--source_host 10.0.0.1 \
88+
--source_index sourceindex \
89+
--target_cluster targetclustername \
90+
--target_host 10.1.0.1 \
91+
--target_index targetindex \
92+
--type force
93+
94+
```
95+
96+
### out put:
97+
```
98+
2016-08-02 15:31:59 [INFO] - {
99+
"help" : false,
100+
"source_cluster" : "sourceclustername",
101+
"source_host" : "10.0.0.1",
102+
"source_port" : 9300,
103+
"source_index" : "sourceindex",
104+
"target_cluster" : "targetclustername",
105+
"target_host" : "10.1.0.1",
106+
"target_port" : 9300,
107+
"target_index" : "targetindex",
108+
"type" : "force"
109+
}
110+
2016-08-02 15:31:59 [INFO] - [Calvin Rankin] modules [], plugins [], sites []
111+
2016-08-02 15:31:59 [INFO] - [Scarlet Spider] modules [], plugins [], sites []
112+
FORCE OK!
113+
2016-08-02 15:31:59 [INFO] - targetindex deleted!
114+
2016-08-02 15:32:00 [INFO] - Target index targetindex create complete!
115+
2016-08-02 15:32:00 [INFO] - 129 documents putted!!
116+
2016-08-02 15:32:00 [INFO] - copy index sourceindex targetindex complete
117+
```
118+
119+
## Example by query
120+
### command
121+
```
122+
java -jar elastictransfor-1.0.jar \
123+
--source_cluster sourceclustername \
124+
--source_host 10.0.0.1 \
125+
--source_index sourceindex \
126+
--target_cluster targetclustername \
127+
--target_host 10.1.0.1 \
128+
--target_index targetindex \
129+
--type data \
130+
--dsl '{"query":{"term":{"word.primitive":{"value":"keyword"}}}}'
131+
```
132+
### out put:
133+
```
134+
2016-08-03 12:35:26 [INFO] - Your setting is:
135+
{
136+
"help" : false,
137+
"source_cluster" : "sourceclustername",
138+
"source_host" : "10.0.0.1",
139+
"source_port" : 9300,
140+
"source_index" : "sourceindex",
141+
"target_cluster" : "targetclustername",
142+
"target_host" : "10.1.0.1",
143+
"target_port" : 9300,
144+
"target_index" : "targetindex",
145+
"type" : "data",
146+
"dsl" : "{\"query\":{\"term\":{\"word.primitive\":{\"value\":\"keyword\"}}}}"
147+
}
148+
2016-08-03 12:35:26 [INFO] - [Jean Grey] modules [], plugins [], sites []
149+
2016-08-03 12:35:27 [INFO] - [Man-Spider] modules [], plugins [], sites []
150+
{"query":{"term":{"word.primitive":{"value":"keyword"}}}}
151+
TYPE IS DATA OK!
152+
2016-08-03 12:35:27 [INFO] - 128 documents putted!!
153+
2016-08-03 12:35:27 [INFO] - copy index sourceindex 1 documents to targetindex complete
154+
```
155+
156+
## Example by use json execute script
157+
### command
158+
```
159+
java -jar elastictransfor-1.0.jar --script_file script.json
160+
```
161+
### script.json
162+
```
163+
{
164+
"source_cluster": "es-source",
165+
"source_host": "10.0.0.22",
166+
"source_port": 9300,
167+
"target_cluster": "es-target",
168+
"target_host": "10.10.10.32",
169+
"target_port": 9300,
170+
"indexes": [
171+
{
172+
"source_index": "index1",
173+
"target_index": "index1",
174+
"type": "force"
175+
},
176+
{
177+
"source_index": "index2",
178+
"target_index": "index2-copy",
179+
"type": "meta"
180+
}
181+
]
182+
}
183+
184+
```
185+
186+
187+
## Use Docker
188+
189+
### command
190+
191+
```
192+
sudo docker run --rm jiashiwen/elastictransfor --source_cluster sourceclustername \
193+
--source_host 10.0.0.1 \
194+
--source_index sourceindex \
195+
--target_cluster targetclustername \
196+
--target_host 10.1.0.1 \
197+
--target_index targetindex \
198+
--type force
199+
```
200+
### out put:
201+
```
202+
2016-08-02 15:31:59 [INFO] - {
203+
"help" : false,
204+
"source_cluster" : "sourceclustername",
205+
"source_host" : "10.0.0.1",
206+
"source_port" : 9300,
207+
"source_index" : "sourceindex",
208+
"target_cluster" : "targetclustername",
209+
"target_host" : "10.1.0.1",
210+
"target_port" : 9300,
211+
"target_index" : "targetindex",
212+
"type" : "force"
213+
}
214+
2016-08-02 15:31:59 [INFO] - [Calvin Rankin] modules [], plugins [], sites []
215+
2016-08-02 15:31:59 [INFO] - [Scarlet Spider] modules [], plugins [], sites []
216+
FORCE OK!
217+
2016-08-02 15:31:59 [INFO] - targetindex deleted!
218+
2016-08-02 15:32:00 [INFO] - Target index targetindex create complete!
219+
2016-08-02 15:32:00 [INFO] - 129 documents putted!!
220+
2016-08-02 15:32:00 [INFO] - copy index sourceindex targetindex complete
221+
```
222+
## Use Docker with script
223+
```
224+
sudo docker run -v ${PWD}:/script --rm jiashiwen/elastictransfor --script_file /script/script.json
225+
```

‎build.gradle

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* This build file was auto generated by running the Gradle 'init' task
3+
* by 'develop' at '16-7-27 上午11:05' with Gradle 2.14
4+
*
5+
* This generated file contains a sample Java project to get you started.
6+
* For more details take a look at the Java Quickstart chapter in the Gradle
7+
* user guide available at https://docs.gradle.org/2.14/userguide/tutorial_java_projects.html
8+
*/
9+
10+
// Apply the java plugin to add support for Java
11+
apply plugin: 'java'
12+
apply plugin: 'eclipse'
13+
14+
15+
sourceCompatibility = 1.8
16+
targetCompatibility = 1.8
17+
18+
//create a single Jar with all dependencies
19+
version=1.0
20+
//task releaseJar(type: Jar) {
21+
// manifest {
22+
// attributes 'Implementation-Title': 'Gradle Jar File Example',
23+
// 'Implementation-Version': version,
24+
// 'Main-Class': 'elastictransfor.ElasticTransforMain'
25+
// }
26+
// baseName = project.name + '-all'
27+
// from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
28+
// with jar
29+
//}
30+
31+
jar {
32+
manifest {
33+
attributes("Implementation-Title": "Gradle",
34+
"Implementation-Version": version,
35+
'Main-Class': 'elastictransfor.ElasticTransforMain')
36+
}
37+
baseName = project.name
38+
from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
39+
}
40+
41+
// In this section you declare where to find the dependencies of your project
42+
repositories {
43+
// Use 'jcenter' for resolving your dependencies.
44+
// You can declare any Maven/Ivy/file repository here.
45+
maven { url "http://10.204.12.34:8081/repository/maven-public" }
46+
jcenter()
47+
}
48+
49+
// In this section you declare the dependencies for your production and test code
50+
dependencies {
51+
// The production code uses the SLF4J logging API at compile time
52+
compile 'org.slf4j:slf4j-api:1.7.21'
53+
// compile 'org.slf4j:slf4j-log4j12:1.7.21'
54+
// compile 'org.slf4j:slf4j-simple:1.7.21'
55+
compile 'com.beust:jcommander:1.55'
56+
compile 'ch.qos.logback:logback-classic:1.1.7'
57+
compile 'ch.qos.logback:logback-core:1.1.7'
58+
compile 'commons-cli:commons-cli:1.3.1'
59+
compile "com.beust:jcommander:1.48"
60+
61+
62+
compile group: 'org.elasticsearch', name: 'elasticsearch', version: '2.3.3'
63+
compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.6.6'
64+
65+
// Declare the dependency for your favourite test framework you want to use in your tests.
66+
// TestNG is also supported by the Gradle Test task. Just change the
67+
// testCompile dependency to testCompile 'org.testng:testng:6.8.1' and add
68+
// 'test.useTestNG()' to your build script.
69+
testCompile 'junit:junit:4.12'
70+
}

‎gradle/wrapper/gradle-wrapper.jar

52.1 KB
Binary file not shown.

‎gradle/wrapper/gradle-wrapper.properties

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#Wed Jul 27 11:05:09 CST 2016
2+
distributionBase=GRADLE_USER_HOME
3+
distributionPath=wrapper/dists
4+
zipStoreBase=GRADLE_USER_HOME
5+
zipStorePath=wrapper/dists
6+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.14-bin.zip

0 commit comments

Comments
 (0)