Skip to content

Commit 0d784f0

Browse files
stefanhansstefanhans
committed
Restore
1 parent 580657d commit 0d784f0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+5504
-1
lines changed

‎README.md

+65-1
Original file line numberDiff line numberDiff line change
@@ -1 +1,65 @@
1-
# cli-chat
1+
### Purpose
2+
3+
Developing distributed systems is a complex project.
4+
Without a central instance which is always available we face some general problems:
5+
6+
- who to contact in the first place
7+
- which peers are currently online
8+
- how does the system scale
9+
10+
To experiment with different libraries and to gain experience about the subject, it is useful to have a command line
11+
running in an online connected peer.
12+
We can now, step-by-step, investigate the libraries and experiment freely and being open to failure.
13+
We gain experience in how different components interact with each other in an online network of peers.
14+
An example application gives us a realistic use case layer.
15+
16+
Here the major features we need:
17+
18+
- command line interface within the running peers
19+
- bootstrap data service
20+
- group membership protocol implementation
21+
22+
The bootstrap service has a separated API. The implementation of the group membership protocol and the CLI tool are libraries.
23+
24+
25+
### CLI Chat
26+
27+
The CLI Chat is a system, which has no central server and every peer can join and leave the chat at any time.
28+
As well a peer can leave the network unforeseeable and without informing any other peer.
29+
30+
31+
### CLI Tool
32+
33+
The interactive command line tool has these basic features:
34+
35+
- command completion and history
36+
- interactive logging
37+
- individual function integration
38+
- script execution
39+
40+
41+
### Bootstrap Data Service
42+
43+
The service consists of
44+
45+
- an API `bootstrap-data-api`
46+
- a backend for production `bootstrap-data-cloudfunctions` which is already deployed as Cloud Functions on GCP
47+
- a backend for development `bootstrap-data-server` which can be started locally
48+
49+
To switch between the backend you can use `source .bootstrap-switch` which is located under `cli-chat/cli-chat`
50+
51+
52+
### HashiCorps ’memberlist’ Library
53+
54+
We use the 'memberlist' library to inform all members about the connection information of the chat layer.
55+
And we get a notification from it about events like joining and leaving peers.
56+
57+
58+
### Chat Application
59+
60+
The chat application is simple - join, leave and send a message.
61+
62+
63+
64+
65+

‎bootstrap-data-api/README.md

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
We test the API against two different backend services:
2+
3+
- a webserver running on localhost:8080
4+
- GCP Cloud Functions
5+
6+
Start the webserver:
7+
8+
```
9+
cd ~/go/src/github.com/stefanhans/cli-chat/bootstrap-data-server
10+
11+
go build
12+
./bootstrap-data-server
13+
```
14+
15+
Now, execute the following tests:
16+
17+
```
18+
cd ~/go/src/github.com/stefanhans/cli-chat/bootstrap-data-api
19+
20+
export BOOTSTRAP_DATA_SERVER="http://localhost:8080"
21+
go test -run TestLocalhost
22+
23+
export BOOTSTRAP_DATA_SERVER="https://europe-west1-bootstrap-data-cloudfunctions.cloudfunctions.net"
24+
go test -run TestCf
25+
```

‎bootstrap-data-api/api.go

+269
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
package bootstrap_data_api
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
uuid "github.com/satori/go.uuid"
7+
"io/ioutil"
8+
"net/http"
9+
"os"
10+
"strings"
11+
"time"
12+
)
13+
14+
// Peer is the struct for the collection
15+
type Peer struct {
16+
ID string `json:"id,omitempty"` // UUID
17+
Name string `json:"name,omitempty"` // chat name
18+
Ip string `json:"ip,omitempty"`
19+
Port string `json:"port,omitempty"`
20+
Protocol string `json:"protocol,omitempty"` // "tcp" or "udp"
21+
// todo get rid of unused field status
22+
Status string `json:"status,omitempty"`
23+
Timestamp string `json:"timestamp,omitempty"` // Unix time in seconds
24+
}
25+
26+
type Config struct {
27+
MaxPeers int `json:"maxpeers,omitempty"` // Max number of bootstrap peers to be saved
28+
MinRefillCandidates int `json:"minrefillcandidates,omitempty"` // Number used to decide peer send refill request
29+
NumPeers int `json:"numpeers,omitempty"` // Number of bootstrap peers
30+
}
31+
32+
// BootstrapData is the complete data structure
33+
type BootstrapData struct {
34+
Config Config
35+
Peers map[string]*Peer
36+
}
37+
38+
// BootstrapDataAPI contains all data needed
39+
type BootstrapDataAPI struct {
40+
ServiceUrl string
41+
Self *Peer
42+
BootstrapData *BootstrapData
43+
}
44+
45+
// Create returns the individual BootstrapDataAPI initially
46+
func Create(peer *Peer) (*BootstrapDataAPI, error) {
47+
48+
// https://europe-west1-bootstrap-peers.cloudfunctions.net
49+
// http://localhost:8080
50+
serviceUrl := os.Getenv("BOOTSTRAP_DATA_SERVER")
51+
if serviceUrl == "" {
52+
return nil, fmt.Errorf("BOOTSTRAP_DATA_SERVER environment variable unset or missing")
53+
}
54+
55+
// Send ping request to service
56+
res, err := http.Post(serviceUrl+"/ping",
57+
"application/x-www-form-urlencoded",
58+
strings.NewReader(""))
59+
if err != nil {
60+
return nil, fmt.Errorf("failed to ping BOOTSTRAP_DATA_SERVER: %v\n", err)
61+
}
62+
fmt.Printf("Received reply from Ping: %v\n", res.Status)
63+
err = res.Body.Close()
64+
if err != nil {
65+
fmt.Printf("cannot close response body\n")
66+
}
67+
68+
// Create UUID for peer
69+
peer.ID = uuid.NewV4().String()
70+
71+
// Create timestamp from now for peer
72+
peer.Timestamp = fmt.Sprintf("%d", time.Now().Unix())
73+
74+
return &BootstrapDataAPI{
75+
ServiceUrl: serviceUrl,
76+
Self: peer,
77+
BootstrapData: &BootstrapData{},
78+
}, nil
79+
}
80+
81+
func (api *BootstrapDataAPI) Join() *BootstrapData {
82+
83+
// Send request to service
84+
res, err := http.Post(api.ServiceUrl+"/join",
85+
"application/x-www-form-urlencoded",
86+
strings.NewReader(fmt.Sprintf("%s %s %s %s %s %s %s",
87+
api.Self.ID, api.Self.Name, api.Self.Ip, api.Self.Port, api.Self.Protocol, "0", api.Self.Timestamp)))
88+
if err != nil {
89+
fmt.Printf("failed to join bootstrap-peers (%v): %v\n", api.Self, err)
90+
return nil
91+
}
92+
if res.StatusCode != http.StatusOK {
93+
b, _ := ioutil.ReadAll(res.Body)
94+
fmt.Printf("Received no \"200 OK\" from Join: %q\n", strings.TrimSuffix(string(b), "\n"))
95+
return nil
96+
}
97+
fmt.Printf("Received reply from Join: %v\n", res.Status)
98+
99+
// Read response body in JSON
100+
body, err := ioutil.ReadAll(res.Body)
101+
res.Body.Close()
102+
if err != nil {
103+
fmt.Printf("failed to read response of bootstrap-peers join (%v): %v\n", api.Self, err)
104+
return nil
105+
}
106+
107+
// Unmarshall bootstrap data
108+
var bootstrapData BootstrapData
109+
err = json.Unmarshal(body, &bootstrapData)
110+
if err != nil {
111+
fmt.Errorf("failed to unmarshall response of bootstrap-peers join (%v): %v\n", res.Proto, err)
112+
return nil
113+
}
114+
115+
return &bootstrapData
116+
}
117+
118+
func (api *BootstrapDataAPI) Leave(id string) *BootstrapData {
119+
120+
// Send request to service
121+
res, err := http.Post(api.ServiceUrl+"/leave",
122+
"application/x-www-form-urlencoded",
123+
strings.NewReader(id))
124+
if err != nil {
125+
fmt.Printf("failed to leave bootstrap-peers (%s): %v\n", id, err)
126+
return nil
127+
}
128+
fmt.Printf("Received reply from Leave: %v\n", res.Status)
129+
130+
// Read response body in JSON
131+
body, err := ioutil.ReadAll(res.Body)
132+
res.Body.Close()
133+
if err != nil {
134+
fmt.Printf("failed to read response of bootstrap-peers leave (%s): %v\n", id, err)
135+
return nil
136+
}
137+
138+
// Unmarshall bootstrap data
139+
var bootstrapData BootstrapData
140+
err = json.Unmarshal(body, &bootstrapData)
141+
if err != nil {
142+
fmt.Errorf("failed to unmarshall response of bootstrap-peers leave (%v): %v\n", res.Proto, err)
143+
return nil
144+
}
145+
146+
return &bootstrapData
147+
}
148+
149+
func (api *BootstrapDataAPI) Refill() *BootstrapData {
150+
151+
// Send request to service
152+
res, err := http.Post(api.ServiceUrl+"/refill",
153+
"application/x-www-form-urlencoded",
154+
strings.NewReader(fmt.Sprintf("%s %s %s %s %s %s %s",
155+
api.Self.ID, api.Self.Name, api.Self.Ip, api.Self.Port, api.Self.Protocol, api.Self.Status, api.Self.Timestamp)))
156+
if err != nil {
157+
fmt.Printf("failed to join bootstrap-peers (%v): %v\n", api.Self, err)
158+
return nil
159+
}
160+
fmt.Printf("Received reply from Refill: %v\n", res.Status)
161+
162+
// Read response body in JSON
163+
body, err := ioutil.ReadAll(res.Body)
164+
res.Body.Close()
165+
if err != nil {
166+
fmt.Printf("failed to read response of bootstrap-peers refill (%v): %v\n", api.Self, err)
167+
return nil
168+
}
169+
170+
// Unmarshall bootstrap data
171+
var bootstrapData BootstrapData
172+
err = json.Unmarshal(body, &bootstrapData)
173+
if err != nil {
174+
fmt.Errorf("failed to unmarshall response of bootstrap-peers leave (%v): %v\n", res.Proto, err)
175+
return nil
176+
}
177+
return &bootstrapData
178+
}
179+
180+
func (api *BootstrapDataAPI) List() *BootstrapData {
181+
182+
// Send request to service
183+
res, err := http.Post(api.ServiceUrl+"/list",
184+
"application/x-www-form-urlencoded",
185+
strings.NewReader(""))
186+
if err != nil {
187+
fmt.Printf("failed to list bootstrap-peers: %v\n", err)
188+
return nil
189+
}
190+
fmt.Printf("Received reply from List: %v\n", res.Status)
191+
192+
// Read response body in JSON
193+
body, err := ioutil.ReadAll(res.Body)
194+
res.Body.Close()
195+
if err != nil {
196+
fmt.Printf("failed to read response of bootstrap-peers list: %v\n", err)
197+
return nil
198+
}
199+
200+
// Unmarshall bootstrap data
201+
var bootstrapData BootstrapData
202+
err = json.Unmarshal(body, &bootstrapData)
203+
if err != nil {
204+
fmt.Errorf("failed to unmarshall response of bootstrap-peers join (%v): %v\n", res.Proto, err)
205+
return nil
206+
}
207+
208+
return &bootstrapData
209+
}
210+
211+
func (api *BootstrapDataAPI) Reset() *BootstrapData {
212+
213+
// Send request to service
214+
res, err := http.Post(api.ServiceUrl+"/reset",
215+
"application/x-www-form-urlencoded",
216+
strings.NewReader(""))
217+
if err != nil {
218+
fmt.Printf("failed to reset bootstrap-peers: %v\n", err)
219+
return nil
220+
}
221+
fmt.Printf("Received reply from Reset: %v\n", res.Status)
222+
223+
// Read response body in JSON
224+
body, err := ioutil.ReadAll(res.Body)
225+
res.Body.Close()
226+
if err != nil {
227+
fmt.Printf("failed to read response of bootstrap-peers list: %v\n", err)
228+
return nil
229+
}
230+
231+
// Unmarshall bootstrap data
232+
var bootstrapData BootstrapData
233+
err = json.Unmarshal(body, &bootstrapData)
234+
if err != nil {
235+
fmt.Errorf("failed to unmarshall response of bootstrap-peers Reset (%v): %v\n", res.Proto, err)
236+
return nil
237+
}
238+
239+
return &bootstrapData
240+
}
241+
242+
func (api *BootstrapDataAPI) UpdateConfig() (*BootstrapData, error) {
243+
244+
// Send request to service
245+
res, err := http.Post(api.ServiceUrl+"/config",
246+
"application/x-www-form-urlencoded",
247+
strings.NewReader(fmt.Sprintf("%d %d",
248+
api.BootstrapData.Config.MaxPeers, api.BootstrapData.Config.MinRefillCandidates)))
249+
if err != nil {
250+
return nil, fmt.Errorf("failed to config bootstrap data: %v", err)
251+
}
252+
fmt.Printf("Received reply from UpdateConfig: %v\n", res.Status)
253+
254+
// Read response body in JSON
255+
body, err := ioutil.ReadAll(res.Body)
256+
res.Body.Close()
257+
if err != nil {
258+
return nil, fmt.Errorf("failed to read response of bootstrap data update config: %v", err)
259+
}
260+
261+
// Unmarshall bootstrap data
262+
var bootstrapData BootstrapData
263+
err = json.Unmarshal(body, &bootstrapData)
264+
if err != nil {
265+
return nil, fmt.Errorf("failed to unmarshall response of bootstrap data UpdateConfig (%v): %v\n", res.Proto, err)
266+
}
267+
268+
return &bootstrapData, nil
269+
}

0 commit comments

Comments
 (0)