Skip to content

Commit 53e577e

Browse files
johannesduesingjohannesduesing
authored
Merge pull request #101 from delphi-hub/develop
Release version 0.9.0
2 parents fc5db9e + 192e1bc commit 53e577e

32 files changed

+2803
-944
lines changed

‎OpenAPISpecification.yaml

Lines changed: 379 additions & 154 deletions
Large diffs are not rendered by default.

‎README.md

Lines changed: 105 additions & 16 deletions
Large diffs are not rendered by default.

‎Setup/Delphi_install.sh

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/usr/bin/env bash
2+
3+
####SETTINGS####
4+
DELPHI_NETWORK_NAME=delphi
5+
6+
####DELPHI INSTALLATION#####
7+
#########################
8+
9+
LOGFILE=$HOME/`date "+%F_%T"`\_$SUDO_USER.log
10+
sudo touch $LOGFILE
11+
sudo chown $SUDO_USER $LOGFILE
12+
13+
echo " Starting installation of Delphi Application"
14+
echo " Logs can be found at: $LOGFILE"
15+
16+
echo " Setting up Traefik..."
17+
sudo -u $SUDO_USER bash -c "docker network create $DELPHI_NETWORK_NAME &>> $LOGFILE"
18+
sudo -u $SUDO_USER bash -c "docker-compose up -d &>> $LOGFILE"
19+
echo " **Setup of Traefik completed**"
20+
21+
echo " Cloning Delphi ..."
22+
sudo -u $SUDO_USER bash -c "git clone --progress https://github.com/delphi-hub/delphi.git --recurse-submodules>> $LOGFILE"
23+
echo " **Cloning of Delphi Repositories completed**"
24+
25+
26+
#######Docker Setup######
27+
#########################
28+
# Testing purposes:
29+
# sudo apt-get purge docker-ce
30+
# sudo rm -rf /var/lib/docker
31+
# sudo rm -rf /etc/docker
32+
# sudo groupdel docker
33+
34+
35+
## Installing Images and Creating Volumes
36+
37+
echo " Building Delphi Images. This step might take several minutes"
38+
sudo -u $SUDO_USER bash -c "cd ..;sbt docker:publishLocal &>> $LOGFILE"
39+
echo " Delphi-Registry Image built"
40+
sudo -u $SUDO_USER bash -c "(cd ./delphi/delphi-webapi;sbt docker:publishLocal) &>> $LOGFILE"
41+
echo " Delphi-WebApi Image built"
42+
sudo -u $SUDO_USER bash -c "(cd ./delphi/delphi-webapp;sbt docker:publishLocal) &>> $LOGFILE"
43+
echo " Delphi-WebApp Image built"
44+
sudo -u $SUDO_USER bash -c "(cd ./delphi/delphi-crawler;sbt docker:publishLocal) &>> $LOGFILE"
45+
echo " Delphi-Crawler Image built"
46+
sudo -u $SUDO_USER bash -c "(cd ./delphi/delphi-management;sbt docker:publishLocal) &>> $LOGFILE"
47+
echo " Delphi-Management Image built"
48+
bash -c "docker images >> $LOGFILE"
49+
50+
51+
echo " **Docker Images for all the components built successfully**"
52+
53+
54+
#########Finished########
55+
#########################
56+
57+
echo " For more information regarding the application, please have a look at the Readme file found at: https://github.com/delphi-hub/delphi-registry/blob/master/README.md"
58+
59+
60+

‎Setup/docker-compose.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
version: '3.5'
2+
3+
services:
4+
5+
traefik:
6+
restart: always
7+
image: traefik:1.6
8+
container_name: traefik
9+
ports:
10+
- "80:80"
11+
- "8080:8080"
12+
networks:
13+
- delphi
14+
volumes:
15+
- ./traefik.toml:/etc/traefik/traefik.toml
16+
- /var/run/docker.sock:/var/run/docker.sock:ro
17+
18+
networks:
19+
delphi:
20+
external: true
21+
22+

‎Setup/traefik.toml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[entryPoints]
2+
[entryPoints.http]
3+
address = ":80"
4+
5+
[api]
6+
7+
[docker]
8+
9+
domain = "delphi.de"
10+
11+

‎build.sbt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test"
1919
libraryDependencies += "org.parboiled" %% "parboiled" % "2.1.4"
2020
libraryDependencies += "com.typesafe.akka" %% "akka-http-testkit" % "10.1.5"
2121

22+
libraryDependencies += "com.pauldijou" %% "jwt-core" % "1.0.0"
2223

2324
lazy val registry = (project in file(".")).
2425
enablePlugins(JavaAppPackaging).

‎src/main/resources/application.conf

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1+
akka.http.client.idle-timeout = infinite
2+
akka.http.host-connection-pool.idle-timeout = infinite
13
akka.http.server.websocket.periodic-keep-alive-max-idle = 10 seconds

‎src/main/scala/de/upb/cs/swt/delphi/instanceregistry/Configuration.scala

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,18 @@ package de.upb.cs.swt.delphi.instanceregistry
22

33
import akka.util.Timeout
44

5-
import scala.concurrent.duration.DurationInt
5+
import scala.concurrent.duration.{DurationInt, FiniteDuration}
66

77
class Configuration( ) {
88
//Where to host the http server
99
val bindHost: String = "0.0.0.0"
1010
val bindPort: Int = 8087
1111

12+
//Traefik data
13+
val traefikBaseHost: String = "delphi.de"
14+
val traefikDockerNetwork: String = "delphi"
15+
val traefikUri: String = "http://172.17.0.1:80"
16+
1217

1318
val recoveryFileName : String = "dump.temp"
1419

@@ -32,16 +37,34 @@ class Configuration( ) {
3237
val maxLabelLength: Int = 50
3338

3439
val dockerOperationTimeout: Timeout = Timeout(20 seconds)
40+
val dockerUri: String = sys.env.getOrElse("DELPHI_DOCKER_HOST", "http://localhost:9095")
3541

36-
//Database configurations
37-
val useInMemoryDB = false
38-
val databaseHost = "jdbc:mysql://localhost/"
39-
val databaseName = ""
40-
val databaseDriver = "com.mysql.jdbc.Driver"
41-
val databaseUsername = ""
42-
val databasePassword = ""
43-
42+
val jwtSecretKey: String = sys.env.getOrElse("JWT_SECRET", "changeme")
4443

45-
}
44+
//Database configurations
45+
val useInMemoryInstanceDB = true
46+
val instanceDatabaseHost = "jdbc:mysql://localhost/"
47+
val instanceDatabaseName = ""
48+
val instanceDatabaseDriver = "com.mysql.jdbc.Driver"
49+
val instanceDatabaseUsername = ""
50+
val instanceDatabasePassword = ""
51+
52+
//Auth database configuration
53+
val useInMemoryAuthDB = true
54+
val authDatabaseHost = "jdbc:mysql://localhost/"
55+
val authDatabaseName = ""
56+
val authDatabaseDriver = "com.mysql.jdbc.Driver"
57+
val authDatabaseUsername = ""
58+
val authDatabasePassword = ""
59+
60+
//Authentication valid for the time
61+
val authenticationValidFor = 30 //minutes
62+
63+
//Request Limiter
64+
val maxTotalNoRequest: Int = 2000
65+
val maxIndividualIpReq: Int = 200
66+
val ipLogRefreshRate: FiniteDuration = 2.minutes
67+
68+
}
4669

4770

‎src/main/scala/de/upb/cs/swt/delphi/instanceregistry/Docker/Container.scala

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,14 @@ case class ContainerConfig(
2525
Image: String,
2626
Entrypoint: Option[Seq[String]] = None,
2727
Cmd: Seq[String] = Seq.empty,
28-
Env: Seq[String] = Seq.empty)
28+
Env: Seq[String] = Seq.empty,
29+
Labels: Map[String, String] = Map.empty[String,String],
30+
ExposedPorts: Map[String, EmptyExposedPortConfig] = Map.empty,
31+
NetworkingConfig: NetworkConfig = NetworkConfig(Map.empty))
32+
33+
case class NetworkConfig(EndpointsConfig: Map[String, EmptyEndpointConfig])
34+
case class EmptyEndpointConfig()
35+
case class EmptyExposedPortConfig()
2936

3037
case class Networks(
3138
IPAddress: String

‎src/main/scala/de/upb/cs/swt/delphi/instanceregistry/Docker/ContainerCommands.scala

Lines changed: 84 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,28 @@
11
package de.upb.cs.swt.delphi.instanceregistry.Docker
22

33

4-
import akka.NotUsed
5-
import akka.actor.ActorSystem
4+
import java.nio.ByteOrder
5+
6+
import akka.{Done, NotUsed}
7+
import akka.actor.{ActorSystem, PoisonPill}
68
import akka.http.scaladsl.client.RequestBuilding._
79
import akka.http.scaladsl.model.MediaTypes.`application/json`
810
import akka.http.scaladsl.model.Uri.{Path, Query}
911
import akka.http.scaladsl.model.{HttpEntity, HttpResponse, StatusCodes}
1012
import akka.http.scaladsl.unmarshalling.Unmarshal
11-
import akka.stream.scaladsl.{Flow, Source}
13+
import akka.stream.scaladsl.{Flow, Framing, Keep, Sink, Source}
1214
import de.upb.cs.swt.delphi.instanceregistry.{AppLogging, Registry}
1315
import spray.json._
1416
import PostDataFormatting.commandJsonRequest
17+
import akka.http.scaladsl.Http
18+
import akka.http.scaladsl.model.ws.{Message, TextMessage, WebSocketRequest}
19+
import akka.stream.OverflowStrategy
20+
import akka.util.ByteString
21+
import org.reactivestreams.Publisher
1522

16-
import scala.concurrent.{ExecutionContext, Future}
23+
import scala.collection.mutable
24+
import scala.concurrent.{Await, ExecutionContext, Future}
25+
import scala.util.{Failure, Success, Try}
1726

1827

1928
class ContainerCommands(connection: DockerConnection) extends JsonSupport with Commands with AppLogging {
@@ -169,10 +178,17 @@ class ContainerCommands(connection: DockerConnection) extends JsonSupport with C
169178
response.status match {
170179
case StatusCodes.OK =>
171180
Unmarshal(response.entity).to[String].map { json =>
172-
val out = json.parseJson.asJsObject.getFields("NetworkSettings")
173-
out match {
174-
case Seq(network) => Networks(network.asJsObject.fields("IPAddress").toString())
175-
case _ => throw DeserializationException("Cannot find required field NetworkSettings/IPAddress")
181+
182+
Try[Networks]{
183+
val ip = json.parseJson.asJsObject.fields("NetworkSettings")
184+
.asJsObject.fields("Networks")
185+
.asJsObject.fields(Registry.configuration.traefikDockerNetwork)
186+
.asJsObject.getFields("IPAddress").head.toString.replace("\"", "")
187+
Networks(ip)
188+
} match {
189+
case Success(network) => network
190+
case Failure(ex) =>
191+
throw DeserializationException(s"Failed to extract IPAddress from docker with message ${ex.getMessage}")
176192
}
177193

178194
}
@@ -184,25 +200,70 @@ class ContainerCommands(connection: DockerConnection) extends JsonSupport with C
184200
}
185201
}
186202

187-
def logs(
188-
containerId: String
189-
)(implicit ec: ExecutionContext): Source[String, NotUsed] = {
190-
val query = Query("stdout" -> "true" )
203+
def retrieveLogs(
204+
containerId: String,
205+
stdErrSelected: Boolean
206+
)(implicit ec: ExecutionContext): Future[String] = {
207+
208+
val query = Query("stdout" -> (!stdErrSelected).toString, "stderr" -> stdErrSelected.toString, "follow" -> "false", "tail" -> "all", "timestamps" -> "true")
191209
val request = Get(buildUri(containersPath / containerId.substring(0,11) / "logs", query))
192210

193-
val flow =
194-
Flow[HttpResponse].map {
195-
case HttpResponse(StatusCodes.OK, _, HttpEntity.Chunked(_, chunks), _) =>
196-
chunks.map(_.data().utf8String)
197-
case HttpResponse(StatusCodes.NotFound, _, HttpEntity.Strict(_, data), _) =>
198-
log.warning(s"DOCKER LOGS FAILED: ${data.utf8String}")
211+
connection.sendRequest(request).flatMap {response =>
212+
response.status match {
213+
case StatusCodes.OK =>
214+
Unmarshal(response.entity).to[String]
215+
case StatusCodes.UpgradeRequired =>
216+
log.warning(s"Unexpected upgrade response while reading logs for container $containerId")
217+
log.warning(s"Got $response")
218+
unknownResponseFuture(response)
219+
case StatusCodes.NotFound =>
199220
throw ContainerNotFoundException(containerId)
200-
case response =>
201-
unknownResponse(response)
202-
}.flatMapConcat(identity)
221+
case _ =>
222+
unknownResponseFuture(response)
223+
}
224+
}
225+
}
226+
227+
def streamLogs(containerId: String, stdErrSelected: Boolean) (implicit ec: ExecutionContext) : Try[Publisher[Message]] = {
228+
229+
// Select stdout / stderr in query params
230+
val queryParams = Query("stdout" -> (!stdErrSelected).toString, "stderr" -> stdErrSelected.toString, "follow" -> "true", "tail" -> "all", "timestamps" -> "false")
231+
232+
// Create actor-publisher pair, publisher will be returned
233+
val (streamActor, streamPublisher) = Source.actorRef[Message](bufferSize = 10, OverflowStrategy.dropNew)
234+
.toMat(Sink.asPublisher(fanout = true))(Keep.both)
235+
.run()
236+
237+
// Delimiter flow splits incoming traffic into lines based on dockers multiplex-protocol
238+
// Docker prepends an 8-byte header, where the last 4 byte encode line length in big endian
239+
// See https://docs.docker.com/engine/api/v1.30/#operation/ContainerAttach
240+
val delimiter: Flow[ByteString, ByteString, NotUsed] = Framing.lengthField(4, 4, 100000, ByteOrder.BIG_ENDIAN)
241+
242+
// Flow that removes header bytes from payload
243+
val removeHeaderFlow: Flow[ByteString, ByteString, NotUsed] = Flow.fromFunction(in => in.slice(8, in.size))
244+
245+
// Build request
246+
val request = Get(buildUri(containersPath / containerId.substring(0,11) / "logs", queryParams))
247+
248+
// Execute request
249+
val res = connection.sendRequest(request).flatMap { res =>
250+
// Extract payload ByteString from data stream using above flows. Map to string.
251+
val logLines = res.entity.dataBytes.via(delimiter).via(removeHeaderFlow).map(_.utf8String)
252+
logLines.runForeach { line =>
253+
// Send each log line to the stream actor, which will publish them
254+
log.debug(s"Streaming log message $line")
255+
streamActor ! TextMessage(line)
256+
}
257+
}
258+
259+
// Kill actor on completion
260+
res.onComplete{ _ =>
261+
log.info("Log stream finished successfully.")
262+
streamActor ! PoisonPill
263+
}
203264

204-
Source.fromFuture(connection.sendRequest(request))
205-
.via(flow)
265+
// Return publish so server can subscribe to it
266+
Success(streamPublisher)
206267
}
207268

208269
def commandCreate(

0 commit comments

Comments
 (0)