Merge branch 'dev'

Author: penguin-boxx

.github/workflows/ci.yaml

@@ -0,0 +1,25 @@
+name: LambdaRPC CI
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+      - name: Build lambdarpc-core
+        uses: gradle/gradle-build-action@937999e9cc2425eddc7fd62d1053baf041147db7
+        with:
+          arguments: |
+            lambdarpc-core:build
+      - name: Build examples
+        uses: gradle/gradle-build-action@937999e9cc2425eddc7fd62d1053baf041147db7
+        with:
+          arguments: |
+            examples:build

README.adoc

@@ -7,12 +7,10 @@ Simple native RPC with high order functions support.
 == Service as a library
 
 .λRPC makes multi-process communication smooth enough to recognize remote service like a common library. It is powered by two main ideas:
-* λRPC does not use standalone declarations (https://en.wikipedia.org/wiki/Interface_description_language[IDL]) 
-and code generation (so λRPC declarations are native for the user's code). 
-Instead it allows working with library-specific data structures and default or custom serializers.
+* λRPC does not use standalone declarations (https://en.wikipedia.org/wiki/Interface_description_language[IDL]) and code generation (so λRPC declarations are native for the user's code).
+Instead, it allows working with library-specific data structures and default or custom serializers.
 * λRPC functions can receive and return other functions as first-class objects.
-Passed lambda will be executed on the client-side, so it can easily capture the state (even mutable) 
-and be "sent" to the other language's process.
+Passed lambda will be executed on the client-side, so it can easily capture the state (even mutable) and be "sent" to the other language's process.
 
 == Example
 
@@ -74,7 +72,7 @@ fun main() = runBlocking(serviceDispatcher) {
     val rawModel = Model(...)
     // Bind dataloader with dataEndpoint, so mlservice will communicate directly
     // with the dataservice on the dataEndpoint without client in the middle
-    val boundLoader = bf(dataloader)
+    val boundLoader = dataloader.toBound()
     val model = fit(rawModel, boundLoader) { epoch, metric ->
         // Lambda will be executed on the client site -- the λRPC magic
         println("Epoch = $epoch, metric = $metric")
@@ -124,10 +122,8 @@ image::https://user-images.githubusercontent.com/25281147/153264790-74784fb7-3be
 * Interactive computations: computing function receives closure as a parameter and calls it periodically, providing computation status information and receiving further directives.
 * Security:
 ** Send closures operating on the sensitive data instead of the data itself.
-** Provide computational resources as a library of functions that are parametrized by client lambdas instead of
-   receiving the client's code and executing it.
-* Choose dynamically computation location: compute something that uses a large amount of data on a client or send data to the server and
-  compute there.
+** Provide computational resources as a library of functions that are parametrized by client lambdas instead of receiving the client's code and executing it.
+* Choose dynamically computation location: compute something that uses a large amount of data on a client or send data to the server and compute there.
 * Load balancing: once a task is finished, request new via client's lambda.
 * Stateful streaming computations: nodes provide their lambdas for a mapper.
 
@@ -143,53 +139,54 @@ Then λRPC provides efficient communication with the corresponding backend part.
 
 == Getting started
 
-.Run both `:detekt` and `:test` tasks
+.Build and run fast tests and checks
 [source,bash]
 ----
-$ ./gradlew :lambdarpc:check
+$ ./gradlew build
 ----
 
 .Run stress tests
 [source,bash]
 ----
-$ ./gradlew :lambdarpc:slow
+$ ./gradlew :lambdarpc-core:slow
 ----
 
 .Generate documentation
 [source,bash]
 ----
-$ ./gradlew :lambdarpc:dokkaHtml
-$ cd ./lambdarpc/build/dokka/html
+$ ./gradlew :lambdarpc-core:dokkaHtml
+$ cd ./lambdarpc-core/build/dokka/html
 ----
 
 === Repository organization
 
 .examples
-* `ml` -- readme example.
+* `interactive_ml` -- readme example.
+
 [sources,bash]
 ----
 $ cd LambdaRPC.kt
 $ ./gradlew :examples:ml.dataservice
 $ ./gradlew :examples:ml.mlservice
 $ ./gradlew :examples:ml.client
 ----
-* `lazy` -- an interesting example that shows the possibility to build lazy
-data processing pipelines using common λRPC functionality.
+
+* `promise_pipeline` -- an interesting example that shows the possibility to build lazy data processing pipelines using common λRPC functionality.
+
 [sources,bash]
 ----
 $ cd LambdaRPC.kt
-$ ./gradlew :examples:lazy.service --args=8090
-$ ./gradlew :examples:lazy.service --args=8091
+$ ./gradlew :examples:promise.service --args=8090
+$ ./gradlew :examples:promise.service --args=8091
 # Any number of services on different ports
-$ ./gradlew :examples:lazy.client --args='8090 8091' # Ports of all services
+$ ./gradlew :examples:promise.client --args='8090 8091' # Ports of all services
 ----
 
-.lambdarpc
+.lambdarpc-core
 * `dsl` -- domain-specific language for λRPC library users.
-* `exceptions` -- base λRPC exception classes.
 * `functions` -- λRPC functions: backend and frontend parts.
-* `coders` -- data coder (serializer) and function coder.
-** λRPC provides some default data coders based on `kotlinx.serialization`, but users can also implement thier own.
+* `coding` -- contains `Coder` definition, it is a thing that can serialize data and work with functions.
+** λRPC provides some default data coders based on `kotlinx.serialization`, but users can also implement their own.
 ** Function encoding saves language closure as backend function to the registry with some `access name`.
 Function decoding creates a frontend function that communicates with the corresponding backend function.
 * `service` -- libservice implementation.

examples/build.gradle.kts

@@ -15,28 +15,28 @@ tasks.withType<KotlinCompile>().all {
 
 dependencies {
     implementation(kotlin("stdlib"))
-    implementation(project(":lambdarpc"))
+    implementation(project(":lambdarpc-core"))
     api("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.0")
     implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.2")
     implementation("org.slf4j:slf4j-simple:1.7.36")
     implementation("io.github.microutils:kotlin-logging-jvm:2.1.21")
 }
 
-val lazy = "io.lambdarpc.examples.lazy"
+val lazy = "io.lambdarpc.examples.promise_pipeline"
 
-tasks.register<JavaExec>("lazy.service") {
+tasks.register<JavaExec>("promise.service") {
     dependsOn("classes")
     classpath = sourceSets["main"].runtimeClasspath
     mainClass.set("$lazy.service.ServiceKt")
 }
 
-tasks.register<JavaExec>("lazy.client") {
+tasks.register<JavaExec>("promise.client") {
     dependsOn("classes")
     classpath = sourceSets["main"].runtimeClasspath
     mainClass.set("$lazy.client.ClientKt")
 }
 
-val ml = "io.lambdarpc.examples.ml"
+val ml = "io.lambdarpc.examples.interactive_ml"
 
 tasks.register<JavaExec>("ml.dataservice") {
     dependsOn("classes")

examples/src/main/kotlin/io/lambdarpc/examples/ml/Defs.kt examples/src/main/kotlin/io/lambdarpc/examples/interactive_ml/Defs.kt

@@ -1,4 +1,4 @@
-package io.lambdarpc.examples.ml
+package io.lambdarpc.examples.interactive_ml
 
 import io.lambdarpc.utils.Endpoint
 

examples/src/main/kotlin/io/lambdarpc/examples/ml/client/Client.kt examples/src/main/kotlin/io/lambdarpc/examples/interactive_ml/client/Client.kt

@@ -1,17 +1,17 @@
-package io.lambdarpc.examples.ml.client
+package io.lambdarpc.examples.interactive_ml.client
 
 import io.lambdarpc.dsl.ServiceDispatcher
-import io.lambdarpc.dsl.bf
-import io.lambdarpc.examples.ml.dataEndpoint
-import io.lambdarpc.examples.ml.dataservice.facade.dataLoader
-import io.lambdarpc.examples.ml.dataservice.facade.dataServiceId
-import io.lambdarpc.examples.ml.mlEndpoint
-import io.lambdarpc.examples.ml.mlservice.Metric
-import io.lambdarpc.examples.ml.mlservice.Model
-import io.lambdarpc.examples.ml.mlservice.facade.fit
-import io.lambdarpc.examples.ml.mlservice.facade.mlServiceId
+import io.lambdarpc.dsl.blockingConnectionPool
+import io.lambdarpc.dsl.toBound
+import io.lambdarpc.examples.interactive_ml.dataEndpoint
+import io.lambdarpc.examples.interactive_ml.dataservice.facade.dataLoader
+import io.lambdarpc.examples.interactive_ml.dataservice.facade.dataServiceId
+import io.lambdarpc.examples.interactive_ml.mlEndpoint
+import io.lambdarpc.examples.interactive_ml.mlservice.Metric
+import io.lambdarpc.examples.interactive_ml.mlservice.Model
+import io.lambdarpc.examples.interactive_ml.mlservice.facade.fit
+import io.lambdarpc.examples.interactive_ml.mlservice.facade.mlServiceId
 import io.lambdarpc.transport.MapServiceRegistry
-import kotlinx.coroutines.runBlocking
 
 /**
  * Can be a simple map that service or client receives with command line options,
@@ -24,14 +24,14 @@ val serviceRegistry = MapServiceRegistry(
 
 val serviceDispatcher = ServiceDispatcher(serviceRegistry)
 
-fun main() = runBlocking(serviceDispatcher) {
+fun main() = blockingConnectionPool(serviceDispatcher) {
     // Keep track of the loss function values
     val history = mutableListOf<Metric>()
     var lastEpoch = 0
     val rawModel = Model()
     // Bind dataloader with dataEndpoint, so mlservice will communicate directly
     // with the dataservice on the dataEndpoint without client in the middle
-    val boundLoader = bf(dataLoader)
+    val boundLoader = dataLoader.toBound()
     val model = fit(rawModel, boundLoader) { epoch, metric ->
         // Lambda will be executed on the client site -- the λRPC magic
         println("Epoch = $epoch, metric = $metric")

examples/src/main/kotlin/io/lambdarpc/examples/ml/dataservice/Facade.kt examples/src/main/kotlin/io/lambdarpc/examples/interactive_ml/dataservice/Facade.kt

@@ -1,10 +1,10 @@
 @file:Suppress("PackageDirectoryMismatch")
 
-package io.lambdarpc.examples.ml.dataservice.facade
+package io.lambdarpc.examples.interactive_ml.dataservice.facade
 
 import io.lambdarpc.dsl.def
 import io.lambdarpc.dsl.j
-import io.lambdarpc.examples.ml.mlservice.Data
+import io.lambdarpc.examples.interactive_ml.mlservice.Data
 import io.lambdarpc.utils.toSid
 
 val dataServiceId = "45b33c9c-ae42-4835-b27c-ad6f57b6a82d".toSid()

examples/src/main/kotlin/io/lambdarpc/examples/interactive_ml/dataservice/Lib.kt

@@ -0,0 +1,5 @@
+package io.lambdarpc.examples.interactive_ml.dataservice
+
+import io.lambdarpc.examples.interactive_ml.mlservice.Data
+
+fun dataloader(): Data = Data(1)

examples/src/main/kotlin/io/lambdarpc/examples/interactive_ml/dataservice/Service.kt

@@ -0,0 +1,14 @@
+package io.lambdarpc.examples.interactive_ml.dataservice
+
+import io.lambdarpc.dsl.LibService
+import io.lambdarpc.examples.interactive_ml.dataEndpoint
+import io.lambdarpc.examples.interactive_ml.dataservice.facade.dataLoader
+import io.lambdarpc.examples.interactive_ml.dataservice.facade.dataServiceId
+
+fun main() {
+    val service = LibService(dataServiceId, dataEndpoint) {
+        dataLoader of ::dataloader
+    }
+    service.start()
+    service.awaitTermination()
+}

examples/src/main/kotlin/io/lambdarpc/examples/interactive_ml/mlservice/Facade.kt

@@ -0,0 +1,37 @@
+@file:Suppress("PackageDirectoryMismatch")
+
+package io.lambdarpc.examples.interactive_ml.mlservice.facade
+
+import io.lambdarpc.coding.Coder
+import io.lambdarpc.coding.CodingContext
+import io.lambdarpc.dsl.def
+import io.lambdarpc.dsl.f
+import io.lambdarpc.dsl.j
+import io.lambdarpc.examples.interactive_ml.mlservice.Data
+import io.lambdarpc.examples.interactive_ml.mlservice.Epoch
+import io.lambdarpc.examples.interactive_ml.mlservice.Metric
+import io.lambdarpc.examples.interactive_ml.mlservice.Model
+import io.lambdarpc.transport.grpc.Entity
+import io.lambdarpc.transport.serialization.Entity
+import io.lambdarpc.transport.serialization.RawData
+import io.lambdarpc.utils.toSid
+
+val mlServiceId = "1a897419-0fd2-4e84-976f-0a2211a48898".toSid()
+
+private val loader = f( // Coder for the function: suspend () -> Data
+    j<Data>() // kotlinx.serialization JSON coder for the @Serializable Data
+)
+
+val fit by mlServiceId.def( // Define declaration for suspend (Epoch, Metric) -> Boolean
+    ModelCoder, // Model may not be @Serializable, so λRPC allows writing custom data coders
+    loader, f(j<Epoch>(), j<Metric>(), j<Boolean>()),
+    ModelCoder
+)
+
+private object ModelCoder : Coder<Model> {
+    override fun encode(value: Model, context: CodingContext): Entity =
+        Entity(RawData.copyFrom(byteArrayOf(value.weight.toByte())))
+
+    override fun decode(entity: Entity, context: CodingContext): Model =
+        Model(entity.data.toByteArray().first().toInt())
+}

examples/src/main/kotlin/io/lambdarpc/examples/ml/mlservice/Lib.kt examples/src/main/kotlin/io/lambdarpc/examples/interactive_ml/mlservice/Lib.kt

@@ -1,4 +1,4 @@
-package io.lambdarpc.examples.ml.mlservice
+package io.lambdarpc.examples.interactive_ml.mlservice
 
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking

examples/src/main/kotlin/io/lambdarpc/examples/ml/mlservice/Service.kt examples/src/main/kotlin/io/lambdarpc/examples/interactive_ml/mlservice/Service.kt

@@ -1,9 +1,9 @@
-package io.lambdarpc.examples.ml.mlservice
+package io.lambdarpc.examples.interactive_ml.mlservice
 
 import io.lambdarpc.dsl.LibService
-import io.lambdarpc.examples.ml.mlEndpoint
-import io.lambdarpc.examples.ml.mlservice.facade.fit
-import io.lambdarpc.examples.ml.mlservice.facade.mlServiceId
+import io.lambdarpc.examples.interactive_ml.mlEndpoint
+import io.lambdarpc.examples.interactive_ml.mlservice.facade.fit
+import io.lambdarpc.examples.interactive_ml.mlservice.facade.mlServiceId
 
 fun main() {
     // Such service that looks like a library is called libservice

examples/src/main/kotlin/io/lambdarpc/examples/ml/dataservice/Lib.kt

@@ -1,11 +1,10 @@
 @file:Suppress("SpellCheckingInspection")
 
-package io.lambdarpc.examples.lazy
+package io.lambdarpc.examples.promise_pipeline
 
-import io.lambdarpc.coders.Coder
+import io.lambdarpc.coding.Coder
 import io.lambdarpc.dsl.f
 import io.lambdarpc.dsl.j
-import io.lambdarpc.functions.frontend.ConnectedFunction
 import kotlinx.coroutines.async
 import kotlinx.coroutines.coroutineScope
 
@@ -20,15 +19,12 @@ fun <R> lazify(f: suspend () -> R): suspend () -> Promise<R> = {
 fun <A, R> lazify(
     f: suspend (A) -> R
 ): suspend (Promise<A>) -> Promise<R> = { a ->
-    require(a is ConnectedFunction);
     { f(a()) }
 }
 
 fun <A, B, R> lazify(
     f: suspend (A, B) -> R
 ): suspend (Promise<A>, Promise<B>) -> Promise<R> = { a, b ->
-    require(a is ConnectedFunction)
-    require(b is ConnectedFunction);
     {
         coroutineScope {
             val aa = async { a() }