diff --git a/basicdataset/build.gradle.kts b/basicdataset/build.gradle.kts index 759e1c33923..a5f91c7d374 100644 --- a/basicdataset/build.gradle.kts +++ b/basicdataset/build.gradle.kts @@ -18,6 +18,7 @@ dependencies { } tasks.register("syncS3") { + workingDir = project.projectDir commandLine( "sh", "-c", diff --git a/buildSrc/src/main/kotlin/ai/djl/cppFormatter.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/cppFormatter.gradle.kts index a96e90cd99f..48276f076a6 100644 --- a/buildSrc/src/main/kotlin/ai/djl/cppFormatter.gradle.kts +++ b/buildSrc/src/main/kotlin/ai/djl/cppFormatter.gradle.kts @@ -33,15 +33,17 @@ project.extensions.create("formatCpp") project.tasks { register("formatCpp") { + val rootProject = project.rootProject + val clang = rootProject.projectDir / ".clang/clang-format" + val formatCpp = project.extensions.getByName("formatCpp") + val files = project.fileTree("src") + val logger = project.logger + doLast { if ("mac" !in os) return@doLast - val rootProject = project.rootProject - val clang = rootProject.projectDir / ".clang/clang-format" checkClang(clang) - val files = project.fileTree("src") files.include("**/*.cc", "**/*.cpp", "**/*.h") - val formatCpp = project.extensions.getByName("formatCpp") if (formatCpp.exclusions.isPresent) for (exclusion in formatCpp.exclusions.get()) files.exclude(exclusion) @@ -49,28 +51,30 @@ project.tasks { for (f in files) { if (!f.isFile()) continue - project.logger.info("formatting cpp file: $f") + logger.info("formatting cpp file: $f") f.text = formatCpp(f, clang) } } } register("verifyCpp") { + val rootProject = project.rootProject + val clang = rootProject.projectDir / ".clang/clang-format" + val files = project.fileTree("src") + val logger = project.logger + doLast { if ("mac" !in os) return@doLast - val rootProject = project.rootProject - val clang = rootProject.projectDir / ".clang/clang-format" checkClang(clang) - val files = project.fileTree("src") files.include("**/*.cc", "**/*.cpp", "**/*.h") for (f in files) { if (!f.isFile()) continue - project.logger.info("checking cpp file: $f") + logger.info("checking cpp file: $f") if (f.text != formatCpp(f, clang)) throw GradleException("File not formatted: " + f.absolutePath) } } } -} \ No newline at end of file +} diff --git a/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts index 50eec3ad4c4..17584a9f89c 100644 --- a/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts +++ b/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts @@ -7,9 +7,10 @@ import java.io.PrintWriter tasks { register("formatJava") { + val sourceSets = project.sourceSets doLast { val formatter = Main(PrintWriter(System.out, true), PrintWriter(System.err, true), System.`in`) - for (item in project.sourceSets) + for (item in sourceSets) for (file in item.allSource) { if (!file.name.endsWith(".java") || "generated-src" in file.absolutePath) continue @@ -24,9 +25,11 @@ tasks { inputs.files(project.sourceSets.flatMap { it.allSource }) inputs.files(project.fileTree("generated-src")) outputs.file(project.file(resultFilePath)) + + val proj = project doLast { val formatter = Main(PrintWriter(System.out, true), PrintWriter(System.err, true), System.`in`) - for (item in project.sourceSets) + for (item in proj.sourceSets) for (file in item.allSource) { if (!file.name.endsWith(".java") || "generated-src" in file.absolutePath) continue @@ -39,7 +42,7 @@ tasks { + "See https://github.com/deepjavalibrary/djl/blob/master/docs/development/development_guideline.md#coding-conventions for more details" ) } - project.file(resultFilePath).writeText("Success") + proj.file(resultFilePath).writeText("Success") } } @@ -47,4 +50,4 @@ tasks { } val Project.sourceSets: SourceSetContainer - get() = extensions.getByName("sourceSets") \ No newline at end of file + get() = extensions.getByName("sourceSets") diff --git a/engines/ml/xgboost/build.gradle.kts b/engines/ml/xgboost/build.gradle.kts index 5dfa5cb0de5..33bec12a3a6 100644 --- a/engines/ml/xgboost/build.gradle.kts +++ b/engines/ml/xgboost/build.gradle.kts @@ -46,12 +46,14 @@ tasks { val jnilibDir = buildDirectory / "resources/main/lib/linux/aarch64/" inputs.properties(mapOf("xgboostVersion" to libs.versions.xgboost.get())) outputs.dir(jnilibDir) + + val logger = project.logger doLast { val url = "https://publish.djl.ai/xgboost/${libs.versions.xgboost.get()}/jnilib/linux/aarch64/libxgboost4j.so" val file = jnilibDir / "libxgboost4j.so" if (!file.exists()) { - project.logger.lifecycle("Downloading $url") + logger.lifecycle("Downloading $url") url.url into file } } @@ -116,4 +118,4 @@ tasks { } } } -} \ No newline at end of file +} diff --git a/engines/mxnet/mxnet-engine/build.gradle.kts b/engines/mxnet/mxnet-engine/build.gradle.kts index 0b825755262..257d7b6e54c 100644 --- a/engines/mxnet/mxnet-engine/build.gradle.kts +++ b/engines/mxnet/mxnet-engine/build.gradle.kts @@ -42,9 +42,12 @@ tasks { .withPropertyName("jna/mapping.properties file") outputs.dir(buildDirectory / "generated-src") outputs.cacheIf { true } + + val dir = project.projectDir doLast { val jnaGenerator = jnaratorJar.get().outputs.files.singleFile - javaexec { + providers.javaexec { + workingDir = dir mainClass = "-jar" args( jnaGenerator.absolutePath, @@ -55,12 +58,12 @@ tasks { "-o", "$buildDirectory/generated-src", "-m", - "${project.projectDir}/src/main/jna/mapping.properties", + "${dir}/src/main/jna/mapping.properties", "-f", "src/main/include/mxnet/c_api.h", "src/main/include/nnvm/c_api.h" ) - } + }.result.get() } } @@ -121,4 +124,4 @@ tasks { } } } -} \ No newline at end of file +} diff --git a/engines/mxnet/mxnet-model-zoo/build.gradle.kts b/engines/mxnet/mxnet-model-zoo/build.gradle.kts index 475c3650ff8..c7f4bb1dffd 100644 --- a/engines/mxnet/mxnet-model-zoo/build.gradle.kts +++ b/engines/mxnet/mxnet-model-zoo/build.gradle.kts @@ -19,6 +19,7 @@ dependencies { } tasks.register("syncS3") { + workingDir = project.projectDir commandLine( "sh", "-c", diff --git a/engines/mxnet/native/build.gradle.kts b/engines/mxnet/native/build.gradle.kts index c168b396f56..1cb34bb3515 100644 --- a/engines/mxnet/native/build.gradle.kts +++ b/engines/mxnet/native/build.gradle.kts @@ -17,8 +17,9 @@ tasks { // this line is to enforce gradle to build the jar otherwise it doesn't generate the placeholder jar at times // when there is no java code inside src/main outputs.dir("build/libs") + + var versionName = project.version.toString() doFirst { - var versionName = project.version.toString() if (!isRelease) versionName += "-$nowFormatted" val dir = placeholder / "native/lib" diff --git a/engines/onnxruntime/onnxruntime-engine/build.gradle.kts b/engines/onnxruntime/onnxruntime-engine/build.gradle.kts index 6346080d179..a5857de78f0 100644 --- a/engines/onnxruntime/onnxruntime-engine/build.gradle.kts +++ b/engines/onnxruntime/onnxruntime-engine/build.gradle.kts @@ -18,8 +18,9 @@ dependencies { } tasks { + val basePath = "${project.projectDir}/build/resources/main/nlp" + val logger = project.logger processResources { - val basePath = "${project.projectDir}/build/resources/main/nlp" outputs.dir(basePath) doLast { val url = "https://mlrepo.djl.ai/model/nlp" @@ -33,9 +34,9 @@ tasks { for (task in tasks) { val file = File("$basePath/$task/ai.djl.huggingface.onnxruntime.json") if (file.exists()) - project.logger.lifecycle("model zoo metadata already exists: $task") + logger.lifecycle("model zoo metadata already exists: $task") else { - project.logger.lifecycle("Downloading model zoo metadata: $task") + logger.lifecycle("Downloading model zoo metadata: $task") file.parentFile.mkdirs() "$url/$task/ai/djl/huggingface/onnxruntime/models.json.gz".url gzipInto file } @@ -55,4 +56,4 @@ publishing { } } } -} \ No newline at end of file +} diff --git a/engines/pytorch/pytorch-jni/build.gradle.kts b/engines/pytorch/pytorch-jni/build.gradle.kts index b6b6342885d..c2c0c0c8472 100644 --- a/engines/pytorch/pytorch-jni/build.gradle.kts +++ b/engines/pytorch/pytorch-jni/build.gradle.kts @@ -21,6 +21,13 @@ tasks { processResources { outputs.dir(buildDirectory / "classes/java/main/jnilib") + + val logger = project.logger + val dir = project.projectDir + val nativeDir = project.parent!!.projectDir / "pytorch-native/jnilib/${libs.versions.djl.get()}/" + val version = project.version + val hasJni = project.hasProperty("jni") + doFirst { val url = "https://publish.djl.ai/pytorch/$ptVersion/jnilib/${libs.versions.djl.get()}" val files = listOf( @@ -49,22 +56,21 @@ tasks { else -> throw GradleException("Unsupported version: $ptVersion.") } - val jnilibDir = project.projectDir / "jnilib" / libs.versions.djl.get() + val jnilibDir = dir / "jnilib" / libs.versions.djl.get() for (entry in files) { val file = jnilibDir / entry if (file.exists()) - project.logger.lifecycle("prebuilt or cached file found for $entry") + logger.lifecycle("prebuilt or cached file found for $entry") else { - val nativeDir = project.parent!!.projectDir / "pytorch-native/jnilib/${libs.versions.djl.get()}/" val jnilibFile = nativeDir / entry if (jnilibFile.exists()) { - project.logger.lifecycle("Copying $jnilibFile") + logger.lifecycle("Copying $jnilibFile") copy { from(jnilibFile) into(file.parent) } - } else if (!project.hasProperty("jni")) { - project.logger.lifecycle("Downloading $url/$entry") + } else if (!hasJni) { + logger.lifecycle("Downloading $url/$entry") file.parentFile.mkdirs() "$url/$entry".url into file } @@ -77,7 +83,7 @@ tasks { // write properties val propFile = buildDirectory / "classes/java/main/jnilib/pytorch.properties" - propFile.text = "jni_version=" + project.version + propFile.text = "jni_version=$version" } } diff --git a/engines/pytorch/pytorch-model-zoo/build.gradle.kts b/engines/pytorch/pytorch-model-zoo/build.gradle.kts index 3aa50c0cc1b..827eade3e89 100644 --- a/engines/pytorch/pytorch-model-zoo/build.gradle.kts +++ b/engines/pytorch/pytorch-model-zoo/build.gradle.kts @@ -18,6 +18,7 @@ dependencies { } tasks.register("syncS3") { + workingDir = project.projectDir commandLine( "sh", "-c", diff --git a/engines/pytorch/pytorch-native/build.gradle.kts b/engines/pytorch/pytorch-native/build.gradle.kts index 4adf96b2523..917feed71be 100644 --- a/engines/pytorch/pytorch-native/build.gradle.kts +++ b/engines/pytorch/pytorch-native/build.gradle.kts @@ -24,9 +24,11 @@ val isAarch64 = project.hasProperty("aarch64") || arch == "aarch64" version = ptVersion + if (isRelease) "" else "-SNAPSHOT" + fun downloadBuild(ver: String, os: String, flavor: String, isPrecxx11: Boolean = false, isAarch64: Boolean = false) { val arch = if (isAarch64) "aarch64" else "x86_64" - exec { + providers.exec { + workingDir = project.projectDir if (os == "win") commandLine(project.projectDir / "build.cmd", ver, flavor) else @@ -50,7 +52,8 @@ fun downloadBuild(ver: String, os: String, flavor: String, isPrecxx11: Boolean = fun downloadBuildAndroid(ver: String) { for (abi in listOf("armeabi-v7a", "arm64-v8a", "x86", "x86_64")) { - exec { + providers.exec { + workingDir = project.projectDir commandLine("bash", "build_android.sh", ver, abi) } val ciDir = project.projectDir / "jnilib/${libs.versions.djl.get()}/android/$abi" @@ -93,7 +96,8 @@ fun prepareNativeLib(binaryRoot: String, ver: String, packageType: String?) { copyNativeLibToOutputDir(files, binaryRoot, officialPytorchUrl) copyNativeLibToOutputDir(aarch64Files, binaryRoot, aarch64PytorchUrl) - exec { + providers.exec { + workingDir = project.projectDir commandLine( "install_name_tool", "-add_rpath", @@ -101,7 +105,8 @@ fun prepareNativeLib(binaryRoot: String, ver: String, packageType: String?) { "$binaryRoot/cpu/osx-aarch64/native/lib/libtorch_cpu.dylib" ) } - exec { + providers.exec { + workingDir = project.projectDir commandLine( "install_name_tool", "-add_rpath", @@ -188,6 +193,8 @@ fun copyNativeLibToOutputDir(fileStoreMap: Map, binaryRoot: Stri } } +open class Cmd @Inject constructor(@Internal val execOperations: ExecOperations) : DefaultTask() + tasks { register("compileAndroidJNI") { doFirst { @@ -228,13 +235,15 @@ tasks { } } - register("uploadS3") { + register("uploadS3") { + val dir = project.projectDir doLast { delete("$binaryRoot") prepareNativeLib("$binaryRoot", ptVersion, "cpu") prepareNativeLib("$binaryRoot", ptVersion, "gpu") - exec { + execOperations.exec { + workingDir = dir commandLine("sh", "-c", "find $binaryRoot -type f | xargs gzip") } @@ -255,13 +264,15 @@ tasks { appendLine(out + "/" + URLEncoder.encode(it, "UTF-8")) } } - exec { + execOperations.exec { + workingDir = dir commandLine("aws", "s3", "sync", "$binaryRoot", "s3://djl-ai/publish/pytorch/$ptVersion/") } } } // Create a placeholder jar without classifier to pass sonatype tests but throws an Exception if loaded + val version = project.version jar { val placeholder = buildDirectory / "placeholder" // this line is to enforce gradle to build the jar @@ -272,7 +283,7 @@ tasks { val dir = placeholder / "native/lib" dir.mkdirs() val propFile = placeholder / "native/lib/pytorch.properties" - propFile.text = "placeholder=true\nversion=${project.version}\n" + propFile.text = "placeholder=true\nversion=${version}\n" } from(placeholder) @@ -442,9 +453,10 @@ tasks { } clean { + val dir = project.projectDir doFirst { - delete(project.projectDir / "jnilib") - delete(project.projectDir.parentFile / "pytorch-jni/jnilib") + delete(dir / "jnilib") + delete(dir.parentFile / "pytorch-jni/jnilib") } } } diff --git a/engines/tensorflow/tensorflow-model-zoo/build.gradle.kts b/engines/tensorflow/tensorflow-model-zoo/build.gradle.kts index eecebdbb39e..284ef20afa0 100644 --- a/engines/tensorflow/tensorflow-model-zoo/build.gradle.kts +++ b/engines/tensorflow/tensorflow-model-zoo/build.gradle.kts @@ -15,6 +15,7 @@ dependencies { } tasks.register("syncS3") { + workingDir = project.projectDir commandLine("sh", "-c", "find . -name .DS_Store | xargs rm && aws s3 sync src/test/resources/mlrepo s3://djl-ai/mlrepo --acl public-read") standardOutput = ByteArrayOutputStream() diff --git a/engines/tensorflow/tensorflow-native/build.gradle.kts b/engines/tensorflow/tensorflow-native/build.gradle.kts index ee2074e352b..3cc774fa207 100644 --- a/engines/tensorflow/tensorflow-native/build.gradle.kts +++ b/engines/tensorflow/tensorflow-native/build.gradle.kts @@ -31,8 +31,11 @@ infix fun URL.unzipInto(dir: File) { } } +open class Cmd @Inject constructor(@Internal val execOperations: ExecOperations) : DefaultTask() + tasks { - register("uploadTensorflowNativeLibs") { + register("uploadTensorflowNativeLibs") { + val baseDir = project.projectDir doLast { val dir = buildDirectory / "download" delete(dir) @@ -54,7 +57,8 @@ tasks { } } - exec { + execOperations.exec { + workingDir = baseDir commandLine( "aws", "s3", @@ -71,11 +75,11 @@ tasks { // otherwise it don't generate the placeholder jar at times // when there is no java code inside src/main outputs.dir("build/libs") + var versionName = project.version.toString() doFirst { val dir = buildDirectory / "classes/java/main/native/lib" dir.mkdirs() val propFile = dir / "tensorflow.properties" - var versionName = project.version.toString() if (!isRelease) versionName += "-$nowFormatted" propFile.text = "placeholder=true\nversion=${versionName}\n" diff --git a/engines/tensorrt/build.gradle.kts b/engines/tensorrt/build.gradle.kts index b10a246e957..99592583ae8 100644 --- a/engines/tensorrt/build.gradle.kts +++ b/engines/tensorrt/build.gradle.kts @@ -17,26 +17,34 @@ dependencies { testRuntimeOnly(project(":engines:pytorch:pytorch-engine")) } +open class Cmd @Inject constructor(@Internal val execOperations: ExecOperations) : DefaultTask() + tasks { compileJava { dependsOn(processResources) } processResources { - inputs.properties(mapOf("djlVersion" to libs.versions.djl, "trtVersion" to libs.versions.tensorrt.get(), - "version" to version)) + inputs.properties( + mapOf( + "djlVersion" to libs.versions.djl, "trtVersion" to libs.versions.tensorrt.get(), + "version" to version + ) + ) val baseResourcePath = "${project.projectDir}/build/resources/main" outputs.dir(file("${baseResourcePath}/native/lib")) + val trtVersion = libs.versions.tensorrt.get() + val djlVersion = libs.versions.djl.get() + val url = "https://publish.djl.ai/tensorrt/${trtVersion}/jnilib/${djlVersion}" + val files = listOf("linux-x86_64/libdjl_trt.so") + val jnilibDir = project.projectDir / "jnilib/${djlVersion}" + + val logger = project.logger doLast { - val trtVersion = libs.versions.tensorrt.get() - val djlVersion = libs.versions.djl.get() - val url = "https://publish.djl.ai/tensorrt/${trtVersion}/jnilib/${djlVersion}" - val files = listOf("linux-x86_64/libdjl_trt.so") - val jnilibDir = project.projectDir / "jnilib/${djlVersion}" for (entry in files) { val file = jnilibDir / entry if (file.exists()) { - project.logger.lifecycle("prebuilt or cached file found for $entry") + logger.lifecycle("prebuilt or cached file found for $entry") } else if (!project.hasProperty("jni")) { - project.logger.lifecycle("Downloading $url/$entry") + logger.lifecycle("Downloading $url/$entry") file.parentFile.mkdirs() "$url/$entry".url into file } @@ -53,10 +61,12 @@ tasks { } } - register("compileJNI") { + register("compileJNI") { + val dir = project.projectDir doFirst { if ("linux" in os) { - exec { + execOperations.exec { + workingDir = dir commandLine("bash", "build.sh") } } else { @@ -65,7 +75,7 @@ tasks { // for nightly ci val classifier = "${os}-x86_64" - val ciDir = project.projectDir / "jnilib/${libs.versions.djl.get()}/${classifier}" + val ciDir = dir / "jnilib/${libs.versions.djl.get()}/${classifier}" copy { from(fileTree(buildDirectory) { include("libdjl_trt.*") diff --git a/extensions/fasttext/build.gradle.kts b/extensions/fasttext/build.gradle.kts index 1c8f463e5bb..ae35e3db8e8 100644 --- a/extensions/fasttext/build.gradle.kts +++ b/extensions/fasttext/build.gradle.kts @@ -14,6 +14,8 @@ dependencies { testImplementation(libs.slf4j.simple) } +open class Cmd @Inject constructor(@Internal val execOperations: ExecOperations) : DefaultTask() + tasks { compileJava { dependsOn(processResources) } @@ -23,6 +25,9 @@ tasks { "version" to version)) val baseResourcePath = "${project.projectDir}/build/resources/main" outputs.dir("$baseResourcePath/native/lib") + val jnilibDir = project.projectDir / "jnilib/${libs.versions.djl.get()}" + val logger = project.logger + val hasJni = project.hasProperty("jni") doLast { val url = "https://publish.djl.ai/fasttext-${libs.versions.fasttext.get()}/jnilib/${libs.versions.djl.get()}" @@ -30,13 +35,12 @@ tasks { "linux-x86_64" to "libjni_fasttext.so", "osx-aarch64" to "libjni_fasttext.dylib" ) - val jnilibDir = project.projectDir / "jnilib/${libs.versions.djl.get()}" for ((key, value) in files) { val file = jnilibDir / key / value if (file.exists()) - project.logger.lifecycle("prebuilt or cached file found for $value") - else if (!project.hasProperty("jni")) { - project.logger.lifecycle("Downloading $url/$key") + logger.lifecycle("prebuilt or cached file found for $value") + else if (!hasJni) { + logger.lifecycle("Downloading $url/$key") file.parentFile.mkdirs() val downloadPath = "$url/$key/$value".url downloadPath into file @@ -53,15 +57,17 @@ tasks { } } - register("compileJNI") { + register("compileJNI") { + val dir = project.projectDir doFirst { check("mac" in os || "linux" in os) { "Unknown Architecture $osName" } - exec { + execOperations.exec { + workingDir = dir commandLine("bash", "build.sh") } // for ci to upload to S3 - val ciDir = project.projectDir / "jnilib/${libs.versions.djl.get()}" + val ciDir = dir / "jnilib/${libs.versions.djl.get()}" copy { from(buildDirectory / "jnilib") into(ciDir) @@ -88,4 +94,4 @@ publishing { } } } -} \ No newline at end of file +} diff --git a/extensions/sentencepiece/build.gradle.kts b/extensions/sentencepiece/build.gradle.kts index ca484d357ee..bceedffe2df 100644 --- a/extensions/sentencepiece/build.gradle.kts +++ b/extensions/sentencepiece/build.gradle.kts @@ -13,6 +13,8 @@ dependencies { testImplementation(libs.slf4j.simple) } +open class Cmd @Inject constructor(@Internal val execOperations: ExecOperations) : DefaultTask() + tasks { compileJava { dependsOn(processResources) } @@ -20,6 +22,8 @@ tasks { val baseResourcePath = "${project.projectDir}/build/resources/main" inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "sentencepieceVersion" to libs.versions.sentencepiece.get())) outputs.dir("$baseResourcePath/native/lib") + val jnilibDir = project.projectDir / "jnilib/${libs.versions.djl.get()}" + val logger = project.logger doLast { val url = "https://publish.djl.ai/sentencepiece-${libs.versions.sentencepiece.get()}/jnilib/${libs.versions.djl.get()}" @@ -29,13 +33,12 @@ tasks { "linux-aarch64" to "libsentencepiece_native.so", "osx-aarch64" to "libsentencepiece_native.dylib" ) - val jnilibDir = project.projectDir / "jnilib/${libs.versions.djl.get()}" for ((key, value) in files) { val file = jnilibDir / key / value if (file.exists()) - project.logger.lifecycle("prebuilt or cached file found for $value") + logger.lifecycle("prebuilt or cached file found for $value") else if (!project.hasProperty("jni")) { - project.logger.lifecycle("Downloading $url/$key") + logger.lifecycle("Downloading $url/$key") file.parentFile.mkdirs() val downloadPath = "$url/$key/$value".url downloadPath into file @@ -52,22 +55,25 @@ tasks { } } - register("compileJNI") { + register("compileJNI") { + val dir = project.projectDir doFirst { if ("win" in os) - exec { - commandLine("${project.projectDir}/build.cmd", "v${libs.versions.sentencepiece.get()}") + execOperations.exec { + workingDir = dir + commandLine("${dir}/build.cmd", "v${libs.versions.sentencepiece.get()}") } else if ("mac" in os || "linux" in os) { val arch = if (arch == "amd64") "x86_64" else arch - exec { + execOperations.exec { + workingDir = dir commandLine("bash", "build.sh", "v${libs.versions.sentencepiece.get()}", arch) } } else throw IllegalStateException("Unknown Architecture $osName") // for ci to upload to S3 - val ciDir = project.projectDir / "jnilib/${libs.versions.djl.get()}/" + val ciDir = dir / "jnilib/${libs.versions.djl.get()}/" copy { from(buildDirectory / "jnilib") into(ciDir) @@ -93,4 +99,4 @@ publishing { } } } -} \ No newline at end of file +} diff --git a/extensions/spark/build.gradle.kts b/extensions/spark/build.gradle.kts index 5f6f57a881d..80bc337012b 100644 --- a/extensions/spark/build.gradle.kts +++ b/extensions/spark/build.gradle.kts @@ -27,12 +27,9 @@ tasks { scalaCompileOptions.setAdditionalParameters(listOf("-target:jvm-1.8")) } - register("formatPython") { - doFirst { - exec { - commandLine("bash", "-c", "find . -name '*.py' -print0 | xargs -0 yapf --in-place") - } - } + register("formatPython") { + workingDir = project.projectDir + commandLine("bash", "-c", "find . -name '*.py' -print0 | xargs -0 yapf --in-place") } } @@ -49,4 +46,4 @@ extensions.getByName("publishing").apply { } } } -} \ No newline at end of file +} diff --git a/extensions/tokenizers/build.gradle.kts b/extensions/tokenizers/build.gradle.kts index dbf32273e71..155954fb863 100644 --- a/extensions/tokenizers/build.gradle.kts +++ b/extensions/tokenizers/build.gradle.kts @@ -21,6 +21,8 @@ dependencies { testImplementation(libs.slf4j.simple) } +open class Cmd @Inject constructor(@Internal val execOperations: ExecOperations) : DefaultTask() + tasks { compileJava { dependsOn(processResources) } @@ -35,6 +37,9 @@ tasks { val baseResourcePath = "${project.projectDir}/build/resources/main" outputs.dirs(File("${baseResourcePath}/native/lib"), File("${baseResourcePath}/nlp")) + val logger = project.logger + val dir = project.projectDir + val hasJni = project.hasProperty("jni") doLast { var url = "https://publish.djl.ai/tokenizers" val (tokenizers, djl) = libs.versions.tokenizers.get() to libs.versions.djl.get() @@ -47,18 +52,18 @@ tasks { "linux-aarch64/cpu/libtokenizers.so" to "$tokenizers/jnilib/$djl", "osx-aarch64/cpu/libtokenizers.dylib" to "$tokenizers/jnilib/$djl" ) - val jnilibDir = project.projectDir / "jnilib/$djl" + val jnilibDir = dir / "jnilib/$djl" for ((key, value) in files) { val file = jnilibDir / URLDecoder.decode(key, "UTF-8") if (file.exists()) - project.logger.lifecycle("prebuilt or cached file found for $key") + logger.lifecycle("prebuilt or cached file found for $key") else if (value.startsWith("extra")) { - project.logger.lifecycle("Downloading $url/$value") + logger.lifecycle("Downloading $url/$value") file.parentFile.mkdirs() val downloadPath = "$url/$value".url downloadPath into file - } else if (!project.hasProperty("jni")) { - project.logger.lifecycle("Downloading $url/$value/$key") + } else if (!hasJni) { + logger.lifecycle("Downloading $url/$value/$key") file.parentFile.mkdirs() val downloadPath = "$url/$value/$key".url downloadPath into file @@ -81,9 +86,9 @@ tasks { for (task in tasks) { var file = prefix / task / "ai.djl.huggingface.pytorch.json" if (file.exists()) - project.logger.lifecycle("PyTorch model zoo metadata already exists: $task") + logger.lifecycle("PyTorch model zoo metadata already exists: $task") else { - project.logger.lifecycle("Downloading PyTorch model zoo metadata: $task") + logger.lifecycle("Downloading PyTorch model zoo metadata: $task") file.parentFile.mkdirs() val downloadPath = "$url/$task/ai/djl/huggingface/pytorch/models.json.gz".url downloadPath gzipInto file @@ -94,9 +99,9 @@ tasks { file = prefix / task / "ai.djl.huggingface.rust.json" if (file.exists()) - project.logger.lifecycle("Rust model zoo metadata alrady exists: $task") + logger.lifecycle("Rust model zoo metadata alrady exists: $task") else { - project.logger.lifecycle("Downloading Rust model zoo metadata: $task") + logger.lifecycle("Downloading Rust model zoo metadata: $task") file.parentFile.mkdirs() val downloadPath = "$url/$task/ai/djl/huggingface/rust/models.json.gz".url downloadPath gzipInto file @@ -109,20 +114,23 @@ tasks { } } - register("compileJNI") { + register("compileJNI") { + val dir = project.projectDir doFirst { if ("mac" in os || "linux" in os) { val arch = if (arch == "amd64") "x86_64" else arch - exec { + execOperations.exec { + workingDir = dir commandLine("bash", "build.sh", arch, flavor) } } else - exec { - commandLine("${project.projectDir}/build.cmd") + execOperations.exec { + workingDir = dir + commandLine("${dir}/build.cmd") } // for ci to upload to S3 - val ciDir = project.projectDir / "jnilib/${libs.versions.djl.get()}/" + val ciDir = dir / "jnilib/${libs.versions.djl.get()}/" copy { from(buildDirectory / "jnilib") into(ciDir) @@ -131,15 +139,17 @@ tasks { } } - register("compileAndroidJNI"){ + register("compileAndroidJNI") { + val dir = project.projectDir doFirst { for (abi in listOf("armeabi-v7a", "arm64-v8a", "x86", "x86_64")) { - exec { + execOperations.exec { + workingDir = dir commandLine("bash", "build_android.sh", abi) } - val ciDir = project.projectDir / "jnilib/${libs.versions.djl.get()}/android/$abi" + val ciDir = dir / "jnilib/${libs.versions.djl.get()}/android/$abi" copy { - from(buildDirectory / "jnilib" / "$abi") + from(buildDirectory / "jnilib" / abi) into(ciDir) } delete("$buildDirectory/jnilib") @@ -147,12 +157,9 @@ tasks { } } - register("formatPython") { - doFirst { - exec { - commandLine("bash", "-c", "find . -name '*.py' -print0 | xargs -0 yapf --in-place") - } - } + register("formatPython") { + workingDir = project.projectDir + commandLine("bash", "-c", "find . -name '*.py' -print0 | xargs -0 yapf --in-place") } clean { @@ -173,4 +180,4 @@ publishing { } } } -} \ No newline at end of file +} diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a407977093c..17dfa12fa96 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip diff --git a/model-zoo/build.gradle.kts b/model-zoo/build.gradle.kts index c5c19752762..87a3aafadbb 100644 --- a/model-zoo/build.gradle.kts +++ b/model-zoo/build.gradle.kts @@ -15,6 +15,7 @@ dependencies { } tasks.register("syncS3") { + workingDir = project.projectDir commandLine( "sh", "-c",