Container.groovy

 * stopAndRemoveContainer() now fills container instead of stopping

DirectorySyncer.groovy
 * Fixed bug that made getDuplicateContainer() return broken containers
 * Added createSyncVolumeToVolume() and syncBetweenVolumesAndUsers()

DockerClientDS.groovy
 * Added getOrCreateVolume()

Author: farthinder

.idea/jarRepositories.xml

@@ -23,7 +23,7 @@
         <spock-core.version>2.3-groovy-${groovy.major.version}</spock-core.version>
         <jiraShortcuts.version>2.0.3-SNAPSHOT-groovy-3.0</jiraShortcuts.version>
         <bitbucketinstancemanager.version>0.0.3-SNAPSHOT-groovy-3.0</bitbucketinstancemanager.version>
-        <jirainstancemanager.version>2.1.8</jirainstancemanager.version>
+        <jirainstancemanager.version>2.1.9</jirainstancemanager.version>
     </properties>
 
 
@@ -111,7 +111,7 @@
         </dependency>
 
 
-        <dependency>
+        <!--dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
             <version>5.10.2</version>
@@ -122,14 +122,15 @@
             <artifactId>junit-jupiter-engine</artifactId>
             <version>5.10.2</version>
             <scope>test</scope>
-        </dependency>
+        </dependency-->
 
 
 
 
     </dependencies>
 
     <repositories>
+
         <repository>
             <id>eficode-github-jiraManagerRest</id>
             <url>https://github.com/eficode/JiraInstanceManagerRest/raw/packages/repository/</url>

pom.xml

@@ -476,10 +476,7 @@ trait Container {
 
 
             if (self.isRunning()) {
-                dockerClient.stop(self.containerId, timeoutS)
-                if (self.isRunning()) {
-                    dockerClient.kill(self.containerId)
-                }
+                dockerClient.kill(self.containerId)
             }
 
             dockerClient.rm(self.containerId)

src/main/groovy/com/eficode/devstack/container/Container.groovy

@@ -96,7 +96,7 @@ class JsmDevDeployment implements Deployment {
     boolean setupDeployment() {
 
 
-        srcSyncer = DirectorySyncer.createSyncToVolume(srcCodePaths, srcCodeVolume.name, "-avh --chown=2001:2001")
+        srcSyncer = DirectorySyncer.createSyncToVolume(srcCodePaths, srcCodeVolume.name, "SrcSyncer", "-avh --chown=2001:2001")
         allureContainer.created ?: allureContainer.createContainer()
         allureContainer.startContainer()
 

src/main/groovy/com/eficode/devstack/deployment/impl/JsmDevDeployment.groovy

@@ -3,6 +3,7 @@ package com.eficode.devstack.util
 import com.eficode.devstack.container.Container
 import com.fasterxml.jackson.databind.ObjectMapper
 import de.gesellix.docker.client.EngineResponseContent
+import de.gesellix.docker.client.network.ManageNetworkClient
 import de.gesellix.docker.remote.api.ContainerSummary
 import de.gesellix.docker.remote.api.Mount
 import de.gesellix.docker.remote.api.MountPoint
@@ -33,13 +34,14 @@ class DirectorySyncer implements Container {
      */
     DirectorySyncer (DockerClientDS dockerClient, ContainerSummary summary) {
 
-        DirectorySyncer syncer = new DirectorySyncer(dockerClient.host, dockerClient.certPath)
-        syncer.containerName = summary.names.first().replaceFirst("/", "")
+        this.dockerClient = dockerClient
+        this.networkClient = dockerClient.getManageNetwork() as ManageNetworkClient
+        this.containerName = summary.names.first().replaceFirst("/", "")
 
 
     }
 
-    static String getSyncScript(String rsyncOptions = "-avh") {
+    static String getSyncScript(String rsyncOptions = "-avh", String rsyncSrc = " /mnt/src/", String rsyncDest = " /mnt/dest/") {
 
         return """
         
@@ -57,7 +59,7 @@ class DirectorySyncer implements Container {
         
         function execute() {
             eval "\$@"
-                rsync $rsyncOptions /mnt/src/*/ /mnt/dest/ 
+                rsync $rsyncOptions $rsyncSrc $rsyncDest
         }
         
         execute""
@@ -105,7 +107,8 @@ class DirectorySyncer implements Container {
      */
     DirectorySyncer getDuplicateContainer() {
 
-        Map filterMap = [name: ["DirectorySyncer.*"], "volume": this.preparedMounts.collect { it.target }]
+
+        Map filterMap = [name: [this.containerName + ".*"], "volume": this.preparedMounts.collect { it.target }]
         String filterString = new ObjectMapper().writeValueAsString(filterMap)
         ArrayList<ContainerSummary> looselyMatchingContainers = dockerClient.ps(true, null, false, filterString).content
         ArrayList<ContainerSummary> matchingContainers = []
@@ -175,25 +178,15 @@ class DirectorySyncer implements Container {
      * @param dockerCertPath Docker certs to use
      * @return
      */
-    static DirectorySyncer createSyncToVolume(ArrayList<String> hostAbsSourcePaths, String destVolumeName, String rsyncOptions = "-avh",  String containerName = "",String dockerHost = "", String dockerCertPath = "") {
+    static DirectorySyncer createSyncToVolume(ArrayList<String> hostAbsSourcePaths, String destVolumeName, String containerName, String rsyncOptions = "-avh",  String dockerHost = "", String dockerCertPath = "") {
 
         DirectorySyncer container = new DirectorySyncer(dockerHost, dockerCertPath)
         Logger log = container.log
 
-        container.containerName = container.getAvailableContainerName()
-        container.prepareCustomEnvVar(["syncScript=${getSyncScript(rsyncOptions)}"])
-
-        Volume volume = container.dockerClient.getVolumesWithName(destVolumeName).find { true }
+        container.containerName = containerName ?: container.getAvailableContainerName()
+        container.prepareCustomEnvVar(["syncScript=${getSyncScript(rsyncOptions, "/mnt/src/*/")}"])
 
-        if (volume) {
-            log.debug("\tFound existing volume:" + volume.name)
-        } else {
-            log.debug("\tCreating new volume $destVolumeName")
-            EngineResponseContent<Volume> volumeResponse = container.dockerClient.createVolume(destVolumeName)
-            volume = volumeResponse?.content
-            assert volume: "Error creating volume $destVolumeName, " + volumeResponse?.getStatus()?.text
-            log.debug("\t\tCreated volume:" + volume.name)
-        }
+        Volume volume = container.dockerClient.getOrCreateVolume(destVolumeName)
 
         container.prepareVolumeMount(volume.name, "/mnt/dest/", false)
 
@@ -224,5 +217,62 @@ class DirectorySyncer implements Container {
         return container
     }
 
+    /**
+     * Creates a DirectorySyncer intended to sync files between two volumes and replacing the owner of the synced files so that
+     * the destination container user has access to.
+     * @param srcVolumeName The volume to sync from (the root of this will be synced)
+     * @param destVolumeName The destination volume where files should be synced to, and where the owner will be changed
+     * @param destUser The destination user and group that the file owner will be changed to, ex: 1001:1001
+     * @return
+     */
+    static DirectorySyncer syncBetweenVolumesAndUsers(String srcVolumeName, String destVolumeName, String destUser) {
+
+        DirectorySyncer syncer =  createSyncVolumeToVolume(srcVolumeName, destVolumeName, "-avhog --chown $destUser")
+
+        return syncer
+    }
+
+
+    /**
+     * Creates a DirectorySyncer which synces files between the roots of two docker volumes
+     * @param srcVolumeName The source volume to sync from
+     * @param destVolumeName The destination volume to sync to
+     * @param rsyncOptions Options to pass to rsync, default: -avh
+     * @param containerName Name of the sync container
+     * @param dockerHost
+     * @param dockerCertPath
+     * @return
+     */
+    static DirectorySyncer createSyncVolumeToVolume(String srcVolumeName, String destVolumeName, String rsyncOptions = "-avh",  String containerName = "",String dockerHost = "", String dockerCertPath = "") {
+
+        DirectorySyncer container = new DirectorySyncer(dockerHost, dockerCertPath)
+        Logger log = container.log
+
+        container.containerName = containerName ?: container.getAvailableContainerName()
+        container.prepareCustomEnvVar(["syncScript=${getSyncScript(rsyncOptions)}"])
+
+        Volume destVolume = container.dockerClient.getOrCreateVolume(destVolumeName)
+        Volume srcVolume = container.dockerClient.getOrCreateVolume(srcVolumeName)
+
+        container.prepareVolumeMount(srcVolume.name, "/mnt/src/", false)
+        container.prepareVolumeMount(destVolume.name, "/mnt/dest/", false)
+
+
+        DirectorySyncer duplicate = container.getDuplicateContainer()
+        if (duplicate) {
+            log.info("\tFound an existing DirectorySyncer with same mount points:" + duplicate.shortId)
+            if (!duplicate.running) {
+                log.debug("\t" * 2 + "Duplicate is not running, starting it")
+                duplicate.startContainer()
+            }
+            log.info("\t" * 2 + "Returning duplicate instead of creating a new one")
+            return duplicate
+        }
+
+        container.createContainer(["/bin/sh", "-c", "echo \"\$syncScript\" > /syncScript.sh && /bin/sh syncScript.sh"], [])
+        container.startContainer()
+
+        return container
+    }
 
 }

src/main/groovy/com/eficode/devstack/util/DirectorySyncer.groovy

@@ -76,6 +76,22 @@ class DockerClientDS extends DockerClientImpl {
     }
 
 
+    Volume getOrCreateVolume(String volumeName) {
+        Volume volume = getVolumesWithName(volumeName).find { true }
+
+        if (volume) {
+            log.debug("\tFound existing volume:" + volume.name)
+        } else {
+            log.debug("\tCreating new volume $volumeName")
+            EngineResponseContent<Volume> volumeResponse = createVolume(volumeName)
+            volume = volumeResponse?.content
+            assert volume: "Error creating volume $volumeName, " + volumeResponse?.getStatus()?.text
+            log.debug("\t\tCreated destination volume:" + volume.name)
+        }
+
+        return volume
+    }
+
     ArrayList<Volume> getVolumesWithName(String name) {
         EngineResponseContent<VolumeListResponse> response = volumes("{\"name\":[\"$name\"]}")
 

src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy

@@ -2,8 +2,10 @@ package com.eficode.devstack.container.impl
 
 import com.eficode.devstack.DevStackSpec
 import com.eficode.devstack.util.DirectorySyncer
+import com.eficode.devstack.util.ImageSummaryDS
 import de.gesellix.docker.remote.api.ContainerInspectResponse
 import de.gesellix.docker.remote.api.MountPoint
+import de.gesellix.docker.remote.api.Volume
 import org.slf4j.LoggerFactory
 
 
@@ -42,13 +44,13 @@ class DirectorySyncerTest extends DevStackSpec {
         !volumeExists(uniqueVolumeName) ?: dockerClient.rmVolume(uniqueVolumeName)
         log.debug("\tWill use sync to Docker volume:" + uniqueVolumeName)
 
-        DirectorySyncer firstSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "-avh --delete", dockerRemoteHost, dockerCertPath )
+        DirectorySyncer firstSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath )
         log.info("\tCreated first sync container: ${firstSyncer.containerName} (${firstSyncer.shortId})")
         Integer containersAfterFirst = firstSyncer.dockerClient.ps(true).content.size()
         log.info("\t\tDocker engine now has a total of ${containersAfterFirst} contianers")
 
         when: "Creating second sync container"
-        DirectorySyncer secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "-avh --delete", dockerRemoteHost, dockerCertPath )
+        DirectorySyncer secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer","-avh --delete", dockerRemoteHost, dockerCertPath )
         log.info("\tCreated second sync container: ${secondSyncer.containerName} (${secondSyncer.shortId})")
 
         then: "They should have the same ID"
@@ -60,7 +62,7 @@ class DirectorySyncerTest extends DevStackSpec {
         when: "Stopping the sync container, and creating another duplicate"
         firstSyncer.stopContainer()
         assert !firstSyncer.running
-        secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "-avh --delete", dockerRemoteHost, dockerCertPath )
+        secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer","-avh --delete", dockerRemoteHost, dockerCertPath )
 
         then:"The duplicate should have been automatically started"
         secondSyncer.running
@@ -74,6 +76,62 @@ class DirectorySyncerTest extends DevStackSpec {
 
 
 
+    }
+
+
+    def "Test create createSyncVolumeToVolume"() {
+
+        setup:
+        log.info("Testing createSyncVolumeToVolume")
+        Volume srcVolume = dockerClient.getOrCreateVolume("srcVolume" + System.nanoTime().toString().takeRight(3))
+        Volume destVolume = dockerClient.getOrCreateVolume("destVolume" + System.nanoTime().toString().takeRight(3))
+        log.info("\tWill use src volume:" + srcVolume)
+        log.info("\tWill use dest volume:" + destVolume)
+
+
+        when: "Creating two containers with two different default users"
+        UbuntuContainer srcContainer = new UbuntuContainer()
+        srcContainer.containerName = "SrcContainer"
+        srcContainer.prepareVolumeMount(srcVolume.name, "/mnt/volume", false)
+        srcContainer.user = "1001:1001"
+        srcContainer.createSleepyContainer()
+        srcContainer.startContainer()
+        srcContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntusrc", "1001", "ubuntusrc", "1001"),10, "root" )
+        srcContainer.runBashCommandInContainer("chown 1001:1001 -R /mnt/volume", 5, "root")
+
+
+        UbuntuContainer destContainer = new UbuntuContainer()
+        destContainer.containerName = "DestContainer"
+        destContainer.prepareVolumeMount(destVolume.name, "/mnt/volume", false)
+        destContainer.user = "1002:1002"
+        destContainer.createSleepyContainer()
+        destContainer.startContainer()
+        destContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntudest", "1002", "ubuntudest", "1002"),10, "root" )
+        destContainer.runBashCommandInContainer("chown 1002:1002 -R /mnt/volume", 5, "root")
+
+        then: "When checking user id, they should be different"
+        srcContainer.runBashCommandInContainer("id -u").contains("1001")
+        destContainer.runBashCommandInContainer("id -u").contains("1002")
+
+
+        when: "Creating the syncer"
+
+        DirectorySyncer syncer = DirectorySyncer.syncBetweenVolumesAndUsers(srcVolume.name, destVolume.name, "1002:1002")
+        srcContainer.runBashCommandInContainer("touch /mnt/volume/createdInSource")
+        sleep(1000)
+        then:
+        destContainer.runBashCommandInContainer("ls -l /mnt/volume/createdInSource && echo Status: \$?").contains("Status: 0")
+        destContainer.runBashCommandInContainer("echo edited >> /mnt/volume/createdInSource && echo Status: \$?").contains("Status: 0")
+        destContainer.runBashCommandInContainer("rm /mnt/volume/createdInSource && echo Status: \$?").contains("Status: 0")
+
+
+        cleanup:
+        srcContainer.stopAndRemoveContainer()
+        destContainer.stopAndRemoveContainer()
+        syncer.stopAndRemoveContainer()
+        dockerClient.manageVolume.rmVolume(srcVolume.name)
+        dockerClient.manageVolume.rmVolume(destVolume.name)
+
     }
 
     def "Test createSyncToVolume"() {
@@ -93,7 +151,7 @@ class DirectorySyncerTest extends DevStackSpec {
         when: "When creating syncer"
 
         assert !volumeExists(uniqueVolumeName): "Destination volume already exists"
-        DirectorySyncer syncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "-avh --delete", dockerRemoteHost, dockerCertPath )
+        DirectorySyncer syncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath )
         log.info("\tCreated sync container: ${syncer.containerName} (${syncer.shortId})")
         ContainerInspectResponse containerInspect = syncer.inspectContainer()