save progress again

Author: AmozPay

examples/message_from_aleph_docker_runtime.json

@@ -54,7 +54,12 @@
         },
         "volumes": [
             {
-                "mount": "/opt/docker",
+                "mount": "/opt/docker/metadata",
+                "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51",
+                "use_latest": false
+            },
+            {
+                "mount": "/opt/docker/layers",
                 "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51",
                 "use_latest": false
             },

run_supervisor_host.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+export PYTHONPATH=$(pwd)
+
+export ALEPH_VM_ALLOW_VM_NETWORKING=False
+export ALEPH_VM_NETWORK_INTERFACE=tap0
+export ALEPH_VM_USE_JAILER=False
+export ALEPH_VM_FAKE_DATA=True
+export ALEPH_VM_SUPERVISOR_HOST=0.0.0.0
+
+export BENCHMARK_FAKE_DATA_PROGRAM=$(pwd)/examples/example_docker_container
+export FAKE_DATA_MESSAGE=$(pwd)/examples/message_from_aleph_docker_runtime.json
+export FAKE_DATA_DATA=$(pwd)/examples/data/
+export FAKE_DATA_RUNTIME=$(pwd)/runtimes/aleph-docker/rootfs.squashfs
+export FAKE_DATA_VOLUME=$(pwd)/examples/volumes/docker/layers:/opt/docker/layers,$(pwd)/examples/volumes/docker/metadata:/opt/docker/metadata
+
+
+python3 -m vm_supervisor --print-settings --very-verbose --system-logs --profile -f ./examples/example_docker_container

runtimes/aleph-docker/create_disk_image.sh

@@ -41,19 +41,28 @@ echo "root:toor" | /usr/sbin/chpasswd
 
 mkdir -p /overlay
 
+mkdir -p /var/lib/docker
+cd /var/lib/docker
+mkdir -m  710 vfs
+mkdir -m 700 image
+mkdir -m 700 image/vfs
+mkdir -m 700 plugins
+mkdir -m 700 swarm
+cmkdir -m 750 network
+mkdir -m 700 trust
+mkdir -m 701 volumes
+mkdir -m 711 buildkit
+mkdir -m 710 containers
+
 # Set up a login terminal on the serial console (ttyS0):
 ln -s agetty /etc/init.d/agetty.ttyS0
 echo ttyS0 > /etc/securetty
 
 update-alternatives --set iptables /usr/sbin/iptables-legacy
 EOT
-# The docker service starts automatically on Debian based distributions. https://docs.docker.com/engine/install/debian/
-
 
 echo "PermitRootLogin yes" >> ./rootfs/etc/ssh/sshd_config
 
-echo -ne '{\n"storage-driver": "vfs"\n}\n' > ./rootfs/etc/docker/daemon.json
-
 # Generate SSH host keys
 #systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
 #systemd-nspawn -D ./rootfs/ ssh-keygen -q -N "" -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key

runtimes/aleph-docker/init0.sh

@@ -48,8 +48,6 @@ cgroupfs-mount
 
 export PATH=$PATH:/usr/local/bin:/usr/bin:/usr/sbin
 
-/usr/sbin/dockerd 2>&1 | tee /log_docker.txt &
-
 log "INIT 0 DONE2"
 # Replace this script with the manager
 exec /root/init1.py

runtimes/aleph-docker/init1.py

@@ -104,6 +104,7 @@ def setup_variables(variables: Optional[Dict[str, str]]):
     if variables is None:
         return
     for key, value in variables.items():
+        print(key)
         os.environ[key] = value
 
 
@@ -121,6 +122,7 @@ def setup_network(
         return
 
     logger.debug("Setting up networking")
+    logger.debug("IP ADDR:" + ip)
     system("ip addr add 127.0.0.1/8 dev lo brd + scope host")
     system("ip addr add ::1/128 dev lo")
     system("ip link set lo up")
@@ -157,9 +159,6 @@ def setup_volumes(volumes: List[Volume]):
         else:
             system(f"mount -o rw /dev/{volume.device} {volume.mount}")
 
-    system("mount")
-
-
 def setup_code_asgi(
     code: bytes, encoding: Encoding, entrypoint: str
 ) -> ASGIApplication:
@@ -450,28 +449,43 @@ def receive_config(client) -> ConfigurationPayload:
 
 
 def setup_docker():
+    logger.debug("Setting up docker")
     docker_mountpoint = os.environ.get("DOCKER_MOUNTPOINT")
     os.makedirs("/docker", exist_ok=True)
     system("bin/mount -t tmpfs -o noatime,mode=0755 tmpfs /docker")
-    os.makedirs("/docker/persist/work", exist_ok=True)
-    os.makedirs("/docker/persist/upper", exist_ok=True)
-    # system("stat -f /")
-    # system(f"stat -f {docker_mountpoint}")
-    # system("stat -f /docker/")
-    # system("stat -f /docker/persist/")
-    # system("stat -f /docker/persist/upper")
-    # system("stat -f /docker/persist/work")
-    # system("stat -f /var/lib/docker")
-    logger.debug(os.path.isdir("/docker/persist"))
-    logger.debug(os.path.isdir("/docker/persist/upper"))
-    system(f'/bin/mount -o noatime,lowerdir={docker_mountpoint},upperdir=/docker/persist/upper,workdir=/docker/persist/work -t overlay "overlayfs:/docker/persist/upper" /var/lib/docker')
-    return subprocess.Popen("/usr/sbin/dockerd", stderr=subprocess.PIPE, encoding='utf-8')
+    os.makedirs("/docker/persist/layers/work", exist_ok=True)
+    os.makedirs("/docker/persist/metadata/work", exist_ok=True)
+    os.makedirs("/docker/persist/layers/upper", exist_ok=True)
+    os.makedirs("/docker/persist/metadata/upper", exist_ok=True)
+    # docker_daemon = subprocess.Popen(["/usr/sbin/dockerd", "--storage-driver=vfs"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
+    system(f'/bin/mount -o noatime,lowerdir={docker_mountpoint}/layers,upperdir=/docker/persist/layers/upper,workdir=/docker/persist/layers/work -t overlay "overlayfs:/docker/persist/layers/upper" /var/lib/docker/vfs')
+    system(f'/bin/mount -o noatime,lowerdir={docker_mountpoint}/metadata,upperdir=/docker/persist/metadata/upper,workdir=/docker/persist/metadata/work -t overlay "overlayfs:/docker/persist/metadata/upper" /var/lib/docker/image/vfs')
+    print("Before daemon:\n")
+    os.system("stat -f /var/lib/docker/image/vfs/repositories.json")
+    os.system("cat /var/lib/docker/image/vfs/repositories.json")
+    print("here")
+    # docker_daemon = subprocess.Popen(["/usr/sbin/dockerd", "--storage-driver=vfs"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
+    print("there")
+    os.system("docker info")
+    docker_daemon = subprocess.Popen(["/usr/sbin/dockerd", "--storage-driver=vfs"], stderr=subprocess.PIPE, encoding='utf-8')
+    # os.system("/usr/sbin/dockerd --storage-driver=vfs")
+    while os.system("docker ps > /dev/null 2>&1") != 0:
+        # print("yulu")
+        # stderr = docker_daemon.communicate()
+        # print("stderr: " + stderr)
+        continue
+    print("After daemon:\n")
+    os.system("stat -f /var/lib/docker/image/vfs/repositories.json")
+    os.system("cat /var/lib/docker/image/vfs/repositories.json")
+    system("mount")
+
 
 def setup_system(config: ConfigurationPayload):
     setup_hostname(config.vm_hash)
     setup_variables(config.variables)
     setup_volumes(config.volumes)
-    docker_daemon = setup_docker()
+    setup_docker()
+    print("dameon ready")
     setup_network(config.ip, config.route, config.dns_servers)
     setup_input_data(config.input_data)
     logger.debug("Setup finished")

vm_supervisor/conf.py

@@ -182,9 +182,15 @@ def check(self):
             assert isfile(
                 self.FAKE_DATA_RUNTIME
             ), "Local runtime .squashfs build is missing"
-            assert isfile(
-                self.FAKE_DATA_VOLUME
-            ), "Local data volume .squashfs is missing"
+            if "," in str(self.FAKE_DATA_VOLUME): # allow multiple volumes with format "host_path:mountpoint,host_path:mountpoint"
+                for volume_bind in str(self.FAKE_DATA_VOLUME).split(","):
+                    assert isfile(
+                        volume_bind.split(":")[0]
+                    ), f"Local data volume {volume_bind.split(':')[0]} is missing"
+            else:
+                assert isfile(
+                    self.FAKE_DATA_VOLUME
+                ), f"Local data volume {volume_bind.split(':')[0]} is missing"
 
     def setup(self):
         os.makedirs(self.MESSAGE_CACHE, exist_ok=True)

vm_supervisor/storage.py

@@ -157,9 +157,14 @@ def create_ext4(path: Path, size_mib: int) -> bool:
 
 async def get_volume_path(volume: MachineVolume, namespace: str) -> Path:
     if isinstance(volume, ImmutableVolume):
+        print(volume)
         ref = volume.ref
         if settings.FAKE_DATA_PROGRAM and settings.FAKE_DATA_VOLUME:
-            return Path(settings.FAKE_DATA_VOLUME)
+            if "," not in str(settings.FAKE_DATA_VOLUME):
+                return Path(settings.FAKE_DATA_VOLUME)
+            for volume_bind in str(settings.FAKE_DATA_VOLUME).split(","):
+                if volume.mount == volume_bind.split(":")[1]:
+                    return volume_bind.split(":")[0]
 
         cache_path = Path(join(settings.DATA_CACHE, ref))
         url = f"{settings.CONNECTOR_URL}/download/data/{ref}"