Skip to content

Commit 96b96ae

Browse files
pierotofypierotofy
authored
Merge pull request #129 from FJEANNOT/master
Add support for docker additionnal args
2 parents 547445c + bcc6618 commit 96b96ae

File tree

8 files changed

+69
-43
lines changed

8 files changed

+69
-43
lines changed

‎docs/aws.md

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -67,31 +67,32 @@ the on-demand instance cost - you'll always pay the current market price, not yo
6767
}
6868
```
6969

70-
| Field | Description |
71-
|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
72-
| accessKey | AWS Access Key |
73-
| secretKey | AWS Secret Key |
74-
| s3 | S3 bucket configuration. |
75-
| vpc | The virtual private cloud in which the instances operate. Not providing this assumes a default setting for VPC within the AWS environment. |
76-
| subnet | The subnet supporting the instances. Not providing this assumes a default setting for the subnet within the AWS environment. |
77-
| usePrivateAddress | Set to true to use the private IP address when communicating with auto-scaled nodes. Useful if ClusterODM is on the same vpc as the auto-scaled nodes. |
78-
| assignPrivateAddressOnly | Set to true to ensure that only a private IP address is assigned to the created node. Only has effect if usePrivateAddress is true. Avoids extra charges. |
79-
| securityGroup | AWS Security Group name (not ID). Must exist and allow incoming connections from your ClusterODM host on port TCP/3000. |
80-
| createRetries | Number of attempts to create a droplet before giving up. Defaults to 1. |
81-
| maxRuntime | Maximum number of seconds an instance is allowed to run ever. Set to -1 for no limit. |
82-
| maxUploadTime | Maximum number of seconds an instance is allowed to receive file uploads. Set to -1 for no limit. |
83-
| monitoring | Set to true to enable detailed Cloudwatch monitoring for the instance. |
84-
| region | Region identifier where the instances should be created. |
85-
| zone | Zone identifier where the instances should be created. |
86-
| ami | The AMI (machine image) to launch this instance from. Note that AMIs are region-specific. |
87-
| engineInstallUrl | Specify installer for Docker engine. This can be cleared if AMI already has Docker engine installed. |
88-
| tags | Comma-separated list of key,value tags to associate to the instance. |
89-
| spot | Whether to request spot instances. If this is true, a `spotPrice` needs to be provided in the `imageSizeMapping`. |
90-
| imageSizeMapping | Max images count to instance size mapping. (See below.) |
91-
| addSwap | Optionally add this much swap space to the instance as a factor of total RAM (`RAM * addSwap`). A value of `1` sets a swapfile equal to the available RAM. |
92-
| dockerImage | Docker image to launch |
93-
| dockerDataDirMountPath | Path on node host to map to NodeODM data directory (/var/www/data). Use local instance storage for much faster I/O. |
94-
| nodeSetupCmd | Can be optionally used to run a setup command on auto-scaled nodes right before we run ODM. |
70+
| Field | Description |
71+
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
72+
| accessKey | AWS Access Key |
73+
| secretKey | AWS Secret Key |
74+
| s3 | S3 bucket configuration. |
75+
| vpc | The virtual private cloud in which the instances operate. Not providing this assumes a default setting for VPC within the AWS environment. |
76+
| subnet | The subnet supporting the instances. Not providing this assumes a default setting for the subnet within the AWS environment. |
77+
| usePrivateAddress | Set to true to use the private IP address when communicating with auto-scaled nodes. Useful if ClusterODM is on the same vpc as the auto-scaled nodes. |
78+
| assignPrivateAddressOnly | Set to true to ensure that only a private IP address is assigned to the created node. Only has effect if usePrivateAddress is true. Avoids extra charges. |
79+
| securityGroup | AWS Security Group name (not ID). Must exist and allow incoming connections from your ClusterODM host on port TCP/3000. |
80+
| createRetries | Number of attempts to create a droplet before giving up. Defaults to 1. |
81+
| maxRuntime | Maximum number of seconds an instance is allowed to run ever. Set to -1 for no limit. |
82+
| maxUploadTime | Maximum number of seconds an instance is allowed to receive file uploads. Set to -1 for no limit. |
83+
| monitoring | Set to true to enable detailed Cloudwatch monitoring for the instance. |
84+
| region | Region identifier where the instances should be created. |
85+
| zone | Zone identifier where the instances should be created. |
86+
| ami | The AMI (machine image) to launch this instance from. Note that AMIs are region-specific. |
87+
| engineInstallUrl | Specify installer for Docker engine. This can be cleared if AMI already has Docker engine installed. |
88+
| tags | Comma-separated list of key,value tags to associate to the instance. |
89+
| spot | Whether to request spot instances. If this is true, a `spotPrice` needs to be provided in the `imageSizeMapping`. |
90+
| imageSizeMapping | Max images count to instance size mapping. (See below.) |
91+
| addSwap | Optionally add this much swap space to the instance as a factor of total RAM (`RAM * addSwap`). A value of `1` sets a swapfile equal to the available RAM. |
92+
| dockerImage | Docker image to launch |
93+
| dockerDataDirMountPath | Path on node host to map to NodeODM data directory (/var/www/data). Use local instance storage for much faster I/O. |
94+
| dockerGpu | Enables GPU acceleration by passing `--gpu all` to docker |
95+
| nodeSetupCmd | Can be optionally used to run a setup command on auto-scaled nodes right before we run ODM. |
9596

9697
## Image Size Mapping
9798

‎docs/digitalocean.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,4 +60,5 @@ Example configuration file:
6060
| minImages | Minimum number of images that a dataset needs to have for the autoscaler to be used (-1 = no minimum). |
6161
| addSwap | Optionally add this much swap space to the droplet as a factor of total RAM (`RAM * addSwap`). A value of `1` sets a swapfile equal to the available RAM. |
6262
| dockerImage | Docker image to launch
63+
| dockerGpu | Enables GPU acceleration by passing `--gpu all` to docker |
6364
|

‎docs/hetzner.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,3 +71,4 @@ Example configuration file:
7171
| minImages | Minimum number of images that a dataset needs to have for the autoscaler to be used (-1 = no minimum). |
7272
| addSwap | Optionally add this much swap space to the machine as a factor of total RAM (`RAM * addSwap`). A value of `1` sets a swapfile equal to the available RAM. |
7373
| dockerImage | Docker image to launch |
74+
| dockerGpu | Enables GPU acceleration by passing `--gpu all` to docker |

‎docs/scaleway.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,4 @@ Example configuration file:
6666
| minImages | Minimum number of images that a dataset needs to have for the autoscaler to be used (-1 = no minimum). |
6767
| addSwap | Optionally add this much swap space to the machine as a factor of total RAM (`RAM * addSwap`). A value of `1` sets a swapfile equal to the available RAM. |
6868
| dockerImage | Docker image to launch |
69+
| dockerGpu | Enables GPU acceleration by passing `--gpu all` to docker |

‎libs/asr-providers/aws.js

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,17 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{
3131
"bucket": "CHANGEME!",
3232
"acl": "public-read"
3333
},
34-
"vpc": "",
35-
"subnet": "",
36-
"usePrivateAddress": false,
37-
"assignPrivateAddressOnly": false,
34+
"vpc": "",
35+
"subnet": "",
36+
"usePrivateAddress": false,
37+
"assignPrivateAddressOnly": false,
3838
"securityGroup": "CHANGEME!",
3939
"maxRuntime": -1,
4040
"maxUploadTime": -1,
4141
"instanceLimit": -1,
4242
"createRetries": 1,
4343
"region": "us-west-2",
44-
"zone": "",
44+
"zone": "",
4545
"monitoring": false,
4646
"tags": ["clusterodm"],
4747
"ami": "ami-07b4f3c02c7f83d59",
@@ -53,9 +53,11 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{
5353

5454
"addSwap": 1,
5555
"dockerImage": "opendronemap/nodeodm",
56-
"dockerDataDirMountPath": "",
57-
"iamrole": "",
58-
"nodeSetupCmd": ""
56+
"dockerDataDirMountPath": "",
57+
"dockerGpu": false,
58+
59+
"iamrole": "",
60+
"nodeSetupCmd": ""
5961
}, userConfig);
6062
}
6163

@@ -121,7 +123,7 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{
121123
const dockerImage = this.getConfig("dockerImage");
122124
const accessKey = this.getConfig("accessKey");
123125
const secretKey = this.getConfig("secretKey");
124-
const dataDirMountPath = this.getConfig("dataDirMountPath");
126+
const dataDirMountPath = this.getConfig("dataDirMountPath");
125127
const s3 = this.getConfig("s3");
126128
const webhook = netutils.publicAddressPath("/commit", req, token);
127129

@@ -136,6 +138,9 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{
136138
if(dataDirMountPath.length > 0){
137139
dockerRunArgs.push(`--mount type=bind,source=${dataDirMountPath},target=/var/www/data`);
138140
}
141+
if (this.getConfig("dockerGpu")){
142+
dockerRunArgs.push(`--gpus all`);
143+
}
139144

140145
dockerRunArgs.push(`${dockerImage} -q 1`);
141146
dockerRunArgs.push(`--s3_access_key ${accessKey}`);
@@ -146,7 +151,7 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{
146151
dockerRunArgs.push(`--webhook ${webhook}`);
147152
dockerRunArgs.push(`--token ${nodeToken}`);
148153

149-
await dm.ssh(dockerRunArgs.join(" "));
154+
await dm.ssh(dockerRunArgs.join(" "));
150155
}
151156

152157
getImagePropertiesFor(imagesCount){

‎libs/asr-providers/digitalocean.js

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ module.exports = class DigitalOceanAsrProvider extends AbstractASRProvider{
5757
"minImages": -1,
5858

5959
"addSwap": 1,
60-
"dockerImage": "opendronemap/nodeodm"
60+
"dockerImage": "opendronemap/nodeodm",
61+
"dockerGpu": false
6162
}, userConfig);
6263
}
6364

@@ -126,8 +127,12 @@ module.exports = class DigitalOceanAsrProvider extends AbstractASRProvider{
126127
const dockerImage = this.getConfig("dockerImage");
127128
const s3 = this.getConfig("s3");
128129
const webhook = netutils.publicAddressPath("/commit", req, token);
130+
let dockerAdditionalArgs = "";
131+
if (this.getConfig("dockerGpu")) {
132+
dockerAdditionalArgs = "--gpus all";
133+
}
129134

130-
await dm.ssh([`docker run -d -p 3000:3000 ${dockerImage} -q 1`,
135+
await dm.ssh([`docker run -d -p 3000:3000 ${dockerAdditionalArgs} ${dockerImage} -q 1`,
131136
`--s3_access_key ${s3.accessKey}`,
132137
`--s3_secret_key ${s3.secretKey}`,
133138
`--s3_endpoint ${s3.endpoint}`,

‎libs/asr-providers/hetzner.js

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ module.exports = class HetznerAsrProvider extends AbstractASRProvider{
5555
"minImages": -1,
5656

5757
"addSwap": 1,
58-
"dockerImage": "opendronemap/nodeodm"
58+
"dockerImage": "opendronemap/nodeodm",
59+
"dockerGpu": false
5960
}, userConfig);
6061
}
6162

@@ -177,15 +178,20 @@ module.exports = class HetznerAsrProvider extends AbstractASRProvider{
177178
const dockerImage = this.getConfig("dockerImage");
178179
const s3 = this.getConfig("s3");
179180
const webhook = netutils.publicAddressPath("/commit", req, token);
181+
let dockerAdditionalArgs = "";
182+
if (this.getConfig("dockerGpu")) {
183+
dockerAdditionalArgs = "--gpus all";
184+
}
180185

181-
await dm.ssh([`docker run -d -p 3000:3000 ${dockerImage} -q 1`,
186+
await dm.ssh([`docker run -d -p 3000:3000 ${dockerAdditionalArgs} ${dockerImage} -q 1`,
182187
`--s3_access_key ${s3.accessKey}`,
183188
`--s3_secret_key ${s3.secretKey}`,
184189
`--s3_endpoint ${s3.endpoint}`,
185190
`--s3_bucket ${s3.bucket}`,
186191
s3.ignoreSSL ? '--s3_ignore_ssl' : '',
187192
`--webhook ${webhook}`,
188-
`--token ${nodeToken}`].join(" "));
193+
`--token ${nodeToken}`
194+
].join(" "));
189195
}
190196

191197
getImageSlugFor(imagesCount){

‎libs/asr-providers/scaleway.js

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ module.exports = class ScalewayAsrProvider extends AbstractASRProvider{
4848
"minImages": -1,
4949

5050
"addSwap": 1,
51-
"dockerImage": "opendronemap/nodeodm"
51+
"dockerImage": "opendronemap/nodeodm",
52+
"dockerGpu": false
5253
}, userConfig);
5354
}
5455

@@ -109,15 +110,20 @@ module.exports = class ScalewayAsrProvider extends AbstractASRProvider{
109110
const dockerImage = this.getConfig("dockerImage");
110111
const s3 = this.getConfig("s3");
111112
const webhook = netutils.publicAddressPath("/commit", req, token);
113+
let dockerAdditionalArgs = "";
114+
if (this.getConfig("dockerGpu")) {
115+
dockerAdditionalArgs = "--gpus all";
116+
}
112117

113-
await dm.ssh([`docker run -d -p 3000:3000 ${dockerImage} -q 1`,
118+
await dm.ssh([`docker run -d -p 3000:3000 ${dockerAdditionalArgs} ${dockerImage} -q 1`,
114119
`--s3_access_key ${s3.accessKey}`,
115120
`--s3_secret_key ${s3.secretKey}`,
116121
`--s3_endpoint ${s3.endpoint}`,
117122
`--s3_bucket ${s3.bucket}`,
118123
s3.ignoreSSL ? '--s3_ignore_ssl' : '',
119124
`--webhook ${webhook}`,
120-
`--token ${nodeToken}`].join(" "));
125+
`--token ${nodeToken}`,
126+
].join(" "));
121127
}
122128

123129
getImageSlugFor(imagesCount){

0 commit comments

Comments
 (0)