Skip to content

Commit d6744fb

Browse files
author
epriestley
committed
Implement bin/aws-s3 get ... and a basic S3 client API
Summary: Ref T5155. This implements pulling file data off S3 using a first-party, Signature v4-compatible API. Test Plan: {F1057744} Reviewers: chad Reviewed By: chad Maniphest Tasks: T5155 Differential Revision: https://secure.phabricator.com/D14979
1 parent 0ca806c commit d6744fb

11 files changed

+299
-78
lines changed

‎bin/aws-s3

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../scripts/utils/aws-s3.php

‎scripts/utils/aws-s3.php

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env php
2+
<?php
3+
4+
$root = dirname(dirname(dirname(__FILE__)));
5+
require_once $root.'/scripts/__init_script__.php';
6+
7+
$args = new PhutilArgumentParser($argv);
8+
$args->setTagline(pht('AWS CLI Client for S3'));
9+
$args->setSynopsis(<<<EOSYNOPSIS
10+
**aws-s3** __command__ [__options__]
11+
Upload and download data from Amazon Simple Storage Service (S3).
12+
13+
EOSYNOPSIS
14+
);
15+
$args->parseStandardArguments();
16+
17+
$workflows = id(new PhutilClassMapQuery())
18+
->setAncestorClass('PhutilAWSS3ManagementWorkflow')
19+
->execute();
20+
21+
$workflows[] = new PhutilHelpArgumentWorkflow();
22+
$args->parseWorkflows($workflows);

‎src/__phutil_library_map__.php

+6
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,10 @@
8585
'PhutilAWSEC2Future' => 'future/aws/PhutilAWSEC2Future.php',
8686
'PhutilAWSException' => 'future/aws/PhutilAWSException.php',
8787
'PhutilAWSFuture' => 'future/aws/PhutilAWSFuture.php',
88+
'PhutilAWSManagementWorkflow' => 'future/aws/management/PhutilAWSManagementWorkflow.php',
8889
'PhutilAWSS3Future' => 'future/aws/PhutilAWSS3Future.php',
90+
'PhutilAWSS3GetManagementWorkflow' => 'future/aws/management/PhutilAWSS3GetManagementWorkflow.php',
91+
'PhutilAWSS3ManagementWorkflow' => 'future/aws/management/PhutilAWSS3ManagementWorkflow.php',
8992
'PhutilAWSv4Signature' => 'future/aws/PhutilAWSv4Signature.php',
9093
'PhutilAWSv4SignatureTestCase' => 'future/aws/__tests__/PhutilAWSv4SignatureTestCase.php',
9194
'PhutilAggregateException' => 'error/PhutilAggregateException.php',
@@ -603,7 +606,10 @@
603606
'PhutilAWSEC2Future' => 'PhutilAWSFuture',
604607
'PhutilAWSException' => 'Exception',
605608
'PhutilAWSFuture' => 'FutureProxy',
609+
'PhutilAWSManagementWorkflow' => 'PhutilArgumentWorkflow',
606610
'PhutilAWSS3Future' => 'PhutilAWSFuture',
611+
'PhutilAWSS3GetManagementWorkflow' => 'PhutilAWSS3ManagementWorkflow',
612+
'PhutilAWSS3ManagementWorkflow' => 'PhutilAWSManagementWorkflow',
607613
'PhutilAWSv4Signature' => 'Phobject',
608614
'PhutilAWSv4SignatureTestCase' => 'PhutilTestCase',
609615
'PhutilAggregateException' => 'Exception',

‎src/future/aws/PhutilAWSFuture.php

+76-77
Original file line numberDiff line numberDiff line change
@@ -3,85 +3,115 @@
33
abstract class PhutilAWSFuture extends FutureProxy {
44

55
private $future;
6-
private $awsAccessKey;
7-
private $awsPrivateKey;
8-
private $awsRegion;
9-
private $builtRequest;
10-
private $params;
6+
private $accessKey;
7+
private $secretKey;
8+
private $region;
9+
private $httpMethod = 'GET';
10+
private $path = '/';
11+
private $params = array();
12+
private $endpoint;
1113

1214
abstract public function getServiceName();
1315

1416
public function __construct() {
1517
parent::__construct(null);
1618
}
1719

18-
public function setAWSKeys($access, $private) {
19-
$this->awsAccessKey = $access;
20-
$this->awsPrivateKey = $private;
20+
public function setAccessKey($access_key) {
21+
$this->accessKey = $access_key;
2122
return $this;
2223
}
2324

24-
public function getAWSAccessKey() {
25-
return $this->awsAccessKey;
25+
public function getAccessKey() {
26+
return $this->accessKey;
2627
}
2728

28-
public function getAWSPrivateKey() {
29-
return $this->awsPrivateKey;
29+
public function setSecretKey(PhutilOpaqueEnvelope $secret_key) {
30+
$this->secretKey = $secret_key;
31+
return $this;
32+
}
33+
34+
public function getSecretKey() {
35+
return $this->secretKey;
3036
}
3137

32-
public function getAWSRegion() {
33-
return $this->awsRegion;
38+
public function getRegion() {
39+
return $this->region;
3440
}
3541

36-
public function setAWSRegion($region) {
37-
$this->awsRegion = $region;
42+
public function setRegion($region) {
43+
$this->region = $region;
3844
return $this;
3945
}
4046

41-
public function getHost() {
42-
$host = $this->getServiceName().'.'.$this->awsRegion.'.amazonaws.com';
43-
return $host;
47+
public function setEndpoint($endpoint) {
48+
$this->endpoint = $endpoint;
49+
return $this;
4450
}
4551

46-
public function setRawAWSQuery($action, array $params = array()) {
47-
$this->params = $params;
48-
$this->params['Action'] = $action;
52+
public function getEndpoint() {
53+
return $this->endpoint;
54+
}
55+
56+
public function setHTTPMethod($method) {
57+
$this->httpMethod = $method;
4958
return $this;
5059
}
5160

52-
protected function getProxiedFuture() {
53-
if (!$this->future) {
54-
$params = $this->params;
61+
public function getHTTPMethod() {
62+
return $this->httpMethod;
63+
}
5564

56-
if (!$this->params) {
57-
throw new Exception(
58-
pht(
59-
'You must %s!',
60-
'setRawAWSQuery()'));
61-
}
65+
public function setPath($path) {
66+
$this->path = $path;
67+
return $this;
68+
}
6269

63-
if (!$this->getAWSAccessKey()) {
64-
throw new Exception(
65-
pht(
66-
'You must %s!',
67-
'setAWSKeys()'));
68-
}
70+
public function getPath() {
71+
return $this->path;
72+
}
6973

70-
$params['AWSAccessKeyId'] = $this->getAWSAccessKey();
71-
$params['Version'] = '2013-10-15';
72-
$params['Timestamp'] = date('c');
74+
protected function getParameters() {
75+
$params = $this->params;
76+
return $params;
77+
}
7378

74-
$params = $this->sign($params);
79+
protected function getProxiedFuture() {
80+
if (!$this->future) {
81+
$params = $this->getParameters();
82+
$method = $this->getHTTPMethod();
83+
$host = $this->getEndpoint();
84+
$path = $this->getPath();
7585

76-
$uri = new PhutilURI('http://'.$this->getHost().'/');
77-
$uri->setQueryParams($params);
86+
$uri = id(new PhutilURI("https://{$host}/"))
87+
->setPath($path)
88+
->setQueryParams($params);
7889

79-
$this->future = new HTTPFuture($uri);
90+
$future = id(new HTTPSFuture($uri))
91+
->setMethod($method);
92+
93+
$this->signRequest($future);
94+
95+
$this->future = $future;
8096
}
8197

8298
return $this->future;
8399
}
84100

101+
protected function signRequest(HTTPSFuture $future) {
102+
$access_key = $this->getAccessKey();
103+
$secret_key = $this->getSecretKey();
104+
105+
$region = $this->getRegion();
106+
107+
id(new PhutilAWSv4Signature())
108+
->setRegion($region)
109+
->setService($this->getServiceName())
110+
->setAccessKey($access_key)
111+
->setSecretKey($secret_key)
112+
->signRequest($future);
113+
}
114+
85115
protected function didReceiveResult($result) {
86116
list($status, $body, $headers) = $result;
87117

@@ -101,7 +131,8 @@ protected function didReceiveResult($result) {
101131
);
102132
if ($xml) {
103133
$params['RequestID'] = $xml->RequestID[0];
104-
foreach ($xml->Errors[0] as $error) {
134+
$errors = array($xml->Error);
135+
foreach ($errors as $error) {
105136
$params['Errors'][] = array($error->Code, $error->Message);
106137
}
107138
}
@@ -112,36 +143,4 @@ protected function didReceiveResult($result) {
112143
return $xml;
113144
}
114145

115-
/**
116-
* http://bit.ly/wU0JFh
117-
*/
118-
private function sign(array $params) {
119-
120-
$params['SignatureMethod'] = 'HmacSHA256';
121-
$params['SignatureVersion'] = '2';
122-
123-
ksort($params);
124-
125-
$pstr = array();
126-
foreach ($params as $key => $value) {
127-
$pstr[] = rawurlencode($key).'='.rawurlencode($value);
128-
}
129-
$pstr = implode('&', $pstr);
130-
131-
$sign = "GET"."\n".
132-
strtolower($this->getHost())."\n".
133-
"/"."\n".
134-
$pstr;
135-
136-
$hash = hash_hmac(
137-
'sha256',
138-
$sign,
139-
$this->getAWSPrivateKey(),
140-
$raw_ouput = true);
141-
142-
$params['Signature'] = base64_encode($hash);
143-
144-
return $params;
145-
}
146-
147146
}

‎src/future/aws/PhutilAWSS3Future.php

+34
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,42 @@
22

33
final class PhutilAWSS3Future extends PhutilAWSFuture {
44

5+
private $bucket;
6+
57
public function getServiceName() {
68
return 's3';
79
}
810

11+
public function setBucket($bucket) {
12+
$this->bucket = $bucket;
13+
return $this;
14+
}
15+
16+
public function getBucket() {
17+
return $this->bucket;
18+
}
19+
20+
public function setParametersForGetObject($key) {
21+
$bucket = $this->getBucket();
22+
23+
$this->setHTTPMethod('GET');
24+
$this->setPath($bucket.'/'.$key);
25+
26+
return $this;
27+
}
28+
29+
protected function didReceiveResult($result) {
30+
list($status, $body, $headers) = $result;
31+
32+
if (!$status->isError()) {
33+
return $body;
34+
}
35+
36+
if ($status->getStatusCode() === 404) {
37+
return null;
38+
}
39+
40+
return parent::didReceiveResult($result);
41+
}
42+
943
}

‎src/future/aws/PhutilAWSv4Signature.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public function setDate($date) {
2828

2929
public function getDate() {
3030
if ($this->date === null) {
31-
$this->date = date('c');
31+
$this->date = gmdate('Ymd\THis\Z', time());
3232
}
3333
return $this->date;
3434
}

‎src/future/aws/management/PhutilAWSManagementWorkflow.php

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
<?php
2+
3+
abstract class PhutilAWSManagementWorkflow
4+
extends PhutilArgumentWorkflow {
5+
6+
public function isExecutable() {
7+
return true;
8+
}
9+
10+
protected function newAWSFuture($template) {
11+
$argv = $this->getArgv();
12+
13+
$access_key = $argv->getArg('access-key');
14+
$secret_key = $argv->getArg('secret-key');
15+
16+
$has_root = (strlen($access_key) || strlen($secret_key));
17+
if ($has_root) {
18+
if (!strlen($access_key) || !strlen($secret_key)) {
19+
throw new PhutilArgumentUsageException(
20+
pht(
21+
'When specifying AWS credentials with --access-key and '.
22+
'--secret-key, you must provide both keys.'));
23+
}
24+
25+
$template->setAccessKey($access_key);
26+
$template->setSecretKey(new PhutilOpaqueEnvelope($secret_key));
27+
}
28+
29+
$has_any = ($has_root);
30+
if (!$has_any) {
31+
throw new PhutilArgumentUsageException(
32+
pht(
33+
'You must specify AWS credentials. Use --access-key and '.
34+
'--secret-key to provide root credentials.'));
35+
}
36+
37+
$region = $argv->getArg('region');
38+
if (!strlen($region)) {
39+
throw new PhutilArgumentUsageException(
40+
pht(
41+
'You must specify an AWS region with --region.'));
42+
}
43+
44+
$template->setRegion($region);
45+
46+
return $template;
47+
}
48+
49+
protected function getAWSArguments() {
50+
return array(
51+
array(
52+
'name' => 'access-key',
53+
'param' => 'key',
54+
'help' => pht('AWS access key.'),
55+
),
56+
array(
57+
'name' => 'secret-key',
58+
'param' => 'file',
59+
'help' => pht('AWS secret key.'),
60+
),
61+
array(
62+
'name' => 'region',
63+
'param' => 'region',
64+
'help' => pht('AWS region.'),
65+
),
66+
);
67+
}
68+
69+
}

0 commit comments

Comments
 (0)