Skip to content

Commit 8d2b125

Browse files
giuseppe-zappiagiuseppe-zappia
authored
Merge pull request #90 from abijango/main
Updated Readme and CloudFormation templates and Agent Eval Notebook
2 parents 6334d6c + db3adbf commit 8d2b125

File tree

14 files changed

+3166
-305
lines changed

14 files changed

+3166
-305
lines changed

‎README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ This repository provides comprehensive resources for working with generative AI
3232
- Implement responsible AI with Bedrock Guardrails
3333
- Develop FMOps fine-tuning workflows with SageMaker Pipelines
3434

35+
#### [Partner AI Apps with SageMaker AI](./workshops/partner-ai-apps-with-sagemakerai/)
36+
- Experiment Management with Comet [Image Classification, Fraud Detection]
37+
- Evaluting LLM applications with Comet Opik
38+
- Evaluating Agents with Opik
39+
- LLM Evaluation with Comet
40+
- Model Monitoring with Fiddler
41+
- RAG chatbot evaluation with Deepchecks
42+
3543
## Getting Started
3644

3745
1. Clone this repository

‎workshops/partner-ai-apps-with-sagemakerai/cloudformation/comet.yaml‎

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
AWSTemplateFormatVersion: '2010-09-09'
2+
Description: 'Deploy Comet Partner AI App in SageMaker with required IAM roles and permissions'
3+
4+
Parameters:
5+
AppName:
6+
Type: String
7+
Default: 'comet'
8+
Description: 'Name for the Comet Partner AI App (alphanumeric only)'
9+
AllowedPattern: '^[a-zA-Z0-9]+$'
10+
MinLength: 1
11+
MaxLength: 256
12+
13+
AppTier:
14+
Type: String
15+
Default: 'comet.large'
16+
Description: 'Tier for the Comet app (impacts speed and capabilities)'
17+
AllowedValues:
18+
- 'comet.small'
19+
- 'comet.medium'
20+
- 'comet.large'
21+
22+
23+
24+
EnableIamSessionBasedIdentity:
25+
Type: String
26+
Default: 'true'
27+
Description: 'Enable IAM session-based identity propagation'
28+
AllowedValues:
29+
- 'true'
30+
- 'false'
31+
32+
Conditions:
33+
EnableSessionIdentity: !Equals [!Ref EnableIamSessionBasedIdentity, 'true']
34+
35+
Resources:
36+
# IAM Role for Partner AI App Execution
37+
PartnerAiAppExecutionRole:
38+
Type: AWS::IAM::Role
39+
Properties:
40+
RoleName: !Sub '${AppName}-execution-role'
41+
AssumeRolePolicyDocument:
42+
Version: '2012-10-17'
43+
Statement:
44+
- Effect: Allow
45+
Principal:
46+
Service: sagemaker.amazonaws.com
47+
Action:
48+
- sts:AssumeRole
49+
- sts:TagSession
50+
Policies:
51+
- PolicyName: LicenseManagerPolicy
52+
PolicyDocument:
53+
Version: '2012-10-17'
54+
Statement:
55+
- Effect: Allow
56+
Action:
57+
- license-manager:CheckoutLicense
58+
- license-manager:CheckInLicense
59+
- license-manager:ExtendLicenseConsumption
60+
- license-manager:GetLicense
61+
- license-manager:GetLicenseUsage
62+
Resource: '*'
63+
- PolicyName: S3AccessPolicy
64+
PolicyDocument:
65+
Version: '2012-10-17'
66+
Statement:
67+
- Effect: Allow
68+
Action:
69+
- s3:GetObject
70+
- s3:PutObject
71+
- s3:DeleteObject
72+
- s3:ListBucket
73+
Resource:
74+
- !Sub 'arn:aws:s3:::${AppName}-data-${AWS::AccountId}-${AWS::Region}/*'
75+
- !Sub 'arn:aws:s3:::${AppName}-data-${AWS::AccountId}-${AWS::Region}'
76+
- PolicyName: BedrockAccessPolicy
77+
PolicyDocument:
78+
Version: '2012-10-17'
79+
Statement:
80+
- Effect: Allow
81+
Action:
82+
- bedrock:InvokeModel
83+
- bedrock:InvokeModelWithResponseStream
84+
- bedrock:GetFoundationModel
85+
- bedrock:ListFoundationModels
86+
Resource: '*'
87+
88+
# IAM Role for Admin Users
89+
PartnerAppAdminRole:
90+
Type: AWS::IAM::Role
91+
Properties:
92+
RoleName: !Sub '${AppName}-admin-role'
93+
AssumeRolePolicyDocument:
94+
Version: '2012-10-17'
95+
Statement:
96+
- Effect: Allow
97+
Principal:
98+
AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
99+
Action: sts:AssumeRole
100+
ManagedPolicyArns:
101+
- arn:aws:iam::aws:policy/AWSMarketplaceManageSubscriptions
102+
Policies:
103+
- PolicyName: PartnerAppAdminPolicy
104+
PolicyDocument:
105+
Version: '2012-10-17'
106+
Statement:
107+
- Effect: Allow
108+
Action:
109+
- sagemaker:CreatePartnerApp
110+
- sagemaker:DeletePartnerApp
111+
- sagemaker:UpdatePartnerApp
112+
- sagemaker:DescribePartnerApp
113+
- sagemaker:ListPartnerApps
114+
- sagemaker:CreatePartnerAppPresignedUrl
115+
- sagemaker:AddTags
116+
- sagemaker:ListTags
117+
- sagemaker:DeleteTags
118+
Resource: '*'
119+
- Effect: Allow
120+
Action:
121+
- iam:PassRole
122+
Resource: !GetAtt PartnerAiAppExecutionRole.Arn
123+
Condition:
124+
StringEquals:
125+
'iam:PassedToService': 'sagemaker.amazonaws.com'
126+
127+
# IAM Role for End Users
128+
PartnerAppUserRole:
129+
Type: AWS::IAM::Role
130+
Properties:
131+
RoleName: !Sub '${AppName}-user-role'
132+
AssumeRolePolicyDocument:
133+
Version: '2012-10-17'
134+
Statement:
135+
- Effect: Allow
136+
Principal:
137+
AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
138+
Action:
139+
- sts:AssumeRole
140+
- sts:TagSession
141+
Policies:
142+
- PolicyName: PartnerAppUserPolicy
143+
PolicyDocument:
144+
Version: '2012-10-17'
145+
Statement:
146+
- Effect: Allow
147+
Action:
148+
- sagemaker:DescribePartnerApp
149+
- sagemaker:ListPartnerApps
150+
- sagemaker:CreatePartnerAppPresignedUrl
151+
- sagemaker:CallPartnerAppApi
152+
Resource: !Sub 'arn:aws:sagemaker:${AWS::Region}:${AWS::AccountId}:partner-app/app-*'
153+
154+
# S3 Bucket for Comet data
155+
S3Bucket:
156+
Type: AWS::S3::Bucket
157+
Properties:
158+
BucketName: !Sub '${AppName}-data-${AWS::AccountId}-${AWS::Region}'
159+
PublicAccessBlockConfiguration:
160+
BlockPublicAcls: true
161+
BlockPublicPolicy: true
162+
IgnorePublicAcls: true
163+
RestrictPublicBuckets: true
164+
BucketEncryption:
165+
ServerSideEncryptionConfiguration:
166+
- ServerSideEncryptionByDefault:
167+
SSEAlgorithm: AES256
168+
169+
# KMS Key for encryption
170+
KMSKey:
171+
Type: AWS::KMS::Key
172+
Properties:
173+
Description: 'KMS Key for Comet Partner AI App encryption'
174+
KeyPolicy:
175+
Version: '2012-10-17'
176+
Statement:
177+
- Sid: Enable IAM User Permissions
178+
Effect: Allow
179+
Principal:
180+
AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
181+
Action: 'kms:*'
182+
Resource: '*'
183+
- Sid: Allow SageMaker Service
184+
Effect: Allow
185+
Principal:
186+
Service: sagemaker.amazonaws.com
187+
Action:
188+
- kms:Decrypt
189+
- kms:GenerateDataKey
190+
Resource: '*'
191+
192+
KMSKeyAlias:
193+
Type: AWS::KMS::Alias
194+
Properties:
195+
AliasName: !Sub 'alias/${AppName}-key'
196+
TargetKeyId: !Ref KMSKey
197+
198+
# Comet Partner AI App
199+
CometPartnerApplication:
200+
Type: AWS::SageMaker::PartnerApp
201+
Properties:
202+
Name: !Ref AppName
203+
Type: 'comet'
204+
AuthType: 'IAM'
205+
ExecutionRoleArn: !GetAtt PartnerAiAppExecutionRole.Arn
206+
Tier: !Ref AppTier
207+
KmsKeyId: !Ref KMSKey
208+
ApplicationConfig:
209+
AdminUsers:
210+
- 'nqmir-Isengard'
211+
EnableIamSessionBasedIdentity: !If [EnableSessionIdentity, true, false]
212+
Tags:
213+
- Key: 'Application'
214+
Value: 'Comet'
215+
- Key: 'Environment'
216+
Value: 'Production'
217+
- Key: 'ManagedBy'
218+
Value: 'CloudFormation'
219+
220+
Outputs:
221+
PartnerAppArn:
222+
Description: 'ARN of the Comet Partner AI App'
223+
Value: !GetAtt CometPartnerApplication.Arn
224+
Export:
225+
Name: !Sub '${AWS::StackName}-PartnerAppArn'
226+
227+
PartnerAppBaseUrl:
228+
Description: 'Base URL of the Comet Partner AI App'
229+
Value: !GetAtt CometPartnerApplication.BaseUrl
230+
Export:
231+
Name: !Sub '${AWS::StackName}-PartnerAppBaseUrl'
232+
233+
ExecutionRoleArn:
234+
Description: 'ARN of the Partner AI App execution role'
235+
Value: !GetAtt PartnerAiAppExecutionRole.Arn
236+
Export:
237+
Name: !Sub '${AWS::StackName}-ExecutionRoleArn'
238+
239+
AdminRoleArn:
240+
Description: 'ARN of the Partner AI App admin role'
241+
Value: !GetAtt PartnerAppAdminRole.Arn
242+
Export:
243+
Name: !Sub '${AWS::StackName}-AdminRoleArn'
244+
245+
UserRoleArn:
246+
Description: 'ARN of the Partner AI App user role'
247+
Value: !GetAtt PartnerAppUserRole.Arn
248+
Export:
249+
Name: !Sub '${AWS::StackName}-UserRoleArn'
250+
251+
S3BucketName:
252+
Description: 'Name of the S3 bucket for Comet data'
253+
Value: !Ref S3Bucket
254+
Export:
255+
Name: !Sub '${AWS::StackName}-S3BucketName'
256+
257+
KMSKeyId:
258+
Description: 'KMS Key ID for encryption'
259+
Value: !Ref KMSKey
260+
Export:
261+
Name: !Sub '${AWS::StackName}-KMSKeyId'

0 commit comments

Comments
 (0)