# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT-0
AWSTemplateFormatVersion: 2010-09-09
Description: Companion infrastructure for M2c/Neptune blog
Parameters:
Env:
Description: "Environment tag, e.g. prod, nonprod."
Default: test
Type: String
AllowedPattern: "[a-z0-9]+"
MaxLength: 15
DbInstanceType:
Description: Neptune DB instance type
Type: String
Default: db.r5.xlarge
IamAuthEnabled:
Type: String
Default: "true"
AllowedValues:
- "true"
- "false"
Description: Enable IAM Auth for Neptune.
NeptuneEnableAuditLog:
Type: Number
Default: 0
AllowedValues:
- 0
- 1
Description: Enable Audit Log. 0 means disable and 1 means enable.
NotebookInstanceType:
Description: >-
SageMaker Notebook instance type. Please refer
https://aws.amazon.com/sagemaker/pricing/ for uptodate allowed instance
type in aws region and https://aws.amazon.com/neptune/pricing/ for
pricing.
Type: String
Default: ml.t3.medium
ConstraintDescription: Must be a valid SageMaker instance type.
M2CAnalysisBucket:
Description: Proxy/analysis bucket of M2C
Type: String
MinLength: 1
Resources:
S3Bucket:
Type: AWS::S3::Bucket
DeletionPolicy: Delete
Properties:
BucketEncryption:
ServerSideEncryptionConfiguration:
- ServerSideEncryptionByDefault:
SSEAlgorithm: 'AES256'
Tags:
- Key: demoname
Value: neptune-m2c-kg
NeptuneStack:
Type: "AWS::CloudFormation::Stack"
Properties:
TemplateURL: "https://aws-neptune-customer-samples.s3.amazonaws.com/v2/cloudformation-templates/neptune-full-stack-nested-template.json"
TimeoutInMinutes: "60"
Parameters:
DBClusterId: !Join [ "-", ["m2ckg", !Join [ "", !Split [ "-", !Select [ 2, !Split [ "/", !Ref AWS::StackId ] ] ] ] ] ]
DbInstanceType: !Ref DbInstanceType
Env: !Ref Env
IamAuthEnabled: !Ref IamAuthEnabled
EC2ClientInstanceType: none
NotebookInstanceType: !Ref NotebookInstanceType
NeptuneEnableAuditLog: !Ref NeptuneEnableAuditLog
NeptuneSagemakerNotebookStartupScript: !GetAtt Repo2S3.NotebookAddScript
DependsOn: Repo2S3
Repo2S3:
Type: 'Custom::EnvSetup'
DependsOn: AWSLambdaExecutionRole
Properties:
ServiceToken: !GetAtt Repo2S3Func.Arn
staging_bucket: !Ref S3Bucket
m2c_analysis_bucket: !Ref M2CAnalysisBucket
Repo2S3Func:
Type: "AWS::Lambda::Function"
Properties:
Description: "Repo2S3Func"
FunctionName: !Sub "Repo2S3Func-lambda-${AWS::StackName}"
Handler: index.lambda_handler
Role: !GetAtt AWSLambdaExecutionRole.Arn
Timeout: 360
Tags:
- Key: demoname
Value: neptune-m2c-kg
Runtime: python3.9
Code:
ZipFile: |
import json
import urllib3
import boto3
import cfnresponse
http = urllib3.PoolManager()
DATA_FILES = {
"data": ["seeddata.ttl"],
"notebook": ["M2CForKnowledgeGraph.ipynb"]
}
def copy_s3_s3(s3_client, source_bucket, target_bucket, folder, key):
s3_path = folder+"/"+key
local_path = "/tmp/" + key
s3_client.download_file(source_bucket, s3_path, local_path)
s3_client.upload_file(Bucket = target_bucket, Key = s3_path, Filename = local_path, ExtraArgs={"ServerSideEncryption": "AES256"})
def copy_web_s3(url, s3_client, bucket, folder, key):
s3_path = folder+"/"+key
local_path = "/tmp/" + key
fileurl = url + "/" + folder + "/" + key
print("loading web file *" + fileurl + "*")
the_file = http.request('GET',fileurl, preload_content=False)
print("Got file")
local_file = open(local_path, 'wb')
local_file.write(the_file.data)
local_file.close()
print("upload *" + s3_path + "*" + local_path + "*")
s3_client.upload_file(Bucket = bucket, Key = s3_path, Filename = local_path, ExtraArgs={"ServerSideEncryption": "AES256"})
def lambda_handler(event, context):
the_event = event['RequestType']
print(event)
print("The event type is: ", str(the_event))
response_data = {}
try:
staging_bucket = event['ResourceProperties']['staging_bucket']
m2c_bucket = event['ResourceProperties']['m2c_analysis_bucket']
if the_event in ('Create', 'Update'):
s3c = boto3.client('s3')
print("Creating/Updating")
for folder in DATA_FILES:
for filename in DATA_FILES[folder]:
#copy_s3_s3(s3c, source_bucket, staging_bucket, folder, filename)
copy_web_s3("https://raw.githubusercontent.com/aws-samples/neptune-m2c-kg/main", s3c, staging_bucket, folder, filename)
#copy_web_s3("https://github.com/aws-samples/neptune-m2c-kg", s3c, staging_bucket, folder, filename)
response_data['Data'] = 'git success'
response_data['NotebookAddScript'] = f'echo "export STAGE_BUCKET={staging_bucket}" >> ~/.bashrc\n'
response_data['NotebookAddScript'] += f'echo "export M2C_ANALYSIS_BUCKET={m2c_bucket}" >> ~/.bashrc\n'
response_data['NotebookAddScript'] += f"if [ ! -f /home/ec2-user/SageMaker/M2CForKnowledgeGraph.ipynb ]\n"
response_data['NotebookAddScript'] += f"then\n"
response_data['NotebookAddScript'] += f" aws s3 cp s3://{staging_bucket}/notebook/M2CForKnowledgeGraph.ipynb /home/ec2-user/SageMaker/M2CForKnowledgeGraph.ipynb\n"
response_data['NotebookAddScript'] += f"fi\n"
response_data['Data'] = 'git success'
cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data)
elif the_event in ('Delete'):
print("Deleting")
s3r = boto3.resource('s3')
s3r.Bucket(str(staging_bucket)).objects.all().delete()
cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data)
else:
response_data['Data'] = "Illegal event " + the_event
cfnresponse.send(event, context, cfnresponse.FAILED, response_data)
except Exception as e:
print("Operation failed...")
print(str(e))
response_data['Data'] = str(e)
cfnresponse.send(event, context, cfnresponse.FAILED, response_data)
AWSLambdaExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Statement:
- Action:
- sts:AssumeRole
Effect: Allow
Principal:
Service:
- lambda.amazonaws.com
Version: "2012-10-17"
Path: "/"
Policies:
- PolicyDocument:
Statement:
- Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Effect: Allow
Resource: arn:aws:logs:*:*:*
Version: "2012-10-17"
PolicyName: !Sub "Repo2S3pol-CW-${AWS::StackName}"
- PolicyDocument:
Statement:
- Action:
- s3:GetObject
- s3:PutObject
- s3:DeleteObject
- s3:List*
Effect: Allow
Resource:
- !Sub arn:aws:s3:::${S3Bucket}/*
- !Sub arn:aws:s3:::${S3Bucket}
- !Sub arn:aws:s3:::${M2CAnalysisBucket}/*
- !Sub arn:aws:s3:::${M2CAnalysisBucket}
Version: "2012-10-17"
PolicyName: !Sub "Repo2S3pol-S3-${AWS::StackName}"
RoleName: !Sub "Repo2S3role-${AWS::StackName}"
Outputs:
S3Bucket:
Value: !Ref S3Bucket
DBClusterEndpoint:
Value: !GetAtt
- NeptuneStack
- Outputs.DBClusterEndpoint
DBClusterId:
Value: !GetAtt
- NeptuneStack
- Outputs.DBClusterId
DBClusterPort:
Value: !GetAtt
- NeptuneStack
- Outputs.DBClusterPort
DBClusterResourceId:
Value: !GetAtt
- NeptuneStack
- Outputs.DBClusterResourceId
NeptuneLoadFromS3IAMRoleArn:
Value: !GetAtt
- NeptuneStack
- Outputs.NeptuneLoadFromS3IAMRoleArn
NeptuneSagemakerNotebook:
Value: !GetAtt
- NeptuneStack
- Outputs.NeptuneSagemakerNotebook
Subnet1:
Value: !GetAtt
- NeptuneStack
- Outputs.Subnet1
Subnet2:
Value: !GetAtt
- NeptuneStack
- Outputs.Subnet2
Subnet4:
Value: !GetAtt
- NeptuneStack
- Outputs.Subnet4
VPC:
Value: !GetAtt
- NeptuneStack
- Outputs.VPC