Adding BindingDB. Adding new RDS source format.

Author: EC2 Default User

DeployChemblOpenTargetsEnv.sh

@@ -2,9 +2,12 @@
 npm run build
 cdk bootstrap
 currentPrincipalArn=$(aws sts get-caller-identity --query Arn --output text)
+#Just in case you are using an IAM role, we will switch the identity from your STS arn to the underlying role ARN.
+currentPrincipalArn=$(sed 's/\(sts\)\(.*\)\(assumed-role\)\(.*\)\(\/.*\)/iam\2role\4/' <<< $currentPrincipalArn)
 jq '.context.starterLakeFormationAdmin = $currentPrincipalArn' --arg currentPrincipalArn $currentPrincipalArn cdk.json > tmp.$$.json && mv tmp.$$.json cdk.json
 cdk deploy BaselineStack --require-approval never
 cdk deploy CoreDataLake --require-approval never
 cdk deploy ChemblStack --require-approval never
 cdk deploy OpenTargetsStack --require-approval never
+cdk deploy BindingDbStack --require-approval never
 cdk deploy AnalyticsStack --require-approval never

RODA_templates/empty.ts

@@ -0,0 +1,19 @@
+aws glue get-tables --database-name opentargets_1911_dl > RODA_templates/open_targets_1911_get_tables.json
+aws glue get-database --name opentargets_1911_dl > RODA_templates/open_targets_1911_get_database.json
+npm run build && cdk synth OpenTargetsRodaTemplate
+aws s3 cp cdk.out/OpenTargetsRodaTemplate.template.json s3://aws-roda-hcls-datalake/OpenTargetsRodaTemplate.json
+
+
+aws glue get-tables --database-name chembl_25_dl > RODA_templates/chembl_25_get_tables.json
+aws glue get-database --name chembl_25_dl > RODA_templates/chembl_25_get_database.json
+npm run build && cdk synth ChemblRodaTemplate
+aws s3 cp cdk.out/ChemblRodaTemplate.template.json s3://aws-roda-hcls-datalake/ChemblRodaTemplate.json
+
+aws glue get-tables --database-name binding_db_dl > RODA_templates/binding_db_get_tables.json
+aws glue get-database --name binding_db_dl > RODA_templates/binding_db_get_database.json
+npm run build && cdk synth BindingDbRodaTemplate
+aws s3 cp cdk.out/BindingDbRodaTemplate.template.json s3://aws-roda-hcls-datalake/BindingDbRodaTemplate.json
+
+#https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateUrl=https%3A%2F%2Faws-roda-hcls-datalake.s3.amazonaws.com%2FChemblRodaTemplate.json&stackName=Chembl25-RODA
+#https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateUrl=https%3A%2F%2Faws-roda-hcls-datalake.s3.amazonaws.com%2FOpenTargetsRodaTemplate.json&stackName=OpenTargets-1911-RODA
+#https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateUrl=https%3A%2F%2Faws-roda-hcls-datalake.s3.amazonaws.com%BindingDbRodaTemplate.json&stackName=BindingDB-RODA

SynthRodaTemplates.sh

@@ -5,10 +5,13 @@ import { BaselineStack } from '../lib/baseline-stack';
 import { DataLakeStack } from '../lib/stacks/datalake-stack';
 import { OpenTargetsStack } from '../lib/opentargets-stack';
 import { ChemblStack } from '../lib/chembl-25-stack';
-import { AnalyticsStack } from '../lib/analytics-stack.js';
+import { BindingDBStack } from '../lib/bindingdb-stack';
+import { AnalyticsStack } from '../lib/analytics-stack';
 import iam = require('@aws-cdk/aws-iam');
 import s3 = require('@aws-cdk/aws-s3');
 import { DataLakeEnrollment } from '../lib/constructs/data-lake-enrollment';
+import { DataSetTemplateStack } from '../lib/stacks/dataset-stack';
+
 
 const app = new cdk.App();
 const baseline = new BaselineStack(app, 'BaselineStack');
@@ -31,22 +34,42 @@ const openTargetsStack = new OpenTargetsStack(app, 'OpenTargetsStack', {
     DataLake: coreDataLake
 });
 
+console.log('basic before binding dbstack');
+
+const bindingDBStack = new BindingDBStack(app, 'BindingDbStack', {
+    database: baseline.BindingDb,
+    accessSecurityGroup: baseline.BindingDBAccessSg,
+    databaseSecret: baseline.BindingDBSecret,
+    DataLake: coreDataLake
+});
+
 const analyticsStack = new AnalyticsStack(app, 'AnalyticsStack', {
     targetVpc: baseline.Vpc,
 });
 
 
+// chemblStack.grantIamRead(analyticsStack.NotebookRole);
+// openTargetsStack.grantIamRead(analyticsStack.NotebookRole);
+// bindingDBStack.grantIamRead(analyticsStack.NotebookRole);
 
 
+const OpenTargetsRodaTemplate = new DataSetTemplateStack(app, 'OpenTargetsRodaTemplate', {
+    DatabaseDescriptionPath: "../../RODA_templates/open_targets_1911_get_database.json",
+    DescribeTablesPath: "../../RODA_templates/open_targets_1911_get_tables.json",
+    DataSetName: openTargetsStack.Enrollment.DataSetName
+});
 
-chemblStack.grantIamRead(analyticsStack.NotebookRole);
-openTargetsStack.grantIamRead(analyticsStack.NotebookRole);
-
-
-
-
-
+const ChemblRodaTemplate = new DataSetTemplateStack(app, 'ChemblRodaTemplate', {
+    DatabaseDescriptionPath: "../../RODA_templates/chembl_25_get_database.json",
+    DescribeTablesPath: "../../RODA_templates/chembl_25_get_tables.json",
+    DataSetName: chemblStack.Enrollment.DataSetName
+});
 
+const BindinbDbRodaTemplate = new DataSetTemplateStack(app, 'BindingDbRodaTemplate', {
+    DatabaseDescriptionPath: "../../RODA_templates/binding_db_get_database.json",
+    DescribeTablesPath: "../../RODA_templates/binding_db_get_tables.json",
+    DataSetName: bindingDBStack.Enrollment.DataSetName
+});
 
 
 // const exampleUser = iam.User.fromUserName(coreDataLake, 'exampleGrantee', 'paul1' );

bin/aws.ts

@@ -4,6 +4,7 @@ import iam = require('@aws-cdk/aws-iam');
 import rds = require('@aws-cdk/aws-rds');
 import ssm = require('@aws-cdk/aws-ssm');
 import s3 = require('@aws-cdk/aws-s3');
+import s3assets = require('@aws-cdk/aws-s3-assets');
 import fs = require('fs');
 
 
@@ -16,6 +17,10 @@ export class BaselineStack extends cdk.Stack {
     public readonly chemblDBSecret: rds.DatabaseSecret; 
     public readonly OpenTargetsSourceBucket: s3.Bucket; 
     public readonly Vpc: ec2.Vpc;
+    public readonly BindingDBSourceBucket: s3.Bucket;
+    public readonly BindingDb: rds.DatabaseInstance;
+    public readonly BindingDBAccessSg: ec2.SecurityGroup;
+    public readonly BindingDBSecret: rds.DatabaseSecret;
 
     constructor(scope: cdk.Construct, id: string, props?: cdk.StackProps) {
 
@@ -166,6 +171,119 @@ export class BaselineStack extends cdk.Stack {
             openTargetsSourceFileTargetBucketLocation: [openTargetsBucket.bucketName]
         });
 
+        //// Start Binding DB  ////
+        
+        const bindingDbAccessSg = new ec2.SecurityGroup(this, 'bindingDbAccessSg', {
+            vpc: baselineVpc,
+            allowAllOutbound: true,
+            description: "Grants access to the BindingDB rds instance",
+            securityGroupName: "BindingDBAccessSecurityGroup"
+        });
+        
+        this.BindingDBSourceBucket = new s3.Bucket(this, 'BindingDbSourceBucket');
+        
+        this.BindingDBAccessSg = bindingDbAccessSg;
+        
+        const bindingDbSg = new ec2.SecurityGroup(this, 'bindingDbSg', {
+            vpc: baselineVpc,
+            allowAllOutbound: true,
+            description: "Security group for binding dbs",
+            securityGroupName: "BindingDbSecurityGroup"
+        });
+        
+        
+
+        
+        bindingDbAccessSg.addIngressRule( bindingDbAccessSg , ec2.Port.allTraffic(),  "Recursive SG rule for Glue" );
+        
+        bindingDbSg.addIngressRule( bindingDbAccessSg , ec2.Port.tcp(1512),  "Gives BindingDB access security group access to oracle port" );
+        
+        importInstance.addSecurityGroup(bindingDbAccessSg);
+        
+        
+        const bindingDBSecret = new rds.DatabaseSecret(this, 'bindingDbSecret', {
+            username: 'master',
+        });
+        this.BindingDBSecret = bindingDBSecret;
+        
+        bindingDBSecret.grantRead(importInstanceRole);
+        this.BindingDBSourceBucket.grantReadWrite(importInstanceRole);
+        
+        
+        const bindingDbOptionGroup = new rds.OptionGroup(this, 'bindingDbRdsOptionGroup',{
+            engine: rds.DatabaseInstanceEngine.oracleSe2({
+                version: rds.OracleEngineVersion.VER_19, // different version class for each engine type
+            }),
+            description: "Binding DB Option Group",
+            configurations: [{
+                name: "S3_INTEGRATION",
+                version: "1.0"
+            }],
+        });
+        
+        const bindingDb = new rds.DatabaseInstance(this, 'bindingDb', {
+            engine: rds.DatabaseInstanceEngine.ORACLE_SE2,
+            masterUsername: 'master',
+            licenseModel: rds.LicenseModel.BRING_YOUR_OWN_LICENSE,
+            vpc: baselineVpc,
+            vpcPlacement: appSubnetSelection, 
+            optionGroup: bindingDbOptionGroup,
+            instanceType: ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize.SMALL),
+            instanceIdentifier: 'binding-db',
+            masterUserPassword: bindingDBSecret.secretValueFromJson('password'),
+            securityGroups: [bindingDbSg, bindingDbAccessSg],
+            deletionProtection: false,
+        });
+        this.BindingDb = bindingDb;
+        
+        var bindingDbCfnDb = this.BindingDb.node.defaultChild as rds.CfnDBInstance;
+        
+        const bindingDbRdsImportRole = new iam.Role(this, 'BindingDbRdsInstanceRole', {
+            assumedBy: new iam.ServicePrincipal('rds.amazonaws.com')
+        });
+        this.BindingDBSourceBucket.grantReadWrite(bindingDbRdsImportRole);
+        
+        bindingDbCfnDb.associatedRoles = [{
+           featureName: "S3_INTEGRATION",
+           roleArn: bindingDbRdsImportRole.roleArn
+        }];
+        
+        
+        const loadBindingDbDoc = new ssm.CfnDocument(this, 'loadBindingDbDoc', {
+            content: JSON.parse(fs.readFileSync('scripts/ssmdoc.importbindingdb.json', { encoding: 'utf-8' })),
+            documentType: "Command"
+        });
+
+
+        const instantClientBasic = new s3assets.Asset(this, `instantClientBasicRpm`, {
+			path: "oracle-instantclient19.8-basic-19.8.0.0.0-1.x86_64.rpm"
+		});
+		instantClientBasic.grantRead(importInstanceRole);
+        const instantClientSqlPlus = new s3assets.Asset(this, `instantClientSqlPlusRpm`, {
+			path: "oracle-instantclient19.8-sqlplus-19.8.0.0.0-1.x86_64.rpm"
+		});
+		instantClientSqlPlus.grantRead(importInstanceRole);
+        
+        
+        const loadBindingDbAssociation = new ssm.CfnAssociation(this, 'loadBindingDbAssociation',{
+            name: loadBindingDbDoc.ref,
+            targets: [
+                { key: "InstanceIds", values: [importInstance.instanceId] }
+            ]
+        });
+        
+        loadBindingDbAssociation.addPropertyOverride('Parameters',{
+            databaseSecretArn: [this.BindingDBSecret.secretArn],
+            databaseHostName: [this.BindingDb.dbInstanceEndpointAddress],
+            databaseDmpS3Location: [this.BindingDBSourceBucket.bucketName],
+            instantClientBasicS3Path: [instantClientBasic.s3ObjectUrl],
+            instantClientSqlPlusS3Path: [instantClientSqlPlus.s3ObjectUrl],
+            executionTimeout: ['7200']
+        });
+        
+        //// End Binding DB  ////
 
    }
+   
+   
 }

lib/baseline-stack.ts

@@ -0,0 +1,53 @@
+import * as cdk from '@aws-cdk/core';
+import ec2 = require('@aws-cdk/aws-ec2');
+import iam = require('@aws-cdk/aws-iam');
+import rds = require('@aws-cdk/aws-rds');
+import glue = require('@aws-cdk/aws-glue');
+import s3 = require('@aws-cdk/aws-s3');
+import s3assets = require('@aws-cdk/aws-s3-assets');
+import { RDSdataSetSetEnrollmentProps, RDSOracleDataSetEnrollment } from './constructs/rds-data-set-enrollment';
+import { DataSetStack, DataSetStackProps} from './stacks/dataset-stack';
+
+
+
+
+export interface BindingDBEnrollmentProps extends DataSetStackProps {
+	databaseSecret: rds.DatabaseSecret;
+	database: rds.DatabaseInstance;
+	accessSecurityGroup: ec2.SecurityGroup;
+}
+
+export class BindingDBStack extends DataSetStack{
+	constructor(scope: cdk.Construct, id: string, props: BindingDBEnrollmentProps) {
+		super(scope, id, props);
+	
+	
+		const dataSetName = "binding_db";
+
+		this.Enrollment = new RDSOracleDataSetEnrollment(this, 'binding-db-enrollment', {
+	    	databaseSecret: props.databaseSecret,
+	    	database: props.database,
+	    	databaseSidOrServiceName: "orcl",
+	    	accessSecurityGroup: props.accessSecurityGroup,
+	    	dataLakeBucket: props.DataLake.DataLakeBucket,
+	    	DataSetName: dataSetName,
+	    	JdbcTargetIncludePaths: ["orcl/%"],
+	    	GlueScriptPath: "scripts/glue.s3import.bindingdb.py",
+			GlueScriptArguments: {
+				"--job-language": "python", 
+				"--job-bookmark-option": "job-bookmark-disable",
+				"--enable-metrics": "",
+				"--DL_BUCKET": props.DataLake.DataLakeBucket.bucketName,
+				"--DL_PREFIX": "/"+dataSetName+"/",
+				"--DL_REGION": cdk.Stack.of(this).region,
+				"--GLUE_SRC_DATABASE": "binding_db_src"
+			}	    	
+		});
+		
+		
+	}
+}
+
+
+
+

lib/bindingdb-stack.ts

@@ -28,6 +28,7 @@ export class ChemblStack extends DataSetStack{
 		this.Enrollment = new RDSPostgresDataSetEnrollment(this, 'chembl-25-enrollment', {
 	    	databaseSecret: props.databaseSecret,
 	    	database: props.database,
+	    	databaseSidOrServiceName: "chembl_25",
 	    	accessSecurityGroup: props.accessSecurityGroup,
 	    	dataLakeBucket: props.DataLake.DataLakeBucket,
 	    	DataSetName: dataSetName,

lib/chembl-25-stack.ts

@@ -398,8 +398,11 @@ export class DataLakeEnrollment extends cdk.Construct {
 
 
         const resolvedPrincipalType = this.determinePrincipalType(principal);
-
+        
+        
+        
 		if(resolvedPrincipalType === iam.Role){
+		    console.log("coarseAthenaAccessPolicy");
 		    this.CoarseAthenaAccessPolicy.attachToRole(principal as iam.Role);
 		    this.CoarseResourceAccessPolicy.attachToRole(principal as iam.Role);
 		    this.CoarseIamPolciesApplied = true;
@@ -416,7 +419,6 @@ export class DataLakeEnrollment extends cdk.Construct {
 
 
 
-
 	}