Completed DynamoDB + DAX Benchmarker with a nice TUI to boot

2023-08-02 18:11:41 -06:00
parent 09862c1b43
commit e42070eefa
55 changed files with 3574 additions and 1 deletions
@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out
@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out
@@ -0,0 +1,88 @@
+# DynamoDB + DAX Benchmarker CDK
+
+This CDK project deploys a DynamoDB table with a DAX cluster on top of it, and an EC2 instance to act as a bastion host for running benchmarking tests agasint DAX.
+
+By default, the name of the DynamoDB table that is created is `$USER-high-velocity-table`.
+By default, the name of the SSH key that is created for you is `$USER-dax-pair`
+
+It should be noted that due to a bug in CDK, if you destroy the stack, you'll have to manually delete the SubnetGroup in DAX once everything else is deleted.
+
+## Prerequisites
+You must be logged into the AWS CLI prior to running the CDK. Ensure you're logged into your target AWS account by running 
+`aws sts get-caller-identity`.
+
+## Getting started
+[NodeJS](https://nodejs.org/en) is required for development. Install NodeJS using the following commands, if it is 
+not already installed:
+
+### Installing NodeJS
+
+#### Windows
+NodeJS can be installed on Windows using the [Chocolatey](https://chocolatey.org) package manager. If Chocolatey is not yet
+installed on your system, first install it in a privileged PowerShell:
+```powershell
+Set-ExecutionPolicy Bypass -Scope Process -Force;
+    [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; 
+    iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+```
+
+Then, in a _non-privileged_ PowerShell session, install node:
+```powershell
+choco install nodejs
+```
+
+#### Linux
+NodeJS can be installed on Linux using [NVM](https://github.com/nvm-sh/nvm). First, install NVM:
+```shell
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash
+```
+
+**Note:** The installation command was _not_ run with `sudo`. This is intentional, because if you install with `sudo`, then
+`sudo` permissions will be required to install any and all new dependencies! You should avoid installing Node for the root
+user!
+
+
+Then, in order to use NVM to install NodeJS, you need to either restart your current shell session, or run the following:
+```shell
+export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+```
+
+Now, install NodeJS:
+```shell
+nvm install node 
+```
+
+### Installing dependent libraries
+
+Once node is installed, run the following commands to install the NPM libraries:
+
+```shell
+cd cdk
+npm install -g aws-cdk
+npm install -g typescript --save-dev
+npm install
+```
+
+## CDK Arguments
+This application depends on a few additional parameters in order to run. They can be specified in one of two ways: environment variables, or via the `-c` argument of the `cdk` command.
+
+**Important:** Only one environment variable is required by the application, regardless of which parameter specification method you choose: `AWS_REGION`.
+
+The following is a table of the **required** parameters for running the CDK
+
+| Parameter Name | Environment Variable Name | Description                                                                                                                                                          |
+|----------------|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `vpcId`        | `VPC_ID`                  | The VPC ID you wish to deploy all of the stack's components into                                                                                                     |
+| `localIp`      | `LOCAL_IP`                | Your local IP; Used to allow SSH and Elasticsearch access in the EC2 security group                                                                                  |
+| `sshKeyName`   | `SSH_KEY_NAME`            | The key name of your ssh key to allow you access to your EC2 instance. This should only be the name of the `.pem` file, and should not include the `.pem` extension. |
+| `awsAccount`   | `AWS_ACCOUNT`             | The account ID of your AWS account.                                                                                                                                  |
+| `awsRegion`    | `AWS_REGION`              | The AWS region to deploy this stack and its components into                                                                                                          |
+
+### Optional Parameters
+It is sometimes necessary to tweak the deployment a bit for different use cases. The CDK can be tweaked with the following parameters:
+
+| Parameter Name  | Default Value         | Description                                                                                                             |
+|-----------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------|
+| `baseTableName` | `high-velocity-table` | This is the base name for the table. All tables created by the stack will be prefixed with `$USER` to prevent conflicts |
+
@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+import * as cdk from 'aws-cdk-lib';
+import {EnvironmentProps} from '../lib/types';
+import { DaxBenchmarkingStack } from '../lib/dax-benchmarking-stack';
+
+const app = new cdk.App();
+const user = process.env.USER || '';
+let vpcId = app.node.tryGetContext('vpcId');
+if (!vpcId) {
+	if (!process.env.VPC_ID) {
+			throw new Error('vpcId is a required parameter. Specify it with `-c vpcId=someId`, or by setting the VPC_ID environment variable');
+	} else {
+		vpcId = process.env.VPC_ID
+	}
+}
+
+let localIp = app.node.tryGetContext('localIp');
+if (!localIp) {
+	if (!process.env.LOCAL_IP) {
+			throw new Error('Local IP is a required parameter. Specify it with `-c localIp=XXX.XXX.XXX.XXX`, or by setting the LOCAL_IP environment variable');
+	} else {
+		localIp = process.env.LOCAL_IP
+	}
+}
+
+let sshKeyName = app.node.tryGetContext('sshKeyName');
+if (!sshKeyName) {
+	if (!process.env.SSH_KEY_NAME) {
+		sshKeyName = `${user}-dax-pair`;
+	} else {
+		sshKeyName = process.env.SSH_KEY_NAME;
+	}
+}
+
+let awsAccount = app.node.tryGetContext('awsAccount');
+if (!awsAccount) {
+	if (!process.env.AWS_ACCOUNT) {
+		throw new Error('awsAccount is a required parameter. Specify it with `-c awsAccount=1234567890`, or by setting the AWS_ACCOUNT environment variable.');
+	} else {
+		awsAccount = process.env.AWS_ACCOUNT;
+	}
+}
+
+let awsRegion = app.node.tryGetContext('awsRegion');
+if (!awsRegion) {
+	if (!process.env.AWS_REGION) {
+		throw new Error('The `AWS_REGION` environment variable was not set. It must be set in order to use this application.');
+	} else {
+		awsRegion = process.env.AWS_REGION
+	}
+}
+
+let baseTableName = app.node.tryGetContext('baseTableName');
+if (!baseTableName) {
+	baseTableName = 'high-velocity-table'
+}
+
+const environmentProps: EnvironmentProps = {
+	env: { account: awsAccount, region: awsRegion },
+	baseTableName,
+	removalPolicy: cdk.RemovalPolicy.DESTROY,
+	user,
+	vpcId,
+	localIp,
+	sshKeyName
+};
+
+new DaxBenchmarkingStack(app, `${user}-dax-benchmark-stack`, environmentProps);
@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};
@@ -0,0 +1,50 @@
+import { Tags } from "aws-cdk-lib";
+import { Construct } from "constructs";
+import { EnvironmentProps } from "./types";
+import { Instance, InstanceClass, InstanceSize, InstanceType, MachineImage, Peer, Port, SecurityGroup, SubnetType, Vpc } from "aws-cdk-lib/aws-ec2";
+import { IRole, Role, ServicePrincipal } from "aws-cdk-lib/aws-iam";
+
+export class DaxBastionHost extends Construct {
+  public readonly instanceRole: IRole;
+  public readonly instance: Instance;
+
+  constructor(scope: Construct, id: string, environmentProps: EnvironmentProps, daxSecurityGroup: SecurityGroup) {
+    super(scope, id);
+
+    Tags.of(this).add('Application', 'dynamodb-dax-benchmarker');
+
+    const { removalPolicy, user, vpcId, localIp, sshKeyName } = environmentProps;
+    const localIpCidr = `${localIp}/32`;
+
+    const vpc = Vpc.fromLookup(this, 'Vpc', { vpcId });
+
+    const bastionHostSecurityGroup = new SecurityGroup(this, `${user}-dax-sg`, {
+      vpc,
+      description: `Allow SSH, Elasticsearch, and DAX access for ${user}`,
+      securityGroupName: `${user}-dax-bastion-host-sg`
+    });
+    bastionHostSecurityGroup.applyRemovalPolicy(removalPolicy);
+    bastionHostSecurityGroup.addIngressRule(Peer.ipv4(localIpCidr), Port.tcp(22), "Allow SSH access to this instance from the users public IP");
+    bastionHostSecurityGroup.addIngressRule(Peer.ipv4(localIpCidr), Port.tcp(9200), "Allow the host to communicate with the users locally running Elasticsearch cluster");
+    bastionHostSecurityGroup.addIngressRule(daxSecurityGroup, Port.allTraffic());
+    daxSecurityGroup.addIngressRule(bastionHostSecurityGroup, Port.allTraffic());
+
+    this.instanceRole = new Role(this, `${user}-bastion-role`, {
+      roleName: `${user}-bastion-role`,
+      assumedBy: new ServicePrincipal('ec2.amazonaws.com'),
+    });
+    this.instanceRole.applyRemovalPolicy(removalPolicy);
+
+    this.instance = new Instance(this, `${user}-dax-bastion-host`, {
+      vpc,
+      instanceType: InstanceType.of(InstanceClass.T2, InstanceSize.SMALL),
+      machineImage: MachineImage.latestAmazonLinux2023(),
+      instanceName: `${user}-dax-bastion-host`,
+      keyName: sshKeyName,
+      vpcSubnets: vpc.selectSubnets({ subnetType: SubnetType.PUBLIC }),
+      securityGroup: bastionHostSecurityGroup,
+      role: this.instanceRole
+    });
+    this.instance.applyRemovalPolicy(removalPolicy);
+  }
+}
@@ -0,0 +1,89 @@
+import { Construct } from "constructs";
+import { EnvironmentProps } from "./types";
+import { CfnOutput, Stack, Tags } from "aws-cdk-lib";
+import { CfnCluster, CfnSubnetGroup } from "aws-cdk-lib/aws-dax";
+import { Effect, PolicyDocument, PolicyStatement, Role, ServicePrincipal } from "aws-cdk-lib/aws-iam";
+import { SecurityGroup, SubnetType, Vpc } from "aws-cdk-lib/aws-ec2";
+import { DynamoDbBenchmarkTable } from "./dynamodb";
+import { DaxBastionHost } from "./bastion-host";
+
+export class DaxBenchmarkingStack extends Stack {
+  constructor(scope: Construct, id: string, environmentProps: EnvironmentProps) {
+    super(scope, id, environmentProps);
+
+    Tags.of(this).add('Application', 'dynamodb-dax-benchmarker');
+
+    const { user, removalPolicy, vpcId } = environmentProps;
+    const { table } = new DynamoDbBenchmarkTable(this, `${user}-dynamodb-benchmark-table`, environmentProps);
+
+    const vpc = Vpc.fromLookup(this, 'Vpc', { vpcId });
+
+    const daxSecurityGroup = new SecurityGroup(this, `${user}-dax-sg`, {
+      vpc,
+      securityGroupName: `${user}-dax-sg`
+    });
+    daxSecurityGroup.applyRemovalPolicy(removalPolicy);
+
+    const { instanceRole, instance } = new DaxBastionHost(this, `${user}-dax-bastion-host`, environmentProps, daxSecurityGroup);
+
+    const daxClusterName = `${user}-high-velocity`;
+      const daxFullAccessPolicy = new PolicyStatement({
+          effect: Effect.ALLOW,
+          actions: [
+              "dynamodb:BatchGetItem",
+              "dynamodb:GetItem",
+              "dynamodb:Query",
+              "dynamodb:Scan",
+              "dynamodb:BatchWriteItem",
+              "dynamodb:DeleteItem",
+              "dynamodb:PutItem",
+              "dynamodb:UpdateItem",
+              "dynamodb:DescribeLimits",
+              "dynamodb:DescribeTimeToLive",
+              "dynamodb:DescribeTable",
+              "dynamodb:ListTables"
+          ],
+          resources: [table.tableArn]
+      });
+
+    const daxServiceRole = new Role(this, `${daxClusterName}-role`, {
+      assumedBy: new ServicePrincipal("dax.amazonaws.com"),
+      inlinePolicies: {
+        DAXFullAccess: new PolicyDocument({
+          statements: [daxFullAccessPolicy]
+        })
+      } 
+    });
+    daxServiceRole.applyRemovalPolicy(removalPolicy);
+
+    instanceRole.addToPrincipalPolicy(daxFullAccessPolicy);
+
+    const subnetGroup = new CfnSubnetGroup(this, `${user}-dax-subnet-group`, {
+      subnetIds: vpc.selectSubnets({
+        subnetType: SubnetType.PRIVATE_ISOLATED
+      }).subnetIds,
+      subnetGroupName: `${user}-dax-subnet-group`,
+    });
+    subnetGroup.applyRemovalPolicy(removalPolicy);
+    
+   const daxCluster = new CfnCluster(this, daxClusterName, {
+      iamRoleArn: daxServiceRole.roleArn,
+      nodeType: 'dax.r5.large',
+      replicationFactor: 3,
+      securityGroupIds: [daxSecurityGroup.securityGroupId],
+      subnetGroupName: subnetGroup.subnetGroupName,
+      availabilityZones: vpc.availabilityZones,
+      clusterEndpointEncryptionType: 'TLS',
+      clusterName: daxClusterName,
+      sseSpecification: {
+        sseEnabled: true,
+      }
+    });
+    daxCluster.applyRemovalPolicy(removalPolicy);
+    daxCluster.addDependency(subnetGroup);
+
+    new CfnOutput(this, 'DaxEndpoint', { value: daxCluster.attrClusterDiscoveryEndpointUrl });
+    new CfnOutput(this, 'InstanceId', { value: instance.instanceId });
+    new CfnOutput(this, 'InstancePublicIp', { value: instance.instancePublicIp });
+  }
+}
@@ -0,0 +1,27 @@
+import {Tags} from "aws-cdk-lib";
+import {Construct} from "constructs";
+import {EnvironmentProps} from "./types";
+import {AttributeType, BillingMode, Table} from "aws-cdk-lib/aws-dynamodb";
+
+export class DynamoDbBenchmarkTable extends Construct {
+	public readonly table: Table;
+
+	constructor(scope: Construct, id: string, environmentProps: EnvironmentProps) {
+		super(scope, id);
+
+		Tags.of(this).add('Application', 'dynamodb-dax-benchmarker');
+
+		const { baseTableName, removalPolicy, user } = environmentProps;
+		const tableName = `${user}-${baseTableName}`;
+
+		this.table =  new Table(this, tableName, {
+			partitionKey: {
+				name: 'id',
+				type: AttributeType.STRING
+			},
+			tableName,
+			removalPolicy,
+			billingMode: BillingMode.PAY_PER_REQUEST
+		});
+	}
+}
@@ -0,0 +1,10 @@
+import {RemovalPolicy, StackProps} from "aws-cdk-lib";
+
+export interface EnvironmentProps extends StackProps {
+	readonly baseTableName: string
+	readonly removalPolicy: RemovalPolicy
+	readonly user: string
+	readonly vpcId: string
+	readonly localIp: string
+	readonly sshKeyName: string
+}