Completed DynamoDB + DAX Benchmarker with a nice TUI to boot
This commit is contained in:
hamilcarBarca17
+18
@@ -0,0 +1,18 @@
|
|||||||
|
/target
|
||||||
|
/.idea/
|
||||||
|
Cargo.lock
|
||||||
|
/.scannerwork/
|
||||||
|
*.exe
|
||||||
|
*.exe~
|
||||||
|
*.dll
|
||||||
|
*.so
|
||||||
|
*.dylib
|
||||||
|
*.test
|
||||||
|
*.out
|
||||||
|
.idea
|
||||||
|
out/
|
||||||
|
main
|
||||||
|
dynamodb-benchmarker
|
||||||
|
dax-benchmarker
|
||||||
|
*.log
|
||||||
|
*.json
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
run:
|
||||||
|
skip-dirs:
|
||||||
|
- cdk
|
||||||
|
- src
|
||||||
|
- scripts
|
||||||
|
- target
|
||||||
+25
@@ -0,0 +1,25 @@
|
|||||||
|
[package]
|
||||||
|
name = "dynamodb-benchmarker"
|
||||||
|
version = "0.1.0"
|
||||||
|
authors = ["Alex Clarke <alex.j.tusa@gmail.com>"]
|
||||||
|
description = "A CLI tool for simulating heavy usage against DynamoDB and publishing metrics to an Elastic Stack for analysis"
|
||||||
|
readme = "README.md"
|
||||||
|
edition = "2021"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
anyhow = "1.0.71"
|
||||||
|
aws-config = "0.55.3"
|
||||||
|
aws-sdk-dynamodb = "0.28.0"
|
||||||
|
aws-types = "0.55.3"
|
||||||
|
chrono = { version = "0.4.26", features = ["serde"] }
|
||||||
|
clap = { version = "4.3.14", features = ["derive"] }
|
||||||
|
elasticsearch = "8.5.0-alpha.1"
|
||||||
|
lipsum = "0.9.0"
|
||||||
|
log = "0.4.19"
|
||||||
|
log4rs = { version = "1.2.0", features = ["console_appender"] }
|
||||||
|
rand = "0.8.5"
|
||||||
|
serde = { version = "1.0.171", features = ["derive"] }
|
||||||
|
serde_json = { version = "1.0.102", features = ["arbitrary_precision"] }
|
||||||
|
tokio = { version = "1.29.1", features = ["full"] }
|
||||||
|
tokio-util = "0.7.8"
|
||||||
|
uuid = { version = "1.4.0", features = ["v4", "fast-rng"] }
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
#!make
|
||||||
|
|
||||||
|
default: build
|
||||||
|
|
||||||
|
.PHONY: init start-elastic-stack stop-elastic-stack build build-dynamodb-benchmarker build-dax-benchmarker run-dynamodb-benchmarker run-dax-benchmarker clean lint
|
||||||
|
|
||||||
|
init: build
|
||||||
|
@[[ -d ../docker-elk ]] || git clone https://github.com/deviantony/docker-elk.git ..
|
||||||
|
@cd ../docker-elk && docker compose up setup
|
||||||
|
@echo "Default login creds: username=elastic, password=changeme"
|
||||||
|
|
||||||
|
start-elastic-stack:
|
||||||
|
@cd ../docker-elk && docker compose up -d
|
||||||
|
|
||||||
|
stop-elastic-stack:
|
||||||
|
@cd ../docker-elk && docker compose down
|
||||||
|
|
||||||
|
build-dynamodb-benchmarker:
|
||||||
|
@cargo clean && rm -f dynamodb-benchmarker && cargo build --release && mv ./target/release/dynamodb-benchmarker .
|
||||||
|
|
||||||
|
build-dax-benchmarker:
|
||||||
|
@rm -f main && rm -f dax-benchmarker && go build -o dax-benchmarker pkg/app/main.go
|
||||||
|
|
||||||
|
build: build-dynamodb-benchmarker build-dax-benchmarker
|
||||||
|
|
||||||
|
run-dynamodb-benchmarker:
|
||||||
|
@cargo run
|
||||||
|
|
||||||
|
run-dax-benchmarker:
|
||||||
|
@go run pkg/app/main.go
|
||||||
|
|
||||||
|
clean:
|
||||||
|
@cargo clean && rm -f main && rm -f dynamodb-benchmarker && rm -f dax-benchmarker && rm -rf cdk/cdk.out && rm -rf cdk/node_modules
|
||||||
|
|
||||||
|
lint:
|
||||||
|
@cargo clippy && golangci-lint run
|
||||||
@@ -1 +1,279 @@
|
|||||||
# dynamodb-dax-benchmarker
|
# DynamoDB + DAX Benchmarker
|
||||||
|
This project houses the Rust and Go code to benchmark the performance of DynamoDB and DAX by simulating heavy loads.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Features
|
||||||
|
* [x] Simulate reads on existing data
|
||||||
|
* [x] Simulate writes
|
||||||
|
* [x] Simulate updates
|
||||||
|
* [x] Simulate deletes
|
||||||
|
* [x] Record the following metrics
|
||||||
|
* [x] The type of operation being simulated
|
||||||
|
* [x] Total simulation time
|
||||||
|
* [x] Read times
|
||||||
|
* [x] Write times
|
||||||
|
* [x] Confirmation of write times (i.e. how long after a write is the item available when performing a read)
|
||||||
|
* [x] Update times
|
||||||
|
* [x] Confirmation of update times (i.e. how long after an update is the item available when performing a read)
|
||||||
|
* [x] Delete times
|
||||||
|
* [x] Confirmation of delete times (i.e. how long after a delete is the item no longer available when performing a read)
|
||||||
|
* [x] Randomized selection of which operation to perform
|
||||||
|
* [x] Multithreaded performance for publishing to a locally running Elasticsearch cluster
|
||||||
|
* [x] Highly performant concurrent operations against DynamoDB - 1,000 concurrent operations
|
||||||
|
* [x] Read-only scenarios for tables that are likely to be hit with mostly reads and very few mutating operations
|
||||||
|
* [x] Randomly generate schemas for DynamoDB with a specified number of attributes and generate random data to query
|
||||||
|
|
||||||
|
**Disclaimer:** This project exists as a proof-of-concept for how to benchmark and evaluate the performance of DynamoDB + DAX. As such,
|
||||||
|
this project does not contain any unit tests, integration tests, or E2E tests, thus regressions with future updates are possible and
|
||||||
|
project stability is _not_ guaranteed in perpetuity.
|
||||||
|
|
||||||
|
## Warning!
|
||||||
|
When making changes to this repository, take extra care to be sure you don't commit the automatically generated variables in
|
||||||
|
the [hosts file](./ansible/inventories/local/hosts.yml) and in the [host_vars](./ansible/inventories/local/host_vars/localhost.yml).
|
||||||
|
|
||||||
|
These files have variables that are populated automatically to make your life easier instead of having to specify variables
|
||||||
|
all the time. You can remove them manually, or you can wipe away everything and have them removed for you:
|
||||||
|
|
||||||
|
* `bastion.hosts.BASTION_HOST_PUBLIC_IP:`
|
||||||
|
* `vpc_id`
|
||||||
|
* `dax_endpoint`
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
The easiest way to use this project is to use the [benchmarker.sh](./benchmarker.sh) script's TUI. This will automate everything for you and make
|
||||||
|
the use of this project as painless as possible! Just ensure it's executable and run the TUI:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
chmod +x benchmarker.sh
|
||||||
|
./benchmarker.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
This project is broken into several distinct pieces. For more information on each the specific pieces, refer to their respective README's:
|
||||||
|
|
||||||
|
* [Ansible](./ansible/README.md) -- Go here if you're looking to have more control over what's going on across the entire deployment process, local and AWS
|
||||||
|
* [CDK](./cdk/README.md) -- Go here if you're looking to discover and tweak the AWS stack and resources
|
||||||
|
|
||||||
|
The vast majority of this project is designed to be automated; however, it is also designed to allow the user to customize it as needed. Naturally, customization
|
||||||
|
is a more challenging course of action here as you'll need to do some of the automated steps manually. I try to detail those steps below.
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
* The commands are being run in a Debian-based Linux environment (i.e. Ubuntu, WSL, etc.)
|
||||||
|
* [AWS CLI v2](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) is installed and configured
|
||||||
|
* Docker is installed (`sudo apt-get update && sudo apt-get install docker-ce docker-ce-cli`)
|
||||||
|
* The docker compose plugin is installed (`sudo apt-get update && sudo apt-get install docker-compose-plugin`)
|
||||||
|
* Rust is installed via rustup (`curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`)
|
||||||
|
* `jq` is installed (`sudo apt-get install jq`)
|
||||||
|
* Go is installed ([instructions here](https://go.dev/doc/install))
|
||||||
|
|
||||||
|
### Setting up your own local Elastic Stack
|
||||||
|
Fortunately, setting up the Elastic Stack locally is super easy thanks to [this repository](https://github.com/deviantony/docker-elk).
|
||||||
|
|
||||||
|
This setup is automated for you via the `Makefile` target: `init`
|
||||||
|
|
||||||
|
It will
|
||||||
|
|
||||||
|
* Clone the repository to the folder above the current directory
|
||||||
|
* Go into the folder
|
||||||
|
* Start the `setup` docker compose to initialize the Elastic Stack. Don't worry. This function is supposed to exit once it's done initializing your local Elastic Stack
|
||||||
|
|
||||||
|
### Starting and Stopping the Elastic Stack
|
||||||
|
To start the Elastic Stack, you can either manually `cd` into the [docker-elk](../docker-elk) folder and run `docker compose up -d`, or you can use the `start-elastic-stack` target in the [`Makefile`](./Makefile).
|
||||||
|
Similarly, you can stop the Elastic Stack by either `cd`into the [docker-elk](../docker-elk) folder and running `docker compose down`, or you can use the `stop-elastic-stack` target in the [`Makefile`](./Makefile).
|
||||||
|
|
||||||
|
### Running the Benchmarkers
|
||||||
|
To run the benchmarker, make sure you've done all the following steps so that the AWS SDK can pick up your credentials:
|
||||||
|
|
||||||
|
* You're logged into your desired AWS account via the AWS CLI
|
||||||
|
* Ensure you're properly connected with `aws sts get-caller-identity`
|
||||||
|
* You've exported all the following AWS environment variables, so they can be picked up by the AWS SDK at runtime to authenticate with AWS:
|
||||||
|
* `AWS_ACCESS_KEY_ID`
|
||||||
|
* `AWS_SECRET_ACCESS_KEY`
|
||||||
|
* `AWS_SESSION_TOKEN`
|
||||||
|
* `AWS_REGION` (not typically defined by the CLI)
|
||||||
|
* `AWS_ACCOUNT` (this is a special variable that is not normally defined by the AWS CLI: definition is achieved by running `export AWS_ACCOUNT=$(aws sts get-caller-identity | jq -r .Account)`)
|
||||||
|
* A nifty little shortcut for exporting all but the `AWS_REGION` and `AWS_ACCOUNT` variables is provided with the following command: ```shell eval $(aws configure export-credentials --format env)```
|
||||||
|
|
||||||
|
There's a few ways to run the benchmarkers, but the easiest is to build them and run their binaries.
|
||||||
|
|
||||||
|
It's as simple as `make build` and running whichever binary that corresponds to the benchmarking you wish to perform; e.g.
|
||||||
|
* `./dynamodb-benchmarker`
|
||||||
|
or
|
||||||
|
* `./dax-benchmarker`
|
||||||
|
|
||||||
|
For both the `dynamodb-benchmarker` and the `dax-benchmarker`, additional help and usage flags can be found using `--help`. This way you can tweak your benchmarking experience as necessary.
|
||||||
|
|
||||||
|
**dynamodb-benchmarker help**:
|
||||||
|
```
|
||||||
|
atusa@atusa-thinkpad:~/code/dynamodb-benchmarker$ ./dynamodb-benchmarker --help
|
||||||
|
A CLI tool for simulating heavy usage against DynamoDB and publishing metrics to an Elastic Stack for analysis
|
||||||
|
|
||||||
|
Usage: dynamodb-benchmarker [OPTIONS]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-c, --concurrent-simulations <CONCURRENT_SIMULATIONS>
|
||||||
|
The number of concurrent simulations to run [default: 1000]
|
||||||
|
-a, --attributes <ATTRIBUTES>
|
||||||
|
The number of attributes to use when populating and querying the DynamoDB table; minimum value of 1 [default: 5]
|
||||||
|
-d, --duration <DURATION>
|
||||||
|
The length of time (in seconds) to run the benchmark for [default: 1800]
|
||||||
|
-b, --buffer <BUFFER>
|
||||||
|
The buffer size of the Elasticsearch thread's MPSC channel [default: 500]
|
||||||
|
-u, --username <USERNAME>
|
||||||
|
Local Elasticsearch cluster username [default: elastic]
|
||||||
|
-p, --password <PASSWORD>
|
||||||
|
Local Elasticsearch cluster password [default: changeme]
|
||||||
|
-i, --index <INDEX>
|
||||||
|
The Elasticsearch Index to insert data into [default: dynamodb]
|
||||||
|
-t, --table-name <TABLE_NAME>
|
||||||
|
The DynamoDB table to perform operations against [default: atusa-high-velocity-table]
|
||||||
|
-r, --read-only
|
||||||
|
Whether to run a read-only scenario for benchmarking
|
||||||
|
-h, --help
|
||||||
|
Print help
|
||||||
|
-V, --version
|
||||||
|
Print version
|
||||||
|
```
|
||||||
|
|
||||||
|
**dax-benchmarker help**:
|
||||||
|
```
|
||||||
|
atusa@atusa-thinkpad:~/code/dynamodb-benchmarker$ ./dynamodb-benchmarker --help
|
||||||
|
A CLI tool for simulating heavy usage against DAX and publishing metrics to an Elastic Stack for analysis
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
dax-benchmarker [flags]
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
-a, --attributes int The number of attributes to use when populating and querying the DynamoDB table; minimum value of 1 (default 5)
|
||||||
|
-b, --buffer int The buffer size of the Elasticsearch goroutine's channel (default 500)
|
||||||
|
-c, --concurrent-simulations int The number of concurrent simulations to run (default 1000)
|
||||||
|
-d, --duration int The length of time (in seconds) to run the bechmark for (default 1800)
|
||||||
|
-e, --endpoint string The DAX endpoint to hit when running simulations (assumes secure endpoint, so do not specify port)
|
||||||
|
-h, --help help for dax-benchmarker
|
||||||
|
-i, --index string The Elasticsearch Index to insert data into (default "dax")
|
||||||
|
-p, --password string Local Elasticsearch cluster password (default "changeme")
|
||||||
|
-r, --read-only Whether to run a read-only scenario for benchmarking
|
||||||
|
-t, --table string The DynamoDB table to perform operations against (default "atusa-high-velocity-table")
|
||||||
|
-u, --username string Local Elasticsearch cluster username (default "elastic")
|
||||||
|
```
|
||||||
|
|
||||||
|
#### DAX Benchmarker Gotcha
|
||||||
|
The nature of DAX os that it does not allow external access from outside the VPC it is deployed in. This is due to the
|
||||||
|
fact that DAX uses a proprietary protocol that does not support TLS. So, in order to run the DAX benchmarker, we must
|
||||||
|
push it out and run it from the bastion host that is created in the CDK. This is why the host is created.
|
||||||
|
|
||||||
|
To do this manually, you need to know a few pieces of information:
|
||||||
|
|
||||||
|
* The SSH key used to connect to the host -- if you did not specify a key manually, then the key is `~/.ssh/$USER-dax-pair.pem`
|
||||||
|
* The bastion host's public IP address
|
||||||
|
* The DAX endpoint URI
|
||||||
|
|
||||||
|
The bastion host's public IP and the DAX Endpoint can be obtained using the following commands
|
||||||
|
(You'll need to copy/paste the DAX endpoint once you've SSH'd into the bastion host):
|
||||||
|
|
||||||
|
```shell
|
||||||
|
dax_endpoint=$(aws cloudformation describe-stacks --stack-name "$USER-dax-benchmark-stack" --query "Stacks[0].Outputs[?OutputKey=='DaxEndpoint'].OutputValue" --output text)
|
||||||
|
bastion_host_ip=$(aws cloudformation describe-stacks --stack-name "$USER-dax-benchmark-stack" --query "Stacks[0].Outputs[?OutputKey=='InstancePublicIp'].OutputValue" --output text)
|
||||||
|
```
|
||||||
|
|
||||||
|
Then, you need to upload the `dax-benchmarker` binary to the bastion host:
|
||||||
|
```shell
|
||||||
|
scp -i ~/.ssh/"$USER"-dax-pair.pem dax-benchmarker ec2-user@"$bastion_host_ip":/home/ec2-user/
|
||||||
|
```
|
||||||
|
|
||||||
|
Additionally, you'll need to configure the bastion host to use your current AWS CLI creds; so you'll need to run the following
|
||||||
|
command locally and paste the output exactly into the SSH session:
|
||||||
|
```shell
|
||||||
|
aws configure export-credentials --format env
|
||||||
|
```
|
||||||
|
|
||||||
|
Finally, you need to SSH into the bastion host with remote port forwarding to your local Elasticsearch cluster (port 9200):
|
||||||
|
```shell
|
||||||
|
ssh -i ~/.ssh/"$USER"-dax-pair.pem -R 9200:localhost:9200 ec2-user@"$bastion_host_ip"
|
||||||
|
```
|
||||||
|
|
||||||
|
Once you're SSH'd into the bastion host, you'll need to set the `DAX_ENDPOINT` environment variable using the DAX endpoint spit out
|
||||||
|
by the previous command:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
export DAX_ENDPOINT='PASTE_DAX_ENDPOINT_HERE'
|
||||||
|
```
|
||||||
|
|
||||||
|
Be sure to paste the output of the `aws configure export-credentials --format env` command as well
|
||||||
|
|
||||||
|
Finally, be sure to also export the `AWS_REGION` environment variable that matches the region you deployed your stack into.
|
||||||
|
|
||||||
|
Once you've done all of this, you're ready to run the `dax-benchmarker` from the bastion host and customize the experience however you need using the
|
||||||
|
configuration parameters provided (`./dax-benchmarker -h`).
|
||||||
|
|
||||||
|
### Scenarios
|
||||||
|
By default, for both benchmarkers, they perform CRUD simulations that randomly choose to
|
||||||
|
* Read an existing item
|
||||||
|
* Write a new item and record how long it takes to confirm it's there (deletes the item afterward)
|
||||||
|
* Create a new item and update it, then record how long it takes to confirm the update is reflected in subsequent API calls (deletes the item afterward)
|
||||||
|
|
||||||
|
However, sometimes a more realistic test is to simply run in `read-only` mode; This is supported by both benchmarkers via the `-r, --read-only` flag.
|
||||||
|
|
||||||
|
`read-only` mode, for each concurrent simulation, randomly select a time between 0 and 15 seconds, and then execute a read on an existing item. This simulates more realistic behavior from applications
|
||||||
|
who are only reading from DAX or DynamoDB and not performing any write, update, or delete operations.
|
||||||
|
|
||||||
|
## Accessing the Elastic Stack and analyzing data
|
||||||
|
By default, the Elastic Stack services are at the following URLs when running locally:
|
||||||
|
|
||||||
|
* Elasticsearch -> `http://localhost:9200`
|
||||||
|
* Kibana -> `http://localhost:5601`
|
||||||
|
|
||||||
|
The default credentials for accessing them are
|
||||||
|
|
||||||
|
* Username -> `elastic`
|
||||||
|
* Password -> `changeme`
|
||||||
|
|
||||||
|
Once you're in, you can use Kibana to analyze the data published by the benchmarker.
|
||||||
|
|
||||||
|
This data lives in the `dynamodb` and the `dax` indices of Elasticsearch by default, unless specified otherwise on the clients by the user.
|
||||||
|
|
||||||
|
**Note:** Sometimes the simulations would reach the provisioned throughput thresholds and would be rate-limited by AWS. I set the DynamoDB table to On-Demand
|
||||||
|
to scale out automatically, however this does not always prevent being rate limited. So that is why I also track the Failed Simulations in the Kibana graphs.
|
||||||
|
|
||||||
|
## Populating the DynamoDB benchmarking table with random data
|
||||||
|
By default, the clients and CDK create a DynamoDB table titled `$USER-high-velocity-table`. To run the clients with a different table name, use the `-t, --table` arguments.
|
||||||
|
|
||||||
|
If you wish to populate the table with some data, the easiest way to achieve this is via the [randomly-generate-high-velocity-data](./scripts/randomly-generate-high-velocity-data.sh) script.
|
||||||
|
Simply run it and specify the number of items you wish to populate (rounded to a multiple of 25) via `-i 50`!
|
||||||
|
|
||||||
|
To follow the progress of the script, tail the `/tmp/benchmarker.log` file.
|
||||||
|
|
||||||
|
You can specify different arguments to the script to tweak the settings for the script as necessary:
|
||||||
|
|
||||||
|
```
|
||||||
|
atusa@atusa-thinkpad:~/code/dynamodb-benchmarker$ ./scripts/randomly-generate-high-velocity-data.sh --help
|
||||||
|
randomly-generate-high-velocity-data: A script to randomly generate high-velocity data for some DynamoDB table with random attributes and values for benchmarking purposes.
|
||||||
|
|
||||||
|
USAGE:
|
||||||
|
randomly-generate-high-velocity-data [OPTIONS] [ARGS]...
|
||||||
|
|
||||||
|
-h, --help Show this usage screen
|
||||||
|
|
||||||
|
ARGS:
|
||||||
|
-a, --attributes <ATTRIBUTES> The number of attributes to populate each item in the table with
|
||||||
|
This defaults to 5
|
||||||
|
|
||||||
|
-i, --items <ITEMS> The number of items to populate the table with
|
||||||
|
Items are populated 25 at a time, so whatever number you provide will be rounded to the nearest multiple of 25
|
||||||
|
|
||||||
|
-t, --table <TABLE_NAME> The name of the DynamoDB table to populate
|
||||||
|
This defaults to atusa-high-velocity-table
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
These arguments are provided as a convenience to the user if they so wish to populate a table other than the default one created by the CDK.
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
In the event you need more information about any of the automation, you can check the various log files created throughout the application:
|
||||||
|
|
||||||
|
* `/tmp/ansible-playbook-output.log` -- Generated whenever ansible-playbooks are run from the TUI
|
||||||
|
* `/tmp/benchmarker.log` -- Generated whenever you run the `randomly-generate-high-velocity-data.sh` script outside the TUI
|
||||||
|
* `/tmp/benchmarker-tui.log` -- Generated by events in the TUI
|
||||||
|
* `/tmp/dynamodb-population.log` -- Generated whenever you run the `randomly-generate-high-velocity-data.sh` script from the TUI
|
||||||
+14
@@ -0,0 +1,14 @@
|
|||||||
|
[defaults]
|
||||||
|
forks = 50
|
||||||
|
gathering = explicit
|
||||||
|
host_key_checking = False
|
||||||
|
nocows = 1
|
||||||
|
retry_files_enabled = False
|
||||||
|
roles_path = ./ansible/roles
|
||||||
|
timeout = 60
|
||||||
|
callback_whitelist = profile_tasks
|
||||||
|
|
||||||
|
[callback_profile_tasks]
|
||||||
|
sort_order = none
|
||||||
|
output_limit = 1000
|
||||||
|
|
||||||
@@ -0,0 +1,322 @@
|
|||||||
|
# Benchmarking Ansible Automation
|
||||||
|
|
||||||
|
This folder houses all the [Ansible](https://www.ansible.com/) roles to automate the configuration of your local
|
||||||
|
environment and to deploy the necessary DynamoDB and DAX components to AWS. AWS Deployments leverage
|
||||||
|
[AWS CDK](https://aws.amazon.com/cdk/) to automate the provisioning of AWS resources. For more information,
|
||||||
|
navigate to the [CDK directory](../cdk/README.md).
|
||||||
|
|
||||||
|
To just see how to run different plays and their corresponding commands without knowing how it all works together,
|
||||||
|
skip down to the [Plays](#plays) section below.
|
||||||
|
|
||||||
|
Note that if no `ssh_key_name` is provided, the default value is `$USER-dax-pair`
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
* You must be logged into the AWS CLI prior to running the CDK. Ensure you're logged into your target AWS account by running
|
||||||
|
`aws sts get-caller-identity`.
|
||||||
|
* Install pip (Assuming python3 is already installed): `sudo apt-get install python3-pip`
|
||||||
|
* Install the most recent version of Ansible and jmespath from pip: `pip3 install --user ansible jmespath`
|
||||||
|
* Export the local bin path: `export PATH=~/.local/bin:$PATH`
|
||||||
|
* Install curl (`sudo apt-get install curl`)
|
||||||
|
* Install the required Ansible dependencies using Ansible Galaxy (`ansible-galaxy install -r requirements.yml`)
|
||||||
|
|
||||||
|
## Initializing the Stack
|
||||||
|
To initialize the stack (including the local Elastic Stack), run the `deploy_benchmarker.yml` playbook with the `init` tag:
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local \
|
||||||
|
--tags init \
|
||||||
|
--ask-become-pass \
|
||||||
|
deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deploying the Stack
|
||||||
|
To deploy the entire benchmarking stack all at once, local and AWS, use the following command:
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local \
|
||||||
|
-e vpc_id={{ vpc_id_to_deploy_into }} \
|
||||||
|
deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
The same prerequisites apply to the CDK with the necessary environment or CDK parameters as is defined in the
|
||||||
|
[CDK Parameters](../cdk/README.md#cdk-arguments) section of the CDK README. Ansible will only resolve the following variables
|
||||||
|
for you; all other variables must be supplied by the user a runtime:
|
||||||
|
|
||||||
|
* `localIp`
|
||||||
|
* `awsAccount`
|
||||||
|
|
||||||
|
## Running the benchmarkers
|
||||||
|
To run the benchmarkers, run the following command:
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local \
|
||||||
|
-e dax_endpoint={{ the_dax_endpoint_uri }} \
|
||||||
|
run_benchmarkers.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Ansible Command Breakdown
|
||||||
|
Let's analyze how an ansible command is formed:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local \
|
||||||
|
-e vpc_id={{ vpc_id_to_deploy_into }} \
|
||||||
|
--ask-become-pass \
|
||||||
|
deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
`ansible-playbook` is the program that runs our playbook, `deploy_benchmarker.yml`. [Playbooks](https://docs.ansible.com/ansible/latest/user_guide/playbooks_intro.html)
|
||||||
|
are the main "blueprints" of automation tasks that Ansible uses.
|
||||||
|
|
||||||
|
`-i inventories/local` tells Ansible that we want to use the hosts and variables associated
|
||||||
|
with the `local` environment. So later in the playbook and
|
||||||
|
[roles](https://docs.ansible.com/ansible/latest/user_guide/playbooks_reuse_roles.html), when we're
|
||||||
|
using variables and hosts, we're pulling the corresponding values for this environment. More
|
||||||
|
information about inventories in Ansible can be found
|
||||||
|
[here](https://docs.ansible.com/ansible/2.3/intro_inventory.html). Inventories would be a good place
|
||||||
|
to start learning about Ansible if you're confused by what's happening in this module.
|
||||||
|
|
||||||
|
[This](./inventories/local/host_vars/localhost.yml) is where you'd put variables to persist between runs of this application.
|
||||||
|
By default, they are only provided for you if you follow the steps in the main repository script.
|
||||||
|
|
||||||
|
`-e vpc_id={{ vpc_id_to_deploy_into }}` is setting an extra variable for the playbook to use (fun fact: `-e` is an alias
|
||||||
|
for `--extra-vars`). This variable is not defined by default in your [local host vars](./inventories/local/host_vars/localhost.yml) because
|
||||||
|
we don't know what VPC you want to deploy the stack into. If you're running this using the main TUI script in the root
|
||||||
|
of this repo, then this is handled graphically for you. This will be set on the first run of the CDK deployment, so you do not have to specify
|
||||||
|
the `vpc_id` between subsequent runs. Otherwise, if you wish to change the VPC ID for any reason (including prior to an initial run), and
|
||||||
|
you wish to run this Ansible playbook manually, you can add it to your host vars file.
|
||||||
|
|
||||||
|
`--ask-become-pass` is telling Ansible to prompt you for your sudo password, so it can run installs and other configuration tasks on your behalf.
|
||||||
|
|
||||||
|
`deploy_benchmarker.yml` is the name of our playbook that we want Ansible to run.
|
||||||
|
|
||||||
|
## Using Tags to Control What is Deployed
|
||||||
|
Each part of the `deploy_benchmarker.yml` playbook has
|
||||||
|
[tags](https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html) associated with them.
|
||||||
|
These tags allow us to tell Ansible which part(s) of the playbook we want to run. In other words, tags
|
||||||
|
allow us to tell Ansible which parts of the overall Logstash deployment pipeline we want to run.
|
||||||
|
|
||||||
|
They `deploy_benchmarker.yml` playbook (and a couple of roles) has the following tags in it:
|
||||||
|
|
||||||
|
* `init`
|
||||||
|
* `init_elk`
|
||||||
|
* `stop_elk`
|
||||||
|
* `prerequisites`
|
||||||
|
* `elk`
|
||||||
|
* `cdk`
|
||||||
|
* `run`
|
||||||
|
* `deploy`
|
||||||
|
* `destroy`
|
||||||
|
* `destroy_key_pair`
|
||||||
|
* `upload`
|
||||||
|
* `dynamodb`
|
||||||
|
* `dax`
|
||||||
|
* `crud`
|
||||||
|
* `read-only`
|
||||||
|
|
||||||
|
To view all these tags and their associated plays from the `ansible` CLI, run
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook deploy_benchmarker.yml --list-tags
|
||||||
|
```
|
||||||
|
|
||||||
|
Using these tags, we can specify that we only want to run specific parts of the Benchmarking Deployment pipeline that's
|
||||||
|
defined in the `deploy_benchmarker.yml` playbook.
|
||||||
|
|
||||||
|
For example: If we only wanted to start the ELK (Elasticsearch-Logstash-Kibana) stack, we would run this:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags elk deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Likewise, if we wanted to stop the ELK stack, we'd run this:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags stop_elk deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Note the `--tags` argument. This allows us to tell Ansible to only run tasks or roles that have the
|
||||||
|
`elk` or `stop_elk` tag on them.
|
||||||
|
|
||||||
|
We can also specify multiple arguments for `--tags` if we wish; for example, if we wanted to simply spin up the local
|
||||||
|
Elastic stack (synonymous with ELK stack), and deploy the CDK, we'd run the following:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local -e vpc_id=vpc-1234567890 --tags 'elk,cdk' deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Plays
|
||||||
|
The following plays can be run from these playbooks using the tags with the following commands:
|
||||||
|
|
||||||
|
#### Initialize Your Local Environment and Elastic Stack
|
||||||
|
A sudo password is required to install applications, so we tell Ansible to prompt us for it at the start:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags init deploy_benchmarker.yml --ask-become-pass
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Deploy CDK and Run the Benchmarkers on the Bastion Host
|
||||||
|
This assumes you already know the VPC ID to deploy into and have already created an SSH key pair and have the key pair
|
||||||
|
locally in your `~/.ssh` directory with a `.pem` extension.
|
||||||
|
|
||||||
|
If you did not do this manually, it was done for you automatically and the created pair is under `~/.ssh/$USER-dax-pair.pem`.
|
||||||
|
|
||||||
|
You can either specify the `vpc_id` argument directly via `-e` in the command, or you can hard code
|
||||||
|
it in your [host_vars](./inventories/local/host_vars/localhost.yml). You must also already be logged into the AWS CLI for
|
||||||
|
your target environment, or specify a `profile_id` either in your `host_vars` or via `-e`, along with an `aws_region`. If you're not
|
||||||
|
already logged into AWS, your `profile_id` must be configured to be picked up automatically from your `~/.aws/config` or
|
||||||
|
`~/.aws/credentials` files with no additional login steps in order to deploy to AWS.
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local -e vpc_id=vpc-1234567890 --tags deploy deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Shut Down Your Local Elastic Stack
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags stop_elk deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Wipe Away everything
|
||||||
|
Once more, this assumes you either have the DAX
|
||||||
|
endpoint and the VPC ID hardcoded in your [host vars](./inventories/local/host_vars/localhost.yml), or you provide them via `-e`.
|
||||||
|
|
||||||
|
If you've already run a CDK deploy via Ansible, then you should not need to specify anything.
|
||||||
|
|
||||||
|
**Note:** For safety purposes, this will _not_ wipe away the `ssk_key_name` in your `~/.ssh` directory. If you specified
|
||||||
|
a pre-existing key to use for this deployment, it will not be touched. If you did not specify a key name, the automatically
|
||||||
|
generated key `$USER-dax-pair` will be left in your `~/.ssh` directory. If you wish to delete this pair from your local machine
|
||||||
|
and remove it from AWS, also specify the `destroy_key_pair` tag as well in the below command.
|
||||||
|
|
||||||
|
You can either specify the `vpc_id` argument directly via `-e` in the command, or you can hard code
|
||||||
|
it in your [host_vars](./inventories/local/host_vars/localhost.yml). You must also already be logged into the AWS CLI for
|
||||||
|
your target environment, or specify a `profile_id` either in your `host_vars` or via `-e`, along with an `aws_region`. If you're not
|
||||||
|
already logged into AWS, your `profile_id` must be configured to be picked up automatically from your `~/.aws/config` or
|
||||||
|
`~/.aws/credentials` files with no additional login steps in order to deploy to AWS.
|
||||||
|
|
||||||
|
**Destroy Everything, But Leave the ssh_key_name Key-Pair Alone:**
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local -e vpc_id=vpc-1234567890 --tags destroy deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
**Destroy Everything, Including the ssh_key_name Key-Pair**
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local -e vpc_id=vpc-1234567890 --tags 'destroy,destroy_key_pair' deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Additional Plays You Can Run
|
||||||
|
|
||||||
|
#### Only Install Prerequisites for Local Machine
|
||||||
|
A sudo password is required to install applications, so we tell Ansible to prompt us for it at the start:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags prerequisites deploy_benchmarker.yml --ask-become-pass
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Start Your Local Elastic Stack
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags elk deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Just Deploy the CDK
|
||||||
|
This assumes you already know the VPC ID to deploy into and have already created an SSH key pair and have the key pair
|
||||||
|
locally in your `~/.ssh` directory with a `.pem` extension. If you did not do this manually, it was done for you automatically
|
||||||
|
and the created pair is under `~/.ssh/$USER-dax-pair.pem`. You can either specify the `vpc_id`
|
||||||
|
argument directly via `-e` in the command, or you can hard code it in your [host_vars](./inventories/local/host_vars/localhost.yml).
|
||||||
|
|
||||||
|
If you've already run a CDK deploy via Ansible, then you should not need to specify anything.
|
||||||
|
|
||||||
|
You must also already be logged into the AWS CLI for your target environment, or specify a `profile_id` either in your
|
||||||
|
`host_vars` or via `-e`, along with an `aws_region`. If you're not already logged into AWS, your `profile_id` must be
|
||||||
|
configured to be picked up automatically from your `~/.aws/config` or `~/.aws/credentials` files with no additional
|
||||||
|
login steps in order to deploy to AWS.
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags cdk deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Only Upload the Benchmarkers to the Bastion Host
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags upload deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Run All Benchmarkers and Scenarios
|
||||||
|
This assumes the CDK is already deployed and an EC2 instance already exists. This also assumes you either have the DAX
|
||||||
|
endpoint and the VPC ID hardcoded in your [host vars](./inventories/local/host_vars/localhost.yml), or you provide them via `-e`.
|
||||||
|
If you've already run a CDK deploy via Ansible, then you should not need to specify anything.
|
||||||
|
|
||||||
|
Additionally, You must also already be logged into the AWS CLI for
|
||||||
|
your target environment, or specify a `profile_id` either in your `host_vars` or via `-e`, along with an `aws_region`. If you're not
|
||||||
|
already logged into AWS, your `profile_id` must be configured to be picked up automatically from your `~/.aws/config` or
|
||||||
|
`~/.aws/credentials` files with no additional login steps in order to deploy to AWS:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags run run_benchmarkers.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Only Run the DynamoDB/DAX Benchmarker
|
||||||
|
This assumes the CDK is already deployed and an EC2 instance already exists. This also assumes you either have the DAX
|
||||||
|
endpoint and the VPC ID hardcoded in your [host vars](./inventories/local/host_vars/localhost.yml), or you provide them via `-e`.
|
||||||
|
If you've already run a CDK deploy via Ansible, then you should not need to specify anything.
|
||||||
|
|
||||||
|
Additionally, You must also already be logged into the AWS CLI for
|
||||||
|
your target environment, or specify a `profile_id` either in your `host_vars` or via `-e`, along with an `aws_region`. If you're not
|
||||||
|
already logged into AWS, your `profile_id` must be configured to be picked up automatically from your `~/.aws/config` or
|
||||||
|
`~/.aws/credentials` files with no additional login steps in order to deploy to AWS:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags dynamodb deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
or
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags dax deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Note the difference in tags: `dynamodb` and `dax`
|
||||||
|
|
||||||
|
#### Only Run the Benchmarkers in CRUD/READONLY mode
|
||||||
|
This assumes the CDK is already deployed and an EC2 instance already exists. This also assumes you either have the DAX
|
||||||
|
endpoint and the VPC ID hardcoded in your [host vars](./inventories/local/host_vars/localhost.yml), or you provide them via `-e`.
|
||||||
|
If you've already run a CDK deploy via Ansible, then you should not need to specify anything.
|
||||||
|
|
||||||
|
Additionally, You must also already be logged into the AWS CLI for
|
||||||
|
your target environment, or specify a `profile_id` either in your `host_vars` or via `-e`, along with an `aws_region`. If you're not
|
||||||
|
already logged into AWS, your `profile_id` must be configured to be picked up automatically from your `~/.aws/config` or
|
||||||
|
`~/.aws/credentials` files with no additional login steps in order to deploy to AWS:
|
||||||
|
|
||||||
|
**CRUD:**
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags crud deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
**read-only:**
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local --tags read-only deploy_benchmarker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Supported Variables
|
||||||
|
The following variables are supported to be specified via the `-e` argument when running the `deploy_benchmarker.yml`
|
||||||
|
playbook:
|
||||||
|
|
||||||
|
| Variable Name | Description | Required? |
|
||||||
|
|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|-----------|
|
||||||
|
| `profile_id` | The name of the AWS CLI profile you wish to deploy with; <br>Defaults to using the `AWS_PROFILE` environment variable | |
|
||||||
|
| `vpc_id` | The ID of the VPC in the AWS account you're deploying to where you want the CDK components created <br>Only required on first run only | * |
|
||||||
|
| `local_ip` | The public IP of your local machine; <br>Defaults to the response from `curl -s -L checkip.amazonaws.com` | |
|
||||||
|
| `ssh_key_name` | The name of the SSH key-pair that will be used when creating the EC2 instance to allow you SSH access to it; <br>Defaults to `$USER-dax-pair` | |
|
||||||
|
| `aws_account` | The account ID of the AWS account you're deploying into; <br>Defaults to the result of `aws sts get-caller-identity \| jq -r .Account` | |
|
||||||
|
| `base_table_name` | The base name to use when creating the DynamoDB table; <br>Defaults to `high-velocity-table` | |
|
||||||
|
| `cdk_action` | The action to perform when deploying the CDK; <br>Defaults to `deploy` | |
|
||||||
|
| `duration` | How long to run each simulation for; <br>Defaults to 1800 seconds | |
|
||||||
|
| `benchmarker` | Which benchmarker to run (i.e. `dynamodb` or `dax`) | |
|
||||||
|
| `dax_endpoint` | The DAX URI to use to hit the DAX cluster; <br>Only required when running the benchmarkers and without an initial CDK deploy) | * |
|
||||||
|
|
||||||
|
## Run Order
|
||||||
|
When first running from scratch, you'll want to run with the `init` tags first to initialize the Elastic Stack and install the prerequisites, then run again without any tags to actually
|
||||||
|
deploy everything and run the benchmarkers. If you only want to run the benchmarkers, run the `run_benchmarkers.yml` playbook, or specify the `run` tag.
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
You can generally get more information about your problem by adding `-vvv` to the end of your
|
||||||
|
`ansible-playbook` command. The more `v`'s you add, the more verbose the output and the more information
|
||||||
|
you will get. For example:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
ansible-playbook -i inventories/local -e cdk_action=destroy --tags 'elk,cdk' deploy_benchmarker.yml -vvv
|
||||||
|
```
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
- name: Deploy the benchmarking components
|
||||||
|
connection: local
|
||||||
|
hosts: local
|
||||||
|
gather_facts: yes
|
||||||
|
roles:
|
||||||
|
- { role: install_prerequisites, tags: [ never, prerequisites, init ] }
|
||||||
|
- { role: configure_elastic_stack, tags: elk }
|
||||||
|
- { role: deploy_cdk, tags: [ cdk, deploy ] }
|
||||||
|
- { role: destroy, tags: [ never, destroy ], cdk_action: destroy }
|
||||||
|
tasks:
|
||||||
|
- name: Populate the DynamoDB table with random data
|
||||||
|
shell:
|
||||||
|
chdir: ../scripts
|
||||||
|
cmd: ./randomly-generate-high-velocity-data.sh -i 5000
|
||||||
|
tags: deploy
|
||||||
|
|
||||||
|
- name: Build the benchmarkers using the Makefile
|
||||||
|
shell:
|
||||||
|
chdir: ../
|
||||||
|
cmd: make build
|
||||||
|
tags: deploy
|
||||||
|
|
||||||
|
- name: Upload the benchmarkers to the bastion host
|
||||||
|
hosts: bastion
|
||||||
|
gather_facts: yes
|
||||||
|
vars:
|
||||||
|
ssh_key_name: "{{ hostvars['localhost']['ssh_key_name'] }}"
|
||||||
|
ansible_ssh_private_key_file: "~/.ssh/{{ ssh_key_name }}.pem"
|
||||||
|
ansible_ssh_common_args: '-o StrictHostKeyChecking=no'
|
||||||
|
remote_user: ec2-user
|
||||||
|
tags: [ upload, deploy ]
|
||||||
|
tasks:
|
||||||
|
- copy:
|
||||||
|
src: "../{{ item }}"
|
||||||
|
dest: .
|
||||||
|
mode: 0777
|
||||||
|
loop:
|
||||||
|
- dynamodb-benchmarker
|
||||||
|
- dax-benchmarker
|
||||||
|
|
||||||
|
- import_playbook: run_benchmarkers.yml
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
user_name: "{{ lookup('env', 'USER') }}"
|
||||||
|
ssh_key_name: "{{ lookup('env', 'USER') }}-dax-pair"
|
||||||
|
profile_id: "{{ lookup('env', 'AWS_PROFILE') }}"
|
||||||
|
aws_region: "{{ lookup('env', 'AWS_REGION') }}"
|
||||||
|
stack_name: "{{ user_name }}-dax-benchmark-stack"
|
||||||
|
vpc_id:
|
||||||
|
base_table_name:
|
||||||
|
dax_endpoint:
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
local:
|
||||||
|
hosts:
|
||||||
|
localhost:
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
---
|
||||||
|
collections:
|
||||||
|
- name: community.general
|
||||||
|
- name: amazon.aws
|
||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,32 @@
|
|||||||
|
- name: Clone the docker-elk repo
|
||||||
|
git:
|
||||||
|
repo: https://github.com/deviantony/docker-elk.git
|
||||||
|
dest: ../../docker-elk
|
||||||
|
ignore_errors: yes
|
||||||
|
|
||||||
|
- name: Build the docker-elk stack just in case a pre-existing version of Elasticsearch needs its nodes upgraded
|
||||||
|
shell:
|
||||||
|
chdir: ../../docker-elk
|
||||||
|
cmd: docker compose build
|
||||||
|
|
||||||
|
- name: Start the docker-elk setup container
|
||||||
|
shell:
|
||||||
|
chdir: ../../docker-elk
|
||||||
|
cmd: docker-compose up setup
|
||||||
|
|
||||||
|
- name: Start the docker-elk stack
|
||||||
|
shell:
|
||||||
|
chdir: ../../docker-elk
|
||||||
|
cmd: docker compose up -d
|
||||||
|
|
||||||
|
- name: Wait 20 seconds for the ELK stack to start
|
||||||
|
pause:
|
||||||
|
seconds: 20
|
||||||
|
|
||||||
|
- name: Import the benchmarking dashboards into Kibana
|
||||||
|
shell:
|
||||||
|
cmd: >
|
||||||
|
curl -X POST http://localhost:5601/api/saved_objects/_import?overwrite=true
|
||||||
|
-H 'kbn-xsrf: true'
|
||||||
|
-u 'elastic:changeme'
|
||||||
|
--form file=@roles/configure_elastic_stack/files/benchmarker-dashboards.ndjson
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
- { import_tasks: init_elk_stack.yml, tags: [ never, init, init_elk ] }
|
||||||
|
- { import_tasks: stop_elk_stack.yml, tags: [ never, stop_elk ] }
|
||||||
|
|
||||||
|
- name: Start the docker-elk stack
|
||||||
|
shell:
|
||||||
|
chdir: ../../docker-elk
|
||||||
|
cmd: docker compose up -d
|
||||||
|
tags: deploy
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
- name: Stop the docker-elk stack
|
||||||
|
shell:
|
||||||
|
chdir: ../../docker-elk
|
||||||
|
cmd: docker compose down
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
- name: Check if a key-pair following the specified format already exists
|
||||||
|
stat:
|
||||||
|
path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
|
||||||
|
register: key_pair
|
||||||
|
changed_when: no
|
||||||
|
when: "'destroy' not in ansible_run_tags"
|
||||||
|
|
||||||
|
- block:
|
||||||
|
- name: Create a new key-pair
|
||||||
|
ec2_key:
|
||||||
|
name: "{{ ssh_key_name }}"
|
||||||
|
register: aws_key_pair
|
||||||
|
|
||||||
|
- name: Create the new pem file
|
||||||
|
file:
|
||||||
|
path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
|
||||||
|
state: touch
|
||||||
|
mode: '0400'
|
||||||
|
|
||||||
|
- name: Add the generated key-pair to the new file
|
||||||
|
blockinfile:
|
||||||
|
path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
|
||||||
|
block: "{{ aws_key_pair.key.private_key }}"
|
||||||
|
|
||||||
|
when:
|
||||||
|
- "'destroy' not in ansible_run_tags"
|
||||||
|
- not key_pair.stat.exists
|
||||||
|
|
||||||
|
- name: Fetch the current system's public IP
|
||||||
|
shell:
|
||||||
|
cmd: curl -s -L checkip.amazonaws.com
|
||||||
|
register: public_ip_resp
|
||||||
|
|
||||||
|
- name: Fetch the current AWS account ID
|
||||||
|
shell:
|
||||||
|
cmd: aws sts get-caller-identity | jq -r .Account
|
||||||
|
register: aws_account_resp
|
||||||
|
|
||||||
|
- name: Install CDK dependencies
|
||||||
|
npm:
|
||||||
|
ci: yes
|
||||||
|
path: ../cdk
|
||||||
|
|
||||||
|
- name: Bootstrapping the AWS environment
|
||||||
|
shell:
|
||||||
|
chdir: ../cdk
|
||||||
|
cmd: >
|
||||||
|
npm run build && yes | npm run cdk bootstrap --
|
||||||
|
--no-color --require-approval never
|
||||||
|
--profile {{ profile_id | default("personal") }}
|
||||||
|
-c vpcId={{ vpc_id }}
|
||||||
|
-c localIp={{ public_ip_resp.stdout }}
|
||||||
|
-c sshKeyName={{ ssh_key_name }}
|
||||||
|
-c awsAccount={{ aws_account_resp.stdout }}
|
||||||
|
-c baseTableName={{ base_table_name | default('') }}
|
||||||
|
|
||||||
|
- name: Deploying Benchmarking CDK
|
||||||
|
shell:
|
||||||
|
chdir: ../cdk
|
||||||
|
cmd: >
|
||||||
|
npm run build && yes | npm run cdk {{ cdk_action | default("deploy") }} --
|
||||||
|
--no-color --require-approval never
|
||||||
|
--profile {{ profile_id | default("personal") }}
|
||||||
|
-c vpcId={{ vpc_id }}
|
||||||
|
-c localIp={{ public_ip_resp.stdout }}
|
||||||
|
-c sshKeyName={{ ssh_key_name }}
|
||||||
|
-c awsAccount={{ aws_account_resp.stdout }}
|
||||||
|
-c baseTableName={{ base_table_name | default('') }}
|
||||||
|
register: cdk_response
|
||||||
|
|
||||||
|
- name: Benchmarking CDK deployment summary
|
||||||
|
debug:
|
||||||
|
msg: "{{ cdk_response.stderr_lines }}"
|
||||||
|
|
||||||
|
- block:
|
||||||
|
- name: Fetch the benchmark stack outputs
|
||||||
|
cloudformation_info:
|
||||||
|
stack_name: "{{ stack_name }}"
|
||||||
|
register: benchmark_stack
|
||||||
|
|
||||||
|
- name: Extracting the bastion host IP
|
||||||
|
set_fact:
|
||||||
|
bastion_host_ip: "{{ benchmark_stack.cloudformation[stack_name].stack_outputs['InstancePublicIp'] }}"
|
||||||
|
|
||||||
|
- name: Extracting DAX endpoint
|
||||||
|
set_fact:
|
||||||
|
dax_endpoint: "{{ benchmark_stack.cloudformation[stack_name].stack_outputs['DaxEndpoint'] }}"
|
||||||
|
|
||||||
|
- name: Setting the dax_endpoint variable in the host vars if it doesn't exist already
|
||||||
|
lineinfile:
|
||||||
|
path: inventories/local/host_vars/localhost.yml
|
||||||
|
line: "dax_endpoint: {{ dax_endpoint }}"
|
||||||
|
regexp: '^dax_endpoint:'
|
||||||
|
|
||||||
|
- name: Setting the vpc_id variable in the host vars if it doesn't exist already
|
||||||
|
lineinfile:
|
||||||
|
path: inventories/local/host_vars/localhost.yml
|
||||||
|
line: "vpc_id: {{ vpc_id }}"
|
||||||
|
regexp: '^vpc_id:'
|
||||||
|
|
||||||
|
- block:
|
||||||
|
- name: Setting the bastion host IP if it doesnt exist in the inventory
|
||||||
|
lineinfile:
|
||||||
|
path: inventories/local/hosts.yml
|
||||||
|
line: |
|
||||||
|
bastion:
|
||||||
|
hosts:
|
||||||
|
{{ bastion_host_ip }}:
|
||||||
|
regexp: 'bastion:\n\s*hosts:\n\s*(?:\d{1,3}\.){3}\d{1,3}:'
|
||||||
|
insertafter: EOF
|
||||||
|
|
||||||
|
- name: Add the bastion host to the bastion group
|
||||||
|
add_host:
|
||||||
|
name: "{{ bastion_host_ip }}"
|
||||||
|
groups: bastion
|
||||||
|
when:
|
||||||
|
- "'bastion' not in groups"
|
||||||
|
- "'bastion' not in group_names"
|
||||||
|
when: "'destroy' not in ansible_run_tags"
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
- name: Wipe away local Elastic Stack
|
||||||
|
shell:
|
||||||
|
chdir: ../../docker-elk
|
||||||
|
cmd: docker compose down -v
|
||||||
|
ignore_errors: yes
|
||||||
|
|
||||||
|
- name: Wipe away the ELK directory
|
||||||
|
file:
|
||||||
|
path: ../../docker-elk
|
||||||
|
state: absent
|
||||||
|
ignore_errors: yes
|
||||||
|
|
||||||
|
- name: Run CDK Destroy
|
||||||
|
import_role:
|
||||||
|
name:
|
||||||
|
deploy_cdk
|
||||||
|
|
||||||
|
- name: Delete the key-pair from AWS
|
||||||
|
ec2_key:
|
||||||
|
name: "{{ ssh_key_name }}"
|
||||||
|
state: absent
|
||||||
|
ignore_errors: yes
|
||||||
|
tags: [ never, destroy_key_pair ]
|
||||||
|
|
||||||
|
- name: Delete the key pair from your local machine
|
||||||
|
file:
|
||||||
|
path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
|
||||||
|
state: absent
|
||||||
|
ignore_errors: yes
|
||||||
|
tags: [ never, destroy_key_pair ]
|
||||||
|
|
||||||
|
- name: Remove the bastion host from the bastion host group
|
||||||
|
replace:
|
||||||
|
path: inventories/local/hosts.yml
|
||||||
|
replace: ''
|
||||||
|
regexp: '^bastion:\n\s*hosts:\n\s*(?:\d{1,3}\.){3}\d{1,3}:'
|
||||||
|
|
||||||
|
- name: Reset the dax_endpoint variable in the host vars
|
||||||
|
lineinfile:
|
||||||
|
path: inventories/local/host_vars/localhost.yml
|
||||||
|
line: 'dax_endpoint:'
|
||||||
|
regexp: '^dax_endpoint:'
|
||||||
|
|
||||||
|
- name: Reset the vpc_id variable in the host vars
|
||||||
|
lineinfile:
|
||||||
|
path: inventories/local/host_vars/localhost.yml
|
||||||
|
line: 'vpc_id:'
|
||||||
|
regexp: '^vpc_id:'
|
||||||
|
|
||||||
|
- name: Clean the repository using the Makefile
|
||||||
|
shell:
|
||||||
|
chdir: ../
|
||||||
|
cmd:
|
||||||
|
make clean
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
- name: Add Docker's official GPG key
|
||||||
|
apt_key:
|
||||||
|
url: https://download.docker.com/linux/ubuntu/gpg
|
||||||
|
keyring: /etc/apt/keyrings/docker.gpg
|
||||||
|
|
||||||
|
- name: Set up docker APT repository
|
||||||
|
apt_repository:
|
||||||
|
repo: "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
|
||||||
|
|
||||||
|
- name: Install the required APT dependencies
|
||||||
|
apt:
|
||||||
|
update_cache: yes
|
||||||
|
name:
|
||||||
|
- docker-ce
|
||||||
|
- docker-ce-cli
|
||||||
|
- docker-compose
|
||||||
|
- containerd.io
|
||||||
|
- docker-compose-plugin
|
||||||
|
- jq
|
||||||
|
- unzip
|
||||||
|
- curl
|
||||||
|
- git
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
- name: Check if AWS CLI is installed
|
||||||
|
shell:
|
||||||
|
cmd: hash aws 2> /dev/null
|
||||||
|
ignore_errors: yes
|
||||||
|
changed_when: no
|
||||||
|
register: awscli_installation_status
|
||||||
|
|
||||||
|
- block:
|
||||||
|
- name: Download the AWS CLI from AWS
|
||||||
|
unarchive:
|
||||||
|
src: https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
|
||||||
|
dest: "{{ ansible_env.HOME }}/Downloads"
|
||||||
|
group: "{{ user_name }}"
|
||||||
|
owner: "{{ user_name }}"
|
||||||
|
remote_src: yes
|
||||||
|
|
||||||
|
- name: Install the AWS CLI
|
||||||
|
shell:
|
||||||
|
cmd: "{{ ansible_env.HOME }}/Downloads/aws/install"
|
||||||
|
|
||||||
|
- name: Cleanup downloaded AWS installation files
|
||||||
|
file:
|
||||||
|
path: "{{ ansible_env.HOME }}/Downloads/aws/"
|
||||||
|
state: absent
|
||||||
|
|
||||||
|
when: awscli_installation_status.rc | int != 0
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
- name: Check if Go is installed
|
||||||
|
shell:
|
||||||
|
cmd: command -v go 2> /dev/null
|
||||||
|
ignore_errors: yes
|
||||||
|
changed_when: no
|
||||||
|
register: go_installation_status
|
||||||
|
|
||||||
|
- name: Install Go 1.20
|
||||||
|
unarchive:
|
||||||
|
src: https://go.dev/dl/go1.20.5.linux-amd64.tar.gz
|
||||||
|
dest: /usr/local
|
||||||
|
creates: /usr/local/go
|
||||||
|
remote_src: yes
|
||||||
|
become: yes
|
||||||
|
when: go_installation_status.rc | int != 0
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
- { import_tasks: aws_cli.yml, become: yes }
|
||||||
|
- import_tasks: rust.yml
|
||||||
|
- import_tasks: go.yml
|
||||||
|
- import_tasks: node.yml
|
||||||
|
- { import_tasks: apt.yml, become: yes }
|
||||||
|
|
||||||
|
- name: Install CDK
|
||||||
|
npm:
|
||||||
|
name: "{{ item }}"
|
||||||
|
global: yes
|
||||||
|
loop:
|
||||||
|
- aws-cdk
|
||||||
|
- typescript
|
||||||
|
|
||||||
|
- name: Check if golangci-lint is installed
|
||||||
|
shell:
|
||||||
|
cmd: command -v golangci-lint 2> /dev/null
|
||||||
|
ignore_errors: yes
|
||||||
|
changed_when: no
|
||||||
|
register: golangci_lint_installation_status
|
||||||
|
|
||||||
|
- name: Install golangci-lint
|
||||||
|
shell:
|
||||||
|
cmd: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.53.3
|
||||||
|
when: golangci_lint_installation_status.rc | int != 0
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
- name: Check if node is installed
|
||||||
|
shell:
|
||||||
|
cmd: hash node 2> /dev/null
|
||||||
|
ignore_errors: yes
|
||||||
|
changed_when: no
|
||||||
|
register: node_installation_status
|
||||||
|
|
||||||
|
- block:
|
||||||
|
- name: Install nvm
|
||||||
|
shell: >
|
||||||
|
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash
|
||||||
|
args:
|
||||||
|
creates: "{{ ansible_env.HOME }}/.nvm/nvm.sh"
|
||||||
|
|
||||||
|
- name: Install Node.JS
|
||||||
|
shell:
|
||||||
|
cmd: |
|
||||||
|
export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
|
||||||
|
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
|
||||||
|
nvm install node
|
||||||
|
|
||||||
|
- name: Add NVM exports to bashrc
|
||||||
|
lineinfile:
|
||||||
|
path: "{{ ansible_env.HOME }}/.bashrc"
|
||||||
|
line: 'export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"'
|
||||||
|
regexp: '^export NVM_DIR=.+'
|
||||||
|
|
||||||
|
- name: Add NVM script to bashrc
|
||||||
|
lineinfile:
|
||||||
|
path: "{{ ansible_env.HOME }}/.bashrc"
|
||||||
|
line: '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"'
|
||||||
|
regexp: '\[ -s |\$NVM_DIR/nvm\.sh \].+'
|
||||||
|
|
||||||
|
when: node_installation_status.rc | int != 0
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
- name: Check if rustup is installed
|
||||||
|
shell:
|
||||||
|
cmd: command -v rustup 2> /dev/null
|
||||||
|
ignore_errors: yes
|
||||||
|
changed_when: no
|
||||||
|
register: rustup_installation_status
|
||||||
|
|
||||||
|
- name: Install Rust via Rustup
|
||||||
|
shell: >
|
||||||
|
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||||
|
when: rustup_installation_status.rc | int != 0
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
- name: Get AWS Credentials
|
||||||
|
connection: local
|
||||||
|
hosts: local
|
||||||
|
gather_facts: yes
|
||||||
|
tags: [ run, deploy ]
|
||||||
|
tasks:
|
||||||
|
- name: Ensure the user is logged into their AWS CLI
|
||||||
|
assert:
|
||||||
|
that:
|
||||||
|
- aws_region is defined
|
||||||
|
- profile_id is defined
|
||||||
|
- dax_endpoint is defined
|
||||||
|
|
||||||
|
- name: Get the environment variables to set on the bastion host for the current AWS profile
|
||||||
|
shell:
|
||||||
|
cmd: aws configure export-credentials
|
||||||
|
register: aws_creds
|
||||||
|
|
||||||
|
- name: Register the aws_creds as a fact for the benchmarkers playbook to receive
|
||||||
|
set_fact:
|
||||||
|
aws_credentials: "{{ aws_creds.stdout }}"
|
||||||
|
|
||||||
|
- name: Run the benchmarkers
|
||||||
|
hosts: bastion
|
||||||
|
gather_facts: no
|
||||||
|
vars:
|
||||||
|
ssh_key_name: "{{ hostvars['localhost']['ssh_key_name'] }}"
|
||||||
|
ansible_ssh_private_key_file: "~/.ssh/{{ ssh_key_name }}.pem"
|
||||||
|
ansible_ssh_common_args: '-o StrictHostKeyChecking=no -R 9200:localhost:9200'
|
||||||
|
tags: [ run, deploy ]
|
||||||
|
remote_user: ec2-user
|
||||||
|
tasks:
|
||||||
|
- name: Run the DynamoDB benchmarker in CRUD mode
|
||||||
|
shell:
|
||||||
|
cmd: >
|
||||||
|
export AWS_ACCESS_KEY="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('AccessKeyId') }}";
|
||||||
|
export AWS_SECRET_ACCESS_KEY="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('SecretAccessKey') }}";
|
||||||
|
export AWS_SESSION_TOKEN="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('SessionToken') }}";
|
||||||
|
export AWS_CREDENTIAL_EXPIRATION="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('Expiration') }}";
|
||||||
|
export AWS_REGION="{{ hostvars['localhost']['aws_region'] }}";
|
||||||
|
./dynamodb-benchmarker -d "{{ duration | default(1800) | int }}" -t "{{ hostvars['localhost']['user_name'] }}"-high-velocity-table
|
||||||
|
executable: /bin/bash
|
||||||
|
tags:
|
||||||
|
- dynamodb
|
||||||
|
- crud
|
||||||
|
|
||||||
|
- name: Run the DynamoDB benchmarker in read-only mode
|
||||||
|
shell:
|
||||||
|
cmd: >
|
||||||
|
export AWS_ACCESS_KEY="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('AccessKeyId') }}";
|
||||||
|
export AWS_SECRET_ACCESS_KEY="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('SecretAccessKey') }}";
|
||||||
|
export AWS_SESSION_TOKEN="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('SessionToken') }}";
|
||||||
|
export AWS_CREDENTIAL_EXPIRATION="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('Expiration') }}";
|
||||||
|
export AWS_REGION="{{ hostvars['localhost']['aws_region'] }}";
|
||||||
|
./dynamodb-benchmarker -d "{{ duration | default(1800) | int }}" -t "{{ hostvars['localhost']['user_name'] }}"-high-velocity-table -r
|
||||||
|
executable: /bin/bash
|
||||||
|
tags:
|
||||||
|
- dynamodb
|
||||||
|
- read-only
|
||||||
|
|
||||||
|
- name: Run the DAX benchmarker in CRUD mode
|
||||||
|
shell:
|
||||||
|
cmd: >
|
||||||
|
export AWS_ACCESS_KEY="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('AccessKeyId') }}";
|
||||||
|
export AWS_SECRET_ACCESS_KEY="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('SecretAccessKey') }}";
|
||||||
|
export AWS_SESSION_TOKEN="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('SessionToken') }}";
|
||||||
|
export AWS_CREDENTIAL_EXPIRATION="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('Expiration') }}";
|
||||||
|
export AWS_REGION="{{ hostvars['localhost']['aws_region'] }}";
|
||||||
|
export DAX_ENDPOINT="{{ hostvars['localhost']['dax_endpoint'] }}";
|
||||||
|
unset cmd;
|
||||||
|
basecmd='./dax-benchmarker -c 100
|
||||||
|
-d 115
|
||||||
|
-t "{{ hostvars['localhost']['user_name'] }}"-high-velocity-table
|
||||||
|
-e "{{ hostvars['localhost']['dax_endpoint'] }}"';
|
||||||
|
for i in $(seq 1 9); do
|
||||||
|
cmd+="$basecmd & ";
|
||||||
|
done;
|
||||||
|
cmd+="$basecmd";
|
||||||
|
timeout -s SIGINT "{{ duration | default(1800) | int }}" bash -c "while :; do $cmd; done"
|
||||||
|
executable: /bin/bash
|
||||||
|
ignore_errors: yes
|
||||||
|
tags:
|
||||||
|
- dax
|
||||||
|
- crud
|
||||||
|
|
||||||
|
- name: Run the DAX benchmarker in read-only mode
|
||||||
|
shell:
|
||||||
|
cmd: >
|
||||||
|
export AWS_ACCESS_KEY="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('AccessKeyId') }}";
|
||||||
|
export AWS_SECRET_ACCESS_KEY="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('SecretAccessKey') }}";
|
||||||
|
export AWS_SESSION_TOKEN="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('SessionToken') }}";
|
||||||
|
export AWS_CREDENTIAL_EXPIRATION="{{ hostvars['localhost']['aws_credentials'] | community.general.json_query('Expiration') }}";
|
||||||
|
export AWS_REGION="{{ hostvars['localhost']['aws_region'] }}";
|
||||||
|
export DAX_ENDPOINT="{{ hostvars['localhost']['dax_endpoint'] }}";
|
||||||
|
unset cmd;
|
||||||
|
basecmd='./dax-benchmarker -c 100
|
||||||
|
-d 115
|
||||||
|
-r
|
||||||
|
-t "{{ hostvars['localhost']['user_name'] }}"-high-velocity-table
|
||||||
|
-e "{{ hostvars['localhost']['dax_endpoint'] }}"';
|
||||||
|
for i in $(seq 1 9); do
|
||||||
|
cmd+="$basecmd & ";
|
||||||
|
done;
|
||||||
|
cmd+="$basecmd";
|
||||||
|
timeout -s SIGINT "{{ duration | default(1800) | int }}" bash -c "while :; do $cmd; done"
|
||||||
|
executable: /bin/bash
|
||||||
|
ignore_errors: yes
|
||||||
|
tags:
|
||||||
|
- dax
|
||||||
|
- read-only
|
||||||
Executable
+264
@@ -0,0 +1,264 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
export PATH="$HOME"/.local/bin:$PATH
|
||||||
|
|
||||||
|
source scripts/logger.sh /tmp/benchmarker-tui.log
|
||||||
|
source scripts/ui_utils.sh
|
||||||
|
|
||||||
|
ANSIBLE_LOG_FILE=/tmp/ansible-playbook-output.log
|
||||||
|
rm "$ANSIBLE_LOG_FILE" > /dev/null 2>&1 &
|
||||||
|
|
||||||
|
verify-prerequisites() {
|
||||||
|
log-info "Verifying prerequisites"
|
||||||
|
declare prerequisites=(whiptail jq dialog)
|
||||||
|
|
||||||
|
if ! (aws sts get-caller-identity > /dev/null 2>&1); then
|
||||||
|
log-error "Must be logged into AWS CLI to use this script. Log into the target AWS account and run this script again" true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for application in "${prerequisites[@]}"; do
|
||||||
|
if ! (command -v "$application" > /dev/null 2>&1); then
|
||||||
|
log-warn "$application is required to run this script. Installing $application..."
|
||||||
|
sudo apt install "$application"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if ! (command -v ansible > /dev/null 2>&1); then
|
||||||
|
log-warn "Ansible is required to run this script. Installing Ansible..." true
|
||||||
|
sudo apt install python3-pip
|
||||||
|
pip3 install --user ansible jmespath
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! (ansible-galaxy collection list | grep -i "community.general\|amazon.aws" > /dev/null 2>&1); then
|
||||||
|
log-warn "Installing Ansible galaxy requirements..." true
|
||||||
|
cd ansible
|
||||||
|
ansible-galaxy install -r requirements.yml
|
||||||
|
cd -
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
initialize-environment() {
|
||||||
|
check-sudo-pass "Installing dependencies requires sudo permissions."
|
||||||
|
if [[ "$?" == 0 ]]; then
|
||||||
|
log-info "Sudo pass: $PASSWORD"
|
||||||
|
declare title="Initialize Local Environment"
|
||||||
|
|
||||||
|
if (prompt-yes-no "$title"); then
|
||||||
|
cd ansible
|
||||||
|
|
||||||
|
ansible-playbook -i inventories/local -e "ansible_become_password=$PASSWORD" --tags init deploy_benchmarker.yml > "$ANSIBLE_LOG_FILE" 2>&1 &
|
||||||
|
pid=$!
|
||||||
|
log-info "Running ansible-playbook 'deploy_benchmarker.yml' with the 'init' tag and logging output to file [$ANSIBLE_LOG_FILE]"
|
||||||
|
|
||||||
|
show-tail-box "$title" $pid "$ANSIBLE_LOG_FILE"
|
||||||
|
|
||||||
|
msg-box "Successfully initialized the local environment!"
|
||||||
|
log-info "Successfully initialized the local environment"
|
||||||
|
|
||||||
|
cd -
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
main-menu
|
||||||
|
}
|
||||||
|
|
||||||
|
deploy-and-run-benchmarkers() {
|
||||||
|
declare title="Deploy and Run Benchmarkers"
|
||||||
|
|
||||||
|
if (prompt-yes-no "$title"); then
|
||||||
|
if [[ -z $VPC_ID ]]; then
|
||||||
|
prompt-for-vpc-id
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd ansible
|
||||||
|
|
||||||
|
ansible-playbook -i inventories/local -e vpc_id="$VPC_ID" deploy_benchmarker.yml > "$ANSIBLE_LOG_FILE" 2>&1 &
|
||||||
|
pid=$!
|
||||||
|
log-info "Running ansible-playbook 'deploy_benchmarker.yml' with no tags and logging output to file [$ANSIBLE_LOG_FILE]"
|
||||||
|
|
||||||
|
show-tail-box "$title" $pid "$ANSIBLE_LOG_FILE"
|
||||||
|
|
||||||
|
msg-box "Successfully deployed and ran benchmarkers!"
|
||||||
|
log-info "Successfully deployed and ran benchmarkers"
|
||||||
|
|
||||||
|
cd -
|
||||||
|
fi
|
||||||
|
|
||||||
|
main-menu
|
||||||
|
}
|
||||||
|
|
||||||
|
destroy-all() {
|
||||||
|
declare title="Destroy Everything (Clean Slate)"
|
||||||
|
|
||||||
|
if (prompt-yes-no "$title"); then
|
||||||
|
cd ansible
|
||||||
|
|
||||||
|
ansible-playbook -i inventories/local --tags 'destroy,destroy_key_pair' deploy_benchmarker.yml > "$ANSIBLE_LOG_FILE" 2>&1 &
|
||||||
|
pid=$!
|
||||||
|
log-info "Running ansible-playbook 'deploy_benchmarker.yml' with [destroy,destroy_key_pair] tags and logging output to file [$ANSIBLE_LOG_FILE]"
|
||||||
|
|
||||||
|
show-tail-box "$title" $pid "$ANSIBLE_LOG_FILE"
|
||||||
|
|
||||||
|
msg-box "Successfully destroyed everything!"
|
||||||
|
log-info "Successfully destroyed everything"
|
||||||
|
|
||||||
|
cd -
|
||||||
|
fi
|
||||||
|
|
||||||
|
main-menu
|
||||||
|
}
|
||||||
|
|
||||||
|
randomly-populate-dynamodb() {
|
||||||
|
declare title="Populate DynamoDB with Random Data"
|
||||||
|
|
||||||
|
if (prompt-yes-no "$title"); then
|
||||||
|
./scripts/randomly-generate-high-velocity-data.sh /tmp/dynamodb-population.log &
|
||||||
|
pid=$!
|
||||||
|
log-info "Running randomly-generate-high-velocity-data script and logging to [$ANSIBLE_LOG_FILE]"
|
||||||
|
|
||||||
|
show-tail-box "$title" $pid "$ANSIBLE_LOG_FILE"
|
||||||
|
|
||||||
|
msg-box "Successfully populated DynamoDB with random data!"
|
||||||
|
log-info "Successfully populated DynamoDB with random data"
|
||||||
|
fi
|
||||||
|
|
||||||
|
main-menu
|
||||||
|
}
|
||||||
|
|
||||||
|
custom-selections() {
|
||||||
|
declare title="Customize What to Run (Advanced Mode)"
|
||||||
|
declare choices
|
||||||
|
declare tags=""
|
||||||
|
|
||||||
|
choices=$(whiptail --separate-output --checklist --fb "$title" "$BOX_HEIGHT" "$BOX_WIDTH" 13 \
|
||||||
|
"PREREQUISITES" "Install Prerequisites for Local Machine" OFF \
|
||||||
|
"INITIALIZE_ELK" "Initialize Local Elastic Stack" OFF \
|
||||||
|
"START_ELK" "Start Local Elastic Stack" OFF \
|
||||||
|
"DEPLOY_CDK" "Deploy CDK" OFF \
|
||||||
|
"UPLOAD_BIN" "Upload Benchmarker binaries" OFF \
|
||||||
|
"RUN_BENCHMARKERS" "Run Benchmarkers" OFF \
|
||||||
|
"STOP_ELK" "Stop Local Elastic Stack" OFF \
|
||||||
|
"DESTROY" "Destroy Everything except the SSH key" OFF \
|
||||||
|
"DESTROY_KEY" "Destroy the SSK Key" OFF \
|
||||||
|
"RUN_DYNAMODB" "Run the DynamoDB Benchmarkers" OFF \
|
||||||
|
"RUN_DAX" "Run the DAX Benchmarkers" OFF \
|
||||||
|
"RUN_CRUD" "Run the CRUD benchmarks for both the DynamoDB and DAX benchmarkers" OFF \
|
||||||
|
"RUN_READ_ONLY" "Run the READ-ONLY benchmarks for both the DynamoDB and DAX benchmarkers" OFF 3>&2 2>&1 1>&3)
|
||||||
|
|
||||||
|
if [[ -n $choices ]]; then
|
||||||
|
for choice in $choices; do
|
||||||
|
case "$choice" in
|
||||||
|
"PREREQUISITES")
|
||||||
|
tags+="prerequisites"
|
||||||
|
;;
|
||||||
|
"INITIALIZE_ELK")
|
||||||
|
tags+="init_elk"
|
||||||
|
;;
|
||||||
|
"START_ELK")
|
||||||
|
tags+="elk"
|
||||||
|
;;
|
||||||
|
"DEPLOY_CDK")
|
||||||
|
tags+="cdk"
|
||||||
|
;;
|
||||||
|
"UPLOAD_BIN")
|
||||||
|
tags+="upload"
|
||||||
|
;;
|
||||||
|
"RUN_BENCHMARKERS")
|
||||||
|
tags+="run"
|
||||||
|
if [[ -z $VPC_ID ]]; then
|
||||||
|
prompt-for-vpc-id
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
"STOP_ELK")
|
||||||
|
tags+="stop_elk"
|
||||||
|
;;
|
||||||
|
"DESTROY")
|
||||||
|
tags+="destroy"
|
||||||
|
;;
|
||||||
|
"DESTROY_KEY")
|
||||||
|
tags+="destroy_key_pair"
|
||||||
|
;;
|
||||||
|
"RUN_DYNAMODB")
|
||||||
|
tags+="dynamodb"
|
||||||
|
;;
|
||||||
|
"RUN_DAX")
|
||||||
|
tags+="dax"
|
||||||
|
;;
|
||||||
|
"RUN_CRUD")
|
||||||
|
tags+="crud"
|
||||||
|
;;
|
||||||
|
"RUN_READ_ONLY")
|
||||||
|
tags+="read-only"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
|
||||||
|
if (prompt-yes-no "$title"); then
|
||||||
|
cd ansible
|
||||||
|
|
||||||
|
ansible-playbook -i inventories/local --tags "$tags" deploy_benchmarker.yml > "$ANSIBLE_LOG_FILE" 2>&1 &
|
||||||
|
pid=$!
|
||||||
|
log-info "Running ansible-playbook 'deploy_benchmarker.yml' with [$tags] tags and logging output to file [$ANSIBLE_LOG_FILE]"
|
||||||
|
|
||||||
|
show-tail-box "$title" $pid "$ANSIBLE_LOG_FILE"
|
||||||
|
|
||||||
|
msg-box "Successfully ran custom tasks!"
|
||||||
|
log-info "Successfully ran custom tasks"
|
||||||
|
|
||||||
|
cd -
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
main-menu
|
||||||
|
}
|
||||||
|
|
||||||
|
prompt-for-vpc-id() {
|
||||||
|
readarray -t vpc_arr < <(aws ec2 describe-vpcs | jq -r '.Vpcs[] | "\(.VpcId) \((.Tags[]? | select(.Key | contains("Name")) | .Value) // "")"' | awk '{print($1, $2 == "" ? "-" : $2);}')
|
||||||
|
declare prompt=""
|
||||||
|
for item in "${vpc_arr[@]}"; do
|
||||||
|
prompt+="$item OFF "
|
||||||
|
done
|
||||||
|
|
||||||
|
VPC_ID=$(whiptail --fb --title "Select VPC" --radiolist "Select which VPC to use to deploy resources into" "$BOX_HEIGHT" "$BOX_WIDTH" "${vpc_arr[@]}" $prompt 3>&2 2>&1 1>&3)
|
||||||
|
}
|
||||||
|
|
||||||
|
main-menu() {
|
||||||
|
declare choice
|
||||||
|
choice=$(whiptail --fb --title "DynamoDB + DAX Benchmarker" --menu "Select an action" "$BOX_HEIGHT" "$BOX_WIDTH" 6 \
|
||||||
|
"I" "(I)nitialize local environment" \
|
||||||
|
"D" "(D)eploy and Run benchmarkers" \
|
||||||
|
"W" "(W)ipe away everything (Clean Slate)" \
|
||||||
|
"R" "(R)andomly populate DynamoDB" \
|
||||||
|
"C" "(C)ustom (Advanced)" \
|
||||||
|
"X" "E(x)it" 3>&2 2>&1 1>&3)
|
||||||
|
|
||||||
|
case $choice in
|
||||||
|
"I")
|
||||||
|
initialize-environment
|
||||||
|
;;
|
||||||
|
"D")
|
||||||
|
deploy-and-run-benchmarkers
|
||||||
|
;;
|
||||||
|
"W")
|
||||||
|
destroy-all
|
||||||
|
;;
|
||||||
|
"R")
|
||||||
|
randomly-populate-dynamodb
|
||||||
|
;;
|
||||||
|
"C")
|
||||||
|
msg-box "This is for advanced users only! Be sure you know what you're doing, as running some things at the same time can cause problems (like destroy and deploy)!"
|
||||||
|
custom-selections
|
||||||
|
;;
|
||||||
|
"X")
|
||||||
|
clear
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
verify-prerequisites
|
||||||
|
|
||||||
|
while :; do
|
||||||
|
main-menu
|
||||||
|
done
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
*.js
|
||||||
|
!jest.config.js
|
||||||
|
*.d.ts
|
||||||
|
node_modules
|
||||||
|
|
||||||
|
# CDK asset staging directory
|
||||||
|
.cdk.staging
|
||||||
|
cdk.out
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
*.ts
|
||||||
|
!*.d.ts
|
||||||
|
|
||||||
|
# CDK asset staging directory
|
||||||
|
.cdk.staging
|
||||||
|
cdk.out
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
# DynamoDB + DAX Benchmarker CDK
|
||||||
|
|
||||||
|
This CDK project deploys a DynamoDB table with a DAX cluster on top of it, and an EC2 instance to act as a bastion host for running benchmarking tests agasint DAX.
|
||||||
|
|
||||||
|
By default, the name of the DynamoDB table that is created is `$USER-high-velocity-table`.
|
||||||
|
By default, the name of the SSH key that is created for you is `$USER-dax-pair`
|
||||||
|
|
||||||
|
It should be noted that due to a bug in CDK, if you destroy the stack, you'll have to manually delete the SubnetGroup in DAX once everything else is deleted.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
You must be logged into the AWS CLI prior to running the CDK. Ensure you're logged into your target AWS account by running
|
||||||
|
`aws sts get-caller-identity`.
|
||||||
|
|
||||||
|
## Getting started
|
||||||
|
[NodeJS](https://nodejs.org/en) is required for development. Install NodeJS using the following commands, if it is
|
||||||
|
not already installed:
|
||||||
|
|
||||||
|
### Installing NodeJS
|
||||||
|
|
||||||
|
#### Windows
|
||||||
|
NodeJS can be installed on Windows using the [Chocolatey](https://chocolatey.org) package manager. If Chocolatey is not yet
|
||||||
|
installed on your system, first install it in a privileged PowerShell:
|
||||||
|
```powershell
|
||||||
|
Set-ExecutionPolicy Bypass -Scope Process -Force;
|
||||||
|
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072;
|
||||||
|
iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
|
||||||
|
```
|
||||||
|
|
||||||
|
Then, in a _non-privileged_ PowerShell session, install node:
|
||||||
|
```powershell
|
||||||
|
choco install nodejs
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Linux
|
||||||
|
NodeJS can be installed on Linux using [NVM](https://github.com/nvm-sh/nvm). First, install NVM:
|
||||||
|
```shell
|
||||||
|
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash
|
||||||
|
```
|
||||||
|
|
||||||
|
**Note:** The installation command was _not_ run with `sudo`. This is intentional, because if you install with `sudo`, then
|
||||||
|
`sudo` permissions will be required to install any and all new dependencies! You should avoid installing Node for the root
|
||||||
|
user!
|
||||||
|
|
||||||
|
|
||||||
|
Then, in order to use NVM to install NodeJS, you need to either restart your current shell session, or run the following:
|
||||||
|
```shell
|
||||||
|
export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
|
||||||
|
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
|
||||||
|
```
|
||||||
|
|
||||||
|
Now, install NodeJS:
|
||||||
|
```shell
|
||||||
|
nvm install node
|
||||||
|
```
|
||||||
|
|
||||||
|
### Installing dependent libraries
|
||||||
|
|
||||||
|
Once node is installed, run the following commands to install the NPM libraries:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
cd cdk
|
||||||
|
npm install -g aws-cdk
|
||||||
|
npm install -g typescript --save-dev
|
||||||
|
npm install
|
||||||
|
```
|
||||||
|
|
||||||
|
## CDK Arguments
|
||||||
|
This application depends on a few additional parameters in order to run. They can be specified in one of two ways: environment variables, or via the `-c` argument of the `cdk` command.
|
||||||
|
|
||||||
|
**Important:** Only one environment variable is required by the application, regardless of which parameter specification method you choose: `AWS_REGION`.
|
||||||
|
|
||||||
|
The following is a table of the **required** parameters for running the CDK
|
||||||
|
|
||||||
|
| Parameter Name | Environment Variable Name | Description |
|
||||||
|
|----------------|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
|
| `vpcId` | `VPC_ID` | The VPC ID you wish to deploy all of the stack's components into |
|
||||||
|
| `localIp` | `LOCAL_IP` | Your local IP; Used to allow SSH and Elasticsearch access in the EC2 security group |
|
||||||
|
| `sshKeyName` | `SSH_KEY_NAME` | The key name of your ssh key to allow you access to your EC2 instance. This should only be the name of the `.pem` file, and should not include the `.pem` extension. |
|
||||||
|
| `awsAccount` | `AWS_ACCOUNT` | The account ID of your AWS account. |
|
||||||
|
| `awsRegion` | `AWS_REGION` | The AWS region to deploy this stack and its components into |
|
||||||
|
|
||||||
|
### Optional Parameters
|
||||||
|
It is sometimes necessary to tweak the deployment a bit for different use cases. The CDK can be tweaked with the following parameters:
|
||||||
|
|
||||||
|
| Parameter Name | Default Value | Description |
|
||||||
|
|-----------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------|
|
||||||
|
| `baseTableName` | `high-velocity-table` | This is the base name for the table. All tables created by the stack will be prefixed with `$USER` to prevent conflicts |
|
||||||
|
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import 'source-map-support/register';
|
||||||
|
import * as cdk from 'aws-cdk-lib';
|
||||||
|
import {EnvironmentProps} from '../lib/types';
|
||||||
|
import { DaxBenchmarkingStack } from '../lib/dax-benchmarking-stack';
|
||||||
|
|
||||||
|
const app = new cdk.App();
|
||||||
|
const user = process.env.USER || '';
|
||||||
|
let vpcId = app.node.tryGetContext('vpcId');
|
||||||
|
if (!vpcId) {
|
||||||
|
if (!process.env.VPC_ID) {
|
||||||
|
throw new Error('vpcId is a required parameter. Specify it with `-c vpcId=someId`, or by setting the VPC_ID environment variable');
|
||||||
|
} else {
|
||||||
|
vpcId = process.env.VPC_ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let localIp = app.node.tryGetContext('localIp');
|
||||||
|
if (!localIp) {
|
||||||
|
if (!process.env.LOCAL_IP) {
|
||||||
|
throw new Error('Local IP is a required parameter. Specify it with `-c localIp=XXX.XXX.XXX.XXX`, or by setting the LOCAL_IP environment variable');
|
||||||
|
} else {
|
||||||
|
localIp = process.env.LOCAL_IP
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let sshKeyName = app.node.tryGetContext('sshKeyName');
|
||||||
|
if (!sshKeyName) {
|
||||||
|
if (!process.env.SSH_KEY_NAME) {
|
||||||
|
sshKeyName = `${user}-dax-pair`;
|
||||||
|
} else {
|
||||||
|
sshKeyName = process.env.SSH_KEY_NAME;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let awsAccount = app.node.tryGetContext('awsAccount');
|
||||||
|
if (!awsAccount) {
|
||||||
|
if (!process.env.AWS_ACCOUNT) {
|
||||||
|
throw new Error('awsAccount is a required parameter. Specify it with `-c awsAccount=1234567890`, or by setting the AWS_ACCOUNT environment variable.');
|
||||||
|
} else {
|
||||||
|
awsAccount = process.env.AWS_ACCOUNT;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let awsRegion = app.node.tryGetContext('awsRegion');
|
||||||
|
if (!awsRegion) {
|
||||||
|
if (!process.env.AWS_REGION) {
|
||||||
|
throw new Error('The `AWS_REGION` environment variable was not set. It must be set in order to use this application.');
|
||||||
|
} else {
|
||||||
|
awsRegion = process.env.AWS_REGION
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let baseTableName = app.node.tryGetContext('baseTableName');
|
||||||
|
if (!baseTableName) {
|
||||||
|
baseTableName = 'high-velocity-table'
|
||||||
|
}
|
||||||
|
|
||||||
|
const environmentProps: EnvironmentProps = {
|
||||||
|
env: { account: awsAccount, region: awsRegion },
|
||||||
|
baseTableName,
|
||||||
|
removalPolicy: cdk.RemovalPolicy.DESTROY,
|
||||||
|
user,
|
||||||
|
vpcId,
|
||||||
|
localIp,
|
||||||
|
sshKeyName
|
||||||
|
};
|
||||||
|
|
||||||
|
new DaxBenchmarkingStack(app, `${user}-dax-benchmark-stack`, environmentProps);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
module.exports = {
|
||||||
|
testEnvironment: 'node',
|
||||||
|
roots: ['<rootDir>/test'],
|
||||||
|
testMatch: ['**/*.test.ts'],
|
||||||
|
transform: {
|
||||||
|
'^.+\\.tsx?$': 'ts-jest'
|
||||||
|
}
|
||||||
|
};
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import { Tags } from "aws-cdk-lib";
|
||||||
|
import { Construct } from "constructs";
|
||||||
|
import { EnvironmentProps } from "./types";
|
||||||
|
import { Instance, InstanceClass, InstanceSize, InstanceType, MachineImage, Peer, Port, SecurityGroup, SubnetType, Vpc } from "aws-cdk-lib/aws-ec2";
|
||||||
|
import { IRole, Role, ServicePrincipal } from "aws-cdk-lib/aws-iam";
|
||||||
|
|
||||||
|
export class DaxBastionHost extends Construct {
|
||||||
|
public readonly instanceRole: IRole;
|
||||||
|
public readonly instance: Instance;
|
||||||
|
|
||||||
|
constructor(scope: Construct, id: string, environmentProps: EnvironmentProps, daxSecurityGroup: SecurityGroup) {
|
||||||
|
super(scope, id);
|
||||||
|
|
||||||
|
Tags.of(this).add('Application', 'dynamodb-dax-benchmarker');
|
||||||
|
|
||||||
|
const { removalPolicy, user, vpcId, localIp, sshKeyName } = environmentProps;
|
||||||
|
const localIpCidr = `${localIp}/32`;
|
||||||
|
|
||||||
|
const vpc = Vpc.fromLookup(this, 'Vpc', { vpcId });
|
||||||
|
|
||||||
|
const bastionHostSecurityGroup = new SecurityGroup(this, `${user}-dax-sg`, {
|
||||||
|
vpc,
|
||||||
|
description: `Allow SSH, Elasticsearch, and DAX access for ${user}`,
|
||||||
|
securityGroupName: `${user}-dax-bastion-host-sg`
|
||||||
|
});
|
||||||
|
bastionHostSecurityGroup.applyRemovalPolicy(removalPolicy);
|
||||||
|
bastionHostSecurityGroup.addIngressRule(Peer.ipv4(localIpCidr), Port.tcp(22), "Allow SSH access to this instance from the users public IP");
|
||||||
|
bastionHostSecurityGroup.addIngressRule(Peer.ipv4(localIpCidr), Port.tcp(9200), "Allow the host to communicate with the users locally running Elasticsearch cluster");
|
||||||
|
bastionHostSecurityGroup.addIngressRule(daxSecurityGroup, Port.allTraffic());
|
||||||
|
daxSecurityGroup.addIngressRule(bastionHostSecurityGroup, Port.allTraffic());
|
||||||
|
|
||||||
|
this.instanceRole = new Role(this, `${user}-bastion-role`, {
|
||||||
|
roleName: `${user}-bastion-role`,
|
||||||
|
assumedBy: new ServicePrincipal('ec2.amazonaws.com'),
|
||||||
|
});
|
||||||
|
this.instanceRole.applyRemovalPolicy(removalPolicy);
|
||||||
|
|
||||||
|
this.instance = new Instance(this, `${user}-dax-bastion-host`, {
|
||||||
|
vpc,
|
||||||
|
instanceType: InstanceType.of(InstanceClass.T2, InstanceSize.SMALL),
|
||||||
|
machineImage: MachineImage.latestAmazonLinux2023(),
|
||||||
|
instanceName: `${user}-dax-bastion-host`,
|
||||||
|
keyName: sshKeyName,
|
||||||
|
vpcSubnets: vpc.selectSubnets({ subnetType: SubnetType.PUBLIC }),
|
||||||
|
securityGroup: bastionHostSecurityGroup,
|
||||||
|
role: this.instanceRole
|
||||||
|
});
|
||||||
|
this.instance.applyRemovalPolicy(removalPolicy);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
import { Construct } from "constructs";
|
||||||
|
import { EnvironmentProps } from "./types";
|
||||||
|
import { CfnOutput, Stack, Tags } from "aws-cdk-lib";
|
||||||
|
import { CfnCluster, CfnSubnetGroup } from "aws-cdk-lib/aws-dax";
|
||||||
|
import { Effect, PolicyDocument, PolicyStatement, Role, ServicePrincipal } from "aws-cdk-lib/aws-iam";
|
||||||
|
import { SecurityGroup, SubnetType, Vpc } from "aws-cdk-lib/aws-ec2";
|
||||||
|
import { DynamoDbBenchmarkTable } from "./dynamodb";
|
||||||
|
import { DaxBastionHost } from "./bastion-host";
|
||||||
|
|
||||||
|
export class DaxBenchmarkingStack extends Stack {
|
||||||
|
constructor(scope: Construct, id: string, environmentProps: EnvironmentProps) {
|
||||||
|
super(scope, id, environmentProps);
|
||||||
|
|
||||||
|
Tags.of(this).add('Application', 'dynamodb-dax-benchmarker');
|
||||||
|
|
||||||
|
const { user, removalPolicy, vpcId } = environmentProps;
|
||||||
|
const { table } = new DynamoDbBenchmarkTable(this, `${user}-dynamodb-benchmark-table`, environmentProps);
|
||||||
|
|
||||||
|
const vpc = Vpc.fromLookup(this, 'Vpc', { vpcId });
|
||||||
|
|
||||||
|
const daxSecurityGroup = new SecurityGroup(this, `${user}-dax-sg`, {
|
||||||
|
vpc,
|
||||||
|
securityGroupName: `${user}-dax-sg`
|
||||||
|
});
|
||||||
|
daxSecurityGroup.applyRemovalPolicy(removalPolicy);
|
||||||
|
|
||||||
|
const { instanceRole, instance } = new DaxBastionHost(this, `${user}-dax-bastion-host`, environmentProps, daxSecurityGroup);
|
||||||
|
|
||||||
|
const daxClusterName = `${user}-high-velocity`;
|
||||||
|
const daxFullAccessPolicy = new PolicyStatement({
|
||||||
|
effect: Effect.ALLOW,
|
||||||
|
actions: [
|
||||||
|
"dynamodb:BatchGetItem",
|
||||||
|
"dynamodb:GetItem",
|
||||||
|
"dynamodb:Query",
|
||||||
|
"dynamodb:Scan",
|
||||||
|
"dynamodb:BatchWriteItem",
|
||||||
|
"dynamodb:DeleteItem",
|
||||||
|
"dynamodb:PutItem",
|
||||||
|
"dynamodb:UpdateItem",
|
||||||
|
"dynamodb:DescribeLimits",
|
||||||
|
"dynamodb:DescribeTimeToLive",
|
||||||
|
"dynamodb:DescribeTable",
|
||||||
|
"dynamodb:ListTables"
|
||||||
|
],
|
||||||
|
resources: [table.tableArn]
|
||||||
|
});
|
||||||
|
|
||||||
|
const daxServiceRole = new Role(this, `${daxClusterName}-role`, {
|
||||||
|
assumedBy: new ServicePrincipal("dax.amazonaws.com"),
|
||||||
|
inlinePolicies: {
|
||||||
|
DAXFullAccess: new PolicyDocument({
|
||||||
|
statements: [daxFullAccessPolicy]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
});
|
||||||
|
daxServiceRole.applyRemovalPolicy(removalPolicy);
|
||||||
|
|
||||||
|
instanceRole.addToPrincipalPolicy(daxFullAccessPolicy);
|
||||||
|
|
||||||
|
const subnetGroup = new CfnSubnetGroup(this, `${user}-dax-subnet-group`, {
|
||||||
|
subnetIds: vpc.selectSubnets({
|
||||||
|
subnetType: SubnetType.PRIVATE_ISOLATED
|
||||||
|
}).subnetIds,
|
||||||
|
subnetGroupName: `${user}-dax-subnet-group`,
|
||||||
|
});
|
||||||
|
subnetGroup.applyRemovalPolicy(removalPolicy);
|
||||||
|
|
||||||
|
const daxCluster = new CfnCluster(this, daxClusterName, {
|
||||||
|
iamRoleArn: daxServiceRole.roleArn,
|
||||||
|
nodeType: 'dax.r5.large',
|
||||||
|
replicationFactor: 3,
|
||||||
|
securityGroupIds: [daxSecurityGroup.securityGroupId],
|
||||||
|
subnetGroupName: subnetGroup.subnetGroupName,
|
||||||
|
availabilityZones: vpc.availabilityZones,
|
||||||
|
clusterEndpointEncryptionType: 'TLS',
|
||||||
|
clusterName: daxClusterName,
|
||||||
|
sseSpecification: {
|
||||||
|
sseEnabled: true,
|
||||||
|
}
|
||||||
|
});
|
||||||
|
daxCluster.applyRemovalPolicy(removalPolicy);
|
||||||
|
daxCluster.addDependency(subnetGroup);
|
||||||
|
|
||||||
|
new CfnOutput(this, 'DaxEndpoint', { value: daxCluster.attrClusterDiscoveryEndpointUrl });
|
||||||
|
new CfnOutput(this, 'InstanceId', { value: instance.instanceId });
|
||||||
|
new CfnOutput(this, 'InstancePublicIp', { value: instance.instancePublicIp });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
import {Tags} from "aws-cdk-lib";
|
||||||
|
import {Construct} from "constructs";
|
||||||
|
import {EnvironmentProps} from "./types";
|
||||||
|
import {AttributeType, BillingMode, Table} from "aws-cdk-lib/aws-dynamodb";
|
||||||
|
|
||||||
|
export class DynamoDbBenchmarkTable extends Construct {
|
||||||
|
public readonly table: Table;
|
||||||
|
|
||||||
|
constructor(scope: Construct, id: string, environmentProps: EnvironmentProps) {
|
||||||
|
super(scope, id);
|
||||||
|
|
||||||
|
Tags.of(this).add('Application', 'dynamodb-dax-benchmarker');
|
||||||
|
|
||||||
|
const { baseTableName, removalPolicy, user } = environmentProps;
|
||||||
|
const tableName = `${user}-${baseTableName}`;
|
||||||
|
|
||||||
|
this.table = new Table(this, tableName, {
|
||||||
|
partitionKey: {
|
||||||
|
name: 'id',
|
||||||
|
type: AttributeType.STRING
|
||||||
|
},
|
||||||
|
tableName,
|
||||||
|
removalPolicy,
|
||||||
|
billingMode: BillingMode.PAY_PER_REQUEST
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import {RemovalPolicy, StackProps} from "aws-cdk-lib";
|
||||||
|
|
||||||
|
export interface EnvironmentProps extends StackProps {
|
||||||
|
readonly baseTableName: string
|
||||||
|
readonly removalPolicy: RemovalPolicy
|
||||||
|
readonly user: string
|
||||||
|
readonly vpcId: string
|
||||||
|
readonly localIp: string
|
||||||
|
readonly sshKeyName: string
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
module github.com/Dark-Alex-17/dynamodb-benchmarker
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/aws/aws-cdk-go/awscdk/v2 v2.88.0
|
||||||
|
github.com/aws/aws-dax-go v1.2.12
|
||||||
|
github.com/aws/aws-sdk-go v1.44.301
|
||||||
|
github.com/aws/constructs-go/constructs/v10 v10.2.69
|
||||||
|
github.com/aws/jsii-runtime-go v1.85.0
|
||||||
|
github.com/elastic/go-elasticsearch/v8 v8.8.2
|
||||||
|
github.com/google/uuid v1.3.0
|
||||||
|
github.com/sirupsen/logrus v1.9.3
|
||||||
|
gopkg.in/loremipsum.v1 v1.1.2
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/Masterminds/semver/v3 v3.2.1 // indirect
|
||||||
|
github.com/cdklabs/awscdk-asset-awscli-go/awscliv1/v2 v2.2.200 // indirect
|
||||||
|
github.com/cdklabs/awscdk-asset-kubectl-go/kubectlv20/v2 v2.1.2 // indirect
|
||||||
|
github.com/cdklabs/awscdk-asset-node-proxy-agent-go/nodeproxyagentv5/v2 v2.0.165 // indirect
|
||||||
|
github.com/fatih/color v1.15.0 // indirect
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
|
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||||
|
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||||
|
github.com/spf13/pflag v1.0.5 // indirect
|
||||||
|
github.com/yuin/goldmark v1.4.13 // indirect
|
||||||
|
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
|
||||||
|
golang.org/x/mod v0.12.0 // indirect
|
||||||
|
golang.org/x/tools v0.11.0 // indirect
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e // indirect
|
||||||
|
github.com/elastic/elastic-transport-go/v8 v8.3.0 // indirect
|
||||||
|
github.com/gofrs/uuid v3.3.0+incompatible // indirect
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
|
github.com/spf13/cobra v1.7.0
|
||||||
|
golang.org/x/sys v0.10.0 // indirect
|
||||||
|
)
|
||||||
+245
@@ -0,0 +1,245 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"math/rand"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/models"
|
||||||
|
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/simulators"
|
||||||
|
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/utils"
|
||||||
|
"github.com/aws/aws-dax-go/dax"
|
||||||
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
|
"github.com/aws/aws-sdk-go/service/dynamodb"
|
||||||
|
"github.com/elastic/go-elasticsearch/v8"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var concurrentSimulations, buffer, attributes, duration int
|
||||||
|
var username, password, index, endpoint, tableName string
|
||||||
|
var readOnly bool
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
rootCmd := &cobra.Command{
|
||||||
|
Use: "dax-benchmarker",
|
||||||
|
Short: "A CLI tool for simulating heavy usage against DAX and publishing metrics to an Elastic Stack for analysis",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
if err := validateFlags(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
execute()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
rootCmd.PersistentFlags().IntVarP(&concurrentSimulations, "concurrent-simulations", "c", 1000, "The number of concurrent simulations to run")
|
||||||
|
rootCmd.PersistentFlags().IntVarP(&buffer, "buffer", "b", 500, "The buffer size of the Elasticsearch goroutine's channel")
|
||||||
|
rootCmd.PersistentFlags().IntVarP(&attributes, "attributes", "a", 5, "The number of attributes to use when populating and querying the DynamoDB table; minimum value of 1")
|
||||||
|
rootCmd.PersistentFlags().IntVarP(&duration, "duration", "d", 1800, "The length of time (in seconds) to run the benchmark for")
|
||||||
|
rootCmd.PersistentFlags().StringVarP(&username, "username", "u", "elastic", "Local Elasticsearch cluster username")
|
||||||
|
rootCmd.PersistentFlags().StringVarP(&password, "password", "p", "changeme", "Local Elasticsearch cluster password")
|
||||||
|
rootCmd.PersistentFlags().StringVarP(&index, "index", "i", "dax", "The Elasticsearch Index to insert data into")
|
||||||
|
rootCmd.PersistentFlags().StringVarP(&tableName, "table", "t", fmt.Sprintf("%s-high-velocity-table", os.Getenv("USER")), "The DynamoDB table to perform operations against")
|
||||||
|
rootCmd.PersistentFlags().StringVarP(&endpoint, "endpoint", "e", "", "The DAX endpoint to hit when running simulations (assumes secure endpoint, so do not specify port)")
|
||||||
|
rootCmd.PersistentFlags().BoolVarP(&readOnly, "read-only", "r", false, "Whether to run a read-only scenario for benchmarking")
|
||||||
|
|
||||||
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
log.Errorf("Something went wrong parsing CLI args and executing the client: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateFlags() error {
|
||||||
|
if len(endpoint) == 0 {
|
||||||
|
daxEndpointEnvironmentVariable := os.Getenv("DAX_ENDPOINT")
|
||||||
|
if len(daxEndpointEnvironmentVariable) == 0 {
|
||||||
|
return errors.New("a DAX endpoint must be specified either via -e, --endpoint or via the DAX_ENDPOINT environment variable")
|
||||||
|
} else {
|
||||||
|
endpoint = daxEndpointEnvironmentVariable
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if attributes < 1 {
|
||||||
|
return errors.New("the number of attributes cannot be lower than 1")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(os.Getenv("AWS_REGION")) == 0 {
|
||||||
|
return errors.New("an AWS region must be specified using the AWS_REGION environment variable")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func execute() {
|
||||||
|
esChan := make(chan models.DynamoDbSimulationMetrics, buffer)
|
||||||
|
defer close(esChan)
|
||||||
|
daxEndpoint := fmt.Sprintf("%s:9111", endpoint)
|
||||||
|
region := os.Getenv("AWS_REGION")
|
||||||
|
sess := session.Must(session.NewSession(&aws.Config{
|
||||||
|
Credentials: credentials.NewChainCredentials([]credentials.Provider{&credentials.EnvProvider{}}),
|
||||||
|
Endpoint: &daxEndpoint,
|
||||||
|
Region: ®ion,
|
||||||
|
}))
|
||||||
|
|
||||||
|
if _, err := sess.Config.Credentials.Get(); err != nil {
|
||||||
|
log.Errorf("credentials were not loaded! %v+", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := dax.NewWithSession(*sess)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("unable to initialize dax client %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
partitionKeys, err := scanAllPartitionKeys(client)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unable to fetch partition keys! Simulation failed! %v+", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
go startElasticsearchPublisher(esChan)
|
||||||
|
|
||||||
|
for i := 0; i < concurrentSimulations; i++ {
|
||||||
|
go simulationLoop(esChan, client, partitionKeys)
|
||||||
|
}
|
||||||
|
|
||||||
|
duration, err := time.ParseDuration(strconv.Itoa(duration) + "s")
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unable to create duration from the provided time: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
<-time.After(duration)
|
||||||
|
}
|
||||||
|
|
||||||
|
func startElasticsearchPublisher(c <-chan models.DynamoDbSimulationMetrics) {
|
||||||
|
config := elasticsearch.Config{
|
||||||
|
Addresses: []string{
|
||||||
|
"http://localhost:9200",
|
||||||
|
},
|
||||||
|
Username: username,
|
||||||
|
Password: password,
|
||||||
|
}
|
||||||
|
esClient, err := elasticsearch.NewClient(config)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("unable to initialize elasticsearch client %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mapping := `{
|
||||||
|
"properties": {
|
||||||
|
"timestamp": {
|
||||||
|
"type": "date"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
|
||||||
|
log.Infof("Setting the explicit mappings for the %s index", index)
|
||||||
|
if _, err := esClient.Indices.Create(index); err != nil {
|
||||||
|
log.Warnf("Unable to create the %s index. Encountered the following error: %v", index, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := esClient.Indices.PutMapping([]string{index}, strings.NewReader(mapping)); err != nil {
|
||||||
|
log.Errorf("unable to create mapping for the %s index! %v+", index, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for metric := range c {
|
||||||
|
log.Info("Publishing metrics to Elasticsearch...")
|
||||||
|
|
||||||
|
data, _ := json.Marshal(metric)
|
||||||
|
_, err := esClient.Index(index, bytes.NewReader(data))
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Was unable to publish metrics to Elasticsearch! Received a non 2XX response")
|
||||||
|
} else {
|
||||||
|
log.Info("Successfully published metrics to Elasticsearch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func simulationLoop(c chan<- models.DynamoDbSimulationMetrics, client *dax.Dax, partitionKeys []string) {
|
||||||
|
for {
|
||||||
|
metrics := new(models.DynamoDbSimulationMetrics)
|
||||||
|
metrics.Successful = true
|
||||||
|
metrics.Timestamp = time.Now().UnixNano() / 1e6
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
if readOnly {
|
||||||
|
log.Info("Running a read-only simulation...")
|
||||||
|
metrics.Scenario = models.ScenarioReadOnly.String()
|
||||||
|
runReadOnlySimulation(client, metrics, partitionKeys)
|
||||||
|
} else {
|
||||||
|
log.Info("Running a CRUD simulation...")
|
||||||
|
metrics.Scenario = models.ScenarioCrud.String()
|
||||||
|
runCrudSimulation(client, metrics, partitionKeys)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("Simulation completed successfully!")
|
||||||
|
|
||||||
|
duration := time.Since(startTime).Microseconds()
|
||||||
|
millisecondDuration := float64(duration) / 1000
|
||||||
|
metrics.SimulationTime = &millisecondDuration
|
||||||
|
|
||||||
|
log.Infof("Metrics: %v+", metrics)
|
||||||
|
|
||||||
|
c <- *metrics
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runReadOnlySimulation(client *dax.Dax, metrics *models.DynamoDbSimulationMetrics, partitionKeys []string) {
|
||||||
|
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||||
|
time.Sleep(time.Duration(r.Intn(16)))
|
||||||
|
|
||||||
|
metrics.Operation = models.DynamoRead.String()
|
||||||
|
simulators.SimulateReadOperation(client, tableName, partitionKeys, metrics)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runCrudSimulation(client *dax.Dax, metrics *models.DynamoDbSimulationMetrics, partitionKeys []string) {
|
||||||
|
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||||
|
operation := r.Intn(3)
|
||||||
|
log.Infof("Operation number: %d", operation)
|
||||||
|
|
||||||
|
switch operation {
|
||||||
|
case int(models.DynamoRead):
|
||||||
|
metrics.Operation = models.DynamoRead.String()
|
||||||
|
simulators.SimulateReadOperation(client, tableName, partitionKeys, metrics)
|
||||||
|
case int(models.DynamoWrite):
|
||||||
|
metrics.Operation = models.DynamoWrite.String()
|
||||||
|
simulators.SimulateWriteOperation(client, tableName, attributes, metrics)
|
||||||
|
case int(models.DynamoUpdate):
|
||||||
|
metrics.Operation = models.DynamoUpdate.String()
|
||||||
|
simulators.SimulateUpdateOperation(client, tableName, attributes, metrics)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanAllPartitionKeys(client *dax.Dax) ([]string, error) {
|
||||||
|
log.Info("Fetching a large list of partition keys to randomly read...")
|
||||||
|
projectionExpression := "id"
|
||||||
|
var limit int64 = 10000
|
||||||
|
|
||||||
|
response, err := client.Scan(&dynamodb.ScanInput{
|
||||||
|
TableName: &tableName,
|
||||||
|
Limit: &limit,
|
||||||
|
ProjectionExpression: &projectionExpression,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unable to fetch partition keys! %v", err)
|
||||||
|
return []string{}, err
|
||||||
|
} else {
|
||||||
|
log.Info("Fetched partition keys!")
|
||||||
|
keys := make([]string, 100)
|
||||||
|
|
||||||
|
for _, itemsMap := range response.Items {
|
||||||
|
keys = append(keys, *utils.MapValues(itemsMap)[0].S)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Found a total of %d keys", len(keys))
|
||||||
|
|
||||||
|
return keys, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package models
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math/rand"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go/service/dynamodb"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"gopkg.in/loremipsum.v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DynamoOperation int
|
||||||
|
|
||||||
|
const (
|
||||||
|
DynamoRead DynamoOperation = iota
|
||||||
|
DynamoWrite
|
||||||
|
DynamoUpdate
|
||||||
|
)
|
||||||
|
|
||||||
|
func (d DynamoOperation) String() string {
|
||||||
|
switch d {
|
||||||
|
case DynamoRead:
|
||||||
|
return "read"
|
||||||
|
case DynamoWrite:
|
||||||
|
return "write"
|
||||||
|
case DynamoUpdate:
|
||||||
|
return "update"
|
||||||
|
default:
|
||||||
|
return "read"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type Scenario int
|
||||||
|
|
||||||
|
const (
|
||||||
|
ScenarioCrud Scenario = iota
|
||||||
|
ScenarioReadOnly
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s Scenario) String() string {
|
||||||
|
switch s {
|
||||||
|
case ScenarioCrud:
|
||||||
|
return "crud"
|
||||||
|
case ScenarioReadOnly:
|
||||||
|
return "readOnly"
|
||||||
|
default:
|
||||||
|
return "crud"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type BenchmarkingItem map[string]*dynamodb.AttributeValue
|
||||||
|
|
||||||
|
func NewBenchmarkingItem(attributes int) BenchmarkingItem {
|
||||||
|
benchmarkingItem := make(map[string]*dynamodb.AttributeValue)
|
||||||
|
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||||
|
loremIpsumGenerator := loremipsum.NewWithSeed(time.Now().UnixNano())
|
||||||
|
id := uuid.New().String()
|
||||||
|
benchmarkingItem["id"] = &dynamodb.AttributeValue{S: &id}
|
||||||
|
|
||||||
|
for i := 0; i < attributes; i++ {
|
||||||
|
switch i % 2 {
|
||||||
|
case 1:
|
||||||
|
float := fmt.Sprintf("%.2f", r.Float64()*32.00)
|
||||||
|
benchmarkingItem[strconv.Itoa(i)] = &dynamodb.AttributeValue{N: &float}
|
||||||
|
default:
|
||||||
|
sentence := loremIpsumGenerator.Sentence()
|
||||||
|
benchmarkingItem[strconv.Itoa(i)] = &dynamodb.AttributeValue{S: &sentence}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return benchmarkingItem
|
||||||
|
}
|
||||||
|
|
||||||
|
type DynamoDbSimulationMetrics struct {
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
Timestamp int64 `json:"timestamp"`
|
||||||
|
Successful bool `json:"successful"`
|
||||||
|
Scenario string `json:"scenario"`
|
||||||
|
SimulationTime *float64 `json:"simulationTime,omitempty"`
|
||||||
|
ReadTime *float64 `json:"readTime,omitempty"`
|
||||||
|
WriteTime *float64 `json:"writeTime,omitempty"`
|
||||||
|
WriteItemConfirmationTime *float64 `json:"writeItemConfirmationTime,omitempty"`
|
||||||
|
UpdateTime *float64 `json:"updateItem,omitempty"`
|
||||||
|
UpdateItemConfirmationTime *float64 `json:"updateItemConfirmationTime,omitempty"`
|
||||||
|
DeleteTime *float64 `json:"deleteTime,omitempty"`
|
||||||
|
DeleteItemConfirmationTime *float64 `json:"deleteItemConfirmationTime,omitempty"`
|
||||||
|
}
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
package simulators
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/models"
|
||||||
|
"github.com/aws/aws-dax-go/dax"
|
||||||
|
"github.com/aws/aws-sdk-go/service/dynamodb"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ReadItem(client *dax.Dax, tableName string, id dynamodb.AttributeValue, metrics *models.DynamoDbSimulationMetrics, recordMetrics bool) (dynamodb.GetItemOutput, error) {
|
||||||
|
partitionKey := *id.S
|
||||||
|
startTime := time.Now()
|
||||||
|
response, err := client.GetItem(&dynamodb.GetItemInput{
|
||||||
|
TableName: &tableName,
|
||||||
|
Key: map[string]*dynamodb.AttributeValue{
|
||||||
|
"id": {S: id.S},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if recordMetrics {
|
||||||
|
duration := time.Since(startTime).Microseconds()
|
||||||
|
millisecondDuration := float64(duration) / 1000
|
||||||
|
metrics.ReadTime = &millisecondDuration
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Could not fetch item with partition key: %v. %v+", partitionKey, err)
|
||||||
|
metrics.Successful = false
|
||||||
|
return dynamodb.GetItemOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(response.Item) == 0 {
|
||||||
|
log.Infof("No items found with partition key: %v", partitionKey)
|
||||||
|
return dynamodb.GetItemOutput{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return *response, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func UpdateItem(client *dax.Dax, tableName string, id dynamodb.AttributeValue, attributes int, metrics *models.DynamoDbSimulationMetrics) {
|
||||||
|
updatedItem := models.NewBenchmarkingItem(attributes)
|
||||||
|
updatedItem["id"] = &id
|
||||||
|
partitionKey := *id.S
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
_, err := client.PutItem(&dynamodb.PutItemInput{
|
||||||
|
TableName: &tableName,
|
||||||
|
Item: updatedItem,
|
||||||
|
})
|
||||||
|
|
||||||
|
duration := time.Since(startTime).Microseconds()
|
||||||
|
millisecondDuration := float64(duration) / 1000
|
||||||
|
metrics.UpdateTime = &millisecondDuration
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Could not update item with partition key: %v. %v+", partitionKey, err)
|
||||||
|
metrics.Successful = false
|
||||||
|
} else {
|
||||||
|
log.Infof("Successfully updated item with partition key: %v", partitionKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func PutItem(client *dax.Dax, tableName string, attributes int, metrics *models.DynamoDbSimulationMetrics) (models.BenchmarkingItem, error) {
|
||||||
|
newItem := models.NewBenchmarkingItem(attributes)
|
||||||
|
partitionKey := *newItem["id"].S
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
_, err := client.PutItem(&dynamodb.PutItemInput{
|
||||||
|
TableName: &tableName,
|
||||||
|
Item: newItem,
|
||||||
|
})
|
||||||
|
|
||||||
|
duration := time.Since(startTime).Microseconds()
|
||||||
|
millisecondDuration := float64(duration) / 1000
|
||||||
|
metrics.WriteTime = &millisecondDuration
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Could not put new item with partition key: %v. %v+", partitionKey, err)
|
||||||
|
metrics.Successful = false
|
||||||
|
return models.BenchmarkingItem{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Successfully put new item with partition key: %v", partitionKey)
|
||||||
|
return newItem, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func DeleteItem(client *dax.Dax, tableName string, id dynamodb.AttributeValue, metrics *models.DynamoDbSimulationMetrics) {
|
||||||
|
partitionKey := *id.S
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
_, err := client.DeleteItem(&dynamodb.DeleteItemInput{
|
||||||
|
TableName: &tableName,
|
||||||
|
Key: map[string]*dynamodb.AttributeValue{
|
||||||
|
"charger_id": &id,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
duration := time.Since(startTime).Microseconds()
|
||||||
|
millisecondDuration := float64(duration) / 1000
|
||||||
|
metrics.DeleteTime = &millisecondDuration
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Could not delete item with partition key: %v. %v+", partitionKey, err)
|
||||||
|
metrics.Successful = false
|
||||||
|
} else {
|
||||||
|
log.Infof("Successfully deleted item with partition key: %v", partitionKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
package simulators
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math/rand"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/models"
|
||||||
|
"github.com/aws/aws-dax-go/dax"
|
||||||
|
"github.com/aws/aws-sdk-go/service/dynamodb"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
func SimulateReadOperation(client *dax.Dax, tableName string, partitionKeys []string, metrics *models.DynamoDbSimulationMetrics) {
|
||||||
|
log.Info("Performing READ operation...")
|
||||||
|
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||||
|
var partitionKey string
|
||||||
|
for {
|
||||||
|
partitionKey = partitionKeys[r.Intn(len(partitionKeys))]
|
||||||
|
if len(strings.TrimSpace(partitionKey)) == 0 {
|
||||||
|
log.Info("Parition key was empty. Trying again to choose a non-empty partition key")
|
||||||
|
} else {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
id := dynamodb.AttributeValue{S: &partitionKey}
|
||||||
|
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
log.Infof("Attempt %d: Fetching existing item with partition key: %v", i, partitionKey)
|
||||||
|
|
||||||
|
response, _ := ReadItem(client, tableName, id, metrics, true)
|
||||||
|
if response.Item["id"] != nil {
|
||||||
|
log.Infof("Successfully read existing item with partition key: %v", partitionKey)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Errorf("Unable to find existing item with partition key: %v", partitionKey)
|
||||||
|
if i == 9 {
|
||||||
|
log.Errorf("All attempts to fetch the existing item with partition key: %v failed!", partitionKey)
|
||||||
|
metrics.Successful = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func SimulateWriteOperation(client *dax.Dax, tableName string, attributes int, metrics *models.DynamoDbSimulationMetrics) {
|
||||||
|
log.Info("Performing WRITE operation...")
|
||||||
|
benchmarkingItem, err := PutItem(client, tableName, attributes, metrics)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unable to complete PUT simulation. %v+", err)
|
||||||
|
metrics.Successful = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
id := *benchmarkingItem["id"]
|
||||||
|
|
||||||
|
AssertItemWasCreated(client, tableName, id, metrics)
|
||||||
|
|
||||||
|
DeleteItem(client, tableName, id, metrics)
|
||||||
|
|
||||||
|
AssertItemWasDeleted(client, tableName, id, metrics)
|
||||||
|
}
|
||||||
|
|
||||||
|
func SimulateUpdateOperation(client *dax.Dax, tableName string, attributes int, metrics *models.DynamoDbSimulationMetrics) {
|
||||||
|
log.Info("Performing UPDATE operation...")
|
||||||
|
newItem, err := PutItem(client, tableName, attributes, metrics)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unable to complete UPDATE simulation. %v+", err)
|
||||||
|
metrics.Successful = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
id := *newItem["id"]
|
||||||
|
partitionKey := *id.S
|
||||||
|
attemptsExhausted := false
|
||||||
|
|
||||||
|
AssertItemWasCreated(client, tableName, id, metrics)
|
||||||
|
UpdateItem(client, tableName, id, attributes, metrics)
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
log.Infof("Attempt %d: Fetching updated item for partition key: %v...", i, partitionKey)
|
||||||
|
|
||||||
|
updatedItem, err := ReadItem(client, tableName, id, metrics, false)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unable to complete UPDATE simulation. %v+", err)
|
||||||
|
metrics.Successful = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if *newItem["1"].N != *updatedItem.Item["1"].N {
|
||||||
|
log.Infof("Confirmed update for partition key: %v", partitionKey)
|
||||||
|
break
|
||||||
|
} else {
|
||||||
|
log.Errorf("Update for partition key %v failed! Values are still equal!", partitionKey)
|
||||||
|
if i == 9 {
|
||||||
|
log.Error("Exhausted attempts to fetch updated item!")
|
||||||
|
metrics.Successful = false
|
||||||
|
attemptsExhausted = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !attemptsExhausted {
|
||||||
|
duration := time.Since(startTime).Microseconds()
|
||||||
|
millisecondDuration := float64(duration) / 1000
|
||||||
|
metrics.UpdateItemConfirmationTime = &millisecondDuration
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteItem(client, tableName, id, metrics)
|
||||||
|
AssertItemWasDeleted(client, tableName, id, metrics)
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package simulators
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/models"
|
||||||
|
"github.com/aws/aws-dax-go/dax"
|
||||||
|
"github.com/aws/aws-sdk-go/service/dynamodb"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
func AssertItemWasCreated(client *dax.Dax, tableName string, id dynamodb.AttributeValue, metrics *models.DynamoDbSimulationMetrics) {
|
||||||
|
partitionKey := *id.S
|
||||||
|
attemptsExhausted := false
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
log.Infof("Attempt %d: Fetching newly added item with partition key: %v", i, partitionKey)
|
||||||
|
|
||||||
|
newItem, err := ReadItem(client, tableName, id, metrics, false)
|
||||||
|
|
||||||
|
if err != nil || newItem.Item["id"].S == nil {
|
||||||
|
log.Errorf("Unable to find new item with partition key: %v", partitionKey)
|
||||||
|
if i == 9 {
|
||||||
|
log.Errorf("All attempts to fetch the newly added item with partition key: %v failed!", partitionKey)
|
||||||
|
attemptsExhausted = true
|
||||||
|
metrics.Successful = false
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.Infof("Successfully read new item with partition key: %v", partitionKey)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !attemptsExhausted {
|
||||||
|
duration := time.Since(startTime).Microseconds()
|
||||||
|
millisecondDuration := float64(duration) / 1000
|
||||||
|
metrics.WriteItemConfirmationTime = &millisecondDuration
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func AssertItemWasDeleted(client *dax.Dax, tableName string, id dynamodb.AttributeValue, metrics *models.DynamoDbSimulationMetrics) {
|
||||||
|
partitionKey := *id.S
|
||||||
|
attemptsExhausted := false
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
log.Infof("Attempt %d: Fetching deleted item with partition key: %v ...", i, partitionKey)
|
||||||
|
|
||||||
|
deletedItem, _ := ReadItem(client, tableName, id, metrics, false)
|
||||||
|
if deletedItem.Item["id"].S == nil {
|
||||||
|
log.Infof("Item with partition key: %v was successfully deleted.", partitionKey)
|
||||||
|
break
|
||||||
|
} else {
|
||||||
|
log.Errorf("Item with partition key %v was not deleted as expected!", partitionKey)
|
||||||
|
if i == 9 {
|
||||||
|
log.Errorf("All attempts to receive an empty response to verify item with partition key: %v was deleted failed!", partitionKey)
|
||||||
|
attemptsExhausted = true
|
||||||
|
metrics.Successful = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !attemptsExhausted {
|
||||||
|
duration := time.Since(startTime).Microseconds()
|
||||||
|
millisecondDuration := float64(duration) / 1000
|
||||||
|
metrics.DeleteItemConfirmationTime = &millisecondDuration
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
package utils
|
||||||
|
|
||||||
|
func MapValues[K comparable, V any](inputMap map[K]V) []V {
|
||||||
|
valuesSlice := make([]V, 0)
|
||||||
|
|
||||||
|
for _, value := range inputMap {
|
||||||
|
valuesSlice = append(valuesSlice, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
return valuesSlice
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
tab_spaces=2
|
||||||
|
edition = "2021"
|
||||||
|
reorder_imports = true
|
||||||
|
imports_granularity = "Crate"
|
||||||
|
group_imports = "StdExternalCrate"
|
||||||
|
reorder_modules = true
|
||||||
|
merge_derives = true
|
||||||
|
use_field_init_shorthand = true
|
||||||
|
format_macro_matchers = true
|
||||||
|
format_macro_bodies = true
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 63 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 35 KiB |
Executable
+41
@@ -0,0 +1,41 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
BENCHMARK_LOG_FILE="$1"
|
||||||
|
if [[ -z $BENCHMARK_LOG_FILE ]]; then
|
||||||
|
BENCHMARK_LOG_FILE=/tmp/benchmarker.log
|
||||||
|
fi
|
||||||
|
|
||||||
|
red=$(tput setaf 1)
|
||||||
|
green=$(tput setaf 2)
|
||||||
|
gold=$(tput setaf 3)
|
||||||
|
blue=$(tput setaf 4)
|
||||||
|
magenta=$(tput setaf 5)
|
||||||
|
cyan=$(tput setaf 6)
|
||||||
|
default=$(tput sgr0)
|
||||||
|
bold=$(tput bold)
|
||||||
|
|
||||||
|
log-error() {
|
||||||
|
if [[ -z $2 ]]; then
|
||||||
|
echo -e "${red}${bold}ERROR:${default}${red} $1${default}"
|
||||||
|
else
|
||||||
|
echo -e "${red}${bold}ERROR:${default}${red} $1${default}"
|
||||||
|
echo -e "${red}${bold}ERROR:${default}${red} $1${default}" >> "$BENCHMARK_LOG_FILE"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
log-warn() {
|
||||||
|
if [[ -z $2 ]]; then
|
||||||
|
echo -e "${gold}${bold}WARN:${default}${gold} $1${default}"
|
||||||
|
else
|
||||||
|
echo -e "${gold}${bold}WARN:${default}${gold} $1${default}"
|
||||||
|
echo -e "${gold}${bold}WARN:${default}${gold} $1${default}" >> "$BENCHMARK_LOG_FILE"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
log-info() {
|
||||||
|
if [[ -z $2 ]]; then
|
||||||
|
echo -e "${cyan}${bold}INFO:${default}${cyan} $1${default}"
|
||||||
|
else
|
||||||
|
echo -e "${cyan}${bold}INFO:${default}${cyan} $1${default}"
|
||||||
|
echo -e "${cyan}${bold}INFO:${default}${cyan} $1${default}" >> "$BENCHMARK_LOG_FILE"
|
||||||
|
fi
|
||||||
|
}
|
||||||
+180
@@ -0,0 +1,180 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
if [[ $(basename "$(pwd)") == scripts ]]; then
|
||||||
|
source logger.sh
|
||||||
|
else
|
||||||
|
source scripts/logger.sh
|
||||||
|
fi
|
||||||
|
|
||||||
|
trap 'echo Stopping...; exit' SIGINT
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat << EOF
|
||||||
|
${blue}${bold}randomly-generate-high-velocity-data${default}: A script to randomly generate high-velocity data for some DynamoDB table with random attributes and values for benchmarking purposes.
|
||||||
|
|
||||||
|
${gold}${bold}USAGE:${default}
|
||||||
|
randomly-generate-high-velocity-data [OPTIONS] [ARGS]...
|
||||||
|
|
||||||
|
${green}-h, --help${default} Show this usage screen
|
||||||
|
|
||||||
|
${gold}${bold}ARGS:${default}
|
||||||
|
${green}-a, --attributes ${magenta}<ATTRIBUTES>${default} The number of attributes to populate each item in the table with
|
||||||
|
This defaults to 5
|
||||||
|
|
||||||
|
${green}-i, --items ${magenta}<ITEMS>${default} The number of items to populate the table with
|
||||||
|
${bold}Note:${default} Items are populated 25 at a time, so whatever number you provide will be rounded to the nearest multiple of 25
|
||||||
|
|
||||||
|
${green}-t, --table ${magenta}<TABLE_NAME>${default} The name of the DynamoDB table to populate
|
||||||
|
This defaults to $USER-high-velocity-table
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure-required-variables-are-set() {
|
||||||
|
declare required_variables=(AWS_PROFILE AWS_REGION ITEMS)
|
||||||
|
|
||||||
|
for variable in "${required_variables[@]}"; do
|
||||||
|
if [[ -z "${!variable}" ]]; then
|
||||||
|
log-error "A required variable environment is not initialized: $variable"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
parse-arguments() {
|
||||||
|
declare parsed_args
|
||||||
|
parsed_args=$(getopt -a -n randomly-generate-high-velocity-data -o :a:hi:t: --long attributes:,help,items:,table: -- "$@")
|
||||||
|
declare valid_arguments=$?
|
||||||
|
|
||||||
|
if [[ $valid_arguments != 0 ]]; then
|
||||||
|
log-error "Invalid arguments passed. See usage below."
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
eval set -- "$parsed_args"
|
||||||
|
while :; do
|
||||||
|
case "$1" in
|
||||||
|
"-a" | "--attributes")
|
||||||
|
ATTRIBUTES="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
"-h" | "--help")
|
||||||
|
usage
|
||||||
|
exit
|
||||||
|
;;
|
||||||
|
"-i" | "--items")
|
||||||
|
ITEMS="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
"-t" | "--table")
|
||||||
|
TABLE_NAME="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--)
|
||||||
|
shift
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log-error "An invalid option was passed, but somehow getopt didn't catch it: $1. Displaying usage and exiting..."
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z $TABLE_NAME ]]; then
|
||||||
|
TABLE_NAME="$USER-high-velocity-table"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z $ATTRIBUTES ]]; then
|
||||||
|
ATTRIBUTES=5
|
||||||
|
fi
|
||||||
|
|
||||||
|
ensure-required-variables-are-set
|
||||||
|
|
||||||
|
if [[ $ATTRIBUTES -lt 1 ]]; then
|
||||||
|
log-error "ATTRIBUTES must be a value of at least 1 so that attributes can be added to the table."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! (aws sts get-caller-identity > /dev/null 2>&1); then
|
||||||
|
log-error "You must be logged into the AWS CLI in order to use this script. Please log into the AWS CLI first and then try again."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
show-properties() {
|
||||||
|
log-info "Using the following settings to randomly populate the DynamoDB benchmarking table:"
|
||||||
|
cat <<-EOF
|
||||||
|
${cyan}
|
||||||
|
ATTRIBUTES=$ATTRIBUTES
|
||||||
|
TABLE_NAME=$TABLE_NAME
|
||||||
|
${default}
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
generate-attribute-value() {
|
||||||
|
declare current_val=$1
|
||||||
|
case "$((current_val % 2))" in
|
||||||
|
"1")
|
||||||
|
echo '"'"$current_val"'": {"N": "'"$(seq 0 .01 32 | shuf | head -1)"'"}'
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo '"'"$current_val"'": {"S": "'"$(base64 /dev/urandom | awk '{print(0==NR%100)?"":$1}' | sed 's/[^[:alpha:]]/ /g' | head -1)"'"}'
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
generate-put-request() {
|
||||||
|
declare attribute_values
|
||||||
|
attribute_values=$(generate-attribute-value 0)
|
||||||
|
|
||||||
|
for j in $(seq 1 $((ATTRIBUTES-1))); do
|
||||||
|
attribute_values="$attribute_values, $(generate-attribute-value "$j")"
|
||||||
|
done
|
||||||
|
|
||||||
|
|
||||||
|
cat <<-EOF
|
||||||
|
{
|
||||||
|
"PutRequest": {
|
||||||
|
"Item": {
|
||||||
|
"id": {"S": "$(cat /proc/sys/kernel/random/uuid)"},
|
||||||
|
$attribute_values
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
generate-batch-json() {
|
||||||
|
declare batch_request='{ "'"$TABLE_NAME"'": ['
|
||||||
|
batch_request="$batch_request $(generate-put-request)"
|
||||||
|
|
||||||
|
for i in $(seq 0 23); do
|
||||||
|
batch_request="$batch_request, $(generate-put-request)"
|
||||||
|
done
|
||||||
|
|
||||||
|
batch_request="$batch_request ]}"
|
||||||
|
|
||||||
|
echo "$batch_request"
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! (command -v aws > /dev/null 2>&1); then
|
||||||
|
log-error "The AWS CLI must be installed first. Install the CLI first and then try again."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
parse-arguments "$@"
|
||||||
|
show-properties
|
||||||
|
|
||||||
|
declare -i i=0
|
||||||
|
declare -i items_written=0
|
||||||
|
while [[ $items_written -lt $ITEMS ]]; do
|
||||||
|
log-info "Writing 25 entries to DynamoDB..."
|
||||||
|
aws dynamodb batch-write-item --request-items "$(generate-batch-json)"
|
||||||
|
log-info 'Entries Written!'
|
||||||
|
((i++))
|
||||||
|
((items_written+=25))
|
||||||
|
log-info "Total entries written: $items_written"
|
||||||
|
log-info "Sleeping for 2 seconds to avoid the partition throughput limits..."
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
Executable
+36
@@ -0,0 +1,36 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
TERMINAL_HEIGHT=$(tput lines)
|
||||||
|
BOX_HEIGHT=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_HEIGHT * .5" | bc)")
|
||||||
|
|
||||||
|
TERMINAL_WIDTH=$(tput cols)
|
||||||
|
BOX_WIDTH=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_WIDTH * .75" | bc)")
|
||||||
|
|
||||||
|
msg-box() {
|
||||||
|
whiptail --fb --msgbox "$1" "$BOX_HEIGHT" "$BOX_WIDTH"
|
||||||
|
}
|
||||||
|
|
||||||
|
check-sudo-pass() {
|
||||||
|
log-info "Prompting user for sudo password with message: $1"
|
||||||
|
if [[ ! "$PASSWORD" ]]; then
|
||||||
|
PASSWORD=$(whiptail --fb --passwordbox "$1 Enter your sudo password" "$BOX_HEIGHT" "$BOX_WIDTH" 3>&2 2>&1 1>&3)
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
show-tail-box() {
|
||||||
|
trap "kill $2 2> /dev/null" EXIT
|
||||||
|
|
||||||
|
while kill -0 "$2" 2> /dev/null; do
|
||||||
|
dialog --title "$1" --exit-label "Finished" --tailbox "$3" "$BOX_HEIGHT" "$BOX_WIDTH"
|
||||||
|
done
|
||||||
|
|
||||||
|
clear
|
||||||
|
|
||||||
|
trap - EXIT
|
||||||
|
}
|
||||||
|
|
||||||
|
prompt-yes-no() {
|
||||||
|
declare action="$1"
|
||||||
|
log-info "Prompting user if they wish to proceed with $action"
|
||||||
|
|
||||||
|
whiptail --fb --title "$action?" --yesno "Are you sure you wish to proceed with the specified action: $action?" --defaultno "$BOX_HEIGHT" "$BOX_WIDTH"
|
||||||
|
}
|
||||||
+315
@@ -0,0 +1,315 @@
|
|||||||
|
use std::{env, time::Duration};
|
||||||
|
|
||||||
|
use anyhow::anyhow;
|
||||||
|
|
||||||
|
use aws_sdk_dynamodb::Client;
|
||||||
|
use chrono::Utc;
|
||||||
|
use clap::Parser;
|
||||||
|
use elasticsearch::{
|
||||||
|
auth::Credentials,
|
||||||
|
http::{
|
||||||
|
transport::{SingleNodeConnectionPool, TransportBuilder},
|
||||||
|
Url,
|
||||||
|
},
|
||||||
|
indices::IndicesPutMappingParts,
|
||||||
|
Elasticsearch,
|
||||||
|
};
|
||||||
|
use log::{error, info, warn, LevelFilter};
|
||||||
|
use log4rs::{
|
||||||
|
append::console::ConsoleAppender,
|
||||||
|
config::{Appender, Root},
|
||||||
|
encode::pattern::PatternEncoder,
|
||||||
|
};
|
||||||
|
use models::{DynamoDbSimulationMetrics, DynamoOperation};
|
||||||
|
use rand::{
|
||||||
|
rngs::{OsRng, StdRng},
|
||||||
|
Rng, SeedableRng,
|
||||||
|
};
|
||||||
|
use serde_json::json;
|
||||||
|
use tokio::{
|
||||||
|
select,
|
||||||
|
sync::mpsc::{self, Receiver, Sender},
|
||||||
|
task::JoinHandle,
|
||||||
|
};
|
||||||
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
|
use crate::{models::Scenario, simulators::Simulator};
|
||||||
|
|
||||||
|
mod models;
|
||||||
|
mod simulators;
|
||||||
|
mod timer_utils;
|
||||||
|
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[command(author, version, about, long_about = None)]
|
||||||
|
struct Cli {
|
||||||
|
/// The number of concurrent simulations to run
|
||||||
|
#[arg(short, long, default_value_t = 1000)]
|
||||||
|
concurrent_simulations: u32,
|
||||||
|
/// The number of attributes to use when populating and querying the DynamoDB table; minimum value of 1
|
||||||
|
#[arg(short, long, default_value_t = 5)]
|
||||||
|
attributes: u32,
|
||||||
|
/// The length of time (in seconds) to run the benchmark for
|
||||||
|
#[arg(short, long, default_value_t = 1800)]
|
||||||
|
duration: u64,
|
||||||
|
/// The buffer size of the Elasticsearch thread's MPSC channel
|
||||||
|
#[arg(short, long, default_value_t = 500)]
|
||||||
|
buffer: usize,
|
||||||
|
/// Local Elasticsearch cluster username
|
||||||
|
#[arg(short, long, default_value_t = String::from("elastic"))]
|
||||||
|
username: String,
|
||||||
|
/// Local Elasticsearch cluster password
|
||||||
|
#[arg(short, long, default_value_t = String::from("changeme"))]
|
||||||
|
password: String,
|
||||||
|
/// The Elasticsearch Index to insert data into
|
||||||
|
#[arg(short, long, default_value_t = String::from("dynamodb"))]
|
||||||
|
index: String,
|
||||||
|
/// The DynamoDB table to perform operations against
|
||||||
|
#[arg(short, long, default_value_t = format!("{}-high-velocity-table", env::var("USER").unwrap()))]
|
||||||
|
table_name: String,
|
||||||
|
/// Whether to run a read-only scenario for benchmarking
|
||||||
|
#[arg(short, long)]
|
||||||
|
read_only: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn main() -> anyhow::Result<()> {
|
||||||
|
let cli = Cli::parse();
|
||||||
|
log4rs::init_config(init_logging_config())?;
|
||||||
|
let cancellation_token = CancellationToken::new();
|
||||||
|
|
||||||
|
let (es_tx, es_rx) = mpsc::channel::<DynamoDbSimulationMetrics>(cli.buffer);
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
start_elasticsearch_publisher(es_rx, cli.username, cli.password, cli.index)
|
||||||
|
});
|
||||||
|
|
||||||
|
let handles: Vec<JoinHandle<_>> = (0..cli.concurrent_simulations)
|
||||||
|
.map(|_| {
|
||||||
|
let tx = es_tx.clone();
|
||||||
|
let token = cancellation_token.clone();
|
||||||
|
let table_name = cli.table_name.clone();
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let config = aws_config::load_from_env().await;
|
||||||
|
let dynamodb_client = Client::new(&config);
|
||||||
|
match scan_all_partition_keys(&dynamodb_client, table_name.clone()).await {
|
||||||
|
Ok(partition_keys_vec) => {
|
||||||
|
let simulator = Simulator::new(
|
||||||
|
&dynamodb_client,
|
||||||
|
table_name.clone(),
|
||||||
|
cli.attributes,
|
||||||
|
&partition_keys_vec,
|
||||||
|
);
|
||||||
|
select! {
|
||||||
|
_ = token.cancelled() => {
|
||||||
|
warn!("Task cancelled. Shutting down...");
|
||||||
|
}
|
||||||
|
_ = simulation_loop(simulator, cli.read_only, tx) => ()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => error!("Unable to fetch partition keys: {e:?}"),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
info!(
|
||||||
|
"Starting timer task. Executing for {} seconds",
|
||||||
|
cli.duration
|
||||||
|
);
|
||||||
|
|
||||||
|
tokio::time::sleep(Duration::from_secs(cli.duration)).await;
|
||||||
|
|
||||||
|
cancellation_token.cancel();
|
||||||
|
});
|
||||||
|
|
||||||
|
for handle in handles {
|
||||||
|
match handle.await {
|
||||||
|
Ok(_) => info!("Task shut down gracefully"),
|
||||||
|
Err(e) => warn!("Task did not shut down gracefully {e:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn start_elasticsearch_publisher(
|
||||||
|
mut elasticsearch_rx: Receiver<DynamoDbSimulationMetrics>,
|
||||||
|
username: String,
|
||||||
|
password: String,
|
||||||
|
index: String,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let url = Url::parse("http://localhost:9200")?;
|
||||||
|
let connection_pool = SingleNodeConnectionPool::new(url);
|
||||||
|
let credentials = Credentials::Basic(username, password);
|
||||||
|
let transport = TransportBuilder::new(connection_pool)
|
||||||
|
.auth(credentials)
|
||||||
|
.build()?;
|
||||||
|
let es_client = Elasticsearch::new(transport);
|
||||||
|
|
||||||
|
info!("Setting the explicit mappings for the {index} index");
|
||||||
|
es_client
|
||||||
|
.indices()
|
||||||
|
.put_mapping(IndicesPutMappingParts::Index(&[&index]))
|
||||||
|
.body(json!({
|
||||||
|
"properties": {
|
||||||
|
"timestamp": {
|
||||||
|
"type": "date"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
while let Some(metric) = elasticsearch_rx.recv().await {
|
||||||
|
info!("Publishing metrics to Elasticsearch...");
|
||||||
|
|
||||||
|
let es_response = es_client
|
||||||
|
.index(elasticsearch::IndexParts::Index(&index))
|
||||||
|
.body(metric)
|
||||||
|
.send()
|
||||||
|
.await;
|
||||||
|
|
||||||
|
match es_response {
|
||||||
|
Ok(resp) => {
|
||||||
|
if resp.status_code().is_success() {
|
||||||
|
info!("Successfully published metrics to Elasticsearch");
|
||||||
|
} else {
|
||||||
|
error!("Was unable to publish metrics to Elasticsearch! Received non 2XX response");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Unable to publish metrics to Elasticsearch! {e:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn simulation_loop(
|
||||||
|
mut simulator: Simulator<'_>,
|
||||||
|
read_only: bool,
|
||||||
|
tx: Sender<DynamoDbSimulationMetrics>,
|
||||||
|
) {
|
||||||
|
let mut rng = StdRng::from_seed(OsRng.gen());
|
||||||
|
loop {
|
||||||
|
let mut metrics = DynamoDbSimulationMetrics::default();
|
||||||
|
metrics.timestamp = Utc::now();
|
||||||
|
|
||||||
|
let simulation_time = time!(match {
|
||||||
|
if read_only {
|
||||||
|
info!("Running a read-only simulation...");
|
||||||
|
metrics.scenario = Scenario::ReadOnly;
|
||||||
|
run_read_only_simulation(&mut simulator, &mut metrics, &mut rng).await
|
||||||
|
} else {
|
||||||
|
info!("Running a CRUD simulation...");
|
||||||
|
metrics.scenario = Scenario::Crud;
|
||||||
|
run_crud_simulation(&mut simulator, &mut metrics, &mut rng).await
|
||||||
|
}
|
||||||
|
} {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Simulation completed successfully!");
|
||||||
|
metrics.successful = true;
|
||||||
|
}
|
||||||
|
Err(e) => error!("Simulation did not complete. Encountered the following error: {e:?}"),
|
||||||
|
});
|
||||||
|
metrics.simulation_time = Some(simulation_time);
|
||||||
|
info!("Metrics: {metrics:?}");
|
||||||
|
|
||||||
|
match tx.send(metrics).await {
|
||||||
|
Ok(_) => info!("Metrics sent down channel successfully"),
|
||||||
|
Err(e) => error!("Metrics were unable to be sent down the channel! {e:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_read_only_simulation(
|
||||||
|
simulator: &mut Simulator<'_>,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
rng: &mut StdRng,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
tokio::time::sleep(Duration::from_secs(rng.gen_range(0..15))).await;
|
||||||
|
|
||||||
|
metrics.operation = DynamoOperation::Read;
|
||||||
|
simulator.simulate_read_operation(metrics).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_crud_simulation(
|
||||||
|
simulator: &mut Simulator<'_>,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
rng: &mut StdRng,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
match DynamoOperation::from(rng.gen_range(0..3)) {
|
||||||
|
DynamoOperation::Read => {
|
||||||
|
metrics.operation = DynamoOperation::Read;
|
||||||
|
simulator.simulate_read_operation(metrics).await?
|
||||||
|
}
|
||||||
|
DynamoOperation::Write => {
|
||||||
|
metrics.operation = DynamoOperation::Write;
|
||||||
|
simulator.simulate_write_operation(metrics).await?;
|
||||||
|
}
|
||||||
|
DynamoOperation::Update => {
|
||||||
|
metrics.operation = DynamoOperation::Update;
|
||||||
|
simulator.simulate_update_operation(metrics).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn scan_all_partition_keys(
|
||||||
|
dynamodb_client: &Client,
|
||||||
|
table_name: String,
|
||||||
|
) -> anyhow::Result<Vec<String>> {
|
||||||
|
info!("Fetching a large list of partition keys to randomly read...");
|
||||||
|
let response = dynamodb_client
|
||||||
|
.scan()
|
||||||
|
.table_name(table_name)
|
||||||
|
.limit(10000)
|
||||||
|
.projection_expression("id")
|
||||||
|
.send()
|
||||||
|
.await;
|
||||||
|
|
||||||
|
match response {
|
||||||
|
Ok(resp) => {
|
||||||
|
info!("Fetched partition keys!");
|
||||||
|
let partition_keys = resp
|
||||||
|
.items()
|
||||||
|
.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.map(|attribute| {
|
||||||
|
attribute
|
||||||
|
.values()
|
||||||
|
.last()
|
||||||
|
.unwrap()
|
||||||
|
.as_s()
|
||||||
|
.unwrap()
|
||||||
|
.to_string()
|
||||||
|
})
|
||||||
|
.collect::<Vec<String>>();
|
||||||
|
info!("Found a total of {} keys", partition_keys.len());
|
||||||
|
Ok(partition_keys)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Unable to fetch partition keys! {e:?}");
|
||||||
|
Err(anyhow!(e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn init_logging_config() -> log4rs::Config {
|
||||||
|
let stdout = ConsoleAppender::builder()
|
||||||
|
.encoder(Box::new(PatternEncoder::new(
|
||||||
|
"{d(%Y-%m-%d %H:%M:%S%.3f)(utc)} <{i}> [{l}] {f}:{L} - {m}{n}",
|
||||||
|
)))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
log4rs::Config::builder()
|
||||||
|
.appender(Appender::builder().build("stdout", Box::new(stdout)))
|
||||||
|
.build(Root::builder().appender("stdout").build(LevelFilter::Info))
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use aws_sdk_dynamodb::types::AttributeValue;
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use rand::Rng;
|
||||||
|
use serde::Serialize;
|
||||||
|
use serde_json::Number;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
#[derive(Serialize, Debug, Default)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub enum DynamoOperation {
|
||||||
|
#[default]
|
||||||
|
Read,
|
||||||
|
Write,
|
||||||
|
Update,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<i32> for DynamoOperation {
|
||||||
|
fn from(value: i32) -> Self {
|
||||||
|
match value {
|
||||||
|
0 => DynamoOperation::Read,
|
||||||
|
1 => DynamoOperation::Write,
|
||||||
|
2 => DynamoOperation::Update,
|
||||||
|
_ => DynamoOperation::Read,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Debug, Default)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub enum Scenario {
|
||||||
|
#[default]
|
||||||
|
Crud,
|
||||||
|
ReadOnly,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct BenchmarkingItem(HashMap<String, AttributeValue>);
|
||||||
|
|
||||||
|
impl From<HashMap<String, AttributeValue>> for BenchmarkingItem {
|
||||||
|
fn from(value: HashMap<String, AttributeValue>) -> BenchmarkingItem {
|
||||||
|
BenchmarkingItem(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BenchmarkingItem {
|
||||||
|
pub fn new(attributes: u32) -> BenchmarkingItem {
|
||||||
|
let mut benchmarking_item = HashMap::<String, AttributeValue>::new();
|
||||||
|
let mut rng = rand::thread_rng();
|
||||||
|
benchmarking_item.insert(
|
||||||
|
"id".to_owned(),
|
||||||
|
AttributeValue::S(Uuid::new_v4().to_string()),
|
||||||
|
);
|
||||||
|
|
||||||
|
(0..attributes).for_each(|i| {
|
||||||
|
if let 0 = i % 2 {
|
||||||
|
benchmarking_item.insert(i.to_string(), AttributeValue::S(lipsum::lipsum_words(15)));
|
||||||
|
} else {
|
||||||
|
benchmarking_item.insert(
|
||||||
|
i.to_string(),
|
||||||
|
AttributeValue::N(rng.gen_range(0.0..=32.0).to_string()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
BenchmarkingItem(benchmarking_item)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_id(&self) -> AttributeValue {
|
||||||
|
self.0.get("id").cloned().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn insert(&mut self, key: &str, val: AttributeValue) -> Option<AttributeValue> {
|
||||||
|
self.0.insert(key.to_owned(), val)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn get(&self, key: &str) -> Option<&AttributeValue> {
|
||||||
|
self.0.get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extract_map(&self) -> HashMap<String, AttributeValue> {
|
||||||
|
self.0.clone()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Debug, Default)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct DynamoDbSimulationMetrics {
|
||||||
|
pub operation: DynamoOperation,
|
||||||
|
pub timestamp: DateTime<Utc>,
|
||||||
|
pub successful: bool,
|
||||||
|
pub scenario: Scenario,
|
||||||
|
pub simulation_time: Option<Number>,
|
||||||
|
pub read_time: Option<Number>,
|
||||||
|
pub write_time: Option<Number>,
|
||||||
|
pub write_item_confirmation_time: Option<Number>,
|
||||||
|
pub update_time: Option<Number>,
|
||||||
|
pub update_item_confirmation_time: Option<Number>,
|
||||||
|
pub delete_time: Option<Number>,
|
||||||
|
pub delete_item_confirmation_time: Option<Number>,
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
use aws_sdk_dynamodb::types::AttributeValue;
|
||||||
|
use log::{error, info};
|
||||||
|
|
||||||
|
use crate::{models::DynamoDbSimulationMetrics, time};
|
||||||
|
|
||||||
|
use super::{utils, Simulator};
|
||||||
|
|
||||||
|
impl<'a> Simulator<'a> {
|
||||||
|
pub(super) async fn assert_item_was_created(
|
||||||
|
&mut self,
|
||||||
|
id: AttributeValue,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let partition_key = utils::extract_partition_key(id.clone());
|
||||||
|
let mut attempts_exhausted = false;
|
||||||
|
|
||||||
|
let write_confirmation_time = time!(for i in 0..10 {
|
||||||
|
info!("Attempt {i}: Fetching newly added item with partition key: {partition_key}");
|
||||||
|
|
||||||
|
match self.read_item(id.clone(), metrics, false).await? {
|
||||||
|
Some(_) => {
|
||||||
|
info!("Successfully read new item with partition key: {partition_key}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
error!("Unable to find new item with partition key: {partition_key}");
|
||||||
|
if i == 9 {
|
||||||
|
error!("All attempts to fetch the newly added item with partition key: {partition_key} failed!");
|
||||||
|
attempts_exhausted = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
if !attempts_exhausted {
|
||||||
|
metrics.write_item_confirmation_time = Some(write_confirmation_time);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn assert_item_was_deleted(
|
||||||
|
&mut self,
|
||||||
|
id: AttributeValue,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let partition_key = utils::extract_partition_key(id.clone());
|
||||||
|
let mut attempts_exhausted = false;
|
||||||
|
let delete_confirmation_time = time!(for i in 0..10 {
|
||||||
|
info!("Attempt {i}: Fetching deleted item with partition key: {partition_key}...");
|
||||||
|
match self.read_item(id.clone(), metrics, false).await? {
|
||||||
|
Some(_) => {
|
||||||
|
error!("Item with partition key {partition_key} was not deleted as expected!");
|
||||||
|
if i == 9 {
|
||||||
|
error!("All attempts to receive an empty response to verify item with partition key: {partition_key} was deleted failed!");
|
||||||
|
attempts_exhausted = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
info!("Item with partition key {partition_key} was successfully deleted.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if !attempts_exhausted {
|
||||||
|
metrics.delete_item_confirmation_time = Some(delete_confirmation_time);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
use aws_sdk_dynamodb::{types::AttributeValue, Client};
|
||||||
|
use log::{error, info};
|
||||||
|
use rand::{
|
||||||
|
rngs::{OsRng, StdRng},
|
||||||
|
Rng, SeedableRng,
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::{models::DynamoDbSimulationMetrics, time};
|
||||||
|
|
||||||
|
mod assertions;
|
||||||
|
mod operations;
|
||||||
|
mod utils;
|
||||||
|
|
||||||
|
pub struct Simulator<'a> {
|
||||||
|
dynamodb_client: &'a Client,
|
||||||
|
table_name: String,
|
||||||
|
attributes: u32,
|
||||||
|
partition_keys_vec: &'a [String],
|
||||||
|
rng: StdRng,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> Simulator<'a> {
|
||||||
|
pub fn new(
|
||||||
|
dynamodb_client: &'a Client,
|
||||||
|
table_name: String,
|
||||||
|
attributes: u32,
|
||||||
|
partition_keys_vec: &'a [String],
|
||||||
|
) -> Simulator<'a> {
|
||||||
|
Simulator {
|
||||||
|
dynamodb_client,
|
||||||
|
table_name,
|
||||||
|
attributes,
|
||||||
|
partition_keys_vec,
|
||||||
|
rng: StdRng::from_seed(OsRng.gen()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn simulate_read_operation(
|
||||||
|
&mut self,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
info!("Performing READ Operation...");
|
||||||
|
let partition_key =
|
||||||
|
self.partition_keys_vec[self.rng.gen_range(0..self.partition_keys_vec.len())].clone();
|
||||||
|
let id = AttributeValue::S(partition_key.clone());
|
||||||
|
|
||||||
|
for i in 0..10 {
|
||||||
|
info!("Attempt {i}: Fetching existing item with partition key: {partition_key}");
|
||||||
|
|
||||||
|
match self.read_item(id.clone(), metrics, true).await? {
|
||||||
|
Some(_) => {
|
||||||
|
info!("Successfully read existing item with partition key: {partition_key}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
error!("Unable to find existing item with partition key: {partition_key}");
|
||||||
|
if i == 9 {
|
||||||
|
error!(
|
||||||
|
"All attempts to fetch the existing item with partition key: {partition_key} failed!"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn simulate_write_operation(
|
||||||
|
&mut self,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
info!("Performing WRITE operation...");
|
||||||
|
let benchmarking_item = self.put_item(metrics).await?;
|
||||||
|
let id = benchmarking_item.get_id();
|
||||||
|
|
||||||
|
self.assert_item_was_created(id.clone(), metrics).await?;
|
||||||
|
|
||||||
|
self.delete_item(id.clone(), metrics).await?;
|
||||||
|
|
||||||
|
self.assert_item_was_deleted(id, metrics).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn simulate_update_operation(
|
||||||
|
&mut self,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
info!("Performing UPDATE operation...");
|
||||||
|
let new_item = self.put_item(metrics).await?;
|
||||||
|
let id = new_item.get_id();
|
||||||
|
let partition_key = utils::extract_partition_key(id.clone());
|
||||||
|
let mut attempts_exhausted = false;
|
||||||
|
|
||||||
|
self.assert_item_was_created(id.clone(), metrics).await?;
|
||||||
|
self.update_item(id.clone(), metrics).await?;
|
||||||
|
|
||||||
|
let update_confirmation_time = time!(for i in 0..10 {
|
||||||
|
info!("Attempt {i}: Fetching updated item for partition key: {partition_key}...");
|
||||||
|
|
||||||
|
let updated_item = self.read_item(id.clone(), metrics, false).await?.unwrap();
|
||||||
|
|
||||||
|
let new_item_attribute_value = new_item
|
||||||
|
.get("1")
|
||||||
|
.cloned()
|
||||||
|
.unwrap()
|
||||||
|
.as_n()
|
||||||
|
.unwrap()
|
||||||
|
.to_string();
|
||||||
|
let updated_item_attribute_value = updated_item
|
||||||
|
.get("1")
|
||||||
|
.cloned()
|
||||||
|
.unwrap()
|
||||||
|
.as_n()
|
||||||
|
.unwrap()
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
if new_item_attribute_value != updated_item_attribute_value {
|
||||||
|
info!("Confirmed update for partition key: {partition_key}");
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
error!("Update for partition key {partition_key} failed! Values are still equal!");
|
||||||
|
if i == 9 {
|
||||||
|
error!("Exhausted attempts to fetch updated item!");
|
||||||
|
attempts_exhausted = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if !attempts_exhausted {
|
||||||
|
metrics.update_item_confirmation_time = Some(update_confirmation_time);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.delete_item(id.clone(), metrics).await?;
|
||||||
|
self.assert_item_was_deleted(id, metrics).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
use anyhow::anyhow;
|
||||||
|
use aws_sdk_dynamodb::types::AttributeValue;
|
||||||
|
use log::{error, info};
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
models::{BenchmarkingItem, DynamoDbSimulationMetrics},
|
||||||
|
time,
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::{utils::extract_partition_key, Simulator};
|
||||||
|
|
||||||
|
impl<'a> Simulator<'a> {
|
||||||
|
pub async fn read_item(
|
||||||
|
&mut self,
|
||||||
|
id: AttributeValue,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
record_metrics: bool,
|
||||||
|
) -> anyhow::Result<Option<BenchmarkingItem>> {
|
||||||
|
let partition_key = extract_partition_key(id.clone());
|
||||||
|
let (read_time, response) = time!(
|
||||||
|
resp,
|
||||||
|
self
|
||||||
|
.dynamodb_client
|
||||||
|
.get_item()
|
||||||
|
.table_name(self.table_name.clone())
|
||||||
|
.key("id", id)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
|
||||||
|
if record_metrics {
|
||||||
|
metrics.read_time = Some(read_time);
|
||||||
|
}
|
||||||
|
|
||||||
|
match response {
|
||||||
|
Ok(resp) => {
|
||||||
|
info!("Found item: {}", partition_key);
|
||||||
|
if let Some(item) = resp.item() {
|
||||||
|
info!("Fetched item: {item:?}");
|
||||||
|
Ok(Some(BenchmarkingItem::from(item.clone())))
|
||||||
|
} else {
|
||||||
|
info!("No items found with partition key: {partition_key}");
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Could not fetch item with partition key: {partition_key}. {e:?}");
|
||||||
|
Err(anyhow!(e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn update_item(
|
||||||
|
&mut self,
|
||||||
|
id: AttributeValue,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let mut updated_item = BenchmarkingItem::new(self.attributes);
|
||||||
|
updated_item.insert("id", id.clone());
|
||||||
|
let partition_key = extract_partition_key(id);
|
||||||
|
let (update_time, response) = time!(
|
||||||
|
resp,
|
||||||
|
self
|
||||||
|
.dynamodb_client
|
||||||
|
.put_item()
|
||||||
|
.table_name(self.table_name.clone())
|
||||||
|
.set_item(Some(updated_item.extract_map()))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
metrics.update_time = Some(update_time);
|
||||||
|
|
||||||
|
match response {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Successfully updated item with partition_key: {partition_key}");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Could not update item with partition key: {partition_key}. {e:?}");
|
||||||
|
Err(anyhow!(e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn put_item(
|
||||||
|
&mut self,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
) -> anyhow::Result<BenchmarkingItem> {
|
||||||
|
let new_item = BenchmarkingItem::new(self.attributes);
|
||||||
|
let partition_key = extract_partition_key(new_item.get("id").cloned().unwrap());
|
||||||
|
let (time, response) = time!(
|
||||||
|
resp,
|
||||||
|
self
|
||||||
|
.dynamodb_client
|
||||||
|
.put_item()
|
||||||
|
.table_name(self.table_name.clone())
|
||||||
|
.set_item(Some(new_item.extract_map()))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
metrics.write_time = Some(time);
|
||||||
|
|
||||||
|
match response {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Successfully put new item with partition key: {partition_key}");
|
||||||
|
Ok(new_item)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Could not put new item with partition key: {partition_key}. {e:?}");
|
||||||
|
Err(anyhow!(e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn delete_item(
|
||||||
|
&mut self,
|
||||||
|
id: AttributeValue,
|
||||||
|
metrics: &mut DynamoDbSimulationMetrics,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let partition_key = extract_partition_key(id.clone());
|
||||||
|
let (delete_time, response) = time!(
|
||||||
|
resp,
|
||||||
|
self
|
||||||
|
.dynamodb_client
|
||||||
|
.delete_item()
|
||||||
|
.table_name(self.table_name.clone())
|
||||||
|
.key("id", id)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
metrics.delete_time = Some(delete_time);
|
||||||
|
|
||||||
|
match response {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Successfully deleted item with partition key: {partition_key}");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Could not delete item with partition key: {partition_key}. {e:?}");
|
||||||
|
Err(anyhow!(e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
use aws_sdk_dynamodb::types::AttributeValue;
|
||||||
|
|
||||||
|
pub(super) fn extract_partition_key(id: AttributeValue) -> String {
|
||||||
|
id.clone().as_s().unwrap().to_string()
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
#[macro_export]
|
||||||
|
macro_rules! time {
|
||||||
|
($x:expr) => {{
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
let _result = $x;
|
||||||
|
serde_json::Number::from(start.elapsed().as_millis())
|
||||||
|
}};
|
||||||
|
|
||||||
|
($resp:ident, $x:expr) => {{
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
let $resp = $x;
|
||||||
|
(serde_json::Number::from(start.elapsed().as_millis()), $resp)
|
||||||
|
}};
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user