Compare commits

..

10 Commits

21 changed files with 205 additions and 95 deletions
+1 -1
View File
@@ -19,7 +19,7 @@ build-dynamodb-benchmarker:
@cargo clean && rm -f dynamodb-benchmarker && cargo build --release && mv ./target/release/dynamodb-benchmarker . @cargo clean && rm -f dynamodb-benchmarker && cargo build --release && mv ./target/release/dynamodb-benchmarker .
build-dax-benchmarker: build-dax-benchmarker:
@rm -f main && rm -f dax-benchmarker && go build -o dax-benchmarker pkg/app/main.go @rm -f main && rm -f dax-benchmarker && go mod tidy && go build -o dax-benchmarker pkg/app/main.go
build: build-dynamodb-benchmarker build-dax-benchmarker build: build-dynamodb-benchmarker build-dax-benchmarker
+9 -2
View File
@@ -2,10 +2,9 @@
This project houses the Rust and Go code to benchmark the performance of DynamoDB and DAX by simulating heavy loads. This project houses the Rust and Go code to benchmark the performance of DynamoDB and DAX by simulating heavy loads.
![main_menu](./screenshots/main-menu.png) ![main_menu](./screenshots/main-menu.png)
![dashboard_screenshot_1](./screenshots/dashboard-screenshot-1.png) ![kibana_dashboard](./screenshots/kibana-dashboard.png)
![advanced_mode](./screenshots/advanced-mode.png) ![advanced_mode](./screenshots/advanced-mode.png)
![ansible_playbook_tail](./screenshots/ansible-playbook-tail.png) ![ansible_playbook_tail](./screenshots/ansible-playbook-tail.png)
![dashboard_screenshot_2](./screenshots/dashboard-screenshot-2.png)
## Features ## Features
* [x] Simulate reads on existing data * [x] Simulate reads on existing data
@@ -281,3 +280,11 @@ In the event you need more information about any of the automation, you can chec
* `/tmp/benchmarker.log` -- Generated whenever you run the `randomly-generate-high-velocity-data.sh` script outside the TUI * `/tmp/benchmarker.log` -- Generated whenever you run the `randomly-generate-high-velocity-data.sh` script outside the TUI
* `/tmp/benchmarker-tui.log` -- Generated by events in the TUI * `/tmp/benchmarker-tui.log` -- Generated by events in the TUI
* `/tmp/dynamodb-population.log` -- Generated whenever you run the `randomly-generate-high-velocity-data.sh` script from the TUI * `/tmp/dynamodb-population.log` -- Generated whenever you run the `randomly-generate-high-velocity-data.sh` script from the TUI
### I can't initialize the Elastic Stack; It Keeps Erroring out
Every once in a blue moon, the Elastic Stack won't start as it's supposed to by the script. The fix is to just start it
manually. It's as simple as
```shell
cd ../docker-elk && docker compose up -d && cd -
```
+2
View File
@@ -1,3 +1,5 @@
local: local:
hosts: hosts:
localhost: localhost:
+7
View File
@@ -21,6 +21,13 @@
blockinfile: blockinfile:
path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem" path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
block: "{{ aws_key_pair.key.private_key }}" block: "{{ aws_key_pair.key.private_key }}"
marker: ""
- name: Remove the blank lines from blockinfile module
lineinfile:
path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
state: absent
regexp: '^$'
when: when:
- "'destroy' not in ansible_run_tags" - "'destroy' not in ansible_run_tags"
+59 -36
View File
@@ -40,7 +40,6 @@ verify-prerequisites() {
initialize-environment() { initialize-environment() {
check-sudo-pass "Installing dependencies requires sudo permissions." check-sudo-pass "Installing dependencies requires sudo permissions."
if [[ "$?" == 0 ]]; then if [[ "$?" == 0 ]]; then
log-info "Sudo pass: $PASSWORD"
declare title="Initialize Local Environment" declare title="Initialize Local Environment"
if (prompt-yes-no "$title"); then if (prompt-yes-no "$title"); then
@@ -62,6 +61,16 @@ initialize-environment() {
main-menu main-menu
} }
prompt-for-vpc-id() {
readarray -t vpc_arr < <(aws ec2 describe-vpcs | jq -r '.Vpcs[] | "\(.VpcId) \((.Tags[]? | select(.Key | contains("Name")) | .Value) // "")"' | awk '{print($1, $2 == "" ? "-" : $2);}')
declare prompt=""
for item in "${vpc_arr[@]}"; do
prompt+="$item OFF "
done
VPC_ID=$(whiptail --fb --title "Select VPC" --radiolist "Select which VPC to use to deploy resources into" "$BOX_HEIGHT" "$BOX_WIDTH" "${#vpc_arr[@]}" $prompt 3>&2 2>&1 1>&3)
}
deploy-and-run-benchmarkers() { deploy-and-run-benchmarkers() {
declare title="Deploy and Run Benchmarkers" declare title="Deploy and Run Benchmarkers"
@@ -72,9 +81,9 @@ deploy-and-run-benchmarkers() {
cd ansible cd ansible
ansible-playbook -i inventories/local -e vpc_id="$VPC_ID" deploy_benchmarker.yml > "$ANSIBLE_LOG_FILE" 2>&1 & ansible-playbook -i inventories/local -e vpc_id="$VPC_ID" --tags deploy deploy_benchmarker.yml > "$ANSIBLE_LOG_FILE" 2>&1 &
pid=$! pid=$!
log-info "Running ansible-playbook 'deploy_benchmarker.yml' with no tags and logging output to file [$ANSIBLE_LOG_FILE]" log-info "Running ansible-playbook 'deploy_benchmarker.yml' with tags [deploy] and logging output to file [$ANSIBLE_LOG_FILE]"
show-tail-box "$title" $pid "$ANSIBLE_LOG_FILE" show-tail-box "$title" $pid "$ANSIBLE_LOG_FILE"
@@ -110,13 +119,14 @@ destroy-all() {
randomly-populate-dynamodb() { randomly-populate-dynamodb() {
declare title="Populate DynamoDB with Random Data" declare title="Populate DynamoDB with Random Data"
declare log_file=/tmp/dynamodb-data-population.log
if (prompt-yes-no "$title"); then if (prompt-yes-no "$title"); then
./scripts/randomly-generate-high-velocity-data.sh /tmp/dynamodb-population.log & ./scripts/randomly-generate-high-velocity-data.sh -i 5000 > "$log_file" 2>&1 &
pid=$! pid=$!
log-info "Running randomly-generate-high-velocity-data script and logging to [$ANSIBLE_LOG_FILE]" log-info "Running randomly-generate-high-velocity-data script and logging to [$log_file]"
show-tail-box "$title" $pid "$ANSIBLE_LOG_FILE" show-tail-box "$title" $pid "$log_file"
msg-box "Successfully populated DynamoDB with random data!" msg-box "Successfully populated DynamoDB with random data!"
log-info "Successfully populated DynamoDB with random data" log-info "Successfully populated DynamoDB with random data"
@@ -128,9 +138,12 @@ randomly-populate-dynamodb() {
custom-selections() { custom-selections() {
declare title="Customize What to Run (Advanced Mode)" declare title="Customize What to Run (Advanced Mode)"
declare choices declare choices
declare prompted_for_sudo_pass=false
declare prompted_for_vpc_id=false
declare requires_vpc_id=false
declare tags="" declare tags=""
choices=$(whiptail --separate-output --checklist --fb "$title" "$BOX_HEIGHT" "$BOX_WIDTH" 13 \ choices=$(whiptail --separate-output --checklist --fb "$title" "$CHECKBOX_HEIGHT" "$CHECKBOX_WIDTH" 13 \
"PREREQUISITES" "Install Prerequisites for Local Machine" OFF \ "PREREQUISITES" "Install Prerequisites for Local Machine" OFF \
"INITIALIZE_ELK" "Initialize Local Elastic Stack" OFF \ "INITIALIZE_ELK" "Initialize Local Elastic Stack" OFF \
"START_ELK" "Start Local Elastic Stack" OFF \ "START_ELK" "Start Local Elastic Stack" OFF \
@@ -149,55 +162,75 @@ custom-selections() {
for choice in $choices; do for choice in $choices; do
case "$choice" in case "$choice" in
"PREREQUISITES") "PREREQUISITES")
tags+="prerequisites" tags+="prerequisites,"
check-sudo-pass "Installing dependencies requires sudo permissions."
prompted_for_sudo_pass=true
;; ;;
"INITIALIZE_ELK") "INITIALIZE_ELK")
tags+="init_elk" tags+="init_elk,"
;; ;;
"START_ELK") "START_ELK")
tags+="elk" tags+="elk,"
;; ;;
"DEPLOY_CDK") "DEPLOY_CDK")
tags+="cdk" tags+="cdk,"
;; requires_vpc_id=true
"UPLOAD_BIN")
tags+="upload"
;;
"RUN_BENCHMARKERS")
tags+="run"
if [[ -z $VPC_ID ]]; then if [[ -z $VPC_ID ]]; then
prompt-for-vpc-id prompt-for-vpc-id
prompted_for_vpc_id=true
fi
;;
"UPLOAD_BIN")
tags+="upload,"
;;
"RUN_BENCHMARKERS")
tags+="run,"
requires_vpc_id=true
if [[ -z $VPC_ID ]]; then
prompt-for-vpc-id
prompted_for_vpc_id=true
fi fi
;; ;;
"STOP_ELK") "STOP_ELK")
tags+="stop_elk" tags+="stop_elk,"
;; ;;
"DESTROY") "DESTROY")
tags+="destroy" tags+="destroy,"
;; ;;
"DESTROY_KEY") "DESTROY_KEY")
tags+="destroy_key_pair" tags+="destroy_key_pair,"
;; ;;
"RUN_DYNAMODB") "RUN_DYNAMODB")
tags+="dynamodb" tags+="dynamodb,"
;; ;;
"RUN_DAX") "RUN_DAX")
tags+="dax" tags+="dax,"
;; ;;
"RUN_CRUD") "RUN_CRUD")
tags+="crud" tags+="crud,"
;; ;;
"RUN_READ_ONLY") "RUN_READ_ONLY")
tags+="read-only" tags+="read-only,"
;; ;;
esac esac
done done
tags=$(echo "$tags" | sed 's/\(.*\),/\1/')
if (prompt-yes-no "$title"); then
if (prompt-yes-no "Advanced Mode: Deploy tasks with tags: [$tags]"); then
cd ansible cd ansible
args=""
ansible-playbook -i inventories/local --tags "$tags" deploy_benchmarker.yml > "$ANSIBLE_LOG_FILE" 2>&1 & if [[ $prompted_for_sudo_pass == true ]]; then
args+=" -e ansible_become_password='$PASSWORD'"
fi
if [[ $requires_vpc_id == true && $prompted_for_vpc_id == true ]]; then
args+=" -e vpc_id=$VPC_ID"
fi
ansible-playbook -i inventories/local $args --tags "$tags" deploy_benchmarker.yml > "$ANSIBLE_LOG_FILE" 2>&1 &
pid=$! pid=$!
log-info "Running ansible-playbook 'deploy_benchmarker.yml' with [$tags] tags and logging output to file [$ANSIBLE_LOG_FILE]" log-info "Running ansible-playbook 'deploy_benchmarker.yml' with [$tags] tags and logging output to file [$ANSIBLE_LOG_FILE]"
@@ -213,16 +246,6 @@ custom-selections() {
main-menu main-menu
} }
prompt-for-vpc-id() {
readarray -t vpc_arr < <(aws ec2 describe-vpcs | jq -r '.Vpcs[] | "\(.VpcId) \((.Tags[]? | select(.Key | contains("Name")) | .Value) // "")"' | awk '{print($1, $2 == "" ? "-" : $2);}')
declare prompt=""
for item in "${vpc_arr[@]}"; do
prompt+="$item OFF "
done
VPC_ID=$(whiptail --fb --title "Select VPC" --radiolist "Select which VPC to use to deploy resources into" "$BOX_HEIGHT" "$BOX_WIDTH" "${vpc_arr[@]}" $prompt 3>&2 2>&1 1>&3)
}
main-menu() { main-menu() {
declare choice declare choice
choice=$(whiptail --fb --title "DynamoDB + DAX Benchmarker" --menu "Select an action" "$BOX_HEIGHT" "$BOX_WIDTH" 7 \ choice=$(whiptail --fb --title "DynamoDB + DAX Benchmarker" --menu "Select an action" "$BOX_HEIGHT" "$BOX_WIDTH" 7 \
+18 -18
View File
@@ -27,24 +27,24 @@ export class DaxBenchmarkingStack extends Stack {
const { instanceRole, instance } = new DaxBastionHost(this, `${user}-dax-bastion-host`, environmentProps, daxSecurityGroup); const { instanceRole, instance } = new DaxBastionHost(this, `${user}-dax-bastion-host`, environmentProps, daxSecurityGroup);
const daxClusterName = `${user}-high-velocity`; const daxClusterName = `${user}-high-velocity`;
const daxFullAccessPolicy = new PolicyStatement({ const daxFullAccessPolicy = new PolicyStatement({
effect: Effect.ALLOW, effect: Effect.ALLOW,
actions: [ actions: [
"dynamodb:BatchGetItem", "dynamodb:BatchGetItem",
"dynamodb:GetItem", "dynamodb:GetItem",
"dynamodb:Query", "dynamodb:Query",
"dynamodb:Scan", "dynamodb:Scan",
"dynamodb:BatchWriteItem", "dynamodb:BatchWriteItem",
"dynamodb:DeleteItem", "dynamodb:DeleteItem",
"dynamodb:PutItem", "dynamodb:PutItem",
"dynamodb:UpdateItem", "dynamodb:UpdateItem",
"dynamodb:DescribeLimits", "dynamodb:DescribeLimits",
"dynamodb:DescribeTimeToLive", "dynamodb:DescribeTimeToLive",
"dynamodb:DescribeTable", "dynamodb:DescribeTable",
"dynamodb:ListTables" "dynamodb:ListTables"
], ],
resources: [table.tableArn] resources: [table.tableArn]
}); });
const daxServiceRole = new Role(this, `${daxClusterName}-role`, { const daxServiceRole = new Role(this, `${daxClusterName}-role`, {
assumedBy: new ServicePrincipal("dax.amazonaws.com"), assumedBy: new ServicePrincipal("dax.amazonaws.com"),
+2 -15
View File
@@ -1,13 +1,10 @@
module github.com/Dark-Alex-17/dynamodb-benchmarker module github.com/Dark-Alex-17/dynamodb-dax-benchmarker
go 1.20 go 1.20
require ( require (
github.com/aws/aws-cdk-go/awscdk/v2 v2.88.0
github.com/aws/aws-dax-go v1.2.12 github.com/aws/aws-dax-go v1.2.12
github.com/aws/aws-sdk-go v1.44.301 github.com/aws/aws-sdk-go v1.44.301
github.com/aws/constructs-go/constructs/v10 v10.2.69
github.com/aws/jsii-runtime-go v1.85.0
github.com/elastic/go-elasticsearch/v8 v8.8.2 github.com/elastic/go-elasticsearch/v8 v8.8.2
github.com/google/uuid v1.3.0 github.com/google/uuid v1.3.0
github.com/sirupsen/logrus v1.9.3 github.com/sirupsen/logrus v1.9.3
@@ -15,19 +12,9 @@ require (
) )
require ( require (
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/cdklabs/awscdk-asset-awscli-go/awscliv1/v2 v2.2.200 // indirect
github.com/cdklabs/awscdk-asset-kubectl-go/kubectlv20/v2 v2.1.2 // indirect
github.com/cdklabs/awscdk-asset-node-proxy-agent-go/nodeproxyagentv5/v2 v2.0.165 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/yuin/goldmark v1.4.13 // indirect github.com/stretchr/testify v1.8.4 // indirect
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/tools v0.11.0 // indirect
) )
require ( require (
+80
View File
@@ -0,0 +1,80 @@
github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e h1:yxMh4HIdsSh2EqxUESWvzszYMNzOugRyYCeohfwNULM=
github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
github.com/aws/aws-dax-go v1.2.12 h1:Ee9jW0lWbztRf7Gh7ErqxUQx4iO2kyrC/GSyAaivSFE=
github.com/aws/aws-dax-go v1.2.12/go.mod h1:SqxIoetx6kj8n3DHuCwOykVfPeuwm8xmabA4Nbf7Yq0=
github.com/aws/aws-sdk-go v1.44.171/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.301 h1:VofuXktwHFTBUvoPiHxQis/3uKgu0RtgUwLtNujd3Zs=
github.com/aws/aws-sdk-go v1.44.301/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo=
github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
github.com/elastic/go-elasticsearch/v8 v8.8.2 h1:3ITzPlRNadzDnbLTnMRjrAN4j4G3LvFo5gCIWDPS6pY=
github.com/elastic/go-elasticsearch/v8 v8.8.2/go.mod h1:GU1BJHO7WeamP7UhuElYwzzHtvf9SDmeVpSSy9+o6Qg=
github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84=
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/loremipsum.v1 v1.1.2 h1:12APklfJKuGszqZsrArW5QoQh03/W+qyCCjvnDuS6Tw=
gopkg.in/loremipsum.v1 v1.1.2/go.mod h1:TuRvzFuzuejXj+odBU6Tubp/EPUyGb9wmSvHenyP2Ts=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+3 -3
View File
@@ -11,9 +11,9 @@ import (
"strings" "strings"
"time" "time"
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/models" "github.com/Dark-Alex-17/dynamodb-dax-benchmarker/pkg/models"
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/simulators" "github.com/Dark-Alex-17/dynamodb-dax-benchmarker/pkg/simulators"
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/utils" "github.com/Dark-Alex-17/dynamodb-dax-benchmarker/pkg/utils"
"github.com/aws/aws-dax-go/dax" "github.com/aws/aws-dax-go/dax"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials"
+1 -1
View File
@@ -3,7 +3,7 @@ package simulators
import ( import (
"time" "time"
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/models" "github.com/Dark-Alex-17/dynamodb-dax-benchmarker/pkg/models"
"github.com/aws/aws-dax-go/dax" "github.com/aws/aws-dax-go/dax"
"github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/models" "github.com/Dark-Alex-17/dynamodb-dax-benchmarker/pkg/models"
"github.com/aws/aws-dax-go/dax" "github.com/aws/aws-dax-go/dax"
"github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
+1 -1
View File
@@ -3,7 +3,7 @@ package simulators
import ( import (
"time" "time"
"github.com/Dark-Alex-17/dynamodb-benchmarker/pkg/models" "github.com/Dark-Alex-17/dynamodb-dax-benchmarker/pkg/models"
"github.com/aws/aws-dax-go/dax" "github.com/aws/aws-dax-go/dax"
"github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
Binary file not shown.

Before

Width:  |  Height:  |  Size: 63 KiB

After

Width:  |  Height:  |  Size: 59 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 75 KiB

After

Width:  |  Height:  |  Size: 141 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 114 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 121 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 183 KiB

+3 -3
View File
@@ -15,7 +15,7 @@ bold=$(tput bold)
log-error() { log-error() {
if [[ -z $2 ]]; then if [[ -z $2 ]]; then
echo -e "${red}${bold}ERROR:${default}${red} $1${default}" echo -e "ERROR: $1"
else else
echo -e "${red}${bold}ERROR:${default}${red} $1${default}" echo -e "${red}${bold}ERROR:${default}${red} $1${default}"
echo -e "${red}${bold}ERROR:${default}${red} $1${default}" >> "$BENCHMARK_LOG_FILE" echo -e "${red}${bold}ERROR:${default}${red} $1${default}" >> "$BENCHMARK_LOG_FILE"
@@ -24,7 +24,7 @@ log-error() {
log-warn() { log-warn() {
if [[ -z $2 ]]; then if [[ -z $2 ]]; then
echo -e "${gold}${bold}WARN:${default}${gold} $1${default}" echo -e "WARN: $1"
else else
echo -e "${gold}${bold}WARN:${default}${gold} $1${default}" echo -e "${gold}${bold}WARN:${default}${gold} $1${default}"
echo -e "${gold}${bold}WARN:${default}${gold} $1${default}" >> "$BENCHMARK_LOG_FILE" echo -e "${gold}${bold}WARN:${default}${gold} $1${default}" >> "$BENCHMARK_LOG_FILE"
@@ -33,7 +33,7 @@ log-warn() {
log-info() { log-info() {
if [[ -z $2 ]]; then if [[ -z $2 ]]; then
echo -e "${cyan}${bold}INFO:${default}${cyan} $1${default}" echo -e "INFO: $1"
else else
echo -e "${cyan}${bold}INFO:${default}${cyan} $1${default}" echo -e "${cyan}${bold}INFO:${default}${cyan} $1${default}"
echo -e "${cyan}${bold}INFO:${default}${cyan} $1${default}" >> "$BENCHMARK_LOG_FILE" echo -e "${cyan}${bold}INFO:${default}${cyan} $1${default}" >> "$BENCHMARK_LOG_FILE"
@@ -105,10 +105,8 @@ parse-arguments() {
show-properties() { show-properties() {
log-info "Using the following settings to randomly populate the DynamoDB benchmarking table:" log-info "Using the following settings to randomly populate the DynamoDB benchmarking table:"
cat <<-EOF cat <<-EOF
${cyan}
ATTRIBUTES=$ATTRIBUTES ATTRIBUTES=$ATTRIBUTES
TABLE_NAME=$TABLE_NAME TABLE_NAME=$TABLE_NAME
${default}
EOF EOF
} }
@@ -129,7 +127,7 @@ generate-put-request() {
attribute_values=$(generate-attribute-value 0) attribute_values=$(generate-attribute-value 0)
for j in $(seq 1 $((ATTRIBUTES-1))); do for j in $(seq 1 $((ATTRIBUTES-1))); do
attribute_values="$attribute_values, $(generate-attribute-value "$j")" attribute_values+=", $(generate-attribute-value "$j")"
done done
@@ -147,13 +145,13 @@ generate-put-request() {
generate-batch-json() { generate-batch-json() {
declare batch_request='{ "'"$TABLE_NAME"'": [' declare batch_request='{ "'"$TABLE_NAME"'": ['
batch_request="$batch_request $(generate-put-request)" batch_request+=" $(generate-put-request)"
for i in $(seq 0 23); do for i in $(seq 0 23); do
batch_request="$batch_request, $(generate-put-request)" batch_request+=", $(generate-put-request)"
done done
batch_request="$batch_request ]}" batch_request+=" ]}"
echo "$batch_request" echo "$batch_request"
} }
@@ -166,15 +164,15 @@ fi
parse-arguments "$@" parse-arguments "$@"
show-properties show-properties
declare -i i=0
declare -i items_written=0 declare -i items_written=0
while [[ $items_written -lt $ITEMS ]]; do while [[ $items_written -lt $ITEMS ]]; do
log-info "Writing 25 entries to DynamoDB..." log-info "Writing 25 entries to DynamoDB..."
aws dynamodb batch-write-item --request-items "$(generate-batch-json)" aws dynamodb batch-write-item --request-items "$(generate-batch-json)"
log-info 'Entries Written!' log-info 'Entries Written!'
((i++))
((items_written+=25)) ((items_written+=25))
log-info "Total entries written: $items_written" log-info "Total entries written: $items_written"
log-info "Sleeping for 2 seconds to avoid the partition throughput limits..." log-info "Sleeping for 2 seconds to avoid the partition throughput limits..."
sleep 2 sleep 2
done done
echo -e "\n\n\nSuccessfully wrote $items_written randomly generated items to DynamoDB!"
+5 -1
View File
@@ -1,9 +1,13 @@
#!/bin/bash #!/bin/bash
TERMINAL_HEIGHT=$(tput lines) TERMINAL_HEIGHT=$(tput lines)
BOX_HEIGHT=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_HEIGHT * .5" | bc)") BOX_HEIGHT=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_HEIGHT * .5" | bc)")
CHECKBOX_HEIGHT=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_HEIGHT * .5" | bc)")
TAIL_BOX_HEIGHT=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_HEIGHT * .8" | bc)")
TERMINAL_WIDTH=$(tput cols) TERMINAL_WIDTH=$(tput cols)
BOX_WIDTH=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_WIDTH * .75" | bc)") BOX_WIDTH=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_WIDTH * .75" | bc)")
CHECKBOX_WIDTH=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_WIDTH * .5" | bc)")
TAIL_BOX_WIDTH=$(printf "%.0f" "$(echo "scale=2; $TERMINAL_WIDTH * .85" | bc)")
msg-box() { msg-box() {
whiptail --fb --msgbox "$1" "$BOX_HEIGHT" "$BOX_WIDTH" whiptail --fb --msgbox "$1" "$BOX_HEIGHT" "$BOX_WIDTH"
@@ -20,7 +24,7 @@ show-tail-box() {
trap "kill $2 2> /dev/null" EXIT trap "kill $2 2> /dev/null" EXIT
while kill -0 "$2" 2> /dev/null; do while kill -0 "$2" 2> /dev/null; do
dialog --title "$1" --exit-label "Finished" --tailbox "$3" "$BOX_HEIGHT" "$BOX_WIDTH" dialog --title "$1" --exit-label "Finished" --tailbox "$3" "$TAIL_BOX_HEIGHT" "$TAIL_BOX_WIDTH"
done done
clear clear
+5 -3
View File
@@ -195,8 +195,10 @@ async fn simulation_loop(
) { ) {
let mut rng = StdRng::from_seed(OsRng.gen()); let mut rng = StdRng::from_seed(OsRng.gen());
loop { loop {
let mut metrics = DynamoDbSimulationMetrics::default(); let mut metrics = DynamoDbSimulationMetrics {
metrics.timestamp = Utc::now(); timestamp: Utc::now(),
..DynamoDbSimulationMetrics::default()
};
let simulation_time = time!(match { let simulation_time = time!(match {
if read_only { if read_only {
@@ -280,7 +282,7 @@ async fn scan_all_partition_keys(
let partition_keys = resp let partition_keys = resp
.items() .items()
.unwrap() .unwrap()
.into_iter() .iter()
.map(|attribute| { .map(|attribute| {
attribute attribute
.values() .values()