Completed DynamoDB + DAX Benchmarker with a nice TUI to boot

2023-08-02 18:11:41 -06:00
parent 09862c1b43
commit e42070eefa
55 changed files with 3574 additions and 1 deletions
@@ -0,0 +1,32 @@
+- name: Clone the docker-elk repo
+  git:
+    repo: https://github.com/deviantony/docker-elk.git
+    dest: ../../docker-elk
+  ignore_errors: yes
+
+- name: Build the docker-elk stack just in case a pre-existing version of Elasticsearch needs its nodes upgraded
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker compose build
+
+- name: Start the docker-elk setup container
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker-compose up setup
+
+- name: Start the docker-elk stack
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker compose up -d
+
+- name: Wait 20 seconds for the ELK stack to start
+  pause:
+    seconds: 20
+
+- name: Import the benchmarking dashboards into Kibana
+  shell:
+    cmd: >
+      curl -X POST http://localhost:5601/api/saved_objects/_import?overwrite=true 
+      -H 'kbn-xsrf: true' 
+      -u 'elastic:changeme' 
+      --form file=@roles/configure_elastic_stack/files/benchmarker-dashboards.ndjson
@@ -0,0 +1,8 @@
+- { import_tasks: init_elk_stack.yml, tags: [ never, init, init_elk ] }
+- { import_tasks: stop_elk_stack.yml, tags: [ never, stop_elk ] }
+
+- name: Start the docker-elk stack
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker compose up -d
+  tags: deploy
@@ -0,0 +1,4 @@
+- name: Stop the docker-elk stack
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker compose down
@@ -0,0 +1,119 @@
+- name: Check if a key-pair following the specified format already exists
+  stat:
+    path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
+  register: key_pair
+  changed_when: no
+  when: "'destroy' not in ansible_run_tags"
+
+- block:
+    - name: Create a new key-pair
+      ec2_key:
+        name: "{{ ssh_key_name }}"
+      register: aws_key_pair
+
+    - name: Create the new pem file
+      file:
+        path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
+        state: touch
+        mode: '0400'
+
+    - name: Add the generated key-pair to the new file
+      blockinfile:
+        path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
+        block: "{{ aws_key_pair.key.private_key }}"
+
+  when:
+   - "'destroy' not in ansible_run_tags"
+   - not key_pair.stat.exists
+
+- name: Fetch the current system's public IP
+  shell:
+    cmd: curl -s -L checkip.amazonaws.com
+  register: public_ip_resp
+
+- name: Fetch the current AWS account ID
+  shell:
+    cmd: aws sts get-caller-identity | jq -r .Account
+  register: aws_account_resp
+
+- name: Install CDK dependencies
+  npm:
+    ci: yes
+    path: ../cdk
+
+- name: Bootstrapping the AWS environment
+  shell:
+    chdir: ../cdk
+    cmd: >
+      npm run build && yes | npm run cdk bootstrap --
+      --no-color --require-approval never
+      --profile {{ profile_id | default("personal") }}
+      -c vpcId={{ vpc_id }}
+      -c localIp={{ public_ip_resp.stdout }}
+      -c sshKeyName={{ ssh_key_name }}
+      -c awsAccount={{ aws_account_resp.stdout }}
+      -c baseTableName={{ base_table_name | default('') }}
+
+- name: Deploying Benchmarking CDK
+  shell:
+    chdir: ../cdk
+    cmd: >
+      npm run build && yes | npm run cdk {{ cdk_action | default("deploy") }} --
+      --no-color --require-approval never
+      --profile {{ profile_id | default("personal") }}
+      -c vpcId={{ vpc_id }}
+      -c localIp={{ public_ip_resp.stdout }}
+      -c sshKeyName={{ ssh_key_name }}
+      -c awsAccount={{ aws_account_resp.stdout }}
+      -c baseTableName={{ base_table_name | default('') }}
+  register: cdk_response
+
+- name: Benchmarking CDK deployment summary
+  debug:
+    msg: "{{ cdk_response.stderr_lines }}"
+
+- block:
+  - name: Fetch the benchmark stack outputs
+    cloudformation_info:
+      stack_name: "{{ stack_name }}"
+    register: benchmark_stack
+
+  - name: Extracting the bastion host IP
+    set_fact:
+      bastion_host_ip: "{{ benchmark_stack.cloudformation[stack_name].stack_outputs['InstancePublicIp'] }}"
+
+  - name: Extracting DAX endpoint
+    set_fact:
+      dax_endpoint: "{{ benchmark_stack.cloudformation[stack_name].stack_outputs['DaxEndpoint'] }}"
+
+  - name: Setting the dax_endpoint variable in the host vars if it doesn't exist already
+    lineinfile:
+      path: inventories/local/host_vars/localhost.yml
+      line: "dax_endpoint: {{ dax_endpoint }}"
+      regexp: '^dax_endpoint:'
+
+  - name: Setting the vpc_id variable in the host vars if it doesn't exist already
+    lineinfile:
+      path: inventories/local/host_vars/localhost.yml
+      line: "vpc_id: {{ vpc_id }}"
+      regexp: '^vpc_id:'
+
+  - block:
+    - name: Setting the bastion host IP if it doesnt exist in the inventory
+      lineinfile:
+        path: inventories/local/hosts.yml
+        line: |
+          bastion:
+            hosts:
+              {{ bastion_host_ip }}:
+        regexp: 'bastion:\n\s*hosts:\n\s*(?:\d{1,3}\.){3}\d{1,3}:'
+        insertafter: EOF
+
+    - name: Add the bastion host to the bastion group
+      add_host:
+        name: "{{ bastion_host_ip }}"
+        groups: bastion
+    when:
+      - "'bastion' not in groups"
+      - "'bastion' not in group_names"
+  when: "'destroy' not in ansible_run_tags"
@@ -0,0 +1,54 @@
+- name: Wipe away local Elastic Stack
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker compose down -v
+  ignore_errors: yes
+
+- name: Wipe away the ELK directory
+  file:
+    path: ../../docker-elk
+    state: absent
+  ignore_errors: yes
+
+- name: Run CDK Destroy
+  import_role:
+    name:
+      deploy_cdk
+
+- name: Delete the key-pair from AWS
+  ec2_key:
+    name: "{{ ssh_key_name }}"
+    state: absent
+  ignore_errors: yes
+  tags: [ never, destroy_key_pair ]
+
+- name: Delete the key pair from your local machine
+  file:
+    path: "{{ ansible_env.HOME }}/.ssh/{{ ssh_key_name }}.pem"
+    state: absent
+  ignore_errors: yes
+  tags: [ never, destroy_key_pair ]
+
+- name: Remove the bastion host from the bastion host group
+  replace:
+    path: inventories/local/hosts.yml
+    replace: ''
+    regexp: '^bastion:\n\s*hosts:\n\s*(?:\d{1,3}\.){3}\d{1,3}:'
+
+- name: Reset the dax_endpoint variable in the host vars
+  lineinfile:
+    path: inventories/local/host_vars/localhost.yml
+    line: 'dax_endpoint:'
+    regexp: '^dax_endpoint:'
+
+- name: Reset the vpc_id variable in the host vars
+  lineinfile:
+    path: inventories/local/host_vars/localhost.yml
+    line: 'vpc_id:'
+    regexp: '^vpc_id:'
+
+- name: Clean the repository using the Makefile
+  shell:
+    chdir: ../
+    cmd:
+      make clean
@@ -0,0 +1,22 @@
+- name: Add Docker's official GPG key
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    keyring: /etc/apt/keyrings/docker.gpg
+
+- name: Set up docker APT repository
+  apt_repository:
+    repo: "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
+
+- name: Install the required APT dependencies
+  apt:
+    update_cache: yes
+    name:
+      - docker-ce
+      - docker-ce-cli
+      - docker-compose
+      - containerd.io
+      - docker-compose-plugin
+      - jq
+      - unzip
+      - curl
+      - git
@@ -0,0 +1,26 @@
+- name: Check if AWS CLI is installed
+  shell:
+    cmd: hash aws 2> /dev/null
+  ignore_errors: yes
+  changed_when: no
+  register: awscli_installation_status
+
+- block:
+    - name: Download the AWS CLI from AWS
+      unarchive:
+        src: https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+        dest: "{{ ansible_env.HOME }}/Downloads"
+        group: "{{ user_name }}"
+        owner: "{{ user_name }}"
+        remote_src: yes
+
+    - name: Install the AWS CLI
+      shell:
+        cmd: "{{ ansible_env.HOME }}/Downloads/aws/install"
+
+    - name: Cleanup downloaded AWS installation files
+      file:
+        path: "{{ ansible_env.HOME }}/Downloads/aws/"
+        state: absent
+
+  when: awscli_installation_status.rc | int != 0
@@ -0,0 +1,15 @@
+- name: Check if Go is installed
+  shell:
+    cmd: command -v go 2> /dev/null
+  ignore_errors: yes
+  changed_when: no
+  register: go_installation_status
+
+- name: Install Go 1.20
+  unarchive:
+    src: https://go.dev/dl/go1.20.5.linux-amd64.tar.gz
+    dest: /usr/local
+    creates: /usr/local/go
+    remote_src: yes
+  become: yes
+  when: go_installation_status.rc | int != 0
@@ -0,0 +1,25 @@
+- { import_tasks: aws_cli.yml, become: yes }
+- import_tasks: rust.yml
+- import_tasks: go.yml
+- import_tasks: node.yml
+- { import_tasks: apt.yml, become: yes }
+
+- name: Install CDK
+  npm:
+    name: "{{ item }}"
+    global: yes
+  loop:
+    - aws-cdk
+    - typescript
+
+- name: Check if golangci-lint is installed
+  shell:
+    cmd: command -v golangci-lint 2> /dev/null
+  ignore_errors: yes
+  changed_when: no
+  register: golangci_lint_installation_status
+
+- name: Install golangci-lint
+  shell:
+    cmd: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.53.3
+  when: golangci_lint_installation_status.rc | int != 0
@@ -0,0 +1,34 @@
+- name: Check if node is installed
+  shell:
+    cmd: hash node 2> /dev/null
+  ignore_errors: yes
+  changed_when: no
+  register: node_installation_status
+
+- block:
+    - name: Install nvm
+      shell: >
+        curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash
+      args:
+        creates: "{{ ansible_env.HOME }}/.nvm/nvm.sh"
+
+    - name: Install Node.JS
+      shell:
+        cmd: |
+          export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
+          [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+          nvm install node
+
+    - name: Add NVM exports to bashrc
+      lineinfile:
+        path: "{{ ansible_env.HOME }}/.bashrc"
+        line: 'export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"'
+        regexp: '^export NVM_DIR=.+'
+
+    - name: Add NVM script to bashrc
+      lineinfile:
+        path: "{{ ansible_env.HOME }}/.bashrc"
+        line: '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"'
+        regexp: '\[ -s |\$NVM_DIR/nvm\.sh \].+'
+
+  when: node_installation_status.rc | int != 0
@@ -0,0 +1,11 @@
+- name: Check if rustup is installed
+  shell:
+    cmd: command -v rustup 2> /dev/null
+  ignore_errors: yes
+  changed_when: no
+  register: rustup_installation_status
+
+- name: Install Rust via Rustup
+  shell: >
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+  when: rustup_installation_status.rc | int != 0