Completed DynamoDB + DAX Benchmarker with a nice TUI to boot

2023-08-02 18:11:41 -06:00
1 parent 09862c1b43
commit e42070eefa
@@ -0,0 +1,5 @@
+{"attributes":{"fieldAttrs":"{}","fieldFormatMap":"{}","fields":"[]","name":"dynamodb","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"timestamp","title":"dynamodb*","typeMeta":"{}"},"coreMigrationVersion":"8.8.0","created_at":"2023-07-17T16:24:37.085Z","id":"2c97a7eb-1968-415f-9281-370373850ec7","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2023-07-17T17:40:31.727Z","version":"WzI0NiwzXQ=="}
+{"attributes":{"controlGroupInput":{"chainingSystem":"HIERARCHICAL","controlStyle":"oneLine","ignoreParentSettingsJSON":"{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}","panelsJSON":"{\"e63ea259-fd86-4227-b1db-33be15a1d500\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"small\",\"explicitInput\":{\"id\":\"e63ea259-fd86-4227-b1db-33be15a1d500\",\"fieldName\":\"successful\",\"title\":\"Simulation Success\",\"exclude\":false,\"singleSelect\":true,\"selectedOptions\":[],\"enhancements\":{}}},\"30bb36b4-23ed-400c-a98d-866a0e4a9f5c\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"small\",\"explicitInput\":{\"id\":\"30bb36b4-23ed-400c-a98d-866a0e4a9f5c\",\"fieldName\":\"operation.keyword\",\"title\":\"Operation\",\"selectedOptions\":[],\"enhancements\":{}}},\"c13132b9-3c01-46a6-8ad5-579b8861cbe7\":{\"type\":\"timeSlider\",\"order\":3,\"grow\":true,\"width\":\"large\",\"explicitInput\":{\"id\":\"c13132b9-3c01-46a6-8ad5-579b8861cbe7\",\"title\":\"Time slider\",\"timesliceStartAsPercentageOfTimeRange\":0,\"timesliceEndAsPercentageOfTimeRange\":0.5,\"enhancements\":{}}},\"bbd18673-d743-46d7-8cb4-d4e25408dc70\":{\"type\":\"optionsListControl\",\"order\":2,\"grow\":true,\"width\":\"small\",\"explicitInput\":{\"id\":\"bbd18673-d743-46d7-8cb4-d4e25408dc70\",\"fieldName\":\"scenario.keyword\",\"title\":\"Scenario\",\"singleSelect\":true,\"selectedOptions\":[],\"existsSelected\":false,\"enhancements\":{}}}}"},"description":"1,000 concurrent tasks run against plain DynamoDB without DAX","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":true,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":7,\"h\":9,\"i\":\"30d02c1c-4524-469a-90f9-4a900880edc3\"},\"panelIndex\":\"30d02c1c-4524-469a-90f9-4a900880edc3\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"description\":\"\",\"visualizationType\":\"lnsLegacyMetric\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"2c97a7eb-1968-415f-9281-370373850ec7\",\"name\":\"indexpattern-datasource-layer-04f4f1c7-234d-420a-99c9-1b7cce672086\"}],\"state\":{\"visualization\":{\"layerId\":\"04f4f1c7-234d-420a-99c9-1b7cce672086\",\"accessor\":\"5d1065c7-dd42-471b-a7cc-c71d6146cb41\",\"layerType\":\"data\",\"textAlign\":\"center\",\"size\":\"m\",\"colorMode\":\"None\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"04f4f1c7-234d-420a-99c9-1b7cce672086\":{\"columns\":{\"5d1065c7-dd42-471b-a7cc-c71d6146cb41\":{\"label\":\"Total Simulations Run\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":false,\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"customLabel\":true,\"filter\":{\"query\":\"\",\"language\":\"kuery\"}}},\"columnOrder\":[\"5d1065c7-dd42-471b-a7cc-c71d6146cb41\"],\"sampling\":1,\"incompleteColumns\":{}}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"enhancements\":{},\"hidePanelTitles\":true}},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":7,\"y\":0,\"w\":7,\"h\":9,\"i\":\"2476847d-06a2-44e0-ba3a-e4e28f4e4f18\"},\"panelIndex\":\"2476847d-06a2-44e0-ba3a-e4e28f4e4f18\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"description\":\"\",\"visualizationType\":\"lnsLegacyMetric\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"2c97a7eb-1968-415f-9281-370373850ec7\",\"name\":\"indexpattern-datasource-layer-69edb1db-56fc-4d65-827d-73d0141c58b3\"}],\"state\":{\"visualization\":{\"layerId\":\"69edb1db-56fc-4d65-827d-73d0141c58b3\",\"accessor\":\"2499fd5c-89d9-48dc-a735-a1c3f3cb1605\",\"layerType\":\"data\",\"textAlign\":\"center\",\"colorMode\":\"Background\",\"palette\":{\"name\":\"custom\",\"type\":\"palette\",\"params\":{\"steps\":3,\"name\":\"custom\",\"reverse\":false,\"rangeType\":\"number\",\"rangeMin\":0,\"rangeMax\":null,\"progression\":\"fixed\",\"stops\":[{\"color\":\"#cc5642\",\"stop\":1},{\"color\":\"#209280\",\"stop\":7844}],\"colorStops\":[{\"color\":\"#cc5642\",\"stop\":0},{\"color\":\"#209280\",\"stop\":1}],\"continuity\":\"above\",\"maxSteps\":5}}},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"69edb1db-56fc-4d65-827d-73d0141c58b3\":{\"columns\":{\"2499fd5c-89d9-48dc-a735-a1c3f3cb1605\":{\"label\":\"Successful Simulations\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"filter\":{\"query\":\"successful: true\",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":false},\"customLabel\":true}},\"columnOrder\":[\"2499Line truncated
+{"attributes":{"fieldAttrs":"{}","fieldFormatMap":"{}","fields":"[]","name":"dax","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"timestamp","title":"dax*","typeMeta":"{}"},"coreMigrationVersion":"8.8.0","created_at":"2023-07-20T19:47:27.359Z","id":"e23af679-3d6b-4f98-9231-5dfdf59cb9aa","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2023-07-20T20:05:34.733Z","version":"WzcyNCw2XQ=="}
+{"attributes":{"controlGroupInput":{"chainingSystem":"HIERARCHICAL","controlStyle":"oneLine","ignoreParentSettingsJSON":"{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}","panelsJSON":"{\"e63ea259-fd86-4227-b1db-33be15a1d500\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"small\",\"explicitInput\":{\"id\":\"e63ea259-fd86-4227-b1db-33be15a1d500\",\"fieldName\":\"successful\",\"title\":\"Simulation Success\",\"exclude\":false,\"singleSelect\":true,\"selectedOptions\":[],\"enhancements\":{}}},\"30bb36b4-23ed-400c-a98d-866a0e4a9f5c\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"small\",\"explicitInput\":{\"id\":\"30bb36b4-23ed-400c-a98d-866a0e4a9f5c\",\"fieldName\":\"operation.keyword\",\"title\":\"Operation\",\"selectedOptions\":[],\"enhancements\":{}}},\"c13132b9-3c01-46a6-8ad5-579b8861cbe7\":{\"type\":\"timeSlider\",\"order\":3,\"grow\":true,\"width\":\"large\",\"explicitInput\":{\"id\":\"c13132b9-3c01-46a6-8ad5-579b8861cbe7\",\"title\":\"Time slider\",\"timesliceStartAsPercentageOfTimeRange\":0,\"timesliceEndAsPercentageOfTimeRange\":0.48333333333333334,\"enhancements\":{}}},\"aed07897-7b46-4d70-abbc-f4f55cb69076\":{\"type\":\"optionsListControl\",\"order\":2,\"grow\":true,\"width\":\"small\",\"explicitInput\":{\"id\":\"aed07897-7b46-4d70-abbc-f4f55cb69076\",\"fieldName\":\"scenario.keyword\",\"title\":\"Scenario\",\"singleSelect\":true,\"enhancements\":{}}}}"},"description":"1,000 concurrent tasks run against DAX","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":true,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":7,\"h\":9,\"i\":\"30d02c1c-4524-469a-90f9-4a900880edc3\"},\"panelIndex\":\"30d02c1c-4524-469a-90f9-4a900880edc3\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"description\":\"\",\"visualizationType\":\"lnsLegacyMetric\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"e23af679-3d6b-4f98-9231-5dfdf59cb9aa\",\"name\":\"indexpattern-datasource-layer-04f4f1c7-234d-420a-99c9-1b7cce672086\"}],\"state\":{\"visualization\":{\"layerId\":\"04f4f1c7-234d-420a-99c9-1b7cce672086\",\"accessor\":\"5d1065c7-dd42-471b-a7cc-c71d6146cb41\",\"layerType\":\"data\",\"textAlign\":\"center\",\"size\":\"m\",\"colorMode\":\"None\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"04f4f1c7-234d-420a-99c9-1b7cce672086\":{\"columns\":{\"5d1065c7-dd42-471b-a7cc-c71d6146cb41\":{\"label\":\"Total Simulations Run\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":false,\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"customLabel\":true,\"filter\":{\"query\":\"\",\"language\":\"kuery\"}}},\"columnOrder\":[\"5d1065c7-dd42-471b-a7cc-c71d6146cb41\"],\"sampling\":1,\"incompleteColumns\":{}}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"enhancements\":{},\"hidePanelTitles\":true}},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":7,\"y\":0,\"w\":7,\"h\":9,\"i\":\"2476847d-06a2-44e0-ba3a-e4e28f4e4f18\"},\"panelIndex\":\"2476847d-06a2-44e0-ba3a-e4e28f4e4f18\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"description\":\"\",\"visualizationType\":\"lnsLegacyMetric\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"e23af679-3d6b-4f98-9231-5dfdf59cb9aa\",\"name\":\"indexpattern-datasource-layer-69edb1db-56fc-4d65-827d-73d0141c58b3\"}],\"state\":{\"visualization\":{\"layerId\":\"69edb1db-56fc-4d65-827d-73d0141c58b3\",\"accessor\":\"2499fd5c-89d9-48dc-a735-a1c3f3cb1605\",\"layerType\":\"data\",\"textAlign\":\"center\",\"colorMode\":\"Background\",\"palette\":{\"name\":\"custom\",\"type\":\"palette\",\"params\":{\"steps\":3,\"name\":\"custom\",\"reverse\":false,\"rangeType\":\"number\",\"rangeMin\":0,\"rangeMax\":null,\"progression\":\"fixed\",\"stops\":[{\"color\":\"#cc5642\",\"stop\":1},{\"color\":\"#209280\",\"stop\":7844}],\"colorStops\":[{\"color\":\"#cc5642\",\"stop\":0},{\"color\":\"#209280\",\"stop\":1}],\"continuity\":\"above\",\"maxSteps\":5}}},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"69edb1db-56fc-4d65-827d-73d0141c58b3\":{\"columns\":{\"2499fd5c-89d9-48dc-a735-a1c3f3cb1605\":{\"label\":\"Successful Simulations\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"filter\":{\"query\":\"successful: true\",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":false},\"customLabel\":true}},\"columnOrder\":[\"2499fd5c-89d9-48dc-a735-a1c3f3cb1605\"],\"sampling\":1,\"inLine truncated
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":4,"missingRefCount":0,"missingReferences":[]}
@@ -0,0 +1,32 @@
+- name: Clone the docker-elk repo
+  git:
+    repo: https://github.com/deviantony/docker-elk.git
+    dest: ../../docker-elk
+  ignore_errors: yes
+
+- name: Build the docker-elk stack just in case a pre-existing version of Elasticsearch needs its nodes upgraded
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker compose build
+
+- name: Start the docker-elk setup container
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker-compose up setup
+
+- name: Start the docker-elk stack
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker compose up -d
+
+- name: Wait 20 seconds for the ELK stack to start
+  pause:
+    seconds: 20
+
+- name: Import the benchmarking dashboards into Kibana
+  shell:
+    cmd: >
+      curl -X POST http://localhost:5601/api/saved_objects/_import?overwrite=true 
+      -H 'kbn-xsrf: true' 
+      -u 'elastic:changeme' 
+      --form file=@roles/configure_elastic_stack/files/benchmarker-dashboards.ndjson
@@ -0,0 +1,8 @@
+- { import_tasks: init_elk_stack.yml, tags: [ never, init, init_elk ] }
+- { import_tasks: stop_elk_stack.yml, tags: [ never, stop_elk ] }
+
+- name: Start the docker-elk stack
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker compose up -d
+  tags: deploy
@@ -0,0 +1,4 @@
+- name: Stop the docker-elk stack
+  shell:
+    chdir: ../../docker-elk
+    cmd: docker compose down