diff --git a/.github/workflows/test_and_release.yml b/.github/workflows/test_and_release.yml index d74c9c2..e0ba32e 100644 --- a/.github/workflows/test_and_release.yml +++ b/.github/workflows/test_and_release.yml @@ -40,7 +40,7 @@ jobs: docker build . -t bbvalabsci/kapow-spec-test-suite:latest - name: Spec test run: | - docker run --mount type=bind,source=$(pwd)/build/kapow,target=/usr/local/bin/kapow bbvalabsci/kapow-spec-test-suite:latest behave --tags=~@skip + docker run --mount type=bind,source=$(pwd)/build/kapow,target=/usr/bin/kapow bbvalabsci/kapow-spec-test-suite:latest "behave --tags=~@skip" doc-test: runs-on: ubuntu-20.04 steps: diff --git a/.gitignore b/.gitignore index 372dd5d..bf081cc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ build docs/build docs/Pipfile.lock + +node_modules + +*.swp diff --git a/Makefile b/Makefile index 4a22e19..ef53067 100644 --- a/Makefile +++ b/Makefile @@ -42,8 +42,8 @@ coverage: test race install: build CGO_ENABLED=0 $(GOINSTALL) ./... -acceptance: install - make -C ./spec/test +acceptance: build + cd ./spec/test && PATH=$(PWD)/build:$$PATH nix-shell --command make deps: @echo "deps here" diff --git a/spec/README.md b/spec/README.md index bd16e4e..9a868ba 100644 --- a/spec/README.md +++ b/spec/README.md @@ -130,6 +130,7 @@ whole lifetime of the server. * Kapow! implementations should follow a general principle of robustness: be conservative in what you do, be liberal in what you accept from others. * We reuse conventions of well-established software projects, such as Docker. +* Secure by default, the Control API can *only* be accessed using mTLS. * All requests and responses will leverage JSON as the data encoding method. * The API calls responses have several parts: * The HTTP status code (e.g., `400`, which is a bad request). The target @@ -178,6 +179,30 @@ Content-Length: 25 ``` +## mTLS + +The Kapow! server generates a pair of keys and certificates, one for the +server, the other for the configuring client. The necessary elements will be +communicated to the client (the init program) via a set of environment +variables. + +The aforementioned variables are named: + +- `KAPOW_CONTROL_SERVER_CERT`: server certificate. +- `KAPOW_CONTROL_CLIENT_CERT`: client certificate. +- `KAPOW_CONTROL_CLIENT_KEY`: client private key. + +Note that all variables contain x509 PEM-encoded values. +Also note that the server private key is not communicated in any way. + +Following the mTLS discipline, the client must ensure upon connecting to the +server that its certificate matches the one stored in +`KAPOW_CONTROL_SERVER_CERT`. + +Conversely, the server must only communicate with clients whose certificate +matches the one stored in `KAPOW_CONTROL_CLIENT_CERT`. + + ## API Elements Kapow! provides a way to control its internal state through these elements. @@ -606,8 +631,6 @@ Commands: ``` -### `kapow server` - This command runs the Kapow! server, which is the core of Kapow!. If run without parameters, it will run an unconfigured server. It can accept a path to an executable file, the init program, which can be a shell script that @@ -615,7 +638,7 @@ contains commands to configure the *Kapow!* server. The init program can leverage the `kapow route` command, which is used to define a route. The `kapow route` command needs a way to reach the *Kapow!* server, -and for that, `kapow` provides the `KAPOW_DATA_URL` variable in the environment +and for that, `kapow` provides the `KAPOW_CONTROL_URL` variable in the environment of the aforementioned init program. Every time the *Kapow!* server receives a request, it will spawn a process to @@ -655,7 +678,10 @@ To deregister a route you must provide a *route_id*. #### **Environment** -- `KAPOW_DATA_URL` +- `KAPOW_CONTROL_URL` +- `KAPOW_CONTROL_SERVER_CERT` +- `KAPOW_CONTROL_CLIENT_CERT` +- `KAPOW_CONTROL_CLIENT_KEY` #### **Help** @@ -696,7 +722,7 @@ Options: $ kapow route add -X GET '/list/{ip}' -c 'nmap -sL $(kapow get /request/matches/ip) | kapow set /response/body' ``` -### `request` +### `kapow get` Exposes the requests' resources. @@ -713,7 +739,7 @@ $ kapow get /request/body ``` -### `response` +### `kapow set` Exposes the response's resources. diff --git a/spec/test/.envrc b/spec/test/.envrc new file mode 100644 index 0000000..4a4726a --- /dev/null +++ b/spec/test/.envrc @@ -0,0 +1 @@ +use_nix diff --git a/spec/test/Dockerfile b/spec/test/Dockerfile index bd8290d..f36ff99 100644 --- a/spec/test/Dockerfile +++ b/spec/test/Dockerfile @@ -1,16 +1,18 @@ -FROM python:3.7-alpine +FROM nixos/nix:2.3.6 # Install CircleCI requirements for base images # https://circleci.com/docs/2.0/custom-images/ -RUN apk upgrade --update-cache \ - && apk add git openssh-server tar gzip ca-certificates +# RUN apk upgrade --update-cache \ +# && apk add git openssh-server tar gzip ca-certificates # Install Kapow! Spec Test Suite RUN mkdir -p /usr/src/ksts WORKDIR /usr/src/ksts COPY features /usr/src/ksts/features -COPY Pipfile Pipfile.lock /usr/src/ksts/ -RUN pip install --upgrade pip \ - && pip install pipenv \ - && pipenv install --deploy --system \ - && rm -f Pipfile Pipfile.lock +# COPY Pipfile Pipfile.lock /usr/src/ksts/ +# RUN pip install --upgrade pip \ +# && pip install pipenv \ +# && pipenv install --deploy --system \ +# && rm -f Pipfile Pipfile.lock +COPY ./*.nix ./ +ENTRYPOINT [ "nix-shell", "--command" ] diff --git a/spec/test/Makefile b/spec/test/Makefile index b11f077..6033104 100644 --- a/spec/test/Makefile +++ b/spec/test/Makefile @@ -1,23 +1,20 @@ -.PHONY: lint wip test fix catalog sync +.PHONY: all lint wip test fix catalog -all: checkbin sync test +all: checkbin test -sync: - pipenv sync lint: gherkin-lint wip: - KAPOW_DEBUG_TESTS=1 pipenv run behave --stop --wip + KAPOW_DEBUG_TESTS=1 behave --stop --wip -k test: lint - pipenv run behave --no-capture --tags=~@skip + behave --no-capture --tags=~@skip fix: lint - KAPOW_DEBUG_TESTS=1 pipenv run behave --stop --no-capture --tags=~@skip + KAPOW_DEBUG_TESTS=1 behave --stop --no-capture --tags=~@skip catalog: - pipenv run behave --format steps.usage --dry-run --no-summary -q -clean: - pipenv --rm + behave --format steps.usage --dry-run --no-summary -q checkbin: @which kapow >/dev/null || (echo "ERROR: Your kapow binary is not present in PATH" && exit 1) -testpoc: sync - pipenv run pip install -r ../../testutils/poc/requirements.txt - PATH=../../testutils/poc:$$PATH KAPOW_CONTROL_URL=http://localhost:8081 KAPOW_DATA_URL=http://localhost:8081 pipenv run behave --no-capture --tags=~@skip +testpoc: + PATH=../../testutils/poc:$$PATH behave --no-capture --tags=~@skip +wippoc: + PATH=../../testutils/poc:$$PATH behave --no-capture --tags=@wip -k diff --git a/spec/test/features/control/list/success.feature b/spec/test/features/control/list/success.feature index 0361043..769f842 100644 --- a/spec/test/features/control/list/success.feature +++ b/spec/test/features/control/list/success.feature @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +@server Feature: Listing routes in a Kapow! server. Listing routes allows users to know what URLs are available on a Kapow! server. The List endpoint returns diff --git a/spec/test/features/control/mtls.feature b/spec/test/features/control/mtls.feature new file mode 100644 index 0000000..3b6a27d --- /dev/null +++ b/spec/test/features/control/mtls.feature @@ -0,0 +1,95 @@ +# +# Copyright 2021 Banco Bilbao Vizcaya Argentaria, S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Feature: Communications with the control interface are secured with mTLS. + Trust is anchored via certificate pinning. + The Kapow! server only allows connections from trusted clients. + The Kapow! clients only establish connections to trusted servers. + + @server + Scenario: Reject clients not providing a certificate. + + Given I have a running Kapow! server + When I try to connect to the control API without providing a certificate + Then I get a connection error + + @server + Scenario: Reject clients providing an invalid certificate. + + Given I have a running Kapow! server + When I try to connect to the control API providing an invalid certificate + Then I get a connection error + + @client + Scenario: Connect to servers providing a valid certificate. + A valid certificate is the one provided via envvars. + + Given a test HTTPS server on the control port + When I run the following command + """ + $ kapow route list + """ + And the HTTPS server receives a "GET" request to "/routes" + And the server responds with + | field | value | + | status | 200 | + | headers.Content-Type | application/json | + | body | [] | + Then the command exits with "0" + + @client + Scenario: Reject servers providing an invalid certificate. + + Given a test HTTPS server on the control port + When I run the following command (with invalid certs) + """ + $ kapow route list + """ + Then the command exits immediately with "1" + + @server + Scenario Outline: The control server is accessible through an alternative address + The automatically generated certificated contains the Alternate Name + provided via the `--control-reachable-addr` parameter. + + Given I launch the server with the following extra arguments + """ + --control-reachable-addr "" + """ + When I inspect the automatically generated control server certificate + Then the extension "Subject Alternative Name" contains "" of type "" + + Examples: + | reachable_addr | value | type | + | localhost:8081 | localhost | DNSName | + | 127.0.0.1:8081 | 127.0.0.1 | IPAddress | + | foo.bar:8081 | foo.bar | DNSName | + | 4.2.2.4:8081 | 4.2.2.4 | IPAddress | + | [2600::]:8081 | 2600:: | IPAddress | + + + @e2e + Scenario: Control server dialog using mTLS + If the user provides the corresponding certificates to the + `kapow route` subcommand, the communication should be possible. + + Given I have a just started Kapow! server + When I run the following command (setting the control certs environment variables) + """ + $ kapow route list + + """ + Then the command exits with "0" diff --git a/spec/test/features/environment.py b/spec/test/features/environment.py index 1dffa4c..52a05f4 100644 --- a/spec/test/features/environment.py +++ b/spec/test/features/environment.py @@ -15,25 +15,43 @@ # import tempfile import os +import signal +from contextlib import suppress - -def before_scenario(context, scenario): - # Create the request_handler FIFO +def tmpfifo(): while True: - context.handler_fifo_path = tempfile.mktemp() # Safe because using - # mkfifo + fifo_path = tempfile.mktemp() # The usage mkfifo make this safe try: - os.mkfifo(context.handler_fifo_path) + os.mkfifo(fifo_path) except OSError: # The file already exist pass else: break + return fifo_path + + +def before_scenario(context, scenario): + context.handler_fifo_path = tmpfifo() + context.init_script_fifo_path = tmpfifo() + def after_scenario(context, scenario): + # Real Kapow! server being tested if hasattr(context, 'server'): context.server.terminate() context.server.wait() os.unlink(context.handler_fifo_path) + os.unlink(context.init_script_fifo_path) + + # Mock HTTP server for testing + if hasattr(context, 'httpserver'): + context.response_ready.set() + context.httpserver.shutdown() + context.httpserver_thread.join() + + if getattr(context, 'testing_handler_pid', None) is not None: + with suppress(ProcessLookupError): + os.kill(int(context.testing_handler_pid), signal.SIGTERM) diff --git a/spec/test/features/steps/get_environment.py b/spec/test/features/steps/get_environment.py new file mode 100755 index 0000000..bb6e09c --- /dev/null +++ b/spec/test/features/steps/get_environment.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +import json +import os +import sys + +if __name__ == '__main__': + with open(os.environ['SPECTEST_FIFO'], 'w') as fifo: + json.dump(dict(os.environ), fifo) diff --git a/spec/test/features/steps/steps.py b/spec/test/features/steps/steps.py index f16468a..21ea795 100644 --- a/spec/test/features/steps/steps.py +++ b/spec/test/features/steps/steps.py @@ -13,26 +13,36 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from contextlib import suppress +from contextlib import suppress, contextmanager +from multiprocessing.pool import ThreadPool from time import sleep +import datetime +import http.server +import ipaddress import json +import logging import os import shlex import signal import socket +import ssl import subprocess import sys import tempfile import threading -from multiprocessing.pool import ThreadPool import time -import requests -from environconfig import EnvironConfig, StringVar, IntVar, BooleanVar from comparedict import is_subset +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import serialization +from cryptography import x509 +from cryptography.x509.oid import NameOID, ExtensionOID +from environconfig import EnvironConfig, StringVar, IntVar, BooleanVar +from requests import exceptions as requests_exceptions import jsonexample - -import logging +import requests WORD2POS = {"first": 0, "second": 1, "last": -1} @@ -44,7 +54,8 @@ class Env(EnvironConfig): KAPOW_SERVER_CMD = StringVar(default="kapow server") #: Where the Control API is - KAPOW_CONTROL_URL = StringVar(default="http://localhost:8081") + KAPOW_CONTROL_URL = StringVar(default="https://localhost:8081") + KAPOW_CONTROL_PORT = IntVar(default=8081) #: Where the Data API is KAPOW_DATA_URL = StringVar(default="http://localhost:8082") @@ -52,7 +63,9 @@ class Env(EnvironConfig): #: Where the User Interface is KAPOW_USER_URL = StringVar(default="http://localhost:8080") - KAPOW_BOOT_TIMEOUT = IntVar(default=1000) + KAPOW_CONTROL_TOKEN = StringVar(default="TEST-SPEC-CONTROL-TOKEN") + + KAPOW_BOOT_TIMEOUT = IntVar(default=3000) KAPOW_DEBUG_TESTS = BooleanVar(default=False) @@ -77,37 +90,134 @@ if Env.KAPOW_DEBUG_TESTS: requests_log.setLevel(logging.DEBUG) requests_log.propagate = True -def run_kapow_server(context): + +def generate_ssl_cert(subject_name, alternate_name): + # Generate our key + key = rsa.generate_private_key( + public_exponent=65537, + key_size=4096, + ) + # Various details about who we are. For a self-signed certificate the + # subject and issuer are always the same. + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, subject_name), + ]) + cert = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + # Our certificate will be valid for 10 days + datetime.datetime.utcnow() + datetime.timedelta(days=10) + ).add_extension( + x509.SubjectAlternativeName([x509.DNSName(alternate_name)]), + critical=True, + ).add_extension( + x509.ExtendedKeyUsage( + [x509.oid.ExtendedKeyUsageOID.SERVER_AUTH + if subject_name.endswith('_server') + else x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH]), + critical=True, + # Sign our certificate with our private key + ).sign(key, hashes.SHA256()) + + key_bytes = key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + crt_bytes = cert.public_bytes(serialization.Encoding.PEM) + + return (key_bytes, crt_bytes) + + +@contextmanager +def mtls_client(context): + with tempfile.NamedTemporaryFile(suffix='.crt', encoding='utf-8', mode='w') as srv_cert, \ + tempfile.NamedTemporaryFile(suffix='.crt', encoding='utf-8', mode='w') as cli_cert, \ + tempfile.NamedTemporaryFile(suffix='.key', encoding='utf-8', mode='w') as cli_key: + srv_cert.write(context.init_script_environ["KAPOW_CONTROL_SERVER_CERT"]) + srv_cert.file.flush() + cli_cert.write(context.init_script_environ["KAPOW_CONTROL_CLIENT_CERT"]) + cli_cert.file.flush() + cli_key.write(context.init_script_environ["KAPOW_CONTROL_CLIENT_KEY"]) + cli_key.file.flush() + session=requests.Session() + session.verify=srv_cert.name + session.cert=(cli_cert.name, cli_key.name) + yield session + + +def is_port_open(port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + return sock.connect_ex(('127.0.0.1', port)) == 0 + + +def run_kapow_server(context, extra_args=""): + assert (not is_port_open(Env.KAPOW_CONTROL_PORT)), "Another process is already bound" + context.server = subprocess.Popen( - shlex.split(Env.KAPOW_SERVER_CMD), + shlex.split(Env.KAPOW_SERVER_CMD) + shlex.split(extra_args) + [os.path.join(HERE, "get_environment.py")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env={'SPECTEST_FIFO': context.init_script_fifo_path, **os.environ}, shell=False) # Check process is running with reachable APIs open_ports = False for _ in range(Env.KAPOW_BOOT_TIMEOUT): - is_running = context.server.poll() is None - assert is_running, "Server is not running!" - with suppress(requests.exceptions.ConnectionError): - open_ports = ( - requests.head(Env.KAPOW_CONTROL_URL, timeout=1).status_code - and requests.head(Env.KAPOW_DATA_URL, timeout=1).status_code) - if open_ports: + with suppress(requests_exceptions.ConnectionError): + if is_port_open(Env.KAPOW_CONTROL_PORT): + open_ports = True break sleep(.01) assert open_ports, "API is unreachable after KAPOW_BOOT_TIMEOUT" + # Get init_script enviroment via fifo + with open(context.init_script_fifo_path, 'r') as fifo: + context.init_script_environ = json.load(fifo) + + @given('I have a just started Kapow! server') @given('I have a running Kapow! server') def step_impl(context): run_kapow_server(context) +@given(u'I launch the server with the following extra arguments') +def step_impl(context): + run_kapow_server(context, context.text) + + +@when('I request a route listing without providing a Control Access Token') +def step_impl(context): + with mtls_client(context) as requests: + context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + + +@when('I request a route listing without providing an empty Control Access Token') +def step_impl(context): + with mtls_client(context) as requests: + context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + + +@when(u'I request a route listing providing a bad Control Access Token') +def step_impl(context): + with mtls_client(context) as requests: + context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + + @when('I request a routes listing') def step_impl(context): - context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + with mtls_client(context) as requests: + context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") @given('I have a Kapow! server with the following routes') @@ -117,10 +227,12 @@ def step_impl(context): if not hasattr(context, 'table'): raise RuntimeError("A table must be set for this step.") - for row in context.table: - response = requests.post(f"{Env.KAPOW_CONTROL_URL}/routes", - json={h: row[h] for h in row.headings}) - response.raise_for_status() + with mtls_client(context) as requests: + for row in context.table: + response = requests.post( + f"{Env.KAPOW_CONTROL_URL}/routes", + json={h: row[h] for h in row.headings}) + response.raise_for_status() @given('I have a Kapow! server with the following testing routes') @@ -130,15 +242,16 @@ def step_impl(context): if not hasattr(context, 'table'): raise RuntimeError("A table must be set for this step.") - for row in context.table: - response = requests.post( - f"{Env.KAPOW_CONTROL_URL}/routes", - json={"entrypoint": " ".join( - [sys.executable, - shlex.quote(os.path.join(HERE, "testinghandler.py")), - shlex.quote(context.handler_fifo_path)]), # Created in before_scenario - **{h: row[h] for h in row.headings}}) - response.raise_for_status() + with mtls_client(context) as requests: + for row in context.table: + response = requests.post( + f"{Env.KAPOW_CONTROL_URL}/routes", + json={"entrypoint": " ".join( + [sys.executable, + shlex.quote(os.path.join(HERE, "testinghandler.py")), + shlex.quote(context.handler_fifo_path)]), # Created in before_scenario + **{h: row[h] for h in row.headings}}) + response.raise_for_status() def testing_request(context, request_fn): # Run the request in background @@ -165,15 +278,17 @@ def step_impl(context, path): @when('I release the testing request') def step_impl(context): os.kill(int(context.testing_handler_pid), signal.SIGTERM) + context.testing_handler_pid = None context.testing_response = context.testing_request.get() @when('I append the route') def step_impl(context): - context.response = requests.post(f"{Env.KAPOW_CONTROL_URL}/routes", - data=context.text, - headers={"Content-Type": "application/json"}) - + with mtls_client(context) as requests: + context.response = requests.post( + f"{Env.KAPOW_CONTROL_URL}/routes", + data=context.text, + headers={"Content-Type": "application/json"}) @then('I get {code} as response code') def step_impl(context, code): @@ -212,50 +327,62 @@ def step_impl(context): @when('I delete the route with id "{id}"') def step_impl(context, id): - context.response = requests.delete(f"{Env.KAPOW_CONTROL_URL}/routes/{id}") + with mtls_client(context) as requests: + context.response = requests.delete( + f"{Env.KAPOW_CONTROL_URL}/routes/{id}") @when('I insert the route') def step_impl(context): - context.response = requests.put(f"{Env.KAPOW_CONTROL_URL}/routes", - headers={"Content-Type": "application/json"}, - data=context.text) + with mtls_client(context) as requests: + context.response = requests.put( + f"{Env.KAPOW_CONTROL_URL}/routes", + headers={"Content-Type": "application/json"}, + data=context.text) @when('I try to append with this malformed JSON document') def step_impl(context): - context.response = requests.post( - f"{Env.KAPOW_CONTROL_URL}/routes", - headers={"Content-Type": "application/json"}, - data=context.text) + with mtls_client(context) as requests: + context.response = requests.post( + f"{Env.KAPOW_CONTROL_URL}/routes", + headers={"Content-Type": "application/json"}, + data=context.text) @when('I delete the {order} route') def step_impl(context, order): - idx = WORD2POS.get(order) - routes = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") - id = routes.json()[idx]["id"] - context.response = requests.delete(f"{Env.KAPOW_CONTROL_URL}/routes/{id}") + with mtls_client(context) as requests: + idx = WORD2POS.get(order) + routes = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + id = routes.json()[idx]["id"] + context.response = requests.delete( + f"{Env.KAPOW_CONTROL_URL}/routes/{id}") @when('I try to insert with this JSON document') def step_impl(context): - context.response = requests.put( - f"{Env.KAPOW_CONTROL_URL}/routes", - headers={"Content-Type": "application/json"}, - data=context.text) + with mtls_client(context) as requests: + context.response = requests.put( + f"{Env.KAPOW_CONTROL_URL}/routes", + headers={"Content-Type": "application/json"}, + data=context.text) @when('I get the route with id "{id}"') def step_impl(context, id): - context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes/{id}") + with mtls_client(context) as requests: + context.response = requests.get( + f"{Env.KAPOW_CONTROL_URL}/routes/{id}") @when('I get the {order} route') def step_impl(context, order): - idx = WORD2POS.get(order) - routes = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") - id = routes.json()[idx]["id"] - context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes/{id}") + with mtls_client(context) as requests: + idx = WORD2POS.get(order) + routes = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + id = routes.json()[idx]["id"] + context.response = requests.get( + f"{Env.KAPOW_CONTROL_URL}/routes/{id}") @when('I get the resource "{resource}"') @@ -316,3 +443,216 @@ def step_impl(context, value, fieldType, elementName): raise ValueError("Unknown fieldtype {fieldType!r}") assert actual == value, f"Expecting {fieldType} {elementName!r} to be {value!r}, got {actual!r} insted" + + +@given('a test HTTPS server on the {port} port') +def step_impl(context, port): + context.request_ready = threading.Event() + context.request_ready.clear() + context.response_ready = threading.Event() + context.response_ready.clear() + + class SaveResponseHandler(http.server.BaseHTTPRequestHandler): + def do_verb(self): + context.request_response = self + context.request_ready.set() + context.response_ready.wait() + do_GET=do_verb + do_POST=do_verb + do_PUT=do_verb + do_DELETE=do_verb + do_HEAD=do_verb + + if port == "control": + port = 8081 + elif port == "data": + port = 8082 + else: + raise ValueError(f"Unknown port {port}") + + context.httpserver = http.server.HTTPServer(('127.0.0.1', port), + SaveResponseHandler) + + context.srv_key, context.srv_crt = generate_ssl_cert("control_server", "localhost") + context.cli_key, context.cli_crt = generate_ssl_cert("control_client", "localhost") + with tempfile.NamedTemporaryFile(suffix=".key") as key_file, \ + tempfile.NamedTemporaryFile(suffix=".crt") as crt_file: + key_file.write(context.srv_key) + key_file.flush() + crt_file.write(context.srv_crt) + crt_file.flush() + context.httpserver.socket = ssl.wrap_socket( + context.httpserver.socket, + keyfile=key_file.name, + certfile=crt_file.name, + server_side=True) + context.httpserver_thread = threading.Thread( + target=context.httpserver.serve_forever, + daemon=True) + context.httpserver_thread.start() + + +def run_command_with_certs(context, srv_crt, cli_crt, cli_key): + _, command = context.text.split('$') + command = command.lstrip() + + def exec_in_thread(): + context.command = subprocess.Popen( + command, + shell=True, + env={'KAPOW_CONTROL_SERVER_CERT': srv_crt, + 'KAPOW_CONTROL_CLIENT_CERT': cli_crt, + 'KAPOW_CONTROL_CLIENT_KEY': cli_key, + **os.environ}) + context.command.wait() + + context.command_thread = threading.Thread(target=exec_in_thread, daemon=True) + context.command_thread.start() + +@step('I run the following command (with invalid certs)') +def step_impl(context): + invalid_srv_crt, _ = generate_ssl_cert("invalid_control_server", + "localhost") + run_command_with_certs(context, + invalid_srv_crt, + context.cli_crt, + context.cli_key) + + +@step('I run the following command') +def step_impl(context): + run_command_with_certs(context, + context.srv_crt, + context.cli_crt, + context.cli_key) + + +@when('I run the following command (setting the control certs environment variables)') +def step_impl(context): + run_command_with_certs( + context, + context.init_script_environ["KAPOW_CONTROL_SERVER_CERT"], + context.init_script_environ["KAPOW_CONTROL_CLIENT_CERT"], + context.init_script_environ["KAPOW_CONTROL_CLIENT_KEY"]) + + +@step('the HTTPS server receives a "{method}" request to "{path}"') +def step_impl(context, method, path): + context.request_ready.wait() + assert context.request_response.command == method, f"Method {context.request_response.command} is not {method}" + assert context.request_response.path == path, f"Method {context.request_response.path} is not {path}" + + + +@then('the received request has the header "{name}" set to "{value}"') +def step_impl(context, name, value): + context.request_ready.wait() + matching = context.request_response.headers[name] + assert matching, f"Header {name} not found" + assert matching == value, f"Value of header doesn't match. {matching} != {value}" + + +@when('the server responds with') +def step_impl(context): + # TODO: set the fields given in the table + has_body = False + for row in context.table: + if row['field'] == 'status': + context.request_response.send_response(int(row['value'])) + elif row['field'].startswith('headers.'): + _, header = row['field'].split('.') + context.request_response.send_header(header, row['value']) + elif row['field'] == 'body': + has_body = True + payload = row['value'].encode('utf-8') + context.request_response.send_header('Content-Length', str(len(payload))) + context.request_response.end_headers() + context.request_response.wfile.write(payload) + + if not has_body: + context.request_response.send_header('Content-Length', '0') + context.request_response.end_headers() + + context.response_ready.set() + +@then('the command exits {immediately} with "{returncode}"') +@then('the command exits with "{returncode}"') +def step_impl(context, returncode, immediately=False): + context.command_thread.join(timeout=3.0 if immediately else None) + if context.command_thread.is_alive(): + try: + print("killing in the name of") + context.command.kill() + finally: + assert False, "The command is still alive" + + else: + context.command.wait() + assert context.command.returncode == int(returncode), f"Command returned {context.command.returncode} instead of {returncode}" + + +@then('the received request doesn\'t have the header "{name}" set') +def step_impl(context, name): + context.request_ready.wait() + assert name not in context.request_response.headers, f"Header {name} found" + + +@when('I try to connect to the control API without providing a certificate') +def step_impl(context): + try: + context.request_response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes", verify=False) + except Exception as exc: + context.request_response = exc + + +@then(u'I get a connection error') +def step_impl(context): + assert issubclass(type(context.request_response), Exception), context.request_response + + +@when(u'I try to connect to the control API providing an invalid certificate') +def step_impl(context): + key, cert = generate_ssl_cert("foo", "localhost") + with tempfile.NamedTemporaryFile(suffix='.crt') as cert_file, \ + tempfile.NamedTemporaryFile(suffix='.key') as key_file: + cert_file.write(cert) + cert_file.flush() + key_file.write(key) + key_file.flush() + with requests.Session() as session: + session.cert = (cert_file.name, key_file.name) + session.verify = False + try: + context.request_response = session.get( + f"{Env.KAPOW_CONTROL_URL}/routes") + except Exception as exc: + context.request_response = exc + + + +@when('I inspect the automatically generated control server certificate') +def step_impl(context): + context.control_server_cert = x509.load_pem_x509_certificate( + context.init_script_environ["KAPOW_CONTROL_SERVER_CERT"].encode('ascii')) + + +@then('the extension "{extension}" contains "{value}" of type "{typename}"') +def step_impl(context, extension, value, typename): + if extension == 'Subject Alternative Name': + oid = ExtensionOID.SUBJECT_ALTERNATIVE_NAME + else: + raise NotImplementedError(f'Unknown extension {extension}') + + if typename == 'DNSName': + type_ = x509.DNSName + converter = lambda x: x + elif typename == 'IPAddress': + type_ = x509.IPAddress + converter = ipaddress.ip_address + else: + raise NotImplementedError(f'Unknown type {typename}') + + ext = context.control_server_cert.extensions.get_extension_for_oid(oid) + values = ext.value.get_values_for_type(type_) + + assert converter(value) in values, f"Value {value} not in {values}" diff --git a/spec/test/node-dependencies.nix b/spec/test/node-dependencies.nix new file mode 100644 index 0000000..c970861 --- /dev/null +++ b/spec/test/node-dependencies.nix @@ -0,0 +1,17 @@ +# This file has been generated by node2nix 1.8.0. Do not edit! + +{pkgs ? import { + inherit system; + }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}: + +let + nodeEnv = import ./node-env.nix { + inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile; + inherit nodejs; + libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; + }; +in +import ./node-packages.nix { + inherit (pkgs) fetchurl fetchgit; + inherit nodeEnv; +} \ No newline at end of file diff --git a/spec/test/node-env.nix b/spec/test/node-env.nix new file mode 100644 index 0000000..e1abf53 --- /dev/null +++ b/spec/test/node-env.nix @@ -0,0 +1,542 @@ +# This file originates from node2nix + +{stdenv, nodejs, python2, utillinux, libtool, runCommand, writeTextFile}: + +let + python = if nodejs ? python then nodejs.python else python2; + + # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise + tarWrapper = runCommand "tarWrapper" {} '' + mkdir -p $out/bin + + cat > $out/bin/tar <> $out/nix-support/hydra-build-products + ''; + }; + + includeDependencies = {dependencies}: + stdenv.lib.optionalString (dependencies != []) + (stdenv.lib.concatMapStrings (dependency: + '' + # Bundle the dependencies of the package + mkdir -p node_modules + cd node_modules + + # Only include dependencies if they don't exist. They may also be bundled in the package. + if [ ! -e "${dependency.name}" ] + then + ${composePackage dependency} + fi + + cd .. + '' + ) dependencies); + + # Recursively composes the dependencies of a package + composePackage = { name, packageName, src, dependencies ? [], ... }@args: + builtins.addErrorContext "while evaluating node package '${packageName}'" '' + DIR=$(pwd) + cd $TMPDIR + + unpackFile ${src} + + # Make the base dir in which the target dependency resides first + mkdir -p "$(dirname "$DIR/${packageName}")" + + if [ -f "${src}" ] + then + # Figure out what directory has been unpacked + packageDir="$(find . -maxdepth 1 -type d | tail -1)" + + # Restore write permissions to make building work + find "$packageDir" -type d -exec chmod u+x {} \; + chmod -R u+w "$packageDir" + + # Move the extracted tarball into the output folder + mv "$packageDir" "$DIR/${packageName}" + elif [ -d "${src}" ] + then + # Get a stripped name (without hash) of the source directory. + # On old nixpkgs it's already set internally. + if [ -z "$strippedName" ] + then + strippedName="$(stripHash ${src})" + fi + + # Restore write permissions to make building work + chmod -R u+w "$strippedName" + + # Move the extracted directory into the output folder + mv "$strippedName" "$DIR/${packageName}" + fi + + # Unset the stripped name to not confuse the next unpack step + unset strippedName + + # Include the dependencies of the package + cd "$DIR/${packageName}" + ${includeDependencies { inherit dependencies; }} + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + ''; + + pinpointDependencies = {dependencies, production}: + let + pinpointDependenciesFromPackageJSON = writeTextFile { + name = "pinpointDependencies.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function resolveDependencyVersion(location, name) { + if(location == process.env['NIX_STORE']) { + return null; + } else { + var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json"); + + if(fs.existsSync(dependencyPackageJSON)) { + var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON)); + + if(dependencyPackageObj.name == name) { + return dependencyPackageObj.version; + } + } else { + return resolveDependencyVersion(path.resolve(location, ".."), name); + } + } + } + + function replaceDependencies(dependencies) { + if(typeof dependencies == "object" && dependencies !== null) { + for(var dependency in dependencies) { + var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency); + + if(resolvedVersion === null) { + process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n"); + } else { + dependencies[dependency] = resolvedVersion; + } + } + } + } + + /* Read the package.json configuration */ + var packageObj = JSON.parse(fs.readFileSync('./package.json')); + + /* Pinpoint all dependencies */ + replaceDependencies(packageObj.dependencies); + if(process.argv[2] == "development") { + replaceDependencies(packageObj.devDependencies); + } + replaceDependencies(packageObj.optionalDependencies); + + /* Write the fixed package.json file */ + fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); + ''; + }; + in + '' + node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"} + + ${stdenv.lib.optionalString (dependencies != []) + '' + if [ -d node_modules ] + then + cd node_modules + ${stdenv.lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies} + cd .. + fi + ''} + ''; + + # Recursively traverses all dependencies of a package and pinpoints all + # dependencies in the package.json file to the versions that are actually + # being used. + + pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args: + '' + if [ -d "${packageName}" ] + then + cd "${packageName}" + ${pinpointDependencies { inherit dependencies production; }} + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + fi + ''; + + # Extract the Node.js source code which is used to compile packages with + # native bindings + nodeSources = runCommand "node-sources" {} '' + tar --no-same-owner --no-same-permissions -xf ${nodejs.src} + mv node-* $out + ''; + + # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty) + addIntegrityFieldsScript = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function augmentDependencies(baseDir, dependencies) { + for(var dependencyName in dependencies) { + var dependency = dependencies[dependencyName]; + + // Open package.json and augment metadata fields + var packageJSONDir = path.join(baseDir, "node_modules", dependencyName); + var packageJSONPath = path.join(packageJSONDir, "package.json"); + + if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored + console.log("Adding metadata fields to: "+packageJSONPath); + var packageObj = JSON.parse(fs.readFileSync(packageJSONPath)); + + if(dependency.integrity) { + packageObj["_integrity"] = dependency.integrity; + } else { + packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads. + } + + if(dependency.resolved) { + packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided + } else { + packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories. + } + + if(dependency.from !== undefined) { // Adopt from property if one has been provided + packageObj["_from"] = dependency.from; + } + + fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2)); + } + + // Augment transitive dependencies + if(dependency.dependencies !== undefined) { + augmentDependencies(packageJSONDir, dependency.dependencies); + } + } + } + + if(fs.existsSync("./package-lock.json")) { + var packageLock = JSON.parse(fs.readFileSync("./package-lock.json")); + + if(packageLock.lockfileVersion !== 1) { + process.stderr.write("Sorry, I only understand lock file version 1!\n"); + process.exit(1); + } + + if(packageLock.dependencies !== undefined) { + augmentDependencies(".", packageLock.dependencies); + } + } + ''; + }; + + # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes + reconstructPackageLock = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + var packageObj = JSON.parse(fs.readFileSync("package.json")); + + var lockObj = { + name: packageObj.name, + version: packageObj.version, + lockfileVersion: 1, + requires: true, + dependencies: {} + }; + + function augmentPackageJSON(filePath, dependencies) { + var packageJSON = path.join(filePath, "package.json"); + if(fs.existsSync(packageJSON)) { + var packageObj = JSON.parse(fs.readFileSync(packageJSON)); + dependencies[packageObj.name] = { + version: packageObj.version, + integrity: "sha1-000000000000000000000000000=", + dependencies: {} + }; + processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); + } + } + + function processDependencies(dir, dependencies) { + if(fs.existsSync(dir)) { + var files = fs.readdirSync(dir); + + files.forEach(function(entry) { + var filePath = path.join(dir, entry); + var stats = fs.statSync(filePath); + + if(stats.isDirectory()) { + if(entry.substr(0, 1) == "@") { + // When we encounter a namespace folder, augment all packages belonging to the scope + var pkgFiles = fs.readdirSync(filePath); + + pkgFiles.forEach(function(entry) { + if(stats.isDirectory()) { + var pkgFilePath = path.join(filePath, entry); + augmentPackageJSON(pkgFilePath, dependencies); + } + }); + } else { + augmentPackageJSON(filePath, dependencies); + } + } + }); + } + } + + processDependencies("node_modules", lockObj.dependencies); + + fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); + ''; + }; + + prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: + let + forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; + in + '' + # Pinpoint the versions of all dependencies to the ones that are actually being used + echo "pinpointing versions of dependencies..." + source $pinpointDependenciesScriptPath + + # Patch the shebangs of the bundled modules to prevent them from + # calling executables outside the Nix store as much as possible + patchShebangs . + + # Deploy the Node.js package by running npm install. Since the + # dependencies have been provided already by ourselves, it should not + # attempt to install them again, which is good, because we want to make + # it Nix's responsibility. If it needs to install any dependencies + # anyway (e.g. because the dependency parameters are + # incomplete/incorrect), it fails. + # + # The other responsibilities of NPM are kept -- version checks, build + # steps, postprocessing etc. + + export HOME=$TMPDIR + cd "${packageName}" + runHook preRebuild + + ${stdenv.lib.optionalString bypassCache '' + ${stdenv.lib.optionalString reconstructLock '' + if [ -f package-lock.json ] + then + echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!" + echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!" + rm package-lock.json + else + echo "No package-lock.json file found, reconstructing..." + fi + + node ${reconstructPackageLock} + ''} + + node ${addIntegrityFieldsScript} + ''} + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild + + if [ "''${dontNpmInstall-}" != "1" ] + then + # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. + rm -f npm-shrinkwrap.json + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install + fi + ''; + + # Builds and composes an NPM package including all its dependencies + buildNodePackage = + { name + , packageName + , version + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , preRebuild ? "" + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" ]; + in + stdenv.mkDerivation ({ + name = "node_${name}-${version}"; + buildInputs = [ tarWrapper python nodejs ] + ++ stdenv.lib.optional (stdenv.isLinux) utillinux + ++ stdenv.lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit nodejs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall preRebuild unpackPhase buildPhase; + + compositionScript = composePackage args; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "compositionScript" "pinpointDependenciesScript" ]; + + installPhase = '' + # Create and enter a root node_modules/ folder + mkdir -p $out/lib/node_modules + cd $out/lib/node_modules + + # Compose the package and all its dependencies + source $compositionScriptPath + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Create symlink to the deployed executable folder, if applicable + if [ -d "$out/lib/node_modules/.bin" ] + then + ln -s $out/lib/node_modules/.bin $out/bin + fi + + # Create symlinks to the deployed manual page folders, if applicable + if [ -d "$out/lib/node_modules/${packageName}/man" ] + then + mkdir -p $out/share + for dir in "$out/lib/node_modules/${packageName}/man/"* + do + mkdir -p $out/share/man/$(basename "$dir") + for page in "$dir"/* + do + ln -s $page $out/share/man/$(basename "$dir") + done + done + fi + + # Run post install hook, if provided + runHook postInstall + ''; + } // extraArgs); + + # Builds a development shell + buildNodeShell = + { name + , packageName + , version + , src + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ]; + + nodeDependencies = stdenv.mkDerivation ({ + name = "node-dependencies-${name}-${version}"; + + buildInputs = [ tarWrapper python nodejs ] + ++ stdenv.lib.optional (stdenv.isLinux) utillinux + ++ stdenv.lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall unpackPhase buildPhase; + + includeScript = includeDependencies { inherit dependencies; }; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "includeScript" "pinpointDependenciesScript" ]; + + installPhase = '' + mkdir -p $out/${packageName} + cd $out/${packageName} + + source $includeScriptPath + + # Create fake package.json to make the npm commands work properly + cp ${src}/package.json . + chmod 644 package.json + ${stdenv.lib.optionalString bypassCache '' + if [ -f ${src}/package-lock.json ] + then + cp ${src}/package-lock.json . + fi + ''} + + # Go to the parent folder to make sure that all packages are pinpointed + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Expose the executables that were installed + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + mv ${packageName} lib + ln -s $out/lib/node_modules/.bin $out/bin + ''; + } // extraArgs); + in + stdenv.mkDerivation { + name = "node-shell-${name}-${version}"; + + buildInputs = [ python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ buildInputs; + buildCommand = '' + mkdir -p $out/bin + cat > $out/bin/shell <