diff --git a/poc/kapow b/poc/bin/kapow similarity index 92% rename from poc/kapow rename to poc/bin/kapow index 99be517..ba8237d 100755 --- a/poc/kapow +++ b/poc/bin/kapow @@ -72,17 +72,17 @@ class Connection: return self.request.content elif res.path == 'request/path': return self.request.path.encode('utf-8') - elif res.path.startswith('request/match/'): + elif res.path.startswith('request/matches/'): return self.request.match_info[nth(2)].encode('utf-8') - elif res.path.startswith('request/param/'): + elif res.path.startswith('request/params/'): return self.request.rel_url.query[nth(2)].encode('utf-8') - elif res.path.startswith('request/header/'): + elif res.path.startswith('request/headers/'): return self.request.headers[nth(2)].encode('utf-8') - elif res.path.startswith('request/cookie/'): + elif res.path.startswith('request/cookies/'): return self.request.cookies[nth(2)].encode('utf-8') elif res.path.startswith('request/form/'): return (await self.request.post())[nth(2)].encode('utf-8') - elif res.path.startswith('request/file/'): + elif res.path.startswith('request/files/'): name = nth(2) content = nth(3) # filename / content field = (await self.request.post())[name] @@ -112,10 +112,10 @@ class Connection: self._status = int((await content.read()).decode('utf-8')) elif res.path == 'response/body': self._body.write(await content.read()) - elif res.path.startswith('response/header/'): + elif res.path.startswith('response/headers/'): clean = (await content.read()).rstrip(b'\n').decode('utf-8') self._headers[nth(2)] = clean - elif res.path.startswith('response/cookie/'): + elif res.path.startswith('response/cookies/'): clean = (await content.read()).rstrip(b'\n').decode('utf-8') self._cookies[nth(2)] = clean elif res.path == 'response/stream': @@ -230,8 +230,8 @@ def handle_route(entrypoint, command): shell_task = await asyncio.create_subprocess_shell( args, env={**os.environ, - "KAPOW_URL": "http://localhost:8080/kapow", - "KAPOW_CONNECTION": id + "KAPOW_URL": "http://localhost:8080", + "KAPOW_HANDLER_ID": id }, stdin=asyncio.subprocess.DEVNULL) @@ -256,7 +256,7 @@ async def get_routes(request): return web.json_response(list(request.app.router)) -async def create_route(request): +async def append_route(request): """Create a new Kapow! route.""" request.app.router._frozen = False content = await request.json() @@ -300,7 +300,7 @@ async def run_init_script(app, scripts): cmd, executable="/bin/bash", env={**os.environ, - "KAPOW_URL": "http://localhost:8080/kapow" + "KAPOW_URL": "http://localhost:8080" }) await shell_task.wait() @@ -316,12 +316,15 @@ async def start_background_tasks(app): def start_kapow_server(scripts): app = web.Application(client_max_size=1024**3) app.add_routes([ - web.get('/kapow/routes', get_routes), - web.post('/kapow/routes', create_route), - web.delete('/kapow/routes/{id}', delete_route), - web.get('/kapow/connections/{id}/{field:.*}', get_field), - # web.post('/kapow/connections/{id}/{field:.*}', append_field), - web.put('/kapow/connections/{id}/{field:.*}', set_field), + # Control API + web.get('/routes', get_routes), + web.post('/routes', append_route), # TODO: return route index + # web.put('/routes', insert_route), # TODO: return route index + web.delete('/routes/{id}', delete_route), + + # Data API + web.get('/handlers/{id}/{field:.*}', get_field), + web.put('/handlers/{id}/{field:.*}', set_field), ]) app["scripts"] = scripts app.on_startup.append(start_background_tasks) diff --git a/poc/bin/request b/poc/bin/request index 3268445..0ebf854 100755 --- a/poc/bin/request +++ b/poc/bin/request @@ -16,4 +16,4 @@ # limitations under the License. # -curl -sf ${KAPOW_URL}/connections/${KAPOW_CONNECTION}/request$1 +curl -sf "${KAPOW_URL}/handlers/${KAPOW_HANDLER_ID}/request$1" diff --git a/poc/bin/response b/poc/bin/response index c0961e4..75dc091 100755 --- a/poc/bin/response +++ b/poc/bin/response @@ -24,17 +24,17 @@ import requests @click.command() @click.option("--url", envvar='KAPOW_URL') -@click.option("--connection", envvar='KAPOW_CONNECTION') +@click.option("--handler-id", envvar='KAPOW_HANDLER_ID') @click.argument("path", nargs=1) @click.argument("value", required=False) -def response(url, connection, path, value): +def response(url, handler_id, path, value): if value is None: data = sys.stdin.buffer else: data = value.encode('utf-8') try: - response = requests.put(f"{url}/connections/{connection}/response{path}", + response = requests.put(f"{url}/handlers/{handler_id}/response{path}", data=data) except requests.exceptions.ConnectionError: return False diff --git a/poc/examples/eval.pow b/poc/examples/eval.pow old mode 100644 new mode 100755 diff --git a/poc/examples/eval.testme b/poc/examples/eval.testme new file mode 100755 index 0000000..a17bfc9 --- /dev/null +++ b/poc/examples/eval.testme @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +curl -X POST --data-binary @- http://localhost:8080/eval <(response /body) \ =(request /body) EOF diff --git a/poc/examples/pandoc/testme b/poc/examples/pandoc/testme new file mode 100755 index 0000000..e2fabd3 --- /dev/null +++ b/poc/examples/pandoc/testme @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +curl -X POST --data-binary @- http://localhost:8080/convert/markdown/man < @@ -28,7 +28,7 @@ kapow route add / - <<-'EOF' EOF kapow route add /save/magnet -e '/bin/bash -c' - <<-'EOF' - link=$(request /param/link) + link=$(request /params/link) [ -z $link ] && response /status 400 && exit 0 watch_folder=/tmp @@ -37,7 +37,7 @@ kapow route add /save/magnet -e '/bin/bash -c' - <<-'EOF' echo "d10:magnet-uri${#link}:${link}e" > "meta-${BASH_REMATCH[1]}.torrent" response /status 302 - response /header/Location /torrent/list + response /headers/Location /torrent/list EOF kapow route add /torrent/list -c 'response /body "Not Implemented Yet"' diff --git a/spec/README.md b/spec/README.md index 5e5c474..baf4294 100644 --- a/spec/README.md +++ b/spec/README.md @@ -5,45 +5,33 @@ Because we think that: -- UNIX is great and we love it -- The UNIX shell is great +- UNIX® is great and we love it +- The UNIX® shell is great - HTTP interfaces are convenient and everywhere - CGI is not a good way to mix them ## How? -So, how we can mix the **web** and the **shell**? Let's see... +So, how we can mix the **web** and the **shell**? Let's see... The **web** and the **shell** are two different beasts, both packed with history. There are some concepts in HTTP and the shell that **resemble each other**. -``` - +------------------------+-------------------------+ - | HTTP | SHELL | - +--------------+------------------------+-------------------------+ - | Input | POST form-encoding | Command line parameters | - | Parameters | GET parameters | Environment variables | - | | Headers | | - | | Serialized body (JSON) | | - +--------------+------------------------+-------------------------+ - | Data Streams | Response/Request Body | Stdin/Stdout/Stderr | - | | Websocket | Input/Output files | - | | Uploaded files | | - +--------------+------------------------+-------------------------+ - | Control | Status codes | Signals | - | | HTTP Methods | Exit Codes | - +--------------+------------------------+-------------------------+ -``` + | | HTTP | Shell | + |------------------------|--------------------------------------------------------------------------------|----------------------------------------------------| + | Input
Parameters | POST form-encoding
Get parameters
Headers
Serialized body (JSON) | Command line parameters
Environment variables | + | Data Streams | Response/Request Body
Websocket
Uploaded files | stdin/stdout/stderr
Input/Output files | + | Control | Status codes
HTTP Methods | Signals
Exit Codes | Any tool designed to give an HTTP interface to an existing shell command -**must map concepts of boths**. For example: +**must map concepts from both domains**. For example: - "GET parameters" to "Command line parameters" - "Headers" to "Environment variables" -- "Stdout" to "Response body" +- "stdout" to "Response body" Kapow! is not opinionated about the different ways you can map both worlds. Instead, it provides a concise set of tools, with a set of sensible defaults, @@ -52,8 +40,8 @@ allowing the user to express the desired mapping in an explicit way. ### Why not tool "X"? -All the alternatives we found are **rigid** about how they match between HTTP -and shell concepts. +All the alternatives we found are **rigid** about the way they match between +HTTP and shell concepts. * [shell2http](https://github.com/msoap/shell2http): HTTP-server to execute shell commands. Designed for development, prototyping or remote control. @@ -78,17 +66,17 @@ incapable in others. * Boilerplate * Custom code = More bugs -* Security issues (Command injection, etc) +* Security issues (command injection, etc) * Dependency on developers -* **"A programming language is low level when its programs require attention to - the irrelevant"** *Alan Perlis* -* **There is more Unix-nature in one line of shell script than there is in ten - thousand lines of C** *Master Foo* +* *"A programming language is low level when its programs require attention to + the irrelevant."
—Alan Perlis* +* *"There is more Unix-nature in one line of shell script than there is in ten + thousand lines of C."
—Master Foo* ### Why not CGI? -* CGI is also **rigid** about how it matches between HTTP and UNIX process +* CGI is also **rigid** about how it matches between HTTP and UNIX® process concepts. Notably CGI *meta-variables* are injected into the script's environment; this behavior can and has been exploited by nasty attacks such as [Shellshock](https://en.wikipedia.org/wiki/Shellshock_(software_bug)). @@ -101,7 +89,7 @@ incapable in others. ## What? We named it Kapow!. It is pronounceable, short and meaningless... like every -good UNIX command ;-) +good UNIX® command ;-) TODO: Definition @@ -111,7 +99,7 @@ TODO: Intro to Architecture ### API Kapow! server interacts with the outside world only through its HTTP API. Any -program making the correct HTTP request to a Kapow! server, can change its +program making the correct HTTP request to a Kapow! server can change its behavior. Kapow! exposes two distinct APIs, a control API and a data API, described @@ -135,16 +123,16 @@ whole lifetime of the server. operating the client. The human can use this information to make a decision on how to proceed. * All successful API calls will return a representation of the *final* state - attained by the objects which have been addressed (requested, set or + attained by the objects which have been addressed (either requested, set or deleted). For instance, given this request: -``` +```http HTTP/1.1 GET /routes ``` an appropiate reponse may look like this: -``` +```http 200 OK Content-Type: application/json Content-Length: 189 @@ -169,7 +157,7 @@ Kapow! provides a way to control its internal state through these elements. ### Routes Routes are the mechanism that allows Kapow! to find the correct program to -respond to an external event (e.g. an incomming HTTP request). +respond to an external event (e.g. an incoming HTTP request). #### List routes @@ -180,8 +168,24 @@ Returns JSON data about the current routes. * **Method**: `GET` * **Success Responses**: * **Code**: `200 OK`
- **Content**: TODO -* **Sample Call**: TODO + **Content**:
+ ```json + [ + { + "method": "GET", + "url_pattern": "/hello", + "entrypoint": null, + "command": "echo Hello World | response /body" + }, + { + "method": "POST", + "url_pattern": "/bye", + "entrypoint": null, + "command": "echo Bye World | response /body" + } + ] + ``` +* **Sample Call**: `$ curl $KAPOW_URL/routes` * **Notes**: Currently all routes are returned; in the future, a filter may be accepted. @@ -193,7 +197,7 @@ Accepts JSON data that defines a new route to be appended to the current routes. * **Method**: `POST` * **Header**: `Content-Type: application/json` * **Data Params**:
- ``` + ```json { "method": "GET", "url_pattern": "/hello", @@ -202,10 +206,10 @@ Accepts JSON data that defines a new route to be appended to the current routes. } ``` * **Success Responses**: - * **Code**: `200 OK`
+ * **Code**: `201 Created`
**Header**: `Content-Type: application/json`
**Content**:
- ``` + ```json { "method": "GET", "url_pattern": "/hello", @@ -217,13 +221,35 @@ Accepts JSON data that defines a new route to be appended to the current routes. * **Error Responses**: * **Code**: `400 Malformed JSON` * **Code**: `400 Invalid Data Type` + * **Code**: `400 Invalid Route Spec` * **Code**: `400 Missing Mandatory Field`
**Header**: `Content-Type: application/json`
- **Content**: `{ "mandatory_fields": ["field1", "field2", "and so on"] }` -* **Sample Call**: TODO + **Content**:
+ ```json + { + "missing_mandatory_fields": [ + "url_pattern", + "command" + ] + } + ``` +* **Sample Call**:
+ ```sh + $ curl -X POST --data-binary @- $KAPOW_URL/routes < - ``` + ```json { "method": "GET", "url_pattern": "/hello", @@ -247,7 +273,7 @@ Accepts JSON data that defines a new route to be appended to the current routes. * **Code**: `200 OK`
**Header**: `Content-Type: application/json`
**Content**:
- ``` + ```json { "method": "GET", "url_pattern": "/hello", @@ -259,15 +285,38 @@ Accepts JSON data that defines a new route to be appended to the current routes. * **Error Responses**: * **Code**: `400 Malformed JSON` * **Code**: `400 Invalid Data Type` + * **Code**: `400 Invalid Route Spec` * **Code**: `400 Missing Mandatory Field`
**Header**: `Content-Type: application/json`
- **Content**: `{ "mandatory_fields": ["field1", "field2", "and so on"] }` + **Content**:
+ ```json + { + "missing_mandatory_fields": [ + "url_pattern", + "command" + ] + } + ``` * **Code**: `400 Invalid Index Type` -* **Sample Call**: TODO + * **Code**: `400 Index Already in Use` + * **Code**: `404 Invalid Index` + * **Code**: `404 Invalid Route Spec` +* **Sample Call**:
+ ```sh + $ curl -X PUT --data-binary @- $KAPOW_URL/routes < - **Content**: TODO + **Content**:
+ ```json + { + "method": "GET", + "url_pattern": "/hello", + "entrypoint": null, + "command": "echo Hello World | response /body", + "index": 0 + } + ``` * **Error Responses**: * **Code**: `404 Not Found` -* **Sample Call**: TODO +* **Sample Call**:
+ ```sh + $ curl -X DELETE $KAPOW_URL/routes/ROUTE_1f186c92_f906_4506_9788_a1f541b11d0f + ``` * **Notes**: @@ -381,7 +442,7 @@ following resource paths: - Comment: That would provide read-only access to the value of the request header `Content-Type`. - Read a field from a form. - Scenario: A request generated by submitting this form:
- ``` + ```html
First name:

@@ -428,8 +489,11 @@ Returns the value of the requested resource path, or an error if the resource pa **Code**: `400 Invalid Resource Path`
**Notes**: Check the list of valid resource paths at the top of this section. * **Code**: `404 Not Found` -* **Sample Call**: TODO -* **Notes**: TODO +* **Sample Call**:
+ ```sh + $ curl /handlers/$KAPOW_HANDLER_ID/request/body + ``` +* **Notes**: The content may be empty. #### Overwrite the value of a resource @@ -447,7 +511,10 @@ Returns the value of the requested resource path, or an error if the resource pa * **Code**: `404 Handler Not Found` * **Code**: `404 Name Not Found`
**Notes**: Although the resource path is correct, no such name is present in the request. For instance, `/request/headers/Foo`, when no `Foo` header is not present in the request. -* **Sample Call**: +* **Sample Call**:
+ ```sh + $ curl -X --data-binary '

Hello!

' PUT /handlers/$KAPOW_HANDLER_ID/response/body + ``` * **Notes**: @@ -472,32 +539,167 @@ Any compliant implementation of Kapow! must provide these commands: ### `kapow server` -This implements the server, XXX +This is the master command, that shows the help if invoked without args, and +runs the sub-commands when provided to it. #### Example +```sh +$ kapow +Usage: kapow [OPTIONS] COMMAND [ARGS]... + +Options: + TBD + +Commands: + start starts a Kapow! server + route operates on routes +... +``` + + +### `kapow start` + +This command runs the Kapow! server, which is the core of Kapow!. If +run without parameters, it will run an unconfigured server. It can accept a path +to a `.pow` file, which is a shell script that contains commands to configure +the Kapow! server. + +The `.pow` can leverage the `kapow route` command, which is used to define a route. +The `kapow route` command needs a way to reach the Kapow! server, and for that, +`kapow` provides the `KAPOW_URL` variable in the environment of the +aforementioned shell script. + +Every time the kapow server receives a request, it will spawn a process to +handle it, according to the specified entrypoint, `/bin/sh -c` by default, and then +execute the specified command. This command is tasked with processing the +incoming request, and can leverage the `request` and `response` commands to +easily access the `HTTP Request` and `HTTP Response`, respectively. + +In order for `request` and `response` to do their job, they require a way to +reach the Kapow! server, as well as a way to identify the current request being +served. Thus, the Kapow! server adds the `KAPOW_URL` and `KAPOW_HANDLER_ID` to the +process' environment. + + +#### Example +```sh +$ kapow start /path/to/service.pow +``` ### `kapow route` +To serve an endpoint, you must first register it. + +`kapow route` registers/deregisters a route, which maps an +`HTTP` method and a URL pattern to the code that will handle the request. + +When registering, you can specify an *entrypoint*, which defaults to `/bin/sh -c`, +and an argument to it, the *command*. + +To deregister a route you must provide a *route_id*. + +**Notes**: + * The entrypoint definition matches *Docker*'s. + * The index matches the way *netfilter*'s `iptables` handles rule numbering. + + +#### **Environment** +- `KAPOW_URL` + + +#### **Help** +```sh +$ kapow route --help +Usage: kapow route [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + add + remove +``` +```sh +$ kapow route add --help +Usage: kapow route add [OPTIONS] URL_PATTERN [COMMAND_FILE] + +Options: + -c, --command TEXT + -e, --entrypoint TEXT + -X, --method TEXT + --url TEXT + --help Show this message and exit. +``` +```sh +$ kapow route remove --help +Usage: kapow route remove [OPTIONS] ROUTE_ID + +Options: + --url TEXT + --help Show this message and exit. +``` #### Example - +```sh +kroute add -X GET '/list/{ip}' -c 'nmap -sL $(request /matches/ip) | response /body' +``` ### `request` +Exposes the requests' resources. + + +#### **Environment** +- `KAPOW_URL` +- `KAPOW_HANDLER_ID` + #### Example +```sh +# Access the body of the request +request /body +``` ### `response` +Exposes the response's resources. + + +#### **Environment** +- `KAPOW_URL` +- `KAPOW_HANDLER_ID` + #### Example +```sh +# Write to the body of the response +echo 'Hello, World!' | response /body +``` ## An End-to-End Example +```sh +$ cat nmap.kpow +kroute add -X GET '/list/{ip}' -c 'nmap -sL $(request /matches/ip) | response /body' +``` +```sh +$ kapow ./nmap.kapow +``` +```sh +$ curl $KAPOW_URL/list/127.0.0.1 +Starting Nmap 7.70 ( https://nmap.org ) at 2019-05-30 14:45 CEST +Nmap scan report for localhost (127.0.0.1) +Host is up (0.00011s latency). +Not shown: 999 closed ports +PORT STATE SERVICE +22/tcp open ssh + +Nmap done: 1 IP address (1 host up) scanned in 0.06 seconds +``` ## Test Suite Notes