Dark-Alex-17/kapow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into server-as-subcommand

Browse Source
This commit is contained in:
Roberto Abdelkader Martínez Pérez
2019-06-05 10:15:35 +02:00
parent 96739189f1 d067b46d11
commit b10765f623
14 changed files with 311 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
poc/kapow → poc/bin/kapow
+20 -17
View File
@@ -72,17 +72,17 @@ class Connection:
return self.request.content
elif res.path == 'request/path':
return self.request.path.encode('utf-8')
elif res.path.startswith('request/match/'):
elif res.path.startswith('request/matches/'):
return self.request.match_info[nth(2)].encode('utf-8')
elif res.path.startswith('request/param/'):
elif res.path.startswith('request/params/'):
return self.request.rel_url.query[nth(2)].encode('utf-8')
elif res.path.startswith('request/header/'):
elif res.path.startswith('request/headers/'):
return self.request.headers[nth(2)].encode('utf-8')
elif res.path.startswith('request/cookie/'):
elif res.path.startswith('request/cookies/'):
return self.request.cookies[nth(2)].encode('utf-8')
elif res.path.startswith('request/form/'):
return (await self.request.post())[nth(2)].encode('utf-8')
elif res.path.startswith('request/file/'):
elif res.path.startswith('request/files/'):
name = nth(2)
content = nth(3) # filename / content
field = (await self.request.post())[name]
@@ -112,10 +112,10 @@ class Connection:
self._status = int((await content.read()).decode('utf-8'))
elif res.path == 'response/body':
self._body.write(await content.read())
elif res.path.startswith('response/header/'):
elif res.path.startswith('response/headers/'):
clean = (await content.read()).rstrip(b'\n').decode('utf-8')
self._headers[nth(2)] = clean
elif res.path.startswith('response/cookie/'):
elif res.path.startswith('response/cookies/'):
clean = (await content.read()).rstrip(b'\n').decode('utf-8')
self._cookies[nth(2)] = clean
elif res.path == 'response/stream':
@@ -230,8 +230,8 @@ def handle_route(entrypoint, command):
shell_task = await asyncio.create_subprocess_shell(
args,
env={**os.environ,
"KAPOW_URL": "http://localhost:8080/kapow",
"KAPOW_CONNECTION": id
"KAPOW_URL": "http://localhost:8080",
"KAPOW_HANDLER_ID": id
},
stdin=asyncio.subprocess.DEVNULL)
@@ -256,7 +256,7 @@ async def get_routes(request):
return web.json_response(list(request.app.router))
async def create_route(request):
async def append_route(request):
"""Create a new Kapow! route."""
request.app.router._frozen = False
content = await request.json()
@@ -300,7 +300,7 @@ async def run_init_script(app, scripts):
cmd,
executable="/bin/bash",
env={**os.environ,
"KAPOW_URL": "http://localhost:8080/kapow"
"KAPOW_URL": "http://localhost:8080"
})
await shell_task.wait()
@@ -316,12 +316,15 @@ async def start_background_tasks(app):
def start_kapow_server(scripts):
app = web.Application(client_max_size=1024**3)
app.add_routes([
web.get('/kapow/routes', get_routes),
web.post('/kapow/routes', create_route),
web.delete('/kapow/routes/{id}', delete_route),
web.get('/kapow/connections/{id}/{field:.*}', get_field),
# web.post('/kapow/connections/{id}/{field:.*}', append_field),
web.put('/kapow/connections/{id}/{field:.*}', set_field),
# Control API
web.get('/routes', get_routes),
web.post('/routes', append_route), # TODO: return route index
# web.put('/routes', insert_route), # TODO: return route index
web.delete('/routes/{id}', delete_route),
# Data API
web.get('/handlers/{id}/{field:.*}', get_field),
web.put('/handlers/{id}/{field:.*}', set_field),
])
app["scripts"] = scripts
app.on_startup.append(start_background_tasks)
poc/bin/request
+1 -1
View File
@@ -16,4 +16,4 @@
# limitations under the License.
#
curl -sf ${KAPOW_URL}/connections/${KAPOW_CONNECTION}/request$1
curl -sf "${KAPOW_URL}/handlers/${KAPOW_HANDLER_ID}/request$1"
poc/bin/response
+3 -3
View File
@@ -24,17 +24,17 @@ import requests
@click.command()
@click.option("--url", envvar='KAPOW_URL')
@click.option("--connection", envvar='KAPOW_CONNECTION')
@click.option("--handler-id", envvar='KAPOW_HANDLER_ID')
@click.argument("path", nargs=1)
@click.argument("value", required=False)
def response(url, connection, path, value):
def response(url, handler_id, path, value):
if value is None:
data = sys.stdin.buffer
else:
data = value.encode('utf-8')
try:
response = requests.put(f"{url}/connections/{connection}/response{path}",
response = requests.put(f"{url}/handlers/{handler_id}/response{path}",
data=data)
except requests.exceptions.ConnectionError:
return False
poc/examples/eval.pow
Regular → Executable
View File
poc/examples/eval.testme
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/usr/bin/env sh
curl -X POST --data-binary @- http://localhost:8080/eval <<EOF
touch /tmp/kapow_was_here
EOF
echo 'Proof of success:'
ls -l /tmp/kapow_was_here
poc/examples/nmap/Dockerfile
+1 -1
View File
@@ -3,4 +3,4 @@ RUN apk update && apk add nmap
COPY nmap.pow /tmp/
RUN cd /tmp && pipenv install --system --deploy
ENTRYPOINT ["/usr/bin/kapow"]
CMD ["/tmp/nmap.pow"]
CMD ["server", "/tmp/nmap.pow"]
poc/examples/nmap/nmap.pow
Regular → Executable
+1 -1
View File
@@ -16,4 +16,4 @@
# limitations under the License.
#
kapow route add -X GET '/list/{ip}' -c 'nmap -sL $(request /match/ip) | response /body'
kapow route add -X GET '/list/{ip}' -c 'nmap -sL $(request /matches/ip) | response /body'
poc/examples/operator.pow
+3 -3
View File
@@ -16,7 +16,7 @@
# limitations under the License.
#
kapow route add /list/files -c 'ls -la $(request /param/path) | response /body'
kapow route add /list/files -c 'ls -la $(request /params/path) | response /body'
kapow route add /list/processes -c 'ps -aux | response /body'
@@ -31,11 +31,11 @@ kapow route add /show/connections -c 'ss -pluton | response /body'
kapow route add /show/mounts -c 'mount | response /body'
kapow route add /tail/dmesg - <<-'EOF'
response /header/Content-Type text/plain
response /headers/Content-Type text/plain
dmesg -w | response /stream
EOF
kapow route add /tail/journal - <<-'EOF'
response /header/Content-Type text/plain
response /headers/Content-Type text/plain
journalctl -f | response /stream
EOF
poc/examples/pandoc/pandoc.pow
+2 -2
View File
@@ -17,8 +17,8 @@
#
kapow route add -X POST --entrypoint '/bin/zsh -c' '/convert/{from}/{to}' - <<-'EOF'
pandoc --from=$(request /match/from) \
--to=$(request /match/to) \
pandoc --from=$(request /matches/from) \
--to=$(request /matches/to) \
--output=>(response /body) \
=(request /body)
EOF
poc/examples/pandoc/testme
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/usr/bin/env sh
curl -X POST --data-binary @- http://localhost:8080/convert/markdown/man <<EOF
# This is not a pipe
1. hello
1. goodbye
EOF
poc/examples/pdfeditor/pdfeditor.pow
Regular → Executable
+1 -1
View File
@@ -17,4 +17,4 @@
#
kapow route add -X POST --entrypoint ./topdf '/editor/pdf'
kapow route add / -c 'response /header/Content-Type text/html && response /body < pdfeditor.html'
kapow route add / -c 'response /headers/Content-Type text/html && response /body < pdfeditor.html'
poc/examples/pdfeditor/topdf
+1 -1
View File
@@ -19,7 +19,7 @@
tmpfile=$(mktemp --suffix=.pdf)
pandoc --from=$(request /form/from) --to=pdf --output=${tmpfile} -t latex =(request /form/content)
if [ $? -eq 0 ]; then
response /header/Content-Type application/pdf
response /headers/Content-Type application/pdf
response /body < ${tmpfile}
response /status 200
else
poc/examples/torrent.pow
Regular → Executable
+3 -3
View File
@@ -17,7 +17,7 @@
#
kapow route add / - <<-'EOF'
response /header/Content-Type text/html
response /headers/Content-Type text/html
response /body <<-HTML
<html>
<body>
@@ -28,7 +28,7 @@ kapow route add / - <<-'EOF'
EOF
kapow route add /save/magnet -e '/bin/bash -c' - <<-'EOF'
link=$(request /param/link)
link=$(request /params/link)
[ -z $link ] && response /status 400 && exit 0
watch_folder=/tmp
@@ -37,7 +37,7 @@ kapow route add /save/magnet -e '/bin/bash -c' - <<-'EOF'
echo "d10:magnet-uri${#link}:${link}e" > "meta-${BASH_REMATCH[1]}.torrent"
response /status 302
response /header/Location /torrent/list
response /headers/Location /torrent/list
EOF
kapow route add /torrent/list -c 'response /body "Not Implemented Yet"'
spec/README.md
+258 -56
View File
@@ -5,8 +5,8 @@
Because we think that:
- UNIX is great and we love it
- The UNIX shell is great
- UNIX® is great and we love it
- The UNIX® shell is great
- HTTP interfaces are convenient and everywhere
- CGI is not a good way to mix them
@@ -20,30 +20,18 @@ history.
There are some concepts in HTTP and the shell that **resemble each other**.
```
+------------------------+-------------------------+
| HTTP | SHELL |
+--------------+------------------------+-------------------------+
| Input | POST form-encoding | Command line parameters |
| Parameters | GET parameters | Environment variables |
| | Headers | |
| | Serialized body (JSON) | |
+--------------+------------------------+-------------------------+
| Data Streams | Response/Request Body | Stdin/Stdout/Stderr |
| | Websocket | Input/Output files |
| | Uploaded files | |
+--------------+------------------------+-------------------------+
| Control | Status codes | Signals |
| | HTTP Methods | Exit Codes |
+--------------+------------------------+-------------------------+
```
| | HTTP | Shell |
|------------------------|--------------------------------------------------------------------------------|----------------------------------------------------|
| Input<br /> Parameters | POST form-encoding<br >Get parameters<br />Headers<br />Serialized body (JSON) | Command line parameters<br />Environment variables |
| Data Streams | Response/Request Body<br />Websocket<br />Uploaded files | stdin/stdout/stderr<br />Input/Output files |
| Control | Status codes<br />HTTP Methods | Signals<br />Exit Codes |
Any tool designed to give an HTTP interface to an existing shell command
**must map concepts of boths**. For example:
**must map concepts from both domains**. For example:
- "GET parameters" to "Command line parameters"
- "Headers" to "Environment variables"
- "Stdout" to "Response body"
- "stdout" to "Response body"
Kapow! is not opinionated about the different ways you can map both worlds.
Instead, it provides a concise set of tools, with a set of sensible defaults,
@@ -52,8 +40,8 @@ allowing the user to express the desired mapping in an explicit way.
### Why not tool "X"?
All the alternatives we found are **rigid** about how they match between HTTP
and shell concepts.
All the alternatives we found are **rigid** about the way they match between
HTTP and shell concepts.
* [shell2http](https://github.com/msoap/shell2http): HTTP-server to execute
shell commands. Designed for development, prototyping or remote control.
@@ -78,17 +66,17 @@ incapable in others.
* Boilerplate
* Custom code = More bugs
* Security issues (Command injection, etc)
* Security issues (command injection, etc)
* Dependency on developers
* **"A programming language is low level when its programs require attention to
the irrelevant"** *Alan Perlis*
* **There is more Unix-nature in one line of shell script than there is in ten
thousand lines of C** *Master Foo*
* *"A programming language is low level when its programs require attention to
the irrelevant."<br />&mdash;Alan Perlis*
* *"There is more Unix-nature in one line of shell script than there is in ten
thousand lines of C."<br />&mdash;Master Foo*
### Why not CGI?
* CGI is also **rigid** about how it matches between HTTP and UNIX process
* CGI is also **rigid** about how it matches between HTTP and UNIX® process
concepts. Notably CGI *meta-variables* are injected into the script's
environment; this behavior can and has been exploited by nasty attacks such as
[Shellshock](https://en.wikipedia.org/wiki/Shellshock_(software_bug)).
@@ -101,7 +89,7 @@ incapable in others.
## What?
We named it Kapow!. It is pronounceable, short and meaningless... like every
good UNIX command ;-)
good UNIX® command ;-)
TODO: Definition
@@ -111,7 +99,7 @@ TODO: Intro to Architecture
### API
Kapow! server interacts with the outside world only through its HTTP API. Any
program making the correct HTTP request to a Kapow! server, can change its
program making the correct HTTP request to a Kapow! server can change its
behavior.
Kapow! exposes two distinct APIs, a control API and a data API, described
@@ -135,16 +123,16 @@ whole lifetime of the server.
operating the client. The human can use this information to make a
decision on how to proceed.
* All successful API calls will return a representation of the *final* state
attained by the objects which have been addressed (requested, set or
attained by the objects which have been addressed (either requested, set or
deleted).
For instance, given this request:
```
```http
HTTP/1.1 GET /routes
```
an appropiate reponse may look like this:
```
```http
200 OK
Content-Type: application/json
Content-Length: 189
@@ -169,7 +157,7 @@ Kapow! provides a way to control its internal state through these elements.
### Routes
Routes are the mechanism that allows Kapow! to find the correct program to
respond to an external event (e.g. an incomming HTTP request).
respond to an external event (e.g. an incoming HTTP request).
#### List routes
@@ -180,8 +168,24 @@ Returns JSON data about the current routes.
* **Method**: `GET`
* **Success Responses**:
* **Code**: `200 OK`<br />
**Content**: TODO
* **Sample Call**: TODO
**Content**:<br />
```json
[
{
"method": "GET",
"url_pattern": "/hello",
"entrypoint": null,
"command": "echo Hello World | response /body"
},
{
"method": "POST",
"url_pattern": "/bye",
"entrypoint": null,
"command": "echo Bye World | response /body"
}
]
```
* **Sample Call**: `$ curl $KAPOW_URL/routes`
* **Notes**: Currently all routes are returned; in the future, a filter may be accepted.
@@ -193,7 +197,7 @@ Accepts JSON data that defines a new route to be appended to the current routes.
* **Method**: `POST`
* **Header**: `Content-Type: application/json`
* **Data Params**:<br />
```
```json
{
"method": "GET",
"url_pattern": "/hello",
@@ -202,10 +206,10 @@ Accepts JSON data that defines a new route to be appended to the current routes.
}
```
* **Success Responses**:
* **Code**: `200 OK`<br />
* **Code**: `201 Created`<br />
**Header**: `Content-Type: application/json`<br />
**Content**:<br />
```
```json
{
"method": "GET",
"url_pattern": "/hello",
@@ -217,13 +221,35 @@ Accepts JSON data that defines a new route to be appended to the current routes.
* **Error Responses**:
* **Code**: `400 Malformed JSON`
* **Code**: `400 Invalid Data Type`
* **Code**: `400 Invalid Route Spec`
* **Code**: `400 Missing Mandatory Field`<br />
**Header**: `Content-Type: application/json`<br />
**Content**: `{ "mandatory_fields": ["field1", "field2", "and so on"] }`
* **Sample Call**: TODO
**Content**:<br />
```json
{
"missing_mandatory_fields": [
"url_pattern",
"command"
]
}
```
* **Sample Call**:<br />
```sh
$ curl -X POST --data-binary @- $KAPOW_URL/routes <<EOF
{
"method": "GET",
"url_pattern": "/hello",
"entrypoint": null,
"command": "echo Hello World | response /body",
"index": 0
}
EOF
```
* **Notes**:
* A successful request will yield a response containing all the effective
parameters that were applied.
* Kapow! won't try to validate the submitted command. Any errors will happen
at runtime, and trigger a `500` status code.
#### Insert a route
@@ -235,7 +261,7 @@ Accepts JSON data that defines a new route to be appended to the current routes.
* **Method**: `PUT`
* **Header**: `Content-Type: application/json`
* **Data Params**:<br />
```
```json
{
"method": "GET",
"url_pattern": "/hello",
@@ -247,7 +273,7 @@ Accepts JSON data that defines a new route to be appended to the current routes.
* **Code**: `200 OK`<br />
**Header**: `Content-Type: application/json`<br />
**Content**:<br />
```
```json
{
"method": "GET",
"url_pattern": "/hello",
@@ -259,15 +285,38 @@ Accepts JSON data that defines a new route to be appended to the current routes.
* **Error Responses**:
* **Code**: `400 Malformed JSON`
* **Code**: `400 Invalid Data Type`
* **Code**: `400 Invalid Route Spec`
* **Code**: `400 Missing Mandatory Field`<br />
**Header**: `Content-Type: application/json`<br />
**Content**: `{ "mandatory_fields": ["field1", "field2", "and so on"] }`
**Content**:<br />
```json
{
"missing_mandatory_fields": [
"url_pattern",
"command"
]
}
```
* **Code**: `400 Invalid Index Type`
* **Sample Call**: TODO
* **Code**: `400 Index Already in Use`
* **Code**: `404 Invalid Index`
* **Code**: `404 Invalid Route Spec`
* **Sample Call**:<br />
```sh
$ curl -X PUT --data-binary @- $KAPOW_URL/routes <<EOF`
{
"method": "GET",
"url_pattern": "/hello",
"entrypoint": null,
"command": "echo Hello World | response /body",
"index": 0
}
EOF
```
* **Notes**:
* Route numbering starts at zero.
* When `index` is not provided or is less than 0 the route will be inserted
first, effectively making it index 0.
* When `index` is not provided or is less than `0` the route will be inserted
first, effectively making it index `0`.
* Conversely, when `index` is greater than the number of entries on the route
table, it will be inserted last.
* A successful request will yield a response containing all the effective
@@ -282,10 +331,22 @@ Removes the route identified by `:id`.
* **Method**: `DELETE`
* **Success Responses**:
* **Code**: `200 OK`<br />
**Content**: TODO
**Content**:<br />
```json
{
"method": "GET",
"url_pattern": "/hello",
"entrypoint": null,
"command": "echo Hello World | response /body",
"index": 0
}
```
* **Error Responses**:
* **Code**: `404 Not Found`
* **Sample Call**: TODO
* **Sample Call**:<br />
```sh
$ curl -X DELETE $KAPOW_URL/routes/ROUTE_1f186c92_f906_4506_9788_a1f541b11d0f
```
* **Notes**:
@@ -381,7 +442,7 @@ following resource paths:
- Comment: That would provide read-only access to the value of the request header `Content-Type`.
- Read a field from a form.
- Scenario: A request generated by submitting this form:<br />
```
```html
<form method="post">
First name:<br>
<input type="text" name="firstname" value="Jane"><br>
@@ -428,8 +489,11 @@ Returns the value of the requested resource path, or an error if the resource pa
**Code**: `400 Invalid Resource Path`<br />
**Notes**: Check the list of valid resource paths at the top of this section.
* **Code**: `404 Not Found`
* **Sample Call**: TODO
* **Notes**: TODO
* **Sample Call**:<br />
```sh
$ curl /handlers/$KAPOW_HANDLER_ID/request/body
```
* **Notes**: The content may be empty.
#### Overwrite the value of a resource
@@ -447,7 +511,10 @@ Returns the value of the requested resource path, or an error if the resource pa
* **Code**: `404 Handler Not Found`
* **Code**: `404 Name Not Found`<br />
**Notes**: Although the resource path is correct, no such name is present in the request. For instance, `/request/headers/Foo`, when no `Foo` header is not present in the request.
* **Sample Call**:
* **Sample Call**:<br />
```sh
$ curl -X --data-binary '<h1>Hello!</h1>' PUT /handlers/$KAPOW_HANDLER_ID/response/body
```
* **Notes**:
@@ -472,32 +539,167 @@ Any compliant implementation of Kapow! must provide these commands:
### `kapow server`
This implements the server, XXX
This is the master command, that shows the help if invoked without args, and
runs the sub-commands when provided to it.
#### Example
```sh
$ kapow
Usage: kapow [OPTIONS] COMMAND [ARGS]...
Options:
TBD
Commands:
start starts a Kapow! server
route operates on routes
...
```
### `kapow start`
This command runs the Kapow! server, which is the core of Kapow!. If
run without parameters, it will run an unconfigured server. It can accept a path
to a `.pow` file, which is a shell script that contains commands to configure
the Kapow! server.
The `.pow` can leverage the `kapow route` command, which is used to define a route.
The `kapow route` command needs a way to reach the Kapow! server, and for that,
`kapow` provides the `KAPOW_URL` variable in the environment of the
aforementioned shell script.
Every time the kapow server receives a request, it will spawn a process to
handle it, according to the specified entrypoint, `/bin/sh -c` by default, and then
execute the specified command. This command is tasked with processing the
incoming request, and can leverage the `request` and `response` commands to
easily access the `HTTP Request` and `HTTP Response`, respectively.
In order for `request` and `response` to do their job, they require a way to
reach the Kapow! server, as well as a way to identify the current request being
served. Thus, the Kapow! server adds the `KAPOW_URL` and `KAPOW_HANDLER_ID` to the
process' environment.
#### Example
```sh
$ kapow start /path/to/service.pow
```
### `kapow route`
To serve an endpoint, you must first register it.
`kapow route` registers/deregisters a route, which maps an
`HTTP` method and a URL pattern to the code that will handle the request.
When registering, you can specify an *entrypoint*, which defaults to `/bin/sh -c`,
and an argument to it, the *command*.
To deregister a route you must provide a *route_id*.
**Notes**:
* The entrypoint definition matches *Docker*'s.
* The index matches the way *netfilter*'s `iptables` handles rule numbering.
#### **Environment**
- `KAPOW_URL`
#### **Help**
```sh
$ kapow route --help
Usage: kapow route [OPTIONS] COMMAND [ARGS]...
Options:
--help Show this message and exit.
Commands:
add
remove
```
```sh
$ kapow route add --help
Usage: kapow route add [OPTIONS] URL_PATTERN [COMMAND_FILE]
Options:
-c, --command TEXT
-e, --entrypoint TEXT
-X, --method TEXT
--url TEXT
--help Show this message and exit.
```
```sh
$ kapow route remove --help
Usage: kapow route remove [OPTIONS] ROUTE_ID
Options:
--url TEXT
--help Show this message and exit.
```
#### Example
```sh
kroute add -X GET '/list/{ip}' -c 'nmap -sL $(request /matches/ip) | response /body'
```
### `request`
Exposes the requests' resources.
#### **Environment**
- `KAPOW_URL`
- `KAPOW_HANDLER_ID`
#### Example
```sh
# Access the body of the request
request /body
```
### `response`
Exposes the response's resources.
#### **Environment**
- `KAPOW_URL`
- `KAPOW_HANDLER_ID`
#### Example
```sh
# Write to the body of the response
echo 'Hello, World!' | response /body
```
## An End-to-End Example
```sh
$ cat nmap.kpow
kroute add -X GET '/list/{ip}' -c 'nmap -sL $(request /matches/ip) | response /body'
```
```sh
$ kapow ./nmap.kapow
```
```sh
$ curl $KAPOW_URL/list/127.0.0.1
Starting Nmap 7.70 ( https://nmap.org ) at 2019-05-30 14:45 CEST
Nmap scan report for localhost (127.0.0.1)
Host is up (0.00011s latency).
Not shown: 999 closed ports
PORT STATE SERVICE
22/tcp open ssh
Nmap done: 1 IP address (1 host up) scanned in 0.06 seconds
```
## Test Suite Notes