diff --git a/kapow2/Pipfile b/kapow2/Pipfile new file mode 100644 index 0000000..33e0c92 --- /dev/null +++ b/kapow2/Pipfile @@ -0,0 +1,14 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +aiohttp = "*" +requests = "*" +click = "*" + +[requires] +python_version = "3.7" diff --git a/kapow2/Pipfile.lock b/kapow2/Pipfile.lock new file mode 100644 index 0000000..26988b2 --- /dev/null +++ b/kapow2/Pipfile.lock @@ -0,0 +1,157 @@ +{ + "_meta": { + "hash": { + "sha256": "ffec9439cc2f8f496cd02959817c2bf50e52d2a23824194004088271137f6057" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "aiohttp": { + "hashes": [ + "sha256:00d198585474299c9c3b4f1d5de1a576cc230d562abc5e4a0e81d71a20a6ca55", + "sha256:0155af66de8c21b8dba4992aaeeabf55503caefae00067a3b1139f86d0ec50ed", + "sha256:09654a9eca62d1bd6d64aa44db2498f60a5c1e0ac4750953fdd79d5c88955e10", + "sha256:199f1d106e2b44b6dacdf6f9245493c7d716b01d0b7fbe1959318ba4dc64d1f5", + "sha256:296f30dedc9f4b9e7a301e5cc963012264112d78a1d3094cd83ef148fdf33ca1", + "sha256:368ed312550bd663ce84dc4b032a962fcb3c7cae099dbbd48663afc305e3b939", + "sha256:40d7ea570b88db017c51392349cf99b7aefaaddd19d2c78368aeb0bddde9d390", + "sha256:629102a193162e37102c50713e2e31dc9a2fe7ac5e481da83e5bb3c0cee700aa", + "sha256:6d5ec9b8948c3d957e75ea14d41e9330e1ac3fed24ec53766c780f82805140dc", + "sha256:87331d1d6810214085a50749160196391a712a13336cd02ce1c3ea3d05bcf8d5", + "sha256:9a02a04bbe581c8605ac423ba3a74999ec9d8bce7ae37977a3d38680f5780b6d", + "sha256:9c4c83f4fa1938377da32bc2d59379025ceeee8e24b89f72fcbccd8ca22dc9bf", + "sha256:9cddaff94c0135ee627213ac6ca6d05724bfe6e7a356e5e09ec57bd3249510f6", + "sha256:a25237abf327530d9561ef751eef9511ab56fd9431023ca6f4803f1994104d72", + "sha256:a5cbd7157b0e383738b8e29d6e556fde8726823dae0e348952a61742b21aeb12", + "sha256:a97a516e02b726e089cffcde2eea0d3258450389bbac48cbe89e0f0b6e7b0366", + "sha256:acc89b29b5f4e2332d65cd1b7d10c609a75b88ef8925d487a611ca788432dfa4", + "sha256:b05bd85cc99b06740aad3629c2585bda7b83bd86e080b44ba47faf905fdf1300", + "sha256:c2bec436a2b5dafe5eaeb297c03711074d46b6eb236d002c13c42f25c4a8ce9d", + "sha256:cc619d974c8c11fe84527e4b5e1c07238799a8c29ea1c1285149170524ba9303", + "sha256:d4392defd4648badaa42b3e101080ae3313e8f4787cb517efd3f5b8157eaefd6", + "sha256:e1c3c582ee11af7f63a34a46f0448fca58e59889396ffdae1f482085061a2889" + ], + "index": "pypi", + "version": "==3.5.4" + }, + "async-timeout": { + "hashes": [ + "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", + "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" + ], + "version": "==3.0.1" + }, + "attrs": { + "hashes": [ + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + ], + "version": "==19.1.0" + }, + "certifi": { + "hashes": [ + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + ], + "version": "==2019.3.9" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "index": "pypi", + "version": "==7.0" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "multidict": { + "hashes": [ + "sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f", + "sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3", + "sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef", + "sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b", + "sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73", + "sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc", + "sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3", + "sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd", + "sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351", + "sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941", + "sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d", + "sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1", + "sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b", + "sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a", + "sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3", + "sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7", + "sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0", + "sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0", + "sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014", + "sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5", + "sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036", + "sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d", + "sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a", + "sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce", + "sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1", + "sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a", + "sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9", + "sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7", + "sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b" + ], + "version": "==4.5.2" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "index": "pypi", + "version": "==2.21.0" + }, + "urllib3": { + "hashes": [ + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + ], + "version": "==1.24.2" + }, + "yarl": { + "hashes": [ + "sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9", + "sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f", + "sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb", + "sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320", + "sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842", + "sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0", + "sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829", + "sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310", + "sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4", + "sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8", + "sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1" + ], + "version": "==1.3.0" + } + }, + "develop": {} +} diff --git a/kapow2/bin/kroute b/kapow2/bin/kroute new file mode 100755 index 0000000..02ae4ba --- /dev/null +++ b/kapow2/bin/kroute @@ -0,0 +1,48 @@ +#!/usr/bin/env python +import sys + +import click +import requests + +@click.group() +def kroute(): + pass + +@kroute.command() +@click.option("-c", "--command", nargs=1) +@click.option("-e", "--entrypoint", default="/bin/sh -c") +@click.option("-X", "--method", default="GET") +@click.option("--url", envvar='KAPOW_URL') +@click.argument("url_pattern", nargs=1) +@click.argument("command_file") +def add(url_pattern, entrypoint, command, method, url, command_file): + if command: + source = command + elif command_file == '-': + source = sys.stdin.read() + elif command_file is not None: + with open(command_file, 'r', encoding='utf-8') as handler: + source = handler.read() + else: + source = "" + + response = requests.post(f"{url}/routes", + json={"method": method, + "url_pattern": url_pattern, + "entrypoint": entrypoint, + "command": source}) + response.raise_for_status() + print(response.json()) + + +@kroute.command() +@click.option("--url", envvar='KAPOW_URL') +@click.argument("route-id") +def remove(route_id, url): + response = requests.delete(f"{url}/routes/{route_id}") + response.raise_for_status() + print(response.json()) + + +if __name__ == '__main__': + kroute() diff --git a/kapow2/bin/request b/kapow2/bin/request new file mode 100755 index 0000000..d21d921 --- /dev/null +++ b/kapow2/bin/request @@ -0,0 +1,2 @@ +#!/bin/sh +curl -sf ${KAPOW_URL}/connections/${KAPOW_CONNECTION}/request$1 diff --git a/kapow2/bin/response b/kapow2/bin/response new file mode 100755 index 0000000..3fe5358 --- /dev/null +++ b/kapow2/bin/response @@ -0,0 +1,2 @@ +#!/bin/sh +curl -sf -X PUT --data-binary @- ${KAPOW_URL}/connections/${KAPOW_CONNECTION}/response$1 diff --git a/kapow2/examples/helloworld.pow b/kapow2/examples/helloworld.pow new file mode 100755 index 0000000..4ee8f58 --- /dev/null +++ b/kapow2/examples/helloworld.pow @@ -0,0 +1,30 @@ +#!/bin/bash + +touch input +touch output +tail -F input | tr '[:lower:]' '[:upper:]' > output & + + +kroute add -X POST --entrypoint /home/nil/Project/kapow/kapow2/examples/topdf '/convert/{from}/pdf' + + +kroute add -X POST --entrypoint /bin/zsh '/convert/{from}/{to}' - <<-'EOF' + pandoc --from=$(request /match/from) --to=$(request /match/to) --output=>(response /body) =(request /body) +EOF + +kroute add -X GET '/formats/input' - <<-EOF + pandoc --list-input-formats | response /body +EOF + +kroute add '/formats/output' - <<-EOF + pandoc --list-output-formats | grep -v pdf | response /body +EOF + +kroute add '/tail' - <<-EOF + tail -f /tmp/mispelotas | response /stream +EOF + +kroute add -X POST '/tr' --command ' + request /body >> input + tail -n1 output | response /body +' diff --git a/kapow2/examples/topdf b/kapow2/examples/topdf new file mode 100755 index 0000000..ae5db11 --- /dev/null +++ b/kapow2/examples/topdf @@ -0,0 +1,12 @@ +#!/usr/bin/zsh + +tmpfile=$(mktemp --suffix=.pdf) +pandoc --from=$(request /match/from) --to=pdf --output=${tmpfile} -t latex =(request /body) +if [ $? -eq 0 ]; then + cat ${tmpfile} | response /body + echo "application/pdf" | response /header/Content-Type + echo 200 | response /status +else + echo 500 | response /status +fi +rm -f ${tmpfile} diff --git a/kapow2/kapow b/kapow2/kapow new file mode 100755 index 0000000..dc2a061 --- /dev/null +++ b/kapow2/kapow @@ -0,0 +1,259 @@ +#!/usr/bin/env python +from functools import partial +from urllib.parse import urlparse +from uuid import uuid4 +import asyncio +import io +import logging +import os +import shlex +import sys + +from aiohttp import web, StreamReader + +log = logging.getLogger('kapow') + + +######################################################################## +# Resource Management # +######################################################################## + +CONNECTIONS = {} + + +class ConnectionHandler: + def __init__(self, request): + self._stream = None + self._body = io.BytesIO() + self._status = 200 + self._headers = dict() + self._cookies = dict() + + self.request = request + + async def get(self, key): + res = urlparse(key) + + def nrd(n): + """Return the nrd element in a path.""" + return res.path.split('/')[n] + + if res.path == 'request/method': + return self.request.method.encode('utf-8') + elif res.path == 'request/body': + return self.request.content + elif res.path == 'request/path': + return self.request.path.encode('utf-8') + elif res.path.startswith('request/match/'): + return self.request.match_info[nrd(2)].encode('utf-8') + elif res.path.startswith('request/param/'): + return self.request.context['request'].rel_url.query[nrd(2)].encode('utf-8') + elif res.path.startswith('request/header/'): + return self.request.headers[nrd(2)].encode('utf-8') + elif res.path.startswith('request/cookie/'): + return self.request.cookies[nrd(2)].encode('utf-8') + elif res.path.startswith('request/form/'): + return (await self.request.post())[nrd(2)].encode('utf-8') + elif res.path.startswith('request/file/'): + name = nrd(2) + content = nrd(3) # filename / content + field = (await self.request.post())[name] + if content == 'filename': + try: + return field.filename.encode('utf-8') + except: + return b'' + elif content == 'content': + try: + return field.file.read() + except: + return b'' + else: + raise ValueError(f'Unknown content type {content!r}') + else: + raise ValueError('Unknown path') + + async def set(self, key, content): + res = urlparse(key) + + def nrd(n): + return res.path.split('/')[n] + + if res.path == 'response/status': + self._status = int((await content.read()).decode('utf-8')) + elif res.path == 'response/body': + self._body.write(await content.read()) + elif res.path.startswith('response/header/'): + clean = (await content.read()).rstrip(b'\n').decode('utf-8') + self._headers[nrd(2)] = clean + elif res.path.startswith('response/cookie/'): + clean = (await content.read()).rstrip(b'\n').decode('utf-8') + self._headers[nrd(2)] = clean + elif res.path == 'response/stream': + if self._stream is None: + self._stream = web.StreamResponse(status=self._status, + reason="OK", + headers=self._headers) + for name, value in self._cookies: + self._stream.set_cookie(name, value) + await self._stream.prepare(self.request) + + chunk = await content.readany() + while chunk: + await self._stream.write(chunk) + chunk = await content.readany() + else: + raise ValueError(f'Unknown path {res.path!r}') + + async def append(self, key, value): + raise NotImplementedError() + + async def build_response(self): + if self._stream is None: + response = web.Response(body=self._body.getvalue(), + status=self._status, + headers=self._headers) + for name, value in self._cookies.items(): + response.set_cookie(name, value) + return response + else: + await self._stream.write_eof() + return self._stream + + +async def get_resource(request): + id = request.match_info["id"] + resource = request.match_info["resource"] + maybe_content = await CONNECTIONS[id].get(resource) + if isinstance(maybe_content, StreamReader): + content = maybe_content + response = web.StreamResponse(status=200, reason="OK") + await response.prepare(request) + chunk = await content.readany() + while chunk: + await response.write(chunk) + chunk = await content.readany() + await response.write_eof() + return response + else: + body = maybe_content + return web.Response(body=body) + + +async def set_resource(request): + id = request.match_info["id"] + resource = request.match_info["resource"] + await CONNECTIONS[id].set(resource, request.content) + return web.Response(body=b'') + + +async def append_resource(request): + pass + + +######################################################################## +# Endpoint Execution # +######################################################################## + + +def handle_route(entrypoint, command): + async def _handle(request): + id = "CONN_" + str(uuid4()).replace('-', '_') + connection = CONNECTIONS[id] = ConnectionHandler(request) + + executable, *params = shlex.split(entrypoint) + args = ' '.join([shlex.quote(token) for token in params] + [shlex.quote(command)]) + # Run the source + shell_task = await asyncio.create_subprocess_exec( + args, + executable=executable, + env={**os.environ, + "KAPOW_URL": "http://localhost:8080/kapow", + "KAPOW_CONNECTION": id, + "PATH": ":".join([os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin"), + os.environ["PATH"]]), + }, + shell=False) + await shell_task.wait() + + del CONNECTIONS[id] + + return await connection.build_response() + + return _handle + + +######################################################################## +# Route Management # +######################################################################## + +async def get_routes(request): + return web.json_response(list(request.app.router)) + + +async def create_route(request): + request.app.router._frozen = False + content = await request.json() + print(f'Defined new route {content["method"]} {content["url_pattern"]}') + name = "ROUTE_" + str(uuid4()).replace('-', '_') + request.app.router.add_route(content["method"], + content["url_pattern"], + handle_route(content["entrypoint"], + content["command"]), + name=name) + return web.json_response(name) + + +async def delete_route(request): + id = request.match_info["id"] + route = request.app.router._named_resources.pop(id) + request.app.router._resources.remove(route) + return web.json_response(id) + + +######################################################################## +# aiohttp webapp # +######################################################################## + +async def run_init_script(): + if len(sys.argv) > 1: + # Script given + # with open(sys.argv[1], 'r') as scriptfile: + # script = scriptfile.read() + pass + else: + # script = sys.stdin.read() + raise RuntimeError("Script file is mandatory.") + + shell_task = await asyncio.create_subprocess_shell( + f"/bin/bash --init-file {sys.argv[1]}", + # script, + # executable=os.environ.get('SHELL', '/bin/sh'), + env={**os.environ, + "KAPOW_URL": "http://localhost:8080/kapow", + "PATH": ":".join([os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin"), + os.environ["PATH"]]), + }) + await shell_task.wait() + +async def start_background_tasks(app): + app["debug_tasks"] = app.loop.create_task(run_init_script()) + + + +def main(): + app = web.Application(client_max_size=1024**3) + app.add_routes([ + web.get('/kapow/routes', get_routes), + web.post('/kapow/routes', create_route), + web.delete('/kapow/routes/{id}', delete_route), + web.get('/kapow/connections/{id}/{resource:.*}', get_resource), + # web.post('/kapow/connections/{id}/{resource:.*}', append_resource), + web.put('/kapow/connections/{id}/{resource:.*}', set_resource), + ]) + app.on_startup.append(start_background_tasks) + web.run_app(app) + + +if __name__ == '__main__': + main()