diff --git a/Dockerfile b/Dockerfile index a39043ec0..5b04d143e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ ENV HASHTOPOLIS_IMPORT_PATH=${HASHTOPOLIS_PATH}/import ENV HASHTOPOLIS_LOG_PATH=${HASHTOPOLIS_PATH}/log ENV HASHTOPOLIS_CONFIG_PATH=${HASHTOPOLIS_PATH}/config ENV HASHTOPOLIS_BINARIES_PATH=${HASHTOPOLIS_PATH}/binaries +ENV HASHTOPOLIS_CRACKERS_PATH=${HASHTOPOLIS_PATH}/crackers ENV HASHTOPOLIS_TUS_PATH=/var/tmp/tus ENV HASHTOPOLIS_TEMP_UPLOADS_PATH=${HASHTOPOLIS_TUS_PATH}/uploads ENV HASHTOPOLIS_TEMP_META_PATH=${HASHTOPOLIS_TUS_PATH}/meta @@ -80,6 +81,7 @@ RUN mkdir -p \ ${HASHTOPOLIS_LOG_PATH} \ ${HASHTOPOLIS_CONFIG_PATH} \ ${HASHTOPOLIS_BINARIES_PATH} \ + ${HASHTOPOLIS_CRACKERS_PATH} \ ${HASHTOPOLIS_TUS_PATH} \ ${HASHTOPOLIS_TEMP_UPLOADS_PATH} \ ${HASHTOPOLIS_TEMP_META_PATH} \ diff --git a/ci/apiv2/test_agent_protocol.py b/ci/apiv2/test_agent_protocol.py index 2eadd7def..1131d1711 100644 --- a/ci/apiv2/test_agent_protocol.py +++ b/ci/apiv2/test_agent_protocol.py @@ -16,12 +16,14 @@ import json import re import unittest +import urllib.parse import requests -from hashtopolis import Agent, Config, HealthCheck, Voucher +from hashtopolis import Agent, Config, Cracker, HealthCheck, Voucher from hashtopolis_agent import DummyAgent -from utils import BaseTest, do_create_agentassignent, do_create_dummy_agent, do_create_voucher, get_hashtopolis_uri +from utils import (BaseTest, SEVEN_ZIP_MAGIC, do_create_agentassignent, do_create_dummy_agent, + do_create_voucher, get_hashtopolis_uri) AGENT_ENDPOINT = '/api/server.php' @@ -32,6 +34,19 @@ def _uri(): return get_hashtopolis_uri() +def fetch_via_test_config(url, **kwargs): + """GET a server-generated url through the configured test server uri. + + The realworld dataset configures baseHost with a url that is only reachable + from outside the container, so server-generated absolute urls cannot be + fetched from within the tests. The authority is rewritten to the uri the + tests run against, path and query (which carry the agent token) are kept. + """ + parts = urllib.parse.urlparse(url) + base = urllib.parse.urlparse(_uri()) + return requests.get(base._replace(path=parts.path, query=parts.query).geturl(), **kwargs) + + def agent_request(payload): """POST a raw JSON payload to the agent API and return (status_code, body_text). @@ -533,6 +548,69 @@ def test_download_cracker_invalid_binary_version_id(self): assert_error_envelope(self, body, "downloadBinary") self.assertEqual(parse_envelope(body)['message'], "Invalid cracker binary type id!") + def test_download_cracker_local_binary(self): + """A locally stored cracker binary is served by the server itself: the + downloadBinary action returns the url of the download endpoint with the + requesting agent's token appended, so the archive can directly be fetched.""" + dummy = self._dummy() + content = SEVEN_ZIP_MAGIC + b'local-binary-download-test' + cracker = self.create_local_cracker(content=content, extra_payload={'version': '7.2.7'}) + + code, body = agent_request({ + "action": "downloadBinary", + "token": dummy.token, + "type": "cracker", + "binaryVersionId": cracker.id, + }) + self.assertEqual(code, 200) + resp = parse_envelope(body) + self.assertEqual(resp['response'], "SUCCESS") + url = resp['url'] + url_parts = urllib.parse.urlparse(url) + self.assertEqual(f'/api/download.php/crackerBinary/{cracker.id}', url_parts.path) + self.assertIn(f'token={dummy.token}', url_parts.query) + + # the agent can fetch the archive with the returned url + r = fetch_via_test_config(url) + self.assertEqual(200, r.status_code) + self.assertEqual(content, r.content) + + def test_download_cracker_local_binary_wrong_token_denied(self): + """The download url of a local binary only works with the agent token it + was issued for.""" + dummy = self._dummy() + cracker = self.create_local_cracker() + + code, body = agent_request({ + "action": "downloadBinary", + "token": dummy.token, + "type": "cracker", + "binaryVersionId": cracker.id, + }) + url = parse_envelope(body)['url'] + + r = fetch_via_test_config(url.replace(f'token={dummy.token}', 'token=wrong-token')) + self.assertEqual(401, r.status_code) + + def test_download_cracker_local_binary_external_unchanged(self): + """Cracker binaries referenced with an external url are answered with the + stored url, no token is appended.""" + dummy = self._dummy() + external_binaries = [c for c in Cracker.objects.filter() if not c.filename] + self.assertTrue(external_binaries, 'no externally referenced cracker binary found') + cracker = external_binaries[0] + + code, body = agent_request({ + "action": "downloadBinary", + "token": dummy.token, + "type": "cracker", + "binaryVersionId": cracker.id, + }) + resp = parse_envelope(body) + self.assertEqual(resp['response'], "SUCCESS") + self.assertEqual(cracker.downloadUrl, resp['url']) + self.assertNotIn('token=', resp['url']) + # --------------------------------------------------------------------------- # clientError diff --git a/ci/apiv2/test_cracker.py b/ci/apiv2/test_cracker.py index 91fe9e2c7..4050f6e4e 100644 --- a/ci/apiv2/test_cracker.py +++ b/ci/apiv2/test_cracker.py @@ -1,5 +1,33 @@ -from hashtopolis import Cracker -from utils import BaseTest +import datetime +import glob +import io +import os +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer + +import requests + +from hashtopolis import Cracker, CrackerType, FileImport, HashtopolisError +from utils import (BaseTest, SEVEN_ZIP_MAGIC, do_create_agent, do_create_local_cracker, + get_bearer_token, get_hashtopolis_uri) + + +CRACKERS_DIR = os.environ.get('HASHTOPOLIS_CRACKERS_PATH', '/usr/local/share/hashtopolis/crackers') +IMPORT_DIR = os.environ.get('HASHTOPOLIS_IMPORT_PATH', '/usr/local/share/hashtopolis/import') +APIV2 = get_hashtopolis_uri() + '/api/v2' + + +def archive_path(obj): + """Absolute path of the locally stored archive of a cracker binary.""" + return os.path.join(CRACKERS_DIR, f'{obj.id}_{obj.filename}') + + +def url_copy_path(obj): + """Absolute path of the local copy of a url-referenced cracker binary, + composed by the server from the cracker type and the version.""" + cracker_type = CrackerType.objects.get(pk=obj.crackerBinaryTypeId) + return os.path.join(CRACKERS_DIR, f'{obj.id}_{cracker_type.typeName}-{obj.version}.7z') class CrackerTest(BaseTest): @@ -27,3 +55,475 @@ def test_expandables(self): model_obj = self.create_test_object() expandables = ['crackerBinaryType'] self._test_expandables(model_obj, expandables) + + +class TestCrackerUpload(BaseTest): + """Cracker binaries created by uploading a 7z archive instead of providing a url.""" + + def _assert_local_binary(self, obj, content, version): + # the archive filename is composed from the cracker type, the version and the extension + cracker_type = CrackerType.objects.get(pk=obj.crackerBinaryTypeId) + self.assertEqual(f'{cracker_type.typeName}-{version}.7z', obj.filename) + # the download url was generated automatically and points to the download endpoint + self.assertTrue(obj.downloadUrl.startswith('http')) + self.assertTrue(obj.downloadUrl.endswith(f'/api/download.php/crackerBinary/{obj.id}')) + # the archive is stored in the crackers directory + self.assertTrue(os.path.isfile(archive_path(obj)), 'archive is not stored in the crackers directory') + with open(archive_path(obj), 'rb') as f: + self.assertEqual(content, f.read()) + + def test_create_with_inline_source(self): + content = SEVEN_ZIP_MAGIC + b'inline-archive-content' + obj = self.create_local_cracker(content=content, extra_payload={'version': '7.2.7'}) + self._assert_local_binary(obj, content, '7.2.7') + + def test_create_with_import_source_chunked_upload(self): + """Arbitrary-size archives are chunk uploaded to the import directory + (the TUS protocol implementation) and then imported on creation.""" + content = SEVEN_ZIP_MAGIC + b'chunked-upload-archive-content' + import_name = f'cracker-upload-{datetime.datetime.now().isoformat()}.7z' + FileImport().do_upload(import_name, io.BytesIO(content)) + # the completed upload landed in the import directory + self.assertTrue(os.path.isfile(os.path.join(IMPORT_DIR, import_name))) + + obj = self.create_local_cracker(source_type='import', source_data=import_name, + extra_payload={'version': '7.2.7'}) + self._assert_local_binary(obj, content, '7.2.7') + # the archive was moved out of the import directory + self.assertFalse(os.path.isfile(os.path.join(IMPORT_DIR, import_name))) + + def test_create_with_url_source(self): + """The server fetches the archive itself from a http url.""" + content = SEVEN_ZIP_MAGIC + b'url-archive-content' + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', str(len(content))) + self.end_headers() + self.wfile.write(content) + + def log_message(self, format, *args): + pass + + server = HTTPServer(('127.0.0.1', 0), Handler) + threading.Thread(target=server.serve_forever).start() + try: + obj = self.create_local_cracker( + source_type='url', + source_data=f'http://127.0.0.1:{server.server_address[1]}/archive.7z', + extra_payload={'version': '7.2.7'}) + self._assert_local_binary(obj, content, '7.2.7') + finally: + server.shutdown() + + def test_create_invalid_archive_rejects(self): + """An archive which is not a 7z file is rejected, the import file is + restored and no binary is created.""" + version = f'9.9.9-{int(time.time())}' + import_name = f'cracker-invalid-{datetime.datetime.now().isoformat()}.txt' + FileImport().do_upload(import_name, io.BytesIO(b'not-a-7z-archive')) + + with self.assertRaises(HashtopolisError) as e: + do_create_local_cracker(source_type='import', source_data=import_name, + extra_payload={'version': version}) + self.assertEqual(400, e.exception.status_code) + + # the import file was put back to the import directory + self.assertTrue(os.path.isfile(os.path.join(IMPORT_DIR, import_name))) + # no binary was created + self.assertEqual([], list(Cracker.objects.filter(version=version))) + os.unlink(os.path.join(IMPORT_DIR, import_name)) + + def test_create_with_both_sources_rejects(self): + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker( + extra_payload={'downloadUrl': 'https://example.org/files/cracker.7z'}, + delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_create_with_missing_source_data_rejects(self): + obj = Cracker(crackerBinaryTypeId=1, version='7.2.7', binaryName='cracker', + sourceType='inline') + with self.assertRaises(HashtopolisError) as e: + obj.save() + self.assertEqual(400, e.exception.status_code) + + def test_create_with_bogus_source_type_rejects(self): + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker(source_type='bogus', source_data='data', delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_create_with_invalid_base64_rejects(self): + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker(source_data='!!!no-base64!!!', delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_create_with_missing_import_file_rejects(self): + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker(source_type='import', source_data='does-not-exist.7z', + delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_create_with_invalid_url_scheme_rejects(self): + """Only http and https urls can be fetched, no local files or stream wrappers.""" + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker(source_type='url', source_data='file:///etc/passwd', + delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_patch_download_url_of_local_binary_rejects(self): + """The download url of a locally stored binary is owned by the server.""" + obj = self.create_local_cracker() + obj.downloadUrl = 'https://evil.example.org/cracker.7z' + with self.assertRaises(HashtopolisError) as e: + obj.save() + self.assertEqual(400, e.exception.status_code) + + # the url was not changed, but other attributes can still be patched + reloaded = Cracker.objects.get(pk=obj.id) + self.assertTrue(reloaded.downloadUrl.endswith(f'/api/download.php/crackerBinary/{obj.id}')) + reloaded.version = '8.0.0' + reloaded.save() + self.assertEqual('8.0.0', Cracker.objects.get(pk=obj.id).version) + + def test_patch_source_type_rejects(self): + """sourceType is only valid at creation, patching it is forbidden.""" + obj = self.create_local_cracker() + headers = {'Authorization': f'Bearer {get_bearer_token()}', + 'Content-Type': 'application/json'} + r = requests.patch( + f'{APIV2}/ui/crackers/{obj.id}', + headers=headers, + json={'data': {'type': 'crackerBinary', 'id': str(obj.id), + 'attributes': {'sourceType': 'url'}}}) + self.assertEqual(403, r.status_code) + + def test_delete_removes_archive(self): + obj = self.create_local_cracker(delete=False) + self.assertTrue(os.path.isfile(archive_path(obj))) + + obj.delete() + + self.assertFalse(os.path.isfile(archive_path(obj))) + # with valid authentication there is no such archive anymore + r = requests.get(f'{get_hashtopolis_uri()}/api/download.php/crackerBinary/{obj.id}', + headers={'Authorization': f'Bearer {get_bearer_token()}'}) + self.assertEqual(404, r.status_code) + + +class TestCrackerUrlCopy(BaseTest): + """Cracker binaries added with a download url: the server downloads a + local copy of the archive, so it has it for later analysis. The agents + still download the archive from the external url. If the download fails, + the create is rejected and nothing is added.""" + + def _serve_archive(self, content, status=200, content_length=None): + """Start a local http server answering every request with the same + response, returns the server and the url it is reachable at.""" + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(status) + if content is not None: + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', + str(len(content) if content_length is None else content_length)) + self.end_headers() + self.wfile.write(content) + else: + self.end_headers() + + def log_message(self, format, *args): + pass + + server = HTTPServer(('127.0.0.1', 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, f'http://127.0.0.1:{server.server_address[1]}/archive.7z' + + def _assert_rejected(self, url, expected_message, version): + """A create with a broken download url reports the failure on the + create and leaves no binary and no local copy of the archive behind.""" + with self.assertRaises(HashtopolisError) as e: + self.create_cracker(extra_payload={'downloadUrl': url, 'version': version}) + self.assertEqual(400, e.exception.status_code) + self.assertIn(expected_message, e.exception.title) + self.assertEqual([], list(Cracker.objects.filter(version=version))) + self.assertEqual([], glob.glob(os.path.join(CRACKERS_DIR, f'*{version}*'))) + + def test_create_with_download_url_stores_local_copy(self): + """The server downloads a local copy of the archive from the download + url, the url itself stays the external reference of the binary.""" + content = SEVEN_ZIP_MAGIC + b'url-copy-archive-content' + server, url = self._serve_archive(content) + try: + version = 'url-copy' + obj = self.create_cracker(extra_payload={'downloadUrl': url, 'version': version}, delete=False) + + # the binary still references the external url, it is not marked as locally stored + self.assertEqual(url, obj.downloadUrl) + self.assertIsNone(obj.filename) + + # the server downloaded a local copy of the archive + path = url_copy_path(obj) + self.assertTrue(os.path.isfile(path), 'local copy of the archive is missing') + with open(path, 'rb') as f: + self.assertEqual(content, f.read()) + + # deleting the binary removes the local copy + obj.delete() + self.assertFalse(os.path.isfile(path)) + finally: + server.shutdown() + + def test_create_with_unreachable_url_rejects(self): + """Nothing is listening on the url, the connection is refused and + the create is rejected.""" + self._assert_rejected('http://127.0.0.1:1/archive.7z', + 'Failed to download the archive from the download url', + f'unreach-{int(time.time())}') + + def test_create_with_not_found_url_rejects(self): + """The url answers with an http error status, the download fails and + the create is rejected.""" + server, url = self._serve_archive(None, status=404) + try: + self._assert_rejected(url, 'Failed to download the archive from the download url', + f'notfound-{int(time.time())}') + finally: + server.shutdown() + + def test_create_with_non_7z_archive_url_rejects(self): + """The url serves something else than a 7z archive, the create is + rejected and nothing is added.""" + server, url = self._serve_archive(b'not-a-7z-archive') + try: + self._assert_rejected(url, 'The archive at the download url is not a valid 7z archive!', + f'non7z-{int(time.time())}') + finally: + server.shutdown() + + def test_create_with_truncated_download_rejects(self): + """The url closes the connection before the announced content is + completely sent, the incomplete download is rejected.""" + server, url = self._serve_archive(SEVEN_ZIP_MAGIC + b'only-half-of-the-archive', + content_length=4096) + try: + self._assert_rejected(url, 'Download incomplete', + f'trunc-{int(time.time())}') + finally: + server.shutdown() + + def test_create_with_invalid_url_scheme_rejects(self): + """Only http and https urls are fetched by the server, no local files + or stream wrappers.""" + self._assert_rejected('file:///etc/passwd', + 'Only http and https download urls are supported!', + f'scheme-{int(time.time())}') + + def _assert_patch_rejected(self, obj, new_url, expected_message): + """Patching the download url to a broken url reports the failure on + the patch, the update is rolled back and the previously stored local + copy is left untouched.""" + old_url = obj.downloadUrl + path = url_copy_path(obj) + with open(path, 'rb') as f: + content_before = f.read() + + with self.assertRaises(HashtopolisError) as e: + obj.downloadUrl = new_url + obj.save() + self.assertEqual(400, e.exception.status_code) + self.assertIn(expected_message, e.exception.title) + + # the update was rolled back, the local copy is untouched + reloaded = Cracker.objects.get(pk=obj.id) + self.assertEqual(old_url, reloaded.downloadUrl) + with open(path, 'rb') as f: + self.assertEqual(content_before, f.read()) + return reloaded + + def test_patch_download_url_refreshes_local_copy(self): + """Changing the download url re-downloads the local copy of the + archive from the new url, the binary itself keeps referencing the + new external url.""" + content_a = SEVEN_ZIP_MAGIC + b'archive-from-url-a' + server_a, url_a = self._serve_archive(content_a) + content_b = SEVEN_ZIP_MAGIC + b'archive-from-url-b' + server_b, url_b = self._serve_archive(content_b) + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url_a, 'version': 'url-patch'}, + delete=False) + path = url_copy_path(obj) + with open(path, 'rb') as f: + self.assertEqual(content_a, f.read()) + + obj.downloadUrl = url_b + obj.save() + + reloaded = Cracker.objects.get(pk=obj.id) + self.assertEqual(url_b, reloaded.downloadUrl) + self.assertIsNone(reloaded.filename) + # the local copy was replaced with the archive from the new url + with open(path, 'rb') as f: + self.assertEqual(content_b, f.read()) + + obj.delete() + self.assertFalse(os.path.isfile(path)) + finally: + server_a.shutdown() + server_b.shutdown() + + def test_patch_download_url_with_version_stores_new_copy(self): + """Changing the download url together with the version stores the + refreshed local copy under the new archive filename and removes the + copy of the previous version.""" + content_a = SEVEN_ZIP_MAGIC + b'archive-old-version' + server_a, url_a = self._serve_archive(content_a) + content_b = SEVEN_ZIP_MAGIC + b'archive-new-version' + server_b, url_b = self._serve_archive(content_b) + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url_a, 'version': 'p-old'}, + delete=False) + cracker_type = CrackerType.objects.get(pk=obj.crackerBinaryTypeId) + old_path = os.path.join(CRACKERS_DIR, f'{obj.id}_{cracker_type.typeName}-p-old.7z') + new_path = os.path.join(CRACKERS_DIR, f'{obj.id}_{cracker_type.typeName}-p-new.7z') + self.assertTrue(os.path.isfile(old_path)) + + obj.downloadUrl = url_b + obj.version = 'p-new' + obj.save() + + reloaded = Cracker.objects.get(pk=obj.id) + self.assertEqual('p-new', reloaded.version) + self.assertEqual(url_b, reloaded.downloadUrl) + with open(new_path, 'rb') as f: + self.assertEqual(content_b, f.read()) + self.assertFalse(os.path.isfile(old_path)) + + obj.delete() + self.assertFalse(os.path.isfile(new_path)) + finally: + server_a.shutdown() + server_b.shutdown() + + def test_patch_download_url_to_unreachable_url_rejects(self): + """Changing the download url to an url the server cannot download + from rejects the patch, the update is rolled back.""" + server, url = self._serve_archive(SEVEN_ZIP_MAGIC + b'url-patch-unreachable') + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url}, delete=False) + self._assert_patch_rejected(obj, 'http://127.0.0.1:1/archive.7z', + 'Failed to download the archive from the download url') + obj.delete() + finally: + server.shutdown() + + def test_patch_download_url_to_non_7z_archive_rejects(self): + """Changing the download url to an url which does not serve a 7z + archive rejects the patch and rolls the update back.""" + server, url = self._serve_archive(SEVEN_ZIP_MAGIC + b'url-patch-non-7z') + bad_server, bad_url = self._serve_archive(b'not-a-7z-archive') + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url}, delete=False) + self._assert_patch_rejected(obj, bad_url, + 'The archive at the download url is not a valid 7z archive!') + obj.delete() + finally: + server.shutdown() + bad_server.shutdown() + + def test_patch_download_url_invalid_scheme_rejects(self): + """The download url can only be changed to an http or https url.""" + server, url = self._serve_archive(SEVEN_ZIP_MAGIC + b'url-patch-scheme') + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url}, delete=False) + self._assert_patch_rejected(obj, 'file:///etc/passwd', + 'Only http and https download urls are supported!') + obj.delete() + finally: + server.shutdown() + + def test_patch_version_keeps_local_copy(self): + """The local copy tracks the download url: patching other attributes + like the version does not re-download it.""" + content = SEVEN_ZIP_MAGIC + b'url-copy-stays-on-version-patch' + server, url = self._serve_archive(content) + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url, 'version': 'keep-1'}, + delete=False) + cracker_type = CrackerType.objects.get(pk=obj.crackerBinaryTypeId) + path = os.path.join(CRACKERS_DIR, f'{obj.id}_{cracker_type.typeName}-keep-1.7z') + + obj.version = 'keep-2' + obj.save() + + reloaded = Cracker.objects.get(pk=obj.id) + self.assertEqual('keep-2', reloaded.version) + # the local copy is untouched, still under the filename of its version + self.assertTrue(os.path.isfile(path)) + with open(path, 'rb') as f: + self.assertEqual(content, f.read()) + + obj.delete() + self.assertFalse(os.path.isfile(path)) + finally: + server.shutdown() + + +class TestDownloadEndpoint(BaseTest): + """The download endpoint serving the locally stored cracker binary archives.""" + + def _download(self, obj, headers=None, params=None, kind='crackerBinary'): + return requests.get(f'{get_hashtopolis_uri()}/api/download.php/{kind}/{obj.id}', + headers=headers, params=params) + + def test_download_without_auth_rejected(self): + obj = self.create_local_cracker() + self.assertEqual(401, self._download(obj).status_code) + + def test_download_with_invalid_agent_token_rejected(self): + obj = self.create_local_cracker() + self.assertEqual(401, self._download(obj, params={'token': 'invalid-token'}).status_code) + + def test_download_with_invalid_bearer_rejected(self): + obj = self.create_local_cracker() + self.assertEqual(401, + self._download(obj, headers={'Authorization': 'Bearer invalid.jwt.token'}).status_code) + + def test_download_with_agent_token(self): + agent = do_create_agent() + self.delete_after_test(agent) + content = SEVEN_ZIP_MAGIC + b'download-endpoint-content' + obj = self.create_local_cracker(content=content) + + r = self._download(obj, params={'token': agent.token}) + self.assertEqual(200, r.status_code) + self.assertEqual(content, r.content) + self.assertEqual('application/x-7z-compressed', r.headers['Content-Type']) + self.assertEqual(f'attachment; filename="{obj.filename}"', r.headers['Content-Disposition']) + + def test_download_with_bearer_token(self): + content = SEVEN_ZIP_MAGIC + b'download-endpoint-content' + obj = self.create_local_cracker(content=content) + + r = self._download(obj, headers={'Authorization': f'Bearer {get_bearer_token()}'}) + self.assertEqual(200, r.status_code) + self.assertEqual(content, r.content) + + def test_download_range_request(self): + content = SEVEN_ZIP_MAGIC + b'download-endpoint-content' + obj = self.create_local_cracker(content=content) + + r = self._download(obj, headers={'Authorization': f'Bearer {get_bearer_token()}', + 'Range': 'bytes=0-5'}) + self.assertEqual(206, r.status_code) + self.assertEqual(content[:6], r.content) + self.assertEqual(f'bytes 0-5/{len(content)}', r.headers['Content-Range']) + + def test_download_unknown_kind_rejected(self): + agent = do_create_agent() + self.delete_after_test(agent) + obj = self.create_local_cracker() + r = self._download(obj, params={'token': agent.token}, kind='unknown') + self.assertEqual(404, r.status_code) diff --git a/ci/apiv2/test_taskwrapper.py b/ci/apiv2/test_taskwrapper.py index 5615c391c..653eb0829 100644 --- a/ci/apiv2/test_taskwrapper.py +++ b/ci/apiv2/test_taskwrapper.py @@ -1,6 +1,6 @@ from hashtopolis import Helper, HashtopolisError, TaskWrapper from hashtopolis import Cracker -from utils import BaseTest +from utils import BaseTest, get_cracker_archive_url class TaskWrapperTest(BaseTest): @@ -59,7 +59,7 @@ def test_helper_create_supertask_generic_cracker(self): cracker = Cracker( crackerBinaryTypeId=crackertype.id, version='1.2.3', - downloadUrl='https://example.org/generic-1.2.3.gz', + downloadUrl=get_cracker_archive_url(), binaryName='generic-x64') cracker.save() self.delete_after_test(cracker) diff --git a/ci/apiv2/utils.py b/ci/apiv2/utils.py index 243ebcd2a..b14e28b4e 100644 --- a/ci/apiv2/utils.py +++ b/ci/apiv2/utils.py @@ -1,10 +1,13 @@ import abc +import base64 import datetime +from http.server import BaseHTTPRequestHandler, HTTPServer from io import BytesIO import json from pathlib import Path import requests import tempfile +import threading import time import unittest import zipfile @@ -51,6 +54,54 @@ def get_hashtopolis_uri(): return get_test_config()['hashtopolis_uri'] +def get_bearer_token(): + """Request an apiv2 JWT with the test credentials, for raw requests.""" + cfg = get_test_config() + r = requests.post(f"{cfg['hashtopolis_uri']}/api/v2/auth/token", + auth=(cfg['username'], cfg['password'])) + return json.loads(r.text)['token'] + + +# magic bytes every valid 7z archive starts with +SEVEN_ZIP_MAGIC = b'\x37\x7A\xBC\xAF\x27\x1C' + +# archive content served by the shared test server for download-url cracker +# creations, the server downloads a local copy of it on creation +CRACKER_URL_ARCHIVE_CONTENT = SEVEN_ZIP_MAGIC + b'cracker-download-url-archive' + + +class _CrackerArchiveHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type', 'application/x-7z-compressed') + self.send_header('Content-Length', str(len(CRACKER_URL_ARCHIVE_CONTENT))) + self.end_headers() + self.wfile.write(CRACKER_URL_ARCHIVE_CONTENT) + + def log_message(self, format, *args): + pass + + +_cracker_archive_server = None + + +def get_cracker_archive_url(): + """Url of a shared local http server which serves a valid 7z archive. + + Cracker binaries created with a download url need a url which really + serves an archive, the server downloads a local copy from it on creation. + The server is started lazily and kept alive for the whole test run, the + hashtopolis server can reach it because the tests run in the same + container/network namespace. + """ + global _cracker_archive_server + if _cracker_archive_server is None: + server = HTTPServer(('127.0.0.1', 0), _CrackerArchiveHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + _cracker_archive_server = server + return f'http://127.0.0.1:{_cracker_archive_server.server_address[1]}/cracker.7z' + + def _do_create_obj_from_file(model_class, file_prefix, extra_payload={}, **kwargs): file_id = kwargs.get('file_id') or '001' p = Path(__file__).parent.joinpath(f'testfiles/{model_class.__name__.lower()}/{file_prefix}_{file_id}.json') @@ -172,7 +223,33 @@ def do_create_accessgroup(**kwargs): def do_create_cracker(**kwargs): - return _do_create_obj_from_file(Cracker, 'create_cracker', **kwargs) + extra_payload = dict(kwargs.pop('extra_payload', None) or {}) + # the server downloads a local copy of the archive on creation, so the + # download url has to serve a real archive unless the test provides its own + extra_payload.setdefault('downloadUrl', get_cracker_archive_url()) + return _do_create_obj_from_file(Cracker, 'create_cracker', extra_payload, **kwargs) + + +def do_create_local_cracker(source_type='inline', source_data=None, + content=SEVEN_ZIP_MAGIC + b'local-cracker-test', + extra_payload={}, **kwargs): + """Create a cracker binary by uploading an archive instead of providing a url. + + source_data defaults to the base64 encoded content for source_type 'inline'. + For 'import' it is the filename of a file in the import directory (e.g. one + uploaded with FileImport.do_upload), for 'url' a http/https url. + """ + if source_data is None: + source_data = base64.b64encode(content).decode() + payload = json.loads( + Path(__file__).parent.joinpath('testfiles/cracker/create_cracker_001.json').read_text('UTF-8')) + # the download url is generated by the server when the archive is uploaded + del payload['downloadUrl'] + final_payload = {**payload, **extra_payload, + 'sourceType': source_type, 'sourceData': source_data} + obj = Cracker(**final_payload) + obj.save() + return obj def do_create_crackertype(**kwargs): @@ -386,6 +463,9 @@ def create_agentassignment(self, **kwargs): def create_cracker(self, **kwargs): return self._create_test_object(do_create_cracker, **kwargs) + def create_local_cracker(self, **kwargs): + return self._create_test_object(do_create_local_cracker, **kwargs) + def create_crackertype(self, **kwargs): return self._create_test_object(do_create_crackertype, **kwargs) diff --git a/ci/phpunit/TestBase.php b/ci/phpunit/TestBase.php index 1669b008b..4340a5f5a 100644 --- a/ci/phpunit/TestBase.php +++ b/ci/phpunit/TestBase.php @@ -213,7 +213,7 @@ protected function createCrackerBinaryType(): CrackerBinaryType { protected function createCrackerBinary(CrackerBinaryType $crackerBinaryType): CrackerBinary { $crackerBinary = $this->createDatabaseObject( Factory::getCrackerBinaryFactory(), - new CrackerBinary(null, $crackerBinaryType->getId(), '1.0.' . uniqid(), 'https://example.invalid/' . uniqid(), 'binary_' . uniqid()) + new CrackerBinary(null, $crackerBinaryType->getId(), '1.0.' . uniqid(), 'https://example.invalid/' . uniqid(), 'binary_' . uniqid(), null) ); $this->assertTrue($crackerBinary instanceof CrackerBinary); return $crackerBinary; diff --git a/ci/phpunit/dba/AbstractModelFactoryTest.php b/ci/phpunit/dba/AbstractModelFactoryTest.php index 1b421fc2f..18adb7feb 100644 --- a/ci/phpunit/dba/AbstractModelFactoryTest.php +++ b/ci/phpunit/dba/AbstractModelFactoryTest.php @@ -1681,7 +1681,7 @@ private function setUpHealthCheck(): array { $crackerBinaryType = new CrackerBinaryType(null, '', 0); $crackerBinaryType = $this->createDatabaseObject(Factory::getCrackerBinaryTypeFactory(), $crackerBinaryType); - $crackerBinary = new CrackerBinary(null, $crackerBinaryType->getId(), '', '', ''); + $crackerBinary = new CrackerBinary(null, $crackerBinaryType->getId(), '', '', '', null); $crackerBinary = $this->createDatabaseObject(Factory::getCrackerBinaryFactory(), $crackerBinary); $healthCheck = new HealthCheck(null, 0, 0, 0, $hashType->getId(), $crackerBinary->getId(), 0, ''); diff --git a/ci/phpunit/dba/MassUpdateSetTest.php b/ci/phpunit/dba/MassUpdateSetTest.php index 66b7e92f8..5bbb199ca 100644 --- a/ci/phpunit/dba/MassUpdateSetTest.php +++ b/ci/phpunit/dba/MassUpdateSetTest.php @@ -159,7 +159,7 @@ public function testMassSingleUpdateWithMappedColumn(): void { $agent = $this->createDatabaseObject(Factory::getAgentFactory(), new Agent(null, '', '', 0, '', '', 0, 0, 0, '', '', 0, '', null, 0, '')); $hashType = $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, $prefix . '_ht', 0, 0)); $cbt = $this->createDatabaseObject(Factory::getCrackerBinaryTypeFactory(), new CrackerBinaryType(null, '', 0)); - $cb = $this->createDatabaseObject(Factory::getCrackerBinaryFactory(), new CrackerBinary(null, $cbt->getId(), '', '', '')); + $cb = $this->createDatabaseObject(Factory::getCrackerBinaryFactory(), new CrackerBinary(null, $cbt->getId(), '', '', '', null)); $healthCheck = $this->createDatabaseObject(Factory::getHealthCheckFactory(), new HealthCheck(null, 0, 0, 0, $hashType->getId(), $cb->getId(), 0, '')); $hca1 = $this->createDatabaseObject(Factory::getHealthCheckAgentFactory(), new HealthCheckAgent(null, $healthCheck->getId(), $agent->getId(), 0, 0, 0, 0, 100, '')); diff --git a/ci/phpunit/downloadapi/DownloadAppTest.php b/ci/phpunit/downloadapi/DownloadAppTest.php new file mode 100644 index 000000000..357685285 --- /dev/null +++ b/ci/phpunit/downloadapi/DownloadAppTest.php @@ -0,0 +1,180 @@ +type = $this->createDatabaseObject( + Factory::getCrackerBinaryTypeFactory(), + new CrackerBinaryType(null, 'download-test-type', 1) + ); + $this->externalBinary = $this->createDatabaseObject( + Factory::getCrackerBinaryFactory(), + new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://example.com/hc.7z', 'testcracker', null) + ); + + // create a locally stored binary through the import source + $this->agentToken = 'dl-test-' . uniqid(); + $this->createDatabaseObject( + Factory::getAgentFactory(), + new Agent(null, 'download-test-agent', '', 0, '', '', 0, 0, 0, $this->agentToken, '', 0, '', null, 0, '') + ); + + $importName = 'download-test-' . uniqid() . '.7z'; + $this->archiveContent = self::SEVEN_ZIP_MAGIC . 'download-test-content'; + file_put_contents(self::getImportPath() . $importName, $this->archiveContent); + $this->localBinary = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $importName); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $this->localBinary); + + if (isset($_SERVER['HTTP_RANGE'])) { + $this->savedHttpRange = true; + $this->savedHttpRangeValue = $_SERVER['HTTP_RANGE']; + } + } + + #[Override] + protected function tearDown(): void { + // remove the archive in case a test failed before it could clean up + $archive = CrackerUtils::getCrackersPath() . $this->localBinary->getId() . '_' . $this->localBinary->getFilename(); + if (file_exists($archive)) { + unlink($archive); + } + if ($this->savedHttpRange) { + $_SERVER['HTTP_RANGE'] = $this->savedHttpRangeValue; + } + else { + unset($_SERVER['HTTP_RANGE']); + } + parent::tearDown(); + } + + private static function getImportPath(): string { + return Factory::getStoredValueFactory()->get(DDirectories::IMPORT)->getVal() . '/'; + } + + private function runDownloadRequest(string $uriWithQuery, array $headers = []): ResponseInterface { + $request = (new ServerRequestFactory())->createServerRequest('GET', $uriWithQuery); + foreach ($headers as $name => $value) { + $request = $request->withHeader($name, $value); + } + return DownloadApp::create()->handle($request); + } + + private function localBinaryUri(): string { + return '/api/download.php/crackerBinary/' . $this->localBinary->getId(); + } + + // A request without any authentication is rejected with 401. + public function testNoAuthenticationIsRejected(): void { + $response = $this->runDownloadRequest($this->localBinaryUri()); + $this->assertEquals(401, $response->getStatusCode()); + $this->assertEquals('No access!', (string)$response->getBody()); + } + + // A request with an invalid agent token is rejected with 401. + public function testInvalidAgentTokenIsRejected(): void { + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=invalid-token'); + $this->assertEquals(401, $response->getStatusCode()); + $this->assertEquals('No access!', (string)$response->getBody()); + } + + // A request with an invalid JWT is rejected with 401. + public function testInvalidBearerTokenIsRejected(): void { + $response = $this->runDownloadRequest($this->localBinaryUri(), ['Authorization' => 'Bearer invalid.jwt.value']); + $this->assertEquals(401, $response->getStatusCode()); + } + + // A valid agent token allows to download the archive of a locally stored + // binary, including the download headers. + public function testAgentTokenCanDownloadArchive(): void { + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken); + $this->assertEquals(200, $response->getStatusCode()); + $this->assertEquals($this->archiveContent, (string)$response->getBody()); + $this->assertEquals('application/x-7z-compressed', $response->getHeaderLine('Content-Type')); + $this->assertEquals( + 'attachment; filename="' . $this->localBinary->getFilename() . '"', + $response->getHeaderLine('Content-Disposition') + ); + $this->assertEquals(strlen($this->archiveContent), (int)$response->getHeaderLine('Content-Length')); + } + + // An unknown download kind is rejected with 404. + public function testUnknownKindIsRejected(): void { + $response = $this->runDownloadRequest('/api/download.php/unknown/' . $this->localBinary->getId() . '?token=' . $this->agentToken); + $this->assertEquals(404, $response->getStatusCode()); + $this->assertEquals('Unknown download kind!', (string)$response->getBody()); + } + + // A non existing binary id is rejected with 404. + public function testUnknownBinaryIdIsRejected(): void { + $response = $this->runDownloadRequest('/api/download.php/crackerBinary/99999999?token=' . $this->agentToken); + $this->assertEquals(404, $response->getStatusCode()); + } + + // Binaries which are not locally stored have no archive to download. + public function testExternalBinaryHasNoArchive(): void { + $response = $this->runDownloadRequest('/api/download.php/crackerBinary/' . $this->externalBinary->getId() . '?token=' . $this->agentToken); + $this->assertEquals(404, $response->getStatusCode()); + $this->assertEquals('No such cracker binary archive!', (string)$response->getBody()); + } + + // When the archive is not present on the server anymore, the download + // results in 404. + public function testMissingArchiveFileIsRejected(): void { + $archive = CrackerUtils::getCrackersPath() . $this->localBinary->getId() . '_' . $this->localBinary->getFilename(); + unlink($archive); + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken); + $this->assertEquals(404, $response->getStatusCode()); + $this->assertEquals('The archive of this cracker binary is not present on the server!', (string)$response->getBody()); + } + + // A range request is answered with partial content. + public function testRangeRequestReturnsPartialContent(): void { + $_SERVER['HTTP_RANGE'] = 'bytes=0-5'; + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken); + $this->assertEquals(206, $response->getStatusCode()); + $this->assertEquals(substr($this->archiveContent, 0, 6), (string)$response->getBody()); + $this->assertEquals('bytes 0-5/' . strlen($this->archiveContent), $response->getHeaderLine('Content-Range')); + } + + // A request with a matching ETag is answered with not modified. + public function testMatchingEtagReturnsNotModified(): void { + $archive = CrackerUtils::getCrackersPath() . $this->localBinary->getId() . '_' . $this->localBinary->getFilename(); + $etag = md5(filemtime($archive) . strlen($this->archiveContent)); + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken, ['If-None-Match' => $etag]); + $this->assertEquals(304, $response->getStatusCode()); + } +} diff --git a/ci/phpunit/fixtures/openapi/crackerbinarytype.spec.json b/ci/phpunit/fixtures/openapi/crackerbinarytype.spec.json index 4188aebcf..d4fe75e31 100644 --- a/ci/phpunit/fixtures/openapi/crackerbinarytype.spec.json +++ b/ci/phpunit/fixtures/openapi/crackerbinarytype.spec.json @@ -1698,7 +1698,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -1708,10 +1709,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -2068,7 +2080,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -2078,10 +2091,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -2438,7 +2462,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -2448,10 +2473,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -2860,7 +2896,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -2870,10 +2907,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } diff --git a/ci/phpunit/inc/UtilTest.php b/ci/phpunit/inc/UtilTest.php index 0a33763a5..97dd3d118 100644 --- a/ci/phpunit/inc/UtilTest.php +++ b/ci/phpunit/inc/UtilTest.php @@ -3,6 +3,7 @@ namespace Hashtopolis\inc; use Exception; +use Override; use Hashtopolis\dba\Factory; use Hashtopolis\dba\models\StoredValue; use Hashtopolis\TestBase; @@ -16,10 +17,44 @@ use Hashtopolis\dba\models\Preprocessor; use Hashtopolis\dba\models\RightGroup; use Hashtopolis\dba\models\HashType; +use Hashtopolis\inc\defines\DConfig; require_once(dirname(__FILE__) . '/../TestBase.php'); final class UtilTest extends TestBase { + private string|false $savedBackendUrl = false; + private array $savedServer = []; + + #[Override] + protected function setUp(): void { + parent::setUp(); + $this->savedBackendUrl = getenv('HASHTOPOLIS_BACKEND_URL'); + $this->savedServer = [ + 'HTTP_HOST' => $_SERVER['HTTP_HOST'] ?? null, + 'SERVER_PORT' => $_SERVER['SERVER_PORT'] ?? null, + 'HTTPS' => $_SERVER['HTTPS'] ?? null, + ]; + } + + #[Override] + protected function tearDown(): void { + if ($this->savedBackendUrl === false) { + putenv('HASHTOPOLIS_BACKEND_URL'); + } + else { + putenv('HASHTOPOLIS_BACKEND_URL=' . $this->savedBackendUrl); + } + foreach ($this->savedServer as $key => $value) { + if ($value === null) { + unset($_SERVER[$key]); + } + else { + $_SERVER[$key] = $value; + } + } + parent::tearDown(); + } + /** * extractFileExtension returns empty string when no dot is present. */ @@ -1034,4 +1069,107 @@ public function testCheckOrCreateInitialObjectAllSetupConfigs(): void { $this->assertNotNull($obj, "Config entry $id should exist"); } } + + /** + * Expected fallback result when HASHTOPOLIS_BACKEND_URL is not usable: + * the server URL derived from the current request plus the configured base URL. + * + * @throws Exception + */ + private function expectedFallbackUrl(): string { + return rtrim(Util::buildServerUrl() . SConfig::getInstance()->getVal(DConfig::BASE_URL), '/'); + } + + /** + * buildBackendBaseUrl takes scheme, host and port from HASHTOPOLIS_BACKEND_URL + * and strips any path it may contain. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnv(): void { + putenv('HASHTOPOLIS_BACKEND_URL=http://localhost:8080/api/v2'); + $this->assertEquals('http://localhost:8080', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl handles a HASHTOPOLIS_BACKEND_URL without path and port. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnvNoPath(): void { + putenv('HASHTOPOLIS_BACKEND_URL=http://hashtopolis.example.com'); + $this->assertEquals('http://hashtopolis.example.com', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl keeps https scheme and non-default ports. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnvHttpsPort(): void { + putenv('HASHTOPOLIS_BACKEND_URL=https://hashtopolis.example.com:8443/hashtopolis/api/v2'); + $this->assertEquals('https://hashtopolis.example.com:8443', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl keeps IPv6 hosts from HASHTOPOLIS_BACKEND_URL. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnvIpv6(): void { + putenv('HASHTOPOLIS_BACKEND_URL=http://[::1]:8080/api/v2'); + $this->assertEquals('http://[::1]:8080', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl strips a trailing slash of the env value. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnvTrailingSlash(): void { + putenv('HASHTOPOLIS_BACKEND_URL=http://localhost:8080/'); + $this->assertEquals('http://localhost:8080', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl falls back to the server URL for a malformed env value + * (missing scheme), instead of returning a broken URL. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlEnvMalformedFallsBack(): void { + $_SERVER['HTTP_HOST'] = 'fallbackhost:1234'; + $_SERVER['SERVER_PORT'] = '1234'; + unset($_SERVER['HTTPS']); + putenv('HASHTOPOLIS_BACKEND_URL=localhost:8080'); + $this->assertEquals($this->expectedFallbackUrl(), Util::buildBackendBaseUrl()); + $this->assertEquals('http://fallbackhost:1234' . SConfig::getInstance()->getVal(DConfig::BASE_URL), Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl falls back to the server URL when HASHTOPOLIS_BACKEND_URL is unset. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlEnvUnsetFallsBack(): void { + $_SERVER['HTTP_HOST'] = 'fallbackhost:1234'; + $_SERVER['SERVER_PORT'] = '1234'; + unset($_SERVER['HTTPS']); + putenv('HASHTOPOLIS_BACKEND_URL'); + $this->assertEquals($this->expectedFallbackUrl(), Util::buildBackendBaseUrl()); + $this->assertEquals('http://fallbackhost:1234' . SConfig::getInstance()->getVal(DConfig::BASE_URL), Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl falls back to the server URL for an empty env value. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlEnvEmptyFallsBack(): void { + $_SERVER['HTTP_HOST'] = 'fallbackhost:1234'; + $_SERVER['SERVER_PORT'] = '1234'; + unset($_SERVER['HTTPS']); + putenv('HASHTOPOLIS_BACKEND_URL='); + $this->assertEquals($this->expectedFallbackUrl(), Util::buildBackendBaseUrl()); + } } diff --git a/ci/phpunit/inc/apiv2/openapi/SpecOverridesTest.php b/ci/phpunit/inc/apiv2/openapi/SpecOverridesTest.php index 56f7bec0c..87874bf1c 100644 --- a/ci/phpunit/inc/apiv2/openapi/SpecOverridesTest.php +++ b/ci/phpunit/inc/apiv2/openapi/SpecOverridesTest.php @@ -142,12 +142,79 @@ public function testTheDefaultsCoverEveryNonPublicUserAttribute(): void { } /** - * Only User carries 'public' features, so nothing else is corrected. + * Only User carries 'public' features, so it is the only model whose + * responses arrive attribute-filtered and need optional corrections. + * CrackerBinary only carries creation documentation. */ - public function testUserIsTheOnlyModelWithDefaults(): void { + public function testTheDefaultsContainTheExpectedModels(): void { $this->assertTrue(SpecOverrides::defaults()->has('User')); + $this->assertTrue(SpecOverrides::defaults()->has('CrackerBinary')); foreach (['Agent', 'Task', 'Hashlist', 'Config', 'ApiToken'] as $model) { $this->assertFalse(SpecOverrides::defaults()->has($model), $model); } } + + public function testAttributeDescriptionsAreAdded(): void { + $overrides = new SpecOverrides([ + 'Foo' => [SpecOverrides::ATTRIBUTE_DESCRIPTIONS => [ + 'email' => 'The email address', + 'state' => 'On or off', + ]], + ]); + $result = $overrides->apply('Foo', $this->attributesSchema()); + + $this->assertSame('The email address', $result['properties']['email']['description']); + $this->assertSame('On or off', $result['properties']['state']['description']); + /* purely additive, the required list and the types stay as they were */ + $this->assertSame(["name", "email", "isValid", "state"], $result['required']); + $this->assertSame(["type" => "string"], ['type' => $result['properties']['email']['type']]); + } + + /** + * Unlike the optional/nullable corrections, a description for an attribute + * which is not part of the schema shape is simply skipped: the same + * description set is applied to responses and requests, and creation-only + * form fields for example are not part of a response. + */ + public function testAttributeDescriptionsSkipAbsentProperties(): void { + $overrides = new SpecOverrides([ + 'Foo' => [SpecOverrides::ATTRIBUTE_DESCRIPTIONS => ['sourceType' => 'Upload source']], + ]); + $result = $overrides->apply('Foo', $this->attributesSchema()); + + $this->assertSame($this->attributesSchema(), $result); + } + + /** + * applyDescriptions() serves the create request schema: descriptions are + * added, but the response oriented optional/nullable corrections must not + * be applied to it. + */ + public function testApplyDescriptionsIgnoresTheOtherCorrections(): void { + $overrides = new SpecOverrides([ + 'Foo' => [ + SpecOverrides::OPTIONAL_ATTRIBUTES => ['email'], + SpecOverrides::NULLABLE_ATTRIBUTES => ['isValid'], + SpecOverrides::ATTRIBUTE_DESCRIPTIONS => ['email' => 'The email address'], + ], + ]); + $result = $overrides->applyDescriptions('Foo', $this->attributesSchema()); + + $this->assertSame('The email address', $result['properties']['email']['description']); + $this->assertSame(["name", "email", "isValid", "state"], $result['required']); + $this->assertSame('string', $result['properties']['email']['type']); + $this->assertSame('boolean', $result['properties']['isValid']['type']); + } + + public function testDescriptionsMustMapToNonEmptyStrings(): void { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("must map attribute names to non-empty descriptions"); + new SpecOverrides(['Foo' => [SpecOverrides::ATTRIBUTE_DESCRIPTIONS => ['email' => '']]]); + } + + public function testDescriptionsMustNameAttributes(): void { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("must map attribute names to non-empty descriptions"); + new SpecOverrides(['Foo' => [SpecOverrides::ATTRIBUTE_DESCRIPTIONS => [42 => 'desc']]]); + } } diff --git a/ci/phpunit/inc/utils/CrackerBinaryUtilsTest.php b/ci/phpunit/inc/utils/CrackerBinaryUtilsTest.php index db7cb5b70..d835092bf 100644 --- a/ci/phpunit/inc/utils/CrackerBinaryUtilsTest.php +++ b/ci/phpunit/inc/utils/CrackerBinaryUtilsTest.php @@ -36,7 +36,7 @@ protected function setUp(): void { private function addBinary(string $version): AbstractModel { return $this->createDatabaseObject( Factory::getCrackerBinaryFactory(), - new CrackerBinary(null, $this->type->getId(), $version, 'http://example.com', 'testcracker') + new CrackerBinary(null, $this->type->getId(), $version, 'http://example.com', 'testcracker', null) ); } diff --git a/ci/phpunit/inc/utils/CrackerUtilsTest.php b/ci/phpunit/inc/utils/CrackerUtilsTest.php index c2bc6497d..06d513719 100644 --- a/ci/phpunit/inc/utils/CrackerUtilsTest.php +++ b/ci/phpunit/inc/utils/CrackerUtilsTest.php @@ -4,8 +4,11 @@ use Hashtopolis\dba\AbstractModel; use Hashtopolis\dba\Factory; +use Hashtopolis\dba\QueryFilter; use Hashtopolis\dba\models\CrackerBinary; use Hashtopolis\dba\models\CrackerBinaryType; +use Hashtopolis\inc\Util; +use Hashtopolis\inc\defines\DDirectories; use Hashtopolis\inc\apiv2\error\HttpConflict; use Hashtopolis\inc\apiv2\error\HttpError; use Hashtopolis\inc\HTException; @@ -36,7 +39,7 @@ protected function setUp(): void { ); $this->binary = $this->createDatabaseObject( Factory::getCrackerBinaryFactory(), - new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://example.com', 'testcracker') + new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://example.com', 'testcracker', null) ); } @@ -89,11 +92,310 @@ public function testCreateBinaryEmptyVersionThrowsHttpError(): void { CrackerUtils::createBinary('', 'testcracker', 'http://example.com', $this->type->getId()); } - // Verifies the full happy path: createBinary() creates and returns a new - // CrackerBinary when all fields are valid. - public function testCreateBinaryValidInputCreatesBinary(): void { - $b = CrackerUtils::createBinary('9.9.9', 'newcracker', 'http://example.com/dl', $this->type->getId()); + // Verifies that createBinary() rejects a download url the server cannot + // fetch, and that the failed creation leaves no binary and no partial + // archive behind. + public function testCreateBinaryUnreachableUrlRollsBack(): void { + $countBefore = $this->countBinariesOfType($this->type->getId()); + try { + CrackerUtils::createBinary('9.9.9', 'testcracker', 'http://127.0.0.1:1/cracker.7z', $this->type->getId()); + $this->fail('Expected HttpError for an unreachable download url'); + } + catch (HttpError $e) { + $this->assertStringContainsString('Failed to download the archive from the download url', $e->getMessage()); + } + $this->assertEquals($countBefore, $this->countBinariesOfType($this->type->getId())); + $this->assertEmpty(glob(CrackerUtils::getCrackersPath() . '*_test-crackerutils-type-9.9.9.7z')); + } + + // Verifies that createBinary() only accepts http and https download urls, + // so the server cannot be pointed at local files or stream wrappers. + public function testCreateBinaryInvalidSchemeThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinary('9.9.9', 'testcracker', 'file:///etc/passwd', $this->type->getId()); + } + + private const SEVEN_ZIP_MAGIC = "\x37\x7A\xBC\xAF\x27\x1C"; + + private function getImportPath(): string { + return Factory::getStoredValueFactory()->get(DDirectories::IMPORT)->getVal() . '/'; + } + + private function countBinariesOfType(int $typeId): int { + $qF = new QueryFilter(CrackerBinary::CRACKER_BINARY_TYPE_ID, $typeId, '='); + return sizeof(Factory::getCrackerBinaryFactory()->filter([Factory::FILTER => $qF])); + } + + // Verifies the full happy path: createBinaryFromUpload() with sourceType 'import' + // moves the archive from the import directory to the crackers directory, composes + // the server-side filename and sets the download url to the download endpoint. + public function testCreateBinaryFromUploadImportSource(): void { + $name = 'test-archive-' . uniqid() . '.7z'; + $content = self::SEVEN_ZIP_MAGIC . 'test-content'; + file_put_contents($this->getImportPath() . $name, $content); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + $this->assertEquals('test-crackerutils-type-7.2.7.7z', $b->getFilename()); + $this->assertEquals( + Util::buildBackendBaseUrl() . '/api/download.php/crackerBinary/' . $b->getId(), + $b->getDownloadUrl() + ); + $archive = CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename(); + $this->assertFileExists($archive); + $this->assertEquals($content, file_get_contents($archive)); + $this->assertFileDoesNotExist($this->getImportPath() . $name); + + unlink($archive); + } + + // Verifies the full happy path: createBinaryFromUpload() with sourceType 'inline' + // stores the base64 decoded archive in the crackers directory. + public function testCreateBinaryFromUploadInlineSource(): void { + $content = self::SEVEN_ZIP_MAGIC . 'inline-content'; + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'inline', base64_encode($content)); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + $this->assertEquals('test-crackerutils-type-7.2.7.7z', $b->getFilename()); + $this->assertStringEndsWith('/api/download.php/crackerBinary/' . $b->getId(), $b->getDownloadUrl()); + $archive = CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename(); + $this->assertFileExists($archive); + $this->assertEquals($content, file_get_contents($archive)); + + unlink($archive); + } + + // Verifies that the composed archive filename sanitizes all characters which are + // problematic in file names. + public function testCreateBinaryFromUploadSanitizesFilename(): void { + $type = $this->createDatabaseObject( + Factory::getCrackerBinaryTypeFactory(), + new CrackerBinaryType(null, 'weird cracker name!', 1) + ); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $type->getId(), 'inline', base64_encode(self::SEVEN_ZIP_MAGIC)); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + $this->assertEquals('weird-cracker-name--7.2.7.7z', $b->getFilename()); + unlink(CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename()); + } + + // Verifies that createBinaryFromUpload() rejects an unsupported sourceType. + public function testCreateBinaryFromUploadInvalidSourceTypeThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'bogus', 'data'); + } + + // Verifies that createBinaryFromUpload() rejects an empty version. + public function testCreateBinaryFromUploadEmptyVersionThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('', 'testcracker', $this->type->getId(), 'inline', base64_encode(self::SEVEN_ZIP_MAGIC)); + } + + // Verifies that createBinaryFromUpload() rejects missing sourceData. + public function testCreateBinaryFromUploadEmptySourceDataThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'inline', ''); + } + + // Verifies that createBinaryFromUpload() rejects sourceData which is not valid base64. + public function testCreateBinaryFromUploadInvalidBase64ThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'inline', '!!!no-base64!!!'); + } + + // Verifies that createBinaryFromUpload() only allows http and https urls, so no + // local files or stream wrappers can be fetched by the server. + public function testCreateBinaryFromUploadUrlSchemeThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'url', 'file:///etc/passwd'); + } + + // Verifies that a non-7z archive is rejected and the import file is restored and + // no leftover binary or archive remains. + public function testCreateBinaryFromUploadImportNot7zRollsBack(): void { + $name = 'test-archive-' . uniqid() . '.txt'; + file_put_contents($this->getImportPath() . $name, 'not-a-7z-archive'); + $countBefore = $this->countBinariesOfType($this->type->getId()); + + try { + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); + $this->fail('Expected HttpError for a non-7z archive'); + } + catch (HttpError $e) { + // expected + } + + $this->assertEquals($countBefore, $this->countBinariesOfType($this->type->getId())); + $this->assertFileExists($this->getImportPath() . $name); + $this->assertEmpty(glob(CrackerUtils::getCrackersPath() . '*_test-crackerutils-type-7.2.7.7z')); + + unlink($this->getImportPath() . $name); + } + + // Verifies that a missing import file results in an error and no leftover binary. + public function testCreateBinaryFromUploadImportFileMissingRollsBack(): void { + $countBefore = $this->countBinariesOfType($this->type->getId()); + + try { + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', 'does-not-exist-' . uniqid() . '.7z'); + $this->fail('Expected HttpError for a missing import file'); + } + catch (HttpError $e) { + // expected + } + + $this->assertEquals($countBefore, $this->countBinariesOfType($this->type->getId())); + } + + // Verifies that deleteBinary() removes the locally stored archive of the binary. + public function testDeleteBinaryRemovesLocalArchive(): void { + $name = 'test-archive-' . uniqid() . '.7z'; + file_put_contents($this->getImportPath() . $name, self::SEVEN_ZIP_MAGIC . 'to-be-deleted'); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); - $this->assertSame('9.9.9', $b->getVersion()); + $archive = CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename(); + $this->assertFileExists($archive); + + CrackerUtils::deleteBinary($b->getId()); + + $this->assertFileDoesNotExist($archive); + $this->assertNull(Factory::getCrackerBinaryFactory()->get($b->getId())); + } + + // Verifies that deleteBinary() also removes the local copy which was + // downloaded from the download url of an externally referenced binary. + public function testDeleteBinaryRemovesDownloadedLocalCopy(): void { + $binary = $this->createDatabaseObject( + Factory::getCrackerBinaryFactory(), + new CrackerBinary(null, $this->type->getId(), '3.0.0', 'http://example.com/cracker.7z', 'testcracker', null) + ); + $copy = CrackerUtils::getCrackersPath() . $binary->getId() . '_test-crackerutils-type-3.0.0.7z'; + file_put_contents($copy, self::SEVEN_ZIP_MAGIC . 'downloaded-local-copy'); + $this->assertFileExists($copy); + + CrackerUtils::deleteBinary($binary->getId()); + + $this->assertFileDoesNotExist($copy); + } + + // Verifies that deleteBinaryType() removes the archives of all locally stored + // binaries of the type. + public function testDeleteBinaryTypeRemovesLocalArchives(): void { + $type = $this->createDatabaseObject( + Factory::getCrackerBinaryTypeFactory(), + new CrackerBinaryType(null, 'type2-' . uniqid(), 1) + ); + $name = 'test-archive-' . uniqid() . '.7z'; + file_put_contents($this->getImportPath() . $name, self::SEVEN_ZIP_MAGIC . 'to-be-deleted'); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $type->getId(), 'import', $name); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + $archive = CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename(); + $this->assertFileExists($archive); + + CrackerUtils::deleteBinaryType($type->getId()); + + $this->assertFileDoesNotExist($archive); + $this->assertEquals(0, $this->countBinariesOfType($type->getId())); + } + + // Verifies that the download url of a locally stored binary cannot be changed. + public function testUpdateBinaryRejectsUrlChangeForLocalBinary(): void { + $name = 'test-archive-' . uniqid() . '.7z'; + file_put_contents($this->getImportPath() . $name, self::SEVEN_ZIP_MAGIC . 'local'); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + try { + CrackerUtils::updateBinary('8.0.0', 'testcracker', 'http://other.example.com/hc.7z', $b->getId()); + $this->fail('Expected HTException when changing the url of a local binary'); + } + catch (HTException $e) { + $this->assertStringContainsString('locally stored', $e->getMessage()); + } + + $reloaded = Factory::getCrackerBinaryFactory()->get($b->getId()); + $this->assertEquals($b->getDownloadUrl(), $reloaded->getDownloadUrl()); + CrackerUtils::deleteBinary($b->getId()); + } + + // Verifies that a locally stored binary can still be updated as long as the + // download url is not changed. + public function testUpdateBinaryAllowsUnchangedUrlForLocalBinary(): void { + $name = 'test-archive-' . uniqid() . '.7z'; + file_put_contents($this->getImportPath() . $name, self::SEVEN_ZIP_MAGIC . 'local'); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + CrackerUtils::updateBinary('8.0.0', 'testcracker', $b->getDownloadUrl(), $b->getId()); + + $reloaded = Factory::getCrackerBinaryFactory()->get($b->getId()); + $this->assertEquals('8.0.0', $reloaded->getVersion()); + $this->assertEquals($b->getDownloadUrl(), $reloaded->getDownloadUrl()); + CrackerUtils::deleteBinary($b->getId()); + } + + // Verifies that changing the download url of an externally referenced binary + // re-downloads the local copy from it, and that a failed download rolls the + // update back so nothing is changed and the previous copy is kept. + public function testUpdateBinaryUrlChangeRollsBackOnFailedDownload(): void { + $binary = $this->createDatabaseObject( + Factory::getCrackerBinaryFactory(), + new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://127.0.0.1:1/original.7z', 'testcracker', null) + ); + $copy = CrackerUtils::getCrackersPath() . $binary->getId() . '_test-crackerutils-type-1.0.0.7z'; + file_put_contents($copy, self::SEVEN_ZIP_MAGIC . 'previous-local-copy'); + + try { + CrackerUtils::updateBinary('2.0.0', 'newcracker', 'http://127.0.0.1:1/changed.7z', $binary->getId()); + $this->fail('Expected HttpError for a failed download from the changed url'); + } + catch (HttpError $e) { + $this->assertStringContainsString('Failed to download the archive from the download url', $e->getMessage()); + } + + // the update was rolled back + $reloaded = Factory::getCrackerBinaryFactory()->get($binary->getId()); + $this->assertEquals('1.0.0', $reloaded->getVersion()); + $this->assertEquals('http://127.0.0.1:1/original.7z', $reloaded->getDownloadUrl()); + $this->assertEquals('testcracker', $reloaded->getBinaryName()); + // the previous local copy is still there, unchanged + $this->assertFileExists($copy); + $this->assertEquals(self::SEVEN_ZIP_MAGIC . 'previous-local-copy', file_get_contents($copy)); + } + + // Verifies that a changed download url of a url-referenced binary has to be + // an http or https url. + public function testUpdateBinaryUrlChangeInvalidSchemeThrowsHttpError(): void { + try { + CrackerUtils::updateBinary('2.0.0', 'testcracker', 'file:///etc/passwd', $this->binary->getId()); + $this->fail('Expected HttpError for an invalid download url scheme'); + } + catch (HttpError $e) { + $this->assertStringContainsString('Only http and https download urls are supported!', $e->getMessage()); + } + // nothing was changed + $reloaded = Factory::getCrackerBinaryFactory()->get($this->binary->getId()); + $this->assertEquals('http://example.com', $reloaded->getDownloadUrl()); + } + + // Verifies that updating a binary without changing its download url does + // not re-download the local copy, the copy tracks the download url. + public function testUpdateBinaryUnchangedUrlKeepsLocalCopy(): void { + $binary = $this->createDatabaseObject( + Factory::getCrackerBinaryFactory(), + new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://127.0.0.1:1/unreachable.7z', 'testcracker', null) + ); + $copy = CrackerUtils::getCrackersPath() . $binary->getId() . '_test-crackerutils-type-1.0.0.7z'; + file_put_contents($copy, self::SEVEN_ZIP_MAGIC . 'kept-local-copy'); + + // the url is not changed, so no download happens even though it is unreachable + CrackerUtils::updateBinary('2.0.0', 'newcracker', 'http://127.0.0.1:1/unreachable.7z', $binary->getId()); + + $reloaded = Factory::getCrackerBinaryFactory()->get($binary->getId()); + $this->assertEquals('2.0.0', $reloaded->getVersion()); + $this->assertEquals('newcracker', $reloaded->getBinaryName()); + // the local copy is untouched, still under the filename of its version + $this->assertFileExists($copy); + $this->assertEquals(self::SEVEN_ZIP_MAGIC . 'kept-local-copy', file_get_contents($copy)); } } diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index fca641e31..d6f7a8858 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -55,6 +55,7 @@ directories=( "${HASHTOPOLIS_LOG_PATH}" "${HASHTOPOLIS_IMPORT_PATH}" "${HASHTOPOLIS_BINARIES_PATH}" + "${HASHTOPOLIS_CRACKERS_PATH}" "${HASHTOPOLIS_TUS_PATH}" "${HASHTOPOLIS_TEMP_UPLOADS_PATH}" "${HASHTOPOLIS_TEMP_META_PATH}" diff --git a/openapi.json b/openapi.json index 9bcd10899..0f763d49f 100644 --- a/openapi.json +++ b/openapi.json @@ -41859,6 +41859,36 @@ "attributes": { "type": "object", "properties": { + "sourceType": { + "oneOf": [ + { + "const": "inline", + "title": "Archive provided as base64 data in sourceData", + "type": "string" + }, + { + "const": "import", + "title": "Archive taken from the import directory, sourceData is the filename", + "type": "string" + }, + { + "const": "url", + "title": "Archive fetched from an http(s) url given in sourceData", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Source the 7z archive is uploaded from: inline (base64 archive data in sourceData), import (filename of a file in the import directory as sourceData) or url (http/https url in sourceData, fetched by the server). Mutually exclusive with downloadUrl." + }, + "sourceData": { + "type": [ + "string", + "null" + ], + "description": "Source of the archive upload, depending on sourceType: base64 encoded archive data, filename of a file in the import directory or a http/https url to fetch the archive from." + }, "crackerBinaryTypeId": { "type": "integer" }, @@ -41866,7 +41896,11 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" @@ -41875,7 +41909,6 @@ "required": [ "crackerBinaryTypeId", "version", - "downloadUrl", "binaryName" ] } @@ -41907,7 +41940,10 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ] }, "version": { "type": "string" @@ -41949,7 +41985,10 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ] }, "version": { "type": "string" @@ -42054,7 +42093,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -42064,10 +42104,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } }, @@ -42428,7 +42479,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -42438,10 +42490,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } }, @@ -42853,7 +42916,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -42863,10 +42927,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } }, @@ -43628,7 +43703,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -43638,10 +43714,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -43998,7 +44085,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -44008,10 +44096,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -44420,7 +44519,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -44430,10 +44530,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -54263,7 +54374,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -54273,10 +54385,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -54717,7 +54840,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -54727,10 +54851,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -55223,7 +55358,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -55233,10 +55369,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -62277,7 +62424,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -62287,10 +62435,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -63240,7 +63399,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -63250,10 +63410,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -64203,7 +64374,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -64213,10 +64385,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -65218,7 +65401,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -65228,10 +65412,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } diff --git a/src/api/download.php b/src/api/download.php new file mode 100644 index 000000000..66a86d44e --- /dev/null +++ b/src/api/download.php @@ -0,0 +1,18 @@ +run(); diff --git a/src/api/v2/index.php b/src/api/v2/index.php index 055215965..5d6809848 100644 --- a/src/api/v2/index.php +++ b/src/api/v2/index.php @@ -17,6 +17,7 @@ // }); use Hashtopolis\inc\apiv2\auth\HashtopolisAuthenticator; +use Hashtopolis\inc\apiv2\auth\JwtAuthenticationFactory; use Hashtopolis\inc\apiv2\auth\JWTBeforeHandler; use Hashtopolis\inc\apiv2\common\ClassMapper; use Hashtopolis\inc\apiv2\error\ErrorHandler; @@ -34,19 +35,12 @@ use Tuupola\Middleware\HttpBasicAuthentication; -use JimTools\JwtAuth\Decoder\FirebaseDecoder; -use JimTools\JwtAuth\Middleware\JwtAuthentication; -use JimTools\JwtAuth\Options; -use JimTools\JwtAuth\Secret; use JimTools\JwtAuth\Exceptions\AuthorizationException; use Middlewares\DeflateEncoder; use Psr\Http\Message\ServerRequestInterface as Request; -use JimTools\JwtAuth\Rules\RequestMethodRule; -use JimTools\JwtAuth\Rules\RequestPathRule; - require_once(__DIR__ . "/../../../vendor/autoload.php"); require_once(__DIR__ . "/../../inc/startup/include.php"); @@ -74,21 +68,12 @@ /* API token validation */ $container->set("JwtAuthentication", function (ContainerInterface $container) { - $decoder = new FirebaseDecoder( - new Secret(StartupConfig::getInstance()->getPepper(0), 'HS256', hash("sha256", StartupConfig::getInstance()->getPepper(0))) - ); - - $options = new Options( - isSecure: false, - attribute: null, - before: new JWTBeforeHandler - ); - - $rules = [ - new RequestPathRule(ignore: ["/api/v2/auth/token", "/api/v2/auth/oauth-token", "/api/v2/helper/resetUserPassword", "/api/v2/openapi.json"]), - new RequestMethodRule(ignore: ["OPTIONS"]) - ]; - return new JwtAuthentication($options, $decoder, $rules); + return JwtAuthenticationFactory::create([ + "/api/v2/auth/token", + "/api/v2/auth/oauth-token", + "/api/v2/helper/resetUserPassword", + "/api/v2/openapi.json" + ]); }); /* diff --git a/src/dba/models/CrackerBinary.php b/src/dba/models/CrackerBinary.php index 337a12331..55c01cfb5 100644 --- a/src/dba/models/CrackerBinary.php +++ b/src/dba/models/CrackerBinary.php @@ -10,13 +10,15 @@ class CrackerBinary extends AbstractModel { private ?string $version; private ?string $downloadUrl; private ?string $binaryName; + private ?string $filename; - function __construct(?int $crackerBinaryId, ?int $crackerBinaryTypeId, ?string $version, ?string $downloadUrl, ?string $binaryName) { + function __construct(?int $crackerBinaryId, ?int $crackerBinaryTypeId, ?string $version, ?string $downloadUrl, ?string $binaryName, ?string $filename) { $this->crackerBinaryId = $crackerBinaryId; $this->crackerBinaryTypeId = $crackerBinaryTypeId; $this->version = $version; $this->downloadUrl = $downloadUrl; $this->binaryName = $binaryName; + $this->filename = $filename; } function getKeyValueDict(): array { @@ -26,6 +28,7 @@ function getKeyValueDict(): array { $dict['version'] = $this->version; $dict['downloadUrl'] = $this->downloadUrl; $dict['binaryName'] = $this->binaryName; + $dict['filename'] = $this->filename; return $dict; } @@ -35,8 +38,9 @@ static function getFeatures(): array { $dict['crackerBinaryId'] = ['read_only' => True, "type" => "int", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => True, "protected" => True, "private" => False, "alias" => "crackerBinaryId", "public" => False, "dba_mapping" => False]; $dict['crackerBinaryTypeId'] = ['read_only' => True, "type" => "int", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => False, "private" => False, "alias" => "crackerBinaryTypeId", "public" => False, "dba_mapping" => False]; $dict['version'] = ['read_only' => False, "type" => "str(20)", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => False, "private" => False, "alias" => "version", "public" => False, "dba_mapping" => False]; - $dict['downloadUrl'] = ['read_only' => False, "type" => "str(150)", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => False, "private" => False, "alias" => "downloadUrl", "public" => False, "dba_mapping" => False]; + $dict['downloadUrl'] = ['read_only' => False, "type" => "str(255)", "subtype" => "unset", "choices" => "unset", "null" => True, "pk" => False, "protected" => False, "private" => False, "alias" => "downloadUrl", "public" => False, "dba_mapping" => False]; $dict['binaryName'] = ['read_only' => False, "type" => "str(50)", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => False, "private" => False, "alias" => "binaryName", "public" => False, "dba_mapping" => False]; + $dict['filename'] = ['read_only' => True, "type" => "str(100)", "subtype" => "unset", "choices" => "unset", "null" => True, "pk" => False, "protected" => True, "private" => False, "alias" => "filename", "public" => False, "dba_mapping" => False]; return $dict; } @@ -97,11 +101,20 @@ function setBinaryName(?string $binaryName): void { $this->binaryName = $binaryName; } + function getFilename(): ?string { + return $this->filename; + } + + function setFilename(?string $filename): void { + $this->filename = $filename; + } + const CRACKER_BINARY_ID = "crackerBinaryId"; const CRACKER_BINARY_TYPE_ID = "crackerBinaryTypeId"; const VERSION = "version"; const DOWNLOAD_URL = "downloadUrl"; const BINARY_NAME = "binaryName"; + const FILENAME = "filename"; const PERM_CREATE = "permCrackerBinaryCreate"; const PERM_READ = "permCrackerBinaryRead"; diff --git a/src/dba/models/CrackerBinaryFactory.php b/src/dba/models/CrackerBinaryFactory.php index 72b7f4878..459d29922 100644 --- a/src/dba/models/CrackerBinaryFactory.php +++ b/src/dba/models/CrackerBinaryFactory.php @@ -32,7 +32,7 @@ function getCacheValidTime(): int { * @return CrackerBinary */ function getNullObject(): CrackerBinary { - return new CrackerBinary(-1, null, null, null, null); + return new CrackerBinary(-1, null, null, null, null, null); } /** @@ -45,6 +45,6 @@ function createObjectFromDict(array $dict): CrackerBinary { $conv[strtolower($key)] = $val; } $dict = $conv; - return new CrackerBinary($dict['crackerbinaryid'], $dict['crackerbinarytypeid'], $dict['version'], $dict['downloadurl'], $dict['binaryname']); + return new CrackerBinary($dict['crackerbinaryid'], $dict['crackerbinarytypeid'], $dict['version'], $dict['downloadurl'], $dict['binaryname'], $dict['filename']); } } diff --git a/src/dba/models/generator.php b/src/dba/models/generator.php index b628d8d0c..0220810b7 100644 --- a/src/dba/models/generator.php +++ b/src/dba/models/generator.php @@ -284,8 +284,10 @@ ['name' => 'crackerBinaryId', 'read_only' => True, 'type' => 'int', 'protected' => True], ['name' => 'crackerBinaryTypeId', 'read_only' => True, 'type' => 'int', 'relation' => 'CrackerBinaryType'], ['name' => 'version', 'read_only' => False, 'type' => 'str(20)'], - ['name' => 'downloadUrl', 'read_only' => False, 'type' => 'str(150)'], + ['name' => 'downloadUrl', 'read_only' => False, 'null' => True, 'type' => 'str(255)'], ['name' => 'binaryName', 'read_only' => False, 'type' => 'str(50)'], + // archive filename of a server-hosted binary; NULL means the binary is downloaded from downloadUrl + ['name' => 'filename', 'read_only' => True, 'null' => True, 'type' => 'str(100)', 'protected' => True], ], ]; $CONF['CrackerBinaryType'] = [ diff --git a/src/inc/StartupConfig.php b/src/inc/StartupConfig.php index 2a40cb015..33ff00fd1 100644 --- a/src/inc/StartupConfig.php +++ b/src/inc/StartupConfig.php @@ -21,6 +21,7 @@ class StartupConfig { private const DIRECTORY_LOG = "log"; private const DIRECTORY_CONFIG = "config"; private const DIRECTORY_TUS = "tus"; + private const DIRECTORY_CRACKERS = "crackers"; private const DB_PROPERTY_TYPE = "type"; private const DB_PROPERTY_USER = "user"; @@ -55,6 +56,7 @@ public function __construct() { "log" => "/usr/local/share/hashtopolis/log", "config" => "/usr/local/share/hashtopolis/config", "tus" => "/var/tmp/tus/", + "crackers" => "/usr/local/share/hashtopolis/crackers", ]; $this->db_properties = [ @@ -134,6 +136,9 @@ private function loadEnv(): void { if (getenv('HASHTOPOLIS_TUS_PATH') !== false) { $this->directories[self::DIRECTORY_TUS] = getenv('HASHTOPOLIS_TUS_PATH'); } + if (getenv('HASHTOPOLIS_CRACKERS_PATH') !== false) { + $this->directories[self::DIRECTORY_CRACKERS] = getenv('HASHTOPOLIS_CRACKERS_PATH'); + } } /** @@ -159,10 +164,14 @@ private function loadLegacyConfig(): void { "log" => dirname(__FILE__) . "/../log/", "config" => dirname(__FILE__) . "/../config/", "tus" => "/var/tmp/tus/", + "crackers" => dirname(__FILE__) . "/../crackers/", ]; } else { $this->directories = $DIRECTORIES; + if (!array_key_exists(self::DIRECTORY_CRACKERS, $this->directories)) { + $this->directories[self::DIRECTORY_CRACKERS] = dirname(__FILE__) . "/../crackers/"; + } } // extract old database settings format @@ -203,6 +212,10 @@ public function getDirectoryTus(): string { return $this->directories[self::DIRECTORY_TUS]; } + public function getDirectoryCrackers(): string { + return $this->directories[self::DIRECTORY_CRACKERS]; + } + public function getDatabaseType(): string { return $this->db_properties[self::DB_PROPERTY_TYPE]; } diff --git a/src/inc/Util.php b/src/inc/Util.php index cf2fd8c2f..cd7bdde87 100755 --- a/src/inc/Util.php +++ b/src/inc/Util.php @@ -1215,11 +1215,23 @@ public static function uploadFile(string $target, string $type, array|string $so break; case "url": - $furl = fopen($sourcedata, "rb"); + error_clear_last(); + $furl = @fopen($sourcedata, "rb"); if (!$furl) { $msg = "Could not open url at source data!"; + $lastError = error_get_last(); + if ($lastError !== null) { + $msg .= " (" . $lastError['message'] . ")"; + } } else { + $contentLength = null; + $meta = stream_get_meta_data($furl); + foreach ($meta['wrapper_data'] ?? [] as $header) { + if (preg_match('/^Content-Length:\s*(\d+)\s*$/i', $header, $matches)) { + $contentLength = (int)$matches[1]; + } + } $fileLocation = fopen($target, "w"); if (!$fileLocation) { $msg = "Could not open target file!"; @@ -1227,18 +1239,41 @@ public static function uploadFile(string $target, string $type, array|string $so else { $buffersize = 131072; $last_logged = time(); + $failed = false; + $received = 0; while (!feof($furl)) { - if (!$data = fread($furl, $buffersize)) { + $data = fread($furl, $buffersize); + if ($data === false) { $msg = "READ ERROR on download"; + $failed = true; break; } - fwrite($fileLocation, $data); + if ($data === '') { + break; + } + if (fwrite($fileLocation, $data) !== strlen($data)) { + $msg = "Failed to write downloaded data to the target file!"; + $failed = true; + break; + } + $received += strlen($data); if ($last_logged < time() - 10) { $last_logged = time(); } } fclose($fileLocation); - $success = true; + if (!$failed && $contentLength !== null && $received !== $contentLength) { + $msg = "Download incomplete, received " . $received . " of " . $contentLength . " bytes!"; + $failed = true; + } + if ($failed) { + if (file_exists($target)) { + unlink($target); + } + } + else { + $success = true; + } } fclose($furl); } @@ -1289,6 +1324,33 @@ public static function buildServerUrl(): string { return $protocol . $hostname . $port; } + /** + * Determines the base URL of the backend as it is reachable from the outside. + * If HASHTOPOLIS_BACKEND_URL is set in the environment, scheme, host and port are + * taken from it and any path is stripped (e.g. "http://localhost:8080/api/v2" + * results in "http://localhost:8080"). If the variable is not set or malformed, + * this falls back to the server URL derived from the current request together + * with the configured base URL, respecting the baseHost config override. + * Used to generate agent reachable URLs for locally hosted files, e.g. the + * download URL of an uploaded cracker binary archive. + * @return string backend base url without trailing slash + * @throws Exception + */ + public static function buildBackendBaseUrl(): string { + $backendUrl = getenv('HASHTOPOLIS_BACKEND_URL'); + if ($backendUrl !== false && strlen($backendUrl) > 0) { + $parts = parse_url($backendUrl); + if ($parts !== false && isset($parts['scheme'], $parts['host'])) { + $url = $parts['scheme'] . '://' . $parts['host']; + if (isset($parts['port'])) { + $url .= ':' . $parts['port']; + } + return rtrim($url, '/'); + } + } + return rtrim(Util::buildServerUrl() . SConfig::getInstance()->getVal(DConfig::BASE_URL), '/'); + } + /** * Round to a specific amount of decimal points * @param $num Number diff --git a/src/inc/agentapi/model/DownloadBinaryAction.php b/src/inc/agentapi/model/DownloadBinaryAction.php index 7026effe6..884b3f4b2 100644 --- a/src/inc/agentapi/model/DownloadBinaryAction.php +++ b/src/inc/agentapi/model/DownloadBinaryAction.php @@ -69,8 +69,14 @@ public function __invoke(Request $request, Response $response): ResponseInterfac $crackerBinaryType = Factory::getCrackerBinaryTypeFactory()->get($crackerBinary->getCrackerBinaryTypeId()); DServerLog::log(DServerLog::TRACE, 'Agent ' . $agent->getId() . ' downloaded cracker binary ' . $crackerBinary->getId()); $ext = Util::getFileExtension($agent->getOs()); + $url = $crackerBinary->getDownloadUrl(); + // locally stored binaries are downloaded from this server, the download + // endpoint requires the token of the requesting agent as authentication + if ($crackerBinary->getFilename() !== null) { + $url .= '?token=' . $agent->getToken(); + } return $this->success($response, PActions::DOWNLOAD_BINARY, [ - PResponseBinaryDownload::URL => $crackerBinary->getDownloadUrl(), + PResponseBinaryDownload::URL => $url, PResponseBinaryDownload::NAME => $crackerBinaryType->getTypeName(), PResponseBinaryDownload::EXECUTABLE => $crackerBinary->getBinaryName() . $ext, ]); diff --git a/src/inc/apiv2/auth/JwtAuthenticationFactory.php b/src/inc/apiv2/auth/JwtAuthenticationFactory.php new file mode 100644 index 000000000..92d9776e3 --- /dev/null +++ b/src/inc/apiv2/auth/JwtAuthenticationFactory.php @@ -0,0 +1,39 @@ +getPepper(0), 'HS256', hash("sha256", StartupConfig::getInstance()->getPepper(0))) + ); + + $options = new Options( + isSecure: false, + attribute: null, + before: new JWTBeforeHandler + ); + + $rules = [ + new RequestPathRule(ignore: $ignorePaths), + new RequestMethodRule(ignore: ["OPTIONS"]) + ]; + return new JwtAuthentication($options, $decoder, $rules); + } +} diff --git a/src/inc/apiv2/common/AbstractHelperAPI.php b/src/inc/apiv2/common/AbstractHelperAPI.php index b912114e1..73563cf56 100644 --- a/src/inc/apiv2/common/AbstractHelperAPI.php +++ b/src/inc/apiv2/common/AbstractHelperAPI.php @@ -15,7 +15,7 @@ use Psr\Http\Message\ServerRequestInterface as Request; use Slim\App; use Slim\Exception\HttpForbiddenException; -use Hashtopolis\inc\Util; +use Hashtopolis\inc\utils\DownloadUtils; abstract class AbstractHelperAPI extends AbstractBaseAPI { abstract public function actionPost(array $data): AbstractModel|array|null; @@ -118,118 +118,16 @@ static public function register(App $app): void { } /** - * Handles HTTP range requests for partial content delivery + * Streams the given file as a download response, handling ETag based caching + * and partial content (range) requests. * - * This method processes the `Range` header from the HTTP request - * to determine the start and end byte positions for the response, - * ensuring the range is valid and updates the file pointer accordingly. - * - * @param int &$start A reference to the starting byte of the range. This value will be updated. - * @param int &$end A reference to the ending byte of the range. This value will be updated. - * @param int $size The total size of the content in bytes. - * @param resource $fp A file pointer resource to seek to the correct position for the range. - * @return bool Returns `true` if the range request is valid and successfully processed, or `false` otherwise. - * - * @throws InvalidArgumentException If the `Range` header is malformed. - * - * @note This function assumes the presence of the `HTTP_RANGE` header in the `$_SERVER` superglobal. - */ - protected function handleRangeRequest(int &$start, int &$end, int $size, $fp): bool { - $c_end = $end; - - list(, $range) = explode('=', $_SERVER['HTTP_RANGE'], 2); - - if (str_contains($range, ',')) { - return false; - } - if ($range == '-') { - $c_start = $size - (int)substr($range, 1); - } - else { - $range = explode('-', $range); - $c_start = (int)$range[0]; - if ((isset($range[1]) && is_numeric($range[1]))) { - $c_end = (int)$range[1]; - } - else { - $c_end = $size; - } - } - if ($c_end > $end) { - $c_end = $end; - } - if ($c_start > $c_end || $c_start > $size - 1 || $c_end >= $size) { - return false; - } - $start = $c_start; - $end = $c_end; - fseek($fp, $start); - return true; - } - - /** * @param Request $request * @param Response $response - * @param string $filename + * @param string $filename Absolute path of the file to stream * @return Response * @throws HttpForbiddenException */ protected function startDownload(Request $request, Response $response, string $filename): Response { - $size = Util::filesize($filename); - $lastModified = filemtime($filename); - - $etag = md5($lastModified . $size); - $ifNoneMatch = $request->getHeaderLine('If-None-Match'); - if ($ifNoneMatch === $etag) { - return $response->withStatus(304); - } - - $exp = explode(".", $filename); - if ($exp[sizeof($exp) - 1] == '7z') { - $contentType = "application/x-7z-compressed"; - } - else { - $contentType = "application/force-download"; - } - $fp = @fopen($filename, "rb"); - - if (!$fp) { - throw new HttpForbiddenException($request, "Can't open the file"); - } - - $start = 0; // Start byte - $end = $size - 1; // End byte - - $status = 200; - if (isset($_SERVER['HTTP_RANGE'])) { - if (!$this->handleRangeRequest($start, $end, $size, $fp)) { - fclose($fp); - return $response->withStatus(416) - ->withHeader("Content-Range", "bytes $start-$end/$size"); - } - else { - $status = 206; - } - } - - $length = $end - $start + 1; //content-length - $buffer = 1024 * 100; - $stream = $response->getBody(); - while (!feof($fp) && ($p = ftell($fp)) <= $end) { - if ($p + $buffer > $end) { - $buffer = $end - $p + 1; - } - $stream->write(fread($fp, $buffer)); - } - fclose($fp); - - return $response->withStatus($status) - ->withHeader("Content-Type", $contentType) - ->withHeader("Content-Description", $filename) - ->withHeader("Content-Disposition", "attachment; filename=\"" . $filename . "\"") - ->withHeader("Accept-Ranges", "Byte") - ->withHeader("Content-Range", "bytes $start-$end/$size") - ->withHeader("Content-Length", $length) - ->withHeader("ETag", $etag); + return DownloadUtils::startDownload($request, $response, $filename); } } diff --git a/src/inc/apiv2/model/CrackerBinaryAPI.php b/src/inc/apiv2/model/CrackerBinaryAPI.php index bdd14e11a..e6132215c 100644 --- a/src/inc/apiv2/model/CrackerBinaryAPI.php +++ b/src/inc/apiv2/model/CrackerBinaryAPI.php @@ -10,6 +10,8 @@ use Hashtopolis\dba\models\Task; use Hashtopolis\inc\apiv2\common\AbstractModelAPI; use Hashtopolis\inc\apiv2\error\HttpError; +use Hashtopolis\inc\apiv2\error\HttpForbidden; +use Hashtopolis\inc\apiv2\error\ResourceNotFoundError; use Hashtopolis\inc\HTException; @@ -20,10 +22,27 @@ class CrackerBinaryAPI extends AbstractModelAPI { public static function getBaseUri(): string { return "/api/v2/ui/crackers"; } - + public static function getDBAclass(): string { return CrackerBinary::class; } + + /** + * Extra fields which are valid for creation of object. With one of the source + * fields given, the archive of the cracker binary is uploaded to the server + * instead of referencing it with an external download url. + */ + public function getFormFields(): array { + return [ + "sourceType" => ['type' => 'str', 'null' => True, + 'choices' => [ + "inline" => "Archive provided as base64 data in sourceData", + "import" => "Archive taken from the import directory, sourceData is the filename", + "url" => "Archive fetched from an http(s) url given in sourceData" + ]], + "sourceData" => ['type' => 'str', 'null' => True] + ]; + } public static function getToOneRelationships(): array { return [ @@ -52,6 +71,25 @@ public static function getToManyRelationships(): array { * @throws HTException */ protected function createObject(array $data): int { + if (isset($data["sourceType"])) { + if (isset($data[CrackerBinary::DOWNLOAD_URL])) { + throw new HttpError("downloadUrl cannot be provided when the archive is uploaded with sourceType!"); + } + if (!isset($data["sourceData"])) { + throw new HttpError("sourceData is required when sourceType is provided!"); + } + $binary = CrackerUtils::createBinaryFromUpload( + $data[CrackerBinary::VERSION], + $data[CrackerBinary::BINARY_NAME], + $data[CrackerBinary::CRACKER_BINARY_TYPE_ID], + $data["sourceType"], + $data["sourceData"] + ); + return $binary->getId(); + } + if (!isset($data[CrackerBinary::DOWNLOAD_URL]) || strlen($data[CrackerBinary::DOWNLOAD_URL]) == 0) { + throw new HttpError("Please provide all information!"); + } $binary = CrackerUtils::createBinary( $data[CrackerBinary::VERSION], $data[CrackerBinary::BINARY_NAME], @@ -60,7 +98,7 @@ protected function createObject(array $data): int { ); return $binary->getId(); } - + /** * @param CrackerBinary $object * @throws HTException @@ -68,4 +106,42 @@ protected function createObject(array $data): int { protected function deleteObject(AbstractModel $object): void { CrackerUtils::deleteBinary($object->getId()); } + + /** + * The download url of locally stored binaries is owned by the server, so it + * cannot be overwritten with a patch. When the download url of a binary which + * is referenced by an external url is changed, the server downloads a new + * local copy of the archive from it; if that download fails, the update is + * rolled back so nothing is changed. + * + * @param int $objectId + * @param array $data + * @throws HttpError + * @throws HttpForbidden + * @throws ResourceNotFoundError + * @throws HTException + */ + protected function updateObject(int $objectId, array $data): void { + $binary = CrackerUtils::getBinary($objectId); + if (array_key_exists(CrackerBinary::DOWNLOAD_URL, $data)) { + if ($binary->getFilename() !== null) { + throw new HttpError("The download url of a locally stored cracker binary cannot be changed!"); + } + } + $refreshLocalCopy = $binary->getFilename() === null + && ($data[CrackerBinary::DOWNLOAD_URL] ?? null) !== null + && $data[CrackerBinary::DOWNLOAD_URL] != $binary->getDownloadUrl(); + if ($refreshLocalCopy) { + CrackerUtils::validateDownloadUrl($data[CrackerBinary::DOWNLOAD_URL]); + } + $previousValues = [ + CrackerBinary::VERSION => $binary->getVersion(), + CrackerBinary::DOWNLOAD_URL => $binary->getDownloadUrl(), + CrackerBinary::BINARY_NAME => $binary->getBinaryName() + ]; + parent::updateObject($objectId, $data); + if ($refreshLocalCopy) { + CrackerUtils::refreshLocalCopy($objectId, $previousValues); + } + } } diff --git a/src/inc/apiv2/openapi/ModelApiPathBuilder.php b/src/inc/apiv2/openapi/ModelApiPathBuilder.php index 621c699ce..6de73eeb8 100644 --- a/src/inc/apiv2/openapi/ModelApiPathBuilder.php +++ b/src/inc/apiv2/openapi/ModelApiPathBuilder.php @@ -181,6 +181,12 @@ public function addRoute(RouteTarget $target, AbstractModelAPI $api, ContainerIn array_filter($createFeatures, fn($f) => !$f['null']) )); $properties_create = $this->jsonApiFragments->buildPatchPost($this->typeMapper->makeProperties($createFeatures), $typeName, null, $requiredCreateAttributes); + /* Descriptions document the creation, the response corrections do not + apply to a request schema */ + $properties_create["data"]["properties"]["attributes"] = $this->overrides->applyDescriptions( + $name, + $properties_create["data"]["properties"]["attributes"] + ); $properties_patch = $this->jsonApiFragments->buildPatchPost($this->typeMapper->makeProperties($class->getPatchValidFeatures(), true), $typeName); $components[$name . "Create"] = diff --git a/src/inc/apiv2/openapi/SpecOverrides.php b/src/inc/apiv2/openapi/SpecOverrides.php index c16f8303f..e963d5733 100644 --- a/src/inc/apiv2/openapi/SpecOverrides.php +++ b/src/inc/apiv2/openapi/SpecOverrides.php @@ -16,14 +16,20 @@ * while the model still declares its column NOT NULL, so the derived schema * demands it and a permission-filtered response fails validation. * - * Two independent corrections are available per model, both keyed by the - * attribute alias as it appears in the response: + * Three independent corrections are available per model, the first two keyed + * by the attribute alias as it appears in the response: * * - OPTIONAL_ATTRIBUTES: the attribute may be missing, so it is dropped from * the "required" list of the attributes object. This is what permission * filtering does: the key is not there at all. * - NULLABLE_ATTRIBUTES: the attribute may be null, so "null" joins its type * (or a "null" branch its oneOf). The key stays required. + * - ATTRIBUTE_DESCRIPTIONS: a map of attribute alias to description text. + * Purely additive documentation, applied to the response schemas and, via + * applyDescriptions(), to the create request schema. Properties the schema + * does not carry are skipped, so one description set can serve the + * different shapes a model's attributes take (responses do not contain + * creation-only form fields, for example). * * Both the openapi.json route and ci/tools/generate-openapi.php build with * defaults(), so the served spec and the committed one agree. @@ -35,15 +41,19 @@ class SpecOverrides { /** Attributes a response may send as null; "null" joins their type. */ public const NULLABLE_ATTRIBUTES = 'nullableAttributes'; - private const KEYS = [self::OPTIONAL_ATTRIBUTES, self::NULLABLE_ATTRIBUTES]; + /** Attribute descriptions; a map of attribute alias to description text. */ + public const ATTRIBUTE_DESCRIPTIONS = 'attributeDescriptions'; - /** @var array>> */ + private const KEYS = [self::OPTIONAL_ATTRIBUTES, self::NULLABLE_ATTRIBUTES, self::ATTRIBUTE_DESCRIPTIONS]; + + /** @var array|array>> */ private array $byModel = []; /** - * @param array>> $overrides model name + * @param array|array>> $overrides model name * (as the component schemas spell it, e.g. "User") to a map of - * OPTIONAL_ATTRIBUTES and/or NULLABLE_ATTRIBUTES to attribute aliases + * OPTIONAL_ATTRIBUTES and/or NULLABLE_ATTRIBUTES to attribute aliases, + * or of ATTRIBUTE_DESCRIPTIONS to attribute alias => description */ public function __construct(array $overrides = []) { foreach ($overrides as $model => $entry) { @@ -63,12 +73,24 @@ public function __construct(array $overrides = []) { if (!is_array($aliases)) { throw new InvalidArgumentException("Spec override '$key' of '$model' must be a list of attribute names"); } - foreach ($aliases as $alias) { - if (!is_string($alias) || $alias === '') { - throw new InvalidArgumentException("Spec override '$key' of '$model' must only name attributes"); + if ($key === self::ATTRIBUTE_DESCRIPTIONS) { + foreach ($aliases as $alias => $description) { + if (!is_string($alias) || $alias === '' || !is_string($description) || $description === '') { + throw new InvalidArgumentException( + "Spec override '$key' of '$model' must map attribute names to non-empty descriptions" + ); + } + } + $normalized[$key] = $aliases; + } + else { + foreach ($aliases as $alias) { + if (!is_string($alias) || $alias === '') { + throw new InvalidArgumentException("Spec override '$key' of '$model' must only name attributes"); + } } + $normalized[$key] = array_values(array_unique($aliases)); } - $normalized[$key] = array_values(array_unique($aliases)); } $this->byModel[$model] = $normalized; } @@ -100,6 +122,14 @@ public static function defaults(): self { 'otp4', ], ], + 'CrackerBinary' => [ + self::ATTRIBUTE_DESCRIPTIONS => [ + 'downloadUrl' => 'External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards.', + 'filename' => 'Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided.', + 'sourceType' => 'Source the 7z archive is uploaded from: inline (base64 archive data in sourceData), import (filename of a file in the import directory as sourceData) or url (http/https url in sourceData, fetched by the server). Mutually exclusive with downloadUrl.', + 'sourceData' => 'Source of the archive upload, depending on sourceType: base64 encoded archive data, filename of a file in the import directory or a http/https url to fetch the archive from.', + ], + ], ]); } @@ -126,6 +156,11 @@ public function apply(string $model, array $schema): array { $properties = $schema['properties'] ?? []; foreach ($entry as $key => $aliases) { + if ($key === self::ATTRIBUTE_DESCRIPTIONS) { + // descriptions are additive documentation, properties which are not + // part of this particular schema shape are simply skipped + continue; + } foreach ($aliases as $alias) { if (!array_key_exists($alias, $properties)) { throw new InvalidArgumentException( @@ -144,6 +179,36 @@ public function apply(string $model, array $schema): array { $schema['properties'][$alias] = $this->makeNullable($schema['properties'][$alias]); } + return $this->applyDescriptionEntries($entry[self::ATTRIBUTE_DESCRIPTIONS] ?? [], $schema); + } + + /** + * Apply only the attribute descriptions of one model to a request attributes + * schema (create), where the response oriented optional/nullable corrections + * must not be applied. Properties the schema does not carry are skipped. + * + * The schema is returned unchanged when nothing is configured for the model. + * + * @param string $model model name as the component schemas spell it + * @param array $schema the "attributes" object, with "required" and "properties" + */ + public function applyDescriptions(string $model, array $schema): array { + if (!$this->has($model)) { + return $schema; + } + $descriptions = $this->byModel[$model][self::ATTRIBUTE_DESCRIPTIONS] ?? []; + return $this->applyDescriptionEntries($descriptions, $schema); + } + + /** + * @param array $descriptions attribute alias to description text + */ + private function applyDescriptionEntries(array $descriptions, array $schema): array { + foreach ($descriptions as $alias => $description) { + if (array_key_exists($alias, $schema['properties'] ?? [])) { + $schema['properties'][$alias]['description'] = $description; + } + } return $schema; } diff --git a/src/inc/defines/DDirectories.php b/src/inc/defines/DDirectories.php index cc0fbeefa..8557bc6e5 100644 --- a/src/inc/defines/DDirectories.php +++ b/src/inc/defines/DDirectories.php @@ -3,9 +3,10 @@ namespace Hashtopolis\inc\defines; class DDirectories { - const FILES = "directory_files"; - const IMPORT = "directory_import"; - const LOG = "directory_log"; - const CONFIG = "directory_config"; - const TUS = "directory_tus"; + const FILES = "directory_files"; + const IMPORT = "directory_import"; + const LOG = "directory_log"; + const CONFIG = "directory_config"; + const TUS = "directory_tus"; + const CRACKERS = "directory_crackers"; } \ No newline at end of file diff --git a/src/inc/downloadapi/CrackerBinaryDownloadHandler.php b/src/inc/downloadapi/CrackerBinaryDownloadHandler.php new file mode 100644 index 000000000..72fb0c03f --- /dev/null +++ b/src/inc/downloadapi/CrackerBinaryDownloadHandler.php @@ -0,0 +1,44 @@ +get((int)$args['id']); + if ($binary === null || $binary->getFilename() === null) { + return CrackerBinaryDownloadHandler::notFound($response, 'No such cracker binary archive!'); + } + $path = CrackerUtils::getCrackersPath() . $binary->getId() . '_' . $binary->getFilename(); + if (!file_exists($path)) { + return CrackerBinaryDownloadHandler::notFound($response, 'The archive of this cracker binary is not present on the server!'); + } + + $agent = $request->getAttribute(AgentAction::AGENT_ATTRIBUTE); + if ($agent instanceof Agent) { + DServerLog::log(DServerLog::TRACE, 'Agent ' . $agent->getId() . ' downloaded the archive of cracker binary ' . $binary->getId()); + } + else { + DServerLog::log(DServerLog::TRACE, 'User ' . ($request->getAttribute('userId') ?? 'unknown') . + ' downloaded the archive of cracker binary ' . $binary->getId()); + } + return DownloadUtils::startDownload($request, $response, $path, $binary->getFilename()); + } + + private static function notFound(Response $response, string $message): Response { + $response->getBody()->write($message); + return $response->withStatus(404)->withHeader('Content-Type', 'text/plain'); + } +} diff --git a/src/inc/downloadapi/DownloadApp.php b/src/inc/downloadapi/DownloadApp.php new file mode 100644 index 000000000..f2798c6b1 --- /dev/null +++ b/src/inc/downloadapi/DownloadApp.php @@ -0,0 +1,79 @@ +add(new DownloadAuthMiddleware()); + + $errorMiddleware = $app->addErrorMiddleware(true, true, true); + $errorMiddleware->setDefaultErrorHandler( + function ( + Request $request, + Throwable $exception, + bool $displayErrorDetails, + bool $logErrors, + bool $logErrorDetails, + ): ResponseInterface { + error_log("DownloadApi: " . $exception->getMessage()); + $response = new Response(500); + $response->getBody()->write('Internal server error'); + return $response; + } + ); + $errorMiddleware->setErrorHandler(AuthorizationException::class, function ( + Request $request, + Throwable $exception, + bool $displayErrorDetails, + bool $logErrors, + bool $logErrorDetails, + ): ResponseInterface { + $response = new Response(401); + $response->getBody()->write('No access!'); + return $response; + }); + $errorMiddleware->setErrorHandler(HttpNotFoundException::class, function ( + Request $request, + Throwable $exception, + bool $displayErrorDetails, + bool $logErrors, + bool $logErrorDetails, + ): ResponseInterface { + $response = new Response(404); + $response->getBody()->write('Not found'); + return $response; + }); + $app->addRoutingMiddleware(); + + $app->get('/api/download.php/{kind}/{id}', function (Request $request, Response $response, array $args): ResponseInterface { + $handlerClass = DownloadRegistry::getHandler($args['kind']); + if ($handlerClass === null) { + $response->getBody()->write('Unknown download kind!'); + return $response->withStatus(404)->withHeader('Content-Type', 'text/plain'); + } + /** @var CrackerBinaryDownloadHandler $handler */ + $handler = new $handlerClass(); + return $handler($request, $response, $args); + }); + + return $app; + } +} diff --git a/src/inc/downloadapi/DownloadAuthMiddleware.php b/src/inc/downloadapi/DownloadAuthMiddleware.php new file mode 100644 index 000000000..22c19fe5b --- /dev/null +++ b/src/inc/downloadapi/DownloadAuthMiddleware.php @@ -0,0 +1,44 @@ +hasHeader('Authorization')) { + // apiv2 JWT authentication, attaches the userId, scope and aud attributes + return JwtAuthenticationFactory::create([])->process($request, $handler); + } + + $token = $request->getQueryParams()['token'] ?? null; + if (is_string($token) && strlen($token) > 0) { + $qF = new QueryFilter(Agent::TOKEN, $token, '='); + $agent = Factory::getAgentFactory()->filter([Factory::FILTER => $qF], true); + if ($agent !== null) { + return $handler->handle($request->withAttribute(AgentAction::AGENT_ATTRIBUTE, $agent)); + } + } + + $response = new Response(401); + $response->getBody()->write('No access!'); + return $response; + } +} diff --git a/src/inc/downloadapi/DownloadRegistry.php b/src/inc/downloadapi/DownloadRegistry.php new file mode 100644 index 000000000..ced811f42 --- /dev/null +++ b/src/inc/downloadapi/DownloadRegistry.php @@ -0,0 +1,19 @@ + CrackerBinaryDownloadHandler::class, + ]; + + public static function getHandler(string $kind): ?string { + return self::HANDLERS[$kind] ?? null; + } +} diff --git a/src/inc/startup/setup.json b/src/inc/startup/setup.json index 50041d2ff..7212225a5 100644 --- a/src/inc/startup/setup.json +++ b/src/inc/startup/setup.json @@ -415,7 +415,8 @@ "crackerBinaryTypeId" : 1, "version" : "7.1.2", "downloadUrl" : "https://hashcat.net/files/hashcat-7.1.2.7z", - "binaryName" : "hashcat" + "binaryName" : "hashcat", + "filename" : null } ], "CrackerBinaryType" : [ diff --git a/src/inc/startup/setup.php b/src/inc/startup/setup.php index 74ff07e83..5d31e8408 100755 --- a/src/inc/startup/setup.php +++ b/src/inc/startup/setup.php @@ -212,4 +212,5 @@ Util::checkDataDirectory(DDirectories::IMPORT, StartupConfig::getInstance()->getDirectoryImport()); Util::checkDataDirectory(DDirectories::LOG, StartupConfig::getInstance()->getDirectoryLog()); Util::checkDataDirectory(DDirectories::CONFIG, StartupConfig::getInstance()->getDirectoryConfig()); -Util::checkDataDirectory(DDirectories::TUS, StartupConfig::getInstance()->getDirectoryTus()); \ No newline at end of file +Util::checkDataDirectory(DDirectories::TUS, StartupConfig::getInstance()->getDirectoryTus()); +Util::checkDataDirectory(DDirectories::CRACKERS, StartupConfig::getInstance()->getDirectoryCrackers()); \ No newline at end of file diff --git a/src/inc/utils/CrackerUtils.php b/src/inc/utils/CrackerUtils.php index a4cfc7f7a..1412897a2 100644 --- a/src/inc/utils/CrackerUtils.php +++ b/src/inc/utils/CrackerUtils.php @@ -10,6 +10,7 @@ use Hashtopolis\dba\ContainFilter; use Hashtopolis\dba\Factory; use Hashtopolis\dba\models\Pretask; +use Hashtopolis\inc\defines\DDirectories; use Hashtopolis\inc\apiv2\error\HttpConflict; use Hashtopolis\inc\apiv2\error\HttpError; use Hashtopolis\inc\HTException; @@ -55,6 +56,12 @@ public static function createBinaryType(string $typeName): CrackerBinaryType { } /** + * Creates a new cracker binary which is referenced by an external download url. + * The server downloads a local copy of the archive into the crackers directory, + * so it has it available for later analysis. The agents still download the + * archive from the external download url. If the download fails or the archive + * is not a valid 7z archive, nothing is added. + * * @param string $version * @param string $name * @param string $url @@ -69,8 +76,231 @@ public static function createBinary(string $version, string $name, string $url, if (strlen($version) == 0 || strlen($name) == 0 || strlen($url) == 0) { throw new HttpError("Please provide all information!"); } - $binary = new CrackerBinary(null, $binaryType->getId(), $version, $url, $name); - return Factory::getCrackerBinaryFactory()->save($binary); + CrackerUtils::validateDownloadUrl($url); + // create the entry first, the id is needed for the filename of the local copy + $binary = Factory::getCrackerBinaryFactory()->save( + new CrackerBinary(null, $binaryType->getId(), $version, $url, $name, null) + ); + try { + CrackerUtils::storeLocalCopy($binary); + } + catch (HttpError $e) { + Factory::getCrackerBinaryFactory()->delete($binary); + throw $e; + } + return $binary; + } + + /** + * Creates a new cracker binary from an uploaded 7z archive. The archive is stored in + * the crackers directory and the downloadUrl is set to the download endpoint of this + * server, so it can directly be used by the agents to download the binary. + * + * @param string $version + * @param string $name + * @param int $binaryTypeId + * @param string $sourceType choices inline, import, url + * @param string $sourceData base64 data, filename in the import directory or download url + * @return CrackerBinary + * @throws HttpError + * @throws HTException + * @throws Exception + */ + public static function createBinaryFromUpload(string $version, string $name, int $binaryTypeId, string $sourceType, string $sourceData): CrackerBinary { + $binaryType = CrackerUtils::getBinaryType($binaryTypeId); + if (strlen($version) == 0 || strlen($name) == 0 || strlen($sourceData) == 0) { + throw new HttpError("Please provide all information!"); + } + + // determine the source of the archive and validate it + switch ($sourceType) { + case "inline": + $archiveData = base64_decode($sourceData, true); + if ($archiveData === false) { + throw new HttpError("sourceData not valid base64 encoding"); + } + $uploadType = "paste"; + $uploadData = $archiveData; + break; + case "import": + $realname = str_replace(" ", "_", htmlentities(basename($sourceData), ENT_QUOTES, "UTF-8")); + if ($sourceData != $realname) { + throw new HttpError("sourceData is invalid filename suggestion '$realname'"); + } + $uploadType = "import"; + $uploadData = $sourceData; + break; + case "url": + $scheme = parse_url($sourceData, PHP_URL_SCHEME); + if ($scheme != "http" && $scheme != "https") { + throw new HttpError("Only http and https URLs are supported as sourceData!"); + } + $uploadType = "url"; + $uploadData = $sourceData; + break; + default: + throw new HttpError("sourceType value '" . $sourceType . "' is not supported (choices inline, import, url"); + } + + $filename = CrackerUtils::buildArchiveFilename($binaryType, $version); + + // create the entry first with a placeholder download url, the final one + // contains the id and can only be set once it is known + $binary = Factory::getCrackerBinaryFactory()->save( + new CrackerBinary(null, $binaryType->getId(), $version, "", $name, null) + ); + + $target = CrackerUtils::getCrackersPath() . $binary->getId() . '_' . $filename; + [$success, $msg] = Util::uploadFile($target, $uploadType, $uploadData); + if (!$success) { + Factory::getCrackerBinaryFactory()->delete($binary); + throw new HttpError("Failed to store the archive: " . $msg); + } + + if (!CrackerUtils::isSevenZipArchive($target)) { + // in case the archive was imported, put the file back to the import directory + if ($sourceType == "import") { + rename($target, CrackerUtils::getImportPath() . $sourceData); + } + else { + unlink($target); + } + Factory::getCrackerBinaryFactory()->delete($binary); + throw new HttpError("The provided archive is not a valid 7z archive!"); + } + + return Factory::getCrackerBinaryFactory()->mset($binary, [ + CrackerBinary::DOWNLOAD_URL => Util::buildBackendBaseUrl() . '/api/download.php/crackerBinary/' . $binary->getId(), + CrackerBinary::FILENAME => $filename + ]); + } + + /** + * @throws Exception + */ + public static function getCrackersPath(): string { + return rtrim(Factory::getStoredValueFactory()->get(DDirectories::CRACKERS)->getVal(), '/') . '/'; + } + + /** + * @throws Exception + */ + private static function getImportPath(): string { + return rtrim(Factory::getStoredValueFactory()->get(DDirectories::IMPORT)->getVal(), '/') . '/'; + } + + /** + * Composed server-side archive filename for a locally stored cracker binary, + * the '.7z' extension is enforced by construction. + */ + private static function buildArchiveFilename(CrackerBinaryType $binaryType, string $version): string { + $sanitized = preg_replace('/[^A-Za-z0-9._-]/', '-', $binaryType->getTypeName() . '-' . $version) ?? ''; + return $sanitized . '.7z'; + } + + private static function isSevenZipArchive(string $path): bool { + $magic = "\x37\x7A\xBC\xAF\x27\x1C"; + $fp = @fopen($path, "rb"); + if ($fp === false) { + return false; + } + $header = fread($fp, strlen($magic)); + fclose($fp); + return $header === $magic; + } + + /** + * Validates that the server is allowed to fetch the given url as the + * download url of a cracker binary archive. + * + * @param string $url + * @throws HttpError + */ + public static function validateDownloadUrl(string $url): void { + $scheme = parse_url($url, PHP_URL_SCHEME); + if ($scheme != "http" && $scheme != "https") { + throw new HttpError("Only http and https download urls are supported!"); + } + } + + /** + * Downloads a local copy of the archive of a url-referenced cracker binary + * from its download url into the crackers directory. The archive is + * downloaded to a temporary file first, so a failed download cannot destroy + * a previously stored local copy, and only moved into place after it was + * validated as a 7z archive. Local copies of previous versions or urls of + * the binary are removed. + * + * @param CrackerBinary $binary the binary to download the archive for + * @throws HttpError + * @throws HTException + * @throws Exception + */ + private static function storeLocalCopy(CrackerBinary $binary): void { + $binaryType = CrackerUtils::getBinaryType($binary->getCrackerBinaryTypeId()); + $crackersPath = CrackerUtils::getCrackersPath(); + $filename = CrackerUtils::buildArchiveFilename($binaryType, $binary->getVersion()); + $target = $crackersPath . $binary->getId() . '_' . $filename; + $temporary = $target . '.part'; + if (file_exists($temporary)) { + unlink($temporary); + } + [$success, $msg] = Util::uploadFile($temporary, "url", $binary->getDownloadUrl()); + if (!$success) { + if (file_exists($temporary)) { + unlink($temporary); + } + throw new HttpError("Failed to download the archive from the download url: " . $msg); + } + if (!CrackerUtils::isSevenZipArchive($temporary)) { + unlink($temporary); + throw new HttpError("The archive at the download url is not a valid 7z archive!"); + } + rename($temporary, $target); + // remove local copies of previous versions or urls of this binary + foreach (glob($crackersPath . $binary->getId() . '_*') ?: [] as $path) { + if ($path != $target) { + unlink($path); + } + } + } + + /** + * Refreshes the local copy of the archive of a url-referenced cracker binary + * from its download url, to be called after the binary was updated in the + * database. If the download fails, the update is rolled back by restoring + * the given previous values before the error is rethrown, so nothing of + * the update remains. + * + * @param int $binaryId + * @param array $previousValues previous values of the updated fields, keyed by the CrackerBinary feature constants + * @throws HttpError + * @throws HTException + * @throws Exception + */ + public static function refreshLocalCopy(int $binaryId, array $previousValues): void { + $binary = CrackerUtils::getBinary($binaryId); + try { + CrackerUtils::storeLocalCopy($binary); + } + catch (HttpError $e) { + Factory::getCrackerBinaryFactory()->mset($binary, $previousValues); + throw $e; + } + } + + /** + * Removes the locally stored archive and any downloaded local copy of a + * cracker binary, all of them are prefixed with the id of the binary. + * + * @throws Exception + */ + private static function deleteLocalArchive(CrackerBinary $binary): void { + foreach (glob(CrackerUtils::getCrackersPath() . $binary->getId() . '_*') ?: [] as $path) { + if (file_exists($path)) { + unlink($path); + } + } } /** @@ -85,6 +315,8 @@ public static function deleteBinary(int $binaryId): void { if (sizeof($check) > 0) { throw new HTException("There are tasks which use this binary!"); } + // remove a locally stored archive if there is one + CrackerUtils::deleteLocalArchive($binary); Factory::getCrackerBinaryFactory()->delete($binary); } @@ -114,17 +346,28 @@ public static function deleteBinaryType(int $binaryTypeId): void { throw new HTException("There are pretasks which use this cracker type!"); } + // remove the archives of locally stored binaries + foreach ($binaries as $binary) { + CrackerUtils::deleteLocalArchive($binary); + } + // delete Factory::getCrackerBinaryFactory()->massDeletion([Factory::FILTER => $qF]); Factory::getCrackerBinaryTypeFactory()->delete($binaryType); } /** + * Updates a cracker binary. When the download url of a binary which is + * referenced by an external url is changed, the server downloads a new local + * copy of the archive from it; if that download fails, the update is rolled + * back so nothing is changed. + * * @param string $version * @param string $name * @param string $url * @param int $binaryId * @return CrackerBinaryType + * @throws HttpError * @throws HTException * @throws Exception */ @@ -133,12 +376,30 @@ public static function updateBinary(string $version, string $name, string $url, if (strlen($version) == 0 || strlen($name) == 0 || strlen($url) == 0) { throw new HTException("Please provide all information!"); } + // locally stored binaries are downloaded from this server, so the url is owned by the server + if ($binary->getFilename() !== null && $url != $binary->getDownloadUrl()) { + throw new HTException("The download url of a locally stored cracker binary cannot be changed!"); + } + // a changed download url of a url-referenced binary requires the server to + // download a new local copy of the archive from it + $refreshLocalCopy = $binary->getFilename() === null && $url != $binary->getDownloadUrl(); + if ($refreshLocalCopy) { + CrackerUtils::validateDownloadUrl($url); + } + $previousValues = [ + CrackerBinary::VERSION => $binary->getVersion(), + CrackerBinary::DOWNLOAD_URL => $binary->getDownloadUrl(), + CrackerBinary::BINARY_NAME => $binary->getBinaryName() + ]; $binary = Factory::getCrackerBinaryFactory()->mset($binary, [ CrackerBinary::BINARY_NAME => htmlentities($name, ENT_QUOTES, "UTF-8"), CrackerBinary::DOWNLOAD_URL => $url, CrackerBinary::VERSION => $version ] ); + if ($refreshLocalCopy) { + CrackerUtils::refreshLocalCopy($binary->getId(), $previousValues); + } return Factory::getCrackerBinaryTypeFactory()->get($binary->getCrackerBinaryTypeId()); } diff --git a/src/inc/utils/DownloadUtils.php b/src/inc/utils/DownloadUtils.php new file mode 100644 index 000000000..1717a3972 --- /dev/null +++ b/src/inc/utils/DownloadUtils.php @@ -0,0 +1,136 @@ + $end) { + $c_end = $end; + } + if ($c_start > $c_end || $c_start > $size - 1 || $c_end >= $size) { + return false; + } + $start = $c_start; + $end = $c_end; + fseek($fp, $start); + return true; + } + + /** + * Streams the given file as a download response, handling ETag based + * caching ('If-None-Match') and partial content ('Range') requests. + * + * @param Request $request + * @param Response $response + * @param string $path Absolute path of the file to stream + * @param string|null $displayName Filename to announce to the client, defaults to the base name of the path + * @return Response + * @throws HttpForbiddenException + */ + public static function startDownload(Request $request, Response $response, string $path, ?string $displayName = null): Response { + if ($displayName === null) { + $displayName = basename($path); + } + $size = Util::filesize($path); + $lastModified = filemtime($path); + + $etag = md5($lastModified . $size); + $ifNoneMatch = $request->getHeaderLine('If-None-Match'); + if ($ifNoneMatch === $etag) { + return $response->withStatus(304); + } + + $exp = explode(".", $path); + if ($exp[sizeof($exp) - 1] == '7z') { + $contentType = "application/x-7z-compressed"; + } + else { + $contentType = "application/force-download"; + } + $fp = @fopen($path, "rb"); + + if (!$fp) { + throw new HttpForbiddenException($request, "Can't open the file"); + } + + $start = 0; // Start byte + $end = $size - 1; // End byte + + $status = 200; + if (isset($_SERVER['HTTP_RANGE'])) { + if (!DownloadUtils::handleRangeRequest($start, $end, $size, $fp)) { + fclose($fp); + return $response->withStatus(416) + ->withHeader("Content-Range", "bytes $start-$end/$size"); + } + else { + $status = 206; + } + } + + $length = $end - $start + 1; //content-length + $buffer = 1024 * 100; + $stream = $response->getBody(); + while (!feof($fp) && ($p = ftell($fp)) <= $end) { + if ($p + $buffer > $end) { + $buffer = $end - $p + 1; + } + $stream->write(fread($fp, $buffer)); + } + fclose($fp); + + return $response->withStatus($status) + ->withHeader("Content-Type", $contentType) + ->withHeader("Content-Description", $displayName) + ->withHeader("Content-Disposition", "attachment; filename=\"" . $displayName . "\"") + ->withHeader("Accept-Ranges", "Byte") + ->withHeader("Content-Range", "bytes $start-$end/$size") + ->withHeader("Content-Length", $length) + ->withHeader("ETag", $etag); + } +} diff --git a/src/migrations/mysql/20260903120000_cracker-binary-local-upload.sql b/src/migrations/mysql/20260903120000_cracker-binary-local-upload.sql new file mode 100644 index 000000000..73540f147 --- /dev/null +++ b/src/migrations/mysql/20260903120000_cracker-binary-local-upload.sql @@ -0,0 +1,6 @@ +-- Cracker binaries can be hosted on the server itself: 'filename' stores the archive +-- filename of the locally stored .7z archive, NULL means the binary is downloaded from downloadUrl. +ALTER TABLE CrackerBinary ADD COLUMN filename VARCHAR(100) NULL AFTER binaryName; + +-- downloadUrl can now point to the hashtopolis server download endpoint, so more space is needed +ALTER TABLE CrackerBinary MODIFY downloadUrl VARCHAR(255) NOT NULL; diff --git a/src/migrations/postgres/20260903120000_cracker-binary-local-upload.sql b/src/migrations/postgres/20260903120000_cracker-binary-local-upload.sql new file mode 100644 index 000000000..57a244fbd --- /dev/null +++ b/src/migrations/postgres/20260903120000_cracker-binary-local-upload.sql @@ -0,0 +1,3 @@ +-- Cracker binaries can be hosted on the server itself: 'filename' stores the archive +-- filename of the locally stored .7z archive, NULL means the binary is downloaded from downloadUrl. +ALTER TABLE CrackerBinary ADD COLUMN filename TEXT NULL;