#!/usr/bin/env python3 # -*- coding: utf-8 -*- #*************************************************************************** # _ _ ____ _ # Project ___| | | | _ \| | # / __| | | | |_) | | # | (__| |_| | _ <| |___ # \___|\___/|_| \_\_____| # # Copyright (C) Daniel Stenberg, , et al. # # This software is licensed as described in the file COPYING, which # you should have received as part of this distribution. The terms # are also available at https://curl.se/docs/copyright.html. # # You may opt to use, copy, modify, merge, publish, distribute and/or sell # copies of the Software, and permit persons to whom the Software is # furnished to do so, under the terms of the COPYING file. # # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY # KIND, either express or implied. # # SPDX-License-Identifier: curl # ########################################################################### # import argparse import json import logging import os import re import sys from statistics import mean from typing import Dict, Any, Optional, List from testenv import Env, Httpd, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile log = logging.getLogger(__name__) class ScoreCardError(Exception): pass class ScoreCard: def __init__(self, env: Env, protocol: str, server_descr: str, server_port: int, verbose: int, curl_verbose: int, download_parallel: int = 0, server_addr: Optional[str] = None): self.verbose = verbose self.env = env self.protocol = protocol self.server_descr = server_descr self.server_addr = server_addr self.server_port = server_port self._silent_curl = not curl_verbose self._download_parallel = download_parallel def info(self, msg): if self.verbose > 0: sys.stderr.write(msg) sys.stderr.flush() def handshakes(self) -> Dict[str, Any]: props = {} sample_size = 5 self.info('TLS Handshake\n') for authority in [ 'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org' ]: self.info(f' {authority}...') props[authority] = {} for ipv in ['ipv4', 'ipv6']: self.info(f'{ipv}...') c_samples = [] hs_samples = [] errors = [] for _ in range(sample_size): curl = CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr) args = [ '--http3-only' if self.protocol == 'h3' else '--http2', f'--{ipv}', f'https://{authority}/' ] r = curl.run_direct(args=args, with_stats=True) if r.exit_code == 0 and len(r.stats) == 1: c_samples.append(r.stats[0]['time_connect']) hs_samples.append(r.stats[0]['time_appconnect']) else: errors.append(f'exit={r.exit_code}') props[authority][f'{ipv}-connect'] = mean(c_samples) \ if len(c_samples) else -1 props[authority][f'{ipv}-handshake'] = mean(hs_samples) \ if len(hs_samples) else -1 props[authority][f'{ipv}-errors'] = errors self.info('ok.\n') return props def _make_docs_file(self, docs_dir: str, fname: str, fsize: int): fpath = os.path.join(docs_dir, fname) data1k = 1024*'x' flen = 0 with open(fpath, 'w') as fd: while flen < fsize: fd.write(data1k) flen += len(data1k) return fpath def setup_resources(self, server_docs: str, downloads: Optional[List[int]] = None): for fsize in downloads: label = self.fmt_size(fsize) fname = f'score{label}.data' self._make_docs_file(docs_dir=server_docs, fname=fname, fsize=fsize) self._make_docs_file(docs_dir=server_docs, fname='reqs10.data', fsize=10*1024) def _check_downloads(self, r: ExecResult, count: int): error = '' if r.exit_code != 0: error += f'exit={r.exit_code} ' if r.exit_code != 0 or len(r.stats) != count: error += f'stats={len(r.stats)}/{count} ' fails = [s for s in r.stats if s['response_code'] != 200] if len(fails) > 0: error += f'{len(fails)} failed' return error if len(error) > 0 else None def transfer_single(self, url: str, count: int): sample_size = count count = 1 samples = [] errors = [] profiles = [] self.info('single...') for _ in range(sample_size): curl = CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr) r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, with_headers=False, with_profile=True) err = self._check_downloads(r, count) if err: errors.append(err) else: total_size = sum([s['size_download'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) return { 'count': count, 'samples': sample_size, 'max-parallel': 1, 'speed': mean(samples) if len(samples) else -1, 'errors': errors, 'stats': RunProfile.AverageStats(profiles), } def transfer_serial(self, url: str, count: int): sample_size = 1 samples = [] errors = [] profiles = [] url = f'{url}?[0-{count - 1}]' self.info('serial...') for _ in range(sample_size): curl = CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr) r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, with_headers=False, with_profile=True) err = self._check_downloads(r, count) if err: errors.append(err) else: total_size = sum([s['size_download'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) return { 'count': count, 'samples': sample_size, 'max-parallel': 1, 'speed': mean(samples) if len(samples) else -1, 'errors': errors, 'stats': RunProfile.AverageStats(profiles), } def transfer_parallel(self, url: str, count: int): sample_size = 1 samples = [] errors = [] profiles = [] max_parallel = self._download_parallel if self._download_parallel > 0 else count url = f'{url}?[0-{count - 1}]' self.info('parallel...') for _ in range(sample_size): curl = CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr) r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, with_headers=False, with_profile=True, extra_args=[ '--parallel', '--parallel-max', str(max_parallel) ]) err = self._check_downloads(r, count) if err: errors.append(err) else: total_size = sum([s['size_download'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) return { 'count': count, 'samples': sample_size, 'max-parallel': max_parallel, 'speed': mean(samples) if len(samples) else -1, 'errors': errors, 'stats': RunProfile.AverageStats(profiles), } def download_url(self, label: str, url: str, count: int): self.info(f' {count}x{label}: ') props = { 'single': self.transfer_single(url=url, count=10), } if count > 1: props['serial'] = self.transfer_serial(url=url, count=count) props['parallel'] = self.transfer_parallel(url=url, count=count) self.info('ok.\n') return props def downloads(self, count: int, fsizes: List[int]) -> Dict[str, Any]: scores = {} for fsize in fsizes: label = self.fmt_size(fsize) fname = f'score{label}.data' url = f'https://{self.env.domain1}:{self.server_port}/{fname}' scores[label] = self.download_url(label=label, url=url, count=count) return scores def _check_uploads(self, r: ExecResult, count: int): error = '' if r.exit_code != 0: error += f'exit={r.exit_code} ' if r.exit_code != 0 or len(r.stats) != count: error += f'stats={len(r.stats)}/{count} ' fails = [s for s in r.stats if s['response_code'] != 200] if len(fails) > 0: error += f'{len(fails)} failed' for f in fails: error += f'[{f["response_code"]}]' return error if len(error) > 0 else None def upload_single(self, url: str, fpath: str, count: int): sample_size = count count = 1 samples = [] errors = [] profiles = [] self.info('single...') for _ in range(sample_size): curl = CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr) r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, with_headers=False, with_profile=True) err = self._check_uploads(r, count) if err: errors.append(err) else: total_size = sum([s['size_upload'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) return { 'count': count, 'samples': sample_size, 'max-parallel': 1, 'speed': mean(samples) if len(samples) else -1, 'errors': errors, 'stats': RunProfile.AverageStats(profiles) if len(profiles) else {}, } def upload_serial(self, url: str, fpath: str, count: int): sample_size = 1 samples = [] errors = [] profiles = [] url = f'{url}?id=[0-{count - 1}]' self.info('serial...') for _ in range(sample_size): curl = CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr) r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, with_headers=False, with_profile=True) err = self._check_uploads(r, count) if err: errors.append(err) else: total_size = sum([s['size_upload'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) return { 'count': count, 'samples': sample_size, 'max-parallel': 1, 'speed': mean(samples) if len(samples) else -1, 'errors': errors, 'stats': RunProfile.AverageStats(profiles) if len(profiles) else {}, } def upload_parallel(self, url: str, fpath: str, count: int): sample_size = 1 samples = [] errors = [] profiles = [] max_parallel = count url = f'{url}?id=[0-{count - 1}]' self.info('parallel...') for _ in range(sample_size): curl = CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr) r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, with_headers=False, with_profile=True, extra_args=[ '--parallel', '--parallel-max', str(max_parallel) ]) err = self._check_uploads(r, count) if err: errors.append(err) else: total_size = sum([s['size_upload'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) return { 'count': count, 'samples': sample_size, 'max-parallel': max_parallel, 'speed': mean(samples) if len(samples) else -1, 'errors': errors, 'stats': RunProfile.AverageStats(profiles) if len(profiles) else {}, } def upload_url(self, label: str, url: str, fpath: str, count: int): self.info(f' {count}x{label}: ') props = { 'single': self.upload_single(url=url, fpath=fpath, count=10), } if count > 1: props['serial'] = self.upload_serial(url=url, fpath=fpath, count=count) props['parallel'] = self.upload_parallel(url=url, fpath=fpath, count=count) self.info('ok.\n') return props def uploads(self, count: int, fsizes: List[int]) -> Dict[str, Any]: scores = {} url = f'https://{self.env.domain2}:{self.server_port}/curltest/put' fpaths = {} for fsize in fsizes: label = self.fmt_size(fsize) fname = f'upload{label}.data' fpaths[label] = self._make_docs_file(docs_dir=self.env.gen_dir, fname=fname, fsize=fsize) for label, fpath in fpaths.items(): scores[label] = self.upload_url(label=label, url=url, fpath=fpath, count=count) return scores def do_requests(self, url: str, count: int, max_parallel: int = 1): sample_size = 1 samples = [] errors = [] profiles = [] url = f'{url}?[0-{count - 1}]' extra_args = [ '-w', '%{response_code},\\n', ] if max_parallel > 1: extra_args.extend([ '--parallel', '--parallel-max', str(max_parallel) ]) self.info(f'{max_parallel}...') for _ in range(sample_size): curl = CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr) r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, with_headers=False, with_profile=True, with_stats=False, extra_args=extra_args) if r.exit_code != 0: errors.append(f'exit={r.exit_code}') else: samples.append(count / r.duration.total_seconds()) non_200s = 0 for line in r.stdout.splitlines(): if not line.startswith('200,'): non_200s += 1 if non_200s > 0: errors.append(f'responses != 200: {non_200s}') profiles.append(r.profile) return { 'count': count, 'samples': sample_size, 'speed': mean(samples) if len(samples) else -1, 'errors': errors, 'stats': RunProfile.AverageStats(profiles), } def requests_url(self, url: str, count: int): self.info(f' {url}: ') props = {} # 300 is max in curl, see tool_main.h for m in [1, 6, 25, 50, 100, 300]: props[str(m)] = self.do_requests(url=url, count=count, max_parallel=m) self.info('ok.\n') return props def requests(self, req_count) -> Dict[str, Any]: url = f'https://{self.env.domain1}:{self.server_port}/reqs10.data' return { 'count': req_count, '10KB': self.requests_url(url=url, count=req_count), } def score(self, handshakes: bool = True, downloads: Optional[List[int]] = None, download_count: int = 50, uploads: Optional[List[int]] = None, upload_count: int = 50, req_count=5000, requests: bool = True): self.info(f"scoring {self.protocol} against {self.server_descr}\n") p = {} if self.protocol == 'h3': p['name'] = 'h3' if not self.env.have_h3_curl(): raise ScoreCardError('curl does not support HTTP/3') for lib in ['ngtcp2', 'quiche', 'msh3', 'nghttp3']: if self.env.curl_uses_lib(lib): p['implementation'] = lib break elif self.protocol == 'h2': p['name'] = 'h2' if not self.env.have_h2_curl(): raise ScoreCardError('curl does not support HTTP/2') for lib in ['nghttp2', 'hyper']: if self.env.curl_uses_lib(lib): p['implementation'] = lib break elif self.protocol == 'h1' or self.protocol == 'http/1.1': proto = 'http/1.1' p['name'] = proto p['implementation'] = 'hyper' if self.env.curl_uses_lib('hyper')\ else 'native' else: raise ScoreCardError(f"unknown protocol: {self.protocol}") if 'implementation' not in p: raise ScoreCardError(f'did not recognized {p} lib') p['version'] = Env.curl_lib_version(p['implementation']) score = { 'curl': self.env.curl_fullname(), 'os': self.env.curl_os(), 'protocol': p, 'server': self.server_descr, } if handshakes: score['handshakes'] = self.handshakes() if downloads and len(downloads) > 0: score['downloads'] = self.downloads(count=download_count, fsizes=downloads) if uploads and len(uploads) > 0: score['uploads'] = self.uploads(count=upload_count, fsizes=uploads) if requests: score['requests'] = self.requests(req_count=req_count) self.info("\n") return score def fmt_ms(self, tval): return f'{int(tval*1000)} ms' if tval >= 0 else '--' def fmt_size(self, val): if val >= (1024*1024*1024): return f'{val / (1024*1024*1024):0.000f}GB' elif val >= (1024 * 1024): return f'{val / (1024*1024):0.000f}MB' elif val >= 1024: return f'{val / 1024:0.000f}KB' else: return f'{val:0.000f}B' def fmt_mbs(self, val): return f'{val/(1024*1024):0.000f} MB/s' if val >= 0 else '--' def fmt_reqs(self, val): return f'{val:0.000f} r/s' if val >= 0 else '--' def print_score(self, score): print(f'{score["protocol"]["name"].upper()} in {score["curl"]}') if 'handshakes' in score: print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}') print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} ' f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}') for key, val in score["handshakes"].items(): print(f' {key:<17} {self.fmt_ms(val["ipv4-connect"]):>12} ' f'{self.fmt_ms(val["ipv4-handshake"]):>12} ' f'{self.fmt_ms(val["ipv6-connect"]):>12} ' f'{self.fmt_ms(val["ipv6-handshake"]):>12} ' f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}' ) if 'downloads' in score: # get the key names of all sizes and measurements made sizes = [] measures = [] m_names = {} mcol_width = 12 mcol_sw = 17 for sskey, ssval in score['downloads'].items(): if isinstance(ssval, str): continue if sskey not in sizes: sizes.append(sskey) for mkey, mval in score['downloads'][sskey].items(): if mkey not in measures: measures.append(mkey) m_names[mkey] = f'{mkey}({mval["count"]}x{mval["max-parallel"]})' print(f'Downloads from {score["server"]}') print(f' {"Size":>8}', end='') for m in measures: print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') print(f' {"Errors":^20}') for size in score['downloads']: size_score = score['downloads'][size] print(f' {size:>8}', end='') errors = [] for val in size_score.values(): if 'errors' in val: errors.extend(val['errors']) for m in measures: if m in size_score: print(f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}', end='') s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' print(f' {s:<{mcol_sw}}', end='') else: print(' '*mcol_width, end='') if len(errors): print(f' {"/".join(errors):<20}') else: print(f' {"-":^20}') if 'uploads' in score: # get the key names of all sizes and measurements made sizes = [] measures = [] m_names = {} mcol_width = 12 mcol_sw = 17 for sskey, ssval in score['uploads'].items(): if isinstance(ssval, str): continue if sskey not in sizes: sizes.append(sskey) for mkey, mval in ssval.items(): if mkey not in measures: measures.append(mkey) m_names[mkey] = f'{mkey}({mval["count"]}x{mval["max-parallel"]})' print(f'Uploads to {score["server"]}') print(f' {"Size":>8}', end='') for m in measures: print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') print(f' {"Errors":^20}') for size in sizes: size_score = score['uploads'][size] print(f' {size:>8}', end='') errors = [] for val in size_score.values(): if 'errors' in val: errors.extend(val['errors']) for m in measures: if m in size_score: print(f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}', end='') stats = size_score[m]["stats"] if 'cpu' in stats: s = f'[{stats["cpu"]:>.1f}%/{self.fmt_size(stats["rss"])}]' else: s = '[???/???]' print(f' {s:<{mcol_sw}}', end='') else: print(' '*mcol_width, end='') if len(errors): print(f' {"/".join(errors):<20}') else: print(f' {"-":^20}') if 'requests' in score: sizes = [] measures = [] m_names = {} mcol_width = 9 mcol_sw = 13 for sskey, ssval in score['requests'].items(): if isinstance(ssval, (str, int)): continue if sskey not in sizes: sizes.append(sskey) for mkey in score['requests'][sskey]: if mkey not in measures: measures.append(mkey) m_names[mkey] = f'{mkey}' print('Requests (max parallel) to {score["server"]}') print(f' {"Size":>6} {"Reqs":>6}', end='') for m in measures: print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') print(f' {"Errors":^10}') for size in sizes: size_score = score['requests'][size] count = score['requests']['count'] print(f' {size:>6} {count:>6}', end='') errors = [] for val in size_score.values(): if 'errors' in val: errors.extend(val['errors']) for m in measures: if m in size_score: print(f' {self.fmt_reqs(size_score[m]["speed"]):>{mcol_width}}', end='') s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' print(f' {s:<{mcol_sw}}', end='') else: print(' '*mcol_width, end='') if len(errors): print(f' {"/".join(errors):<10}') else: print(f' {"-":^10}') def parse_size(s): m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE) if m is None: raise Exception(f'unrecognized size: {s}') size = int(m.group(1)) if not m.group(2): pass elif m.group(2).lower() == 'kb': size *= 1024 elif m.group(2).lower() == 'mb': size *= 1024 * 1024 elif m.group(2).lower() == 'gb': size *= 1024 * 1024 * 1024 return size def main(): parser = argparse.ArgumentParser(prog='scorecard', description=""" Run a range of tests to give a scorecard for a HTTP protocol 'h3' or 'h2' implementation in curl. """) parser.add_argument("-v", "--verbose", action='count', default=1, help="log more output on stderr") parser.add_argument("-j", "--json", action='store_true', default=False, help="print json instead of text") parser.add_argument("-H", "--handshakes", action='store_true', default=False, help="evaluate handshakes only") parser.add_argument("-d", "--downloads", action='store_true', default=False, help="evaluate downloads") parser.add_argument("--download", action='append', type=str, default=None, help="evaluate download size") parser.add_argument("--download-count", action='store', type=int, default=50, help="perform that many downloads") parser.add_argument("--download-parallel", action='store', type=int, default=0, help="perform that many downloads in parallel (default all)") parser.add_argument("-u", "--uploads", action='store_true', default=False, help="evaluate uploads") parser.add_argument("--upload", action='append', type=str, default=None, help="evaluate upload size") parser.add_argument("--upload-count", action='store', type=int, default=50, help="perform that many uploads") parser.add_argument("-r", "--requests", action='store_true', default=False, help="evaluate requests") parser.add_argument("--request-count", action='store', type=int, default=5000, help="perform that many requests") parser.add_argument("--httpd", action='store_true', default=False, help="evaluate httpd server only") parser.add_argument("--caddy", action='store_true', default=False, help="evaluate caddy server only") parser.add_argument("--curl-verbose", action='store_true', default=False, help="run curl with `-v`") parser.add_argument("protocol", default='h2', nargs='?', help="Name of protocol to score") parser.add_argument("--start-only", action='store_true', default=False, help="only start the servers") parser.add_argument("--remote", action='store', type=str, default=None, help="score against the remote server at :") args = parser.parse_args() if args.verbose > 0: console = logging.StreamHandler() console.setLevel(logging.INFO) console.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) logging.getLogger('').addHandler(console) protocol = args.protocol handshakes = True downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] if args.download is not None: downloads = [] for x in args.download: downloads.extend([parse_size(s) for s in x.split(',')]) uploads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] if args.upload is not None: uploads = [] for x in args.upload: uploads.extend([parse_size(s) for s in x.split(',')]) requests = True if args.downloads or args.uploads or args.requests or args.handshakes: handshakes = args.handshakes if not args.downloads: downloads = None if not args.uploads: uploads = None requests = args.requests test_httpd = protocol != 'h3' test_caddy = True if args.caddy or args.httpd: test_caddy = args.caddy test_httpd = args.httpd rv = 0 env = Env() env.setup() env.test_timeout = None httpd = None nghttpx = None caddy = None try: cards = [] if args.remote: m = re.match(r'^(.+):(\d+)$', args.remote) if m is None: raise ScoreCardError(f'unable to parse ip:port from --remote {args.remote}') test_httpd = False test_caddy = False remote_addr = m.group(1) remote_port = int(m.group(2)) card = ScoreCard(env=env, protocol=protocol, server_descr=f'Server at {args.remote}', server_addr=remote_addr, server_port=remote_port, verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel) cards.append(card) if test_httpd: httpd = Httpd(env=env) assert httpd.exists(), \ f'httpd not found: {env.httpd}' httpd.clear_logs() server_docs = httpd.docs_dir assert httpd.start() if protocol == 'h3': nghttpx = NghttpxQuic(env=env) nghttpx.clear_logs() assert nghttpx.start() server_descr = f'nghttpx: https:{env.h3_port} [backend httpd: {env.httpd_version()}, https:{env.https_port}]' server_port = env.h3_port else: server_descr = f'httpd: {env.httpd_version()}, http:{env.http_port} https:{env.https_port}' server_port = env.https_port card = ScoreCard(env=env, protocol=protocol, server_descr=server_descr, server_port=server_port, verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel) card.setup_resources(server_docs, downloads) cards.append(card) if test_caddy and env.caddy: backend = '' if uploads and httpd is None: backend = f' [backend httpd: {env.httpd_version()}, http:{env.http_port} https:{env.https_port}]' httpd = Httpd(env=env) assert httpd.exists(), \ f'httpd not found: {env.httpd}' httpd.clear_logs() assert httpd.start() caddy = Caddy(env=env) caddy.clear_logs() assert caddy.start() server_descr = f'Caddy: {env.caddy_version()}, http:{env.caddy_http_port} https:{env.caddy_https_port}{backend}' server_port = caddy.port server_docs = caddy.docs_dir card = ScoreCard(env=env, protocol=protocol, server_descr=server_descr, server_port=server_port, verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel) card.setup_resources(server_docs, downloads) cards.append(card) if args.start_only: print('started servers:') for card in cards: print(f'{card.server_descr}') sys.stderr.write('press [RETURN] to finish') sys.stderr.flush() sys.stdin.readline() else: for card in cards: score = card.score(handshakes=handshakes, downloads=downloads, download_count=args.download_count, uploads=uploads, upload_count=args.upload_count, req_count=args.request_count, requests=requests) if args.json: print(json.JSONEncoder(indent=2).encode(score)) else: card.print_score(score) except ScoreCardError as ex: sys.stderr.write(f"ERROR: {ex}\n") rv = 1 except KeyboardInterrupt: log.warning("aborted") rv = 1 finally: if caddy: caddy.stop() if nghttpx: nghttpx.stop(wait_dead=False) if httpd: httpd.stop() sys.exit(rv) if __name__ == "__main__": main()