936 lines
37 KiB
Python
936 lines
37 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
#***************************************************************************
|
|
# _ _ ____ _
|
|
# Project ___| | | | _ \| |
|
|
# / __| | | | |_) | |
|
|
# | (__| |_| | _ <| |___
|
|
# \___|\___/|_| \_\_____|
|
|
#
|
|
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
|
#
|
|
# This software is licensed as described in the file COPYING, which
|
|
# you should have received as part of this distribution. The terms
|
|
# are also available at https://curl.se/docs/copyright.html.
|
|
#
|
|
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
|
# copies of the Software, and permit persons to whom the Software is
|
|
# furnished to do so, under the terms of the COPYING file.
|
|
#
|
|
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
|
# KIND, either express or implied.
|
|
#
|
|
# SPDX-License-Identifier: curl
|
|
#
|
|
###########################################################################
|
|
#
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import sys
|
|
from statistics import mean
|
|
from typing import Dict, Any, Optional, List
|
|
|
|
from testenv import Env, Httpd, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile, Dante
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class ScoreCardError(Exception):
|
|
pass
|
|
|
|
|
|
class Card:
|
|
@classmethod
|
|
def fmt_ms(cls, tval):
|
|
return f'{int(tval*1000)} ms' if tval >= 0 else '--'
|
|
|
|
@classmethod
|
|
def fmt_size(cls, val):
|
|
if val >= (1024*1024*1024):
|
|
return f'{val / (1024*1024*1024):0.000f}GB'
|
|
elif val >= (1024 * 1024):
|
|
return f'{val / (1024*1024):0.000f}MB'
|
|
elif val >= 1024:
|
|
return f'{val / 1024:0.000f}KB'
|
|
else:
|
|
return f'{val:0.000f}B'
|
|
|
|
@classmethod
|
|
def fmt_mbs(cls, val):
|
|
if val is None or val < 0:
|
|
return '--'
|
|
if val >= (1024*1024):
|
|
return f'{val/(1024*1024):0.000f} MB/s'
|
|
elif val >= 1024:
|
|
return f'{val / 1024:0.000f} KB/s'
|
|
else:
|
|
return f'{val:0.000f} B/s'
|
|
|
|
@classmethod
|
|
def fmt_reqs(cls, val):
|
|
return f'{val:0.000f} r/s' if val >= 0 else '--'
|
|
|
|
@classmethod
|
|
def mk_mbs_cell(cls, samples, profiles, errors):
|
|
val = mean(samples) if len(samples) else -1
|
|
cell = {
|
|
'val': val,
|
|
'sval': Card.fmt_mbs(val) if val >= 0 else '--',
|
|
}
|
|
if len(profiles):
|
|
cell['stats'] = RunProfile.AverageStats(profiles)
|
|
if len(errors):
|
|
cell['errors'] = errors
|
|
return cell
|
|
|
|
@classmethod
|
|
def mk_reqs_cell(cls, samples, profiles, errors):
|
|
val = mean(samples) if len(samples) else -1
|
|
cell = {
|
|
'val': val,
|
|
'sval': Card.fmt_reqs(val) if val >= 0 else '--',
|
|
}
|
|
if len(profiles):
|
|
cell['stats'] = RunProfile.AverageStats(profiles)
|
|
if len(errors):
|
|
cell['errors'] = errors
|
|
return cell
|
|
|
|
@classmethod
|
|
def parse_size(cls, s):
|
|
m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE)
|
|
if m is None:
|
|
raise Exception(f'unrecognized size: {s}')
|
|
size = int(m.group(1))
|
|
if not m.group(2):
|
|
pass
|
|
elif m.group(2).lower() == 'kb':
|
|
size *= 1024
|
|
elif m.group(2).lower() == 'mb':
|
|
size *= 1024 * 1024
|
|
elif m.group(2).lower() == 'gb':
|
|
size *= 1024 * 1024 * 1024
|
|
return size
|
|
|
|
@classmethod
|
|
def print_score(cls, score):
|
|
print(f'Scorecard curl, protocol {score["meta"]["protocol"]} '
|
|
f'via {score["meta"]["implementation"]}/'
|
|
f'{score["meta"]["implementation_version"]}')
|
|
print(f'Date: {score["meta"]["date"]}')
|
|
if 'curl_V' in score["meta"]:
|
|
print(f'Version: {score["meta"]["curl_V"]}')
|
|
if 'curl_features' in score["meta"]:
|
|
print(f'Features: {score["meta"]["curl_features"]}')
|
|
if 'limit-rate' in score['meta']:
|
|
print(f'--limit-rate: {score["meta"]["limit-rate"]}')
|
|
print(f'Samples Size: {score["meta"]["samples"]}')
|
|
if 'handshakes' in score:
|
|
print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}')
|
|
print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} '
|
|
f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}')
|
|
for key, val in score["handshakes"].items():
|
|
print(f' {key:<17} {Card.fmt_ms(val["ipv4-connect"]):>12} '
|
|
f'{Card.fmt_ms(val["ipv4-handshake"]):>12} '
|
|
f'{Card.fmt_ms(val["ipv6-connect"]):>12} '
|
|
f'{Card.fmt_ms(val["ipv6-handshake"]):>12} '
|
|
f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}'
|
|
)
|
|
for name in ['downloads', 'uploads', 'requests']:
|
|
if name in score:
|
|
Card.print_score_table(score[name])
|
|
|
|
@classmethod
|
|
def print_score_table(cls, score):
|
|
cols = score['cols']
|
|
rows = score['rows']
|
|
colw = []
|
|
statw = 13
|
|
errors = []
|
|
col_has_stats = []
|
|
for idx, col in enumerate(cols):
|
|
cellw = max([len(r[idx]["sval"]) for r in rows])
|
|
colw.append(max(cellw, len(col)))
|
|
col_has_stats.append(False)
|
|
for row in rows:
|
|
if 'stats' in row[idx]:
|
|
col_has_stats[idx] = True
|
|
break
|
|
if 'title' in score['meta']:
|
|
print(score['meta']['title'])
|
|
for idx, col in enumerate(cols):
|
|
if col_has_stats[idx]:
|
|
print(f' {col:>{colw[idx]}} {"[cpu/rss]":<{statw}}', end='')
|
|
else:
|
|
print(f' {col:>{colw[idx]}}', end='')
|
|
print('')
|
|
for row in rows:
|
|
for idx, cell in enumerate(row):
|
|
print(f' {cell["sval"]:>{colw[idx]}}', end='')
|
|
if col_has_stats[idx]:
|
|
if 'stats' in cell:
|
|
s = f'[{cell["stats"]["cpu"]:>.1f}%' \
|
|
f'/{Card.fmt_size(cell["stats"]["rss"])}]'
|
|
else:
|
|
s = ''
|
|
print(f' {s:<{statw}}', end='')
|
|
if 'errors' in cell:
|
|
errors.extend(cell['errors'])
|
|
print('')
|
|
if len(errors):
|
|
print(f'Errors: {errors}')
|
|
|
|
|
|
class ScoreRunner:
|
|
|
|
def __init__(self, env: Env,
|
|
protocol: str,
|
|
server_descr: str,
|
|
server_port: int,
|
|
verbose: int,
|
|
curl_verbose: int,
|
|
download_parallel: int = 0,
|
|
upload_parallel: int = 0,
|
|
server_addr: Optional[str] = None,
|
|
with_flame: bool = False,
|
|
socks_args: Optional[List[str]] = None,
|
|
limit_rate: Optional[str] = None):
|
|
self.verbose = verbose
|
|
self.env = env
|
|
self.protocol = protocol
|
|
self.server_descr = server_descr
|
|
self.server_addr = server_addr
|
|
self.server_port = server_port
|
|
self._silent_curl = not curl_verbose
|
|
self._download_parallel = download_parallel
|
|
self._upload_parallel = upload_parallel
|
|
self._with_flame = with_flame
|
|
self._socks_args = socks_args
|
|
self._limit_rate = limit_rate
|
|
|
|
def info(self, msg):
|
|
if self.verbose > 0:
|
|
sys.stderr.write(msg)
|
|
sys.stderr.flush()
|
|
|
|
def mk_curl_client(self):
|
|
return CurlClient(env=self.env, silent=self._silent_curl,
|
|
server_addr=self.server_addr,
|
|
with_flame=self._with_flame,
|
|
socks_args=self._socks_args)
|
|
|
|
def handshakes(self) -> Dict[str, Any]:
|
|
props = {}
|
|
sample_size = 5
|
|
self.info('TLS Handshake\n')
|
|
for authority in [
|
|
'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org'
|
|
]:
|
|
self.info(f' {authority}...')
|
|
props[authority] = {}
|
|
for ipv in ['ipv4', 'ipv6']:
|
|
self.info(f'{ipv}...')
|
|
c_samples = []
|
|
hs_samples = []
|
|
errors = []
|
|
for _ in range(sample_size):
|
|
curl = self.mk_curl_client()
|
|
args = [
|
|
'--http3-only' if self.protocol == 'h3' else '--http2',
|
|
f'--{ipv}', f'https://{authority}/'
|
|
]
|
|
r = curl.run_direct(args=args, with_stats=True)
|
|
if r.exit_code == 0 and len(r.stats) == 1:
|
|
c_samples.append(r.stats[0]['time_connect'])
|
|
hs_samples.append(r.stats[0]['time_appconnect'])
|
|
else:
|
|
errors.append(f'exit={r.exit_code}')
|
|
props[authority][f'{ipv}-connect'] = mean(c_samples) \
|
|
if len(c_samples) else -1
|
|
props[authority][f'{ipv}-handshake'] = mean(hs_samples) \
|
|
if len(hs_samples) else -1
|
|
props[authority][f'{ipv}-errors'] = errors
|
|
self.info('ok.\n')
|
|
return props
|
|
|
|
def _make_docs_file(self, docs_dir: str, fname: str, fsize: int):
|
|
fpath = os.path.join(docs_dir, fname)
|
|
data1k = 1024*'x'
|
|
flen = 0
|
|
with open(fpath, 'w') as fd:
|
|
while flen < fsize:
|
|
fd.write(data1k)
|
|
flen += len(data1k)
|
|
return fpath
|
|
|
|
def setup_resources(self, server_docs: str,
|
|
downloads: Optional[List[int]] = None):
|
|
if downloads is not None:
|
|
for fsize in downloads:
|
|
label = Card.fmt_size(fsize)
|
|
fname = f'score{label}.data'
|
|
self._make_docs_file(docs_dir=server_docs,
|
|
fname=fname, fsize=fsize)
|
|
self._make_docs_file(docs_dir=server_docs,
|
|
fname='reqs10.data', fsize=10*1024)
|
|
|
|
def _check_downloads(self, r: ExecResult, count: int):
|
|
error = ''
|
|
if r.exit_code != 0:
|
|
error += f'exit={r.exit_code} '
|
|
if r.exit_code != 0 or len(r.stats) != count:
|
|
error += f'stats={len(r.stats)}/{count} '
|
|
fails = [s for s in r.stats if s['response_code'] != 200]
|
|
if len(fails) > 0:
|
|
error += f'{len(fails)} failed'
|
|
return error if len(error) > 0 else None
|
|
|
|
def dl_single(self, url: str, nsamples: int = 1):
|
|
count = 1
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
self.info('single...')
|
|
for _ in range(nsamples):
|
|
curl = self.mk_curl_client()
|
|
r = curl.http_download(urls=[url], alpn_proto=self.protocol,
|
|
no_save=True, with_headers=False,
|
|
with_profile=True,
|
|
limit_rate=self._limit_rate)
|
|
err = self._check_downloads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s['size_download'] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return Card.mk_mbs_cell(samples, profiles, errors)
|
|
|
|
def dl_serial(self, url: str, count: int, nsamples: int = 1):
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
url = f'{url}?[0-{count - 1}]'
|
|
self.info('serial...')
|
|
for _ in range(nsamples):
|
|
curl = self.mk_curl_client()
|
|
r = curl.http_download(urls=[url], alpn_proto=self.protocol,
|
|
no_save=True,
|
|
with_headers=False,
|
|
with_profile=True,
|
|
limit_rate=self._limit_rate)
|
|
err = self._check_downloads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s['size_download'] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return Card.mk_mbs_cell(samples, profiles, errors)
|
|
|
|
def dl_parallel(self, url: str, count: int, nsamples: int = 1):
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
max_parallel = self._download_parallel if self._download_parallel > 0 else count
|
|
url = f'{url}?[0-{count - 1}]'
|
|
self.info('parallel...')
|
|
for _ in range(nsamples):
|
|
curl = self.mk_curl_client()
|
|
r = curl.http_download(urls=[url], alpn_proto=self.protocol,
|
|
no_save=True,
|
|
with_headers=False,
|
|
with_profile=True,
|
|
limit_rate=self._limit_rate,
|
|
extra_args=[
|
|
'--parallel',
|
|
'--parallel-max', str(max_parallel)
|
|
])
|
|
err = self._check_downloads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s['size_download'] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return Card.mk_mbs_cell(samples, profiles, errors)
|
|
|
|
def downloads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]:
|
|
nsamples = meta['samples']
|
|
max_parallel = self._download_parallel if self._download_parallel > 0 else count
|
|
cols = ['size']
|
|
if not self._download_parallel:
|
|
cols.append('single')
|
|
if count > 1:
|
|
cols.append(f'serial({count})')
|
|
if count > 1:
|
|
cols.append(f'parallel({count}x{max_parallel})')
|
|
rows = []
|
|
for fsize in fsizes:
|
|
row = [{
|
|
'val': fsize,
|
|
'sval': Card.fmt_size(fsize)
|
|
}]
|
|
self.info(f'{row[0]["sval"]} downloads...')
|
|
url = f'https://{self.env.domain1}:{self.server_port}/score{row[0]["sval"]}.data'
|
|
if 'single' in cols:
|
|
row.append(self.dl_single(url=url, nsamples=nsamples))
|
|
if count > 1:
|
|
if 'single' in cols:
|
|
row.append(self.dl_serial(url=url, count=count, nsamples=nsamples))
|
|
row.append(self.dl_parallel(url=url, count=count, nsamples=nsamples))
|
|
rows.append(row)
|
|
self.info('done.\n')
|
|
title = f'Downloads from {meta["server"]}'
|
|
if self._socks_args:
|
|
title += f' via {self._socks_args}'
|
|
return {
|
|
'meta': {
|
|
'title': title,
|
|
'count': count,
|
|
'max-parallel': max_parallel,
|
|
},
|
|
'cols': cols,
|
|
'rows': rows,
|
|
}
|
|
|
|
def _check_uploads(self, r: ExecResult, count: int):
|
|
error = ''
|
|
if r.exit_code != 0:
|
|
error += f'exit={r.exit_code} '
|
|
if r.exit_code != 0 or len(r.stats) != count:
|
|
error += f'stats={len(r.stats)}/{count} '
|
|
fails = [s for s in r.stats if s['response_code'] != 200]
|
|
if len(fails) > 0:
|
|
error += f'{len(fails)} failed'
|
|
for f in fails:
|
|
error += f'[{f["response_code"]}]'
|
|
return error if len(error) > 0 else None
|
|
|
|
def ul_single(self, url: str, fpath: str, nsamples: int = 1):
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
self.info('single...')
|
|
for _ in range(nsamples):
|
|
curl = self.mk_curl_client()
|
|
r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol,
|
|
with_headers=False, with_profile=True)
|
|
err = self._check_uploads(r, 1)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s['size_upload'] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return Card.mk_mbs_cell(samples, profiles, errors)
|
|
|
|
def ul_serial(self, url: str, fpath: str, count: int, nsamples: int = 1):
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
url = f'{url}?id=[0-{count - 1}]'
|
|
self.info('serial...')
|
|
for _ in range(nsamples):
|
|
curl = self.mk_curl_client()
|
|
r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol,
|
|
with_headers=False, with_profile=True)
|
|
err = self._check_uploads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s['size_upload'] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return Card.mk_mbs_cell(samples, profiles, errors)
|
|
|
|
def ul_parallel(self, url: str, fpath: str, count: int, nsamples: int = 1):
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
max_parallel = self._download_parallel if self._download_parallel > 0 else count
|
|
url = f'{url}?id=[0-{count - 1}]'
|
|
self.info('parallel...')
|
|
for _ in range(nsamples):
|
|
curl = self.mk_curl_client()
|
|
r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol,
|
|
with_headers=False, with_profile=True,
|
|
extra_args=[
|
|
'--parallel',
|
|
'--parallel-max', str(max_parallel)
|
|
])
|
|
err = self._check_uploads(r, count)
|
|
if err:
|
|
errors.append(err)
|
|
else:
|
|
total_size = sum([s['size_upload'] for s in r.stats])
|
|
samples.append(total_size / r.duration.total_seconds())
|
|
profiles.append(r.profile)
|
|
return Card.mk_mbs_cell(samples, profiles, errors)
|
|
|
|
def uploads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]:
|
|
nsamples = meta['samples']
|
|
max_parallel = self._upload_parallel if self._upload_parallel > 0 else count
|
|
cols = ['size']
|
|
if not self._upload_parallel:
|
|
cols.append('single')
|
|
if count > 1:
|
|
cols.append(f'serial({count})')
|
|
if count > 1:
|
|
cols.append(f'parallel({count}x{max_parallel})')
|
|
rows = []
|
|
for fsize in fsizes:
|
|
row = [{
|
|
'val': fsize,
|
|
'sval': Card.fmt_size(fsize)
|
|
}]
|
|
self.info(f'{row[0]["sval"]} uploads...')
|
|
url = f'https://{self.env.domain2}:{self.server_port}/curltest/put'
|
|
fname = f'upload{row[0]["sval"]}.data'
|
|
fpath = self._make_docs_file(docs_dir=self.env.gen_dir,
|
|
fname=fname, fsize=fsize)
|
|
if 'single' in cols:
|
|
row.append(self.ul_single(url=url, fpath=fpath, nsamples=nsamples))
|
|
if count > 1:
|
|
if 'single' in cols:
|
|
row.append(self.ul_serial(url=url, fpath=fpath, count=count, nsamples=nsamples))
|
|
row.append(self.ul_parallel(url=url, fpath=fpath, count=count, nsamples=nsamples))
|
|
rows.append(row)
|
|
self.info('done.\n')
|
|
title = f'Uploads to {meta["server"]}'
|
|
if self._socks_args:
|
|
title += f' via {self._socks_args}'
|
|
return {
|
|
'meta': {
|
|
'title': title,
|
|
'count': count,
|
|
'max-parallel': max_parallel,
|
|
},
|
|
'cols': cols,
|
|
'rows': rows,
|
|
}
|
|
|
|
def do_requests(self, url: str, count: int, max_parallel: int = 1, nsamples: int = 1):
|
|
samples = []
|
|
errors = []
|
|
profiles = []
|
|
url = f'{url}?[0-{count - 1}]'
|
|
extra_args = [
|
|
'-w', '%{response_code},\\n',
|
|
]
|
|
if max_parallel > 1:
|
|
extra_args.extend([
|
|
'--parallel', '--parallel-max', str(max_parallel)
|
|
])
|
|
self.info(f'{max_parallel}...')
|
|
for _ in range(nsamples):
|
|
curl = self.mk_curl_client()
|
|
r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True,
|
|
with_headers=False, with_profile=True,
|
|
with_stats=False, extra_args=extra_args)
|
|
if r.exit_code != 0:
|
|
errors.append(f'exit={r.exit_code}')
|
|
else:
|
|
samples.append(count / r.duration.total_seconds())
|
|
non_200s = 0
|
|
for line in r.stdout.splitlines():
|
|
if not line.startswith('200,'):
|
|
non_200s += 1
|
|
if non_200s > 0:
|
|
errors.append(f'responses != 200: {non_200s}')
|
|
profiles.append(r.profile)
|
|
return Card.mk_reqs_cell(samples, profiles, errors)
|
|
|
|
def requests(self, count: int, meta: Dict[str, Any]) -> Dict[str, Any]:
|
|
url = f'https://{self.env.domain1}:{self.server_port}/reqs10.data'
|
|
fsize = 10*1024
|
|
cols = ['size', 'total']
|
|
rows = []
|
|
mparallel = meta['request_parallels']
|
|
cols.extend([f'{mp} max' for mp in mparallel])
|
|
row = [{
|
|
'val': fsize,
|
|
'sval': Card.fmt_size(fsize)
|
|
},{
|
|
'val': count,
|
|
'sval': f'{count}',
|
|
}]
|
|
self.info('requests, max parallel...')
|
|
row.extend([self.do_requests(url=url, count=count,
|
|
max_parallel=mp, nsamples=meta["samples"])
|
|
for mp in mparallel])
|
|
rows.append(row)
|
|
self.info('done.\n')
|
|
title = f'Requests in parallel to {meta["server"]}'
|
|
if self._socks_args:
|
|
title += f' via {self._socks_args}'
|
|
return {
|
|
'meta': {
|
|
'title': title,
|
|
'count': count,
|
|
},
|
|
'cols': cols,
|
|
'rows': rows,
|
|
}
|
|
|
|
def score(self,
|
|
handshakes: bool = True,
|
|
downloads: Optional[List[int]] = None,
|
|
download_count: int = 50,
|
|
uploads: Optional[List[int]] = None,
|
|
upload_count: int = 50,
|
|
req_count=5000,
|
|
request_parallels=None,
|
|
nsamples: int = 1,
|
|
requests: bool = True):
|
|
self.info(f"scoring {self.protocol} against {self.server_descr}\n")
|
|
|
|
score = {
|
|
'meta': {
|
|
'curl_version': self.env.curl_version(),
|
|
'curl_V': self.env.curl_fullname(),
|
|
'curl_features': self.env.curl_features_string(),
|
|
'os': self.env.curl_os(),
|
|
'server': self.server_descr,
|
|
'samples': nsamples,
|
|
'date': f'{datetime.datetime.now(tz=datetime.timezone.utc).isoformat()}',
|
|
}
|
|
}
|
|
if self._limit_rate:
|
|
score['meta']['limit-rate'] = self._limit_rate
|
|
|
|
if self.protocol == 'h3':
|
|
score['meta']['protocol'] = 'h3'
|
|
if not self.env.have_h3_curl():
|
|
raise ScoreCardError('curl does not support HTTP/3')
|
|
for lib in ['ngtcp2', 'quiche', 'nghttp3']:
|
|
if self.env.curl_uses_lib(lib):
|
|
score['meta']['implementation'] = lib
|
|
break
|
|
elif self.protocol == 'h2':
|
|
score['meta']['protocol'] = 'h2'
|
|
if not self.env.have_h2_curl():
|
|
raise ScoreCardError('curl does not support HTTP/2')
|
|
for lib in ['nghttp2']:
|
|
if self.env.curl_uses_lib(lib):
|
|
score['meta']['implementation'] = lib
|
|
break
|
|
elif self.protocol == 'h1' or self.protocol == 'http/1.1':
|
|
score['meta']['protocol'] = 'http/1.1'
|
|
score['meta']['implementation'] = 'native'
|
|
else:
|
|
raise ScoreCardError(f"unknown protocol: {self.protocol}")
|
|
|
|
if 'implementation' not in score['meta']:
|
|
raise ScoreCardError('did not recognized protocol lib')
|
|
score['meta']['implementation_version'] = Env.curl_lib_version(score['meta']['implementation'])
|
|
|
|
if handshakes:
|
|
score['handshakes'] = self.handshakes()
|
|
if downloads and len(downloads) > 0:
|
|
score['downloads'] = self.downloads(count=download_count,
|
|
fsizes=downloads,
|
|
meta=score['meta'])
|
|
if uploads and len(uploads) > 0:
|
|
score['uploads'] = self.uploads(count=upload_count,
|
|
fsizes=uploads,
|
|
meta=score['meta'])
|
|
if requests:
|
|
if request_parallels is None:
|
|
request_parallels = [1, 6, 25, 50, 100, 300]
|
|
score['meta']['request_parallels'] = request_parallels
|
|
score['requests'] = self.requests(count=req_count, meta=score['meta'])
|
|
return score
|
|
|
|
|
|
def run_score(args, protocol):
|
|
if protocol not in ['http/1.1', 'h1', 'h2', 'h3']:
|
|
sys.stderr.write(f'ERROR: protocol "{protocol}" not known to scorecard\n')
|
|
sys.exit(1)
|
|
if protocol == 'h1':
|
|
protocol = 'http/1.1'
|
|
|
|
handshakes = True
|
|
downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024]
|
|
if args.download_sizes is not None:
|
|
downloads = []
|
|
for x in args.download_sizes:
|
|
downloads.extend([Card.parse_size(s) for s in x.split(',')])
|
|
|
|
uploads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024]
|
|
if args.upload_sizes is not None:
|
|
uploads = []
|
|
for x in args.upload_sizes:
|
|
uploads.extend([Card.parse_size(s) for s in x.split(',')])
|
|
|
|
requests = True
|
|
request_parallels = None
|
|
if args.request_parallels:
|
|
request_parallels = []
|
|
for x in args.request_parallels:
|
|
request_parallels.extend([int(s) for s in x.split(',')])
|
|
|
|
|
|
if args.downloads or args.uploads or args.requests or args.handshakes:
|
|
handshakes = args.handshakes
|
|
if not args.downloads:
|
|
downloads = None
|
|
if not args.uploads:
|
|
uploads = None
|
|
requests = args.requests
|
|
|
|
test_httpd = protocol != 'h3'
|
|
test_caddy = protocol == 'h3'
|
|
if args.caddy or args.httpd:
|
|
test_caddy = args.caddy
|
|
test_httpd = args.httpd
|
|
|
|
rv = 0
|
|
env = Env()
|
|
env.setup()
|
|
env.test_timeout = None
|
|
|
|
sockd = None
|
|
socks_args = None
|
|
if args.socks4 and args.socks5:
|
|
raise ScoreCardError('unable to run --socks4 and --socks5 together')
|
|
elif args.socks4 or args.socks5:
|
|
sockd = Dante(env=env)
|
|
if sockd:
|
|
assert sockd.initial_start()
|
|
socks_args = [
|
|
'--socks4' if args.socks4 else '--socks5',
|
|
f'127.0.0.1:{sockd.port}',
|
|
]
|
|
|
|
httpd = None
|
|
nghttpx = None
|
|
caddy = None
|
|
try:
|
|
cards = []
|
|
|
|
if args.remote:
|
|
m = re.match(r'^(.+):(\d+)$', args.remote)
|
|
if m is None:
|
|
raise ScoreCardError(f'unable to parse ip:port from --remote {args.remote}')
|
|
test_httpd = False
|
|
test_caddy = False
|
|
remote_addr = m.group(1)
|
|
remote_port = int(m.group(2))
|
|
card = ScoreRunner(env=env,
|
|
protocol=protocol,
|
|
server_descr=f'Server at {args.remote}',
|
|
server_addr=remote_addr,
|
|
server_port=remote_port,
|
|
verbose=args.verbose,
|
|
curl_verbose=args.curl_verbose,
|
|
download_parallel=args.download_parallel,
|
|
upload_parallel=args.upload_parallel,
|
|
with_flame=args.flame,
|
|
socks_args=socks_args,
|
|
limit_rate=args.limit_rate)
|
|
cards.append(card)
|
|
|
|
if test_httpd:
|
|
httpd = Httpd(env=env)
|
|
assert httpd.exists(), \
|
|
f'httpd not found: {env.httpd}'
|
|
httpd.clear_logs()
|
|
server_docs = httpd.docs_dir
|
|
assert httpd.initial_start()
|
|
if protocol == 'h3':
|
|
nghttpx = NghttpxQuic(env=env)
|
|
nghttpx.clear_logs()
|
|
assert nghttpx.initial_start()
|
|
server_descr = f'nghttpx: https:{env.h3_port} [backend httpd/{env.httpd_version()}]'
|
|
server_port = env.h3_port
|
|
else:
|
|
server_descr = f'httpd/{env.httpd_version()}'
|
|
server_port = env.https_port
|
|
card = ScoreRunner(env=env,
|
|
protocol=protocol,
|
|
server_descr=server_descr,
|
|
server_port=server_port,
|
|
verbose=args.verbose, curl_verbose=args.curl_verbose,
|
|
download_parallel=args.download_parallel,
|
|
upload_parallel=args.upload_parallel,
|
|
with_flame=args.flame,
|
|
socks_args=socks_args,
|
|
limit_rate=args.limit_rate)
|
|
card.setup_resources(server_docs, downloads)
|
|
cards.append(card)
|
|
|
|
if test_caddy and env.caddy:
|
|
backend = ''
|
|
if uploads and httpd is None:
|
|
backend = f' [backend httpd: {env.httpd_version()}]'
|
|
httpd = Httpd(env=env)
|
|
assert httpd.exists(), \
|
|
f'httpd not found: {env.httpd}'
|
|
httpd.clear_logs()
|
|
assert httpd.initial_start()
|
|
caddy = Caddy(env=env)
|
|
caddy.clear_logs()
|
|
assert caddy.initial_start()
|
|
server_descr = f'Caddy/{env.caddy_version()} {backend}'
|
|
server_port = caddy.port
|
|
server_docs = caddy.docs_dir
|
|
card = ScoreRunner(env=env,
|
|
protocol=protocol,
|
|
server_descr=server_descr,
|
|
server_port=server_port,
|
|
verbose=args.verbose, curl_verbose=args.curl_verbose,
|
|
download_parallel=args.download_parallel,
|
|
upload_parallel=args.upload_parallel,
|
|
with_flame=args.flame,
|
|
socks_args=socks_args,
|
|
limit_rate=args.limit_rate)
|
|
card.setup_resources(server_docs, downloads)
|
|
cards.append(card)
|
|
|
|
if args.start_only:
|
|
print('started servers:')
|
|
for card in cards:
|
|
print(f'{card.server_descr}')
|
|
sys.stderr.write('press [RETURN] to finish')
|
|
sys.stderr.flush()
|
|
sys.stdin.readline()
|
|
else:
|
|
for card in cards:
|
|
score = card.score(handshakes=handshakes,
|
|
downloads=downloads,
|
|
download_count=args.download_count,
|
|
uploads=uploads,
|
|
upload_count=args.upload_count,
|
|
req_count=args.request_count,
|
|
requests=requests,
|
|
request_parallels=request_parallels,
|
|
nsamples=args.samples)
|
|
if args.json:
|
|
print(json.JSONEncoder(indent=2).encode(score))
|
|
else:
|
|
Card.print_score(score)
|
|
|
|
except ScoreCardError as ex:
|
|
sys.stderr.write(f"ERROR: {ex}\n")
|
|
rv = 1
|
|
except KeyboardInterrupt:
|
|
log.warning("aborted")
|
|
rv = 1
|
|
finally:
|
|
if caddy:
|
|
caddy.stop()
|
|
if nghttpx:
|
|
nghttpx.stop(wait_dead=False)
|
|
if httpd:
|
|
httpd.stop()
|
|
if sockd:
|
|
sockd.stop()
|
|
return rv
|
|
|
|
|
|
def print_file(filename):
|
|
if not os.path.exists(filename):
|
|
sys.stderr.write(f"ERROR: file does not exist {filename}\n")
|
|
return 1
|
|
with open(filename) as file:
|
|
data = json.load(file)
|
|
Card.print_score(data)
|
|
return 0
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(prog='scorecard', description="""
|
|
Run a range of tests to give a scorecard for a HTTP protocol
|
|
'h3' or 'h2' implementation in curl.
|
|
""")
|
|
parser.add_argument("-v", "--verbose", action='count', default=1,
|
|
help="log more output on stderr")
|
|
parser.add_argument("-j", "--json", action='store_true',
|
|
default=False, help="print json instead of text")
|
|
parser.add_argument("--samples", action='store', type=int, metavar='number',
|
|
default=1, help="how many sample runs to make")
|
|
parser.add_argument("--httpd", action='store_true', default=False,
|
|
help="evaluate httpd server only")
|
|
parser.add_argument("--caddy", action='store_true', default=False,
|
|
help="evaluate caddy server only")
|
|
parser.add_argument("--curl-verbose", action='store_true',
|
|
default=False, help="run curl with `-v`")
|
|
parser.add_argument("--print", type=str, default=None, metavar='filename',
|
|
help="print the results from a JSON file")
|
|
parser.add_argument("protocol", default=None, nargs='?',
|
|
help="Name of protocol to score")
|
|
parser.add_argument("--start-only", action='store_true', default=False,
|
|
help="only start the servers")
|
|
parser.add_argument("--remote", action='store', type=str,
|
|
default=None, help="score against the remote server at <ip>:<port>")
|
|
parser.add_argument("--flame", action='store_true',
|
|
default = False, help="produce a flame graph on curl")
|
|
parser.add_argument("--limit-rate", action='store', type=str,
|
|
default=None, help="use curl's --limit-rate")
|
|
|
|
parser.add_argument("-H", "--handshakes", action='store_true',
|
|
default=False, help="evaluate handshakes only")
|
|
|
|
parser.add_argument("-d", "--downloads", action='store_true',
|
|
default=False, help="evaluate downloads")
|
|
parser.add_argument("--download-sizes", action='append', type=str,
|
|
metavar='numberlist',
|
|
default=None, help="evaluate download size")
|
|
parser.add_argument("--download-count", action='store', type=int,
|
|
metavar='number',
|
|
default=50, help="perform that many downloads")
|
|
parser.add_argument("--download-parallel", action='store', type=int,
|
|
metavar='number', default=0,
|
|
help="perform that many downloads in parallel (default all)")
|
|
|
|
parser.add_argument("-u", "--uploads", action='store_true',
|
|
default=False, help="evaluate uploads")
|
|
parser.add_argument("--upload-sizes", action='append', type=str,
|
|
metavar='numberlist',
|
|
default=None, help="evaluate upload size")
|
|
parser.add_argument("--upload-count", action='store', type=int,
|
|
metavar='number', default=50,
|
|
help="perform that many uploads")
|
|
parser.add_argument("--upload-parallel", action='store', type=int,
|
|
metavar='number', default=0,
|
|
help="perform that many uploads in parallel (default all)")
|
|
|
|
parser.add_argument("-r", "--requests", action='store_true',
|
|
default=False, help="evaluate requests")
|
|
parser.add_argument("--request-count", action='store', type=int,
|
|
metavar='number',
|
|
default=5000, help="perform that many requests")
|
|
parser.add_argument("--request-parallels", action='append', type=str,
|
|
metavar='numberlist',
|
|
default=None, help="evaluate request with these max-parallel numbers")
|
|
parser.add_argument("--socks4", action='store_true',
|
|
default=False, help="test with SOCKS4 proxy")
|
|
parser.add_argument("--socks5", action='store_true',
|
|
default=False, help="test with SOCKS5 proxy")
|
|
args = parser.parse_args()
|
|
|
|
if args.verbose > 0:
|
|
console = logging.StreamHandler()
|
|
console.setLevel(logging.INFO)
|
|
console.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
|
|
logging.getLogger('').addHandler(console)
|
|
|
|
if args.print:
|
|
rv = print_file(args.print)
|
|
elif not args.protocol:
|
|
parser.print_usage()
|
|
rv = 1
|
|
else:
|
|
rv = run_score(args, args.protocol)
|
|
|
|
sys.exit(rv)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|