Revision of speedtest-hd.py

mreschke revised this gist 5 days ago. Go to revision

1 file changed, 1 insertion, 2 deletions

speedtest-hd.py

			@@ -2,5 +2,4 @@
2	2		# See https://git.mreschke.net/mreschke/speedtest-hd
3	3
4	4		# Quick Install
5		-
6		-	wget https://git.mreschke.net/mreschke/speedtest-hd/raw/branch/master/speedtest-hd.py
	5	+	wget https://git.mreschke.net/mreschke/speedtest-hd/raw/branch/master/speedtest-hd.py && chmod a+x speedtest-hd.py

mreschke revised this gist 5 days ago. Go to revision

1 file changed, 4 insertions, 729 deletions

speedtest-hd.py

			@@ -1,731 +1,6 @@
1		-	#!/usr/bin/env python3
2		-	"""speedtest-hd — a CrystalDiskMark-style storage benchmark for Linux, on fio.
	1	+	# Latest speedtest-hd.py moved to Gitea
	2	+	# See https://git.mreschke.net/mreschke/speedtest-hd
3	3
4		-	This is the Python successor to ``speedtest-hd.sh``. It measures storage the way
5		-	CrystalDiskMark does, and adds a dedicated SLOG / sync-write latency profile
6		-	for diagnosing ZFS ZIL performance (NFS / iSCSI / VM sync workloads).
	4	+	# Quick Install
7	5
8		-	Three profiles
9		-	--------------
10		-	* cdm — the four CrystalDiskMark default tests (``SEQ1M Q8T1``, ``SEQ1M
11		-	Q1T1``, ``RND4K Q32T16``, ``RND4K Q1T1``), each measured for read and
12		-	write, reported in MB/s and IOPS.
13		-	* slog — synchronous 4K random writes swept across thread counts, reporting
14		-	IOPS, MB/s and p50/p99 commit latency. This is the load a ZFS SLOG sees.
15		-	* dd — a dependency-free fallback when ``fio`` isn't installed.
16		-
17		-	Why fio JSON?
18		-	-------------
19		-	Every measurement asks fio for ``--output-format=json`` and parses it with the
20		-	standard library. That's the whole reason this is Python and not bash: robust,
21		-	unit-safe parsing of bandwidth / IOPS / latency percentiles with no fragile
22		-	text scraping.
23		-
24		-	See README.md for the full case study (diagnosing a "slow" Optane SLOG that
25		-	turned out to be CPU power management, not the disk).
26		-	"""
27		-
28		-	from __future__ import annotations
29		-
30		-	import argparse
31		-	import glob
32		-	import json
33		-	import os
34		-	import shutil
35		-	import statistics
36		-	import subprocess
37		-	import sys
38		-	import time
39		-	from dataclasses import dataclass, field
40		-	from typing import Callable, Optional, Sequence
41		-
42		-	# --------------------------------------------------------------------------- #
43		-	# Color / styling (standard library only -- raw ANSI escape codes)
44		-	# --------------------------------------------------------------------------- #
45		-
46		-	# SGR escape sequences. We stick to the basic 16-color set so output respects
47		-	# the user's terminal theme instead of hard-coding RGB.
48		-	RESET = "\033[0m"
49		-	BOLD = "\033[1m"
50		-	DIM = "\033[2m"
51		-	ITALIC = "\033[3m"
52		-	RED = "\033[31m"
53		-	GREEN = "\033[32m"
54		-	YELLOW = "\033[33m"
55		-	BLUE = "\033[34m"
56		-	MAGENTA = "\033[35m"
57		-	CYAN = "\033[36m"
58		-	WHITE = "\033[37m"
59		-	BRIGHT_CYAN = "\033[96m"
60		-
61		-
62		-	def supports_color(stream) -> bool:
63		-	"""Decide per-stream whether to emit ANSI codes.
64		-
65		-	Honors the de-facto ``NO_COLOR`` / ``FORCE_COLOR`` conventions and otherwise
66		-	colors only when the stream is an interactive terminal.
67		-	"""
68		-	if os.environ.get("NO_COLOR"):
69		-	return False
70		-	if os.environ.get("FORCE_COLOR"):
71		-	return True
72		-	if os.environ.get("TERM") == "dumb":
73		-	return False
74		-	return bool(getattr(stream, "isatty", None)) and stream.isatty()
75		-
76		-
77		-	class Painter:
78		-	"""Wraps text in SGR codes, or returns it untouched when color is off."""
79		-
80		-	def __init__(self, enabled: bool):
81		-	self.enabled = enabled
82		-
83		-	def paint(self, text: str, *codes: str) -> str:
84		-	if not self.enabled or not codes:
85		-	return text
86		-	return "".join(codes) + text + RESET
87		-
88		-
89		-	# stdout carries results (banner + tables); stderr carries progress + verbose
90		-	# fio dumps. Each gets its own enable flag so piping one but not the other works.
91		-	OUT = Painter(supports_color(sys.stdout))
92		-	ERR = Painter(supports_color(sys.stderr))
93		-
94		-
95		-	# --------------------------------------------------------------------------- #
96		-	# Constants
97		-	# --------------------------------------------------------------------------- #
98		-
99		-	# Preference order for the IO engine. io_uring is the modern, lowest-overhead
100		-	# Linux engine; libaio is older (and only truly async with O_DIRECT); posixaio
101		-	# and sync are the portable fallbacks (e.g. some NFS mounts).
102		-	ENGINE_CANDIDATES = ("io_uring", "libaio", "posixaio", "sync")
103		-
104		-	# Binary unit multipliers (fio treats "1g" as 1 GiB, so we match that).
105		-	_SIZE_UNITS = {"k": 1024, "m": 10242, "g": 10243, "t": 1024**4}
106		-
107		-
108		-	@dataclass(frozen=True)
109		-	class CdmTest:
110		-	"""One CrystalDiskMark test. ``seq`` picks sequential vs random patterns."""
111		-
112		-	label: str
113		-	bs: str
114		-	iodepth: int
115		-	numjobs: int
116		-	seq: bool
117		-
118		-	@property
119		-	def read_rw(self) -> str:
120		-	return "read" if self.seq else "randread"
121		-
122		-	@property
123		-	def write_rw(self) -> str:
124		-	return "write" if self.seq else "randwrite"
125		-
126		-
127		-	# The four CrystalDiskMark default tests, as data, in CrystalDiskMark's own
128		-	# display order. Q = queue depth (iodepth), T = threads (numjobs).
129		-	CDM_TESTS: tuple[CdmTest, ...] = (
130		-	CdmTest("SEQ1M Q8T1", bs="1m", iodepth=8, numjobs=1, seq=True),
131		-	CdmTest("SEQ1M Q1T1", bs="1m", iodepth=1, numjobs=1, seq=True),
132		-	CdmTest("RND4K Q32T16", bs="4k", iodepth=32, numjobs=16, seq=False),
133		-	CdmTest("RND4K Q1T1", bs="4k", iodepth=1, numjobs=1, seq=False),
134		-	)
135		-
136		-	# Thread counts for the SLOG sweep. T1 is the headline single-stream latency;
137		-	# the higher counts show how the SLOG scales as concurrent sync writers pile on.
138		-	SLOG_THREADS: tuple[int, ...] = (1, 4, 8, 16)
139		-
140		-
141		-	# --------------------------------------------------------------------------- #
142		-	# Configuration
143		-	# --------------------------------------------------------------------------- #
144		-
145		-
146		-	@dataclass
147		-	class Config:
148		-	"""Resolved run settings. ``engine`` and ``direct`` may start unset and get
149		-	filled in by :func:`detect_io_settings`."""
150		-
151		-	path: str
152		-	mode: Optional[str] # "cdm" \| "slog" \| "dd" \| None (auto)
153		-	engine: Optional[str] # None => auto-detect
154		-	direct: Optional[bool] # None => auto-detect
155		-	runtime: int
156		-	size: str
157		-	verbose: bool
158		-	assume_yes: bool
159		-	benchfile: str = field(default="")
160		-
161		-	@property
162		-	def direct_flag(self) -> int:
163		-	"""fio's --direct takes 0/1; default to 0 until detection runs."""
164		-	return 1 if self.direct else 0
165		-
166		-
167		-	# --------------------------------------------------------------------------- #
168		-	# Small helpers
169		-	# --------------------------------------------------------------------------- #
170		-
171		-
172		-	def log(message: str) -> None:
173		-	"""Progress/diagnostic line -> stderr, so stdout stays pure results."""
174		-	print(message, file=sys.stderr, flush=True)
175		-
176		-
177		-	def step(message: str) -> None:
178		-	"""A single colored progress line ('measuring ...') -> stderr."""
179		-	log(f" {ERR.paint('▶', BOLD, CYAN)} {ERR.paint(message, DIM)}")
180		-
181		-
182		-	def vsection(title: str) -> None:
183		-	"""A clear, ruled header for one --verbose fio section -> stderr."""
184		-	width = 78
185		-	head = f"─── {title} "
186		-	head += "─" * max(3, width - len(head))
187		-	log("")
188		-	log(ERR.paint(head, BOLD, CYAN))
189		-
190		-
191		-	def parse_size_bytes(size: str) -> int:
192		-	"""Turn an fio-style size ("4k", "1g", "512m", "2048") into bytes."""
193		-	s = size.strip().lower()
194		-	if s and s[-1] in _SIZE_UNITS:
195		-	return int(float(s[:-1]) * _SIZE_UNITS[s[-1]])
196		-	return int(s)
197		-
198		-
199		-	def render_table(
200		-	headers: Sequence[str],
201		-	rows: Sequence[Sequence[str]],
202		-	aligns: Optional[Sequence[str]] = None,
203		-	col_styles: Optional[Sequence[Optional[str]]] = None,
204		-	) -> str:
205		-	"""Render a bordered ASCII table whose columns auto-size to their content.
206		-
207		-	``aligns`` is a per-column "<" (left) or ">" (right); by default the first
208		-	column is left-aligned (the label) and the rest are right-aligned (numbers).
209		-	``col_styles`` is an optional per-column ANSI style applied to body cells
210		-	(header is always bold, borders dim). Widths are computed on the plain
211		-	text and color is applied after padding, so escape codes never skew layout.
212		-	"""
213		-	ncols = len(headers)
214		-	if aligns is None:
215		-	aligns = ["<"] + [">"] * (ncols - 1)
216		-	if col_styles is None:
217		-	col_styles = [None] * ncols
218		-
219		-	# Widest cell (header or any row) sets each column's width.
220		-	widths = [
221		-	max([len(str(headers[c]))] + [len(str(r[c])) for r in rows])
222		-	for c in range(ncols)
223		-	]
224		-
225		-	bar = OUT.paint("\|", DIM)
226		-	sep = OUT.paint(" \| ", DIM)
227		-
228		-	def rule() -> str:
229		-	return OUT.paint("+" + "+".join("-" * (w + 2) for w in widths) + "+", DIM)
230		-
231		-	def line(cells: Sequence[str], *, header: bool = False) -> str:
232		-	parts = []
233		-	for c in range(ncols):
234		-	padded = f"{str(cells[c]):{aligns[c]}{widths[c]}}"
235		-	if header:
236		-	parts.append(OUT.paint(padded, BOLD))
237		-	elif col_styles[c]:
238		-	parts.append(OUT.paint(padded, col_styles[c]))
239		-	else:
240		-	parts.append(padded)
241		-	return f"{bar} " + sep.join(parts) + f" {bar}"
242		-
243		-	out = [rule(), line(headers, header=True), rule()]
244		-	out += [line(r) for r in rows]
245		-	out.append(rule())
246		-	return "\n".join(out)
247		-
248		-
249		-	# --------------------------------------------------------------------------- #
250		-	# fio: results, invocation, parsing
251		-	# --------------------------------------------------------------------------- #
252		-
253		-
254		-	@dataclass(frozen=True)
255		-	class FioResult:
256		-	"""Parsed metrics for one direction (read or write) of an fio run.
257		-
258		-	Latencies are completion latency (clat) in microseconds; for synchronous
259		-	writes that is effectively the durable-commit latency.
260		-	"""
261		-
262		-	bw_mbps: float # decimal MB/s, matching CrystalDiskMark's SI figure
263		-	iops: float
264		-	p50_us: float
265		-	p99_us: float
266		-	mean_us: float
267		-
268		-	@classmethod
269		-	def zero(cls) -> "FioResult":
270		-	return cls(0.0, 0.0, 0.0, 0.0, 0.0)
271		-
272		-
273		-	def _direction_of(rw: str) -> str:
274		-	"""fio reports 'read' and 'write' sub-objects; pick the active one."""
275		-	return "read" if "read" in rw else "write"
276		-
277		-
278		-	def _aggregate(jobs: list[dict], direction: str) -> FioResult:
279		-	"""Combine per-job fio JSON stats into a single :class:`FioResult`.
280		-
281		-	We deliberately run fio without --group_reporting and aggregate ourselves,
282		-	which sidesteps fio's group-merge quirks: sum throughput, average the median
283		-	latency, and take the worst-case tail (p99) across jobs.
284		-	"""
285		-
286		-	def section(job: dict) -> dict:
287		-	return job.get(direction, {})
288		-
289		-	bw_mbps = sum(section(j).get("bw_bytes", 0) for j in jobs) / 1e6
290		-	iops = sum(section(j).get("iops", 0.0) for j in jobs)
291		-
292		-	def percentiles(key: str) -> list[float]:
293		-	vals = []
294		-	for j in jobs:
295		-	pct = section(j).get("clat_ns", {}).get("percentile", {})
296		-	v = pct.get(key)
297		-	if v is not None:
298		-	vals.append(v)
299		-	return vals
300		-
301		-	p50_ns = percentiles("50.000000")
302		-	p99_ns = percentiles("99.000000")
303		-	mean_ns = [section(j).get("clat_ns", {}).get("mean", 0.0) for j in jobs]
304		-
305		-	return FioResult(
306		-	bw_mbps=bw_mbps,
307		-	iops=iops,
308		-	p50_us=(statistics.mean(p50_ns) / 1000.0) if p50_ns else 0.0, # avg of medians
309		-	p99_us=(max(p99_ns) / 1000.0) if p99_ns else 0.0, # worst tail
310		-	mean_us=(statistics.mean(mean_ns) / 1000.0) if mean_ns else 0.0,
311		-	)
312		-
313		-
314		-	def run_fio(
315		-	cfg: Config,
316		-	*,
317		-	rw: str,
318		-	bs: str,
319		-	iodepth: int,
320		-	numjobs: int,
321		-	engine: Optional[str] = None,
322		-	direct: Optional[bool] = None,
323		-	sync: bool = False,
324		-	end_fsync: bool = False,
325		-	) -> FioResult:
326		-	"""Run a single fio job against the shared bench file and parse its JSON.
327		-
328		-	``engine``/``direct`` default to the detected config values; the SLOG path
329		-	overrides them (psync + buffered + O_SYNC). ``end_fsync`` flushes the device
330		-	cache at the end of a write run so cached writes can't inflate the result.
331		-	"""
332		-	engine = engine if engine is not None else cfg.engine
333		-	direct = direct if direct is not None else cfg.direct
334		-
335		-	cmd = [
336		-	"sudo", "fio",
337		-	"--name=speedtest",
338		-	f"--filename={cfg.benchfile}",
339		-	f"--ioengine={engine}",
340		-	f"--direct={1 if direct else 0}",
341		-	f"--rw={rw}",
342		-	f"--bs={bs}",
343		-	f"--size={cfg.size}",
344		-	f"--numjobs={numjobs}",
345		-	f"--iodepth={iodepth}",
346		-	"--time_based",
347		-	f"--runtime={cfg.runtime}",
348		-	"--output-format=json",
349		-	]
350		-	if sync:
351		-	cmd.append("--sync=1") # O_SYNC: every write is a durable commit
352		-	if end_fsync:
353		-	cmd.append("--end_fsync=1") # flush device cache once at the end
354		-
355		-	proc = subprocess.run(cmd, capture_output=True, text=True)
356		-
357		-	if cfg.verbose:
358		-	sync_tag = " sync=1" if sync else ""
359		-	vsection(f"fio · {rw} bs={bs} iodepth={iodepth} numjobs={numjobs}{sync_tag}")
360		-	log(" " + ERR.paint("$ " + " ".join(cmd[1:]), DIM, ITALIC))
361		-	log(ERR.paint(proc.stdout.strip() or "(no stdout)", DIM))
362		-	if proc.stderr.strip():
363		-	log(ERR.paint(proc.stderr.strip(), YELLOW))
364		-
365		-	try:
366		-	data = json.loads(proc.stdout)
367		-	return _aggregate(data["jobs"], _direction_of(rw))
368		-	except (json.JSONDecodeError, KeyError, ValueError):
369		-	# fio failed or produced no parseable JSON; report zeros rather than
370		-	# crashing the whole run (verbose mode above shows what went wrong).
371		-	return FioResult.zero()
372		-
373		-
374		-	def fio_probe(path: str, engine: str, direct: bool) -> bool:
375		-	"""Throwaway 1s fio job to see if an (engine, O_DIRECT) combo works here.
376		-
377		-	This is how we stay accurate and portable across ext4/xfs/btrfs/ZFS/NFS
378		-	without the caller needing to know what each filesystem supports.
379		-	"""
380		-	cmd = [
381		-	"sudo", "fio", "--name=probe",
382		-	f"--directory={path}",
383		-	f"--ioengine={engine}",
384		-	"--rw=write", "--bs=4k", "--size=1m",
385		-	f"--direct={1 if direct else 0}",
386		-	"--time_based", "--runtime=1",
387		-	]
388		-	proc = subprocess.run(cmd, capture_output=True, text=True)
389		-	for leftover in glob.glob(os.path.join(path, "probe*")):
390		-	try:
391		-	os.remove(leftover)
392		-	except OSError:
393		-	pass
394		-	return proc.returncode == 0
395		-
396		-
397		-	def detect_io_settings(cfg: Config) -> None:
398		-	"""Fill in ``cfg.engine`` and ``cfg.direct`` if the user didn't force them.
399		-
400		-	O_DIRECT bypasses the OS page cache so we measure the device, not RAM. Not
401		-	every filesystem supports it (older OpenZFS, some NFS), so we probe and fall
402		-	back to buffered rather than erroring out.
403		-	"""
404		-	if cfg.engine is None:
405		-	for engine in ENGINE_CANDIDATES:
406		-	if fio_probe(cfg.path, engine, direct=False):
407		-	cfg.engine = engine
408		-	break
409		-	else:
410		-	cfg.engine = "sync"
411		-
412		-	if cfg.direct is None:
413		-	cfg.direct = fio_probe(cfg.path, cfg.engine, direct=True)
414		-
415		-
416		-	# --------------------------------------------------------------------------- #
417		-	# Profiles
418		-	# --------------------------------------------------------------------------- #
419		-
420		-
421		-	def _meta_line(text: str) -> str:
422		-	"""Color the 'Label :' prefix of a banner metadata line, leave value as-is.
423		-
424		-	The value may already contain its own ANSI codes (e.g. a red O_DIRECT
425		-	warning); since these lines aren't width-aligned that's harmless.
426		-	"""
427		-	key, sep, val = text.partition(":")
428		-	if not sep:
429		-	return " " + text
430		-	return " " + OUT.paint(key + ":", BOLD, BRIGHT_CYAN) + val
431		-
432		-
433		-	def banner(title: str, cfg: Config, extra: Sequence[str] = ()) -> None:
434		-	full = f"speedtest-hd · {title}"
435		-	inner = len(full) + 4
436		-
437		-	print()
438		-	print(OUT.paint("╭" + "─" * inner + "╮", BOLD, CYAN))
439		-	print(
440		-	OUT.paint("│", BOLD, CYAN)
441		-	+ " " + OUT.paint(full, BOLD, WHITE) + " "
442		-	+ OUT.paint("│", BOLD, CYAN)
443		-	)
444		-	print(OUT.paint("╰" + "─" * inner + "╯", BOLD, CYAN))
445		-	print(_meta_line(f"Target : {OUT.paint(cfg.path, CYAN)}"))
446		-	for line_ in extra:
447		-	print(_meta_line(line_))
448		-	print()
449		-
450		-
451		-	def cdm_profile(cfg: Config) -> None:
452		-	"""The CrystalDiskMark-style profile: 4 tests x (read, write), MB/s + IOPS."""
453		-	detect_io_settings(cfg)
454		-
455		-	if cfg.direct:
456		-	direct_label = OUT.paint("enabled (device)", GREEN)
457		-	else:
458		-	direct_label = OUT.paint(
459		-	"DISABLED (buffered -- may reflect RAM cache!)", BOLD, RED
460		-	)
461		-
462		-	banner(
463		-	"CrystalDiskMark-style storage benchmark",
464		-	cfg,
465		-	extra=[
466		-	f"Engine : {cfg.engine} O_DIRECT: {direct_label}",
467		-	f"Profile : size={cfg.size.upper()} runtime={cfg.runtime}s/run (8 runs)",
468		-	],
469		-	)
470		-
471		-	rows: list[list[str]] = []
472		-	for test in CDM_TESTS:
473		-	step(f"measuring {test.label.strip()} ...")
474		-	r = run_fio(cfg, rw=test.read_rw, bs=test.bs,
475		-	iodepth=test.iodepth, numjobs=test.numjobs)
476		-	w = run_fio(cfg, rw=test.write_rw, bs=test.bs,
477		-	iodepth=test.iodepth, numjobs=test.numjobs, end_fsync=True)
478		-	rows.append([
479		-	test.label,
480		-	f"{r.bw_mbps:.2f}", f"{w.bw_mbps:.2f}",
481		-	f"{r.iops:.0f}", f"{w.iops:.0f}",
482		-	])
483		-
484		-	_cleanup(cfg)
485		-
486		-	print()
487		-	print(render_table(
488		-	["Test", "Read (MB/s)", "Write (MB/s)", "Read (IOPS)", "Write (IOPS)"],
489		-	rows,
490		-	col_styles=[CYAN, GREEN, YELLOW, GREEN, YELLOW],
491		-	))
492		-	print()
493		-
494		-
495		-	def slog_profile(cfg: Config) -> None:
496		-	"""SLOG / sync-write latency profile.
497		-
498		-	Forces synchronous 4K random writes (O_SYNC) via the portable psync engine,
499		-	so every write is a ZIL commit -- exercising a ZFS SLOG exactly the way a
500		-	sync=always dataset (NFS/iSCSI/VM) does, regardless of the dataset's own
501		-	sync property. Engine/direct detection is only for the banner here.
502		-	"""
503		-	detect_io_settings(cfg)
504		-
505		-	banner(
506		-	"SLOG / sync-write latency profile",
507		-	cfg,
508		-	extra=[
509		-	"Method : fio randwrite bs=4k --sync=1 (O_SYNC), psync engine",
510		-	f"Profile : runtime={cfg.runtime}s/run size={cfg.size.upper()}",
511		-	"Note : every write is a synchronous ZIL commit -- the load your",
512		-	" SLOG actually sees. Watch it live in another shell with:",
513		-	" zpool iostat -vl <pool> 1",
514		-	],
515		-	)
516		-
517		-	# Measure everything first (progress -> stderr), then draw the whole table,
518		-	# so the "measuring..." lines can't interleave into the middle of it.
519		-	rows: list[list[str]] = []
520		-	for threads in SLOG_THREADS:
521		-	step(f"measuring 4K sync randwrite T{threads} ...")
522		-	res = run_fio(
523		-	cfg, rw="randwrite", bs="4k", iodepth=1, numjobs=threads,
524		-	engine="psync", direct=False, sync=True,
525		-	)
526		-	rows.append([
527		-	f"4K sync T{threads}",
528		-	f"{res.iops:.0f}", f"{res.bw_mbps:.2f}",
529		-	f"{res.p50_us:.1f}", f"{res.p99_us:.1f}",
530		-	])
531		-
532		-	_cleanup(cfg)
533		-
534		-	print()
535		-	print(render_table(
536		-	["Test", "IOPS", "MB/s", "p50 lat(us)", "p99 lat(us)"],
537		-	rows,
538		-	col_styles=[CYAN, GREEN, GREEN, YELLOW, MAGENTA],
539		-	))
540		-	print()
541		-	print(OUT.paint(" Healthy Optane SLOG (eg P1600X) single-stream (T1) target:", BOLD))
542		-	print(OUT.paint(" ~15-25k IOPS, p50 latency ~40-65us.", GREEN)
543		-	+ OUT.paint(" Much higher latency usually means", DIM))
544		-	print(OUT.paint(" CPU C-states / PCIe ASPM / BIOS power profile (eg Dell DAPC) throttling.", DIM))
545		-	print()
546		-
547		-
548		-	def dd_profile(cfg: Config) -> None:
549		-	"""Dependency-free fallback when fio isn't installed.
550		-
551		-	Far cruder than fio: a single sequential stream, and the cached-read figure
552		-	will reflect RAM. Use it only as a rough sanity check.
553		-	"""
554		-	count_mib = max(1, parse_size_bytes(cfg.size) // (1024**2))
555		-	nbytes = count_mib * 1024**2
556		-
557		-	banner(
558		-	"basic dd benchmark (fio not installed)",
559		-	cfg,
560		-	extra=[f"Profile : {count_mib} MiB sequential stream"],
561		-	)
562		-
563		-	def timed_dd(args: list[str]) -> float:
564		-	start = time.monotonic()
565		-	subprocess.run(args, capture_output=True, text=True)
566		-	elapsed = time.monotonic() - start
567		-	return (nbytes / elapsed / 1e6) if elapsed > 0 else 0.0 # decimal MB/s
568		-
569		-	step("measuring sequential write ...")
570		-	write_mbps = timed_dd([
571		-	"dd", "if=/dev/zero", f"of={cfg.benchfile}",
572		-	"bs=1M", f"count={count_mib}", "conv=fdatasync,notrunc",
573		-	])
574		-
575		-	step("dropping caches for uncached read ...")
576		-	subprocess.run(["sudo", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"],
577		-	capture_output=True, text=True)
578		-
579		-	step("measuring uncached read ...")
580		-	uncached_mbps = timed_dd([
581		-	"dd", f"if={cfg.benchfile}", "of=/dev/null", "bs=1M", f"count={count_mib}",
582		-	])
583		-
584		-	step("measuring cached read ...")
585		-	cached_mbps = timed_dd([
586		-	"dd", f"if={cfg.benchfile}", "of=/dev/null", "bs=1M", f"count={count_mib}",
587		-	])
588		-
589		-	_cleanup(cfg)
590		-
591		-	print()
592		-	print(render_table(
593		-	["Test", "MB/s"],
594		-	[
595		-	["Sequential write", f"{write_mbps:.2f}"],
596		-	["Uncached read", f"{uncached_mbps:.2f}"],
597		-	["Cached read (RAM)", f"{cached_mbps:.2f}"],
598		-	],
599		-	col_styles=[CYAN, GREEN],
600		-	))
601		-	print()
602		-
603		-
604		-	def _cleanup(cfg: Config) -> None:
605		-	"""Remove the shared bench file."""
606		-	try:
607		-	os.remove(cfg.benchfile)
608		-	except OSError:
609		-	pass
610		-
611		-
612		-	# --------------------------------------------------------------------------- #
613		-	# CLI
614		-	# --------------------------------------------------------------------------- #
615		-
616		-
617		-	def build_parser() -> argparse.ArgumentParser:
618		-	p = argparse.ArgumentParser(
619		-	prog="speedtest-hd.py",
620		-	description="CrystalDiskMark-style storage benchmark built on fio.",
621		-	epilog=(
622		-	"Examples:\n"
623		-	" speedtest-hd.py .\n"
624		-	" speedtest-hd.py /mnt/nvmepool --runtime=10 --size=4g\n"
625		-	" speedtest-hd.py /mnt/nfsshare --buffered\n"
626		-	" speedtest-hd.py /mnt/nvme-ultra-r10/vm-root --slog --runtime=30\n"
627		-	),
628		-	formatter_class=argparse.RawDescriptionHelpFormatter,
629		-	)
630		-	p.add_argument("path", help="directory/mount to benchmark ('.' for cwd)")
631		-
632		-	mode = p.add_mutually_exclusive_group()
633		-	mode.add_argument("--fio", action="store_const", const="cdm", dest="mode",
634		-	help="force the fio CrystalDiskMark-style profile")
635		-	mode.add_argument("--dd", action="store_const", const="dd", dest="mode",
636		-	help="force the basic dd fallback test")
637		-	mode.add_argument("--slog", action="store_const", const="slog", dest="mode",
638		-	help="SLOG / sync-write latency profile (ZFS ZIL)")
639		-
640		-	direct = p.add_mutually_exclusive_group()
641		-	direct.add_argument("--direct", action="store_const", const=True, dest="direct",
642		-	help="force O_DIRECT (bypass page cache)")
643		-	direct.add_argument("--buffered", action="store_const", const=False, dest="direct",
644		-	help="force buffered IO (e.g. if O_DIRECT unsupported)")
645		-
646		-	p.add_argument("--engine", choices=ENGINE_CANDIDATES,
647		-	help="force a specific IO engine (default: auto-detect)")
648		-	p.add_argument("--runtime", type=int, default=5, metavar="SEC",
649		-	help="seconds per run (default: 5, like CrystalDiskMark)")
650		-	p.add_argument("--size", default="1g", metavar="SIZE",
651		-	help="test file size (default: 1g)")
652		-	p.add_argument("--verbose", action="store_true",
653		-	help="also print the full fio output for every run")
654		-	p.add_argument("-y", "--yes", action="store_true", dest="assume_yes",
655		-	help="skip the confirmation prompt")
656		-
657		-	p.set_defaults(mode=None, direct=None)
658		-	return p
659		-
660		-
661		-	def confirm(cfg: Config) -> None:
662		-	"""Guard prompt -- we're about to write a multi-GB file to the target."""
663		-	if cfg.assume_yes:
664		-	return
665		-	print(OUT.paint("NOTICE:", BOLD, YELLOW)
666		-	+ f" {cfg.size.upper()} free space on "
667		-	+ OUT.paint(f"'{cfg.path}'", CYAN)
668		-	+ " is required to perform the benchmark.")
669		-	answer = input(f"Are you ready to start a storage benchmark against "
670		-	f"'{cfg.path}' ? ")
671		-	if not answer.strip().lower().startswith("y"):
672		-	print(OUT.paint("Ok, cancelled!", YELLOW))
673		-	sys.exit(0)
674		-	print(OUT.paint("Great! Starting benchmark now!", BOLD, GREEN))
675		-
676		-
677		-	def main(argv: Optional[Sequence[str]] = None) -> int:
678		-	args = build_parser().parse_args(argv)
679		-
680		-	path = os.getcwd() if args.path == "." else args.path
681		-	if not os.path.exists(path):
682		-	print(f"Path {path} does not exist", file=sys.stderr)
683		-	return 1
684		-
685		-	cfg = Config(
686		-	path=path,
687		-	mode=args.mode,
688		-	engine=args.engine,
689		-	direct=args.direct,
690		-	runtime=args.runtime,
691		-	size=args.size,
692		-	verbose=args.verbose,
693		-	assume_yes=args.assume_yes,
694		-	benchfile=os.path.join(path, "speedtest-hd.bench"),
695		-	)
696		-
697		-	have_fio = shutil.which("fio") is not None
698		-
699		-	# Resolve the mode: explicit flag wins; otherwise fio if available, else dd.
700		-	mode = cfg.mode
701		-	if mode in ("cdm", "slog") and not have_fio:
702		-	print(ERR.paint("ERROR:", BOLD, RED)
703		-	+ " --fio/--slog require fio (apt install fio / pacman -S fio).",
704		-	file=sys.stderr)
705		-	return 1
706		-	if mode is None:
707		-	if have_fio:
708		-	mode = "cdm"
709		-	else:
710		-	print(OUT.paint("\nfio is not installed -- falling back to basic dd test.",
711		-	YELLOW))
712		-	print(OUT.paint("Install fio for the full CrystalDiskMark-style benchmark.",
713		-	DIM))
714		-	mode = "dd"
715		-
716		-	confirm(cfg)
717		-
718		-	# Dictionary dispatch: map each mode name to its handler function (the values
719		-	# are the functions themselves -- no parentheses), look up the one for `mode`,
720		-	# then call it with (cfg). Equivalent to an if/elif chain over `mode`. `mode`
721		-	# is always one of these three keys by now, so the lookup can't KeyError.
722		-	{"cdm": cdm_profile, "slog": slog_profile, "dd": dd_profile}[mode](cfg)
723		-	return 0
724		-
725		-
726		-	if __name__ == "__main__":
727		-	try:
728		-	sys.exit(main())
729		-	except KeyboardInterrupt:
730		-	print(ERR.paint("\nInterrupted.", YELLOW), file=sys.stderr)
731		-	sys.exit(130)
	6	+	wget https://git.mreschke.net/mreschke/speedtest-hd/raw/branch/master/speedtest-hd.py

mreschke revised this gist 5 days ago. Go to revision

2 files changed, 731 insertions, 366 deletions

speedtest-hd.py(file created)

		@@ -0,0 +1,731 @@
1	+	#!/usr/bin/env python3
2	+	"""speedtest-hd — a CrystalDiskMark-style storage benchmark for Linux, on fio.
3	+
4	+	This is the Python successor to ``speedtest-hd.sh``. It measures storage the way
5	+	CrystalDiskMark does, and adds a dedicated SLOG / sync-write latency profile
6	+	for diagnosing ZFS ZIL performance (NFS / iSCSI / VM sync workloads).
7	+
8	+	Three profiles
9	+	--------------
10	+	* cdm — the four CrystalDiskMark default tests (``SEQ1M Q8T1``, ``SEQ1M
11	+	Q1T1``, ``RND4K Q32T16``, ``RND4K Q1T1``), each measured for read and
12	+	write, reported in MB/s and IOPS.
13	+	* slog — synchronous 4K random writes swept across thread counts, reporting
14	+	IOPS, MB/s and p50/p99 commit latency. This is the load a ZFS SLOG sees.
15	+	* dd — a dependency-free fallback when ``fio`` isn't installed.
16	+
17	+	Why fio JSON?
18	+	-------------
19	+	Every measurement asks fio for ``--output-format=json`` and parses it with the
20	+	standard library. That's the whole reason this is Python and not bash: robust,
21	+	unit-safe parsing of bandwidth / IOPS / latency percentiles with no fragile
22	+	text scraping.
23	+
24	+	See README.md for the full case study (diagnosing a "slow" Optane SLOG that
25	+	turned out to be CPU power management, not the disk).
26	+	"""
27	+
28	+	from __future__ import annotations
29	+
30	+	import argparse
31	+	import glob
32	+	import json
33	+	import os
34	+	import shutil
35	+	import statistics
36	+	import subprocess
37	+	import sys
38	+	import time
39	+	from dataclasses import dataclass, field
40	+	from typing import Callable, Optional, Sequence
41	+
42	+	# --------------------------------------------------------------------------- #
43	+	# Color / styling (standard library only -- raw ANSI escape codes)
44	+	# --------------------------------------------------------------------------- #
45	+
46	+	# SGR escape sequences. We stick to the basic 16-color set so output respects
47	+	# the user's terminal theme instead of hard-coding RGB.
48	+	RESET = "\033[0m"
49	+	BOLD = "\033[1m"
50	+	DIM = "\033[2m"
51	+	ITALIC = "\033[3m"
52	+	RED = "\033[31m"
53	+	GREEN = "\033[32m"
54	+	YELLOW = "\033[33m"
55	+	BLUE = "\033[34m"
56	+	MAGENTA = "\033[35m"
57	+	CYAN = "\033[36m"
58	+	WHITE = "\033[37m"
59	+	BRIGHT_CYAN = "\033[96m"
60	+
61	+
62	+	def supports_color(stream) -> bool:
63	+	"""Decide per-stream whether to emit ANSI codes.
64	+
65	+	Honors the de-facto ``NO_COLOR`` / ``FORCE_COLOR`` conventions and otherwise
66	+	colors only when the stream is an interactive terminal.
67	+	"""
68	+	if os.environ.get("NO_COLOR"):
69	+	return False
70	+	if os.environ.get("FORCE_COLOR"):
71	+	return True
72	+	if os.environ.get("TERM") == "dumb":
73	+	return False
74	+	return bool(getattr(stream, "isatty", None)) and stream.isatty()
75	+
76	+
77	+	class Painter:
78	+	"""Wraps text in SGR codes, or returns it untouched when color is off."""
79	+
80	+	def __init__(self, enabled: bool):
81	+	self.enabled = enabled
82	+
83	+	def paint(self, text: str, *codes: str) -> str:
84	+	if not self.enabled or not codes:
85	+	return text
86	+	return "".join(codes) + text + RESET
87	+
88	+
89	+	# stdout carries results (banner + tables); stderr carries progress + verbose
90	+	# fio dumps. Each gets its own enable flag so piping one but not the other works.
91	+	OUT = Painter(supports_color(sys.stdout))
92	+	ERR = Painter(supports_color(sys.stderr))
93	+
94	+
95	+	# --------------------------------------------------------------------------- #
96	+	# Constants
97	+	# --------------------------------------------------------------------------- #
98	+
99	+	# Preference order for the IO engine. io_uring is the modern, lowest-overhead
100	+	# Linux engine; libaio is older (and only truly async with O_DIRECT); posixaio
101	+	# and sync are the portable fallbacks (e.g. some NFS mounts).
102	+	ENGINE_CANDIDATES = ("io_uring", "libaio", "posixaio", "sync")
103	+
104	+	# Binary unit multipliers (fio treats "1g" as 1 GiB, so we match that).
105	+	_SIZE_UNITS = {"k": 1024, "m": 10242, "g": 10243, "t": 1024**4}
106	+
107	+
108	+	@dataclass(frozen=True)
109	+	class CdmTest:
110	+	"""One CrystalDiskMark test. ``seq`` picks sequential vs random patterns."""
111	+
112	+	label: str
113	+	bs: str
114	+	iodepth: int
115	+	numjobs: int
116	+	seq: bool
117	+
118	+	@property
119	+	def read_rw(self) -> str:
120	+	return "read" if self.seq else "randread"
121	+
122	+	@property
123	+	def write_rw(self) -> str:
124	+	return "write" if self.seq else "randwrite"
125	+
126	+
127	+	# The four CrystalDiskMark default tests, as data, in CrystalDiskMark's own
128	+	# display order. Q = queue depth (iodepth), T = threads (numjobs).
129	+	CDM_TESTS: tuple[CdmTest, ...] = (
130	+	CdmTest("SEQ1M Q8T1", bs="1m", iodepth=8, numjobs=1, seq=True),
131	+	CdmTest("SEQ1M Q1T1", bs="1m", iodepth=1, numjobs=1, seq=True),
132	+	CdmTest("RND4K Q32T16", bs="4k", iodepth=32, numjobs=16, seq=False),
133	+	CdmTest("RND4K Q1T1", bs="4k", iodepth=1, numjobs=1, seq=False),
134	+	)
135	+
136	+	# Thread counts for the SLOG sweep. T1 is the headline single-stream latency;
137	+	# the higher counts show how the SLOG scales as concurrent sync writers pile on.
138	+	SLOG_THREADS: tuple[int, ...] = (1, 4, 8, 16)
139	+
140	+
141	+	# --------------------------------------------------------------------------- #
142	+	# Configuration
143	+	# --------------------------------------------------------------------------- #
144	+
145	+
146	+	@dataclass
147	+	class Config:
148	+	"""Resolved run settings. ``engine`` and ``direct`` may start unset and get
149	+	filled in by :func:`detect_io_settings`."""
150	+
151	+	path: str
152	+	mode: Optional[str] # "cdm" \| "slog" \| "dd" \| None (auto)
153	+	engine: Optional[str] # None => auto-detect
154	+	direct: Optional[bool] # None => auto-detect
155	+	runtime: int
156	+	size: str
157	+	verbose: bool
158	+	assume_yes: bool
159	+	benchfile: str = field(default="")
160	+
161	+	@property
162	+	def direct_flag(self) -> int:
163	+	"""fio's --direct takes 0/1; default to 0 until detection runs."""
164	+	return 1 if self.direct else 0
165	+
166	+
167	+	# --------------------------------------------------------------------------- #
168	+	# Small helpers
169	+	# --------------------------------------------------------------------------- #
170	+
171	+
172	+	def log(message: str) -> None:
173	+	"""Progress/diagnostic line -> stderr, so stdout stays pure results."""
174	+	print(message, file=sys.stderr, flush=True)
175	+
176	+
177	+	def step(message: str) -> None:
178	+	"""A single colored progress line ('measuring ...') -> stderr."""
179	+	log(f" {ERR.paint('▶', BOLD, CYAN)} {ERR.paint(message, DIM)}")
180	+
181	+
182	+	def vsection(title: str) -> None:
183	+	"""A clear, ruled header for one --verbose fio section -> stderr."""
184	+	width = 78
185	+	head = f"─── {title} "
186	+	head += "─" * max(3, width - len(head))
187	+	log("")
188	+	log(ERR.paint(head, BOLD, CYAN))
189	+
190	+
191	+	def parse_size_bytes(size: str) -> int:
192	+	"""Turn an fio-style size ("4k", "1g", "512m", "2048") into bytes."""
193	+	s = size.strip().lower()
194	+	if s and s[-1] in _SIZE_UNITS:
195	+	return int(float(s[:-1]) * _SIZE_UNITS[s[-1]])
196	+	return int(s)
197	+
198	+
199	+	def render_table(
200	+	headers: Sequence[str],
201	+	rows: Sequence[Sequence[str]],
202	+	aligns: Optional[Sequence[str]] = None,
203	+	col_styles: Optional[Sequence[Optional[str]]] = None,
204	+	) -> str:
205	+	"""Render a bordered ASCII table whose columns auto-size to their content.
206	+
207	+	``aligns`` is a per-column "<" (left) or ">" (right); by default the first
208	+	column is left-aligned (the label) and the rest are right-aligned (numbers).
209	+	``col_styles`` is an optional per-column ANSI style applied to body cells
210	+	(header is always bold, borders dim). Widths are computed on the plain
211	+	text and color is applied after padding, so escape codes never skew layout.
212	+	"""
213	+	ncols = len(headers)
214	+	if aligns is None:
215	+	aligns = ["<"] + [">"] * (ncols - 1)
216	+	if col_styles is None:
217	+	col_styles = [None] * ncols
218	+
219	+	# Widest cell (header or any row) sets each column's width.
220	+	widths = [
221	+	max([len(str(headers[c]))] + [len(str(r[c])) for r in rows])
222	+	for c in range(ncols)
223	+	]
224	+
225	+	bar = OUT.paint("\|", DIM)
226	+	sep = OUT.paint(" \| ", DIM)
227	+
228	+	def rule() -> str:
229	+	return OUT.paint("+" + "+".join("-" * (w + 2) for w in widths) + "+", DIM)
230	+
231	+	def line(cells: Sequence[str], *, header: bool = False) -> str:
232	+	parts = []
233	+	for c in range(ncols):
234	+	padded = f"{str(cells[c]):{aligns[c]}{widths[c]}}"
235	+	if header:
236	+	parts.append(OUT.paint(padded, BOLD))
237	+	elif col_styles[c]:
238	+	parts.append(OUT.paint(padded, col_styles[c]))
239	+	else:
240	+	parts.append(padded)
241	+	return f"{bar} " + sep.join(parts) + f" {bar}"
242	+
243	+	out = [rule(), line(headers, header=True), rule()]
244	+	out += [line(r) for r in rows]
245	+	out.append(rule())
246	+	return "\n".join(out)
247	+
248	+
249	+	# --------------------------------------------------------------------------- #
250	+	# fio: results, invocation, parsing
251	+	# --------------------------------------------------------------------------- #
252	+
253	+
254	+	@dataclass(frozen=True)
255	+	class FioResult:
256	+	"""Parsed metrics for one direction (read or write) of an fio run.
257	+
258	+	Latencies are completion latency (clat) in microseconds; for synchronous
259	+	writes that is effectively the durable-commit latency.
260	+	"""
261	+
262	+	bw_mbps: float # decimal MB/s, matching CrystalDiskMark's SI figure
263	+	iops: float
264	+	p50_us: float
265	+	p99_us: float
266	+	mean_us: float
267	+
268	+	@classmethod
269	+	def zero(cls) -> "FioResult":
270	+	return cls(0.0, 0.0, 0.0, 0.0, 0.0)
271	+
272	+
273	+	def _direction_of(rw: str) -> str:
274	+	"""fio reports 'read' and 'write' sub-objects; pick the active one."""
275	+	return "read" if "read" in rw else "write"
276	+
277	+
278	+	def _aggregate(jobs: list[dict], direction: str) -> FioResult:
279	+	"""Combine per-job fio JSON stats into a single :class:`FioResult`.
280	+
281	+	We deliberately run fio without --group_reporting and aggregate ourselves,
282	+	which sidesteps fio's group-merge quirks: sum throughput, average the median
283	+	latency, and take the worst-case tail (p99) across jobs.
284	+	"""
285	+
286	+	def section(job: dict) -> dict:
287	+	return job.get(direction, {})
288	+
289	+	bw_mbps = sum(section(j).get("bw_bytes", 0) for j in jobs) / 1e6
290	+	iops = sum(section(j).get("iops", 0.0) for j in jobs)
291	+
292	+	def percentiles(key: str) -> list[float]:
293	+	vals = []
294	+	for j in jobs:
295	+	pct = section(j).get("clat_ns", {}).get("percentile", {})
296	+	v = pct.get(key)
297	+	if v is not None:
298	+	vals.append(v)
299	+	return vals
300	+
301	+	p50_ns = percentiles("50.000000")
302	+	p99_ns = percentiles("99.000000")
303	+	mean_ns = [section(j).get("clat_ns", {}).get("mean", 0.0) for j in jobs]
304	+
305	+	return FioResult(
306	+	bw_mbps=bw_mbps,
307	+	iops=iops,
308	+	p50_us=(statistics.mean(p50_ns) / 1000.0) if p50_ns else 0.0, # avg of medians
309	+	p99_us=(max(p99_ns) / 1000.0) if p99_ns else 0.0, # worst tail
310	+	mean_us=(statistics.mean(mean_ns) / 1000.0) if mean_ns else 0.0,
311	+	)
312	+
313	+
314	+	def run_fio(
315	+	cfg: Config,
316	+	*,
317	+	rw: str,
318	+	bs: str,
319	+	iodepth: int,
320	+	numjobs: int,
321	+	engine: Optional[str] = None,
322	+	direct: Optional[bool] = None,
323	+	sync: bool = False,
324	+	end_fsync: bool = False,
325	+	) -> FioResult:
326	+	"""Run a single fio job against the shared bench file and parse its JSON.
327	+
328	+	``engine``/``direct`` default to the detected config values; the SLOG path
329	+	overrides them (psync + buffered + O_SYNC). ``end_fsync`` flushes the device
330	+	cache at the end of a write run so cached writes can't inflate the result.
331	+	"""
332	+	engine = engine if engine is not None else cfg.engine
333	+	direct = direct if direct is not None else cfg.direct
334	+
335	+	cmd = [
336	+	"sudo", "fio",
337	+	"--name=speedtest",
338	+	f"--filename={cfg.benchfile}",
339	+	f"--ioengine={engine}",
340	+	f"--direct={1 if direct else 0}",
341	+	f"--rw={rw}",
342	+	f"--bs={bs}",
343	+	f"--size={cfg.size}",
344	+	f"--numjobs={numjobs}",
345	+	f"--iodepth={iodepth}",
346	+	"--time_based",
347	+	f"--runtime={cfg.runtime}",
348	+	"--output-format=json",
349	+	]
350	+	if sync:
351	+	cmd.append("--sync=1") # O_SYNC: every write is a durable commit
352	+	if end_fsync:
353	+	cmd.append("--end_fsync=1") # flush device cache once at the end
354	+
355	+	proc = subprocess.run(cmd, capture_output=True, text=True)
356	+
357	+	if cfg.verbose:
358	+	sync_tag = " sync=1" if sync else ""
359	+	vsection(f"fio · {rw} bs={bs} iodepth={iodepth} numjobs={numjobs}{sync_tag}")
360	+	log(" " + ERR.paint("$ " + " ".join(cmd[1:]), DIM, ITALIC))
361	+	log(ERR.paint(proc.stdout.strip() or "(no stdout)", DIM))
362	+	if proc.stderr.strip():
363	+	log(ERR.paint(proc.stderr.strip(), YELLOW))
364	+
365	+	try:
366	+	data = json.loads(proc.stdout)
367	+	return _aggregate(data["jobs"], _direction_of(rw))
368	+	except (json.JSONDecodeError, KeyError, ValueError):
369	+	# fio failed or produced no parseable JSON; report zeros rather than
370	+	# crashing the whole run (verbose mode above shows what went wrong).
371	+	return FioResult.zero()
372	+
373	+
374	+	def fio_probe(path: str, engine: str, direct: bool) -> bool:
375	+	"""Throwaway 1s fio job to see if an (engine, O_DIRECT) combo works here.
376	+
377	+	This is how we stay accurate and portable across ext4/xfs/btrfs/ZFS/NFS
378	+	without the caller needing to know what each filesystem supports.
379	+	"""
380	+	cmd = [
381	+	"sudo", "fio", "--name=probe",
382	+	f"--directory={path}",
383	+	f"--ioengine={engine}",
384	+	"--rw=write", "--bs=4k", "--size=1m",
385	+	f"--direct={1 if direct else 0}",
386	+	"--time_based", "--runtime=1",
387	+	]
388	+	proc = subprocess.run(cmd, capture_output=True, text=True)
389	+	for leftover in glob.glob(os.path.join(path, "probe*")):
390	+	try:
391	+	os.remove(leftover)
392	+	except OSError:
393	+	pass
394	+	return proc.returncode == 0
395	+
396	+
397	+	def detect_io_settings(cfg: Config) -> None:
398	+	"""Fill in ``cfg.engine`` and ``cfg.direct`` if the user didn't force them.
399	+
400	+	O_DIRECT bypasses the OS page cache so we measure the device, not RAM. Not
401	+	every filesystem supports it (older OpenZFS, some NFS), so we probe and fall
402	+	back to buffered rather than erroring out.
403	+	"""
404	+	if cfg.engine is None:
405	+	for engine in ENGINE_CANDIDATES:
406	+	if fio_probe(cfg.path, engine, direct=False):
407	+	cfg.engine = engine
408	+	break
409	+	else:
410	+	cfg.engine = "sync"
411	+
412	+	if cfg.direct is None:
413	+	cfg.direct = fio_probe(cfg.path, cfg.engine, direct=True)
414	+
415	+
416	+	# --------------------------------------------------------------------------- #
417	+	# Profiles
418	+	# --------------------------------------------------------------------------- #
419	+
420	+
421	+	def _meta_line(text: str) -> str:
422	+	"""Color the 'Label :' prefix of a banner metadata line, leave value as-is.
423	+
424	+	The value may already contain its own ANSI codes (e.g. a red O_DIRECT
425	+	warning); since these lines aren't width-aligned that's harmless.
426	+	"""
427	+	key, sep, val = text.partition(":")
428	+	if not sep:
429	+	return " " + text
430	+	return " " + OUT.paint(key + ":", BOLD, BRIGHT_CYAN) + val
431	+
432	+
433	+	def banner(title: str, cfg: Config, extra: Sequence[str] = ()) -> None:
434	+	full = f"speedtest-hd · {title}"
435	+	inner = len(full) + 4
436	+
437	+	print()
438	+	print(OUT.paint("╭" + "─" * inner + "╮", BOLD, CYAN))
439	+	print(
440	+	OUT.paint("│", BOLD, CYAN)
441	+	+ " " + OUT.paint(full, BOLD, WHITE) + " "
442	+	+ OUT.paint("│", BOLD, CYAN)
443	+	)
444	+	print(OUT.paint("╰" + "─" * inner + "╯", BOLD, CYAN))
445	+	print(_meta_line(f"Target : {OUT.paint(cfg.path, CYAN)}"))
446	+	for line_ in extra:
447	+	print(_meta_line(line_))
448	+	print()
449	+
450	+
451	+	def cdm_profile(cfg: Config) -> None:
452	+	"""The CrystalDiskMark-style profile: 4 tests x (read, write), MB/s + IOPS."""
453	+	detect_io_settings(cfg)
454	+
455	+	if cfg.direct:
456	+	direct_label = OUT.paint("enabled (device)", GREEN)
457	+	else:
458	+	direct_label = OUT.paint(
459	+	"DISABLED (buffered -- may reflect RAM cache!)", BOLD, RED
460	+	)
461	+
462	+	banner(
463	+	"CrystalDiskMark-style storage benchmark",
464	+	cfg,
465	+	extra=[
466	+	f"Engine : {cfg.engine} O_DIRECT: {direct_label}",
467	+	f"Profile : size={cfg.size.upper()} runtime={cfg.runtime}s/run (8 runs)",
468	+	],
469	+	)
470	+
471	+	rows: list[list[str]] = []
472	+	for test in CDM_TESTS:
473	+	step(f"measuring {test.label.strip()} ...")
474	+	r = run_fio(cfg, rw=test.read_rw, bs=test.bs,
475	+	iodepth=test.iodepth, numjobs=test.numjobs)
476	+	w = run_fio(cfg, rw=test.write_rw, bs=test.bs,
477	+	iodepth=test.iodepth, numjobs=test.numjobs, end_fsync=True)
478	+	rows.append([
479	+	test.label,
480	+	f"{r.bw_mbps:.2f}", f"{w.bw_mbps:.2f}",
481	+	f"{r.iops:.0f}", f"{w.iops:.0f}",
482	+	])
483	+
484	+	_cleanup(cfg)
485	+
486	+	print()
487	+	print(render_table(
488	+	["Test", "Read (MB/s)", "Write (MB/s)", "Read (IOPS)", "Write (IOPS)"],
489	+	rows,
490	+	col_styles=[CYAN, GREEN, YELLOW, GREEN, YELLOW],
491	+	))
492	+	print()
493	+
494	+
495	+	def slog_profile(cfg: Config) -> None:
496	+	"""SLOG / sync-write latency profile.
497	+
498	+	Forces synchronous 4K random writes (O_SYNC) via the portable psync engine,
499	+	so every write is a ZIL commit -- exercising a ZFS SLOG exactly the way a
500	+	sync=always dataset (NFS/iSCSI/VM) does, regardless of the dataset's own
501	+	sync property. Engine/direct detection is only for the banner here.
502	+	"""
503	+	detect_io_settings(cfg)
504	+
505	+	banner(
506	+	"SLOG / sync-write latency profile",
507	+	cfg,
508	+	extra=[
509	+	"Method : fio randwrite bs=4k --sync=1 (O_SYNC), psync engine",
510	+	f"Profile : runtime={cfg.runtime}s/run size={cfg.size.upper()}",
511	+	"Note : every write is a synchronous ZIL commit -- the load your",
512	+	" SLOG actually sees. Watch it live in another shell with:",
513	+	" zpool iostat -vl <pool> 1",
514	+	],
515	+	)
516	+
517	+	# Measure everything first (progress -> stderr), then draw the whole table,
518	+	# so the "measuring..." lines can't interleave into the middle of it.
519	+	rows: list[list[str]] = []
520	+	for threads in SLOG_THREADS:
521	+	step(f"measuring 4K sync randwrite T{threads} ...")
522	+	res = run_fio(
523	+	cfg, rw="randwrite", bs="4k", iodepth=1, numjobs=threads,
524	+	engine="psync", direct=False, sync=True,
525	+	)
526	+	rows.append([
527	+	f"4K sync T{threads}",
528	+	f"{res.iops:.0f}", f"{res.bw_mbps:.2f}",
529	+	f"{res.p50_us:.1f}", f"{res.p99_us:.1f}",
530	+	])
531	+
532	+	_cleanup(cfg)
533	+
534	+	print()
535	+	print(render_table(
536	+	["Test", "IOPS", "MB/s", "p50 lat(us)", "p99 lat(us)"],
537	+	rows,
538	+	col_styles=[CYAN, GREEN, GREEN, YELLOW, MAGENTA],
539	+	))
540	+	print()
541	+	print(OUT.paint(" Healthy Optane SLOG (eg P1600X) single-stream (T1) target:", BOLD))
542	+	print(OUT.paint(" ~15-25k IOPS, p50 latency ~40-65us.", GREEN)
543	+	+ OUT.paint(" Much higher latency usually means", DIM))
544	+	print(OUT.paint(" CPU C-states / PCIe ASPM / BIOS power profile (eg Dell DAPC) throttling.", DIM))
545	+	print()
546	+
547	+
548	+	def dd_profile(cfg: Config) -> None:
549	+	"""Dependency-free fallback when fio isn't installed.
550	+
551	+	Far cruder than fio: a single sequential stream, and the cached-read figure
552	+	will reflect RAM. Use it only as a rough sanity check.
553	+	"""
554	+	count_mib = max(1, parse_size_bytes(cfg.size) // (1024**2))
555	+	nbytes = count_mib * 1024**2
556	+
557	+	banner(
558	+	"basic dd benchmark (fio not installed)",
559	+	cfg,
560	+	extra=[f"Profile : {count_mib} MiB sequential stream"],
561	+	)
562	+
563	+	def timed_dd(args: list[str]) -> float:
564	+	start = time.monotonic()
565	+	subprocess.run(args, capture_output=True, text=True)
566	+	elapsed = time.monotonic() - start
567	+	return (nbytes / elapsed / 1e6) if elapsed > 0 else 0.0 # decimal MB/s
568	+
569	+	step("measuring sequential write ...")
570	+	write_mbps = timed_dd([
571	+	"dd", "if=/dev/zero", f"of={cfg.benchfile}",
572	+	"bs=1M", f"count={count_mib}", "conv=fdatasync,notrunc",
573	+	])
574	+
575	+	step("dropping caches for uncached read ...")
576	+	subprocess.run(["sudo", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"],
577	+	capture_output=True, text=True)
578	+
579	+	step("measuring uncached read ...")
580	+	uncached_mbps = timed_dd([
581	+	"dd", f"if={cfg.benchfile}", "of=/dev/null", "bs=1M", f"count={count_mib}",
582	+	])
583	+
584	+	step("measuring cached read ...")
585	+	cached_mbps = timed_dd([
586	+	"dd", f"if={cfg.benchfile}", "of=/dev/null", "bs=1M", f"count={count_mib}",
587	+	])
588	+
589	+	_cleanup(cfg)
590	+
591	+	print()
592	+	print(render_table(
593	+	["Test", "MB/s"],
594	+	[
595	+	["Sequential write", f"{write_mbps:.2f}"],
596	+	["Uncached read", f"{uncached_mbps:.2f}"],
597	+	["Cached read (RAM)", f"{cached_mbps:.2f}"],
598	+	],
599	+	col_styles=[CYAN, GREEN],
600	+	))
601	+	print()
602	+
603	+
604	+	def _cleanup(cfg: Config) -> None:
605	+	"""Remove the shared bench file."""
606	+	try:
607	+	os.remove(cfg.benchfile)
608	+	except OSError:
609	+	pass
610	+
611	+
612	+	# --------------------------------------------------------------------------- #
613	+	# CLI
614	+	# --------------------------------------------------------------------------- #
615	+
616	+
617	+	def build_parser() -> argparse.ArgumentParser:
618	+	p = argparse.ArgumentParser(
619	+	prog="speedtest-hd.py",
620	+	description="CrystalDiskMark-style storage benchmark built on fio.",
621	+	epilog=(
622	+	"Examples:\n"
623	+	" speedtest-hd.py .\n"
624	+	" speedtest-hd.py /mnt/nvmepool --runtime=10 --size=4g\n"
625	+	" speedtest-hd.py /mnt/nfsshare --buffered\n"
626	+	" speedtest-hd.py /mnt/nvme-ultra-r10/vm-root --slog --runtime=30\n"
627	+	),
628	+	formatter_class=argparse.RawDescriptionHelpFormatter,
629	+	)
630	+	p.add_argument("path", help="directory/mount to benchmark ('.' for cwd)")
631	+
632	+	mode = p.add_mutually_exclusive_group()
633	+	mode.add_argument("--fio", action="store_const", const="cdm", dest="mode",
634	+	help="force the fio CrystalDiskMark-style profile")
635	+	mode.add_argument("--dd", action="store_const", const="dd", dest="mode",
636	+	help="force the basic dd fallback test")
637	+	mode.add_argument("--slog", action="store_const", const="slog", dest="mode",
638	+	help="SLOG / sync-write latency profile (ZFS ZIL)")
639	+
640	+	direct = p.add_mutually_exclusive_group()
641	+	direct.add_argument("--direct", action="store_const", const=True, dest="direct",
642	+	help="force O_DIRECT (bypass page cache)")
643	+	direct.add_argument("--buffered", action="store_const", const=False, dest="direct",
644	+	help="force buffered IO (e.g. if O_DIRECT unsupported)")
645	+
646	+	p.add_argument("--engine", choices=ENGINE_CANDIDATES,
647	+	help="force a specific IO engine (default: auto-detect)")
648	+	p.add_argument("--runtime", type=int, default=5, metavar="SEC",
649	+	help="seconds per run (default: 5, like CrystalDiskMark)")
650	+	p.add_argument("--size", default="1g", metavar="SIZE",
651	+	help="test file size (default: 1g)")
652	+	p.add_argument("--verbose", action="store_true",
653	+	help="also print the full fio output for every run")
654	+	p.add_argument("-y", "--yes", action="store_true", dest="assume_yes",
655	+	help="skip the confirmation prompt")
656	+
657	+	p.set_defaults(mode=None, direct=None)
658	+	return p
659	+
660	+
661	+	def confirm(cfg: Config) -> None:
662	+	"""Guard prompt -- we're about to write a multi-GB file to the target."""
663	+	if cfg.assume_yes:
664	+	return
665	+	print(OUT.paint("NOTICE:", BOLD, YELLOW)
666	+	+ f" {cfg.size.upper()} free space on "
667	+	+ OUT.paint(f"'{cfg.path}'", CYAN)
668	+	+ " is required to perform the benchmark.")
669	+	answer = input(f"Are you ready to start a storage benchmark against "
670	+	f"'{cfg.path}' ? ")
671	+	if not answer.strip().lower().startswith("y"):
672	+	print(OUT.paint("Ok, cancelled!", YELLOW))
673	+	sys.exit(0)
674	+	print(OUT.paint("Great! Starting benchmark now!", BOLD, GREEN))
675	+
676	+
677	+	def main(argv: Optional[Sequence[str]] = None) -> int:
678	+	args = build_parser().parse_args(argv)
679	+
680	+	path = os.getcwd() if args.path == "." else args.path
681	+	if not os.path.exists(path):
682	+	print(f"Path {path} does not exist", file=sys.stderr)
683	+	return 1
684	+
685	+	cfg = Config(
686	+	path=path,
687	+	mode=args.mode,
688	+	engine=args.engine,
689	+	direct=args.direct,
690	+	runtime=args.runtime,
691	+	size=args.size,
692	+	verbose=args.verbose,
693	+	assume_yes=args.assume_yes,
694	+	benchfile=os.path.join(path, "speedtest-hd.bench"),
695	+	)
696	+
697	+	have_fio = shutil.which("fio") is not None
698	+
699	+	# Resolve the mode: explicit flag wins; otherwise fio if available, else dd.
700	+	mode = cfg.mode
701	+	if mode in ("cdm", "slog") and not have_fio:
702	+	print(ERR.paint("ERROR:", BOLD, RED)
703	+	+ " --fio/--slog require fio (apt install fio / pacman -S fio).",
704	+	file=sys.stderr)
705	+	return 1
706	+	if mode is None:
707	+	if have_fio:
708	+	mode = "cdm"
709	+	else:
710	+	print(OUT.paint("\nfio is not installed -- falling back to basic dd test.",
711	+	YELLOW))
712	+	print(OUT.paint("Install fio for the full CrystalDiskMark-style benchmark.",
713	+	DIM))
714	+	mode = "dd"
715	+
716	+	confirm(cfg)
717	+
718	+	# Dictionary dispatch: map each mode name to its handler function (the values
719	+	# are the functions themselves -- no parentheses), look up the one for `mode`,
720	+	# then call it with (cfg). Equivalent to an if/elif chain over `mode`. `mode`
721	+	# is always one of these three keys by now, so the lookup can't KeyError.
722	+	{"cdm": cdm_profile, "slog": slog_profile, "dd": dd_profile}[mode](cfg)
723	+	return 0
724	+
725	+
726	+	if __name__ == "__main__":
727	+	try:
728	+	sys.exit(main())
729	+	except KeyboardInterrupt:
730	+	print(ERR.paint("\nInterrupted.", YELLOW), file=sys.stderr)
731	+	sys.exit(130)

speedtest-hd.sh (file deleted)

		@@ -1,366 +0,0 @@
1	-	#!/usr/bin/env bash
2	-
3	-	# Robust HD/SDD/NVMe performance CLI utility
4	-	# Utilizing FIO for sequential/random writes/writes
5	-	# Dependencies: fio (apt install fio)
6	-	# See: https://cloud.google.com/compute/docs/disks/benchmarking-pd-performance
7	-	# See: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
8	-	# mReschke 2024-01-18
9	-
10	-	# CLI Parameters
11	-	path="$1"
12	-	option=""
13	-	simple=false
14	-	for arg in "${@:2}"; do
15	-	case "$arg" in
16	-	--simple) simple=true ;;
17	-	--dd) option="--dd" ;;
18	-	--fio) option="--fio" ;;
19	-	esac
20	-	done
21	-
22	-	# Main application flow
23	-	function main {
24	-
25	-	# Show usage if no params
26	-	if [ ! "$path" ]; then
27	-	usage
28	-	fi
29	-
30	-	# Understand . path
31	-	if [ "$path" == '.' ]; then
32	-	path=$(pwd)
33	-	fi
34	-
35	-	# Check if path exists
36	-	if [ ! -e "$path" ]; then
37	-	echo "Path $path does not exist"
38	-	exit 1
39	-	fi
40	-
41	-	# Must type y or n THEN press enter (which I like better)
42	-	echo "NOTICE: 1GB free space on '$path' is required to perform the benchmark."
43	-	echo -n "Are you ready to start a robust IO benchmark against '$path' ?"; read answer
44	-	if [ "$answer" != "${answer#[Yy]}" ]; then
45	-	echo "Great! Starting benchmark now!"
46	-	echo "------------------------------"
47	-	else
48	-	echo "Ok, cancelled!"
49	-	exit 0
50	-	fi
51	-
52	-	# Use dd of fio based on param or defaults
53	-	if [ "$option" == "--dd" ]; then
54	-	dd_speedtest
55	-	elif [ "$option" == "--fio" ]; then
56	-	fio_speedtest
57	-	elif [ "$option" == "" ]; then
58	-	# If fio is installed, use it, else use dd
59	-	[ "$simple" != true ] && echo ""
60	-	if ! command -v fio &> /dev/null; then
61	-	dd_speedtest
62	-	else
63	-	fio_speedtest
64	-	fi
65	-	fi
66	-	}
67	-
68	-	function print_result {
69	-	# Print a single fio test result, either as the full "Run status group"
70	-	# line or, when --simple is given, as a compact aligned "Label value" row.
71	-	# The simple value is taken from the parenthetical (MB/s) figure that
72	-	# follows the first bw= value in fio's output.
73	-	local label="$1"
74	-	local output="$2"
75	-	local status
76	-	status=$(echo " - $output " \| /usr/bin/grep -A1 'Run status group' \| tail -n1)
77	-	if [ "$simple" == true ]; then
78	-	local value
79	-	# Grab the parenthetical (MB/s) figure after the first bw=, then put a
80	-	# space between the number and the unit, eg "98.4MB/s" -> "98.4 MB/s".
81	-	value=$(echo "$status" \| /usr/bin/grep -oP 'bw=\S+\s+\(\K[^)]+' \| sed -E 's/([0-9.]+)([A-Za-z])/\1 \2/')
82	-	printf "%-28s%s\n" "$label" "$value"
83	-	else
84	-	echo "$status"
85	-	fi
86	-	}
87	-
88	-	function fio_write_single_random_4k {
89	-	# Single 4k Random Writes
90	-
91	-	# This is a single process doing random 4K writes. This is where the pain
92	-	# really, really lives; it's basically the worst possible thing you can ask a
93	-	# disk to do. Where this happens most frequently in real life: copying home
94	-	# directories and dotfiles, manipulating email stuff, some database operations,
95	-	# source code trees.
96	-
97	-	# When I ran this test against the high-performance SSDs in my Ubuntu
98	-	# workstation, they pushed 127MiB/sec. The server just beneath it in the rack
99	-	# only managed 33MiB/sec on its "high-performance" 7200RPM rust disks... but
100	-	# even then, the vast majority of that speed is because the data is being
101	-	# written asynchronously, allowing the operating system to batch it up into
102	-	# larger, more efficient write operations.
103	-
104	-	# If we add the argument --fsync=1, forcing the operating system to perform
105	-	# synchronous writes (calling fsync after each block of data is written) the
106	-	# picture gets much more grim: 2.6MiB/sec on the high-performance SSDs but
107	-	# only 184KiB/sec on the "high-performance" rust. The SSDs were about four
108	-	# times faster than the rust when data was written asynchronously but a
109	-	# whopping fourteen times faster when
110	-
111	-	# --name= is a required argument, but it's basically human-friendly fluff—fio will create files based on that name to test with, inside the working directory you're currently in.
112	-	# --ioengine=posixaio sets the mode fio interacts with the filesystem. POSIX is a standard Windows, Macs, Linux, and BSD all understand, so it's great for portability—although inside fio itself, Windows users need to invoke --ioengine=windowsaio, not --ioengine=posixaio, unfortunately. AIO stands for Asynchronous Input Output and means that we can queue up multiple operations to be completed in whatever order the OS decides to complete them. (In this particular example, later arguments effectively nullify this.)
113	-	# --rw=randwrite means exactly what it looks like it means: we're going to do random write operations to our test files in the current working directory. Other options include seqread, seqwrite, randread, and randrw, all of which should hopefully be fairly self-explanatory.
114	-	# --bs=4k blocksize 4K. These are very small individual operations. This is where the pain lives; it's hard on the disk, and it also means a ton of extra overhead in the SATA, USB, SAS, SMB, or whatever other command channel lies between us and the disks, since a separate operation has to be commanded for each 4K of data.
115	-	# --size=1g our test file(s) will be 1GB in size apiece. (We're only creating one, see next argument.)
116	-	# --numjobs=1 we're only creating a single file, and running a single process commanding operations within that file. If we wanted to simulate multiple parallel processes, we'd do, eg, --numjobs=16, which would create 16 separate test files of --size size, and 16 separate processes operating on them at the same time.
117	-	# --iodepth=1 this is how deep we're willing to try to stack commands in the OS's queue. Since we set this to 1, this is effectively pretty much the same thing as the sync IO engine—we're only asking for a single operation at a time, and the OS has to acknowledge receipt of every operation we ask for before we can ask for another. (It does not have to satisfy the request itself before we ask it to do more operations, it just has to acknowledge that we actually asked for it.)
118	-	# --runtime=15 --time_based Run and even if we complete sooner, just start over again and keep going until 60 seconds is up.
119	-	# --end_fsync=1 After all operations have been queued, keep the timer going until the OS reports that the very last one of them has been successfully completed—ie, actually written to disk.
120	-	[ "$simple" != true ] && { echo ""; echo "Single 4K Random Writes (size=1G, time=15sec, jobs=1, iodepth=1)"; }
121	-	x=`sudo fio \
122	-	--name=fio-write-random-4k \
123	-	--directory=$path \
124	-	--ioengine=posixaio \
125	-	--rw=randwrite \
126	-	--bs=4k \
127	-	--size=1g \
128	-	--numjobs=1 \
129	-	--iodepth=1 \
130	-	--time_based --runtime=15 \
131	-	--end_fsync=1`
132	-	print_result "Single 4K Random Writes" "$x"
133	-
134	-	# Cleanup my test files
135	-	rm -rf $path/fio-write-random-4k*
136	-	}
137	-
138	-	function fio_write_parallel_random_64k {
139	-	# Parallel 64k Random Writes
140	-
141	-	# This time, we're creating 16 separate 64MB files (still totaling 1GB, when
142	-	# all put together) and we're issuing 64KB blocksized random write operations.
143	-	# We're doing it with sixteen separate processes running in parallel, and
144	-	# we're queuing up to 16 simultaneous asynchronous ops before we pause and wait
145	-	# for the OS to start acknowledging their receipt.
146	-
147	-	# This is a pretty decent approximation of a significantly busy system. It's
148	-	# not doing any one particularly nasty thing—like running a database engine or
149	-	# copying tons of dotfiles from a user's home directory—but it is coping with
150	-	# a bunch of applications doing moderately demanding stuff all at once.
151	-
152	-	# This is also a pretty good, slightly pessimistic approximation of a busy,
153	-	# multi-user system like a NAS, which needs to handle multiple 1MB operations
154	-	# simultaneously for different users. If several people or processes are trying
155	-	# to read or write big files (photos, movies, whatever) at once, the OS tries
156	-	# to feed them all data simultaneously. This pretty quickly devolves down to a
157	-	# pattern of multiple random small block access. So in addition to "busy desktop
158	-	# with lots of apps," think "busy fileserver with several people actively using it."
159	-
160	-	# You will see a lot more variation in speed as you watch this operation play
161	-	# out on the console. For example, the 4K single process test we tried first
162	-	# wrote a pretty consistent 11MiB/sec on my MacBook Air's internal drive—but
163	-	# this 16-process job fluctuated between about 10MiB/sec and 300MiB/sec during
164	-	# the run, finishing with an average of 126MiB/sec.
165	-
166	-	# Most of the variation you're seeing here is due to the operating system and
167	-	# SSD firmware sometimes being able to aggregate multiple writes. When it
168	-	# manages to aggregate them helpfully, it can write them in a way that allows
169	-	# parallel writes to all the individual physical media stripes inside the SSD.
170	-	# Sometimes, it still ends up having to give up and write to only a single
171	-	# physical media stripe at a time—or a garbage collection or other maintenance
172	-	# operation at the SSD firmware level needs to run briefly in the background,
173	-	# slowing things down.
174	-	[ "$simple" != true ] && { echo ""; echo "Parallel 64K Random Writes (size=1G, time=15sec, jobs=16, iodepth=16)"; }
175	-	x=`sudo fio \
176	-	--name=fio-write-random-64k \
177	-	--directory=$path \
178	-	--ioengine=posixaio \
179	-	--rw=randwrite \
180	-	--bs=64k \
181	-	--size=64m \
182	-	--numjobs=16 \
183	-	--iodepth=16 \
184	-	--time_based --runtime=15 \
185	-	--end_fsync=1`
186	-	print_result "Parallel 64K Random Writes" "$x"
187	-
188	-	# Cleanup my test files
189	-	rm -rf $path/fio-write-random-64k*
190	-	}
191	-
192	-	function fio_write_single_sequential_1m {
193	-	# Single 1M Random Writes
194	-
195	-	# This is pretty close to the best-case scenario for a real-world system
196	-	# doing real-world things. No, it's not quite as fast as a single, truly
197	-	# contiguous write... but the 1MiB blocksize is large enough that it's quite
198	-	# close. Besides, if literally any other disk activity is requested simultaneously
199	-	# with a contiguous write, the "contiguous" write devolves to this level of
200	-	# performance pretty much instantly, so this is a much more realistic test of
201	-	# the upper end of storage performance on a typical system.
202	-
203	-	# You'll see some kooky fluctuations on SSDs when doing this test. This is largely
204	-	# due to the SSD's firmware having better luck or worse luck at any given time,
205	-	# when it's trying to queue operations so that it can write across all physical
206	-	# media stripes cleanly at once. Rust disks will tend to provide a much more
207	-	# consistent, though typically lower, throughput across the run.
208	-
209	-	# You can also see SSD performance fall off a cliff here if you exhaust an
210	-	# onboard write cache—TLC and QLC drives tend to have small write cache areas
211	-	# made of much faster MLC or SLC media. Once those get exhausted, the disk has
212	-	# to drop to writing directly to the much slower TLC/QLC media where the data
213	-	# eventually lands. This is the major difference between, for example, Samsung
214	-	# EVO and Pro SSDs—the EVOs have slow TLC media with a fast MLC cache, where
215	-	# the Pros use the higher-performance, higher-longevity MLC media throughout
216	-	# the entire SSD.
217	-
218	-	# If you have any doubt at all about a TLC or QLC disk's ability to sustain
219	-	# heavy writes, you may want to experimentally extend your time duration here.
220	-	# If you watch the throughput live as the job progresses, you'll see the impact
221	-	# immediately when you run out of cache—what had been a fairly steady,
222	-	# several-hundred-MiB/sec throughput will suddenly plummet to half the speed
223	-	# or less and get considerably less stable as well.
224	-
225	-	# However, you might choose to take the opposite position—you might not
226	-	# expect to do sustained heavy writes very frequently, in which case you
227	-	# actually are more interested in the on-cache behavior. What's important
228	-	# here is that you understand both what you want to test, and how to test
229	-	# it accurately.
230	-
231	-	[ "$simple" != true ] && { echo ""; echo "Single 1M Sequential Writes (size=1G, time=15sec, jobs=1, iodepth=1)"; }
232	-	x=`sudo fio \
233	-	--name=fio-write-random-1m \
234	-	--directory=$path \
235	-	--ioengine=posixaio \
236	-	--rw=write \
237	-	--bs=1m \
238	-	--size=1g \
239	-	--numjobs=1 \
240	-	--iodepth=1 \
241	-	--time_based --runtime=15 \
242	-	--end_fsync=1`
243	-	print_result "Single 1M Sequential Writes" "$x"
244	-
245	-	# Cleanup my test files
246	-	rm -rf $path/fio-write-random-1m*
247	-	}
248	-
249	-	function fio_read_sequential_1m {
250	-	# Sequential Parallel Reads
251	-
252	-	[ "$simple" != true ] && { echo ""; echo "Sequential 4x 1M Reads"; }
253	-	x=`sudo fio \
254	-	--name=fio-read-sequential-1m \
255	-	--directory=$path \
256	-	--ioengine=posixaio \
257	-	--bs=1M \
258	-	--numjobs=4 \
259	-	--size=256M \
260	-	--time_based --runtime=30s \
261	-	--ramp_time=2s \
262	-	--direct=1 \
263	-	--verify=0 \
264	-	--iodepth=64 \
265	-	--rw=read \
266	-	--group_reporting=1 \
267	-	--iodepth_batch_submit=64 \
268	-	--iodepth_batch_complete_max=64`
269	-	print_result "Sequential 4x 1M Reads" "$x"
270	-	rm -rf $path/fio-read-sequential-1m*
271	-	}
272	-
273	-	function fio_read_random_4k {
274	-	# Random 4k Reads
275	-
276	-	[ "$simple" != true ] && { echo ""; echo "Random 4k Reads"; }
277	-	x=`sudo fio \
278	-	--name=fio-read-random-4k \
279	-	--directory=$path \
280	-	--ioengine=posixaio \
281	-	--rw=randread \
282	-	--bs=4k \
283	-	--size=1g \
284	-	--time_based --runtime=30s \
285	-	--ramp_time=2s \
286	-	--direct=1 \
287	-	--verify=0 \
288	-	--iodepth=256 \
289	-	--rw=read \
290	-	--group_reporting=1 \
291	-	--iodepth_batch_submit=256 \
292	-	--iodepth_batch_complete_max=256`
293	-	print_result "Random 4k Reads" "$x"
294	-	rm -rf $path/fio-read-random-4k*
295	-	}
296	-
297	-	function fio_speedtest {
298	-	# Write tests
299	-	fio_write_single_random_4k
300	-	fio_write_parallel_random_64k
301	-	fio_write_single_sequential_1m
302	-
303	-	# Read Tests
304	-	fio_read_sequential_1m
305	-	fio_read_random_4k
306	-	}
307	-
308	-	function dd_speedtest {
309	-	# Basic HD speed test using DD
310	-	# mReschke 2017-07-11
311	-
312	-	file=$path/bigfile
313	-	size=1024
314	-
315	-	echo "Running dd based HD/SSD/NVMe Benchmarks"
316	-	echo "---------------------------------------"
317	-
318	-	printf "Cached write speed...\n"
319	-	dd if=/dev/zero of=$file bs=1M count=$size
320	-
321	-	printf "\nUncached write speed...\n"
322	-	dd if=/dev/zero of=$file bs=1M count=$size conv=fdatasync,notrunc
323	-
324	-	printf "\nUncached read speed...\n"
325	-	echo 3 \| sudo tee /proc/sys/vm/drop_caches > /dev/null
326	-	dd if=$file of=/dev/null bs=1M count=$size
327	-
328	-	printf "\nCached read speed...\n"
329	-	dd if=$file of=/dev/null bs=1M count=$size
330	-
331	-	rm $file
332	-	printf "\nDone\n"
333	-	}
334	-
335	-	# Show help and usage information
336	-	function usage {
337	-	echo "Robust Flexible Input/Output HD Speedtest"
338	-	echo " If FIO is installed, we use FIO for more detailed performance analysis."
339	-	echo " If FIO is not installed, we use basic DD analysis."
340	-	echo " You should apt install fio (pacman -S fio) for detailed analysis."
341	-	echo "mReschke 2024-01-18"
342	-	echo ""
343	-	echo "NOTICE, this creates a 1GB file on the desired destination disk."
344	-	echo "Please ensure you have write access with 1GB free space on destination."
345	-	echo ""
346	-	echo "Usage:"
347	-	echo " This will use FIO if installed, else DD"
348	-	echo " ./speedtest-hd /mnt/somedisk"
349	-	echo " ./speedtest-hd ."
350	-	echo ""
351	-	echo " This will force FIO"
352	-	echo " ./speedtest-hd /mnt/somedisk --fio"
353	-	echo " ./speedtest-hd . --fio"
354	-	echo ""
355	-	echo " This will force DD"
356	-	echo " ./speedtest-hd /mnt/somedisk --dd"
357	-	echo " ./speedtest-hd . --dd"
358	-	echo ""
359	-	echo " Add --simple (FIO only) for a compact, aligned summary of MB/s values"
360	-	echo " ./speedtest-hd . --simple"
361	-	echo " ./speedtest-hd . --fio --simple"
362	-	exit 0
363	-	}
364	-
365	-	# Go
366	-	main

mreschke revised this gist 2 weeks ago. Go to revision

1 file changed, 47 insertions, 19 deletions

speedtest-hd.sh

			@@ -9,7 +9,15 @@
9	9
10	10		# CLI Parameters
11	11		path="$1"
12		-	option="$2"
	12	+	option=""
	13	+	simple=false
	14	+	for arg in "${@:2}"; do
	15	+	case "$arg" in
	16	+	--simple) simple=true ;;
	17	+	--dd) option="--dd" ;;
	18	+	--fio) option="--fio" ;;
	19	+	esac
	20	+	done
13	21
14	22		# Main application flow
15	23		function main {
			@@ -34,7 +42,8 @@ function main {
34	42		echo "NOTICE: 1GB free space on '$path' is required to perform the benchmark."
35	43		echo -n "Are you ready to start a robust IO benchmark against '$path' ?"; read answer
36	44		if [ "$answer" != "${answer#[Yy]}" ]; then
37		-	echo "Great! Starting benchmark now!";
	45	+	echo "Great! Starting benchmark now!"
	46	+	echo "------------------------------"
38	47		else
39	48		echo "Ok, cancelled!"
40	49		exit 0
			@@ -47,7 +56,7 @@ function main {
47	56		fio_speedtest
48	57		elif [ "$option" == "" ]; then
49	58		# If fio is installed, use it, else use dd
50		-	echo ""
	59	+	[ "$simple" != true ] && echo ""
51	60		if ! command -v fio &> /dev/null; then
52	61		dd_speedtest
53	62		else
			@@ -56,6 +65,26 @@ function main {
56	65		fi
57	66		}
58	67
	68	+	function print_result {
	69	+	# Print a single fio test result, either as the full "Run status group"
	70	+	# line or, when --simple is given, as a compact aligned "Label value" row.
	71	+	# The simple value is taken from the parenthetical (MB/s) figure that
	72	+	# follows the first bw= value in fio's output.
	73	+	local label="$1"
	74	+	local output="$2"
	75	+	local status
	76	+	status=$(echo " - $output " \| /usr/bin/grep -A1 'Run status group' \| tail -n1)
	77	+	if [ "$simple" == true ]; then
	78	+	local value
	79	+	# Grab the parenthetical (MB/s) figure after the first bw=, then put a
	80	+	# space between the number and the unit, eg "98.4MB/s" -> "98.4 MB/s".
	81	+	value=$(echo "$status" \| /usr/bin/grep -oP 'bw=\S+\s+\(\K[^)]+' \| sed -E 's/([0-9.]+)([A-Za-z])/\1 \2/')
	82	+	printf "%-28s%s\n" "$label" "$value"
	83	+	else
	84	+	echo "$status"
	85	+	fi
	86	+	}
	87	+
59	88		function fio_write_single_random_4k {
60	89		# Single 4k Random Writes
61	90
			@@ -88,8 +117,7 @@ function fio_write_single_random_4k {
88	117		# --iodepth=1 this is how deep we're willing to try to stack commands in the OS's queue. Since we set this to 1, this is effectively pretty much the same thing as the sync IO engine—we're only asking for a single operation at a time, and the OS has to acknowledge receipt of every operation we ask for before we can ask for another. (It does not have to satisfy the request itself before we ask it to do more operations, it just has to acknowledge that we actually asked for it.)
89	118		# --runtime=15 --time_based Run and even if we complete sooner, just start over again and keep going until 60 seconds is up.
90	119		# --end_fsync=1 After all operations have been queued, keep the timer going until the OS reports that the very last one of them has been successfully completed—ie, actually written to disk.
91		-	echo ""
92		-	echo "Single 4K Random Writes (size=1G, time=15sec, jobs=1, iodepth=1)"
	120	+	[ "$simple" != true ] && { echo ""; echo "Single 4K Random Writes (size=1G, time=15sec, jobs=1, iodepth=1)"; }
93	121		x=`sudo fio \
94	122		--name=fio-write-random-4k \
95	123		--directory=$path \
			@@ -101,8 +129,8 @@ function fio_write_single_random_4k {
101	129		--iodepth=1 \
102	130		--time_based --runtime=15 \
103	131		--end_fsync=1`
104		-	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
105		-
	132	+	print_result "Single 4K Random Writes" "$x"
	133	+
106	134		# Cleanup my test files
107	135		rm -rf $path/fio-write-random-4k*
108	136		}
			@@ -143,8 +171,7 @@ function fio_write_parallel_random_64k {
143	171		# physical media stripe at a time—or a garbage collection or other maintenance
144	172		# operation at the SSD firmware level needs to run briefly in the background,
145	173		# slowing things down.
146		-	echo ""
147		-	echo "Parallel 64K Random Writes (size=1G, time=15sec, jobs=16, iodepth=16)"
	174	+	[ "$simple" != true ] && { echo ""; echo "Parallel 64K Random Writes (size=1G, time=15sec, jobs=16, iodepth=16)"; }
148	175		x=`sudo fio \
149	176		--name=fio-write-random-64k \
150	177		--directory=$path \
			@@ -156,7 +183,7 @@ function fio_write_parallel_random_64k {
156	183		--iodepth=16 \
157	184		--time_based --runtime=15 \
158	185		--end_fsync=1`
159		-	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
	186	+	print_result "Parallel 64K Random Writes" "$x"
160	187
161	188		# Cleanup my test files
162	189		rm -rf $path/fio-write-random-64k*
			@@ -201,8 +228,7 @@ function fio_write_single_sequential_1m {
201	228		# here is that you understand both what you want to test, and how to test
202	229		# it accurately.
203	230
204		-	echo ""
205		-	echo "Single 1M Sequential Writes (size=1G, time=15sec, jobs=1, iodepth=1)"
	231	+	[ "$simple" != true ] && { echo ""; echo "Single 1M Sequential Writes (size=1G, time=15sec, jobs=1, iodepth=1)"; }
206	232		x=`sudo fio \
207	233		--name=fio-write-random-1m \
208	234		--directory=$path \
			@@ -214,7 +240,7 @@ function fio_write_single_sequential_1m {
214	240		--iodepth=1 \
215	241		--time_based --runtime=15 \
216	242		--end_fsync=1`
217		-	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
	243	+	print_result "Single 1M Sequential Writes" "$x"
218	244
219	245		# Cleanup my test files
220	246		rm -rf $path/fio-write-random-1m*
			@@ -223,8 +249,7 @@ function fio_write_single_sequential_1m {
223	249		function fio_read_sequential_1m {
224	250		# Sequential Parallel Reads
225	251
226		-	echo ""
227		-	echo "Sequential 4x 1M Reads"
	252	+	[ "$simple" != true ] && { echo ""; echo "Sequential 4x 1M Reads"; }
228	253		x=`sudo fio \
229	254		--name=fio-read-sequential-1m \
230	255		--directory=$path \
			@@ -241,15 +266,14 @@ function fio_read_sequential_1m {
241	266		--group_reporting=1 \
242	267		--iodepth_batch_submit=64 \
243	268		--iodepth_batch_complete_max=64`
244		-	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
	269	+	print_result "Sequential 4x 1M Reads" "$x"
245	270		rm -rf $path/fio-read-sequential-1m*
246	271		}
247	272
248	273		function fio_read_random_4k {
249	274		# Random 4k Reads
250	275
251		-	echo ""
252		-	echo "Random 4k Reads"
	276	+	[ "$simple" != true ] && { echo ""; echo "Random 4k Reads"; }
253	277		x=`sudo fio \
254	278		--name=fio-read-random-4k \
255	279		--directory=$path \
			@@ -266,7 +290,7 @@ function fio_read_random_4k {
266	290		--group_reporting=1 \
267	291		--iodepth_batch_submit=256 \
268	292		--iodepth_batch_complete_max=256`
269		-	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
	293	+	print_result "Random 4k Reads" "$x"
270	294		rm -rf $path/fio-read-random-4k*
271	295		}
272	296
			@@ -331,6 +355,10 @@ function usage {
331	355		echo " This will force DD"
332	356		echo " ./speedtest-hd /mnt/somedisk --dd"
333	357		echo " ./speedtest-hd . --dd"
	358	+	echo ""
	359	+	echo " Add --simple (FIO only) for a compact, aligned summary of MB/s values"
	360	+	echo " ./speedtest-hd . --simple"
	361	+	echo " ./speedtest-hd . --fio --simple"
334	362		exit 0
335	363		}
336	364

mreschke revised this gist 1 month ago. Go to revision

No changes

mreschke revised this gist 1 month ago. Go to revision

1 file changed, 338 insertions

speedtest-hd.sh(file created)

		@@ -0,0 +1,338 @@
1	+	#!/usr/bin/env bash
2	+
3	+	# Robust HD/SDD/NVMe performance CLI utility
4	+	# Utilizing FIO for sequential/random writes/writes
5	+	# Dependencies: fio (apt install fio)
6	+	# See: https://cloud.google.com/compute/docs/disks/benchmarking-pd-performance
7	+	# See: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
8	+	# mReschke 2024-01-18
9	+
10	+	# CLI Parameters
11	+	path="$1"
12	+	option="$2"
13	+
14	+	# Main application flow
15	+	function main {
16	+
17	+	# Show usage if no params
18	+	if [ ! "$path" ]; then
19	+	usage
20	+	fi
21	+
22	+	# Understand . path
23	+	if [ "$path" == '.' ]; then
24	+	path=$(pwd)
25	+	fi
26	+
27	+	# Check if path exists
28	+	if [ ! -e "$path" ]; then
29	+	echo "Path $path does not exist"
30	+	exit 1
31	+	fi
32	+
33	+	# Must type y or n THEN press enter (which I like better)
34	+	echo "NOTICE: 1GB free space on '$path' is required to perform the benchmark."
35	+	echo -n "Are you ready to start a robust IO benchmark against '$path' ?"; read answer
36	+	if [ "$answer" != "${answer#[Yy]}" ]; then
37	+	echo "Great! Starting benchmark now!";
38	+	else
39	+	echo "Ok, cancelled!"
40	+	exit 0
41	+	fi
42	+
43	+	# Use dd of fio based on param or defaults
44	+	if [ "$option" == "--dd" ]; then
45	+	dd_speedtest
46	+	elif [ "$option" == "--fio" ]; then
47	+	fio_speedtest
48	+	elif [ "$option" == "" ]; then
49	+	# If fio is installed, use it, else use dd
50	+	echo ""
51	+	if ! command -v fio &> /dev/null; then
52	+	dd_speedtest
53	+	else
54	+	fio_speedtest
55	+	fi
56	+	fi
57	+	}
58	+
59	+	function fio_write_single_random_4k {
60	+	# Single 4k Random Writes
61	+
62	+	# This is a single process doing random 4K writes. This is where the pain
63	+	# really, really lives; it's basically the worst possible thing you can ask a
64	+	# disk to do. Where this happens most frequently in real life: copying home
65	+	# directories and dotfiles, manipulating email stuff, some database operations,
66	+	# source code trees.
67	+
68	+	# When I ran this test against the high-performance SSDs in my Ubuntu
69	+	# workstation, they pushed 127MiB/sec. The server just beneath it in the rack
70	+	# only managed 33MiB/sec on its "high-performance" 7200RPM rust disks... but
71	+	# even then, the vast majority of that speed is because the data is being
72	+	# written asynchronously, allowing the operating system to batch it up into
73	+	# larger, more efficient write operations.
74	+
75	+	# If we add the argument --fsync=1, forcing the operating system to perform
76	+	# synchronous writes (calling fsync after each block of data is written) the
77	+	# picture gets much more grim: 2.6MiB/sec on the high-performance SSDs but
78	+	# only 184KiB/sec on the "high-performance" rust. The SSDs were about four
79	+	# times faster than the rust when data was written asynchronously but a
80	+	# whopping fourteen times faster when
81	+
82	+	# --name= is a required argument, but it's basically human-friendly fluff—fio will create files based on that name to test with, inside the working directory you're currently in.
83	+	# --ioengine=posixaio sets the mode fio interacts with the filesystem. POSIX is a standard Windows, Macs, Linux, and BSD all understand, so it's great for portability—although inside fio itself, Windows users need to invoke --ioengine=windowsaio, not --ioengine=posixaio, unfortunately. AIO stands for Asynchronous Input Output and means that we can queue up multiple operations to be completed in whatever order the OS decides to complete them. (In this particular example, later arguments effectively nullify this.)
84	+	# --rw=randwrite means exactly what it looks like it means: we're going to do random write operations to our test files in the current working directory. Other options include seqread, seqwrite, randread, and randrw, all of which should hopefully be fairly self-explanatory.
85	+	# --bs=4k blocksize 4K. These are very small individual operations. This is where the pain lives; it's hard on the disk, and it also means a ton of extra overhead in the SATA, USB, SAS, SMB, or whatever other command channel lies between us and the disks, since a separate operation has to be commanded for each 4K of data.
86	+	# --size=1g our test file(s) will be 1GB in size apiece. (We're only creating one, see next argument.)
87	+	# --numjobs=1 we're only creating a single file, and running a single process commanding operations within that file. If we wanted to simulate multiple parallel processes, we'd do, eg, --numjobs=16, which would create 16 separate test files of --size size, and 16 separate processes operating on them at the same time.
88	+	# --iodepth=1 this is how deep we're willing to try to stack commands in the OS's queue. Since we set this to 1, this is effectively pretty much the same thing as the sync IO engine—we're only asking for a single operation at a time, and the OS has to acknowledge receipt of every operation we ask for before we can ask for another. (It does not have to satisfy the request itself before we ask it to do more operations, it just has to acknowledge that we actually asked for it.)
89	+	# --runtime=15 --time_based Run and even if we complete sooner, just start over again and keep going until 60 seconds is up.
90	+	# --end_fsync=1 After all operations have been queued, keep the timer going until the OS reports that the very last one of them has been successfully completed—ie, actually written to disk.
91	+	echo ""
92	+	echo "Single 4K Random Writes (size=1G, time=15sec, jobs=1, iodepth=1)"
93	+	x=`sudo fio \
94	+	--name=fio-write-random-4k \
95	+	--directory=$path \
96	+	--ioengine=posixaio \
97	+	--rw=randwrite \
98	+	--bs=4k \
99	+	--size=1g \
100	+	--numjobs=1 \
101	+	--iodepth=1 \
102	+	--time_based --runtime=15 \
103	+	--end_fsync=1`
104	+	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
105	+
106	+	# Cleanup my test files
107	+	rm -rf $path/fio-write-random-4k*
108	+	}
109	+
110	+	function fio_write_parallel_random_64k {
111	+	# Parallel 64k Random Writes
112	+
113	+	# This time, we're creating 16 separate 64MB files (still totaling 1GB, when
114	+	# all put together) and we're issuing 64KB blocksized random write operations.
115	+	# We're doing it with sixteen separate processes running in parallel, and
116	+	# we're queuing up to 16 simultaneous asynchronous ops before we pause and wait
117	+	# for the OS to start acknowledging their receipt.
118	+
119	+	# This is a pretty decent approximation of a significantly busy system. It's
120	+	# not doing any one particularly nasty thing—like running a database engine or
121	+	# copying tons of dotfiles from a user's home directory—but it is coping with
122	+	# a bunch of applications doing moderately demanding stuff all at once.
123	+
124	+	# This is also a pretty good, slightly pessimistic approximation of a busy,
125	+	# multi-user system like a NAS, which needs to handle multiple 1MB operations
126	+	# simultaneously for different users. If several people or processes are trying
127	+	# to read or write big files (photos, movies, whatever) at once, the OS tries
128	+	# to feed them all data simultaneously. This pretty quickly devolves down to a
129	+	# pattern of multiple random small block access. So in addition to "busy desktop
130	+	# with lots of apps," think "busy fileserver with several people actively using it."
131	+
132	+	# You will see a lot more variation in speed as you watch this operation play
133	+	# out on the console. For example, the 4K single process test we tried first
134	+	# wrote a pretty consistent 11MiB/sec on my MacBook Air's internal drive—but
135	+	# this 16-process job fluctuated between about 10MiB/sec and 300MiB/sec during
136	+	# the run, finishing with an average of 126MiB/sec.
137	+
138	+	# Most of the variation you're seeing here is due to the operating system and
139	+	# SSD firmware sometimes being able to aggregate multiple writes. When it
140	+	# manages to aggregate them helpfully, it can write them in a way that allows
141	+	# parallel writes to all the individual physical media stripes inside the SSD.
142	+	# Sometimes, it still ends up having to give up and write to only a single
143	+	# physical media stripe at a time—or a garbage collection or other maintenance
144	+	# operation at the SSD firmware level needs to run briefly in the background,
145	+	# slowing things down.
146	+	echo ""
147	+	echo "Parallel 64K Random Writes (size=1G, time=15sec, jobs=16, iodepth=16)"
148	+	x=`sudo fio \
149	+	--name=fio-write-random-64k \
150	+	--directory=$path \
151	+	--ioengine=posixaio \
152	+	--rw=randwrite \
153	+	--bs=64k \
154	+	--size=64m \
155	+	--numjobs=16 \
156	+	--iodepth=16 \
157	+	--time_based --runtime=15 \
158	+	--end_fsync=1`
159	+	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
160	+
161	+	# Cleanup my test files
162	+	rm -rf $path/fio-write-random-64k*
163	+	}
164	+
165	+	function fio_write_single_sequential_1m {
166	+	# Single 1M Random Writes
167	+
168	+	# This is pretty close to the best-case scenario for a real-world system
169	+	# doing real-world things. No, it's not quite as fast as a single, truly
170	+	# contiguous write... but the 1MiB blocksize is large enough that it's quite
171	+	# close. Besides, if literally any other disk activity is requested simultaneously
172	+	# with a contiguous write, the "contiguous" write devolves to this level of
173	+	# performance pretty much instantly, so this is a much more realistic test of
174	+	# the upper end of storage performance on a typical system.
175	+
176	+	# You'll see some kooky fluctuations on SSDs when doing this test. This is largely
177	+	# due to the SSD's firmware having better luck or worse luck at any given time,
178	+	# when it's trying to queue operations so that it can write across all physical
179	+	# media stripes cleanly at once. Rust disks will tend to provide a much more
180	+	# consistent, though typically lower, throughput across the run.
181	+
182	+	# You can also see SSD performance fall off a cliff here if you exhaust an
183	+	# onboard write cache—TLC and QLC drives tend to have small write cache areas
184	+	# made of much faster MLC or SLC media. Once those get exhausted, the disk has
185	+	# to drop to writing directly to the much slower TLC/QLC media where the data
186	+	# eventually lands. This is the major difference between, for example, Samsung
187	+	# EVO and Pro SSDs—the EVOs have slow TLC media with a fast MLC cache, where
188	+	# the Pros use the higher-performance, higher-longevity MLC media throughout
189	+	# the entire SSD.
190	+
191	+	# If you have any doubt at all about a TLC or QLC disk's ability to sustain
192	+	# heavy writes, you may want to experimentally extend your time duration here.
193	+	# If you watch the throughput live as the job progresses, you'll see the impact
194	+	# immediately when you run out of cache—what had been a fairly steady,
195	+	# several-hundred-MiB/sec throughput will suddenly plummet to half the speed
196	+	# or less and get considerably less stable as well.
197	+
198	+	# However, you might choose to take the opposite position—you might not
199	+	# expect to do sustained heavy writes very frequently, in which case you
200	+	# actually are more interested in the on-cache behavior. What's important
201	+	# here is that you understand both what you want to test, and how to test
202	+	# it accurately.
203	+
204	+	echo ""
205	+	echo "Single 1M Sequential Writes (size=1G, time=15sec, jobs=1, iodepth=1)"
206	+	x=`sudo fio \
207	+	--name=fio-write-random-1m \
208	+	--directory=$path \
209	+	--ioengine=posixaio \
210	+	--rw=write \
211	+	--bs=1m \
212	+	--size=1g \
213	+	--numjobs=1 \
214	+	--iodepth=1 \
215	+	--time_based --runtime=15 \
216	+	--end_fsync=1`
217	+	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
218	+
219	+	# Cleanup my test files
220	+	rm -rf $path/fio-write-random-1m*
221	+	}
222	+
223	+	function fio_read_sequential_1m {
224	+	# Sequential Parallel Reads
225	+
226	+	echo ""
227	+	echo "Sequential 4x 1M Reads"
228	+	x=`sudo fio \
229	+	--name=fio-read-sequential-1m \
230	+	--directory=$path \
231	+	--ioengine=posixaio \
232	+	--bs=1M \
233	+	--numjobs=4 \
234	+	--size=256M \
235	+	--time_based --runtime=30s \
236	+	--ramp_time=2s \
237	+	--direct=1 \
238	+	--verify=0 \
239	+	--iodepth=64 \
240	+	--rw=read \
241	+	--group_reporting=1 \
242	+	--iodepth_batch_submit=64 \
243	+	--iodepth_batch_complete_max=64`
244	+	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
245	+	rm -rf $path/fio-read-sequential-1m*
246	+	}
247	+
248	+	function fio_read_random_4k {
249	+	# Random 4k Reads
250	+
251	+	echo ""
252	+	echo "Random 4k Reads"
253	+	x=`sudo fio \
254	+	--name=fio-read-random-4k \
255	+	--directory=$path \
256	+	--ioengine=posixaio \
257	+	--rw=randread \
258	+	--bs=4k \
259	+	--size=1g \
260	+	--time_based --runtime=30s \
261	+	--ramp_time=2s \
262	+	--direct=1 \
263	+	--verify=0 \
264	+	--iodepth=256 \
265	+	--rw=read \
266	+	--group_reporting=1 \
267	+	--iodepth_batch_submit=256 \
268	+	--iodepth_batch_complete_max=256`
269	+	echo " - $x " \| /usr/bin/grep -A1 'Run status group' \| tail -n1
270	+	rm -rf $path/fio-read-random-4k*
271	+	}
272	+
273	+	function fio_speedtest {
274	+	# Write tests
275	+	fio_write_single_random_4k
276	+	fio_write_parallel_random_64k
277	+	fio_write_single_sequential_1m
278	+
279	+	# Read Tests
280	+	fio_read_sequential_1m
281	+	fio_read_random_4k
282	+	}
283	+
284	+	function dd_speedtest {
285	+	# Basic HD speed test using DD
286	+	# mReschke 2017-07-11
287	+
288	+	file=$path/bigfile
289	+	size=1024
290	+
291	+	echo "Running dd based HD/SSD/NVMe Benchmarks"
292	+	echo "---------------------------------------"
293	+
294	+	printf "Cached write speed...\n"
295	+	dd if=/dev/zero of=$file bs=1M count=$size
296	+
297	+	printf "\nUncached write speed...\n"
298	+	dd if=/dev/zero of=$file bs=1M count=$size conv=fdatasync,notrunc
299	+
300	+	printf "\nUncached read speed...\n"
301	+	echo 3 \| sudo tee /proc/sys/vm/drop_caches > /dev/null
302	+	dd if=$file of=/dev/null bs=1M count=$size
303	+
304	+	printf "\nCached read speed...\n"
305	+	dd if=$file of=/dev/null bs=1M count=$size
306	+
307	+	rm $file
308	+	printf "\nDone\n"
309	+	}
310	+
311	+	# Show help and usage information
312	+	function usage {
313	+	echo "Robust Flexible Input/Output HD Speedtest"
314	+	echo " If FIO is installed, we use FIO for more detailed performance analysis."
315	+	echo " If FIO is not installed, we use basic DD analysis."
316	+	echo " You should apt install fio (pacman -S fio) for detailed analysis."
317	+	echo "mReschke 2024-01-18"
318	+	echo ""
319	+	echo "NOTICE, this creates a 1GB file on the desired destination disk."
320	+	echo "Please ensure you have write access with 1GB free space on destination."
321	+	echo ""
322	+	echo "Usage:"
323	+	echo " This will use FIO if installed, else DD"
324	+	echo " ./speedtest-hd /mnt/somedisk"
325	+	echo " ./speedtest-hd ."
326	+	echo ""
327	+	echo " This will force FIO"
328	+	echo " ./speedtest-hd /mnt/somedisk --fio"
329	+	echo " ./speedtest-hd . --fio"
330	+	echo ""
331	+	echo " This will force DD"
332	+	echo " ./speedtest-hd /mnt/somedisk --dd"
333	+	echo " ./speedtest-hd . --dd"
334	+	exit 0
335	+	}
336	+
337	+	# Go
338	+	main

Newer Older