附录：随书实验完整代码
以下文件在构建时从随书实验与测试目录读取，与下载区实验包同源。它们是本书教学实现；实际验证范围以实验章节和验证记录为准。
lab/README.md
# MySQL 恢复实验
需要 linux/amd64、Python 3.12+、Docker Engine、可下载固定官方 MySQL 镜像，约 2 GB 可用内存与 3 GB 磁盘。工具RPM固定为x86_64，ARM原生环境未验证；本书正式证据来自 Ubuntu 24.04 GitHub Actions，不把不同平台结果混为一谈。
```sh
python3 lab/run.py --evidence evidence/runtime.json
```
程序创建带随机前缀的私有 bridge network 和四个实例，端口不发布到宿主机，使用教学专用口令，完成后只删除本次记录的容器（含匿名卷）及网络。不会执行 Docker prune。遇异常不跳过测试，返回非零，证据 status 为 FAIL；进程被强杀则需按本次随机前缀手工清理其资源，不能清理其他项目。
10个场景必须全部 PASS：GTID复制建立、写入期间的一致快照、干净基线恢复、事务边界PITR、缺失日志不可达到目标、误删同样复制、应用暂停再追平、未隔离拒绝提升、隔离后手工提升并观测已确认事务丢失、返回旧主断网且只读。
业务写入由ledger_app执行，仅具有ledger.*的SELECT/INSERT/UPDATE/DELETE，没有SUPER或CONNECTION_ADMIN。提升显式关闭super_read_only和read_only并检查0/0，再用该普通业务账号提交与查询。角色管理、备份和核对使用root。
事务业务不变量：账户余额=100000+分录delta之和；订单与分录逐行一一对应；分录金额等于订单金额相反数。恢复目标还比较全部有序订单、余额和分录总和的SHA-256，不能仅用行数或线程状态判定成功。
RTO仅计本次docker stop发起到新主成功提交并查询确认的手工流程，排除故障发现、路由、DNS与客户端重试。RPO报告实际缺失事务ID和数量；观察窗口是停止复制后相邻客户端观测间隔，不是线上RPO保证。
使用的逻辑备份参数为 `--single-transaction --source-data=2 --set-gtid-purged=OFF --databases ledger`，实验只针对InnoDB事务表且没有并发DDL。恢复目的不是直接加入原复制拓扑，不能照搬此GTID选项到生产副本初始化。回放前FLUSH封口并复制真实日志到任务私有归档，记录字节与hash，停止源实例后用无网络工具容器只读挂载归档。缺失日志负例临时移走实际必需文件并验证基线未变化。回放使用同一实际binlog文件中的开始位置与误删事务之前的提交结束位置，生产跨文件日志需逐文件验证连续性。
每次运行均重新拉起干净实例，固定镜像digest，报告实际VERSION和变量。不包含自动选主、代理改路由、物理备份、跨地域容灾、故障磁盘持久性或生产密钥管理。lab/run.py
#!/usr/bin/env python3
"""Real, isolated MySQL recovery exercises. Requires Docker; never skips tests."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import secrets
import subprocess
import tempfile
import time
import urllib.request
IMAGE = 'mysql@sha256:0f51e63e537a1f176423fa03ef559aabe6fa4372a53160a44bc7e75da7d15364'
PASSWORD = 'recovery-lab-only'
CLIENT_URL = 'https://repo.mysql.com/yum/mysql-8.4-community/el/9/x86_64/mysql-community-client-8.4.11-1.el9.x86_64.rpm'
CLIENT_SHA256 = '2f1e8317c557363dd1554b29a9f6d69403802dc623fe72d9995f20f542771bb1'
def call(args, data=None, check=True, timeout=180):
p = subprocess.run(args, input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
if check and p.returncode:
raise RuntimeError(f'{args[0:3]} failed ({p.returncode}): {p.stderr.decode(errors="replace")[-3000:]}')
return p
class Lab:
def __init__(self, output):
self.prefix = 'mysql-recovery-' + secrets.token_hex(6)
self.network = self.prefix + '-net'
self.names = []
self.tmp = tempfile.TemporaryDirectory(prefix=self.prefix)
self.report = {'status': 'RUNNING', 'image': IMAGE, 'scenarios': [], 'scope':
'Isolated Linux Docker bridge; teaching data; manual failover; no production routing, automatic HA or cross-region claim.'}
self.report['source_sha'] = os.environ.get('GITHUB_SHA', 'local-unattributed-run')
self.output = Path(output)
self.output.parent.mkdir(parents=True, exist_ok=True)
def sql(self, name, query, check=True):
return self.sql_result(name, query, check=check).stdout.decode().strip()
def sql_result(self, name, query, check=True, user='root'):
return call(['docker', 'exec', '-e', 'MYSQL_PWD=' + PASSWORD, self.prefix + '-' + name,
'mysql', '-u' + user, '--batch', '--raw', '--skip-column-names', '-e', query],
check=check)
def business(self, name, query):
return self.sql_result(name, query, user='ledger_app').stdout.decode().strip()
def readonly_rejected(self, name, query):
result = self.sql_result(name, query, check=False, user='ledger_app')
error = result.stderr.decode(errors='replace')
assert result.returncode != 0 and 'ERROR 1290' in error and 'read-only' in error, error
return {'user': 'ledger_app', 'exit': result.returncode, 'mysql_error': error.strip()}
def prepare_tools(self):
context = Path(self.tmp.name) / 'tools'
context.mkdir()
rpm = context / 'client.rpm'
with urllib.request.urlopen(CLIENT_URL, timeout=90) as response:
rpm.write_bytes(response.read())
assert hashlib.sha256(rpm.read_bytes()).hexdigest() == CLIENT_SHA256
dockerfile = Path(__file__).with_name('tools.Dockerfile')
(context / 'Dockerfile').write_bytes(dockerfile.read_bytes())
call(['docker', 'build', '-t', self.prefix + '-tools', str(context)], timeout=240)
version = call(['docker', 'run', '--rm', '--network', 'none', self.prefix + '-tools',
'mysqlbinlog', '--version']).stdout.decode().strip()
assert '8.4.11' in version
self.report['binlog_tool'] = {'url': CLIENT_URL, 'rpm_sha256': CLIENT_SHA256, 'version': version,
'architecture': 'linux/amd64'}
def archived_binlog(self, directory, filename, startpos, stoppos, check=True):
return call(['docker', 'run', '--rm', '--network', 'none', '--mount',
f'type=bind,source={directory},target=/archive,readonly', self.prefix + '-tools',
'mysqlbinlog', '--start-position=' + str(startpos), '--stop-position=' + str(stoppos),
'/archive/' + filename], check=check)
def stop_source(self, stage):
call(['docker', 'stop', '--time', '10', self.prefix + '-source'])
state = json.loads(call(['docker', 'inspect', '--format', '{{json .State}}',
self.prefix + '-source']).stdout)
assert state['Running'] is False and state['OOMKilled'] is False
self.report.setdefault('stop_observations', []).append({'stage': stage,
'running': state['Running'], 'exit_code': state['ExitCode'],
'finished_at': state['FinishedAt'], 'grace_seconds': 10})
def feed(self, name, data):
return call(['docker', 'exec', '-i', '-e', 'MYSQL_PWD=' + PASSWORD, self.prefix + '-' + name,
'mysql', '-uroot', '--binary-mode'], data=data)
def check(self, name, **values):
self.report['scenarios'].append({'name': name, 'status': 'PASS', **values})
print('PASS', name, flush=True)
def start(self, name, server_id):
full = self.prefix + '-' + name
call(['docker', 'run', '-d', '--name', full, '--network', self.network,
'--network-alias', name, '--label', 'aias.mysql-recovery=' + self.prefix,
'-e', 'MYSQL_ROOT_PASSWORD=' + PASSWORD, '-e', 'MYSQL_ROOT_HOST=%', IMAGE,
'--server-id=' + str(server_id), '--log-bin=mysql-bin', '--gtid-mode=ON',
'--enforce-gtid-consistency=ON', '--binlog-row-image=FULL',
'--innodb-buffer-pool-size=64M', '--performance-schema=OFF'])
self.names.append(name)
for _ in range(100):
try:
if self.sql(name, 'SELECT 1') == '1':
return
except RuntimeError:
pass
time.sleep(1)
raise RuntimeError('MySQL readiness timed out: ' + name)
def state(self, name):
rows = self.sql(name, 'SELECT id,amount,tag FROM ledger.orders ORDER BY id;').splitlines()
account = int(self.sql(name, 'SELECT balance FROM ledger.accounts WHERE id=1'))
delta = int(self.sql(name, 'SELECT COALESCE(SUM(delta),0) FROM ledger.entries'))
entry_count = int(self.sql(name, 'SELECT COUNT(*) FROM ledger.entries'))
orphan = int(self.sql(name, 'SELECT COUNT(*) FROM ledger.entries e LEFT JOIN ledger.orders o ON e.order_id=o.id WHERE o.id IS NULL OR e.delta<>-o.amount'))
assert account == 100000 + delta, (name, account, delta)
assert entry_count == len(rows) and orphan == 0, (name, entry_count, rows, orphan)
canonical = '\n'.join(rows) + f'\nbalance={account}\ndelta={delta}'
return {'orders': len(rows), 'balance': account, 'entry_sum': delta,
'sha256': hashlib.sha256(canonical.encode()).hexdigest(), 'rows': rows}
@staticmethod
def order(ident, tag, amount=1):
assert re.fullmatch('[a-z0-9-]+', tag)
return (f"START TRANSACTION; INSERT INTO ledger.orders VALUES({ident},{amount},'{tag}'); "
f"INSERT INTO ledger.entries VALUES({ident},{ident},-{amount}); "
f"UPDATE ledger.accounts SET balance=balance-{amount} WHERE id=1; COMMIT;")
def dump(self):
p = call(['docker', 'exec', '-e', 'MYSQL_PWD=' + PASSWORD, self.prefix + '-source',
'mysqldump', '-uroot', '--single-transaction', '--source-data=2',
'--set-gtid-purged=OFF', '--routines', '--triggers', '--events', '--hex-blob',
'--databases', 'ledger'])
return p.stdout
def wait_replica(self):
gtid = self.sql('source', 'SELECT @@GLOBAL.gtid_executed')
result = self.sql('replica', f"SELECT WAIT_FOR_EXECUTED_GTID_SET('{gtid}',30)")
assert result == '0', self.sql('replica', 'SHOW REPLICA STATUS\\G')
assert self.state('source') == self.state('replica')
return gtid
def promote(self):
running = call(['docker', 'inspect', '--format', '{{.State.Running}}', self.prefix + '-source']).stdout.strip()
if running != b'false':
raise RuntimeError('FENCE_REQUIRED: old primary remains running; promotion refused')
self.sql('replica', 'STOP REPLICA; RESET REPLICA ALL; SET GLOBAL super_read_only=OFF; SET GLOBAL read_only=OFF;')
assert self.sql('replica', 'SELECT @@GLOBAL.read_only,@@GLOBAL.super_read_only') == '0\t0'
def run(self):
self.prepare_tools()
call(['docker', 'network', 'create', '--label', 'aias.mysql-recovery=' + self.prefix, self.network])
for i, name in enumerate(['source', 'replica', 'restore', 'pitr'], 11):
self.start(name, i)
self.report['mysql_version'] = self.sql('source', 'SELECT VERSION()')
self.report['server_variables'] = self.sql('source', 'SELECT @@gtid_mode,@@enforce_gtid_consistency,@@binlog_format,@@binlog_row_image,@@sync_binlog,@@innodb_flush_log_at_trx_commit')
self.report['image_inspect'] = json.loads(call(['docker', 'image', 'inspect', IMAGE, '--format', '{{json .RepoDigests}}']).stdout)
self.sql('source', "CREATE USER 'repl'@'%' IDENTIFIED BY 'replica-lab-only'; GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%';")
self.sql('source', f"CREATE USER 'ledger_app'@'%' IDENTIFIED BY '{PASSWORD}'; GRANT SELECT,INSERT,UPDATE,DELETE ON ledger.* TO 'ledger_app'@'%';")
grants = self.sql('source', "SHOW GRANTS FOR 'ledger_app'@'%'")
assert 'SUPER' not in grants and 'CONNECTION_ADMIN' not in grants and 'ALL PRIVILEGES' not in grants
self.report['business_identity'] = {'user': 'ledger_app', 'grants': grants,
'scope': 'ledger.* DML; no administrative privileges'}
self.sql('replica', "CHANGE REPLICATION SOURCE TO SOURCE_HOST='source',SOURCE_USER='repl',SOURCE_PASSWORD='replica-lab-only',SOURCE_AUTO_POSITION=1,SOURCE_CONNECT_RETRY=1,GET_SOURCE_PUBLIC_KEY=1; START REPLICA; SET GLOBAL super_read_only=ON;")
self.sql('source', '''CREATE DATABASE ledger; USE ledger;
CREATE TABLE accounts(id INT PRIMARY KEY,balance BIGINT NOT NULL) ENGINE=InnoDB;
CREATE TABLE orders(id INT PRIMARY KEY,amount INT NOT NULL,tag VARCHAR(40) NOT NULL) ENGINE=InnoDB;
CREATE TABLE entries(id INT PRIMARY KEY,order_id INT UNIQUE NOT NULL,delta INT NOT NULL) ENGINE=InnoDB;
CREATE TABLE padding(id INT PRIMARY KEY,payload LONGTEXT NOT NULL) ENGINE=InnoDB;
INSERT INTO accounts VALUES(1,100000);''')
self.business('source', self.order(1, 'opening-order', 100))
# Slow enough for real dump/commit overlap, but padding is never a business invariant.
self.sql('source', "SET SESSION cte_max_recursion_depth=3000; INSERT INTO ledger.padding WITH RECURSIVE seq AS (SELECT 1 n UNION ALL SELECT n+1 FROM seq WHERE n<2000) SELECT n,REPEAT('x',8192) FROM seq;")
self.wait_replica()
self.check('replication-bootstrap', source_equals_replica=True)
writer_sql = ''.join(self.order(i, 'backup-concurrent') + ' DO SLEEP(0.08);' for i in range(2, 122))
writer = subprocess.Popen(['docker', 'exec', '-i', '-e', 'MYSQL_PWD=' + PASSWORD,
self.prefix + '-source', 'mysql', '-uledger_app'], stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
writer.stdin.write(writer_sql.encode()); writer.stdin.close(); writer.stdin = None
deadline = time.monotonic() + 15
while int(self.sql('source', 'SELECT COUNT(*) FROM ledger.orders')) < 2:
assert time.monotonic() < deadline
time.sleep(.03)
before = int(self.sql('source', 'SELECT COUNT(*) FROM ledger.orders'))
dump_started = time.monotonic()
snapshot = self.dump()
dump_seconds = time.monotonic() - dump_started
after = int(self.sql('source', 'SELECT COUNT(*) FROM ledger.orders'))
writer_out, writer_err = writer.communicate(timeout=40)
assert writer.returncode == 0, writer_err.decode()
source_final = self.state('source')
assert after > before, ('No observed commits during actual dump', before, after)
self.feed('restore', snapshot)
restored = self.state('restore')
assert before <= restored['orders'] <= after < source_final['orders'], (before, restored, after)
self.check('consistent-snapshot-under-writes', source_count_before=before,
source_count_after=after, restored=restored, final_source_orders=source_final['orders'],
dump_seconds=round(dump_seconds, 4), dump_sha256=hashlib.sha256(snapshot).hexdigest())
# A second quiet baseline makes the exact recovery target independently unambiguous.
self.sql('source', 'FLUSH BINARY LOGS')
baseline = self.dump()
pos = re.search(rb"SOURCE_LOG_FILE='([^']+)', SOURCE_LOG_POS=(\d+)", baseline)
assert pos, 'No source coordinates in dump'
filename, startpos = pos[1].decode(), int(pos[2])
self.feed('pitr', baseline)
assert self.state('pitr') == source_final
self.check('clean-baseline-restore', baseline=source_final, binlog=filename, start_position=startpos)
self.business('source', self.order(2001, 'keep-before-delete', 17))
self.business('source', self.order(2002, 'keep-second', 23))
target = self.state('source')
status = self.sql('source', 'SHOW BINARY LOG STATUS').split('\t')
assert status[0] == filename
stoppos = int(status[1])
target_gtid = self.sql('source', 'SELECT @@GLOBAL.gtid_executed')
# This unwanted transaction is valid SQL and replicates: replication is not a backup.
self.business('source', 'START TRANSACTION; DELETE FROM ledger.entries; DELETE FROM ledger.orders; UPDATE ledger.accounts SET balance=100000 WHERE id=1; COMMIT;')
deleted = self.state('source')
assert deleted['orders'] == 0
self.business('source', self.order(3001, 'after-delete', 7))
self.sql('source', 'FLUSH BINARY LOGS')
archive = Path(self.tmp.name) / 'archive'
archive.mkdir()
archived_file = archive / filename
call(['docker', 'cp', self.prefix + '-source:/var/lib/mysql/' + filename, str(archived_file)])
self.report['binlog_archive'] = {'file': filename, 'bytes': archived_file.stat().st_size,
'sha256': hashlib.sha256(archived_file.read_bytes()).hexdigest(),
'sealed_by_flush': True, 'replay_mount_readonly': True, 'replay_network': 'none'}
self.stop_source('offline-archive-replay')
# Remove the actual required archive: the failure must not mutate the clean baseline.
held = archived_file.with_suffix('.held')
archived_file.rename(held)
missing = self.archived_binlog(archive, filename, startpos, stoppos, check=False)
assert missing.returncode != 0
assert self.state('pitr') == source_final and source_final['sha256'] != target['sha256']
self.check('missing-log-cannot-reach-target', mysqlbinlog_exit=missing.returncode,
required_file=filename, baseline_unchanged=True, snapshot_differs_from_target=True)
held.rename(archived_file)
replay = self.archived_binlog(archive, filename, startpos, stoppos).stdout
self.feed('pitr', replay)
assert self.state('pitr') == target
assert not self.sql('pitr', 'SELECT id FROM ledger.orders WHERE id=3001')
self.check('transaction-boundary-pitr', start_position=startpos, stop_position=stoppos,
binlog=filename, expected=target, actual=self.state('pitr'), excluded=['delete-transaction','after-delete'],
target_source_gtid=target_gtid, replay_sha256=hashlib.sha256(replay).hexdigest(),
source_running_during_replay=False, archive=self.report['binlog_archive'])
call(['docker', 'start', self.prefix + '-source'])
for _ in range(60):
try:
if self.sql('source', 'SELECT 1') == '1':
break
except RuntimeError:
time.sleep(1)
else:
raise RuntimeError('Source did not restart after offline archive replay')
# Restart receiver explicitly; do not let its default reconnect backoff
# become an undocumented 60-second wait in a 30-second assertion.
self.sql('replica', 'STOP REPLICA IO_THREAD; START REPLICA IO_THREAD;')
self.wait_replica()
assert self.state('replica')['orders'] == 1 # it received the accidental deletion too
self.check('accidental-delete-replicates', replica_orders=1, recovered_pitr_orders=target['orders'])
self.sql('replica', 'STOP REPLICA SQL_THREAD')
applied = self.state('replica')
self.business('source', self.order(4001, 'delayed-apply', 11))
wanted = self.sql('source', 'SELECT @@GLOBAL.gtid_executed')
assert self.state('replica') == applied
assert self.sql('replica', f"SELECT GTID_SUBSET('{wanted}',@@GLOBAL.gtid_executed)") == '0'
self.sql('replica', 'START REPLICA SQL_THREAD')
self.wait_replica()
self.check('pause-and-catch-up', during_pause=applied, after_resume=self.state('replica'))
self.sql('replica', 'STOP REPLICA')
failover_prefix = self.state('replica')
prefix_time = time.monotonic()
self.business('source', self.order(5001, 'acknowledged-but-not-replicated', 13))
source_ack_time = time.monotonic()
source_before_failure = self.state('source')
assert source_before_failure['orders'] == failover_prefix['orders'] + 1
assert not self.sql('replica', 'SELECT id FROM ledger.orders WHERE id=5001')
try:
self.promote()
except RuntimeError as error:
assert 'FENCE_REQUIRED' in str(error)
else:
raise AssertionError('Unsafe promotion was accepted')
denied = self.readonly_rejected('replica', self.order(6000, 'must-reject'))
assert self.state('replica') == failover_prefix
assert self.sql('replica', 'SELECT @@GLOBAL.super_read_only') == '1'
self.check('unfenced-promotion-refused', old_primary_running=True, replica_remains_read_only=True,
actual_write_rejection=denied)
failure_start = time.monotonic()
self.stop_source('failover-fence')
self.promote()
self.business('replica', self.order(6001, 'new-primary-only', 19))
promoted = self.state('replica')
assert not self.sql('replica', 'SELECT id FROM ledger.orders WHERE id=5001')
assert self.business('replica', 'SELECT tag FROM ledger.orders WHERE id=6001') == 'new-primary-only'
assert promoted['orders'] == failover_prefix['orders'] + 1
expected_rows = failover_prefix['rows'] + ['6001\t19\tnew-primary-only']
assert promoted['rows'] == expected_rows
assert promoted['balance'] == failover_prefix['balance'] - 19
new_primary_ready = time.monotonic()
self.check('fenced-manual-failover-with-observed-loss', source_before_failure=source_before_failure,
surviving_prefix=failover_prefix, promoted=promoted, lost_acknowledged_order_ids=[5001],
rpo_lost_transactions=1, observed_window_seconds=round(source_ack_time-prefix_time, 4),
manual_rto_seconds=round(new_primary_ready-failure_start, 4),
business_user='ledger_app', read_only=self.sql('replica', 'SELECT @@GLOBAL.read_only'),
super_read_only=self.sql('replica', 'SELECT @@GLOBAL.super_read_only'),
rto_boundary='docker-stop invocation to verified new-primary business commit; excludes detection/routing/client retry')
# A returning old primary stays disconnected; fence its writes before any possible rejoin.
call(['docker', 'network', 'disconnect', self.network, self.prefix + '-source'])
call(['docker', 'start', self.prefix + '-source'])
for _ in range(60):
try:
self.sql('source', 'SET GLOBAL super_read_only=ON')
break
except RuntimeError:
time.sleep(1)
else:
raise RuntimeError('Returning source did not start')
denied = self.readonly_rejected('source', self.order(7001, 'old-primary-reject'))
assert not self.sql('source', 'SELECT id FROM ledger.orders WHERE id=7001')
assert not self.sql('source', 'SELECT id FROM ledger.orders WHERE id=6001')
networks = json.loads(call(['docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', self.prefix + '-source']).stdout)
assert not networks
self.check('returning-old-primary-quarantined', networks=networks,
super_read_only=self.sql('source', 'SELECT @@GLOBAL.super_read_only'),
divergent_old_state=True, automatically_rejoined=False, actual_write_rejection=denied)
self.report['status'] = 'PASS'
self.report['scenario_count'] = len(self.report['scenarios'])
assert self.report['scenario_count'] == 10
def close(self):
self.output.write_text(json.dumps(self.report, ensure_ascii=False, indent=2) + '\n')
for name in self.names:
call(['docker', 'rm', '-f', '-v', self.prefix + '-' + name], check=False)
call(['docker', 'network', 'rm', self.network], check=False)
call(['docker', 'image', 'rm', self.prefix + '-tools'], check=False)
self.tmp.cleanup()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--evidence', default='evidence/runtime.json')
args = parser.parse_args()
lab = Lab(args.evidence)
try:
lab.run()
except BaseException as error:
lab.report['status'] = 'FAIL'
lab.report['error'] = str(error)
for name in lab.names:
logs = call(['docker', 'logs', '--tail', '40', lab.prefix + '-' + name], check=False)
print(name, logs.stderr.decode(errors='replace')[-5000:])
raise
finally:
lab.close()
if __name__ == '__main__':
main()lab/tools.Dockerfile
FROM mysql@sha256:0f51e63e537a1f176423fa03ef559aabe6fa4372a53160a44bc7e75da7d15364
# Only extraction uses cpio. The actual mysqlbinlog comes from an exact Oracle
# client RPM whose SHA-256 is checked by run.py before this build.
COPY client.rpm /tmp/client.rpm
RUN microdnf install -y cpio && \
cd /tmp && rpm2cpio client.rpm | cpio -id ./usr/bin/mysqlbinlog && \
cp /tmp/usr/bin/mysqlbinlog /usr/local/bin/mysqlbinlog && \
rm -rf /tmp/client.rpm /tmp/usr && microdnf clean all