61 lines
2.0 KiB
Python
61 lines
2.0 KiB
Python
#!/usr/bin/env python3
|
|
import hashlib, os, tempfile
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
SIZE = 64 * 1024 * 1024 + 12345
|
|
CHUNK = 8 * 1024 * 1024
|
|
WORKERS = 8
|
|
IO = 1024 * 1024
|
|
|
|
|
|
def sha(path):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb", buffering=0) as f:
|
|
while True:
|
|
b = f.read(4 * 1024 * 1024)
|
|
if not b: break
|
|
h.update(b)
|
|
return h.digest()
|
|
|
|
|
|
def main():
|
|
with tempfile.TemporaryDirectory() as d:
|
|
src = os.path.join(d, "source.bin")
|
|
dst = os.path.join(d, "received.qftpart")
|
|
# deterministic 1 MiB pattern, streamed to avoid holding the file in RAM
|
|
pattern = bytes((i * 131 + 17) & 0xFF for i in range(IO))
|
|
remaining = SIZE
|
|
with open(src, "wb", buffering=0) as f:
|
|
while remaining:
|
|
n = min(remaining, len(pattern))
|
|
f.write(pattern[:n]); remaining -= n
|
|
with open(dst, "wb") as f: f.truncate(SIZE)
|
|
|
|
src_fd = os.open(src, os.O_RDONLY)
|
|
dst_fd = os.open(dst, os.O_RDWR)
|
|
try:
|
|
total = (SIZE + CHUNK - 1) // CHUNK
|
|
def worker(index):
|
|
offset = index * CHUNK
|
|
remaining = min(CHUNK, SIZE - offset)
|
|
pos = 0
|
|
while remaining:
|
|
n = min(IO, remaining)
|
|
block = os.pread(src_fd, n, offset + pos)
|
|
assert len(block) == n
|
|
wrote = os.pwrite(dst_fd, block, offset + pos)
|
|
assert wrote == n
|
|
remaining -= n; pos += n
|
|
return index
|
|
with ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
|
completed = sorted(pool.map(worker, reversed(range(total))))
|
|
assert completed == list(range(total))
|
|
finally:
|
|
os.close(src_fd); os.close(dst_fd)
|
|
|
|
assert sha(src) == sha(dst)
|
|
print(f"PASS parallel upload: {SIZE} bytes, {CHUNK//(1024*1024)} MiB chunks, {WORKERS} workers, 1 MiB worker I/O, SHA-256 identical")
|
|
|
|
|
|
if __name__ == "__main__": main()
|