V0.3.0版本优化传输速度和提示
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib, os, tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
SIZE = 64 * 1024 * 1024 + 12345
|
||||
CHUNK = 8 * 1024 * 1024
|
||||
WORKERS = 8
|
||||
IO = 1024 * 1024
|
||||
|
||||
|
||||
def sha(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb", buffering=0) as f:
|
||||
while True:
|
||||
b = f.read(4 * 1024 * 1024)
|
||||
if not b: break
|
||||
h.update(b)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def main():
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
src = os.path.join(d, "source.bin")
|
||||
dst = os.path.join(d, "received.qftpart")
|
||||
# deterministic 1 MiB pattern, streamed to avoid holding the file in RAM
|
||||
pattern = bytes((i * 131 + 17) & 0xFF for i in range(IO))
|
||||
remaining = SIZE
|
||||
with open(src, "wb", buffering=0) as f:
|
||||
while remaining:
|
||||
n = min(remaining, len(pattern))
|
||||
f.write(pattern[:n]); remaining -= n
|
||||
with open(dst, "wb") as f: f.truncate(SIZE)
|
||||
|
||||
src_fd = os.open(src, os.O_RDONLY)
|
||||
dst_fd = os.open(dst, os.O_RDWR)
|
||||
try:
|
||||
total = (SIZE + CHUNK - 1) // CHUNK
|
||||
def worker(index):
|
||||
offset = index * CHUNK
|
||||
remaining = min(CHUNK, SIZE - offset)
|
||||
pos = 0
|
||||
while remaining:
|
||||
n = min(IO, remaining)
|
||||
block = os.pread(src_fd, n, offset + pos)
|
||||
assert len(block) == n
|
||||
wrote = os.pwrite(dst_fd, block, offset + pos)
|
||||
assert wrote == n
|
||||
remaining -= n; pos += n
|
||||
return index
|
||||
with ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
||||
completed = sorted(pool.map(worker, reversed(range(total))))
|
||||
assert completed == list(range(total))
|
||||
finally:
|
||||
os.close(src_fd); os.close(dst_fd)
|
||||
|
||||
assert sha(src) == sha(dst)
|
||||
print(f"PASS parallel upload: {SIZE} bytes, {CHUNK//(1024*1024)} MiB chunks, {WORKERS} workers, 1 MiB worker I/O, SHA-256 identical")
|
||||
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
Reference in New Issue
Block a user