优化大文件稳定并发传输与打包流程
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded Range + retry + out-of-order positioned-write download self-test."""
|
||||
import hashlib
|
||||
import http.client
|
||||
import random
|
||||
import tempfile
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
SIZE = 64 * 1024 * 1024 + 12345
|
||||
CHUNK = 16 * 1024 * 1024
|
||||
WORKERS = 8
|
||||
TOKEN = "parallel_download_test"
|
||||
|
||||
|
||||
def digest(path):
|
||||
value = hashlib.sha256()
|
||||
with open(path, "rb", buffering=0) as stream:
|
||||
while block := stream.read(4 * 1024 * 1024):
|
||||
value.update(block)
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
assert (1024 * 1024 * 1024 + CHUNK - 1) // CHUNK == 64
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
source = root / "source.bin"
|
||||
target = root / "target.qftpart"
|
||||
pattern = bytes((index * 73 + 19) & 0xFF for index in range(1024 * 1024))
|
||||
with source.open("wb", buffering=0) as stream:
|
||||
remaining = SIZE
|
||||
while remaining:
|
||||
block = pattern[:min(len(pattern), remaining)]
|
||||
stream.write(block)
|
||||
remaining -= len(block)
|
||||
with target.open("wb") as stream:
|
||||
stream.truncate(SIZE)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def do_GET(self):
|
||||
if self.path != "/qft/download" or self.headers.get("X-QFT-Token") != TOKEN:
|
||||
self.send_error(403)
|
||||
return
|
||||
value = self.headers.get("Range", "")
|
||||
assert value.startswith("bytes=") and "-" in value
|
||||
start_text, end_text = value[6:].split("-", 1)
|
||||
start, end = int(start_text), int(end_text)
|
||||
assert 0 <= start <= end < SIZE
|
||||
length = end - start + 1
|
||||
self.send_response(206)
|
||||
self.send_header("Content-Length", str(length))
|
||||
self.send_header("Content-Range", f"bytes {start}-{end}/{SIZE}")
|
||||
self.end_headers()
|
||||
with source.open("rb", buffering=0) as stream:
|
||||
stream.seek(start)
|
||||
remaining = length
|
||||
while remaining:
|
||||
block = stream.read(min(1024 * 1024, remaining))
|
||||
assert block
|
||||
self.wfile.write(block)
|
||||
remaining -= len(block)
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
host, port = server.server_address
|
||||
indices = list(range((SIZE + CHUNK - 1) // CHUNK))
|
||||
random.Random(42).shuffle(indices)
|
||||
|
||||
def receive(index):
|
||||
start = index * CHUNK
|
||||
end = min(SIZE, start + CHUNK) - 1
|
||||
connection = http.client.HTTPConnection(host, port, timeout=10)
|
||||
connection.request("GET", "/qft/download", headers={
|
||||
"X-QFT-Token": TOKEN,
|
||||
"Range": f"bytes={start}-{end}",
|
||||
})
|
||||
response = connection.getresponse()
|
||||
assert response.status == 206
|
||||
assert response.getheader("Content-Range") == f"bytes {start}-{end}/{SIZE}"
|
||||
offset = start
|
||||
with target.open("r+b", buffering=0) as output:
|
||||
output.seek(start)
|
||||
while block := response.read(1024 * 1024):
|
||||
assert output.write(block) == len(block)
|
||||
offset += len(block)
|
||||
connection.close()
|
||||
assert offset == end + 1
|
||||
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
||||
list(pool.map(receive, indices))
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
assert digest(source) == digest(target)
|
||||
print(f"PASS parallel download: {SIZE} bytes, 16 MiB bounded ranges, {WORKERS} workers, SHA-256 identical; 1 GiB maps to 64 resumable ranges")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user