108 lines
4.1 KiB
Python
108 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded Range + retry + out-of-order positioned-write download self-test."""
|
|
import hashlib
|
|
import http.client
|
|
import random
|
|
import tempfile
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
|
|
SIZE = 64 * 1024 * 1024 + 12345
|
|
CHUNK = 16 * 1024 * 1024
|
|
WORKERS = 8
|
|
TOKEN = "parallel_download_test"
|
|
|
|
|
|
def digest(path):
|
|
value = hashlib.sha256()
|
|
with open(path, "rb", buffering=0) as stream:
|
|
while block := stream.read(4 * 1024 * 1024):
|
|
value.update(block)
|
|
return value.hexdigest()
|
|
|
|
|
|
def main():
|
|
assert (1024 * 1024 * 1024 + CHUNK - 1) // CHUNK == 64
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root = Path(temp)
|
|
source = root / "source.bin"
|
|
target = root / "target.qftpart"
|
|
pattern = bytes((index * 73 + 19) & 0xFF for index in range(1024 * 1024))
|
|
with source.open("wb", buffering=0) as stream:
|
|
remaining = SIZE
|
|
while remaining:
|
|
block = pattern[:min(len(pattern), remaining)]
|
|
stream.write(block)
|
|
remaining -= len(block)
|
|
with target.open("wb") as stream:
|
|
stream.truncate(SIZE)
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
if self.path != "/qft/download" or self.headers.get("X-QFT-Token") != TOKEN:
|
|
self.send_error(403)
|
|
return
|
|
value = self.headers.get("Range", "")
|
|
assert value.startswith("bytes=") and "-" in value
|
|
start_text, end_text = value[6:].split("-", 1)
|
|
start, end = int(start_text), int(end_text)
|
|
assert 0 <= start <= end < SIZE
|
|
length = end - start + 1
|
|
self.send_response(206)
|
|
self.send_header("Content-Length", str(length))
|
|
self.send_header("Content-Range", f"bytes {start}-{end}/{SIZE}")
|
|
self.end_headers()
|
|
with source.open("rb", buffering=0) as stream:
|
|
stream.seek(start)
|
|
remaining = length
|
|
while remaining:
|
|
block = stream.read(min(1024 * 1024, remaining))
|
|
assert block
|
|
self.wfile.write(block)
|
|
remaining -= len(block)
|
|
|
|
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
host, port = server.server_address
|
|
indices = list(range((SIZE + CHUNK - 1) // CHUNK))
|
|
random.Random(42).shuffle(indices)
|
|
|
|
def receive(index):
|
|
start = index * CHUNK
|
|
end = min(SIZE, start + CHUNK) - 1
|
|
connection = http.client.HTTPConnection(host, port, timeout=10)
|
|
connection.request("GET", "/qft/download", headers={
|
|
"X-QFT-Token": TOKEN,
|
|
"Range": f"bytes={start}-{end}",
|
|
})
|
|
response = connection.getresponse()
|
|
assert response.status == 206
|
|
assert response.getheader("Content-Range") == f"bytes {start}-{end}/{SIZE}"
|
|
offset = start
|
|
with target.open("r+b", buffering=0) as output:
|
|
output.seek(start)
|
|
while block := response.read(1024 * 1024):
|
|
assert output.write(block) == len(block)
|
|
offset += len(block)
|
|
connection.close()
|
|
assert offset == end + 1
|
|
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
|
list(pool.map(receive, indices))
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
assert digest(source) == digest(target)
|
|
print(f"PASS parallel download: {SIZE} bytes, 16 MiB bounded ranges, {WORKERS} workers, SHA-256 identical; 1 GiB maps to 64 resumable ranges")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|