V0.3.0版本优化传输速度和提示
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Loopback integration test for QFTDISC2 + Parallel v2 init/chunk/finish semantics."""
|
||||
import hashlib, http.client, json, os, random, socket, tempfile, threading, time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
TOKEN = "integration_session_128bit_ABC"
|
||||
DISCOVERY_MAGIC = f"QFTDISC2:{TOKEN}".encode()
|
||||
CHUNK = 8 * 1024 * 1024
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root = Path(td)
|
||||
source = root / "src.bin"
|
||||
source.write_bytes(os.urandom(40 * 1024 * 1024 + 12345))
|
||||
source_sha = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
part = root / "recv.qftpart"
|
||||
part.write_bytes(b"")
|
||||
part_fp = open(part, "r+b", buffering=0)
|
||||
part_fp.truncate(source.stat().st_size)
|
||||
lock = threading.Lock()
|
||||
received = set()
|
||||
upload_id = "up_test_01"
|
||||
total_chunks = (source.stat().st_size + CHUNK - 1) // CHUNK
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_): pass
|
||||
def _auth(self): return self.headers.get("X-QFT-Token") == TOKEN
|
||||
def _reply(self, code, body=b"{}"):
|
||||
self.send_response(code); self.send_header("Content-Length", str(len(body))); self.end_headers(); self.wfile.write(body)
|
||||
def do_GET(self):
|
||||
if not self._auth(): return self._reply(403)
|
||||
if self.path == "/qft/ping": return self._reply(200, b'{"ok":true,"protocol":2,"parallel":true}')
|
||||
self._reply(404)
|
||||
def do_POST(self):
|
||||
if not self._auth(): return self._reply(403)
|
||||
if self.path == "/qft/upload/init":
|
||||
body = json.dumps({"ok":True,"upload_id":upload_id,"chunk_size":CHUNK,"total_chunks":total_chunks}).encode()
|
||||
return self._reply(200, body)
|
||||
prefix = f"/qft/upload/chunk/{upload_id}/"
|
||||
if self.path.startswith(prefix):
|
||||
idx = int(self.path[len(prefix):]); n = int(self.headers["Content-Length"]); data = self.rfile.read(n)
|
||||
offset = idx * CHUNK
|
||||
assert n == min(CHUNK, source.stat().st_size - offset)
|
||||
with lock:
|
||||
part_fp.seek(offset); part_fp.write(data); received.add(idx)
|
||||
return self._reply(200)
|
||||
if self.path == f"/qft/upload/finish/{upload_id}":
|
||||
if len(received) != total_chunks: return self._reply(409)
|
||||
with lock: part_fp.flush(); os.fsync(part_fp.fileno())
|
||||
return self._reply(200, json.dumps({"ok":True}).encode())
|
||||
self._reply(404)
|
||||
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
port = httpd.server_address[1]
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
|
||||
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); udp.bind(("127.0.0.1", 0)); udp_port = udp.getsockname()[1]
|
||||
stop = threading.Event()
|
||||
def udp_server():
|
||||
while not stop.is_set():
|
||||
udp.settimeout(.2)
|
||||
try: data, peer = udp.recvfrom(512)
|
||||
except socket.timeout: continue
|
||||
except OSError:
|
||||
if stop.is_set(): break
|
||||
raise
|
||||
if data == DISCOVERY_MAGIC: udp.sendto(f"QFTHERE2:{TOKEN}:{port}".encode(), peer)
|
||||
threading.Thread(target=udp_server, daemon=True).start()
|
||||
|
||||
# UDP discovery client.
|
||||
c = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); c.settimeout(1); c.sendto(DISCOVERY_MAGIC, ("127.0.0.1", udp_port))
|
||||
resp, _ = c.recvfrom(512); assert resp == f"QFTHERE2:{TOKEN}:{port}".encode(); c.close()
|
||||
|
||||
def request(method, path, body=b"", headers=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
|
||||
h = {"X-QFT-Token": TOKEN}; h.update(headers or {})
|
||||
conn.request(method, path, body=body, headers=h); r = conn.getresponse(); payload = r.read(); status = r.status; conn.close(); return status, payload
|
||||
assert request("GET", "/qft/ping")[0] == 200
|
||||
status, body = request("POST", "/qft/upload/init", headers={"Content-Length":"0"}); assert status == 200
|
||||
init = json.loads(body); assert init["chunk_size"] == CHUNK and init["total_chunks"] == total_chunks
|
||||
|
||||
indices = list(range(total_chunks)); random.Random(42).shuffle(indices)
|
||||
def send(idx):
|
||||
with open(source, "rb") as f:
|
||||
f.seek(idx * CHUNK); data = f.read(min(CHUNK, source.stat().st_size - idx * CHUNK))
|
||||
status, _ = request("POST", f"/qft/upload/chunk/{upload_id}/{idx}", data, {"Content-Length":str(len(data))})
|
||||
assert status == 200
|
||||
started = time.perf_counter()
|
||||
with ThreadPoolExecutor(max_workers=8) as ex: list(ex.map(send, indices))
|
||||
assert request("POST", f"/qft/upload/finish/{upload_id}", headers={"Content-Length":"0"})[0] == 200
|
||||
elapsed = time.perf_counter() - started
|
||||
part_fp.close(); recv_sha = hashlib.sha256(part.read_bytes()).hexdigest(); assert recv_sha == source_sha
|
||||
stop.set(); udp.close(); httpd.shutdown(); httpd.server_close()
|
||||
print(f"PASS loopback Parallel v2 integration: UDP discovery + HTTP auth + {total_chunks} out-of-order chunks / 8 workers; "
|
||||
f"{source.stat().st_size} bytes SHA-256 identical in {elapsed:.2f}s")
|
||||
Reference in New Issue
Block a user