优化大文件稳定并发传输与打包流程

This commit is contained in:
zhy200308
2026-08-11 11:27:40 +08:00
parent e326611c82
commit a4edf734b6
13 changed files with 715 additions and 101 deletions
+29 -2
View File
@@ -34,6 +34,7 @@ pub enum ReceiverEvent {
#[derive(Debug)]
struct UploadSession {
upload_key: Option<String>,
device: String,
file_name: String,
size: u64,
@@ -254,9 +255,34 @@ fn init_parallel_upload(
let total_chunks_u64 = if size == 0 { 0 } else { (size + chunk_size - 1) / chunk_size };
if total_chunks_u64 > u32::MAX as u64 { bail!("文件分块数过大") }
let total_chunks = total_chunks_u64 as u32;
let upload_key = headers.get("x-qft-upload-key")
.map(|value| value.trim().to_owned())
.filter(|value| !value.is_empty() && value.len() <= 128);
let mut map = sessions.lock().map_err(|_| anyhow::anyhow!("上传会话锁损坏"))?;
// init is idempotent: if the response packet was lost, the Android client
// may repeat init. Return the existing session so already received chunks
// and the preallocated file are preserved instead of leaking a second part.
for (existing_id, existing) in map.iter() {
if let Ok(session) = existing.lock() {
if upload_key.is_some() && session.upload_key == upload_key {
let missing: Vec<usize> = session.received.iter().enumerate()
.filter_map(|(index, received)| if *received { None } else { Some(index) })
.collect();
let body = json!({
"ok": true,
"upload_id": existing_id,
"chunk_size": session.chunk_size,
"total_chunks": session.total_chunks,
"resumed": true,
"missing": missing,
}).to_string();
return write_response(stream, 200, "application/json", body.as_bytes());
}
}
}
let upload_id = random_id();
let mut map = sessions.lock().map_err(|_| anyhow::anyhow!("上传会话锁损坏"))?;
let final_path = unique_path_reserved(save_dir, &file_name, map.values());
let final_name = final_path.file_name().and_then(|v| v.to_str()).unwrap_or("qft-upload.bin");
let part_path = final_path.with_file_name(format!("{final_name}.{upload_id}.qftpart"));
@@ -266,6 +292,7 @@ fn init_parallel_upload(
drop(part);
let session = UploadSession {
upload_key,
device: device.to_owned(),
file_name: file_name.clone(),
size,
@@ -338,7 +365,7 @@ fn receive_parallel_chunk(
let mut out = OpenOptions::new().write(true).open(&part_path)?;
out.seek(SeekFrom::Start(offset))?;
copy_exact(reader, &mut out, content_len)?;
out.flush()?;
out.sync_data()?;
let received = {
let mut session = session_arc.lock().map_err(|_| anyhow::anyhow!("上传会话损坏"))?;
+47 -21
View File
@@ -168,18 +168,19 @@ fn handle_client(
if input.metadata()?.len() != size {
bail!("发送文件在二维码生成后发生变化,请重新选择文件并生成二维码")
}
let offset = parse_range_start(headers.get("range"), size)?;
let remaining = size.saturating_sub(offset);
let range = parse_range(headers.get("range"), size)?;
let offset = range.start;
let end = range.end;
let remaining = if size == 0 { 0 } else { end.saturating_sub(offset).saturating_add(1) };
let _ = events.send(SenderEvent::DownloadStarted { device: device.clone(), file_name: file_name.to_owned(), size });
if offset == 0 {
if headers.get("range").is_none() {
write!(stream,
"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {remaining}\r\nAccept-Ranges: bytes\r\nConnection: close\r\nX-QFT-Filename: {}\r\n\r\n",
URL_SAFE_NO_PAD.encode(file_name.as_bytes()),
)?;
} else {
write!(stream,
"HTTP/1.1 206 Partial Content\r\nContent-Type: application/octet-stream\r\nContent-Length: {remaining}\r\nContent-Range: bytes {offset}-{}/{}\r\nAccept-Ranges: bytes\r\nConnection: close\r\nX-QFT-Filename: {}\r\n\r\n",
size - 1,
"HTTP/1.1 206 Partial Content\r\nContent-Type: application/octet-stream\r\nContent-Length: {remaining}\r\nContent-Range: bytes {offset}-{end}/{}\r\nAccept-Ranges: bytes\r\nConnection: close\r\nX-QFT-Filename: {}\r\n\r\n",
size,
URL_SAFE_NO_PAD.encode(file_name.as_bytes()),
)?;
@@ -187,11 +188,15 @@ fn handle_client(
use std::io::{Seek, SeekFrom};
input.seek(SeekFrom::Start(offset))?;
let mut buf = vec![0u8; IO_BUFFER];
loop {
let n = input.read(&mut buf)?;
let mut left = remaining;
while left > 0 {
let want = left.min(buf.len() as u64) as usize;
let n = input.read(&mut buf[..want])?;
if n == 0 { break }
stream.write_all(&buf[..n])?;
left -= n as u64;
}
if left != 0 { bail!("发送文件读取提前结束:还缺 {left} 字节") }
stream.flush()?;
let _ = events.send(SenderEvent::DownloadCompleted { device, file_name: file_name.to_owned() });
Ok(())
@@ -210,18 +215,34 @@ fn decode_header(value: Option<&String>) -> Option<String> {
String::from_utf8(bytes).ok()
}
fn parse_range_start(value: Option<&String>, size: u64) -> Result<u64> {
let Some(value) = value else { return Ok(0) };
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct ByteRange {
start: u64,
end: u64,
}
fn parse_range(value: Option<&String>, size: u64) -> Result<ByteRange> {
if size == 0 {
if value.is_some() { bail!("空文件没有可下载的字节范围") }
return Ok(ByteRange { start: 0, end: 0 });
}
let Some(value) = value else { return Ok(ByteRange { start: 0, end: size - 1 }) };
let suffix = value.trim().strip_prefix("bytes=").context("Range 格式无效")?;
let start = suffix.strip_suffix('-').context("只支持单段 Range")?
.parse::<u64>().context("Range 起点无效")?;
if start >= size { bail!("Range 超出文件范围") }
Ok(start)
let (start_text, end_text) = suffix.split_once('-').context("Range 格式无效")?;
if start_text.is_empty() { bail!("不支持后缀 Range") }
let start = start_text.parse::<u64>().context("Range 起点无效")?;
let end = if end_text.is_empty() {
size - 1
} else {
end_text.parse::<u64>().context("Range 终点无效")?
};
if start >= size || end >= size || end < start { bail!("Range 超出文件范围") }
Ok(ByteRange { start, end })
}
#[cfg(test)]
mod tests {
use super::{handle_client, parse_range_start, SenderEvent};
use super::{handle_client, parse_range, ByteRange, SenderEvent};
use std::{
fs,
io::{Read, Write},
@@ -231,12 +252,15 @@ mod tests {
};
#[test]
fn accepts_only_a_valid_resumable_byte_range() {
assert_eq!(parse_range_start(None, 10).unwrap(), 0);
assert_eq!(parse_range_start(Some(&"bytes=4-".to_owned()), 10).unwrap(), 4);
assert!(parse_range_start(Some(&"bytes=10-".to_owned()), 10).is_err());
assert!(parse_range_start(Some(&"bytes=-4".to_owned()), 10).is_err());
assert!(parse_range_start(Some(&"items=4-".to_owned()), 10).is_err());
fn accepts_open_and_bounded_byte_ranges() {
assert_eq!(parse_range(None, 10).unwrap(), ByteRange { start: 0, end: 9 });
assert_eq!(parse_range(None, 0).unwrap(), ByteRange { start: 0, end: 0 });
assert_eq!(parse_range(Some(&"bytes=4-".to_owned()), 10).unwrap(), ByteRange { start: 4, end: 9 });
assert_eq!(parse_range(Some(&"bytes=2-6".to_owned()), 10).unwrap(), ByteRange { start: 2, end: 6 });
assert!(parse_range(Some(&"bytes=10-".to_owned()), 10).is_err());
assert!(parse_range(Some(&"bytes=6-2".to_owned()), 10).is_err());
assert!(parse_range(Some(&"bytes=-4".to_owned()), 10).is_err());
assert!(parse_range(Some(&"items=4-".to_owned()), 10).is_err());
}
#[test]
@@ -248,7 +272,7 @@ mod tests {
let (events, _receiver) = mpsc::channel::<SenderEvent>();
let path_for_server = path.clone();
let server = thread::spawn(move || {
for _ in 0..2 {
for _ in 0..3 {
let (mut stream, _) = listener.accept().unwrap();
handle_client(&mut stream, &path_for_server, "range.bin", 6, "session", &events).unwrap();
}
@@ -260,6 +284,8 @@ mod tests {
assert_response(&resumed, "HTTP/1.1 206 Partial Content", "cdef");
let header = String::from_utf8_lossy(&resumed[..header_end(&resumed)]);
assert!(header.contains("Content-Range: bytes 2-5/6"));
let bounded = request(address, Some("bytes=1-3"));
assert_response(&bounded, "HTTP/1.1 206 Partial Content", "bcd");
server.join().unwrap();
fs::remove_file(path).unwrap();