优化大文件稳定并发传输与打包流程
This commit is contained in:
+29
-2
@@ -34,6 +34,7 @@ pub enum ReceiverEvent {
|
||||
|
||||
#[derive(Debug)]
|
||||
struct UploadSession {
|
||||
upload_key: Option<String>,
|
||||
device: String,
|
||||
file_name: String,
|
||||
size: u64,
|
||||
@@ -254,9 +255,34 @@ fn init_parallel_upload(
|
||||
let total_chunks_u64 = if size == 0 { 0 } else { (size + chunk_size - 1) / chunk_size };
|
||||
if total_chunks_u64 > u32::MAX as u64 { bail!("文件分块数过大") }
|
||||
let total_chunks = total_chunks_u64 as u32;
|
||||
let upload_key = headers.get("x-qft-upload-key")
|
||||
.map(|value| value.trim().to_owned())
|
||||
.filter(|value| !value.is_empty() && value.len() <= 128);
|
||||
|
||||
let mut map = sessions.lock().map_err(|_| anyhow::anyhow!("上传会话锁损坏"))?;
|
||||
// init is idempotent: if the response packet was lost, the Android client
|
||||
// may repeat init. Return the existing session so already received chunks
|
||||
// and the preallocated file are preserved instead of leaking a second part.
|
||||
for (existing_id, existing) in map.iter() {
|
||||
if let Ok(session) = existing.lock() {
|
||||
if upload_key.is_some() && session.upload_key == upload_key {
|
||||
let missing: Vec<usize> = session.received.iter().enumerate()
|
||||
.filter_map(|(index, received)| if *received { None } else { Some(index) })
|
||||
.collect();
|
||||
let body = json!({
|
||||
"ok": true,
|
||||
"upload_id": existing_id,
|
||||
"chunk_size": session.chunk_size,
|
||||
"total_chunks": session.total_chunks,
|
||||
"resumed": true,
|
||||
"missing": missing,
|
||||
}).to_string();
|
||||
return write_response(stream, 200, "application/json", body.as_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let upload_id = random_id();
|
||||
let mut map = sessions.lock().map_err(|_| anyhow::anyhow!("上传会话锁损坏"))?;
|
||||
let final_path = unique_path_reserved(save_dir, &file_name, map.values());
|
||||
let final_name = final_path.file_name().and_then(|v| v.to_str()).unwrap_or("qft-upload.bin");
|
||||
let part_path = final_path.with_file_name(format!("{final_name}.{upload_id}.qftpart"));
|
||||
@@ -266,6 +292,7 @@ fn init_parallel_upload(
|
||||
drop(part);
|
||||
|
||||
let session = UploadSession {
|
||||
upload_key,
|
||||
device: device.to_owned(),
|
||||
file_name: file_name.clone(),
|
||||
size,
|
||||
@@ -338,7 +365,7 @@ fn receive_parallel_chunk(
|
||||
let mut out = OpenOptions::new().write(true).open(&part_path)?;
|
||||
out.seek(SeekFrom::Start(offset))?;
|
||||
copy_exact(reader, &mut out, content_len)?;
|
||||
out.flush()?;
|
||||
out.sync_data()?;
|
||||
|
||||
let received = {
|
||||
let mut session = session_arc.lock().map_err(|_| anyhow::anyhow!("上传会话损坏"))?;
|
||||
|
||||
+47
-21
@@ -168,18 +168,19 @@ fn handle_client(
|
||||
if input.metadata()?.len() != size {
|
||||
bail!("发送文件在二维码生成后发生变化,请重新选择文件并生成二维码")
|
||||
}
|
||||
let offset = parse_range_start(headers.get("range"), size)?;
|
||||
let remaining = size.saturating_sub(offset);
|
||||
let range = parse_range(headers.get("range"), size)?;
|
||||
let offset = range.start;
|
||||
let end = range.end;
|
||||
let remaining = if size == 0 { 0 } else { end.saturating_sub(offset).saturating_add(1) };
|
||||
let _ = events.send(SenderEvent::DownloadStarted { device: device.clone(), file_name: file_name.to_owned(), size });
|
||||
if offset == 0 {
|
||||
if headers.get("range").is_none() {
|
||||
write!(stream,
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {remaining}\r\nAccept-Ranges: bytes\r\nConnection: close\r\nX-QFT-Filename: {}\r\n\r\n",
|
||||
URL_SAFE_NO_PAD.encode(file_name.as_bytes()),
|
||||
)?;
|
||||
} else {
|
||||
write!(stream,
|
||||
"HTTP/1.1 206 Partial Content\r\nContent-Type: application/octet-stream\r\nContent-Length: {remaining}\r\nContent-Range: bytes {offset}-{}/{}\r\nAccept-Ranges: bytes\r\nConnection: close\r\nX-QFT-Filename: {}\r\n\r\n",
|
||||
size - 1,
|
||||
"HTTP/1.1 206 Partial Content\r\nContent-Type: application/octet-stream\r\nContent-Length: {remaining}\r\nContent-Range: bytes {offset}-{end}/{}\r\nAccept-Ranges: bytes\r\nConnection: close\r\nX-QFT-Filename: {}\r\n\r\n",
|
||||
size,
|
||||
URL_SAFE_NO_PAD.encode(file_name.as_bytes()),
|
||||
)?;
|
||||
@@ -187,11 +188,15 @@ fn handle_client(
|
||||
use std::io::{Seek, SeekFrom};
|
||||
input.seek(SeekFrom::Start(offset))?;
|
||||
let mut buf = vec![0u8; IO_BUFFER];
|
||||
loop {
|
||||
let n = input.read(&mut buf)?;
|
||||
let mut left = remaining;
|
||||
while left > 0 {
|
||||
let want = left.min(buf.len() as u64) as usize;
|
||||
let n = input.read(&mut buf[..want])?;
|
||||
if n == 0 { break }
|
||||
stream.write_all(&buf[..n])?;
|
||||
left -= n as u64;
|
||||
}
|
||||
if left != 0 { bail!("发送文件读取提前结束:还缺 {left} 字节") }
|
||||
stream.flush()?;
|
||||
let _ = events.send(SenderEvent::DownloadCompleted { device, file_name: file_name.to_owned() });
|
||||
Ok(())
|
||||
@@ -210,18 +215,34 @@ fn decode_header(value: Option<&String>) -> Option<String> {
|
||||
String::from_utf8(bytes).ok()
|
||||
}
|
||||
|
||||
fn parse_range_start(value: Option<&String>, size: u64) -> Result<u64> {
|
||||
let Some(value) = value else { return Ok(0) };
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct ByteRange {
|
||||
start: u64,
|
||||
end: u64,
|
||||
}
|
||||
|
||||
fn parse_range(value: Option<&String>, size: u64) -> Result<ByteRange> {
|
||||
if size == 0 {
|
||||
if value.is_some() { bail!("空文件没有可下载的字节范围") }
|
||||
return Ok(ByteRange { start: 0, end: 0 });
|
||||
}
|
||||
let Some(value) = value else { return Ok(ByteRange { start: 0, end: size - 1 }) };
|
||||
let suffix = value.trim().strip_prefix("bytes=").context("Range 格式无效")?;
|
||||
let start = suffix.strip_suffix('-').context("只支持单段 Range")?
|
||||
.parse::<u64>().context("Range 起点无效")?;
|
||||
if start >= size { bail!("Range 超出文件范围") }
|
||||
Ok(start)
|
||||
let (start_text, end_text) = suffix.split_once('-').context("Range 格式无效")?;
|
||||
if start_text.is_empty() { bail!("不支持后缀 Range") }
|
||||
let start = start_text.parse::<u64>().context("Range 起点无效")?;
|
||||
let end = if end_text.is_empty() {
|
||||
size - 1
|
||||
} else {
|
||||
end_text.parse::<u64>().context("Range 终点无效")?
|
||||
};
|
||||
if start >= size || end >= size || end < start { bail!("Range 超出文件范围") }
|
||||
Ok(ByteRange { start, end })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{handle_client, parse_range_start, SenderEvent};
|
||||
use super::{handle_client, parse_range, ByteRange, SenderEvent};
|
||||
use std::{
|
||||
fs,
|
||||
io::{Read, Write},
|
||||
@@ -231,12 +252,15 @@ mod tests {
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn accepts_only_a_valid_resumable_byte_range() {
|
||||
assert_eq!(parse_range_start(None, 10).unwrap(), 0);
|
||||
assert_eq!(parse_range_start(Some(&"bytes=4-".to_owned()), 10).unwrap(), 4);
|
||||
assert!(parse_range_start(Some(&"bytes=10-".to_owned()), 10).is_err());
|
||||
assert!(parse_range_start(Some(&"bytes=-4".to_owned()), 10).is_err());
|
||||
assert!(parse_range_start(Some(&"items=4-".to_owned()), 10).is_err());
|
||||
fn accepts_open_and_bounded_byte_ranges() {
|
||||
assert_eq!(parse_range(None, 10).unwrap(), ByteRange { start: 0, end: 9 });
|
||||
assert_eq!(parse_range(None, 0).unwrap(), ByteRange { start: 0, end: 0 });
|
||||
assert_eq!(parse_range(Some(&"bytes=4-".to_owned()), 10).unwrap(), ByteRange { start: 4, end: 9 });
|
||||
assert_eq!(parse_range(Some(&"bytes=2-6".to_owned()), 10).unwrap(), ByteRange { start: 2, end: 6 });
|
||||
assert!(parse_range(Some(&"bytes=10-".to_owned()), 10).is_err());
|
||||
assert!(parse_range(Some(&"bytes=6-2".to_owned()), 10).is_err());
|
||||
assert!(parse_range(Some(&"bytes=-4".to_owned()), 10).is_err());
|
||||
assert!(parse_range(Some(&"items=4-".to_owned()), 10).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -248,7 +272,7 @@ mod tests {
|
||||
let (events, _receiver) = mpsc::channel::<SenderEvent>();
|
||||
let path_for_server = path.clone();
|
||||
let server = thread::spawn(move || {
|
||||
for _ in 0..2 {
|
||||
for _ in 0..3 {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
handle_client(&mut stream, &path_for_server, "range.bin", 6, "session", &events).unwrap();
|
||||
}
|
||||
@@ -260,6 +284,8 @@ mod tests {
|
||||
assert_response(&resumed, "HTTP/1.1 206 Partial Content", "cdef");
|
||||
let header = String::from_utf8_lossy(&resumed[..header_end(&resumed)]);
|
||||
assert!(header.contains("Content-Range: bytes 2-5/6"));
|
||||
let bounded = request(address, Some("bytes=1-3"));
|
||||
assert_response(&bounded, "HTTP/1.1 206 Partial Content", "bcd");
|
||||
|
||||
server.join().unwrap();
|
||||
fs::remove_file(path).unwrap();
|
||||
|
||||
Reference in New Issue
Block a user