优化大文件稳定并发传输与打包流程

This commit is contained in:
zhy200308
2026-08-11 11:27:40 +08:00
parent e326611c82
commit a4edf734b6
13 changed files with 715 additions and 101 deletions
@@ -20,12 +20,14 @@ import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.UUID
data class SelectedFile(
val uri: Uri,
val name: String,
val size: Long,
val uploadKey: String = UUID.randomUUID().toString(),
)
object UploadClient {
@@ -42,7 +44,7 @@ object UploadClient {
// QFT Direct can briefly reset a TCP flow while the Android network route
// settles. Chunk PUTs and the init/finish control calls are safe to retry;
// the Windows receiver treats a repeated chunk as an idempotent success.
private const val LOCAL_REQUEST_ATTEMPTS = 3
private const val LOCAL_REQUEST_ATTEMPTS = 8
fun describeFile(resolver: ContentResolver, uri: Uri): SelectedFile? {
var name = "qft-upload.bin"
@@ -220,16 +222,20 @@ object UploadClient {
fun upload(
resolver: ContentResolver,
network: Network?,
networkProvider: () -> Network?,
baseUrl: String,
pairing: PairingPayload,
deviceName: String,
file: SelectedFile,
abortReason: () -> String? = { null },
onProgress: (sent: Long, total: Long) -> Unit,
) {
if (pairing.version >= 2 && file.size >= PARALLEL_MIN_FILE_SIZE && isSeekable(resolver, file)) {
if (pairing.version >= 2 && file.size >= PARALLEL_MIN_FILE_SIZE) {
require(isSeekable(resolver, file)) {
"大文件必须使用可随机读取的本地文件才能并行直传;请先把云盘文件下载到手机本地后再选择"
}
try {
uploadParallel(resolver, network, baseUrl, pairing, deviceName, file, onProgress)
uploadParallel(resolver, networkProvider, baseUrl, pairing, deviceName, file, abortReason, onProgress)
return
} catch (e: Exception) {
// Never mask a transient v2 failure by immediately switching to a
@@ -242,14 +248,20 @@ object UploadClient {
}
}
Log.i(TAG, "using single-stream upload: size=${file.size}, endpoint=$baseUrl")
uploadLegacy(resolver, network, baseUrl, pairing, deviceName, file, onProgress)
uploadLegacy(resolver, networkProvider(), baseUrl, pairing, deviceName, file, abortReason, onProgress)
}
private data class InitReply(val uploadId: String, val chunkSize: Long, val totalChunks: Int)
private data class InitReply(
val uploadId: String,
val chunkSize: Long,
val totalChunks: Int,
val pendingChunks: IntArray,
)
private fun <T> retryLocalRequest(label: String, block: () -> T): T {
private fun <T> retryLocalRequest(label: String, abortReason: () -> String?, block: () -> T): T {
var last: Throwable? = null
repeat(LOCAL_REQUEST_ATTEMPTS) { attempt ->
abortReason()?.let { throw IllegalStateException(it, last) }
try {
return block()
} catch (e: Throwable) {
@@ -260,7 +272,7 @@ object UploadClient {
e,
)
if (attempt + 1 < LOCAL_REQUEST_ATTEMPTS) {
try { Thread.sleep(120L * (attempt + 1)) } catch (_: InterruptedException) {
try { Thread.sleep((250L shl attempt).coerceAtMost(3_000L)) } catch (_: InterruptedException) {
Thread.currentThread().interrupt()
throw e
}
@@ -288,27 +300,37 @@ object UploadClient {
private fun uploadParallel(
resolver: ContentResolver,
network: Network?,
networkProvider: () -> Network?,
baseUrl: String,
pairing: PairingPayload,
deviceName: String,
file: SelectedFile,
abortReason: () -> String?,
onProgress: (sent: Long, total: Long) -> Unit,
) {
Log.i(TAG, "parallel init: size=${file.size}, endpoint=$baseUrl")
val init = initUpload(network, baseUrl, pairing, deviceName, file)
Log.i(TAG, "parallel init complete: chunks=${init.totalChunks}, chunkSize=${init.chunkSize}")
val init = initUpload(networkProvider, baseUrl, pairing, deviceName, file, file.uploadKey, abortReason)
Log.i(TAG, "parallel init complete: chunks=${init.totalChunks}, pending=${init.pendingChunks.size}, chunkSize=${init.chunkSize}")
if (init.totalChunks == 0) {
finishUpload(network, baseUrl, pairing, deviceName, init.uploadId)
finishUpload(networkProvider, baseUrl, pairing, deviceName, init.uploadId, abortReason)
onProgress(0, 0)
return
}
val cpuWorkers = Runtime.getRuntime().availableProcessors().coerceIn(2, MAX_WORKERS)
val workerCount = init.totalChunks.coerceAtMost(cpuWorkers).coerceAtLeast(1)
// Upload workers are network/disk bound, not CPU bound. Large files
// therefore use all eight lanes even on a 4-core phone.
if (init.pendingChunks.isEmpty()) {
finishUpload(networkProvider, baseUrl, pairing, deviceName, init.uploadId, abortReason)
onProgress(file.size, file.size)
return
}
val workerCount = init.pendingChunks.size.coerceAtMost(MAX_WORKERS).coerceAtLeast(1)
val pool = Executors.newFixedThreadPool(workerCount)
val next = AtomicInteger(0)
val sent = AtomicLong(0)
val pendingBytes = init.pendingChunks.sumOf { index ->
minOf(init.chunkSize, file.size - index.toLong() * init.chunkSize)
}
val sent = AtomicLong((file.size - pendingBytes).coerceAtLeast(0L))
val lastReport = AtomicLong(0)
val failures = mutableListOf<Throwable>()
val futures = (0 until workerCount).map {
@@ -318,13 +340,15 @@ object UploadClient {
val channel = input.channel
val buffer = ByteBuffer.allocate(STREAM_BUFFER)
while (!Thread.currentThread().isInterrupted) {
val index = next.getAndIncrement()
if (index >= init.totalChunks) break
abortReason()?.let { error(it) }
val pendingIndex = next.getAndIncrement()
if (pendingIndex >= init.pendingChunks.size) break
val index = init.pendingChunks[pendingIndex]
val offset = index.toLong() * init.chunkSize
val length = minOf(init.chunkSize, file.size - offset)
sendChunk(
network, baseUrl, pairing, deviceName,
init.uploadId, index, channel, offset, length, buffer,
networkProvider, baseUrl, pairing, deviceName,
init.uploadId, index, channel, offset, length, buffer, abortReason,
)
val totalSent = sent.addAndGet(length)
val now = android.os.SystemClock.elapsedRealtime()
@@ -345,7 +369,7 @@ object UploadClient {
pool.shutdownNow()
}
if (failures.isNotEmpty()) throw failures.first()
finishUpload(network, baseUrl, pairing, deviceName, init.uploadId)
finishUpload(networkProvider, baseUrl, pairing, deviceName, init.uploadId, abortReason)
onProgress(file.size, file.size)
}
@@ -357,10 +381,10 @@ object UploadClient {
}
private fun initUpload(
network: Network?, baseUrl: String, pairing: PairingPayload,
deviceName: String, file: SelectedFile,
): InitReply = retryLocalRequest("本地上传初始化失败") {
val conn = open(network, URL("$baseUrl/qft/upload/init"))
networkProvider: () -> Network?, baseUrl: String, pairing: PairingPayload,
deviceName: String, file: SelectedFile, requestKey: String, abortReason: () -> String?,
): InitReply = retryLocalRequest("本地上传初始化失败", abortReason) {
val conn = open(networkProvider(), URL("$baseUrl/qft/upload/init"))
try {
conn.requestMethod = "POST"
conn.doOutput = true
@@ -371,15 +395,27 @@ object UploadClient {
conn.setRequestProperty("X-QFT-Filename", b64(file.name))
conn.setRequestProperty("X-QFT-Size", file.size.toString())
conn.setRequestProperty("X-QFT-Chunk-Size", desiredChunkSize(file.size).toString())
conn.setRequestProperty("X-QFT-Upload-Key", requestKey)
conn.outputStream.close()
val code = conn.responseCode
if (code != 200) error("初始化并行上传失败 HTTP $code")
val body = conn.inputStream.bufferedReader().use { it.readText() }
val obj = JSONObject(body)
val totalChunks = obj.getInt("total_chunks")
val missing = obj.optJSONArray("missing")
val pending = if (missing == null) {
IntArray(totalChunks) { it }
} else {
IntArray(missing.length()) { missing.getInt(it) }
}
if (pending.any { it !in 0 until totalChunks } || pending.distinct().size != pending.size) {
error("电脑返回了无效的缺块列表")
}
InitReply(
uploadId = obj.getString("upload_id"),
chunkSize = obj.getLong("chunk_size"),
totalChunks = obj.getInt("total_chunks"),
totalChunks = totalChunks,
pendingChunks = pending,
)
} finally {
conn.disconnect()
@@ -387,17 +423,17 @@ object UploadClient {
}
private fun sendChunk(
network: Network?, baseUrl: String, pairing: PairingPayload, deviceName: String,
networkProvider: () -> Network?, baseUrl: String, pairing: PairingPayload, deviceName: String,
uploadId: String, index: Int, channel: FileChannel, offset: Long, length: Long,
buffer: ByteBuffer,
) = retryLocalRequest("分块 $index 上传失败") {
buffer: ByteBuffer, abortReason: () -> String?,
) = retryLocalRequest("分块 $index 上传失败", abortReason) {
Log.d(TAG, "chunk start: index=$index, bytes=$length, endpoint=$baseUrl")
val conn = open(network, URL("$baseUrl/qft/upload/chunk/$uploadId/$index"))
val conn = open(networkProvider(), URL("$baseUrl/qft/upload/chunk/$uploadId/$index"))
try {
conn.requestMethod = "POST"
conn.doOutput = true
conn.connectTimeout = 4_000
conn.readTimeout = 15_000
conn.connectTimeout = 8_000
conn.readTimeout = 90_000
conn.setFixedLengthStreamingMode(length)
conn.setRequestProperty("Content-Type", "application/octet-stream")
commonHeaders(conn, pairing, deviceName)
@@ -406,6 +442,7 @@ object UploadClient {
var remaining = length
val array = buffer.array()
while (remaining > 0) {
abortReason()?.let { error(it) }
buffer.clear()
buffer.limit(minOf(array.size.toLong(), remaining).toInt())
val n = channel.read(buffer)
@@ -428,10 +465,10 @@ object UploadClient {
}
private fun finishUpload(
network: Network?, baseUrl: String, pairing: PairingPayload,
deviceName: String, uploadId: String,
) = retryLocalRequest("本地上传完成确认失败") {
val conn = open(network, URL("$baseUrl/qft/upload/finish/$uploadId"))
networkProvider: () -> Network?, baseUrl: String, pairing: PairingPayload,
deviceName: String, uploadId: String, abortReason: () -> String?,
) = retryLocalRequest("本地上传完成确认失败", abortReason) {
val conn = open(networkProvider(), URL("$baseUrl/qft/upload/finish/$uploadId"))
try {
conn.requestMethod = "POST"
conn.doOutput = true
@@ -458,6 +495,7 @@ object UploadClient {
pairing: PairingPayload,
deviceName: String,
file: SelectedFile,
abortReason: () -> String?,
onProgress: (sent: Long, total: Long) -> Unit,
) {
val conn = open(network, URL("$baseUrl/qft/upload"))
@@ -478,6 +516,7 @@ object UploadClient {
var sent = 0L
var lastReport = 0L
while (true) {
abortReason()?.let { error(it) }
val n = input.read(buf)
if (n <= 0) break
output.write(buf, 0, n)
@@ -520,7 +559,7 @@ object UploadClient {
private fun commonHeaders(conn: HttpURLConnection, pairing: PairingPayload, deviceName: String) {
conn.useCaches = false
conn.setRequestProperty("Connection", "close")
conn.setRequestProperty("Connection", "keep-alive")
conn.setRequestProperty("X-QFT-Token", pairing.session)
conn.setRequestProperty("X-QFT-Device", b64(deviceName))
}