Allow for reads if one volume is down

src/server.rs

@@ -19,17 +19,38 @@ pub async fn get_key(
     State(state): State<AppState>,
     Path(key): Path<String>,
 ) -> Result<Response, AppError> {
+    use rand::seq::SliceRandom;
+
     let record = state.db.get(&key).await?;
-    let vol = record
-        .volumes
-        .first()
-        .ok_or_else(|| AppError::CorruptRecord { key: key.clone() })?;
-    let location = format!("{vol}/{key}");
-    Ok((
-        StatusCode::FOUND,
-        [(axum::http::header::LOCATION, location)],
-    )
-        .into_response())
+    if record.volumes.is_empty() {
+        return Err(AppError::CorruptRecord { key });
+    }
+
+    // Shuffle volumes for load balancing
+    let mut volumes = record.volumes.clone();
+    volumes.shuffle(&mut rand::thread_rng());
+
+    // Probe each volume until we find one that's reachable
+    for vol in &volumes {
+        let url = format!("{vol}/{key}");
+        match state.http.head(&url).send().await {
+            Ok(resp) if resp.status().is_success() => {
+                return Ok((
+                    StatusCode::FOUND,
+                    [(axum::http::header::LOCATION, url)],
+                )
+                    .into_response());
+            }
+            Ok(resp) => {
+                tracing::warn!("volume {vol} returned {} for {key}", resp.status());
+            }
+            Err(e) => {
+                tracing::warn!("volume {vol} unreachable for {key}: {e}");
+            }
+        }
+    }
+
+    Err(AppError::AllVolumesUnreachable)
 }
 
 pub async fn put_key(