Allow for reads if one volume is down

src/error.rs

@@ -35,6 +35,7 @@ pub enum AppError {
     Db(rusqlite::Error),
     InsufficientVolumes { need: usize, have: usize },
     PartialWrite,
+    AllVolumesUnreachable,
 }
 
 impl From<rusqlite::Error> for AppError {
@@ -58,6 +59,7 @@ impl std::fmt::Display for AppError {
                 write!(f, "need {need} volumes but only {have} available")
             }
             AppError::PartialWrite => write!(f, "not all volume writes succeeded"),
+            AppError::AllVolumesUnreachable => write!(f, "all volume replicas are unreachable"),
         }
     }
 }
@@ -70,6 +72,7 @@ impl IntoResponse for AppError {
             AppError::Db(_) => StatusCode::INTERNAL_SERVER_ERROR,
             AppError::InsufficientVolumes { .. } => StatusCode::SERVICE_UNAVAILABLE,
             AppError::PartialWrite => StatusCode::BAD_GATEWAY,
+            AppError::AllVolumesUnreachable => StatusCode::BAD_GATEWAY,
         };
         (status, self.to_string()).into_response()
     }

src/server.rs

@@ -19,17 +19,38 @@ pub async fn get_key(
     State(state): State<AppState>,
     Path(key): Path<String>,
 ) -> Result<Response, AppError> {
+    use rand::seq::SliceRandom;
+
     let record = state.db.get(&key).await?;
-    let vol = record
-        .volumes
-        .first()
-        .ok_or_else(|| AppError::CorruptRecord { key: key.clone() })?;
-    let location = format!("{vol}/{key}");
-    Ok((
-        StatusCode::FOUND,
-        [(axum::http::header::LOCATION, location)],
-    )
-        .into_response())
+    if record.volumes.is_empty() {
+        return Err(AppError::CorruptRecord { key });
+    }
+
+    // Shuffle volumes for load balancing
+    let mut volumes = record.volumes.clone();
+    volumes.shuffle(&mut rand::thread_rng());
+
+    // Probe each volume until we find one that's reachable
+    for vol in &volumes {
+        let url = format!("{vol}/{key}");
+        match state.http.head(&url).send().await {
+            Ok(resp) if resp.status().is_success() => {
+                return Ok((
+                    StatusCode::FOUND,
+                    [(axum::http::header::LOCATION, url)],
+                )
+                    .into_response());
+            }
+            Ok(resp) => {
+                tracing::warn!("volume {vol} returned {} for {key}", resp.status());
+            }
+            Err(e) => {
+                tracing::warn!("volume {vol} unreachable for {key}: {e}");
+            }
+        }
+    }
+
+    Err(AppError::AllVolumesUnreachable)
 }
 
 pub async fn put_key(