feat: 新增nginx存储后端支持并优化代码结构

- 新增nginx存储后端支持，支持通过nginx提供文件服务
- 重构存储模块，支持动态选择存储后端（local或nginx）
- 优化错误处理，使用anyhow统一错误处理
- 新增nginx测试环境，包含Dockerfile和docker-compose配置
- 优化目录列表渲染逻辑，修复URL拼接问题
- 新增更多文件扩展名支持
- 优化配置文件结构，支持nginx存储配置

Signed-off-by: longjin <longjin@DragonOS.org>

Cargo.toml

@@ -17,4 +17,6 @@ lazy_static = "1.5.0"
 log = "0.4.27"
 tokio = { version = "1.44.2", features = ["full"] }
 actix-files = "0.6.6"
-
+scraper = "0.18.1"
+url = "2.5.0"
+reqwest = { version = "0.12.15", features = ["json"] }

config.toml

@@ -1,6 +1,19 @@
 [storage]
-# 存储后端类型，目前支持local
-backend = "local"
+# 存储后端类型，支持local或nginx
+backend = "nginx"
+
+# 本地存储配置（当backend=local时必需）
+[storage.local]
+# 本地存储根目录
+root_path = "/tmp/test-mirror-proxy"
+
+# nginx存储配置（当backend=nginx时生效）
+[storage.nginx]
+# nginx服务器基础URL (例如: "http://nginx.example.com/files")
+base_url = "http://127.0.0.1:18080/"
+
+# 用于对外返回的url base (可以是公开域名)
+public_url = "http://127.0.0.1:18080/"
 
 [download_rules]
 # 需要特殊处理的文件后缀列表
@@ -12,10 +25,25 @@ extensions = [
     "rpm",
     "zip",
     "html",
-    "md"
+    "md",
+    "txt",
+    "json",
+    "xml",
+    "png",
+    "jpg",
+    "jpeg",
+    "gif",
+    "svg",
+    "mp4",
+    "avi",
+    "mkv",
+    "mov",
+    "wmv",
+    "flv",
+    "mpeg",
+    "mpg",
+    "m4v",
+    "webm",
+    "ogg",
+    "oga",
 ]
-
-# 本地存储配置（当backend=local时生效）
-[storage.local]
-# 本地存储根目录
-root_path = "/tmp/test-mirror-proxy"

src/config.rs

@@ -8,9 +8,25 @@ pub struct Config {
     pub download_rules: DownloadRules,
 }
 
+#[derive(Debug, Deserialize, PartialEq)]
+pub enum StorageBackend {
+    #[serde(rename = "local")]
+    Local,
+    #[serde(rename = "nginx")]
+    Nginx,
+}
+
 #[derive(Debug, Deserialize)]
 pub struct StorageConfig {
+    pub backend: StorageBackend,
     pub local: Option<LocalStorageConfig>,
+    pub nginx: Option<NginxStorageConfig>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct NginxStorageConfig {
+    pub base_url: String,
+    pub public_url: String, // 用于对外返回的url
 }
 
 #[derive(Debug, Deserialize)]

src/main.rs

@@ -23,7 +23,107 @@ mod storage;
 
 const BASE_PATH: &str = "/pub";
 
-#[get("/pub{path:.*}")]
+async fn handle_download_request(
+    path_str: &str,
+    req: &HttpRequest,
+) -> Result<HttpResponse, HttpError> {
+    let config = CONFIG.get().expect("Config not initialized");
+    if !has_matching_extension(path_str, &config.download_rules.extensions) {
+        return Err(HttpError::not_found("路径不存在", "请求的资源不存在"));
+    }
+
+    match select_provider(path_str) {
+        Some((provider, path_in_provider)) => {
+            if provider.is_local() {
+                log::debug!("Local storage provider selected, attempting to stream file (path in provider: {:?})", path_in_provider);
+                match provider.stream_file(&path_in_provider).await {
+                    Ok(Some(file)) => {
+                        return named_file_to_response(
+                            file,
+                            req.headers().get("range").and_then(|h| h.to_str().ok()),
+                            req,
+                        )
+                        .await
+                        .map_err(|_| HttpError::internal_error("服务器错误", "文件处理失败"));
+                    }
+                    Ok(None) => {
+                        return Err(HttpError::not_found("文件不存在", "请求的下载文件不存在"));
+                    }
+                    Err(e) => {
+                        log::error!("Failed to stream file: {}", e);
+                        return Err(HttpError::internal_error("服务器错误", "文件处理失败"));
+                    }
+                }
+            } else {
+                match provider.get_download_url(path_str).await {
+                    Ok(Some(download_url)) => {
+                        return Ok(HttpResponse::Found()
+                            .append_header((header::LOCATION, download_url))
+                            .finish());
+                    }
+                    Ok(None) => {
+                        return Err(HttpError::not_found("文件不存在", "请求的下载文件不存在"));
+                    }
+                    Err(e) => {
+                        log::error!("Failed to get download URL: {}", e);
+                        return Err(HttpError::internal_error("服务器错误", "获取下载链接失败"));
+                    }
+                }
+            }
+        }
+        None => Err(HttpError::not_found("路径不存在", "请求的资源不存在")),
+    }
+}
+
+async fn handle_directory_listing(
+    path_str: &str,
+    full_path: &PathBuf,
+) -> Result<HttpResponse, HttpError> {
+    let entries = match select_provider(path_str) {
+        Some((provider, path_in_provider)) => {
+            match provider.list_directory(&path_in_provider).await {
+                Ok(Some(entries)) => entries,
+                Ok(None) => return Err(HttpError::not_found("目录不存在", "请求的目录不存在")),
+                Err(e) => {
+                    if e.to_string().contains("Failed to connect to nginx") {
+                        log::error!("Nginx connection failed: {}", e);
+                        return Err(HttpError::internal_error(
+                            "服务器错误",
+                            "无法连接到存储服务",
+                        ));
+                    }
+                    log::error!("Failed to list directory: {}", e);
+                    return Err(HttpError::internal_error("服务器错误", "获取目录列表失败"));
+                }
+            }
+        }
+        None => return Err(HttpError::not_found("路径不存在", "请求的资源不存在")),
+    };
+
+    render::render_list(BASE_PATH, full_path.to_str().unwrap(), entries)
+        .map(|html| HttpResponse::Ok().content_type("text/html").body(html))
+        .map_err(|e| {
+            log::error!("渲染目录失败: {}", e);
+            HttpError::internal_error("服务器错误", "渲染目录时发生内部错误")
+        })
+}
+
+fn validate_path(full_path: &PathBuf) -> Result<&str, HttpError> {
+    if full_path
+        .components()
+        .any(|c| c == std::path::Component::ParentDir)
+    {
+        log::warn!("检测到非法路径访问尝试: {:?}", full_path);
+        return Err(HttpError::forbidden("访问被拒绝", "请求的路径无效"));
+    }
+
+    full_path.to_str().ok_or_else(|| {
+        log::warn!("检测到无效路径编码: {:?}", full_path);
+        HttpError::bad_request("无效请求", "请求路径无效")
+    })
+}
+
+#[get("/pub/{path:.*}")]
 async fn autoindex(req: HttpRequest, path: web::Path<String>) -> HttpResponse {
     let base_path = BASE_PATH.to_string();
     log::debug!("Base path: {:?}", base_path);
@@ -38,76 +138,24 @@ async fn autoindex(req: HttpRequest, path: web::Path<String>) -> HttpResponse {
         req_path = req_path.trim_start_matches('/').to_string();
     }
 
-    // 构造完整路径
     let full_path = PathBuf::from(format!("{}/{}", base_path, req_path));
     log::debug!("Full path: {:?}", full_path);
 
-    // 检查路径是否包含 `..`，防止跳出 `/pub` 目录
-    if full_path
-        .components()
-        .any(|c| c == std::path::Component::ParentDir)
-    {
-        return HttpError::forbidden("访问被拒绝", "路径中包含'..'，可能试图访问上级目录")
-            .to_http_response();
-    }
-
-    let path_str = match full_path.to_str() {
-        Some(s) => s,
-        None => {
-            return HttpError::bad_request("无效路径编码", "请求路径包含无效字符")
-                .to_http_response()
-        }
+    let path_str = match validate_path(&full_path) {
+        Ok(s) => s,
+        Err(e) => return e.to_http_response(),
     };
 
-    // 检查是否匹配下载规则
     let config = CONFIG.get().expect("Config not initialized");
     if has_matching_extension(path_str, &config.download_rules.extensions) {
-        match select_provider(path_str) {
-            Some((provider, path_in_provider)) => {
-                if provider.is_local() {
-                    log::debug!("Local storage provider selected, attempting to stream file (path in provider: {:?})", path_in_provider);
-                    if let Some(file) = provider.stream_file(&path_in_provider).await {
-                        return named_file_to_response(
-                            file,
-                            req.headers().get("range").and_then(|h| h.to_str().ok()),
-                            &req,
-                        )
-                        .await
-                        .unwrap_or_else(|_| {
-                            HttpError::internal_error("服务器错误", "文件处理失败")
-                                .to_http_response()
-                        });
-                    }
-                } else if let Some(download_url) = provider.get_download_url(path_str).await {
-                    return HttpResponse::Found()
-                        .append_header((header::LOCATION, download_url))
-                        .finish();
-                }
-                return HttpError::not_found("文件不存在", "请求的下载文件不存在")
-                    .to_http_response();
-            }
-            None => {
-                return HttpError::not_found("路径不存在", "请求的资源不存在").to_http_response()
-            }
+        match handle_download_request(path_str, &req).await {
+            Ok(resp) => resp,
+            Err(e) => e.to_http_response(),
         }
-    }
-
-    let entries = match select_provider(path_str) {
-        Some((provider, path_in_provider)) => provider.list_directory(&path_in_provider).await,
-        None => return HttpError::not_found("路径不存在", "请求的资源不存在").to_http_response(),
-    };
-
-    if entries.is_none() {
-        return HttpError::not_found("目录不存在", "请求的目录不存在").to_http_response();
-    }
-
-    let entries = entries.unwrap();
-
-    match render::render_list(BASE_PATH, full_path.to_str().unwrap(), entries) {
-        Ok(html) => HttpResponse::Ok().content_type("text/html").body(html),
-        Err(e) => {
-            log::error!("渲染目录失败: {}", e);
-            HttpError::internal_error("服务器错误", "渲染目录时发生内部错误").to_http_response()
+    } else {
+        match handle_directory_listing(path_str, &full_path).await {
+            Ok(resp) => resp,
+            Err(e) => e.to_http_response(),
         }
     }
 }

src/render/mod.rs

@@ -1,6 +1,5 @@
 use std::time::SystemTime;
 
-use actix_web::HttpResponse;
 use askama::Template;
 
 use crate::storage::StorageEntry;
@@ -69,7 +68,7 @@ pub fn render_list(
         .into_iter()
         .map(|e| {
             let mut et: IndexDirEntry = e.into();
-            et.url = format!("{}/{}", base_path, et.name);
+            et.url = format!("{}/{}", base_path, et.url);
             et
         })
         .collect();

src/storage/local.rs

@@ -85,11 +85,14 @@ impl StorageProvider for LocalStorageProvider {
         true
     }
 
-    async fn stream_file(&self, path_in_provider: &str) -> Option<NamedFile> {
-        let file_path = self.abs_path(path_in_provider).ok()?;
+    async fn stream_file(&self, path_in_provider: &str) -> anyhow::Result<Option<NamedFile>> {
+        let file_path = self.abs_path(path_in_provider)?;
         match NamedFile::open_async(file_path).await {
-            Ok(file) => Some(file),
-            Err(_) => None,
+            Ok(file) => Ok(Some(file)),
+            Err(e) => {
+                log::debug!("Failed to open file {}: {}", path_in_provider, e);
+                Ok(None)
+            }
         }
     }
 
@@ -101,31 +104,30 @@ impl StorageProvider for LocalStorageProvider {
         }
     }
 
-    async fn get_download_url(&self, _full_path: &str) -> Option<String> {
+    async fn get_download_url(&self, _full_path: &str) -> anyhow::Result<Option<String>> {
         // should not impl for local storage
-        None
+        Ok(None)
     }
 
-    async fn list_directory(&self, path_in_provider: &str) -> Option<Vec<super::StorageEntry>> {
+    async fn list_directory(
+        &self,
+        path_in_provider: &str,
+    ) -> anyhow::Result<Option<Vec<super::StorageEntry>>> {
         let mut entries = Vec::new();
-        let full_path = self.abs_path(path_in_provider);
-
-        if full_path.is_err() {
-            let e = full_path.unwrap_err();
-            log::error!("list_directory failed: {:?}", e);
-            // 路径解析失败
-            return None;
-        }
-
-        let full_path = full_path.unwrap();
+        let full_path = self.abs_path(path_in_provider).map_err(|e| {
+            e.context(format!(
+                "list_directory: Failed to resolve path '{}'",
+                path_in_provider
+            ))
+        })?;
         if !full_path.is_dir() {
             log::debug!("Path {} is not a directory", full_path.display());
-            return None;
+            return Ok(None);
         }
 
         if !full_path.exists() {
             log::debug!("Path {} does not exist", full_path.display());
-            return None;
+            return Ok(None);
         }
         log::debug!(
             "Listing directory {}/{}, full_path={}",
@@ -133,26 +135,20 @@ impl StorageProvider for LocalStorageProvider {
             path_in_provider,
             full_path.display()
         );
-        let dir_entries = fs::read_dir(full_path).await.inspect_err(|e| {
-            log::error!("Failed to read directory {}: {:?}", path_in_provider, e);
-        });
-        if dir_entries.is_err() {
-            return None;
-        }
-
-        if let Ok(mut dir_entries) = dir_entries {
-            while let Ok(Some(entry)) = dir_entries.next_entry().await {
-                let ent = self.process_entry(path_in_provider, &entry).await;
-                if ent.is_err() {
-                    log::info!(
-                        "Failed to process entry: {:?}, err: {:?}",
-                        entry.path(),
-                        ent
-                    );
-                    continue;
+        let mut dir_entries = fs::read_dir(full_path)
+            .await
+            .map_err(|e| anyhow::anyhow!("Failed to read directory {}: {}", path_in_provider, e))?;
+
+        while let Some(entry) = dir_entries
+            .next_entry()
+            .await
+            .map_err(|e| anyhow::anyhow!("Failed to read directory entry: {}", e))?
+        {
+            match self.process_entry(path_in_provider, &entry).await {
+                Ok(ent) => entries.push(ent),
+                Err(e) => {
+                    log::warn!("Failed to process entry {:?}: {}", entry.path(), e);
                 }
-                let ent = ent.unwrap();
-                entries.push(ent);
             }
         }
 
@@ -164,6 +160,6 @@ impl StorageProvider for LocalStorageProvider {
             path_in_provider,
             entries
         );
-        Some(entries)
+        Ok(Some(entries))
     }
 }

src/storage/mod.rs

@@ -1,28 +1,49 @@
 use std::{sync::Arc, time::SystemTime};
 
-use crate::BASE_PATH;
+use crate::{config::StorageBackend, BASE_PATH};
 use actix_files::NamedFile;
 use async_trait::async_trait;
 
 pub mod local;
+pub mod nginx;
 
 lazy_static! {
     static ref STORAGE_PROVIDER: Arc<dyn StorageProvider> = {
         let config = crate::CONFIG.get().expect("Config not initialized");
-        Arc::new(local::LocalStorageProvider::new(
-            config.storage.local.as_ref().unwrap().root_path.clone(),
-            BASE_PATH.to_string(),
-        ))
+        match config.storage.backend {
+            StorageBackend::Nginx => {
+                let nginx_config = config
+                    .storage
+                    .nginx
+                    .as_ref()
+                    .expect("Nginx storage config not found");
+                Arc::new(
+                    nginx::NginxStorageProvider::new(
+                        nginx_config.base_url.clone(),
+                        BASE_PATH.to_string(),
+                        nginx_config.public_url.clone(),
+                    )
+                    .expect("Failed to create Nginx storage provider"),
+                )
+            }
+            StorageBackend::Local => Arc::new(local::LocalStorageProvider::new(
+                config.storage.local.as_ref().unwrap().root_path.clone(),
+                BASE_PATH.to_string(),
+            )),
+        }
     };
 }
 
 #[async_trait]
 pub trait StorageProvider: Sync + Send {
-    async fn list_directory(&self, path_in_provider: &str) -> Option<Vec<StorageEntry>>;
+    async fn list_directory(
+        &self,
+        path_in_provider: &str,
+    ) -> anyhow::Result<Option<Vec<StorageEntry>>>;
     /// 根据完整的请求路径，返回在存储提供者中的路径
     fn path_in_provider(&self, full_path: &str) -> Option<String>;
     /// 获取文件的下载URL（适用于特定后缀的文件）
-    async fn get_download_url(&self, full_path: &str) -> Option<String>;
+    async fn get_download_url(&self, full_path: &str) -> anyhow::Result<Option<String>>;
 
     /// 是否是本地存储
     fn is_local(&self) -> bool {
@@ -31,8 +52,8 @@ pub trait StorageProvider: Sync + Send {
 
     /// 流式返回文件内容
     #[allow(unused)]
-    async fn stream_file(&self, path_in_provider: &str) -> Option<NamedFile> {
-        None
+    async fn stream_file(&self, path_in_provider: &str) -> anyhow::Result<Option<NamedFile>> {
+        Ok(None)
     }
 }
 

src/storage/nginx.rs

@@ -0,0 +1,156 @@
+use std::time::SystemTime;
+
+use async_trait::async_trait;
+use reqwest::Url;
+use scraper::{Html, Selector};
+use url::Url as UrlParser;
+
+use super::{StorageEntry, StorageProvider};
+
+pub struct NginxStorageProvider {
+    base_url: String,
+    req_path_prefix: String,
+    public_url: String, // 用于对外返回的url_base
+}
+
+impl NginxStorageProvider {
+    pub fn new(
+        mut base_url: String,
+        req_path_prefix: String,
+        mut public_url: String,
+    ) -> anyhow::Result<Self> {
+        // 验证base_url格式
+        let url = UrlParser::parse(&base_url)
+            .map_err(|e| anyhow::anyhow!("Invalid nginx base_url: {}", e))?;
+
+        // 验证协议是http或https
+        if url.scheme() != "http" && url.scheme() != "https" {
+            return Err(anyhow::anyhow!(
+                "nginx base_url must use http or https protocol"
+            ));
+        }
+
+        if !base_url.is_empty() && !base_url.ends_with('/') {
+            base_url.push('/');
+        }
+        if !public_url.is_empty() && !public_url.ends_with('/') {
+            public_url.push('/');
+        }
+
+        Ok(Self {
+            base_url,
+            req_path_prefix,
+            public_url,
+        })
+    }
+
+    async fn fetch_autoindex(&self, path: &str) -> anyhow::Result<String> {
+        let url = format!("{}/{}", self.base_url, path.trim_start_matches('/'));
+        let resp = reqwest::get(&url).await.map_err(|e| {
+            if e.is_connect() {
+                anyhow::anyhow!("Failed to connect to nginx: {}", e)
+            } else {
+                anyhow::anyhow!(e)
+            }
+        })?;
+
+        if resp.status().is_success() {
+            resp.text().await.map_err(Into::into)
+        } else if resp.status() == reqwest::StatusCode::NOT_FOUND {
+            Err(anyhow::anyhow!("nginx returned 404"))
+        } else {
+            Err(anyhow::anyhow!("nginx returned {}", resp.status()))
+        }
+    }
+
+    fn parse_entries(&self, html: &str, path: &str) -> anyhow::Result<Vec<StorageEntry>> {
+        let document = Html::parse_document(html);
+        let selector = Selector::parse("a").unwrap();
+        let mut entries = Vec::new();
+
+        for element in document.select(&selector) {
+            if let Some(href) = element.value().attr("href") {
+                if href == "../" {
+                    continue;
+                }
+
+                let name = element.text().collect::<String>();
+                let url = format!("{}/{}", path.trim_end_matches('/'), href);
+                let entry = StorageEntry {
+                    name,
+                    url,
+                    modified: SystemTime::now(), // nginx autoindex doesn't provide modified time
+                    size: None,                  // nginx autoindex doesn't provide file size
+                };
+                entries.push(entry);
+            }
+        }
+        log::debug!("Parsed nginx autoindex entries: {:?}", entries);
+        Ok(entries)
+    }
+}
+
+#[async_trait]
+impl StorageProvider for NginxStorageProvider {
+    async fn list_directory(
+        &self,
+        path_in_provider: &str,
+    ) -> anyhow::Result<Option<Vec<StorageEntry>>> {
+        match self.fetch_autoindex(path_in_provider).await {
+            Ok(html) => match self.parse_entries(&html, path_in_provider) {
+                Ok(entries) => Ok(Some(entries)),
+                Err(e) => {
+                    log::error!("Failed to parse nginx autoindex: {}", e);
+                    Ok(None)
+                }
+            },
+            Err(e) => {
+                if e.to_string().contains("Failed to connect to nginx") {
+                    log::error!("Nginx connection failed: {}", e);
+                    Err(anyhow::anyhow!("Internal server error").context(e))
+                } else if e.to_string().contains("nginx returned 404") {
+                    log::debug!("Nginx returned 404: {}", e);
+                    Ok(None)
+                } else {
+                    log::error!("Failed to fetch nginx autoindex: {}", e);
+                    Ok(None)
+                }
+            }
+        }
+    }
+
+    fn path_in_provider(&self, full_path: &str) -> Option<String> {
+        if full_path.starts_with(&self.req_path_prefix) {
+            Some(full_path[self.req_path_prefix.len()..].to_string())
+        } else {
+            None
+        }
+    }
+
+    async fn get_download_url(&self, full_path: &str) -> anyhow::Result<Option<String>> {
+        log::debug!("Getting download URL for {}", full_path);
+        let path_in_provider = match self.path_in_provider(full_path) {
+            Some(path) => path,
+            None => return Ok(None),
+        };
+
+        // 使用public_url构建对外URL
+        let url = Url::parse(&self.public_url)
+            .map_err(|e| anyhow::anyhow!("Invalid URL: {}", e))?
+            .join(&path_in_provider.strip_prefix("/").unwrap_or_default())
+            .map_err(|e| anyhow::anyhow!("Failed to join URLs: {}", e))?;
+
+        log::debug!(
+            "Download URL for {} is {} (public_host: {})",
+            full_path,
+            url,
+            self.public_url
+        );
+
+        Ok(Some(url.to_string()))
+    }
+
+    fn is_local(&self) -> bool {
+        false
+    }
+}

tests/nginx/Dockerfile

@@ -0,0 +1,18 @@
+# 使用官方nginx镜像作为基础
+FROM nginx:latest
+
+# 复制测试用的nginx配置
+COPY nginx-test.conf /etc/nginx/conf.d/default.conf
+
+# 创建测试目录结构
+RUN mkdir -p /usr/share/nginx/html/test && \
+    mkdir -p /usr/share/nginx/html/dir1/dir2 && \
+    echo "test1" > /usr/share/nginx/html/test/file1.txt && \
+    echo "test2" > /usr/share/nginx/html/test/file2.txt && \
+    echo "nested" > /usr/share/nginx/html/dir1/dir2/file3.txt
+
+# 暴露80端口
+EXPOSE 80
+
+# 启动nginx并保持前台运行
+CMD ["nginx", "-g", "daemon off;"]

tests/nginx/README.md

@@ -0,0 +1,37 @@
+# Nginx 测试环境
+
+## 快速启动 (使用docker-compose)
+
+1. 启动服务：
+```bash
+docker-compose up -d
+```
+
+2. 测试访问：
+```bash
+curl http://localhost:8080
+```
+或在浏览器打开 http://localhost:8080
+
+## 文件结构
+容器内预创建了以下测试文件：
+- /test/file1.txt
+- /test/file2.txt
+- /dir1/dir2/file3.txt
+
+## 管理命令
+
+停止服务：
+```bash
+docker-compose down
+```
+
+重启服务：
+```bash
+docker-compose restart
+```
+
+重新构建并启动：
+```bash
+docker-compose up -d --build
+```

tests/nginx/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3.8'
+
+services:
+  mirror-proxy-nginx-test-backend:
+    build: .
+    ports:
+      - "127.0.0.1:18080:80"
+    volumes:
+      - ./nginx-test.conf:/etc/nginx/conf.d/default.conf

tests/nginx/nginx-test.conf

@@ -0,0 +1,16 @@
+server {
+    listen 80;
+    server_name localhost;
+    autoindex on;
+
+    location / {
+        root   /usr/share/nginx/html;
+        index  index.html index.htm;
+    }
+
+    # 错误页面配置
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}