From 2c6e1c67b22109be4c20ec3cc2db5b7fdfb1e1fd Mon Sep 17 00:00:00 2001 From: ALEZ-DEV Date: Sat, 27 Apr 2024 15:49:52 +0200 Subject: [PATCH] can now resume download by checking game file and md5 checksum --- babylonia-terminal-cli/Cargo.lock | 47 ++++++++++++++ babylonia-terminal-sdk/Cargo.lock | 47 ++++++++++++++ babylonia-terminal-sdk/Cargo.toml | 1 + .../src/components/game_component.rs | 64 ++++++++++++++++--- 4 files changed, 149 insertions(+), 10 deletions(-) diff --git a/babylonia-terminal-cli/Cargo.lock b/babylonia-terminal-cli/Cargo.lock index 8ff4c95..9338401 100644 --- a/babylonia-terminal-cli/Cargo.lock +++ b/babylonia-terminal-cli/Cargo.lock @@ -75,6 +75,7 @@ name = "babylonia-terminal-sdk" version = "0.1.0" dependencies = [ "anyhow", + "chksum-md5", "dirs", "dotenv", "downloader", @@ -159,6 +160,52 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chksum-core" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6db20071fdeca52ed6a7745519fb2d343fddcb93af81448373b851f072aaec5" +dependencies = [ + "chksum-hash-core", + "thiserror", +] + +[[package]] +name = "chksum-hash-core" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221456234d441c788a2c51a27b91c4380f499de560670a67d3303e621d37b3bd" + +[[package]] +name = "chksum-hash-md5" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80c33d01c33c9e193fe33e719a29a7eb900c08583375dd1d3269991aacbe434a" +dependencies = [ + "chksum-hash-core", + "thiserror", +] + +[[package]] +name = "chksum-md5" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95dda0f76fbb6069e042c370a928457086e1b4eabc7e75f5f49fe1b913634351" +dependencies = [ + "chksum-core", + "chksum-hash-md5", + "chksum-reader", +] + +[[package]] +name = "chksum-reader" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bcdf6bcd410820599b7501d9bc3346964c4d472f48905685e08e07359d2fff9" +dependencies = [ + "chksum-core", +] + [[package]] name = "colored" version = "2.1.0" diff --git a/babylonia-terminal-sdk/Cargo.lock b/babylonia-terminal-sdk/Cargo.lock index 819259c..48e6665 100644 --- a/babylonia-terminal-sdk/Cargo.lock +++ b/babylonia-terminal-sdk/Cargo.lock @@ -62,6 +62,7 @@ name = "babylonia-terminal-sdk" version = "0.1.0" dependencies = [ "anyhow", + "chksum-md5", "dirs", "dotenv", "downloader", @@ -146,6 +147,52 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chksum-core" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6db20071fdeca52ed6a7745519fb2d343fddcb93af81448373b851f072aaec5" +dependencies = [ + "chksum-hash-core", + "thiserror", +] + +[[package]] +name = "chksum-hash-core" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221456234d441c788a2c51a27b91c4380f499de560670a67d3303e621d37b3bd" + +[[package]] +name = "chksum-hash-md5" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80c33d01c33c9e193fe33e719a29a7eb900c08583375dd1d3269991aacbe434a" +dependencies = [ + "chksum-hash-core", + "thiserror", +] + +[[package]] +name = "chksum-md5" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95dda0f76fbb6069e042c370a928457086e1b4eabc7e75f5f49fe1b913634351" +dependencies = [ + "chksum-core", + "chksum-hash-md5", + "chksum-reader", +] + +[[package]] +name = "chksum-reader" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bcdf6bcd410820599b7501d9bc3346964c4d472f48905685e08e07359d2fff9" +dependencies = [ + "chksum-core", +] + [[package]] name = "constant_time_eq" version = "0.3.0" diff --git a/babylonia-terminal-sdk/Cargo.toml b/babylonia-terminal-sdk/Cargo.toml index 64a1226..e147996 100644 --- a/babylonia-terminal-sdk/Cargo.toml +++ b/babylonia-terminal-sdk/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] anyhow = "1.0.81" +chksum-md5 = { version = "0.0.0", features = ["reader"] } dirs = "5.0.1" dotenv = "0.15.0" downloader = { git = "https://github.com/ALEZ-DEV/downloader" } # version = "0.2.7", diff --git a/babylonia-terminal-sdk/src/components/game_component.rs b/babylonia-terminal-sdk/src/components/game_component.rs index 8b9f74e..3a3d2f6 100644 --- a/babylonia-terminal-sdk/src/components/game_component.rs +++ b/babylonia-terminal-sdk/src/components/game_component.rs @@ -6,16 +6,20 @@ use log::info; use serde::Deserialize; use serde::Serialize; use std::collections::TryReserveError; +use std::fs as std_fs; use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; use std::time::Instant; use std::vec; -use tokio::fs::create_dir_all; +use tokio::fs::File; +use tokio::fs::{create_dir_all, remove_file}; use super::component_downloader::ComponentDownloader; use crate::utils::github_requester::GithubRequester; use crate::utils::kuro_prod_api; +use crate::utils::kuro_prod_api::Resource; +use crate::utils::kuro_prod_api::Resources; pub struct GameComponent { game_dir: PathBuf, @@ -25,6 +29,43 @@ impl GameComponent { pub fn new(game_dir: PathBuf) -> Self { Self { game_dir } } + + async fn check_and_get_resources_to_download( + game_dir: &std::path::PathBuf, + resources: &Resources, + ) -> anyhow::Result> { + info!("checking all files..."); + let mut to_download: Vec = vec![]; + + for r in &resources.resource { + let file_path = game_dir.join(r.dest.clone().strip_prefix("/").unwrap()); + + if file_path.try_exists()? { + let blocking_file_path = file_path.clone(); + let file = + tokio::task::spawn_blocking(move || std_fs::File::open(blocking_file_path)) + .await??; // only the std::File is supported by chksum_md5, that's why I block + let digest = chksum_md5::chksum(file)?; + + debug!( + "{} = {} -> {}", + digest.to_hex_lowercase(), + r.md5, + digest.to_hex_lowercase() == r.md5 + ); + + if digest.to_hex_lowercase() != r.md5 { + to_download.push(r.clone()); + remove_file(file_path).await?; + } + } else { + debug!("{:?} don't exist", file_path); + to_download.push(r.clone()); + } + } + + Ok(to_download) + } } impl GithubRequester for GameComponent {} @@ -50,18 +91,21 @@ impl ComponentDownloader for GameComponent { .parallel_requests(5) .build()?; - if let Some(ref p) = progress { - let mut size: u64 = 0; + //if let Some(ref p) = progress { + // let mut size: u64 = 0; - resources - .resource - .iter() - .for_each(|r| size += r.size as u64); + // resources + // .resource + // .iter() + // .for_each(|r| size += r.size as u64); - p.setup(Some(size), "download has started"); - } + // p.setup(Some(size), "download has started"); + //} - for chunk_resource in resources.resource.chunks(5) { + let checked_resources = + GameComponent::check_and_get_resources_to_download(output_dir, &resources).await?; + + for chunk_resource in checked_resources.chunks(5) { let mut output_path: Vec = vec![]; for path in chunk_resource