From 85a0e8557bc722120a7021e677801e2cd5c47e9d Mon Sep 17 00:00:00 2001 From: Jean-Marie Mineau Date: Mon, 3 Feb 2025 11:03:35 +0100 Subject: [PATCH] add feature to use an actual zip parsing lib when reading --- Cargo.lock | 379 ++++++++++++++++++++++++++++++++- androscalpel/Cargo.toml | 4 +- androscalpel/src/apk.rs | 82 ++++--- androscalpel/src/dex_string.rs | 1 + 4 files changed, 429 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c06f2ea..8610ed3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "androscalpel" version = "0.1.0" @@ -33,6 +44,7 @@ dependencies = [ "serde", "serde_json", "sha1", + "zip", ] [[package]] @@ -72,6 +84,15 @@ dependencies = [ "rand", ] +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.6.0" @@ -108,12 +129,46 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ + "jobserver", "libc", ] @@ -123,6 +178,22 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "cpufeatures" version = "0.2.11" @@ -133,10 +204,25 @@ dependencies = [ ] [[package]] -name = "crc32fast" -version = "1.3.2" +name = "crc" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -162,9 +248,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -176,6 +262,32 @@ dependencies = [ "typenum", ] +[[package]] +name = "deflate64" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "diff" version = "0.1.13" @@ -190,6 +302,18 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -198,6 +322,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "flate2" version = "1.0.28" @@ -235,30 +365,79 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indoc" version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "itoa" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "libc" version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.25" @@ -266,10 +445,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] -name = "memchr" -version = "2.5.0" +name = "lzma-rs" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -289,6 +478,12 @@ dependencies = [ "adler", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "object" version = "0.32.1" @@ -300,9 +495,25 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "portable-atomic" @@ -310,6 +521,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -523,6 +740,18 @@ dependencies = [ "digest", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.96" @@ -540,6 +769,45 @@ version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +[[package]] +name = "thiserror" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + [[package]] name = "typenum" version = "1.17.0" @@ -575,3 +843,94 @@ name = "yansi" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae9c1ea7b3a5e1f4b922ff856a129881167511563dc219869afe3787fc0c1a45" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "rand", + "sha1", + "thiserror", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/androscalpel/Cargo.toml b/androscalpel/Cargo.toml index cd179a3..4369668 100644 --- a/androscalpel/Cargo.toml +++ b/androscalpel/Cargo.toml @@ -20,6 +20,7 @@ rayon = "1.9.0" serde = { version = "1.0.195", features = ["derive"] } serde_json = "1.0.111" sha1 = "0.10.6" +zip = {version = "2.2.2", optional = true} [dev-dependencies] pretty_assertions = "1.4.1" @@ -27,4 +28,5 @@ pretty_assertions = "1.4.1" [features] default = [] # TODO: need refactoring to https://github.com/PyO3/pyo3/issues/2935#issuecomment-2560930677 or cfg_eval https://github.com/rust-lang/rust/issues/82679 -python = ["pyo3", "pyo3-log"] +python = ["pyo3", "pyo3-log"] # Currently not supported +external-zip-reader = ["zip"] diff --git a/androscalpel/src/apk.rs b/androscalpel/src/apk.rs index ae8528f..38fbbba 100644 --- a/androscalpel/src/apk.rs +++ b/androscalpel/src/apk.rs @@ -4,7 +4,7 @@ use anyhow::{anyhow, bail, Context}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fs::File; -use std::io::Cursor; +use std::io::{Cursor, Read, Seek}; use std::path::PathBuf; use log::{error, info}; @@ -17,8 +17,11 @@ use crate::Result; use crate::*; use androscalpel_serializer::Instruction as InsFormat; use androscalpel_serializer::*; +#[cfg(not(feature = "external-zip-reader"))] use apk_frauder::ZipFileReader; use rayon::prelude::*; +#[cfg(feature = "external-zip-reader")] +use zip::ZipArchive; #[derive(Debug, Clone, PartialEq, Default, Deserialize, Serialize)] pub struct DexFile { @@ -3043,6 +3046,55 @@ impl Apk { None } + + /// Load all android files in an application. + /// This **does not include any .dex file that android would not load. + #[cfg(feature = "external-zip-reader")] + pub fn load_apk(apk: impl Read + Seek, label_each_ins: bool, cache: bool) -> Result { + let mut zip = ZipArchive::new(apk)?; + let mut apk = Self::default(); + let file_names: HashSet = zip.file_names().map(|s| s.into()).collect(); + let mut dex_names: HashSet = HashSet::new(); + let mut i = 0; + loop { + let name = if i == 0 { + "classes.dex".into() + } else { + format!("classes{}.dex", i + 1) + }; + i += 1; + if file_names.contains(&name) { + dex_names.insert(name); + } else { + break; + } + } + for name in dex_names.into_iter() { + let mut data = vec![]; + // TODO: It sould not be necessary to clone data, but right now DexFileReader takes a &[u8] + std::io::copy(&mut zip.by_name(&name)?, &mut data)?; + apk.add_dex_file(&name, &data, label_each_ins, cache)?; + } + Ok(apk) + } + + /// Load all android files in an application. + /// This **does not include any .dex file that android would not load. + #[cfg(not(feature = "external-zip-reader"))] + pub fn load_apk(apk: impl Read + Seek, label_each_ins: bool, cache: bool) -> Result { + let mut apk_z = ZipFileReader::new(apk); + let mut apk = Self::default(); + let dex_names = apk_z + .get_classes_file_info() + .into_iter() + .map(|info| info.get_name()) + .collect::>(); + for name in dex_names { + let data = apk_z.read_file_as_vec(&name); + apk.add_dex_file(&name, &data, label_each_ins, cache)?; + } + Ok(apk) + } } #[cfg_attr(feature = "python", pymethods)] @@ -3059,20 +3111,9 @@ impl Apk { /// This **does not include any .dex file that android would not load. #[cfg_attr(feature = "python", staticmethod)] #[cfg_attr(feature = "python", pyo3(signature = (apk, label_each_ins=false, cache=false)))] - pub fn load_apk(apk: PathBuf, label_each_ins: bool, cache: bool) -> Result { + pub fn load_apk_path(apk: PathBuf, label_each_ins: bool, cache: bool) -> Result { let file = File::open(apk)?; - let mut apk_z = ZipFileReader::new(file); - let mut apk = Self::default(); - let dex_names = apk_z - .get_classes_file_info() - .into_iter() - .map(|info| info.get_name()) - .collect::>(); - for name in dex_names { - let data = apk_z.read_file_as_vec(&name); - apk.add_dex_file(&name, &data, label_each_ins, cache)?; - } - Ok(apk) + Self::load_apk(file, label_each_ins, cache) } /// Load all android files in an application. @@ -3080,18 +3121,7 @@ impl Apk { #[cfg_attr(feature = "python", staticmethod)] #[cfg_attr(feature = "python", pyo3(signature = (apk, label_each_ins=false, cache=false)))] pub fn load_apk_bin(apk: &[u8], label_each_ins: bool, cache: bool) -> Result { - let mut apk_z = ZipFileReader::new(Cursor::new(apk)); - let mut apk = Self::default(); - let dex_names = apk_z - .get_classes_file_info() - .into_iter() - .map(|info| info.get_name()) - .collect::>(); - for name in dex_names { - let data = apk_z.read_file_as_vec(&name); - apk.add_dex_file(&name, &data, label_each_ins, cache)?; - } - Ok(apk) + Self::load_apk(Cursor::new(apk), label_each_ins, cache) } // TODO: change cache to true when cache invalidation is setup diff --git a/androscalpel/src/dex_string.rs b/androscalpel/src/dex_string.rs index 26d47c9..5637ac0 100644 --- a/androscalpel/src/dex_string.rs +++ b/androscalpel/src/dex_string.rs @@ -26,6 +26,7 @@ impl VisitableMut for DexString { impl std::fmt::Debug for DexString { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + #[allow(clippy::unnecessary_fallible_conversions)] if let Ok(string) = TryInto::::try_into(self) { f.write_str(&format!( "DexString({}, {:#x})",