diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7646990..1e183a46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,76 +70,39 @@ jobs: restore-keys: | ${{ runner.os }}-cargo-build- - # Add disk space cleanup before linting - - name: Check disk space before build - run: df -h - - - name: Aggressive pre-build cleanup - run: | - # Clean package manager caches - sudo apt-get clean - sudo apt-get autoremove -y - - sudo rm -rf /opt/hostedtoolcache - sudo rm -rf /usr/local/.ghcup - sudo rm -rf /opt/hostedtoolcache/ - sudo rm -rf /usr/local/lib/android/sdk/ndk - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf /usr/local/share/boost - rm -rf target/debug/deps - rm -rf target/debug/incremental - - # Clean up unused swap and memory - sudo swapoff -a - sudo swapon -a - - echo "Disk space after aggressive cleanup:" - df -h - - - name: Build and Test run: | gnome-keyring-daemon --components=secrets --daemonize --unlock <<< 'foobar' cargo test working-directory: crates - - name: Lint - run: cargo clippy -- -D warnings - # Add disk space cleanup before linting - name: Check disk space before cleanup run: df -h - - name: Clean up disk space after build + - name: Clean up disk space run: | - echo "Cleaning up disk space after build..." + echo "Cleaning up disk space..." # Remove debug artifacts that are no longer needed after tests - rm -rf target/debug/deps || true - rm -rf target/debug/build || true - rm -rf target/debug/incremental || true - - # Clean Cargo cache - rm -rf ~/.cargo/registry/src || true - rm -rf ~/.cargo/registry/index || true - rm -rf ~/.cargo/git/checkouts || true - rm -rf ~/.cargo/git/db || true - - # Clean package manager caches + rm -rf target/debug/deps + rm -rf target/debug/build + rm -rf target/debug/incremental + # Clean npm cache if it exists npm cache clean --force || true + # Clean apt cache sudo apt-get clean - sudo apt-get autoremove -y - - # Remove Docker images if any + # Remove unnecessary large directories + rm -rf ~/.cargo/registry/index || true + # Remove docker images if any docker system prune -af || true - - # Remove temp files - sudo rm -rf /tmp/* || true - - # Check disk space after all cleanup - echo "Final disk space:" - df -h + # Remove unused packages + sudo apt-get autoremove -y || true + - name: Check disk space after cleanup + run: df -h + + - name: Lint + run: cargo clippy -- -D warnings desktop-lint: name: Lint Electron Desktop App diff --git a/Cargo.lock b/Cargo.lock index dcbf593d..fbb41a55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1075,25 +1075,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e" -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "cast" version = "0.3.0" @@ -1228,7 +1209,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading 0.8.6", + "libloading", ] [[package]] @@ -1469,7 +1450,7 @@ dependencies = [ "bitflags 2.9.0", "core-foundation 0.10.0", "core-graphics-types", - "foreign-types 0.5.0", + "foreign-types", "libc", ] @@ -1493,21 +1474,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.4.2" @@ -1696,12 +1662,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" -[[package]] -name = "deflate64" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" - [[package]] name = "deranged" version = "0.3.11" @@ -1975,26 +1935,6 @@ dependencies = [ "zune-inflate", ] -[[package]] -name = "extractous" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "082fd3334d09f6722e230d3a824e4ebab34bcf88e8d40b40c0bdb806d436d3f4" -dependencies = [ - "bytemuck", - "flate2", - "fs_extra", - "jni", - "libc", - "reqwest 0.12.12", - "strum", - "strum_macros", - "tar", - "thiserror 1.0.69", - "walkdir", - "zip 2.2.3", -] - [[package]] name = "fancy-regex" version = "0.14.0" @@ -2019,7 +1959,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947" dependencies = [ "cfg-if", - "rustix 0.38.44", + "rustix", "windows-sys 0.52.0", ] @@ -2032,18 +1972,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "filetime" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" -dependencies = [ - "cfg-if", - "libc", - "libredox", - "windows-sys 0.59.0", -] - [[package]] name = "flate2" version = "1.1.0" @@ -2060,15 +1988,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared 0.1.1", -] - [[package]] name = "foreign-types" version = "0.5.0" @@ -2076,7 +1995,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared 0.3.1", + "foreign-types-shared", ] [[package]] @@ -2090,12 +2009,6 @@ dependencies = [ "syn 2.0.99", ] -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -2117,12 +2030,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.31" @@ -2500,7 +2407,6 @@ dependencies = [ "chrono", "docx-rs", "etcetera", - "extractous", "google-drive3", "google-sheets4", "http-body-util", @@ -2880,22 +2786,6 @@ dependencies = [ "tokio-io-timeout", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.6.0", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.10" @@ -3302,15 +3192,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" -[[package]] -name = "java-locator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09c46c1fe465c59b1474e665e85e1256c3893dd00927b8d55f63b09044c1e64f" -dependencies = [ - "glob", -] - [[package]] name = "jni" version = "0.21.1" @@ -3320,9 +3201,7 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "java-locator", "jni-sys", - "libloading 0.7.4", "log", "thiserror 1.0.69", "walkdir", @@ -3470,16 +3349,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - [[package]] name = "libloading" version = "0.8.6" @@ -3498,7 +3367,6 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.0", "libc", - "redox_syscall", ] [[package]] @@ -3525,12 +3393,6 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" -[[package]] -name = "linux-raw-sys" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9" - [[package]] name = "litemap" version = "0.7.5" @@ -3598,16 +3460,6 @@ dependencies = [ "weezl", ] -[[package]] -name = "lzma-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" -dependencies = [ - "byteorder", - "crc", -] - [[package]] name = "macro_rules_attribute" version = "0.2.0" @@ -3854,23 +3706,6 @@ dependencies = [ "rand", ] -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework 2.11.1", - "security-framework-sys", - "tempfile", -] - [[package]] name = "ndk-context" version = "0.1.1" @@ -4137,50 +3972,12 @@ version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" -[[package]] -name = "openssl" -version = "0.10.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" -dependencies = [ - "bitflags 2.9.0", - "cfg-if", - "foreign-types 0.3.2", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.99", -] - [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" -[[package]] -name = "openssl-sys" -version = "0.9.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "option-ext" version = "0.2.0" @@ -4253,16 +4050,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest", - "hmac", -] - [[package]] name = "pem" version = "3.0.5" @@ -4888,7 +4675,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration 0.5.1", + "system-configuration", "tokio", "tokio-rustls 0.24.1", "tokio-util", @@ -4914,7 +4701,6 @@ dependencies = [ "cookie", "cookie_store", "encoding_rs", - "futures-channel", "futures-core", "futures-util", "h2 0.4.8", @@ -4923,13 +4709,11 @@ dependencies = [ "http-body-util", "hyper 1.6.0", "hyper-rustls 0.27.5", - "hyper-tls", "hyper-util", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -4941,9 +4725,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.2", - "system-configuration 0.6.1", "tokio", - "tokio-native-tls", "tokio-rustls 0.26.2", "tokio-util", "tower 0.5.2", @@ -5038,20 +4820,7 @@ dependencies = [ "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustix" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825" -dependencies = [ - "bitflags 2.9.0", - "errno", - "libc", - "linux-raw-sys 0.9.2", + "linux-raw-sys", "windows-sys 0.59.0", ] @@ -5605,25 +5374,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.99", -] - [[package]] name = "subtle" version = "2.6.1" @@ -5735,18 +5485,7 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.4", - "system-configuration-sys 0.5.0", -] - -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags 2.9.0", - "core-foundation 0.9.4", - "system-configuration-sys 0.6.0", + "system-configuration-sys", ] [[package]] @@ -5759,16 +5498,6 @@ dependencies = [ "libc", ] -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "system-deps" version = "6.2.2" @@ -5782,17 +5511,6 @@ dependencies = [ "version-compare", ] -[[package]] -name = "tar" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "target-lexicon" version = "0.12.16" @@ -5819,7 +5537,7 @@ dependencies = [ "fastrand", "getrandom 0.3.1", "once_cell", - "rustix 0.38.44", + "rustix", "windows-sys 0.59.0", ] @@ -5838,7 +5556,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9" dependencies = [ - "rustix 0.38.44", + "rustix", "windows-sys 0.59.0", ] @@ -6113,16 +5831,6 @@ dependencies = [ "syn 2.0.99", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -6784,7 +6492,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.44", + "rustix", ] [[package]] @@ -7258,16 +6966,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" -[[package]] -name = "xattr" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" -dependencies = [ - "libc", - "rustix 1.0.2", -] - [[package]] name = "xcap" version = "0.0.14" @@ -7482,27 +7180,15 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b280484c454e74e5fff658bbf7df8fdbe7a07c6b2de4a53def232c15ef138f3a" dependencies = [ - "aes", "arbitrary", - "bzip2", - "constant_time_eq", "crc32fast", "crossbeam-utils", - "deflate64", "displaydoc", "flate2", - "hmac", "indexmap 2.7.1", - "lzma-rs", "memchr", - "pbkdf2", - "rand", - "sha1", "thiserror 2.0.12", - "time", - "zeroize", "zopfli", - "zstd", ] [[package]] diff --git a/crates/goose-mcp/Cargo.toml b/crates/goose-mcp/Cargo.toml index ef6495da..cd718134 100644 --- a/crates/goose-mcp/Cargo.toml +++ b/crates/goose-mcp/Cargo.toml @@ -42,7 +42,6 @@ ignore = "0.4" lopdf = "0.35.0" docx-rs = "0.4.7" image = "0.24.9" -extractous = "0.3.0" umya-spreadsheet = "2.2.3" keyring = { version = "3.6.1", features = ["apple-native", "windows-native", "sync-secret-service"] } diff --git a/crates/goose-mcp/src/computercontroller/document_tool.rs b/crates/goose-mcp/src/computercontroller/document_tool.rs deleted file mode 100644 index d0d28573..00000000 --- a/crates/goose-mcp/src/computercontroller/document_tool.rs +++ /dev/null @@ -1,269 +0,0 @@ -use extractous::Extractor; -use mcp_core::{Content, ToolError}; -use std::{ - fs, - io::Read, - path::{Path, PathBuf}, -}; - -// Threshold for large text files (0.22MB - about 1/18 of the 4,194,304 bytes limit) -const LARGE_TEXT_THRESHOLD: usize = (2 * 1024 * 1024) / 9; // ~0.22MB in bytes - -pub async fn document_tool( - path: &str, - operation: &str, - cache_dir: &Path, -) -> Result, ToolError> { - match operation { - "get_text" => { - // Extract text from a local file (PDF, DOCX, XLSX, etc.) - extract_text_from_file(path, cache_dir) - } - "get_text_url" => { - // Extract text from a URL - extract_text_from_url(path, cache_dir) - } - _ => Err(ToolError::InvalidParameters(format!( - "Invalid operation: {}. Valid operations are: 'get_text', 'get_text_url'", - operation - ))), - } -} - -fn extract_text_from_file(path: &str, cache_dir: &Path) -> Result, ToolError> { - // Use extractous library for text extraction - let extractor = Extractor::new(); - - // Extract text from the file - let (text, metadata) = extractor.extract_file_to_string(path).map_err(|e| { - ToolError::ExecutionError(format!("Failed to extract text from file: {}", e)) - })?; - - process_extracted_text(text, metadata, path, cache_dir) -} - -fn extract_text_from_url(url: &str, cache_dir: &Path) -> Result, ToolError> { - // Validate that the input is actually a URL - if !url.starts_with("http://") && !url.starts_with("https://") { - return Err(ToolError::InvalidParameters(format!( - "Invalid URL: {}. URL must start with http:// or https://", - url - ))); - } - - // Use extractous library for text extraction - let extractor = Extractor::new(); - - // Handle URL extraction - let (mut stream_reader, metadata) = extractor.extract_url(url).map_err(|e| { - ToolError::ExecutionError(format!("Failed to extract text from URL: {}", e)) - })?; - - // Convert StreamReader to String - let mut text = String::new(); - stream_reader - .read_to_string(&mut text) - .map_err(|e| ToolError::ExecutionError(format!("Failed to read text from URL: {}", e)))?; - - process_extracted_text(text, metadata, url, cache_dir) -} - -fn process_extracted_text( - text: String, - metadata: std::collections::HashMap>, - source_path: &str, - cache_dir: &Path, -) -> Result, ToolError> { - // Check if the extracted text is large - let text_size = text.len(); - if text_size > LARGE_TEXT_THRESHOLD { - // Create a directory for large text files if it doesn't exist - let large_text_dir = cache_dir.join("large_document_texts"); - fs::create_dir_all(&large_text_dir).map_err(|e| { - ToolError::ExecutionError(format!("Failed to create directory for large text: {}", e)) - })?; - - // Create a filename based on the original document name - let doc_path = PathBuf::from(source_path); - let doc_filename = doc_path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("unnamed_document"); - - let text_file_path = large_text_dir.join(format!("{}.txt", doc_filename)); - - // Write the text to a file - fs::write(&text_file_path, &text).map_err(|e| { - ToolError::ExecutionError(format!("Failed to write large text to file: {}", e)) - })?; - - // Format size in human-readable form - let size_str = if text_size < 1024 * 1024 { - format!("{:.2} KB", text_size as f64 / 1024.0) - } else { - format!("{:.2} MB", text_size as f64 / (1024.0 * 1024.0)) - }; - - Ok(vec![Content::text(format!( - "Large text extracted from document ({})\n\n\ - The extracted text is too large to display directly.\n\ - Text has been written to: {}\n\n\ - You can search through this file using ripgrep:\n\ - rg 'search term' {}\n\n\ - Or view portions of it:\n\ - head -n 50 {}\n\ - tail -n 50 {}\n\ - less {}", - size_str, - text_file_path.display(), - text_file_path.display(), - text_file_path.display(), - text_file_path.display(), - text_file_path.display() - ))]) - } else { - // Include metadata information in the output - let metadata_info = if metadata.is_empty() { - "Document Metadata: None\n\n".to_string() - } else { - let mut formatted_metadata = String::from("Document Metadata:\n"); - - // Format each metadata entry - for (key, values) in &metadata { - formatted_metadata.push_str(&format!(" {}: ", key)); - - // Single value case - if values.len() == 1 { - formatted_metadata.push_str(&format!("{}\n", values[0])); - continue; - } - - // Multiple values case - formatted_metadata.push_str("[\n"); - for value in values { - formatted_metadata.push_str(&format!(" {}\n", value)); - } - formatted_metadata.push_str(" ]\n"); - } - - formatted_metadata.push('\n'); - formatted_metadata - }; - - Ok(vec![Content::text(format!( - "{}Extracted text from document:\n\n{}", - metadata_info, text - ))]) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::path::PathBuf; - - #[tokio::test] - async fn test_docx_text_extraction() { - let test_docx_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("src/computercontroller/tests/data/sample.docx"); - let cache_dir = tempfile::tempdir().unwrap().into_path(); - - println!( - "Testing text extraction from DOCX: {}", - test_docx_path.display() - ); - - let result = document_tool(test_docx_path.to_str().unwrap(), "get_text", &cache_dir).await; - - assert!(result.is_ok(), "DOCX text extraction should succeed"); - let content = result.unwrap(); - assert!(!content.is_empty(), "Extracted text should not be empty"); - let text = content[0].as_text().unwrap(); - println!("Extracted text:\n{}", text); - assert!( - text.contains("Document Metadata") || !text.is_empty(), - "Should contain metadata or at least some text content" - ); - } - - #[tokio::test] - async fn test_url_text_extraction() { - // Skip this test if we're not online - // This is a simple test URL that should be stable - let test_url = "https://example.com"; - let cache_dir = tempfile::tempdir().unwrap().into_path(); - - println!("Testing text extraction from URL: {}", test_url); - - let result = document_tool(test_url, "get_text_url", &cache_dir).await; - - // If the test fails due to network issues, just skip it - if let Err(err) = &result { - if err.to_string().contains("network") || err.to_string().contains("connection") { - println!("Skipping URL extraction test due to network issues"); - return; - } - } - - assert!(result.is_ok(), "URL text extraction should succeed"); - let content = result.unwrap(); - assert!(!content.is_empty(), "Extracted text should not be empty"); - let text = content[0].as_text().unwrap(); - println!("Extracted text from URL:\n{}", text); - assert!( - text.contains("Example Domain"), - "Should contain expected content from example.com" - ); - } - - #[tokio::test] - async fn test_document_invalid_path() { - let cache_dir = tempfile::tempdir().unwrap().into_path(); - let result = document_tool("nonexistent.pdf", "get_text", &cache_dir).await; - - assert!(result.is_err(), "Should fail with invalid path"); - } - - #[tokio::test] - async fn test_document_invalid_operation() { - let test_pdf_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("src/computercontroller/tests/data/test.pdf"); - let cache_dir = tempfile::tempdir().unwrap().into_path(); - - let result = document_tool( - test_pdf_path.to_str().unwrap(), - "invalid_operation", - &cache_dir, - ) - .await; - - assert!(result.is_err(), "Should fail with invalid operation"); - } - - #[tokio::test] - async fn test_url_with_get_text() { - let test_url = "https://example.com"; - let cache_dir = tempfile::tempdir().unwrap().into_path(); - - let result = document_tool(test_url, "get_text", &cache_dir).await; - - // This should fail since URLs should use get_text_url - assert!(result.is_err(), "Using get_text with URL should fail"); - } - - #[tokio::test] - async fn test_file_with_get_text_url() { - let test_docx_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("src/computercontroller/tests/data/sample.docx"); - let cache_dir = tempfile::tempdir().unwrap().into_path(); - - let result = - document_tool(test_docx_path.to_str().unwrap(), "get_text_url", &cache_dir).await; - - // This should fail since local files should use get_text - assert!( - result.is_err(), - "Using get_text_url with local file should fail" - ); - } -} diff --git a/crates/goose-mcp/src/computercontroller/mod.rs b/crates/goose-mcp/src/computercontroller/mod.rs index f55ad4b4..692605cb 100644 --- a/crates/goose-mcp/src/computercontroller/mod.rs +++ b/crates/goose-mcp/src/computercontroller/mod.rs @@ -19,7 +19,6 @@ use mcp_core::{ use mcp_server::router::CapabilitiesBuilder; use mcp_server::Router; -mod document_tool; mod docx_tool; mod pdf_tool; mod presentation_tool; @@ -68,10 +67,10 @@ impl ComputerControllerRouter { }), ); - let web_fetch_tool = Tool::new( - "web_fetch", + let web_scrape_tool = Tool::new( + "web_scrape", indoc! {r#" - Fetch and save content from a web page using http(s). The content can be saved as: + Fetch and save content from a web page. The content can be saved as: - text (for HTML pages) - json (for API responses) - binary (for images and other files) @@ -123,7 +122,6 @@ impl ComputerControllerRouter { - File Operations: Organize files/folders - Integration: Calendar, reminders, messages - Data: Interact with spreadsheets and documents - - Text: extract content from many file formats Can be combined with screenshot tool for visual task assistance. "#}, @@ -244,10 +242,10 @@ impl ComputerControllerRouter { indoc! {r#" Process PDF files to extract text and images. Supports operations: - - extract_text: Extract all text content from the PDF (file or url to file) + - extract_text: Extract all text content from the PDF - extract_images: Extract and save embedded images to PNG files - Use this when there is a .pdf file or files that need to be processed. + Use this when there is a .pdf file or files that need to be processed. "#}, json!({ "type": "object", @@ -255,7 +253,7 @@ impl ComputerControllerRouter { "properties": { "path": { "type": "string", - "description": "Path to the PDF file or URL to pdf" + "description": "Path to the PDF file" }, "operation": { "type": "string", @@ -266,74 +264,6 @@ impl ComputerControllerRouter { }), ); - // Check if Tesseract OCR is installed - let has_tesseract = match std::env::consts::OS { - "macos" | "linux" => { - let output = std::process::Command::new("which") - .arg("tesseract") - .output() - .map(|output| output.status.success()) - .unwrap_or(false); - output - } - "windows" => { - let output = std::process::Command::new("where") - .arg("tesseract") - .output() - .map(|output| output.status.success()) - .unwrap_or(false); - output - } - _ => false, - }; - - // Conditionally include OCR information in the description - let image_formats_desc = if has_tesseract { - "This will also extract any embedded text via OCR for the following: png, jpeg, tiff, bmp, gif, ico, psd, svg and pdf (use this if there are embedded images in PDF)" - } else { - "metadata only: png, jpeg, tiff, bmp, gif, ico, psd, svg (metadata only, OCR not available as tesseract not installed)" - }; - - let document_tool = Tool::new( - "document_tool", - formatdoc! {r#" - Extract plain text from various file formats. Use this when you see a file extension of the following, - OR a url to treat as a document to get text from. - Formats: - doc, docx, ppt, pptx, xls, xlsx, rtf, odt, ods, odp - (consider using docx and xlsx tools for those first) - csv, tsv - (when not handled by other tools) - html, xml,epub, txt - - {image_formats_desc} - E-Mail: eml, msg, mbox, pst (extracts content, headers, attachments) - - Supports operations: - - get_text: Extract all text content from local document files - - get_text_url: Extract all text content from a document at a URL - - Use this for general text extraction from misc document types. - "#, - image_formats_desc = image_formats_desc - }, - json!({ - "type": "object", - "required": ["path", "operation"], - "properties": { - "path": { - "type": "string", - "description": "Path to the document file or URL to load content from" - }, - "operation": { - "type": "string", - "enum": ["get_text", "get_text_url"], - "description": "Operation to perform on the document" - } - } - }), - ); - let docx_tool = Tool::new( "docx_tool", indoc! {r#" @@ -634,13 +564,14 @@ impl ComputerControllerRouter { {os_instructions} - This extension has many tools to automate, for example: - - web_search, web_fetch, quick_script, computer_control for automation, - pdf_tool (pdfs text), - document_tool (many doc types and URLs), docx_tool, xlsx_tool, make_presentation - - cache of content: + web_search + - Search the web using DuckDuckGo's API for general topics or keywords + web_scrape + - Fetch content from html websites and APIs + - Save as text, JSON, or binary files + - Content is cached locally for later use + - This is not optimised for complex websites, so don't use this as the first tool. + cache - Manage your cached files - List, view, delete files - Clear all cached data @@ -655,12 +586,11 @@ impl ComputerControllerRouter { Self { tools: vec![ web_search_tool, - web_fetch_tool, + web_scrape_tool, quick_script_tool, computer_control_tool, cache_tool, pdf_tool, - document_tool, docx_tool, xlsx_tool, make_presentation_tool, @@ -755,7 +685,7 @@ impl ComputerControllerRouter { ))]) } - async fn web_fetch(&self, params: Value) -> Result, ToolError> { + async fn web_scrape(&self, params: Value) -> Result, ToolError> { let url = params .get("url") .and_then(|v| v.as_str()) @@ -1152,21 +1082,6 @@ impl ComputerControllerRouter { crate::computercontroller::pdf_tool::pdf_tool(path, operation, &self.cache_dir).await } - async fn document_tool(&self, params: Value) -> Result, ToolError> { - let path = params - .get("path") - .and_then(|v| v.as_str()) - .ok_or_else(|| ToolError::InvalidParameters("Missing 'path' parameter".into()))?; - - let operation = params - .get("operation") - .and_then(|v| v.as_str()) - .ok_or_else(|| ToolError::InvalidParameters("Missing 'operation' parameter".into()))?; - - crate::computercontroller::document_tool::document_tool(path, operation, &self.cache_dir) - .await - } - async fn cache(&self, params: Value) -> Result, ToolError> { let command = params .get("command") @@ -1274,12 +1189,11 @@ impl Router for ComputerControllerRouter { Box::pin(async move { match tool_name.as_str() { "web_search" => this.web_search(arguments).await, - "web_fetch" => this.web_fetch(arguments).await, + "web_scrape" => this.web_scrape(arguments).await, "automation_script" => this.quick_script(arguments).await, "computer_control" => this.computer_control(arguments).await, "cache" => this.cache(arguments).await, "pdf_tool" => this.pdf_tool(arguments).await, - "document_tool" => this.document_tool(arguments).await, "docx_tool" => this.docx_tool(arguments).await, "xlsx_tool" => this.xlsx_tool(arguments).await, "make_presentation" => { diff --git a/crates/goose-mcp/src/computercontroller/pdf_tool.rs b/crates/goose-mcp/src/computercontroller/pdf_tool.rs index 83e1e43c..f25dde64 100644 --- a/crates/goose-mcp/src/computercontroller/pdf_tool.rs +++ b/crates/goose-mcp/src/computercontroller/pdf_tool.rs @@ -1,124 +1,117 @@ -use extractous::Extractor; -use lopdf::{Document, Object}; +use lopdf::{content::Content as PdfContent, Document, Object}; use mcp_core::{Content, ToolError}; -use std::{ - fs, - io::Read, - path::{Path, PathBuf}, -}; - -// Threshold for large text files (0.22MB - about 1/18 of the 4,194,304 bytes limit) -const LARGE_TEXT_THRESHOLD: usize = (2 * 1024 * 1024) / 9; // ~0.22MB in bytes +use std::{fs, path::Path}; pub async fn pdf_tool( path: &str, operation: &str, cache_dir: &Path, ) -> Result, ToolError> { - match operation { + // Open and parse the PDF file + let doc = Document::load(path) + .map_err(|e| ToolError::ExecutionError(format!("Failed to open PDF file: {}", e)))?; + + let result = match operation { "extract_text" => { - // Use extractous library for text extraction - let extractor = Extractor::new(); + let mut text = String::new(); - // Check if the path is a URL or a file - let (text, metadata) = if path.starts_with("http://") || path.starts_with("https://") { - // Handle URL extraction - let (mut stream_reader, metadata) = extractor.extract_url(path).map_err(|e| { - ToolError::ExecutionError(format!("Failed to extract text from URL: {}", e)) - })?; + // Iterate over each page in the document + for (page_num, page_id) in doc.get_pages() { + text.push_str(&format!("Page {}:\n", page_num)); - // Convert StreamReader to String - assuming it has a read_to_string method - let mut text = String::new(); - stream_reader.read_to_string(&mut text).map_err(|e| { - ToolError::ExecutionError(format!("Failed to read text from URL: {}", e)) - })?; + // Try to get text from page contents + if let Ok(page_obj) = doc.get_object(page_id) { + if let Ok(page_dict) = page_obj.as_dict() { + // Try to get text from Contents stream + if let Ok(contents) = + page_dict.get(b"Contents").and_then(|c| c.as_reference()) + { + if let Ok(content_obj) = doc.get_object(contents) { + if let Ok(stream) = content_obj.as_stream() { + if let Ok(content_data) = stream.get_plain_content() { + if let Ok(content) = PdfContent::decode(&content_data) { + // Process each operation in the content stream + for operation in content.operations { + match operation.operator.as_ref() { + // "Tj" operator: show text + "Tj" => { + for operand in operation.operands { + if let Object::String(ref bytes, _) = + operand + { + if let Ok(s) = + std::str::from_utf8(bytes) + { + text.push_str(s); + } + } + } + text.push(' '); + } + // "TJ" operator: show text with positioning + "TJ" => { + if let Some(Object::Array(ref arr)) = + operation.operands.first() + { + let mut last_was_text = false; + for element in arr { + match element { + Object::String( + ref bytes, + _, + ) => { + if let Ok(s) = + std::str::from_utf8( + bytes, + ) + { + if last_was_text { + text.push(' '); + } + text.push_str(s); + last_was_text = true; + } + } + Object::Integer(offset) => { + // Large negative offsets often indicate word spacing + if *offset < -100 { + text.push(' '); + last_was_text = false; + } + } + Object::Real(offset) => { + if *offset < -100.0 { + text.push(' '); + last_was_text = false; + } + } + _ => {} + } + } + text.push(' '); + } + } + _ => (), // Ignore other operators + } + } + } + } + } + } + } + } + } + text.push('\n'); + } - (text, metadata) + if text.trim().is_empty() { + "No text found in PDF".to_string() } else { - // Extract text from the file (PDF or other) - extractor.extract_file_to_string(path).map_err(|e| { - ToolError::ExecutionError(format!("Failed to extract text from file: {}", e)) - })? - }; - - // Check if the extracted text is large - let text_size = text.len(); - if text_size > LARGE_TEXT_THRESHOLD { - // Create a directory for large text files if it doesn't exist - let large_text_dir = cache_dir.join("large_pdf_texts"); - fs::create_dir_all(&large_text_dir).map_err(|e| { - ToolError::ExecutionError(format!( - "Failed to create directory for large text: {}", - e - )) - })?; - - // Create a filename based on the original PDF name - let pdf_path = PathBuf::from(path); - let pdf_filename = pdf_path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("unnamed_pdf"); - - let text_file_path = large_text_dir.join(format!("{}.txt", pdf_filename)); - - // Write the text to a file - fs::write(&text_file_path, &text).map_err(|e| { - ToolError::ExecutionError(format!("Failed to write large text to file: {}", e)) - })?; - - // Format size in human-readable form - let size_str = if text_size < 1024 * 1024 { - format!("{:.2} KB", text_size as f64 / 1024.0) - } else { - format!("{:.2} MB", text_size as f64 / (1024.0 * 1024.0)) - }; - - Ok(vec![Content::text(format!( - "Large text extracted from PDF ({})\n\n\ - The extracted text is too large to display directly.\n\ - Text has been written to: {}\n\n\ - You can search through this file using ripgrep:\n\ - rg 'search term' {}\n\n\ - Or view portions of it:\n\ - head -n 50 {}\n\ - tail -n 50 {}\n\ - less {}", - size_str, - text_file_path.display(), - text_file_path.display(), - text_file_path.display(), - text_file_path.display(), - text_file_path.display() - ))]) - } else { - // Include metadata information in the output - let metadata_info = format!( - "PDF Metadata:\n{}\n\n", - serde_json::to_string_pretty(&metadata) - .unwrap_or_else(|_| "Unable to format metadata".to_string()) - ); - - Ok(vec![Content::text(format!( - "{}Extracted text from PDF:\n\n{}", - metadata_info, text - ))]) + format!("Extracted text from PDF:\n\n{}", text) } } "extract_images" => { - // Check if the path is a URL (not supported for image extraction) - if path.starts_with("http://") || path.starts_with("https://") { - return Err(ToolError::InvalidParameters( - "Image extraction is not supported for URLs. Please provide a local PDF file path.".to_string(), - )); - } - - // Open and parse the PDF file for image extraction - let doc = Document::load(path).map_err(|e| { - ToolError::ExecutionError(format!("Failed to open PDF file: {}", e)) - })?; - let cache_dir = cache_dir.join("pdf_images"); fs::create_dir_all(&cache_dir).map_err(|e| { ToolError::ExecutionError(format!("Failed to create image cache directory: {}", e)) @@ -312,21 +305,21 @@ pub async fn pdf_tool( } if images.is_empty() { - Ok(vec![Content::text("No images found in PDF".to_string())]) + "No images found in PDF".to_string() } else { - Ok(vec![Content::text(format!( - "Found {} images:\n{}", - image_count, - images.join("\n") - ))]) + format!("Found {} images:\n{}", image_count, images.join("\n")) } } - _ => Err(ToolError::InvalidParameters(format!( - "Invalid operation: {}. Valid operations are: 'extract_text', 'extract_images'", - operation - ))), - } + _ => { + return Err(ToolError::InvalidParameters(format!( + "Invalid operation: {}. Valid operations are: 'extract_text', 'extract_images'", + operation + ))) + } + }; + + Ok(vec![Content::text(result)]) } #[cfg(test)] @@ -349,39 +342,10 @@ mod tests { assert!(!content.is_empty(), "Extracted text should not be empty"); let text = content[0].as_text().unwrap(); println!("Extracted text:\n{}", text); + assert!(text.contains("Page 1"), "Should contain page marker"); assert!( - text.contains("This is a test PDF") || text.contains("PDF Metadata"), - "Should contain expected test content or metadata" - ); - } - - #[tokio::test] - async fn test_url_text_extraction() { - // Skip this test if we're not online - // This is a simple test URL that should be stable - let test_url = "https://example.com"; - let cache_dir = tempfile::tempdir().unwrap().into_path(); - - println!("Testing text extraction from URL: {}", test_url); - - let result = pdf_tool(test_url, "extract_text", &cache_dir).await; - - // If the test fails due to network issues, just skip it - if let Err(err) = &result { - if err.to_string().contains("network") || err.to_string().contains("connection") { - println!("Skipping URL extraction test due to network issues"); - return; - } - } - - assert!(result.is_ok(), "URL text extraction should succeed"); - let content = result.unwrap(); - assert!(!content.is_empty(), "Extracted text should not be empty"); - let text = content[0].as_text().unwrap(); - println!("Extracted text from URL:\n{}", text); - assert!( - text.contains("Example Domain"), - "Should contain expected content from example.com" + text.contains("This is a test PDF"), + "Should contain expected test content" ); } @@ -432,29 +396,6 @@ mod tests { } } - #[tokio::test] - async fn test_url_image_extraction_fails() { - // Test that image extraction from URLs is properly rejected - let test_url = "https://example.com"; - let cache_dir = tempfile::tempdir().unwrap().into_path(); - - println!( - "Testing image extraction from URL (should fail): {}", - test_url - ); - - let result = pdf_tool(test_url, "extract_images", &cache_dir).await; - assert!(result.is_err(), "URL image extraction should fail"); - - let error = result.unwrap_err(); - assert!( - error - .to_string() - .contains("Image extraction is not supported for URLs"), - "Should return the correct error message for URL image extraction" - ); - } - #[tokio::test] async fn test_pdf_invalid_path() { let cache_dir = tempfile::tempdir().unwrap().into_path(); @@ -478,65 +419,4 @@ mod tests { assert!(result.is_err(), "Should fail with invalid operation"); } - - #[tokio::test] - async fn test_large_pdf_text_extraction() { - let large_pdf_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("src/computercontroller/tests/data/visa-rules-public.pdf"); - - // Skip test if the large PDF file doesn't exist (may not be committed to git) - if !large_pdf_path.exists() { - println!( - "Skipping large PDF test as file doesn't exist: {}", - large_pdf_path.display() - ); - return; - } - - let cache_dir = tempfile::tempdir().unwrap().into_path(); - - println!( - "Testing large text extraction from: {}", - large_pdf_path.display() - ); - - let result = pdf_tool(large_pdf_path.to_str().unwrap(), "extract_text", &cache_dir).await; - - assert!(result.is_ok(), "Large PDF text extraction should succeed"); - let content = result.unwrap(); - assert!(!content.is_empty(), "Extracted text should not be empty"); - let text = content[0].as_text().unwrap(); - - // Check if the text is large enough to be written to a file - if text.contains("Large text extracted from PDF") { - // For large PDFs, we should get the message about writing to a file - assert!( - text.contains("Text has been written to:"), - "Should indicate where text was written" - ); - - // Extract the file path from the output and verify it exists - let file_path = text - .lines() - .find(|line| line.contains("Text has been written to:")) - .and_then(|line| line.split(": ").nth(1)) - .expect("Should have a valid file path"); - - println!("Verifying text file exists: {}", file_path); - assert!(PathBuf::from(file_path).exists(), "Text file should exist"); - - // Verify file contains actual content - let file_content = - fs::read_to_string(file_path).expect("Should be able to read text file"); - assert!(!file_content.is_empty(), "Text file should not be empty"); - } else { - // If the text is not written to a file, it should contain PDF content directly - assert!( - text.contains("PDF Metadata:"), - "Should contain PDF metadata" - ); - // The text should not be empty (beyond just metadata) - assert!(text.len() > 100, "Should contain substantial text content"); - } - } } diff --git a/crates/goose-mcp/src/computercontroller/tests/data/visa-rules-public.pdf b/crates/goose-mcp/src/computercontroller/tests/data/visa-rules-public.pdf deleted file mode 100644 index 458e79a6..00000000 Binary files a/crates/goose-mcp/src/computercontroller/tests/data/visa-rules-public.pdf and /dev/null differ