chore(page): add safe char handling extensions

spider-rs · Nov 9, 2024 · 1b2c556 · 1b2c556
1 parent 62e5807
commit 1b2c556
Show file tree

Hide file tree

Showing 9 changed files with 38 additions and 33 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.13.6"
+version = "2.13.7"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider/src/page.rs b/spider/src/page.rs
@@ -245,10 +245,17 @@ pub fn push_link<A: PartialEq + Eq + std::hash::Hash + From<String>>(
                 let hchars = abs.path();
 
                 if let Some(position) = hchars.rfind('.') {
-                    let resource_ext = &hchars[position + 1..hchars.len()];
+                    let hlen = hchars.len();
+                    let has_asset = hlen - position;
 
-                    if !ONLY_RESOURCES.contains::<CaseInsensitiveString>(&resource_ext.into()) {
-                        can_process = false;
+                    if has_asset >= 3 {
+                        let next_position = position + 1;
+
+                        if !ONLY_RESOURCES
+                            .contains::<CaseInsensitiveString>(&hchars[next_position..].into())
+                        {
+                            can_process = false;
+                        }
                     }
                 }
 
@@ -673,15 +680,7 @@ impl Page {
                                         let href = auto_encode_bytes(&matched.as_bytes())
                                             .replace(r#"\u002F"#, "/");
 
-                                        fn get_last_segment(path: &str) -> &str {
-                                            if let Some(pos) = path.rfind('/') {
-                                                &path[pos + 1..]
-                                            } else {
-                                                path
-                                            }
-                                        }
-
-                                        let last_segment = get_last_segment(&href);
+                                        let last_segment = crate::utils::get_last_segment(&href);
 
                                         // we can pass in a static map of the dynamic SSG routes pre-hand, custom API endpoint to seed, or etc later.
                                         if !(last_segment.starts_with("[")
@@ -1385,15 +1384,7 @@ impl Page {
                                 let href = auto_encode_bytes(&matched.as_bytes())
                                     .replace(r#"\u002F"#, "/");
 
-                                fn get_last_segment(path: &str) -> &str {
-                                    if let Some(pos) = path.rfind('/') {
-                                        &path[pos + 1..]
-                                    } else {
-                                        path
-                                    }
-                                }
-
-                                let last_segment = get_last_segment(&href);
+                                let last_segment = crate::utils::get_last_segment(&href);
 
                                 // we can pass in a static map of the dynamic SSG routes pre-hand, custom API endpoint to seed, or etc later.
                                 if !(last_segment.starts_with("[") && last_segment.ends_with("]")) {

diff --git a/spider/src/utils/mod.rs b/spider/src/utils/mod.rs
@@ -2853,3 +2853,17 @@ pub(crate) fn modify_selectors(
         }
     }
 }
+
+/// Get the last segment path.
+pub fn get_last_segment(path: &str) -> &str {
+    if let Some(pos) = path.rfind('/') {
+        let next_position = pos + 1;
+        if next_position < path.len() {
+            &path[next_position..]
+        } else {
+            ""
+        }
+    } else {
+        path
+    }
+}
diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.13.6"
+version = "2.13.7"
 rust-version = "1.70"
 authors = [
     "j-mendez <jeff@spider.cloud>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.13.6"
+version = "2.13.7"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.13.6"
+version = "2.13.7"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.13.6"
+version = "2.13.7"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.13.6"
+version = "2.13.7"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]