feat: prune some deps

tests/parser/file/test_html_parser.py

@@ -1,16 +1,19 @@
 import pytest
-from pathlib import Path
-from unittest.mock import patch, MagicMock
-
-import sys
-import types
 
 from application.parser.file.html_parser import HTMLParser
 
 
+HTML = (
+    "<html><head><title>My Page</title></head>"
+    "<body><h1>Heading</h1><p>Hello world.</p></body></html>"
+)
+
+
 @pytest.fixture
-def html_parser():
-    return HTMLParser()
+def html_file(tmp_path):
+    path = tmp_path / "page.html"
+    path.write_text(HTML)
+    return path
 
 
 def test_html_init_parser():
@@ -21,23 +24,33 @@ def test_html_init_parser():
     assert parser.parser_config_set
 
 
-def test_html_parser_parse_file():
-    parser = HTMLParser()
-    mock_doc = MagicMock()
-    mock_doc.page_content = "Extracted HTML content"
-    mock_doc.metadata = {"source": "test.html"}
+def test_html_parser_extracts_text(html_file):
+    text = HTMLParser().parse_file(html_file)
+    assert "Heading" in text
+    assert "Hello world." in text
 
-    fake_lc = types.ModuleType("langchain_community")
-    fake_dl = types.ModuleType("langchain_community.document_loaders")
 
-    bshtml_mock = MagicMock(return_value=MagicMock(load=MagicMock(return_value=[mock_doc])))
-    fake_dl.BSHTMLLoader = bshtml_mock
-    fake_lc.document_loaders = fake_dl
+def test_html_parser_returns_plain_text(html_file):
+    """bulk.py stringifies parser output, so it must not be a repr.
 
-    with patch.dict(sys.modules, {
-        "langchain_community": fake_lc,
-        "langchain_community.document_loaders": fake_dl,
-    }):
-        result = parser.parse_file(Path("test.html"))
-        assert result == [mock_doc]
-        bshtml_mock.assert_called_once_with(Path("test.html"))
+    The previous langchain BSHTMLLoader returned a Document whose ``str()`` is
+    ``page_content='...' metadata={...}``, which leaked into indexed content.
+    """
+    text = HTMLParser().parse_file(html_file)
+    assert isinstance(text, str)
+    assert "page_content" not in str(text)
+    assert "metadata=" not in str(text)
+
+
+def test_html_parser_reports_title_metadata(html_file):
+    assert HTMLParser().get_file_metadata(html_file) == {"title": "My Page"}
+
+
+def test_html_parser_metadata_without_title(tmp_path):
+    path = tmp_path / "no_title.html"
+    path.write_text("<html><body><p>Just text.</p></body></html>")
+    assert HTMLParser().get_file_metadata(path) == {}
+
+
+def test_html_parser_metadata_unreadable_file(tmp_path):
+    assert HTMLParser().get_file_metadata(tmp_path / "missing.html") == {}

tests/parser/remote/test_reddit_loader.py

@@ -1,83 +1,103 @@
 import json
-from unittest.mock import patch, MagicMock
+import sys
+import types
+from unittest.mock import MagicMock, patch
+
 import pytest
 
 from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
 
 
+def _post(post_id, title, body):
+    post = MagicMock()
+    post.id = post_id
+    post.title = title
+    post.selftext = body
+    post.subreddit_name_prefixed = "r/python"
+    post.score = 42
+    post.url = f"https://reddit.com/{post_id}"
+    post.author = "someone"
+    return post
+
+
+def _fake_praw(posts):
+    """Install a stub ``praw`` module whose listings yield ``posts``."""
+    praw = types.ModuleType("praw")
+    reddit = MagicMock()
+    listing = MagicMock()
+    listing.new.return_value = posts
+    listing.hot.return_value = posts
+    reddit.subreddit.return_value = listing
+    reddit.redditor.return_value = MagicMock(submissions=listing)
+    praw.Reddit = MagicMock(return_value=reddit)
+    return praw, reddit, listing
+
+
+BASE_PAYLOAD = {
+    "client_id": "cid",
+    "client_secret": "csecret",
+    "user_agent": "ua",
+    "search_queries": ["python"],
+}
+
+
+@pytest.mark.unit
 class TestRedditPostsLoaderRemote:
     def test_invalid_json_raises(self):
-        loader = RedditPostsLoaderRemote()
-        with pytest.raises(ValueError) as exc:
-            loader.load_data("not a json")
-        assert "Invalid JSON input" in str(exc.value)
+        with pytest.raises(ValueError, match="Invalid JSON input"):
+            RedditPostsLoaderRemote().load_data("not a json")
 
     def test_missing_required_fields_raises(self):
-        loader = RedditPostsLoaderRemote()
-        payload = json.dumps({"client_id": "id"})
-        with pytest.raises(ValueError) as exc:
-            loader.load_data(payload)
-        assert "Missing required fields" in str(exc.value)
+        with pytest.raises(ValueError, match="Missing required fields") as exc:
+            RedditPostsLoaderRemote().load_data(json.dumps({"client_id": "id"}))
         assert "client_secret" in str(exc.value)
 
-    @patch("application.parser.remote.reddit_loader.RedditPostsLoader")
-    def test_constructs_loader_and_loads_with_defaults(self, MockRedditLoader):
-        loader = RedditPostsLoaderRemote()
+    def test_invalid_mode_raises(self):
+        payload = {**BASE_PAYLOAD, "mode": "comments"}
+        praw, _, _ = _fake_praw([])
+        with patch.dict(sys.modules, {"praw": praw}):
+            with pytest.raises(ValueError, match="username.*subreddit|subreddit"):
+                RedditPostsLoaderRemote().load_data(json.dumps(payload))
 
-        instance = MagicMock()
-        docs = [MagicMock(), MagicMock()]
-        instance.load.return_value = docs
-        MockRedditLoader.return_value = instance
+    def test_subreddit_mode_returns_documents(self):
+        praw, reddit, listing = _fake_praw([_post("p1", "First", "Body one")])
+        with patch.dict(sys.modules, {"praw": praw}):
+            docs = RedditPostsLoaderRemote().load_data(json.dumps(BASE_PAYLOAD))
 
-        payload = {
-            "client_id": "cid",
-            "client_secret": "csecret",
-            "user_agent": "ua",
-            "search_queries": ["r/langchain"],
-        }
-
-        result = loader.load_data(json.dumps(payload))
-
-        MockRedditLoader.assert_called_once_with(
-            client_id="cid",
-            client_secret="csecret",
-            user_agent="ua",
-            categories=["new", "hot"],
-            mode="subreddit",
-            search_queries=["r/langchain"],
-            number_posts=10,
+        praw.Reddit.assert_called_once_with(
+            client_id="cid", client_secret="csecret", user_agent="ua"
         )
-        instance.load.assert_called_once()
-        assert result == docs
+        reddit.subreddit.assert_called_with("python")
+        # Default categories are new + hot, so the single post arrives twice.
+        assert len(docs) == 2
+        assert docs[0].text == "Body one"
+        assert docs[0].doc_id == "p1"
+        assert docs[0].extra_info["title"] == "First"
+        assert docs[0].extra_info["source"] == "https://reddit.com/p1"
+        assert docs[0].extra_info["post_category"] == "new"
 
-    @patch("application.parser.remote.reddit_loader.RedditPostsLoader")
-    def test_constructs_loader_and_loads_with_overrides(self, MockRedditLoader):
-        loader = RedditPostsLoaderRemote()
+    def test_documents_convert_to_vector_format(self):
+        """The remote base class calls to_vector_format on every result."""
+        praw, _, _ = _fake_praw([_post("p1", "First", "Body one")])
+        with patch.dict(sys.modules, {"praw": praw}):
+            docs = RedditPostsLoaderRemote().load_data(json.dumps(BASE_PAYLOAD))
+        converted = docs[0].to_vector_format()
+        assert converted.page_content == "Body one"
+        assert converted.metadata["title"] == "First"
 
-        instance = MagicMock()
-        instance.load.return_value = []
-        MockRedditLoader.return_value = instance
+    def test_username_mode_uses_redditor_submissions(self):
+        payload = {**BASE_PAYLOAD, "mode": "username", "categories": ["new"]}
+        praw, reddit, _ = _fake_praw([_post("p2", "Second", "Body two")])
+        with patch.dict(sys.modules, {"praw": praw}):
+            docs = RedditPostsLoaderRemote().load_data(json.dumps(payload))
 
-        payload = {
-            "client_id": "cid",
-            "client_secret": "csecret",
-            "user_agent": "ua",
-            "search_queries": ["python"],
-            "categories": ["hot"],
-            "mode": "comments",
-            "number_posts": 3,
-        }
-
-        loader.load_data(json.dumps(payload))
-
-        MockRedditLoader.assert_called_once_with(
-            client_id="cid",
-            client_secret="csecret",
-            user_agent="ua",
-            categories=["hot"],
-            mode="comments",
-            search_queries=["python"],
-            number_posts=3,
-        )
-        instance.load.assert_called_once()
+        reddit.redditor.assert_called_with("python")
+        assert len(docs) == 1 and docs[0].text == "Body two"
 
+    def test_number_posts_and_categories_are_forwarded(self):
+        payload = {**BASE_PAYLOAD, "categories": ["hot"], "number_posts": 3}
+        praw, _, listing = _fake_praw([])
+        with patch.dict(sys.modules, {"praw": praw}):
+            RedditPostsLoaderRemote().load_data(json.dumps(payload))
+        listing.hot.assert_called_once_with(limit=3)
+        listing.new.assert_not_called()

tests/parser/remote/test_web_loader.py

@@ -5,7 +5,7 @@ from urllib.parse import urlparse
 from application.core.url_validation import SSRFError
 from application.parser.remote.web_loader import WebLoader, headers
 from application.parser.schema.base import Document
-from langchain_core.documents import Document as LCDocument
+from application.vectorstore.document_class import Document as LCDocument
 
 
 def _mock_validate_url(url):
@@ -301,13 +301,13 @@ class TestWebLoaderIntegration:
 
     @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
     @patch("application.parser.remote.web_loader.pinned_request")
-    def test_load_langchain_documents_method(self, mock_pinned_request, mock_validate, web_loader):
+    def test_load_vector_documents_method(self, mock_pinned_request, mock_validate, web_loader):
         mock_pinned_request.return_value = _fake_response(
             "<html><head><title>Test Page</title></head>"
             "<body><p>Test web page content</p></body></html>"
         )
 
-        result = web_loader.load_langchain_documents(inputs="https://example.com")
+        result = web_loader.load_vector_documents(inputs="https://example.com")
 
         assert len(result) == 1
         assert isinstance(result[0], LCDocument)

tests/parser/test_chunking_strategies.py

@@ -115,7 +115,7 @@ class TestParentChild:
     def test_parent_text_reaches_vectorstore_metadata(self):
         chunker = ParentChildChunker(max_tokens=80, min_tokens=20)
         out = chunker.chunk([Document(text="beta " * 150, doc_id="d")])
-        lc = out[0].to_langchain_format()
+        lc = out[0].to_vector_format()
         # parent_text must survive the langchain conversion into metadata.
         assert "parent_text" in lc.metadata
         assert lc.metadata["parent_text"]

tests/parser/test_schema.py

@@ -73,30 +73,30 @@ class TestDocumentBase:
         with pytest.raises(ValueError, match="text field not set"):
             Document(text=None)
 
-    def test_document_to_langchain_format(self):
-        """Cover line 27: Document.to_langchain_format converts correctly."""
+    def test_document_to_vector_format(self):
+        """Cover line 27: Document.to_vector_format converts correctly."""
         from application.parser.schema.base import Document
 
         doc = Document(text="hello world", extra_info={"source": "test"})
-        lc_doc = doc.to_langchain_format()
+        lc_doc = doc.to_vector_format()
         assert lc_doc.page_content == "hello world"
         assert lc_doc.metadata == {"source": "test"}
 
-    def test_document_to_langchain_format_no_extra_info(self):
-        """Cover: to_langchain_format with no extra_info uses empty dict."""
+    def test_document_to_vector_format_no_extra_info(self):
+        """Cover: to_vector_format with no extra_info uses empty dict."""
         from application.parser.schema.base import Document
 
         doc = Document(text="hello")
-        lc_doc = doc.to_langchain_format()
+        lc_doc = doc.to_vector_format()
         assert lc_doc.metadata == {}
 
-    def test_document_from_langchain_format(self):
-        """Cover line 34: Document.from_langchain_format creates Document."""
+    def test_document_from_vector_format(self):
+        """Cover line 34: Document.from_vector_format creates Document."""
         from application.parser.schema.base import Document
-        from langchain_core.documents import Document as LCDocument
+        from application.vectorstore.document_class import Document as LCDocument
 
         lc_doc = LCDocument(page_content="test content", metadata={"key": "val"})
-        doc = Document.from_langchain_format(lc_doc)
+        doc = Document.from_vector_format(lc_doc)
         assert doc.text == "test content"
         assert doc.extra_info == {"key": "val"}