chore: more tests

tests/parser/file/test_bulk.py

@@ -0,0 +1,367 @@
+"""Comprehensive tests for application/parser/file/bulk.py
+
+Covers: SimpleDirectoryReader (init, file discovery, load_data, directory
+structure building), get_default_file_extractor.
+"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from application.parser.schema.base import Document
+
+
+# =====================================================================
+# Helpers
+# =====================================================================
+
+
+@pytest.fixture
+def temp_dir(tmp_path):
+    """Create a temporary directory with test files."""
+    (tmp_path / "file1.md").write_text("# Heading\n\nContent 1")
+    (tmp_path / "file2.txt").write_text("Plain text content")
+    (tmp_path / ".hidden").write_text("hidden file")
+    sub = tmp_path / "subdir"
+    sub.mkdir()
+    (sub / "file3.md").write_text("Nested content")
+    return tmp_path
+
+
+@pytest.fixture
+def temp_dir_with_types(tmp_path):
+    """Directory with multiple file types."""
+    (tmp_path / "doc.md").write_text("markdown")
+    (tmp_path / "data.json").write_text('{"key": "value"}')
+    (tmp_path / "notes.txt").write_text("text")
+    return tmp_path
+
+
+# =====================================================================
+# SimpleDirectoryReader - Init
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestSimpleDirectoryReaderInit:
+
+    def test_init_with_dir(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        reader = SimpleDirectoryReader(input_dir=str(temp_dir))
+        assert len(reader.input_files) >= 2
+
+    def test_init_with_files(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        files = [str(temp_dir / "file1.md")]
+        reader = SimpleDirectoryReader(input_files=files)
+        assert len(reader.input_files) == 1
+
+    def test_init_requires_input(self):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        with pytest.raises(ValueError, match="Must provide"):
+            SimpleDirectoryReader()
+
+    def test_exclude_hidden(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        reader = SimpleDirectoryReader(input_dir=str(temp_dir), exclude_hidden=True)
+        filenames = [f.name for f in reader.input_files]
+        assert ".hidden" not in filenames
+
+    def test_include_hidden(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        reader = SimpleDirectoryReader(input_dir=str(temp_dir), exclude_hidden=False)
+        filenames = [f.name for f in reader.input_files]
+        assert ".hidden" in filenames
+
+    def test_recursive(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        reader = SimpleDirectoryReader(input_dir=str(temp_dir), recursive=True)
+        filenames = [f.name for f in reader.input_files]
+        assert "file3.md" in filenames
+
+    def test_non_recursive(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        reader = SimpleDirectoryReader(input_dir=str(temp_dir), recursive=False)
+        filenames = [f.name for f in reader.input_files]
+        assert "file3.md" not in filenames
+
+    def test_required_exts(self, temp_dir_with_types):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir_with_types), required_exts=[".md"]
+        )
+        filenames = [f.name for f in reader.input_files]
+        assert "doc.md" in filenames
+        assert "data.json" not in filenames
+        assert "notes.txt" not in filenames
+
+    def test_required_exts_case_insensitive(self, tmp_path):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        (tmp_path / "FILE.MD").write_text("content")
+        reader = SimpleDirectoryReader(
+            input_dir=str(tmp_path), required_exts=[".md"]
+        )
+        assert len(reader.input_files) == 1
+
+    def test_num_files_limit(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir), num_files_limit=1, recursive=False
+        )
+        assert len(reader.input_files) <= 1
+
+    def test_custom_file_extractor(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        mock_parser = MagicMock()
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir),
+            file_extractor={".md": mock_parser},
+        )
+        assert ".md" in reader.file_extractor
+
+
+# =====================================================================
+# SimpleDirectoryReader - load_data
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestSimpleDirectoryReaderLoadData:
+
+    def test_load_data_returns_documents(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = True
+        mock_parser.parse_file.return_value = "parsed content"
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir),
+            file_extractor={".md": mock_parser, ".txt": mock_parser},
+            recursive=False,
+            exclude_hidden=True,
+        )
+        docs = reader.load_data()
+        assert len(docs) >= 1
+        for doc in docs:
+            assert isinstance(doc, Document)
+
+    def test_load_data_concatenate(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = True
+        mock_parser.parse_file.return_value = "content"
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir),
+            file_extractor={".md": mock_parser, ".txt": mock_parser},
+            recursive=False,
+            exclude_hidden=True,
+        )
+        docs = reader.load_data(concatenate=True)
+        assert len(docs) == 1
+
+    def test_load_data_with_file_metadata(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        def custom_metadata(filename):
+            return {"custom_key": f"meta_{filename}"}
+
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = True
+        mock_parser.parse_file.return_value = "parsed"
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir),
+            file_extractor={".md": mock_parser, ".txt": mock_parser},
+            file_metadata=custom_metadata,
+            recursive=False,
+            exclude_hidden=True,
+        )
+        docs = reader.load_data()
+        assert len(docs) >= 1
+        for doc in docs:
+            assert doc.extra_info is not None
+            assert "custom_key" in doc.extra_info
+
+    def test_load_data_inits_parser_if_not_set(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = False
+        mock_parser.parse_file.return_value = "content"
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir),
+            file_extractor={".md": mock_parser, ".txt": mock_parser},
+            recursive=False,
+            exclude_hidden=True,
+        )
+        reader.load_data()
+        mock_parser.init_parser.assert_called()
+
+    def test_load_data_standard_read_for_unknown_ext(self, tmp_path):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        (tmp_path / "file.xyz").write_text("xyz content")
+        reader = SimpleDirectoryReader(
+            input_dir=str(tmp_path),
+            file_extractor={},
+        )
+        docs = reader.load_data()
+        assert len(docs) == 1
+        assert "xyz content" in docs[0].text
+
+    def test_load_data_list_return_from_parser(self, tmp_path):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        (tmp_path / "multi.md").write_text("content")
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = True
+        mock_parser.parse_file.return_value = ["part1", "part2"]
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(tmp_path),
+            file_extractor={".md": mock_parser},
+        )
+        docs = reader.load_data()
+        assert len(docs) == 2
+
+    def test_load_data_tracks_token_counts(self, tmp_path):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        (tmp_path / "test.md").write_text("hello world")
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = True
+        mock_parser.parse_file.return_value = "hello world"
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(tmp_path),
+            file_extractor={".md": mock_parser},
+        )
+        reader.load_data()
+        assert hasattr(reader, "file_token_counts")
+        assert len(reader.file_token_counts) >= 1
+
+
+# =====================================================================
+# Directory Structure Building
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestBuildDirectoryStructure:
+
+    def test_builds_structure(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = True
+        mock_parser.parse_file.return_value = "content"
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir),
+            file_extractor={".md": mock_parser, ".txt": mock_parser},
+            exclude_hidden=True,
+        )
+        reader.load_data()
+        assert hasattr(reader, "directory_structure")
+        assert isinstance(reader.directory_structure, dict)
+
+    def test_structure_contains_files_and_dirs(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = True
+        mock_parser.parse_file.return_value = "content"
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir),
+            file_extractor={".md": mock_parser, ".txt": mock_parser},
+            exclude_hidden=True,
+        )
+        reader.load_data()
+        struct = reader.directory_structure
+        # Should contain subdir
+        assert "subdir" in struct
+        # Files should have metadata
+        for key, val in struct.items():
+            if isinstance(val, dict) and "type" in val:
+                assert "size_bytes" in val
+
+    def test_structure_excludes_hidden(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = True
+        mock_parser.parse_file.return_value = "c"
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_dir=str(temp_dir),
+            file_extractor={".md": mock_parser, ".txt": mock_parser},
+            exclude_hidden=True,
+        )
+        reader.load_data()
+        assert ".hidden" not in reader.directory_structure
+
+    def test_no_structure_without_input_dir(self, temp_dir):
+        from application.parser.file.bulk import SimpleDirectoryReader
+
+        files = [str(temp_dir / "file1.md")]
+        mock_parser = MagicMock()
+        mock_parser.parser_config_set = True
+        mock_parser.parse_file.return_value = "content"
+        mock_parser.get_file_metadata.return_value = {}
+
+        reader = SimpleDirectoryReader(
+            input_files=files,
+            file_extractor={".md": mock_parser},
+        )
+        reader.load_data()
+        assert reader.directory_structure == {}
+
+
+# =====================================================================
+# get_default_file_extractor
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestGetDefaultFileExtractor:
+
+    def test_returns_dict(self):
+        from application.parser.file.bulk import get_default_file_extractor
+
+        with patch.dict("sys.modules", {"docling": None, "docling.document_converter": None}):
+            result = get_default_file_extractor()
+            assert isinstance(result, dict)
+            assert ".pdf" in result
+
+    def test_fallback_parsers_on_import_error(self):
+        with patch(
+            "application.parser.file.bulk.get_default_file_extractor"
+        ) as mock_fn:
+            mock_fn.return_value = {".pdf": MagicMock(), ".md": MagicMock()}
+            result = mock_fn()
+            assert ".pdf" in result

tests/parser/file/test_docling_parser.py

@@ -0,0 +1,382 @@
+"""Comprehensive tests for application/parser/file/docling_parser.py
+
+Covers: DoclingParser (init, _init_parser, _get_ocr_options, _export_content,
+parse_file), subclass initialization, error handling.
+"""
+
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+# =====================================================================
+# DoclingParser - Init
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestDoclingParserInit:
+
+    def test_default_init(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser()
+        assert parser.ocr_enabled is True
+        assert parser.table_structure is True
+        assert parser.export_format == "markdown"
+        assert parser.use_rapidocr is True
+        assert parser.ocr_languages == ["english"]
+        assert parser.force_full_page_ocr is False
+        assert parser._converter is None
+
+    def test_custom_init(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(
+            ocr_enabled=False,
+            table_structure=False,
+            export_format="text",
+            use_rapidocr=False,
+            ocr_languages=["german"],
+            force_full_page_ocr=True,
+        )
+        assert parser.ocr_enabled is False
+        assert parser.table_structure is False
+        assert parser.export_format == "text"
+        assert parser.use_rapidocr is False
+        assert parser.ocr_languages == ["german"]
+        assert parser.force_full_page_ocr is True
+
+
+# =====================================================================
+# Init Parser
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestDoclingParserInitParser:
+
+    def test_init_parser_raises_without_docling(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser()
+
+        with patch("importlib.util.find_spec", return_value=None):
+            with pytest.raises(ImportError, match="docling is required"):
+                parser._init_parser()
+
+    def test_init_parser_success(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser()
+
+        mock_converter = MagicMock()
+        with patch("importlib.util.find_spec", return_value=MagicMock()), \
+             patch.object(parser, "_create_converter", return_value=mock_converter):
+            result = parser._init_parser()
+
+            assert isinstance(result, dict)
+            assert result["ocr_enabled"] is True
+            assert result["table_structure"] is True
+            assert parser._converter is mock_converter
+
+
+# =====================================================================
+# Get OCR Options
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestGetOCROptions:
+
+    def test_returns_none_when_rapidocr_disabled(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(use_rapidocr=False)
+        assert parser._get_ocr_options() is None
+
+    def test_returns_options_when_available(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(use_rapidocr=True, ocr_languages=["english"])
+
+        mock_options = MagicMock()
+        with patch(
+            "application.parser.file.docling_parser.DoclingParser._get_ocr_options",
+            return_value=mock_options,
+        ):
+            result = parser._get_ocr_options()
+            assert result is mock_options
+
+    def test_returns_none_on_import_error(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(use_rapidocr=True)
+
+        # Simulate the ImportError path
+        original = parser._get_ocr_options
+
+        def patched_get_ocr():
+            try:
+                raise ImportError("No RapidOcrOptions")
+            except ImportError:
+                return None
+
+        parser._get_ocr_options = patched_get_ocr
+        assert parser._get_ocr_options() is None
+        parser._get_ocr_options = original
+
+
+# =====================================================================
+# Export Content
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestExportContent:
+
+    def test_export_markdown(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(export_format="markdown")
+        mock_doc = MagicMock()
+        mock_doc.export_to_markdown.return_value = "# Title\n\nContent here"
+        mock_doc.texts = []
+
+        result = parser._export_content(mock_doc)
+        assert "# Title" in result
+        mock_doc.export_to_markdown.assert_called_once()
+
+    def test_export_html(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(export_format="html")
+        mock_doc = MagicMock()
+        mock_doc.export_to_html.return_value = "<h1>Title</h1>"
+        mock_doc.texts = []
+
+        result = parser._export_content(mock_doc)
+        assert "<h1>" in result
+
+    def test_export_text(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(export_format="text")
+        mock_doc = MagicMock()
+        mock_doc.export_to_text.return_value = "Plain text content"
+        mock_doc.texts = []
+
+        result = parser._export_content(mock_doc)
+        assert "Plain text" in result
+
+    def test_fallback_to_texts_on_minimal_content(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(export_format="markdown")
+        mock_doc = MagicMock()
+        mock_doc.export_to_markdown.return_value = "<!-- image -->"
+
+        text1 = MagicMock()
+        text1.text = "OCR extracted text 1"
+        text2 = MagicMock()
+        text2.text = "OCR extracted text 2"
+        mock_doc.texts = [text1, text2]
+
+        result = parser._export_content(mock_doc)
+        assert "OCR extracted text 1" in result
+        assert "OCR extracted text 2" in result
+
+    def test_no_fallback_for_substantial_content(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(export_format="markdown")
+        mock_doc = MagicMock()
+        mock_doc.export_to_markdown.return_value = "A" * 100
+        mock_doc.texts = []
+
+        result = parser._export_content(mock_doc)
+        assert result == "A" * 100
+
+    def test_fallback_skipped_when_no_texts(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(export_format="markdown")
+        mock_doc = MagicMock()
+        mock_doc.export_to_markdown.return_value = "short"
+        mock_doc.texts = []
+
+        result = parser._export_content(mock_doc)
+        assert result == "short"
+
+    def test_fallback_skips_empty_texts(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser(export_format="markdown")
+        mock_doc = MagicMock()
+        mock_doc.export_to_markdown.return_value = ""
+
+        empty_text = MagicMock()
+        empty_text.text = ""
+        mock_doc.texts = [empty_text]
+
+        result = parser._export_content(mock_doc)
+        assert result == ""
+
+
+# =====================================================================
+# Parse File
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestDoclingParserParseFile:
+
+    def test_parse_file_success(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser()
+
+        mock_converter = MagicMock()
+        mock_result = MagicMock()
+        mock_doc = MagicMock()
+        mock_doc.export_to_markdown.return_value = "Parsed document content"
+        mock_doc.texts = []
+        mock_result.document = mock_doc
+        mock_converter.convert.return_value = mock_result
+        parser._converter = mock_converter
+
+        result = parser.parse_file(Path("test.pdf"))
+        assert "Parsed document content" in result
+
+    def test_parse_file_inits_converter_on_first_call(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser()
+        parser._converter = None
+
+        mock_converter = MagicMock()
+        mock_result = MagicMock()
+        mock_doc = MagicMock()
+        mock_doc.export_to_markdown.return_value = "content"
+        mock_doc.texts = []
+        mock_result.document = mock_doc
+        mock_converter.convert.return_value = mock_result
+
+        with patch.object(parser, "_init_parser") as mock_init:
+            parser._converter = mock_converter
+            mock_init.return_value = {}
+            result = parser.parse_file(Path("test.pdf"))
+            assert "content" in result
+
+    def test_parse_file_error_ignore(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser()
+        mock_converter = MagicMock()
+        mock_converter.convert.side_effect = Exception("Parse failed")
+        parser._converter = mock_converter
+
+        result = parser.parse_file(Path("bad.pdf"), errors="ignore")
+        assert "Error" in result
+
+    def test_parse_file_error_raise(self):
+        from application.parser.file.docling_parser import DoclingParser
+
+        parser = DoclingParser()
+        mock_converter = MagicMock()
+        mock_converter.convert.side_effect = Exception("Parse failed")
+        parser._converter = mock_converter
+
+        with pytest.raises(Exception, match="Parse failed"):
+            parser.parse_file(Path("bad.pdf"), errors="strict")
+
+
+# =====================================================================
+# Subclass Init
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestDoclingSubclasses:
+
+    def test_pdf_parser_init(self):
+        from application.parser.file.docling_parser import DoclingPDFParser
+
+        parser = DoclingPDFParser()
+        assert parser.ocr_enabled is True
+        assert parser.export_format == "markdown"
+
+    def test_pdf_parser_custom_ocr(self):
+        from application.parser.file.docling_parser import DoclingPDFParser
+
+        parser = DoclingPDFParser(ocr_enabled=False, force_full_page_ocr=True)
+        assert parser.ocr_enabled is False
+        assert parser.force_full_page_ocr is True
+
+    def test_docx_parser_init(self):
+        from application.parser.file.docling_parser import DoclingDocxParser
+
+        parser = DoclingDocxParser()
+        assert parser.export_format == "markdown"
+
+    def test_pptx_parser_init(self):
+        from application.parser.file.docling_parser import DoclingPPTXParser
+
+        parser = DoclingPPTXParser()
+        assert parser.export_format == "markdown"
+
+    def test_xlsx_parser_init(self):
+        from application.parser.file.docling_parser import DoclingXLSXParser
+
+        parser = DoclingXLSXParser()
+        assert parser.table_structure is True
+
+    def test_html_parser_init(self):
+        from application.parser.file.docling_parser import DoclingHTMLParser
+
+        parser = DoclingHTMLParser()
+        assert parser.export_format == "markdown"
+
+    def test_image_parser_init(self):
+        from application.parser.file.docling_parser import DoclingImageParser
+
+        parser = DoclingImageParser()
+        assert parser.ocr_enabled is True
+        assert parser.force_full_page_ocr is True
+
+    def test_image_parser_custom(self):
+        from application.parser.file.docling_parser import DoclingImageParser
+
+        parser = DoclingImageParser(ocr_enabled=False)
+        assert parser.ocr_enabled is False
+
+    def test_csv_parser_init(self):
+        from application.parser.file.docling_parser import DoclingCSVParser
+
+        parser = DoclingCSVParser()
+        assert parser.table_structure is True
+
+    def test_markdown_parser_init(self):
+        from application.parser.file.docling_parser import DoclingMarkdownParser
+
+        parser = DoclingMarkdownParser()
+        assert parser.export_format == "markdown"
+
+    def test_asciidoc_parser_init(self):
+        from application.parser.file.docling_parser import DoclingAsciiDocParser
+
+        parser = DoclingAsciiDocParser()
+        assert parser.export_format == "markdown"
+
+    def test_vtt_parser_init(self):
+        from application.parser.file.docling_parser import DoclingVTTParser
+
+        parser = DoclingVTTParser()
+        assert parser.export_format == "markdown"
+
+    def test_xml_parser_init(self):
+        from application.parser.file.docling_parser import DoclingXMLParser
+
+        parser = DoclingXMLParser()
+        assert parser.export_format == "markdown"

tests/parser/file/test_docs_parser.py

@@ -1,117 +1,187 @@
-import pytest
+"""Comprehensive tests for application/parser/file/docs_parser.py
+
+Covers: PDFParser (init, parse with pypdf, parse as image, import error),
+DocxParser (init, parse, import error).
+"""
+
 from pathlib import Path
-from unittest.mock import patch, MagicMock
+from unittest.mock import MagicMock, patch, mock_open
+
+import pytest
 
 from application.parser.file.docs_parser import PDFParser, DocxParser
 
 
-@pytest.fixture
-def pdf_parser():
-    return PDFParser()
+# =====================================================================
+# PDFParser - Init
+# =====================================================================
 
 
-@pytest.fixture
-def docx_parser():
-    return DocxParser()
+@pytest.mark.unit
+class TestPDFParserInit:
+
+    def test_init_parser(self):
+        parser = PDFParser()
+        result = parser._init_parser()
+        assert isinstance(result, dict)
+        assert result == {}
+
+    def test_parser_config_not_set_initially(self):
+        parser = PDFParser()
+        assert not parser.parser_config_set
+
+    def test_parser_config_set_after_init(self):
+        parser = PDFParser()
+        parser.init_parser()
+        assert parser.parser_config_set
 
 
-def test_pdf_init_parser():
-    parser = PDFParser()
-    assert isinstance(parser._init_parser(), dict)
-    assert not parser.parser_config_set
-    parser.init_parser()
-    assert parser.parser_config_set
+# =====================================================================
+# PDFParser - Parse File
+# =====================================================================
 
 
-def test_docx_init_parser():
-    parser = DocxParser()
-    assert isinstance(parser._init_parser(), dict)
-    assert not parser.parser_config_set
-    parser.init_parser()
-    assert parser.parser_config_set
+@pytest.mark.unit
+class TestPDFParserParse:
+
+    @patch("application.parser.file.docs_parser.settings")
+    def test_parse_with_pypdf(self, mock_settings):
+        mock_settings.PARSE_PDF_AS_IMAGE = False
+
+        parser = PDFParser()
+
+        mock_page1 = MagicMock()
+        mock_page1.extract_text.return_value = "Page 1 content"
+        mock_page2 = MagicMock()
+        mock_page2.extract_text.return_value = "Page 2 content"
+
+        mock_reader = MagicMock()
+        mock_reader.pages = [mock_page1, mock_page2]
+
+        with patch("application.parser.file.docs_parser.PdfReader",
+                   create=True), \
+             patch("builtins.open", mock_open()):
+            # Need to patch the import inside the function
+            import sys
+            mock_pypdf = MagicMock()
+            mock_pypdf.PdfReader = MagicMock(return_value=mock_reader)
+            sys.modules["pypdf"] = mock_pypdf
+
+            try:
+                result = parser.parse_file(Path("test.pdf"))
+                assert "Page 1 content" in result
+                assert "Page 2 content" in result
+            finally:
+                del sys.modules["pypdf"]
+
+    @patch("application.parser.file.docs_parser.settings")
+    @patch("application.parser.file.docs_parser.requests")
+    def test_parse_as_image(self, mock_requests, mock_settings):
+        mock_settings.PARSE_PDF_AS_IMAGE = True
+
+        mock_response = MagicMock()
+        mock_response.json.return_value = {"markdown": "# OCR Result"}
+        mock_requests.post.return_value = mock_response
+
+        parser = PDFParser()
+
+        with patch("builtins.open", mock_open(read_data=b"fake pdf")):
+            result = parser.parse_file(Path("test.pdf"))
+            assert result == "# OCR Result"
+
+    @patch("application.parser.file.docs_parser.settings")
+    def test_parse_raises_on_missing_pypdf(self, mock_settings):
+        mock_settings.PARSE_PDF_AS_IMAGE = False
+
+        parser = PDFParser()
+
+        # Simulate the import error path
+        original = parser.parse_file
+
+        def mock_parse(*args, **kwargs):
+            raise ValueError("pypdf is required to read PDF files.")
+
+        parser.parse_file = mock_parse
+
+        try:
+            with pytest.raises(ValueError, match="pypdf is required"):
+                parser.parse_file(Path("test.pdf"))
+        finally:
+            parser.parse_file = original
 
 
-@patch("application.parser.file.docs_parser.settings")
-def test_parse_pdf_with_pypdf(mock_settings, pdf_parser):
-    mock_settings.PARSE_PDF_AS_IMAGE = False
-
-    # Create mock pages with text content
-    mock_page1 = MagicMock()
-    mock_page1.extract_text.return_value = "Test PDF content page 1"
-    mock_page2 = MagicMock()
-    mock_page2.extract_text.return_value = "Test PDF content page 2"
-
-    mock_reader_instance = MagicMock()
-    mock_reader_instance.pages = [mock_page1, mock_page2]
-
-    original_parse_file = pdf_parser.parse_file
-
-    def mock_parse_file(*args, **kwargs):
-        _ = args, kwargs
-        text_list = []
-        num_pages = len(mock_reader_instance.pages)
-        for page_index in range(num_pages):
-            page = mock_reader_instance.pages[page_index]
-            page_text = page.extract_text()
-            text_list.append(page_text)
-        text = "\n".join(text_list)
-        return text
-
-    pdf_parser.parse_file = mock_parse_file
-
-    try:
-        result = pdf_parser.parse_file(Path("test.pdf"))
-        assert result == "Test PDF content page 1\nTest PDF content page 2"
-    finally:
-        pdf_parser.parse_file = original_parse_file
+# =====================================================================
+# DocxParser - Init
+# =====================================================================
 
 
-@patch("application.parser.file.docs_parser.settings")
-def test_parse_pdf_pypdf_import_error(mock_settings, pdf_parser):
-    mock_settings.PARSE_PDF_AS_IMAGE = False
+@pytest.mark.unit
+class TestDocxParserInit:
 
-    original_parse_file = pdf_parser.parse_file
+    def test_init_parser(self):
+        parser = DocxParser()
+        result = parser._init_parser()
+        assert isinstance(result, dict)
+        assert result == {}
 
-    def mock_parse_file(*args, **kwargs):
-        _ = args, kwargs
-        raise ValueError("pypdf is required to read PDF files.")
+    def test_parser_config_not_set_initially(self):
+        parser = DocxParser()
+        assert not parser.parser_config_set
 
-    pdf_parser.parse_file = mock_parse_file
-
-    try:
-        with pytest.raises(ValueError, match="pypdf is required to read PDF files"):
-            pdf_parser.parse_file(Path("test.pdf"))
-    finally:
-        pdf_parser.parse_file = original_parse_file
+    def test_parser_config_set_after_init(self):
+        parser = DocxParser()
+        parser.init_parser()
+        assert parser.parser_config_set
 
 
-def test_parse_docx(docx_parser):
-    original_parse_file = docx_parser.parse_file
-
-    def mock_parse_file(*args, **kwargs):
-        _ = args, kwargs
-        return "Test DOCX content"
-
-    docx_parser.parse_file = mock_parse_file
-
-    try:
-        result = docx_parser.parse_file(Path("test.docx"))
-        assert result == "Test DOCX content"
-    finally:
-        docx_parser.parse_file = original_parse_file
+# =====================================================================
+# DocxParser - Parse File
+# =====================================================================
 
 
-def test_parse_docx_import_error(docx_parser):
-    original_parse_file = docx_parser.parse_file
+@pytest.mark.unit
+class TestDocxParserParse:
 
-    def mock_parse_file(*args, **kwargs):
-        _ = args, kwargs
-        raise ValueError("docx2txt is required to read Microsoft Word files.")
+    def test_parse_file_success(self):
+        parser = DocxParser()
 
-    docx_parser.parse_file = mock_parse_file
+        import sys
+        mock_docx2txt = MagicMock()
+        mock_docx2txt.process.return_value = "DOCX content here"
+        sys.modules["docx2txt"] = mock_docx2txt
 
-    try:
-        with pytest.raises(ValueError, match="docx2txt is required to read Microsoft Word files"):
-            docx_parser.parse_file(Path("test.docx"))
-    finally:
-        docx_parser.parse_file = original_parse_file
+        try:
+            result = parser.parse_file(Path("test.docx"))
+            assert result == "DOCX content here"
+        finally:
+            del sys.modules["docx2txt"]
+
+    def test_parse_raises_on_missing_docx2txt(self):
+        parser = DocxParser()
+
+        original = parser.parse_file
+
+        def mock_parse(*args, **kwargs):
+            raise ValueError("docx2txt is required to read Microsoft Word files.")
+
+        parser.parse_file = mock_parse
+
+        try:
+            with pytest.raises(ValueError, match="docx2txt is required"):
+                parser.parse_file(Path("test.docx"))
+        finally:
+            parser.parse_file = original
+
+
+# =====================================================================
+# BaseParser properties
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestBaseParserProperties:
+
+    def test_get_file_metadata_default(self):
+        parser = PDFParser()
+        meta = parser.get_file_metadata(Path("test.pdf"))
+        assert meta == {}

tests/parser/remote/test_github_loader.py

@@ -116,6 +116,159 @@ class TestGitHubLoaderLoadData:
 
 
 
+class TestGitHubLoaderIsTextFile:
+    def test_known_extension(self):
+        loader = GitHubLoader()
+        assert loader.is_text_file("app.py") is True
+        assert loader.is_text_file("data.json") is True
+
+    def test_unknown_extension_with_text_mime(self):
+        loader = GitHubLoader()
+        assert loader.is_text_file("file.xml") is True
+
+    def test_binary_file(self):
+        loader = GitHubLoader()
+        assert loader.is_text_file("image.png") is False
+
+    @patch("application.parser.remote.github_loader.mimetypes.guess_type")
+    def test_mime_fallback_text(self, mock_mime):
+        mock_mime.return_value = ("text/plain", None)
+        loader = GitHubLoader()
+        assert loader.is_text_file("unknownfile.xyz") is True
+
+
+class TestGitHubLoaderMakeRequest:
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_success(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response({"ok": True}, 200)
+        resp = loader._make_request("http://example.com")
+        assert resp.status_code == 200
+
+    @patch("application.parser.remote.github_loader.time.sleep")
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_rate_limit_retry(self, mock_get, mock_sleep):
+        loader = GitHubLoader()
+        rate_resp = MagicMock()
+        rate_resp.status_code = 403
+        rate_resp.json.return_value = {"message": "API rate limit exceeded"}
+        rate_resp.headers = {
+            "X-RateLimit-Remaining": "0",
+            "X-RateLimit-Reset": "9999999",
+        }
+        ok_resp = make_response({"ok": True}, 200)
+        mock_get.side_effect = [rate_resp, ok_resp]
+
+        resp = loader._make_request("http://example.com", max_retries=2)
+        assert resp.status_code == 200
+        mock_sleep.assert_called_once()
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_rate_limit_exhausted(self, mock_get):
+        loader = GitHubLoader()
+        rate_resp = MagicMock()
+        rate_resp.status_code = 403
+        rate_resp.json.return_value = {"message": "API rate limit exceeded"}
+        rate_resp.headers = {
+            "X-RateLimit-Remaining": "0",
+            "X-RateLimit-Reset": "9999",
+        }
+        mock_get.return_value = rate_resp
+
+        with pytest.raises(Exception, match="rate limit exceeded"):
+            loader._make_request("http://example.com", max_retries=1)
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_403_non_rate_limit(self, mock_get):
+        loader = GitHubLoader()
+        resp = MagicMock()
+        resp.status_code = 403
+        resp.json.return_value = {"message": "Forbidden - need auth"}
+        resp.headers = {"X-RateLimit-Remaining": "50", "X-RateLimit-Reset": "9999"}
+        mock_get.return_value = resp
+
+        with pytest.raises(Exception, match="GitHub API error"):
+            loader._make_request("http://example.com", max_retries=1)
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_other_error_raises(self, mock_get):
+        loader = GitHubLoader()
+        resp = make_response(
+            status_code=500,
+            raise_error=requests.HTTPError("Server Error"),
+        )
+        mock_get.return_value = resp
+
+        with pytest.raises(requests.HTTPError):
+            loader._make_request("http://example.com", max_retries=1)
+
+
+class TestGitHubLoaderFetchRepoFilesErrors:
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_api_error_message_in_dict(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response(
+            {"message": "Not Found"}, 200
+        )
+
+        with pytest.raises(Exception, match="GitHub API error"):
+            loader.fetch_repo_files("owner/repo")
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_non_list_response(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response("not a list", 200)
+
+        with pytest.raises(TypeError, match="Expected list"):
+            loader.fetch_repo_files("owner/repo")
+
+
+class TestGitHubLoaderFetchFileContentEdgeCases:
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_empty_base64_text_returns_none(self, mock_get):
+        loader = GitHubLoader()
+        b64 = base64.b64encode(b"").decode("utf-8")
+        mock_get.return_value = make_response(
+            {"encoding": "base64", "content": b64}
+        )
+        result = loader.fetch_file_content("owner/repo", "empty.py")
+        assert result is None
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_empty_non_base64_returns_none(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response(
+            {"encoding": "none", "content": "   "}
+        )
+        result = loader.fetch_file_content("owner/repo", "empty.txt")
+        assert result is None
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_decode_failure_returns_none(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response(
+            {"encoding": "base64", "content": "invalid!!base64"}
+        )
+        result = loader.fetch_file_content("owner/repo", "broken.py")
+        assert result is None
+
+
+class TestGitHubLoaderLoadDataSkipsNone:
+    def test_skips_binary_files(self, monkeypatch):
+        loader = GitHubLoader()
+        monkeypatch.setattr(
+            loader, "fetch_repo_files", lambda repo, path="": ["a.py", "b.png"]
+        )
+
+        def fake_content(repo, fp):
+            return "code" if fp == "a.py" else None
+
+        monkeypatch.setattr(loader, "fetch_file_content", fake_content)
+        docs = loader.load_data("https://github.com/o/r")
+        assert len(docs) == 1
+        assert docs[0].doc_id == "a.py"
+
+
 class TestGitHubLoaderRobustness:
     @patch("application.parser.remote.github_loader.requests.get")
     def test_fetch_repo_files_non_json_raises(self, mock_get):

tests/parser/remote/test_sitemap_loader.py

@@ -0,0 +1,306 @@
+"""Comprehensive tests for application/parser/remote/sitemap_loader.py
+
+Covers: SitemapLoader (init, load_data, _extract_urls, _is_sitemap,
+_parse_sitemap, URL validation, error handling).
+"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+import requests
+
+from application.parser.remote.sitemap_loader import SitemapLoader
+
+
+# =====================================================================
+# SitemapLoader - Init
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestSitemapLoaderInit:
+
+    def test_default_limit(self):
+        loader = SitemapLoader()
+        assert loader.limit == 20
+
+    def test_custom_limit(self):
+        loader = SitemapLoader(limit=5)
+        assert loader.limit == 5
+
+    def test_has_loader_class(self):
+        loader = SitemapLoader()
+        assert loader.loader is not None
+
+
+# =====================================================================
+# _is_sitemap
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestIsSitemap:
+
+    def test_xml_content_type(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "application/xml"}
+        response.url = "https://example.com/sitemap.xml"
+        response.text = ""
+        assert loader._is_sitemap(response) is True
+
+    def test_xml_url_extension(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/sitemap.xml"
+        response.text = ""
+        assert loader._is_sitemap(response) is True
+
+    def test_sitemapindex_in_body(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/sitemap"
+        response.text = "<sitemapindex><sitemap></sitemap></sitemapindex>"
+        assert loader._is_sitemap(response) is True
+
+    def test_urlset_in_body(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/page"
+        response.text = "<urlset><url></url></urlset>"
+        assert loader._is_sitemap(response) is True
+
+    def test_regular_page(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/about"
+        response.text = "<html><body>About us</body></html>"
+        assert loader._is_sitemap(response) is False
+
+    def test_text_xml_content_type(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/xml; charset=utf-8"}
+        response.url = "https://example.com/feed"
+        response.text = ""
+        assert loader._is_sitemap(response) is True
+
+
+# =====================================================================
+# _parse_sitemap
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestParseSitemap:
+
+    def test_parse_basic_sitemap(self):
+        loader = SitemapLoader()
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://example.com/page1</loc></url>
+            <url><loc>https://example.com/page2</loc></url>
+        </urlset>"""
+
+        urls = loader._parse_sitemap(sitemap_xml)
+        assert "https://example.com/page1" in urls
+        assert "https://example.com/page2" in urls
+
+    def test_parse_nested_sitemap(self):
+        loader = SitemapLoader()
+
+        parent_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+        <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <sitemap>
+                <loc>https://example.com/sitemap-child.xml</loc>
+            </sitemap>
+        </sitemapindex>"""
+
+        with patch.object(
+            loader, "_extract_urls",
+            return_value=["https://example.com/page1"]
+        ):
+            urls = loader._parse_sitemap(parent_xml)
+            assert "https://example.com/page1" in urls
+
+    def test_parse_empty_sitemap(self):
+        loader = SitemapLoader()
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        </urlset>"""
+
+        urls = loader._parse_sitemap(sitemap_xml)
+        assert urls == []
+
+
+# =====================================================================
+# _extract_urls
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestExtractUrls:
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    @patch("application.parser.remote.sitemap_loader.requests.get")
+    def test_extract_urls_from_sitemap(self, mock_get, mock_validate):
+        loader = SitemapLoader()
+
+        response = MagicMock()
+        response.headers = {"Content-Type": "application/xml"}
+        response.url = "https://example.com/sitemap.xml"
+        response.text = "<urlset><url><loc>https://example.com/p</loc></url></urlset>"
+        response.content = b"""<?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://example.com/p</loc></url>
+        </urlset>"""
+        mock_get.return_value = response
+
+        urls = loader._extract_urls("https://example.com/sitemap.xml")
+        assert "https://example.com/p" in urls
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    @patch("application.parser.remote.sitemap_loader.requests.get")
+    def test_extract_urls_not_sitemap(self, mock_get, mock_validate):
+        loader = SitemapLoader()
+
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/page"
+        response.text = "<html>Normal page</html>"
+        mock_get.return_value = response
+
+        urls = loader._extract_urls("https://example.com/page")
+        assert urls == ["https://example.com/page"]
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    @patch("application.parser.remote.sitemap_loader.requests.get")
+    def test_extract_urls_http_error(self, mock_get, mock_validate):
+        loader = SitemapLoader()
+        mock_get.side_effect = requests.exceptions.HTTPError("404")
+
+        urls = loader._extract_urls("https://example.com/missing")
+        assert urls == []
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    @patch("application.parser.remote.sitemap_loader.requests.get")
+    def test_extract_urls_connection_error(self, mock_get, mock_validate):
+        loader = SitemapLoader()
+        mock_get.side_effect = requests.exceptions.ConnectionError()
+
+        urls = loader._extract_urls("https://example.com/bad")
+        assert urls == []
+
+    def test_extract_urls_ssrf_blocked(self):
+        from application.core.url_validation import SSRFError
+
+        loader = SitemapLoader()
+
+        with patch(
+            "application.parser.remote.sitemap_loader.validate_url",
+            side_effect=SSRFError("blocked"),
+        ):
+            urls = loader._extract_urls("http://169.254.169.254/")
+            assert urls == []
+
+
+# =====================================================================
+# load_data
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestSitemapLoaderLoadData:
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    def test_load_data_success(self, mock_validate):
+        loader = SitemapLoader(limit=10)
+
+        mock_doc = MagicMock()
+        mock_loader_instance = MagicMock()
+        mock_loader_instance.load.return_value = [mock_doc]
+        loader.loader = MagicMock(return_value=mock_loader_instance)
+
+        with patch.object(
+            loader, "_extract_urls",
+            return_value=["https://example.com/page1"]
+        ):
+            docs = loader.load_data("https://example.com/sitemap.xml")
+            assert len(docs) == 1
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    def test_load_data_no_urls(self, mock_validate):
+        loader = SitemapLoader()
+
+        with patch.object(loader, "_extract_urls", return_value=[]):
+            docs = loader.load_data("https://example.com/empty-sitemap.xml")
+            assert docs == []
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    def test_load_data_list_input(self, mock_validate):
+        loader = SitemapLoader()
+
+        mock_loader_instance = MagicMock()
+        mock_loader_instance.load.return_value = [MagicMock()]
+        loader.loader = MagicMock(return_value=mock_loader_instance)
+
+        with patch.object(
+            loader, "_extract_urls",
+            return_value=["https://example.com/page1"]
+        ):
+            docs = loader.load_data(["https://example.com/sitemap.xml"])
+            assert len(docs) == 1
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    def test_load_data_respects_limit(self, mock_validate):
+        loader = SitemapLoader(limit=2)
+
+        mock_loader_instance = MagicMock()
+        mock_loader_instance.load.return_value = [MagicMock()]
+        loader.loader = MagicMock(return_value=mock_loader_instance)
+
+        urls = [f"https://example.com/page{i}" for i in range(10)]
+        with patch.object(loader, "_extract_urls", return_value=urls):
+            docs = loader.load_data("https://example.com/sitemap.xml")
+            assert len(docs) == 2
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    def test_load_data_handles_url_error(self, mock_validate):
+        loader = SitemapLoader()
+        loader.loader = MagicMock(side_effect=Exception("Load failed"))
+
+        with patch.object(
+            loader, "_extract_urls",
+            return_value=["https://example.com/broken"]
+        ):
+            docs = loader.load_data("https://example.com/sitemap.xml")
+            assert docs == []
+
+    def test_load_data_ssrf_blocked(self):
+        from application.core.url_validation import SSRFError
+
+        loader = SitemapLoader()
+
+        with patch(
+            "application.parser.remote.sitemap_loader.validate_url",
+            side_effect=SSRFError("blocked"),
+        ):
+            docs = loader.load_data("http://169.254.169.254/")
+            assert docs == []
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    def test_load_data_no_limit(self, mock_validate):
+        loader = SitemapLoader(limit=None)
+
+        mock_loader_instance = MagicMock()
+        mock_loader_instance.load.return_value = [MagicMock()]
+        loader.loader = MagicMock(return_value=mock_loader_instance)
+
+        urls = [f"https://example.com/page{i}" for i in range(5)]
+        with patch.object(loader, "_extract_urls", return_value=urls):
+            docs = loader.load_data("https://example.com/sitemap.xml")
+            assert len(docs) == 5

tests/parser/test_chunking.py

@@ -0,0 +1,279 @@
+"""Comprehensive tests for application/parser/chunking.py
+
+Covers: Chunker (init, separate_header_and_body, split_document,
+classic_chunk, chunk), edge cases, token counting.
+"""
+
+import pytest
+
+from application.parser.chunking import Chunker
+from application.parser.schema.base import Document
+
+
+# =====================================================================
+# Chunker - Init
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestChunkerInit:
+
+    def test_default_init(self):
+        chunker = Chunker()
+        assert chunker.chunking_strategy == "classic_chunk"
+        assert chunker.max_tokens == 2000
+        assert chunker.min_tokens == 150
+        assert chunker.duplicate_headers is False
+
+    def test_custom_init(self):
+        chunker = Chunker(
+            chunking_strategy="classic_chunk",
+            max_tokens=1000,
+            min_tokens=50,
+            duplicate_headers=True,
+        )
+        assert chunker.max_tokens == 1000
+        assert chunker.min_tokens == 50
+        assert chunker.duplicate_headers is True
+
+    def test_invalid_strategy_raises(self):
+        with pytest.raises(ValueError, match="Unsupported chunking strategy"):
+            Chunker(chunking_strategy="unknown_strategy")
+
+
+# =====================================================================
+# Separate Header and Body
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestSeparateHeaderAndBody:
+
+    def test_with_header(self):
+        chunker = Chunker()
+        text = "line1\nline2\nline3\nbody content here"
+        header, body = chunker.separate_header_and_body(text)
+        assert "line1" in header
+        assert "line2" in header
+        assert "line3" in header
+        assert "body content here" in body
+
+    def test_without_header(self):
+        chunker = Chunker()
+        text = "short"
+        header, body = chunker.separate_header_and_body(text)
+        assert header == ""
+        assert body == "short"
+
+    def test_empty_text(self):
+        chunker = Chunker()
+        header, body = chunker.separate_header_and_body("")
+        assert header == ""
+        assert body == ""
+
+    def test_exactly_three_lines(self):
+        chunker = Chunker()
+        text = "line1\nline2\nline3\n"
+        header, body = chunker.separate_header_and_body(text)
+        assert header == "line1\nline2\nline3\n"
+        assert body == ""
+
+
+# =====================================================================
+# Split Document
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestSplitDocument:
+
+    def test_split_large_document(self):
+        chunker = Chunker(max_tokens=50, min_tokens=5)
+        long_text = "word " * 200
+        doc = Document(text=long_text, doc_id="doc1")
+
+        result = chunker.split_document(doc)
+        assert len(result) > 1
+        for split_doc in result:
+            assert split_doc.doc_id.startswith("doc1-")
+            assert split_doc.extra_info is not None
+            assert "token_count" in split_doc.extra_info
+
+    def test_split_preserves_header_on_first(self):
+        chunker = Chunker(max_tokens=50, min_tokens=5, duplicate_headers=False)
+        text = "h1\nh2\nh3\n" + "word " * 200
+        doc = Document(text=text, doc_id="doc1")
+
+        result = chunker.split_document(doc)
+        assert len(result) > 1
+        # First chunk should contain header
+        assert "h1" in result[0].text
+
+    def test_split_duplicates_header(self):
+        chunker = Chunker(max_tokens=50, min_tokens=5, duplicate_headers=True)
+        text = "h1\nh2\nh3\n" + "word " * 200
+        doc = Document(text=text, doc_id="doc1")
+
+        result = chunker.split_document(doc)
+        assert len(result) > 1
+        # First chunk should contain header
+        assert "h1" in result[0].text
+
+    def test_split_preserves_embedding(self):
+        chunker = Chunker(max_tokens=50, min_tokens=5)
+        doc = Document(
+            text="word " * 200,
+            doc_id="doc1",
+            embedding=[0.1, 0.2],
+        )
+
+        result = chunker.split_document(doc)
+        for split_doc in result:
+            assert split_doc.embedding == [0.1, 0.2]
+
+    def test_split_preserves_extra_info(self):
+        chunker = Chunker(max_tokens=50, min_tokens=5)
+        doc = Document(
+            text="word " * 200,
+            doc_id="doc1",
+            extra_info={"source": "test"},
+        )
+
+        result = chunker.split_document(doc)
+        for split_doc in result:
+            assert split_doc.extra_info["source"] == "test"
+            assert "token_count" in split_doc.extra_info
+
+
+# =====================================================================
+# Classic Chunk
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestClassicChunk:
+
+    def test_small_doc_passes_through(self):
+        chunker = Chunker(max_tokens=2000, min_tokens=1)
+        doc = Document(text="Short text", doc_id="d1")
+
+        result = chunker.classic_chunk([doc])
+        assert len(result) == 1
+        assert result[0].extra_info is not None
+        assert "token_count" in result[0].extra_info
+
+    def test_large_doc_gets_split(self):
+        chunker = Chunker(max_tokens=50, min_tokens=5)
+        doc = Document(text="word " * 200, doc_id="d1")
+
+        result = chunker.classic_chunk([doc])
+        assert len(result) > 1
+
+    def test_medium_doc_within_range(self):
+        chunker = Chunker(max_tokens=2000, min_tokens=5)
+        doc = Document(text="Hello " * 50, doc_id="d1")
+
+        result = chunker.classic_chunk([doc])
+        assert len(result) == 1
+
+    def test_multiple_docs(self):
+        chunker = Chunker(max_tokens=2000, min_tokens=1)
+        docs = [
+            Document(text="Doc 1 content", doc_id="d1"),
+            Document(text="Doc 2 content", doc_id="d2"),
+        ]
+
+        result = chunker.classic_chunk(docs)
+        assert len(result) == 2
+
+    def test_empty_docs_list(self):
+        chunker = Chunker()
+        result = chunker.classic_chunk([])
+        assert result == []
+
+    def test_very_small_doc_below_min(self):
+        chunker = Chunker(max_tokens=2000, min_tokens=500)
+        doc = Document(text="tiny", doc_id="d1")
+
+        result = chunker.classic_chunk([doc])
+        assert len(result) == 1
+        assert result[0].extra_info["token_count"] < 500
+
+    def test_existing_extra_info_preserved(self):
+        chunker = Chunker(max_tokens=2000, min_tokens=1)
+        doc = Document(
+            text="Hello world",
+            doc_id="d1",
+            extra_info={"source": "test"},
+        )
+
+        result = chunker.classic_chunk([doc])
+        assert result[0].extra_info["source"] == "test"
+        assert "token_count" in result[0].extra_info
+
+    def test_none_extra_info_initialized(self):
+        chunker = Chunker(max_tokens=2000, min_tokens=1)
+        doc = Document(text="Hello", doc_id="d1", extra_info=None)
+
+        result = chunker.classic_chunk([doc])
+        assert result[0].extra_info is not None
+        assert "token_count" in result[0].extra_info
+
+
+# =====================================================================
+# Chunk (dispatcher)
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestChunkDispatcher:
+
+    def test_dispatch_classic_chunk(self):
+        chunker = Chunker(chunking_strategy="classic_chunk")
+        doc = Document(text="content", doc_id="d1")
+
+        result = chunker.chunk([doc])
+        assert len(result) == 1
+
+    def test_dispatch_unknown_raises(self):
+        chunker = Chunker()
+        chunker.chunking_strategy = "nonexistent"
+
+        with pytest.raises(ValueError, match="Unsupported chunking strategy"):
+            chunker.chunk([Document(text="x", doc_id="d")])
+
+
+# =====================================================================
+# Integration-like test
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestChunkerIntegration:
+
+    def test_mixed_document_sizes(self):
+        chunker = Chunker(max_tokens=50, min_tokens=5)
+        docs = [
+            Document(text="small text", doc_id="small"),
+            Document(text="word " * 200, doc_id="large"),
+            Document(text="medium " * 20, doc_id="medium"),
+        ]
+
+        result = chunker.chunk(docs)
+        # Small and medium should pass through, large should be split
+        assert len(result) >= 3
+        doc_ids = [d.doc_id for d in result]
+        assert "small" in doc_ids
+
+    def test_all_chunks_have_token_counts(self):
+        chunker = Chunker(max_tokens=50, min_tokens=1)
+        docs = [
+            Document(text="word " * 200, doc_id="big"),
+            Document(text="tiny", doc_id="small"),
+        ]
+
+        result = chunker.chunk(docs)
+        for doc in result:
+            assert doc.extra_info is not None
+            assert "token_count" in doc.extra_info
+            assert doc.extra_info["token_count"] > 0

tests/parser/test_schema.py

@@ -0,0 +1,58 @@
+import pytest
+
+from application.parser.schema.schema import BaseDocument
+
+
+class ConcreteDoc(BaseDocument):
+    @classmethod
+    def get_type(cls) -> str:
+        return "test"
+
+
+@pytest.mark.unit
+class TestBaseDocument:
+
+    def test_get_text(self):
+        doc = ConcreteDoc(text="hello")
+        assert doc.get_text() == "hello"
+
+    def test_get_text_raises_when_none(self):
+        doc = ConcreteDoc()
+        with pytest.raises(ValueError, match="text field not set"):
+            doc.get_text()
+
+    def test_get_doc_id(self):
+        doc = ConcreteDoc(text="x", doc_id="doc1")
+        assert doc.get_doc_id() == "doc1"
+
+    def test_get_doc_id_raises_when_none(self):
+        doc = ConcreteDoc(text="x")
+        with pytest.raises(ValueError, match="doc_id not set"):
+            doc.get_doc_id()
+
+    def test_is_doc_id_none(self):
+        doc = ConcreteDoc(text="x")
+        assert doc.is_doc_id_none is True
+
+    def test_is_doc_id_not_none(self):
+        doc = ConcreteDoc(text="x", doc_id="y")
+        assert doc.is_doc_id_none is False
+
+    def test_get_embedding(self):
+        doc = ConcreteDoc(text="x", embedding=[1.0, 2.0])
+        assert doc.get_embedding() == [1.0, 2.0]
+
+    def test_get_embedding_raises_when_none(self):
+        doc = ConcreteDoc(text="x")
+        with pytest.raises(ValueError, match="embedding not set"):
+            doc.get_embedding()
+
+    def test_extra_info_str(self):
+        doc = ConcreteDoc(text="x", extra_info={"key": "value", "num": 42})
+        result = doc.extra_info_str
+        assert "key: value" in result
+        assert "num: 42" in result
+
+    def test_extra_info_str_none(self):
+        doc = ConcreteDoc(text="x")
+        assert doc.extra_info_str is None