diff --git a/pyproject.toml b/pyproject.toml index 756380a..c5bd58b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "SpeechRecognition", "pathvalidate", "charset-normalizer", + "openai", ] [project.urls] @@ -77,3 +78,6 @@ exclude_lines = [ "if __name__ == .__main__.:", "if TYPE_CHECKING:", ] + +[tool.hatch.build.targets.sdist] +only-include = ["src/markitdown"] diff --git a/src/markitdown/__about__.py b/src/markitdown/__about__.py index d1621f5..a365900 100644 --- a/src/markitdown/__about__.py +++ b/src/markitdown/__about__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2024-present Adam Fourney # # SPDX-License-Identifier: MIT -__version__ = "0.0.1a1" +__version__ = "0.0.1a3" diff --git a/src/markitdown/_markitdown.py b/src/markitdown/_markitdown.py index 041e56e..f6d6da0 100644 --- a/src/markitdown/_markitdown.py +++ b/src/markitdown/_markitdown.py @@ -15,7 +15,7 @@ import traceback import zipfile from typing import Any, Dict, List, Optional, Union from urllib.parse import parse_qs, quote, unquote, urlparse, urlunparse -from warnings import catch_warnings +from warnings import warn, resetwarnings, catch_warnings import mammoth import markdownify @@ -44,6 +44,8 @@ try: IS_AUDIO_TRANSCRIPTION_CAPABLE = True except ModuleNotFoundError: pass +finally: + resetwarnings() # Optional YouTube transcription support try: @@ -847,8 +849,6 @@ class ImageConverter(MediaConverter): if prompt is None or prompt.strip() == "": prompt = "Write a detailed caption for this image." - sys.stderr.write(f"llm Prompt:\n{prompt}\n") - data_uri = "" with open(local_path, "rb") as image_file: content_type, encoding = mimetypes.guess_type("_dummy" + extension) @@ -1010,14 +1010,46 @@ class MarkItDown: self, requests_session: Optional[requests.Session] = None, llm_client: Optional[Any] = None, - llm_model: Optional[Any] = None, + llm_model: Optional[str] = None, style_map: Optional[str] = None, + # Deprecated + mlm_client: Optional[Any] = None, + mlm_model: Optional[str] = None, ): if requests_session is None: self._requests_session = requests.Session() else: self._requests_session = requests_session + # Handle deprecation notices + ############################# + if mlm_client is not None: + if llm_client is None: + warn( + "'mlm_client' is deprecated, and was renamed 'llm_client'.", + DeprecationWarning, + ) + llm_client = mlm_client + mlm_client = None + else: + raise ValueError( + "'mlm_client' is deprecated, and was renamed 'llm_client'. Do not use both at the same time. Just use 'llm_client' instead." + ) + + if mlm_model is not None: + if llm_model is None: + warn( + "'mlm_model' is deprecated, and was renamed 'llm_model'.", + DeprecationWarning, + ) + llm_model = mlm_model + mlm_model = None + else: + raise ValueError( + "'mlm_model' is deprecated, and was renamed 'llm_model'. Do not use both at the same time. Just use 'llm_model' instead." + ) + ############################# + self._llm_client = llm_client self._llm_model = llm_model self._style_map = style_map diff --git a/tests/test_files/test.docx b/tests/test_files/test.docx old mode 100755 new mode 100644 diff --git a/tests/test_files/test.jpg b/tests/test_files/test.jpg old mode 100755 new mode 100644 diff --git a/tests/test_files/test.xlsx b/tests/test_files/test.xlsx old mode 100755 new mode 100644 diff --git a/tests/test_files/test_llm.jpg b/tests/test_files/test_llm.jpg new file mode 100644 index 0000000..1f358fe Binary files /dev/null and b/tests/test_files/test_llm.jpg differ diff --git a/tests/test_files/test_with_comment.docx b/tests/test_files/test_with_comment.docx old mode 100755 new mode 100644 diff --git a/tests/test_markitdown.py b/tests/test_markitdown.py index 76bd302..7db904d 100644 --- a/tests/test_markitdown.py +++ b/tests/test_markitdown.py @@ -6,11 +6,23 @@ import shutil import pytest import requests +from warnings import catch_warnings, resetwarnings + from markitdown import MarkItDown skip_remote = ( True if os.environ.get("GITHUB_ACTIONS") else False ) # Don't run these tests in CI + + +# Don't run the llm tests without a key and the client library +skip_llm = False if os.environ.get("OPENAI_API_KEY") else True +try: + import openai +except ModuleNotFoundError: + skip_llm = True + +# Skip exiftool tests if not installed skip_exiftool = shutil.which("exiftool") is None TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") @@ -107,6 +119,10 @@ CSV_CP932_TEST_STRINGS = [ "髙橋淳,35,名古屋", ] +LLM_TEST_STRINGS = [ + "5bda1dd6", +] + @pytest.mark.skipif( skip_remote, @@ -229,8 +245,63 @@ def test_markitdown_exiftool() -> None: assert target in result.text_content +def test_markitdown_deprecation() -> None: + try: + with catch_warnings(record=True) as w: + test_client = object() + markitdown = MarkItDown(mlm_client=test_client) + assert len(w) == 1 + assert w[0].category is DeprecationWarning + assert markitdown._llm_client == test_client + finally: + resetwarnings() + + try: + with catch_warnings(record=True) as w: + markitdown = MarkItDown(mlm_model="gpt-4o") + assert len(w) == 1 + assert w[0].category is DeprecationWarning + assert markitdown._llm_model == "gpt-4o" + finally: + resetwarnings() + + try: + test_client = object() + markitdown = MarkItDown(mlm_client=test_client, llm_client=test_client) + assert False + except ValueError: + pass + + try: + markitdown = MarkItDown(mlm_model="gpt-4o", llm_model="gpt-4o") + assert False + except ValueError: + pass + + +@pytest.mark.skipif( + skip_llm, + reason="do not run llm tests without a key", +) +def test_markitdown_llm() -> None: + client = openai.OpenAI() + markitdown = MarkItDown(llm_client=client, llm_model="gpt-4o") + + result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_llm.jpg")) + + for test_string in LLM_TEST_STRINGS: + assert test_string in result.text_content + + # This is not super precise. It would also accept "red square", "blue circle", + # "the square is not blue", etc. But it's sufficient for this test. + for test_string in ["red", "circle", "blue", "square"]: + assert test_string in result.text_content.lower() + + if __name__ == "__main__": """Runs this file's tests from the command line.""" test_markitdown_remote() test_markitdown_local() test_markitdown_exiftool() + test_markitdown_deprecation() + test_markitdown_llm()