Bases: TextSplitter
Basic wrapper around langchain's text splitter.
TODO: Figure out how to make this metadata aware.
Source code in llama-index-core/llama_index/core/node_parser/text/langchain.py
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50 | class LangchainNodeParser(TextSplitter):
"""
Basic wrapper around langchain's text splitter.
TODO: Figure out how to make this metadata aware.
"""
_lc_splitter: "LC_TextSplitter" = PrivateAttr()
def __init__(
self,
lc_splitter: "LC_TextSplitter",
callback_manager: Optional[CallbackManager] = None,
include_metadata: bool = True,
include_prev_next_rel: bool = True,
id_func: Optional[Callable[[int, Document], str]] = None,
):
"""Initialize with parameters."""
try:
from langchain.text_splitter import TextSplitter as LC_TextSplitter # noqa
except ImportError:
raise ImportError(
"Could not run `from langchain.text_splitter import TextSplitter`, "
"please run `pip install langchain`"
)
id_func = id_func or default_id_func
super().__init__(
callback_manager=callback_manager or CallbackManager(),
include_metadata=include_metadata,
include_prev_next_rel=include_prev_next_rel,
id_func=id_func,
)
self._lc_splitter = lc_splitter
def split_text(self, text: str) -> List[str]:
"""Split text into sentences."""
return self._lc_splitter.split_text(text)
|
split_text
split_text(text: str) -> List[str]
Split text into sentences.
Source code in llama-index-core/llama_index/core/node_parser/text/langchain.py
| def split_text(self, text: str) -> List[str]:
"""Split text into sentences."""
return self._lc_splitter.split_text(text)
|