fix: getting empty chunks in semantic splitter (#85)

elisalimli · web-flow · commit ea5e0411b85e · 2024-03-07T09:16:04.000-08:00
diff --git a/service/splitter.py b/service/splitter.py
@@ -188,6 +188,8 @@ def _append_chunks(
                             # TODO: Think of how to pass this to LLM
                             metadata={"table_content": table, **metadata},
                         )
+                else:
+                    accumulated_element_texts.append(element.get("text"))
 
             # Process any remaining accumulated text after the last table
             # or if no table was encountered

Original file line number	Diff line number	Diff line change
`@@ -188,6 +188,8 @@ def _append_chunks(`
`188`	`188`	`# TODO: Think of how to pass this to LLM`
`189`	`189`	`metadata={"table_content": table, **metadata},`
`190`	`190`	`)`
	`191`	`+ else:`
	`192`	`+ accumulated_element_texts.append(element.get("text"))`
`191`	`193`
`192`	`194`	`# Process any remaining accumulated text after the last table`
`193`	`195`	`# or if no table was encountered`