tokenizer updated

app.py

@@ -1,6 +1,6 @@
 import gradio as gr
 import pkg_resources
-from turkish_tokenizer import TokenType, TurkishTokenizer
+from turkish_tokenizer import TurkishTokenizer
 
 # Get the version from the installed package
 try:
@@ -12,11 +12,12 @@ tokenizer = TurkishTokenizer()
 
 # Define colors for each token type
 color_map = {
-    TokenType.ROOT.name: "#FF6B6B",      # Red
-    TokenType.SUFFIX.name: "#4ECDC4",    # Teal
-    TokenType.BPE.name: "#FFE66D",       # Yellow
+    "ROOT": "#FF6B6B",  # Red
+    "SUFFIX": "#4ECDC4",  # Teal
+    "BPE": "#FFE66D",  # Yellow
 }
 
+
 def tokenize_and_display(text):
     """
     Tokenizes the input text and prepares it for display in Gradio's HighlightedText component.
@@ -25,7 +26,7 @@ def tokenize_and_display(text):
         # Return a structure that matches all outputs to avoid errors
         return [], "", "", ""
 
-    tokens, _ = tokenizer.tokenize_text(text)
+    tokens = tokenizer.tokenize_text(text)
 
     # Create the list of (token, label) for HighlightedText
     highlighted_tokens = []
@@ -33,7 +34,7 @@ def tokenize_and_display(text):
 
     for t in tokens:
         token_text = t["token"]
-        token_type = t["type"].name
+        token_type = t["type"]
 
         # Count token types for statistics
         token_stats[token_type] = token_stats.get(token_type, 0) + 1
@@ -49,7 +50,12 @@ def tokenize_and_display(text):
     compression_ratio = (1 - total_tokens / total_chars) * 100 if total_chars > 0 else 0
 
     # Define colors for the stats block
-    bg_col, text_col, card_col, border_col = ('#f8f9fa', '#2d3748', '#ffffff', '#e2e8f0')
+    bg_col, text_col, card_col, border_col = (
+        "#f8f9fa",
+        "#2d3748",
+        "#ffffff",
+        "#e2e8f0",
+    )
 
     # Create statistics HTML
     stats_html = f"""
@@ -71,6 +77,7 @@ def tokenize_and_display(text):
     </div>"""
     return highlighted_tokens, str(encoded_ids), decoded_text, stats_html
 
+
 # Custom CSS for better styling
 custom_css = """
 .gradio-container{font-family:'Inter',-apple-system,BlinkMacSystemFont,sans-serif;}
@@ -81,30 +88,38 @@ custom_css = """
 """
 
 # Create the Gradio Interface
-with gr.Blocks(theme=gr.themes.Soft(), title="Turkish Tokenizer", css=custom_css) as demo:
+with gr.Blocks(
+    theme=gr.themes.Soft(), title="Turkish Tokenizer", css=custom_css
+) as demo:
     with gr.Row():
         with gr.Column(scale=3):
-            gr.Markdown(f"""
+            gr.Markdown(
+                f"""
                 # Turkish Tokenizer
                 ### Advanced Turkish Text Tokenization with Visual Analysis
                 Enter text to see how it's tokenized. Tokens are color-coded by type.                
-            """)
+            """
+            )
 
     input_text = gr.Textbox(
         label="📝 Input Text",
         placeholder="Merhaba Dünya, kitapları okumak güzeldir.",
         lines=4,
-        elem_classes=["input-textbox"]
+        elem_classes=["input-textbox"],
     )
 
     with gr.Row():
-        process_button = gr.Button("🚀 Tokenize", variant="primary", elem_classes=["custom-button"], size="lg")
+        process_button = gr.Button(
+            "🚀 Tokenize", variant="primary", elem_classes=["custom-button"], size="lg"
+        )
         clear_button = gr.Button("🗑️ Clear", variant="secondary", size="lg")
 
     gr.Markdown("---")
     gr.Markdown("### 🔄 Encoded & Decoded Output")
     with gr.Row():
-        encoded_output = gr.Textbox(label="🔢 Encoded Token IDs", interactive=False, lines=2)
+        encoded_output = gr.Textbox(
+            label="🔢 Encoded Token IDs", interactive=False, lines=2
+        )
         decoded_output = gr.Textbox(label="📝 Decoded Text", interactive=False, lines=2)
 
     gr.Markdown("### 💡 Example Texts")
@@ -117,23 +132,22 @@ with gr.Blocks(theme=gr.themes.Soft(), title="Turkish Tokenizer", css=custom_css
             ["Yapay zeka ve makine öğrenmesi teknolojileri gelişiyor."],
         ],
         inputs=input_text,
-        label="Try these examples:"
+        label="Try these examples:",
     )
 
     gr.Markdown("---")
     gr.Markdown("### 🎨 Tokenization Output")
     highlighted_output = gr.HighlightedText(
-        label="Colorized Tokens",
-        color_map=color_map,
-        show_legend=True
+        label="Colorized Tokens", color_map=color_map, show_legend=True
     )
 
     gr.Markdown("---")
     gr.Markdown("### 📊 Statistics")
     stats_output = gr.HTML(label="")
 
-
-    gr.Markdown(f"--- \n **Turkish Tokenizer v{VERSION}** - Advanced tokenization for Turkish text.")
+    gr.Markdown(
+        f"--- \n **Turkish Tokenizer v{VERSION}** - Advanced tokenization for Turkish text."
+    )
 
     # --- Event Handlers ---
     def process_with_theme(text):
@@ -146,18 +160,24 @@ with gr.Blocks(theme=gr.themes.Soft(), title="Turkish Tokenizer", css=custom_css
     process_button.click(
         fn=process_with_theme,
         inputs=[input_text],
-        outputs=[highlighted_output, encoded_output, decoded_output, stats_output]
+        outputs=[highlighted_output, encoded_output, decoded_output, stats_output],
     )
 
     clear_button.click(
         fn=clear_all,
-        outputs=[input_text, highlighted_output, encoded_output, decoded_output, stats_output]
+        outputs=[
+            input_text,
+            highlighted_output,
+            encoded_output,
+            decoded_output,
+            stats_output,
+        ],
     )
 
     # Auto-process on load with a default example
     demo.load(
         fn=lambda: tokenize_and_display("Merhaba Dünya!"),
-        outputs=[highlighted_output, encoded_output, decoded_output, stats_output]
+        outputs=[highlighted_output, encoded_output, decoded_output, stats_output],
     )
 
 if __name__ == "__main__":

requirements.txt

@@ -1,3 +1,3 @@
 gradio
 
-turkish-tokenizer==0.2.26
+turkish-tokenizer>=1.0.4