Document Loading, Parsing, and Cleaning in AI Applications

Document Loading, Parsing, and Cleaning in AI Applications | Tiger Data

# Using Docling for Document Conversion with OCR
from docling.document_converter import DocumentConverter

# Initialize the converter
converter = DocumentConverter()

# Load PDF and extract text with OCR enabled
result = converter.convert(
    "document.pdf",  # Can be local path or URL
    enable_ocr=True  # Enable OCR for scanned documents
)

# Get the converted markdown content
markdown_text = result.document.export_to_markdown()

# Preview the first 500 characters
print(markdown_text[:500])

import os
from mistralai import Mistral

api_key = os.environ["MISTRAL_API_KEY"]
client = Mistral(api_key=api_key)

ocr_response = client.ocr.process(
    model="mistral-ocr-latest",
    document={
        "type": "document_url",
        "document_url": "https://arxiv.org/pdf/2201.04234"
    },
    include_image_base64=True
)

import os
from mistralai import Mistral

api_key = os.environ["MISTRAL_API_KEY"]
client = Mistral(api_key=api_key)

ocr_response = client.ocr.process(
    model="mistral-ocr-latest",
    document={
        "type": "image_url",
        "image_url": "https://raw.githubusercontent.com/mistralai/cookbook/refs/heads/main/mistral/ocr/receipt.png"
    }
)

import os
from mistralai import Mistral

# Retrieve the API key from environment variables
api_key = os.environ["MISTRAL_API_KEY"]

# Specify model
model = "mistral-small-latest"

# Initialize the Mistral client
client = Mistral(api_key=api_key)

# Define the messages for the chat
messages = [
    {
        "role": "user",
        "content": [
            {
                "type": "text",
                "text": "In JSON format, extract the following metadata from the provided document: title, author, and created data. "
            },
            {
                "type": "document_url",
                "document_url": "https://arxiv.org/pdf/1805.04770"
            }
        ]
    }
]

# Get the chat response
chat_response = client.chat.complete(
    model=model,
    messages=messages
)

# Print the content of the response
print(chat_response.choices[0].message.content)

# Save the metadata somewhere for later ingest into your RAG pipeline

from firecrawl import FirecrawlApp

app = FirecrawlApp(api_key="fc-YOUR_API_KEY")

# Crawl a website:
crawl_status = app.crawl_url(
  'https://firecrawl.dev', 
  params={
    'limit': 100, 
    'scrapeOptions': {'formats': ['markdown', 'html']}
  },
  poll_interval=30
)
print(crawl_status)

from firecrawl import FirecrawlApp

app = FirecrawlApp(api_key="fc-YOUR_API_KEY")

# Scrape a website:
scrape_result = app.scrape_url('firecrawl.dev', params={'formats': ['markdown', 'html']})
print(scrape_result

from firecrawl import FirecrawlApp
from pydantic import BaseModel, Field

# Initialize the FirecrawlApp with your API key
app = FirecrawlApp(api_key='your_api_key')

class ExtractSchema(BaseModel):
    company_mission: str
    supports_sso: bool
    is_open_source: bool
    is_in_yc: bool

data = app.scrape_url('https://docs.firecrawl.dev/', {
    'formats': ['json'],
    'jsonOptions': {
        'schema': ExtractSchema.model_json_schema(),
    }
})
print(data["json"])

// Enable pgai and pgvector on your Postgres database
CREATE EXTENSION IF NOT EXISTS vector;  
CREATE EXTENSION IF NOT EXISTS ai;  

// Create a table to store Wikipedia articles
CREATE TABLE wiki (
    id      TEXT PRIMARY KEY,
    url     TEXT,
    title   TEXT,
    text    TEXT
);

// Load Wikipedia dataset directly from HuggingFace 
SELECT ai.load_dataset('wikimedia/wikipedia', '20231101.en', table_name=>'wiki', batch_size=>5, max_batches=>1, if_table_exists=>'append');

// Create Vectorizer that
// 1. Chunks the data using the chunking recursive character text splitter
// 2. Embeds it using Mini LM at 384 token size per chunk
// 3. Continuously monitors the wiki table for new text incoming
SELECT ai.create_vectorizer(
     'wiki'::regclass,
     embedding => ai.embedding_ollama('all-minilm', 384),
     formatting=> ai.formatting_python_template('url: $url  title: $title $chunk')
     chunking => ai.chunking_recursive_character_text_splitter('text'),

);

// Check status of Vectorizxer embedding creation
select * from ai.vectorizer_status;

version: '3'
services:
  db:
    image: timescale/timescaledb-ha:pg17
    environment:
      POSTGRES_PASSWORD: postgres
    ports:
      - "5432:5432"
    volumes:
      - data:/home/postgres/pgdata/data
      
  vectorizer-worker:
    image: timescale/pgai-vectorizer-worker:latest
    environment:
      PGAI_VECTORIZER_WORKER_DB_URL: postgres://postgres:postgres@db:5432/postgres
      OPENAI_API_KEY: your_openai_api_key_here
    command: [ "--poll-interval", "5s" ]
    
  ollama:
    image: ollama/ollama
    
volumes:
  data:

import re

def clean_ocr_text(text):
    # Replace hyphenated line breaks (e.g., "exam-\nple" -> "example")
    text = re.sub(r'(\w)-\n(\w)', r'\1\2', text)

    # Merge lines that are broken in the middle of sentences
    text = re.sub(r'\n(?=\w)', ' ', text)

    # More cleaning steps go here

    return text

from markitdown import MarkItDown
from openai import OpenAI

# Set up OpenAI client
client = OpenAI(api_key="your-openai-api-key")

# Initialize MarkItDown with LLM capabilities
md = MarkItDown(llm_client=client, llm_model="gpt-4o")

# Convert an image file
result = md.convert("path_to_your_image.jpg")

# Print the generated description
print(result.text_content)

from transformers import BlipProcessor, BlipForConditionalGeneration
from PIL import Image

# Load the pre-trained BLIP model and processor
processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")

# Open an image file
image_path = "path_to_your_image.jpg"  # Replace with your image file path
image = Image.open(image_path)

# Preprocess the image and prepare inputs for the model
inputs = processor(images=image, return_tensors="pt")

# Generate caption
outputs = model.generate(**inputs)

# Decode the generated caption
caption = processor.decode(outputs[0], skip_special_tokens=True)

print("Generated Caption:", caption)

from google import genai
from google.genai.types import HttpOptions

client = genai.Client(http_options=HttpOptions(api_version="v1"))
response = client.models.generate_content(
    model="gemini-2.0-flash-001",
    contents="Summarize in a technical tone the following piece of text: Attention Is All You Need...",
)
print(response.text)

Document Loading, Parsing, and Cleaning in AI Applications

Gathering the Right Data for AI Applications

Data selection matters

Choosing the right data

Where to collect data

Ensure data freshness

Extracting Text From Documents

Using MarkItDown for Document Conversion

Using Visual Language Models for OCR

Metadata Extraction

Extracting built-in metadata

Contextual metadata extraction

Extracting Text From the Web

Firecrawl for Web Scraping

Other Web Scraping Options

Direct Data Loading

Storing Data

Cleaning Messy Data

Cleaning webpages

Cleaning PDFs

Fixing Text Formatting

Handling Images in Documents

Summarizing or Cleaning Text With an LLM

Conclusion

Get Involved