ColPali联手DocLayNet：打造能“看懂”文档布局的视觉问答神器！ 原创

Option Awards Stock Awards
 Name
 Option/ 
PSU/RSU 
...
...
... (truncated)
Shares, Units, or 
Other Rights That 
Have Not Vested 
($)(1)
 D. Ralston 1/20/2021 67,220(2) 3,498,129
 1/20/2021 33,610(3) 1,749,064
 2/3/2021 29,390(4) 1,529,456
 1/19/2022 64,587(5) 3,361,107
 1/19/2022 22,321(6) 1,161,585
 1/18/2023 41,668(7) 2,168,403
 1/18/2023 14,427(8) 750,781

Total Shareholder Return
 Indexed ($100)
 $50.0 \n$40.0 \n$30.0 \n$20.0 \n$10.0 \n$0.0-$10.0-$20.0
 2020 2021 2023 2022
 CEO CAP Avg. NEO CAP SLB TSR-$10.6
 $2.2 \n$32.4 \n$13.1 \n$39.4 \n$26.2 \n$17.3 \n$9.3 
 ...
 ...
 ... (truncated)
 $40.00 \n$20.00 \n$0.00
 OSX TSR
 CAP vs. Total Shareholder Return
 (SLB and OSX)
 (in millions of US dollars)
 Compensation Actually Paid

conda create -n multimodal_rag pythnotallow=3.11 
conda activate multimodal_rag

ultralytics 
git+https://github.com/illuin-tech/colpali 
groq 
pill 
pymilvus 
sentence-transformers 
uvicorn 
fastapi 
opencv-python 
pytesseract 
PyMuPDF 
pydantic 
chainlit 
pybase64 
huggingface_hub[hf_transfer]

pip install -r requirements.txt

# 下载YOLO DocLayNet用于布局检测
HF_HUB_ENABLE_HF_TRANSFER=1 hf download hantian/yolo-doclaynet --local-dir pretrained_models/yolo-doclaynet

# 下载 ColQwen 2.5（用于 ColPALI）用于基于图像的检索
HF_HUB_ENABLE_HF_TRANSFER=1 hf download vidore/colqwen2.5-v0.2 --local-dir pretrained_models/colqwen2.5-v0.2

# 下载 Mixedbread Embed Large 模型用于基于文本的检索
HF_HUB_ENABLE_HF_TRANSFER=1 hf download mixedbread-ai/mxbai-embed-large-v1 --local-dir pretrained_models/mxbai-embed-large-v1

import torch 
from Typing import Cast 
from Ultralytics import YOLO 
from transforms.utils.import_utils import is_flash_attn_2_available 
from colpali_engine.models.paligemma.colpali.processing_colpali import ColPaliProcessor 
from colpali_engine.models import ColQwen2_5, ColQwen2_5_Processor 
from sentence_transformers import SentenceTransformer 

# 定义设备
device = "cuda" if torch.cuda.is_available() else "cpu" 

# 定义知识库源和目标图像目录
document_source_dir = "document_sources"
 img_dir = "image_database"
 os.makedirs(img_dir, exist_ok= True ) # 确保目录存在

# YOLO-12L-Doclaynet
 yolo_model = YOLO( "pretrained_models/yolo-doclaynet/yolov12l-doclaynet.pt" ) 
yolo_model = yolo_model.to(device) 

# ColQwen2.5-Colpali
 colpali_model = ColQwen2_5.from_pretrained( 
 "pretrained_models/colqwen2.5-v0.2" , 
 torch_dtype=torch.bfloat16, 
 device_map=device, # 或 "mps" 如果在 Apple Silicon 上
 attn_implementatinotallow= "flash_attention_2" 如果is_flash_attn_2_available() else None , 
 )。eval () 
colpali_processor = ColQwen2_5_Processor.from_pretrained( "pretrained_models/colqwen2.5-v0.2" ) 
processor = cast( 
 ColPaliProcessor, 
 colpali_processor) 

# Mxbai-embed-large-v1
 embed_model = SentenceTransformer( "pretrained_models/mxbai-embed-large-v1" ,device=device) 

# 定义实体颜色
ENTITIES_COLORS = { 
 "Picture" : ( 255 , 72 , 88 ), 
 "Table" : ( 128 , 0 , 128 ) 
} 

print ( "FINISH SETUP..." )

import matplotlib.pyplot as plt 
import cv2 

def display_img_array ( img_arr ): 
 image_rgb = cv2.cvtColor(img_arr, cv2.COLOR_BGR2RGB) 
 plt.figure(figsize=( 30 , 30 )) 
 plt.imshow(image_rgb) 
 plt.axis( 'off' ) 
 plt.show() 

def show_layout_detection ( detection_results, img_arr ): 
 for result indetection_results : 
 boxes = result.boxes # 获取检测框
 for box in boxes: 
 x, y, w, h = box.xywh[ 0 ] # 框坐标（中心 x, y, 宽度, 高度）
 x, y, w, h = int (x), int (y), int (w), int (h) # 转换为整数
 conf = box.conf.item() # 置信度得分
 cls = int (box.cls.item()) # 类 ID
 label = f" {yolo_model.model.names[cls]} {conf: .2 f} "
 color = ENTITIES_COLORS[yolo_model.model.names[cls]] # 获取此类的颜色
 top_left = (x - w // 2 , y - h // 2 ) 
 bottom_right = (x + w // 2 , y + h // 2 ) # 特定类的彩色框
 cv2.rectangle(img_arr, top_left, bottom_right, color, 2 ) 
 cv2.putText(img_arr, label, (top_left[ 0 ], top_left[ 1 ] - 10 ), cv2.FONT_HERSHEY_SIMPLEX, 0.9 , color, 2 ) # 匹配文本颜色
 display_img_array(img_arr)

import fitz # PyMuPDF
import numpy as np
import cv2
import os

# 定义文件名和页面
page_id = 38
 filename = "SLB-2023-Annual-Report.pdf" 

# 阅读文档
doc = fitz. open (os.path.join(document_source_dir,filename)) 
page = doc.load_page(page_id) 
pix = page.get_pixmap(dpi= 300 ) 
img_rgb = np.frombuffer(pix.samples, dtype=np.uint8).reshape((pix.height, pix.width, pix.n)) 
img_page = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2BGR)

def layout_detection(img_doc):
 return yolo_model.predict(
 source=img_doc, 
 classes=[6,8],
 cnotallow=0.25, 
 iou=0.45)

layout_results = layout_detection(img_page)
show_layout_detection(layout_results, img_page)

def extract_and_masking_images(img_doc, layout_results):
 height, width, _ = img_doc.shape
 extracted_imgs = []
 for box in layout_results[0].boxes:
 x, y, w, h = map(int,box.xywh[0]) #矩形坐标（中心x、y、宽度、高度）
 # 计算左上角(x_min, y_min)
 x_min = x - w // 2
 y_min = y - h // 2
 x_max = x_min + w
 y_max = y_min + h
 # 将坐标夹紧到图像边界
 x_start = max(0, x_min)
 y_start = max(0, y_min)
 x_end = min(width, x_max)
 y_end = min(height, y_max)
 # 如果区域无效，则跳过
 if x_start >= x_end or y_start >= y_end:
 continue
 # 将图像提取到extracted_imgs数组中
 extracted_imgs.append(img_doc[y_start:y_end, x_start:x_end].copy())
 #将区域设置为白色
 img_doc[y_start:y_end, x_start:x_end] = [255, 255, 255]
 return extracted_imgs, img_doc

extracted_imgs, img_page = extract_and_masking_images(img_page, layout_results)
display_img_array(img_page)

import os
import cv2

def save_img_files(extracted_imgs, filename, page_id):
 #目标路径
 save_path = os.path.join(img_dir, filename, f"page_{page_id}/")
 # 确保目录存在
 os.makedirs(os.path.dirname(save_path), exist_ok=True)
 # 保存图像
 for i in range(len(extracted_imgs)):
 cv2.imwrite(save_path+f"fig_{i}.jpg", extracted_imgs[i])

save_img_files(extracted_imgs, filename, page_id)

import pytesseract

text = pytesseract.image_to_string(img_page)
print(text)

Short-Term Cash Incentive Awards

We pay performance-based short-term (annual) cash incentives to
our executives to foster a results-driven, pay-for-performance culture,
and to align executives’ interests with those of our shareholders. STI
awards are earned according to the achievement of quantitative
Company financial and non-financial objectives, as well as strategic
objectives. Our Compensation Committee selects performance
measures that it believes support our strategy and strike a balance
between motivating our executives to increase near-term financial and
operating results and driving profitable long-term Company growth
and value for shareholders.

2022 STI Opportunity Mix

Compensation Discussion and Analysis

For 2023, 70% of our NEOs’ target STI opportunity was based on
achieving quantitative Company financial objectives, 10% was based
on achieving quantitative Company non-financial objectives, and 20%
was based on strategic personal objectives. The financial portion of
the target plan was evenly split between adjusted EBITDA and free
cash flow performance goals. The total maximum STI payout for 2023
was 200% of target—consistent with 2022—and the weighted payout
range for each metric as a percentage of target is reflected by the outer
bars in the 2023 STI Opportunity Mix chart below.

2023 STI Opportunity Mix

»>

»>

In January 2023, our Compensation Committee determined to leave the target STI opportunity for all NEOs unchanged from 2022, following
a review of market data indicating that our NEOs’ target STI opportunity (as a percentage of base salary) was competitively positioned. As a
result, the 2023 target STI opportunity for our CEO was 150% of his base salary and for our other NEOs it was 100% of base salary.

The following table reflects our NEOs’ full-year 2023 STI results, together with relevant weightings of the different components and payouts

under each component.

(1) Equals the sum of the financial, non-financial, and personal portions of the STI achieved, shown as a percentage of base salary.
(2) In January 2024, due to factors not contemplated in the 2023 forecast, our Compensation Committee applied a discretionary downward
adjustment to reduce all executive payouts by 5% under our 2023 STI plan.

2024 Proxy Statement

from pymilvus import MilvusClient, DataType
import numpy as np
import concurrent.futures
import os
import base64

class MilvusColbertRetriever:
 def __init__(self, milvus_client, collection_name, img_dir, dim=128):
 #使用Milvus客户端、集合名称和向量嵌入的维度初始化检索器。
 # If the collection exists, load it.
 self.collection_name = collection_name
 self.client = milvus_client
 if self.client.has_collection(collection_name=self.collection_name):
 self.client.load_collection(collection_name)
 self.dim = dim
 self.img_dir = img_dir

 def create_collection(self):
 # 在Milvus中创建一个新的集合来存储嵌入。
 #如果现有集合已存在，请删除该集合，并为该集合定义架构。
 if self.client.has_collection(collection_name=self.collection_name):
 self.client.drop_collection(collection_name=self.collection_name)
 schema = self.client.create_schema(auto_id=True, enable_dynamic_field=True)
 schema.add_field(field_name="pk", datatype=DataType.INT64, is_primary=True)
 schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self.dim)
 schema.add_field(field_name="seq_id", datatype=DataType.INT16)
 schema.add_field(field_name="doc_id", datatype=DataType.INT64)
 schema.add_field(field_name="doc", datatype=DataType.VARCHAR, max_length=65535)
 self.client.create_collection(collection_name=self.collection_name, schema=schema)

 def create_index(self):
 # 在向量字段上创建索引，以实现快速相似性搜索。
 # 在使用指定参数创建新索引之前，释放并删除任何现有索引。
 self.client.release_collection(collection_name=self.collection_name)
 self.client.drop_index(collection_name=self.collection_name, index_name="vector")
 index_params = self.client.prepare_index_params()
 index_params.add_index(
 field_name="vector",
 index_name="vector_index",
 index_type="IVF_FLAT", 
 metric_type="IP", 
 )
 self.client.create_index(collection_name=self.collection_name, index_params=index_params, sync=True)

 def search(self, data, topk):
 # 对集合执行向量搜索，以找到前k个最相似的文档。
 search_params = {"metric_type": "IP", "params": {}}
 results = self.client.search(
 self.collection_name,
 data,
 limit=int(50),
 output_fields=["vector", "seq_id", "doc_id","$meta"],
 search_params=search_params,
 )
 doc_meta = {}
 for r_id in range(len(results)):
 for r in range(len(results[r_id])):
 entity = results[r_id][r]["entity"]
 doc_id = entity["doc_id"]
 if doc_id not in doc_meta:
 doc_meta[doc_id] = {
 "page_id": entity["page_id"],
 "fig_id": entity["fig_id"],
 "filename": entity["filename"],
 }
 scores = []

 def rerank_single_doc(doc_id, data, client, collection_name):
 #通过检索单个文档的嵌入并计算其与查询的相似度来对其重新排序。
 doc_colbert_vecs = client.query(
 collection_name=collection_name,
 filter=f"doc_id in [{doc_id}]",
 output_fields=["seq_id", "vector", "doc"],
 limit=1000,
 )
 doc_vecs = np.vstack(
 [doc_colbert_vecs[i]["vector"] for i in range(len(doc_colbert_vecs))]
 )
 score = np.dot(data, doc_vecs.T).max(1).sum()
 return (score, doc_id)

 with concurrent.futures.ThreadPoolExecutor(max_workers=300) as executor:
 futures = {
 executor.submit(
 rerank_single_doc, doc_id, data, self.client, self.collection_name
 ): doc_id
 for doc_id in doc_meta.keys()
 }
 for future in concurrent.futures.as_completed(futures):
 score, doc_id = future.result()
 meta = doc_meta[doc_id]
 img_path = os.path.join(self.img_dir, meta["filename"], f"page_{meta['page_id']}", f"fig_{meta['fig_id']}.jpg")
 with open(img_path, "rb") as f:
 img_base64 = base64.b64encode(f.read()).decode('utf-8')
 scores.append({
 "score":float(score), 
 "page_id": meta["page_id"],
 "fig_id": meta["fig_id"],
 "filename": meta["filename"],
 "content": img_base64})

 scores.sort(key=lambda x: x["score"], reverse=True)
 if len(scores) >= topk:
 return scores[:topk]
 else:
 return scores

 def insert(self, data):
 # 将文档的ColBERT嵌入和元数据插入集合中。
 #将数据作为多个向量（每个序列一个）与相应的元数据一起插入。
 colbert_vecs = [vec for vec in data["colbert_vecs"]]
 seq_length = len(colbert_vecs)
 self.client.insert(
 self.collection_name,
 [
 {
 "vector": colbert_vecs[i],
 "seq_id": i,
 "doc_id": data["doc_id"] ,
 "doc": "",
 "page_id": data["page_id"],
 "fig_id": data["fig_id"],
 "filename": data["filename"],
 }
 for i in range(seq_length)
 ],
 )

class MilvusBasicRetriever:
 def __init__(self, milvus_client, collection_name, dim=1024):
 # 使用Milvus客户端、集合名称和向量嵌入的维度初始化检索器。
 #如果集合存在，则加载之。
 self.collection_name = collection_name
 self.client = milvus_client
 if self.client.has_collection(collection_name=self.collection_name):
 self.client.load_collection(collection_name)
 self.dim = dim

 def normalize(self, vec):
 #把向量规范化
 norm = np.linalg.norm(vec)
 if norm == 0:
 return vec
 return vec / norm

 def create_collection(self):
 # 在Milvus中创建一个新的集合来存储嵌入。
 #如果现有集合已存在，请删除该集合，并为该集合定义架构。
 if self.client.has_collection(collection_name=self.collection_name):
 self.client.drop_collection(collection_name=self.collection_name)
 schema = self.client.create_schema(auto_id=True, enable_dynamic_field=True)
 schema.add_field(field_name="pk", datatype=DataType.INT64, is_primary=True)
 schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self.dim)
 schema.add_field(field_name="content", datatype=DataType.VARCHAR, max_length=65535)
 self.client.create_collection(collection_name=self.collection_name, schema=schema)

 def create_index(self):
 #在向量字段上创建索引，以实现快速相似性搜索。
 # 在使用指定参数创建新索引之前，释放并删除任何现有索引。
 self.client.release_collection(collection_name=self.collection_name)
 self.client.drop_index(collection_name=self.collection_name, index_name="vector")
 index_params = self.client.prepare_index_params()
 index_params.add_index(
 field_name="vector",
 index_name="vector_index",
 index_type="IVF_FLAT", # or any other index type you want
 metric_type="IP", # or the appropriate metric type
 )
 self.client.create_index(collection_name=self.collection_name, index_params=index_params, sync=True)

 def search(self, data, topk):
 #对集合执行向量搜索，以找到前k个最相似的文档。
 normalized_data = self.normalize(data)
 search_params = {"metric_type": "IP", "params": {}}
 results = self.client.search(
 self.collection_name,
 [normalized_data],
 limit=topk,
 output_fields=["vector", "content","$meta"],
 search_params=search_params,
 )
 return_arr = []
 for hit in results[0]:
 return_arr.append({
 "score":hit.distance,
 "page_id":hit["entity"]["page_id"],
 "filename":hit["entity"]["filename"],
 "content":hit["entity"]["content"]
 })
 return return_arr

 def insert(self, data):
 data["vector"] = self.normalize(np.array(data["vector"])).tolist()
 self.client.insert(
 self.collection_name,
 [data]
 )

client = MilvusClient("milvus_file.db")
colbert_retriever = MilvusColbertRetriever(collection_name="colbert", milvus_client=client,img_dir=img_dir)
basic_retriever = MilvusBasicRetriever(collection_name="basic", milvus_client=client)

colbert_retriever.create_collection() 
colbert_retriever.create_index() 
basic_retriever.create_collection() 
basic_retriever.create_index()

from colpali_engine.utils.torch_utils import ListDataset
from torch.utils.data import DataLoader
from typing import List
from tqdm import tqdm
from PIL import Image

def create_image_embedding_loader(extracted_imgs):
 images = [Image.fromarray(img_arr) for img_arr in extracted_imgs]
 dataloader_images = DataLoader(
 dataset=ListDataset[str](images),
 batch_size=1,
 shuffle=False,
 collate_fn=lambda x: processor.process_images(x),
 )
 ds: List[torch.Tensor] = []
 for batch_doc in tqdm(dataloader_images):
 with torch.no_grad():
 batch_doc = {k: v.to(colpali_model.device) for k, v in batch_doc.items()}
 embeddings_doc = colpali_model(**batch_doc)
 ds.extend(list(torch.unbind(embeddings_doc.to("cpu"))))
 return ds

embedding_loader = create_image_embedding_loader(extracted_imgs)
embedding_loader

[tensor([[-0.0055, 0.0991, -0.0903, ..., -0.0474, -0.0042, -0.1138],
 [-0.0067, 0.1064, -0.0488, ..., -0.0723, 0.0535, -0.0986],
 [-0.0200, 0.1113, -0.1084, ..., -0.0747, 0.0447, -0.0786],
 ...,
 [-0.0027, 0.0811, -0.1602, ..., 0.0354, -0.0112, -0.1670],
 [-0.0557, -0.1099, 0.0128, ..., 0.0203, -0.0728, -0.0688],
 [ 0.1025, 0.0145, -0.0420, ..., 0.0894, -0.0413, 0.1650]], dtype=torch.bfloat16),
 tensor([[-0.0055, 0.0991, -0.0903, ..., -0.0474, -0.0042, -0.1138],
 [-0.0067, 0.1064, -0.0488, ..., -0.0723, 0.0535, -0.0986],
 [-0.0200, 0.1113, -0.1084, ..., -0.0747, 0.0447, -0.0786],
 ...,
 [-0.0141, 0.0645, -0.1377, ..., 0.0430, -0.0061, -0.1338],
 [-0.0835, -0.1094, 0.0049, ..., 0.0211, -0.0608, -0.0645],
 [ 0.1396, 0.0549, -0.0669, ..., 0.0942, 0.0038, 0.1514]], dtype=torch.bfloat16),
 tensor([[-0.0053, 0.0996, -0.0894, ..., -0.0471, -0.0042, -0.1128],
 [-0.0068, 0.1060, -0.0491, ..., -0.0713, 0.0532, -0.0986],
 [-0.0204, 0.1118, -0.1089, ..., -0.0752, 0.0444, -0.0791],
 ...,
 [ 0.0330, 0.0398, -0.0505, ..., 0.0586, 0.0250, -0.1099],
 [-0.0508, -0.0981, -0.0126, ..., 0.0183, -0.0791, -0.0713],
 [ 0.1387, 0.0698, -0.0330, ..., 0.0238, 0.0923, 0.0337]], dtype=torch.bfloat16)]

import random

for i in range(len(extracted_imgs)):
 data = {
 "colbert_vecs": embedding_loader[i].float().numpy(),
 "doc_id": random.getrandbits(63),
 "page_id": page_id,
 "fig_id": i,
 "filename": filename,
 }
 colbert_retriever.insert(data)

data = {
 "vector": embed_model.encode(text),
 "content": text,
 "page_id": page_id,
 "filename": filename
}
basic_retriever.insert(data)

query = "O.Le Peuch Payout Results in percentage according to SLB Financial Objectives"
batch_query = colpali_processor.process_queries([query]).to(device)
embeddings_query = torch.unbind(colpali_model(**batch_query).to("cpu"))[0].float().numpy()
colbert_retriever_result = colbert_retriever.search(embeddings_query, topk=3)
colbert_retriever_result

[{ 'score' : 20.13466208751197, 
 'page_id' : 38, 
 'fig_id' : 1, 
 'filename' : 'SLB-2023-Annual-Report.pdf' , 
 'content' : '/9j/4AAQSkZJRgABAQAAAQABAAD/...' }, 
{ 'score' : 20.13466208751197, 
 'page_id' : 38, 
 'fig_id' : 1, 
 'filename' : 'SLB-2023-Annual-Report.pdf' , 
 'content' : '/9j/4AAQSkZJRgABAQAAAQABAAD/...' }, 
{ 'score' : 15.088707166083623，
 'page_id'：41，
 'fig_id'：1，
 'filename'：'SLB-2023-Annual-Report.pdf'，
 'content'：'/9j/4AAQSkZJRgABAQAAAQABAAD/...' }]

query = "Potential Payout as a % of Target Opportunity"
basic_retriever_result = basic_retriever.search(embed_model.encode(query), topk=3)
basic_retriever_result

[{'score': 0.6565427184104919,
 'page_id': 38,
 'filename': 'SLB-2023-Annual-Report.pdf',
 'content': 'Short-Term Cash Incentive Awards\n\nWe pay performance-based short-term (annual) cash incentives to\nour ... (truncated)'},
 {'score': 0.6533020734786987,
 'page_id': 40,
 'filename': 'SLB-2023-Annual-Report.pdf',
 'content': "Free Cash Flow Targets and Results\n\nIn January 2023, our Compensation Committee considered SLB's\n2022 ... (truncated)"},
 {'score': 0.6505128145217896,
 'page_id': 59,
 'filename': 'SLB-2023-Annual-Report.pdf',
 'content': "Executive Compensation Tables\n\nChange in Control\n\nUnder our omnibus incentive plans, in the event of ... (truncated)"}]

import os
from groq import Groq

# Groq API-Llama4
os.environ["GROQ_API_KEY"] = "<your-api-key>"
client_groq = Groq()

def url_conversion(img_base64):
 return f"data:image/jpeg;base64,{img_base64}"

def llama4_inference(messages, token=1024):
 completion = client_groq.chat.completions.create(
 model="meta-llama/llama-4-maverick-17b-128e-instruct",
 messages=messages,
 temperature=0.1,
 max_completion_tokens=token,
 top_p=1,
 stream=True,
 stop=None,
 )
 inference_result = ""
 for chunk in completion:
 chunk_inference = chunk.choices[0].delta.content or ""
 inference_result += chunk_inference
 text = inference_result
 return text

user_query = "I want to know the payout"
batch_query = colpali_processor.process_queries([user_query]).to(device)
embeddings_query = torch.unbind(colpali_model(**batch_query).to("cpu"))[0].float().numpy()
colbert_retriever_result = colbert_retriever.search(embeddings_query, topk=3)
basic_retriever_result = basic_retriever.search(embed_model.encode(user_query), topk=3)

system_instruction = """
You are a helpful assistant designed to answer user queries based on document-related content.

You will be provided with two types of context:
1. Text-based context — extracted textual content from documents.
2. Image-based context — visual content (e.g., figures, tables, or screenshots) extracted from documents.

Your tasks are:
- Analyze the user query and determine the appropriate response using the available context.
- Decide whether the answer requires information from the image-based context.

If the image context is necessary to answer the query:
- Set "need_image" to True.
- Set "image_index" to the appropriate index of the image used (e.g., 0 for the first image, 1 for the second, and so on).
- Include a clear explanation or reasoning in the response.

If the image context is **not** needed:
- Set "need_image" to False.
- Set "image_index" to -1.

All responses **must be returned in strict JSON format**:
{"response": <string>, "need_image": <true|false>, "image_index": <int>}

If you are unsure or cannot answer based on the given context, clearly state that you do not know.

Examples:
{"response": "The chart in image 1 shows the revenue trend.", "need_image": true, "image_index": 1}
{"response": "The policy details are outlined in the text section.", "need_image": false, "image_index": -1}
"""

#定义有效载荷内容
payload_content = [{
 "type": "text",
 "text": f"User Query: {user_query}"
 }]

# 构造正在检索的图像URL
for i in range(len(colbert_retriever_result)):
 img_payload = {
 "type": "image_url",
 "image_url": {"url":url_conversion(colbert_retriever_result[i]["content"])}
 }
 payload_content.append(img_payload)

# 构建基于文本的上下文
for i in range(len(basic_retriever_result)):
 txt_payload = {
 "type": "text",
 "text": f"Text-based Context #{i+1}:\n{basic_retriever_result[i]['content']}"
 }
 payload_content.append(txt_payload)

#创建最终消息形式
messages = [
 {
 "role": "system",
 "content": system_instruction
 },
 {
 "role": "user",
 "content": payload_content
 }
]

[{'role': 'system',
 'content': '\nYou are a helpful assistant designed to answer user queries based on document-related content.\n\nYou will be provided with two types of context:\n1. Text-based ... (truncated)'},
 {'role': 'user',
 'content': [{'type': 'text',
 'text': 'User Query: I want to know the payout'},
 {'type': 'image_url',
 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/...'}},
 {'type': 'image_url',
 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/...'}},
 {'type': 'image_url',
 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/...'}},
 {'type': 'text',
 'text': 'Text-based Context #1:\nExecutive Compensation Tables\n\nSummary Compensation Table\n\nThe following table sets forth information regarding the total compensation ... (truncated)'},
 {'type': 'text',
 'text': 'Text-based Context #2:\nExecutive Compensation Tables\n\nGrants of Plan-Based Awards in 2023\n\nThe following table provides additional information regarding cash ... (truncated)'},
 {'type': 'text',
 'text': 'Text-based Context #3:\nPay vs. Performance Comparison\n\nPay vs. Performance Comparison\n\nAs discussed in the CD&A above, our Compensation Committee has implemented ... (truncated)'}]}

import json
import re 

chat_result = llama4_inference(messages) 
chat_result = re.findall( r'\{[^{}]+\}' , chat_result) 
chat_result = json.loads(chat_result[- 1 ]) 
chat_result

{'response': 'The payout varies based on the performance metric. For Relative TSR Percentile Rank, Delta ROCE, and FCF Margin, the payouts are illustrated in the provided graphs. For example, at a Relative TSR Percentile Rank of 60%, the payout is 60%; at a Delta ROCE of 0 bps, the payout is 100%; and at an FCF Margin of 10%, the payout is 100%.',
 'need_image': True,
 'image_index': 0}

if chat_result[ "need_image" ]: 
 img_content = colbert_retriever_result[chat_result[ 'image_index' ]][ 'content' ] 
else : 
 img_content = ""

 output_response = { 
 "response" :chat_result[ "response" ], 
 "need_image" :chat_result[ "need_image" ], 
 "img_base64" :img_content 
 } 
output_response

{'response': 'The payout varies based on the performance metric. For Relative TSR Percentile Rank, Delta ROCE, and FCF Margin, the payouts are illustrated in the provided graphs. For example, at a Relative TSR Percentile Rank of 60%, the payout is 60%; at a Delta ROCE of 0 bps, the payout is 100%; and at an FCF Margin of 10%, the payout is 100%.',
 'need_image': True,
 'img_base64': '/9j/4AAQSkZJRgABAQAAAQABAAD/...'}