Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,21 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-BF16.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ1_M.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ1_S.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ2_M.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ2_XXS.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ3_M.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ3_S.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ3_XS.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ3_XXS.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ4_NL.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
+jina-reranker-v3-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,99 @@
----
-license: cc-by-nc-4.0
----
+---
+pipeline_tag: text-ranking
+tags:
+- gguf
+- reranker
+- qwen3
+- llama-cpp
+language:
+- multilingual
+base_model: jinaai/jina-reranker-v3
+base_model_relation: quantized
+inference: false
+license: cc-by-nc-4.0
+library_name: llama.cpp
+---
+
+# jina-reranker-v3-GGUF
+
+GGUF quantizations of [jina-reranker-v3](https://huggingface.co/jinaai/jina-reranker-v3) using llama.cpp. A 0.6B parameter multilingual listwise reranker quantized for efficient inference.
+
+## Requirements
+
+- Python 3.8+
+- llama.cpp binaries (`llama-embedding` and `llama-tokenize`)
+- Hanxiao's llama.cpp fork recommended: https://github.com/hanxiao/llama.cpp
+
+## Installation
+
+```bash
+pip install numpy safetensors
+```
+
+## Files
+
+- `jina-reranker-v3-BF16.gguf` - Quantized model weights (BF16, 1.1GB)
+- `projector.safetensors` - MLP projector weights (3MB)
+- `rerank.py` - Reranker implementation
+
+## Usage
+
+```python
+from rerank import GGUFReranker
+
+# Initialize reranker
+reranker = GGUFReranker(
+    model_path="jina-reranker-v3-BF16.gguf",
+    projector_path="projector.safetensors",
+    llama_embedding_path="/path/to/llama-embedding"
+)
+
+# Rerank documents
+query = "What is the capital of France?"
+documents = [
+    "Paris is the capital and largest city of France.",
+    "Berlin is the capital of Germany.",
+    "The Eiffel Tower is located in Paris."
+]
+
+results = reranker.rerank(query, documents)
+
+for result in results:
+    print(f"Score: {result['relevance_score']:.4f}, Doc: {result['document'][:50]}...")
+```
+
+## API
+
+### `GGUFReranker.rerank(query, documents, top_n=None, return_embeddings=False, instruction=None)`
+
+**Arguments:**
+- `query` (str): Search query
+- `documents` (List[str]): Documents to rerank
+- `top_n` (int, optional): Return only top N results
+- `return_embeddings` (bool): Include embeddings in output
+- `instruction` (str, optional): Custom ranking instruction
+
+**Returns:**
+List of dicts with keys: `index`, `relevance_score`, `document`, and optionally `embedding`
+
+
+
+## Citation
+
+If you find `jina-reranker-v3` useful in your research, please cite the [original paper](https://arxiv.org/abs/2509.25085):
+
+```bibtex
+@misc{wang2025jinarerankerv3lateinteractiondocument,
+      title={jina-reranker-v3: Last but Not Late Interaction for Document Reranking},
+      author={Feng Wang and Yuqing Li and Han Xiao},
+      year={2025},
+      eprint={2509.25085},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2509.25085},
+}
+```
+
+## License
+
+This MLX implementation follows the same CC BY-NC 4.0 license as the original model. For commercial usage inquiries, please [contact Jina AI](https://jina.ai/contact-sales/).

jina-reranker-v3-BF16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e0e94dd584f84bab2a83254d38b93699ae5f40405422b4f419874aca68e6313
+size 1198785888

jina-reranker-v3-IQ1_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ed1e761349bcabe9d91efb865996eade10731f9a5dbbd07ebd1eb8a97ebd77b
+size 216655456

jina-reranker-v3-IQ1_S.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5dcad4805b2c4a8d86eb584fd14586d278a6b971b9ab89950d109a2d3a9f555
+size 208619104

jina-reranker-v3-IQ2_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff7c65d50b7263fe213f228bd5d68629cf958cb2b1cc04896a47d2ffe4815fea
+size 265512544

jina-reranker-v3-IQ2_XXS.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:557af10823f1195edbffa02404572242f53caee200199c8ef153701cfb56efc8
+size 230049376

jina-reranker-v3-IQ3_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f072998e87864ad5475732dd747fd8e20e959fc89a3f3ca25684726696f9fbe
+size 336630368

jina-reranker-v3-IQ3_S.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8733f7f0f640b9ffc1b7978fdd69d9271cba9f1d8069d2c7ce0311b52496ac2
+size 323678816

jina-reranker-v3-IQ3_XS.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52d4227ffb43eeb0bceaab9441512010948aefb4be3994ffd4f132f248eebeff
+size 313356896

jina-reranker-v3-IQ3_XXS.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3bbd42148f40b8d22347e8c25ecdbffd9a30025ded775c2c5953ba8bacf887d
+size 279619168

jina-reranker-v3-IQ4_NL.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10a5b0f0d8ed9dcd4b5adeddfe371b12fadb584bcafa69f5ceacd5e48104c1ce
+size 382169696

jina-reranker-v3-IQ4_XS.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0480feaf9b3bcff664707c7263d1249489afffc7170c558301efdec4c7cbfa70
+size 368407136

jina-reranker-v3-Q2_K.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51198d403351d4596e6290b69aec04bdb9b4e9c7451b626941c6e2ac5b5d8a51
+size 296841824

jina-reranker-v3-Q3_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed67b76ff409aea52355b7db49fa61959ae9376055d1f3b57812587b2469c5dc
+size 347730528

jina-reranker-v3-Q4_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac307418158012c3c87a83e40835b2090cf979c5ac8e38f36322bc2e1ca43f51
+size 397308512

jina-reranker-v3-Q5_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b31ac0efae5de481f6d4661d2eac03d66392439b5a4fe977a87212b5227168f
+size 445018720

jina-reranker-v3-Q5_K_S.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0e024cbc5fa37a2fe5202b96357a3d834aeceb88a81497152873ec638c4e119
+size 437219936

jina-reranker-v3-Q6_K.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b53bb9e0081896b8dc5e116c0978b1bc3248db4205f341ca7a9e908ed1e191ff
+size 495710816

jina-reranker-v3-Q8_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac96f5c868b78e4f369c78386f7767ea6ed92167b349cf6006944541f620991e
+size 640050528

projector.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d67c38edb8f2010e26b9877aef98c0d1fd975ffb734a03597e318cb5de74a09
+size 3145912

rerank.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+import numpy as np
+import subprocess
+import tempfile
+import os
+from typing import Optional, List, Dict
+from safetensors import safe_open
+import json
+
+
+class MLPProjector:
+    """MLP projector to project hidden states to embedding space."""
+    def __init__(self, linear1_weight, linear2_weight):
+        self.linear1_weight = linear1_weight
+        self.linear2_weight = linear2_weight
+
+    def __call__(self, x):
+        # Linear 1
+        x = x @ self.linear1_weight.T
+        # ReLU
+        x = np.maximum(0, x)
+        # Linear 2
+        x = x @ self.linear2_weight.T
+        return x
+
+
+def load_projector(projector_path: str) -> MLPProjector:
+    """Load projector weights from safetensors file."""
+    with safe_open(projector_path, framework="numpy") as f:
+        w0 = f.get_tensor("projector.0.weight")
+        w2 = f.get_tensor("projector.2.weight")
+
+    return MLPProjector(w0, w2)
+
+
+def sanitize_input(text: str, special_tokens: Dict[str, str]) -> str:
+    """Remove special tokens from input text."""
+    for token in special_tokens.values():
+        text = text.replace(token, "")
+    return text
+
+
+def format_docs_prompts_func(
+    query: str,
+    docs: list[str],
+    instruction: Optional[str] = None,
+    special_tokens: Dict[str, str] = {},
+) -> str:
+    """Format query and documents into a prompt for the model."""
+    query = sanitize_input(query, special_tokens)
+    docs = [sanitize_input(doc, special_tokens) for doc in docs]
+
+    prefix = (
+        "<|im_start|>system\n"
+        "You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. "
+        "If the query is a question, how relevant a passage is depends on how well it answers the question. "
+        "If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. "
+        "If an instruction is provided, you should follow the instruction when determining the ranking."
+        "<|im_end|>\n<|im_start|>user\n"
+    )
+    suffix = "<|im_end|>\n<|im_start|>assistant\n"
+
+    doc_emb_token = special_tokens["doc_embed_token"]
+    query_emb_token = special_tokens["query_embed_token"]
+
+    prompt = (
+        f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. "
+        f"Rank the passages based on their relevance to query: {query}\n"
+    )
+
+    if instruction:
+        prompt += f'<instruct>\n{instruction}\n</instruct>\n'
+
+    doc_prompts = [f'<passage id="{i}">\n{doc}{doc_emb_token}\n</passage>' for i, doc in enumerate(docs)]
+    prompt += "\n".join(doc_prompts) + "\n"
+    prompt += f"<query>\n{query}{query_emb_token}\n</query>"
+
+    return prefix + prompt + suffix
+
+
+class GGUFReranker:
+    """GGUF-based implementation of jina-reranker-v3."""
+
+    def __init__(self, model_path: str = "jina-reranker-v3-BF16.gguf", projector_path: str = "projector.safetensors",
+                 llama_embedding_path: str = "/tmp/hanxiao-llama.cpp/build/bin/llama-embedding"):
+        """Initialize GGUF-based reranker."""
+        self.model_path = model_path
+        self.llama_embedding_path = llama_embedding_path
+        self.projector = load_projector(projector_path)
+
+        # Special tokens
+        self.special_tokens = {
+            "query_embed_token": "<|rerank_token|>",
+            "doc_embed_token": "<|embed_token|>"
+        }
+        self.doc_embed_token_id = 151670
+        self.query_embed_token_id = 151671
+
+    def _get_hidden_states(self, prompt: str) -> np.ndarray:
+        """Get per-token hidden states using llama-embedding CLI."""
+        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f:
+            f.write(prompt)
+            prompt_file = f.name
+
+        try:
+            result = subprocess.run(
+                [
+                    self.llama_embedding_path,
+                    '-m', self.model_path,
+                    '-f', prompt_file,
+                    '--pooling', 'none',
+                    '--embd-separator', '<#JINA_SEP#>',  # Preserve internal newlines
+                    '--embd-normalize', '-1',
+                    '--embd-output-format', 'json',
+                    '--ubatch-size', '512',
+                    '--ctx-size', '8192',
+                    '--flash-attn',
+                    '-ngl', '99'
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=True
+            )
+
+            output = json.loads(result.stdout)
+            embeddings = [item['embedding'] for item in output['data']]
+            return np.array(embeddings)
+        finally:
+            os.unlink(prompt_file)
+
+    def _tokenize(self, prompt: str) -> List[int]:
+        """Tokenize prompt to find special token positions."""
+        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f:
+            f.write(prompt)
+            prompt_file = f.name
+
+        try:
+            result = subprocess.run(
+                ['llama-tokenize', '-m', self.model_path, '-f', prompt_file],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.DEVNULL,
+                text=True,
+                check=True
+            )
+
+            tokens = []
+            for line in result.stdout.strip().split('\n'):
+                if '->' in line:
+                    token_id = int(line.split('->')[0].strip())
+                    tokens.append(token_id)
+            return tokens
+        finally:
+            os.unlink(prompt_file)
+
+    def rerank(
+        self,
+        query: str,
+        documents: List[str],
+        top_n: Optional[int] = None,
+        return_embeddings: bool = False,
+        instruction: Optional[str] = None
+    ) -> List[Dict]:
+        """Rerank documents based on relevance to query."""
+        # Format prompt
+        prompt = format_docs_prompts_func(
+            query,
+            documents,
+            instruction=instruction,
+            special_tokens=self.special_tokens
+        )
+
+        # Get per-token hidden states using llama-embedding CLI
+        embeddings = self._get_hidden_states(prompt)
+
+        # Tokenize to find special token positions
+        tokens = self._tokenize(prompt)
+        tokens_array = np.array(tokens)
+
+        query_embed_positions_in_tokens = np.where(tokens_array == self.query_embed_token_id)[0]
+        doc_embed_positions_in_tokens = np.where(tokens_array == self.doc_embed_token_id)[0]
+
+        if len(query_embed_positions_in_tokens) == 0:
+            raise ValueError(f"Query embed token (ID {self.query_embed_token_id}) not found in input")
+
+        if len(doc_embed_positions_in_tokens) == 0:
+            raise ValueError(f"Document embed tokens (ID {self.doc_embed_token_id}) not found in input")
+
+        # llama-embedding strips trailing newlines but preserves internal newlines (via --embd-separator)
+        # Token positions map directly to embedding indices
+        query_pos = query_embed_positions_in_tokens[0]
+        doc_positions = doc_embed_positions_in_tokens
+
+        # Extract embeddings at special token positions
+        query_hidden = embeddings[query_pos:query_pos+1]  # [1, hidden_size]
+        doc_hidden = embeddings[doc_positions]  # [num_docs, hidden_size]
+
+        # Project embeddings
+        query_embeds = self.projector(query_hidden)  # [1, 512]
+        doc_embeds = self.projector(doc_hidden)  # [num_docs, 512]
+
+        # Compute cosine similarity scores
+        # Broadcast query to match doc shape
+        query_expanded = np.tile(query_embeds, (len(doc_embeds), 1))  # [num_docs, 512]
+
+        # Cosine similarity
+        dot_product = np.sum(doc_embeds * query_expanded, axis=-1)  # [num_docs]
+        doc_norm = np.sqrt(np.sum(doc_embeds * doc_embeds, axis=-1))  # [num_docs]
+        query_norm = np.sqrt(np.sum(query_expanded * query_expanded, axis=-1))  # [num_docs]
+        scores = dot_product / (doc_norm * query_norm)  # [num_docs]
+
+        # Create results
+        results = []
+        for idx, (doc, score, embed) in enumerate(zip(documents, scores, doc_embeds)):
+            result = {
+                "index": idx,
+                "relevance_score": float(score),
+                "document": doc
+            }
+            if return_embeddings:
+                result["embedding"] = embed.tolist()
+            results.append(result)
+
+        # Sort by score descending
+        results.sort(key=lambda x: x["relevance_score"], reverse=True)
+
+        # Return top_n if specified
+        if top_n is not None:
+            results = results[:top_n]
+
+        return results
+
+
+if __name__ == "__main__":
+    # Test the reranker
+    reranker = GGUFReranker()
+
+    query = "What is the capital of France?"
+    documents = [
+        "Paris is the capital and largest city of France.",
+        "Berlin is the capital of Germany.",
+        "The Eiffel Tower is located in Paris."
+    ]
+
+    results = reranker.rerank(query, documents)
+    for result in results:
+        print(f"Doc {result['index']}: {result['relevance_score']:.4f} - {result['document'][:50]}...")