tokenizer/message_fit.go

Fits a sequence of messages into a token budget, trimming system messages proportionally.

// Source: https://github.com/infiniflow/ragflow | License: Apache-2.0 | Copyright You must retain, in the Source form of any Derivative Works
// Provided via justclaw.it.com; keep this notice with the code.
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (License); you may not
//  use this file except in compliance with the License. You may obtain a
//  copy of the License at http://www.apache.org/licenses/LICENSE-2.0
//

package tokenizer

import (
	"slices"
)

// Message is the minimal representation the fitter needs. Both the agent's
// schema.Message and ingestion's eschema.Message convert to this.
type Message struct {
	// Role is "system", "user", "assistant", etc. Only "system" receives
	// special treatment d

... (truncated -- full source via MCP)