newConnectionEvictingClient (model/connection_evicting_client.go)

Wraps a gRPC backend to evict models on connection errors, forwarding all inference methods.

// Source: https://github.com/mudler/LocalAI | License: MIT | Copyright 2023-2025 Ettore Di Giacinto (mudler@localai.io)
// Provided via justclaw.it.com; keep this notice with the code.
package model

import (
	"context"
	"fmt"
	"sync"

	grpc "github.com/mudler/LocalAI/pkg/grpc"
	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
	"github.com/mudler/xlog"
	ggrpc "google.golang.org/grpc"
)

// ConnectionEvictingClient wraps a grpc.Backend. When any inference method
// fails with a connection error (server unreachable), it calls the evict
// callback to remove the model from the ModelLoader's cache. The error is
// still returned to the caller — the NEXT request will trigger rescheduling
// via SmartRouter.
type ConnectionEvictingClient struct {
	grpc.Backend
	modelID string
	evict   func()
	once

... (truncated -- full source via MCP)