Illia Dulskyi GuuD

Which GGUF is right for me? (Opinionated)

Good question! I am collecting human data on how quantization affects outputs. See here for more information: ggml-org/llama.cpp#5962

In the meantime, use the largest that fully fits in your GPU. If you can comfortably fit Q4_K_S, try using a model with more parameters.

	module Theli.Collections

	open System
	open System.Collections.Generic
	open System.ComponentModel
	open System.Runtime.CompilerServices
	open System.Text
	open System.Threading
	open Microsoft.FSharp.Collections // For ValueOption, etc.

	module InlineIfLambdaTests.Parsing
	open System
	open System.Collections.Generic
	open System.Numerics
	open System.Runtime.CompilerServices
	open Microsoft.FSharp.Core
	open System.Runtime.InteropServices
	type Inl = InlineIfLambdaAttribute
	#nowarn "9"
	module Core =

	open System
	open System.Runtime.CompilerServices
	open Microsoft.FSharp.Core
	open System.Runtime.InteropServices
	open Microsoft.FSharp.NativeInterop

	[<RequireQualifiedAccess>]
	module Stack =
	let inline alloc<'t when 't: unmanaged> (count: int) =
	let mutable ref = NativePtr.toByRef (NativePtr.stackalloc<'t> (count))

	module Threading =
	let private update =
	let event = Event<_>()
	EditorApplication.update <- System.Delegate.Combine(EditorApplication.update, EditorApplication.CallbackFunction event.Trigger) :?> EditorApplication.CallbackFunction
	event.Publish


	let private waitTask (op : 't when 't :> System.Threading.Tasks.Task) = async {
	while ((op.IsCompleted \|\| op.IsFaulted \|\| op.IsCanceled )\|> not) do
	do! Async.AwaitEvent update

	open System
	open System.Runtime.InteropServices
	open System.Runtime.CompilerServices
	module EntityId =
	type T = int
	let inline toValue T = T

	let inline key (batch: ^a when ^a : (member EntityId: EntityId.T)): EntityId.T =
	batch.EntityId

	open System

	// Replace with your type
	type MyType<'a> = Async<'a>
	type Internal<'a> = MyType<'a>

	// Replace with types that feel similar and can be converted to `MyType`
	type External1<'a> = System.Threading.Tasks.Task<'a>
	type External2<'a> = System.Threading.Tasks.ValueTask<'a>


	open System
	open Giraffe
	open Microsoft.AspNetCore.Http
	open System.Threading
	open System.Threading.Tasks

	module HttpHandlerHelpers=
	/// Converts an Async HttpHandler to a normal Giraffe Task based HttpHandler
	let inline convertAsyncToTask (asyncHandler : HttpFunc -> HttpContext -> Async<HttpContext option>) (next : HttpFunc) (ctx : HttpContext) : HttpFuncResult =

	// step0: quick intro about paket, fake, \|> operator and unit

	// ------------------ begin ----------------


	(*
	The problem: lots of boilerplate code to make sure the exit code is being handled properly in our build script.
	when we have more steps, the code will smell worse.
	*)