chenkovsky · December 16, 2024 07:52 · chenkovsky · Dec 16, 2024
diff --git a/lance_tokenizer_plugin_draft.h b/lance_tokenizer_plugin_draft.h
 #ifndef __LANCE_TOKENIZER_H__
 #define __LANCE_TOKENIZER_H__ 1
 #include <stddef.h>
 #include <stdint.h>

 typedef struct LanceTokenizerFactoryT* LanceTokenizerFactory;
 typedef struct LanceTokenizerT* LanceTokenizer;
 typedef struct LanceTokenStreamT* LanceTokenStream;

 struct LanceTokenT {
    size_t offset_from;
    /// Offset (byte index) of the last character of the token + 1.
    /// The text that generated the token should be obtained by
    /// &text[token.offset_from..token.offset_to]
    size_t offset_to;

    /// Position, expressed in number of tokens.
    size_t position;

    /// Actual text content of the token.
    const char* text;

    /// Is the length expressed in term of number of original tokens.
    size_t position_length;
 };

 typedef struct LanceTokenT* LanceToken;

 struct LanceTokenizerPluginT {

    // lance will call this when load dynamic library
    int (*create_factory)(int argc, const char * const* argv, LanceTokenizerFactory *factory_ptr);

    // lance will call this when unload dynamic library
    void (*destroy_factory)(LanceTokenizerFactory *factory_ptr);

    // create tokenizer
    int (*create_tokenizer)(LanceTokenizerFactory factory, int argc, const char * const* argv, LanceTokenizer *tokenizer_ptr);

    // destroy tokenizer
    void (*destroy_tokenizer)(LanceTokenizer *tokenizer_ptr);

    // create token stream
    int (*create_stream)(const LanceTokenizer tokenizer, const char* text, size_t length, LanceTokenStream *stream_ptr);

    // destroy token stream
    void (*destroy_stream)(LanceTokenStream *stream_ptr);

    // if there's token, return 1, and update token. otherwise return 0.
    int (*iter_token)(LanceTokenStream stream, LanceToken token);

    const char* (*strerror)(int errnum);
 };

 typedef struct LanceTokenizerPluginT* LanceTokenizerPlugin;

 // return 0 if init succeeded.
 typedef int(*plugin_metadata)(LanceTokenizerPlugin plugin);

 #endif
	#ifndef __LANCE_TOKENIZER_H__
	#define __LANCE_TOKENIZER_H__ 1
	#include <stddef.h>
	#include <stdint.h>

	typedef struct LanceTokenizerFactoryT* LanceTokenizerFactory;
	typedef struct LanceTokenizerT* LanceTokenizer;
	typedef struct LanceTokenStreamT* LanceTokenStream;

	struct LanceTokenT {
	size_t offset_from;
	/// Offset (byte index) of the last character of the token + 1.
	/// The text that generated the token should be obtained by
	/// &text[token.offset_from..token.offset_to]
	size_t offset_to;

	/// Position, expressed in number of tokens.
	size_t position;

	/// Actual text content of the token.
	const char* text;

	/// Is the length expressed in term of number of original tokens.
	size_t position_length;
	};

	typedef struct LanceTokenT* LanceToken;

	struct LanceTokenizerPluginT {

	// lance will call this when load dynamic library
	int (create_factory)(int argc, const char const* argv, LanceTokenizerFactory *factory_ptr);

	// lance will call this when unload dynamic library
	void (destroy_factory)(LanceTokenizerFactory factory_ptr);

	// create tokenizer
	int (create_tokenizer)(LanceTokenizerFactory factory, int argc, const char const* argv, LanceTokenizer *tokenizer_ptr);

	// destroy tokenizer
	void (destroy_tokenizer)(LanceTokenizer tokenizer_ptr);

	// create token stream
	int (create_stream)(const LanceTokenizer tokenizer, const char text, size_t length, LanceTokenStream *stream_ptr);

	// destroy token stream
	void (destroy_stream)(LanceTokenStream stream_ptr);

	// if there's token, return 1, and update token. otherwise return 0.
	int (*iter_token)(LanceTokenStream stream, LanceToken token);

	const char* (*strerror)(int errnum);
	};

	typedef struct LanceTokenizerPluginT* LanceTokenizerPlugin;

	// return 0 if init succeeded.
	typedef int(*plugin_metadata)(LanceTokenizerPlugin plugin);

	#endif