adamdebreceni commented on code in PR #2107:
URL: https://github.com/apache/nifi-minifi-cpp/pull/2107#discussion_r3301683713
##########
extensions/llamacpp/processors/DefaultLlamaContext.cpp:
##########
@@ -73,9 +78,27 @@ DefaultLlamaContext::DefaultLlamaContext(const
std::filesystem::path& model_path
llama_sampler_chain_add(llama_sampler_,
llama_sampler_init_temp(*llama_sampler_params.temperature));
}
llama_sampler_chain_add(llama_sampler_,
llama_sampler_init_dist(LLAMA_DEFAULT_SEED));
+
+ if (!multimodal_model_path) {
+ logger->log_info("No multimodal model path provided");
+ return;
+ }
+
+ mtmd_context_params mparams = mtmd_context_params_default();
+ mparams.use_gpu = false;
+ mparams.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;
+
+ multimodal_ctx_ =
mtmd_init_from_file(multimodal_model_path->string().c_str(), llama_model_,
mparams);
+ if (!multimodal_ctx_) {
+ throw Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION,
fmt::format("Failed to load multimodal model from '{}'",
multimodal_model_path->string()));
+ }
+
+ logger->log_info("Successfully loaded multimodal model from '{}'",
multimodal_model_path->string());
Review Comment:
done
##########
extensions/llamacpp/processors/DefaultLlamaContext.cpp:
##########
@@ -85,47 +108,96 @@ DefaultLlamaContext::~DefaultLlamaContext() {
}
std::optional<std::string> DefaultLlamaContext::applyTemplate(const
std::vector<LlamaChatMessage>& messages) {
- std::vector<llama_chat_message> llama_messages;
- llama_messages.reserve(messages.size());
- std::transform(messages.begin(), messages.end(),
std::back_inserter(llama_messages),
- [](const LlamaChatMessage& msg) { return
llama_chat_message{.role = msg.role.c_str(), .content = msg.content.c_str()};
});
- std::string text;
- text.resize(DEFAULT_BUFFER_SIZE);
- const char * chat_template = llama_model_chat_template(llama_model_,
nullptr);
- int32_t res_size = llama_chat_apply_template(chat_template,
llama_messages.data(), llama_messages.size(), true, text.data(),
gsl::narrow<int32_t>(text.size()));
- if (res_size < 0) {
+ if (!chat_template_) {
return std::nullopt;
}
- if (res_size > gsl::narrow<int32_t>(text.size())) {
- text.resize(res_size);
- res_size = llama_chat_apply_template(chat_template, llama_messages.data(),
llama_messages.size(), true, text.data(), gsl::narrow<int32_t>(text.size()));
- if (res_size < 0) {
- return std::nullopt;
- }
+ common_chat_templates_inputs inputs;
+ for (auto& msg : messages) {
+ common_chat_msg chat_msg;
+ chat_msg.role = msg.role;
+ chat_msg.content = msg.content;
+ inputs.messages.push_back(std::move(chat_msg));
}
- text.resize(res_size);
+ inputs.enable_thinking = false; // TODO(adebreceni): MINIFICPP-2800
common_chat_templates_support_enable_thinking(chat_template_.get());
- return text;
+ return common_chat_templates_apply(chat_template_.get(), inputs).prompt;
}
-std::expected<GenerationResult, std::string>
DefaultLlamaContext::generate(const std::string& input,
std::function<void(std::string_view/*token*/)> token_handler) {
+namespace {
+
+struct mtmd_bitmap_deleter {
+ void operator()(mtmd_bitmap* val) { mtmd_bitmap_free(val); }
+};
+using unique_bitmap_ptr = std::unique_ptr<mtmd_bitmap, mtmd_bitmap_deleter>;
+
+struct mtmd_input_chunks_deleter {
+ void operator()(mtmd_input_chunks* val) { mtmd_input_chunks_free(val); }
+};
+using unique_mtmd_input_chunks_ptr = std::unique_ptr<mtmd_input_chunks,
mtmd_input_chunks_deleter>;
+
+} // namespace
+
+std::expected<GenerationResult, std::string>
DefaultLlamaContext::generate(const std::string& prompt, const
std::vector<std::vector<std::byte>>& files,
+ std::function<void(std::string_view/*token*/)> token_handler) {
GenerationResult result{};
auto start_time = std::chrono::steady_clock::now();
+ llama_memory_seq_rm(llama_get_memory(llama_ctx_), 0, -1, -1);
const llama_vocab * vocab = llama_model_get_vocab(llama_model_);
- std::vector<llama_token> tokenized_input = tokenizeInput(vocab, input);
- result.num_tokens_in = gsl::narrow<uint64_t>(tokenized_input.size());
+ llama_pos n_past = 0;
+ std::vector<llama_token> tokenized_input;
+ llama_batch batch = llama_batch_init(1, 0, 1);
+ auto batch_deleter = gsl::finally([&] {llama_batch_free(batch);});
+ batch.n_tokens = 1;
+ batch.n_seq_id[0] = 1;
+ batch.seq_id[0][0] = 0;
+ batch.logits[0] = true;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
