From 081bee8c643b1f6302e9edfe789ce2d5f0be6c77 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 14 Mar 2025 09:03:24 +0200 Subject: [PATCH 01/55] hparams : add SWA rope parameters (#12374) ggml-ci --- src/llama-context.cpp | 14 +++++--------- src/llama-graph.cpp | 4 ++-- src/llama-hparams.cpp | 2 +- src/llama-hparams.h | 4 +++- src/llama-model.cpp | 22 +++++++++++++++------- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 89fb33cbcd..4df6b18ec1 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -537,16 +537,12 @@ llm_graph_result_ptr llama_context::build_kv_self_shift( const int64_t n_head_kv = hparams.n_head_kv(il); const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); - float freq_base_l = cparams.rope_freq_base; - float freq_scale_l = cparams.rope_freq_scale; + const bool is_swa = hparams.is_swa(il); - // TODO: improve - if (model.arch == LLM_ARCH_GEMMA3) { - const bool is_sliding = hparams.is_sliding(il); - - freq_base_l = is_sliding ? 10000.0f : cparams.rope_freq_base; - freq_scale_l = is_sliding ? 1.0f : cparams.rope_freq_scale; - } + // note: the swa rope params could become part of the cparams in the future + // if we decide to make them configurable, like the non-sliding ones + const float freq_base_l = is_swa ? hparams.rope_freq_base_train_swa : cparams.rope_freq_base; + const float freq_scale_l = is_swa ? hparams.rope_freq_scale_train_swa : cparams.rope_freq_scale; ggml_tensor * rope_factors = kv_self->cbs.get_rope_factors(n_ctx_per_seq(), il); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 4a53e83929..1041ba29fb 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1403,9 +1403,9 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, ggml_cpy(ctx0, v_cur, v_cache_view)); } - const bool is_sliding = hparams.is_sliding(il); + const bool is_swa = hparams.is_swa(il); - const auto & kq_mask = is_sliding ? inp->get_kq_mask_swa() : inp->get_kq_mask(); + const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask(); const auto n_kv = kv_self->n; diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 58e98bf231..90dfe7a7fc 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -70,7 +70,7 @@ uint32_t llama_hparams::n_embd_v_s() const { return ssm_d_state * ssm_d_inner; } -bool llama_hparams::is_sliding(uint32_t il) const { +bool llama_hparams::is_swa(uint32_t il) const { if (il < n_layer) { return n_swa > 0 && n_swa_pattern > 0 && il % n_swa_pattern < (n_swa_pattern - 1); } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index e3091c8127..dbb7abd317 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -79,7 +79,9 @@ struct llama_hparams { float rope_attn_factor = 1.0f; float rope_freq_base_train; + float rope_freq_base_train_swa; float rope_freq_scale_train; + float rope_freq_scale_train_swa; uint32_t n_ctx_orig_yarn; float rope_yarn_log_mul; @@ -135,7 +137,7 @@ struct llama_hparams { // dimension of the recurrent state embeddings uint32_t n_embd_v_s() const; - bool is_sliding(uint32_t il) const; + bool is_swa(uint32_t il) const; }; static_assert(std::is_trivially_copyable::value, "llama_hparams must be trivially copyable"); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 5647d2ad62..cce943df08 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -475,6 +475,10 @@ void llama_model::load_hparams(llama_model_loader & ml) { } hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale; + // by default assume that the sliding-window layers use the same scaling type as the non-sliding-window layers + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false); // non-transformer models do not have attention heads @@ -877,6 +881,9 @@ void llama_model::load_hparams(llama_model_loader & ml) { { hparams.n_swa_pattern = 6; + hparams.rope_freq_base_train_swa = 10000.0f; + hparams.rope_freq_scale_train_swa = 1.0f; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -1346,13 +1353,14 @@ bool llama_model::load_tensors(llama_model_loader & ml) { const int i_gpu_start = std::max((int) hparams.n_layer - n_gpu_layers, (int) 0); const int act_gpu_layers = devices.empty() ? 0 : std::min(n_gpu_layers, (int)n_layer + 1); auto get_layer_buft_list = [&](int il) -> llama_model::impl::layer_dev { + const bool is_swa = il < (int) hparams.n_layer && hparams.is_swa(il); if (il < i_gpu_start || (il - i_gpu_start) >= act_gpu_layers) { - LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s\n", il, ggml_backend_dev_name(cpu_dev)); + LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s, is_swa = %d\n", il, ggml_backend_dev_name(cpu_dev), is_swa); return {cpu_dev, &pimpl->cpu_buft_list}; } const int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + n_devices(), float(il - i_gpu_start)/act_gpu_layers) - splits.begin(); auto * dev = devices.at(layer_gpu); - LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s\n", il, ggml_backend_dev_name(dev)); + LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s, is_swa = %d\n", il, ggml_backend_dev_name(dev), is_swa); return {dev, &pimpl->gpu_buft_list.at(dev)}; }; @@ -7381,10 +7389,10 @@ struct llm_build_gemma3 : public llm_graph_context { auto * inp_attn = build_attn_inp_kv_unified(true, true); for (int il = 0; il < n_layer; ++il) { - const bool is_sliding = hparams.is_sliding(il); + const bool is_swa = hparams.is_swa(il); - const float freq_base_l = is_sliding ? 10000.0f : freq_base; - const float freq_scale_l = is_sliding ? 1.0f : freq_scale; + const float freq_base_l = is_swa ? hparams.rope_freq_base_train_swa : cparams.rope_freq_base; + const float freq_scale_l = is_swa ? hparams.rope_freq_scale_train_swa : cparams.rope_freq_scale; // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); @@ -7973,7 +7981,7 @@ struct llm_build_cohere2 : public llm_graph_context { auto * inp_attn = build_attn_inp_kv_unified(true, true); for (int il = 0; il < n_layer; ++il) { - const bool is_sliding = hparams.is_sliding(il); + const bool is_swa = hparams.is_swa(il); // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il); @@ -8007,7 +8015,7 @@ struct llm_build_cohere2 : public llm_graph_context { cb(Vcur, "Vcur", il); } - if (is_sliding) { + if (is_swa) { Qcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); From c522ce4143a2b5c277f1e5f65cd570dbd0626466 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 14 Mar 2025 10:47:44 +0200 Subject: [PATCH 02/55] graph : simplify attn input build for unified KV cache (#12381) ggml-ci --- src/llama-graph.cpp | 14 ++----- src/llama-graph.h | 4 +- src/llama-model.cpp | 93 +++++++++++++++++++++++---------------------- 3 files changed, 53 insertions(+), 58 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 1041ba29fb..e4af507780 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1311,29 +1311,23 @@ ggml_tensor * llm_graph_context::build_attn( return cur; } -llm_graph_input_attn_kv_unified * llm_graph_context::build_attn_inp_kv_unified( - bool causal, - bool swa) const { +llm_graph_input_attn_kv_unified * llm_graph_context::build_attn_inp_kv_unified() const { const llama_kv_cache_unified * kv_self = static_cast(memory); auto inp = std::make_unique(hparams, cparams, kv_self); const auto n_kv = kv_self->n; - inp->self_kq_mask = causal - ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)) - : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); + inp->self_kq_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); //cb(inp->self_kq_mask, "KQ_mask", -1); ggml_set_input(inp->self_kq_mask); inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask; - if (swa) { + if (hparams.n_swa_pattern > 1) { GGML_ASSERT(hparams.n_swa > 0); - inp->self_kq_mask_swa = causal - ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)) - : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); + inp->self_kq_mask_swa = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); //cb(inp->self_kq_mask_swa, "KQ_mask_swa", -1); ggml_set_input(inp->self_kq_mask_swa); diff --git a/src/llama-graph.h b/src/llama-graph.h index b7a66d1898..c4328e6f9e 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -509,9 +509,7 @@ struct llm_graph_context { float kq_scale, int il) const; - llm_graph_input_attn_kv_unified * build_attn_inp_kv_unified( - bool causal, - bool swa) const; + llm_graph_input_attn_kv_unified * build_attn_inp_kv_unified() const; ggml_tensor * build_attn( llm_graph_input_attn_kv_unified * inp, diff --git a/src/llama-model.cpp b/src/llama-model.cpp index cce943df08..750a702ff7 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -784,9 +784,11 @@ void llama_model::load_hparams(llama_model_loader & ml) { hparams.n_swa = 2047; } else if (hparams.n_layer == 32 && hparams.n_head_kv(0) == 32 && hparams.n_ctx_train == 131072) { // default value for Phi-3-mini-128k-instruct + // note: this seems incorrect because the window is bigger than the train context? hparams.n_swa = 262144; } else if (hparams.n_layer == 40 && hparams.n_ctx_train == 131072) { // default value for Phi-3-medium-128k-instruct + // note: this seems incorrect because the window is equal to the train context? hparams.n_swa = 131072; } bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); @@ -3710,6 +3712,7 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: n_head_kv = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_head_kv(il); }, hparams.n_layer).c_str()); LLAMA_LOG_INFO("%s: n_rot = %u\n", __func__, hparams.n_rot); LLAMA_LOG_INFO("%s: n_swa = %u\n", __func__, hparams.n_swa); + LLAMA_LOG_INFO("%s: n_swa_pattern = %u\n", __func__, hparams.n_swa_pattern); LLAMA_LOG_INFO("%s: n_embd_head_k = %u\n", __func__, hparams.n_embd_head_k); LLAMA_LOG_INFO("%s: n_embd_head_v = %u\n", __func__, hparams.n_embd_head_v); LLAMA_LOG_INFO("%s: n_gqa = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_gqa(il); }, hparams.n_layer).c_str()); @@ -3871,7 +3874,7 @@ struct llm_build_llama : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; for (int il = 0; il < n_layer; ++il) { @@ -4034,7 +4037,7 @@ struct llm_build_deci : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; for (int il = 0; il < n_layer; ++il) { @@ -4192,7 +4195,7 @@ struct llm_build_baichuan : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = model.type == LLM_TYPE_7B ? build_inp_pos() : nullptr; - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -4310,7 +4313,7 @@ struct llm_build_xverse : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -4418,7 +4421,7 @@ struct llm_build_falcon : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * attn_norm; @@ -4543,7 +4546,7 @@ struct llm_build_grok : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -4697,7 +4700,7 @@ struct llm_build_dbrx : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -4821,7 +4824,7 @@ struct llm_build_starcoder : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); cb(pos, "pos_embd", -1); @@ -4924,7 +4927,7 @@ struct llm_build_refact : public llm_graph_context { inpL = build_inp_embd(model.tok_embd); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -5187,7 +5190,7 @@ struct llm_build_bloom : public llm_graph_context { inpL = build_inp_embd(model.tok_embd); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); inpL = build_norm(inpL, model.tok_norm, @@ -5292,7 +5295,7 @@ struct llm_build_mpt : public llm_graph_context { inpL = build_inp_embd(model.tok_embd); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); if (model.pos_embd) { // inp_pos - contains the positions @@ -5436,7 +5439,7 @@ struct llm_build_stablelm : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { // norm @@ -5587,7 +5590,7 @@ struct llm_build_qwen : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -5703,7 +5706,7 @@ struct llm_build_qwen2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -5818,7 +5821,7 @@ struct llm_build_qwen2vl : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); int sections[4]; std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); @@ -5938,7 +5941,7 @@ struct llm_build_qwen2moe : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -6087,7 +6090,7 @@ struct llm_build_phi2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { attn_norm_output = build_norm(inpL, @@ -6211,7 +6214,7 @@ struct llm_build_phi3 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, true); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { auto * residual = inpL; @@ -6357,7 +6360,7 @@ struct llm_build_plamo : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { @@ -6465,7 +6468,7 @@ struct llm_build_gpt2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); cb(pos, "pos_embd", -1); @@ -6573,7 +6576,7 @@ struct llm_build_codeshell : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { cur = build_norm(inpL, @@ -6686,7 +6689,7 @@ struct llm_build_orion : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -6807,7 +6810,7 @@ struct llm_build_internlm2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -6937,7 +6940,7 @@ struct llm_build_minicpm3 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -7141,7 +7144,7 @@ struct llm_build_gemma : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { // norm @@ -7251,7 +7254,7 @@ struct llm_build_gemma2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, true); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { // norm @@ -7386,7 +7389,7 @@ struct llm_build_gemma3 : public llm_graph_context { ggml_tensor * inp_pos = build_inp_pos(); // TODO: is causal == true correct? might need some changes - auto * inp_attn = build_attn_inp_kv_unified(true, true); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { const bool is_swa = hparams.is_swa(il); @@ -7515,7 +7518,7 @@ struct llm_build_starcoder2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -7828,7 +7831,7 @@ struct llm_build_command_r : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { @@ -7978,7 +7981,7 @@ struct llm_build_cohere2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, true); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { const bool is_swa = hparams.is_swa(il); @@ -8110,7 +8113,7 @@ struct llm_build_olmo : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -8232,7 +8235,7 @@ struct llm_build_olmo2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -8358,7 +8361,7 @@ struct llm_build_olmoe : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -8481,7 +8484,7 @@ struct llm_build_openelm : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { const int64_t n_head = hparams.n_head(il); @@ -8611,7 +8614,7 @@ struct llm_build_gptneox : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { cur = build_norm(inpL, @@ -8757,7 +8760,7 @@ struct llm_build_arctic : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -8889,7 +8892,7 @@ struct llm_build_deepseek : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; @@ -9054,7 +9057,7 @@ struct llm_build_deepseek2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -9274,7 +9277,7 @@ struct llm_build_bitnet : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -9532,7 +9535,7 @@ struct llm_build_t5_dec : public llm_graph_context { const int64_t n_outputs_enc = embd_enc->ne[1]; - auto * inp_attn_self = build_attn_inp_kv_unified(true, false); + auto * inp_attn_self = build_attn_inp_kv_unified(); auto * inp_attn_cross = build_attn_inp_cross(); for (int il = 0; il < n_layer; ++il) { @@ -9698,7 +9701,7 @@ struct llm_build_jais : public llm_graph_context { inpL = build_inp_embd(model.tok_embd); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { cur = build_norm(inpL, @@ -9794,7 +9797,7 @@ struct llm_build_chatglm : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -9926,7 +9929,7 @@ struct llm_build_nemotron : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -10049,7 +10052,7 @@ struct llm_build_exaone : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -10565,7 +10568,7 @@ struct llm_build_chameleon : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; From add2a3aa5a1571211aa5c7303b8e80c8d1824b91 Mon Sep 17 00:00:00 2001 From: Victor <194116445+dodekapod@users.noreply.github.com> Date: Fri, 14 Mar 2025 11:21:17 +0100 Subject: [PATCH 03/55] server: fix "--grammar-file" parameter (#12285) --- examples/server/utils.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 36ad276fd3..58cdd6af92 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -621,7 +621,9 @@ static json oaicompat_completion_params_parse( llama_params["chat_format"] = static_cast(chat_params.format); llama_params["prompt"] = chat_params.prompt; - llama_params["grammar"] = chat_params.grammar; + if (!chat_params.grammar.empty()) { + llama_params["grammar"] = chat_params.grammar; + } llama_params["grammar_lazy"] = chat_params.grammar_lazy; auto grammar_triggers = json::array(); for (const auto & trigger : chat_params.grammar_triggers) { From 8fcb563613e20a04dd9791f0a9b8a41086428c09 Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:47:05 +0100 Subject: [PATCH 04/55] Load all MoE experts during warmup (#11571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * llama : introduce llama_set_warmup() API call that controls warmup mode; use all MoE experts during warmup * common : use new API to enable warmup mode during model warmup --------- Co-authored-by: Stanisław Szymczyk --- common/common.cpp | 3 +++ include/llama.h | 4 ++++ src/llama-context.cpp | 13 ++++++++++++- src/llama-context.h | 1 + src/llama-cparams.h | 1 + src/llama-graph.cpp | 2 +- 6 files changed, 22 insertions(+), 2 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 8487e3834b..18ffb4e738 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1033,6 +1033,8 @@ struct common_init_result common_init_from_params(common_params & params) { if (params.warmup) { LOG_WRN("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__); + llama_set_warmup(lctx, true); + std::vector tmp; llama_token bos = llama_vocab_bos(vocab); llama_token eos = llama_vocab_eos(vocab); @@ -1063,6 +1065,7 @@ struct common_init_result common_init_from_params(common_params & params) { llama_kv_self_clear(lctx); llama_synchronize(lctx); llama_perf_context_reset(lctx); + llama_set_warmup(lctx, false); } iparams.model.reset(model); diff --git a/include/llama.h b/include/llama.h index e5286f0616..6a44be404d 100644 --- a/include/llama.h +++ b/include/llama.h @@ -945,6 +945,10 @@ extern "C" { // If set to true, the model will only attend to the past tokens LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); + // Set whether the model is in warmup mode or not + // If true, all model tensors are activated during llama_decode() to load and cache their weights. + LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup); + // Set abort callback LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 4df6b18ec1..c2fcce42a7 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -39,6 +39,7 @@ llama_context::llama_context( cparams.flash_attn = params.flash_attn; cparams.no_perf = params.no_perf; cparams.pooling_type = params.pooling_type; + cparams.warmup = false; cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx; cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base; @@ -948,6 +949,12 @@ void llama_context::set_causal_attn(bool value) { cparams.causal_attn = value; } +void llama_context::set_warmup(bool value) { + LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value); + + cparams.warmup = value; +} + void llama_context::set_adapter_lora( llama_adapter_lora * adapter, float scale) { @@ -1594,7 +1601,7 @@ void llama_context::output_reorder() { // int32_t llama_context::graph_max_nodes() const { - return std::max(8192, 5*model.n_tensors()); + return std::max(65536, 5*model.n_tensors()); } ggml_cgraph * llama_context::graph_init() { @@ -2372,6 +2379,10 @@ void llama_set_causal_attn(llama_context * ctx, bool causal_attn) { ctx->set_causal_attn(causal_attn); } +void llama_set_warmup(llama_context * ctx, bool warmup) { + ctx->set_warmup(warmup); +} + void llama_synchronize(llama_context * ctx) { ctx->synchronize(); } diff --git a/src/llama-context.h b/src/llama-context.h index 88df8950e4..04facb544c 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -64,6 +64,7 @@ struct llama_context { void set_embeddings (bool value); void set_causal_attn(bool value); + void set_warmup(bool value); void set_adapter_lora( llama_adapter_lora * adapter, diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 252012f3d9..30e550f023 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -29,6 +29,7 @@ struct llama_cparams { bool offload_kqv; bool flash_attn; bool no_perf; + bool warmup; enum llama_pooling_type pooling_type; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index e4af507780..4e90873397 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -577,7 +577,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) : n_embd_head_v (hparams.n_embd_head_v), n_embd_v_gqa (hparams.n_embd_v_gqa()), n_expert (hparams.n_expert), - n_expert_used (hparams.n_expert_used), + n_expert_used (cparams.warmup ? hparams.n_expert : hparams.n_expert_used), freq_base (cparams.rope_freq_base), freq_scale (cparams.rope_freq_scale), ext_factor (cparams.yarn_ext_factor), From 774973b8f3d5e375b0b74d58638eeb1817e950a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Fri, 14 Mar 2025 16:57:05 +0100 Subject: [PATCH 05/55] main : add -sysf / --system-prompt-file (#12249) (#12250) * add system_prompt_file * add -sysf / --system-prompt-file * remove system_prompt_file --- common/arg.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/common/arg.cpp b/common/arg.cpp index fe6a1eece7..240c699a2c 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -853,6 +853,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_excludes({LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"-sysf", "--system-prompt-file"}, "FNAME", + "a file containing the system prompt (default: none)", + [](common_params & params, const std::string & value) { + std::ifstream file(value); + if (!file) { + throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str())); + } + std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.system_prompt)); + if (!params.system_prompt.empty() && params.system_prompt.back() == '\n') { + params.system_prompt.pop_back(); + } + } + ).set_examples({LLAMA_EXAMPLE_MAIN})); add_opt(common_arg( {"--in-file"}, "FNAME", "an input file (repeat to specify multiple files)", From 9f2250ba722738ec0e6ab684636268a79160c854 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Fri, 14 Mar 2025 16:41:20 +0000 Subject: [PATCH 06/55] Add CLI arg to llama-run to adjust the number of threads used (#12370) We default to 4, sometimes we want to manually adjust this Signed-off-by: Eric Curtin --- examples/run/run.cpp | 131 +++++++++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 43 deletions(-) diff --git a/examples/run/run.cpp b/examples/run/run.cpp index 437f2533e5..462a6d1519 100644 --- a/examples/run/run.cpp +++ b/examples/run/run.cpp @@ -79,6 +79,7 @@ class Opt { ctx_params = llama_context_default_params(); model_params = llama_model_default_params(); context_size_default = ctx_params.n_batch; + n_threads_default = ctx_params.n_threads; ngl_default = model_params.n_gpu_layers; common_params_sampling sampling; temperature_default = sampling.temp; @@ -104,6 +105,7 @@ class Opt { ctx_params.n_batch = context_size >= 0 ? context_size : context_size_default; ctx_params.n_ctx = ctx_params.n_batch; + ctx_params.n_threads = ctx_params.n_threads_batch = n_threads >= 0 ? n_threads : n_threads_default; model_params.n_gpu_layers = ngl >= 0 ? ngl : ngl_default; temperature = temperature >= 0 ? temperature : temperature_default; @@ -116,12 +118,12 @@ class Opt { std::string chat_template_file; std::string user; bool use_jinja = false; - int context_size = -1, ngl = -1; + int context_size = -1, ngl = -1, n_threads = -1; float temperature = -1; bool verbose = false; private: - int context_size_default = -1, ngl_default = -1; + int context_size_default = -1, ngl_default = -1, n_threads_default = -1; float temperature_default = -1; bool help = false; @@ -159,53 +161,94 @@ class Opt { return 0; } + int parse_options_with_value(int argc, const char ** argv, int & i, bool & options_parsing) { + if (options_parsing && (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--context-size") == 0)) { + if (handle_option_with_value(argc, argv, i, context_size) == 1) { + return 1; + } + } else if (options_parsing && + (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "-ngl") == 0 || strcmp(argv[i], "--ngl") == 0)) { + if (handle_option_with_value(argc, argv, i, ngl) == 1) { + return 1; + } + } else if (options_parsing && (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--threads") == 0)) { + if (handle_option_with_value(argc, argv, i, n_threads) == 1) { + return 1; + } + } else if (options_parsing && strcmp(argv[i], "--temp") == 0) { + if (handle_option_with_value(argc, argv, i, temperature) == 1) { + return 1; + } + } else if (options_parsing && strcmp(argv[i], "--chat-template-file") == 0) { + if (handle_option_with_value(argc, argv, i, chat_template_file) == 1) { + return 1; + } + use_jinja = true; + } else { + return 2; + } + + return 0; + } + + int parse_options(const char ** argv, int & i, bool & options_parsing) { + if (options_parsing && (parse_flag(argv, i, "-v", "--verbose") || parse_flag(argv, i, "-v", "--log-verbose"))) { + verbose = true; + } else if (options_parsing && strcmp(argv[i], "--jinja") == 0) { + use_jinja = true; + } else if (options_parsing && parse_flag(argv, i, "-h", "--help")) { + help = true; + return 0; + } else if (options_parsing && strcmp(argv[i], "--") == 0) { + options_parsing = false; + } else { + return 2; + } + + return 0; + } + + int parse_positional_args(const char ** argv, int & i, int & positional_args_i) { + if (positional_args_i == 0) { + if (!argv[i][0] || argv[i][0] == '-') { + return 1; + } + + ++positional_args_i; + model_ = argv[i]; + } else if (positional_args_i == 1) { + ++positional_args_i; + user = argv[i]; + } else { + user += " " + std::string(argv[i]); + } + + return 0; + } + int parse(int argc, const char ** argv) { bool options_parsing = true; for (int i = 1, positional_args_i = 0; i < argc; ++i) { - if (options_parsing && (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--context-size") == 0)) { - if (handle_option_with_value(argc, argv, i, context_size) == 1) { - return 1; - } - } else if (options_parsing && - (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "-ngl") == 0 || strcmp(argv[i], "--ngl") == 0)) { - if (handle_option_with_value(argc, argv, i, ngl) == 1) { - return 1; - } - } else if (options_parsing && strcmp(argv[i], "--temp") == 0) { - if (handle_option_with_value(argc, argv, i, temperature) == 1) { - return 1; - } - } else if (options_parsing && - (parse_flag(argv, i, "-v", "--verbose") || parse_flag(argv, i, "-v", "--log-verbose"))) { - verbose = true; - } else if (options_parsing && strcmp(argv[i], "--jinja") == 0) { - use_jinja = true; - } else if (options_parsing && strcmp(argv[i], "--chat-template-file") == 0){ - if (handle_option_with_value(argc, argv, i, chat_template_file) == 1) { - return 1; - } - use_jinja = true; - } else if (options_parsing && parse_flag(argv, i, "-h", "--help")) { - help = true; - return 0; - } else if (options_parsing && strcmp(argv[i], "--") == 0) { - options_parsing = false; - } else if (positional_args_i == 0) { - if (!argv[i][0] || argv[i][0] == '-') { - return 1; - } + int ret = parse_options_with_value(argc, argv, i, options_parsing); + if (ret == 0) { + continue; + } else if (ret == 1) { + return ret; + } - ++positional_args_i; - model_ = argv[i]; - } else if (positional_args_i == 1) { - ++positional_args_i; - user = argv[i]; - } else { - user += " " + std::string(argv[i]); + ret = parse_options(argv, i, options_parsing); + if (ret == 0) { + continue; + } else if (ret == 1) { + return ret; + } + + if (parse_positional_args(argv, i, positional_args_i)) { + return 1; } } - if (model_.empty()){ + if (model_.empty()) { return 1; } @@ -232,6 +275,8 @@ class Opt { " Number of GPU layers (default: %d)\n" " --temp \n" " Temperature (default: %.1f)\n" + " -t, --threads \n" + " Number of threads to use during generation (default: %d)\n" " -v, --verbose, --log-verbose\n" " Set verbosity level to infinity (i.e. log all messages, useful for debugging)\n" " -h, --help\n" @@ -260,7 +305,7 @@ class Opt { " llama-run file://some-file3.gguf\n" " llama-run --ngl 999 some-file4.gguf\n" " llama-run --ngl 999 some-file5.gguf Hello World\n", - context_size_default, ngl_default, temperature_default); + context_size_default, ngl_default, temperature_default, n_threads_default); } }; From 92a391327e9201b9b5b32fdd3afb452026c22d4c Mon Sep 17 00:00:00 2001 From: Chenguang Li <757486878@qq.com> Date: Sat, 15 Mar 2025 09:31:08 +0800 Subject: [PATCH 07/55] [CANN]MUL_MAT optimization (#12382) --- ggml/src/ggml-cann/aclnn_ops.cpp | 8 ++++++-- ggml/src/ggml-cann/ggml-cann.cpp | 5 ----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cann/aclnn_ops.cpp b/ggml/src/ggml-cann/aclnn_ops.cpp index b2d857e1e5..6bb5d08349 100644 --- a/ggml/src/ggml-cann/aclnn_ops.cpp +++ b/ggml/src/ggml-cann/aclnn_ops.cpp @@ -2790,10 +2790,14 @@ static void ggml_cann_mul_mat_quant(ggml_backend_cann_context& ctx, (char*)output_buffer + batch1 * output_stride, ACL_FLOAT16, output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND, output_ne_offset); + int64_t antiquantGroupSize = 0; + if (src0->ne[0] > QK8_0) { + antiquantGroupSize = QK8_0; + } ACL_CHECK(aclnnWeightQuantBatchMatmulV2GetWorkspaceSize( acl_input_tensor, acl_weight_tensor, acl_scale_tensor, nullptr, - nullptr, nullptr, nullptr, QK8_0, acl_output_tensor, + nullptr, nullptr, nullptr, antiquantGroupSize, acl_output_tensor, &workspaceSize, &executor)); if (workspaceAddr == nullptr) { workspaceAddr = workspace_allocator.alloc(workspaceSize); @@ -2833,7 +2837,7 @@ static void ggml_cann_mul_mat_quant(ggml_backend_cann_context& ctx, ACL_CHECK(aclnnWeightQuantBatchMatmulV2GetWorkspaceSize( acl_input_tensor, acl_weight_tensor, acl_scale_tensor, - nullptr, nullptr, nullptr, nullptr, QK8_0, + nullptr, nullptr, nullptr, nullptr, antiquantGroupSize, acl_output_tensor, &workspaceSize, &executor)); ACL_CHECK(aclnnWeightQuantBatchMatmulV2( workspaceAddr, workspaceSize, executor, ctx.stream())); diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index b8d272cda6..68cd9920d1 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -1689,11 +1689,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, case GGML_OP_MUL_MAT: { switch (op->src[0]->type) { case GGML_TYPE_Q8_0: - // Current groupsize should not be greater than k-1 in - // aclnnWeightQuantBatchMatmulV2GetWorkspaceSize - if (op->src[0]->ne[0] <= QK8_0) { - return false; - } case GGML_TYPE_F16: case GGML_TYPE_F32: case GGML_TYPE_Q4_0: From b19bd064c09822cb81efe4a38abafab3e979c9ce Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Sat, 15 Mar 2025 15:19:30 +0100 Subject: [PATCH 08/55] SYCL : support non-contiguous tensors in binary ops (add, sub, etc) (#12399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * sycl : support non-contiguous tensors in binary ops * sycl : silence unused variable warning --------- Co-authored-by: Stanisław Szymczyk --- ggml/src/ggml-sycl/common.hpp | 87 ++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index a92988b7db..fdd07d9caf 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -474,6 +474,7 @@ static void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst, int ne0, int ne1, int ne2, int ne3, int ne10, int ne11, int ne12, int ne13, /*int s0, */ int s1, int s2, int s3, + /*int s00,*/ int s01, int s02, int s03, /*int s10,*/ int s11, int s12, int s13, const sycl::nd_item<3> &item_ct1) { const int i0s = item_ct1.get_local_range(2) * item_ct1.get_group(2) + @@ -495,9 +496,9 @@ static void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst, const int i12 = i2 % ne12; const int i13 = i3 % ne13; - const size_t i_src0 = i3*s3 + i2*s2 + i1*s1; + const size_t i_src0 = i3*s03 + i2*s02 + i1*s01; const size_t i_src1 = i13*s13 + i12*s12 + i11*s11; - const size_t i_dst = i_src0; + const size_t i_dst = i3*s3 + i2*s2 + i1*s1; const src0_t * src0_row = src0 + i_src0; const src1_t * src1_row = src1 + i_src1; @@ -515,6 +516,7 @@ static void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t int ne0, int ne1, int ne2, int ne3, int ne10, int ne11, int ne12, int ne13, /*int s0, */ int s1, int s2, int s3, + /*int s00,*/ int s01, int s02, int s03, /*int s10,*/ int s11, int s12, int s13, const sycl::nd_item<3> &item_ct1) { @@ -534,9 +536,9 @@ static void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t const int i12 = i2 % ne12; const int i13 = i3 % ne13; - const size_t i_src0 = i3*s3 + i2*s2 + i1*s1; + const size_t i_src0 = i3*s03 + i2*s02 + i1*s01; const size_t i_src1 = i13*s13 + i12*s12 + i11*s11; - const size_t i_dst = i_src0; + const size_t i_dst = i3*s3 + i2*s2 + i1*s1; const src0_t * src0_row = src0 + i_src0; const src1_t * src1_row = src1 + i_src1; @@ -566,9 +568,11 @@ struct bin_bcast_sycl { int nr[4] = { nr0, nr1, nr2, nr3 }; // collapse dimensions until first broadcast dimension - int64_t cne0[] = {ne0, ne1, ne2, ne3}; + int64_t cne[] = {ne0, ne1, ne2, ne3}; + int64_t cne0[] = {ne00, ne01, ne02, ne03}; int64_t cne1[] = {ne10, ne11, ne12, ne13}; - size_t cnb0[] = {nb0, nb1, nb2, nb3}; + size_t cnb[] = {nb0, nb1, nb2, nb3}; + size_t cnb0[] = {nb00, nb01, nb02, nb03}; size_t cnb1[] = {nb10, nb11, nb12, nb13}; auto collapse = [](int64_t cne[]) { cne[0] *= cne[1]; @@ -583,32 +587,41 @@ struct bin_bcast_sycl { cnb[3] *= cne[3]; }; - for (int i = 0; i < 4; i++) { - if (nr[i] != 1) { - break; - } - if (i > 0) { - collapse_nb(cnb0, cne0); - collapse_nb(cnb1, cne1); - collapse(cne0); - collapse(cne1); + if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)) { + for (int i = 0; i < 4; i++) { + if (nr[i] != 1) { + break; + } + if (i > 0) { + collapse_nb(cnb, cne); + collapse_nb(cnb0, cne0); + collapse_nb(cnb1, cne1); + collapse(cne); + collapse(cne0); + collapse(cne1); + } } } { - int64_t ne0 = cne0[0]; - int64_t ne1 = cne0[1]; - int64_t ne2 = cne0[2]; - int64_t ne3 = cne0[3]; + int64_t ne0 = cne[0]; + int64_t ne1 = cne[1]; + int64_t ne2 = cne[2]; + int64_t ne3 = cne[3]; int64_t ne10 = cne1[0]; int64_t ne11 = cne1[1]; int64_t ne12 = cne1[2]; int64_t ne13 = cne1[3]; - size_t nb0 = cnb0[0]; - size_t nb1 = cnb0[1]; - size_t nb2 = cnb0[2]; - size_t nb3 = cnb0[3]; + size_t nb0 = cnb[0]; + size_t nb1 = cnb[1]; + size_t nb2 = cnb[2]; + size_t nb3 = cnb[3]; + + size_t nb00 = cnb0[0]; + size_t nb01 = cnb0[1]; + size_t nb02 = cnb0[2]; + size_t nb03 = cnb0[3]; size_t nb10 = cnb1[0]; size_t nb11 = cnb1[1]; @@ -625,6 +638,28 @@ struct bin_bcast_sycl { size_t s12 = nb12 / sizeof(src1_t); size_t s13 = nb13 / sizeof(src1_t); + size_t s00 = nb00 / sizeof(src0_t); + size_t s01 = nb01 / sizeof(src0_t); + size_t s02 = nb02 / sizeof(src0_t); + size_t s03 = nb03 / sizeof(src0_t); + + GGML_UNUSED(s00); + + GGML_ASSERT(nb0 % sizeof(dst_t) == 0); + GGML_ASSERT(nb1 % sizeof(dst_t) == 0); + GGML_ASSERT(nb2 % sizeof(dst_t) == 0); + GGML_ASSERT(nb3 % sizeof(dst_t) == 0); + + GGML_ASSERT(nb00 % sizeof(src0_t) == 0); + GGML_ASSERT(nb01 % sizeof(src0_t) == 0); + GGML_ASSERT(nb02 % sizeof(src0_t) == 0); + GGML_ASSERT(nb03 % sizeof(src0_t) == 0); + + GGML_ASSERT(nb10 % sizeof(src1_t) == 0); + GGML_ASSERT(nb11 % sizeof(src1_t) == 0); + GGML_ASSERT(nb12 % sizeof(src1_t) == 0); + GGML_ASSERT(nb13 % sizeof(src1_t) == 0); + GGML_ASSERT(s0 == 1); GGML_ASSERT(s10 == 1); @@ -661,8 +696,8 @@ struct bin_bcast_sycl { [=](sycl::nd_item<3> item_ct1) { k_bin_bcast_unravel( src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3, - ne10, ne11, ne12, ne13, s1, s2, s3, s11, s12, - s13, item_ct1); + ne10, ne11, ne12, ne13, s1, s2, s3, s01, s02, + s03, s11, s12, s13, item_ct1); }); } } else { @@ -680,7 +715,7 @@ struct bin_bcast_sycl { [=](sycl::nd_item<3> item_ct1) { k_bin_bcast(src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3, ne10, ne11, ne12, ne13, - s1, s2, s3, s11, s12, s13, + s1, s2, s3, s01, s02, s03, s11, s12, s13, item_ct1); }); } From 3d35d87b4113648e224b837bb88e6b2c4c7f29e5 Mon Sep 17 00:00:00 2001 From: aubreyli Date: Sat, 15 Mar 2025 22:49:03 +0800 Subject: [PATCH 09/55] SYCL: Delete redundant plus sign and space (#12391) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 6977b705e4..ef7d5fa01a 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -3113,8 +3113,8 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, const int64_t i2 = i12; src0_row.data = src0_original + i02*nb02; - src1_row.data = src1_original + + i11*nb11 + i12*nb12; - dst_row.data = dst_original + i1*nb1 + i2*nb2; + src1_row.data = src1_original + i11*nb11 + i12*nb12; + dst_row.data = dst_original + i1*nb1 + i2*nb2; ggml_sycl_mul_mat(ctx, &src0_row, &src1_row, &dst_row); } From f4c3dd5daa3a79f713813cf1aabdc5886071061d Mon Sep 17 00:00:00 2001 From: marcoStocchi Date: Sat, 15 Mar 2025 17:23:11 +0100 Subject: [PATCH 10/55] llama-tts : add '-o' option (#12398) * added -o option to specify an output file name * llama-tts returns ENOENT in case of file write error note : PR #12042 is closed as superseded with this one. --- common/arg.cpp | 2 +- examples/tts/tts.cpp | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 240c699a2c..b6bfe6f89b 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1889,7 +1889,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.out_file = value; } - ).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA})); + ).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_TTS})); add_opt(common_arg( {"-ofreq", "--output-frequency"}, "N", string_format("output the imatrix every N iterations (default: %d)", params.n_out_freq), diff --git a/examples/tts/tts.cpp b/examples/tts/tts.cpp index c658f3182f..d953cadd62 100644 --- a/examples/tts/tts.cpp +++ b/examples/tts/tts.cpp @@ -87,11 +87,11 @@ struct wav_header { uint32_t data_size; }; -static void save_wav16(const std::string & fname, const std::vector & data, int sample_rate) { +static bool save_wav16(const std::string & fname, const std::vector & data, int sample_rate) { std::ofstream file(fname, std::ios::binary); if (!file) { - LOG_ERR("%s: Failed to open file '%s' for writing", __func__, fname.c_str()); - return; + LOG_ERR("%s: Failed to open file '%s' for writing.\n", __func__, fname.c_str()); + return false; } wav_header header; @@ -108,7 +108,7 @@ static void save_wav16(const std::string & fname, const std::vector & dat file.write(reinterpret_cast(&pcm_sample), sizeof(pcm_sample)); } - file.close(); + return file.good(); } static void fill_hann_window(int length, bool periodic, float * output) { @@ -536,6 +536,7 @@ static std::string audio_data_from_speaker(json speaker, const outetts_version t int main(int argc, char ** argv) { common_params params; + params.out_file = "output.wav"; params.prompt = ""; params.n_predict = 4096; @@ -1060,8 +1061,6 @@ lovely<|t_0.56|><|code_start|><|634|><|596|><|1766|><|1556|><|1306|><|1285|><|14 } #endif - const std::string fname = "output.wav"; - const int n_sr = 24000; // sampling rate // zero out first 0.25 seconds @@ -1072,11 +1071,15 @@ lovely<|t_0.56|><|code_start|><|634|><|596|><|1766|><|1556|><|1306|><|1285|><|14 LOG_INF("%s: time for spectral ops: %.3f ms\n", __func__, (ggml_time_us() - t_spec_start) / 1000.0f); LOG_INF("%s: total time: %.3f ms\n", __func__, (ggml_time_us() - t_main_start) / 1000.0f); - save_wav16(fname, audio, n_sr); + int retval = 0; - LOG_INF("%s: audio written to file '%s'\n", __func__, fname.c_str()); + if (save_wav16(params.out_file, audio, n_sr)) { + LOG_INF("%s: audio written to file '%s'\n", __func__, params.out_file.c_str()); + } else { + retval = ENOENT; + } llama_backend_free(); - return 0; + return retval; } From 7b61bcc87cfdeab88350e82df1c4b7be64331ea6 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Sun, 16 Mar 2025 18:22:05 +0100 Subject: [PATCH 11/55] ci : add --symlinks to xcframework zip command (#12409) This commit adds the --symlinks option to the zip command used to create the xcframework zip file. This is necessary to create symlinks in the zip file. Without this option, the Versions symlink is stored as a regular directory entry in the zip file, rather than as a symlink in the zip which causes the followig error in xcode: ```console Couldn't resolve framework symlink for '/Users/danbev/work/ai/llama.cpp/tmp_1/build-apple/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current': readlink(/Users/danbev/work/ai/llama.cpp/tmp_1/build-apple/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current): Invalid argument (22) ``` Refs: https://github.com/ggml-org/llama.cpp/pull/11996#issuecomment-2727026377 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e24293645..03cde0a484 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1379,7 +1379,7 @@ jobs: id: pack_artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} run: | - zip -r llama-${{ steps.tag.outputs.name }}-xcframework.zip build-apple/llama.xcframework + zip --symlinks -r llama-${{ steps.tag.outputs.name }}-xcframework.zip build-apple/llama.xcframework - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} From dc079cfdffa1141a6caf5d41a33d73a1ef03da55 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 16 Mar 2025 19:29:36 +0200 Subject: [PATCH 12/55] context : fix init of n_outputs (#12397) ggml-ci --- src/llama-context.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index c2fcce42a7..abb7e526f6 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -285,11 +285,15 @@ llama_context::llama_context( // reserve worst-case graph if (!hparams.vocab_only) { - uint32_t n_seqs = 1; // TODO: worst-case number of sequences - uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); + const uint32_t n_seqs = 1; // TODO: worst-case number of sequences + const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); llama_token token = model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph + // restore later + // TODO: something cleaner + const auto n_outputs_save = n_outputs; + // max number of outputs n_outputs = n_tokens; @@ -341,6 +345,8 @@ llama_context::llama_context( } } + n_outputs = n_outputs_save; + for (size_t i = 0; i < backend_ptrs.size(); ++i) { ggml_backend_t backend = backend_ptrs[i]; ggml_backend_buffer_type_t buft = backend_buft[i]; From 8ba95dca2065c0073698afdfcda4c8a8f08bf0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 16 Mar 2025 18:46:36 +0100 Subject: [PATCH 13/55] llama : fix OLMo-2-0325-32B-Instruct K-norm size (#12400) --- src/llama-model.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 750a702ff7..4b288d8f66 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1005,6 +1005,7 @@ void llama_model::load_hparams(llama_model_loader & ml) { case 16: type = LLM_TYPE_1B; break; case 32: type = LLM_TYPE_7B; break; case 40: type = LLM_TYPE_13B; break; + case 64: type = LLM_TYPE_32B; break; default: type = LLM_TYPE_UNKNOWN; } } break; @@ -2726,6 +2727,8 @@ bool llama_model::load_tensors(llama_model_loader & ml) { } break; case LLM_ARCH_OLMO2: { + const int64_t n_embd_head = n_embd / n_head; + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); // output @@ -2740,7 +2743,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0); layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0); layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_head_kv * n_embd_head}, 0); layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); From b3c9a65673a63a6c9a75da24ee00d13499494e0c Mon Sep 17 00:00:00 2001 From: Akarshan Biswas Date: Mon, 17 Mar 2025 07:15:12 +0530 Subject: [PATCH 14/55] SYCL: set extras only on GGML_TYPE_Q4_0 (#12366) * SYCL: set extras only on GGML_TYPE_Q4_0 * release tensor_extras in reset buffer interface --- ggml/src/ggml-sycl/ggml-sycl.cpp | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index ef7d5fa01a..05984d8c5a 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -333,10 +333,11 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, assert(tensor->view_src->buffer->buft == buffer->buft); return GGML_STATUS_SUCCESS; } - - ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; - tensor->extra = extra; - ctx->tensor_extras.push_back(extra); //used to release it when destroy ctx. + if (tensor->type == GGML_TYPE_Q4_0) { + ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; + tensor->extra = extra; + ctx->tensor_extras.push_back(extra); //used to release it when destroy ctx. + } if (ggml_is_quantized(tensor->type)) { // initialize padding to 0 to avoid possible NaN values @@ -486,6 +487,22 @@ catch (sycl::exception const &exc) { std::exit(1); } +static void ggml_backend_sycl_buffer_reset(ggml_backend_buffer_t buffer) { + GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); + if (buffer == nullptr) { + return; + } + + ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *) buffer->context; + + if (ctx != nullptr) { + for (ggml_tensor_extra_gpu * extra : ctx->tensor_extras) { + release_extra_gpu(extra); + } + ctx->tensor_extras.clear(); // reset the tensor_extras vector + } +} + static const ggml_backend_buffer_i ggml_backend_sycl_buffer_interface = { /* .free_buffer = */ ggml_backend_sycl_buffer_free_buffer, /* .get_base = */ ggml_backend_sycl_buffer_get_base, @@ -495,7 +512,7 @@ static const ggml_backend_buffer_i ggml_backend_sycl_buffer_interface = { /* .get_tensor = */ ggml_backend_sycl_buffer_get_tensor, /* .cpy_tensor = */ ggml_backend_sycl_buffer_cpy_tensor, /* .clear = */ ggml_backend_sycl_buffer_clear, - /* .reset = */ NULL, + /* .reset = */ ggml_backend_sycl_buffer_reset, }; // sycl buffer type @@ -576,7 +593,6 @@ ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { static std::mutex mutex; std::lock_guard lock(mutex); - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_buffer_type\n"); auto dev_count = ggml_backend_sycl_get_device_count(); @@ -3761,7 +3777,6 @@ bool ggml_backend_is_sycl(ggml_backend_t backend) { } int ggml_backend_sycl_get_device_count() { - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_get_device_count\n"); return ggml_sycl_info().device_count; } From 374101fd742bb35cb9bf46c86836e54d51191ffd Mon Sep 17 00:00:00 2001 From: Christian Kastner Date: Mon, 17 Mar 2025 10:05:23 +0100 Subject: [PATCH 15/55] cmake : enable building llama.cpp using system libggml (#12321) * cmake: Factor out compiler flag function from ggml llama.cpps's build requires it, too, and we may want to make use of it without add_subdirectory(ggml). * cmake: Enable building against system ggml This facilitates package maintenance for Linux distributions, where the libggml library most likely will be shipped as an individual package upon which a llama.cpp package depends. --- CMakeLists.txt | 10 +++++++++- cmake/common.cmake | 2 ++ ggml/cmake/common.cmake | 26 ++++++++++++++++++++++++++ ggml/src/CMakeLists.txt | 28 +--------------------------- 4 files changed, 38 insertions(+), 28 deletions(-) create mode 100644 ggml/cmake/common.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b2a1845e5..23cfbce5ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,8 @@ else() set(LLAMA_STANDALONE OFF) endif() +option(LLAMA_USE_SYSTEM_GGML "Use system libggml" OFF) + if (EMSCRIPTEN) set(BUILD_SHARED_LIBS_DEFAULT OFF) @@ -145,7 +147,13 @@ endif() # 3rd-party # -if (NOT TARGET ggml) +if (LLAMA_USE_SYSTEM_GGML) + message(STATUS "Using system-provided libggml, skipping ggml build") + find_package(ggml REQUIRED) + add_library(ggml ALIAS ggml::ggml) +endif() + +if (NOT TARGET ggml AND NOT LLAMA_USE_SYSTEM_GGML) add_subdirectory(ggml) # ... otherwise assume ggml is added by a parent CMakeLists.txt endif() diff --git a/cmake/common.cmake b/cmake/common.cmake index 0f54871e41..a5bb787f15 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1,3 +1,5 @@ +include("ggml/cmake/common.cmake") + function(llama_add_compile_flags) if (LLAMA_FATAL_WARNINGS) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake new file mode 100644 index 0000000000..1976d0ae9b --- /dev/null +++ b/ggml/cmake/common.cmake @@ -0,0 +1,26 @@ +function(ggml_get_flags CCID CCVER) + set(C_FLAGS "") + set(CXX_FLAGS "") + + if (CCID MATCHES "Clang") + set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return) + set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi) + + if ( + (CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR + (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0) + ) + list(APPEND C_FLAGS -Wdouble-promotion) + endif() + elseif (CCID STREQUAL "GNU") + set(C_FLAGS -Wdouble-promotion) + set(CXX_FLAGS -Wno-array-bounds) + + if (CCVER VERSION_GREATER_EQUAL 8.1.0) + list(APPEND CXX_FLAGS -Wextra-semi) + endif() + endif() + + set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE) + set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE) +endfunction() diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 52817510f6..a797e2b187 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -1,4 +1,5 @@ include(CheckCXXCompilerFlag) +include("../cmake/common.cmake") add_compile_definitions(GGML_SCHED_MAX_COPIES=${GGML_SCHED_MAX_COPIES}) @@ -24,33 +25,6 @@ if (NOT MSVC) endif() endif() -function(ggml_get_flags CCID CCVER) - set(C_FLAGS "") - set(CXX_FLAGS "") - - if (CCID MATCHES "Clang") - set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return) - set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi) - - if ( - (CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR - (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0) - ) - list(APPEND C_FLAGS -Wdouble-promotion) - endif() - elseif (CCID STREQUAL "GNU") - set(C_FLAGS -Wdouble-promotion) - set(CXX_FLAGS -Wno-array-bounds) - - if (CCVER VERSION_GREATER_EQUAL 8.1.0) - list(APPEND CXX_FLAGS -Wextra-semi) - endif() - endif() - - set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE) - set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE) -endfunction() - if (GGML_FATAL_WARNINGS) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") list(APPEND C_FLAGS -Werror) From 2f21123c1deb3ce1be3c0578c5f6980fe19ed077 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Mon, 17 Mar 2025 04:35:00 -0500 Subject: [PATCH 16/55] vulkan: Adjust coopmat2 tile sizes and selection heuristic (#12258) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 42 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index ff53bdfbe1..e46007a52f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1476,26 +1476,26 @@ static void ggml_vk_load_shaders(vk_device& device) { // spec constants and tile sizes for quant matmul (non-Qi_K) l_warptile_mmq = { 256, 128, 256, 64 }; m_warptile_mmq = { 256, 128, 128, 64 }; - s_warptile_mmq = { 256, 128, 128, 64 }; + s_warptile_mmq = { 256, 32, 64, 128 }; l_mmq_wg_denoms = { 128, 256, 1 }; m_mmq_wg_denoms = { 128, 128, 1 }; - s_mmq_wg_denoms = { 128, 128, 1 }; + s_mmq_wg_denoms = { 32, 64, 1 }; // spec constants and tile sizes for quant matmul (Qi_K) - l_warptile_mmq_k = { 256, 128, 512, 16 }; - m_warptile_mmq_k = { 256, 128, 256, 16 }; - s_warptile_mmq_k = { 256, 32, 128, 64 }; - l_mmq_wg_denoms_k = { 128, 512, 1 }; - m_mmq_wg_denoms_k = { 128, 256, 1 }; - s_mmq_wg_denoms_k = { 32, 128, 1 }; + l_warptile_mmq_k = { 256, 64, 128, 64 }; + m_warptile_mmq_k = { 256, 32, 64, 64 }; + s_warptile_mmq_k = { 256, 32, 32, 128 }; + l_mmq_wg_denoms_k = { 64, 128, 1 }; + m_mmq_wg_denoms_k = { 32, 64, 1 }; + s_mmq_wg_denoms_k = { 32, 32, 1 }; // spec constants and tile sizes for quant matmul_id - l_warptile_mmqid = { 256, 128, 128, 16 }; + l_warptile_mmqid = { 256, 128, 64, 16 }; m_warptile_mmqid = { 256, 128, 64, 16 }; - s_warptile_mmqid = { 256, 64, 64, 16 }; - l_mmqid_wg_denoms = { 128, 128, 1 }; + s_warptile_mmqid = { 256, 128, 64, 16 }; + l_mmqid_wg_denoms = { 128, 64, 1 }; m_mmqid_wg_denoms = { 128, 64, 1 }; - s_mmqid_wg_denoms = { 64, 64, 1 }; + s_mmqid_wg_denoms = { 128, 64, 1 }; l_align = 128; m_align = 64; @@ -3850,10 +3850,14 @@ static vk_pipeline ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ")"); if (ctx->device->coopmat2) { - if ((ctx->device->mul_mat_l[src0_type] && (m % mmp->l->wg_denoms[0]) == 0 && (n % mmp->l->wg_denoms[1]) == 0) || (!ctx->device->mul_mat_m[src0_type] && !ctx->device->mul_mat_s[src0_type])) { + // Use large shader when the N dimension is greater than the medium shader's tile size + uint32_t crossover_large = mmp->m->wg_denoms[1]; + if ((ctx->device->mul_mat_l[src0_type] && (n > crossover_large)) || (!ctx->device->mul_mat_m[src0_type] && !ctx->device->mul_mat_s[src0_type])) { return aligned ? mmp->a_l : mmp->l; } - if ((ctx->device->mul_mat_m[src0_type] && (m % mmp->m->wg_denoms[0]) == 0 && (n % mmp->m->wg_denoms[1]) == 0) || !ctx->device->mul_mat_s[src0_type]) { + // Use medium shader when the N dimension is greater than the small shader's tile size + uint32_t crossover_medium = mmp->s->wg_denoms[1]; + if ((ctx->device->mul_mat_m[src0_type] && (n > crossover_medium)) || !ctx->device->mul_mat_s[src0_type]) { return aligned ? mmp->a_m : mmp->m; } return aligned ? mmp->a_s : mmp->s; @@ -3898,13 +3902,17 @@ static void ggml_vk_matmul( } static vk_pipeline ggml_vk_guess_matmul_id_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, bool aligned, ggml_type src0_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ")"); + VK_LOG_DEBUG("ggml_vk_guess_matmul_id_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ")"); if (ctx->device->coopmat2) { - if ((ctx->device->mul_mat_id_l[src0_type] && (m % mmp->l->wg_denoms[0]) == 0 && (n % mmp->l->wg_denoms[1]) == 0) || (!ctx->device->mul_mat_id_m[src0_type] && !ctx->device->mul_mat_id_s[src0_type])) { + // Use large shader when the N dimension is greater than the medium shader's tile size + uint32_t crossover_large = mmp->m->wg_denoms[1]; + if ((ctx->device->mul_mat_id_l[src0_type] && (n > crossover_large)) || (!ctx->device->mul_mat_id_m[src0_type] && !ctx->device->mul_mat_id_s[src0_type])) { return aligned ? mmp->a_l : mmp->l; } - if ((ctx->device->mul_mat_id_m[src0_type] && (m % mmp->m->wg_denoms[0]) == 0 && (n % mmp->m->wg_denoms[1]) == 0) || !ctx->device->mul_mat_id_s[src0_type]) { + // Use medium shader when the N dimension is greater than the small shader's tile size + uint32_t crossover_medium = mmp->s->wg_denoms[1]; + if ((ctx->device->mul_mat_id_m[src0_type] && (n > crossover_medium)) || !ctx->device->mul_mat_id_s[src0_type]) { return aligned ? mmp->a_m : mmp->m; } return aligned ? mmp->a_s : mmp->s; From 891c63956dbfbdf7ed2ecd0b5882cff49dbfe90f Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Mon, 17 Mar 2025 04:41:59 -0500 Subject: [PATCH 17/55] vulkan: Pad N dimension of B matrix for coopmat2 perf, to avoid bounds checking (#12273) * vulkan: Pad N dimension of B matrix for coopmat2 perf, to avoid bounds checking --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 43 +++++++++++-------- .../vulkan-shaders/mul_mm_cm2.comp | 13 +++--- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index e46007a52f..a837b0dda4 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -29,6 +29,7 @@ #include "ggml-vulkan-shaders.hpp" +#define ROUNDUP_POW2(M, N) (((M) + (N) - 1) & ~((N) - 1)) #define CEIL_DIV(M, N) (((M) + (N)-1) / (N)) #define VK_VENDOR_ID_AMD 0x1002 @@ -368,6 +369,7 @@ struct vk_mat_mat_push_constants { uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; uint32_t k_split; uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3; + uint32_t padded_N; }; struct vk_mat_vec_push_constants { uint32_t ncols; uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; @@ -380,6 +382,7 @@ struct vk_mat_mat_id_push_constants { uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11; + uint32_t padded_N; }; struct vk_mat_vec_id_push_constants { uint32_t ncols; uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; @@ -3882,18 +3885,19 @@ static void ggml_vk_matmul( vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, - uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3) { + uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, + uint32_t padded_n) { VK_LOG_DEBUG("ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << (split_k_buffer.buffer != nullptr ? split_k_buffer.buffer->buffer : VK_NULL_HANDLE) << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ")"); ggml_vk_sync_buffers(subctx); if (split_k == 1) { - const vk_mat_mat_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, k, ne02, ne12, broadcast2, broadcast3 }; + const vk_mat_mat_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, k, ne02, ne12, broadcast2, broadcast3, padded_n }; ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d }, sizeof(vk_mat_mat_push_constants), &pc, { m, n, batch }); return; } GGML_ASSERT(batch_stride_d == m * n); - const vk_mat_mat_push_constants pc1 = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, CEIL_DIV(k, split_k), ne02, ne12, broadcast2, broadcast3 }; + const vk_mat_mat_push_constants pc1 = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, CEIL_DIV(k, split_k), ne02, ne12, broadcast2, broadcast3, padded_n }; // Make sure enough workgroups get assigned for split k to work ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, sizeof(vk_mat_mat_push_constants), &pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, batch }); ggml_vk_sync_buffers(subctx); @@ -3937,14 +3941,15 @@ static void ggml_vk_matmul_id( vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& ids, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, - uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11) { + uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11, + uint32_t padded_n) { VK_LOG_DEBUG("ggml_vk_matmul_id(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), ids: (" << ids.buffer->buffer << ", " << ids.offset << ", " << ids.size << "), " << "m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", " << "batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", " << "n_as: " << n_as << ", nei0: " << nei0 << ", nei1: " << nei1 << ", nbi1: " << nbi1 << ", ne11: " << ne11 << ")"); ggml_vk_sync_buffers(subctx); const vk_mat_mat_id_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, - nei0, nei1, nbi1, ne11 }; + nei0, nei1, nbi1, ne11, padded_n }; ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids }, sizeof(vk_mat_mat_id_push_constants), &pc, { m, nei1, n_as }); } @@ -4106,15 +4111,17 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub // Not implemented GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT - const int x_ne = ne01 * ne00; - const int y_ne = ne11 * ne10; - const int d_ne = ne11 * ne01; - const uint32_t kpad = ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ctx, mmp, ne01, ne11, qx_needs_dequant ? GGML_TYPE_F16 : src0->type)); const bool aligned = ne10 == kpad && ne01 > 8 && ne11 > 8; vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline(ctx, mmp, ne01, ne11, aligned, qx_needs_dequant ? GGML_TYPE_F16 : src0->type); + // Reserve extra storage in the N dimension for the Y matrix, so we can avoid bounds-checking + uint32_t padded_n = qy_needs_dequant ? ROUNDUP_POW2(ne11, pipeline->wg_denoms[1]) :ne11; + const int x_ne = ne01 * ne00; + const int y_ne = padded_n * ne10; + const int d_ne = ne11 * ne01; + const uint32_t split_k = ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, pipeline); const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type); @@ -4237,7 +4244,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub { d_D, d_buf_offset, d_sz * ne12 * ne13 }, { ctx->prealloc_split_k, 0, d_sz * ne12 * ne13 * split_k }, ne01, ne11, ne10, ne10, ne10, ne01, stride_batch_x, stride_batch_y, ne20*ne21, - split_k, ne12*ne13, ne02, ne12, r2, r3 + split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n ); // NOLINT } @@ -4688,15 +4695,17 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& // Not implemented GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT - const uint64_t x_ne = ne01 * ne00; - const uint64_t y_ne = ne11 * ne10; - const uint64_t d_ne = ne21 * ne20; - const uint32_t kpad = ggml_vk_align_size(ne10, ggml_vk_guess_matmul_id_pipeline_align(ctx, mmp, ne01, nei1, qx_needs_dequant ? GGML_TYPE_F16 : src0->type)); const bool aligned = ne10 == kpad && ne01 > 8 && nei1 > 8; vk_pipeline pipeline = ggml_vk_guess_matmul_id_pipeline(ctx, mmp, ne01, nei1, aligned, qx_needs_dequant ? GGML_TYPE_F16 : src0->type); + // Reserve extra storage in the N dimension for the Y matrix, so we can avoid bounds-checking + uint32_t padded_n = qy_needs_dequant ? ROUNDUP_POW2(ne11, pipeline->wg_denoms[1]) :ne11; + const uint64_t x_ne = ne01 * ne00; + const uint64_t y_ne = padded_n * ne10; + const uint64_t d_ne = ne21 * ne20; + const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type); const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne; @@ -4815,7 +4824,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& { d_D, d_buf_offset, d_sz * ne22 * ne23 }, { d_ids, ids_buf_offset, ids_sz }, ne01, ne21, ne10, ne10, ne10, ne01, stride_batch_x, stride_batch_y, ne20*ne21, - n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11 + n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, padded_n ); // NOLINT } @@ -6775,7 +6784,7 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t ctx, subctx, p, ggml_vk_subbuffer(d_X), ggml_vk_subbuffer(d_Y), ggml_vk_subbuffer(d_D), ggml_vk_subbuffer(ctx->prealloc_split_k), m, n, k, k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1 + split_k, batch, batch, batch, 1, 1, n ); } ggml_vk_ctx_end(subctx); @@ -7120,7 +7129,7 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, ctx, subctx, p, ggml_vk_subbuffer(qx_buf), ggml_vk_subbuffer(y_buf), ggml_vk_subbuffer(d_buf), ggml_vk_subbuffer(ctx->prealloc_split_k), m, n, k, k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1 + split_k, batch, batch, batch, 1, 1, n ); } ggml_vk_ctx_end(subctx); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 66dd2c860d..5b7a4efe2c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -48,6 +48,8 @@ layout (push_constant) uniform parameter uint broadcast2; uint broadcast3; #endif + // N dimension for the B matrix can be >= p.N + uint padded_N; } p; @@ -202,18 +204,19 @@ void main() { #endif // Use end_k rather than p.K as the dimension because that's what - // we need to bound check against when using split_k + // we need to bound check against when using split_k. + // Bounds check B against padded_N, but bounds check D against N. tensorLayoutA = setTensorLayoutDimensionNV(tensorLayoutA, p.M, end_k); - tensorLayoutB = setTensorLayoutDimensionNV(tensorLayoutB, p.N, end_k); + tensorLayoutB = setTensorLayoutDimensionNV(tensorLayoutB, p.padded_N, end_k); tensorLayoutD = setTensorLayoutDimensionNV(tensorLayoutD, p.N, p.M); tensorLayoutAClamp = setTensorLayoutDimensionNV(tensorLayoutAClamp, p.M, end_k); - tensorLayoutBClamp = setTensorLayoutDimensionNV(tensorLayoutBClamp, p.N, end_k); + tensorLayoutBClamp = setTensorLayoutDimensionNV(tensorLayoutBClamp, p.padded_N, end_k); tensorViewNV<2, false, 1, 0> tensorViewTranspose = createTensorViewNV(2, false, 1, 0); #if !defined(MUL_MAT_ID) // Detect a fast path where all loads are entirely in bounds and no clamping is required - if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.N && (start_k % BK) == 0 && (end_k % BK) == 0 && + if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % BK) == 0 && (end_k % BK) == 0 && #if QUANT_K == 1 (stride_a % 8) == 0 && #endif @@ -263,7 +266,7 @@ void main() { #ifdef MUL_MAT_ID bool unclampedB = true; #else - bool unclampedB = (ic + 1) * BN <= p.N && block_k + BK <= end_k && (block_k % 8) == 0; + bool unclampedB = (ic + 1) * BN <= p.padded_N && block_k + BK <= end_k && (block_k % 8) == 0; #endif if (unclampedA && unclampedB) { coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, (block_k & ~7), BK) DECODEFUNCA); From f07690c930f74d82d4f108e567c7092544847f77 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Mon, 17 Mar 2025 04:43:35 -0500 Subject: [PATCH 18/55] vulkan: use fp32 in coopmat2 q4_k dequant function (#12309) --- .../ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp index 4ccbe613af..8efe4653ff 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp @@ -178,7 +178,7 @@ float16_t dequantFuncQ4_K(const in decodeBufQ4_K bl, const in uint blockCoords[2 uvec4 v = bl128.block.q4k[0]; - const f16vec2 loadd = unpackFloat2x16(v.x); + const vec2 loadd = vec2(unpackFloat2x16(v.x)); uint32_t sc; uint32_t mbyte; @@ -199,15 +199,15 @@ float16_t dequantFuncQ4_K(const in decodeBufQ4_K bl, const in uint blockCoords[2 sc &= 0x3F; mbyte &= 0x3F; - const float16_t d = loadd.x * float16_t(sc); - const float16_t m = loadd.y * float16_t(mbyte); + const float d = loadd.x * float(sc); + const float m = loadd.y * float(mbyte); uint qs = uint32_t(bl16.block.qs[((idx & 0xC0) >> 2) + ((idx & 0x1E) >> 1)]); qs = (qs >> (b * 4 + 8 * (idx & 1))) & 0xF; - float16_t ret = d * float16_t(qs) - m; + float ret = d * float(qs) - m; - return ret; + return float16_t(ret); } layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K { From cf2270e4d3685ac46f4a166d8718997ba7cbc45a Mon Sep 17 00:00:00 2001 From: Daniele Date: Mon, 17 Mar 2025 12:42:33 +0100 Subject: [PATCH 19/55] vulkan: subgroup size tuning (#12087) * vulkan: subgroup size test * Vulkan: Add device architecture enum and logic to recognize AMD generations * vulkan: use new architecture logic to specify subgroup size * Initial vulkan subgroup size tuning for RDNA3 * vulkan: commonize RDNA subgroup tuning * vulkan: override subgroup size if required_subgroup_size = 0 * vulkan: disable warp 32 for RDNA3 * vulkan: fine tuned RDNA1 subgroup sizes * vulkan: adjusted subgroup size map * vulkan: fixed RDNA2 subgroup map --------- Co-authored-by: 0cc4m --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 155 +++++++++++++++++++++++++-- 1 file changed, 145 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index a837b0dda4..aa7281acbf 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -150,6 +150,66 @@ static void ggml_vk_destroy_buffer(vk_buffer& buf); static constexpr uint32_t mul_mat_vec_max_cols = 8; +enum vk_device_architecture { + OTHER, + AMD_GCN, + AMD_RDNA1, + AMD_RDNA2, + AMD_RDNA3, +}; + +static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& device) { + vk::PhysicalDeviceProperties props = device.getProperties(); + + if (props.vendorID == VK_VENDOR_ID_AMD) { + const std::vector ext_props = device.enumerateDeviceExtensionProperties(); + + bool amd_shader_core_properties = false; + bool integer_dot_product = false; + bool subgroup_size_control = false; + + for (const auto& properties : ext_props) { + if (strcmp("VK_AMD_shader_core_properties", properties.extensionName) == 0) { + amd_shader_core_properties = true; + } else if (strcmp("VK_KHR_shader_integer_dot_product", properties.extensionName) == 0) { + integer_dot_product = true; + } else if (strcmp("VK_EXT_subgroup_size_control", properties.extensionName) == 0) { + subgroup_size_control = true; + } + } + + if (!amd_shader_core_properties || !integer_dot_product || !subgroup_size_control) { + return vk_device_architecture::OTHER; + } + + vk::PhysicalDeviceProperties2 props2; + vk::PhysicalDeviceShaderCorePropertiesAMD shader_core_props_amd; + vk::PhysicalDeviceShaderIntegerDotProductPropertiesKHR integer_dot_props; + vk::PhysicalDeviceSubgroupSizeControlPropertiesEXT subgroup_size_control_props; + + props2.pNext = &shader_core_props_amd; + shader_core_props_amd.pNext = &integer_dot_props; + integer_dot_props.pNext = &subgroup_size_control_props; + + device.getProperties2(&props2); + + if (subgroup_size_control_props.maxSubgroupSize == 64 && subgroup_size_control_props.minSubgroupSize == 64) { + return vk_device_architecture::AMD_GCN; + } + if (subgroup_size_control_props.maxSubgroupSize == 64 && subgroup_size_control_props.minSubgroupSize == 32) { + // RDNA + if (shader_core_props_amd.wavefrontsPerSimd == 20) { + return vk_device_architecture::AMD_RDNA1; + } + if (integer_dot_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated) { + return vk_device_architecture::AMD_RDNA3; + } + return vk_device_architecture::AMD_RDNA2; + } + } + return vk_device_architecture::OTHER; +} + struct vk_device_struct { std::mutex mutex; @@ -162,6 +222,7 @@ struct vk_device_struct { bool pipeline_robustness; vk::Device device; uint32_t vendor_id; + vk_device_architecture architecture; vk_queue compute_queue; vk_queue transfer_queue; bool single_queue; @@ -1448,6 +1509,73 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec return supported; } +struct GpuPipelineConfig { + // GPU architecture identifier. + // Example: vk_device_architecture::AMD_GCN + vk_device_architecture arch; + + // Mapping of pipeline names to their specific subgroup sizes. + // Example: {"soft_max_f32", 64} + std::unordered_map pipelines; + + // Default subgroup size for this GPU. + // Defaults to 0 if not explicitly provided. + uint32_t default_subgroup_size = 0; +}; + +// Pipeline configuration for RDNA1 GPUs. +static const std::unordered_map rdna1_pipelines = { + {"soft_max", 64}, {"im2col", 64}, + {"argmax", 64}, {"mul_mat_vec", 64}, + {"mul_mat_vec_f16", 32}, {"mul_mat_vec_f32_f16", 32} +}; + +// Pipeline configuration for RDNA2 GPUs. +static const std::unordered_map rdna2_pipelines = { + {"soft_max", 64}, {"im2col", 64}, +}; + +static constexpr uint32_t RDNA_DEFAULT_SUBGROUP_SIZE = 32; + +// Define configurations for different GPUs. +static std::vector gpu_pipeline_configs = { + { + vk_device_architecture::AMD_RDNA1, + { + rdna1_pipelines, + }, + RDNA_DEFAULT_SUBGROUP_SIZE + }, + { + vk_device_architecture::AMD_RDNA2, + { + rdna2_pipelines, + }, + RDNA_DEFAULT_SUBGROUP_SIZE + }, +}; + +static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch) { + for (const auto &config : gpu_pipeline_configs) { + if (config.arch == arch) { + auto pipIt = config.pipelines.find(pipeline_name); + if (pipIt != config.pipelines.end()) { + return pipIt->second; + } + std::vector> sorted_pipelines(config.pipelines.begin(), config.pipelines.end()); + std::sort(sorted_pipelines.begin(), sorted_pipelines.end(), + [](const auto &a, const auto &b) { return a.first.size() > b.first.size(); }); + for (const auto &entry : sorted_pipelines) { + if (pipeline_name.find(entry.first) != std::string::npos) { + return entry.second; + } + } + return config.default_subgroup_size; + } + } + return 0; // If no matching configuration is found +} + static void ggml_vk_load_shaders(vk_device& device) { VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")"); @@ -1574,6 +1702,10 @@ static void ggml_vk_load_shaders(vk_device& device) { uint32_t parameter_count, uint32_t push_constant_size, std::array wg_denoms, const std::vector& specialization_constants, uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { + if (!require_full_subgroups && required_subgroup_size == 0) { + required_subgroup_size = get_subgroup_size(name, device->architecture); + } + if (!pipeline) { pipeline = std::make_shared(); pipeline->name = name; @@ -2250,7 +2382,7 @@ static void ggml_vk_load_shaders(vk_device& device) { device->need_compiles = false; } -static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props); +static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch); static vk_device ggml_vk_get_device(size_t idx) { VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")"); @@ -2279,6 +2411,8 @@ static vk_device ggml_vk_get_device(size_t idx) { device->physical_device = physical_devices[dev_num]; const std::vector ext_props = device->physical_device.enumerateDeviceExtensionProperties(); + device->architecture = get_device_architecture(device->physical_device); + const char* GGML_VK_PREFER_HOST_MEMORY = getenv("GGML_VK_PREFER_HOST_MEMORY"); device->prefer_host_memory = GGML_VK_PREFER_HOST_MEMORY != nullptr; @@ -2291,7 +2425,6 @@ static vk_device ggml_vk_get_device(size_t idx) { bool coopmat2_support = false; device->coopmat_support = false; - // Check if maintenance4 is supported for (const auto& properties : ext_props) { if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) { maintenance4_support = true; @@ -2404,7 +2537,7 @@ static vk_device ggml_vk_get_device(size_t idx) { device->fp16 = !force_disable_f16 && fp16_storage && fp16_compute; - if (!ggml_vk_khr_cooperative_matrix_support(device->properties, driver_props)) { + if (!ggml_vk_khr_cooperative_matrix_support(device->properties, driver_props, device->architecture)) { device->coopmat_support = false; } @@ -2782,7 +2915,10 @@ static void ggml_vk_print_gpu_info(size_t idx) { subgroup_props.pNext = &driver_props; physical_device.getProperties2(&props2); - const size_t subgroup_size = subgroup_props.subgroupSize; + vk_device_architecture arch = get_device_architecture(physical_device); + uint32_t default_subgroup_size = get_subgroup_size("", arch); + const size_t subgroup_size = (default_subgroup_size != 0) ? default_subgroup_size : subgroup_props.subgroupSize; + const bool uma = props2.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; bool fp16_storage = false; @@ -2808,7 +2944,9 @@ static void ggml_vk_print_gpu_info(size_t idx) { } } - if (!ggml_vk_khr_cooperative_matrix_support(props2.properties, driver_props)) { + const vk_device_architecture device_architecture = get_device_architecture(physical_device); + + if (!ggml_vk_khr_cooperative_matrix_support(props2.properties, driver_props, device_architecture)) { coopmat_support = false; } @@ -8843,7 +8981,7 @@ static bool ggml_vk_instance_portability_enumeration_ext_available(const std::ve UNUSED(instance_extensions); } -static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props) { +static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch) { switch (props.vendorID) { case VK_VENDOR_ID_INTEL: // Intel drivers don't support coopmat properly yet @@ -8851,10 +8989,7 @@ static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDevicePrope case VK_VENDOR_ID_AMD: if (driver_props.driverID == vk::DriverId::eAmdProprietary || driver_props.driverID == vk::DriverId::eAmdOpenSource) { // Workaround for AMD proprietary driver reporting support on all GPUs - const std::string name = props.deviceName; - return name.rfind("AMD Radeon RX 7", 0) == 0 || name.rfind("AMD Radeon(TM) RX 7", 0) == 0 || // RDNA 3 consumer GPUs - name.rfind("AMD Radeon PRO W7", 0) == 0 || name.rfind("AMD Radeon(TM) PRO W7", 0) == 0 || // RDNA 3 workstation GPUs - name.rfind("AMD Radeon 7", 0) == 0 || name.rfind("AMD Radeon(TM) 7", 0) == 0; // RDNA 3 APUs + return arch == vk_device_architecture::AMD_RDNA3; } return true; default: From 484a8ab513bbd740cc49f30280c1acf52cb4e7e9 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Mon, 17 Mar 2025 09:26:18 -0500 Subject: [PATCH 20/55] vulkan: Add N/2 and N/4 optimized paths in coopmat2 shader (#12312) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 24 +++--- .../vulkan-shaders/mul_mm_cm2.comp | 79 ++++++++++++++----- 2 files changed, 72 insertions(+), 31 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index aa7281acbf..97398f071b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1597,33 +1597,33 @@ static void ggml_vk_load_shaders(vk_device& device) { uint32_t l_align, m_align, s_align; if (device->coopmat2) { // spec constants and tile sizes for non-quant matmul/matmul_id - l_warptile = { 256, 128, 256, 64 }; - m_warptile = { 256, 128, 128, 64 }; - s_warptile = { 128, 64, 64, 64 }; + l_warptile = { 256, 128, 256, 64, 1 }; + m_warptile = { 256, 128, 128, 64, 0 }; + s_warptile = { 128, 64, 64, 64, 0 }; l_wg_denoms = {128, 256, 1 }; m_wg_denoms = {128, 128, 1 }; s_wg_denoms = { 64, 64, 1 }; // spec constants and tile sizes for quant matmul (non-Qi_K) - l_warptile_mmq = { 256, 128, 256, 64 }; - m_warptile_mmq = { 256, 128, 128, 64 }; - s_warptile_mmq = { 256, 32, 64, 128 }; + l_warptile_mmq = { 256, 128, 256, 64, 1 }; + m_warptile_mmq = { 256, 128, 128, 64, 1 }; + s_warptile_mmq = { 256, 32, 64, 128, 0 }; l_mmq_wg_denoms = { 128, 256, 1 }; m_mmq_wg_denoms = { 128, 128, 1 }; s_mmq_wg_denoms = { 32, 64, 1 }; // spec constants and tile sizes for quant matmul (Qi_K) - l_warptile_mmq_k = { 256, 64, 128, 64 }; - m_warptile_mmq_k = { 256, 32, 64, 64 }; - s_warptile_mmq_k = { 256, 32, 32, 128 }; + l_warptile_mmq_k = { 256, 64, 128, 64, 1 }; + m_warptile_mmq_k = { 256, 32, 64, 64, 0 }; + s_warptile_mmq_k = { 256, 32, 32, 128, 0 }; l_mmq_wg_denoms_k = { 64, 128, 1 }; m_mmq_wg_denoms_k = { 32, 64, 1 }; s_mmq_wg_denoms_k = { 32, 32, 1 }; // spec constants and tile sizes for quant matmul_id - l_warptile_mmqid = { 256, 128, 64, 16 }; - m_warptile_mmqid = { 256, 128, 64, 16 }; - s_warptile_mmqid = { 256, 128, 64, 16 }; + l_warptile_mmqid = { 256, 128, 64, 16, 0 }; + m_warptile_mmqid = { 256, 128, 64, 16, 0 }; + s_warptile_mmqid = { 256, 128, 64, 16, 0 }; l_mmqid_wg_denoms = { 128, 64, 1 }; m_mmqid_wg_denoms = { 128, 64, 1 }; s_mmqid_wg_denoms = { 128, 64, 1 }; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 5b7a4efe2c..7649febb07 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -23,6 +23,10 @@ layout (constant_id = 1) const uint BM = 64; layout (constant_id = 2) const uint BN = 64; layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working with a quant +layout (constant_id = 4) const bool enable_smaller_matrices = false; +const uint BNover2 = enable_smaller_matrices ? (BN / 2) : BN; +const uint BNover4 = enable_smaller_matrices ? (BN / 4) : BN; + layout (push_constant) uniform parameter { uint M; @@ -168,15 +172,13 @@ void main() { const uint end_k = min(p.K, (ik + 1) * p.k_split); #endif - coopmat sum; - sum = coopmat(0.0); - #ifdef MUL_MAT_ID uint pos_a = (expert_idx * p.batch_stride_a) / QUANT_K; uint pos_b = 0; #else uint pos_a = (batch_idx_a * p.batch_stride_a) / QUANT_K; uint pos_b = batch_idx * p.batch_stride_b; + uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * gl_NumWorkGroups.z; #endif uint stride_a = p.stride_a / QUANT_K; @@ -197,6 +199,7 @@ void main() { tensorLayoutNV<2> tensorLayoutB = createTensorLayoutNV(2); tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutBClamp = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV); tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutD = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV); + tensorLayoutD = setTensorLayoutStrideNV(tensorLayoutD, p.stride_d, 1); #if QUANT_K > 1 tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K); @@ -232,16 +235,54 @@ void main() { tensorLayoutB = setTensorLayoutStrideNV(tensorLayoutB, stride_b, 1); uint k_iters = (end_k - start_k + BK - 1) / BK; + if (enable_smaller_matrices && ic * BN + BNover4 >= p.N) { + coopmat sum = coopmat(0.0); + for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { + coopmat mat_a; + coopmat mat_b; - coopmat mat_a; - coopmat mat_b; + coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); + coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose); - coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); - coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose); + sum = coopMatMulAdd(mat_a, mat_b, sum); + } + coopmat mat_d = coopmat(sum); - sum = coopMatMulAdd(mat_a, mat_b, sum); + coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover4, ir * BM, BM), tensorViewTranspose); + return; + } else if (enable_smaller_matrices && ic * BN + BNover2 >= p.N) { + coopmat sum = coopmat(0.0); + for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { + + coopmat mat_a; + coopmat mat_b; + + coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); + coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose); + + sum = coopMatMulAdd(mat_a, mat_b, sum); + } + coopmat mat_d = coopmat(sum); + + coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover2, ir * BM, BM), tensorViewTranspose); + return; + } else { + coopmat sum = coopmat(0.0); + for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { + + coopmat mat_a; + coopmat mat_b; + + coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); + coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose); + + sum = coopMatMulAdd(mat_a, mat_b, sum); + } + coopmat mat_d = coopmat(sum); + + coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose); + return; } } else #endif // !defined(MUL_MAT_ID) @@ -254,6 +295,9 @@ void main() { tensorLayoutBClamp = setTensorLayoutStrideNV(tensorLayoutBClamp, stride_b, 1); + coopmat sum; + sum = coopmat(0.0); + [[dont_unroll]] for (uint block_k = start_k; block_k < end_k; block_k += BK) { @@ -296,19 +340,16 @@ void main() { sum = coopMatMulAdd(mat_a, mat_b, sum); } } - } - // Convert from ACC_TYPE to D_TYPE - coopmat mat_d; - mat_d = coopmat(sum); + // Convert from ACC_TYPE to D_TYPE + coopmat mat_d; + mat_d = coopmat(sum); #ifdef MUL_MAT_ID - // Call callback to store each element, remapping row through shared memory - coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic); + // Call callback to store each element, remapping row through shared memory + coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic); #else - tensorLayoutD = setTensorLayoutStrideNV(tensorLayoutD, p.stride_d, 1); - - uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * gl_NumWorkGroups.z; - coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose); + coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose); #endif + } } From 01e8f2138b2e40902afe2983ecbf503a08d74b1d Mon Sep 17 00:00:00 2001 From: Guus Waals <_@guusw.nl> Date: Tue, 18 Mar 2025 00:35:43 +0800 Subject: [PATCH 21/55] ggml-vulkan: remove unused find_program(glslc) (#12416) It's already found by FindVulkan.cmake in the parent CMakeLists --- ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt b/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt index 074031087f..51c78b7d22 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +++ b/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt @@ -1,8 +1,4 @@ find_package (Threads REQUIRED) -find_program(GLSLC_EXECUTABLE glslc) -if(NOT GLSLC_EXECUTABLE) - message(FATAL_ERROR "glslc not found.") -endif() set(TARGET vulkan-shaders-gen) add_executable(${TARGET} vulkan-shaders-gen.cpp) From b1b132efcba216c873715c483809730bb253f4a1 Mon Sep 17 00:00:00 2001 From: Gaurav Garg <52341457+gaugarg-nv@users.noreply.github.com> Date: Mon, 17 Mar 2025 23:55:13 +0530 Subject: [PATCH 22/55] cuda : enable CUDA Graph on CUDA Toolkit < 12.x (#12394) * Enable CUDA Graph on CTK < 12.x `cudaGraphExecUpdate` API was changed on 12.x. For this reason CUDA graph support was disabled on older CUDA toolkit. This change enables CUDA support in CTK version < 12.x by using older API if CTK < 12.x. * Fix compilation errors with MUSA * Disable CUDA Graph for MUSA --- ggml/src/ggml-cuda/common.cuh | 2 +- ggml/src/ggml-cuda/ggml-cuda.cu | 12 +++++++----- ggml/src/ggml-cuda/vendors/hip.h | 2 +- ggml/src/ggml-cuda/vendors/musa.h | 3 ++- ggml/src/ggml-musa/CMakeLists.txt | 4 ---- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 4d4ac47c03..e78205e5d5 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -678,7 +678,7 @@ struct ggml_tensor_extra_gpu { }; -#if ((CUDART_VERSION >= 12000) && defined(GGML_CUDA_USE_GRAPHS)) || defined(GGML_HIP_GRAPHS) +#if (defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)) #define USE_CUDA_GRAPH #endif diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 497de37be8..9bba398ce6 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2610,13 +2610,15 @@ static bool is_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx, static void update_cuda_graph_executable(ggml_backend_cuda_context * cuda_ctx) { +#if CUDART_VERSION >= 12000 cudaGraphExecUpdateResultInfo result_info; -#ifdef __HIP_PLATFORM_AMD__ - hipGraphNode_t errorNode; - hipError_t stat = hipGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &errorNode, &result_info); -#else cudaError_t stat = cudaGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &result_info); -#endif +#else + cudaGraphNode_t errorNode; + cudaGraphExecUpdateResult result_info; + cudaError_t stat = cudaGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &errorNode, &result_info); +#endif // CUDART_VERSION >= 12000 + if (stat == cudaErrorGraphExecUpdateFailure) { #ifndef NDEBUG GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__); diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index 81964611c6..aace21e3a8 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -112,7 +112,7 @@ #define cudaGraphExecDestroy hipGraphExecDestroy #define cudaGraphLaunch hipGraphLaunch #define cudaErrorGraphExecUpdateFailure hipErrorGraphExecUpdateFailure -#define cudaGraphExecUpdateResultInfo hipGraphExecUpdateResult +#define cudaGraphExecUpdateResult hipGraphExecUpdateResult #define cudaGraphNodeType hipGraphNodeType #define cudaGraphNodeTypeKernel hipGraphNodeTypeKernel #define cudaGraphInstantiate hipGraphInstantiate diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 6cc1b69ee3..997f671431 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -119,7 +119,7 @@ #define cudaGraphExecDestroy musaGraphExecDestroy #define cudaGraphExec_t musaGraphExec_t #define cudaGraphExecUpdate musaGraphExecUpdate -#define cudaGraphExecUpdateResultInfo musaGraphExecUpdateResult +#define cudaGraphExecUpdateResult musaGraphExecUpdateResult #define cudaGraphGetNodes musaGraphGetNodes #define cudaGraphInstantiate musaGraphInstantiate #define cudaGraphKernelNodeGetParams musaGraphKernelNodeGetParams @@ -132,6 +132,7 @@ #define cudaGraph_t musaGraph_t #define cudaKernelNodeParams musaKernelNodeParams #define cudaStreamCaptureModeRelaxed musaStreamCaptureModeRelaxed +#define cudaStreamBeginCapture musaStreamBeginCapture #define cudaStreamEndCapture musaStreamEndCapture typedef mt_bfloat16 nv_bfloat16; diff --git a/ggml/src/ggml-musa/CMakeLists.txt b/ggml/src/ggml-musa/CMakeLists.txt index 166970ca6b..92f05d5558 100644 --- a/ggml/src/ggml-musa/CMakeLists.txt +++ b/ggml/src/ggml-musa/CMakeLists.txt @@ -67,10 +67,6 @@ if (MUSAToolkit_FOUND) add_compile_definitions(GGML_USE_MUSA) add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE}) - if (GGML_CUDA_GRAPHS) - add_compile_definitions(GGML_CUDA_USE_GRAPHS) - endif() - if (GGML_CUDA_FORCE_MMQ) add_compile_definitions(GGML_CUDA_FORCE_MMQ) endif() From 60c902926c928f9c2cd6390ce411876f92feeaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Mon, 17 Mar 2025 21:14:32 +0100 Subject: [PATCH 23/55] docs : bring llama-cli conversation/template docs up-to-date (#12426) --- examples/main/README.md | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/examples/main/README.md b/examples/main/README.md index f7c2497294..e4b3590b5d 100644 --- a/examples/main/README.md +++ b/examples/main/README.md @@ -27,12 +27,24 @@ Once downloaded, place your model in the models folder in llama.cpp. ##### Input prompt (One-and-done) ```bash -./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf --prompt "Once upon a time" +./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf -no-cnv --prompt "Once upon a time" ``` ##### Conversation mode (Allow for continuous interaction with the model) ```bash -./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf -cnv --chat-template gemma +./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf --chat-template gemma +``` + +##### Conversation mode using built-in jinja chat template + +```bash +./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf --jinja +``` + +##### One-and-done query using jinja with custom system prompt and a starting prompt + +```bash +./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf --jinja --single-turn -sys "You are a helpful assistant" -p "Hello" ``` ##### Infinite text from a starting prompt (you can use `Ctrl-C` to stop it): @@ -44,12 +56,24 @@ Once downloaded, place your model in the models folder in llama.cpp. ##### Input prompt (One-and-done) ```powershell -./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --prompt "Once upon a time" +./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf -no-cnv --prompt "Once upon a time" ``` ##### Conversation mode (Allow for continuous interaction with the model) ```powershell -./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf -cnv --chat-template gemma +./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --chat-template gemma +``` + +##### Conversation mode using built-in jinja chat template + +```powershell +./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --jinja +``` + +##### One-and-done query using jinja with custom system prompt and a starting prompt + +```powershell +./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --jinja --single-turn -sys "You are a helpful assistant" -p "Hello" ``` #### Infinite text from a starting prompt (you can use `Ctrl-C` to stop it): @@ -77,6 +101,8 @@ The `llama-cli` program provides several ways to interact with the LLaMA models - `--prompt PROMPT`: Provide a prompt directly as a command-line option. - `--file FNAME`: Provide a file containing a prompt or multiple prompts. +- `--system-prompt PROMPT`: Provide a system prompt (will otherwise use the default one in the chat template (if provided)). +- `--system-prompt-file FNAME`: Provide a file containing a system prompt. - `--interactive-first`: Run the program in interactive mode and wait for input right away. (More on this below.) ## Interaction @@ -89,7 +115,10 @@ In interactive mode, users can participate in text generation by injecting their - `-i, --interactive`: Run the program in interactive mode, allowing users to engage in real-time conversations or provide specific instructions to the model. - `--interactive-first`: Run the program in interactive mode and immediately wait for user input before starting the text generation. -- `-cnv, --conversation`: Run the program in conversation mode (does not print special tokens and suffix/prefix, use default chat template) (default: false) +- `-cnv, --conversation`: Run the program in conversation mode (does not print special tokens and suffix/prefix, use default or provided chat template) (default: true if chat template found) +- `-no-cnv`: Disable conversation mode (default: false) +- `-st, --single-turn`: Only process a single conversation turn (user input) and then exit. +- `--jinja`: Enable jinja chat template parser, will use the model's built-in template or a user-provided one (default: false) - `--color`: Enable colorized output to differentiate visually distinguishing between prompts, user input, and generated text. By understanding and utilizing these interaction options, you can create engaging and dynamic experiences with the LLaMA models, tailoring the text generation process to your specific needs. @@ -125,6 +154,8 @@ When --in-prefix or --in-suffix options are enabled the chat template ( --chat-t Example usage: `--chat-template gemma` +`--chat-template-file FNAME`: Load a custom jinja chat template from an external file, useful if the model contains outdated or incompatible template, some examples can be found in models/templates. Up-to-date chat templates can be downloaded from Hugging Face using scripts/get_chat_template.py + ## Context Management During text generation, LLaMA models have a limited context size, which means they can only consider a certain number of tokens from the input and generated text. When the context fills up, the model resets internally, potentially losing some information from the beginning of the conversation or instructions. Context management options help maintain continuity and coherence in these situations. From 7dfad387e3f6ac98d383ded2d175eb59736a3993 Mon Sep 17 00:00:00 2001 From: Molly Sophia Date: Tue, 18 Mar 2025 07:27:50 +0800 Subject: [PATCH 24/55] llama: Add support for RWKV v7 architecture (#12412) * ggml: Add op l2_norm Signed-off-by: Molly Sophia * ggml: Add op rwkv_wkv7 Signed-off-by: Molly Sophia * llama: Add support for RWKV7 and ARWKV7 models Signed-off-by: Molly Sophia * llama: fix inference with RWKV6Qwen2 Signed-off-by: Molly Sophia * llama: add more (a)rwkv7 variants in size Signed-off-by: Molly Sophia * Apply code-format changes Signed-off-by: Molly Sophia * fix MUSA build Signed-off-by: Molly Sophia * llama: fix shape error with rwkv using llama-parallel Signed-off-by: Molly Sophia --------- Signed-off-by: Molly Sophia --- convert_hf_to_gguf.py | 229 ++++++- ggml/include/ggml.h | 24 + ggml/src/ggml-cpu/ggml-cpu.c | 255 +++++++- ggml/src/ggml-cuda/ggml-cuda.cu | 10 +- ggml/src/ggml-cuda/norm.cu | 116 ++++ ggml/src/ggml-cuda/norm.cuh | 2 + ggml/src/ggml-cuda/wkv.cu | 199 ++++++ ggml/src/ggml-cuda/{wkv6.cuh => wkv.cuh} | 2 + ggml/src/ggml-cuda/wkv6.cu | 89 --- ggml/src/ggml-metal/ggml-metal-impl.h | 7 + ggml/src/ggml-metal/ggml-metal.m | 122 ++++ ggml/src/ggml-metal/ggml-metal.metal | 221 +++++++ ggml/src/ggml-sycl/backend.hpp | 2 +- ggml/src/ggml-sycl/ggml-sycl.cpp | 14 + ggml/src/ggml-sycl/norm.cpp | 108 ++++ ggml/src/ggml-sycl/norm.hpp | 6 + ggml/src/ggml-sycl/wkv.cpp | 305 +++++++++ ggml/src/ggml-sycl/wkv.hpp | 10 + ggml/src/ggml-sycl/wkv6.cpp | 143 ----- ggml/src/ggml-sycl/wkv6.hpp | 9 - ggml/src/ggml-vulkan/ggml-vulkan.cpp | 206 ++++--- .../ggml-vulkan/vulkan-shaders/l2_norm.comp | 41 ++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 3 + ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp | 91 +++ ggml/src/ggml.c | 87 ++- gguf-py/gguf/constants.py | 126 +++- gguf-py/gguf/gguf_writer.py | 12 + gguf-py/gguf/tensor_mapping.py | 131 +++- src/llama-arch.cpp | 118 +++- src/llama-arch.h | 18 + src/llama-hparams.h | 4 + src/llama-model.cpp | 581 +++++++++++++++++- src/llama-model.h | 16 + src/llama-quant.cpp | 11 +- tests/test-backend-ops.cpp | 68 ++ 35 files changed, 2948 insertions(+), 438 deletions(-) create mode 100644 ggml/src/ggml-cuda/wkv.cu rename ggml/src/ggml-cuda/{wkv6.cuh => wkv.cuh} (62%) delete mode 100644 ggml/src/ggml-cuda/wkv6.cu create mode 100644 ggml/src/ggml-sycl/wkv.cpp create mode 100644 ggml/src/ggml-sycl/wkv.hpp delete mode 100644 ggml/src/ggml-sycl/wkv6.cpp delete mode 100644 ggml/src/ggml-sycl/wkv6.hpp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index b5d95bd563..d13d57c541 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -908,6 +908,40 @@ class Model: special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) special_vocab.add_to_gguf(self.gguf_writer) + def _set_vocab_rwkv_world(self): + assert (self.dir_model / "rwkv_vocab_v20230424.txt").is_file() + vocab_size = self.hparams.get("vocab_size", 65536) + + tokens: list[bytes] = [''.encode("utf-8")] + toktypes: list[int] = [gguf.TokenType.CONTROL] + + with open(self.dir_model / "rwkv_vocab_v20230424.txt", "r", encoding="utf-8") as f: + lines = f.readlines() + for line in lines: + parts = line.split(' ') + assert len(parts) >= 3 + token, token_len = ast.literal_eval(' '.join(parts[1:-1])), int(parts[-1]) + token = token.encode("utf-8") if isinstance(token, str) else token + assert isinstance(token, bytes) + assert len(token) == token_len + token_text: str = repr(token)[2:-1] # "b'\xff'" -> "\xff" + tokens.append(token_text.encode("utf-8")) + toktypes.append(gguf.TokenType.NORMAL) + remainder = vocab_size - len(tokens) + assert remainder >= 0 + for i in range(len(tokens), vocab_size): + tokens.append(f"[PAD{i}]".encode("utf-8")) + toktypes.append(gguf.TokenType.UNUSED) + + self.gguf_writer.add_tokenizer_model("rwkv") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False) + special_vocab.chat_template = "rwkv-world" + # hack: Add '\n\n' as the EOT token to make it chat normally + special_vocab._set_special_token("eot", 261) + special_vocab.add_to_gguf(self.gguf_writer) + def _set_vocab_builtin(self, model_name: Literal["gpt-neox", "llama-spm"], vocab_size: int): tokenizer_path = Path(sys.path[0]) / "models" / f"ggml-vocab-{model_name}.gguf" logger.warning(f"Using tokenizer from '{os.path.relpath(tokenizer_path, os.getcwd())}'") @@ -3412,38 +3446,7 @@ class Rwkv6Model(Model): model_arch = gguf.MODEL_ARCH.RWKV6 def set_vocab(self): - assert (self.dir_model / "rwkv_vocab_v20230424.txt").is_file() - vocab_size = self.hparams.get("vocab_size", 65536) - - tokens: list[bytes] = [''.encode("utf-8")] - toktypes: list[int] = [gguf.TokenType.CONTROL] - - with open(self.dir_model / "rwkv_vocab_v20230424.txt", "r", encoding="utf-8") as f: - lines = f.readlines() - for line in lines: - parts = line.split(' ') - assert len(parts) >= 3 - token, token_len = ast.literal_eval(' '.join(parts[1:-1])), int(parts[-1]) - token = token.encode("utf-8") if isinstance(token, str) else token - assert isinstance(token, bytes) - assert len(token) == token_len - token_text: str = repr(token)[2:-1] # "b'\xff'" -> "\xff" - tokens.append(token_text.encode("utf-8")) - toktypes.append(gguf.TokenType.NORMAL) - remainder = vocab_size - len(tokens) - assert remainder >= 0 - for i in range(len(tokens), vocab_size): - tokens.append(f"[PAD{i}]".encode("utf-8")) - toktypes.append(gguf.TokenType.UNUSED) - - self.gguf_writer.add_tokenizer_model("rwkv") - self.gguf_writer.add_token_list(tokens) - self.gguf_writer.add_token_types(toktypes) - special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False) - special_vocab.chat_template = "rwkv-world" - # hack: Add '\n\n' as the EOT token to make it chat normally - special_vocab._set_special_token("eot", 261) - special_vocab.add_to_gguf(self.gguf_writer) + self._set_vocab_rwkv_world() def set_gguf_parameters(self): block_count = self.hparams["num_hidden_layers"] @@ -3565,6 +3568,168 @@ class RWKV6Qwen2Model(Rwkv6Model): yield (new_name, data) +@Model.register("Rwkv7ForCausalLM", "RWKV7ForCausalLM") +class Rwkv7Model(Model): + model_arch = gguf.MODEL_ARCH.RWKV7 + + def set_vocab(self): + self._set_vocab_rwkv_world() + + def calc_lora_rank(self, hidden_size, exponent, multiplier): + return max(1, round(hidden_size ** exponent * multiplier / 32)) * 32 + + def set_gguf_parameters(self): + block_count = self.hparams["num_hidden_layers"] + try: + head_size = self.hparams["head_size"] + layer_norm_eps = self.hparams["layer_norm_epsilon"] + except KeyError: + head_size = self.hparams["head_dim"] + layer_norm_eps = self.hparams["norm_eps"] + hidden_size = self.hparams["hidden_size"] + intermediate_size = self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else (hidden_size * 4) + + # ICLR: In-Context-Learning-Rate + try: + lora_rank_decay = self.hparams["lora_rank_decay"] if self.hparams["lora_rank_decay"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8) + lora_rank_iclr = self.hparams["lora_rank_iclr"] if self.hparams["lora_rank_iclr"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8) + lora_rank_value_residual_mix = self.hparams["lora_rank_value_residual_mix"] if self.hparams["lora_rank_value_residual_mix"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3) + lora_rank_gate = self.hparams["lora_rank_gate"] if self.hparams["lora_rank_gate"] is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6) + except KeyError: + lora_rank_decay = self.hparams["decay_low_rank_dim"] if self.hparams["decay_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8) + lora_rank_iclr = self.hparams["a_low_rank_dim"] if self.hparams["a_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8) + lora_rank_value_residual_mix = self.hparams["v_low_rank_dim"] if self.hparams["v_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3) + lora_rank_gate = self.hparams["gate_low_rank_dim"] if self.hparams["gate_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6) + + # RWKV isn't context limited + self.gguf_writer.add_context_length(1048576) + self.gguf_writer.add_embedding_length(hidden_size) + self.gguf_writer.add_block_count(block_count) + self.gguf_writer.add_layer_norm_eps(layer_norm_eps) + self.gguf_writer.add_wkv_head_size(head_size) + self.gguf_writer.add_decay_lora_rank(lora_rank_decay) + self.gguf_writer.add_iclr_lora_rank(lora_rank_iclr) + self.gguf_writer.add_value_residual_mix_lora_rank(lora_rank_value_residual_mix) + self.gguf_writer.add_gate_lora_rank(lora_rank_gate) + self.gguf_writer.add_feed_forward_length(intermediate_size) + self.gguf_writer.add_file_type(self.ftype) + + # required by llama.cpp, unused + self.gguf_writer.add_head_count(0) + + lerp_weights: dict[int, dict[str, Tensor]] = {} + lora_needs_transpose: bool = True + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # unify tensor names here to make life easier + name = name.replace("blocks", "layers").replace("ffn", "feed_forward") + name = name.replace("self_attn", "attention").replace("attn", "attention") + name = name.replace("time_mixer.", "") + # lora layer names in fla-hub's impl + if "_lora.lora" in name: + self.lora_needs_transpose = False + name = name.replace("_lora.lora.0.weight", "1.weight") + name = name.replace("_lora.lora.2.weight", "2.weight") + name = name.replace("_lora.lora.2.bias", "0.weight") + + name = name.replace("feed_forward_norm", "ln2") + name = name.replace("g_norm", "ln_x") + + if "attention.v" in name and "value" not in self.map_tensor_name(name) and bid == 0: + # some models have dummy v0/v1/v2 on first layer while others don't + # ignore them all since they are not used + return + + wkv_has_gate = self.hparams.get("wkv_has_gate", True) + lerp_list = ["r", "w", "k", "v", "a", "g"] if wkv_has_gate else ["r", "w", "k", "v", "a"] + + if bid is not None and "attention.x_" in name: + if "attention.x_x" in name: + # already concatenated + new_name = f"blk.{bid}.time_mix_lerp_fused.weight" + data = data_torch.reshape(len(lerp_list), 1, 1, -1) + yield (new_name, data) + else: + try: + self.lerp_weights[bid][name] = data_torch + except KeyError: + self.lerp_weights[bid] = {name: data_torch} + if all(f"model.layers.{bid}.attention.x_{i}" in self.lerp_weights[bid].keys() for i in lerp_list): + new_name = f"blk.{bid}.time_mix_lerp_fused.weight" + data = torch.stack([self.lerp_weights[bid][f"model.layers.{bid}.attention.x_{i}"] for i in lerp_list], dim=0) + yield (new_name, data) + return + else: + data_torch = data_torch.squeeze() + new_name = self.map_tensor_name(name) + + if not (new_name.endswith(".weight") or new_name.endswith(".bias")): + new_name += ".weight" + + if self.lora_needs_transpose and any( + new_name.endswith(t) for t in [ + "time_mix_w1.weight", "time_mix_w2.weight", + "time_mix_a1.weight", "time_mix_a2.weight", + "time_mix_v1.weight", "time_mix_v2.weight", + "time_mix_g1.weight", "time_mix_g2.weight", + ] + ): + data_torch = data_torch.transpose(0, 1) + + if 'r_k' in new_name: + data_torch = data_torch.flatten() + + if bid == 0 and "time_mix_a" in new_name: + # dummy v0/v1/v2 on first layer + # easist way to make llama happy + yield (new_name.replace("time_mix_a", "time_mix_v"), data_torch) + + yield (new_name, data_torch) + + +@Model.register("RwkvHybridForCausalLM") +class ARwkv7Model(Rwkv7Model): + model_arch = gguf.MODEL_ARCH.ARWKV7 + + def set_vocab(self): + try: + self._set_vocab_sentencepiece() + except FileNotFoundError: + self._set_vocab_gpt2() + + def set_gguf_parameters(self): + block_count = self.hparams["num_hidden_layers"] + hidden_size = self.hparams["hidden_size"] + head_size = self.hparams["head_size"] + rms_norm_eps = self.hparams["rms_norm_eps"] + intermediate_size = self.hparams["intermediate_size"] + wkv_has_gate = self.hparams["wkv_has_gate"] + assert self.hparams["wkv_version"] == 7 + + # ICLR: In-Context-Learning-Rate + lora_rank_decay = 64 + lora_rank_iclr = 64 + lora_rank_value_residual_mix = 32 + lora_rank_gate = 128 if wkv_has_gate else 0 + + # RWKV isn't context limited + self.gguf_writer.add_context_length(1048576) + self.gguf_writer.add_embedding_length(hidden_size) + self.gguf_writer.add_block_count(block_count) + self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps) + self.gguf_writer.add_wkv_head_size(head_size) + self.gguf_writer.add_decay_lora_rank(lora_rank_decay) + self.gguf_writer.add_iclr_lora_rank(lora_rank_iclr) + self.gguf_writer.add_value_residual_mix_lora_rank(lora_rank_value_residual_mix) + self.gguf_writer.add_gate_lora_rank(lora_rank_gate) + self.gguf_writer.add_feed_forward_length(intermediate_size) + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_token_shift_count(1) + + # required by llama.cpp, unused + self.gguf_writer.add_head_count(0) + + @Model.register("MambaForCausalLM", "MambaLMHeadModel", "FalconMambaForCausalLM") class MambaModel(Model): model_arch = gguf.MODEL_ARCH.MAMBA diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 2e5076d36a..cb3edb10d4 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -454,6 +454,7 @@ extern "C" { GGML_OP_RMS_NORM, GGML_OP_RMS_NORM_BACK, GGML_OP_GROUP_NORM, + GGML_OP_L2_NORM, GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID, @@ -502,6 +503,7 @@ extern "C" { GGML_OP_ADD_REL_POS, GGML_OP_RWKV_WKV6, GGML_OP_GATED_LINEAR_ATTN, + GGML_OP_RWKV_WKV7, GGML_OP_UNARY, @@ -1095,6 +1097,18 @@ extern "C" { int n_groups, float eps); + // l2 normalize along rows + // used in rwkv v7 + GGML_API struct ggml_tensor * ggml_l2_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_l2_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + // a - x // b - dy GGML_API struct ggml_tensor * ggml_rms_norm_back( @@ -1890,6 +1904,16 @@ extern "C" { struct ggml_tensor * state, float scale); + GGML_API struct ggml_tensor * ggml_rwkv_wkv7( + struct ggml_context * ctx, + struct ggml_tensor * r, + struct ggml_tensor * w, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * state); + // custom operators typedef void (*ggml_unary_op_f32_t) (const int, float *, const float *); diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index f2ab4c5d69..75dc96b478 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -8548,6 +8548,69 @@ static void ggml_compute_forward_group_norm( } } +// ggml_compute_forward_l2_norm + +static void ggml_compute_forward_l2_norm_f32( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + GGML_ASSERT(ggml_are_same_shape(src0, dst)); + + GGML_ASSERT(src0->nb[0] == sizeof(float)); + + const int ith = params->ith; + const int nth = params->nth; + + GGML_TENSOR_UNARY_OP_LOCALS + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + + GGML_ASSERT(eps >= 0.0f); + + // TODO: optimize + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + for (int64_t i01 = ith; i01 < ne01; i01 += nth) { + const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03); + + ggml_float sum = 0.0; + for (int64_t i00 = 0; i00 < ne00; i00++) { + sum += (ggml_float)(x[i00] * x[i00]); + } + + float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); + + memcpy(y, x, ne00 * sizeof(float)); + + const float scale = 1.0f/fmaxf(sqrtf(sum), eps); + + ggml_vec_scale_f32(ne00, y, scale); + } + } + } +} + +static void ggml_compute_forward_l2_norm( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_l2_norm_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + // ggml_compute_forward_mul_mat static void ggml_compute_forward_mul_mat_one_chunk( @@ -13604,6 +13667,184 @@ static void ggml_compute_forward_gla( } } +// ggml_compute_forward_rwkv_wkv7 + +static void ggml_compute_forward_rwkv_wkv7_f32( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + const int64_t T = dst->src[1]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t HEADS = dst->src[1]->ne[1]; + const int64_t n_seqs = dst->src[6]->ne[1]; + const int64_t head_size = C / HEADS; + + float * dst_data = (float *) dst->data; + float * state = ((float *) dst->data) + C * T; + + const int ith = params->ith; + const int nth = params->nth; + + if (ith >= HEADS) { + return; + } + + const int h_start = (HEADS * ith) / nth; + const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ? + (HEADS * (ith + 1)) / nth : HEADS; + + float * r = (float *) dst->src[0]->data; + float * w = (float *) dst->src[1]->data; + float * k = (float *) dst->src[2]->data; + float * v = (float *) dst->src[3]->data; + float * a = (float *) dst->src[4]->data; + float * b = (float *) dst->src[5]->data; + + int64_t t_stride = HEADS * head_size; // Same to C + + int64_t h_stride = C / HEADS; + GGML_ASSERT(C % HEADS == 0); // C must be divisible by HEADS + int64_t h_stride_2d = head_size * head_size; + + #if defined(GGML_SIMD) + for (int64_t t = 0; t < T; t++) { + int64_t t_offset = t * t_stride; + int64_t state_offset = head_size * C * (t / (T / n_seqs)); + float * state_cur = state + state_offset; + float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset; + + for (int64_t h = h_start; h < h_end; h++) { + int64_t h_offset = h * h_stride; + int64_t t_h_offset = t_offset + h_offset; + int64_t h_2d_offset = h * h_stride_2d; + + for (int64_t ii = 0; ii < head_size; ii++) { + int64_t t_h_i_offset = t_h_offset + ii; + int64_t h_2d_i_offset = h_2d_offset + ii * h_stride; + + GGML_F32_VEC v_vec = GGML_F32_VEC_SET1(v[t_h_i_offset]); + + float sa = 0; + { + GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO }; + GGML_F32_VEC ax[GGML_F32_ARR]; + GGML_F32_VEC ay[GGML_F32_ARR]; + for (int64_t j = 0; j < head_size; j += GGML_F32_STEP) { + for (int64_t kk = 0; kk < GGML_F32_ARR; kk++) { + ax[kk] = GGML_F32_VEC_LOAD(&a[t_h_offset + j + kk * GGML_F32_EPR]); + ay[kk] = GGML_F32_VEC_LOAD(&state_prev[h_2d_i_offset + j + kk * GGML_F32_EPR]); + sum[kk] = GGML_F32_VEC_FMA(sum[kk], ax[kk], ay[kk]); + } + } + GGML_F32_VEC_REDUCE(sa, sum); + } + + GGML_F32_VEC sa_vec = GGML_F32_VEC_SET1(sa); + + int64_t j = 0; + GGML_F32_VEC result_vec[GGML_F32_ARR] = { GGML_F32_VEC_ZERO }; + for (; j < head_size; j += GGML_F32_STEP) { + for (int64_t kk = 0; kk < GGML_F32_ARR; kk++) { + int64_t t_h_j_offset = t_h_offset + j + kk * GGML_F32_EPR; + int64_t h_2d_i_j_offset = h_2d_i_offset + j + kk * GGML_F32_EPR; + + GGML_F32_VEC r_vec = GGML_F32_VEC_LOAD(&r[t_h_j_offset]); + GGML_F32_VEC w_vec = GGML_F32_VEC_LOAD(&w[t_h_j_offset]); + GGML_F32_VEC k_vec = GGML_F32_VEC_LOAD(&k[t_h_j_offset]); + GGML_F32_VEC b_vec = GGML_F32_VEC_LOAD(&b[t_h_j_offset]); + + k_vec = GGML_F32_VEC_MUL(v_vec, k_vec); + + GGML_F32_VEC state_vec = GGML_F32_VEC_LOAD(&state_prev[h_2d_i_j_offset]); + // kv + s * decay + sa * b + state_vec = GGML_F32_VEC_FMA(k_vec, state_vec, w_vec); + state_vec = GGML_F32_VEC_FMA(state_vec, sa_vec, b_vec); + GGML_F32_VEC_STORE(&state_cur[h_2d_i_j_offset], state_vec); + + result_vec[kk] = GGML_F32_VEC_FMA(result_vec[kk], state_vec, r_vec); + } + } + GGML_F32_VEC_REDUCE(dst_data[t_h_i_offset], result_vec); + + // There shouldn't be left-overs though. + for (; j < head_size; j++) { + int64_t t_h_j_offset = t_h_offset + j; + int64_t h_2d_i_j_offset = h_2d_i_offset + j; + + float r_val = r[t_h_j_offset]; + float w_val = w[t_h_j_offset]; + float k_val = k[t_h_j_offset]; + float b_val = b[t_h_j_offset]; + float kv_val = v[t_h_i_offset] * k_val; + + float prev_state_val = state_prev[h_2d_i_j_offset]; + state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val; + dst_data[t_h_i_offset] += state_cur[h_2d_i_j_offset] * r_val; + } + } + } + } + #else + for (int64_t t = 0; t < T; t++) { + int64_t t_offset = t * t_stride; + int64_t state_offset = head_size * C * (t / (T / n_seqs)); + float * state_cur = state + state_offset; + float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset; + + for (int64_t h = h_start; h < h_end; h++) { + int64_t h_offset = h * h_stride; + int64_t t_h_offset = t_offset + h_offset; + int64_t h_2d_offset = h * h_stride_2d; + + for (int64_t i = 0; i < head_size; i++) { + int64_t t_h_i_offset = t_h_offset + i; + int64_t h_2d_i_offset = h_2d_offset + i * h_stride; + + float v_val = v[t_h_i_offset]; + + float sa = 0, result = 0; + for (int64_t j = 0; j < head_size; j++) { + sa += a[t_h_offset + j] * state_prev[h_2d_i_offset + j]; + } + + for (int64_t j = 0; j < head_size; j++) { + int64_t t_h_j_offset = t_h_offset + j; + int64_t h_2d_i_j_offset = h_2d_i_offset + j; + + float r_val = r[t_h_j_offset]; + float w_val = w[t_h_j_offset]; + float k_val = k[t_h_j_offset]; + float b_val = b[t_h_j_offset]; + float kv_val = v_val * k_val; + float prev_state_val = state_prev[h_2d_i_j_offset]; + state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val; + result += state_cur[h_2d_i_j_offset] * r_val; + } + dst_data[t_h_i_offset] = result; + } + } + } + #endif +} + + +static void ggml_compute_forward_rwkv_wkv7( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_rwkv_wkv7_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + // ggml_compute_forward_map_unary static void ggml_compute_forward_map_unary_f32( @@ -14170,6 +14411,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_group_norm(params, tensor); } break; + case GGML_OP_L2_NORM: + { + ggml_compute_forward_l2_norm(params, tensor); + } break; case GGML_OP_MUL_MAT: { ggml_compute_forward_mul_mat(params, tensor); @@ -14357,6 +14602,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_gla(params, tensor); } break; + case GGML_OP_RWKV_WKV7: + { + ggml_compute_forward_rwkv_wkv7(params, tensor); + } break; case GGML_OP_MAP_UNARY: { ggml_unary_op_f32_t fun; @@ -14582,6 +14831,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_GROUP_NORM: case GGML_OP_CONCAT: case GGML_OP_MUL_MAT: @@ -14648,14 +14898,15 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_FLASH_ATTN_BACK: case GGML_OP_SSM_CONV: case GGML_OP_SSM_SCAN: + case GGML_OP_RWKV_WKV6: + case GGML_OP_GATED_LINEAR_ATTN: + case GGML_OP_RWKV_WKV7: { n_tasks = n_threads; } break; case GGML_OP_WIN_PART: case GGML_OP_WIN_UNPART: case GGML_OP_GET_REL_POS: - case GGML_OP_RWKV_WKV6: - case GGML_OP_GATED_LINEAR_ATTN: case GGML_OP_MAP_UNARY: case GGML_OP_MAP_BINARY: case GGML_OP_MAP_CUSTOM1_F32: diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 9bba398ce6..8fb063822c 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -36,7 +36,7 @@ #include "ggml-cuda/tsembd.cuh" #include "ggml-cuda/unary.cuh" #include "ggml-cuda/upscale.cuh" -#include "ggml-cuda/wkv6.cuh" +#include "ggml-cuda/wkv.cuh" #include "ggml-cuda/gla.cuh" #include "ggml.h" @@ -2196,6 +2196,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_GROUP_NORM: ggml_cuda_op_group_norm(ctx, dst); break; + case GGML_OP_L2_NORM: + ggml_cuda_op_l2_norm(ctx, dst); + break; case GGML_OP_CONCAT: ggml_cuda_op_concat(ctx, dst); break; @@ -2304,6 +2307,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_GATED_LINEAR_ATTN: ggml_cuda_op_gated_linear_attn(ctx, dst); break; + case GGML_OP_RWKV_WKV7: + ggml_cuda_op_rwkv_wkv7(ctx, dst); + break; case GGML_OP_CROSS_ENTROPY_LOSS_BACK: ggml_cuda_cross_entropy_loss_back(ctx, dst); break; @@ -3161,6 +3167,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g break; case GGML_OP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: return true; case GGML_OP_RMS_NORM_BACK: return ggml_is_contiguous(op->src[0]) && op->ne[0] % WARP_SIZE == 0; @@ -3215,6 +3222,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_LEAKY_RELU: case GGML_OP_RWKV_WKV6: case GGML_OP_GATED_LINEAR_ATTN: + case GGML_OP_RWKV_WKV7: return true; case GGML_OP_FLASH_ATTN_EXT: { #ifndef FLASH_ATTN_AVAILABLE diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index f127616edd..0020dbcec5 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -201,6 +201,85 @@ static __global__ void rms_norm_back_f32( } } +// template +// static __global__ void l2_norm_f32(const float * x, float * dst, const int ncols, const float eps) { +// const int row = blockIdx.x*blockDim.y + threadIdx.y; +// const int tid = threadIdx.x; + +// float tmp = 0.0f; // partial sum for thread in warp + +// for (int col = tid; col < ncols; col += block_size) { +// const float xi = x[row*ncols + col]; +// tmp += xi * xi; +// } + +// // sum up partial sums +// tmp = warp_reduce_sum(tmp); +// if (block_size > WARP_SIZE) { +// __shared__ float s_sum[32]; +// int warp_id = threadIdx.x / WARP_SIZE; +// int lane_id = threadIdx.x % WARP_SIZE; +// if (lane_id == 0) { +// s_sum[warp_id] = tmp; +// } +// __syncthreads(); +// tmp = s_sum[lane_id]; +// tmp = warp_reduce_sum(tmp); +// } + +// // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html +// const float scale = rsqrtf(fmaxf(tmp, eps * eps)); + +// for (int col = tid; col < ncols; col += block_size) { +// dst[row*ncols + col] = scale * x[row*ncols + col]; +// } +// } + +template +static __global__ void l2_norm_f32( + const float * x, float * dst, const int ncols, const int64_t stride_row, const int64_t stride_channel, + const int64_t stride_sample, const float eps) { + const int nrows = gridDim.x; + const int nchannels = gridDim.y; + + const int row = blockIdx.x; + const int channel = blockIdx.y; + const int sample = blockIdx.z; + const int tid = threadIdx.x; + + x += sample*stride_sample + channel*stride_channel + row*stride_row; + dst += ((sample*nchannels + channel)*nrows + row)*ncols; + + float tmp = 0.0f; // partial sum for thread in warp + + for (int col = tid; col < ncols; col += block_size) { + const float xi = x[col]; + tmp += xi * xi; + } + + // sum up partial sums + tmp = warp_reduce_sum(tmp); + if constexpr (block_size > WARP_SIZE) { + static_assert(block_size == 1024, "unexpected block_size"); + __shared__ float s_sum[32]; + const int warp_id = threadIdx.x / WARP_SIZE; + const int lane_id = threadIdx.x % WARP_SIZE; + if (lane_id == 0) { + s_sum[warp_id] = tmp; + } + __syncthreads(); + tmp = s_sum[lane_id]; + tmp = warp_reduce_sum(tmp); + } + + // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html + const float scale = rsqrtf(fmaxf(tmp, eps * eps)); + + for (int col = tid; col < ncols; col += block_size) { + dst[col] = scale * x[col]; + } +} + static void norm_f32_cuda( const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { @@ -248,6 +327,19 @@ static void rms_norm_back_f32_cuda(const float * grad, const float * xf, float * } } +static void l2_norm_f32_cuda( + const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { + const dim3 blocks_num(nrows, nchannels, nsamples); + if (ncols < 1024) { + const dim3 block_dims(WARP_SIZE, 1, 1); + l2_norm_f32<<>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + } else { + const dim3 block_dims(1024, 1, 1); + l2_norm_f32<1024><<>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + } +} + void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const float * src0_d = (const float *) src0->data; @@ -340,3 +432,27 @@ void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * d rms_norm_back_f32_cuda(grad_d, src0f_d, dst_d, ne00, nrows, eps, stream); } + +void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const float * src0_d = (const float *) src0->data; + float * dst_d = (float *) dst->data; + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + GGML_TENSOR_UNARY_OP_LOCALS; + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + GGML_ASSERT(eps >= 0.0f); + + const size_t ts0 = ggml_type_size(src0->type); + GGML_ASSERT(nb00 == ts0); + const int64_t s01 = nb01 / ts0; + const int64_t s02 = nb02 / ts0; + const int64_t s03 = nb03 / ts0; + + l2_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream); +} diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh index d63d34380b..706a5660a6 100644 --- a/ggml/src/ggml-cuda/norm.cuh +++ b/ggml/src/ggml-cuda/norm.cuh @@ -7,3 +7,5 @@ void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/wkv.cu b/ggml/src/ggml-cuda/wkv.cu new file mode 100644 index 0000000000..d2fced705e --- /dev/null +++ b/ggml/src/ggml-cuda/wkv.cu @@ -0,0 +1,199 @@ +#include "common.cuh" +#include "wkv.cuh" + +template +static __global__ void rwkv_wkv_f32(const int B, const int T, const int C, const int H, const float * k, const float * v, const float * r, const float * tf, const float * td, const float * s, float * dst) { + const int tid = threadIdx.x; + const int bid = blockIdx.x; + + const int head_size = block_size; + const int batch_i = bid / H; + const int head_i = bid % H; + const int state_size = C * head_size; + const int n_seq_tokens = T / B; + + float state[head_size]; + __shared__ float _k[head_size], _r[head_size], _tf[head_size], _td[head_size]; + + #pragma unroll + for (int i = 0; i < head_size; i++) { + state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid]; + } + + __syncthreads(); + _tf[tid] = tf[head_i * head_size + tid]; + __syncthreads(); + + for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) { + __syncthreads(); + _k[tid] = k[t]; + _r[tid] = r[t]; + _td[tid] = td[t]; + __syncthreads(); + + const float _v = v[t]; + float y = 0; + for (int j = 0; j < head_size; j += 4) { + const float4& k = (float4&)(_k[j]); + const float4& r = (float4&)(_r[j]); + const float4& tf = (float4&)(_tf[j]); + const float4& td = (float4&)(_td[j]); + float4& s = (float4&)(state[j]); + float4 kv; + + kv.x = k.x * _v; + kv.y = k.y * _v; + kv.z = k.z * _v; + kv.w = k.w * _v; + + y += r.x * (tf.x * kv.x + s.x); + y += r.y * (tf.y * kv.y + s.y); + y += r.z * (tf.z * kv.z + s.z); + y += r.w * (tf.w * kv.w + s.w); + + s.x = s.x * td.x + kv.x; + s.y = s.y * td.y + kv.y; + s.z = s.z * td.z + kv.z; + s.w = s.w * td.w + kv.w; + } + dst[t] = y; + } + + #pragma unroll + for (int i = 0; i < head_size; i++) { + dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i]; + } +} + +template +static __global__ void rwkv_wkv7_f32(const int B, const int T, const int C, const int H, const float * r, const float * w, const float * k, const float * v, const float * a, const float * b, const float * s, float * dst) { + const int tid = threadIdx.x; + const int bid = blockIdx.x; + + const int head_size = block_size; + const int batch_i = bid / H; + const int head_i = bid % H; + const int state_size = C * head_size; + const int n_seq_tokens = T / B; + + float state[head_size]; + __shared__ float _r[head_size], _w[head_size], _k[head_size], _a[head_size], _b[head_size]; + +#ifndef GGML_USE_MUSA + #pragma unroll +#endif + for (int i = 0; i < head_size; i++) { + state[i] = s[batch_i * state_size + head_i * head_size * head_size + tid * head_size + i]; + } + + for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) { + __syncthreads(); + _r[tid] = r[t]; + _w[tid] = w[t]; + _k[tid] = k[t]; + _a[tid] = a[t]; + _b[tid] = b[t]; + __syncthreads(); + + float sa = 0; + #pragma unroll + for (int j = 0; j < head_size; j += 4) + { + const float4& a = (float4&)(_a[j]); + const float4& s = (float4&)(state[j]); + sa += a.x * s.x; + sa += a.y * s.y; + sa += a.z * s.z; + sa += a.w * s.w; + } + + const float _v = v[t]; + float y = 0; + for (int j = 0; j < head_size; j += 4) { + const float4& r = (float4&)(_r[j]); + const float4& w = (float4&)(_w[j]); + const float4& k = (float4&)(_k[j]); + const float4& b = (float4&)(_b[j]); + float4& s = (float4&)(state[j]); + float4 kv; + + kv.x = k.x * _v; + kv.y = k.y * _v; + kv.z = k.z * _v; + kv.w = k.w * _v; + + s.x = s.x * w.x + kv.x + sa * b.x; + s.y = s.y * w.y + kv.y + sa * b.y; + s.z = s.z * w.z + kv.z + sa * b.z; + s.w = s.w * w.w + kv.w + sa * b.w; + + y += s.x * r.x; + y += s.y * r.y; + y += s.z * r.z; + y += s.w * r.w; + } + dst[t] = y; + } + + #pragma unroll + for (int i = 0; i < head_size; i++) { + dst[T * C + batch_i * state_size + head_i * head_size * head_size + tid * head_size + i] = state[i]; + } +} + +void ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const float * k_d = (const float *)dst->src[0]->data; + const float * v_d = (const float *)dst->src[1]->data; + const float * r_d = (const float *)dst->src[2]->data; + const float * tf_d = (const float *)dst->src[3]->data; + const float * td_d = (const float *)dst->src[4]->data; + const float * s_d = (const float *)dst->src[5]->data; + + const int64_t B = dst->src[5]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + float * dst_d = (float *)dst->data; + + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE || C / H == CUDA_WKV_BLOCK_SIZE * 2); + + if (C / H == CUDA_WKV_BLOCK_SIZE) { + rwkv_wkv_f32<<>>(B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d); + } else { + rwkv_wkv_f32<<>>(B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d); + } +} + +void ggml_cuda_op_rwkv_wkv7(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const float * r_d = (const float *)dst->src[0]->data; + const float * w_d = (const float *)dst->src[1]->data; + const float * k_d = (const float *)dst->src[2]->data; + const float * v_d = (const float *)dst->src[3]->data; + const float * a_d = (const float *)dst->src[4]->data; + const float * b_d = (const float *)dst->src[5]->data; + const float * s_d = (const float *)dst->src[6]->data; + + const int64_t B = dst->src[6]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + float * dst_d = (float *)dst->data; + + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(dst->src[6]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE || C / H == CUDA_WKV_BLOCK_SIZE * 2); + + if (C / H == CUDA_WKV_BLOCK_SIZE) { + rwkv_wkv7_f32<<>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d); + } else { + rwkv_wkv7_f32<<>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d); + } +} diff --git a/ggml/src/ggml-cuda/wkv6.cuh b/ggml/src/ggml-cuda/wkv.cuh similarity index 62% rename from ggml/src/ggml-cuda/wkv6.cuh rename to ggml/src/ggml-cuda/wkv.cuh index a7124ee517..9623dd7f8c 100644 --- a/ggml/src/ggml-cuda/wkv6.cuh +++ b/ggml/src/ggml-cuda/wkv.cuh @@ -3,3 +3,5 @@ #define CUDA_WKV_BLOCK_SIZE 64 void ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +void ggml_cuda_op_rwkv_wkv7(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/wkv6.cu b/ggml/src/ggml-cuda/wkv6.cu deleted file mode 100644 index bbdafbee58..0000000000 --- a/ggml/src/ggml-cuda/wkv6.cu +++ /dev/null @@ -1,89 +0,0 @@ -#include "common.cuh" -#include "wkv6.cuh" - -static __global__ void rwkv_wkv_f32(const int B, const int T, const int C, const int H, const float * k, const float * v, const float * r, const float * tf, const float * td, const float * s, float * dst) { - const int tid = threadIdx.x; - const int bid = blockIdx.x; - - const int head_size = CUDA_WKV_BLOCK_SIZE; - const int batch_i = bid / H; - const int head_i = bid % H; - const int state_size = C * head_size; - const int n_seq_tokens = T / B; - - float state[head_size]; - __shared__ float _k[head_size], _r[head_size], _tf[head_size], _td[head_size]; - - #pragma unroll - for (int i = 0; i < head_size; i++) { - state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid]; - } - - __syncthreads(); - _tf[tid] = tf[head_i * head_size + tid]; - __syncthreads(); - - for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) { - __syncthreads(); - _k[tid] = k[t]; - _r[tid] = r[t]; - _td[tid] = td[t]; - __syncthreads(); - - const float _v = v[t]; - float y = 0; - for (int j = 0; j < head_size; j += 4) { - const float4& k = (float4&)(_k[j]); - const float4& r = (float4&)(_r[j]); - const float4& tf = (float4&)(_tf[j]); - const float4& td = (float4&)(_td[j]); - float4& s = (float4&)(state[j]); - float4 kv; - - kv.x = k.x * _v; - kv.y = k.y * _v; - kv.z = k.z * _v; - kv.w = k.w * _v; - - y += r.x * (tf.x * kv.x + s.x); - y += r.y * (tf.y * kv.y + s.y); - y += r.z * (tf.z * kv.z + s.z); - y += r.w * (tf.w * kv.w + s.w); - - s.x = s.x * td.x + kv.x; - s.y = s.y * td.y + kv.y; - s.z = s.z * td.z + kv.z; - s.w = s.w * td.w + kv.w; - } - dst[t] = y; - } - - #pragma unroll - for (int i = 0; i < head_size; i++) { - dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i]; - } -} - -void ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const float * k_d = (const float *)dst->src[0]->data; - const float * v_d = (const float *)dst->src[1]->data; - const float * r_d = (const float *)dst->src[2]->data; - const float * tf_d = (const float *)dst->src[3]->data; - const float * td_d = (const float *)dst->src[4]->data; - const float * s_d = (const float *)dst->src[5]->data; - - const int64_t B = dst->src[5]->ne[1]; - const int64_t T = dst->src[0]->ne[2]; - const int64_t C = dst->ne[0]; - const int64_t H = dst->src[0]->ne[1]; - - float * dst_d = (float *)dst->data; - - cudaStream_t stream = ctx.stream(); - - GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); - GGML_ASSERT(C % H == 0); - GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE); // The current cuda kernel is designed for RWKV6, HEAD_SIZE == 64 - - rwkv_wkv_f32<<>>(B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d); -} diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index a58c474eb0..1e954b4cea 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -285,6 +285,13 @@ typedef struct { float eps; } ggml_metal_kargs_rms_norm; +typedef struct { + int32_t ne00; + int32_t ne00_4; + uint64_t nb01; + float eps; +} ggml_metal_kargs_l2_norm; + typedef struct { int64_t ne00; int64_t ne01; diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m index e51a4169a2..af65e7d9f5 100644 --- a/ggml/src/ggml-metal/ggml-metal.m +++ b/ggml/src/ggml-metal/ggml-metal.m @@ -184,10 +184,13 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, GGML_METAL_KERNEL_TYPE_RMS_NORM, + GGML_METAL_KERNEL_TYPE_L2_NORM, GGML_METAL_KERNEL_TYPE_GROUP_NORM, GGML_METAL_KERNEL_TYPE_NORM, GGML_METAL_KERNEL_TYPE_SSM_CONV_F32, GGML_METAL_KERNEL_TYPE_SSM_SCAN_F32, + GGML_METAL_KERNEL_TYPE_RWKV_WKV6_F32, + GGML_METAL_KERNEL_TYPE_RWKV_WKV7_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, @@ -810,10 +813,13 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, get_rows_iq4_xs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, has_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_L2_NORM, l2_norm, has_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, has_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SSM_CONV_F32, ssm_conv_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SSM_SCAN_F32, ssm_scan_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RWKV_WKV6_F32, rwkv_wkv6_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RWKV_WKV7_F32, rwkv_wkv7_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, has_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_BF16_F32, mul_mv_bf16_f32, has_simdgroup_reduction && use_bfloat); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_BF16_F32_1ROW, mul_mv_bf16_f32_1row, has_simdgroup_reduction && use_bfloat); @@ -1251,6 +1257,7 @@ static bool ggml_metal_supports_op(const struct ggml_backend_metal_device_contex case GGML_OP_GROUP_NORM: return has_simdgroup_reduction && ggml_is_contiguous(op->src[0]); case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: return has_simdgroup_reduction && (op->ne[0] % 4 == 0 && ggml_is_contiguous_1(op->src[0])); case GGML_OP_ARGMAX: return true; @@ -1288,6 +1295,8 @@ static bool ggml_metal_supports_op(const struct ggml_backend_metal_device_contex return has_simdgroup_mm; // TODO: over-restricted for vec-kernels case GGML_OP_SSM_CONV: case GGML_OP_SSM_SCAN: + case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: return true; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: @@ -2216,6 +2225,83 @@ static void ggml_metal_encode_node( [encoder dispatchThreadgroups:MTLSizeMake(d_inner, n_seqs, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; } break; + case GGML_OP_RWKV_WKV6: + { + const int64_t B = dst->src[5]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == 64); + + size_t offs_src3 = 0; + size_t offs_src4 = 0; + size_t offs_src5 = 0; + + id id_src3 = dst->src[3] ? ggml_metal_get_buffer(dst->src[3], &offs_src3) : nil; + id id_src4 = dst->src[4] ? ggml_metal_get_buffer(dst->src[4], &offs_src4) : nil; + id id_src5 = dst->src[5] ? ggml_metal_get_buffer(dst->src[5], &offs_src5) : nil; + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RWKV_WKV6_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2]; + [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3]; + [encoder setBuffer:id_src4 offset:offs_src4 atIndex:4]; + [encoder setBuffer:id_src5 offset:offs_src5 atIndex:5]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:6]; + + [encoder setBytes:&B length:sizeof(B) atIndex:7]; + [encoder setBytes:&T length:sizeof(T) atIndex:8]; + [encoder setBytes:&C length:sizeof(C) atIndex:9]; + [encoder setBytes:&H length:sizeof(H) atIndex:10]; + + [encoder dispatchThreadgroups:MTLSizeMake(B * H, 1, 1) threadsPerThreadgroup:MTLSizeMake(C/ H, 1, 1)]; + } break; + case GGML_OP_RWKV_WKV7: + { + const int64_t B = dst->src[6]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + GGML_ASSERT(dst->src[6]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == 64); + + size_t offs_src3 = 0; + size_t offs_src4 = 0; + size_t offs_src5 = 0; + size_t offs_src6 = 0; + + id id_src3 = dst->src[3] ? ggml_metal_get_buffer(dst->src[3], &offs_src3) : nil; + id id_src4 = dst->src[4] ? ggml_metal_get_buffer(dst->src[4], &offs_src4) : nil; + id id_src5 = dst->src[5] ? ggml_metal_get_buffer(dst->src[5], &offs_src5) : nil; + id id_src6 = dst->src[6] ? ggml_metal_get_buffer(dst->src[6], &offs_src6) : nil; + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RWKV_WKV7_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2]; + [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3]; + [encoder setBuffer:id_src4 offset:offs_src4 atIndex:4]; + [encoder setBuffer:id_src5 offset:offs_src5 atIndex:5]; + [encoder setBuffer:id_src6 offset:offs_src6 atIndex:6]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:7]; + + [encoder setBytes:&B length:sizeof(B) atIndex:8]; + [encoder setBytes:&T length:sizeof(T) atIndex:9]; + [encoder setBytes:&C length:sizeof(C) atIndex:10]; + [encoder setBytes:&H length:sizeof(H) atIndex:11]; + + [encoder dispatchThreadgroups:MTLSizeMake(B * H, 1, 1) threadsPerThreadgroup:MTLSizeMake(C/ H, 1, 1)]; + } break; case GGML_OP_MUL_MAT: { GGML_ASSERT(ne00 == ne10); @@ -3122,6 +3208,42 @@ static void ggml_metal_encode_node( const int64_t nrows = ggml_nrows(src0); + [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_L2_NORM: + { + GGML_ASSERT(ne00 % 4 == 0); + GGML_ASSERT(ggml_is_contiguous_1(src0)); + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_L2_NORM].pipeline; + + int nth = 32; // SIMD width + + while (nth < ne00/4 && nth < (int) pipeline.maxTotalThreadsPerThreadgroup) { + nth *= 2; + } + + nth = MIN(nth, ne00/4); + + ggml_metal_kargs_l2_norm args = { + /*.ne00 =*/ ne00, + /*.ne00_4 =*/ ne00/4, + /*.nb01 =*/ nb01, + /*.eps =*/ eps, + }; + + [encoder setComputePipelineState:pipeline]; + [encoder setBytes:&args length:sizeof(args) atIndex:0]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + + [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; + + const int64_t nrows = ggml_nrows(src0); + [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; } break; case GGML_OP_GROUP_NORM: diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index ad9d42a3ea..3cef81b797 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -1295,6 +1295,184 @@ kernel void kernel_ssm_scan_f32( } } +kernel void kernel_rwkv_wkv6_f32( + device const float * k, + device const float * v, + device const float * r, + device const float * tf, + device const float * td, + device const float * state_in, + device float * dst, + constant uint & B, + constant uint & T, + constant uint & C, + constant uint & H, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const uint head_size = 64; // TODO: support head_size = 128 + const uint batch_id = tgpig.x / H; + const uint head_id = tgpig.x % H; + const uint tid = tpitg.x; + + if (batch_id >= B || head_id >= H) { + return; + } + + const uint state_size = C * head_size; + const uint n_seq_tokens = T / B; + + threadgroup float _k[head_size]; + threadgroup float _r[head_size]; + threadgroup float _tf[head_size]; + threadgroup float _td[head_size]; + + float state[head_size]; + + for (uint i = 0; i < head_size; i++) { + state[i] = state_in[batch_id * state_size + head_id * head_size * head_size + + i * head_size + tid]; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + _tf[tid] = tf[head_id * head_size + tid]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; + const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; + + for (uint t = start_t; t < end_t; t += C) { + threadgroup_barrier(mem_flags::mem_threadgroup); + _k[tid] = k[t]; + _r[tid] = r[t]; + _td[tid] = td[t]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + const float v_val = v[t]; + float y = 0.0; + + for (uint j = 0; j < head_size; j += 4) { + float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + float4 tf_vec = float4(_tf[j], _tf[j+1], _tf[j+2], _tf[j+3]); + float4 td_vec = float4(_td[j], _td[j+1], _td[j+2], _td[j+3]); + float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); + + float4 kv = k_vec * v_val; + + float4 temp = tf_vec * kv + s_vec; + y += dot(r_vec, temp); + + s_vec = s_vec * td_vec + kv; + state[j] = s_vec[0]; + state[j+1] = s_vec[1]; + state[j+2] = s_vec[2]; + state[j+3] = s_vec[3]; + } + + dst[t] = y; + } + + for (uint i = 0; i < head_size; i++) { + dst[T * C + batch_id * state_size + head_id * head_size * head_size + + i * head_size + tid] = state[i]; + } +} + +kernel void kernel_rwkv_wkv7_f32( + device const float * r, + device const float * w, + device const float * k, + device const float * v, + device const float * a, + device const float * b, + device const float * state_in, + device float * dst, + constant uint & B, + constant uint & T, + constant uint & C, + constant uint & H, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const uint head_size = 64; // TODO: support head_size = 128 + const uint batch_id = tgpig.x / H; + const uint head_id = tgpig.x % H; + const uint tid = tpitg.x; + + if (batch_id >= B || head_id >= H) { + return; + } + + const uint state_size = C * head_size; + const uint n_seq_tokens = T / B; + + threadgroup float _r[head_size]; + threadgroup float _w[head_size]; + threadgroup float _k[head_size]; + threadgroup float _a[head_size]; + threadgroup float _b[head_size]; + + float state[head_size]; + + for (uint i = 0; i < head_size; i++) { + state[i] = state_in[batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i]; + } + + const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; + const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; + + for (uint t = start_t; t < end_t; t += C) { + threadgroup_barrier(mem_flags::mem_threadgroup); + _r[tid] = r[t]; + _w[tid] = w[t]; + _k[tid] = k[t]; + _a[tid] = a[t]; + _b[tid] = b[t]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + const float v_val = v[t]; + float y = 0.0, sa = 0.0; + + float4 sa_vec(0.0); + + for (int j = 0; j < head_size; j += 4) { + float4 a_vec = float4(_a[j], _a[j+1], _a[j+2], _a[j+3]); + float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); + sa_vec += a_vec * s_vec; + } + sa = sa_vec[0] + sa_vec[1] + sa_vec[2] + sa_vec[3]; + + for (uint j = 0; j < head_size; j += 4) { + float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + float4 w_vec = float4(_w[j], _w[j+1], _w[j+2], _w[j+3]); + float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + float4 b_vec = float4(_b[j], _b[j+1], _b[j+2], _b[j+3]); + float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); + + float4 kv = k_vec * v_val; + + s_vec = s_vec * w_vec + kv + sa * b_vec; + y += dot(s_vec, r_vec); + + state[j] = s_vec[0]; + state[j+1] = s_vec[1]; + state[j+2] = s_vec[2]; + state[j+3] = s_vec[3]; + } + + dst[t] = y; + } + + for (uint i = 0; i < head_size; i++) { + dst[T * C + batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i] = state[i]; + } +} + kernel void kernel_argmax( device const void * x, device int32_t * dst, @@ -1463,6 +1641,49 @@ kernel void kernel_rms_norm( } } +kernel void kernel_l2_norm( + constant ggml_metal_kargs_l2_norm & args, + device const char * src0, + device char * dst, + threadgroup float * shmem_f32 [[threadgroup(0)]], + uint tgpig[[threadgroup_position_in_grid]], + ushort tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort ntg[[threads_per_threadgroup]]) { + if (sgitg == 0) { + shmem_f32[tiisg] = 0.0f; + } + + device const float4 * x = (device const float4 *) (src0 + tgpig*args.nb01); + + float sumf = 0.0f; + + // parallel sum + for (int i00 = tpitg; i00 < args.ne00_4; i00 += ntg) { + sumf += dot(x[i00], x[i00]); + } + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_f32[tiisg]; + sumf = simd_sum(sumf); + + const float scale = 1.0f/sqrt(max(sumf, args.eps)); + + device float4 * y = (device float4 *) dst + tgpig*args.ne00_4; + for (int i00 = tpitg; i00 < args.ne00_4; i00 += ntg) { + y[i00] = x[i00] * scale; + } +} + kernel void kernel_group_norm( device const float * src0, device float * dst, diff --git a/ggml/src/ggml-sycl/backend.hpp b/ggml/src/ggml-sycl/backend.hpp index 577ff51fde..73d807cab0 100644 --- a/ggml/src/ggml-sycl/backend.hpp +++ b/ggml/src/ggml-sycl/backend.hpp @@ -26,7 +26,7 @@ #include "softmax.hpp" #include "tsembd.hpp" #include "im2col.hpp" -#include "wkv6.hpp" +#include "wkv.hpp" #include "outprod.hpp" #include "element_wise.hpp" #include "cpy.hpp" diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 05984d8c5a..477652ab28 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -2696,6 +2696,12 @@ static void ggml_sycl_rms_norm(ggml_backend_sycl_context & ctx, ggml_tensor * ds GGML_SYCL_DEBUG("call %s done\n", __func__); } +static void ggml_sycl_l2_norm(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + GGML_SYCL_DEBUG("call %s\n", __func__); + ggml_sycl_op_flatten(ctx, dst->src[0], dst->src[1], dst, ggml_sycl_op_l2_norm); + GGML_SYCL_DEBUG("call %s done\n", __func__); +} + static void ggml_sycl_group_norm(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(ctx, dst->src[0], dst->src[1], dst, ggml_sycl_op_group_norm); @@ -3410,6 +3416,9 @@ bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tens case GGML_OP_RMS_NORM: ggml_sycl_rms_norm(ctx, dst); break; + case GGML_OP_L2_NORM: + ggml_sycl_l2_norm(ctx, dst); + break; case GGML_OP_MUL_MAT: if (dst->src[0]->ne[3] != dst->src[1]->ne[3]) { return false; @@ -3487,6 +3496,9 @@ bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tens case GGML_OP_RWKV_WKV6: ggml_sycl_op_rwkv_wkv6(ctx, dst); break; + case GGML_OP_RWKV_WKV7: + ggml_sycl_op_rwkv_wkv7(ctx, dst); + break; case GGML_OP_GATED_LINEAR_ATTN: ggml_sycl_op_gated_linear_attn(ctx, dst); break; @@ -4012,6 +4024,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g return (op->src[0]->type == GGML_TYPE_F32); case GGML_OP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: case GGML_OP_GROUP_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_SCALE: @@ -4045,6 +4058,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_LEAKY_RELU: case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: case GGML_OP_GATED_LINEAR_ATTN: return true; default: diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp index 9cf2be1557..6439db21b2 100644 --- a/ggml/src/ggml-sycl/norm.cpp +++ b/ggml/src/ggml-sycl/norm.cpp @@ -180,6 +180,50 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, const floa } } +static void l2_norm_f32(const float* x, float* dst, const int ncols, const float eps, + const sycl::nd_item<3>& item_ct1, float* s_sum, int block_size) { + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + const int tid = item_ct1.get_local_id(2); + const int nthreads = item_ct1.get_local_range(2); + const int nwarps = nthreads / WARP_SIZE; + float tmp = 0.0f; // partial sum for thread in warp + + for (int col = tid; col < ncols; col += block_size) { + const float xi = x[row * ncols + col]; + tmp += xi * xi; + } + + // sum up partial sums + tmp = warp_reduce_sum(tmp, item_ct1); + if (block_size > WARP_SIZE) { + + int warp_id = item_ct1.get_local_id(2) / WARP_SIZE; + int lane_id = item_ct1.get_local_id(2) % WARP_SIZE; + if (lane_id == 0) { + s_sum[warp_id] = tmp; + } + /* + DPCT1118:3: SYCL group functions and algorithms must be encountered in + converged control flow. You may need to adjust the code. + */ + item_ct1.barrier(sycl::access::fence_space::local_space); + size_t nreduce = nwarps / WARP_SIZE; + tmp = 0.f; + for (size_t i = 0; i < nreduce; i += 1) + { + tmp += s_sum[lane_id + i * WARP_SIZE]; + } + tmp = warp_reduce_sum(tmp, item_ct1); + } + + const float scale = sycl::rsqrt(sycl::max(tmp, eps * eps)); + + for (int col = tid; col < ncols; col += block_size) { + dst[row * ncols + col] = scale * x[row * ncols + col]; + } +} + static void norm_f32_sycl(const float* x, float* dst, const int ncols, const int nrows, const float eps, queue_ptr stream, int device) { @@ -311,6 +355,48 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, } } +static void l2_norm_f32_sycl(const float* x, float* dst, const int ncols, + const int nrows, const float eps, + queue_ptr stream, int device) { + GGML_ASSERT(ncols % WARP_SIZE == 0); + // printf("%s ncols=%d, nrows=%d, WARP_SIZE=%d\n", __func__, ncols, nrows, WARP_SIZE); + if (ncols < 1024) { + const sycl::range<3> block_dims(1, 1, WARP_SIZE); + stream->submit([&](sycl::handler& cgh) { + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, + block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(WARP_SIZE)]] { + l2_norm_f32(x, dst, ncols, eps, item_ct1, + nullptr, WARP_SIZE); + }); + }); + } + else { + const int work_group_size = ggml_sycl_info().max_work_group_sizes[device]; + assert(work_group_size % (WARP_SIZE * WARP_SIZE) == 0); + const sycl::range<3> block_dims(1, 1, work_group_size); + /* + DPCT1049:19: The work-group size passed to the SYCL kernel may exceed + the limit. To get the device limit, query + info::device::max_work_group_size. Adjust the work-group size if needed. + */ + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor s_sum_acc_ct1(sycl::range<1>(work_group_size / WARP_SIZE), + cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, + block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(WARP_SIZE)]] { + l2_norm_f32(x, dst, ncols, eps, item_ct1, + get_pointer(s_sum_acc_ct1), work_group_size); + }); + }); + } +} + void ggml_sycl_op_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor* dst, const float* src0_dd, const float* src1_dd, float* dst_dd, @@ -376,3 +462,25 @@ void ggml_sycl_op_rms_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* sr (void)dst; (void)src1_dd; } + +void ggml_sycl_op_l2_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* src0, + const ggml_tensor* src1, ggml_tensor* dst, + const float* src0_dd, const float* src1_dd, + float* dst_dd, + const queue_ptr& main_stream) { + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + const int64_t ne00 = src0->ne[0]; + const int64_t nrows = ggml_nrows(src0); + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + + l2_norm_f32_sycl(src0_dd, dst_dd, ne00, nrows, eps, main_stream, ctx.device); + + (void)src1; + (void)dst; + (void)src1_dd; +} diff --git a/ggml/src/ggml-sycl/norm.hpp b/ggml/src/ggml-sycl/norm.hpp index a9ad9156fa..11e91680cc 100644 --- a/ggml/src/ggml-sycl/norm.hpp +++ b/ggml/src/ggml-sycl/norm.hpp @@ -32,4 +32,10 @@ void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* float* dst_dd, const queue_ptr& main_stream); +void ggml_sycl_op_l2_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* src0, + const ggml_tensor* src1, ggml_tensor* dst, + const float* src0_dd, const float* src1_dd, + float* dst_dd, + const queue_ptr& main_stream); + #endif // GGML_SYCL_NORM_HPP diff --git a/ggml/src/ggml-sycl/wkv.cpp b/ggml/src/ggml-sycl/wkv.cpp new file mode 100644 index 0000000000..540f6fbf5f --- /dev/null +++ b/ggml/src/ggml-sycl/wkv.cpp @@ -0,0 +1,305 @@ +#include +#include "wkv.hpp" + +constexpr int WKV_BLOCK_SIZE = 64; // Matching CUDA_WKV_BLOCK_SIZE + +// Helper function for the main kernel +template +static void rwkv_wkv6_f32_kernel( + const int B, const int T, const int C, const int H, + const float* k, const float* v, const float* r, + const float* tf, const float* td, const float* s, + float* dst, const sycl::nd_item<3>& item_ct1, float* shared_mem) { + + const int tid = item_ct1.get_local_id(2); + const int bid = item_ct1.get_group(2); + + const int head_size = block_size; + const int batch_i = bid / H; + const int head_i = bid % H; + const int state_size = C * head_size; + const int n_seq_tokens = T / B; + + // Set up shared memory pointers + float* _k = shared_mem; + float* _r = _k + head_size; + float* _tf = _r + head_size; + float* _td = _tf + head_size; + + // Local state array + float state[block_size]; + + // Load initial state + #pragma unroll + for (int i = 0; i < head_size; i++) { + state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid]; + } + + // Sync threads before shared memory operations + item_ct1.barrier(sycl::access::fence_space::local_space); + + // Load time-mixing parameters + _tf[tid] = tf[head_i * head_size + tid]; + item_ct1.barrier(sycl::access::fence_space::local_space); + + // Main sequence processing loop + for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; + t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; + t += C) { + + item_ct1.barrier(sycl::access::fence_space::local_space); + + // Load current timestep data to shared memory + _k[tid] = k[t]; + _r[tid] = r[t]; + _td[tid] = td[t]; + + item_ct1.barrier(sycl::access::fence_space::local_space); + + const float _v = v[t]; + float y = 0; + + // Process in chunks of 4 for better vectorization + sycl::float4 k4, r4, tf4, td4, s4; + #pragma unroll + for (int j = 0; j < head_size; j += 4) { + // Load data in vec4 chunks + k4 = sycl::float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + r4 = sycl::float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + tf4 = sycl::float4(_tf[j], _tf[j+1], _tf[j+2], _tf[j+3]); + td4 = sycl::float4(_td[j], _td[j+1], _td[j+2], _td[j+3]); + s4 = sycl::float4(state[j], state[j+1], state[j+2], state[j+3]); + + // Compute key-value product + sycl::float4 kv4 = k4 * _v; + + // Accumulate weighted sum + y += sycl::dot(r4, tf4 * kv4 + s4); + + // Update state + s4 = s4 * td4 + kv4; + + // Store updated state + state[j] = s4.x(); + state[j+1] = s4.y(); + state[j+2] = s4.z(); + state[j+3] = s4.w(); + } + + dst[t] = y; + } + + // Save final state + #pragma unroll + for (int i = 0; i < head_size; i++) { + dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i]; + } +} + +template +static void rwkv_wkv7_f32_kernel( + const int B, const int T, const int C, const int H, + const float* r, const float* w, const float* k, const float* v, + const float* a, const float* b, const float* s, + float* dst, const sycl::nd_item<3>& item_ct1, float* shared_mem) { + + const int tid = item_ct1.get_local_id(2); + const int bid = item_ct1.get_group(2); + + const int head_size = block_size; + const int batch_i = bid / H; + const int head_i = bid % H; + const int state_size = C * head_size; + const int n_seq_tokens = T / B; + + float* _r = shared_mem; + float* _w = _r + head_size; + float* _k = _w + head_size; + float* _a = _k + head_size; + float* _b = _a + head_size; + + float state[block_size]; + + #pragma unroll + for (int i = 0; i < head_size; i++) { + state[i] = s[batch_i * state_size + head_i * head_size * head_size + tid * head_size + i]; + } + + for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; + t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; + t += C) { + + item_ct1.barrier(sycl::access::fence_space::local_space); + + _r[tid] = r[t]; + _w[tid] = w[t]; + _k[tid] = k[t]; + _a[tid] = a[t]; + _b[tid] = b[t]; + + item_ct1.barrier(sycl::access::fence_space::local_space); + + const float _v = v[t]; + float y = 0, sa = 0; + sycl::float4 a4, s4; + + #pragma unroll + for (int j = 0; j < head_size; j += 4) { + a4 = sycl::float4(_a[j], _a[j+1], _a[j+2], _a[j+3]); + s4 = sycl::float4(state[j], state[j+1], state[j+2], state[j+3]); + sa += sycl::dot(a4, s4); + } + + sycl::float4 r4, w4, k4, b4; + #pragma unroll + for (int j = 0; j < head_size; j += 4) { + r4 = sycl::float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + w4 = sycl::float4(_w[j], _w[j+1], _w[j+2], _w[j+3]); + k4 = sycl::float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + b4 = sycl::float4(_b[j], _b[j+1], _b[j+2], _b[j+3]); + s4 = sycl::float4(state[j], state[j+1], state[j+2], state[j+3]); + + sycl::float4 kv4 = k4 * _v; + + s4 = s4 * w4 + kv4 + sa * b4; + y += sycl::dot(r4, s4); + + state[j] = s4.x(); + state[j+1] = s4.y(); + state[j+2] = s4.z(); + state[j+3] = s4.w(); + } + + dst[t] = y; + } + + #pragma unroll + for (int i = 0; i < head_size; i++) { + dst[T * C + batch_i * state_size + head_i * head_size * head_size + tid * head_size + i] = state[i]; + } +} + +void ggml_sycl_op_rwkv_wkv6(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { + + const ggml_tensor *src0 = dst->src[0]; + const ggml_tensor *src1 = dst->src[1]; + + const float* k_d = (const float*)dst->src[0]->data; + const float* v_d = (const float*)dst->src[1]->data; + const float* r_d = (const float*)dst->src[2]->data; + const float* tf_d = (const float*)dst->src[3]->data; + const float* td_d = (const float*)dst->src[4]->data; + const float* s_d = (const float*)dst->src[5]->data; + float* dst_d = (float*)dst->data; + + const int64_t B = dst->src[5]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == WKV_BLOCK_SIZE || C / H == WKV_BLOCK_SIZE * 2); // The current sycl kernel is designed for RWKV6, HEAD_SIZE == 64 + + dpct::queue_ptr stream = ctx.stream(); + + // Calculate execution configuration + const size_t shared_mem_size = C / H * 4 * sizeof(float); // For k, r, tf, td + sycl::range<3> block_dims(1, 1, C / H); + sycl::range<3> grid_dims(1, 1, B * H); + + // Submit kernel + if (C / H == WKV_BLOCK_SIZE) { + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + rwkv_wkv6_f32_kernel( + B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d, + item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() + ); + }); + }); + } else { + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + rwkv_wkv6_f32_kernel( + B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d, + item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() + ); + }); + }); + } + + GGML_UNUSED(src0); + GGML_UNUSED(src1); +} + +void ggml_sycl_op_rwkv_wkv7(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { + + const ggml_tensor *src0 = dst->src[0]; + const ggml_tensor *src1 = dst->src[1]; + + const float* r_d = (const float*)dst->src[0]->data; + const float* w_d = (const float*)dst->src[1]->data; + const float* k_d = (const float*)dst->src[2]->data; + const float* v_d = (const float*)dst->src[3]->data; + const float* a_d = (const float*)dst->src[4]->data; + const float* b_d = (const float*)dst->src[5]->data; + const float* s_d = (const float*)dst->src[6]->data; + float* dst_d = (float*)dst->data; + + const int64_t B = dst->src[6]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + GGML_ASSERT(dst->src[6]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == WKV_BLOCK_SIZE || C / H == WKV_BLOCK_SIZE * 2); + + dpct::queue_ptr stream = ctx.stream(); + + // Calculate execution configuration + const size_t shared_mem_size = C / H * 5 * sizeof(float); // For r, w, k, a, b + sycl::range<3> block_dims(1, 1, C / H); + sycl::range<3> grid_dims(1, 1, B * H); + + // Submit kernel + if (C / H == WKV_BLOCK_SIZE) { + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + rwkv_wkv7_f32_kernel( + B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d, + item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() + ); + }); + }); + } else { + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + rwkv_wkv7_f32_kernel( + B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d, + item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() + ); + }); + }); + } + + GGML_UNUSED(src0); + GGML_UNUSED(src1); +} diff --git a/ggml/src/ggml-sycl/wkv.hpp b/ggml/src/ggml-sycl/wkv.hpp new file mode 100644 index 0000000000..9f34a1001f --- /dev/null +++ b/ggml/src/ggml-sycl/wkv.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_WKV_HPP +#define GGML_SYCL_WKV_HPP + +#include "common.hpp" + +void ggml_sycl_op_rwkv_wkv6(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +void ggml_sycl_op_rwkv_wkv7(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_WKV_HPP diff --git a/ggml/src/ggml-sycl/wkv6.cpp b/ggml/src/ggml-sycl/wkv6.cpp deleted file mode 100644 index b54c20964e..0000000000 --- a/ggml/src/ggml-sycl/wkv6.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include -#include "wkv6.hpp" - -constexpr int WKV_BLOCK_SIZE = 64; // Matching CUDA_WKV_BLOCK_SIZE - -// Helper function for the main kernel -static void rwkv_wkv_f32_kernel( - const int B, const int T, const int C, const int H, - const float* k, const float* v, const float* r, - const float* tf, const float* td, const float* s, - float* dst, const sycl::nd_item<3>& item_ct1, float* shared_mem) { - - const int tid = item_ct1.get_local_id(2); - const int bid = item_ct1.get_group(2); - - const int head_size = WKV_BLOCK_SIZE; - const int batch_i = bid / H; - const int head_i = bid % H; - const int state_size = C * head_size; - const int n_seq_tokens = T / B; - - // Set up shared memory pointers - float* _k = shared_mem; - float* _r = _k + head_size; - float* _tf = _r + head_size; - float* _td = _tf + head_size; - - // Local state array - float state[WKV_BLOCK_SIZE]; - - // Load initial state - #pragma unroll - for (int i = 0; i < head_size; i++) { - state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid]; - } - - // Sync threads before shared memory operations - item_ct1.barrier(sycl::access::fence_space::local_space); - - // Load time-mixing parameters - _tf[tid] = tf[head_i * head_size + tid]; - item_ct1.barrier(sycl::access::fence_space::local_space); - - // Main sequence processing loop - for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; - t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; - t += C) { - - item_ct1.barrier(sycl::access::fence_space::local_space); - - // Load current timestep data to shared memory - _k[tid] = k[t]; - _r[tid] = r[t]; - _td[tid] = td[t]; - - item_ct1.barrier(sycl::access::fence_space::local_space); - - const float _v = v[t]; - float y = 0; - - // Process in chunks of 4 for better vectorization - sycl::float4 k4, r4, tf4, td4, s4; - #pragma unroll - for (int j = 0; j < head_size; j += 4) { - // Load data in vec4 chunks - k4 = sycl::float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); - r4 = sycl::float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); - tf4 = sycl::float4(_tf[j], _tf[j+1], _tf[j+2], _tf[j+3]); - td4 = sycl::float4(_td[j], _td[j+1], _td[j+2], _td[j+3]); - s4 = sycl::float4(state[j], state[j+1], state[j+2], state[j+3]); - - // Compute key-value product - sycl::float4 kv4 = k4 * _v; - - // Accumulate weighted sum - y += sycl::dot(r4, tf4 * kv4 + s4); - - // Update state - s4 = s4 * td4 + kv4; - - // Store updated state - state[j] = s4.x(); - state[j+1] = s4.y(); - state[j+2] = s4.z(); - state[j+3] = s4.w(); - } - - dst[t] = y; - } - - // Save final state - #pragma unroll - for (int i = 0; i < head_size; i++) { - dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i]; - } -} - -void ggml_sycl_op_rwkv_wkv6(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { - - const ggml_tensor *src0 = dst->src[0]; - const ggml_tensor *src1 = dst->src[1]; - - const float* k_d = (const float*)dst->src[0]->data; - const float* v_d = (const float*)dst->src[1]->data; - const float* r_d = (const float*)dst->src[2]->data; - const float* tf_d = (const float*)dst->src[3]->data; - const float* td_d = (const float*)dst->src[4]->data; - const float* s_d = (const float*)dst->src[5]->data; - float* dst_d = (float*)dst->data; - - const int64_t B = dst->src[5]->ne[1]; - const int64_t T = dst->src[0]->ne[2]; - const int64_t C = dst->ne[0]; - const int64_t H = dst->src[0]->ne[1]; - - GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); - GGML_ASSERT(C % H == 0); - GGML_ASSERT(C / H == WKV_BLOCK_SIZE); // The current sycl kernel is designed for RWKV6, HEAD_SIZE == 64 - - dpct::queue_ptr stream = ctx.stream(); - - // Calculate execution configuration - const size_t shared_mem_size = WKV_BLOCK_SIZE * 4 * sizeof(float); // For k, r, tf, td - sycl::range<3> block_dims(1, 1, C / H); - sycl::range<3> grid_dims(1, 1, B * H); - - // Submit kernel - stream->submit([&](sycl::handler& cgh) { - sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); - - cgh.parallel_for( - sycl::nd_range<3>(grid_dims * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - rwkv_wkv_f32_kernel( - B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d, - item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() - ); - }); - }); - - GGML_UNUSED(src0); - GGML_UNUSED(src1); -} diff --git a/ggml/src/ggml-sycl/wkv6.hpp b/ggml/src/ggml-sycl/wkv6.hpp deleted file mode 100644 index 8c596a9972..0000000000 --- a/ggml/src/ggml-sycl/wkv6.hpp +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef GGML_SYCL_WKV6_HPP -#define GGML_SYCL_WKV6_HPP - -#include "common.hpp" - -void ggml_sycl_op_rwkv_wkv6(ggml_backend_sycl_context & ctx, ggml_tensor * dst); - - -#endif // GGML_SYCL_WKV6_HPP diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 97398f071b..c0ee5dadef 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -304,6 +304,7 @@ struct vk_device_struct { vk_pipeline pipeline_group_norm_f32; vk_pipeline pipeline_rms_norm_f32; vk_pipeline pipeline_rms_norm_back_f32; + vk_pipeline pipeline_l2_norm_f32; vk_pipeline pipeline_gelu_f32; vk_pipeline pipeline_gelu_quick_f32; vk_pipeline pipeline_silu_f32; @@ -328,6 +329,7 @@ struct vk_device_struct { vk_pipeline pipeline_timestep_embedding_f32; vk_pipeline pipeline_pool2d_f32; vk_pipeline pipeline_rwkv_wkv6_f32; + vk_pipeline pipeline_rwkv_wkv7_f32; vk_pipeline pipeline_opt_step_adamw_f32; // [2][2][2] is for {f16acc,f32acc}x{large,small_rows}x{unaligned, aligned} @@ -629,6 +631,13 @@ struct vk_op_rwkv_wkv6_push_constants { uint32_t H; }; +struct vk_op_rwkv_wkv7_push_constants { + uint32_t B; + uint32_t T; + uint32_t C; + uint32_t H; +}; + // Allow pre-recording command buffers struct vk_staging_memcpy { vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {} @@ -2263,6 +2272,7 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); @@ -2374,6 +2384,8 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv6_f32, "rwkv_wkv6_f32", rwkv_wkv6_f32_len, rwkv_wkv6_f32_data, "main", 7, sizeof(vk_op_rwkv_wkv6_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv7_f32, "rwkv_wkv7_f32", rwkv_wkv7_f32_len, rwkv_wkv7_f32_data, "main", 8, sizeof(vk_op_rwkv_wkv7_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); + ggml_vk_create_pipeline(device, device->pipeline_opt_step_adamw_f32, "opt_step_adamw_f32", opt_step_adamw_f32_len, opt_step_adamw_f32_data, "main", 5, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); for (auto &c : compiles) { @@ -5473,6 +5485,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return ctx->device->pipeline_rms_norm_back_f32; } return nullptr; + case GGML_OP_L2_NORM: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_l2_norm_f32; + } + return nullptr; case GGML_OP_UNARY: switch (ggml_get_unary_op(dst)) { case GGML_UNARY_OP_SILU: @@ -5612,6 +5629,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return ctx->device->pipeline_rwkv_wkv6_f32; } return nullptr; + case GGML_OP_RWKV_WKV7: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rwkv_wkv7_f32; + } + return nullptr; case GGML_OP_OPT_STEP_ADAMW: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_opt_step_adamw_f32; @@ -5859,6 +5881,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co case GGML_OP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_SOFT_MAX: case GGML_OP_SOFT_MAX_BACK: case GGML_OP_SUM_ROWS: @@ -6108,23 +6131,17 @@ static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const }, dryrun); } -static void ggml_vk_op_f32_rwkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, const vk_op_rwkv_wkv6_push_constants&& pc, bool dryrun = false) { - const ggml_tensor * k = dst->src[0]; - const ggml_tensor * v = dst->src[1]; - const ggml_tensor * r = dst->src[2]; - const ggml_tensor * tf = dst->src[3]; - const ggml_tensor * td = dst->src[4]; - const ggml_tensor * state = dst->src[5]; +static void ggml_vk_op_f32_wkv(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, const vk_op_rwkv_wkv6_push_constants&& pc, int version, bool dryrun = false) { + GGML_ASSERT(version == 6 || version == 7); + int num_srcs = version == 6 ? 6 : 7; + + for (int i = 0; i < num_srcs; i++) { + GGML_ASSERT(!ggml_is_quantized(dst->src[i]->type)); + } - GGML_ASSERT(!ggml_is_quantized(k->type)); - GGML_ASSERT(!ggml_is_quantized(v->type)); - GGML_ASSERT(!ggml_is_quantized(r->type)); - GGML_ASSERT(!ggml_is_quantized(tf->type)); - GGML_ASSERT(!ggml_is_quantized(td->type)); - GGML_ASSERT(!ggml_is_quantized(state->type)); GGML_ASSERT(dst->buffer != nullptr); - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, k, v, r, dst, GGML_OP_RWKV_WKV6); + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); GGML_ASSERT(pipeline != nullptr); if (dryrun) { @@ -6133,89 +6150,73 @@ static void ggml_vk_op_f32_rwkv6(ggml_backend_vk_context * ctx, vk_context& subc } ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; - ggml_backend_vk_buffer_context * k_buf_ctx = (ggml_backend_vk_buffer_context *)k->buffer->context; - ggml_backend_vk_buffer_context * v_buf_ctx = (ggml_backend_vk_buffer_context *)v->buffer->context; - ggml_backend_vk_buffer_context * r_buf_ctx = (ggml_backend_vk_buffer_context *)r->buffer->context; - ggml_backend_vk_buffer_context * tf_buf_ctx = (ggml_backend_vk_buffer_context *)tf->buffer->context; - ggml_backend_vk_buffer_context * td_buf_ctx = (ggml_backend_vk_buffer_context *)td->buffer->context; - ggml_backend_vk_buffer_context * state_buf_ctx = (ggml_backend_vk_buffer_context *)state->buffer->context; + ggml_backend_vk_buffer_context * src_buf_ctxs[7] = { nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr }; + for (int i = 0; i < num_srcs; i++) { + src_buf_ctxs[i] = (ggml_backend_vk_buffer_context *)dst->src[i]->buffer->context; + } ggml_vk_sync_buffers(subctx); - vk_buffer d_D = nullptr, d_K = nullptr, d_V = nullptr, d_R = nullptr, d_TF = nullptr, d_TD = nullptr, d_State = nullptr; - size_t k_offset = 0, v_offset = 0, r_offset = 0, tf_offset = 0, td_offset = 0, state_offset = 0, dst_offset = 0; - bool K_uma = false, V_uma = false, R_uma = false, TF_uma = false, TD_uma = false, STATE_uma = false, DST_uma = false; + vk_buffer d_D = nullptr, d_srcs[7] = { nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr }; + size_t dst_offset = 0, src_offsets[7] = { 0, 0, 0, 0, 0, 0, 0 }; + bool dst_uma = false, srcs_uma[7] = { false, false, false, false, false, false, false }; if (ctx->device->uma) { - ggml_vk_host_get(ctx->device, k->data, d_K, k_offset); - ggml_vk_host_get(ctx->device, v->data, d_V, v_offset); - ggml_vk_host_get(ctx->device, r->data, d_R, r_offset); - ggml_vk_host_get(ctx->device, tf->data, d_TF, tf_offset); - ggml_vk_host_get(ctx->device, td->data, d_TD, td_offset); - ggml_vk_host_get(ctx->device, state->data, d_State, state_offset); + for (int i = 0; i < num_srcs; i++) { + ggml_vk_host_get(ctx->device, dst->src[i]->data, d_srcs[i], src_offsets[i]); + srcs_uma[i] = d_srcs[i] != nullptr; + } + ggml_vk_host_get(ctx->device, dst->data, d_D, dst_offset); - - K_uma = d_K != nullptr; - V_uma = d_V != nullptr; - R_uma = d_R != nullptr; - TF_uma = d_TF != nullptr; - TD_uma = d_TD != nullptr; - STATE_uma = d_State != nullptr; - DST_uma = d_D != nullptr; + dst_uma = d_D != nullptr; } - if (!K_uma) { - d_K = k_buf_ctx->dev_buffer; - k_offset = vk_tensor_offset(k) + k->view_offs; + uint64_t src_sizes[7] = { 0, 0, 0, 0, 0, 0, 0 }; + for (int i = 0; i < num_srcs; i++) { + src_sizes[i] = ggml_nbytes(dst->src[i]); + if (!srcs_uma[i]) { + d_srcs[i] = src_buf_ctxs[i]->dev_buffer; + src_offsets[i] = vk_tensor_offset(dst->src[i]) + dst->src[i]->view_offs; + } } - if (!V_uma) { - d_V = v_buf_ctx->dev_buffer; - v_offset = vk_tensor_offset(v) + v->view_offs; - } - if (!R_uma) { - d_R = r_buf_ctx->dev_buffer; - r_offset = vk_tensor_offset(r) + r->view_offs; - } - if (!TF_uma) { - d_TF = tf_buf_ctx->dev_buffer; - tf_offset = vk_tensor_offset(tf) + tf->view_offs; - } - if (!TD_uma) { - d_TD = td_buf_ctx->dev_buffer; - td_offset = vk_tensor_offset(td) + td->view_offs; - } - if (!STATE_uma) { - d_State = state_buf_ctx->dev_buffer; - state_offset = vk_tensor_offset(state) + state->view_offs; - } - if (!DST_uma) { + + const uint64_t dst_size = ggml_nbytes(dst); + if (!dst_uma) { d_D = dst_buf_ctx->dev_buffer; dst_offset = vk_tensor_offset(dst) + dst->view_offs; } - const uint64_t k_size = ggml_nbytes(k); - const uint64_t v_size = ggml_nbytes(v); - const uint64_t r_size = ggml_nbytes(r); - const uint64_t tf_size = ggml_nbytes(tf); - const uint64_t td_size = ggml_nbytes(td); - const uint64_t state_size = ggml_nbytes(state); - const uint64_t dst_size = ggml_nbytes(dst); - std::array elements = { (uint32_t)(pc.B * pc.H), 1, 1 }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { - vk_subbuffer{ d_K, k_offset, k_size }, - vk_subbuffer{ d_V, v_offset, v_size }, - vk_subbuffer{ d_R, r_offset, r_size }, - vk_subbuffer{ d_TF, tf_offset, tf_size }, - vk_subbuffer{ d_TD, td_offset, td_size }, - vk_subbuffer{ d_State, state_offset, state_size }, - vk_subbuffer{ d_D, dst_offset, dst_size } - }, sizeof(vk_op_rwkv_wkv6_push_constants), &pc, elements); + if (version == 6) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { + vk_subbuffer{ d_srcs[0], src_offsets[0], src_sizes[0] }, + vk_subbuffer{ d_srcs[1], src_offsets[1], src_sizes[1] }, + vk_subbuffer{ d_srcs[2], src_offsets[2], src_sizes[2] }, + vk_subbuffer{ d_srcs[3], src_offsets[3], src_sizes[3] }, + vk_subbuffer{ d_srcs[4], src_offsets[4], src_sizes[4] }, + vk_subbuffer{ d_srcs[5], src_offsets[5], src_sizes[5] }, + vk_subbuffer{ d_D, dst_offset, dst_size } + }, sizeof(vk_op_rwkv_wkv6_push_constants), &pc, elements); + } else if (version == 7) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { + vk_subbuffer{ d_srcs[0], src_offsets[0], src_sizes[0] }, + vk_subbuffer{ d_srcs[1], src_offsets[1], src_sizes[1] }, + vk_subbuffer{ d_srcs[2], src_offsets[2], src_sizes[2] }, + vk_subbuffer{ d_srcs[3], src_offsets[3], src_sizes[3] }, + vk_subbuffer{ d_srcs[4], src_offsets[4], src_sizes[4] }, + vk_subbuffer{ d_srcs[5], src_offsets[5], src_sizes[5] }, + vk_subbuffer{ d_srcs[6], src_offsets[6], src_sizes[6] }, + vk_subbuffer{ d_D, dst_offset, dst_size } + }, sizeof(vk_op_rwkv_wkv7_push_constants), &pc, elements); + } else { + // shouldn't happen + GGML_ASSERT(false); + } } static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, bool dryrun = false) { @@ -6224,7 +6225,7 @@ static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, const size_t n_heads = dst->src[0]->ne[1]; const size_t n_seqs = dst->src[5]->ne[1]; - ggml_vk_op_f32_rwkv6( + ggml_vk_op_f32_wkv( ctx, subctx, dst, { (uint32_t)n_seqs, @@ -6232,6 +6233,26 @@ static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, (uint32_t)n_embed, (uint32_t)n_heads, }, + 6, + dryrun + ); +} + +static void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, bool dryrun = false) { + const size_t seq_length = dst->src[0]->ne[2]; + const size_t n_embed = dst->ne[0]; + const size_t n_heads = dst->src[0]->ne[1]; + const size_t n_seqs = dst->src[6]->ne[1]; + + ggml_vk_op_f32_wkv( + ctx, subctx, dst, + { + (uint32_t)n_seqs, + (uint32_t)seq_length, + (uint32_t)n_embed, + (uint32_t)n_heads, + }, + 7, dryrun ); } @@ -6533,6 +6554,11 @@ static void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& sub ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }, dryrun); } +static void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst, bool dryrun = false) { + float * op_params = (float *)dst->op_params; + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_L2_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }, dryrun); +} + static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst, bool dryrun = false) { ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }, dryrun); } @@ -7528,6 +7554,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: case GGML_OP_SOFT_MAX_BACK: @@ -7544,6 +7571,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_POOL_2D: case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: case GGML_OP_LEAKY_RELU: case GGML_OP_FLASH_ATTN_EXT: case GGML_OP_OPT_STEP_ADAMW: @@ -7590,6 +7618,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_UNARY: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: @@ -7707,6 +7736,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_RMS_NORM_BACK: ggml_vk_rms_norm_back(ctx, compute_ctx, src0, src1, node, dryrun); + break; + case GGML_OP_L2_NORM: + ggml_vk_l2_norm(ctx, compute_ctx, src0, node, dryrun); + break; case GGML_OP_UNARY: switch (ggml_get_unary_op(node)) { @@ -7797,6 +7830,11 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod break; + case GGML_OP_RWKV_WKV7: + ggml_vk_rwkv_wkv7(ctx, compute_ctx, node, dryrun); + + break; + case GGML_OP_OPT_STEP_ADAMW: ggml_vk_opt_step_adamw(ctx, compute_ctx, node, dryrun); @@ -7870,6 +7908,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_tensor * case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: case GGML_OP_SOFT_MAX_BACK: @@ -7889,6 +7928,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_tensor * case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_POOL_2D: case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: case GGML_OP_LEAKY_RELU: case GGML_OP_REPEAT: case GGML_OP_REPEAT_BACK: @@ -8806,6 +8846,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_NORM: case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_ADD: case GGML_OP_SUB: @@ -8835,6 +8876,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_POOL_2D: case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: case GGML_OP_LEAKY_RELU: case GGML_OP_OPT_STEP_ADAMW: return true; @@ -9219,6 +9261,9 @@ static void ggml_vk_check_results_0(ggml_tensor * tensor) { tensor_clone = ggml_rms_norm_back(ggml_ctx, src_clone[0], src_clone[1], eps); } else if (tensor->op == GGML_OP_SILU_BACK) { tensor_clone = ggml_silu_back(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_L2_NORM) { + const float eps = ((float *) tensor->op_params)[0]; + tensor_clone = ggml_l2_norm(ggml_ctx, src_clone[0], eps); } else if (tensor->op == GGML_OP_SOFT_MAX) { if (src1 != nullptr) { tensor_clone = ggml_soft_max_ext(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); @@ -9338,6 +9383,9 @@ static void ggml_vk_check_results_0(ggml_tensor * tensor) { } else if (tensor->op == GGML_OP_RWKV_WKV6) { tensor_clone = ggml_rwkv_wkv6(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], src_clone[4], src_clone[5]); + } else if (tensor->op == GGML_OP_RWKV_WKV7) { + tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], + src_clone[4], src_clone[5], src_clone[6]); } else if (tensor->op == GGML_OP_OPT_STEP_ADAMW) { src_clone[0]->flags = src0->flags; tensor_clone = ggml_opt_step_adamw(ggml_ctx, src_clone[0], src_clone[1], diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp new file mode 100644 index 0000000000..deba8c3985 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp @@ -0,0 +1,41 @@ +#version 450 + +#include "generic_head.comp" +#include "types.comp" + +#extension GL_EXT_control_flow_attributes : enable +#define BLOCK_SIZE 512 + +layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; + +shared FLOAT_TYPE sum[BLOCK_SIZE]; + +void main() { + const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; + const uint tid = gl_LocalInvocationID.x; + + sum[tid] = FLOAT_TYPE(0.0f); // partial sum for thread in warp + + [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + const FLOAT_TYPE xi = FLOAT_TYPE(data_a[row*p.KX + col]); + sum[tid] += xi * xi; + } + + // sum up partial sums and write back result + barrier(); + [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) { + sum[tid] += sum[tid + s]; + } + barrier(); + } + + const FLOAT_TYPE scale = inversesqrt(max(sum[0], FLOAT_TYPE(p.param1))); + + [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + data_d[row*p.KX + col] = D_TYPE(scale * FLOAT_TYPE(data_a[row*p.KX + col])); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index ee1fec4e11..eb2ad63ff6 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -434,6 +434,7 @@ void process_shaders() { string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rms_norm_back_f32", "rms_norm_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); + string_to_spv("l2_norm_f32", "l2_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("cpy_f32_f32", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("cpy_f32_f16", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float16_t"}}); @@ -528,6 +529,8 @@ void process_shaders() { string_to_spv("rwkv_wkv6_f32", "wkv6.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); + string_to_spv("rwkv_wkv7_f32", "wkv7.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); + string_to_spv("opt_step_adamw_f32", "opt_step_adamw.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); for (auto &c : compiles) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp b/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp new file mode 100644 index 0000000000..88c1c02b32 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp @@ -0,0 +1,91 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : require + +#define BLOCK_SIZE 64 +layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; + +layout(push_constant) uniform Parameters { + uint B; + uint T; + uint C; + uint H; +}; + +layout(binding = 0) readonly buffer RBuf { A_TYPE r[]; }; +layout(binding = 1) readonly buffer WBuf { A_TYPE w[]; }; +layout(binding = 2) readonly buffer KBuf { A_TYPE k[]; }; +layout(binding = 3) readonly buffer VBuf { A_TYPE v[]; }; +layout(binding = 4) readonly buffer ABuf { A_TYPE a[]; }; +layout(binding = 5) readonly buffer BBuf { A_TYPE b[]; }; +layout(binding = 6) readonly buffer StateBuf { A_TYPE state_in[]; }; +layout(binding = 7) buffer DstBuf { A_TYPE dst[]; }; + +shared A_TYPE _r[BLOCK_SIZE], _w[BLOCK_SIZE], _k[BLOCK_SIZE], _a[BLOCK_SIZE], _b[BLOCK_SIZE]; + +void main() { + const uint head_size = BLOCK_SIZE; + const uint batch_id = gl_WorkGroupID.x / H; + const uint head_id = gl_WorkGroupID.x % H; + const uint tid = gl_LocalInvocationID.x; + + const uint state_size = C * head_size; + const uint n_seq_tokens = T / B; + + if (batch_id >= B || head_id >= H) { + return; + } + + A_TYPE state[BLOCK_SIZE]; + [[unroll]] for (uint i = 0; i < head_size; i++) { + state[i] = state_in[batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i]; + } + + const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; + const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; + + for (uint t = start_t; t < end_t; t += C) { + barrier(); + _r[tid] = r[t]; + _w[tid] = w[t]; + _k[tid] = k[t]; + _a[tid] = a[t]; + _b[tid] = b[t]; + barrier(); + + A_TYPE sa = 0.0; + [[unroll]] for (uint j = 0; j < head_size; j += 4) { + vec4 s_vec = vec4(state[j], state[j+1], state[j+2], state[j+3]); + vec4 a_vec = vec4(_a[j], _a[j+1], _a[j+2], _a[j+3]); + sa += dot(s_vec, a_vec); + } + + const A_TYPE v_val = v[t]; + A_TYPE y = 0.0; + + [[unroll]] for (uint j = 0; j < head_size; j += 4) { + vec4 r_vec = vec4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + vec4 w_vec = vec4(_w[j], _w[j+1], _w[j+2], _w[j+3]); + vec4 k_vec = vec4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + vec4 b_vec = vec4(_b[j], _b[j+1], _b[j+2], _b[j+3]); + vec4 s_vec = vec4(state[j], state[j+1], state[j+2], state[j+3]); + + vec4 kv = k_vec * v_val; + s_vec = s_vec * w_vec + kv + sa * b_vec; + y += dot(r_vec, s_vec); + + state[j] = s_vec.x; + state[j+1] = s_vec.y; + state[j+2] = s_vec.z; + state[j+3] = s_vec.w; + } + + dst[t] = y; + } + + [[unroll]] for (uint i = 0; i < head_size; i++) { + dst[T * C + batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i] = state[i]; + } +} diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 89409bb0e4..2e081d5910 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -929,6 +929,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "RMS_NORM", "RMS_NORM_BACK", "GROUP_NORM", + "L2_NORM", "MUL_MAT", "MUL_MAT_ID", @@ -977,6 +978,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "ADD_REL_POS", "RWKV_WKV6", "GATED_LINEAR_ATTN", + "RWKV_WKV7", "UNARY", @@ -996,7 +998,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "OPT_STEP_ADAMW", }; -static_assert(GGML_OP_COUNT == 83, "GGML_OP_COUNT != 83"); +static_assert(GGML_OP_COUNT == 85, "GGML_OP_COUNT != 85"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1026,6 +1028,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "rms_norm(x)", "rms_norm_back(x)", "group_norm(x)", + "l2_norm(x)", "X*Y", "X[i]*Y", @@ -1074,6 +1077,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "add_rel_pos(x)", "rwkv_wkv6(k, v, r, tf, td, s)", "gated_linear_attn(k, v, q, gate, s)", + "rwkv_wkv7(r, w, k, v, a, b, s)", "unary(x)", @@ -1093,7 +1097,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "adamw(x)", }; -static_assert(GGML_OP_COUNT == 83, "GGML_OP_COUNT != 83"); +static_assert(GGML_OP_COUNT == 85, "GGML_OP_COUNT != 85"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -2686,6 +2690,37 @@ struct ggml_tensor * ggml_group_norm_inplace( return ggml_group_norm_impl(ctx, a, n_groups, eps, true); } +// ggml_l2_norm + +static struct ggml_tensor * ggml_l2_norm_impl( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps, + bool inplace) { + struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); + + ggml_set_op_params_f32(result, 0, eps); + + result->op = GGML_OP_L2_NORM; + result->src[0] = a; + + return result; +} + +struct ggml_tensor * ggml_l2_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps) { + return ggml_l2_norm_impl(ctx, a, eps, false); +} + +struct ggml_tensor * ggml_l2_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps) { + return ggml_l2_norm_impl(ctx, a, eps, true); +} + // ggml_mul_mat static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { @@ -4720,6 +4755,54 @@ struct ggml_tensor * ggml_gated_linear_attn( return result; } +// ggml_rwkv_wkv7 + +struct ggml_tensor * ggml_rwkv_wkv7( + struct ggml_context * ctx, + struct ggml_tensor * r, + struct ggml_tensor * w, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * state) { + GGML_ASSERT(ggml_is_contiguous(r)); + GGML_ASSERT(ggml_is_contiguous(w)); + GGML_ASSERT(ggml_is_contiguous(k)); + GGML_ASSERT(ggml_is_contiguous(v)); + GGML_ASSERT(ggml_is_contiguous(a)); + GGML_ASSERT(ggml_is_contiguous(b)); + GGML_ASSERT(ggml_is_contiguous(state)); + + const int64_t S = k->ne[0]; + const int64_t H = k->ne[1]; + const int64_t n_tokens = k->ne[2]; + const int64_t n_seqs = state->ne[1]; + { + GGML_ASSERT(w->ne[0] == S && w->ne[1] == H && w->ne[2] == n_tokens); + GGML_ASSERT(k->ne[0] == S && k->ne[1] == H && k->ne[2] == n_tokens); + GGML_ASSERT(v->ne[0] == S && v->ne[1] == H && v->ne[2] == n_tokens); + GGML_ASSERT(a->ne[0] == S && a->ne[1] == H && a->ne[2] == n_tokens); + GGML_ASSERT(b->ne[0] == S && b->ne[1] == H && b->ne[2] == n_tokens); + GGML_ASSERT(ggml_nelements(state) == S * S * H * n_seqs); + } + + // concat output and new_state + const int64_t ne[4] = { S * H, n_tokens + S * n_seqs, 1, 1 }; + struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); + + result->op = GGML_OP_RWKV_WKV7; + result->src[0] = r; + result->src[1] = w; + result->src[2] = k; + result->src[3] = v; + result->src[4] = a; + result->src[5] = b; + result->src[6] = state; + + return result; +} + // ggml_unary static struct ggml_tensor * ggml_unary_impl( diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 19624eae04..cc48913d97 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -118,22 +118,26 @@ class Keys: TOKEN_SHIFT_COUNT = "{arch}.token_shift_count" class Attention: - HEAD_COUNT = "{arch}.attention.head_count" - HEAD_COUNT_KV = "{arch}.attention.head_count_kv" - MAX_ALIBI_BIAS = "{arch}.attention.max_alibi_bias" - CLAMP_KQV = "{arch}.attention.clamp_kqv" - KEY_LENGTH = "{arch}.attention.key_length" - VALUE_LENGTH = "{arch}.attention.value_length" - LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon" - LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon" - GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon" - GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups" - CAUSAL = "{arch}.attention.causal" - Q_LORA_RANK = "{arch}.attention.q_lora_rank" - KV_LORA_RANK = "{arch}.attention.kv_lora_rank" - REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count" - SLIDING_WINDOW = "{arch}.attention.sliding_window" - SCALE = "{arch}.attention.scale" + HEAD_COUNT = "{arch}.attention.head_count" + HEAD_COUNT_KV = "{arch}.attention.head_count_kv" + MAX_ALIBI_BIAS = "{arch}.attention.max_alibi_bias" + CLAMP_KQV = "{arch}.attention.clamp_kqv" + KEY_LENGTH = "{arch}.attention.key_length" + VALUE_LENGTH = "{arch}.attention.value_length" + LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon" + LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon" + GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon" + GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups" + CAUSAL = "{arch}.attention.causal" + Q_LORA_RANK = "{arch}.attention.q_lora_rank" + KV_LORA_RANK = "{arch}.attention.kv_lora_rank" + DECAY_LORA_RANK = "{arch}.attention.decay_lora_rank" + ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank" + VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank" + GATE_LORA_RANK = "{arch}.attention.gate_lora_rank" + REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count" + SLIDING_WINDOW = "{arch}.attention.sliding_window" + SCALE = "{arch}.attention.scale" class Rope: DIMENSION_COUNT = "{arch}.rope.dimension_count" @@ -257,6 +261,8 @@ class MODEL_ARCH(IntEnum): STARCODER2 = auto() RWKV6 = auto() RWKV6QWEN2 = auto() + RWKV7 = auto() + ARWKV7 = auto() MAMBA = auto() XVERSE = auto() COMMAND_R = auto() @@ -329,8 +335,20 @@ class MODEL_TENSOR(IntEnum): SSM_A = auto() SSM_D = auto() SSM_OUT = auto() + TIME_MIX_W0 = auto() TIME_MIX_W1 = auto() TIME_MIX_W2 = auto() + TIME_MIX_A0 = auto() + TIME_MIX_A1 = auto() + TIME_MIX_A2 = auto() + TIME_MIX_V0 = auto() + TIME_MIX_V1 = auto() + TIME_MIX_V2 = auto() + TIME_MIX_G1 = auto() + TIME_MIX_G2 = auto() + TIME_MIX_K_K = auto() + TIME_MIX_K_A = auto() + TIME_MIX_R_K = auto() TIME_MIX_LERP_X = auto() TIME_MIX_LERP_K = auto() TIME_MIX_LERP_V = auto() @@ -445,6 +463,8 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.STARCODER2: "starcoder2", MODEL_ARCH.RWKV6: "rwkv6", MODEL_ARCH.RWKV6QWEN2: "rwkv6qwen2", + MODEL_ARCH.RWKV7: "rwkv7", + MODEL_ARCH.ARWKV7: "arwkv7", MODEL_ARCH.MAMBA: "mamba", MODEL_ARCH.XVERSE: "xverse", MODEL_ARCH.COMMAND_R: "command-r", @@ -517,8 +537,20 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.SSM_A: "blk.{bid}.ssm_a", MODEL_TENSOR.SSM_D: "blk.{bid}.ssm_d", MODEL_TENSOR.SSM_OUT: "blk.{bid}.ssm_out", + MODEL_TENSOR.TIME_MIX_W0: "blk.{bid}.time_mix_w0", MODEL_TENSOR.TIME_MIX_W1: "blk.{bid}.time_mix_w1", MODEL_TENSOR.TIME_MIX_W2: "blk.{bid}.time_mix_w2", + MODEL_TENSOR.TIME_MIX_A0: "blk.{bid}.time_mix_a0", + MODEL_TENSOR.TIME_MIX_A1: "blk.{bid}.time_mix_a1", + MODEL_TENSOR.TIME_MIX_A2: "blk.{bid}.time_mix_a2", + MODEL_TENSOR.TIME_MIX_V0: "blk.{bid}.time_mix_v0", + MODEL_TENSOR.TIME_MIX_V1: "blk.{bid}.time_mix_v1", + MODEL_TENSOR.TIME_MIX_V2: "blk.{bid}.time_mix_v2", + MODEL_TENSOR.TIME_MIX_G1: "blk.{bid}.time_mix_g1", + MODEL_TENSOR.TIME_MIX_G2: "blk.{bid}.time_mix_g2", + MODEL_TENSOR.TIME_MIX_K_K: "blk.{bid}.time_mix_k_k", + MODEL_TENSOR.TIME_MIX_K_A: "blk.{bid}.time_mix_k_a", + MODEL_TENSOR.TIME_MIX_R_K: "blk.{bid}.time_mix_r_k", MODEL_TENSOR.TIME_MIX_LERP_X: "blk.{bid}.time_mix_lerp_x", MODEL_TENSOR.TIME_MIX_LERP_K: "blk.{bid}.time_mix_lerp_k", MODEL_TENSOR.TIME_MIX_LERP_V: "blk.{bid}.time_mix_lerp_v", @@ -1172,6 +1204,68 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.RWKV7: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_NORM_2, + MODEL_TENSOR.TIME_MIX_LERP_FUSED, + MODEL_TENSOR.TIME_MIX_W0, + MODEL_TENSOR.TIME_MIX_W1, + MODEL_TENSOR.TIME_MIX_W2, + MODEL_TENSOR.TIME_MIX_A0, + MODEL_TENSOR.TIME_MIX_A1, + MODEL_TENSOR.TIME_MIX_A2, + MODEL_TENSOR.TIME_MIX_V0, + MODEL_TENSOR.TIME_MIX_V1, + MODEL_TENSOR.TIME_MIX_V2, + MODEL_TENSOR.TIME_MIX_G1, + MODEL_TENSOR.TIME_MIX_G2, + MODEL_TENSOR.TIME_MIX_K_K, + MODEL_TENSOR.TIME_MIX_K_A, + MODEL_TENSOR.TIME_MIX_R_K, + MODEL_TENSOR.TIME_MIX_KEY, + MODEL_TENSOR.TIME_MIX_VALUE, + MODEL_TENSOR.TIME_MIX_RECEPTANCE, + MODEL_TENSOR.TIME_MIX_LN, + MODEL_TENSOR.TIME_MIX_OUTPUT, + MODEL_TENSOR.CHANNEL_MIX_LERP_K, + MODEL_TENSOR.CHANNEL_MIX_KEY, + MODEL_TENSOR.CHANNEL_MIX_VALUE, + ], + MODEL_ARCH.ARWKV7: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.TIME_MIX_LERP_FUSED, + MODEL_TENSOR.TIME_MIX_W0, + MODEL_TENSOR.TIME_MIX_W1, + MODEL_TENSOR.TIME_MIX_W2, + MODEL_TENSOR.TIME_MIX_A0, + MODEL_TENSOR.TIME_MIX_A1, + MODEL_TENSOR.TIME_MIX_A2, + MODEL_TENSOR.TIME_MIX_V0, + MODEL_TENSOR.TIME_MIX_V1, + MODEL_TENSOR.TIME_MIX_V2, + MODEL_TENSOR.TIME_MIX_G1, + MODEL_TENSOR.TIME_MIX_G2, + MODEL_TENSOR.TIME_MIX_K_K, + MODEL_TENSOR.TIME_MIX_K_A, + MODEL_TENSOR.TIME_MIX_R_K, + MODEL_TENSOR.TIME_MIX_KEY, + MODEL_TENSOR.TIME_MIX_VALUE, + MODEL_TENSOR.TIME_MIX_RECEPTANCE, + MODEL_TENSOR.TIME_MIX_LN, + MODEL_TENSOR.TIME_MIX_OUTPUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], MODEL_ARCH.MAMBA: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 080d2b9dce..af8b388dfa 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -767,6 +767,18 @@ class GGUFWriter: def add_kv_lora_rank(self, length: int) -> None: self.add_uint32(Keys.Attention.KV_LORA_RANK.format(arch=self.arch), length) + def add_decay_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.DECAY_LORA_RANK.format(arch=self.arch), length) + + def add_iclr_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.ICLR_LORA_RANK.format(arch=self.arch), length) + + def add_value_residual_mix_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.VALUE_RESIDUAL_MIX_LORA_RANK.format(arch=self.arch), length) + + def add_gate_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.GATE_LORA_RANK.format(arch=self.arch), length) + def add_relative_attn_buckets_count(self, value: int) -> None: self.add_uint32(Keys.Attention.REL_BUCKETS_COUNT.format(arch=self.arch), value) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 617791e240..8d4a2b0320 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -27,7 +27,8 @@ class TensorNameMap: "embedding.word_embeddings", # chatglm "transformer.token_embeddings", # openelm "shared", # t5 - "rwkv.embeddings", # rwkv + "rwkv.embeddings", # rwkv6 + "model.embeddings", # rwkv7 ), # Token type embeddings @@ -42,6 +43,9 @@ class TensorNameMap: "emb_ln", # nomic-bert "transformer.norm", # openelm "rwkv.blocks.0.pre_ln", # rwkv + "rwkv.blocks.0.pre_ln", # rwkv6 + "model.pre_ln", # rwkv7 + "model.layers.0.pre_norm", # rwkv7 "backbone.norm", # wavtokenizer ), @@ -81,7 +85,8 @@ class TensorNameMap: "encoder.final_layernorm", # chatglm "transformer.norm", # openelm "model.norm", # nemotron - "rwkv.ln_out", # rwkv + "rwkv.ln_out", # rwkv6 + "model.ln_out", # rwkv7 "backbone.final_layer_norm", # wavtokenizer ), @@ -122,14 +127,16 @@ class TensorNameMap: "transformer.blocks.{bid}.norm_attn_norm.norm_1", # dbrx "encoder.layers.{bid}.input_layernorm", # chatglm "transformer.layers.{bid}.attn_norm", # openelm - "rwkv.blocks.{bid}.ln1", # rwkv + "rwkv.blocks.{bid}.ln1", # rwkv6 + "model.layers.{bid}.ln1", # rwkv7 ), # Attention norm 2 MODEL_TENSOR.ATTN_NORM_2: ( "transformer.h.{bid}.ln_attn", # falcon40b "encoder.layer.{bid}.layer_norm_1", # jina-v2-code - "rwkv.blocks.{bid}.ln2", # rwkv + "rwkv.blocks.{bid}.ln2", # rwkv6 + "model.layers.{bid}.ln2", # rwkv7 ), # Attention query-key-value @@ -462,112 +469,174 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.out_proj", ), + MODEL_TENSOR.TIME_MIX_W0: ( + "model.layers.{bid}.attention.w0", # rwkv7 + ), + MODEL_TENSOR.TIME_MIX_W1: ( - "rwkv.blocks.{bid}.attention.time_maa_w1", # rwkv v6 - "model.layers.{bid}.self_attn.time_maa_w1", # rwkv6qwen2 + "rwkv.blocks.{bid}.attention.time_maa_w1", # rwkv6 + "model.layers.{bid}.self_attn.time_maa_w1", # rwkv6qwen2 + "model.layers.{bid}.attention.w1", # rwkv7 ), MODEL_TENSOR.TIME_MIX_W2: ( - "rwkv.blocks.{bid}.attention.time_maa_w2", # rwkv v6 - "model.layers.{bid}.self_attn.time_maa_w2", # rwkv6qwen2 + "rwkv.blocks.{bid}.attention.time_maa_w2", # rwkv6 + "model.layers.{bid}.self_attn.time_maa_w2", # rwkv6qwen2 + "model.layers.{bid}.attention.w2", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_A0: ( + "model.layers.{bid}.attention.a0", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_A1: ( + "model.layers.{bid}.attention.a1", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_A2: ( + "model.layers.{bid}.attention.a2", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_V0: ( + "model.layers.{bid}.attention.v0", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_V1: ( + "model.layers.{bid}.attention.v1", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_V2: ( + "model.layers.{bid}.attention.v2", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_G1: ( + "model.layers.{bid}.attention.g1", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_G2: ( + "model.layers.{bid}.attention.g2", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_K_K: ( + "model.layers.{bid}.attention.k_k", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_K_A: ( + "model.layers.{bid}.attention.k_a", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_R_K: ( + "model.layers.{bid}.attention.r_k", # rwkv7 ), MODEL_TENSOR.TIME_MIX_LERP_X: ( - "rwkv.blocks.{bid}.attention.time_maa_x", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_x", # rwkv6 "model.layers.{bid}.self_attn.time_maa_x", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_K: ( - "rwkv.blocks.{bid}.attention.time_maa_k", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_k", # rwkv6 "model.layers.{bid}.self_attn.time_maa_k", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_V: ( - "rwkv.blocks.{bid}.attention.time_maa_v", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_v", # rwkv6 "model.layers.{bid}.self_attn.time_maa_v", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_R: ( - "rwkv.blocks.{bid}.attention.time_maa_r", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_r", # rwkv6 "model.layers.{bid}.self_attn.time_maa_r", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_G: ( - "rwkv.blocks.{bid}.attention.time_maa_g", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_g", # rwkv6 "model.layers.{bid}.self_attn.time_maa_g", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_W: ( - "rwkv.blocks.{bid}.attention.time_maa_w", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_w", # rwkv6 "model.layers.{bid}.self_attn.time_maa_w", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_FIRST: ( - "rwkv.blocks.{bid}.attention.time_faaaa", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_faaaa", # rwkv6 ), MODEL_TENSOR.TIME_MIX_DECAY: ( - "rwkv.blocks.{bid}.attention.time_decay", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_decay", # rwkv6 "model.layers.{bid}.self_attn.time_decay", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_DECAY_W1: ( - "rwkv.blocks.{bid}.attention.time_decay_w1", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_decay_w1", # rwkv6 "model.layers.{bid}.self_attn.time_decay_w1", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_DECAY_W2: ( - "rwkv.blocks.{bid}.attention.time_decay_w2", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_decay_w2", # rwkv6 "model.layers.{bid}.self_attn.time_decay_w2", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_KEY: ( - "rwkv.blocks.{bid}.attention.key", # rwkv + "rwkv.blocks.{bid}.attention.key", # rwkv6 "model.layers.{bid}.self_attn.k_proj", # rwkv6qwen2 + "model.layers.{bid}.attention.key", # rwkv7 + "model.layers.{bid}.attention.k_proj", # rwkv7 ), MODEL_TENSOR.TIME_MIX_VALUE: ( - "rwkv.blocks.{bid}.attention.value", # rwkv + "rwkv.blocks.{bid}.attention.value", # rwkv6 "model.layers.{bid}.self_attn.v_proj", # rwkv6qwen2 + "model.layers.{bid}.attention.value", # rwkv7 + "model.layers.{bid}.attention.v_proj", # rwkv7 ), MODEL_TENSOR.TIME_MIX_RECEPTANCE: ( - "rwkv.blocks.{bid}.attention.receptance", # rwkv - "model.layers.{bid}.self_attn.q_proj", # rwkv6qwen2 + "rwkv.blocks.{bid}.attention.receptance", # rwkv6 + "model.layers.{bid}.self_attn.q_proj", # rwkv6qwen2 + "model.layers.{bid}.attention.receptance", # rwkv7 + "model.layers.{bid}.attention.r_proj", # rwkv7 ), MODEL_TENSOR.TIME_MIX_GATE: ( - "rwkv.blocks.{bid}.attention.gate", # rwkv - "model.layers.{bid}.self_attn.gate", # rwkv6qwen2 + "rwkv.blocks.{bid}.attention.gate", # rwkv6 + "model.layers.{bid}.self_attn.gate", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LN: ( - "rwkv.blocks.{bid}.attention.ln_x", # rwkv + "rwkv.blocks.{bid}.attention.ln_x", # rwkv6 + "model.layers.{bid}.attention.ln_x" # rwkv7 ), MODEL_TENSOR.TIME_MIX_OUTPUT: ( - "rwkv.blocks.{bid}.attention.output", # rwkv + "rwkv.blocks.{bid}.attention.output", # rwkv6 "model.layers.{bid}.self_attn.o_proj", # rwkv6qwen2 + "model.layers.{bid}.attention.output", # rwkv7 + "model.layers.{bid}.attention.o_proj", # rwkv7 ), MODEL_TENSOR.CHANNEL_MIX_LERP_K: ( - "rwkv.blocks.{bid}.feed_forward.time_maa_k", # rwkv v6 + "rwkv.blocks.{bid}.feed_forward.time_maa_k", # rwkv6 + "model.layers.{bid}.feed_forward.x_k", # rwkv7 ), MODEL_TENSOR.CHANNEL_MIX_LERP_R: ( - "rwkv.blocks.{bid}.feed_forward.time_maa_r", # rwkv v6 + "rwkv.blocks.{bid}.feed_forward.time_maa_r", # rwkv6 ), MODEL_TENSOR.CHANNEL_MIX_KEY: ( - "rwkv.blocks.{bid}.feed_forward.key", # rwkv + "rwkv.blocks.{bid}.feed_forward.key", # rwkv6 + "model.layers.{bid}.feed_forward.key", # rwkv7 ), MODEL_TENSOR.CHANNEL_MIX_RECEPTANCE: ( - "rwkv.blocks.{bid}.feed_forward.receptance", # rwkv + "rwkv.blocks.{bid}.feed_forward.receptance", # rwkv6 ), MODEL_TENSOR.CHANNEL_MIX_VALUE: ( - "rwkv.blocks.{bid}.feed_forward.value", # rwkv + "rwkv.blocks.{bid}.feed_forward.value", # rwkv6 + "model.layers.{bid}.feed_forward.value", # rwkv7 ), MODEL_TENSOR.ATTN_Q_A: ( diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 28f2bbc8f7..9debb56cc8 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -59,6 +59,8 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_EXAONE, "exaone" }, { LLM_ARCH_RWKV6, "rwkv6" }, { LLM_ARCH_RWKV6QWEN2, "rwkv6qwen2" }, + { LLM_ARCH_RWKV7, "rwkv7" }, + { LLM_ARCH_ARWKV7, "arwkv7" }, { LLM_ARCH_GRANITE, "granite" }, { LLM_ARCH_GRANITE_MOE, "granitemoe" }, { LLM_ARCH_CHAMELEON, "chameleon" }, @@ -110,22 +112,26 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_EMBEDDING_SCALE, "%s.embedding_scale" }, { LLM_KV_TOKEN_SHIFT_COUNT, "%s.token_shift_count" }, - { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, - { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, - { LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" }, - { LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" }, - { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" }, - { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" }, - { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" }, - { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" }, - { LLM_KV_ATTENTION_GROUPNORM_EPS, "%s.attention.group_norm_epsilon" }, - { LLM_KV_ATTENTION_GROUPNORM_GROUPS, "%s.attention.group_norm_groups" }, - { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" }, - { LLM_KV_ATTENTION_Q_LORA_RANK, "%s.attention.q_lora_rank" }, - { LLM_KV_ATTENTION_KV_LORA_RANK, "%s.attention.kv_lora_rank" }, - { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" }, - { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" }, - { LLM_KV_ATTENTION_SCALE, "%s.attention.scale" }, + { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, + { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, + { LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" }, + { LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" }, + { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" }, + { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" }, + { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" }, + { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" }, + { LLM_KV_ATTENTION_GROUPNORM_EPS, "%s.attention.group_norm_epsilon" }, + { LLM_KV_ATTENTION_GROUPNORM_GROUPS, "%s.attention.group_norm_groups" }, + { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" }, + { LLM_KV_ATTENTION_Q_LORA_RANK, "%s.attention.q_lora_rank" }, + { LLM_KV_ATTENTION_KV_LORA_RANK, "%s.attention.kv_lora_rank" }, + { LLM_KV_ATTENTION_DECAY_LORA_RANK, "%s.attention.decay_lora_rank" }, + { LLM_KV_ATTENTION_ICLR_LORA_RANK, "%s.attention.iclr_lora_rank" }, + { LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, "%s.attention.value_residual_mix_lora_rank" }, + { LLM_KV_ATTENTION_GATE_LORA_RANK, "%s.attention.gate_lora_rank" }, + { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" }, + { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" }, + { LLM_KV_ATTENTION_SCALE, "%s.attention.scale" }, { LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" }, { LLM_KV_ROPE_DIMENSION_SECTIONS, "%s.rope.dimension_sections" }, @@ -1238,6 +1244,74 @@ static const std::map> LLM_TENSOR_N { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_RWKV7, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" }, + { LLM_TENSOR_TIME_MIX_W0, "blk.%d.time_mix_w0" }, + { LLM_TENSOR_TIME_MIX_W1, "blk.%d.time_mix_w1" }, + { LLM_TENSOR_TIME_MIX_W2, "blk.%d.time_mix_w2" }, + { LLM_TENSOR_TIME_MIX_A0, "blk.%d.time_mix_a0" }, + { LLM_TENSOR_TIME_MIX_A1, "blk.%d.time_mix_a1" }, + { LLM_TENSOR_TIME_MIX_A2, "blk.%d.time_mix_a2" }, + { LLM_TENSOR_TIME_MIX_V0, "blk.%d.time_mix_v0" }, + { LLM_TENSOR_TIME_MIX_V1, "blk.%d.time_mix_v1" }, + { LLM_TENSOR_TIME_MIX_V2, "blk.%d.time_mix_v2" }, + { LLM_TENSOR_TIME_MIX_G1, "blk.%d.time_mix_g1" }, + { LLM_TENSOR_TIME_MIX_G2, "blk.%d.time_mix_g2" }, + { LLM_TENSOR_TIME_MIX_K_K, "blk.%d.time_mix_k_k" }, + { LLM_TENSOR_TIME_MIX_K_A, "blk.%d.time_mix_k_a" }, + { LLM_TENSOR_TIME_MIX_R_K, "blk.%d.time_mix_r_k" }, + { LLM_TENSOR_TIME_MIX_LERP_FUSED, "blk.%d.time_mix_lerp_fused" }, + { LLM_TENSOR_TIME_MIX_KEY, "blk.%d.time_mix_key" }, + { LLM_TENSOR_TIME_MIX_VALUE, "blk.%d.time_mix_value" }, + { LLM_TENSOR_TIME_MIX_RECEPTANCE, "blk.%d.time_mix_receptance" }, + { LLM_TENSOR_TIME_MIX_LN, "blk.%d.time_mix_ln" }, + { LLM_TENSOR_TIME_MIX_OUTPUT, "blk.%d.time_mix_output" }, + { LLM_TENSOR_CHANNEL_MIX_LERP_K, "blk.%d.channel_mix_lerp_k" }, + { LLM_TENSOR_CHANNEL_MIX_KEY, "blk.%d.channel_mix_key" }, + { LLM_TENSOR_CHANNEL_MIX_VALUE, "blk.%d.channel_mix_value" }, + }, + }, + { + LLM_ARCH_ARWKV7, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_TIME_MIX_W0, "blk.%d.time_mix_w0" }, + { LLM_TENSOR_TIME_MIX_W1, "blk.%d.time_mix_w1" }, + { LLM_TENSOR_TIME_MIX_W2, "blk.%d.time_mix_w2" }, + { LLM_TENSOR_TIME_MIX_A0, "blk.%d.time_mix_a0" }, + { LLM_TENSOR_TIME_MIX_A1, "blk.%d.time_mix_a1" }, + { LLM_TENSOR_TIME_MIX_A2, "blk.%d.time_mix_a2" }, + { LLM_TENSOR_TIME_MIX_V0, "blk.%d.time_mix_v0" }, + { LLM_TENSOR_TIME_MIX_V1, "blk.%d.time_mix_v1" }, + { LLM_TENSOR_TIME_MIX_V2, "blk.%d.time_mix_v2" }, + { LLM_TENSOR_TIME_MIX_G1, "blk.%d.time_mix_g1" }, + { LLM_TENSOR_TIME_MIX_G2, "blk.%d.time_mix_g2" }, + { LLM_TENSOR_TIME_MIX_K_K, "blk.%d.time_mix_k_k" }, + { LLM_TENSOR_TIME_MIX_K_A, "blk.%d.time_mix_k_a" }, + { LLM_TENSOR_TIME_MIX_R_K, "blk.%d.time_mix_r_k" }, + { LLM_TENSOR_TIME_MIX_LERP_FUSED, "blk.%d.time_mix_lerp_fused" }, + { LLM_TENSOR_TIME_MIX_KEY, "blk.%d.time_mix_key" }, + { LLM_TENSOR_TIME_MIX_VALUE, "blk.%d.time_mix_value" }, + { LLM_TENSOR_TIME_MIX_RECEPTANCE, "blk.%d.time_mix_receptance" }, + { LLM_TENSOR_TIME_MIX_LN, "blk.%d.time_mix_ln" }, + { LLM_TENSOR_TIME_MIX_OUTPUT, "blk.%d.time_mix_output" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + }, + }, { LLM_ARCH_GRANITE, { @@ -1397,6 +1471,12 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_SSM_OUT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_W1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_W2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_A1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_A2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_V1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_V2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_G1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_G2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_DECAY_W1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_DECAY_W2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_KEY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, @@ -1415,6 +1495,9 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_TIME_MIX_LN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_CHANNEL_MIX_LERP_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_CHANNEL_MIX_LERP_R, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_TIME_MIX_K_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_TIME_MIX_K_A, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_TIME_MIX_R_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_TIME_MIX_LERP_W, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_LERP_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_LERP_V, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, @@ -1422,6 +1505,9 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_TIME_MIX_LERP_G, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_LERP_FUSED, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_DECAY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_TIME_MIX_W0, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_TIME_MIX_A0, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_TIME_MIX_V0, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_FIRST, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_RWKV_WKV6}}, {LLM_TENSOR_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_ATTN_NORM_2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, diff --git a/src/llama-arch.h b/src/llama-arch.h index 2ec2e2362e..a28815d8a1 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -63,6 +63,8 @@ enum llm_arch { LLM_ARCH_EXAONE, LLM_ARCH_RWKV6, LLM_ARCH_RWKV6QWEN2, + LLM_ARCH_RWKV7, + LLM_ARCH_ARWKV7, LLM_ARCH_GRANITE, LLM_ARCH_GRANITE_MOE, LLM_ARCH_CHAMELEON, @@ -127,6 +129,10 @@ enum llm_kv { LLM_KV_ATTENTION_CAUSAL, LLM_KV_ATTENTION_Q_LORA_RANK, LLM_KV_ATTENTION_KV_LORA_RANK, + LLM_KV_ATTENTION_DECAY_LORA_RANK, + LLM_KV_ATTENTION_ICLR_LORA_RANK, + LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, + LLM_KV_ATTENTION_GATE_LORA_RANK, LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, LLM_KV_ATTENTION_SLIDING_WINDOW, LLM_KV_ATTENTION_SCALE, @@ -250,8 +256,20 @@ enum llm_tensor { LLM_TENSOR_SSM_A, LLM_TENSOR_SSM_D, LLM_TENSOR_SSM_OUT, + LLM_TENSOR_TIME_MIX_W0, LLM_TENSOR_TIME_MIX_W1, LLM_TENSOR_TIME_MIX_W2, + LLM_TENSOR_TIME_MIX_A0, + LLM_TENSOR_TIME_MIX_A1, + LLM_TENSOR_TIME_MIX_A2, + LLM_TENSOR_TIME_MIX_V0, + LLM_TENSOR_TIME_MIX_V1, + LLM_TENSOR_TIME_MIX_V2, + LLM_TENSOR_TIME_MIX_G1, + LLM_TENSOR_TIME_MIX_G2, + LLM_TENSOR_TIME_MIX_K_K, + LLM_TENSOR_TIME_MIX_K_A, + LLM_TENSOR_TIME_MIX_R_K, LLM_TENSOR_TIME_MIX_LERP_X, LLM_TENSOR_TIME_MIX_LERP_W, LLM_TENSOR_TIME_MIX_LERP_K, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index dbb7abd317..bb17ba86dc 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -76,6 +76,10 @@ struct llama_hparams { uint32_t time_decay_extra_dim = 0; uint32_t wkv_head_size = 0; uint32_t token_shift_count = 2; + uint32_t n_lora_decay = 0; + uint32_t n_lora_iclr = 0; + uint32_t n_lora_value_res_mix = 0; + uint32_t n_lora_gate = 0; float rope_attn_factor = 1.0f; float rope_freq_base_train; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 4b288d8f66..c571aa69b6 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -32,6 +32,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_109M: return "109M"; case LLM_TYPE_137M: return "137M"; case LLM_TYPE_160M: return "160M"; + case LLM_TYPE_190M: return "190M"; case LLM_TYPE_220M: return "220M"; case LLM_TYPE_250M: return "250M"; case LLM_TYPE_270M: return "270M"; @@ -48,6 +49,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_1_6B: return "1.6B"; case LLM_TYPE_2B: return "2B"; case LLM_TYPE_2_8B: return "2.8B"; + case LLM_TYPE_2_9B: return "2.9B"; case LLM_TYPE_3B: return "3B"; case LLM_TYPE_4B: return "4B"; case LLM_TYPE_6B: return "6B"; @@ -1250,6 +1252,36 @@ void llama_model::load_hparams(llama_model_loader & ml) { default: type = LLM_TYPE_UNKNOWN; } } break; + case LLM_ARCH_RWKV7: + case LLM_ARCH_ARWKV7: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps, false); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps, false); + ml.get_key(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size); + ml.get_key(LLM_KV_ATTENTION_DECAY_LORA_RANK, hparams.n_lora_decay); + ml.get_key(LLM_KV_ATTENTION_ICLR_LORA_RANK, hparams.n_lora_iclr); + ml.get_key(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix); + ml.get_key(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate, false); + ml.get_key(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count, false); + + switch (hparams.n_layer) { + case 12: type = LLM_TYPE_190M; break; + case 24: + switch (hparams.n_embd) { + case 1024: type = LLM_TYPE_450M; break; + case 2048: type = LLM_TYPE_1_5B; break; + default: type = LLM_TYPE_UNKNOWN; + } break; + case 28: + switch (hparams.n_embd) { + case 1536: type = LLM_TYPE_1_5B; break; + case 3584: type = LLM_TYPE_7B; break; + default: type = LLM_TYPE_UNKNOWN; + } break; + case 32: type = LLM_TYPE_2_9B; break; // RWKV-7-World + default: type = LLM_TYPE_UNKNOWN; + } + } break; case LLM_ARCH_GRANITE: case LLM_ARCH_GRANITE_MOE: { @@ -3366,6 +3398,146 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); } } break; + case LLM_ARCH_RWKV7: + { + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + // Block 0, LN0 + tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, 0); + tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + const int n_lora_decay = hparams.n_lora_decay; + const int n_lora_iclr = hparams.n_lora_iclr; + const int n_lora_value_res_mix = hparams.n_lora_value_res_mix; + const int n_lora_gate = hparams.n_lora_gate; + const int attn_hidden_size = n_embd; + const int ffn_size = hparams.n_ff_arr[0]; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0); + + layer.attn_norm_2 = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, 0); + layer.attn_norm_2_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, 0); + + layer.time_mix_w0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W0, "weight", i), {n_embd}, 0); + layer.time_mix_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W1, "weight", i), {n_embd, n_lora_decay}, 0); + layer.time_mix_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W2, "weight", i), {n_lora_decay, n_embd}, 0); + + layer.time_mix_a0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A0, "weight", i), {n_embd}, 0); + layer.time_mix_a1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A1, "weight", i), {n_embd, n_lora_iclr}, 0); + layer.time_mix_a2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A2, "weight", i), {n_lora_iclr, n_embd}, 0); + + if (i == 0) { + // actually not used + layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0); + layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_iclr}, 0); + layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_iclr, n_embd}, 0); + } else { + layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0); + layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_value_res_mix}, 0); + layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_value_res_mix, n_embd}, 0); + } + + layer.time_mix_g1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G1, "weight", i), {n_embd, n_lora_gate}, 0); + layer.time_mix_g2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G2, "weight", i), {n_lora_gate, n_embd}, 0); + + layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, "weight", i), {n_embd, 1, 1, 6}, 0); + + layer.time_mix_k_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_K, "weight", i), {attn_hidden_size}, 0); + layer.time_mix_k_a = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_A, "weight", i), {attn_hidden_size}, 0); + layer.time_mix_r_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_R_K, "weight", i), {attn_hidden_size}, 0); + + layer.time_mix_key = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, "weight", i), {attn_hidden_size, n_embd}, 0); + layer.time_mix_value = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, "weight", i), {attn_hidden_size, n_embd}, 0); + layer.time_mix_receptance = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, "weight", i), {attn_hidden_size, n_embd}, 0); + + layer.time_mix_ln = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "weight", i), {n_embd}, 0); + layer.time_mix_ln_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "bias", i), {n_embd}, 0); + layer.time_mix_output = create_tensor(tn(LLM_TENSOR_TIME_MIX_OUTPUT, "weight", i), {n_embd, attn_hidden_size}, 0); + + layer.channel_mix_lerp_k = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_LERP_K, "weight", i), {n_embd, 1, 1}, 0); + + layer.channel_mix_key = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_KEY, "weight", i), {n_embd, ffn_size}, 0); + layer.channel_mix_value = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_VALUE, "weight", i), {ffn_size, n_embd}, 0); + } + + } break; + case LLM_ARCH_ARWKV7: + { + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + const int n_lora_decay = hparams.n_lora_decay; + const int n_lora_iclr = hparams.n_lora_iclr; + const int n_lora_value_res_mix = hparams.n_lora_value_res_mix; + const int n_lora_gate = hparams.n_lora_gate; + const int attn_hidden_size = n_embd; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + layer.time_mix_w0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W0, "weight", i), {n_embd}, 0); + layer.time_mix_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W1, "weight", i), {n_embd, n_lora_decay}, 0); + layer.time_mix_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W2, "weight", i), {n_lora_decay, n_embd}, 0); + + layer.time_mix_a0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A0, "weight", i), {n_embd}, 0); + layer.time_mix_a1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A1, "weight", i), {n_embd, n_lora_iclr}, 0); + layer.time_mix_a2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A2, "weight", i), {n_lora_iclr, n_embd}, 0); + + if (i == 0) { + // actually not used + layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0); + layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_iclr}, 0); + layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_iclr, n_embd}, 0); + } else { + layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0); + layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_value_res_mix}, 0); + layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_value_res_mix, n_embd}, 0); + } + + layer.time_mix_g1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G1, "weight", i), {n_embd, n_lora_gate}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.time_mix_g2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G2, "weight", i), {n_lora_gate, n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + try { + layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, "weight", i), {n_embd, 1, 1, 6}, 0); + } catch(std::runtime_error & e) { + // ARWKV models may not have gate tensors + layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, "weight", i), {n_embd, 1, 1, 5}, 0); + } + + layer.time_mix_k_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_K, "weight", i), {attn_hidden_size}, 0); + layer.time_mix_k_a = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_A, "weight", i), {attn_hidden_size}, 0); + layer.time_mix_r_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_R_K, "weight", i), {attn_hidden_size}, 0); + + layer.time_mix_key = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, "weight", i), {attn_hidden_size, n_embd}, 0); + layer.time_mix_value = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, "weight", i), {attn_hidden_size, n_embd}, 0); + layer.time_mix_receptance = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, "weight", i), {attn_hidden_size, n_embd}, 0); + + layer.time_mix_ln = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.time_mix_ln_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.time_mix_output = create_tensor(tn(LLM_TENSOR_TIME_MIX_OUTPUT, "weight", i), {n_embd, attn_hidden_size}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } + + } break; case LLM_ARCH_CHAMELEON: { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); @@ -10212,6 +10384,7 @@ struct llm_build_rwkv6_base : public llm_graph_context { const auto n_tokens = ubatch.n_tokens; const auto n_seqs = ubatch.n_seqs; + const auto n_seq_tokens = ubatch.n_seq_tokens; const auto n_embd = hparams.n_embd; const auto head_size = hparams.wkv_head_size; const auto n_head = n_embd / head_size; @@ -10224,6 +10397,10 @@ struct llm_build_rwkv6_base : public llm_graph_context { bool is_qrwkv = layer.time_mix_first == nullptr; ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur); + + sx = ggml_reshape_2d(ctx0, sx, n_embd, n_tokens); + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + ggml_tensor * xxx = ggml_add(ctx0, ggml_mul(ctx0, sx, layer.time_mix_lerp_x), cur); xxx = ggml_reshape_4d( @@ -10366,7 +10543,7 @@ struct llm_build_rwkv6_base : public llm_graph_context { cur = ggml_mul(ctx0, cur, g); cur = build_lora_mm(layer.time_mix_output, cur); - return cur; + return ggml_reshape_3d(ctx0, cur, n_embd, n_seq_tokens, n_seqs); } }; @@ -10389,6 +10566,7 @@ struct llm_build_rwkv6 : public llm_build_rwkv6_base { for (int il = 0; il < n_layer; ++il) { const llama_layer * layer = &model.layers[il]; + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); ggml_tensor * token_shift = build_rwkv_token_shift_load( gf, state_copy, state_mask, ubatch, il @@ -10422,9 +10600,6 @@ struct llm_build_rwkv6 : public llm_build_rwkv6_base { 1 ); - cur = build_rwkv6_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV6); - cur = ggml_add(ctx0, cur, ffn_inp); - token_shift = ggml_concat(ctx0, ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm)), ggml_view_3d(ctx0, ffn_norm, n_embd, 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(ffn_norm)), @@ -10432,6 +10607,18 @@ struct llm_build_rwkv6 : public llm_build_rwkv6_base { ); ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il)); + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + ffn_inp = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens), inp_out_ids); + ffn_norm = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_norm, n_embd, n_tokens), inp_out_ids); + x_prev = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, x_prev, n_embd, n_tokens), inp_out_ids); + cur = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, cur, n_embd, n_tokens), inp_out_ids); + } + + cur = build_rwkv6_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV6); + cur = ggml_add(ctx0, cur, ffn_inp); + if (hparams.rescale_every_n_layers != 0 && (il + 1) % hparams.rescale_every_n_layers == 0) { cur = ggml_scale(ctx0, cur, 0.5F); } @@ -10444,12 +10631,6 @@ struct llm_build_rwkv6 : public llm_build_rwkv6_base { } cur = inpL; - - ggml_tensor * inp_out_ids = build_inp_out_ids(); - - cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM, -1); cb(cur, "result_norm", -1); @@ -10481,10 +10662,9 @@ struct llm_build_rwkv6qwen2 : public llm_build_rwkv6_base { const auto n_seq_tokens = ubatch.n_seq_tokens; const auto n_seqs = ubatch.n_seqs; - inpL = build_inp_embd(model.tok_embd); - for (int il = 0; il < n_layer; ++il) { const llama_layer * layer = &model.layers[il]; + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); ggml_tensor * token_shift = build_rwkv_token_shift_load( gf, state_copy, state_mask, ubatch, il @@ -10508,6 +10688,13 @@ struct llm_build_rwkv6qwen2 : public llm_build_rwkv6_base { ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); cb(ffn_inp, "ffn_inp", il); + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, cur, n_embd, n_tokens), inp_out_ids); + ffn_inp = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens), inp_out_ids); + } + // feed-forward network cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, @@ -10532,10 +10719,358 @@ struct llm_build_rwkv6qwen2 : public llm_build_rwkv6_base { } cur = inpL; - ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); + cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_rwkv7_base : public llm_graph_context { + const llama_model & model; + + llm_build_rwkv7_base(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params), model(model) { + } + + ggml_tensor * build_rwkv7_channel_mix( + const llama_layer * layer, + ggml_tensor * cur, + ggml_tensor * x_prev, + llm_arch arch) const { + ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur); + switch (arch) { + case LLM_ARCH_RWKV7: + { + ggml_tensor * xk = ggml_add(ctx0, ggml_mul(ctx0, sx, layer->channel_mix_lerp_k), cur); + + ggml_tensor * k = ggml_sqr( + ctx0, + ggml_relu( + ctx0, + build_lora_mm(layer->channel_mix_key, xk) + ) + ); + + cur = build_lora_mm(layer->channel_mix_value, k); + } break; + default: + GGML_ABORT("fatal error"); + } + + return cur; + } + + ggml_tensor * build_rwkv7_time_mix( + ggml_cgraph * gf, + ggml_tensor * cur, + ggml_tensor * x_prev, + ggml_tensor * state_copy, + ggml_tensor * state_mask, + ggml_tensor *& first_layer_value, + const llama_ubatch & ubatch, + int il) const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + const auto n_tokens = ubatch.n_tokens; + const auto n_seqs = ubatch.n_seqs; + const auto n_embd = hparams.n_embd; + const auto head_size = hparams.wkv_head_size; + const auto head_count = n_embd / head_size; + const auto n_seq_tokens = ubatch.n_seq_tokens; + + const auto kv_head = kv_self->head; + + const auto & layer = model.layers[il]; + + bool has_gating = layer.time_mix_g1 && layer.time_mix_g2; + + ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur); + ggml_tensor * dummy = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_embd, n_seq_tokens, n_seqs, has_gating ? 6 : 5); + sx = ggml_repeat(ctx0, sx, dummy); + + ggml_tensor * xxx = ggml_add(ctx0, ggml_mul(ctx0, sx, layer.time_mix_lerp_fused), cur); + + ggml_tensor * xr = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], 0); + ggml_tensor * xw = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * sizeof(float)); + ggml_tensor * xk = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 2 * sizeof(float)); + ggml_tensor * xv = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 3 * sizeof(float)); + ggml_tensor * xa = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 4 * sizeof(float)); + ggml_tensor * xg = has_gating ? ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 5 * sizeof(float)) : nullptr; + + ggml_tensor * r = build_lora_mm(layer.time_mix_receptance, xr); + ggml_tensor * w = ggml_add( + ctx0, + ggml_mul_mat(ctx0, layer.time_mix_w2, ggml_tanh(ctx0, ggml_mul_mat(ctx0, layer.time_mix_w1, xw))), + layer.time_mix_w0 + ); + w = ggml_exp(ctx0, ggml_scale(ctx0, ggml_sigmoid(ctx0, w), -0.606531)); + + ggml_tensor * k = build_lora_mm(layer.time_mix_key, xk); + ggml_tensor * v = build_lora_mm(layer.time_mix_value, xv); + if (first_layer_value == nullptr) { + first_layer_value = v; + } else { + // Add the first layer value as a residual connection. + v = ggml_add(ctx0, v, + ggml_mul(ctx0, + ggml_sub(ctx0, first_layer_value, v), + ggml_sigmoid(ctx0, ggml_add(ctx0, + ggml_mul_mat(ctx0, layer.time_mix_v2, ggml_mul_mat(ctx0, layer.time_mix_v1, xv)), + layer.time_mix_v0 + ) + ) + ) + ); + } + + ggml_tensor * g = nullptr; + if (layer.time_mix_g1 && layer.time_mix_g2) { + g = ggml_mul_mat(ctx0, layer.time_mix_g2, ggml_sigmoid(ctx0, ggml_mul_mat(ctx0, layer.time_mix_g1, xg))); + } + + ggml_tensor * a = ggml_sigmoid(ctx0, + ggml_add( + ctx0, + ggml_mul_mat(ctx0, layer.time_mix_a2, ggml_mul_mat(ctx0, layer.time_mix_a1, xa)), + layer.time_mix_a0 + ) + ); + + ggml_tensor * kk = ggml_reshape_3d(ctx0, ggml_mul(ctx0, k, layer.time_mix_k_k), head_size, head_count, n_tokens); + kk = ggml_l2_norm(ctx0, kk, 1e-12); + + ggml_tensor * ka = ggml_mul(ctx0, k, layer.time_mix_k_a); + k = ggml_add(ctx0, k, ggml_sub(ctx0, ggml_mul(ctx0, a, ka), ka)); + + r = ggml_reshape_3d(ctx0, r, head_size, head_count, n_tokens); + w = ggml_reshape_3d(ctx0, w, head_size, head_count, n_tokens); + k = ggml_reshape_3d(ctx0, k, head_size, head_count, n_tokens); + v = ggml_reshape_3d(ctx0, v, head_size, head_count, n_tokens); + a = ggml_reshape_3d(ctx0, a, head_size, head_count, n_tokens); + + ggml_tensor * wkv_state = build_copy_mask_state( + gf, kv_self->v_l[il], state_copy, state_mask, + hparams.n_embd_v_s(), n_seqs); + + ggml_tensor * wkv_output = ggml_rwkv_wkv7(ctx0, r, w, k, v, ggml_neg(ctx0, kk), ggml_mul(ctx0, kk, a), wkv_state); + cur = ggml_view_1d(ctx0, wkv_output, n_embd * n_tokens, 0); + wkv_state = ggml_view_1d(ctx0, wkv_output, n_embd * head_size * n_seqs, n_embd * n_tokens * sizeof(float)); + + ggml_build_forward_expand( + gf, + ggml_cpy( + ctx0, + wkv_state, + ggml_view_1d( + ctx0, + kv_self->v_l[il], + hparams.n_embd_v_s() * n_seqs, + hparams.n_embd_v_s() * kv_head * ggml_element_size(kv_self->v_l[il]) + ) + ) + ); + + if (layer.time_mix_ln && layer.time_mix_ln_b) { + // group norm with head_count groups + cur = ggml_reshape_3d(ctx0, cur, n_embd / head_count, head_count, n_tokens); + cur = ggml_norm(ctx0, cur, 64e-5f); + + // Convert back to regular vectors. + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.time_mix_ln), layer.time_mix_ln_b); + } else { + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + } + + ggml_tensor * rk = ggml_sum_rows(ctx0, + ggml_mul(ctx0, ggml_mul(ctx0, k, r), ggml_reshape_2d(ctx0, layer.time_mix_r_k, head_size, head_count))); + cur = ggml_add(ctx0, cur, ggml_reshape_2d(ctx0, ggml_mul(ctx0, v, rk), n_embd, n_tokens)); + + if (has_gating) { + cur = ggml_mul(ctx0, cur, g); + } + cur = build_lora_mm(layer.time_mix_output, cur); + + return ggml_reshape_3d(ctx0, cur, n_embd, n_seq_tokens, n_seqs); + } +}; + +struct llm_build_rwkv7 : public llm_build_rwkv7_base { + llm_build_rwkv7(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_build_rwkv7_base(model, params) { + GGML_ASSERT(hparams.token_shift_count == 2); + + ggml_tensor * cur; + ggml_tensor * inpL; + ggml_tensor * v_first = nullptr; + + inpL = build_inp_embd(model.tok_embd); + inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1); + + ggml_tensor * state_copy = build_inp_s_copy(); + ggml_tensor * state_mask = build_inp_s_mask(); + + const auto n_embd = hparams.n_embd; + const auto n_seq_tokens = ubatch.n_seq_tokens; + const auto n_seqs = ubatch.n_seqs; + + for (int il = 0; il < n_layer; ++il) { + const llama_layer * layer = &model.layers[il]; + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); + + ggml_tensor * token_shift = build_rwkv_token_shift_load( + gf, state_copy, state_mask, ubatch, il + ); + + ggml_tensor * att_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], 0); + ggml_tensor * ffn_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], n_embd * ggml_element_size(token_shift)); + + ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM, il); + cb(att_norm, "attn_norm", il); + + ggml_tensor * x_prev = ggml_concat( + ctx0, + att_shift, + ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0), + 1 + ); + + cur = build_rwkv7_time_mix(gf, att_norm, x_prev, state_copy, state_mask, v_first, ubatch, il); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + ggml_tensor * ffn_norm = build_norm(ffn_inp, layer->attn_norm_2, layer->attn_norm_2_b, LLM_NORM, il); + cb(ffn_norm, "ffn_norm", il); + + x_prev = ggml_concat( + ctx0, + ffn_shift, + ggml_view_3d(ctx0, ffn_norm, n_embd, n_seq_tokens - 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], 0), + 1 + ); + + token_shift = ggml_concat(ctx0, + ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm)), + ggml_view_3d(ctx0, ffn_norm, n_embd, 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(ffn_norm)), + 1 + ); + ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il)); + + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + ffn_inp = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens), inp_out_ids); + ffn_norm = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_norm, n_embd, n_tokens), inp_out_ids); + x_prev = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, x_prev, n_embd, n_tokens), inp_out_ids); + } + + cur = build_rwkv7_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV7); + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + + +struct llm_build_arwkv7 : public llm_build_rwkv7_base { + llm_build_arwkv7(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_build_rwkv7_base(model, params) { + GGML_ASSERT(n_embd == hparams.n_embd_k_s()); + + ggml_tensor * cur; + ggml_tensor * inpL; + ggml_tensor * v_first = nullptr; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * state_copy = build_inp_s_copy(); + ggml_tensor * state_mask = build_inp_s_mask(); + + const auto n_embd = hparams.n_embd; + const auto n_seq_tokens = ubatch.n_seq_tokens; + const auto n_seqs = ubatch.n_seqs; + + for (int il = 0; il < n_layer; ++il) { + const llama_layer * layer = &model.layers[il]; + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); + + ggml_tensor * token_shift = build_rwkv_token_shift_load( + gf, state_copy, state_mask, ubatch, il + ); + + ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM_RMS, il); + cb(att_norm, "attn_norm", il); + + ggml_tensor * x_prev = ggml_concat( + ctx0, + token_shift, + ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0), + 1 + ); + + cur = build_rwkv7_time_mix(gf, att_norm, x_prev, state_copy, state_mask, v_first, ubatch, il); + + token_shift = ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm)); + ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il)); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, cur, n_embd, n_tokens), inp_out_ids); + ffn_inp = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens), inp_out_ids); + } + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM_RMS, -1); cb(cur, "result_norm", -1); @@ -10883,9 +11418,11 @@ llama_memory_i * llama_model::create_memory() const { llama_memory_i * res; switch (arch) { + case LLM_ARCH_MAMBA: case LLM_ARCH_RWKV6: case LLM_ARCH_RWKV6QWEN2: - case LLM_ARCH_MAMBA: + case LLM_ARCH_RWKV7: + case LLM_ARCH_ARWKV7: { res = new llama_kv_cache_unified(hparams, { /*.get_rope_factors =*/ nullptr @@ -11132,6 +11669,14 @@ llm_graph_result_ptr llama_model::build_graph( { llm = std::make_unique(*this, params, gf); } break; + case LLM_ARCH_RWKV7: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_ARWKV7: + { + llm = std::make_unique(*this, params, gf); + } break; case LLM_ARCH_CHAMELEON: { llm = std::make_unique(*this, params, gf); @@ -11245,6 +11790,8 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_JAIS: case LLM_ARCH_RWKV6: case LLM_ARCH_RWKV6QWEN2: + case LLM_ARCH_RWKV7: + case LLM_ARCH_ARWKV7: case LLM_ARCH_WAVTOKENIZER_DEC: return LLAMA_ROPE_TYPE_NONE; @@ -11399,6 +11946,8 @@ bool llama_model_is_recurrent(const llama_model * model) { case LLM_ARCH_MAMBA: return true; case LLM_ARCH_RWKV6: return true; case LLM_ARCH_RWKV6QWEN2: return true; + case LLM_ARCH_RWKV7: return true; + case LLM_ARCH_ARWKV7: return true; default: return false; } } diff --git a/src/llama-model.h b/src/llama-model.h index 55c26a92b0..a9da1215ab 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -29,6 +29,7 @@ enum llm_type { LLM_TYPE_109M, LLM_TYPE_137M, LLM_TYPE_160M, + LLM_TYPE_190M, LLM_TYPE_220M, LLM_TYPE_250M, LLM_TYPE_270M, @@ -45,6 +46,7 @@ enum llm_type { LLM_TYPE_1_6B, LLM_TYPE_2B, LLM_TYPE_2_8B, + LLM_TYPE_2_9B, LLM_TYPE_3B, LLM_TYPE_4B, LLM_TYPE_6B, @@ -260,6 +262,20 @@ struct llama_layer { struct ggml_tensor * time_mix_receptance_b = nullptr; struct ggml_tensor * time_mix_gate = nullptr; + // rwkv7 + struct ggml_tensor * time_mix_w0 = nullptr; + struct ggml_tensor * time_mix_a0 = nullptr; + struct ggml_tensor * time_mix_a1 = nullptr; + struct ggml_tensor * time_mix_a2 = nullptr; + struct ggml_tensor * time_mix_v0 = nullptr; + struct ggml_tensor * time_mix_v1 = nullptr; + struct ggml_tensor * time_mix_v2 = nullptr; + struct ggml_tensor * time_mix_g1 = nullptr; + struct ggml_tensor * time_mix_g2 = nullptr; + struct ggml_tensor * time_mix_k_k = nullptr; + struct ggml_tensor * time_mix_k_a = nullptr; + struct ggml_tensor * time_mix_r_k = nullptr; + struct ggml_tensor * time_mix_ln = nullptr; struct ggml_tensor * time_mix_ln_b = nullptr; struct ggml_tensor * time_mix_output = nullptr; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index fb7982655a..09eb570779 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -756,10 +756,19 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: // NOTE: can't use LLM_TN here because the layer number is not known quantize &= name.find("ssm_conv1d.weight") == std::string::npos; - // do not quantize RWKV's time_mix_first tensors + // do not quantize RWKV's small yet 2D weights quantize &= name.find("time_mix_first.weight") == std::string::npos; + quantize &= name.find("time_mix_w0.weight") == std::string::npos; quantize &= name.find("time_mix_w1.weight") == std::string::npos; quantize &= name.find("time_mix_w2.weight") == std::string::npos; + quantize &= name.find("time_mix_v0.weight") == std::string::npos; + quantize &= name.find("time_mix_v1.weight") == std::string::npos; + quantize &= name.find("time_mix_v2.weight") == std::string::npos; + quantize &= name.find("time_mix_a0.weight") == std::string::npos; + quantize &= name.find("time_mix_a1.weight") == std::string::npos; + quantize &= name.find("time_mix_a2.weight") == std::string::npos; + quantize &= name.find("time_mix_g1.weight") == std::string::npos; + quantize &= name.find("time_mix_g2.weight") == std::string::npos; quantize &= name.find("time_mix_decay_w1.weight") == std::string::npos; quantize &= name.find("time_mix_decay_w2.weight") == std::string::npos; quantize &= name.find("time_mix_lerp_fused.weight") == std::string::npos; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c86ffb64e9..adb749bd5e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1916,6 +1916,40 @@ struct test_gla : public test_case { } }; +// GGML_OP_RWKV_WKV7 +struct test_rwkv_wkv7 : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + + std::string vars() override { + return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); + } + + test_rwkv_wkv7(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t n_tokens = n_seq_tokens * n_seqs; + ggml_tensor * r = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * w = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * a = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + // Outputs may become NaN with long seqlen without these normalization + a = ggml_l2_norm(ctx, a, 1e-7F); + b = ggml_l2_norm(ctx, b, 1e-7F); + ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector{ head_size * head_size * head_count, n_seqs }.data()); + ggml_tensor * out = ggml_rwkv_wkv7(ctx, r, w, k, v, a, b, s); + return out; + } +}; + // GGML_OP_MUL_MAT struct test_mul_mat : public test_case { const ggml_type type_a; @@ -2972,6 +3006,32 @@ struct test_group_norm : public test_case { } }; +// GGML_OP_L2_NORM +struct test_l2_norm : public test_case { + const ggml_type type; + const std::array ne; + const float eps; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_l2_norm(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 64, 320, 1}, + float eps = 1e-12f) + : type(type), ne(ne), eps(eps) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_l2_norm(ctx, a, eps); + ggml_set_name(out, "out"); + + return out; + } +}; + // GGML_OP_ACC struct test_acc : public test_case { const ggml_type type; @@ -4006,8 +4066,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, v, eps)); } test_cases.emplace_back(new test_rms_norm_back(GGML_TYPE_F32, {64, 5, 4, 3}, eps)); + test_cases.emplace_back(new test_l2_norm (GGML_TYPE_F32, {64, 5, 4, 3}, eps)); } + test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, {64, 5, 4, 3}, 1e-12f)); + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {4, 1536, 1, 1}, {4, 1536, 1, 1})); test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {8, 1536, 1, 1}, {4, 1536, 1, 1})); test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {4, 1536, 4, 1}, {4, 1536, 1, 1})); @@ -4019,6 +4082,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 4)); test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 128, 4)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 1, 1)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 1)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 4)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 128, 4)); + test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 1, 1)); test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 32, 1)); test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 32, 4)); From a53f7f7b8859f3e634415ab03e1e295b9861d7e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= <112692748+lslusarczyk@users.noreply.github.com> Date: Tue, 18 Mar 2025 01:51:25 +0100 Subject: [PATCH 25/55] fixed compilation warnings in ggml-sycl (#12424) --- ggml/src/ggml-sycl/convert.cpp | 2 +- ggml/src/ggml-sycl/dmmv.cpp | 25 +++++---- ggml/src/ggml-sycl/element_wise.cpp | 80 ++++++++++++++--------------- ggml/src/ggml-sycl/getrows.cpp | 3 +- ggml/src/ggml-sycl/ggml-sycl.cpp | 43 ++++++++-------- ggml/src/ggml-sycl/mmq.cpp | 1 - ggml/src/ggml-sycl/mmvq.cpp | 39 +++++++------- ggml/src/ggml-sycl/norm.cpp | 12 ++--- ggml/src/ggml-sycl/softmax.cpp | 2 +- 9 files changed, 101 insertions(+), 106 deletions(-) diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index 86b200e070..76ac6a4dd1 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -138,7 +138,7 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * sycl::range<3>(1, 1, WARP_SIZE), sycl::range<3>(1, 1, WARP_SIZE)), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]]{ + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ dequantize_block_q4_0_reorder(vx, y, k, item_ct1); }); diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index 99d3859de8..04a85fa35f 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -210,7 +210,7 @@ static void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec<1, 1, convert_f16>(vx, y, dst, ncols, nrows, item_ct1); }); @@ -879,7 +879,7 @@ static void dequantize_mul_mat_vec_q4_0_sycl_reorder(const void *vx, const dfloa stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec_reorder( vx, y, dst, ncols, nrows, item_ct1); }); @@ -902,7 +902,7 @@ static void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -923,7 +923,7 @@ static void dequantize_mul_mat_vec_q4_1_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -944,7 +944,7 @@ static void dequantize_mul_mat_vec_q5_0_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -965,7 +965,7 @@ static void dequantize_mul_mat_vec_q5_1_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -986,7 +986,7 @@ static void dequantize_mul_mat_vec_q8_0_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -1004,7 +1004,7 @@ static void dequantize_mul_mat_vec_q2_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q2_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1020,7 +1020,7 @@ static void dequantize_mul_mat_vec_q3_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q3_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1036,7 +1036,7 @@ static void dequantize_mul_mat_vec_q4_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q4_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1049,7 +1049,7 @@ static void dequantize_mul_mat_vec_q5_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, 1, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q5_k(vx, y, dst, ncols, item_ct1); }); } @@ -1065,7 +1065,7 @@ static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q6_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1143,7 +1143,6 @@ void ggml_sycl_op_dequantize_mul_mat_vec( default: printf("ggml_sycl_op_dequantize_mul_mat_vec unsupported GGML_TYPE %d\n", src0->type); GGML_ABORT("fatal error"); - break; } GGML_UNUSED(src1); diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cpp index 4bcd74376e..1e12cb220e 100644 --- a/ggml/src/ggml-sycl/element_wise.cpp +++ b/ggml/src/ggml-sycl/element_wise.cpp @@ -1,7 +1,7 @@ #include "common.hpp" #include "element_wise.hpp" -void acc_f32(const float * x, const float * y, float * dst, const int ne, +static void acc_f32(const float * x, const float * y, float * dst, const int ne, const int ne10, const int ne11, const int ne12, const int nb1, const int nb2, int offset, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + @@ -20,7 +20,7 @@ void acc_f32(const float * x, const float * y, float * dst, const int ne, } } -void gelu_f32(const float * x, float * dst, const int k, +static void gelu_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const float GELU_COEF_A = 0.044715f; const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; @@ -37,7 +37,7 @@ void gelu_f32(const float * x, float * dst, const int k, sycl::tanh(SQRT_2_OVER_PI * xi * (1.0f + GELU_COEF_A * xi * xi))); } -void silu_f32(const float * x, float * dst, const int k, +static void silu_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -48,7 +48,7 @@ void silu_f32(const float * x, float * dst, const int k, dst[i] = x[i] / (1.0f + sycl::native::exp(-x[i])); } -void gelu_quick_f32(const float *x, float *dst, int k, +static void gelu_quick_f32(const float *x, float *dst, int k, const sycl::nd_item<3> &item_ct1) { const float GELU_QUICK_COEF = -1.702f; const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + @@ -59,7 +59,7 @@ void gelu_quick_f32(const float *x, float *dst, int k, dst[i] = x[i] * (1.0f / (1.0f + sycl::native::exp(GELU_QUICK_COEF * x[i]))); } -void tanh_f32(const float *x, float *dst, int k, +static void tanh_f32(const float *x, float *dst, int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -69,7 +69,7 @@ void tanh_f32(const float *x, float *dst, int k, dst[i] = sycl::tanh((float)(x[i])); } -void relu_f32(const float * x, float * dst, const int k, +static void relu_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -80,7 +80,7 @@ void relu_f32(const float * x, float * dst, const int k, dst[i] = sycl::fmax((float)(x[i]), (float)0); } -void sigmoid_f32(const float * x, float * dst, const int k, +static void sigmoid_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -91,7 +91,7 @@ void sigmoid_f32(const float * x, float * dst, const int k, dst[i] = 1.0f / (1.0f + sycl::native::exp(-x[i])); } -void sqrt_f32(const float * x, float * dst, const int k, +static void sqrt_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -102,7 +102,7 @@ void sqrt_f32(const float * x, float * dst, const int k, dst[i] = sycl::sqrt(x[i]); } -void sin_f32(const float * x, float * dst, const int k, +static void sin_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -113,7 +113,7 @@ void sin_f32(const float * x, float * dst, const int k, dst[i] = sycl::sin(x[i]); } -void cos_f32(const float * x, float * dst, const int k, +static void cos_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -124,7 +124,7 @@ void cos_f32(const float * x, float * dst, const int k, dst[i] = sycl::cos(x[i]); } -void hardsigmoid_f32(const float * x, float * dst, const int k, +static void hardsigmoid_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -135,7 +135,7 @@ void hardsigmoid_f32(const float * x, float * dst, const int k, dst[i] = sycl::fmin(1.0f, sycl::fmax(0.0f, (x[i] + 3.0f) / 6.0f)); } -void hardswish_f32(const float * x, float * dst, const int k, +static void hardswish_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -146,7 +146,7 @@ void hardswish_f32(const float * x, float * dst, const int k, dst[i] = x[i] * sycl::fmin(1.0f, sycl::fmax(0.0f, (x[i] + 3.0f) / 6.0f)); } -void exp_f32(const float * x, float * dst, const int k, +static void exp_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -157,7 +157,7 @@ void exp_f32(const float * x, float * dst, const int k, dst[i] = sycl::exp(x[i]); } -void log_f32(const float * x, float * dst, const int k, +static void log_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -173,7 +173,7 @@ void log_f32(const float * x, float * dst, const int k, } } -void neg_f32(const float * x, float * dst, const int k, +static void neg_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -184,7 +184,7 @@ void neg_f32(const float * x, float * dst, const int k, dst[i] = -x[i]; } -void step_f32(const float * x, float * dst, const int k, +static void step_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -195,7 +195,7 @@ void step_f32(const float * x, float * dst, const int k, dst[i] = x[i] > 0.0f; } -void leaky_relu_f32(const float *x, float *dst, const int k, const float negative_slope, +static void leaky_relu_f32(const float *x, float *dst, const int k, const float negative_slope, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -206,7 +206,7 @@ void leaky_relu_f32(const float *x, float *dst, const int k, const float negativ sycl::fmin((float)(x[i]), 0.0f) * negative_slope; } -void sqr_f32(const float * x, float * dst, const int k, +static void sqr_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -217,7 +217,7 @@ void sqr_f32(const float * x, float * dst, const int k, dst[i] = x[i] * x[i]; } -void upscale_f32(const float *x, float *dst, const int nb00, const int nb01, +static void upscale_f32(const float *x, float *dst, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int ne13, const float sf0, const float sf1, const float sf2, const float sf3, const sycl::nd_item<1> &item_ct1) { @@ -240,7 +240,7 @@ void upscale_f32(const float *x, float *dst, const int nb00, const int nb01, dst[index] = *(const float *)((const char *)x + i03 * nb03 + i02 * nb02 + i01 * nb01 + i00 * nb00); } -void pad_f32(const float *x, float *dst, const int ne0, const int ne00, const int ne01, const int ne02, +static void pad_f32(const float *x, float *dst, const int ne0, const int ne00, const int ne01, const int ne02, const sycl::nd_item<3> &item_ct1) { int nidx = item_ct1.get_local_id(2) + item_ct1.get_group(2) * item_ct1.get_local_range(2); @@ -262,7 +262,7 @@ void pad_f32(const float *x, float *dst, const int ne0, const int ne00, const i -void acc_f32_sycl(const float *x, const float *y, float *dst, +static void acc_f32_sycl(const float *x, const float *y, float *dst, const int n_elements, const int ne10, const int ne11, const int ne12, const int nb1, const int nb2, const int offset, queue_ptr stream) { @@ -277,7 +277,7 @@ void acc_f32_sycl(const float *x, const float *y, float *dst, }); } -void gelu_f32_sycl(const float *x, float *dst, const int k, +static void gelu_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_GELU_BLOCK_SIZE - 1) / SYCL_GELU_BLOCK_SIZE; stream->parallel_for( @@ -289,7 +289,7 @@ void gelu_f32_sycl(const float *x, float *dst, const int k, }); } -void silu_f32_sycl(const float *x, float *dst, const int k, +static void silu_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SILU_BLOCK_SIZE - 1) / SYCL_SILU_BLOCK_SIZE; stream->parallel_for( @@ -301,7 +301,7 @@ void silu_f32_sycl(const float *x, float *dst, const int k, }); } -void gelu_quick_f32_sycl(const float *x, float *dst, const int k, +static void gelu_quick_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_GELU_BLOCK_SIZE - 1) / SYCL_GELU_BLOCK_SIZE; stream->parallel_for( @@ -313,7 +313,7 @@ void gelu_quick_f32_sycl(const float *x, float *dst, const int k, }); } -void tanh_f32_sycl(const float *x, float *dst, const int k, +static void tanh_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_TANH_BLOCK_SIZE - 1) / SYCL_TANH_BLOCK_SIZE; stream->parallel_for( @@ -325,7 +325,7 @@ void tanh_f32_sycl(const float *x, float *dst, const int k, }); } -void relu_f32_sycl(const float *x, float *dst, const int k, +static void relu_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_RELU_BLOCK_SIZE - 1) / SYCL_RELU_BLOCK_SIZE; stream->parallel_for( @@ -337,7 +337,7 @@ void relu_f32_sycl(const float *x, float *dst, const int k, }); } -void hardsigmoid_f32_sycl(const float *x, float *dst, const int k, +static void hardsigmoid_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_HARDSIGMOID_BLOCK_SIZE - 1) / SYCL_HARDSIGMOID_BLOCK_SIZE; stream->parallel_for( @@ -349,7 +349,7 @@ void hardsigmoid_f32_sycl(const float *x, float *dst, const int k, }); } -void hardswish_f32_sycl(const float *x, float *dst, const int k, +static void hardswish_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_HARDSWISH_BLOCK_SIZE - 1) / SYCL_HARDSWISH_BLOCK_SIZE; stream->parallel_for( @@ -361,7 +361,7 @@ void hardswish_f32_sycl(const float *x, float *dst, const int k, }); } -void exp_f32_sycl(const float *x, float *dst, const int k, +static void exp_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_EXP_BLOCK_SIZE - 1) / SYCL_EXP_BLOCK_SIZE; stream->parallel_for( @@ -373,7 +373,7 @@ void exp_f32_sycl(const float *x, float *dst, const int k, }); } -void log_f32_sycl(const float *x, float *dst, const int k, +static void log_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_EXP_BLOCK_SIZE - 1) / SYCL_EXP_BLOCK_SIZE; stream->parallel_for( @@ -385,7 +385,7 @@ void log_f32_sycl(const float *x, float *dst, const int k, }); } -void neg_f32_sycl(const float *x, float *dst, const int k, +static void neg_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_NEG_BLOCK_SIZE - 1) / SYCL_NEG_BLOCK_SIZE; stream->parallel_for( @@ -397,7 +397,7 @@ void neg_f32_sycl(const float *x, float *dst, const int k, }); } -void step_f32_sycl(const float *x, float *dst, const int k, +static void step_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_NEG_BLOCK_SIZE - 1) / SYCL_NEG_BLOCK_SIZE; stream->parallel_for( @@ -409,7 +409,7 @@ void step_f32_sycl(const float *x, float *dst, const int k, }); } -void sigmoid_f32_sycl(const float *x, float *dst, const int k, +static void sigmoid_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SIGMOID_BLOCK_SIZE - 1) / SYCL_SIGMOID_BLOCK_SIZE; stream->parallel_for( @@ -421,7 +421,7 @@ void sigmoid_f32_sycl(const float *x, float *dst, const int k, }); } -void sqrt_f32_sycl(const float *x, float *dst, const int k, +static void sqrt_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SQRT_BLOCK_SIZE - 1) / SYCL_SQRT_BLOCK_SIZE; stream->parallel_for( @@ -433,7 +433,7 @@ void sqrt_f32_sycl(const float *x, float *dst, const int k, }); } -void sin_f32_sycl(const float *x, float *dst, const int k, +static void sin_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SIN_BLOCK_SIZE - 1) / SYCL_SIN_BLOCK_SIZE; stream->parallel_for( @@ -445,7 +445,7 @@ void sin_f32_sycl(const float *x, float *dst, const int k, }); } -void cos_f32_sycl(const float *x, float *dst, const int k, +static void cos_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SIN_BLOCK_SIZE - 1) / SYCL_SIN_BLOCK_SIZE; stream->parallel_for( @@ -457,7 +457,7 @@ void cos_f32_sycl(const float *x, float *dst, const int k, }); } -void leaky_relu_f32_sycl(const float *x, float *dst, const int k, +static void leaky_relu_f32_sycl(const float *x, float *dst, const int k, const float negative_slope, queue_ptr stream) { const int num_blocks = (k + SYCL_RELU_BLOCK_SIZE - 1) / SYCL_RELU_BLOCK_SIZE; @@ -470,7 +470,7 @@ void leaky_relu_f32_sycl(const float *x, float *dst, const int k, }); } -void sqr_f32_sycl(const float *x, float *dst, const int k, +static void sqr_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SQR_BLOCK_SIZE - 1) / SYCL_SQR_BLOCK_SIZE; stream->parallel_for( @@ -482,7 +482,7 @@ void sqr_f32_sycl(const float *x, float *dst, const int k, }); } -void upscale_f32_sycl(const float *x, float *dst, const int nb00, const int nb01, +static void upscale_f32_sycl(const float *x, float *dst, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int ne13, const float sf0, const float sf1, const float sf2, const float sf3, queue_ptr stream) { @@ -496,7 +496,7 @@ void upscale_f32_sycl(const float *x, float *dst, const int nb00, const int nb01 }); } -void pad_f32_sycl(const float *x, float *dst, const int ne00, +static void pad_f32_sycl(const float *x, float *dst, const int ne00, const int ne01, const int ne02, const int ne0, const int ne1, const int ne2, queue_ptr stream) { int num_blocks = (ne0 + SYCL_PAD_BLOCK_SIZE - 1) / SYCL_PAD_BLOCK_SIZE; diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp index 51c19f6b3b..b9cf8767cb 100644 --- a/ggml/src/ggml-sycl/getrows.cpp +++ b/ggml/src/ggml-sycl/getrows.cpp @@ -207,7 +207,7 @@ static void get_rows_sycl_reorder(ggml_backend_sycl_context & ctx, const ggml_te const size_t nrows = ne01; const sycl::half* src0_dq = (const sycl::half*)(src0_q + nrows * ncols / 2); stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]]{ + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ k_get_rows_reorder( src0_dd, src0_dq, src1_dd, dst_dd, ne00, ne12, s1, s2, s3, nb01, nb02, nb03, s10, s11, s12, item_ct1); @@ -302,7 +302,6 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, const ggml_tensor *s // TODO: k-quants GGML_LOG_ERROR("%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type)); GGML_ABORT("fatal error"); - break; } } diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 477652ab28..207c0b440a 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -95,7 +95,7 @@ const ggml_sycl_device_info & ggml_sycl_info() { return info; } -void print_device_detail(int id, sycl::device &device, std::string device_type) { +static void print_device_detail(int id, sycl::device &device, std::string device_type) { dpct::device_info prop; SYCL_CHECK(CHECK_TRY_ERROR( @@ -118,7 +118,7 @@ void print_device_detail(int id, sycl::device &device, std::string device_type) global_mem_size, device.get_info().c_str()); } -void print_device_opt_feature(int device_count) { +static void print_device_opt_feature(int device_count) { GGML_LOG_INFO("SYCL Optimization Feature:\n"); GGML_LOG_INFO( "|ID| Device Type|Reorder|\n"); @@ -401,7 +401,7 @@ catch (sycl::exception const &exc) { std::exit(1); } -void dev2dev_memcpy(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst, +static void dev2dev_memcpy(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst, const void *ptr_src, size_t size) { char *host_buf = (char *)malloc(size); q_src.memcpy(host_buf, (const char *)ptr_src, size).wait(); @@ -620,7 +620,7 @@ ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { return &ggml_backend_sycl_buffer_types[device]; } -ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(ggml_backend_sycl_context * ctx) { +static ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(ggml_backend_sycl_context * ctx) { GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_buffer_type\n"); int device = ctx->device; @@ -1682,7 +1682,7 @@ static void quantize_row_q8_1_sycl(const float *x, void *vy, const int kx, stream->parallel_for( sycl::nd_range<3>(num_blocks * block_size, block_size), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { quantize_q8_1(x, vy, kx, kx_padded, item_ct1); }); } @@ -1703,7 +1703,7 @@ static void ggml_mul_mat_p021_f16_f32_sycl(const void *vx, const float *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_p021_f16_f32(vx, y, dst, ncols_x, nrows_x, nchannels_x, nchannels_y, item_ct1); }); @@ -1723,7 +1723,7 @@ static void ggml_mul_mat_vec_nc_f16_f32_sycl( stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_nc_f16_f32(vx, y, dst, ncols_x, nrows_x, row_stride_x, channel_stride_x, nchannels_y / nchannels_x, item_ct1); @@ -1764,7 +1764,7 @@ static void sum_rows_f32_sycl(const float *x, float *dst, const int ncols, const sycl::range<3> block_nums(1, nrows, 1); stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { k_sum_rows_f32(x, dst, ncols, item_ct1); }); } @@ -2920,7 +2920,7 @@ inline bool ggml_sycl_supports_mmq(enum ggml_type type) { return false; } -bool ggml_sycl_supports_dmmv(enum ggml_type type) { +static bool ggml_sycl_supports_dmmv(enum ggml_type type) { switch (type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -3293,7 +3293,7 @@ static void ggml_sycl_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * dst) } -void ggml_sycl_set_main_device(const int main_device) try { +static void ggml_sycl_set_main_device(const int main_device) try { if (dpct::get_current_device_id() == static_cast (main_device)) { return; } @@ -3314,7 +3314,7 @@ catch (sycl::exception const &exc) { std::exit(1); } -bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tensor * dst) { +static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tensor * dst) { if (!g_sycl_loaded) return false; if (dst->src[0] != nullptr && ggml_backend_buffer_is_sycl_split(dst->src[0]->buffer)) { @@ -3638,7 +3638,7 @@ catch (sycl::exception const &exc) { std::exit(1); } -void reorder_qw(char *data_device, const int ncols, const int nrows, +static void reorder_qw(char *data_device, const int ncols, const int nrows, size_t size, size_t offset, dpct::queue_ptr stream) { auto tmp_buf = sycl::malloc_shared(size, *stream); SYCL_CHECK( @@ -3652,7 +3652,7 @@ void reorder_qw(char *data_device, const int ncols, const int nrows, stream->parallel_for( size / sizeof(block_q4_0), - [=](auto i) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { const block_q4_0* x = (const block_q4_0*)tmp_buf; const int ib = i; @@ -3666,7 +3666,7 @@ void reorder_qw(char *data_device, const int ncols, const int nrows, sycl::free(tmp_buf, *stream); } -void reorder_qw(ggml_tensor * src0, dpct::queue_ptr stream) { +static void reorder_qw(ggml_tensor * src0, dpct::queue_ptr stream) { char*data_device = (char*)src0->data; size_t ncols = src0->ne[0]; size_t nrows = src0->ne[1]; @@ -3675,7 +3675,7 @@ void reorder_qw(ggml_tensor * src0, dpct::queue_ptr stream) { reorder_qw(data_device, ncols, nrows, size, 0, stream); } -void opt_for_reorder(ggml_tensor * dst, dpct::queue_ptr stream) { +static void opt_for_reorder(ggml_tensor * dst, dpct::queue_ptr stream) { ggml_tensor *src0 = dst->src[0]; ggml_tensor *src1 = dst->src[1]; @@ -3688,7 +3688,7 @@ void opt_for_reorder(ggml_tensor * dst, dpct::queue_ptr stream) { } } -void optimize_graph_once(ggml_cgraph * cgraph, ggml_backend_sycl_context * ctx) { +static void optimize_graph_once(ggml_cgraph * cgraph, ggml_backend_sycl_context * ctx) { dpct::queue_ptr stream = ctx->stream(); if (ctx->optimized_graph) { return; @@ -3878,7 +3878,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g return true; } return false; - } break; + } case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { case GGML_UNARY_OP_NEG: @@ -3896,7 +3896,6 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g default: return false; } - break; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: { @@ -3927,7 +3926,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g return false; } return true; - } break; + } case GGML_OP_OUT_PROD: return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->ne[2] == 1 && op->ne[3] == 1; case GGML_OP_GET_ROWS: @@ -3944,7 +3943,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g default: return false; } - } break; + } case GGML_OP_CPY: { ggml_type src0_type = op->src[0]->type; @@ -3995,12 +3994,12 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g return true; } return false; - } break; + } case GGML_OP_CONCAT: { ggml_type src0_type = op->src[0]->type; return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; - } break; + } case GGML_OP_DUP: case GGML_OP_ARGMAX: case GGML_OP_NONE: diff --git a/ggml/src/ggml-sycl/mmq.cpp b/ggml/src/ggml-sycl/mmq.cpp index 8ea82c940c..ffb272aa28 100644 --- a/ggml/src/ggml-sycl/mmq.cpp +++ b/ggml/src/ggml-sycl/mmq.cpp @@ -3017,7 +3017,6 @@ void ggml_sycl_op_mul_mat_q( break; default: GGML_ABORT("fatal error"); - break; } GGML_UNUSED(src1); diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index a96286d710..1b92ba2d60 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -495,7 +495,7 @@ static void mul_mat_vec_q4_0_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -519,7 +519,7 @@ static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -543,7 +543,7 @@ static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -567,7 +567,7 @@ static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -591,7 +591,7 @@ static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -615,7 +615,7 @@ static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -639,7 +639,7 @@ static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -663,7 +663,7 @@ static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -687,7 +687,7 @@ static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -711,7 +711,7 @@ static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -734,7 +734,7 @@ static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq2_xxs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -755,7 +755,7 @@ static void mul_mat_vec_iq2_xs_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq2_xs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -777,7 +777,7 @@ static void mul_mat_vec_iq2_s_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq2_s_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -799,7 +799,7 @@ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq3_xxs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -821,7 +821,7 @@ static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq3_s_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -843,7 +843,7 @@ static void mul_mat_vec_iq1_s_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq1_s_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -864,7 +864,7 @@ static void mul_mat_vec_iq1_m_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq1_m_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -886,7 +886,7 @@ static void mul_mat_vec_iq4_nl_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq4_nl_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -908,7 +908,7 @@ static void mul_mat_vec_iq4_xs_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq4_xs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -1003,7 +1003,6 @@ void ggml_sycl_op_mul_mat_vec_q( break; default: GGML_ABORT("fatal error"); - break; } } GGML_UNUSED(src1); diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp index 6439db21b2..d9678da8f0 100644 --- a/ggml/src/ggml-sycl/norm.cpp +++ b/ggml/src/ggml-sycl/norm.cpp @@ -235,7 +235,7 @@ static void norm_f32_sycl(const float* x, float* dst, const int ncols, sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { norm_f32(x, dst, ncols, eps, item_ct1, nullptr, WARP_SIZE); }); @@ -258,7 +258,7 @@ static void norm_f32_sycl(const float* x, float* dst, const int ncols, sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { norm_f32(x, dst, ncols, eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); }); @@ -277,7 +277,7 @@ static void group_norm_f32_sycl(const float* x, float* dst, sycl::nd_range<3>(sycl::range<3>(1, 1, num_groups) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { group_norm_f32( x, dst, group_size, ne_elements, eps_ct4, item_ct1, nullptr, WARP_SIZE); @@ -304,7 +304,7 @@ static void group_norm_f32_sycl(const float* x, float* dst, sycl::nd_range<3>(sycl::range<3>(1, 1, num_groups) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { group_norm_f32(x, dst, group_size, ne_elements, eps_ct4, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); @@ -325,7 +325,7 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { rms_norm_f32(x, dst, ncols, eps, item_ct1, nullptr, WARP_SIZE); }); @@ -347,7 +347,7 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { rms_norm_f32(x, dst, ncols, eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); }); diff --git a/ggml/src/ggml-sycl/softmax.cpp b/ggml/src/ggml-sycl/softmax.cpp index eb20bd251e..7563d9ceda 100644 --- a/ggml/src/ggml-sycl/softmax.cpp +++ b/ggml/src/ggml-sycl/softmax.cpp @@ -132,7 +132,7 @@ static void soft_max_f32_submitter(const float * x, const T * mask, float * dst, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { soft_max_f32(x, mask, dst, ncols_par, nrows_y, scale, max_bias, m0, m1, n_head_log2, item_ct1, From fd123cfead49eb32e386e26b8ef7a6d41554dda5 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Tue, 18 Mar 2025 07:21:40 +0100 Subject: [PATCH 26/55] Vulkan: Default to 1GB allocations instead of 4GB to avoid fragmentation and driver issues (#12434) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index c0ee5dadef..dd680aa522 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -2524,13 +2524,9 @@ static vk_device ggml_vk_get_device(size_t idx) { if (GGML_VK_SUBALLOCATION_BLOCK_SIZE != nullptr) { device->suballocation_block_size = std::stoul(GGML_VK_SUBALLOCATION_BLOCK_SIZE); -#if defined(_WIN32) - } else if (device->vendor_id == VK_VENDOR_ID_NVIDIA) { + } else { // Limit batching of allocations to 1GB by default to avoid fragmentation issues device->suballocation_block_size = 1024*1024*1024; -#endif - } else { - device->suballocation_block_size = device->max_memory_allocation_size; } device->suballocation_block_size = std::min(device->suballocation_block_size, device->max_memory_allocation_size); From d9a14523bb9074eef42d1b43ae4a1e149f81b7e2 Mon Sep 17 00:00:00 2001 From: fj-y-saito <85871716+fj-y-saito@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:14:39 +0900 Subject: [PATCH 27/55] ggml : add SVE support for q6_K_q8_K (#12361) --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 151 +++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 8c7dbd1ccb..4e0ae05724 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -8158,7 +8158,156 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi const int nb = n / QK_K; -#ifdef __ARM_NEON +#ifdef __ARM_FEATURE_SVE + const int vector_length = ggml_cpu_get_sve_cnt()*8; + float sum = 0; + svuint8_t m4b = svdup_n_u8(0xf); + svint32_t vzero = svdup_n_s32(0); + svuint8_t mone = svdup_n_u8(0x30); + svint8_t q6bytes_1, q6bytes_2, q6bytes_3, q6bytes_4; + svuint8_t q6h_1, q6h_2, q6h_3, q6h_4; + + for (int i = 0; i < nb; ++i) { + const float d_all = GGML_FP16_TO_FP32(x[i].d); + + const uint8_t * GGML_RESTRICT q6 = x[i].ql; + const uint8_t * GGML_RESTRICT qh = x[i].qh; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + const int8_t * GGML_RESTRICT scale = x[i].scales; + + const svbool_t pg16_8 = svptrue_pat_b16(SV_VL8); + const svint16_t q8sums_1 = svld1_s16(pg16_8, y[i].bsums); + const svint16_t q8sums_2 = svld1_s16(pg16_8, y[i].bsums + 8); + const svint16_t q6scales_1 = svunpklo_s16(svld1_s8(svptrue_pat_b8(SV_VL8), scale)); + const svint16_t q6scales_2 = svunpklo_s16(svld1_s8(svptrue_pat_b8(SV_VL8), scale + 8)); + const svint64_t prod = svdup_n_s64(0); + int32_t isum_mins = svaddv_s64(svptrue_b64(), svadd_s64_x(svptrue_b64(), svdot_s64(prod, q8sums_1, q6scales_1), + svdot_s64(prod, q8sums_2, q6scales_2))); + int32_t isum = 0; + + switch (vector_length) { + case 128: + { + const svbool_t pg32_4 = svptrue_pat_b32(SV_VL4); + const svbool_t pg8_16 = svptrue_pat_b8(SV_VL16); + svint32_t isum_tmp = svdup_n_s32(0); + for (int j = 0; j < QK_K/128; ++j) { + svuint8_t qhbits_1 = svld1_u8(pg8_16, qh); + svuint8_t qhbits_2 = svld1_u8(pg8_16, qh+16); + qh += 32; + svuint8_t q6bits_1 = svld1_u8(pg8_16, q6); + svuint8_t q6bits_2 = svld1_u8(pg8_16, q6+16); + svuint8_t q6bits_3 = svld1_u8(pg8_16, q6+32); + svuint8_t q6bits_4 = svld1_u8(pg8_16, q6+48); + q6 += 64; + svint8_t q8bytes_1 = svld1_s8(pg8_16, q8); + svint8_t q8bytes_2 = svld1_s8(pg8_16, q8+16); + svint8_t q8bytes_3 = svld1_s8(pg8_16, q8+32); + svint8_t q8bytes_4 = svld1_s8(pg8_16, q8+48); + q8 += 64; + + q6h_1 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_1, 4)); + q6h_2 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_2, 4)); + q6h_3 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_1, 2)); + q6h_4 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_2, 2)); + q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_1, m4b), q6h_1)); + q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_2, m4b), q6h_2)); + q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_3, m4b), q6h_3)); + q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_4, m4b), q6h_4)); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale[0]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale[1]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale[2]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale[3]); + + scale += 4; + q8bytes_1 = svld1_s8(pg8_16, q8); + q8bytes_2 = svld1_s8(pg8_16, q8+16); + q8bytes_3 = svld1_s8(pg8_16, q8+32); + q8bytes_4 = svld1_s8(pg8_16, q8+48); + q8 += 64; + + q6h_1 = svand_u8_x(pg16_8, mone, qhbits_1); + q6h_2 = svand_u8_x(pg16_8, mone, qhbits_2); + q6h_3 = svand_u8_x(pg16_8, mone, svlsr_n_u8_x(pg16_8, qhbits_1, 2)); + q6h_4 = svand_u8_x(pg16_8, mone, svlsr_n_u8_x(pg16_8, qhbits_2, 2)); + q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_1, 4), q6h_1)); + q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_2, 4), q6h_2)); + q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_3, 4), q6h_3)); + q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_4, 4), q6h_4)); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale[0]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale[1]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale[2]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale[3]); + scale += 4; + } + isum += svaddv_s32(pg32_4, isum_tmp); + sum += d_all * y[i].d * (isum - 32 * isum_mins); + } + break; + case 256: + case 512: + { + const svbool_t pg8_2 = svptrue_pat_b8(SV_VL2); + const svbool_t pg32_8 = svptrue_pat_b32(SV_VL8); + const svbool_t pg8_32 = svptrue_pat_b8(SV_VL32); + svint32_t isum_tmp = svdup_n_s32(0); + for (int j = 0; j < QK_K/128; j++) { + svuint8_t qhbits_1 = svld1_u8(pg8_32, qh); + qh += 32; + svuint8_t q6bits_1 = svld1_u8(pg8_32, q6); + svuint8_t q6bits_2 = svld1_u8(pg8_32, q6+32); + q6 += 64; + svint8_t q8bytes_1 = svld1_s8(pg8_32, q8); + svint8_t q8bytes_2 = svld1_s8(pg8_32, q8+32); + svint8_t q8bytes_3 = svld1_s8(pg8_32, q8+64); + svint8_t q8bytes_4 = svld1_s8(pg8_32, q8+96); + q8 += 128; + q6h_1 = svand_u8_x(pg8_32, mone, svlsl_n_u8_x(pg8_32, qhbits_1, 4)); + q6h_2 = svand_u8_x(pg8_32, mone, svlsl_n_u8_x(pg8_32, qhbits_1, 2)); + q6h_3 = svand_u8_x(pg8_32, mone, qhbits_1); + q6h_4 = svand_u8_x(pg8_32, mone, svlsr_n_u8_x(pg8_32, qhbits_1, 2)); + q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svand_u8_x(pg8_32, q6bits_1, m4b), q6h_1)); + q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svand_u8_x(pg8_32, q6bits_2, m4b), q6h_2)); + q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svlsr_n_u8_x(pg8_32, q6bits_1, 4), q6h_3)); + q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svlsr_n_u8_x(pg8_32, q6bits_2, 4), q6h_4)); + + svint8_t scale_lane_1_tmp = svld1_s8(pg8_2, scale); + scale_lane_1_tmp= svzip1_s8(scale_lane_1_tmp, scale_lane_1_tmp); + scale_lane_1_tmp= svzip1_s8(scale_lane_1_tmp, scale_lane_1_tmp); + svint8_t scale_lane_2_tmp = svld1_s8(pg8_2, scale+2); + scale_lane_2_tmp = svzip1_s8(scale_lane_2_tmp, scale_lane_2_tmp); + scale_lane_2_tmp = svzip1_s8(scale_lane_2_tmp, scale_lane_2_tmp); + svint8_t scale_lane_3_tmp = svld1_s8(pg8_2, scale+4); + scale_lane_3_tmp = svzip1_s8(scale_lane_3_tmp, scale_lane_3_tmp); + scale_lane_3_tmp = svzip1_s8(scale_lane_3_tmp, scale_lane_3_tmp); + svint8_t scale_lane_4_tmp = svld1_s8(pg8_2, scale+6); + scale_lane_4_tmp = svzip1_s8(scale_lane_4_tmp, scale_lane_4_tmp); + scale_lane_4_tmp = svzip1_s8(scale_lane_4_tmp, scale_lane_4_tmp); + svint32_t scale_lane_1 = svunpklo_s32(svunpklo_s16(scale_lane_1_tmp)); + svint32_t scale_lane_2 = svunpklo_s32(svunpklo_s16(scale_lane_2_tmp)); + svint32_t scale_lane_3 = svunpklo_s32(svunpklo_s16(scale_lane_3_tmp)); + svint32_t scale_lane_4 = svunpklo_s32(svunpklo_s16(scale_lane_4_tmp)); + + isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale_lane_1); + isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale_lane_2); + isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale_lane_3); + isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale_lane_4); + scale += 8; + } + isum += svaddv_s32(pg32_8, isum_tmp); + sum += d_all * y[i].d * (isum - 32 * isum_mins); + } + break; + default: + assert(false && "Unsupported vector length"); + break; + } + } + + *s = sum; + +#elif __ARM_NEON float sum = 0; const uint8x16_t m4b = vdupq_n_u8(0xF); From eba92d64c3f6d86de2e6b4dd3a540d2805a22b0c Mon Sep 17 00:00:00 2001 From: Prajwal B Mehendarkar Date: Tue, 18 Mar 2025 15:07:33 +0530 Subject: [PATCH 28/55] cmake : fix PowerPC build (#12241) Closes #12240 --- ggml/src/ggml-cpu/CMakeLists.txt | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index d6c4a9c299..6aa078a93e 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -287,17 +287,25 @@ function(ggml_add_cpu_backend_variant_impl tag_name) endif() endif() endif() - elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64") + elseif ("${CMAKE_SYSTEM_PROCESSOR} " STREQUAL "ppc64le " OR "${CMAKE_SYSTEM_PROCESSOR} " STREQUAL "powerpc ") message(STATUS "PowerPC detected") - execute_process(COMMAND bash -c "grep POWER /proc/cpuinfo | head -n 1" OUTPUT_VARIABLE POWER_M) - if (${POWER_M} MATCHES "POWER10") - list(APPEND ARCH_FLAGS -mcpu=power10) - elseif (${POWER_M} MATCHES "POWER9") - list(APPEND ARCH_FLAGS -mcpu=power9) + if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64") + file(READ "/proc/cpuinfo" POWER10_M) + elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "powerpc") + execute_process(COMMAND bash -c "prtconf |grep 'Implementation' | head -n 1" OUTPUT_VARIABLE POWER10_M) + endif() + + string(REGEX MATCHALL "POWER *([0-9]+)" MATCHED_STRING "${POWER10_M}") + string(REGEX REPLACE "POWER *([0-9]+)" "\\1" EXTRACTED_NUMBER "${MATCHED_STRING}") + + if (EXTRACTED_NUMBER GREATER_EQUAL 10) + list(APPEND ARCH_FLAGS -mcpu=power10 -mpowerpc64) + elseif (EXTRACTED_NUMBER EQUAL 9) + list(APPEND ARCH_FLAGS -mcpu=power9 -mpowerpc64) elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le") list(APPEND ARCH_FLAGS -mcpu=powerpc64le -mtune=native) else() - list(APPEND ARCH_FLAGS -mcpu=powerpc64 -mtune=native) + list(APPEND ARCH_FLAGS -mcpu=native -mtune=native -mpowerpc64) endif() elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64") message(STATUS "loongarch64 detected") From 810e0af3f50379682dd46b7967c4aadf3f8286f6 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 18 Mar 2025 12:05:42 +0200 Subject: [PATCH 29/55] server : fix warmup draft cache type (#12446) ggml-ci --- examples/server/server.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 71e053b202..c2f1afeca4 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1872,6 +1872,10 @@ struct server_context { params_dft.n_gpu_layers = params_base.speculative.n_gpu_layers; params_dft.n_parallel = 1; + // force F16 KV cache for the draft model for extra performance + params_dft.cache_type_k = GGML_TYPE_F16; + params_dft.cache_type_v = GGML_TYPE_F16; + llama_init_dft = common_init_from_params(params_dft); model_dft = llama_init_dft.model.get(); @@ -1892,10 +1896,6 @@ struct server_context { cparams_dft = common_context_params_to_llama(params_dft); cparams_dft.n_batch = n_ctx_dft; - // force F16 KV cache for the draft model for extra performance - cparams_dft.type_k = GGML_TYPE_F16; - cparams_dft.type_v = GGML_TYPE_F16; - // the context is not needed - we will create one for each slot llama_init_dft.context.reset(); } From 35cae5ba05a5292dc3108636a71ec59fa2f80ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= <112692748+lslusarczyk@users.noreply.github.com> Date: Tue, 18 Mar 2025 11:16:31 +0100 Subject: [PATCH 30/55] SYCL: using graphs is configurable by environment variable and compile option (#12371) * alberto changes * enable sycl graphs by env variable * fixed compilation warnings in ggml-sycl.cpp * renamed graph variables * fix markdown in docs/backend/SYCL.md Co-authored-by: Romain Biessy * fix markdown in docs/backend/SYCL.md again * compiling graphs by default, renamed graph_enable to graph_disable --------- Co-authored-by: Romain Biessy --- docs/backend/SYCL.md | 4 ++- ggml/CMakeLists.txt | 1 + ggml/src/ggml-sycl/CMakeLists.txt | 3 ++ ggml/src/ggml-sycl/common.hpp | 5 ++++ ggml/src/ggml-sycl/ggml-sycl.cpp | 47 +++++++++++++++++++++++++++++-- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 5da439e94e..184cd41955 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -660,8 +660,9 @@ use 1 SYCL GPUs: [0] with Max compute units:512 |--------------------|---------------------------------------|---------------------------------------------| | GGML_SYCL | ON (mandatory) | Enable build with SYCL code path.
FP32 path - recommended for better perforemance than FP16 on quantized model| | GGML_SYCL_TARGET | INTEL *(default)* \| NVIDIA \| AMD | Set the SYCL target device type. | -| GGML_SYCL_DEVICE_ARCH | Optional (except for AMD) | Set the SYCL device architecture, optional except for AMD. Setting the device architecture can improve the performance. See the table [--offload-arch](https://github.com/intel/llvm/blob/sycl/sycl/doc/design/OffloadDesign.md#--offload-arch) for a list of valid architectures. | +| GGML_SYCL_DEVICE_ARCH | Optional (except for AMD) | Set the SYCL device architecture, optional except for AMD. Setting the device architecture can improve the performance. See the table [--offload-arch](https://github.com/intel/llvm/blob/sycl/sycl/doc/design/OffloadDesign.md#--offload-arch) for a list of valid architectures. | | GGML_SYCL_F16 | OFF *(default)* \|ON *(optional)* | Enable FP16 build with SYCL code path. | +| GGML_SYCL_GRAPH | ON *(default)* \|OFF *(Optional)* | Enable build with [SYCL Graph extension](https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/experimental/sycl_ext_oneapi_graph.asciidoc). | | CMAKE_C_COMPILER | `icx` *(Linux)*, `icx/cl` *(Windows)* | Set `icx` compiler for SYCL code path. | | CMAKE_CXX_COMPILER | `icpx` *(Linux)*, `icx` *(Windows)* | Set `icpx/icx` compiler for SYCL code path. | @@ -671,6 +672,7 @@ use 1 SYCL GPUs: [0] with Max compute units:512 |-------------------|------------------|---------------------------------------------------------------------------------------------------------------------------| | GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG | | GGML_SYCL_DISABLE_OPT | 0 (default) or 1 | Disable optimize features based on Intel GPU type, to compare the performance increase | +| GGML_SYCL_DISABLE_GRAPH | 0 or 1 (default) | Disable running computations through SYCL Graphs feature. Disabled by default because graph performance isn't yet better than non-graph performance. | | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
Recommended to use when --split-mode = layer | diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 9a4ee4992d..740f9f69cf 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -186,6 +186,7 @@ option(GGML_OPENMP "ggml: use OpenMP" option(GGML_RPC "ggml: use RPC" OFF) option(GGML_SYCL "ggml: use SYCL" OFF) option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF) +option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON) set (GGML_SYCL_TARGET "INTEL" CACHE STRING "ggml: sycl target device") set (GGML_SYCL_DEVICE_ARCH "" CACHE STRING diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index 3ad044432a..271413ca41 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -66,6 +66,9 @@ if (WIN32) find_package(MKL REQUIRED) target_link_libraries(ggml-sycl PRIVATE IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL) else() + if (GGML_SYCL_GRAPH) + add_compile_definitions(GGML_SYCL_GRAPH) + endif() if (GGML_SYCL_TARGET STREQUAL "INTEL") target_link_libraries(ggml-sycl PRIVATE sycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread) elseif (GGML_SYCL_TARGET STREQUAL "NVIDIA") diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index fdd07d9caf..7cc5e14f9a 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -301,6 +301,7 @@ inline optimize_feature check_gpu_optimize_feature(syclex::architecture &arch) { return opt; } +namespace sycl_ex = sycl::ext::oneapi::experimental; struct ggml_backend_sycl_context { int device; std::string name; @@ -392,6 +393,10 @@ struct ggml_backend_sycl_context { return pool(device); } +#ifdef GGML_SYCL_GRAPH + std::unique_ptr> exec_graph = nullptr; +#endif + ggml_sycl_pool & host_pool(int device) { if (host_pools[device] == nullptr) { host_pools[device] = new_pool_for_host(stream(device, 0), device); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 207c0b440a..360e3f166c 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -46,6 +46,7 @@ static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; int g_ggml_sycl_disable_optimize = 0; +int g_ggml_sycl_disable_graph = 0; static ggml_sycl_device_info ggml_sycl_init() { ggml_sycl_device_info info = {}; @@ -191,10 +192,12 @@ static void ggml_check_sycl() try { if (!initialized) { g_ggml_sycl_debug = get_sycl_env("GGML_SYCL_DEBUG", 0); g_ggml_sycl_disable_optimize= get_sycl_env("GGML_SYCL_DISABLE_OPT", 0); + g_ggml_sycl_disable_graph = get_sycl_env("GGML_SYCL_DISABLE_GRAPH", 1); GGML_SYCL_DEBUG("[SYCL] call ggml_check_sycl\n"); GGML_LOG_INFO("Running with Environment Variables:\n"); GGML_LOG_INFO(" GGML_SYCL_DEBUG: %d\n", g_ggml_sycl_debug); GGML_LOG_INFO(" GGML_SYCL_DISABLE_OPT: %d\n", g_ggml_sycl_disable_optimize); + GGML_LOG_INFO(" GGML_SYCL_DISABLE_GRAPH: %d\n", g_ggml_sycl_disable_graph); GGML_LOG_INFO("Build with Macros:\n"); #if defined(GGML_SYCL_FORCE_MMQ) GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: yes\n"); @@ -3699,10 +3702,9 @@ static void optimize_graph_once(ggml_cgraph * cgraph, ggml_backend_sycl_context if (ctx->opt_feature.reorder) opt_for_reorder(cgraph->nodes[i], stream); } } -static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; - ggml_sycl_set_main_device(sycl_ctx->device); +static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { + ggml_sycl_set_main_device(sycl_ctx->device); if (!g_ggml_sycl_disable_optimize) optimize_graph_once(cgraph, sycl_ctx); for (int i = 0; i < cgraph->n_nodes; i++) { @@ -3724,7 +3726,46 @@ static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_ } GGML_ASSERT(ok); } +} +static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { + auto * sycl_ctx = static_cast(backend->context); + +#ifdef GGML_SYCL_GRAPH + if (!g_ggml_sycl_disable_graph) { + if (!sycl_ctx->exec_graph && !dpct::get_device(sycl_ctx->device).has(sycl::aspect::ext_oneapi_graph)) { + GGML_SYCL_DEBUG("[SYCL-GRAPH] can not use graphs on device:%d\n", sycl_ctx->device); + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + return GGML_STATUS_SUCCESS; + } + + sycl_ex::command_graph model_sycl_graph(*(sycl_ctx->stream())); + model_sycl_graph.begin_recording(*(sycl_ctx->stream())); + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + model_sycl_graph.end_recording(); + + if (!sycl_ctx->exec_graph) { + auto exec_graph = model_sycl_graph.finalize({sycl_ex::property::graph::updatable{}}); + sycl_ctx->exec_graph = std::make_unique< + sycl_ex::command_graph>(exec_graph); + } else { + try { + sycl_ctx->exec_graph->update(model_sycl_graph); + GGML_SYCL_DEBUG("[SYCL-GRAPH] update success\n"); + } catch (sycl::exception const & e) { + GGML_SYCL_DEBUG("[SYCL-GRAPH] Exception when updating graph, %s\n", e.what()); + auto exec_graph = model_sycl_graph.finalize({sycl_ex::property::graph::updatable{}}); + sycl_ctx->exec_graph = std::make_unique< + sycl_ex::command_graph>(exec_graph); + } + } + + sycl_ctx->stream()->ext_oneapi_graph(*(sycl_ctx->exec_graph)); + } else +#endif + { + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + } return GGML_STATUS_SUCCESS; } From 8551c44d840a7db50adb958ccaf464dc3ded82e7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 18 Mar 2025 13:05:49 +0200 Subject: [PATCH 31/55] context : always use non-causal attention for encoder graphs (#12447) * context : always use non-causal attention for encoder graphs ggml-ci * context : move the change to llama_context::encode() ggml-ci --- src/llama-context.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index abb7e526f6..42332acf1e 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1057,6 +1057,13 @@ int llama_context::encode(llama_batch & inp_batch) { ggml_backend_sched_reset(sched.get()); ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data); + const auto causal_attn_org = cparams.causal_attn; + + // always use non-causal attention for encoder graphs + // TODO: this is a tmp solution until we have a proper way to support enc-dec models + // ref: https://github.com/ggml-org/llama.cpp/pull/12181#issuecomment-2730451223 + cparams.causal_attn = false; + auto * gf = graph_init(); auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_ENCODER); @@ -1064,6 +1071,8 @@ int llama_context::encode(llama_batch & inp_batch) { res->set_inputs(&ubatch); + cparams.causal_attn = causal_attn_org; + const auto compute_status = graph_compute(gf, n_tokens > 1); switch (compute_status) { case GGML_STATUS_SUCCESS: From 99aa304fb900654ec338749f64e62895b9a88afd Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Tue, 18 Mar 2025 17:24:33 +0100 Subject: [PATCH 32/55] llama : add support for EXAONE tied word embeddings (#12451) --- src/llama-model.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index c571aa69b6..9171585bd9 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -3264,7 +3264,12 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; From c6af2161b200284d55633cf184a07406ca89908e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 18 Mar 2025 19:35:11 +0200 Subject: [PATCH 33/55] speculative : fix seg fault in certain cases (#12454) --- examples/speculative/speculative.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index bfddc67e03..627d01bbcb 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -331,11 +331,11 @@ int main(int argc, char ** argv) { } active_seqs.erase(s); - for(int i = 0; i < n_seq_dft; i++) { + for (int i = 0; i < n_seq_dft; i++) { if (i == s) { continue; } - if (drafts[i].tokens[i_dft] == drafts[s].tokens[i_dft]) { + if (drafts[i].active && drafts[i].tokens[i_dft] == drafts[s].tokens[i_dft]) { // synchronize active status for sequences with the same drafted token drafts[i].active = drafts[i].active && accept; if (!drafts[i].active) { From 29fff308c704c1c752cdb5153361e545e2bac09d Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Tue, 18 Mar 2025 19:16:19 +0100 Subject: [PATCH 34/55] llama : support converting Mistral Small text-only (#12450) --- convert_hf_to_gguf.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index d13d57c541..7a2ef4c7e3 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -1747,6 +1747,25 @@ class LlamaModel(Model): raise ValueError(f"Unprocessed experts: {experts}") +@Model.register("Mistral3ForConditionalGeneration") +class Mistral3Model(LlamaModel): + model_arch = gguf.MODEL_ARCH.LLAMA + + # we need to merge the text_config into the root level of hparams + def __init__(self, *args, **kwargs): + hparams = Model.load_hparams(kwargs["dir_model"]) + if "text_config" in hparams: + hparams = {**hparams, **hparams["text_config"]} + kwargs["hparams"] = hparams + super().__init__(*args, **kwargs) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None): + name = name.replace("language_model.", "") + if "multi_modal_projector" in name or "vision_tower" in name: + return [] + return super().modify_tensors(data_torch, name, bid) + + @Model.register("DeciLMForCausalLM") class DeciModel(Model): model_arch = gguf.MODEL_ARCH.DECI From bb115d2bf7ed2cdd7dccd7ae74cc9cfe4b0adb71 Mon Sep 17 00:00:00 2001 From: R0CKSTAR Date: Wed, 19 Mar 2025 02:28:26 +0800 Subject: [PATCH 35/55] musa: override warp_size of musa device to 32 (#12445) Signed-off-by: Xiaodong Ye --- ggml/src/ggml-cuda/ggml-cuda.cu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 8fb063822c..5cb56df9a8 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -262,6 +262,8 @@ static ggml_cuda_device_info ggml_cuda_init() { id, prop.name, prop.gcnArchName, info.devices[id].cc & 0xffff, device_vmm ? "yes" : "no", prop.warpSize); #elif defined(GGML_USE_MUSA) + // FIXME: Ensure compatibility with varying warp sizes across different MUSA archs. + info.devices[id].warp_size = 32; // TODO: refine the .cc to reflect MUSA's actual CC capabilities info.devices[id].smpbo = prop.sharedMemPerBlockOptin; info.devices[id].cc = 100*prop.major + 10*prop.minor; From 75422e8bc42646005be0754f7aa438b97a5e777e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 18 Mar 2025 21:35:19 +0200 Subject: [PATCH 36/55] graph : normalize Q, K, V shapes + sync cross attention (#12449) * graph : normalize Q, K, V shapes and add comments ggml-ci * context : synchronize before getting cross attention data * model : fix command-r attention norm check --- src/llama-context.cpp | 2 + src/llama-graph.cpp | 2 +- src/llama-graph.h | 24 +- src/llama-model.cpp | 682 ++++++++++++++++++++++++++---------------- 4 files changed, 433 insertions(+), 277 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 42332acf1e..664703a896 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1143,6 +1143,8 @@ int llama_context::encode(llama_batch & inp_batch) { if (model.arch == LLM_ARCH_T5 && t_embd) { //cross.t_embd = t_embd; + synchronize(); + cross.n_embd = t_embd->ne[0]; cross.n_enc = t_embd->ne[1]; cross.v_embd.resize(cross.n_embd*cross.n_enc); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 4e90873397..0bd4017443 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1378,7 +1378,7 @@ ggml_tensor * llm_graph_context::build_attn( // note: storing RoPE-ed version of K in the KV cache ggml_build_forward_expand(gf, ggml_cpy(ctx0, k_cur, k_cache_view)); - assert(v_cur->ne[0] == n_embd_v_gqa && v_cur->ne[1] == n_tokens); + v_cur = ggml_reshape_2d(ctx0, v_cur, n_embd_v_gqa, n_tokens); ggml_tensor * v_cache_view = nullptr; diff --git a/src/llama-graph.h b/src/llama-graph.h index c4328e6f9e..bdf19ed015 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -487,9 +487,9 @@ struct llm_graph_context { ggml_tensor * build_attn_mha( ggml_cgraph * gf, - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, + ggml_tensor * q, // [n_embd_head_q, n_tokens, n_head_q] + ggml_tensor * k, // [n_embd_head_k, n_tokens, n_head_k] + ggml_tensor * v, // [n_embd_head_v, n_tokens, n_head_v] (v_trans == false) ggml_tensor * kq_b, ggml_tensor * kq_mask, bool v_trans, @@ -502,9 +502,9 @@ struct llm_graph_context { ggml_cgraph * gf, ggml_tensor * wo, ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * k_cur, - ggml_tensor * v_cur, + ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens] + ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] + ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] ggml_tensor * kq_b, float kq_scale, int il) const; @@ -516,9 +516,9 @@ struct llm_graph_context { ggml_cgraph * gf, ggml_tensor * wo, ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * k_cur, - ggml_tensor * v_cur, + ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens] + ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] + ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] ggml_tensor * kq_b, float kq_scale, int il) const; @@ -530,9 +530,9 @@ struct llm_graph_context { ggml_cgraph * gf, ggml_tensor * wo, ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * k_cur, - ggml_tensor * v_cur, + ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens] + ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] + ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] ggml_tensor * kq_b, float kq_scale, int il) const; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 9171585bd9..d286176c1f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -4093,19 +4093,25 @@ struct llm_build_llama : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -4267,19 +4273,25 @@ struct llm_build_deci : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -4396,28 +4408,32 @@ struct llm_build_baichuan : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + switch (model.type) { case LLM_TYPE_7B: Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); break; case LLM_TYPE_13B: - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd/n_head, n_head, n_tokens); break; default: GGML_ABORT("fatal error"); } + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -4514,19 +4530,25 @@ struct llm_build_xverse : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -4632,25 +4654,26 @@ struct llm_build_falcon : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // using mode = 2 for neox mode Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -4762,19 +4785,25 @@ struct llm_build_grok : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -4907,24 +4936,26 @@ struct llm_build_dbrx : public llm_graph_context { Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -5031,12 +5062,14 @@ struct llm_build_starcoder : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -5128,11 +5161,13 @@ struct llm_build_refact : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - cb(Kcur, "Kcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -5267,21 +5302,21 @@ struct llm_build_bert : public llm_graph_context { Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Kcur, "Kcur", il); - - Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); } cb(Qcur, "Qcur", il); @@ -5397,12 +5432,14 @@ struct llm_build_bloom : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -5534,20 +5571,19 @@ struct llm_build_mpt : public llm_graph_context { model.layers[il].attn_k_norm_b, LLM_NORM, il); cb(Kcur, "Kcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - cur = build_attn(inp_attn, gf, - model.layers[il].wo, model.layers[il].bo, - Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); - } else { - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - cur = build_attn(inp_attn, gf, - model.layers[il].wo, model.layers[il].bo, - Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); } + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); } if (il == n_layer - 1) { @@ -5656,9 +5692,8 @@ struct llm_build_stablelm : public llm_graph_context { } Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cb(Qcur, "Qcur", il); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - cb(Kcur, "Kcur", il); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); if (model.layers[il].attn_q_norm) { Qcur = build_norm(Qcur, @@ -5667,6 +5702,7 @@ struct llm_build_stablelm : public llm_graph_context { LLM_NORM, il); cb(Qcur, "Qcur", il); } + if (model.layers[il].attn_k_norm) { Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, @@ -5675,20 +5711,21 @@ struct llm_build_stablelm : public llm_graph_context { cb(Kcur, "Kcur", il); } - Qcur = ggml_rope_ext( ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -5792,25 +5829,26 @@ struct llm_build_qwen : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 2*sizeof(float)*(n_embd))); - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // using mode = 2 for neox mode Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -5901,33 +5939,36 @@ struct llm_build_qwen2 : public llm_graph_context { { // compute Q and K and RoPE them ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); cb(Kcur, "Kcur", il); ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6019,35 +6060,36 @@ struct llm_build_qwen2vl : public llm_graph_context { { // compute Q and K and RoPE them ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); cb(Kcur, "Kcur", il); ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_multi( - ctx0, - ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_multi( - ctx0, - ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6136,33 +6178,36 @@ struct llm_build_qwen2moe : public llm_graph_context { { // compute Q and K and RoPE them ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); cb(Kcur, "Kcur", il); ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6307,23 +6352,27 @@ struct llm_build_phi2 : public llm_graph_context { Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); // with phi2, we scale the Q to avoid precision issues // ref: https://github.com/ml-explore/mlx-examples/blob/08e862336ade809bc37d1035f94b359e7d1a5152/phi2/phi2.py#L64-L66 Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head))); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6433,22 +6482,27 @@ struct llm_build_phi3 : public llm_graph_context { Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head))); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f, il); @@ -6564,17 +6618,25 @@ struct llm_build_plamo : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); + ext_factor, attn_factor, beta_fast, beta_slow + ); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + ext_factor, attn_factor, beta_fast, beta_slow + ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -6679,7 +6741,9 @@ struct llm_build_gpt2 : public llm_graph_context { cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6773,27 +6837,29 @@ struct llm_build_codeshell : public llm_graph_context { cur = ggml_add(ctx0, cur, model.layers[il].bqkv); cb(cur, "bqkv", il); - ggml_tensor * tmpq = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - ggml_tensor * tmpk = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - cb(tmpq, "tmpq", il); - cb(tmpk, "tmpk", il); - cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - ggml_tensor * Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); - - ggml_tensor * Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6904,19 +6970,25 @@ struct llm_build_orion : public llm_graph_context { // cb(Vcur, "Vcur", il); // } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -7025,19 +7097,25 @@ struct llm_build_internlm2 : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -7311,7 +7389,7 @@ struct llm_build_minicpm3 : public llm_graph_context { struct llm_build_gemma : public llm_graph_context { llm_build_gemma(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { - const int64_t n_embd_head_k = hparams.n_embd_head_k; + const int64_t n_embd_head = hparams.n_embd_head_v; ggml_tensor * cur; ggml_tensor * inpL; @@ -7345,20 +7423,26 @@ struct llm_build_gemma : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - - Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); - cb(Qcur, "Qcur_scaled", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head))); + cb(Qcur, "Qcur_scaled", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -7421,7 +7505,7 @@ struct llm_build_gemma : public llm_graph_context { struct llm_build_gemma2 : public llm_graph_context { llm_build_gemma2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { - const int64_t n_embd_head_k = hparams.n_embd_head_k; + const int64_t n_embd_head = hparams.n_embd_head_k; ggml_tensor * cur; ggml_tensor * inpL; @@ -7455,27 +7539,33 @@ struct llm_build_gemma2 : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); // ref: https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e switch (model.type) { case LLM_TYPE_2B: - case LLM_TYPE_9B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); break; - case LLM_TYPE_27B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd / n_head))); break; + case LLM_TYPE_9B: + case LLM_TYPE_27B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head))); break; default: GGML_ABORT("fatal error"); }; cb(Qcur, "Qcur_scaled", il); - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, Qcur, Kcur, Vcur, nullptr, 1.0f, il); @@ -7552,7 +7642,7 @@ struct llm_build_gemma2 : public llm_graph_context { struct llm_build_gemma3 : public llm_graph_context { llm_build_gemma3(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { - const int64_t n_embd_head_k = hparams.n_embd_head_k; + const int64_t n_embd_head = hparams.n_embd_head_k; ggml_tensor * cur; ggml_tensor * inpL; @@ -7593,7 +7683,10 @@ struct llm_build_gemma3 : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); @@ -7601,9 +7694,7 @@ struct llm_build_gemma3 : public llm_graph_context { ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); @@ -7611,7 +7702,10 @@ struct llm_build_gemma3 : public llm_graph_context { ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -7733,19 +7827,25 @@ struct llm_build_starcoder2 : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -8046,24 +8146,25 @@ struct llm_build_command_r : public llm_graph_context { cb(Vcur, "Vcur", il); } - if (model.layers[il].attn_q_norm) { - Qcur = ggml_view_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens, - ggml_element_size(Qcur) * n_embd_head, - ggml_element_size(Qcur) * n_embd_head * n_head, - 0); - cb(Qcur, "Qcur", il); - Kcur = ggml_view_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens, - ggml_element_size(Kcur) * n_embd_head, - ggml_element_size(Kcur) * n_embd_head * n_head_kv, - 0); - cb(Kcur, "Kcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + if (model.layers[il].attn_q_norm) { Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM, il); cb(Qcur, "Qcur", il); + } + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + if (model.layers[il].attn_k_norm) { Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, @@ -8071,19 +8172,15 @@ struct llm_build_command_r : public llm_graph_context { cb(Kcur, "Kcur", il); } - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -8198,25 +8295,28 @@ struct llm_build_cohere2 : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + if (is_swa) { - Qcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, - beta_fast, beta_slow); - cb(Qcur, "Qcur", il); + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); - Kcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur", il); - } else { - // For non-sliding layers, just reshape without applying RoPE - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cb(Qcur, "Qcur", il); - - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - cb(Kcur, "Kcur", il); + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); } + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -8328,19 +8428,25 @@ struct llm_build_olmo : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, nullptr, @@ -8442,22 +8548,25 @@ struct llm_build_olmo2 : public llm_graph_context { LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_ext( ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur_rope", il); Kcur = ggml_rope_ext( ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Kcur, "Kcur_rope", il); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -8572,22 +8681,25 @@ struct llm_build_olmoe : public llm_graph_context { LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_ext( ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur_rope", il); Kcur = ggml_rope_ext( ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Kcur, "Kcur_rope", il); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -8687,7 +8799,7 @@ struct llm_build_openelm : public llm_graph_context { cur = ggml_reshape_3d(ctx0, cur, n_embd_head_k, n_head_qkv, n_tokens); - ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, cur->nb[1], cur->nb[2], 0)); + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, cur->nb[1], cur->nb[2], 0)); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*n_head)); @@ -8707,18 +8819,19 @@ struct llm_build_openelm : public llm_graph_context { cb(Kcur, "Kcur", il); Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, NULL, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Kcur, inp_pos, NULL, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Kcur, "Kcur", il); - Vcur = ggml_reshape_2d(ctx0, Vcur, n_embd_head * n_head_kv, n_tokens); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); cb(Qcur, "Vcur", il); cur = build_attn(inp_attn, gf, @@ -8815,24 +8928,26 @@ struct llm_build_gptneox : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -8963,19 +9078,25 @@ struct llm_build_arctic : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -9112,19 +9233,25 @@ struct llm_build_deepseek : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -9502,19 +9629,25 @@ struct llm_build_bitnet : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, NULL, NULL, @@ -9906,7 +10039,9 @@ struct llm_build_jais : public llm_graph_context { cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -10019,29 +10154,30 @@ struct llm_build_chatglm : public llm_graph_context { Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + //printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor); + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - //printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor); - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur_rope", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur_rope", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); - } if (il == n_layer - 1) { @@ -10145,19 +10281,25 @@ struct llm_build_nemotron : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -10270,19 +10412,25 @@ struct llm_build_exaone : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -11166,19 +11314,25 @@ struct llm_build_chameleon : public llm_graph_context { cb(Kcur, "Kcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, nullptr, From d84635b1b085d54d6a21924e6171688d6e3dfb46 Mon Sep 17 00:00:00 2001 From: lhez Date: Tue, 18 Mar 2025 12:54:55 -0700 Subject: [PATCH 37/55] opencl: improve profiling (#12442) * opencl: more profiling timing * opencl: generate trace for profiling * opencl: reduce profiling overhead * Populate profiling timing info at the end rather than after each kernel run * opencl: fix for chrome tracing --- ggml/src/ggml-opencl/ggml-opencl.cpp | 115 ++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 21 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 14d9934fb1..efaf7f4790 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -297,8 +297,27 @@ static int ggml_backend_opencl_n_devices = 0; struct ProfilingInfo { std::string op_name; std::string kernel_name; - // Kernel execution time in nanoseconds. - cl_ulong duration_ns; + + cl_kernel kernel; + cl_event evt; + + cl_ulong cmd_queued; + cl_ulong cmd_submit; + cl_ulong cmd_start; + cl_ulong cmd_end; + cl_ulong overhead_start; + cl_ulong overhead_end; + // For the times below, see spec for clGetEventProfilingInfo + // The time kernel spent in cmd queue - SUBMIT - QUEUED + cl_ulong cmd_queued_duration_ns; + // The time kernel spent for submission - START - SUBMIT + cl_ulong cmd_submit_duration_ns; + // Kernel execution time in nanoseconds - END - START + cl_ulong cmd_duration_ns; + // The time for the kernel to complete - COMPLETE - END + cl_ulong cmd_complete_duration_ns; + // Total time to finish the kernel - COMPELTE - QUEUED + cl_ulong cmd_total_duration_ns; // Global and local work sizes. size_t global_size[3]; size_t local_size[3]; @@ -903,12 +922,56 @@ static void ggml_cl2_free(void) { return; } + // Populate profiling info + for (ProfilingInfo & info : g_profiling_info) { + cl_ulong cmd_queued; + cl_ulong cmd_submit; + cl_ulong cmd_start; + cl_ulong cmd_end; + cl_ulong cmd_complete; + + CL_CHECK(clWaitForEvents(1, &info.evt)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_QUEUED, sizeof(cl_ulong), &cmd_queued, NULL)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_SUBMIT, sizeof(cl_ulong), &cmd_submit, NULL)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_START, sizeof(cl_ulong), &cmd_start, NULL)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &cmd_end, NULL)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_COMPLETE, sizeof(cl_ulong), &cmd_complete, NULL)); + CL_CHECK(clReleaseEvent(info.evt)); + + char kernel_name[512]; + CL_CHECK(clGetKernelInfo(info.kernel, CL_KERNEL_FUNCTION_NAME, + sizeof(kernel_name), kernel_name, NULL)); + info.kernel_name = kernel_name; + + info.cmd_queued = cmd_queued; + info.cmd_submit = cmd_submit; + info.cmd_start = cmd_start; + info.cmd_end = cmd_end; + + info.cmd_queued_duration_ns = cmd_submit - cmd_queued; + info.cmd_submit_duration_ns = cmd_start - cmd_submit; + info.cmd_duration_ns = cmd_end - cmd_start; + info.cmd_complete_duration_ns = cmd_complete - cmd_end; + info.cmd_total_duration_ns = cmd_complete - cmd_queued; + } + + // Dump a csv float total_kernel_time = 0; - fprintf(fperf, "op name, kernel name, duration (ms), global size, local size, output size\n"); + fprintf(fperf, "op name, kernel name, queued duration (ms), submit duration(ms), exec duration (ms), complete duration (ms), total duration (ms), global size, local size, output size\n"); for (const ProfilingInfo & info : g_profiling_info) { - total_kernel_time += info.duration_ns/1.e6f; - fprintf(fperf, "%s,%s,%f,%zux%zux%zu,%zux%zux%zu,%zux%zux%zux%zu\n", - info.op_name.c_str(), info.kernel_name.c_str(), info.duration_ns/1.e6f, + total_kernel_time += info.cmd_duration_ns/1.e6f; + fprintf(fperf, "%s,%s,%f,%f,%f,%f,%f,%zux%zux%zu,%zux%zux%zu,%zux%zux%zux%zu\n", + info.op_name.c_str(), info.kernel_name.c_str(), + info.cmd_queued_duration_ns/1.e6f, + info.cmd_submit_duration_ns/1.e6f, + info.cmd_duration_ns/1.e6f, + info.cmd_complete_duration_ns/1.e6f, + info.cmd_total_duration_ns/1.e6f, info.global_size[0], info.global_size[1], info.global_size[2], info.local_size[0], info.local_size[2], info.local_size[2], info.output_size[0], info.output_size[1], info.output_size[2], info.output_size[3]); @@ -916,6 +979,27 @@ static void ggml_cl2_free(void) { fclose(fperf); GGML_LOG_INFO("ggml_opencl: total kernel time: %f\n", total_kernel_time); + + // Dump a simple chrome trace + FILE* ftrace = fopen("cl_trace.json", "w"); + if (!ftrace) { + GGML_LOG_ERROR("Failed to open cl_trace.json\n"); + return; + } + + fprintf(ftrace, "[\n"); + for (const ProfilingInfo & info : g_profiling_info) { + fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"B\", \"ts\": %lu, \"pid\": \"\", \"tid\": \"Host\"},\n", + info.kernel_name.c_str(), info.cmd_queued/1000); + fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"E\", \"ts\": %lu, \"pid\": \"\", \"tid\": \"Host\"},\n", + info.kernel_name.c_str(), info.cmd_submit/1000); + + fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"B\", \"ts\": %lu, \"pid\": \"\", \"tid\": \"Device\"},\n", + info.kernel_name.c_str(), info.cmd_start/1000); + fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"E\", \"ts\": %lu, \"pid\": \"\", \"tid\": \"Device\"},\n", + info.kernel_name.c_str(), info.cmd_end/1000); + } + fclose(ftrace); #endif } @@ -2062,25 +2146,14 @@ static void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tenso // Profiling utility //------------------------------------------------------------------------------ #ifdef GGML_OPENCL_PROFILING -void populateProfilingInfo( +static void populateProfilingInfo( ProfilingInfo& info, cl_event evt, cl_kernel kernel, size_t global_size[3], size_t local_size[3], const ggml_tensor * tensor) { - cl_ulong start; - cl_ulong end; - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clGetEventProfilingInfo( - evt, CL_PROFILING_COMMAND_START, sizeof(cl_ulong), &start, NULL)); - CL_CHECK(clGetEventProfilingInfo( - evt, CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &end, NULL)); + info.op_name = tensor->name; + info.kernel = kernel; + info.evt = evt; - char kernel_name[512]; - CL_CHECK(clGetKernelInfo(kernel, CL_KERNEL_FUNCTION_NAME, - sizeof(kernel_name), kernel_name, NULL)); - - info.duration_ns = end - start; - info.op_name = tensor->name; - info.kernel_name = kernel_name; info.local_size[0] = local_size[0]; info.local_size[1] = local_size[1]; info.local_size[2] = local_size[2]; From c446b2edd2a9fe2772a1a18923c3e54a6749c364 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Wed, 19 Mar 2025 02:26:26 -0500 Subject: [PATCH 38/55] vulkan: Submit once enough matmul work has been recorded (#12406) I've been seeing significantly worse performance for tg with flash attention enabled vs disabled, and it seems to be related to the submit heuristic. Change the heuristic to check how many bytes worth of weight matrix are used and flush every 100MB, and ramp up after the first few submits. This seems to resolve the issue, and also increases perf for non-FA a bit. --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 32 ++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index dd680aa522..d450fe9a2f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -8436,8 +8436,12 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)"); ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + uint64_t total_mat_mul_bytes = 0; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_vk_build_graph(ctx, cgraph->nodes[i], i, nullptr, 0, true, false, false); + if (cgraph->nodes[i]->op == GGML_OP_MUL_MAT || cgraph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { + total_mat_mul_bytes += ggml_nbytes(cgraph->nodes[i]->src[0]); + } } if (ctx->device->need_compiles) { ggml_vk_load_shaders(ctx->device); @@ -8458,17 +8462,27 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg bool first_node_in_batch = true; // true if next node will be first node in a batch int submit_node_idx = 0; // index to first node in a batch - // Submit work every nodes_per_submit nodes to overlap CPU cmdbuffer generation with GPU execution. - // Start with a smaller count to get work submitted right away, and increase it after each submit. - int nodes_per_submit = 20; + // Submit after enough work has accumulated, to overlap CPU cmdbuffer generation with GPU execution. + // Estimate the amount of matmul work by looking at the weight matrix size, and submit every 100MB + // (and scaled down based on model size, so smaller models submit earlier). + // Also submit at least every 100 nodes, in case there are workloads without as much matmul. + int nodes_per_submit = 100; int submitted_nodes = 0; int submit_count = 0; + uint64_t mul_mat_bytes = 0; + uint64_t mul_mat_bytes_per_submit = std::min(uint64_t(100*1000*1000), total_mat_mul_bytes / 40u); for (int i = 0; i < cgraph->n_nodes; i++) { if (first_node_in_batch) { submit_node_idx = i; } - bool submit = (submitted_nodes >= nodes_per_submit) || (i == last_node); + if (cgraph->nodes[i]->op == GGML_OP_MUL_MAT || cgraph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { + mul_mat_bytes += ggml_nbytes(cgraph->nodes[i]->src[0]); + } + + bool submit = (submitted_nodes >= nodes_per_submit) || + (mul_mat_bytes >= mul_mat_bytes_per_submit) || + (i == last_node); bool enqueued = ggml_vk_build_graph(ctx, cgraph->nodes[i], i, cgraph->nodes[submit_node_idx], submit_node_idx, false, i == last_node, submit); @@ -8485,13 +8499,9 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg if (submit) { first_node_in_batch = true; submitted_nodes = 0; - switch (submit_count) { - case 0: - nodes_per_submit = 50; - break; - default: - nodes_per_submit = 100; - break; + mul_mat_bytes = 0; + if (submit_count < 3) { + mul_mat_bytes_per_submit *= 2; } submit_count++; } From a686171ea71ed8cb8a324850d146cb65a001e141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 19 Mar 2025 08:58:13 +0100 Subject: [PATCH 39/55] convert : Support chat_template.json (#12460) --- gguf-py/gguf/vocab.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index 2ef7d14ab1..cca0979862 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -154,7 +154,12 @@ class SpecialVocab: return True with open(tokenizer_config_file, encoding = 'utf-8') as f: tokenizer_config = json.load(f) - chat_template = tokenizer_config.get('chat_template') + chat_template_alt = None + chat_template_file = path / 'chat_template.json' + if chat_template_file.is_file(): + with open(chat_template_file, encoding = 'utf-8') as f: + chat_template_alt = json.load(f).get('chat_template') + chat_template = tokenizer_config.get('chat_template', chat_template_alt) if chat_template is None or isinstance(chat_template, (str, list)): self.chat_template = chat_template else: From 108e53c2f1b57b79c8bcf9773fba5a82e2f1eeb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 19 Mar 2025 09:08:49 +0100 Subject: [PATCH 40/55] llama : add support for GPT2, Bloom and CodeShell tied word embeddings (#12456) * Add support for GPT2, Bloom and CodeShell tied word embeddings * Deduplicate tied word embeddings weights * Workaround for incorrect weight map It appears transformer.wte.weight is in the weight map even though the weights are not there, remove it if output weights are encountered first. * check++ * fatfingers-- --- convert_hf_to_gguf.py | 37 ++++++++++++++++--------------------- src/llama-model.cpp | 21 ++++++++++++++++++--- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 7a2ef4c7e3..7574218e24 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -180,7 +180,8 @@ class Model: extra = sorted(tensor_names_from_parts.difference(self.tensor_names)) missing_files = sorted(set(weight_map[n] for n in missing if n in weight_map)) if len(extra) == 0 and len(missing_files) > 0: - raise ValueError(f"Missing or incomplete model files: {missing_files}") + raise ValueError(f"Missing or incomplete model files: {missing_files}\n" + f"Missing tensors: {missing}") else: raise ValueError("Mismatch between weight map and model parts for tensor names:\n" f"Missing tensors: {missing}\n" @@ -1099,13 +1100,6 @@ class BloomModel(Model): tensors.append((self.map_tensor_name(name), data_torch)) - if name == "word_embeddings.weight": - assert self.tensor_names is not None - - # TODO: tie them at runtime, don't duplicate in the model file - if all(s not in self.tensor_names for s in ("lm_head.weight", "output.weight")): - tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch)) - return tensors @@ -2423,10 +2417,6 @@ class GPT2Model(Model): tensors.append((new_name, data_torch)) - # note: GPT2 output is tied to (same as) wte in original model - if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD): - tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch)) - return tensors @@ -2756,21 +2746,26 @@ class CodeShellModel(Model): self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR) self.gguf_writer.add_rope_scaling_factor(1.0) + _has_tok_embd = False + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: del bid # unused + output_name = self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT) + tok_embd_name = self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD) + new_name = self.map_tensor_name(name) - tensors: list[tuple[str, Tensor]] = [(new_name, data_torch)] + # assuming token_embd.weight is seen before output.weight + if not self._has_tok_embd and new_name == self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT): + # even though the tensor file(s) does not contain the word embeddings they are still in the weight map + if self.tensor_names and "transformer.wte.weight" in self.tensor_names: + logger.debug(f"{tok_embd_name} not found before {output_name}, assuming they are tied") + self.tensor_names.remove("transformer.wte.weight") + elif new_name == tok_embd_name: + self._has_tok_embd = True - if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD): - assert self.tensor_names is not None - - if all(s not in self.tensor_names for s in ("lm_head.weight", "output.weight")): - # copy tok_embd.weight to output.weight - tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch)) - - return tensors + return [(new_name, data_torch)] @Model.register("InternLM2ForCausalLM") diff --git a/src/llama-model.cpp b/src/llama-model.cpp index d286176c1f..17af8cc30b 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2020,7 +2020,12 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; @@ -2381,7 +2386,12 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; @@ -2407,7 +2417,12 @@ bool llama_model::load_tensors(llama_model_loader & ml) { } break; case LLM_ARCH_CODESHELL: { - tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if tok embd is NULL, init from output + if (tok_embd == NULL) { + tok_embd = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); From 0fd8487b142b2b92565bc95b39ddc440955a237c Mon Sep 17 00:00:00 2001 From: Guus Waals <_@guusw.nl> Date: Wed, 19 Mar 2025 10:15:23 +0000 Subject: [PATCH 41/55] Fix visionOS build and add CI (#12415) * ci: add visionOS build workflow Add a new GitHub Actions workflow for building on visionOS with CMake and Xcode. * ggml: Define _DARWIN_C_SOURCE for visionOS to fix missing u_xxx typedefs * ci: remove define hacks for u_xxx system types --------- Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com> --- .github/workflows/build.yml | 29 +++++++++++++++++++++++++++++ build-xcframework.sh | 8 ++++---- ggml/src/CMakeLists.txt | 4 ++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03cde0a484..7db8552865 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -676,6 +676,35 @@ jobs: -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO + macOS-latest-cmake-visionos: + runs-on: macos-latest + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + + - name: Dependencies + id: depends + continue-on-error: true + run: | + brew update + + - name: Build + id: cmake_build + run: | + sysctl -a + cmake -B build -G Xcode \ + -DGGML_METAL_USE_BF16=ON \ + -DGGML_METAL_EMBED_LIBRARY=ON \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DCMAKE_SYSTEM_NAME=visionOS \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \ + -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO + macOS-latest-swift: runs-on: macos-latest diff --git a/build-xcframework.sh b/build-xcframework.sh index 37833dc4ea..2ce3939c43 100755 --- a/build-xcframework.sh +++ b/build-xcframework.sh @@ -432,8 +432,8 @@ cmake -B build-visionos -G Xcode \ -DCMAKE_SYSTEM_NAME=visionOS \ -DCMAKE_OSX_SYSROOT=xros \ -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \ - -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 -Du_int=unsigned\ int -Du_char=unsigned\ char -Du_short=unsigned\ short ${COMMON_C_FLAGS}" \ - -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 -Du_int=unsigned\ int -Du_char=unsigned\ char -Du_short=unsigned\ short ${COMMON_CXX_FLAGS}" \ + -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}" \ + -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}" \ -S . cmake --build build-visionos --config Release -- -quiet @@ -445,8 +445,8 @@ cmake -B build-visionos-sim -G Xcode \ -DCMAKE_SYSTEM_NAME=visionOS \ -DCMAKE_OSX_SYSROOT=xrsimulator \ -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \ - -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 -Du_int=unsigned\ int -Du_char=unsigned\ char -Du_short=unsigned\ short ${COMMON_C_FLAGS}" \ - -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 -Du_int=unsigned\ int -Du_char=unsigned\ char -Du_short=unsigned\ short ${COMMON_CXX_FLAGS}" \ + -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}" \ + -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}" \ -S . cmake --build build-visionos-sim --config Release -- -quiet diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index a797e2b187..c1c7498694 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -325,6 +325,10 @@ if (CMAKE_SYSTEM_NAME MATCHES "Android") target_link_libraries(ggml-base PRIVATE dl) endif() +if(CMAKE_SYSTEM_NAME MATCHES "visionOS") + target_compile_definitions(ggml-base PUBLIC _DARWIN_C_SOURCE) +endif() + if (BUILD_SHARED_LIBS) foreach (target ggml-base ggml) set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON) From a9b59288e222f39fc0311dc66944ed5a86c815fa Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Wed, 19 Mar 2025 13:56:23 -0500 Subject: [PATCH 42/55] vulkan: optimize iq1 coopmat2 dequant functions (#12427) --- .../vulkan-shaders/dequant_funcs_cm2.comp | 18 ++++++++++++------ ggml/src/ggml-vulkan/vulkan-shaders/types.comp | 7 +++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp index 8efe4653ff..b3fad35e21 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp @@ -311,8 +311,8 @@ float16_t dequantFuncIQ1_S(const in decodeBufIQ1_S bl, const in uint blockCoords const float16_t d = bl.block.d; const uint idx = coordInBlock[1]; - const uint ib32 = idx / 32; - const uint ib8 = idx / 8; + const uint ib32 = (idx & 0xE0) >> 5; + const uint ib8 = (idx & 0xF8) >> 3; const uint qh = bl.block.qh[ib32]; const uint qs = bl.block.qs[ib8]; @@ -330,14 +330,20 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1 block_iq1_m block; }; +layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufIQ1_M_packed64 { + block_iq1_m_packed64 block; +}; + float16_t dequantFuncIQ1_M(const in decodeBufIQ1_M bl, const in uint blockCoords[2], const in uint coordInBlock[2]) { - const u16vec4 scales = u16vec4(bl.block.scales[0], bl.block.scales[1], bl.block.scales[2], bl.block.scales[3]) >> 12; - const float16_t d = uint16BitsToHalf(scales.x | (scales.y << 4) | (scales.z << 8) | (scales.w << 12)); + decodeBufIQ1_M_packed64 bl64 = decodeBufIQ1_M_packed64(bl); const uint idx = coordInBlock[1]; - const uint ib8 = idx / 8; - const uint ib16 = idx / 16; + uvec2 scales = unpack32(bl64.block.scales); + const float16_t d = uint16BitsToHalf(uint16_t(((scales.x & 0xF000) >> 12) | ((scales.x & 0xF0000000) >> 24) | ((scales.y & 0xF000) >> 4) | ((scales.y & 0xF0000000) >> 16))); + + const uint ib8 = (idx & 0xF8) >> 3; + const uint ib16 = (idx & 0xF0) >> 4; const int i8 = int(idx % 8); const uint sc = bl.block.scales[ib8 / 8]; const uint qs = bl.block.qs[ib8]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp index f01179326e..789776816b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp @@ -2,6 +2,7 @@ #if !defined(GGML_TYPES_COMP) #define GGML_TYPES_COMP +#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #extension GL_EXT_shader_explicit_arithmetic_types_int8 : require @@ -312,6 +313,12 @@ struct block_iq1_m { uint16_t scales[QUANT_K_IQ1_M/64]; }; +struct block_iq1_m_packed64 { + uint64_t qs[QUANT_K_IQ1_M/8/8]; + uint64_t qh[QUANT_K_IQ1_M/16/8]; + uint64_t scales; +}; + #if defined(DATA_A_IQ1_S) #define QUANT_K QUANT_K_IQ1_S #define QUANT_R QUANT_R_IQ1_S From 517b5ddbf002b91fd6d6daf5d8db8c88a0173039 Mon Sep 17 00:00:00 2001 From: Gaurav Garg <52341457+gaugarg-nv@users.noreply.github.com> Date: Thu, 20 Mar 2025 01:22:06 +0530 Subject: [PATCH 43/55] CUDA: Improve flash decoding kernel GPU occupancy for BS=1 case (#12183) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Find out active blocks per SM using cudaOccupancyMaxActiveBlocksPerMultiprocessor API. Use this value to determine the optimal parallel_blocks value. - Prefer vector flash attention kernels over MMA kernel for BS=1 Fixes Issue: #12182 --------- Co-authored-by: Johannes Gäßler --- ggml/src/ggml-cuda/fattn-common.cuh | 88 +++++++++++++++++++--------- ggml/src/ggml-cuda/fattn-mma-f16.cuh | 3 +- ggml/src/ggml-cuda/fattn-tile-f16.cu | 63 ++++++++------------ ggml/src/ggml-cuda/fattn-tile-f32.cu | 63 ++++++++------------ ggml/src/ggml-cuda/fattn-vec-f16.cuh | 73 +++++++++-------------- ggml/src/ggml-cuda/fattn-vec-f32.cuh | 73 +++++++++-------------- ggml/src/ggml-cuda/fattn-wmma-f16.cu | 65 +++++--------------- ggml/src/ggml-cuda/fattn.cu | 18 +++--- ggml/src/ggml-cuda/ggml-cuda.cu | 3 + ggml/src/ggml-cuda/vendors/hip.h | 1 + ggml/src/ggml-cuda/vendors/musa.h | 1 + tests/test-backend-ops.cpp | 29 ++++++--- 12 files changed, 214 insertions(+), 266 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 4067fd41bc..1c2a2a138f 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -606,48 +606,47 @@ static __global__ void flash_attn_stream_k_fixup( *dst = dst_val / rowsum; } -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(D, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) static __global__ void flash_attn_combine_results( const float * __restrict__ VKQ_parts, const float2 * __restrict__ VKQ_meta, - float * __restrict__ dst) { - VKQ_parts += parallel_blocks*D * gridDim.y*blockIdx.x; - VKQ_meta += parallel_blocks * gridDim.y*blockIdx.x; - dst += D * gridDim.y*blockIdx.x; + float * __restrict__ dst, + const int parallel_blocks) { + VKQ_parts += parallel_blocks*D * gridDim.z*blockIdx.x; + VKQ_meta += parallel_blocks * gridDim.z*blockIdx.x; + dst += D * gridDim.z*blockIdx.x; const int tid = threadIdx.x; __builtin_assume(tid < D); - __shared__ float2 meta[parallel_blocks]; + extern __shared__ float2 meta[]; if (tid < 2*parallel_blocks) { - ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.y*(2*parallel_blocks) + tid]; + ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.z*(2*parallel_blocks) + tid]; } __syncthreads(); float kqmax = meta[0].x; -#pragma unroll for (int l = 1; l < parallel_blocks; ++l) { kqmax = max(kqmax, meta[l].x); } float VKQ_numerator = 0.0f; float VKQ_denominator = 0.0f; -#pragma unroll for (int l = 0; l < parallel_blocks; ++l) { const float diff = meta[l].x - kqmax; const float KQ_max_scale = expf(diff); const uint32_t ftz_mask = 0xFFFFFFFF * (diff > SOFTMAX_FTZ_THRESHOLD); *((uint32_t *) &KQ_max_scale) &= ftz_mask; - VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.y*D + blockIdx.y*D + tid]; + VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.z*D + blockIdx.z*D + tid]; VKQ_denominator += KQ_max_scale * meta[l].y; } - dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator; + dst[blockIdx.z*D + tid] = VKQ_numerator / VKQ_denominator; } static void on_no_fattn_vec_case(const int D) { @@ -671,12 +670,10 @@ static void on_no_fattn_vec_case(const int D) { } } -// parallel_blocks == 0 is stream-k decomposition -template +template void launch_fattn( - ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, - const int nwarps, const size_t nbytes_shared, const bool need_f16_K, const bool need_f16_V, - const int warp_size = WARP_SIZE + ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared, + const int KQ_row_granularity, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const int warp_size = WARP_SIZE ) { constexpr int ncols = ncols1 * ncols2; @@ -748,12 +745,14 @@ void launch_fattn( nb23 = nb23*bs*sizeof(half)/ts; } + int parallel_blocks = 1; + const int ntiles_x = ((Q->ne[1] + ncols1 - 1) / ncols1); const int ntiles_total = ntiles_x * (Q->ne[2] / ncols2) * Q->ne[3]; const dim3 block_dim(warp_size, nwarps, 1); dim3 blocks_num; - if (parallel_blocks == 0) { + if (stream_k) { // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup. const int max_blocks = 2*nsm; const int tiles_nwaves = (ntiles_total + max_blocks - 1) / max_blocks; @@ -769,9 +768,43 @@ void launch_fattn( dst_tmp_meta.alloc(blocks_num.x*ncols * (2*2 + D) * sizeof(float)); } else { - blocks_num.x = parallel_blocks*ntiles_x; - blocks_num.y = Q->ne[2]; - blocks_num.z = Q->ne[3]; + GGML_ASSERT(K->ne[1] % KQ_row_granularity == 0); + const int ntiles_KQ = K->ne[1] / KQ_row_granularity; // Max. number of parallel blocks limited by tensor size. + + int max_blocks_per_sm = 1; // Max. number of active blocks limited by occupancy. + CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, fattn_kernel, block_dim.x * block_dim.y * block_dim.z, nbytes_shared)); + + // parallel_blocks should be at least large enough to achieve max. occupancy for a single wave: + parallel_blocks = std::max((nsm * max_blocks_per_sm) / ntiles_total, 1); + + // parallel_blocks must not be larger than what the tensor size allows: + parallel_blocks = std::min(parallel_blocks, ntiles_KQ); + + // If ntiles_total % blocks_per_wave != 0 then some efficiency is lost due to tail effects. + // Test whether parallel_blocks can be set to a higher value for better efficiency. + const int blocks_per_wave = nsm * max_blocks_per_sm; + int nwaves_best = 0; + int efficiency_percent_best = 0; + for (int parallel_blocks_test = parallel_blocks; parallel_blocks_test <= ntiles_KQ; ++parallel_blocks_test) { + const int nblocks_total = ntiles_total * parallel_blocks_test; + const int nwaves = (nblocks_total + blocks_per_wave - 1) / blocks_per_wave; + const int efficiency_percent = 100 * nblocks_total / (nwaves*blocks_per_wave); + + // Stop trying configurations with more waves if we already have good efficiency to avoid excessive overhead. + if (efficiency_percent_best >= 90 && nwaves > nwaves_best) { + break; + } + + if (efficiency_percent > efficiency_percent_best) { + nwaves_best = nwaves; + efficiency_percent_best = efficiency_percent; + parallel_blocks = parallel_blocks_test; + } + } + + blocks_num.x = ntiles_x; + blocks_num.y = parallel_blocks; + blocks_num.z = Q->ne[2]*Q->ne[3]; if (parallel_blocks > 1) { dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); @@ -803,7 +836,7 @@ void launch_fattn( K_data, V_data, mask ? ((const char *) mask->data) : nullptr, - (parallel_blocks) > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, + !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], K->ne[0], K->ne[1], K->ne[2], K->ne[3], @@ -815,7 +848,7 @@ void launch_fattn( ); CUDA_CHECK(cudaGetLastError()); - if constexpr (parallel_blocks == 0) { + if (stream_k) { if (ntiles_total % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles. const dim3 block_dim_combine(D, 1, 1); const dim3 blocks_num_combine = {blocks_num.x, ncols1, ncols2}; @@ -824,13 +857,14 @@ void launch_fattn( <<>> ((float *) KQV->data, dst_tmp_meta.ptr, Q->ne[1], Q->ne[2], K->ne[1]); } - } else if constexpr (parallel_blocks > 1) { + } else if (parallel_blocks > 1) { const dim3 block_dim_combine(D, 1, 1); - const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); + const dim3 blocks_num_combine(Q->ne[1], 1, blocks_num.z); + const size_t nbytes_shared_combine = parallel_blocks*sizeof(float2); - flash_attn_combine_results - <<>> - (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); + flash_attn_combine_results + <<>> + (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data, parallel_blocks); } CUDA_CHECK(cudaGetLastError()); } diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 718ee5402d..024032f622 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -970,7 +970,8 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml fattn_kernel = flash_attn_ext_f16; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, true, true); + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, FATTN_KQ_STRIDE, true, true, true); } diff --git a/ggml/src/ggml-cuda/fattn-tile-f16.cu b/ggml/src/ggml-cuda/fattn-tile-f16.cu index ef3569fab2..77455d8e4f 100644 --- a/ggml/src/ggml-cuda/fattn-tile-f16.cu +++ b/ggml/src/ggml-cuda/fattn-tile-f16.cu @@ -4,7 +4,7 @@ #define FATTN_KQ_STRIDE_TILE_F16 64 -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(nwarps*WARP_SIZE, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) @@ -58,18 +58,17 @@ static __global__ void flash_attn_tile_ext_f16( //In this kernel Q, K, V are matrices while i, j, k are matrix indices. - const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on. const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0); - const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio)); - const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.z + nb01*ic0); + const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.z / gqa_ratio)); + const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.z / gqa_ratio)); // K and V have same shape const half * maskh = (const half *) mask + ne11*ic0; const int stride_KV2 = nb11 / sizeof(half2); - const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slopef = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); const half slopeh = __float2half(slopef); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); @@ -105,8 +104,7 @@ static __global__ void flash_attn_tile_ext_f16( __syncthreads(); - const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F16; - for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F16) { + for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE_TILE_F16; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE_TILE_F16) { // Calculate KQ tile and keep track of new maximum KQ values: half kqmax_new[ncols/nwarps]; @@ -271,16 +269,16 @@ static __global__ void flash_attn_tile_ext_f16( const int i0 = i00 + 2*threadIdx.x; half2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)]; - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val /= __half2half2(kqsum_j); } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; - dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = __low2float(dst_val); - dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = __high2float(dst_val); + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; + dst[j_dst*D*gridDim.z + D*blockIdx.z + i0 + 0] = __low2float(dst_val); + dst[j_dst*D*gridDim.z + D*blockIdx.z + i0 + 1] = __high2float(dst_val); } - if (parallel_blocks != 1 && threadIdx.x == 0) { - dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); + if (gridDim.y != 1 && threadIdx.x == 0) { + dst_meta[((ic0 + j_VKQ)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); } } #else @@ -288,7 +286,7 @@ static __global__ void flash_attn_tile_ext_f16( #endif // defined(FLASH_ATTN_AVAILABLE) && defined(FP16_AVAILABLE) } -template +template void launch_fattn_tile_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * Q = dst->src[0]; switch (Q->ne[0]) { @@ -296,15 +294,17 @@ void launch_fattn_tile_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * constexpr int D = 64; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; - fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, FATTN_KQ_STRIDE_TILE_F16, true, true, false); } break; case 128: { constexpr int D = 128; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; - fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, FATTN_KQ_STRIDE_TILE_F16, true, true, false); } break; default: { GGML_ABORT("FlashAttention without tensor cores only supports head sizes 64 and 128."); @@ -324,37 +324,22 @@ void ggml_cuda_flash_attn_ext_tile_f16(ggml_backend_cuda_context & ctx, ggml_ten if (Q->ne[1] <= 16) { constexpr int cols_per_block = 16; - constexpr int parallel_blocks = 4; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - launch_fattn_tile_f16_64_128(ctx, dst); + launch_fattn_tile_f16_64_128(ctx, dst); } else { constexpr bool use_logit_softcap = true; - launch_fattn_tile_f16_64_128(ctx, dst); - } - return; - } - - if (Q->ne[1] <= 32) { - constexpr int cols_per_block = 32; - constexpr int parallel_blocks = 4; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - launch_fattn_tile_f16_64_128(ctx, dst); - } else { - constexpr bool use_logit_softcap = true; - launch_fattn_tile_f16_64_128(ctx, dst); + launch_fattn_tile_f16_64_128(ctx, dst); } return; } constexpr int cols_per_block = 32; - constexpr int parallel_blocks = 1; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - launch_fattn_tile_f16_64_128(ctx, dst); + launch_fattn_tile_f16_64_128(ctx, dst); } else { constexpr bool use_logit_softcap = true; - launch_fattn_tile_f16_64_128(ctx, dst); + launch_fattn_tile_f16_64_128(ctx, dst); } } diff --git a/ggml/src/ggml-cuda/fattn-tile-f32.cu b/ggml/src/ggml-cuda/fattn-tile-f32.cu index 04b69c83be..85fea4404d 100644 --- a/ggml/src/ggml-cuda/fattn-tile-f32.cu +++ b/ggml/src/ggml-cuda/fattn-tile-f32.cu @@ -4,7 +4,7 @@ #define FATTN_KQ_STRIDE_TILE_F32 32 -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(nwarps*WARP_SIZE, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) @@ -58,18 +58,17 @@ static __global__ void flash_attn_tile_ext_f32( // In this kernel Q, K, V are matrices while i, j, k are matrix indices. - const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on. const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0); - const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio)); - const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.z + nb01*ic0); + const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.z / gqa_ratio)); + const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.z / gqa_ratio)); // K and V have same shape const half * maskh = (const half *) mask + ne11*ic0; const int stride_KV2 = nb11 / sizeof(half2); - const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slope = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); @@ -103,8 +102,7 @@ static __global__ void flash_attn_tile_ext_f32( __syncthreads(); - const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F32; - for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F32) { + for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE_TILE_F32; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE_TILE_F32) { // Calculate KQ tile and keep track of new maximum KQ values: float kqmax_new[ncols/nwarps]; @@ -269,17 +267,17 @@ static __global__ void flash_attn_tile_ext_f32( const int i0 = i00 + 2*threadIdx.x; float2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)]; - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val.x /= kqsum_j; dst_val.y /= kqsum_j; } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; - dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = dst_val.x; - dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = dst_val.y; + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; + dst[j_dst*D*gridDim.z + D*blockIdx.z + i0 + 0] = dst_val.x; + dst[j_dst*D*gridDim.z + D*blockIdx.z + i0 + 1] = dst_val.y; } - if (parallel_blocks != 1 && threadIdx.x == 0) { - dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); + if (gridDim.y != 1 && threadIdx.x == 0) { + dst_meta[((ic0 + j_VKQ)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); } } #else @@ -287,7 +285,7 @@ static __global__ void flash_attn_tile_ext_f32( #endif // FLASH_ATTN_AVAILABLE } -template +template void launch_fattn_tile_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * Q = dst->src[0]; switch (Q->ne[0]) { @@ -295,15 +293,17 @@ void launch_fattn_tile_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * constexpr int D = 64; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; - fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, FATTN_KQ_STRIDE_TILE_F32, true, true, false); } break; case 128: { constexpr int D = 128; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; - fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, FATTN_KQ_STRIDE_TILE_F32, true, true, false); } break; default: { GGML_ABORT("FlashAttention without tensor cores only supports head sizes 64 and 128."); @@ -320,37 +320,22 @@ void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_ten if (Q->ne[1] <= 16) { constexpr int cols_per_block = 16; - constexpr int parallel_blocks = 4; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - launch_fattn_tile_f32_64_128(ctx, dst); + launch_fattn_tile_f32_64_128(ctx, dst); } else { constexpr bool use_logit_softcap = true; - launch_fattn_tile_f32_64_128(ctx, dst); - } - return; - } - - if (Q->ne[1] <= 32) { - constexpr int cols_per_block = 32; - constexpr int parallel_blocks = 4; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - launch_fattn_tile_f32_64_128(ctx, dst); - } else { - constexpr bool use_logit_softcap = true; - launch_fattn_tile_f32_64_128(ctx, dst); + launch_fattn_tile_f32_64_128(ctx, dst); } return; } constexpr int cols_per_block = 32; - constexpr int parallel_blocks = 1; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - launch_fattn_tile_f32_64_128(ctx, dst); + launch_fattn_tile_f32_64_128(ctx, dst); } else { constexpr bool use_logit_softcap = true; - launch_fattn_tile_f32_64_128(ctx, dst); + launch_fattn_tile_f32_64_128(ctx, dst); } } diff --git a/ggml/src/ggml-cuda/fattn-vec-f16.cuh b/ggml/src/ggml-cuda/fattn-vec-f16.cuh index b7686c1ec3..32c52ebe33 100644 --- a/ggml/src/ggml-cuda/fattn-vec-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-vec-f16.cuh @@ -1,7 +1,7 @@ #include "common.cuh" #include "fattn-common.cuh" -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(D, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) @@ -55,17 +55,16 @@ static __global__ void flash_attn_vec_ext_f16( constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16; constexpr dequantize_1_f16_t dequantize_1_v = get_dequantize_1_f16(type_V); - const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on. const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - Q += nb02* blockIdx.y + nb01*ic0; - K += nb12*(blockIdx.y / gqa_ratio); - V += nb22*(blockIdx.y / gqa_ratio); + Q += nb02* blockIdx.z + nb01*ic0; + K += nb12*(blockIdx.z / gqa_ratio); + V += nb22*(blockIdx.z / gqa_ratio); const half * maskh = (const half *) mask + ne11*ic0; - const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slopef = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); const half slopeh = __float2half(slopef); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); @@ -172,8 +171,7 @@ static __global__ void flash_attn_vec_ext_f16( half2 VKQ[ncols] = {{0.0f, 0.0f}}; - const int k_start = parallel_blocks == 1 ? 0 : ip*D; - for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) { + for (int k_VKQ_0 = blockIdx.y*D; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*D) { // Calculate KQ tile and keep track of new maximum KQ values: // For unknown reasons using a half array of size 1 for kqmax_new causes a performance regression, @@ -283,29 +281,29 @@ static __global__ void flash_attn_vec_ext_f16( kqsum[j_VKQ] = warp_reduce_sum((float)kqsum[j_VKQ]); half dst_val = (__low2half(VKQ[j_VKQ]) + __high2half(VKQ[j_VKQ])); - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val /= kqsum[j_VKQ]; } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; - dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; + dst[j_dst*D*gridDim.z + D*blockIdx.z + tid] = dst_val; } - if (parallel_blocks != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { - dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); + if (gridDim.y != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { + dst_meta[((ic0 + tid)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = make_float2(kqmax[tid], kqsum[tid]); } #else NO_DEVICE_CODE; #endif // defined(FLASH_ATTN_AVAILABLE) && defined(FP16_AVAILABLE) } -template +template void ggml_cuda_flash_attn_ext_vec_f16_case_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { constexpr int nwarps = D/WARP_SIZE; - fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16; constexpr bool need_f16_K = D != 128; constexpr bool need_f16_V = D != 128 && D != 64; constexpr size_t nbytes_shared = 0; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, need_f16_K, need_f16_V); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false); } template @@ -325,65 +323,48 @@ void ggml_cuda_flash_attn_ext_vec_f16_case(ggml_backend_cuda_context & ctx, ggml memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); if (Q->ne[1] == 1) { - constexpr int cols_per_block = 1; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 1; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } return; } if (Q->ne[1] == 2) { - constexpr int cols_per_block = 2; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 2; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } return; } if (Q->ne[1] <= 4) { - constexpr int cols_per_block = 4; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 4; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } return; } - if (Q->ne[1] <= 8) { - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 4; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); - } else { - constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); - } - return; - } - - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 1; + constexpr int cols_per_block = 8; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } } diff --git a/ggml/src/ggml-cuda/fattn-vec-f32.cuh b/ggml/src/ggml-cuda/fattn-vec-f32.cuh index c1d2dd8d19..336c136d19 100644 --- a/ggml/src/ggml-cuda/fattn-vec-f32.cuh +++ b/ggml/src/ggml-cuda/fattn-vec-f32.cuh @@ -1,7 +1,7 @@ #include "common.cuh" #include "fattn-common.cuh" -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(D, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) @@ -55,16 +55,15 @@ static __global__ void flash_attn_vec_ext_f32( constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16; constexpr dequantize_1_f32_t dequantize_1_v = get_dequantize_1_f32(type_V); - const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on. const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - Q += nb02* blockIdx.y + nb01*ic0; - K += nb12*(blockIdx.y / gqa_ratio); - V += nb22*(blockIdx.y / gqa_ratio); // K and V have same shape + Q += nb02* blockIdx.z + nb01*ic0; + K += nb12*(blockIdx.z / gqa_ratio); + V += nb22*(blockIdx.z / gqa_ratio); // K and V have same shape const half * maskh = (const half *) mask + ne11*ic0; - const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slope = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); constexpr int nwarps = D / WARP_SIZE; @@ -167,8 +166,7 @@ static __global__ void flash_attn_vec_ext_f32( float VKQ[ncols] = {0.0f}; - const int k_start = parallel_blocks == 1 ? 0 : ip*D; - for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) { + for (int k_VKQ_0 = blockIdx.y*D; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*D) { // Calculate KQ tile and keep track of new maximum KQ values: float kqmax_new_arr[ncols]; @@ -268,29 +266,29 @@ static __global__ void flash_attn_vec_ext_f32( kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]); float dst_val = VKQ[j_VKQ]; - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val /= kqsum[j_VKQ]; } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; - dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; + dst[j_dst*D*gridDim.z + D*blockIdx.z + tid] = dst_val; } - if (parallel_blocks != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { - dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); + if (gridDim.y != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { + dst_meta[((ic0 + tid)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = make_float2(kqmax[tid], kqsum[tid]); } #else NO_DEVICE_CODE; #endif // FLASH_ATTN_AVAILABLE } -template +template void ggml_cuda_flash_attn_ext_vec_f32_case_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { constexpr int nwarps = D/WARP_SIZE; - fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f32; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f32; constexpr bool need_f16_K = D != 128; constexpr bool need_f16_V = D != 128 && D != 64; constexpr size_t nbytes_shared = 0; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, need_f16_K, need_f16_V); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false); } template @@ -307,65 +305,48 @@ void ggml_cuda_flash_attn_ext_vec_f32_case(ggml_backend_cuda_context & ctx, ggml memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); if (Q->ne[1] == 1) { - constexpr int cols_per_block = 1; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 1; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } return; } if (Q->ne[1] == 2) { - constexpr int cols_per_block = 2; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 2; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } return; } if (Q->ne[1] <= 4) { - constexpr int cols_per_block = 4; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 4; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } return; } - if (Q->ne[1] <= 8) { - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 4; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); - } else { - constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); - } - return; - } - - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 1; + constexpr int cols_per_block = 8; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } } diff --git a/ggml/src/ggml-cuda/fattn-wmma-f16.cu b/ggml/src/ggml-cuda/fattn-wmma-f16.cu index dab1d5cbca..5c214ea310 100644 --- a/ggml/src/ggml-cuda/fattn-wmma-f16.cu +++ b/ggml/src/ggml-cuda/fattn-wmma-f16.cu @@ -18,7 +18,7 @@ namespace wmma = rocwmma; #endif // FP16_MMA_AVAILABLE // D == head size, VKQ_stride == num VKQ rows calculated in parallel: -template +template __launch_bounds__(nwarps*ggml_cuda_get_physical_warp_size(), 1) static __global__ void flash_attn_ext_f16( const char * __restrict__ Q, @@ -67,8 +67,7 @@ static __global__ void flash_attn_ext_f16( constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - const int ic0 = ncols*(blockIdx.x / parallel_blocks); // Index of the first Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = ncols*blockIdx.x; // Index of the first Q/QKV column to work on. static_assert(D <= FATTN_KQ_STRIDE, "D must be <= FATTN_KQ_STRIDE."); static_assert(ncols == 8 || ncols % 16 == 0, "ncols must be 8 or a multiple of 16."); @@ -91,16 +90,16 @@ static __global__ void flash_attn_ext_f16( constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half); const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const float * Q_f = (const float *) (Q + nb02* blockIdx.y + nb01*ic0); - const half * K_h = (const half *) (K + nb12*(blockIdx.y / gqa_ratio)); - const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const float * Q_f = (const float *) (Q + nb02* blockIdx.z + nb01*ic0); + const half * K_h = (const half *) (K + nb12*(blockIdx.z / gqa_ratio)); + const half * V_h = (const half *) (V + nb12*(blockIdx.z / gqa_ratio)); // K and V have same shape const half * maskh = (const half *) mask + (nb31/sizeof(half))* ic0; const half2 * mask2 = (const half2 *) mask + (nb31/sizeof(half))*(ic0/2); const int stride_Q = nb01 / sizeof(float); const int stride_KV = nb11 / sizeof(half); - const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slopef = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); const half slopeh = __float2half(slopef); const half2 slope2 = make_half2(slopef, slopef); @@ -176,7 +175,7 @@ static __global__ void flash_attn_ext_f16( __syncthreads(); // Iterate over ne11 == previous tokens: - for (int k_VKQ_0 = ip*FATTN_KQ_STRIDE; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE) { + for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE) { // Calculate tile of KQ: #pragma unroll for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) { @@ -395,7 +394,7 @@ static __global__ void flash_attn_ext_f16( if (ic0 + j_VKQ >= ne01) { return; } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; float KQ_rowsum_j; if (std::is_same::value) { @@ -411,13 +410,13 @@ static __global__ void flash_attn_ext_f16( break; } float dst_val = VKQ[j_VKQ*D_padded + i]; - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val /= KQ_rowsum_j; } - dst[j_dst*gridDim.y*D + blockIdx.y*D + i] = dst_val; + dst[j_dst*gridDim.z*D + blockIdx.z*D + i] = dst_val; } - if (parallel_blocks == 1 || threadIdx.x != 0) { + if (gridDim.y == 1 || threadIdx.x != 0) { continue; } @@ -428,7 +427,7 @@ static __global__ void flash_attn_ext_f16( dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]); } dst_meta_val.y = KQ_rowsum_j; - dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = dst_meta_val; + dst_meta[((ic0 + j_VKQ)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = dst_meta_val; } #else NO_DEVICE_CODE; @@ -462,60 +461,26 @@ static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed."); template void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * KQV = dst; - const ggml_tensor * Q = dst->src[0]; constexpr int nwarps = 4; constexpr int frag_m = cols_per_block == 8 && D % 32 == 0 ? 32 : 16; - const int blocks_num_pb1 = ((Q->ne[1] + cols_per_block - 1) / cols_per_block)*Q->ne[2]*Q->ne[3]; - const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; float logit_softcap; memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); - if (4*blocks_num_pb1 < 2*nsm) { - constexpr int parallel_blocks = 4; - fattn_kernel_t fattn_kernel; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; - } else { - constexpr bool use_logit_softcap = true; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; - } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true, warp_size); - return; - } - if (2*blocks_num_pb1 < 2*nsm) { - constexpr int parallel_blocks = 2; - fattn_kernel_t fattn_kernel; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; - } else { - constexpr bool use_logit_softcap = true; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; - } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true, warp_size); - return; - } - constexpr int parallel_blocks = 1; fattn_kernel_t fattn_kernel; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; + D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>; } else { constexpr bool use_logit_softcap = true; fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; + D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true, warp_size); + launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, FATTN_KQ_STRIDE, true, true, false, warp_size); } void ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 2e72fc8fd3..973541893e 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -281,13 +281,13 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst if (!fp16_mma_available(cc)) { if (prec == GGML_PREC_DEFAULT) { - if (Q->ne[1] <= 8) { + if (Q->ne[1] <= 8 || Q->ne[0] == 256) { ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); } else { ggml_cuda_flash_attn_ext_tile_f16(ctx, dst); } } else { - if (Q->ne[1] <= 8) { + if (Q->ne[1] <= 8 || Q->ne[0] == 256) { ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); } else { ggml_cuda_flash_attn_ext_tile_f32(ctx, dst); @@ -296,17 +296,17 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst return; } - const int gqa_ratio = Q->ne[2] / K->ne[2]; - const bool mma_fast_for_bs1 = fp16_mma_available(cc) && gqa_ratio % 2 == 0 && - K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && mask; - if (Q->ne[1] == 1 && Q->ne[0] % (2*warp_size) == 0 && !mma_fast_for_bs1) { + const bool gqa_opt_applies = ((Q->ne[2] / K->ne[2]) % 2 == 0) && mask; // The mma-based kernels have GQA-specific optimizations + const bool mma_needs_data_conversion = K->type != GGML_TYPE_F16 || V->type != GGML_TYPE_F16; + const bool mma_faster_for_bs1 = new_mma_available(cc) && gqa_opt_applies && cc < GGML_CUDA_CC_ADA_LOVELACE && !mma_needs_data_conversion; + const bool can_use_vector_kernel = (Q->ne[0] % (2*warp_size) == 0) && (prec == GGML_PREC_DEFAULT || Q->ne[0] <= 128); + if (Q->ne[1] == 1 && can_use_vector_kernel && !mma_faster_for_bs1) { if (prec == GGML_PREC_DEFAULT) { ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); - return; - } else if(Q->ne[0] <= 128) { + } else { ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); - return; } + return; } // The MMA implementation needs Turing or newer, use the old WMMA code for Volta: diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 5cb56df9a8..b783310ef7 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3230,6 +3230,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g #ifndef FLASH_ATTN_AVAILABLE return false; #endif // FLASH_ATTN_AVAILABLE + if (op->src[0]->ne[3] != 1) { + return false; + } if (op->src[1]->type == GGML_TYPE_BF16 || op->src[2]->type == GGML_TYPE_BF16) { return false; } diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index aace21e3a8..a4c717a321 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -129,6 +129,7 @@ #define cudaGraph_t hipGraph_t #define cudaStream_t hipStream_t #define cudaSuccess hipSuccess +#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor #define __trap() do { abort(); __builtin_unreachable(); } while(0) #define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS #define CUBLAS_STATUS_NOT_INITIALIZED HIPBLAS_STATUS_NOT_INITIALIZED diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 997f671431..f2d55796e7 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -134,5 +134,6 @@ #define cudaStreamCaptureModeRelaxed musaStreamCaptureModeRelaxed #define cudaStreamBeginCapture musaStreamBeginCapture #define cudaStreamEndCapture musaStreamEndCapture +#define cudaOccupancyMaxActiveBlocksPerMultiprocessor musaOccupancyMaxActiveBlocksPerMultiprocessor typedef mt_bfloat16 nv_bfloat16; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index adb749bd5e..d48cd21723 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -259,6 +259,10 @@ static std::string var_to_str(ggml_type type) { return ggml_type_name(type); } +static std::string var_to_str(ggml_prec prec) { + return prec == GGML_PREC_F32 ? "f32" : "def"; +} + static std::string var_to_str(ggml_op_pool pool) { switch (pool) { case GGML_OP_POOL_AVG: return "avg"; @@ -3206,11 +3210,12 @@ struct test_flash_attn_ext : public test_case { const float max_bias; // ALiBi const float logit_softcap; // Gemma 2 + const ggml_prec prec; const ggml_type type_KV; std::array permute; std::string vars() override { - return VARS_TO_STR10(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, type_KV, permute); + return VARS_TO_STR11(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, prec, type_KV, permute); } double max_nmse_err() override { @@ -3225,9 +3230,9 @@ struct test_flash_attn_ext : public test_case { } test_flash_attn_ext(int64_t hs = 128, int64_t nh = 32, int64_t nr = 1, int64_t kv = 96, int64_t nb = 8, - bool mask = true, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_type type_KV = GGML_TYPE_F16, - std::array permute = {0, 1, 2, 3}) - : hs(hs), nh(nh), nr(nr), kv(kv), nb(nb), mask(mask), max_bias(max_bias), logit_softcap(logit_softcap), type_KV(type_KV), permute(permute) {} + bool mask = true, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32, + ggml_type type_KV = GGML_TYPE_F16, std::array permute = {0, 1, 2, 3}) + : hs(hs), nh(nh), nr(nr), kv(kv), nb(nb), mask(mask), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec), type_KV(type_KV), permute(permute) {} ggml_tensor * build_graph(ggml_context * ctx) override { const int64_t hs_padded = GGML_PAD(hs, ggml_blck_size(type_KV)); @@ -3261,6 +3266,7 @@ struct test_flash_attn_ext : public test_case { } ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hs), max_bias, logit_softcap); + ggml_flash_attn_ext_set_prec(out, prec); ggml_set_name(out, "out"); return out; @@ -4376,11 +4382,16 @@ static std::vector> make_test_cases_eval() { for (int kv : { 512, 1024, }) { if (nr != 1 && kv != 512) continue; for (int nb : { 1, 3, 32, 35, }) { - for (ggml_type type_KV : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { - test_cases.emplace_back(new test_flash_attn_ext(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, type_KV)); - // run fewer test cases permuted - if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) { - test_cases.emplace_back(new test_flash_attn_ext(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, type_KV, {0, 2, 1, 3})); + for (ggml_prec prec : {GGML_PREC_F32, GGML_PREC_DEFAULT}) { + if (hs != 128 && prec == GGML_PREC_DEFAULT) continue; + for (ggml_type type_KV : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { + test_cases.emplace_back(new test_flash_attn_ext( + hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, prec, type_KV)); + // run fewer test cases permuted + if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) { + test_cases.emplace_back(new test_flash_attn_ext( + hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, prec, type_KV, {0, 2, 1, 3})); + } } } } From 568013d0cd3d5add37c376b3d5e959809b711fc7 Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Wed, 19 Mar 2025 21:01:57 +0100 Subject: [PATCH 44/55] context : clear sets containing encoder output sequence ids before storing new values (#12470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: StanisÅ‚aw Szymczyk --- src/llama-context.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 664703a896..5bec63e2e7 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1153,6 +1153,7 @@ int llama_context::encode(llama_batch & inp_batch) { // remember the sequence ids used during the encoding - needed for cross attention later cross.seq_ids_enc.resize(n_tokens); for (int32_t i = 0; i < n_tokens; i++) { + cross.seq_ids_enc[i].clear(); for (int s = 0; s < ubatch.n_seq_id[i]; s++) { llama_seq_id seq_id = ubatch.seq_id[i][s]; cross.seq_ids_enc[i].insert(seq_id); From 732b5fbf5e7f9cf069942f0c5850ee959ef321ba Mon Sep 17 00:00:00 2001 From: Bartowski Date: Thu, 20 Mar 2025 02:36:37 -0400 Subject: [PATCH 45/55] convert : avoid calls to tokenizer.added_tokens_decoder (#12473) tokenizer.added_tokens_decoder returns a fresh dict every time relatively slowly (~0.04s on average) which results in massive slowdowns when we have a huge number of added tokens --- convert_hf_to_gguf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 7574218e24..d21edce16b 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -529,6 +529,8 @@ class Model: reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} added_vocab = tokenizer.get_added_vocab() + added_tokens_decoder = tokenizer.added_tokens_decoder + for i in range(vocab_size): if i not in reverse_vocab: tokens.append(f"[PAD{i}]") @@ -538,13 +540,13 @@ class Model: if token in added_vocab: # The tokenizer in llama.cpp assumes the CONTROL and USER_DEFINED tokens are pre-normalized. # To avoid unexpected issues - we make sure to normalize non-normalized tokens - if not tokenizer.added_tokens_decoder[i].normalized: + if not added_tokens_decoder[i].normalized: previous_token = token token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) if previous_token != token: logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer") - if tokenizer.added_tokens_decoder[i].special or self.does_token_look_special(token): + if added_tokens_decoder[i].special or self.does_token_look_special(token): toktypes.append(gguf.TokenType.CONTROL) else: # NOTE: this was added for Gemma. From 3d82dbcbce2c677fc35fbf99574ccd107d95a1f8 Mon Sep 17 00:00:00 2001 From: Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com> Date: Thu, 20 Mar 2025 17:05:34 +0530 Subject: [PATCH 46/55] ggml : block interleaving support for Q4_K quantization for x86 AVX2 architecture (#12332) * Add block interleaving support for Q4_K quantization * Remove whitespaces and fix CI/CD issues * Update pointer of bsums from int16_t to const int16_t * Add vector version of quantize_q8_K_4x8 function * Update code formatting based on review comments --- ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp | 1505 +++++++++++++++++++++++- 1 file changed, 1493 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp b/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp index c24fd56e20..e852c8253b 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp @@ -45,6 +45,24 @@ using block_q4_0x8 = block<4, 8>; using block_q8_0x4 = block<8, 4>; using block_q8_0x8 = block<8, 8>; + +struct block_q4_Kx8 { + ggml_half d[8]; // super-block scale for quantized scales + ggml_half dmin[8]; // super-block scale for quantized mins + uint8_t scales[96]; // scales and mins, quantized with 6 bits + uint8_t qs[1024]; // 4--bit quants +}; + +static_assert(sizeof(block_q4_Kx8) == sizeof(ggml_half) * 16 + K_SCALE_SIZE * 8 + QK_K * 4, "wrong q4_K block size/padding"); + +struct block_q8_Kx4 { + float d[4]; // delta + int8_t qs[QK_K * 4]; // quants + int16_t bsums[QK_K / 4]; // sum of quants in groups of 16 +}; + +static_assert(sizeof(block_q8_Kx4) == sizeof(float) * 4 + QK_K * 4 + (QK_K / 4) * sizeof(int16_t), "wrong q8_K block size/padding"); + struct block_iq4_nlx4 { ggml_half d[4]; // deltas for 4 iq4_nl blocks uint8_t qs[QK4_NL * 2]; // nibbles / quants for 4 iq4_nl blocks @@ -60,6 +78,13 @@ static_assert(sizeof(block_iq4_nlx4) == 4 * sizeof(ggml_half) + QK4_NL * 2, "wro #define UNUSED GGML_UNUSED +static inline int nearest_int(float fval) { + assert(fabsf(fval) <= 4194303.f); + float val = fval + 12582912.f; + int i; memcpy(&i, &val, sizeof(int)); + return (i & 0x007fffff) - 0x00400000; +} + // Functions to create the interleaved data layout formats // interleave 4 block_q4_0s in blocks of blck_size_interleave @@ -534,6 +559,270 @@ static void quantize_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRIC #endif } +static void quantize_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(QK_K == 256); + assert(k % QK_K == 0); + const int nb = k / QK_K; + + block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy; + +#if defined(__AVX2__) + float iscale[4]; + __m256 srcv[4][32]; + __m256 iscale_vec[4]; + + for (int i = 0; i < nb; i++) { + for (int row_iter = 0; row_iter < 4; row_iter++) { + // Load elements into 4 AVX vectors + __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 ); + __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 8 ); + __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 16 ); + __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 24 ); + + // Compute max(abs(e)) for the block + const __m256 signBit = _mm256_set1_ps( -0.0f ); + __m256 abs0 = _mm256_andnot_ps( signBit, v0 ); + __m256 abs1 = _mm256_andnot_ps( signBit, v1 ); + __m256 abs2 = _mm256_andnot_ps( signBit, v2 ); + __m256 abs3 = _mm256_andnot_ps( signBit, v3 ); + + __m256 maxAbs = _mm256_max_ps( abs0, abs1 ); + maxAbs = _mm256_max_ps( maxAbs, abs2 ); + maxAbs = _mm256_max_ps( maxAbs, abs3 ); + + __m256 mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ ); + __m256 mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ ); + __m256 mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ ); + __m256 mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ ); + + __m256 maskAbs = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3)); + + srcv[row_iter][0] = v0; + srcv[row_iter][1] = v1; + srcv[row_iter][2] = v2; + srcv[row_iter][3] = v3; + + for (int sb = 1; sb < 8; sb++) { + // Temporarily stores absolute quant values + __m256 tempAbs = maxAbs; + + // Load elements into 4 AVX vectors + __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32); + __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 8 ); + __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 16 ); + __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 24 ); + + // Compute max(abs(e)) for the block + __m256 abs0 = _mm256_andnot_ps( signBit, v0 ); + __m256 abs1 = _mm256_andnot_ps( signBit, v1 ); + __m256 abs2 = _mm256_andnot_ps( signBit, v2 ); + __m256 abs3 = _mm256_andnot_ps( signBit, v3 ); + + maxAbs = _mm256_max_ps( maxAbs, abs0 ); + maxAbs = _mm256_max_ps( maxAbs, abs1 ); + maxAbs = _mm256_max_ps( maxAbs, abs2 ); + maxAbs = _mm256_max_ps( maxAbs, abs3 ); + + __m256 mask_prev = _mm256_cmp_ps( tempAbs, maxAbs, _CMP_EQ_OQ ); + maskAbs = _mm256_and_ps( maskAbs, mask_prev ); + + mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ ); + mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ ); + mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ ); + mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ ); + + __m256 mask_curr = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3)); + maskAbs = _mm256_or_ps(maskAbs, mask_curr); + + srcv[row_iter][sb * 4] = v0; + srcv[row_iter][sb * 4 + 1] = v1; + srcv[row_iter][sb * 4 + 2] = v2; + srcv[row_iter][sb * 4 + 3] = v3; + } + + __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) ); + max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); + max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); + const float maxScalar = _mm_cvtss_f32( max4 ); + + __m256 maxScalarVec = _mm256_set1_ps(maxScalar); + + __m256 mask_next = _mm256_cmp_ps( maxScalarVec, maxAbs, _CMP_EQ_OQ ); + __m256 finalMask = _mm256_and_ps(maskAbs, mask_next); + + const int mask = _mm256_movemask_ps(finalMask); + iscale[row_iter] = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; + + if(mask) { + iscale[row_iter] = ( maxScalar != 0.0f ) ? -127.f / maxScalar: 0.0f; + } + + y[i].d[row_iter] = maxScalar ? 1/iscale[row_iter] : 0; + iscale_vec[row_iter] = _mm256_set1_ps(iscale[row_iter]); + } + + __m256i quants_interleaved[32]; + for (int j = 0; j < 32; j++) { + // Apply the multiplier + __m256 v0 = _mm256_mul_ps(srcv[0][j], iscale_vec[0]); + __m256 v1 = _mm256_mul_ps(srcv[1][j], iscale_vec[1]); + __m256 v2 = _mm256_mul_ps(srcv[2][j], iscale_vec[2]); + __m256 v3 = _mm256_mul_ps(srcv[3][j], iscale_vec[3]); + + // Round to nearest integer + v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST ); + v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST ); + v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST ); + v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST ); + + // Convert floats to integers + __m256i i0 = _mm256_cvtps_epi32( v0 ); + __m256i i1 = _mm256_cvtps_epi32( v1 ); + __m256i i2 = _mm256_cvtps_epi32( v2 ); + __m256i i3 = _mm256_cvtps_epi32( v3 ); + + // Convert int32 to int16 + i0 = _mm256_packs_epi32( i0, i1 ); + i2 = _mm256_packs_epi32( i2, i3 ); + // Convert int16 to int8 + i0 = _mm256_packs_epi16( i0, i2 ); + + // Permute and store the quantized weights in the required order after the pack instruction + const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 ); + i0 = _mm256_permutevar8x32_epi32( i0, perm ); + + _mm256_storeu_si256((__m256i *)(y[i].qs + 32 * j), i0); + quants_interleaved[j] = i0; + } + + // Masks to shuffle the quants of corresonding sub blocks for rearraning quants for vectorized bsums computation + __m256i shuffle_mask_sb2 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 0, 1, 4, 5, 6, 7, 8, 9, 8, 9, 12, 13, 14, 15)); + shuffle_mask_sb2 = _mm256_permute2f128_si256(shuffle_mask_sb2, shuffle_mask_sb2, 0); + __m256i shuffle_mask_sb3 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 0, 1, 6, 7, 8, 9, 10, 11, 8, 9, 14, 15)); + shuffle_mask_sb3 = _mm256_permute2f128_si256(shuffle_mask_sb3, shuffle_mask_sb3, 0); + __m256i shuffle_mask_sb4 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 4, 5, 0, 1, 8, 9, 10, 11, 12, 13, 8, 9)); + shuffle_mask_sb4 = _mm256_permute2f128_si256(shuffle_mask_sb4, shuffle_mask_sb4, 0); + + for (int k = 0; k < 4; k++) { + // Quants from four different sub blocks are taken + __m256i q0 = quants_interleaved[k * 8 + 0]; + __m256i q1 = quants_interleaved[k * 8 + 1]; + __m256i q2 = quants_interleaved[k * 8 + 2]; + __m256i q3 = quants_interleaved[k * 8 + 3]; + __m256i q4 = quants_interleaved[k * 8 + 4]; + __m256i q5 = quants_interleaved[k * 8 + 5]; + __m256i q6 = quants_interleaved[k * 8 + 6]; + __m256i q7 = quants_interleaved[k * 8 + 7]; + + + // The below code block has the first half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time + __m256i sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2); + __m256i sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34); + __m256i sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3); + sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68); + __m256i sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4); + sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136); + + __m256i one = _mm256_set1_epi8(1); + __m256i bsums_r1 = _mm256_maddubs_epi16(one, sb_h1_interleaved); + + for (int l = 0; l < 3; l++) { + // Quants value shifted to process next two values from each sub block + q0 = _mm256_srli_epi64(q0, 16); + q2 = _mm256_srli_epi64(q2, 16); + q4 = _mm256_srli_epi64(q4, 16); + q6 = _mm256_srli_epi64(q6, 16); + + sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2); + sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34); + sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3); + sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68); + sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4); + sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136); + + bsums_r1 = _mm256_add_epi16(bsums_r1, _mm256_maddubs_epi16(one, sb_h1_interleaved)); + } + + // The below code block has the second half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time + __m256i sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2); + __m256i sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34); + __m256i sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3); + sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68); + __m256i sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4); + sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136); + + __m256i bsums_r2 = _mm256_maddubs_epi16(one, sb_h2_interleaved); + + for (int l = 0; l < 3; l++) { + // Quants value shifted to process next two values from each sub block + q1 = _mm256_srli_epi64(q1, 16); + q3 = _mm256_srli_epi64(q3, 16); + q5 = _mm256_srli_epi64(q5, 16); + q7 = _mm256_srli_epi64(q7, 16); + + sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2); + sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34); + sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3); + sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68); + sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4); + sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136); + + bsums_r2 = _mm256_add_epi16(bsums_r2, _mm256_maddubs_epi16(one, sb_h2_interleaved)); + } + + // Overall bsums in interleaved fashion computed by adding results of both halves + __m256i bsums_r = _mm256_add_epi16(bsums_r1, bsums_r2); + _mm256_storeu_si256((__m256i *)(y[i].bsums + 16 * k), bsums_r); + } + } + +#else + + // scalar + const int blck_size_interleave = 8; + float srcv[4][QK_K]; + float iscale[4]; + + for (int i = 0; i < nb; i++) { + for (int row_iter = 0; row_iter < 4; row_iter++) { + float amax = 0.0f; // absolute max + float max = 0; + + for (int j = 0; j < QK_K; j++) { + srcv[row_iter][j] = x[row_iter * k + i * QK_K + j]; + // Update the maximum value of the corresponding super block + if(amax < fabsf(srcv[row_iter][j])) { + amax = fabsf(srcv[row_iter][j]); + max = srcv[row_iter][j]; + } + } + + iscale[row_iter] = amax ? -127.f/max : 0; + + y[i].d[row_iter] = amax ? 1/iscale[row_iter] : 0; + } + + for (int j = 0; j < QK_K / 4; j++) { + y[i].bsums[j] = 0; + } + + // Quants values are interleaved in sequence of eight bytes from corresponding super blocks + // Bsums values are interleaved in sequence of four bsums from each super block taken for interleaving + // i.e first four bsums from the first super block, followed by first four bsums from second super block and so on + for (int j = 0; j < QK_K * 4; j++) { + int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave; + int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave; + src_offset += (j % blck_size_interleave); + int index = (((j & 31) >> 3) << 2) + ((j >> 8) << 4) + ((j >> 6) & 3); + + float x0 = srcv[src_id][src_offset] * iscale[src_id]; + y[i].qs[j] = nearest_int(x0); + y[i].bsums[index] += y[i].qs[j]; + } + } +#endif +} + static void quantize_mat_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row, int64_t blck_size_interleave) { assert(nrow == 4); UNUSED(nrow); @@ -546,6 +835,16 @@ static void quantize_mat_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRIC } } +static void quantize_mat_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row, int64_t blck_size_interleave) { + assert(nrow == 4); + UNUSED(nrow); + if (blck_size_interleave == 8) { + quantize_q8_K_4x8(x, vy, n_per_row); + } else { + assert(false); + } +} + static void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -994,6 +1293,281 @@ static void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, c } } +static void ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK_K; + const int nb = n / qk; + const int ncols_interleaved = 8; + const int blocklen = 8; + static const uint32_t kmask1 = 0x3f3f3f3f; + static const uint32_t kmask2 = 0x0f0f0f0f; + static const uint32_t kmask3 = 0x03030303; + + assert (n % qk == 0); + assert (nc % ncols_interleaved == 0); + + UNUSED(s); + UNUSED(bs); + UNUSED(vx); + UNUSED(vy); + UNUSED(nr); + UNUSED(nc); + UNUSED(nb); + UNUSED(ncols_interleaved); + UNUSED(blocklen); + +#if defined(__AVX2__) + // Lookup table to convert signed nibbles to signed bytes + __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0)); + signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0); + // Shuffle masks to rearrange delta and scale values to multiply with appropriate scales + __m128i deltamask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0); + __m128i scalemask = _mm_set_epi8(7, 7, 3, 3, 6, 6, 2, 2, 5, 5, 1, 1, 4, 4, 0, 0); + // Permute mask used for easier vector processing at later stages + __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); + + // Mask to extract nibbles from bytes + const __m256i m4b = _mm256_set1_epi8(0x0F); + + int64_t b_nb = n / QK_K; + + const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 *)vx; + const block_q8_K * a_ptr_start = (const block_q8_K *)vy; + + // Process Q8_K blocks one by one + for (int64_t y = 0; y < nr; y++) { + + // Pointers to LHS blocks of block_q8_K format + const block_q8_K * a_ptr = a_ptr_start + (y * nb); + + // Take group of eight interleaved block_q4_K structures at each pass of the loop and perform dot product operation + for (int64_t x = 0; x < nc / 8; x++) { + + // Pointers to RHS blocks + const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb); + + // Master FP accumulators + __m256 acc_row = _mm256_setzero_ps(); + __m256 acc_min_rows = _mm256_setzero_ps(); + + for (int64_t b = 0; b < nb; b++) { + + // Load and convert to FP32 scale from block_q8_K + const __m256 row_scale_f32 = _mm256_set1_ps((a_ptr[b].d)); + + // Load the scale values for the 8 blocks interleaved in block_q4_Kx8 + // col_scale_f32 rearranged so as to multiply with appropriate quants + const __m256 col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, deltamask); + const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin); + + __m256i iacc_b = _mm256_setzero_si256(); + __m256i iacc_min_b = _mm256_setzero_si256(); + + const __m256i q8sums = _mm256_loadu_si256((const __m256i * )(a_ptr[b].bsums)); + __m256i q8s = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(q8sums), _mm256_extracti128_si256(q8sums, 1))); + q8s = _mm256_permute2f128_si256(q8s, q8s, 0); + + // Processes two sub blocks from each Q4_K in each iteration + for (int sb = 0; sb < QK_K / 64; sb++) { + + // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7 + const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); + const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); + const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); + const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); + const __m256i rhs_raw_vec_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); + const __m256i rhs_raw_vec_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); + const __m256i rhs_raw_vec_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); + const __m256i rhs_raw_vec_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); + + // 4-bit -> 8-bit + // Values of the first sub block of eight block_q4_K structures for the sb loop + const __m256i rhs_vec_0123_00 = _mm256_and_si256(rhs_raw_vec_0123_0, m4b); + const __m256i rhs_vec_4567_00 = _mm256_and_si256(rhs_raw_vec_4567_0, m4b); + const __m256i rhs_vec_0123_01 = _mm256_and_si256(rhs_raw_vec_0123_1, m4b); + const __m256i rhs_vec_4567_01 = _mm256_and_si256(rhs_raw_vec_4567_1, m4b); + const __m256i rhs_vec_0123_02 = _mm256_and_si256(rhs_raw_vec_0123_2, m4b); + const __m256i rhs_vec_4567_02 = _mm256_and_si256(rhs_raw_vec_4567_2, m4b); + const __m256i rhs_vec_0123_03 = _mm256_and_si256(rhs_raw_vec_0123_3, m4b); + const __m256i rhs_vec_4567_03 = _mm256_and_si256(rhs_raw_vec_4567_3, m4b); + + // Values of the second sub block of eight block_q4_K structures when sb = 1 + const __m256i rhs_vec_0123_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m4b); + const __m256i rhs_vec_4567_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m4b); + const __m256i rhs_vec_0123_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m4b); + const __m256i rhs_vec_4567_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m4b); + const __m256i rhs_vec_0123_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 4), m4b); + const __m256i rhs_vec_4567_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 4), m4b); + const __m256i rhs_vec_0123_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 4), m4b); + const __m256i rhs_vec_4567_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 4), m4b); + + uint32_t utmp_0[4], utmp_1[4]; + + // Scales and Mins of corresponding sub blocks from different Q8_K structures are stored together + // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12); + utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp_0[1] & kmask1; + utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4); + utmp_0[2] = uaux_0; + utmp_0[0] &= kmask1; + + // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12); + utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4); + const uint32_t uaux_1 = utmp_1[1] & kmask1; + utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4); + utmp_1[2] = uaux_1; + utmp_1[0] &= kmask1; + + // Scales of first sub block in the sb loop + const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]); + __m128i scales_rearrange_0 = _mm_shuffle_epi8(mins_and_scales_0, scalemask); + __m256i scales_0 = _mm256_cvtepu8_epi16(scales_rearrange_0); + + // Scales of second sub block in the sb loop + __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]); + __m128i scales_rearrange_1 = _mm_shuffle_epi8(mins_and_scales_1, scalemask); + __m256i scales_1 = _mm256_cvtepu8_epi16(scales_rearrange_1); + + // Mins of first and second sub block of Q4_K block are arranged side by side + __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78))); + + // Load the two sub block values corresponding to sb in block_q8_K in batches of 16 bytes and replicate the same across 256 bit vector + __m256i lhs_vec_00 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + sb * 64))); + __m256i lhs_vec_01 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16 + sb * 64))); + __m256i lhs_vec_10 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 32 + sb * 64))); + __m256i lhs_vec_11 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 48 + sb * 64))); + + lhs_vec_00 = _mm256_permute2f128_si256(lhs_vec_00, lhs_vec_00, 0); + lhs_vec_01 = _mm256_permute2f128_si256(lhs_vec_01, lhs_vec_01, 0); + lhs_vec_10 = _mm256_permute2f128_si256(lhs_vec_10, lhs_vec_10, 0); + lhs_vec_11 = _mm256_permute2f128_si256(lhs_vec_11, lhs_vec_11, 0); + + // Dot product done within 32 bit lanes and accumulated in the same vector + // First done for first sub block and thenn for second sub block in each sb + // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3) + // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7) + // ........................................................................... + // B0(28-31) B4(28-31) B1(28-31) B5(28-31) B2(28-31) B6(28-31) B3(28-31) B7(28-31) with A0(28-31) + + + __m256i iacc_0 = _mm256_setzero_si256(); + __m256i iacc_1 = _mm256_setzero_si256(); + + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_00 ,_mm256_shuffle_epi32(rhs_vec_4567_00, 177), 170), _mm256_shuffle_epi32(lhs_vec_00, 0))); + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_00, 177) ,rhs_vec_4567_00, 170), _mm256_shuffle_epi32(lhs_vec_00, 85))); + + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_01 ,_mm256_shuffle_epi32(rhs_vec_4567_01, 177), 170), _mm256_shuffle_epi32(lhs_vec_00, 170))); + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_01, 177) ,rhs_vec_4567_01, 170), _mm256_shuffle_epi32(lhs_vec_00, 255))); + + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_02 ,_mm256_shuffle_epi32(rhs_vec_4567_02, 177), 170), _mm256_shuffle_epi32(lhs_vec_01, 0))); + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_02, 177) ,rhs_vec_4567_02, 170), _mm256_shuffle_epi32(lhs_vec_01, 85))); + + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_03 ,_mm256_shuffle_epi32(rhs_vec_4567_03, 177), 170), _mm256_shuffle_epi32(lhs_vec_01, 170))); + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_03, 177) ,rhs_vec_4567_03, 170), _mm256_shuffle_epi32(lhs_vec_01, 255))); + + iacc_0 = _mm256_madd_epi16(iacc_0, scales_0); + + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_10 ,_mm256_shuffle_epi32(rhs_vec_4567_10, 177), 170), _mm256_shuffle_epi32(lhs_vec_10, 0))); + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_10, 177) ,rhs_vec_4567_10, 170), _mm256_shuffle_epi32(lhs_vec_10, 85))); + + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_11 ,_mm256_shuffle_epi32(rhs_vec_4567_11, 177), 170), _mm256_shuffle_epi32(lhs_vec_10, 170))); + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_11, 177) ,rhs_vec_4567_11, 170), _mm256_shuffle_epi32(lhs_vec_10, 255))); + + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_12 ,_mm256_shuffle_epi32(rhs_vec_4567_12, 177), 170), _mm256_shuffle_epi32(lhs_vec_11, 0))); + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_12, 177) ,rhs_vec_4567_12, 170), _mm256_shuffle_epi32(lhs_vec_11, 85))); + + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_13 ,_mm256_shuffle_epi32(rhs_vec_4567_13, 177), 170), _mm256_shuffle_epi32(lhs_vec_11, 170))); + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_13, 177) ,rhs_vec_4567_13, 170), _mm256_shuffle_epi32(lhs_vec_11, 255))); + + iacc_1 = _mm256_madd_epi16(iacc_1, scales_1); + + // Accumulate the iacc value for one sb + __m256i iacc_sb = _mm256_add_epi32(iacc_0, iacc_1); + + // Broadcast the bsums of the two sub blocks of the iteration of Q8_K across the vector + // Multiply-Add with corresponding mins of Q4_Kx8 with bsums + __m256i q8s_sb = _mm256_shuffle_epi32(q8s, 0); + __m256i iacc_min_sb = _mm256_madd_epi16(q8s_sb, mins_01); + q8s = _mm256_bsrli_epi128(q8s, 4); + + // Accumulate for the complete block + iacc_b = _mm256_add_epi32(iacc_b, iacc_sb); + iacc_min_b = _mm256_add_epi32(iacc_min_b, iacc_min_sb); + } + + // Multiply-Add with scale values for the complete super block + acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_b), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row); + acc_min_rows = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_min_b), _mm256_mul_ps(col_dmin_f32, row_scale_f32), acc_min_rows); + + } + + // Accumulated output values permuted so as to be stored in appropriate order post accumulation + acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask); + _mm256_storeu_ps(s + (y * nr + x * 8), _mm256_sub_ps(acc_row, acc_min_rows)); + } + } + +#else + + float sumf[8]; + float sum_minf[8]; + uint32_t utmp[32]; + int sumi1; + int sumi2; + int sumi; + + const block_q8_K * a_ptr = (const block_q8_K *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb); + + for (int j = 0; j < ncols_interleaved; j++) { + sumf[j] = 0.0; + sum_minf[j] = 0.0; + } + for (int l = 0; l < nb; l++) { + for (int sb = 0; sb < 8; sb++) { + memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12); + utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1; + utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4); + utmp[sb * 4 + 2] = uaux_0; + utmp[sb * 4 + 0] &= kmask1; + } + for (int k = 0; k < (qk / (2 * blocklen)); k++) { + uint8_t *scales_0 = (uint8_t*) utmp + (k / 4) * 32; + uint8_t *scales_1 = (uint8_t*) utmp + (k / 4) * 32 + 16; + for (int j = 0; j < ncols_interleaved; j++) { + sumi1 = 0; + sumi2 = 0; + sumi = 0; + for (int i = 0; i < blocklen; ++i) { + const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF); + const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4); + sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 64 + (k % 4) * blocklen + i]); + sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 64 + (k % 4) * blocklen + i + 32]); + sumi1 = sumi1 * scales_0[j]; + sumi2 = sumi2 * scales_1[j]; + sumi += sumi1 + sumi2; + } + sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d; + } + } + for (int sb = 0; sb < 8; sb++) { + uint8_t *mins = (uint8_t*) utmp + 8 + sb * 16; + for (int j = 0; j < ncols_interleaved; j++) { + sum_minf[j] += mins[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) * GGML_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d; + } + } + } + for (int j = 0; j < ncols_interleaved; j++) { + s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j]; + } + } +#endif +} + + static void ggml_gemv_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -3480,6 +4054,781 @@ static void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, c } } +static void ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK_K; + const int nb = n / qk; + const int ncols_interleaved = 8; + const int blocklen = 8; + static const uint32_t kmask1 = 0x3f3f3f3f; + static const uint32_t kmask2 = 0x0f0f0f0f; + static const uint32_t kmask3 = 0x03030303; + + assert (n % qk == 0); + assert (nr % 4 == 0); + assert (nc % ncols_interleaved == 0); + + UNUSED(s); + UNUSED(bs); + UNUSED(vx); + UNUSED(vy); + UNUSED(nr); + UNUSED(nc); + UNUSED(nb); + UNUSED(ncols_interleaved); + UNUSED(blocklen); + +#if defined(__AVX2__) + const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 * ) vx; + const block_q8_Kx4 * a_ptr_start = (const block_q8_Kx4 * ) vy; + int64_t b_nb = n / QK_K; + int64_t y = 0; + + // Mask to mask out nibbles from packed bytes + const __m256i m4b = _mm256_set1_epi8(0x0F); + // Permute mask used for easier vector processing at later stages + __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4); + + int anr = nr - nr % 16;; // Used to align nr with boundary of 16 + // Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation + for (; y < anr / 4; y += 4) { + + const block_q8_Kx4 * a_ptrs[4]; + + a_ptrs[0] = a_ptr_start + (y * nb); + for (int i = 0; i < 3; ++i) { + a_ptrs[i + 1] = a_ptrs[i] + nb; + } + + // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation + for (int64_t x = 0; x < nc / 8; x++) { + + const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb); + + // Master FP accumulators + __m256 acc_rows[16]; + for (int i = 0; i < 16; i++) { + acc_rows[i] = _mm256_setzero_ps(); + } + + __m256 acc_min_rows[16]; + for (int i = 0; i < 16; i++) { + acc_min_rows[i] = _mm256_setzero_ps(); + } + + // For super block + for (int64_t b = 0; b < nb; b++) { + + // Scale values - Load the eight scale values of block_q4_kx8 + const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d); + + // dmin values - Load the eight dmin values of block_q4_kx8 + const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin); + + // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration + for (int sb = 0; sb < QK_K / 64; sb++) { + + // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 + const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); + const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); + const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); + const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); + const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); + const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); + const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); + const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); + + // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values + const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240); + const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240); + const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240); + const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240); + + // 4-bit -> 8-bit + // First sub block of the two sub blocks processed in the iteration + const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) + const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) + + const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) + const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) + + const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) + const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) + + const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) + const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) + + // Second sub block of the two sub blocks processed in the iteration + const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) + const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) + + const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) + const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) + + const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) + const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) + + const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) + const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) + + // Shuffle pattern one - right side input + const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) + const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) + + const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) + const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) + + const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) + const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) + + const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) + const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) + + const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) + const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) + + const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) + const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) + + const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) + const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) + + const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) + const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) + + + // Shuffle pattern two - right side input + const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) + const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) + + const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) + const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) + + const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) + const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) + + const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) + const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) + + const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) + const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) + + const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) + const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) + + const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) + const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) + + const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) + const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) + + uint32_t utmp_0[4], utmp_1[4]; + + // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together + // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12); + utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp_0[1] & kmask1; + utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4); + utmp_0[2] = uaux_0; + utmp_0[0] &= kmask1; + + // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12); + utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4); + const uint32_t uaux_1 = utmp_1[1] & kmask1; + utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4); + utmp_1[2] = uaux_1; + utmp_1[0] &= kmask1; + + // Scales of first sub block in the sb loop + const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]); + const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0)); + + // Scales of second sub block in the sb loop + const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]); + const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1)); + + // Mins of first and second sub block of Q4_K block are arranged side by side + const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78))); + + const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68); + const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238); + + const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68); + const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238); + + for (int rp = 0; rp < 4; rp++) { + + // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 + // Loaded as set of 128 bit vectors and repeated into a 256 bit vector + __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 * sb))); + __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0); + __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17); + __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 256 * sb))); + __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0); + __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17); + __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 256 * sb))); + __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0); + __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17); + __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 256 * sb))); + __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0); + __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17); + __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 256 * sb))); + __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0); + __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17); + __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 256 * sb))); + __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0); + __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17); + __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 256 * sb))); + __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0); + __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17); + __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 256 * sb))); + __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0); + __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17); + + // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks + __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].bsums + 16 * sb))); + __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1))); + lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0); + + // Shuffle pattern one - left side input + const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) + const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) + + const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) + const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) + + const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) + const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160); //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) + + const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) + const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) + + const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) + const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) + + const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) + const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) + + const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) + const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160); //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) + + const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) + const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) + + // Shuffle pattern two- left side input + const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) + const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) + + const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) + const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) + + const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) + const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) + + const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) + const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) + + const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) + const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) + + const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) + const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) + + const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) + const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) + + const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) + const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) + + // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane + __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1)); + __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1)); + __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1)); + __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1)); + __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1)); + __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1)); + __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1)); + __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1)); + + __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2)); + __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2)); + __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2)); + __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2)); + __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2)); + __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2)); + __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2)); + __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2)); + + // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block + __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); + __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); + __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); + __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); + + __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); + __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); + __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); + __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); + + // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block + iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0); + iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0); + iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0); + iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0); + + iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1); + iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1); + iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1); + iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1); + + // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step) + __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204); + __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204); + __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204); + __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204); + __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204); + __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204); + __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204); + __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204); + + __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1); + __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1); + __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1); + __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1); + + // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes + const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d); + const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);//GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask); + + // Multiply with appropiate scales and accumulate (for both d and dmin) below + acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]); + acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]); + acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]); + acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]); + + __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0), mins_01); + __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85), mins_01); + __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170), mins_01); + __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255), mins_01); + + acc_min_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]); + acc_min_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]); + acc_min_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]); + acc_min_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]); + + } + } + } + // Store the accumulated values + for (int i = 0; i < 16; i++) { + _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i])); + } + } + } + for (; y < nr / 4; y++) { + + const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb); + + for (int64_t x = 0; x < nc / 8; x++) { + + const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb); + + // Master FP accumulators + __m256 acc_rows[4]; + for (int i = 0; i < 4; i++) { + acc_rows[i] = _mm256_setzero_ps(); + } + + __m256 acc_min_rows[4]; + for (int i = 0; i < 4; i++) { + acc_min_rows[i] = _mm256_setzero_ps(); + } + + for (int64_t b = 0; b < nb; b++) { + + // Scale values - Load the eight scale values of block_q4_Kx8 + const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d); + + // dmin values - Load the eight dmin values of block_q4_Kx8 + const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin); + + // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration + for (int sb = 0; sb < QK_K / 64; sb++) { + + // Load the eight block_q4_k for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 + const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); + const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); + const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); + const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); + const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); + const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); + const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); + const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); + + // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values + const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240); + const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240); + const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240); + const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240); + + // 4-bit -> 8-bit + // First sub block of the two sub blocks processed in the iteration + const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) + const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) + + const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) + const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) + + const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) + const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) + + const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) + const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) + + // Second sub block of the two sub blocks processed in the iteration + const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) + const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) + + const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) + const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) + + const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) + const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) + + const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) + const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) + + // Shuffle pattern one - right side input + const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) + const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) + + const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) + const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) + + const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) + const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) + + const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) + const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) + + const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) + const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) + + const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) + const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) + + const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) + const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) + + const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) + const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) + + // Shuffle pattern two - right side input + const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) + const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) + + const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) + const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) + + const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) + const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) + + const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) + const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) + + const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) + const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) + + const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) + const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) + + const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) + const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) + + const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) + const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) + + uint32_t utmp_0[4], utmp_1[4]; + + // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together + // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12); + utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp_0[1] & kmask1; + utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4); + utmp_0[2] = uaux_0; + utmp_0[0] &= kmask1; + + // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures when sb = 1 + memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12); + utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4); + const uint32_t uaux_1 = utmp_1[1] & kmask1; + utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4); + utmp_1[2] = uaux_1; + utmp_1[0] &= kmask1; + + // Scales of first sub block in the sb loop + const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]); + const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0)); + + // Scales of second sub block in the sb loop + const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]); + const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1)); + + // Mins of first and second sub block of Q4_K block are arranged side by side + const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78))); + + const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68); + const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238); + + const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68); + const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238); + + // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 + // Loaded as set of 128 bit vectors and repeated into a 256 bit vector + __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 * sb))); + __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0); + __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17); + __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 256 * sb))); + __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0); + __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17); + __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 256 * sb))); + __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0); + __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17); + __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 256 * sb))); + __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0); + __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17); + __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 256 * sb))); + __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0); + __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17); + __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 256 * sb))); + __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0); + __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17); + __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 256 * sb))); + __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0); + __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17); + __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 256 * sb))); + __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0); + __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17); + + // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks + __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].bsums + 16 * sb))); + __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1))); + lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0); + + // Shuffle pattern one - left side input + const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) + const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) + + const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) + const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) + + const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) + const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160); //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) + + const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) + const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) + + const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) + const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) + + const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) + const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) + + const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) + const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160); //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) + + const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) + const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) + + // Shuffle pattern two- left side input + const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) + const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) + + const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) + const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) + + const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) + const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) + + const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) + const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) + + const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) + const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) + + const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) + const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) + + const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) + const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) + + const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) + const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) + + // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane + __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1)); + __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1)); + __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1)); + __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1)); + __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1)); + __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1)); + __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1)); + __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1)); + + __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2)); + __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2)); + __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2)); + __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2)); + __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2)); + __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2)); + __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2)); + __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2)); + + // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block + __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); + __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); + __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); + __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); + + __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); + __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); + __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); + __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); + + // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block + iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0); + iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0); + iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0); + iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0); + + iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1); + iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1); + iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1); + iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1); + + // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step) + __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204); + __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204); + __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204); + __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204); + __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204); + __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204); + __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204); + __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204); + + __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1); + __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1); + __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1); + __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1); + + // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes + const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d); + const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); //GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask); + + // Multiply with appropiate scales and accumulate (for both d and dmin) below + acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]); + acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]); + acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]); + acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]); + + __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0), mins_01); + __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85), mins_01); + __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170), mins_01); + __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255), mins_01); + + acc_min_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]); + acc_min_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]); + acc_min_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]); + acc_min_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]); + } + } + + // Store the accumulated values + for (int i = 0; i < 4; i++) { + _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i])); + } + } + } + +#else + + float sumf[4][8]; + float sum_minf[4][8]; + uint32_t utmp[32]; + int sumi1; + int sumi2; + int sumi; + + for (int y = 0; y < nr / 4; y++) { + const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb); + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + sumf[m][j] = 0.0; + sum_minf[m][j] = 0.0; + } + } + for (int l = 0; l < nb; l++) { + for (int sb = 0; sb < 8; sb++) { + memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12); + utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1; + utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4); + utmp[sb * 4 + 2] = uaux_0; + utmp[sb * 4 + 0] &= kmask1; + } + for (int k = 0; k < (qk / (2 * blocklen)); k++) { + uint8_t *scales_0 = (uint8_t*) utmp + (k / 4) * 32; + uint8_t *scales_1 = (uint8_t*) utmp + (k / 4) * 32 + 16; + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + sumi1 = 0; + sumi2 = 0; + sumi = 0; + for (int i = 0; i < blocklen; ++i) { + const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF); + const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4); + sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 256 + (k % 4) * 4 * blocklen + m * blocklen + i]); + sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 256 + (k % 4) * 4 * blocklen + m * blocklen + i + 128]); + sumi1 = sumi1 * scales_0[j]; + sumi2 = sumi2 * scales_1[j]; + sumi += sumi1 + sumi2; + } + sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m]; + } + } + } + for (int sb = 0; sb < 8; sb++) { + uint8_t *mins = (uint8_t*) utmp + 8 + sb * 16; + for(int m = 0; m < 4; m++) { + const int16_t *bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) * 6); + for(int j = 0; j < ncols_interleaved; j++) { + sum_minf[m][j] += mins[j] * (bsums[0] + bsums[1]) * GGML_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m]; + } + } + } + } + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j]; + } + } + } + } +#endif +} + static void ggml_gemm_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -3660,6 +5009,82 @@ static block_q4_0x8 make_block_q4_0x8(block_q4_0 * in, unsigned int blck_size_in return out; } +static block_q4_Kx8 make_block_q4_Kx8(block_q4_K * in, unsigned int blck_size_interleave) { + block_q4_Kx8 out; + //Delta(scale) and dmin values of the eight Q4_K structures are copied onto the output interleaved structure + for (int i = 0; i < 8; i++) { + out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d; + } + + for (int i = 0; i < 8; i++) { + out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin; + } + + const int end = QK_K * 4 / blck_size_interleave; + + // Interleave Q4_K quants by taking 8 bytes at a time + for (int i = 0; i < end; ++i) { + int src_id = i % 8; + int src_offset = (i / 8) * blck_size_interleave; + int dst_offset = i * blck_size_interleave; + + uint64_t elems; + memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t)); + memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t)); + } + + // The below logic is designed so as to unpack and rearrange scales and mins values in Q4_K + // Currently the Q4_K structure has 8 scales and 8 mins packed in 12 bytes ( 6 bits for each value) + // The output Q4_Kx8 structure has 96 bytes + // Every 12 byte is packed such that it contains scales and mins for corresponding sub blocks from Q4_K structure + // For eg - First 12 bytes contains 8 scales and 8 mins - each of first sub block from different Q4_K structures + uint8_t s[8], m[8]; + + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 8; j++) { + s[j] = in[j].scales[i] & 63; + m[j] = in[j].scales[i + 4] & 63; + } + + out.scales[i * 12] = (s[0] & 63) + ((s[4] & 48) << 2); + out.scales[i * 12 + 1] = (s[1] & 63) + ((s[5] & 48) << 2); + out.scales[i * 12 + 2] = (s[2] & 63) + ((s[6] & 48) << 2); + out.scales[i * 12 + 3] = (s[3] & 63) + ((s[7] & 48) << 2); + out.scales[i * 12 + 4] = (m[0] & 63) + ((m[4] & 48) << 2); + out.scales[i * 12 + 5] = (m[1] & 63) + ((m[5] & 48) << 2); + out.scales[i * 12 + 6] = (m[2] & 63) + ((m[6] & 48) << 2); + out.scales[i * 12 + 7] = (m[3] & 63) + ((m[7] & 48) << 2); + out.scales[i * 12 + 8] = (s[4] & 15) + ((m[4] & 15) << 4); + out.scales[i * 12 + 9] = (s[5] & 15) + ((m[5] & 15) << 4); + out.scales[i * 12 + 10] = (s[6] & 15) + ((m[6] & 15) << 4); + out.scales[i * 12 + 11] = (s[7] & 15) + ((m[7] & 15) << 4); + + } + + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 8; j++) { + s[j] = ((in[j].scales[i] & 192) >> 2) | (in[j].scales[i+8] & 15); + m[j] = ((in[j].scales[i + 4] & 192) >> 2) | ((in[j].scales[i+8] & 240) >> 4); + } + + out.scales[i * 12 + 48] = (s[0] & 63) + ((s[4] & 48) << 2); + out.scales[i * 12 + 49] = (s[1] & 63) + ((s[5] & 48) << 2); + out.scales[i * 12 + 50] = (s[2] & 63) + ((s[6] & 48) << 2); + out.scales[i * 12 + 51] = (s[3] & 63) + ((s[7] & 48) << 2); + out.scales[i * 12 + 52] = (m[0] & 63) + ((m[4] & 48) << 2); + out.scales[i * 12 + 53] = (m[1] & 63) + ((m[5] & 48) << 2); + out.scales[i * 12 + 54] = (m[2] & 63) + ((m[6] & 48) << 2); + out.scales[i * 12 + 55] = (m[3] & 63) + ((m[7] & 48) << 2); + out.scales[i * 12 + 56] = (s[4] & 15) + ((m[4] & 15) << 4); + out.scales[i * 12 + 57] = (s[5] & 15) + ((m[5] & 15) << 4); + out.scales[i * 12 + 58] = (s[6] & 15) + ((m[6] & 15) << 4); + out.scales[i * 12 + 59] = (s[7] & 15) + ((m[7] & 15) << 4); + + } + + return out; +} + static int repack_q4_0_to_q4_0_4_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) { GGML_ASSERT(t->type == GGML_TYPE_Q4_0); GGML_ASSERT(interleave_block == 4 || interleave_block == 8); @@ -3690,6 +5115,36 @@ static int repack_q4_0_to_q4_0_4_bl(struct ggml_tensor * t, int interleave_block GGML_UNUSED(data_size); } +static int repack_q4_K_to_q4_K_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_K); + GGML_ASSERT(interleave_block == 8); + constexpr int nrows_interleaved = 8; + + block_q4_Kx8 * dst = (block_q4_Kx8*)t->data; + const block_q4_K * src = (const block_q4_K*) data; + block_q4_K dst_tmp[8]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_K; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_K)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++ ) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q4_Kx8(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} static int repack_q4_0_to_q4_0_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) { GGML_ASSERT(t->type == GGML_TYPE_Q4_0); @@ -3807,6 +5262,10 @@ template <> int repack(struct ggml_tensor * t, const void * da return repack_q4_0_to_q4_0_8_bl(t, 8, data, data_size); } +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_q4_K_to_q4_K_8_bl(t, 8, data, data_size); +} + template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { return repack_iq4_nl_to_iq4_nl_4_bl(t, 4, data, data_size); } @@ -3832,6 +5291,10 @@ template <> void gemv(int n, float * s, size_t bs, const void ggml_gemv_q4_0_8x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_q4_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc); +} + template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemv_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc); @@ -3853,6 +5316,10 @@ template <> void gemm(int n, float * s, size_t bs, const void ggml_gemm_q4_0_8x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_q4_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc); +} + template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc); @@ -3863,16 +5330,16 @@ class tensor_traits_base : public ggml::cpu::tensor_traits { virtual int repack(struct ggml_tensor * t, const void * data, size_t data_size) = 0; }; -template class tensor_traits : public tensor_traits_base { +template class tensor_traits : public tensor_traits_base { bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override { // not realy a GGML_TYPE_Q8_0 but same size. switch (op->op) { case GGML_OP_MUL_MAT: - size = ggml_row_size(GGML_TYPE_Q8_0, ggml_nelements(op->src[1])); + size = ggml_row_size(PARAM_TYPE, ggml_nelements(op->src[1])); return true; case GGML_OP_MUL_MAT_ID: - size = ggml_row_size(GGML_TYPE_Q8_0, ggml_nelements(op->src[1])); + size = ggml_row_size(PARAM_TYPE, ggml_nelements(op->src[1])); size = GGML_PAD(size, sizeof(int64_t)); // + padding for next bloc. size += sizeof(int64_t) * (1+op->src[0]->ne[2]) * op->src[1]->ne[2]; return true; @@ -3925,16 +5392,23 @@ template class tensor_ // GGML_ASSERT(ggml_n_dims(op->src[1]) == 2); char * wdata = static_cast(params->wdata); - const size_t nbw1 = ggml_row_size(GGML_TYPE_Q8_0, ne10); + const size_t nbw1 = ggml_row_size(PARAM_TYPE, ne10); assert(params->wsize >= nbw1 * ne11); - const ggml_from_float_t from_float = ggml_get_type_traits_cpu(GGML_TYPE_Q8_0)->from_float; + const ggml_from_float_t from_float = ggml_get_type_traits_cpu(PARAM_TYPE)->from_float; int64_t i11_processed = 0; - for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) { - quantize_mat_q8_0((float *) ((char *) src1->data + i11 * nb11), (void *) (wdata + i11 * nbw1), 4, ne10, + if(PARAM_TYPE == GGML_TYPE_Q8_K) { + for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) { + quantize_mat_q8_K((float *) ((char *) src1->data + i11 * nb11), (void *) (wdata + i11 * nbw1), 4, ne10, INTER_SIZE); + } + } else { + for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) { + quantize_mat_q8_0((float *) ((char *) src1->data + i11 * nb11), (void *) (wdata + i11 * nbw1), 4, ne10, + INTER_SIZE); + } } i11_processed = ne11 - ne11 % 4; for (int64_t i11 = i11_processed + ith; i11 < ne11; i11 += nth) { @@ -3944,7 +5418,7 @@ template class tensor_ ggml_barrier(params->threadpool); const void * src1_wdata = params->wdata; - const size_t src1_col_stride = ggml_row_size(GGML_TYPE_Q8_0, ne10); + const size_t src1_col_stride = ggml_row_size(PARAM_TYPE, ne10); int64_t src0_start = (ith * ne01) / nth; int64_t src0_end = ((ith + 1) * ne01) / nth; src0_start = (src0_start % NB_COLS) ? src0_start + NB_COLS - (src0_start % NB_COLS) : src0_start; @@ -4098,12 +5572,13 @@ template class tensor_ }; // instance for Q4 -static const tensor_traits q4_0_4x4_q8_0; -static const tensor_traits q4_0_4x8_q8_0; -static const tensor_traits q4_0_8x8_q8_0; +static const tensor_traits q4_0_4x4_q8_0; +static const tensor_traits q4_0_4x8_q8_0; +static const tensor_traits q4_0_8x8_q8_0; +static const tensor_traits q4_K_8x8_q8_K; // instance for IQ4 -static const tensor_traits iq4_nl_4x4_q8_0; +static const tensor_traits iq4_nl_4x4_q8_0; } // namespace ggml::cpu::aarch64 @@ -4124,6 +5599,12 @@ static const ggml::cpu::tensor_traits * ggml_aarch64_get_optimal_repack_type(con return &ggml::cpu::aarch64::q4_0_4x4_q8_0; } } + } else if (cur->type == GGML_TYPE_Q4_K) { + if (ggml_cpu_has_avx2()) { + if (cur->ne[1] % 8 == 0) { + return &ggml::cpu::aarch64::q4_K_8x8_q8_K; + } + } } else if (cur->type == GGML_TYPE_IQ4_NL) { if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) { if (cur->ne[1] % 4 == 0) { From dbb3a4739e53226346c772dd72666e68b78dc583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Thu, 20 Mar 2025 12:49:59 +0100 Subject: [PATCH 47/55] llama : make Qwen2MoE QKV bias optional (#12477) --- src/llama-model.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 17af8cc30b..cd7e0a0c4d 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2210,9 +2210,9 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0); // optional bias tensors - layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0); - layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0); - layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0); + layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); + layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED); + layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED); layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); @@ -6193,16 +6193,25 @@ struct llm_build_qwen2moe : public llm_graph_context { { // compute Q and K and RoPE them ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); From e04643063b3d240b8c0fdba98677dff6ba346784 Mon Sep 17 00:00:00 2001 From: Woof Dog <197125663+woof-dog@users.noreply.github.com> Date: Thu, 20 Mar 2025 14:57:43 +0000 Subject: [PATCH 48/55] webui : Prevent rerendering on textarea input (#12299) * webui: Make textarea uncontrolled to eliminate devastating lag * Update index.html.gz * use signal-style implementation * rm console log * no duplicated savedInitValue set --------- Co-authored-by: Xuan Son Nguyen --- examples/server/public/index.html.gz | Bin 1260534 -> 1260624 bytes .../webui/src/components/ChatScreen.tsx | 78 ++++++++++++------ .../server/webui/src/utils/llama-vscode.ts | 12 ++- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/examples/server/public/index.html.gz b/examples/server/public/index.html.gz index c7a3c426b623c707bf17b900886abb3a3a57c8d6..d0e6da8e4a1e0646ad83f40bceea6462506ffc8a 100644 GIT binary patch delta 1176841 zcmV(+K;6Ig_e{|EOpqiGvXf4HDLYOj8&G2+_@ELB6$+F{^mFzBxcTKI%YB&usEJML!NNKBoCdN85U5bCn6p2EQ3<} zsgtDA;b9~l2@PT+VI=rK#xuzAId#VDjyr=9ONKlnj^MXkI5Ncw-QW0)bMe7-GCq(| znmAE{@nu=5GyT5I(h3P=fCfpgxas2LY+v>-0wnvAEPpVMp=yz4St_Dol)RfV;runGoW6UMBaz-apJx-^7a|oN_ZMAA zJ`AbHi6n_n-^td9WghqE^S{QPyT=k(PIo==G3S0#=JXCj{B{LrA#9zoGZE30N!VFD z4qcxly`%*@$K!q=TSG2A?)&6pwA@_JYK?^($3dE8(vdVw2l<#MvNaH#N#2h+ggiHnk~`OL z2_AonGTDu8J%8C~$X1q5V1s7%x2G3;B6ys#(Di-6WiFELZ7Mv~P6|i6XzH(=i`hAk z`9P*(Cyu@U39VfU9@AP?SO3%ZBQN#43BPxaa>=0gCpQ@vQ!c!ezI!a&h@p+tk8ZuV zh5L|FcQA_L5C-7-WyTCL#;`U#JWfNN`B_V{;V~QzN`Ku?AC5n|jYbAzhj6t=jYi~U z{=#pisu!La6?rTg3)b2s%WtIMJXmWl{H~dIC-G7<^)Y>1bk#&TX)8)3ABsrM8VxTl zL)1#~N!mgnCAc_E<7lArR-Y0<(^Ti*CGnz|1z6F?=+;YHd}8(reABgd*Gvw~RQHz4 z!N^NnBf)P;>@OC+rvSh&QwmF*-a5lLynBHTe`V1;1tx_f- zqHJ?=m1MdOVIZ(U#W>b(R*OB9uytPi{qKKs{_gx& z91VDqap#P)frJcX% zPB@t(~R07;tBhh8!+s-E!#UNeEZxh&x9gKAULgHqD_ryf~qX zPapRB$LEv61Sfx??7kn<4sH%k&u8PCG;YC>chz)o1=n1k$Q7wDw=4!Axgzmau%Rv+ ztw_$Khz9a{oQB->NxG_bk;0gT>yvnO@-zxDIa`s8_wxg^K3LVcRNONWUJHKf`XpS{ zzdW13dbmEBtVka7aq9YHwjwEJ-)F8*h9)VJfOVy!T!()L@31%yhm_JJkK_4VAEU!H z>ZW)k+#VzZ-XR?}8i$v|tNHx!(*3XhG8MX4#FU}bHMP6F!vMu6LL&# z$guAnk}-cLyZ$3=U|7>hMh^)F^a|23S7E_#GkUDPbL^;~yYu4J)pg-$vtb4VTPU7@5MxTvM~xYPI~pk?f9V4q!T1 z7o5=ylAS3N5xa?b255^Khm#72ImM*hkD|fIQ7eD+d=)L%@7nXxs;z|wi3F#(=%4FX zXXLhAcxzJhg4F7@Hi@a)waUA`wX|oj{%2)F)M~WM_36K31HeW^-nKxCZm2*GBpD8pY8bNYkJihQAugIoxnNIZlK zGT?vo@s7^`w-Njney*?af>d7!u>spDr{tIya{|2>N^*yFRt#8^_-U03mt?#EnRxMNO+(r+}@2K6y| z81LQ6%b~oYV-7zGy?Bt6i6NqOSu5trP>z3^8_>@ONh$64x3Y!Ep5O`H?eaL|PIT*a zwjZRA2r1ODt0&!ikUse;X}kMXZa>KGSGm6K@AAv9@~Uj)@~d3mxawBw<%{UnJ8U!# z4Uz!;&a4@FkhGdc9;B~FW6g|?d5~&+>Nr0B@{0lq0Y}&y91>XESmNRyOM)9XxRpWV zI=xT02)?xRc36~n+#T0I=czgo566nf94kJA<srl&fbo{gXZ zpWq4{OWGzwX^%J<%5^tzVX%k3Zy?J90Fwu~eW*cG_W?l4V=U?6<%7KHYUneRFk!I- zsL(x@>+~~*Z~}HYY@daJT)UhCDrc~;U2dac*zO_S)$sJte<#?1HP~y1t%N^F&-Ytl zn((e4OB&18l*KtG)h?{B`($M&3h=2ms@^>sB^h(7&%vQjG=8FPuFPSABa?5+;wW?& za1`)EK*agqD?My!*SO&;Pg?UW-9M~luei%jJOB}lGrJ`5WSZsx6v@T;gzHlbq`wz)~V+|R#0~~kchPd zbq+^EC3EOFlHnysK=I3w4TcT0Tz^r}(XBm$fjo}cBsk;bHWhs~7zLAO3u}KLBCn=Z zqT~AzJkaa}nn37O^9E>&p;04Kxb(uOq_;od*0)M zX!OtrFo$>%!TSdAM?exZwpbw^9x+fS8)8E?K)C{qT}4COR%i%?y#kts^ic%~*Vpih zBxHK;az(+|M93~*O@v^)H(T`KS6(}z!gqA{_qJx zWtaPp4^k5-yWH;@+`Ldgs$}hs=8E^VhMvx zK*R9EmZ`$(M@+6YaUXviLKagmJ+#JTPS-In;|>^dJn0H>c+6WjQ4)ICkHZ3*z-0>@ zfkT9cMMY=)mM*cws&^9R){MeR_QPe$Z9o(%6KLQSWnTcjGDG+ValjXRwvrjSp0F8+ zOM*vzwFb=&(HAGo)HMh%gG4Vq@F44ata_ajL#7VW@q`H;AX0y)PX@9V$6VExNhajy z;tF0h#W%cF-qGDk2)KGu%pMOte{mZnAa8!GW-#sKAn?11`E>zH^NjDYIKE+nyVb*S z0BEYEu`{bY3sc3@N1#?*W%LeF-wzQm%f`rPqA{Nh!>zaRN4^5FLG6KH9|8AnYkJik+*P^?Rh3M|DtoyMP z-NDSRl+l$EydMhbe#F~ODgO^J=hhN2M&f^$C7yW$5|W9!h-PF6?C=N04Zl;&@QIC( zJ^(&C@kxJJ$5b&PRCsDwM|kfjMk%BYxD8I|YbBD9%m53G>qzJfanQ@z6%8+ES3Y2n zgGOVp%z0MT(P|0_eJpGB!K$l%lS_4z4-hYWz?r)=K)D`YUVwnLW@< ze`NTDr}T@p*Z!QAr}PV)(l2mIe=ASvdpxCO1lE7>7lm>p(R=ZH0E_gsE`t{k`+F_+ z*YH`%Jg8iOs(|?4?}#5-)Be$#_CuKVLzwmg4s-vp$SS=XPA<4Q52t{aX_n@VW{+NQ zzsm#IdENH*X;)=Dq^Hf!yLaz81aY~`!zrxQ-f-&;pke)E**0iWcP4;w`)XbbmQ;d;d;k!c7QIvL75d0#zVQ@0s8ia zU&c~R$Ga3b)~CK7%k435G$7&_#Rff3P0)X_Bq~3kuQZl^pc8H}VW<;^6(+4+*QuH6 zdQG*t){}E#NoLZemCsTkKO`Z4XkJ(WL;k?TwR@+*4R1vuaB<*fW!V5OXG0m9P_3 znNL<5AI4)I0<6&rQDp???dqHfn?^%W<$!FBMiFdjkQXIG>xNpD+={Xf$t0JjJPH5p z+?rrskZ#0!1hOMj29wMinqSl9Qsaxjg?KzfW5ySvRB zZnqb6a{|@w0s%XGoO#FGC&wJP?6CWk3c)FTK(ZL7?*O1O>`^cDyThVIZ_OLlM*i*s zG&OP5R;PJ!lXwtdGf ziSjj9==i|w9JN9>%Tw3=MsK*e+Sb~*8!k8u`{E*P(@s+&iYU=r z(n`6=xY&);!CjOLi?SPbjsaY#@iJ5F22p>Y>H%L=3Z{vSlAQM+_#m&-@q6?6-F)s@ zpwO=rMoq2*!N&Zo!j^zm+wX4peBRl3xqZWXpjDN|yv3DNqgu{rzB!W=_ybcwi$RSGLHlpL5jhDoMVo@@5ZlR*95oZCsZB$RXg;EvNabmWN1QR{y; zPE+A^_~wfn?guX`LmF~Jd7j!Ja0O!VP%n27iAHn~3S`@2vOdt$MWqW91{}my>tD$x_+L5C zN4agTsH5yJ~d!v!#As2Gt?naJp#?@uo7wi)V*u2*Ond zgj{fPYVOM=V5~c=q2Z~G^qy)+-t$iBX)EM0n~_s$<w0F3E8XWq;DLo|C z==IIo>{yCD&Ia(@H(uJxKF&*s>$G2QzTWD*+1Rv`wlIlr)po>~)0cd6QAmF^=;`{H zqu4wOgW38zF+d@>;G}ZjgFw8vJA+eFz4JlEtU$toW@kZ8-yH&_W1(iJM0yh#CxH!5 z&CPtn@y4G36Xdw(*%(l83_wO*-KWh%-$xqBs#2qOQ0EG;*sC`l4V-I0`&EO_w(=j_j{ zm5SkuFdbws@P=xJ9L6bHU}@Nk5>%fC)4g>;a@y(U+jU`ZH(y`(**br98%zwd%PZ=x z^91U+IQvj28hE92H!t{g?opIPw-FDG>>fJc{110sth-%j8f6jKhNS53BbM*iG{Hs)}P^OvoJ({Jd2$Xk0-Zz?A$YvM9DB~ zxwum!MDotLBxywpKlXnFVg6%=KP2%)-A^2AufxQe06+1ZbgbGc#>s zecg}HdXHTSEo&dpHLqe*^v=!Q!rDPawTv~`b7@tvFysWwjUO?sC-lYtaT)A1f4^qU z?|0HV?3wt@hR5e|bS>z|b95w$lTgPJNg|#Xuht;rG_sfey|v*a;>r$usE* zzc+;awAK#1nfiYqYaMIj-ND8qIT<0S$P3yqHx`SAk+{~WUGYro;1*<((!$M965HX0ZJz|(a7;!-cHaLS?XRffvy)jjU{% z#+gAo^q`^1wT?kkbYzqrvty6>D*9DDdOq*GV~b)mr745yCYh2xInqWRwOORl;>LUq zKbz`jOBs1EDMfUXOO7_CqZN_agbg^Isua+hy2!XW`ZOnnO$mgt5~1R7+c5Q=F|Y{# z&M(eB(Y}ANG#RH;-XBloOzF|mBZOu9u+Lqi!sCQs1257%i)ZIten=87-hV#&9~~nn{lD^P>Jt2JatvHut?Zq;x5Z`qs=I&5Oh*~%U`H=ISM|QSgX%1~ z#0>=-4)u37;c<*p>=GCCsNJg{Ps;|jRz1K=4TBGbGY-;iXozd^`59M;O?C$36gcjp5g6%x370K%{YI zwj2C%;e-)(C^R3Z(EzUX4~hOT@n?LTN)9yv9u4Tl_&7xsCZOm?Q5f>XiPLoAB&o97 zb&@hYo$$n&#B7#*Na85rfV+k#pav@R>^Ofy&0y$cgLDEvoQ-2JxQ`?spF`Zgx1%?I zX6Xj%%(=uJ=dp|?xH98hVpN8P^gR5T#n~hvt6r|(AW3eR-Y3fP{>PPVtvrn2ULtde zE7E)T$)fLSU$f+A>U_1jPLbZ9Lk+}50`O;QdileNMa2A!zPS9O8C-dntoi-Qx2=D9 zK3Ug49OafwFD4UH*8+h|BAX+Qqzj9qA>y>n8*hzZirrM9EN(c+A8d7fqgo=5rhspSTm9bjp7PuB+k_ zf@IUg$Wb)7OVAkyFsGWS0FrdVk~HJZjx$MdEt`A_q?c3l370sLPBvl_yQ`T@C%Rww zX%+&kvhFyjXXnFF*^O#+*_TX;^o}<}mW`A(iWQOG-ez2y7#PHWP3$0;!EYv)4F6si6uN3}c$b`j_o)brEd-uiU=_h3fb%eplPq`u_a@GRKJY)hA2d!;SdV|5kx%|A?4o|h zVlx37$QG>_ed~S`xfz|eZ{NPa zwp<&)KaAr5pp#4RGiJ$9eJT!`=B zcY^Wi`xq@+_dMvul#C#dD_W6^=JTnhI(rWuQ%O23G-b47XfTiYfdHopRt$07_1yP^ zX`LYJiOXq)8oQ!J%~M0kZvxHjx`(_!eN`qS*sVo=7 z4^Mkfgp7~Hj_h{m4Z8e{5#=g@`Ef`1Ko_lbBt z9u&?1AIwK)m?grg4wC3eQug~L7%ln2K$54E%q5Q*m%)=2Eec(iGkyyMqAeoSQKeTA z{}KAP6zwr`d1YpPs6|mDI^>Jxtb$x#0h1k7V4yapSfqAwEGksIH6*LhswFQ=!nn#f z(7@r1#;DNq8*;TLC^fDMN`K+Xl$unH-Td=!|MW?pj+P^j=+xNKb;t$n7lCePn;wDK zlwLF%)5{~cf%=yhS6%<2*+gw?D@XUJ*(4XuCM>v>7rTA5Mrmh0zknYOYpO_I~A=k%7BXj7fYk@wWS@pWq+Q#U3hSNhZpBp z`n9akP9&8|e4l(>+>fG|d*uLqCD>go3ZWvLCi2ud$o269R&%dX`+P21fP@9E>-P{A z1g=~1UT=Y~ZY1;qJoRbLZ^6`sGKwXv*t(rpR*L%xL8VUFv{TLtD6Lc9u1Zzz167W8 zMK7RJ^gvL69qMuhRDVq93Og$@v=qTp$+xneHl$ZM?5=BT@JS6eMb$@uajLTlB>SuO zh*^O7d__@nl1xD(mk@dt2Hf<`=;ym?P+v^^sk-h}>|K%Fm_F--<07FndDcnyWs+Gj zmGF5f0T*jg9f5HvQEZ`_Ly~h-i&vW!ch4XK>uzD%shj>*mVYrOpwARGrlY#>H4jSC z1vW!*bZMmvM9k-&psZRm(~IT0L{JpRxIOovu@i=%@W(<295ua4xr(&OIP|Es;-GIB znU1^11|zfh<`%Ge?B`dl(=H4B`^(mOWBRerM<~tUlb<0bBBiM*X+Wp@8783)Ph(lg z&feH^wqxG2k8R*ur*~t+RC!X9=OG~hLX!z1M}G~Gdi5KP8pi+}?nI!W?=~7J{8ciH zoAkj|l2KQ%FiKrZHvZLu1=wt_kb4AL6h?AGL=3sK*bMtz`JYaOCu~=F7S_ysG6JJH zammm;W=$-!Dhfn0S7r<*1g+7EwWr+1do6+*Kc69_bIToBu+J7i;h)&(rU z$4fLdYDRGB+@#@5-^WEJj4ON77m5TnbD@V>SYd%@7DgVcx&XRgq~bju?$GAjG7J?s_XGOKQ^o6`sBF|~3< zZq@PJC$*l^$7)xD5z!H9+&R%A>=>IWJ&~8-uZUDIuvhD9Q0a=bIDcyX1SC}vf?qc3_6bq#u}m!tM^!haCFD}5WJC&04NTrfRkp0)wdPu0v**M&``yP5+z zY0{8!8Wd?5t|u239oR~i-+%}4`5Z%_b0A$f{dG2|z6cDhyI^%FZZh;j+{v0>>Icgi z?fnA8uw4LW#?IjTq?mJ~v{m0nh1)<;OQ36O^}N>F943m3uJojy8V_un=YL0FxHIKC zu6d)ia&8+945_(=>dfjGL%T8tSKoMMXLxO{KCbGp&2z%&f=O%B?(~1I%rqrTM-Ftzcg#w1_}+;4A(9 z!4w45?Dm$gSCeag`?;t(<$tx@x+N9WIY6}138ZA|@PmqC!PRPar`x$vTb+Wo?#*^b24|`Bia)!M9N^fo%fAvE%5}CsX9VsdzoobCV3h zB4q5TKDh4sgY^(z6-nWINZzWRRLmh{}IDkStn8LO;@&8fPaNG!_8JXwf4|_ z?t!w_^vW-C0D5XyZ<|Q|8WsFt)_OumZiNT!#CzNE{rZ(zxd>O?ltvpb_`zPY@5bM2 z*EKeRAe@#i(bPSzJyEjS4N7v$c^X?inhhUq*UyEQ`YP2cb4GzW`$M)L(L zCmGLoPTAtA`&3N^ZhsML4X!IAtln75P`aBJKo~b%j_^T)BkKl&UAO{~fZMHhj#nJw z-1o_78Q2#md}J-qucAUf@dv53B)bY?KE2G12GgL;+NvdBwP;0d^GA)V-s#W~cq@d* z8|`-c1!PeKA!tB6lN+VvV4;Tpjy}0{g7GQoG$>cv#N-FA)_<}TIq5SBcmA`L^d6Li zc6G*LIE@C9mbariHVarR;D`AM12-9lH_jS@%r zqE$pt)-6Evoqwcvaz20RMW8U6g1l8)h28CJI(+A@>oa+B>-nYUYJc6`eu3$P(L33? zMLR@*LtSA9e1qj-(6iH&3!5&hWrq`!getq82Gp+QhNP6{74W0dFn~rSI~T2(=~&Rk z2e!s%EyedARam1h=7L#N4l`xSra7UHEQ!Xb-1GruqkmDFC=Y~dZNf5eC3*&t5WGQs zun(Y+c9Kh==BttCpnzYC`wdj8RNTWs%pa`JLy_ha9X=5uh-8YWK^o`dvLh;(!RM{6 zYSf?oB(y@H&}j!joz8^! zu!q%m;D=O#pdPnx6!5B1N(HZsJ1Vs7%DAPv%(hsN z-x7KWkJsHrpawq%4cw#jyGz^?_o{3Aw!TkR>wk9rmMljrl5-uH9!%^=ukBY=Vb?_~ z%Wo9((d+o+J^>Z1`{Yu9uYBuBek(|wl6PFS3b9<(Yd)b1)O}v&ShMhWrhG%+Cv{vF zP>2kzLJueAgMw(d?IEZO3o#x4NIM!nPmv(CXBP)Y0SmFESimA4|;3MaRo+ibeP z^nYxVDgBmsIboOi6&V3C1Q$}AGUSBj)E#8m0S0}er&Cr;rhWjZG@UAk4~d|s-s6}j zIcS&T0zWc+yg*wHe}z8M@B^>Y;VBtmY&0I?4=`fkuMg}`98rHEj_x45U4d2Fjg|Ke= za);ptYZe~Li@Kw%lq=Mvvo1bmsMMn$0NKIFinh3p)9T!VrEn9J3hQyXh?e5ee}9r( ziUv$YM~(s5t+G_L966Ketn3nHRMPCUdgks2%vzduO!RJ)gmIMR= zLhx*oqKlx5%wRHrM$;!#fd^sV|2@$HMTSIz=LC%_{;plH3?ubBi)uanN6FLO%YZ#? zJh#27k(v zn(%{JL@&ivFpQ|MK%n*jGmI2e6Lc66MrRQmoLNN8AaP8Dp%=gv*0TQ)+EmsO=+w}C z&!$>-B=~5624zxF!V)$+OeVgK6#HGs+_u@=su7ZxrtMfC@rF^s1DKglV1MvH!(_;iM>`m{J<(QROkJ9Oin3Y*CCiU>jY<;k1a072d*G2`$#YY7Oy+=oM z%|jCICYLgW8(x%hLLE@77=IH292+h?*9VEp+Ni5t%9S7k{0YH9t1G|crI7Bq^S0R` z%!pI+l4+BxooOD&8K62M+VSJti>oDSToM;N+DB9FqWr9)ou?kV@rgDDXJOa}!w-;D z!*j--3l_vK;gWD!*xzewG=QPsmm0feJfb%1YtvD!HXNfY%fX;RG9XVI-Rs(e7zP?ooEe80=1 z`P_pcw_(V`imSEAW>5l=oTB7Wv{;NLn3DUN%GC^apV8tQiE}|&H%zfGYu(0dm^HTA zZ{Hw3X>!8(C`u_+JyE>s!9bY?Z?in8DjM7{b5rRooz0S61q#T~cOH`qEieHqlRPa$ ze{|F}p05^4yAtYFNG_$cufCL4E7Xs{ZeRH>>?6Zm4`Ciba3Q`PsOfBh$`uaEvMUiOQb%`8a(|y&J%O>!&xWo!*y#46DyqrN*X0*@T`yQ98U- z=LKu&D)xW%n0Apa*0W`wQ?#5-seIa%71oL?U*Ngq>+b8l}j%2$Xz9Ul~X&ynVqHTEsq>d%gXo3}(ATS|9l>3_DwB6Ai6C0Q1iDe$@4dQ2Iva0*q=cVB zO%uI8N{SBY2J!dJjb8Z~FhOLbf70S~NbZ^X{&c)i_MZuR7r%S=j<$(FZ_1k+!DgrF zN!V^L?%BNU&)eAEg`}^xND|Sl&fBfc_LgN}CT!V8qbUjxWx9cmaZRSf8#N~&TQz0H zH+7B1&MU18SyZv9>v(0f|2E!et-zPBvQZewfX%^TC$)|;2FUzzsazV5s%`YakS zbsJl4J$yBfx*MBs-n?p?N?yHw{d%LLYuVg<`EqM(Q#btTb*KI2&8w}l(#~ePy|KBq zv1Jr6w>CPjUvCt%Sw!ene|Ys~YxCvS%VKhi2(6X6`RdK<_S-`3vy3p>x!R9h(SG)xvfNjqef%RV~r^gvr{^1G)JUQS)<9w^d0Rho50SiM#JlyP2ppwOQ;SlG`674 zIn5iLSAZ13)5wy+t7dxW*rIQ)4wI;;1JoOs2H6y`7WMoB|6I z^89PsEi?v;#X3o~f0M4N89~+KV(G+Z9UKnoGi;8oIk5k~JEm;?CExsSJfh7`{b1J4 zVeQzhK4r~12d-)Rar%&!&e96@NcGf=YNrMcR@FJ9K3Vp;Y;X5926CkLP~RsPu!oMw zR|=+1-LJ^+?0IY6MI-m;^NR)=_(B>zNxchTdIev%!N$+ie=X`R%;|+s`VIQ83u`2j z)%b^Ye;UnVUS1;7)$WNvG|I{9S@YH|E8R~$!#BGqG<`4R_BQR1HSa6p;9c*B&6Dc-HL%b)m&&IPvtz4p3gTEDKxq(f zkEcL_Ya6rM-p&h(PK`JRyTH!&_O#0`r&qL1M)1=Fe@^N*Z_t1BZEh>8&PTO}bIQ6a z!D!bsJSrPTAUf(!8*DBvM^_Ep+LxoNIq1?syIL~ze>=R>S8AD5EO{HWDM)8q^tQG| zPn7ppuk$J>VZm0Q9C1G|@;MR{nD@Y%?HX{LB@lf-*GmWc;`uBrWPPW;E#7Xga zmq3|MDRsewkb@^72fy-(Bh!2^QZ#`22A3-Mp}eTD{9yoGO((pG)ra$z~CI$+804^se=!3nkivJM$cKKl{ zi?y=3!f~1{CB(_9+N_gwn%X?O6~^hH`GQ*QaInEdw&w}@cmwnr4^NVSkuV)&B0k-i z6NCWgvrO{w0T4aGh*&iQeWHX&pvb|Zgo`Y=oL()6tsV$69Ohe5pFt$<6X6G*1~FCx ze{(uQfK;!SOxH@qVI$O=`uc@p4d{VoV{x_4Ef!|6eOP__7Z@qUT9!jambI?f zR=TE2^(0QpdgJ7|dRMr0ypKz+;n&nNe~VUShG?)Jf&8lCjZ$$Hk^h5SMPQv`)D1&5 z*CL;26&sW?atq4G#<8{i+%Zc)^|8=lEna6wf0Ce=w8TTIGFn?vR@x<2Tg^)buw9GU zoF9_gRJmv|Z7=N7Ri@MTv|u_FS*na95a$&r&GJbA%E5>Z&Fmh)?Au3|ET1d_f6ib? z1p@dL`P`$@EiY0v8UPnmZkvT^2cjTPoF#Nba@|EV;sZVimi9(I(K2YY6ot;pUS_Ya zh6kiF-(;=RWi9me<{38pnuRt##0Gi8>4eoakkLB-yMSD(8~Lz&+T}Z(a(n?@zpX(s zX(wh967{{{ldzH)Vq$0#KgdGef9E0g;WSx6%Wtj5aO^Qsjdxe~3F(6%q>qA-XV`<@ z)9kd})VDeN-wF%SFYZ=Cw6I+JU4oB!R#Z@k^naIhN~(oy{Cnqr zt`DnK#Q^g)5b_!Ys>ch$)e>BF%nh!c)VE2#4b_OkJX3|>jn2}qXNkg4mT z$f-*b1}vOrXZurHpc9~Jg4z(YaKsku>dQpm$LN@RF*1f>6N6m-Y`}Ho0@)t;LgBDXSo@M!$K<) zw}MsK{fHO42Aj4Ah7tBqbW3&~9i?+sgk-LMyTdi=YB#axYwr9*R1>(}MKuWw_vt42k93Mz`CWjV-Akw1U_W(0D(LFlVk1GI|Zq(~EN5-(<5^1HjKFfa{pY4G%qPDjJHGBx;FnU_F zucDNwRoOvxHt|a7Z2Dk1#W1PUHgRNgV;ejUVC-g3*i7wnRz4wUMgNkP(H0dQiyDSr ze<$G@{SxA{1^Ow2YejHh;SvuXUXE6|kHT^tHZ> zwk@P<9lv+Zf?raitGDo_bD2&iJfu2tQP_>zPX8V!s&3t_s^7mTTk1jSudqI$U9f1( zTh;n{^$5=o3LhudP}ZoMCwB>4kXVa7O7Lg))Z~AiGhev{vBn+L-m!YJtF>~yd>P{` zJF70%-TdYzUh1u#2ByBxECzOX!bKKkmC0U^C_7crz&<=@3!fF+C;UBURK&fMq_Y`H=YD5j15JrNS@E(Nn!ocFj?ARZ4zalMV{gJf{Zg73-6lw zBw~LQO^&a@<~ZW_0azPL7R5n(;gccei@l*Nr~(lqkLq^*8nlV}`Ubs^ z!NvO5;tS#?Y{B!;Wo4??mw1A5@2NPh>Fj6tow1%W$vjKf}T-7OLo z02Zw>H~R0|1I7sa6NRUb5{>A2f2GhZHPg*KW5V-wpz$V-! z=U{q!!$jaZF8Rc7W#H}zk>10nOD;_&I(RX0$r+j&sxW@L1eEf13~KC?eI5tyKJx!u z@>`Sy?#a1Jj(Cy>#vdI7T(UEnWVNVsL>>cI{eDW}<(Q-N`>BW$WUs-&b1#x8lOa4U z0q2uXJQaT;p+z>@0X({vsh(`SE+=hs1E*_q6Y>thtblB8sc$t6n=hfo&?TF%0{49i zPxWlRuFUM_8*64a-&Ut}t6iDYt(SrOA<4Laq*vwqc3_ML9r)M`+=C7H*b3Z(P55{j zxCdMC@hWf+Uc$%gz&&^cA8!Kp;5B@_4cvn_Fjar;z&&^iA04Q$4PRI_)V%?9ccAjE zt-w9b$7()0&?|eXHa4~b_lQg8F35FGA9rFIxC(N*L{D_!YWNC!LbAZs0M#Y9D1mE$ zu4}2#AGJ#ia_M3%m=$$^+*^+FP|iiKb1?7et*rqTT=8J_u|x}8P!{P3@%J>M47{?S zQU{X#s6SiA|Y$$eJdO@+WOKuc^n z3fe?HrvY~fw3!RUu7GurfD?^VD>8}^e}P}XR^&Zkb#0<61yeu7r%pvQW@3iGe+Pm8 zedd#scDa`orzXRO;QISqgcZ{wEF)H|_R3wcV!`3=!K?SA?UN7f@@lf*9QcVF^)r7~ z@}cdM@)bMU|BdE7FS= ze@EOlmh4FF9-r*1&6E3NC#RQhNoSL6ye69)SL91d2aSfiS2DH`5<>ObV3endkc4c; z2gZ@6vQ4mn8h)pF_!-NOs!ByTi6CQ@g2{s9S!*l;a%TLO-7m+IFk)Vi!*1=vQP z+Vm0a-oc{*aJ0=Y5Zv(Xnm`k4ca=K@pqs%>LYgSQE6VkR@|cii@bD9SJc|>gNGHFPK(xfzkJzgKGr~T|2W>Q3Ozq#<#BhUG=fswHz>OnM!YM?NfjIQeClE z0jrqMTFM7mDRTP_glvqS=J4K4n(QgwyH83T4fUiBzjwx&SLPr_1fj)yAUzaHiAk|R zf}F(KWHk%e!n`u0%|Vard|autN+$5gr^N?NqjFmIV{NDO9<)-6LuBdV=B3{R<#SUF zra&E`p4-Bv7pomx-}lr_6E%O47R*N*!h4nDnm+3tse4_0U#F;!xY|?@9*m(H0$#Ut zvTe$<7}I+t6dWVmqhq)(R^B-C`P=Q)S5Y}7pFyuE0xh-(;jhtcanr;g6J{Mxd{`B| zKvoqe1=3!5fR-9wdZUp>-ZAsdGF@JEA2N@TV@3o)pA9|{i6>Kd9mIc9BPz6)mCG(A zFYI}rJ)f|MLV21K7Qs77&v#q<1uJf+M&q+*I|MFYNuW!jGHqwB?$R=Vm!8=-YIv6D zzqZRzK5AvQ^KxFzT(Nbp?ax}#bje$qQn*<|pEsQsNaMVKH;aW59|O^(_V`D$#k8S1 zdI%|>M;;3y3e>n=Eq8MqNPnD{dI`+GcDtSPyq3JceG}SD&q?&*0&uYNV>eYJblU zT&)<96^uI_R`0O^62JsEPFhbjLeN`V^PWruKkl7Z;A6YOq!8fd#4LrN8s}$J3eWE{ zCjyyxGQcg~0tY`I4g?z;@ZMGLBWWpx@w=m6a{f(m7Y+5_g6H>2hdkP_G5|e5!oSyk zDsmrC$Vy42wxU#Pus@;z=A{9DIgkW#gkvs_-jsCllvQ9+n`#yRKrV;mY z(l=6b42}Hmw9z=iorp*@`Y2SdV~Ky3dNuY~5vfz$+-39$0zgbqun>}Pujm3tMFEU0 z@*nB^KAr}q5josUfO-H1him4k7Ky1J@LTPUJdQDY=J}-mZqx%`9$~0zKt*6R4W8-& zDH`+x^%g*XMH;Ep;cHz}vlER@P%52?KLD9#ZFpoW2O@TvAXtAF5Ih}w$csfxu*_6}g znJvHX4xGg}kUTs9ec!dwEbhcHQrMKJRmb z?(|`QN#Vh3rQV44`6?(dLP&oy@_QrthzBVc)u005NoYs*lW+n4le|$KjE(9rZ28># zwL?xu;EdE9pkQR3CSsqUB1HvBafE_Vd7OxKgiO2&EqwBYHyRoDzHox`@rC2DKh3>U z4kV@O6OxpplB8r7NvU&~Nz^1I>YY|eN@kIiBaplswU9oq>Wo(~vB!T!;fCX~_Dszc zXo^-|y3Gssnd{<;y7&5~Qv#Mm)>FrvIf{mCt_udt%uxaaPe2$z)KAlvdukvB3P zVxKNiIFYr>sm=6c8MOg@QYtQ+PJJ$7Tw(>VTMH6_Yb){z9b5sA@{@d?*i}9a?cr4! zp7Xh<-*)+vx9kZj17v?ihGJHp&xVsw2K~x_>K!S|S0=>@S=M^r%>y!#1c{g4Iv7C0 z;!kcx3+=2ABT25@?LX%_w9&xiPqI=mg3^+tfI5GX^t<$2d*WQ53`I3IvNp(mtwg3b z!q)`jD?@jE@_T8z`CCd|c9SYY%spl3s0=R$EIEzY%=~^<&&huqV@OahHnxx;jV^5_ zL5mXq^|gH_O5)VlL!{C_ff6EqolJGwd1=KN=GuL-vbf~qiHvyY@MIun6Nz6T{0DbC z&M=jyN|>Wv(`jo?tz8Tjp8MM5^VFDpUQ8T#z7GF!)Ctg^AY%W=2*GVq*2aJDetUTsvBT`gGc0dgo|G|6l6H~2i=(=wix?;`x8|pTnP`mzy`b9d# zk%5Y`o-<{cs??GlrI3t2&}=3dyilXdG4NNv)eZ|yJ}99RmR=r2SLExAeoJcr{db7FY+uz}7Jo~%5@yHm_7m^x46Z77{r8!_=qKJU5`s0r z-A}xssKndgxu1CZ)lvo!_Y<$wqbD`jjw;!(zaaGh)uFBU6d8zVvJ0Bj82>eaa1DYJ zyh|-tjO2f^)K(^zrt+1Y6%XMLv+BKH)N3gBR7pzYhN~I|8aY}s0FV?N93|D%OIsFhFCHtq9r%Q zj98*dELF2>FUInOJ$os=TkfD$x7wv-$voh{tA2k4O{xtjn}Ywa1-x>50g|;P_)stWwcNlPKkC*&9!+ZLi zOV2g8GG4sqa>HA!0)XxEz;_ui>D^_%1$4!vN*F07&U|hZW}Ap`yIoOJwa&>uB5y>P z@hN`>x;87cu`Q2Um)<2Z-2uvS3QPx7R|57!f%$iNppDw})9UDj zy!1slD&OmFZ_^`qhub)88eJY;k+1ZkW!QaW$mvD3|6Oz+cu}-RzJi;9Mq|2CQP}f+ zX=PDkOOVtbJscWIJ}Wfs-3p0 z8?lYPu19mat&8e$w36wKjI4SGMhY~Ca)3N>f$mT0wU+^0?u91Hg zk3(L5%CF#csZTk)lSQKCXAg4!5nmUp!}~_V`*nsR1gAu&Z4a!y;UuU}I22&UZ;Ur8 zoCOio9vg$_LN@=cE1-Ju8;e}%@#qB$8TV6Yytzb%b zOpFnlPwDUQKHIc)6D6T%h;SI$RIh(OS(9=;N9{bzN=-jiR2nPk2!-^`=iU^x?MJ=o znx&|{ zhOb`KLm3wjQ2Nr<(wvIqt83trWJB~J2hx!F9DcS6$n=!4XjgkPozCYM_Evv-(`Z~+ z96`%GSGTa`E8t^Xw|0-JHo;%%k?D~4RrTmA7Hc#-NPD6`N56}&Oqr$>d8T!*7WS4)Jo7!@O4h|23q z+NE=!oN%x$&V91aL0GM?lO2Cfe^>rPl#;@+uKevBd|2gYjYvi^y zZz}1YU7PcZR92Hip?>!z3(Hnt+e&lnqp8p z`%hgu{rkq$|DJ8EWr3Yax}$UgXR^d5$w5NbX0p*x`Wr)l%$6hLZGU0SwjfLsZ zdvqqfkDN%p?kf08QmJ?94mT2rW-+Ts=`RmGnJ=?{i7NjU?V*jj?{=^pw*+U+uF} zAqybGuOTmW*ELmivOoh-`cK|-m}_gIyQ{sVz)I;;^9+t|)u!yckwf$)^^?a>B?8W* zlkQIxpM?+jdBK8;eHNeiVq2PKGW**lYHgFvrk0G+%> z8uQx16Uq#T0ltVl0dmkNw>U^w{&tT&^lX8k}xCzUy zf9zvRwS4LIw=i|e(&kAf^a-F4!$K`nD0iin=J&*AKbdLjQ)YQmAxAJzc`5qg6@q6>xHY=@nmV z?tDI4U+?C0x=?^I_q$j}IjAHpO)=EAK_%FVSwmgKV&16hDE;%zk|WM=THcD#nZ`b;ubx{ihoi3;6fWC-HaR z#Gd6v&wp~$vq#;jcBz5Jiuu-^Z8X^W`eH?AmfrCC5YwNKpAp8Wd#&ysd|K)kcl|?5 zuCpQ*q>-75-6``HE}vD(`*x9n@)unonikPUBBN7Snev_`Va!#0c}~9CUeYcZyHc{? z;-MS+%@Rv{THDAlnghRSKF^`3GMk~yx-~V#EPvjSwQk!=ihd;r_kfMMV1?K((J$(15@y5V@;OPc|Y5U!$7@m_Y zUVqU0vwis~z9Q;JU8JAlX4C4+NK3On#p~-PUs1l)T+n87Q3@KcGeJe_MX68z!Y|;3 z{GQS%EjxS~S0DVqcT_c%3db5>omwm2VvN$_L6Qm%1{&fmy}>^uxa;C0?P2yJ8MEw; zhX`V|!ymA>GIc&#@Mx)+Hurd3kUpM>zJG%@D$~n8=$-Zpz0;9?*3*7l`rqjtdpiGe zr_dzCGSU<^X8{8%FHR}Zw}9m%5zwfc#QRJ5XeQ8E4n_TrQXC41czW8D}PjGgHI)LqEYx47w3lcMol&l67|gB?M0Ygy{D5l z5s3~Nx*rHHfv*wwjhO&2Z5+3n&7Mgb0Ur|>ju~&Uh=0jDCI+!5&Ncu{2Y+Z%nuuLR zx`4a-Xa?~1UZdqp`BVEVtgN2N%+96aE_)>XNe_}dD!|gsw(qmknfEJY(F`}RK}(Os z_^c{S#$RKg`7g%2#aIO%LKooK;&s>-kUGl)2jW zS@PR&yvNEXyobGh{@hfyES^6vmkA6+xS0vwPLv7BBXWL!My#*Qx zo3-dM{d>#s*PQd$MaO?#@W00APoBIOUx*RD(6d!{zoTFNIP3TyXZ&S){^ZH0^g@iT zeEB4*zkgbPA6Lv+`lT)8-!sHJ>8Fm6JekvD_%!wQgr*DsYx$i*g`Xj-y>S%23qOTN z8;s;Y!N5#0Y>-_WQGaU2F=}u%ygbDB>APISyj_V@KM-gvw>v#CuIDyL4X{$u9cwS3 zOoM^&FtO4i`mwNW5};RDV>*%eCpsSqG?(^oobn7ua^^)OfZa#usfNUJ)+sw9lg|yA zdJri%~<%WbI@I{`YyT5yXH7hbyH*-aqi2l@Mn6D9ZcBr5u zkQ(G`ShqncER>wn;C!4KObHmmGmXoH4Zy~=n1rpsQT9YOBa!VJodCJf=3-2v*wrlJ z=>M3+(Mb71ew~eOHk!=_DE34lvvhHz`H;~A>R6=JX@4x|5Nd3VDH7kP<4YpQBC)~g z+=7>f=!+8ObT$lU-+-Q5?KL2FdJl=!ojJ3*(|+pv9SJ$%Xl2;nRdgublF(}03QA=w zdbIs2TBBNO5GB)csk9%Yij~}{GON}i<>R!A-!XoU^oukb7G+prA_$DcB89GI)fN+R z7y#<=dw=9CGmi(j!TYH?Yx*fNk^NLNk$51McckDk2jZs%k$N&ztY0y+a|nhD3G%KY zyGR6)OZ2LdY7F&LR<5$%x80S|dgd#dLWt<9mYLsu3XHd8wg7qnwD9^!BueEhzH&Q( zH|=P3ytQ>H^xmiNl!sls8F&%bD%74bl9cdh(SK2{r(GE(+CL930{D;jRxG0Rg#I5n#H5rjZGVLXY1Q6oq8MW(T2advDyeS$|-)G(_93X=`=L}vR<}z zIe(9CAOwZH@pJBE1VA?(u_{GPb8wk4TvZ1tggE!+i!GKy8iEUhk&VH6vB5n&XgxCw6t<_-chuK)5V zCUi!9S3l$PnQHe)R>cASQq>|;w30ePQGXts3QaZgO*ntLo|pxr@SJOZNq96@EuP2E z^B5;aPQx~KY##f8d(*ljQ(?n>)2bf$BM}@S+5nw>idO4eo$BjOv13z@es#IbW2OD< zdU?zrkGwg!T_VQ?uQ2UU$;EI6P^|RCKvkX+N^O5xx**MCi;*uuNjC)7L|qzT(|;D) z_$J&u5db;JiH$XnEop`GGS}8QsMVO^O5;*c=39>+i$X8}d9(R-?v*((k-2D?-v#GW z**(z|@IXcC`DtNRh`6UZehkBM9P{g;Cp#$HtSe!1xtVc&NTozXiXS|YyLi1O9-+bc z+_~ixPbYIRt{5yO%G>vFq5Sw;3x7pV=W=;|ao4IjvHHo%{LZdE#ioqMc2nT3f4ppu zVnPPY;RRn;r&qIy3vt7kgYl`f5&NdSk4N6d#%6BXR`cnO^9{Xh^n@y*d|k}N)anyb62X>;K>_2_e8I`m`Y-P0RMd&cDrAT_5yJFM?D{E$D zZq5AMnfZCm%xi)pgmguMH#RoKm)*+sl+PwU zGchAPzaeG`n~UqdFP0X}j`%{y`GTIEAa1Az3}gf3*X>z* za4pwyO*L{Kp5^@5*@tvs&#-bcsb#Jv^;k~ov652qf?#z_4YD1yw+vThnbYIS6Q=pnKzr$X$@BviCU3;4ZZ1< z<-&>bOjXu}-cEzF+xG2+U^W9+MQtrPholu$A5j9S7V(gGtHOaEKg?s zsyZ1&N0-;0J%V5oic{JKPe%c;*Aim#JmzIC1X^*J%jtHDw|!$2Jp=#xJN6WtPGOl% zg(;ORhQ@TdHJVLhoGYu;-`#1Gnb~R6sqi{hdu=j%@LrqD?%ivXS?yd;W^}G6Gk7|g znY|`^`j0l*9e=|#oyy%tJI$t3XTSX|0`c(uc6Y<*##`IG%qDO(TW>mbww}20Pe|W= zTd!Z*Td!ZFVi;>u2_*$efIA#bW$^HL}Xyz&tQLc&T`8? z>MYk>9R4;EHIt&WDkc=zPyJh5#b%swN=ol6XLX&R9Dnr)V>y*7S!aAc%m1^2H?$Q5 z@B14n+o!}9d~#=-WsQn{h}_ZY-dWDg$5wR8jZo?8B8ZGBbrLn7|8GzQj37c*b5f`h zQjDTpJ7;R$SmC&^*464g%YE@vtc+cdFxP1bcvVZ_6t^`3yhckPC)DWeX<53uJD ztz}BEtA9)++_RK}tNT?GSbxn}8rkQJ$Ud5UoZTh+s3XH+YY~mY5{eu~peg|UZK)G= zk3zO2q*>4Ne%HHW9@ebif~b5SKVUR{hU4f5t_1#SiuRcPnE#xEUk&>;$E&u_elNgV zP)PEBNI#dyWAXQk>)-#e<@vW?TKw^t%d&>Rzkel}+kP*U<5cJC)cYL~Q7`2kk#?21 zy~HDB{!wnzG>8VgtMrkF=1ghK29x-r8KI9s*Sfg`XL56FCiWxW57s=CmDRk0CW=D( zSCilx;><36EB^qu9x+0~(vQ9w;F=ImhpF%f)D~gk)}TD>^fA9R7OiPEy>A&}(AY)Y zRDT!mQ_Ipp#~B9fOA8V$_-I?ZmF}KF#*`%$%LNewW<=yKg85hHC0KL|Ud!0-w7{VI zc+4~QM`3l6-(nAf7NgAu-&iCWkH`G+7?~e#>}I$?y2#*5D?K}e9p?^T#DDe*~CH3%vy@|Zca!SM-5cVQw zZpovLvKgDDix{hBmJ3}L@#0yH(*WRQ+pF5Dky*7l3)ZbtveJ?o?R1Hi(1_6h`=QRK2%IqAVT-x6)xp+$2M+-had; zv|{nqRXofWfs>mml-^3vOVUCXS?$FXBVM$=5Bj>&e!1z)9k(+u-PUv2v206R=$2<& z@`{GP97Qx%O2ms{PP`1e4t_wO`%UDsF+MSGuu*C3)?ag z;PdNhr`yte!FVCr@otg5Ow8tiN3!Q$(cEo6+uh!6p_7xZMllk*6&xzHqZw{CRxEkK zx4(@6#w9q4?s(*T%tg>QD zUyJo72of$jBolh!L9P|NGOMrHQkP^6c0w&w0@_s&@=(lGkEuj1Dr`!%%zh;@uSDXc zz`LD-?%3px9b39PG*nwMakpfmwM&v};Mouj8O6 zo?OG{2l(bmY%SXpVh%n*TpPM<`*6c;G~~0u0A2Z^lejXV*KyF-U#?Wa2R^d>q3LLp zJvxd31yC)IPLH<$H4Qo`I@af*iZDEc&|+Nhvz+(MUO)~rX&xeK%YR|)-&T8vjUqc~ zmWvJ4!oAbC_D;XLclvdE$5(i60My2Aa^6>HZHM5>IRiEY(P6GNJ|bxAat>Hb@(WTZ znOLEv{_PP!!U!$pEN2LpVm#CEp#{+JXeXHD((zGUG@tD6?d?2Wwzm&NnO61dhcT~h zK)3SoU;?HrIdAIIM}M(M^=h|Id~G{`ZP8k-%+TpM5tC>;q%QgXE_|U>dmKL}>5RI^ z&-)j_p>)dW`@SGJv)Y+GjQ@?Pg(axYwrf~GfYUSJN3kD}^)$;_Z^;j1h2x56U7xby z)tzph3&8}|H0vWw47@G>ily%o4t@wJVrpo^O44YEzDTdU6My2}f{kWmhw%X`Yc_86 zV<+HTsy|21eG@rS@(3b7dP^j7(-_b@1#J=y0>I2jPE%+arE3)@aLeyx<0bDg$&2Ha z?QDG%({!(M%l9zSOGaNwIgSQRg0E;~5P&0!02~PfV6_ksSf@dz(L53rCm?`GwcTAt zhi7e!j4H)uN`L&7NJjI10TFVu2!#0e3!fb>)KyPi5owAdKfGVKpLn`!La&H#UpBJB z0+OlMTgZ^J!-Wi%nneu0nGyGRh0F8knE(lFw*x>^hE%k6Dk~p2$!HQU=Fu?jpS}s+ zhPOT(?M zq?>jo>7?*M*A@?ZoMb&3-Pqi)-811cl2g=G=|;#4Y|COz5PLL27)R}6F|u5=Tb=BA zG@=o5(daz8*t+~V%f7#X)GUbM{yEMT>1--fz<-O5=+}4)KlHJ1y|9nc#T*>5%V8bq0C*axjK+HTVYzZmJ%oz<+H zHrl$$>gyn{V+f*$!0V8V#$?H?@q#Z#2B3z8_}%H4htR=@Ir~9$(bgNtTk@b`AGMN` zn}3v|?qMTO{;M)bN=S+30XPmjUHG@5fV%+S0yBi$d(U}T(X)X8Io#xJ7?tWYj WYxIPc!^nR6Cw4lJk)ZmLh<~CA!9+`Jhf`-_78%ciMaCCWZp+w$H0dxE z;jJHy?}=#OAu+&qtnia+%LUDX5fprk41`+Sra-nuC5OcCE;1g$*2_97U@2Rpj%~v< ze8g}scN{-DUm|1raY_VgGq&=!kl}u)TX;cv+rKlwD?KShbGonqgH)7+B|t=8=6?r7 zh8~TKx*O(+!D048skhcCtJ|=NglK*Wis~#LoyMbL7-u?+_Tv;;H`DJB^2{2kI;4DD zX-S?v#j8s7lu%9{GNM^9qkpk{dkDewyweb8bN4u5KJ9dC&%mPCs%x|XPoaNoMYV(XdtMTJ?GaQD5g zRXE7Jgy~QE67*6?@Sg*)}!X*w&mMv|$xAUFyCT|3o$jg@AP0ytik zt2G;|s4BsFTn_+_Q|5)Nrwn{y~LTT5+(Z(?VNUFOD1~(q(>0@WaORBNooz? zo-!ZPoH0cIIGaZEMS`Hb(Q+9smkVR!TsX%fNCVBb48>ECGOgVMcC_%Gv7Y;#62we} zaE2D@V%_1l3juu7adW4&TpmIK7RaEkjSb{r*YZAzhTl2Zg)@_MCVz;SK;tY{EP_Md zG94IO1yHL+9-^i|8n^OU7E5ClvtL9mQ#=PNCRiM~x}im^E-eDOcVXwGp-9BgDM3Sd z!6G^rJYgq*?$>O-v0*z-=QaxFqk@98S?|zj5G$09dlKq85*gg9T%uWn1a|BzvNkTH@e zvSW;!4v7lhELdeL2N{H9;(n55Syc}o(X=&(Q`}lEZOzE;KU$c{*C;r*pq9mD7KMS zJJ@;F+TYt}2Y)QHbZ5!P=HrJRl6AM(UvszEF57m>yr?O&>y&vl*P4Smk-;~JT#y?b zpack&i-fn+N{U(37A{{fa=rDQk(_Old%r?ddC^dYLYc=7>Ai^X5*>OqxAIkGzGb?e zkE}!fv30J1n9yv_9qSh)w1z`9LpQz1^C@ao!R)dL zS&rWxNPicHrR^zlHIq4{3MqpnN(3t$(W}v>!j_|ZVaISl`=23$Ug7IZ2L&e%TEsU& zkj&G}^E=;CnK-ZX+3$jfOQVLkGf(E#m^TOMgj3Ta+;StmmubXtv9nk#?n8qSU@_ z@`eypnUq&c*VnDor-xwS^b%a@=`87VBA7YMC99oFI+@27Pf2uY20Hq1Dw6g|Qh|2} zifpI~o!v}_;V*%6cm_Q~Ak=;LwNB$Av|8cI{K1KHLBaIb^sSsk8nn;_PO1W%8_F-;o%zqLV5{=p2S}!PDRJC3!Nh&Kd2BX6wUSwP8D4yo& zRT^g>viK_f)gkD61z?9FOlSeno(KtQP(pnh;&ZcL5#Ih8lJx%)79JbzzH&XZ9s zIq%QwNa$bl3PFA`*JSg@oRYcGNNxIebF*>DRH>N#k@sdHi0|Ln&qjN$hU_&ksuhb~ z=HI)$MFY-T7fe)GE1V%Aui3<(=Ji|m{amiiZ}YonZvOwhnY()6%w7G5W=>m^yA_fj z{Nq`@`hRm)jl0L=RI6k#U4M{t*%Ba;w~_GfjC>goWu6&F4}b{IeDLC+iQDlv_;(nl z!DJyYI2I$K6R*HT7U&9rQ89&@fIu{e6*clavn4&tMVoLV5^Xyh^4oOLn-x=lo~)k5 z2sLRVeJ}iO3efT3RfL-(p5p0o7U#|yR7_^I3Wfcgp|MBE!XZD3&;#kC6O$cr9e)dP z?fbfory{%1f2^J0Al7F8eeHcU%Z4f*z~PM7G`0|pH428K$b*$7(|2vNsA0TMbps>a zKx|31fq+e`8i=y6**t&Dsb>=FSvdtvVgqI3*jgmB0>I%Xb4a|GOD|jMG_j_PmtggP z{;Z%G!7Uvw#Bh+f^R}{uc~#g9 zF{%WtQ~j30h`bS4zE&B}hRnu{j@%W*GxqOhq$vMZMvBm9G{A%y2Q$@8W`A;NsX6^4 z9gk1rVVst+1kAQEN!J#{gdkb(Su)(JAw6ECN--xK_Igx} zv7ssWF#4YtOeW^5oE8FL72K*Z0J5X9*wLkY=Nc|mQMH!C+O=$o;j#^Ve{GZC^Ve~H z6lZBE@4d=q6C5!{+CmMV@_$GSf$FTOm(1h3+G(W*UxM0zMFPhv2_mKICLPbqT1ek= z>+XWfr5CBJitLVe?D428;rZD-Ii)Lq;Q8JVTCv}?wcpliKBE> z(>xObBlSdNUwo4x>c&#P0eLGi>Y~`>S0D|dcZwL@!fyhNp)y*KXslumura9*0}pnuO9aujEA?w#jJy14Lz zVzw2HMjrf^by4QAc|7C4C(UIBL168jGF6F;;x zghHx{WV%RZIe#z36A+A~%c_;0Kt!j=d<6FKClV7#!P$hR$a{gX-Ri`G1fN5uuhrqk zwHS!`gyyX^dMV6x(Ir8JdwVkbPZt^F4 z+Xt;ZrZ5@WxI$e>wXS&FehOglVz0JwY*7#y&(N~g#(#{o%Dmn7nZ)0}uHf&VAQ(Q_ zu?fyYFQX1+B2$R=>;46L3FVRJ2mLWdHDMtqHP_S{N0A^GbnBXd%Hk_E13$*No*??o z1ZALLS8#|MATTM7Z&}v?aBu-1)PQcc_IA_+HTi#)>|G>mlwk1$XtJW&{M>?-9_dM8 zviE4Ilz+V^oyx3EQSWLN=>|ot5&(}SVQB{l)iFP z3hhWP1wbJCz|a&XS8uS0S>g3qg1Ol+`GNSh+JCQlinLb8Oic#p@!*nlZ=*>RU1h== z_Jn2C#xG+0ird2U3Y~h}EV6S8B|KzgR1hda=fc5Fu916$`-AS8ARY2mR_Xdz5}-tL ztxas|Z#PGTn<{9u@a~WlJS}i3K+Il}jz;lxL!{`7RCXWVRZb%M!aHeK8^X!*VKy5} zZ+|AOnN(ha$<+HiW<~0=B3<%AhpjZcqG>W1`NrkS$o^aUL(3v4v<*EX;5OV;!R}~h zc7Ho-0;ue!XH10C{d=QP+dc46vA&haxMrQmAmP?%)P|?|0R#^_6)8$2$yzCnVjMsM z=sdAfFC%5vluhKMk>P9vtxBXbDe#C^dw+DVy=`decI(;G_TDzl!9f?a+<&h#uW&?R z&C`J-9D%+@oexFjd`SK~&xd6F`B2oK4`Y|yVE0sC8?DY`Fl(!*b&<14buJ>BBuS&9 zpF#7VsZq4yAVL+2<-5q(~}bZVGM#lupK>^*@0V-$xj@@ zxDS>~lZte?oXxsk?T87j4=k6H7=PAIrBh4piM6OB&l34uXmzlFOc_l!M4}7M!EBJUXB-Do9 zvH-D>@czHo3{cZcPYIiLNL;ll4S{_fDR|b1qR)2w!Q<2u&sg8I{yK)u?SHT%WM?3#&Oj0!z|2`obUFZRB0o0E|2ft1|IgAK*r0+U zP@*%CG>28`4XdX&9SJq$QYY&hlYA}NndS9lXO;umsaEVWI4cO@ffWptXOgo4n3dXo z*E_k7`iS9w$S_t*uBAWKIe)DXAZqk;c{uV7yuKvySu9buj1gt87Wbou{R<01&X09Ce*>*=&$v0C=IyXAs)ES-O3G2r7V{z*ze_KvngL!Cn}7-Ky0#s!)c64ox&3m zezHN!D~a&4y|>SZa(__w5jyp2WMwUu;;%_uEL6cdcX5k&JA5A*=a|1Gvy<4B`&1(D zro89MB)~y9`X|w~cY;FsM59@b55b%h2+1`F8Rl>DcFiH;v zx^>jjCR6YIl$Fi4uaPCyZu%3Y(5!Nt(?l$sro1&l|6Q%g>3?PvovEwTDF=bNGP)FV zOIRfXuSc~?8BeyZ6trCk+6k#o43%<*_4qB^36xTash5M^iZ#5X+4hNv96WoBPNxO# zKUBlk^MjDz3_)+mloMD?WwD`#H6qxr(!-@itn+7E%F z>4`)yWz|Q(%zp#m@{xJa(3=`+UPLWHSNf=q!Jtln`5 zN_itUydI9X0m~}Iq=~8%d+-x`@CPg+sZ8Z^NI(@^0{fF}EsC$O(_bM*WNH0uws-dr zTCM$EUk^k#gja!*f+NIu3(RP(84k$)$kRZElb%G<+zm>;Eg>-lrsjT7@&kHje^;VyZXbw??Y(_* zv47XzmXBK7vJeEJh^yV!Gg<7Q{Y;euToiG+v;A~mmJ}v#Q2=dSva0><_O2|wx4pZ4 zaG*Yh$&}Ui&_EzRiJB<+%E8XT!G24P;0e=yZmp~{N#RBtH$dm~^zuO8*Jl0#yCj7yr zESOhxMApA}DUjw%4027-^B96?bgq?==eygw?QCleOheH$yPma&D;u2mEH4Z2786dQ zr_eMWsa+3;3E{-qBIVPWXRU6ZjZ+@6vnepJ{j_p0yiCtOrJI`<{4`?v`h;A6oPV)X z5NpyAJAuEmGS05Xv!CqaOgKU#Wlab%HL`E(*}etlFq#-h9xH!_jgD_6Z85nsAbX64 z+q5tg&T5QL4j*Um=UtZnHeI}khTnfi*=PYlk0@;Fs6+kpERQmQ72U#U!XROJl#YV} z|MY;ymaD)Ew@7u8PRbf2lK)najy1CoX^GEIy`0o+;=+Jv zIAinDaI${gLmL@3vyov%JZBXd7F~EW4!KA{RErIpx!ADehN*yfA7w5stbbmp=p8O` zVUs3rLpy{tm{>fWU&nWd4$IS_6i36x)hxrswSj|PXx3O{is%$3r3@!dMr6+;f-r&A z1||}mI+O@f@RD$drT%~-Qxh^4_Q$w2l`4m&5w4iN;y4iNSb_+)5>0JMImKyhKD&mVae<6P(0TpU9$~{-fin=3YYZuSiq8Y3JVJI{B z3?iINuZMyan&ULPmaO&6EJm|1>+2*n*$a2=M3owkL~h22Bxu+P78B6nfL_hwQ-YU( zvSx8{0fYw#r0`}F9R}#*-5?v8XoqT+K)bG1piTgE;Js9O!O}J7Ne|rw^0TqTP8(b1 zyEoLvkssL!kX?1PQ<`^GTyoV}vn%PGIbND=zfO43D~+tTu2%a6wbf)?FLFlQpyXr* zLKvud(dyeX(KmOgBd<=6H{{q=PxblRpz z*-m&epfQ=S%MqJT*ksa4Cfp91Gq~L9TrVtotlU}m032TCKJ>G#Xb!s#1D8CKdyL=% zLE#1_#4H?=P}m1~fqS`nkQe407RPu|dyE(EF<#JPT*f{K?^Qg-3-=UPPbf)Qt=OgI z2k7EDC=bvL)jEtegVUj~06(2~HtT|8zPp@{OS6VJ0M#L|i2xxu0Lq z!)pS-mgWi}n+{TGG5dK=4#OfXM%3Yj%@7~h=5`J3r9&R&1&y;B}m8;S|nHx|Ww4oLqf%}w)glwvUUA5k_H zNlEQuuaVrQv#EbuAS&|NtJ$zvz+ld5#-JN6mmbU-Dvm5*D_SeCnKCF#Hj>F zMdYWE05~{Mfr3t~9d7XH=mQ*eLvF&p9Q(@PL(PKf$Wd?94Eu69Xf_9R_iLuZOyR&E z)JU_doIiv7I4;q*-;#Mw=~jfMCDs*X^Yzz1ry>((fw1S={7Bi9ie%@`5^i0O{ID6U9q zoL{Zjq>g`j!%3GoMU7eW-9jTEG|r~JB+;1`*d?u&=9@r-y`*E3=|cD$>*F-4$@LDR zolo zZJHqV-LF|KIvmk6=zCHW(1Hdlz@#Ny6e3?Ibk~2fzY!k`m1=6SED5W}lI1$9mUwh! z-z$xVp(Hjt-2M3Zf9KQWi|an5BbMlUfI&xf&43V1hPS*+U8J~+-=!SaMjFQ`~w(9c@WR3lL%AZ z{}?K8ZvcRa`AB3|rHZ4wqYc>KHz{OHcZ7d>&dIEWV-1mxbm~=?bgY@pV5`(+=TUS4 zwyE>W#Ra55;6}v&NIr$=cto?UMQa0(5h=`bE|V0bqC9M2Xp{GvoPy~?6V zyn!oRIj)Ky@#HIzT6hz}VBHtXbQI5I(Wn@uGob*3L`voJ?+cabsNCH2%c=K$ab8|H zHHbieXz>M%X0EhtaxFiQPNHko$Z}xx~n!LgM7(qKP@B$xNl?x z6dHj43xT@~t1O`6_`~zwMW;^ zX6ZcN=z1Br%_G9U%$w+KjM8!gXmhIM3BK`Ml>hDd|NSp{wITDsPiAE-^e-~8IsWkV z3QM-q1zFV9&lT)8g~Z9}Vue=5%5$H8i582gYBs$dfZ^zk-0kU0-e@83_ZQe!dw9Xc zZ{PQWO4aYr5JHz_l744MzY~^-Gpf->uSa^yWROo+8PIu_30RrC$a`jycia}qKP2(h zm;VWw2jm&MmKsuD)6R#H`;3zNYAv-Szvg3_?4!=7)LJKt3TM_%R-4!DoZR4l+1=+t zLeGVDnyNd#ZXFo1(Mn}i|C(_gNW@`MLTpTlCrKz&5}>1S>1C)fx%=ng1sa;ADB67q zUmk#q(! z&$IMvk;mQ!KE%WGSMJHOzF)t-4G|=(8BOv686=cyv=7i?y9_ixNE9@{XWF7_X}cUb zao6c%@6@Y)kFwhSgl-tol7L85%;jJ zk49c2Zte#HsTw5k38Y zJ&{X7L}_PyoEpDys*o~9N@NN3o2&p}10oSQrb}l?9>j0nQTZyYVYMEC?ak)3ilDH0 zEno5npWhSiIHIsUZ-=$qJ$lUIohieRnyg^*Fi?v7Y6UVtFX2ud#&p zET{=fg0zK8I;D{vLILXPg4G$+L%gs>bO(tiq;*wg-{=f~LaC8=W?-N44Inl|AWaq}nPA)iu`+P^q-P=Ny!ks10`t zHCny0Cp8o??>&=kiX?yHtg_rTad^~?P&kH~WZt>w|Mt5@OJL;!lVz5#B8Soy+aykC zBtD)ZUkdQN&*;%ot*&tJR*H3-ps+T9Xhq9czO9;k84p(jQ3R2uX1zC?U|_4Ni}C2# z{$J4dCv_gRI3UnG6ZrP`2B>2>@^(>mA?#PKr(zG&APWCoX5fFYbopMOODZqlo@Q08 zxrDro;)dlm9I0{BmcV)^;*W?+YKH(7mUpl4JM9**wt@oPvHZZR96MEqq8Gx`y*@&P z-FKS0wp2JreZWrE;fqDuv?fY4$tIr7{Yo8v;2uTW`p6{r_UOHp>z%kS?OrrQTBdee zYaLr}z=E?z*zkWc$)u<@^Q`#Oh+ThZjzdwhVYna_MQ`Dg0N8CI z6J82*+r+0WInEFBV?REio0?@F~A}pSc=*@6CS_#Q04sHwX!JybOA^}nSoW?(w%b)3V zH2bMjQv!eRf~pP5(%&NIPy!NG1`4{D0?&xI0N66wil)P4mI*v@G_tB&Cs$VxG=_n9 zpy$9MiTWj;Vpc~Z;j&O)$}D>YaoTtkuLJ}Gcm+LavjO~f4Y467@ZS;ST{@I;0XZs_ zJ00VsImz zAHsb=z`}!M0QcoLX{eLhAwrRHgitv6{Vf2ioTC0ay?E! zOL>wr;}GvT$6vc8#Um3U;FD*M|U``&dNKbaqoqle$$rhn#D({$ zk*-?A(Zxw6rabMGbna+)9R=hLq#Y4N{5VD`9_iR>b{zY{;lgl~840FRYf2Tzr6qr& zX<*svpeJzvPI7)dVR$aENlsEqFI>+fA|VLLUkTCOWXnxQo;Yb^fwID0?~Gs3KHVa_{Wegw+xW<8 ze8J&m+W4Zc1pd9{G7-@(_Uh|e){KL0 zK^Tv%6w}v<0|?1VP}JP>oGO33h|^jyYqRxE0Upv3YFudKP38+NlZ+dm&G3}MqfBMp zS}r4H_tPlAUPMiSCT1rQ@j6~+fK0Tt38q{Rmp#Rr2dRB-M*Hv|j&|Lc+N1E`9vt4& zgWBiFvClKHS%0LKXtbEAK8ZlrqN4Ntse-et_#&f)n2<2e7rA!lf02KK8Fx0Br1`7# zYn=6`6Bh>*g~2uDybn2qf5|l8%Ghfi2S2mr@_6iF%(Y}b#JtSGANfbxQC-%3WOGnn z|44bs<|N|uM+)9d^GtM*B+0VYct*Dg<%|kH$ebH~Fn4Z|5HmZx?;!a5P5IICXlod@ zfbx9JCKSQ^Zg3L%!H<8I2bHdC#crTS4aTTg$6Mm+Y1IuVt<7b{cQs26#O-G$eD>EF zDu9PFtcqc=z<W@A$&d`6Y{+bSKw}blo+d-l8 ziu}wVb=}X*hX}-44XT;;pBYd5N}Y-;)Az|q^~>VA93DF&Y`x1$bgIl@!AbOq#f`*2 zAN>-5fwL@(hKWg*q+=aV?bI-d-zhMKOoG8#m_i1O_aa$LiJ2Q|KVGLcQoZH@E$x>H zU}<%d=Y{g8OE!NuRWpk7s!Qcp7B0efr6@2CzeM$ zPMNx$Y`?MlH{MwFd0%l;>6JIn*7=x#V)f?)LDov77vvNY{%8a*p))XaU_jYe-vpZ= ze<@b}wU8N0hS=__m2mC1)DhbUd%HLtkv3U#-H@Ux(FYeJ7?|flp zhxj{9*=ui`L<`L(5;_%G>`3}G9l7&u9j3msU6%n7I&z$QC4?FoGPtZ^f&U>5fi_S@GJ8+ilDE5Ps zyMsGVa;u#2(mYlVUU?;8^&iE!p%mq@{TDw^u_J%=Zi|;$<)M!778wXi+o5)g_|tao zc*bcAh5}R%C1}JfH$I9bqa^_X9GnypL7YCo_=KwCVoeQU#cwgFE2Q1Z&o6?F4fv?P zDn($=gSX}W8X|#f@m_bnZ9tm0c1AV35FR;3IsP}=wo}_Sy6cNOLd$T!{26%_6yUo) zZ*hO9)~6lx54%|gvsA@Kf*lqkam#wnrX+`6x_FH?ejc($xLlSs(Vpa9ZGFq7cQQgt zw=V|?*&m1XO0NL)f9>#TyS2rkUB`k&JBgX<^v?_-!riGs+w-jmU$3`#GrFiuzcKu{ zaT24UTzE$UaRU7+7p0QcZ4`p~PJB%N6W`0|`veir>U0gFt*b)~pxA`Za#l5uz4UMIQa++EZ3 zn(ukevIh)dcf0LX`&L;#hxN{x=vfI*AktjAs4eRyE$48h>EuAJ>H^{w>; z;))jYadfl6Hm0*FxV5K~`7Fz$X};kH;g)>N#Zy1<`u4{M@Y8V=#&-0$cH6H2FX)f@ z-GTao(3|(g+xoMAPDu;F?f*`Sx>A3t9zBZt&M3n0(e@K+eI0iBQ?`r5A8-1zf$g5KGwqxkBULxWn{?=l!5F&p-YH4l@ zNgxggpplwK2hpgU=!{;8vWPs2R|F!cMg~*7Tw+O)%2~kB%0#p3pzV-cIZHG~@Xn6H z{Xrdl2NjNxx}jsS3JM+;T?LZ0B4uCJf{It$UFB*E1}0w@TSU$Mnu$0aH;TDgX2 zm<`d3yr=Pfh{!qMea$EI*S!BT>-70-s053jm=t+7x;{3r=$Yn`U`{id#_-yD0;#GpVifX5duYuTrNlqOLpl+krF)9hzmTs{J z`5%COdOY;@n2WBbEX@{&qUzsmHXA=PU*d-$?MkZYv^?oquCiO#dL6jJu5m$dp7@=l zCa;WFvZ^NX0UitfN`ZeKP;uJi$G(IlC7~TSFZ~*NRr`C7I`S0?PtrP!^lXR|i4r*N z#7+j=nv0qOt4oo{s_<=oOzcW$+KetzPawEtskDY`fFGnc=06Kv8)%Q@(4Qc1pBOIV#&K#q~s%A^d}FVpp*3WrXU_cT?z z=fxg=#km;~*5GrrvR>!!J1H_cGZS;>1*o@@={bj%h*H&y#lx|l=r;*9l1 z^?BTTseAtgyzz22y-Kgy+Z+&WqygZybWfRNa5LpC_IZEIvzgcO+1n{^vA>kF$rzpw zd5fKnc#FM|w~u8tCvfxkF>n8?#Xdyh-Vi;U-$*x|VkZ8M@a|Clx`G;CLFmqrESC(y zlXX7iEn^%%%{=Y9@-}bocR{5SB*O+eVd0X8{ z)_gL_f^mOI^0;V*W2&tEO1644q8O&AaMe82M|MPPGqplx{26JJEJ6`3!i z3k$G}%4cxk43~h&h5?4V&1M4DzzKMIe3l3Yg)o0nzs*Ho?VK^ck8wyh_gXFXqi8lG z7zHxISP3!93}rqMBnNM1CQDqx`%qLgusg|umalv@TE5Y`9YW?#X4j(>Z|dHMES+U( ze#1?+C2sAJ$n`!1Rd$gq!2gvRz$nMbuU>FJgC$Jhf*a_c@uom2m>dl!WR9-cIJTFYh9 z%(miR^E6YbIhK(h@yQUEvKaV?C(`ZyZQg%*n=kpc6p?!q&+m8z?3Qe~#Qh(^8dsEx zuJuR+RQR3I>=B+4rQ+#~ng0+*tM5yf1prX^*z$a~7-~ny5U6a)UTcaH@mAm|qKm{f z;%}$oJQ48&@?_aYvsx|`=)!^9Lr^0AZiHC6zFNa#VhaA(LNvPNGu-2e+~KJAW`%#^ zO!j^GRL8=ieqJ<~h`0?LaF9|jORvNzZ8gkHH87M740QuT)xgj+@WCzA=G&`unl6%f z^v5jw&P!%)h~Ur!mZ-(f1aABgP)Z3&ID=0fXY51dNI0;jWHTWD+)k^ty|cT$>&t+F zh@XsLDLlatxpwQh1YN)t+Y$%`qCS5)hIY;v=9SIF7qHE6aDne;I=xoQwo<;Sa}R9g z=_H;N`Riyp8pr%`%bSfYCkfFREyu3b^l;bdAw(%|@3ydi1P2>5UwCvuJRsx(dD8B* zo+llpAK7U?`I-aVR><4WpZCBs0&JhX-%4DfJy62&{|Z+kXA(d>LwqIVVDo?4+pXV{ zZhN~G>>j|??m;VPx9}4F1np;{z_aaEuoLh6mUMUGonUtd@4}y;{cOMWzs^c8`4?RI zP;a~S9FVXdI!ehsqqJAEKd|GBTAj$q0FkR7*Fpnzk6Ec#&GaNQgTx;-`G1{`82l-Q%q2NXWCjRTYV!ZW zU_5ccB4P^?g-+{v)KN1k=m9$2wVyvPI@x(~0jvUiB%#jGD4_z@!aDW=*o2N)Lz1Y< zd~GCE8`)*V{Y)>=eBhDXSWVe+j9#5u@3|9mGkQL%&N_=0X16Y4}M%BZ!z{9id zpwXMxI=AW$Rre%|7$H8)>8lyazbtYgxtcbvboFhJVXx9@G#=mFzRh{T6gywf!IODP z$1qRBnckPfnc|8;7?2DunJfK$(VSjNBqx^eFS*)+))T?tOY-Q}6SbkSa+iZ=I=;@`ODXEzQnaCySz4PY}GtIdz`2 zSzaPuqh|4Z;kf;JT4OCzHX;)uE_@qFh&XuEV$?AbB?wxPxhZt_@%TKtKtJIWbmikX zD&r9e7_;shPuEg9II|EUDn#rJLL*t4QgC@jH)!z(xFCOV(Oc=ac(RJeR>B~12p>wh zOS)0R@===zMT(Ck27AcfT2`>bsiVnFDrT^)A9(s&nPpV7Wab#z&(+%kK$g-gZwdpw z$oR)8Ka`vIbOAb>6VOfcMpjU`PW@%2OA!O~cFNl@iIGw=Nq#I2h_kIun$qV58}LX- zE0c`%6_bC1IVPELNF|_z5^mGnLvP`#y74;Z18|w?Gojj+;UW$1BqkaUw2!x>vAx|^ z7kkpidx)uGHNvFEK?-NG~08#6!`NQ_S znwqhArRnl*!+SNQNMJD8>W*Zh^NIIrYQ|^K^1h9|0qgsw9jlNJxV*~{Mrv1}c5VBh z4wZjg^os>Q4=;ij3z204Ev!9=l8~f7v{3eRwp?r%a8geqg?gm2U5;Y@1%_q-e9swN zL3$+1VZ0sm`2Y*&F&F-?Yz|kJg1+H}Hu{>}H93Y+AH`W(%H{Pcn@w=g=~h}U=drs9 z;kY82`7L_91_rKaTlF=yG3o{y4U#L|LAtE=H5*)d_fK zNR>^QLs)y610h1#03wULo^o?Q5mzlitF(s4kZe#U;cCmcQTQlUkf?yBLuN3ah5;e^ zu)J?Q0sOXH;@=(l7ZtZ}r+|Z07{IuOmyMe$dF>vkX1roZZY-C{%qvj4YrY~`Wt)F@ z+?Lw6`!r&%7VM>~1uLZ%tfX2nb!s6WN`@jKT$0U4Z-Xm|8azuq_I65aUZp2ZUb>!+ ze>(AAPf342RSPwnjVplcsFXb=Q84c&G|rf1tVsQi8<}gz-=cmCC-wv?3SNV7=$wq^ zV<$A^rlU`K$5NXSIZylkJl#K#_V$0Dr3iSUwnItaq1aPc+C|5(Sp@XKI!7}XGR+O2 z76R_uDc{$}wlYrXlx+MfwgyIuG{|WIkyc5F`)ka|Cl3z(h5nD3xdpt%tnUVUZyQiF#lu+1Kej} zt5&vK`@4dAskmymv0RRZ%cTc~!4~^4@#2UPON$(YKNMWk%7-IW*U;X3_N*n7bU1n( zc(PoE{%z=ps?`-D^2LD!3rl~cBaFF{WbRgYUW73!eD)noFPYD7V((SUN2r#@ixzv; z;(%@>hgH5jvy@TrQ9-cha2;CWw7^gt@##lhCs!Ni_oF*e$r0R zhiIhxc4MBZ+oA=M6?iq}H@S8kF_#kZBt0P zWVFg?BI+}^&1%ZLd*Ii|EVUE+jD;-Jv5!({38^p4$c*twk8w!6VU$1MqZoT;ImF~K z*#mMl=JsmrskMNBta|8sbz^^{#Tb7cTkf*+m<~_R{RZZ;Omd}}5JN=Vt@?Kk(|G7T z1Di1sN#0DoK2yFytjtk84Y7HEXEhE4fPvJ5Mbs1ow1`ZjMF3{1;-(U+?=75jFVf}=;w69cMmv5-Aa)l* zL-A#fcQL8+c&4r+E-bA^tV3^^wpQmy)OAII6v~X}3INB)55mDt2u{?njXA39?HdeP z)-m^*H̯R+;OHLw8K1}Ye^{jiKtet?QorwX=V9-9!vEevqzzgYl zS4q#}p=HURWXeq$p2(;|o zr1Q>AxGHun!8dheb<9RcrS;|VhY^UrWdNRFbdM5&yIJ^^9&lL5Ji7MFKJK`cMtZS?l+rr*mmCvL}VJpzkK6?u9L4@`7{;rz~qIIGVr?z zIR(S`VeCch#`LlC18x)$fLlmMgoq_9W2MS=8}cEjSnX5@OWsxyfhAS0m#)@NWv92P zPr3E@u_$2NASF3$v${kNj;2_+qe6Z+2vKab*keq05x$Ks)xmLNZ<1~qVE;>>eHCb2iP!)(Cig$Bb3Cckxf8?zP?ZIs6tXo) zEIS9FgK<;q)+E9xd8j_m!ns%!g96v2u)N7E_wIj-!jH0N-nHZZk`yog`Sis>lI(s&W{w$?slhP8uoW;JN%R?r`A0p5ZYXRR{iE4z5I(SW8=0LSYVSa^f=z#v zYvhw)TB2q%_^!65&8X;%&aH1{dC`DdLeL!OH00<;jGfYQ<$xbW-jg;fnokO!4e-IX ziZ%{ilRb$5yvFFiMG|H4sAIwbm3NCqKIjgF6U!j*)V~_iC*-K)=M6@#N_targ+WUFUsi8Azl!~0Ww9-^##E$o=SE6r`SCW5pZf&-n zZ0^>OO{9qS$9#EP7V%kDrIu}zz+@uH7?!S3O*eJfKc(1;Wmm$E6k;+W#GqiP#^=vD za&e`vWDJxO90?;c4b*dRWglk~qegPPb0zKN^1~M5xETRqg~i`WuE!rXjyX}%kWfHn?u~!a z88Bu0BO22P8n$ef&v3yuyhgTMHnOdBA)-E&qL6AmOX-#^rFctxSIU15{6@DOU?Rn= zK{H2YTE0i_=rUICwA&q-TJqG~+f{UA+q7UlTZ81N9YA!tx`M;VAcQQqs(+7fB>tAq z3Ty3T^cO3{3dMRxbGg)r5b3kB!K3ntJVR;0;&_ z3Hq$n%D+QgfSB!&W#Z+Z&dWREW%t#eHLrehUVT>o>Zh7lhdM#k%@8t~i%u%ttvz;H zFPJ&FABI54G5j;=@wS6)dj@0n%)WOC_u39@?kV2fwQo*yxVg^~+c@SNdOIzkP(ol; z$*eP5u>=FuQ{#VQpY55^r$u_4B9i7~Tp`&w-cJF=7&Uh~nJn@7yBXHMwY_a#qpcX` zv>rmpZWTUY&}gsd7anU=d|LlL%Y@jvu3bwl|L)k1R6S0h32W$II9)gUD~-7xq;*oKPVmAG#rGkKN*9D}%P$7jK#CvEj? znn%;?aZH&I+dE9KIYkamKBsX+@6ym}GdR`_+c0PK@g3!LR6Ahetj4FKc$%kIX`Fq? z;;ZzRVzz&(&PTjtw5?~eA{)kGbRB0pqxeyjM-DUfdPGkO^yVIr-2jZ0UN-PZgeil8 zuZ`YQ4Gb7t@&mNwL6Ys$)Y}o;$Lt{!;!SheW|dU3>vXJzRI!RBKs2RzW4*!^`wFIv z&-rJ0xc!a-T*d8s%wwtN+xmJu)e$nvs} zDoMIPdA@j#o2W%l83ysYqhN8u7j)Fhqeez%xu?zLUNoP z(XxL+8ze__1Okc7X+}chCUK9md%-=&WJPpoxZk^@pf1gQpuq5(Z;&Gr(~0y#DMcy^V&nfN zy4nK=**^<$?fa}pQY#JRZpg?;H{=;>x&CVQstT)SqQ@O~b)F@;M&hg>71GDPP-kGe$=kxzb;VXgAh3_q-;Fu&Q94@j_jXgUji+{y%gJgVNK zUlO#bTAf1Ws8K@hww?h|6Z=6QKK_bX-!CL&vef`}llV^y7uWg*m^6Mh{ndWaAQjq~ z4bw5j!YPLed4V+6XVLj#bRkY?GQ593hYuUxV!FVi|8g-*;!!b<^}%lqFLXi;@?TT* zxVim0o(;dpqZ7$h=23PX=RwBilEVt3RmXt4f|K+2sW~denvI@+pPm;NTk0Kjf9`yr zZY}2HbQpUDYoRS$I1Bw8Big0uiQNotem; z166~B+vgjP@N|2EYv;*)mQM30YWyElKz4QT_quBD z;D`-`y3uC?l6z%HCs@UU)HDJ^qjgL)8J)6jbjnWYj?O@?0XaB`^*2O7K8!Jw#+1KA z+>ziS)TrQ#t6z~t7;3!TgtMEK0#X3J-VeQ^PP$jXAyS#g@5n>W`y+n><^HlQ{Ut@5 zg+?&2z!)Tyngi`=&j|AJ=segK$XHH>Q*h@gC|P(G116sG90`eMOk8ga=g ztXAx=DX)eFeaTnsDzkq)1Xbf@@P;l3iX9tvqFAlim$>3ysALE`+e&)neV%&K8?3;L zGD98cvK+Jj#U{LJy6nTFO0oKktliAtLG<+@$n=sTz3ee&MjA)}#nEWjfBy3r& z*dXSAjo4w%Q^AD1DEV!%06u=4Lh9pbp51(alL~y6LeP~gg7cRPb~I+M3ibzk8MD7e z7c16Z@Y}0&IzoRJ^%pm?Buu(h+*-e7{C zi?a{W@OyL}zmFzyup!RM(QLA@Vy{YmORDHEw#+YGU4OA9FKa98FScrK>!Jd*la`V& zo}e(v(oy_+Hv4{Nd>t#dkI=LBLzE|L3Y^BU&Z-57i;sVGYh8u-(5-9{3-()R%QAga zDLCx=lGhXX)#BdJMLj)@uY&cEs4&-bl3(25%v;>Q7KeLCkTT1vK@+^Pbt|Wjnzgz| z%jJi%(ul>Y88kbK7m3}``kS*^uE$B1u!iY6Rvj086Wn(^oK1)6ILQ4ZY_#Ah++$DF~*R5$T<|7f;GW=RYD*N4~Hst zMCmqu&$ccv&%&3d;it~j7?+@RdT^^lWzCVBYblE=(FHcS{|2*km{_*DByUQ2h z<*PTR;SqmPvfneABKremC>hrt5!-j_%C>8DW!q9$2HpIOplVJ;d#hz4+NE+%`fILC zqfq3rn zpc8)_Al&EhPE|om-vUidwFII-B4gXO8QTJWqWVa%tJVy>%*ltQ}bs+?#nKS5&|m`WC2=5KwQrBiQ% zZTQRpARIt!jbZVDD4~1iTC&74(fe_+NX~z5rbAvs+Ys9ei%Ax?E{LM>Yk1H4{jNp( z2$Y6}Rb^!jt7_|3@osC+S9ri0k+)D)jf94Ov0REYV6f0bSm-@i=oG})EACq4B`)&f z&P5IqOm&eL_bhTiDq6G1QAm};)MkZLgqE<(QOj7j%mJPflb`ceCwrbKmMz=d^b3EU zoM#shyRmh79p{poL!Ml(*t%TcK*6TG&o-iTq4$+LFIagI+*?lI*`S{^aa8#^RGYD) z`}(KLtgV)rRAy;kmczVmr#DbPQdw>O-9mHMv*j(-0@;j4+kzmd6}v@IqvQh78FCVW zSr%I_SLs*eOk^&P1#q?WsTj61;Yfe-W}45;2kfDgVByf+qJA(9*#iZS8_?oVF%ynp zO@sIp5NI(6l5yqt@)GR#OeixZMho^wDFW^X6GdqUBQP`qAJ7?0I7xpVi;|6pu_M22 z6ecY4?xI`6Bp!Z0dKoq<>7O6?q8BR|q1;s*Ls~D{v)J>Q=^g*Qw0`5zd?9~B^QRn| zFM4RshMg}Hi9HejKg7NJew#?PF#7*I1t!@Vh|5^_3vR4!OV*^j9Xsu~dv>?tu@Mw9 zV+3kIkrjz2IDeh<0^g&YSNa~|{MM=}C?q+Tnf;yJnMp)Z)OFRWbzc@N-AVpNCK{;p zKt4DnWK88J^1)#{^2vhU$lHI!T4&@JX%K(#LHv>r;%##B%gm|~er>tb^Fx}aVtbnq z_5PkF?;;>mRkB8I&OE4Q&^lbz3AGaVnb6p`1OGF%<`7XSNfl}x|6}}dQLWZ%NAOSM z4F4YB-xpZaxwt!geN569YKl5nds2GEuJg zb*T3Z)!w{5_KBbOU6uPD%6&)TQZIznY4icPa}rpZrdy|LHs@zqV3CRKd@eo|EwLTt znz!dn*YskO6788h>Ab}ODzO7{Wj%{4!}qkSAL?QQ2Yk7QmiEMN0Cw!nC&Q%<7|?At ztCN7H%8gY8jVY^tX+3{4E39izQ!SO;AUe6UAp9i{a?2`?+4Qi+ zwX`=sT$5Y6EdqWRlZv`11lOA%^OLr^=K<@Jp1Zgh?^qcz(4CbLFJNT^o{|hhwP!p! z(zW^@lWx3X0b`TVydDqUiekkBn>WP|;RQ5t`QwxRydfG>wEXVJ+rKDpj)5xvX=bVO zXpTY^wV<|s`I9-lSOGnglD&cf?~@h2eF2k`lfEzkPm{#H$N_JYjlYHouOHsBZZ54q zJ(C>3ECOWblS;r*HhV@}l}9_!hU}Q$$d_Og`%@p006>V^-+5(j-&+#2s4GwdDB8>HfobA?y^{sOCIT>#lP$p~AJdFX zb>dn>f&#jT+w<+O#Ee#n1Qx>r$!ifA>f+_ElXJm&0YQ`P!Oj8Ilf1%)e}Q*(@WAoj zELDXB5t?JM5?yG@!M4T;7A{SYnA7zL+bANA;8B2$4V%0*UWd4#%ajPVuvyTZ6iwh$ z1Y@)x;=3VbZXMsj9*;vH#Qi%fC6N^xG;}%q_f?$)R#;q9ANz1mvF7~U=9H*E4v4DW znd9J&BbBhJ`P`b~Sk-|Ge;wb6MRVA4Bat_==bCGsy0}jLz#Ip}y^}uFNdzmEf1ndb zd_#8+EpJ9WR!vuCm|L(s)JUQ@)qwNUt<%)NZ3gpJ_N&kTJ>qXnr=_JvvG3TeZ=L!^ z38&~J{Dw+83zr$UtmA)}t3m9RRVPu)s*#{&)v6?Ee_2O$lGZG0og&Tl zMiPH4T8x7eJVmk9#J9rSN`d2eBZ+2IOEGzpe`qBwouuri-GXs@KDhK~k#%~I*0f01 zjhMO%hib^3Tc^>X7PpvqI?(%m%M951MlCaF>44-5{Yy(`EfrcoNt#=3(OhV7RMWwK zHeP9(Pa@qq^`YW8f4bBAbS%C-s@)x>qT1L|iMB@rl71I85s z?1=*fdCQ$2{NfgKg#5!FJ9lJm!DZv;osyLmFmXgbHB+JN61b9Guv07Ltf*N=ms=N< zTfU$O4P9VuRyuhGoD35{ZOOD!nO7ncnylbX&)=Qf2{l(OZMzDT-{~Bun$MZO_3i-I zyHe2t>xEARagj*G#exCgt~3G|c#AC*-CCQ28e5KPrgMzw-&tNuW6}#kx0eo(q)s9$ zHD<_sG1BTFf7+p{0xKFElc&x#?3<5Qx zfRN+B&E|kgchnmd^r2G0@f%eq4>u)xfM%+)^n*Z|pu8FE*c3Zad{6OHWl3X-m85$P z%m=WgOjRb-u@?zNQtRWxn~x6)NPuhyB<)DH*um0XC5jgXL9|qC<>SNg#|I(&=94p& z+eiWae=7FfTmlYqKQKm6=8VE|9_?^Sxy(Ete;R#ZbjE{_2#xPZWU2p959ON2Z$7@^ zcXmTkn+gFeK+V~$Ie?K1I9qN76RFCr~wXkv8+T{M84%9bl zrjzK<%0UBv9KTWfs%E^=U#q67_05kTk2T{^-+cUd3}w0*_M<~fg~!*biC9LhR0?bZ zFSWj@^{tN|k2kM%UDe4L7~LB>y?CB$5lMwkUiIO^y^~V=B84Gof#jk*ezj1yq@o=KTqfsrFl#bf zC>MUG?A8>RkqS!%S$ilGaB2CKzJpRje=C)byQ=pK(!dly35ZC!q&Cc9%CA&>w#`h~ zCkT;{UEq>2217r{m6sGsp%4Z5xo9~;WzrtuHwuAGx+7Nzfx;|Pto_iOwX~2hxi{|)Z=B3a@+D^Y)`3_6GWv`e)AK1dYYN(3Sn*dfDE`#HZ>t5iZ`9?dn~e}oh~ zs-a>59KQ%Sj-p~eEFjb@E#XgYA+KN~co^6rz$E8(6kGW#fa_l+eCme?nBf9Ezm#N7 z9kBg%m5&c;WB{and{8`}2Cg#R(d_toc4sJpTTCe6i4?&-TBHEM41r@qVcw9ToN;${ z{}T;X$|_nYGkXQ#|H*e5GWHfhf56U{4F`vH31j`?(hr}MG=k+b)KkR{-2!{Z?8-r7 zI~WB>=2p?dc#tI0h%E{hl9Az@G2vMUIqQLh2ym#yQ_fvWYBMKfK%E^bg~ENBa4B(7Fi&@rx^$|4`fl4^iXDef5}fS{7Y2v zS|utw3o40p{Tmy4#T$B6ZV1ki+ze|N8(}|RXhIC2Xh7$L9px;2F$ydIS`Qh|qMV={ zAmM^H#|l}vXlQ0C?E_*)AGEV2-LfGxv(Hu{A8WKhdmdNs`0+i_pN-_v_fkzK(<7^P zA`R?WzDKO-sa5T#_F%p}bPoB1)ek7A5c+Cg zRn4;8ps(tEidsKPCgtj{_Edkxs=rbXsT4Dzbe<2&$rzZ1b&;SRe*vtm)SXwi7#pSF zSMt{)0E1o{yL|u{aPP zfKbV0_-2}+9{D=d9v7NU)J4LSHacFp6RIr0*a+1}oPFZ)fl#Wg{J_BERLRpT87gAq zHJG??`YdEC8pR;~3Bj-?ANbtX$(qq@GCRuYtt7MNi>=`TcOK+ow46h5a#>J_a*vP% zAjj}!{bWJ!e>r&%oM4nyj<66#Kv{XT-hxYrCH-VnRF)SpS7`k^3rTjm&vlQ8UxKEk z{vIuqx?Ki!x(yD0fA|G$-2b(u>E?Y)lUjkN60BYo zv;FtrRiRr$NfMnDK%7cI9PR~Tahm--bCL|tqRax2A%V3;E5}U@0AWC$zidDf{`C#@ zw1Zsw1gnp_*CNm&dXDg(cuXx#bIBZzGm}02?rV8<5;S#kkU#20ibsB-xy2o*uS3v} zx}tL=M^={K)PG!pgGTyU3v?2Zc`n|B(+9Iu^%I)sJj##fsvvw;CE^T)w{T!nvXTUq zI??8WZ0P6e4wG>_y0#*7Zi6z{GcpDuI3N+BcTAQ)XuXWGMaJt^L6z_9m%-5VNnx9m zH+R8eooR!X@t*M`vGC=g8Ii5cu38-81!U}lc9L2psegLjWk2NtUUMyRnLF4pHv$r! zgkJ_fpMzht2mC@^a!-UNAXFcM9da7d)<-(=Nm}eU8B-EL(VaB^v$EP@f!vDpe$gzr zQOs9%A|Gr#r^&y64Hd^WA`DfT!57bOvTp$$D;X!nE)myq`cAPypqjvEzRI9^Se1%U zuF4b=4u2@km_}jE>}jTz$~4!VpEylzP7bD<&1hSuS}FkU>;aI!K-P*&6NiSb9j1Ie zl2b*CV!RfRVV9Qm5(P_4Rs`si(R!0ZtvIriJF3ao&$MD9M#H zwta|6t*CEQE~}bRJ49J7uZTjnR;qGFzcLWc4VFqTk3qz zUbgfAFkZ11Hycg6v_?4;=w*0 zsHk){Pgklkx2M#|ed&fRZCI%cQI= zK$Eaaaq)~^)b)-^R}n{etAC+46n{eQ>yhbyGoWGdQkSTwoBj^{WHFn z_lXAJE3f?T|Mh?WUySvA>3<`jdK^2<6#}-hL?RK?y2`Kr1y@wa)wwyBYtQEj+Q%je zPC0=|vIgLUW1VOjD(aq&;UuVJousP(>=TJvS_f4psuIO2QNQhD?4G~IZ{aReB0f%r zIKeFIUs>13x>%!ML;5_Y4ye`7<`8$4;aC=%W$*GnHu7Pvy(mmS;D1lmPwl{W;QWB= zMSst>#KrugjD$Lmzr|Rwo7OIXylb&Qx8%BIIf0!d3Vlw2v<1E+$KuTEI$og16xG+s zVou=)DUu^u16?sh2Y*UoWj|3OrwTr!%W~ z3TC)vc20}|Eq{zfEK}jX3Gvp@-c*}gr}KQ7(2}myN-!?o-swdg24E}QQ>#iI2DvE- zG>Y2!)Oo~nf;2T~wfP+f>SrwbahQPni5yZbkV2|~S4xx{y;TW%FwXoCym6WFhIBn>K+Vp#nWbnt)&BpMs9=ArLTv7k+u8nYXo2=U;&j zDD!40R(J(qNQRasR_-P@gY4f}Pv&#~#u}LOZGS$sf3r)4>{MbzN&YdPegDSVDlu;? zcNe?;*y1uob52SqT{n`tIF#-693G z6uxk~hh=Ihzq-sPPB~4BQ4TeI|3pj5&GK9is#&JPml;oWP`Hgg7Q_~?V3`n1G0875 z>VK%t0tGua<}noI1EmUtB}vTVV~n@J-#F3in1;s?)8&{ICC78$dGf>Qk zFKPp%TlG5BQi9_wIe5H(@msh0wQoOvNel6nZ$CX2-(ou?6vodq8K+yfnxtE}4z(~x zB7`kAuu~{qFGTG^)Pw>*p2I@1l||_PxqqOF>bvSg?qFf~u9QcoRS;*N9<^MW|D!8M zN&u!C<^KL&_1p9jSn~I+;&bj6OF~O-l$V|q79KfiYihog3c8?t$@=3^vJm}MXlfW> zfTHG-bu842Dh!5qE0u*`R@Q}IRMy$aDs;e`gKO-5_qDL3uFIc*WMksysL;~&!GCj@ zD?cOMw87Ltk;}*$+_3n{n<>aff=mZ>8AU}6tbJGmU--gPgsWuq^+5`RQf*A{Disa? z1q9CQTSHyXLW&Xpwbw04r}kPL2Q+jd8Ie@y zX9wzl#aNA~XIzwj3Q}c+UsMdFQGX7{b|R?|BCpVlaAZzOx&+XoU{sV~l{>jDV|&hc z^SrMxS2-o8h78niTUwzrW^oS+mcZgko&*lwmQ(rM8GoP33Viq3teJ+4i$H(~6oSn{ z3Fe0hibcW*^eB0`p-8p@Y}P16oiRNNVgQT~BLpTwHJ(^>uL+l9ZRrW6$$uw%ry-zX z4JjAK35mc;L)a{aCBKjmtQL86g+d2f#wJ0JgvJj$wA&f?_BiEI^B^!`dJ2ZMS_n^$usfq^3HzuwI?Cn=li6HH3ZdE z0vapsj4E8qnYv6UX27jA1AjFh*QpdH<+-eU%Q;oFPn3jN;O2nC)N#3_d>b9=*~37WM83tbAnc2$Ob$Zx&z4qHp^{jlKqF@@?X8FMAo&1#oP!jW*oA8& z$d-f?tgyAFggd!0t`ZWL{08KY7_u*H-MNA37{a5F=#F_?@&p!=Tz?KsC%{6%u;=q& zrG?}z)GPD*B*^vb*>zYn^Rq)9#;=8wlqk-%i>$z8|6tIF>@J6l95ogCZ{-ZUa&y}r z(R&F83KFaMJ_2-s7DHl6rw`0NS4;za+%}toooT6OpsLFqhy{Au^&ZZ#%G!6$yjgd% z2{=P~1h}Q-43>9R_J8@}^tgB{PI^7~7bau}8vVSBO&f zNcQtGd9D4g$riJfSKlfpGI3u_R4OE8p}eS4Dg`{XzuNW-^vvIE_O^1s^@jUK!EqS7 zTy6vpQEc4y3&Kp-sZ*M& z| zQ*>_p!DBYk1}{Q08dQ7>@--e85t-&S?4=xnRfin?oWC7GtXh4IMaEs;+M~e{(>!m>y4-k2Eycr@9{577GQU)c@k?yLd46>GNH zI4`cIr}>uKN5b^w1%<|nMRmS=xzW{7T5{@X&UfvI0}9{+IGBumM(TJ@kWN`~vTmAL z=wG;!_JX_4%t6;a&x{9CRZjvSnP}*04-onI`GKeqd4H)~x?-ga4CWBkguk7Wsh(jm zU*?JIjVebgPm|@PSz%0HsxEGFWTM zN3_Xc4;fgp#Z=ToTO0#elyY;SPI$1cdy#(vSe#r3_Jx;*pM03xkQs$+$NOUaYAk}+ z`#75U31ysQR*HVuknuccoepXy#ZYAY8CYs`S$~+bR!hD`0AC+s=L3Dg_XywnpJ`IO z&SO2YcK3Y6I?u_Jt!+;mLiWRyi)>^0Bn^mtOVP%OeHL-TW3DkgzD5$KV zpITZ#3b)_H2>kFV>GOXz-5lR#A_7iU7$VyZ%{RQjPPLjYoH1!bm%^h&Vja-}A$g=U zMSl;#t<`0o-Y?YegI{DsVj7t60bZplu@jS3QZsX3g%ZOW*Mv1Qks`o?mIwSs$Md_s zo39T9wi(a!pdk?Cf;gfs%0yQ=(+c&3dot$>Rn;)*(O8esQ3~txfVvA*z(f8v7h6=Q0-GU^fA% z9YHb5^>=wO#Ei;xaiHbnBcV(XvIr`i>eAT#h8)6`gry8&RA(`y^gNu9kMX(mbnM?* zbnF+t6-A47L-yL|oys|yU<|2FXn!8{g^P>}Fu?`iR}&h4qOo~=4Cdki!&S$sxqX*$ zz{`UnDCv%0PEnxVL&R_ckGLt7KfGQSPqODPbNtQliw9Y26*O|Q%NG@GNPo?sh#Lh| zTpo!AeS&NgB5vh}!dF$?-$KZk%lOq&AmHSs5lHH~e{lq83ijn;06JXm@!%B32FooU z&QWa7=rW3*ctuGt;Wl5G$nx=Dn+TUZxk4Pgclj1uM`o*IGWJPy$IDp|*n^r*#sNuo zOV?32ikwAaIwT6ugGFq*B!3FuMc}On?$zd8JYG`!6EGD~cpEJo2)7%jz+0r`S<=BV zA20UIeGoM+++~@DnH^cBl4y|Y4$D6IbP2eLvn5y7jdi-d$tA>>;B=>5^3*4BLLIOO z*E2mMLqr%waTH!p*x}Qz2t*C1$zv3_rqRmuB7sq^(;FHQtFGmRI)4kWIs_1KqA)~% z%?#8BV9&jGUcMN1LepD+MxzCPhf~Z*t(rqM%dQkqI~EnMSmU6NfJUB}Sw+OY9OO?m zoh0ygVU*Cv&^+J*AEqiq70)dD2^aZ5FK_Ta6K&4wNZb6+ zL|d_%(q{iN(Wc}{hksrUYOFs=T~M~Wgv*RpDiB46gd!UD`g8Eru?DfExCk#PQvhho1~g>`FVR<$w2qU9qt%Y)kYg0Tpc#EZGx|s~mT9FYL0gc4#i-ge$1p=q zBwwD0FxQ;^8h=Xk*TKzZb8u5o$eWFJ;IbH!BkOdO*R(8aC>ypF55UI*S*BfDf!uT+ z^Yn6;jNWXpOO`gAb=)6V2apgA2pXv;fQcs~fvk1gu}qaK3(1*WzgnrVnh=Bbimw_Y zeT{^s8qgXrf0g9C#LzlL51>+yNVX2GQ>m;lXw=F0s(;uU(aRkwVu3?*NlGXvpUHH4 zZ^MQ*Cy`FZa}wQQ{Ktkim(R^ttnybluqW$u*Hj(|K9wN#_q31O4dS)Sq_y2D=iy_G01e8UVm&r$k#&3kpjleK^pL{q|`5R{6bC* zVekez{(sy+CWRXni6gIRm55rFm!njar&amVyYhsfCQ7ZmW~HL-lm7fBvUOsn9yQPL zQTSdrWZMqOu7k63JBHank?1FWK1VU@qU4uPn#kS|7Ks{wXVW>0mDk-vK?&JmBs7t^ zmb?glpA}Xbx#5uurb#?>P7#&`VL#Dav=Bqlhht9Rcs*y3sIn9{Ww6gp{Yo5;xh&#)Y_dqe za~;w)h#O{4#j8q18(MSXHCY;0yql;c_{nV^5%U_5->rE$I37mFmWj#^;yMI#oz;$8$x%Gh_{w^-k-whViAE*p=7`b|^ zi1CK*>!-pxF##`4o<0n1W0sG;^T=bqiP(u~mCYQ(9DoDrDn}yP=K}K#v(Sn~D;U^~ z`+{FuA0YO45rQECL#7cV%nazAwP-Xpt-Nh)5Ad@e_ndXXS`pJAN z*c8A93zRf#yodu-JNE4(j#^bnN`C~cs#{hkA$mH1gBr#*#rTpClQ@EM?$8J4JlWnX zr@(bCtS?5A=E3FW3oVi0rwE{u0jgnJ`+fg$ z%ZL=ks2G8ZBY=bqC$q**khcO+Ot%q*w$4cdT95|Z!s2JheI}?92!C(3MHTotRS;F+T!*|ovKx?=7igQ?R~8k2xp znoY|Z6#GjC5Bg@4oPP{7n?hI|4m-DaHIa2`qxjMXw7g~8JM_IfAiL<$y#`0H4ZwTx z0Sv{)Ulkkg_g(W6_j30(y{S~_7jDx`Kbda9G}vWG;^9@}f6}97*8FCVn#PW23rL>U zoX|+~%?kNu`TKS#Dt0i~-mD}KlchKo1fnA@yJ+Xz;RS$A~8=;jiK%u(MAk_^qm~nJx`MJLwb2UAxL5xNuP!<|Qk}}kC7*YtR0!ZXU_Dv5& zw$FtB5Lw$8Tl_(ib!wt6U?XOC7X+G$7U|3ckjr;h)+rj#feoW$xHO<}7xH?<6~x)p zI-M3lx6v{rYkRfyZ2qLVOmH=Hzd)8gF(4G5HWW#bv%7eFOXx6k#tWz;@70$#*M=D)URze zntxxjY9#fnwf*P9HrJ{H(U6W3=aJ5+YO1asKY`58pJ70wSJfVMyat4+XGKToe+XQO z`l?!OHCiojK64@-0O?h)-S#F8l|W|AFg4Xzo=1DV9>f7!q_G{C>Z``2YdcMqV9xKO zTD1yoHJUBg?x@5My~tEw)m!Zbov0*PIDZaJ5>tIuYuk;ctrDJ(##CRqy-ugvR=FRx z8a)NhqtK&ue(AA7g0&pn#>RcNg_*0&bD9h&~3QD7|;(wyB&I)Ty z_#+A%tgzmOKcY}tOt%exO6$aX;!6TOc%Hl@)oK@eEtbUdJP%6vlDvW^9M{nihJU`I z?$IQAR!w!QEDd+dYqoA?*&^-fqum)nI$;wMJd(wT*U7-wT|qdZTOf>QKGfZ0EJ5 z?d6SDjYf~bOjGGpjZRy~qzb&+ZhxyiqtV4y+ugjr01)0}&8T;<$yP^cwT)I?m$sJG zsnv}Rf@-zXQtI8B(d+4DxZ?Gy)&EecN);g4nXR6_zE-cAt+R`Zi!vl%berAonFPtB z(Wu_2%Me$y;m?=HAi%MCcGewKWpM1)8}(iZ94{`qwML@=$BSNL)M&soqJMZEfo09N z?t5EoQN7+(>g}GfwUMtnlU{vNmuA=QI1{^Cnq9YM*I{dD>L6Yvw4d< z$IGa*=yW@W%&6^o!;hN$u6$Ia=KlSYm=R|&ckHKcSo~M*meX*$<)x-x&zm^XQd4Ss z4O?F7dau@KY@>M3_Bvi`3x7#ckJ_}og(PW*)?IJUg6HjSFKf-{$tyRBVh|Bv_`iM2 z*@4O#aaI!=&llm#M+(BejqI5ptW0&`AI+jLO6I^2d>Flp!szIRP8We4lXo->BJwT@ zZ6_iFG`zPHa_LVfivpuSZ8C@!u}@>=iY`gs)DEn7nnoRwR=wI&+J8o)wHIeKj79?y zOt-1jt4*U*-$6p1n$c}DWZ7vejat=cHg?fay@* z^SQM!Ue~MVw?=nD zd!8%b8m+2lcP1qwrsmmQMFx&uujkp=<9A&+d5ix&_kd@IshzC8^%8D$ zj3%S4yFH~*ZyDA4zVp*+aH6Z$t}FFs&1me>N1c|@V~4ET1p25`qK{tHyn3(k{ns)3 zDg(g|SkQg-1b@`%q}yt~#N05_Sq78Z@Vwi-kYE7{&dy6C?={->;r8+F)ateR%hy&$ zrgrSo@SCZjI^T$Tr>iuKZlm33SF4?7qt>g}dY%0gF#}_HI^BzAtu50`EvMrFXqPVX z3uGYn(M&zsq1&8LxlrDmYSpWHYDOMyh-gXo&2}%=?tgTQW)3Rts?n~$Fuz)()~>bp z0Aw=+88f%NMQrPgR0)lQ??={1^-R=e543vusp-#YR2QSH3i+QyBr z`L%zawQaYI4m(#piMK@v`BuPb_KX@f(Wrj$3T*d`4qkALdat+yzrYgpy8MG`t@`q{ z*)}?DL4ULCyEe}o&E^ZhHtYkjS)#1<9^?A{{XkWCyggb)B^S|MSsSe@K z7g$eeSS1v4R_oPzUtCM+%lu^KVuKwh-1(+h&s-|;5-qp0bpQIgwl7*2tv%~1#=wA| z!$HqzA;np%Hk5j`WdPrs9|sOlRinn}+g3xVwSSvNt*J{VgcHGCqrtfBMoa1Rj2_e} zi-WpGn}Hm#*haIXe;2n_{Sv!Y{Q|#M`|oAgg2nS9UVS6NdELs(K%~u%e{KlUUjlfw{~jnT6bISMeA;TA4}&tcB|>+*s(-o_yCv!K+RntSN$WFd(q362*Qwf$)A^pm zQYQ2GTDUd08|Fv7Ua!}Bf&T5*oAoWJt=)7cljckGulOQgnc<+@ zZB_T)iK4EA)i3JZ`q_SVMFNv&{_ArrVutGeg7k!CCKYpPYb?;{U{z zyAOXfUqTh)@#jAPsF zNe$E?#TwgQv)y?C$9h$_O}jf{GurW*d)Bz!s5_0xe~Zeb+F?M^*$d>~hMSgtOg-?; z{3c=A;L#4ACXpSx5L4H8lmt@K%zqN{SNL(1(Af9>Dlhk>3xBqx6}+rK*}R}wZaRTK z--EI#{?@xSlV#n^4T-RgadF2<<8gg)voqmTMal47Mu&_0HR8^162G?X%Wk8&qrXr%(~pnW(E+rDm_!=$2Ikz0L@fWPc@yY~>Ky ztSR-TQEQ_X57K=>J74GP2hEG_7Vq8}^r}5s{c&;D?lsHIZlm6BpM6hr0fziD*Yb=hku^`6ma_G;ZutJ`jMyVd4CQh%WK;*Qo{+RoaS z(P}_vQE0p0tSM^=l_!uawMNru@7>%E*FAzLyjH6j)!wcSvn6zH)f%o(XOH%vQRT7~ zP-51qUF>d;HXxS|)RkJD6CU!mcH2g+$27GarC#U!-uAZ2m1?by(QWNv3QLu$b)&Yk zujS5K*i1&k{gRX&D}RCR#Pzl=qHe7_7|9pWsM)EVm&oASMZMS9mit#nwGPn60=W#% zS{J=NWKu4S%&|^=IBcGeWUMo4o{w6R?tgUNsdihaZ1p_)4P@l-yjO3YmnT+lH!d0@ zS>jo3b=t$y#Ol>St-gabYwg--57FYZdS1Igr`_S;Y*3RY)_)tGcgpB=G#b`hBWYrT zUaMO}iUU37zx80Wg*L3!QtGXa(duk*$jx@+>`a!24r%B%}TT)Q1 zw~AYMPrHKN?b1n^25Z-;H*rhsf#+6R9hnBRX>S5&J&M0YkJP&8)&Ky?XZ@@;kcp#i zcTgRc#K6r#tKEKqak$84GSy}xzpLS zX}JsSzM-RqVz3~|g7^vX0XT3uUO)$^?kjB)Wl@i~DBJtfi6}3zr zgz9yoK_DO2s%}7cR#$dcO}bWt^g^-Kxxp^8zxiUIHJq^^F(o`DO+N&+%vx z>izyO6cu$n%Yp@(VE>)kPw(tGNy?9YQhM})(&M723;{g;=~=lSDdsMRaATn*`U@8N zVSjjX!ZO~02Y<+nrLR5h;V%zpl`EZV918P|rmr)dxwS(Dc8*<}I14g~$ZtLg60&NNPfdQ5uNtKz zi$OH=Q7vwIS`MlYazr1e?D1=aoJWet9t3$K+6eESCJaVb{)U|aZ zrV#$M2xZHn=}L<$;`%z$)>ezfWPfd7uz}0Sd4fo?Sss251_bHa{`Jx2NAnOn_s?l%(R$!9=r|v|d=T8^;^D_2R@td_hX`JYnMYwl6?hGyz*(Aiu{){7J~Zb3oZddh(PH|j30IUMMOPF?x9K}o1`Uix z$~^rvzGe`W^j$+sbsh7T%8^yT71zW>2%!{ztYAF6zPh+mb$#X2z*R8l=-iGIs)>8C zrR#aJ({hWPFwT#zZ|(WKt?MV-%i@cy=9`WBYxopGZfweOtz~@Et{bouu(o+@3d0Hf zIqEs`5Ziw8fjWy=xMJyh>KXuy6xI_q5W~2OwSOt412kpF)K>MA)Ij*PC`k>B5@5`K zBj)d9srDp^f<;Qz{9Qo3)KZUn_#c={=RL4Zjw$-gr8;~;1B}2})S=JR8AP+04Q_Q# zG+)7LNa=e^fxj`a_)QT9Yah=H;lCKEn_)aKw*{hk4|fW0q&MnyW= zigs3xW5>NTLkNqTWkFOrAw%7_ZES-4~B8Ts4*Ja^v62LQgO1(@0i{q^QY z9fu)(xEpOQUJ zv*%PLDV@#1PXO}38C~M8LK}+iV>_Huvsx6;XP(8WM$%}0|713+_+xGwy%w4IA^fVg z$aCu4v+z!S;HN9IiX~F~)f$8c@f|z3pPMysccylFAJ|EHADZo^D45fK&<@g-Sp$ds zgoX~yi>kzn>7NT4IxDHnjaT<9|4ZQ0m>0nIm^Gu*!Y5|@>E1Ku&8%Y~Z$XUUs77Y? z^L>h|4NuM@NuwET*?a}jW#=(XX?zjFbq%|akx`@A*bkJYPCtA9qw+IdDZa;?P-Adn zg%k-#NA}!5dZH^OnzPt{oKcKluvoRm4rJWIJGevOYkmu`JQ<9%NtDNE3Aq_)AhHQg zD5UvTfU_?{0wxAw|VcuWPjj8y?QrsBq{ zqf>;0qtkRXzn`KvU);PuxMtf_DV#ZkZ#LdJG~=teV8up%aYq{@_r z6vH1YfPR(9O0hBA9jpSpmq1K${e(p&!C>9&`)HA3q7d9iLwV~@A^D=k(krs=hyvh{ z7oz1Gb>VcE4}p5-y3$5Tvk+@&D6qb^3t5@*`h%cld|p!uLxLv3(m?u{@O`k|e=d?V zk!Ix61SSmVIGV42a7Jtz>Yn5HEu;G|iy--A0;{sKdND1n9~>D>I1BY=QId*aZIq3| zQ2F^+f2MTK^L#@5w|Oy2h8NUJf99tNFJ?;;B^-FuXi703CT66w6=yM9FlXt~;35wI z7Fdr|nX4SlY&BdSu^;-g#jMl_5`C;TPeuxtijNsaF?|Vt>zD7@R`>Zb1pX5ZR{0`q z<2m)eZU7hSokuo4>le1IB4gy1|0+hop zB+7r?4tW0sR{i&SXGa*|@Vv#olo^$nm)P0rCl8maE|9y>WfdqcPxe zWkORh!&icTXnG{~3oUTCH`J}mLA|t*VlUuHY$NV4B69yT=BuG|nj+DuZr>+ccqgxz zpE4z3cMDB0*Ukv$Ur=Oq$|L%6!I&FGJ1+)NOfeWdv3Is8U<`0c_|HF zZWi!F`8HICH@`FRV9Nv96B2p$ez=WsfzwoMCytVTLNBI zHjVBi_?Eoo|e8ornYFfY?k8h}@HKO;-4M(!;; zo+1yGh;Wa~AMQ_VYT< z6Z|A5e=?$F_%mH4`b%pt&^)pvun0Pjoc(0&6;c<2A23*i!~29kU2R#ZOtmF{!C;Yk z5p5mTq!ev)4bl3r7=$e55-`iQ*f}$~5PbP1fpkXhWx>%EoMo=DfUrqsKF4C2y#j`} zq|P%*kw*raRFSBX)=nj!PS#8#Z$29nKXfsj43R#7Y$>>mLkTV<7#PyTiT!yR#pK2s z-?76~tnH;Z=X}gJIcSijG~H2u34UgpC%v;NtOZE3yqCdUY60539+dE*&N7dFdFq-f-Dc=?9~ zzFS)3J92MT$!&r1?ti#FIXt{)ci6RsBN*S`=_gB;LhD*+C6y&^$!BYSR%p--ra=G9 z)zt9*t>ukp5O6d}W35U)>v=k`w`DVe-?XrttwZ0_*&OkdknR9e1JX~}Y(;gzv}_1JjF_jhCo zi^o!=;);^bT7WGwOQU#SYc3NliDITvYMK*LNoTWi3sIYHzIr4ff~yPTy<}0GzJ%KE7UO=3DZaEOp)d zYHu^Sw^^yT8T2;0vzF3UUg3!lgf!kkqVjJ0pJvWmnVZZ2&gOw!{UJd zULY#viSm1Yd`CQne4nYMp8L~Z{V;7e@unl^{`AM+0m5 zZsi(ZQmNKGEn5d!MT}dGA%cTto@94f5GMltEl=11y+>VtskTJabT8iCqt0Dh>iYT~ z(ICYnIB)~!hnrD@OPLxxJiOKK877<|t*jT>u|FVnVejpL8c zNS#}6Ez8;_cKhqhEMKQ@b$N(nsRM2B8P3Z6`173wgZT`D5!1^wFJEB7tmSSgkWWH$ zQLOrnyy~}qoC@`HuB7aeqZ&I$VV)C_DgKeRWIqi*1=bW)p8xru;SA5B zFM|Xkf}BFm!@18aZycs~dZjYs=h@RUbB7O6FhHSXe?ZHZxaH-TcBYf+57w+wDd8$C z!`b~x5gCb=-b+}D+>=`Xmu)`AvaA{cA|S^6*5&Je#xjLI`iW&(ztWt3lQIE|Y)u~C z+qagJLNjM;OU+?gA_%I(z>k&fvqsd4w)OTqX^iZxgh*m`FbtTneBmta!qK>4!{^~r8p6Y;6WM}#frx_ zr&wuIMSD-`4$o0VIAd=N`b6`r&F32N>;rRbMPVB~KGo!bazuaeed? z14HzB1b=fap*1V$t7%$7f!?8NFh(VRF%|Vt7N*Vg@&#RRNWi*ED(-Ce;zyiVTpBr+ zT>_1IpK+T3u)LLW6Q%`|i=&5CTf{W6W+Z>6F0j_tJuBfh=!0kxrq;4NXFgGh*Gjin z3C|BsUM1xozkMCR+FbtheDyxcM_d|9shXZ$Y3I5De6xll&<)1qYHP8ud#}W%54pBhuv7>`>*4;$>tF`EOF|oSf4aKW5Bg z{#gN+_u4JFmREd9Zu=am-%5#-wbE*-sz3v`Ir)0sfB1{?aCmKk0R`6f!;r?-^=5;Y znklu)3B5|?#b$9Z1WZzA)?pV)*u>m0@^w(xcMm8~Ba z_YI5$E;zhE?j#DyVutV9Za2D^W4`ro!4|qWd}%D-*iwM-xI6)i$C~zkEXzvcz9dvy z%CZ+A%q6Dr&#h%B4vx29%u8l3h<~NrwdwZAthcE=AOZI;baAqjgb)R7IIiAVOP%u` z2t`rI(HOZE1!&QCQhv)wb-4oLj+h|kh4~1A>Rud%qPg;8|IbjnC*?BTyw}NX@g?h2 zet@rTOJmZ_Tb*1Nxi+qUO_{m>E&S@5zO zuPx90*7DrTF!r~vt;do9+5L+^<}#L)v7=k7b6s8?`40ISRFUON5G4b|%WHMJvsM5W zUt6o|t+l$=$$cSSk|a%BCb%?R?H+`*QX1y?s}Lrz$o}~qfHgCJaDiXL4MK z$nbNyJ@{V8f_q8tgbPJ(cbTediK)8PPi}#!x(3Bkfo^UKri!_Dyu1?k23TG=}uqXIGVV4$nDdWd$ih=ld{EN5%~{S(e!2e`rAEZjjW~O-K}~ zL5<)sO43cO-uY;MRPlc`{g~yv+A{deH{=%|u9)|s#l#tZ#MU|^_pw#0R>?E9c6Ckt z@!j@rc|y@-#59S5Pl8Y9DY#I;#?({?>;dGjX+U#UnSh3fcy}z*p;fA6xc(F-pniCJ zcGRk2(DS8&Qa-kYpzszbdpsj^5K4G=nWh7m`V9JT;QyX~K}`-bdZyz#mb{KDmDe$D zFZq#Lzr>ZwFR?6UB2TGxaPUmYb-Jx~j+w+@>=C%dmtm!%p@-Ky--)BtPM);06v`p9 zPey)D!B}^RCcA$RK`{0^{ym7TG$vtet$o*2nF>-R>|f9#LCUQvS%i-$?sOA#d`F@% ze>UHyWImmLqs40Y$y{f|^U;Qfns3vL{6wuVCUFcstwO6xKKmd5$#?z_p?=}}1RyI3 z{0fasPGe~*-{&MxEs`%6nC|&D)CVheKuGs2r!I{X7=b97fL$}&W>)n6!_fMPO7P+x zl+H8AU=A$nk9hxxuA)@2m7;zLVc%oUAL(4z#6XyTm3{G|euw@^n*15VX+Oo>53Kl3 z*E6|ZaT)MtvEcAG0oLG+bz9(*YnVO zZQLV&PK1{xYXN^}x>-!*pi1t=dR&vgYxlZNK36I+y2z2AVo50p`cdYDcCD{}iWx@D zxThtPl1_aKHL^R1=O;>L)+hgDdzBe|hfRF84m^V$Dxks-EbAw_*`NSJK)k=Q%h$oaIyz~Sc)ychnED1AyvMD_Im6h827e@|TNwxAt;iY3OG+@Jg&a-D%ryz==0WtZjsf8@RUcH+p=C;BReRfkb-(iHCg zX4Z^wZl*5oO*hqr>7w0jXg)1$QnsoP@|J{EV0gB9r1J`AMdV!)F8997I&;oHHoB!! zd5MgS_{A@*Ax~rWE_H5foWXA~K4TiNkp8w7e<`YTTiK=1nqM@=ka@i@DNSLzJ7n)6)sanEigz zQKH`?AnX}~?NhtGUDHu+D=(*~aJAcNEg*W3&gOP)-*N34SMAhL0!e9Q;&eZ@hlC$n ze|?S`8gs1Xp7ogQa6guPO+3{Nqz5ANAzPx@RCWKI(^)}rofSSIyXjQ_n@{*%`?NvM zD*RC_U(svCvL>z8wTeP-wLDyoL2bTVgKxy1kbU>ClWyvi!O)w4_=A5`y<)$=HgWJ1 z(hy7i8sprq5&)j8oS4K zks=EqxLZ^cy+`8N>1i?d>_*Hj>yKQTrYVI6wC>`$ZBG12=OZgEl>tLJIDtE0YADNF z>J=CoeQfKd2U6T>#r7cjn0gb4fAojVyjax_U%e@}5J^w4F8Oy6MRmbsRcTCr0u_tmtDOb!n7hzka78pemPDkY8PdYh0edA3p0hfY3$c1!O0&S$hvI(W z=Q)8nNBkE&+iC?JnGeu@RO;@ltzYA-GsV{638rS(6@p9i*mf1)@ee1nXvBTNr0 zMA+YI#jbNAmv#oB9~z&0b!&0t%djp88zjCZ=DHx}s93%HJf|t7b^RANs^g*x>e?zG zcKx6|Elo7pEVDa%EoCFR(|QQ|yCahj0!gPr+6-(U{nRSbnlb%}NBogzUj?htf$k!q zLYmc-d*o^05xAlhf2>Grahwy-wg`QZW__@}fY1=m3P&w$w+Q_a_)&i3z$5jE=lO~+ zh47(GbEZlEWdr`W46zidz%q>jpz5CzDawn2*hf%_H<{lI)uy2-nL2g{Yh60_R4O+Y z5RbOVk~KVaY^^#yEkJ|?V!9G+>CR%k)spGW-CfEdM>Segf5b{48-Bh07q)m+ZWp+y zx$lQ=D9ds>vHBsX?S~QF)%C-u+$sah0XVQ#fB|`$0LX11WPWYcCd6q(uq^!|UX*5F z%qX2=<$L6ZUFGZvSwk|vOSuq!T^cR@?tvy?nVg_v&m-2QkMQR{V_gfUks64D*#7>+ z>8MOIqx)e5e}o`>owErY!KMcAhSL56n~zZ^TC#TgK9kEf9pU2rCGS7U72Th5Dt`hP zUZMP_{CSu^toc)%&SI&8UV(IjW;V;q@`x_hTRgMg>NVE%q+0m!4E#EPfj4fomfncQ zAc1h}Pe(>bI>Ja8P_QAy&dxqz?eA#7ZX$0)Lu_B!e_}XTN48BNu*@)#Sud{gPPE+b zJ^atw?PLu#yUIJFyXjTl5&hj}K}zv1QxF*46sD}9pJ}zc(^GvU3>l*ME-~dXeaBw{ z&=sr=(apkIcJ%|T7TU)w^FCwjN8b0~b7*)}y++GmT&p_crErakG4ZWU?VCi)j!8P3 zq?+z-f8Lv`wjgC`#q1^?N$rA=O^wrc(mW2p7KbY zA^UyF2NCxjV^+KU7*VT1u$jd{gLOqOmRpzUrk_Zk$4?w9Kfy*Pc5Dgp-vCu38dzH` z4>TV1M6Od{e=#~TpbeyBy=N*vM5(3L%;l ze_{z}vrPsg9P_5QUlJqR-Ceok27;Z1yDXz2qiD5o`hHBk7+z^tDhj~|f}0gD8)VO>Cy*KD?5rNl(7%+`WP0a&=^ zO^qA+DPmE|2Dw;E!!^@O0xcre{J;O#fB$OIan2Ov^aRLJ9@6A7TJmKFSX7L+4!nHZ z2OvbjXbc&GOiv&z0}z*}aGZ;)Cwmp4yvOT*?KJ7liZ#VP_=JCbK#dHj6u_@YzeKDV z1j`#RZe&f$7pw`G&IG}}KX7q`+*lUW?V$u0>c{;W<7$0R8WelEi~Z0S@bk-@e_>pr z|2;*n694q|&E|9QQ=YeBqP7{%8k`b_x8J}1qa&T{A?eped>rb6 z{Nh7V(5b#!`$e=$nUn`pgA%~f@%E-NUaysQq@Wf(Vjv7rw>EM0ubUs_G<>uglfc@aOmb= z_xqoC2X9pPL_e&!*oLg4O}+ z>RJ#B{<9N4ZaNgPU&9V^xi26F%ODTl@{S78@R}I^aMEl~+D#x|e*s2}92XWVxIyp5 zVsF@qmN%jyhN3|S=!xhl88u~S6JOqRf_3s3JphCiYN}8415|HCYxaXqAh8c#h{2S; zZD2CygYs<$Dmcgefa3oZaJ{Kg;Q)FJTzHlx2-<~aoQ|<^5)PlTyaCQ?E`5BR z$)E-7^A*n}Y36P-+4ukB zzX$Q+>xU$HVBbpz&Bm}1Zn$zVe9Sv`JWZBVA+i(H_=Px^5K9XIFN%FSvA8+$G)wVE5r@Ru^wCAK!b~j2o(Pfb1ll=@@RbrH}IvH<>ul- z_SLuEMS7+mf5m_sFR-pl3*5d-{(^=lokj}J_n$w-J=v09-BRXYMSrrSe zqu-&1Zw|X4wA+Sw9U^3>dJizrfFwu40~y3Oy<1glg4nJJ2V!y0Q&{UdVU%Xzyz(VT zzi*ltwKuJ{EBzJb7s^-+uq90*-HKw_@hqjf#&H2*f3m4~Qj5DUeCvmmF+Z`MxlfWW z$OaCvRAH2IL^G&IfUU-99{e^|^2;8L#KaXuN%Y^~(RZuGCgTd<)hmU1K6`s}W)?ol zpxSN{+MFwXqL^i#C^J6hAQoK{g9D($|GrtQG6>C>b?0*jFE18Cu{-0Lvxv$|kmqE` zx79eVe*w1000hPZESj)W;g3As)(+L~IU7LC)a4?-w>=F9VidzE>-MJS8ZOXoPYHga zx?LAAo6g0?k*HuzIba*zgAU&$oh(T$=LMqt6oG`UiZohHOD7Iw)}1RDwpys;#n&O< zHW(`+WfUm%b1FMRtYH_`?ilI57`aZlsTHO(e}w?|L!)T5K#Ou$C}7t5`-Z1mqZ%PO zv>1$02>Kjt0s~x)Mnsz@g{1ym^DxR!AQtIlandC1rs~8KZ0i#<0OQ&L7`UTn`StDg z<07_*QzlMx-{g1*<^#?AQI0nn!u@cOb2bDxa{~X*=5)w{*>p}b&Y*1+1e){xen{bO ze`DBx$Whj|TEh-9%9;J?Sl409uDmgw5_;>6Nvkz3!QdipIps68hWa%H;bl=4_;M3I zKhT**cFYMKu^rTZ$95J9V`NLrkHJk+2vS1bX+-gB8%?=E{Z)>ai4-`*tad&%sMe@T+4IlwQ znsIg{NVU{f@@}tOO1yZOB<~@QFe?3e~ zw3IKm^2*rT^REyOVI(-^+*T1dZ#WcTT!Tz&vF`XPon1=M12@|;ikfrMZUWKj8p8lA zeGIulH>*|C+UzFWlXzxZaho1w9qH79PjiJO?z?_Q&WR72w|%Dt7lO(soE*FfxHAZ? zcUnwr-Y!Tde*}(E9H^c6;;h0ze@5uUoT<|o7Iau`@AsO>aH z$OOYV!JS#c!Pv#7@VB%hGv+RP?XdeSbsR)l3M`h(qq`L9sv%NSr4tt5mXD5Dpw?9w z1J*_h0u^`cE#QSgDW{dahqNr!01V{jBi@K?0Al|!K-`_*TtEtLUwDjg-2_XZu&?n0wpm`i|VX!okrXMYvMyDa27$X5;SLlVv23ONrs(*yJREBVx z&0~pR#B2U70_s{(hm&Q(g-itc+C{X6c%T!Rmk}I9dRO_a3YMGVe_@mVA2jG5fFi~W zuln)#|I$Gx96{9u-myY_v!Sf`|GI`MWJ8IU$q9-?VwJBbtR}B-lfrEinS`vOKLb{m zRX|pM>Z@$3%$l!?ov#v%3{)f|xXf!l&2q{e`Ly^&pLVlZefe8SuYW0;!@puG3m#$ZRm)31uOP@ zZ=&MPfzl8$6yTDUx4pQ>cX|gLl;z6+G%yq}9F;LCV!%Yfd8;*X9T^q5DZkSrJ_yFl z*x@2+m;)D>$Q&RdE7vz@4uG2lGv}n$Qf|8jGn8P@zgGRqf27D3?Ehc1w;uC4U}BR+ z%q8u)jOLsgoFIO2t2O8gDD&GK)gxca)}U^B>?%w0L+CMq!XWSoKX}earXNYG4X`|^_U1f|oL$KH~Ie$hdC+%|ECRe*;W72Vg74bjkQ-N zWjBC^gm74Q1uIbD-abs?v1B*v?y#Ii$QF5i z&y{69Mt&XB^+8d*$-PJLj3I!-nO%lv(QCdFf&XG_ zVypJBUpBoB4kQsbd&|KGy5;l{vU>_FXC{mu@N*0y?BXYQUk_pqCY(*pr^&AQ)-;=1 z(@fMafAgPI_hBP{df>O2KdEB)hobE==ns3^K}Xrs3UPQAyhWQfBIu-z+eu^6i}?9Q z_fe$#DCN+h8$OHpoNhTA%|0R!$-C!d-rI7`oSJerp55~~l`PESU<{qgQVw(wwNpB_ zr?l!2dsb(>JgpOcWL}Stt7-3_+bUS`vM-*9f5G#+hM;cO5Y{NxJB8KUIlVbzJ6me* z*v{$Pi6b+6FKJuR0&d$*%1v7wY1y4@Rm*1etINZ-MhKKel?;>*=;O0pbQPi@&aE(x z>FF8iot{p6g!9=HGK`y$)TyBxn(gKtZHux4Ptb>bFlN+U)zbClV8LUaKM ze{Yc~Z)DM{pViPJX`1zsMY|RluJG8j&IrDrPHm&^&SH*{Dok&{2K)UoRf6P6e^!c2 zHn92dr);|44|1xn`L--bc*lB3dpn038(X>o#n%u)Cr~;Yc=8t=f+fSU@;&n!Rw}gY z-OY?YECbW^@L*n8dQ^JykR0V;cQK3Me^0^>V9#WAHdP}DIiC&Z!b_fULPj1q_h!R6 zc+$j=v$;$(i;}T{nNAlE&2Ok?3V47ykxztWS*l)R25-roKz~cZRtEaU9_X+pMjS}B zgkZJCdBo?vvD>)i?*>%Qio2*0dm3!Gl7j~uNB)D$EA|nyhybDFlNOaAp{T{_fBLKQ zNW%+dPeatfVs)3KQGOr8L6SoFLEH~JkZBO}iusLDq$u6~fX`T{#Ze-PRmCyDILqG- zv+9qDyVr0_?74?5r~yVl>&9;Z?o*4^}8>2as zeP_XJ=fDz+Pn00D2`7Z!c_TW6LPxXV9C)B&ReM6T5L6H`Ak3J-XLka@a1Vqg1)*;v z4?7-+3aK?8>5f{x2^cKwe`68VOy(Ms<~)xbV2S{amgh<_UHi7uwLJH%nTUx;%%;T) zyb-}u2fwf~RJlsgNScGS%M8=gmgAduHGZNIrlOA48*mBt;#R8>L*!o7YQ19vH_~$? zhQol+9RqGS_Zkibb&c%z6K^CMgdQ!O(48m5FpkKSdA}`eSix6*f4e!VlFpj{^M5|1 z$wQnhn{$EHS}d2E{!~*V!&#kCLh1Tm0$E7V#S(%V0g*<9L^Rp7g9R&7@_SOb56OPt zCGDbg^G$RI1op&=Arcu)ypXgDX|H!LKq-e>ueDlSbDqWqkp%p^__-;MlSf|q;()0O z;-dro(c)_>nuxaRe>Q?nw8os(8JcrtYxpxOTfs?CX^<1$Ag4j;#^)O2G&s^2r@`}$ z0fdg};-0UUrbz;mUZVh8TC7rkU#2{az7nRmjcbTZK?5r4l0aZS3Yb;dZ&40WCjC;e zv$J_Slv#iPN1)P?Ko}?eZDH*Me#0Py%BWRznK|>u+zt+af9=#!4_PO;&A*zBw4iV; zgx_9m$V4pV+6t_cH$vJV_g33`V1YH=3gO=F$V7DN6U+L<->Ofn>l4@1N9`qPbCukF z!Ngsq8ZDN~H%b^kTd~*(ZP22uL!<_AOO?w9Umj0()_$QTeO zpd0HPe`(~!7$_b)tyXEb3PSFf6Rh$gB?hsdz;SHe0|M$-Act&pwOpJ?fSp$O9HlK^ z085b46r~0iXbtWtVa3k?T8*Q%x1)i;%@Ju18!>Ile@`QuCsRG!ZbN|UffjUfG~i)m z8A{4c2Xfm1s2TYYs`ONyriBcn0Yfo3(HWIa&+Lu5&>A)HzCZG&vjQM9uy;H?BmQ(2 z%q8;F3@ND^LzWx}-?NJxxJBON9=rs2sv(kSLdais2N%*N@qSgX&{Ql_0|#mDG)En_ z@E;mW2biVG`K(q;myjm`BY$0Ytd71lJS|898%PHPo%9E2gr5+9AXTxQWpH=LsHMOO zLkZA<6wYTI9kV-R4=Z<14UF}OTnW#SU?7%{v{^`h1wPLHj|qsB3i)@1#LiBP0-6An z2LI3cr>BKKv8x<(7RyIKB6m;V!I-oXOQ`yZrk5=_dlp8AWW#uxzfkoB} zL1kG$i%DOIQD53m#7)V1i76VRWGx&$5^Gt#;5nt%w$qYlx5p}HA)_mL&X8|krtYf! zKWmsS!`rx5Op9sRdzi9DtX1bXp5lS+Ynv8xJ9q$&^_TjO&`uhaz{l7Cg7SZ2U~q-!>6 zMr8lMlt8y_jeqz~v;4U^rk5A-{6Da^JXd;Yd8wVU%P7mE^=)2{Zs5-+g9 z-++-+H2_y_kT#}z6E-WEys+qTWKtR@E}XcPB<*{*TwJ3U+KTC2f>|40(Ix&RSnV!=-^#jET~#}u zdoDMO16Ek#fDYNfMt(@aJ7vR8vdIx|1wP{;B(+Nyw=l#YMB=+{mkwEsftJYwIggAI=6MpnAU_&YFK|&iDK8x?O*0*|;+KB^tQ$Xh2MW zUU_sdP=KFcDL;KQsEZ3gU0iaOd0TAM^Ggl3xX@sWE6y(Y>;heYC}XyKcE#sJWT77e zWJjSljF^gr6~-gBtgny>;80LsbI(2XDHq&m-+u`Z0H{6kqsj6*C_{|`foA~8j)AL> z&@^r6WQjeBVt_+H7SM`dytKyT6-QE2{5#-m3W;;jK68zyKEq)B%+D`)uigGED;(v} zMxXhde#n{R9W1yP<%CM8ke~9|BX57^^FwbZ2**F<^g}KU_8)SUi^)^xWEj)6MEB_D ziGM%c?{5G~hvjpwpTL~x-M+VzkLdGs+G-6doIa^To{EBNb{=pHj z38Gcy0(!-(=9}EK{44M%p6qgyH}aY*Hdmct{Z$TJuDy(V`sH*7mY!pao!33zsXO=k zt($#VVIPjy-{7MLTz$um(_0XWX|(XS+J_BbD&$_Av9s4eC|TFMB7Gu$t<=N{u?kCltdROxPYZdkTdiFbdfoGJ6&xGpo_|RG zcp^kMaHu6~WL=BFDK6o$O**C#r&bnf{hbQ_RoW!E(66ex0j!ExaaADT9stl_0Ri^= zxFeIsd@m!H%ImcSQ>f^^_F^_+vE}$QcRpvRRo?X)K;X!V^sq{ME{es8_m~{Rqzy`- zGZ&Gj5x%<;bZuAa(sxK302l^|41a{!c@d?WHK~n49FSwg1_LBEprccZ@#J(Tx?4nP z$#AT%3+#S7dmBWw;#mK>s5kG#9g~yFBDI^8awrN)wGgusbp10cA4aLT^YF-xI}M%J zx&0i@t}I7RuFwvz4v8N^8XnjP$TW()@QVFjI}Zsa?Xl7wrzQJ`TVB;7e}D2fjywP= z0^Vkdy(qi)DLV8Ktbn9g+NdrdFJ%%Biy1)*l!zNAdh>J7jjGo1}j&*p8gslWiQ*uR)Fh{SL>NbFtL2vEmYY7aar z39#hmCqi3|<-@<{jWRq`=G_CBGvJ%iDB^87ozk%ysg15y=^tepyMGJL#-Py;du`Bx z@r9RG^{+VHaXR4in$ypmMwS}(!LofD9ZN6pgzj>7dPZO6tlN8)pMS`E?e;6_gSkuD zj?Z4@xC6dR8;o7!p_)>kL2ZK*bRf>3cl%dddi1?6eU+28xY8cz)35UO8S%}d*GC_{ zu79+9&UU$T^DZat9eoqVB?DF$&A1dRQD}G&D@q>4daRLL_tI;X`-GR@qcWLsgJ!*P{ zF}vXHGvCu^_WSPn4t7njZEm&T!lmJJ+klqVP+VA+-oy>KNq?$wK1d5Bk+T+n`?O3L zEErzDlMJt+lrOPG^IBNcu7RCMn485!lyX)oB@~;}EN4gPrK`{nrR zf9sd48dTqHkYTm9-U`FBuEfns>f>_afdGoW zFl!<4FQU@t+1jdM+=!}D^%!@k40X^1K`b?$GhOo|Na`fBx*rT#|5I-09jGfYuKpu8+#hp z*Vh=|<9`Pf__5Hgi)RF{%ao|YuGCjbPP;!h^}x+S<5_I-UVzpfJgWvns!BGpwk(}s zk=^n&@D%uACjf7-IHKmOWH!{Uf#Docm?co##A+d}T(5gxcvBh!+cBoyY_}&Eohel( zy$czD9-?uZ>JbS;+1chF3!;tpNbm_HP;t9=oqyy&y#ZyTcp&7%^$!m>hRcFtw?kdip5&Qk|1e#k(>%jH$0V{ z%*1XVweR@}vJ;))fp}czI>(z%^83YF{C{lHYJi&}5op9<-E?|TsK%T#$Z6Vf`Vy68 zVf&dl<{A+=t(t2no8ysV9yu8A;C{R3mC-AoD|^bemV@m4l3SbWE4YYmGQcc)_y&3< zn!SryML2hc=el!3p71~0c~@4A!~<;L;eUz`klszo7heu)Cw2iFwo49uEoR0eCx3Ly zMN()nc_BRL+4(wVU2xiQ-me=}-E$Z=G~u%`1}R!5Cg(0zx(!@_(<3F0%&pyLN-W(* zQ?QYMm-f%CWC^sUpj@3L+?6(sDQzH4uW<-GyTU;Qhiq#Z&pc793Gv|wscw2O zdL3ENw#xIYN7f~$ICOAfOTp}*)_=4yV@y|G&DuxqU2tnzYeeAZW>=iFT8$0*ya8e< zc7<0~?}$~ee4(o@JUX$PKz}jmZ>_z(zTK~$J+s!!d;YvK_or33lI*K6xW%^RwqLLE zm6wJR1%@tj7Sp(0L_k;UYw=k60FRtN=*CHrq+cZGigY6eLj3 zDv1tyvLD2plsOs*ZG^2>qaZTBkWRNP^9;FJ4X~W1?*uZJax4Tgd(Y2NmPPBkH7>nHeCJN0Vqm zG?FyK66(pJq8f*V`+vR4wSaEa{k|X_D{qUWK7}A{BzvX0r|8oXtdP+v%C|xa-rlE4 zY^O1KuWtYdMY0zDJX<_X$kS?#;BgpK$g3Ah!GvCtWWBga()`#n;`>2kQ%}oQdUJy= zBo&pWZv`bU?j&okiRCtl;k(1P$(G~J3(1rFc%VQkgf^w%x_<;Y-i$DKD~dXEALJ`nb+hQ?lIhYxoJrxq#Amq;pQu;XdY}!2(;gl^(4RQk4 z;1SjH&Lh3&C&jS{uX2TOy71f^SwXl1z|+w`@$jzvFsU2iZig^7!K|kCIb2%=63H1! zLUvn4S${$W5Lf011go|8H@6W*5WF{o|Io+Vm>d00K_~_q8(piTAfTSf~C{! zNyr!gLz`?gkfpJj6$Vb$kf2nPq=hd@l2$M~kOv?2Aj(!KrGns?+3U9WT);(R63GG@ z!`d!_8ij}-|@L4Ofz?gtd;CjALjDNy_o&C**RGQTu- z{bsA_H=CsG1^wn<1}7HmFn3E zD*>i0bq$VtI;)T7%Rs=E{Ea}6Sls>=g;m69qmgpr+qcEy1>%k|Z8SC<=pJt5F@HX@ zkq;iM9`hLBLo>#a*2s{4`s(z#(#w3M$yECI3WC!X8T@N5K&dk+j20j?e|Y&8v2mN z%YYLBto+=Y(a?>@jUj6&J z)%txso34%9roZFBQ|i6X;eRw0SBdYOC4_$Fdb|E1MzG5Tln!I+&EtN`EowQQD#u+j3$> zVS#6;w-`H#vk&=Pcc^y($Uxjui7SL)`Rqdu(RZc|KXY*` znm&TJ4mjxzxIn7O7k`Upeqj^;xfYh5yK2+Q%A=St^>xC*w1d^VYFA?;yZi7pE>Ab=N5rzC2f8*O>lw5HUE~DtSt^ni9 zK64oq^BI5Rq6@_js>ALYB%Ta15z!(20rv;80EqNK%Pf=>cz>UTQR&Jdc22qXB-+ZK z%@3qE?t9)21`R~B-SPa*YUO--P79e=S+Apn*!cE$VFM_xdJYwXPrxkV*N{L(!KQ7`AyVaHF+H@sw;$sxA2 zhLup;WF)oJ+r+N(&ns@%=6fp!TSiRmQm@F<-&*4;JVeP z(T@uQxN%=Q9tN=Dm&QW?(u!Vl;t!lrq0jd|JFS1kiGRDK*yd_8ROl|JuX1{qf)epr z51aRS-+P~Tf*^>Ntlj>7LEq<{B@ml6&OkuFHzPq9h~DQ2%mATc51(kO~_{nG*0Z-$Dpox2*%Z%MAhu?6;u0wr&tk=K%m5VgjuLO+&+7#OP zecpjCEq^R8YI5?jPY5X0O4Dsa)+m`nG z+S-Lqr4LefF>*BizE*k3Q6rI`h5868bh^{KHRT^~ts8zHt(PaD2%YeCo^G3}g7_uS zHoqmTcFta`DX3K!YiL_gkK9rHLqv4r{}Ac!)_;$KCioAHfxDDm;Nu5lN9T=tMf_sCC{zFP@>?Jmlh`A7F#vizvzD0q&bD9O2kgN9!J4~ic0bp&(L>`_W*S9GLlnIzRAaycWW^l91L`9_e zgPL_&)=-!st=0!RIK=?rkEOXmTU{frZGW_5>b%)cxn{2JQo|bIe_42AP{pMc^m^fC zvU5d~xgUCOGuq2Ba9?eY3qufbAwTUAe@e#eOFTf^@dd~VSl=&9%U!B=fk=1tZMTR* zQlEFBKj)ECQ;>hjOCNW^4VCnU&(?Tm&Mx_Z&{P1V}F{* zddni78k5o;gr=xA-7m7gp%UtM&j%XE(G-d)=mYMUgGfPuTdAPE_2MzQTjWW4-rO`> zt=FFx;UbM*cXFQPp1#=GWITOy2N1_rtMS@xs^!bS66 znszFd|AOj42qP0*<5tb)2m14mAA{k=wU6-PpWpnB2XDd4p)8tH%iZy&TYshwFFKZq zx93JO@a6lr^htawxFtSzy_imD06+zCw&IC!{SZ5eyP=U6c~1vG67IYa_{o4^*;H!^ z?D>I0AqOH&G}IM9#mnW27Yhh;FDOFpA2M)MiyVu1GqTu(s*@|-IZ?j&iOf3fums4b z+`tGsZ4^Ai7#kfR>}(?58h_dzuMwf)Y&5r`%0uBV13H7*Xf7TRPvVxvgnBQm;J9Zh z9?T~5UR=FFyYEd)dKe_GtN&ge@AnW({D2wCS}k?g!Q0UZ>3NO8e&5865pEB_^)E?U z6csq8LWTTl7Vg4$I`{kCiTnBHuKqiC$+Llnq8TUTKr{QMDS7%LP=AUODp_Zy6i_8H z@UR?+=!7YDF&Brwn@#32Mq0QqCv+sXjaHTpnH!^~feV9V4S>C+AjK@G?P*E00GI8u zzieDagHCW8-gRK3871q}*g3S4H!P~j8-nGHNN3S2D#}JjCe+Qm=Bt?J3t)U{Ogi`* z2QB_$Usvin4Hm#Z0Dm^!*&JiR4QfH`;#EP4Vi;umOcE^4-3Nf~6pcs;8be zWg}wYGYmbw`r5u+D)b37r8Q*-HOqS>YoaB*-{)Y-umCAA*?*2dk-xue5=vk`h<&6-hy;bTBAx;}=hQA-h@6uO1a%H|8FQjD?td)!9vRaR8PM*H3 zz$18#UWjXOL4W8Bl;TM0l%Nb;aG5l(z9e*AbW$z`rwf(%_2DjEEVw@( z8PK4YYpVx+bH!)Zd=8N&*Br=aen`9_oek?3ojIWwi|4M>46P?uST8X=*%g=WRWQB% z4Xt*H?}oQpKSdteK&R0Bt5s5z$Ne?$2~)2(hIJI4@vb?GQVYf;V6HnZX8tvdS|T=; znwv{rbbn|CEx7@f;|qS2^_t_-Mk`a(kPA}bbg4{EBt3u++;Zwd7%nj8zp{y~cGHy+ zE04&Or??kH31V1{teJoYhJUsvhtA?+2@#stCt$3o8|d z9k@Cml}vS1CSr*wAVzL7OSoAc#F4c;D3t&e+uDh4!Jj%0$-`y^7=Hv%kBD6v0p6vs z>VF&6?I9Bi->->h>1;Fx0Js@K|JPZ!0dYz33x*=x1rR%2uj)n>EHKt+wcqpltb2M! ztge=*K~y3_sYJvCR&Aubvg&IkX6(`lEYpR32a)R(BxuyR8WpMLh}l~dZrQeM|bkHSY5u+iiRr#onm07uyrMCo^dd%EN4&DSaVH$d+k$*+` zr~%{nS=L^Ays+|j(11NH`3hhri{%pjiDm=GCJulCc0D19XT^NK4{+s1v04ywIo6w$ zSuv+01~+5IRFyMK7}yHB-JWwl; zV|n5dgEZMyrw;4nAH0JgqU0Ct^^EQf$v zSK4hebL4f;zHc_26KVHVO_p$|Vz zOhi6KAkZ*ztEp;svvL-7~;|33iD2zC4UE)HPgMA#hpBjAos3HE$F@WX05->_kXniXp`4>KuAOC zF_VoEkba=5`JaExyv|?9AM;nQp^ECA?I!U9?HpJ^PKNy+H@mO@Xvtz?LSO&!7yF?7 zN823Ug5ba$Ns0$K&X^Z1oXA69!NxJ40o`>^=1XcJg`7V zB7;_6^+D2OMw0waIJPw)Z34g#5h>(?g8q$-q++L9sq(bhkqcr1I7 ze&I{NH#?&9#A~;SfHyacysa%67Z3|Jy*Pnscd}S7txs62Pk8*0Z%-5xCuE?W6)4yW z^zT4+59;&{ZhuEDFj(QlGA(a=;fDgF)r3$Hj%-;3@T`cGboxc^E1t<_QZeMQPk*U*>>6gh56zn_6nwOB;D`6B%oY}9fDK>8HXqu$f< z2F7&eH{oCN0I_J&W-qVU0W*KMypoqr|08(tIAzGHN@5PomhCF!ODo>hzD#9mtoIh#uv+$}S676&njXgTkF|7q!; zQFxyp(me6c&MSAset7VomOePiOb`FGaP`KU9M=|xWejJ9(lj6F^~sDlC0rVX_=qFx zpFzeu`7wDS=&Z^b=4C?!Y(4f^Sk{h=IQ#%vqWN(2Y}&nt0XTS)Yn)I zu`=PiILJlF+7rYSt^M$zqsjQ!+^=;XvuxPssE+j=BsPn!onrfVN>p_zYnFp)Cixg&SstIpJ`zN3u*v#y92h{l!;D}U`` zac51TYk*cxJpG+`dffm@K()U~VLzR*@2UHylxBYS&{O!t13vV<4=U>BK{oM|D6)Gv zSOfgi%8K`=0rNZ$8Td@r?YNS4JMJtofBLEC;i9v8Hbu9*nNHZ*dC+gBO+Pq4J?l4D z5YXF$=<=9NfX1RJ&ZSLSFhsC~F#bc>rAdED5Qq_n^E`lI2^Ymat`NWCIrYL>JZA;z zEZL_G&Ao*zkh>1tV(&#;ggsL(2;yLngD)oDP3x zqd9Br_Iu^!YG`xhe*XzESzrh@NvTPH=Lcv>S3bCdhqZ9YwYOObL$SayApY6OEFdu0 zf$>;mv9n^{YF%uwe6+~GV*S;7Yb#@(_bO$enfNv8)aOJpvMKFov0MV~ujbaD7V_5e z&6)pv&fu`NmrSWJ9}2_Ua=*yRH4uMgUaSN)m=JEE6ldg(jZ!CL7&a3TjQ?-85r9WYM8EnUF*Orn@48waM2kJFnB z?r(TlBw)q5#F`~HaxfSXfU_0sG)8gy#B)Ipx>&7<>&KAeMMQyklLaX|FoJ&#j^|)5 za*7ipP;njzd$=%c(*b+^pV9K>5C4zXf4s(xv;6xSv$TQbX3%PRBQ^l)s{tF#zME4h zelcrC%NzgoYi;o#(dKJo>>m&?n?O}$Ey$s$bvP(wzpHx&8)L3D3-Sz%^IcenMH>Kg zKH`}dBEUQ1nE-l|-spU%T4a9&E*)cI+QB@ZGE64H6<`JOz`+Oh5`I2Z4<TPQiMsMzlC zhs@w7MV8Mfd&{Hs9Yn(O<$fxP0l1VrqFFBqEbJuFcQ#uj3GExk%KkNP_gbmp`fz*fgMBjpM+@FKVFC!X?<8xm!V&n86yA6AzkNsgUEjG(hb-miXUdt)^f%H`tA)<^1kh)& zwN3f!Gi1>n!>WIxx~ncigghl8(M(SS!BCNVL<4K8FT3LG1BB965Hw;xYBHrjSd=7q zxOBSLgOns!+#xhU+FXb_Xc}%bAyCXnAomFB*+Tp~1;*Bph)D_^XJ7y_WT%89#P5PO zcZ^LRyn9~fCj!$}%IOSl*TyZdSZ$G`S1y%N=vCkZ5JP{6k=5gchcAZTA&~%t9Y&s7 z*aG7CG7X^5)$Xh~o&k5B7;xA5c>u6hV^j|{5Dt3r=J!z;`tmo*B>pw$>m^?z6>749 z6prAk6JEc2ezM-gH#|LgSY#Pr0$jmhFFevY5oH(AYIU+$F15c9h?g?7%0}AAyIueb z)rovq6`X%3NX5E92`o%{O<)+%Gk5~SNIuj)1Gx;^l{WzHFNC`6G+XlFYa}sISOW6B zWQG#B93mBW9&dk$Qn7}F$XMt+pqL8>T#z0BcOg`&QjmBSuer}adRQ0-64bv#-eD21 zW8x1)xQ=BQq<>EY5AY7~>0hG#{x0(n)+TUH0&Rb=Gq|BaP6(B0*$sfliO7(<@Jyv8 zl^aDF`ypvMe_elBC&gM)ZQ{I+JD0zleg8lHJMf)5olLvqK#*)3%4s@K7y=}u?$1X4hp)&wC?eXz+h1+ z`RIR;ndAcSG`Nyx0j)#@{k6Gqz>CCh)cC<*Nzh(9(2t+>+hIF+d>?IM@{ODIKrt5G zer@<+W@qCa8?B+X;tK%Y`N$gQ*n|4^OBBI>YJs`ThffrjtB!+U(Pu47bm>c zGOCo4sLHsE`8k~001+&I{-g#L&N0my$T=*svNM%>v%NhXS5L_z$Xt3ZkdKnifE;d)@aI2WGDO_{sYm=J zUuDhu8XO3XBJayuu0I(kU}wlUz)!%~WbHVH==#xzv6Jl;4JsocKMOVy_VCG(%&@R+ zK+LFR60z^F_VHqSJ@Tdy%s6oEH3NUNiT9`eRQs={HTZEcU|`Sh-+EKhSK^$YI7UV!>jD6Q1!VYC9$3~Z?9Pq{& zZ=cBxmYYb`%~vAQLO|tDAN5a>Wg%ueFR)^nH8=?nn|6rvL2Yb6~=~p%T1w%!a}iB#PCHh8^7-ec6BKLqeaNRMO*) z(}OY#G{|pNq#%hIv$1LhDpEgYn&|DU|D8IL=^#I@Abws@F=iI`Vn6JtK;W<=j!;Z7^e{ zCN&>{;>3E=+$=xD8NNVec5s+4>Z6kHZ5oBc0px1sRv89)?+~|b#7xCBOc0D?HN@dS zMeUEjQ$qETYy{O_3!G78yi4C#Fs~k@{-m&F(>5d4&f32HnN@vSdc|~uqtJrmoineFCx}D^Tj{#EeijT z09tm;N-7k;(J{?tkR)HSs!H?Szd@z3257G;fy)%O8eo5{!lV(6p^EL?C&?FtV_jc+``&k_k4VG>rh?EEpiBs9pp0VorpiLdT)#R0>Yecx4(Sf(W3| zl_^eu)wWbJ`hWh_?~x+`UN!hP6)p&2|21zgyA@9c%T*S(C+uiY3&miDcpFD638Wq} zK2W4u^do;>*><`_P%i0T!abp+sK%o>iBeA0n-9;dZ{72I-mz#>`^G$icB*^cQLL$$ zm@8Gr-1CU{pERNBY))N<-zr_Mk8Nb><3fE(`7E`Pr#b2oY!|tpiGYEQGiD-{v@`x6 zqHhnn!Nqj*Cf362s*WO<(ab)Kh8MEt4Zx{~!q_nRDT22;q|n|_iaeHhGl;hpM}$yOPrL(usO#_>&8n;rJmA#x#%xHO zTJ_5&ui2b{5-W*k)es%aR>6u!oZc>O@44TMVrW;0tU1&wN}7nk#ly=CbUOPS6~W&3 zn7)4ym=vF+!$|=Cv-F#AqDdxqsc zE@h~EGy(yf1e${s>0%!pSpw||JmXpvfN!DO+wpTf<9qG)PL7fRXWk{>?^8}@J26DI z9DX5}FDl5_+DqALncv<8-X@4t)i9Db$Rp34mO!5XkMwo>?)cmXZ-~UhX}Na6HBo=` zC3i&J6?bGDKUA%l?jF0+T^BY1p7}8ITX6o;l0<%#60xeBzS}6H5tvFrl3{vN7%)m` zhowx9=v_4?rWu-)0H4yEoI6jpS|j~1nwt=(gfjzBc(akn@z`p4(hgPh3+^cm4i!xM zfVSHsE3!~KTh%N2fsg}ELJK^Ds^5Q)Rn<@Ec&E?UtzMwn)#z{)cJ4r+#DfmXN22GC zlBR*eR=?YLfk;RY2+7(B@DVB>h0%DsO4i(DdzsiSoC$0=O{t}6IkGZN*`+hF(8xDS zoul$b=$sMpM+eADb*SW82<*b;A_Y`^%>b1#YKAxD3iN?wL8 z^;WCV2qhE=aTVe|{`FxM-A0&FQk;Q|e+qJemB1x5%AgOSijBCcoNJM7nOJ4Lq#_+k z9U67Brmy!Xq6M*3Nk)vf-SNEXt6$JV@(9fy^0zS-sPYma$p6+|d1cYP!C$76=(y+VAh2oe@a=$2D z+0JDIP`y?3$j8vICm87RHSe5^5!aQTWEdOd8<5IRTw=abVr6KxAsV8Fhcc(16%;Mz za{vITi84O#WCXh8Z5*l?R-WOW%j_<3*QLgb*mXI9V6E1$LIyH6zO#Rh;;#hruG9t$ z#nnznjsa!1wogxLc8g=n{Sk3o-8FF#D|=*O#~BkiN1su(fpves_rwo5P{c+W`8@;( zg()TQ5q{cv3io@@Jpr&#fCWSxILts7fM3%41IoZySV^zduX0X~{%kCC`ME!?sChGN zy9_Q#BY<4N{%f~Cd$WJ?c63ZCz2G#_0Z3`Yvvb&=?11DOz=g>64M@;y{kc+_so{1D z+&@UAE&;>Hz6}&ZS9T=<`(T!?f#zs=w*h4fNW4)x?)R6Kf0~! z;tWKWyvHXRLfd$sD@A>mBf0qRYwN`CYr(1miCB2w8*mwLW&?dzH^;yL@il2d8=w z*Su}|yFcXErd#d`x&TbrC1D6)d*t31EMPiWVsuC!A;pU3&TXVB%Jt48-IpG!> zN;}ukcItl&_0}Hz{-hQyfeGJh8&qo~ATlw@*ga>HjrSAagj=nT8%#e&`jLXSv!{c< z;Uf8(HdWL^7ys0fIzadAY=BFu_@S3nSUb03QM?ep)S`$!so8k$Nh(GGcHbzldlZ84 zhB2vwg5$|?0JmC=NM+9GUjSD&q?FT(fDj8{$7p|)9z<y>zHztv>rDn0wb$bkk^1~Q!f@_h)Ad*-avm4^MihFg|?*Ovg20n zRvS-5(b6ugtNN#lw*!i=5@%0du#kFQz+xg+9&I??@4di6G{~rGd5ZWVT+N(7vtw!-H2;et{rSk6YlO7hs@i#z>@ubN9;Q?gd&tV+J(1BE^K314raIn zM7(6kwoZVmremt^xm9-cBS;X=WNS+r>71fskw2s<#X7zaSoQq-aST!!n8+J zH-teXc!hcc7>$pd^dHOEe@i{RNtEo(sKpR)OMIwMWdsq5X@G>yy z_+_t0O+}u0+e=>IK+Vw&E8|uwjbnzux2i;X*fP&lY^N)x+f6*|i*Kqn>Pvr`!ub~E z>Xo+ty3awu`pvoyp08{5ozeqr6a}G-f9pM0K(&S)bB!i=4K@K|TF=mlGZywkfR&X{ zcaD4bCijLAXEnq)FO2%V-jr5ae{LqL) zU5dWzT%YnLon@5fTZZu4QD1*5En&*}4$--OX5;XFb}hLeu_f+JW(1Zl0K6d3Y)YVu z3t-^Ys|jO`Mr;nnRlPoltNPGX;+K*+Sx0DFYl7h#(7YpNM|67TW0KgQa?=@V;DKDz z2v!=)o3vUwYN;zYHSI^#wbL`h*;sBGYJEAiK3m6q|M?F&_8q5x{yTq8o$oktzr*QC zYV{GdI>;I(BRlNbi7l4)4y3xgc;xH+LzLxw%~P+5h<*oRfwo*a>bXV847vqyH`wR1 zXeTp%dJ`4`{ppeD_Q{dQZn!rB+9~=y|DLGWoF6yeu+zT-v*Ana^Dni);|Q<`A6)ad zxZJaX9xRXw@<4bak;Z=&+Vowr&eH^9AxI~S?$(PHov@nNb*qU4Z<>r&EtPMKKAfu{ ze-(XRd?WOnj_X41-EDTp32H$G5U#BG5G;Tq8hk%-l;zRw=plQ4Eth#I7#kcA+A-(^ z#-B8W1S_^xM{8Coe9t{Eu@sEU0s1LF*Qs-RDSWOfKxG2;FvWk`Eah{eaQnS2KiA2( zds}#Fn_=eCM-D0(k(LWus#~k&-2ja{hH37;5wq{Cy*TaU`m46I-E&PT4$&!2ll~)3 z+z?G{;RL#%TYjSfFZlVTZi)-L0To_E=N^WZyhQAXX~}DT{#ieM4Ua=Je#W!~K)jeK zOR(FNtri#^K*fK-#ShKGvZ7J;Lz?XP+S{Rt05gN~L>^K6_jjN}k7V_*CuAxAeDC;Z zP8Kgi{0IC1Bf}0h;s^p@mVa1fWraOBn3h#lP?r6ZRRJIFF{`icdQZR*=-{|KpLmTk zfTE+|@plJOdo!XPUV5|4V8ikLu5&-3gNj#B^Qb8~Qb>O;%5UH|gBp+ANKt7k1esv6 zG;p9@)j7><=QP}*1Q7gIto`T>Z@7Y1JK&{(e?%l3`Zr~0LFPJWsqgKWRRkTC+yJAJ z8=&F+*X}tRxtmFZ$AL~#+WuM0C&Ko8Af)dNO5y`|eE4d27EV^4dy3Fq$?i8LPu=sW zZG;hGu=IZl$Y|zG*_(~mr6aNNtgXcKC6z$OL>bCMp6cKF?c!iw*L;>-L9xmvEt|5i zj%s&u38%wy0OHiZGl%nj!hyNtyb99>~`67S&QWIzTlGcOXAigz(E5Rhv4kX+F z0om^xA2*&fF+-X$J&}Y#d!!Oxc|eq`X0jnFqUtYHStaNEnM6-~rky#6*UghPnZ9m5no?%z&Ts zo-ls`fwKY3GFZ#L`XQ`i9i4)dfJ;HV;*lRMQCe_P?t*75s34|atVwx)iEF9Yh>9#O za!)ERNOLD%Q7(y}H=ev|TkP}~;2|-P#tYt`T|j85APt%Ovu7&gHSzsmd=>BL+cosh zUU$h~ty+2wT`zR+n<|r5>z9|+Q?}$IpZ9-k=#Z}cv+ei7wxHMojSESfGCv%Uz0AxG z(A_z@p@#7PY!22+&VeCo4FA8M8=QHoHG~Wa(z7;_KYVY<%-kwTI@cL0y)IGcL192T z0}2j!tafH!p_K29rCOig^Ip51a^hMjHbS-TmCq{Ijd~+1@T`det70`R)65t`%u#=d z8lxaA;V%)94$bx?zh5;3l{%U~q?XR*t4)%?=*xyr`QQB!MRg6_X>{BlYkGU%8#CN4 zd1yG^@4c6-`0Po_0AB??DnIPg;Xsp{&!Q$jFCM5Pt^pdeSqniWr7k={NQrrCRFM-X zA=nr!M;llw#uPH50MpS!ErFJ4Vx@n%^2Cb_Q4p=+{csoD0BD1DL?K@tAJX4kmxd&t z___u5TgTZYch~mzd3n?9$Nn!oT){AcT!Z4KgERvK5%V&OqEIotIDF4wx<`865a?jf zJ7+FRZnAygUh{9!BVTKFnJB}YvWt_4B#V%DM)1uDdZDn)`bB_12+3!VuxEe27yQvk zdgih!)YFo`6WrrKfum|r-1GH`V%&iGfSa>P1dq7jFVcLM<{a2NE#7fM5Ds+V<=vu< zCABkOft~aKxi4L&YN#`oDrr@KCMQW6inc969 zz&9f`1$&+b(zJA|<=HQ=nIeB6E2hOjt@;GmGy0)N{4!e_TE)`aTDMwm7~$hRXmFUpHY8znu&y~IIeV2 z(u`L~BVW+(pLMPBOZI#_$?{0l`E)1#Yi>}t4c6ao1;p_Nx;ZSe(7LbC1GDsTbtkJH zV>2=P6Zt+M3-lX}0`MPDl?G!4?5k_O1_lQ!*=s{*0^LO3I<1MCL(;}WWnK!}s@f72 zvt!J@Mq&jTg@OWA5W9b|xYha^VV;az$bNK2EIBE{A2?{MpxuD4%i+eUl?Gu)P%fg|BS5ZIr(sI%qS=Wp??C@ani$^584kg7J}@1- zr)gm~XD{x1h5NCsR$HWS`Pg<0z zGs`cAJxr+~h40{))DLYc+mO|@s3Nzma)|3lZGB;{x{%;(M>P; z|73LtriXC`52OjGS-*C~-+5KJbhEhq!q?09Z`qN&=_oo%4@KFiBpxeINV81o!TUw_ zmh-ikD9;K*d$?N(D#Q{3TCF?jn$;J9P0f9Yle47qmim7hFSqMO96=D{R#JLBU!YOX z4l1N^$-`*PFVp0Kr}@@VwP{n(YBgyc?#*3qGAULxDP}oElbrxw-!zLMRlJk6{;ZtS z9*7U@Z?@cV)jo1^LYY*3m9zt3J3jyJ8zX&^Bqt#+j9h6NFN^f@rRiepO-|29@AP!i zBcs{m^lX1_UN@V}jjwP7sZ>2eXED8oi5Sceq#mf}Xu*N?!S?MHH7}@b87H3!J~%NZ z^axGb@KqlO3Hky!S%N+Od zxF=&H0Tr}>=IniQ9oYylR3oM1*Od>f=j{e&f-EK+`g z%)*%IN;_*%;f34?`#r2&d_hKxFGSu<*gQ5^GcB5}R@h)a@E3|xJ8BC&68=fE9g^ON z{lE_*lPLN-Wau7Wv|1y^pm^M73`)Ds&5f08N$);6kTIStPO{C-sr^1+c0Kv!;?4Wt ze)(~7fv*>TyVHk7ifv{`lx2A5=;Md$Ch3WxahO25&WgiI_9_lCE19zzI-H9U_M$Hk zJ=1J+-frT?T^i7VKenct9aFahrg?$fGLsHmeB_8z5K4;BL$Rhh40K``C`7^<*9S0! zyAQ+<@dE{osF6E>w}Sgos1P16Ak+vR)c&%-enZ05+Tzl^ zvhQWnI;pzV`uH(d6X6p)i@cIz+4cw^a!yZ=mD~4fS!6$Q!9AREP)$#kH%&B#2@S5Q zPM)EE6;bW6j}VVqJ&EH$ChHC{xcnecpaB2;8(7zRI@r9``YA%DaaNmJTQ~xGKsz2u z{(%%a8WEdY-Usk!ly+{w_>8X=QNH(5QF=taC!$FBfIx(Q&21V)tq$WOX;Qcs;l;xQ z*dt)G1NL9l1$h2YZ(#l(?%s8~ZDUy&{69}Y;dr=# zs6mms*#-^s*pd}Hkz_|wlqfVKgFur+1fm%L6iJIbZ#HWlXBmKX zBU$Y1-bLZv8TZrfO&|3RFrw>bGVV*lWw6_=mLVGb7FXIpGaf1Rp01oP8W`Y(N&^8z zAPD|v%p>l-uzg;EN3l)R<~{j?Sip!|jweY$rytuUS=ex-zcs{a+{`ZL8%n?m4+YDi zDnZ>ER(Uuwa5nB= z_?jPh`>{Ng!)eZc(8Y}(DQ;{Q2iD8RvRR@WX`x>BS4nxnN`z$Sm9csSQ^TuYGHTI^ zNRqPqc*2KQDER?HN6--VgK^&AmSIXqH{i|J3=0bjGKM?=otl@#G2YvPln;G8mlk$F zijuU3C(y?gn5ObZ;O*%-94lePs!i$F1V%qQ4?2;Q82!8g_1~|TfN%j6e+~FIOZ_zF zY`ZmKlW7uw4vi&_;TEZwl_$bfJb6Il)m|4ZOZQ|&%BZ!v>he$QM`FkZyN?z3A$bZ0 zr>lDuH`g?iVv}iET8=I@40Ks8*ZCVz8vLbR2h&pyQtdV55so^l=Cq|_8zZ-kzwnlG zMt)f?f3{8Yk7&JFb$yzuXU>Z}o8!=xj>Dgj;@};DrY1i<*;o9ibduSB6vlzrFunv^ z_JeT!>R+|FaqtfXG!b7y@n?^~NB!bwO5%w|Cu2|>5U>B?Ui&sI=KM(_C(LjIn^81L zhwYBjAzCYv9%hTVFYKvgmwIslEPrR71&oI!P3l?}36dep;$k%Bicf{|UEM6;r}8#G{PAq&o}Hf-4#!DYDLpla-Z} zne($FDCEky46;pgDUBe@$$T%Cb20aSF_!Yr{98EB&BZE*NdnKhdT0B&*MF&mF_D+X zoDX=AMg_g@Uj-~{BIelZzR5_D^@y}H=yiWgF@dy6=yk7CP6%`#jWfUfS@rY-d%UOI z5}Pt=r?=Xu?|w<6&cqt?CattS+qrMcwJED%ap!~VCnB!7F>3^~hQf|msVXnAi z_U*R{YsdLUtkp~Nt@ak}KZrdOR6Yz(*PMt*C)c2f!G)MyC&TpgT%i`fjc7)Xk^t@r z?LSnT^5RQ0Bj#6O(6&Fw8hQR%P-lFYghsmg8`c~t*)Nj03&#^7Tn_S(ra%k)exZ0u z1uN^ZnQwtmE!vUgjemprV`=dfDF=Zy0}upAE<61~`5f?~SV45x7Lz9 z=0{9T!n%u{5*{>P{M<*lz)J#jd=mpli8O98!KZd z6@zIG;+8pvU^P>RUi@tM73*w}w8)#C4c*`#>|~8=gCrK+rU8(dYW+oRZ6rr50pwR& zGnQdbTeo$m)pS(PVFCP17yfHzmrh!Ws#e8m{`wACQf8Vp)?{zLkeQ*B`uGlLuFKL@ zn{Q6ZtTbD~Lx0E54w=Vmngh9RTCU`2GvT}hwTx%=`c|&QXTpQks&XGwF6L3sRzKsN z99R9wsr|0$7=ft~I5uN0%)t95zd2z1B*qoPkh+#{l?;-~=_M%gSlYL>f^(%`r+Sz% zDAte6o{1L+Am9RhY3TydR5TMcYSl#Z>LJKFKZ1%JzigODroRr40T=SH21#OXO z9owwVx(o-a+I$&=rGa7Ml~fIjCq4s9hHC12O%&l7<18UhGGEB|E*IlIOI}!0 zt6sO5Fm#ttP`%|*9sbkMah~;`|7oNor%m6!NDN_;K802f|+0;zqbB(SpW- zC!B{1hJSxdBuC@T01eGryHB#-BbS|!&=4$LpbOqg8~nivj%H_xDXvMhxMsOb8e{hP zof)3d{#{&@K;!(~MgdP^k{u(jA@va&Fy$5??_w6jSrpZmd2aXx2Y&4q*VJiq-PpbV zBYK=CZz0(qAP}PHD~WL_jA1-$9Omh zhE%*8D4nmt#pJ3DH$OpE@GGDo1FsaE9Ds8*Qr0;VEqRrpo-g^uFRifK#Qp>2)r;Jw zp!5Lb4!GQSO>s#o`M0;;*6U}+w8HZ<7=$SfMX5iZH$B^glX%HrgizxzQk!39zUR{Z zwtvvNe+^zA7t~OL_uqHkQ04S0&K&OzU5VfWTt#ImxOxCzG7o(0yeTE}S(SXmAl@|q z<0$MidIMo%oIm{`RWsA;l6q+V44vWn3rgN{sQ?zw#{jjhaC9CqKMXf*_tc38_E1=u z7Bsl6o-W(yJlI(-e_k$a^fm?7#(#!4?tkP}wk}(T`MJUy4EYPqL^Z5xyIZ7N(W-bO zPKr2ZJkZncY8DlR=7q4|wp_klF3W9syIfv1gau&n!J7dM(Z&VTi{_kyCs1ZsaJ~hV zj%$!fY{2K)irm6}^H}_NMYy}DAog!#O}NejttG``?t(3b30z-y+`An*^ctY)3V(@V zEcVB%)!YtZ`O@i0>W~M{#cguN0E-}qL2+A%%)6kaAoY1x1m*hqx98k=#b)}Uka}pK z=ppV{tg<&`9WN|CP0O0EuSCzC0>!(e=~dLteJ10 znLZM${W8_9OB{k$_vEUb8!VF;Nq@FLxeKt*8|F5+K5rCE3ny*IcsOY^!YLbU))#R4 zn8g`_F`2ChSNl0if;{&g_pJb97%6p=6^Vx_%S>Ko6Cyl*b>(;iz@hti;=N@g$QNn2 zsQ{xr(pT^p?koqJ86?M4)a87|c>NZ0>=_d3u@uK@CS{PK5v>);C&A3@mw&eE&2O8G zhRnzq+@t9h+^QDbuD9UUY{9K;!Ijy9EA;v^%|OIrj`AZW!{q34sn}{oKtOe3x^F7^ zd@&y|O9rge!XR#}&9Du#<}?aH`^L-VA>DU~nq>LW5Ov#Q#~ZKYe&qc@Nkse8;bUF+ zkb+&rqJ##Q7EM6>vwdVlAHyCC)=c;z$6y*Xj`G|S?MdM5;5DCb@p6P9{$Oq97} zZ=Q>cC4vvE4nTVZcj8iYumJ-Y zoT`$wxAUj0R10X!@!|+nhejJpT*EP{DiwQTa9(W8h_I?ql1d_iGbR|-RRob z7OXL0d2z<;<=rMvbua zLdv9ch(3DnFnZetSFUjgQ zVG4FGAF)}vbAMmyS-oz5ch|5Iha69BhnxrYLy#n1*>f!Vjl~&K?z3Z-@{2cr(=M|$ zSS|;q9OZQ-tTZ$$@XJM71alFyH4v{Hw|+0W{JRHsy~c>j8V1|ZpdnBgjQUvkng=NF zpk6oXR`gE$OuOJ^Ao+OZ!pI*8|+_iX4c%fQ@-nY23~TyZ*xqQ_Wq z$c536E~8_1PrsB@rhwED9R%=1^fo3)U17_7osu!lhTMKz?)}C1y>R~io@Plf@!m{V zc&?B+SAPzHKP&sU&~eCbA*dy?tjLiAeY@xXy=OaZK-Qx6+rW>e_S-#&u$C{xPq45! z@>&l0CG>|0>j%lzD9*dA1@Fqr0ROiEsz3fVXz;yj%lCf?#rK*+sZUokR*T}nfTG%{ z)UBzp0ap#KB`gq{r67RMgE6fIlZjV*@}MeM)qjFCsy%s7wX8mg{ybc4!r88phfSasvGLYBf1VGEALrHtGP!Mo=DyT}9Sq_ZNo{JQg((6&yK zV1FEhlgV$iKaJrXNGczn+AT*tA2rL)J*yr^qz0~ic(38+LM(0>$?GGvF8|n$qaLXN zbD9^ycw((2QjJsp0S_3pAuV1d$&p!+UjG4bt+l`Q)JscAKT!(c5Su5sh09f=fJVW% z>c8Xghlw^Nh2y_08fN2NS_;B?l`VRoQh&2WHm$U1t-=}HPTE!sw*&O=X^`YLMHziw z*iNUz--f8-#parCDpwAuC{oX$qkFXy+8eJIr!*LxvxvTnpo=7R9{BjiaG}mQi?d*Y z8zGMfVHr_}HF8?mVIz)6=pgPTK0~Q{!5_^g&I3S0d^(_VM?81bP*eIctk>sQ>R{!g?5oca?u{CUC`;kGY)wdAf`1W zk>kG%b;YjP2olRN=R*GKl=40m{xMv{q-Zeve=W8jo_w`W!<9RDYHdG0ic|2sej-qe4_~ynIm2-4^*AL2s5w;T_5j9@lKg^PAPae`$MSt)06VJ)k zbAI426-52DcgHnAfJ%1Qc*1HGpPcq1laJ~`)K1Am35^V-Q}_gW8LLfrBFZSpgP!0< zqrn<7NFZx(^(c-t8VxGZq$p#_${|V1P8Ip%I~A;te|hE~84>v!a#L&9sh$#UVmsmH zfu%=ldtPKN=+u8n?Nl7#8h^Yjr=5y7?OjTWo9xQIT(Dc&H=YbV|P;$ z_ZbF$ZZS@?2i~tHxkMOmfE$gBFPnYk{8+d%T6BY)^B^Ncr zqIy@*F_t=rhZa~VdRVDvQGfw6MgR|1DAExHQ|F;nOkOF5V0~PY5`PaCHrU^Vc3q9= znCgvsb5jaPge?IDU<>Qz89`4M&3FU6Q_9VUs#JitxrscMfamHq$6Y0y2bKZI+~#(T z<6#DZ`78GB__gJfXkQJDjq5EBm%CYvQ}C;@{I|Fm@kw+or(PHKyT3L+h12Qa#5iqf z#fSUr`IlE4Y`KNsG=JKj^6i@A4(u~=c`IX@eM?q;N^(B|b4Q_{G=jp0{T7B$+D6`8 zF1PaKax3Ho0vndFq5bLdE}l%_^jaZ`@ru7(d-5QQSV_v7b8Z|RqOSyN;pcKo>U1nS zWh$*>wZE{hZtRw#sL0{qCLJl>bAr8W#F4w(etb)ws4e-C{eO7!efx3F2{g_Q%+$hf zwihoFW$hF!!t(YF{iM6-wI8SCL(!rFHGFk5RnvyhYte`W*zoO4yrjEuTRmj>WSbx= zQAY7Q-JaJ?x{Ga01sg-Ly-i9peJ0k;_s#Tba~Zg^mgcht4ya0N4Ro#475*oOY^DR{ zc`;!uh&awz)Nxai>aCV`e#NhU z$*?tw~Uf5KD`df5eav9W+5i=ND7k+6hNNvE%WHhW&^z-@c=M zT)sk$f$Rihu-9jT@2J=TNjD)m_~HS*AAHzn^T%u5kbgvFIE+wtLKYjKLozQzd9UR@ z<&f?jIoepuIXN!Fw^~k{hu;hapsk8z?1-lqi}glK(vYu?og>T=GOA47H`;vBUss9p}<1HoK zo1S+G{(q})H9F0x&tp~dgsR|-c5ixQ2G7>WsV;Q!Z!7e~zwuvgZG`{{C?7zqhTKw# zdcRWg0jT?leN);0X4EfFDKg7nS{gUbx>N8YeoI!S6vC+20L*q-YFx3t4F}qdi|WRK zetVEA)En*FeOAF5_s z^?>_(w`X5~!OR)FL`GeVs}We|D&D6T^zVAz{?R|)vww7ypoaPnip>2;@*<c}h$d6fqctEY z*yQ#O3x^L3PGoQbmu@_WA23~Djzgr70WvQ_=t*DPK|*8&gh`4~1~U01k_i+5#C|j= zZh;NCO(D7jkkJ#G1_>{}3?Y+@<}?kdnSZ1TBLmbkO2dVjY7I!DY;lWjd?pEKBEhJY zAeaDDBxDqPBT(2!AZ}I(8KY!N9 zKaqzqEk-QztPk%muPid6K?Jz_U>)(*ks%H4QSF)t)@4B&$y=Y+9d13$JpP8dFEAlBI=E_Tt9)U&9fjn1b#2-DXoV zh6Bl=yly~SK*y~@-E8|;>JN%C1Aj0mgt)W}{*_A35)Np;rgtfur8WM(M{-ZY!oS$F zUnAGbg21e4%nF0WTnPIAU=~!DbXl0*XK$63BVs_mvb2 z3~siQ0!?7!4%w%Q5S|y?Nujv^IhHC=unn4Rn^6yolIi>hPUvn^Z;lU6@PC_#k8y55 z$2lzKC=Lr6$>b_%M9EA#ymE-x0LU%KJA?+xk2uB0wp&vDIFzC5e3sj0@2XyQOvrvXZ32a*!D%SIBs9egnC% zqW9&T2jB~5p2OUr^+M)$LR}{_$#2BMX?89nB&M6MbIBjS- z*o{Nx!17DB`tQbH`P`oGH$G(SK8|Q64A6TIasDHS@AwnY6LKHZS)2}!MnT~vWWwKq zNp{r>Ne(^dLC^xq3x75-@?XmfLy_cF8i`m6dQM@G-Um4-P84Yl zkP3q5iAuHc%6}W!aB^L!q35X=4oU5Wv)XVJ#5a-ch=r(1*nfaM6!N_)iY)F=3u;@S z_TmaTVBwIjCJkA_G8Z2oQj6I3!XNH;i(X}OT6C>a!XD5`6jeGa0?Ea(y@U!$2^DhD z-_TENJFggT#shgHjvOR~B*F=R1r>sbt$Z9?_>bjo&*C&VA*hIx7lp&V47JRf$!MXA@I=ga8&T`L7XI> zB{XG^U@4W`5!d>$n}7)#Tr{uQ+_RdsX03HtgY4DjfOyOz4^ZmG6!ryrPg52}7KuFT zY`1YxYwZS&gT_G(3#l~@+{VGE)ihzvTI=ne0e^5Y(YWdE|F6zzaoa;gbZ!kxhJYME zh27nGQX8Q9VR5MX1uh$>V7-eh%%c&+mT$)B3 z@UK5D3OE?SqiU4ZdQGu=%a{rVj@_0;q~9Me{GUNSIJ#3eVny!W)2skNz<1#yAl`F8 zynnr*1wS~&ty;_7gI@&!*j}wUZn_8m&U7Ud%57_!l*fO=G`fO|s*PznHAzYht?1Lp z?X!99v)gU;YBTp>w>EKiYxyXMK+$oX=)@LzVCfN1OSmej3{qejH`Ug$Aj+1B_E>_^ zPDFTNT|yQdeXU^bCl!kA;s1}Wse&;Oe}B}(uiIxu!NwL53`9J!JWKK_>vOJ9yo|}Z zp|EGh=yI!^HUD3P%|jYwUqRpQI{rSZ?Khqt><4>|-TlKF|I5i%vv%0nf4*BQ$KZc1 z#+)5M&f2qJ7vpwS-d1fNb2e+`7@&S4#?tdtT1-!$AJ$rpz5Tt=Z5-@Abz6<6&wt&$ zM)UAMejY&f#BDU6?>F{Z&x6LZr+e^El;3LX9vn0db`!U;`+V1Jv<^eJ)i~UD8_%Ba zx_gcNy}ics=B~TjIM}bjpH|Xn9XyA`)uI{)yEQB>aiJ!tF~nA=mesIjwPp>QmB>_; zf@)?#twX3P#OBnXVU6d_Rt?%Enty|T6VdpCYX54ge{ujqXid#%jrio_1gYE&2YkSAyw*#Dat^Uf z#F}R~X3!L9o^BncJbb#Z5pcaM7u&?!mvHoIh0rF^IK(m+o5Y)kBi7=`wO*R;=b>6; z&q7zW%f)tyFFt6l#<9)C?SJwn%9%P0U1%CK5z2rjV)OQSBNK^dtyi1I?fL#e?OEgb zo`FOmPNJ)3))iyNTd0w8LwSIgwf)B4)2FyJU9mQyX}pclv_n3jypf4Qx6L>rI{Vqe zz37WP-nZ;WcsyR)jlP*x(*ard+Frm5`Yyihe}q|^1PLt)y8GQ6T7R?FH~>dLxW9N+ zV$y5P(LtGQZyekoGy9dt_cfZkW z9n|(e<6zylb{qT84?d5ZHU4$o+WY*kaGR3R-1`6 zGLdfOkYD%k_1(gM`K2buDR>u^Qjsi(VrB^s+9=m@<&c)v1&<2VwRi2idPuAwivyQS z4!i|7*XJ!rIHF+T20OeUQWe4*`Y(r#0-q@~M3Q=)fy~ZXk%c38Z^?sea~MCGa>Mfj zc$y}}hoB-^5+8!gp)P+T$p*y{?GmIa9(+VmN2t3P${yhEq$}Z|Ub$8jZ)NUlZMwLm zf3W^U;TyD0K#Se}s-8)XdL})!y#8*SWWf+*Po}nS1zX~1P0qs+2<8JVelUEWv-rB$14V5iOB5B_Z8mG;ICV$vV45T~+2q<};qKQab9cS*-zmnE>#R?U z84S+2@9x*eg}Z4~M!S__O-YZL!78DiaeVsUR`eHS8JoHNAafHs__qiFTPOuQSQED0 zyDd`!2m9iEYp;L09iO$DwP)*lG;ICJF0chP_EIxF1}0FI@&d*y2P`rejs z$z{s2ageOH3e;hJ-?v%$LV>}p$Xa{Voh;;9eR8;g+X{bUAn|t+R3tS;HU)!p)JU5P zGaoAgB|2Z=(?KdF);506!4G5*#>K+jZ^FukR@BDxFDzKpFoQcfbb-*%*es{nKXR?u zYRRp$w0lll=JSGXkEA|#dt#$|QT}Qx_EYWag@9KDzey5*i$B_GZzXN#Ax1YI@R;6D z#YH+KX-a>Mh}=&bIeLd(n+@pX!2^*rh{{$g=z+i7DGL>E9+I#$i{EX&!ph_%h(rxa9N z{H<2UvOKG1IkM5>y8dR*-}$rq{M{jvux zfN+08R=5Ca<0+rMY#J6UvkQ`vSXgSC2!B+*FoVihxu23)x?RjCK}h}BS*^xX!x){) z_^@i1B18#QFfRrzcB>BD?N@ONp6sn~JuvxRvAw;`UMM%o(m68dV-8b0CA7E zIgh(P{II=E`(kPSKJd#%1~&U17~d?ef}zx0vCddDP2jr3A(`qD*p0V;A2@A}t9PRsx5~2# zZAnZRe;1s%R}WIjCs({A0CR-HkTkL{m@Zpe5f!r}er`yG74WC8aI;{=*6^!o8oRa`JS} zX>M-24=a*N+7csSiQzq%B4jbm!Qx)g=LUpKIEfuu4)e{3ZrmdY>O|a)fYyKT2VVo0 zjcuw)K)zf8VRHy0Vr{{LvC8^@S7#NFHnA6jmPfL_q`>-XH}3ffNxSLPj-8$#^Up+A z55QHK@&N+Krih`u-1n!s{eUpfnkR93XAz@2dhg?rp0!;FUgN;2&`P35$a|Qd#_8Rw zu$Y1(6s{%?$-)Z|QEJHBj1GT{U?mn*kc{9}T$G|g$yb^gC%5IkjFU%uGER(s?0vZ( z-Rp#Ra34p9$W8l%gSsj0XNIQVTp21;Vl~eId<)+A4I?tYAeEA!l}cT zlOWYW6u#x&L}cejUR@P55lpQ{uNN}SF}_I7jWkjXLbTyHV-qfyl37r{S1J&yd~fEH z0A#5$-$?=r%rei9S&Ul3{(()V6P^7n9{y##a2utgf2NDe5zC5jS{Q9T1!+Si zn3%mgMc3eBcuv7KoJxGgq9Bp=oxxLt(>SFiz$FC>ifGbUfZt~^O$)ACuS$A9OVyBn z0L~TOUnq}60A%^Q6il{=hJ46qp6kH?Z#L*BMTHZ3UqYh``uBeneJ9BjETMcZ(Vn=h z{DRF&=yJhkC3Lx9GZS5S_jIY3w785vQx(lJqY2mCe~K!LhvT#qK4-I1_zH|ObAK+9 z3c^}U`{wEgQBisE2VqDjML2>76|f#L=!&qA-Ees{ep&>I7=W~C0J0qr5y61nb7_%_ z-&M*{*@L{UCFFm|7O}*`6pEh(f^sodRNexf()=(1USVyDM%PgvZloq zkQXS@6L|cTPR7{Nuj>MOaU*DWf2s)Omh-uEYII(@h}!8!*!+l zqF{knglNdnX^seSyHh7%)NnvE{D{R|MsbFX7Xd*JA%##I0!BB=g5gj$m!_fGxe3Z< zex{4JvFv{!zW(`;#o+B|AwYnK>g$z(6%c-7mIr_{{y0d5&)GE)NZ>_I3WRW@O+1Bs zJTytAAoK|(T)Ya&NAYlk5XGs$m*xtj@D{Qc#pDgHr3u0 zn&H^u<-i#eu2V>y7Idh`AeQ;_vtWYB3p#!$D=B~I7*;1%T+s0a9mvnikWHvekK;5+ z$z7a^oQd8Q64fX8nyL*86g|Zk3p7?F;3n>*APJ{q;~F5t0QX+kcf70$O{)KjC-1_P?TCDg}Qj(X(IE5g!3HB09#Tan~%Fjw^%s8~5a06ZM#?>NSs6lB6COd*;Gn45YDWHJn399QMb zsP0K74;Ci{$$-nrXvU{Z#wXCQOr9<<8QcKHn?tS~H(`G)2R|`O zPJ=Nz499PAynt_1rnz9O0OQoiHI<;CHa!dqnk!CJ$7%K<9|fvPaZt%M7m_DTj8raL zJE2K}M{3~F@+=h5S$NF!x8{3tgN4BhLoB4A6A4;KTvDn=PbevjIV!X`hg-2%xL7E5 z4!+K&dBJz&lY#&V3Pc@TF8Y60D(d5|Cr zM{wW;7Jn`|t$q;R4N>Vygok(^E~Ud+#3j4@s}u$LdS^j4RVHV^+MCoW}zrXmZ6QbGK> z9DyrA{QNEBd@sK?lPj1FVu?h;n1xua_u>O8)J@H%iCO~qI8c5A(T4c|+%I6^ie;Zq z$KVV?V52&s7i^*nMLd7h1265l^S~>OaQ7PF?$uD@5fwcG;G0-UZurSF%T!mpT!}rCWu^5=snk@@GJh2Q)wuZYkkVU#Y4rNv1wj+2GZuNq*0N1*(TgMoBt;HwBUS&S$*K!`|DWFTRX z(L5xh7KthnzB%|BWOS)4@ScZw)NcaYFlHvuZI~=3Bk_M25I$WvwFp8E*lebOz^m8~Rr?B7)&h!?+QeQ>)zk3#bxQ}?&(ApAO#&3FpS{Z86a@e0z4G} z#kWBnS}^0+)qE1c7W~txH89MnH7$rHsU4DdnFQ z7nL9+l_4sB!y+Cf2LO@JJNd!GV(9=r6y#m1U>rLT`koItf*BuoL&{>YbQI(x-9|2N zu~;f}rM1unVfbkovKNb`cXYv*vmt`vV6efJTS^qefm0kWl)PAszicf_Uj&p!uyI=J zau^jq77(i>h7r+frldT^Qtn`^k_ZOi5RiQ^WHB;-N@12Em634@hqQb~MuF7>%M{ee z2r&;@V51`Lje7WqA)JpHLiuT4M$8~&8NwZlrHp@CTE+vwRKk>`$6~4WG-F771e`l* zk&+(-=7mT(%>`u2DS#w6lG;c)kO1L6V~B$Q%qHl<*D}RI02kj@Sv+L16qIx{o?f5@ zIE3SWm!*V62*S%ZB`M+{WKxv8$YLo!c34^p1Tg%v3|Wu>iarRD5CY)~VPe$eh)J;! zfcU3nNQV%LpOzL90T90|LsW!7{ItpvBMa0f$CTSOx3ugC;Nq2nk`q}hovKwp*U!;=yFHJ9;-*J1U81P1UJU6oZc9J zz`V%V9X$^65jwZPl@Bu32UAY-I3g*jI2j&hlVj6~6rrDa&)rW)0!EtCelO2ZMp z+yLeB00TVYn1(D9$3={c6o4l%rWxaZjagQA@A6!y0IodT!Ex^Wz_YlDo<-ngUS&}z zkTJaqW=yS4XH4rfjZwA~M=_+pfZLCwSVeIU$|^47D|nj51sLIbGsdVQcPFq!#uf-d zCZnKFi#YVGdpA%ai`>5r(jkt-y*uKc7P)^HQ8vti$%to&bPvr{k#>xq^JM&^kYBVj2N|R#ON?jKNRgG!jZMN@63_Dxf}$k+LDJCZIrCXcl~q z$5So~><4MYCAvZkYl?gn=tLS$)u0ciG_Vh%cm73{zC%9_P|78xdts1-T4b)JMV$;TELUw7Lnuc%(rP2R1fd>c-m*kgO+R=73(!-Txd04x$CkCcunfRKqP^_xI`%eH4sYV(L`8;AxCTc8s* z+{M32h$aJ@MQ1$YV*)ZAdm+_#hVV#RdH~j%MG|ZbI==wTQ~>vXC{P80%P3)KxRksI zO-e8_#^F)zAtjH4a!nyzIeY_QrBY<tMcULPJWvGnz zgpxb_)55>I_;(Ng?&DvYr8GGUiY%VPhQho`G%O2$5QS2bDKr$Iam+y}eU-v-1n#DoLIW*GG=gVwDQGBCaKKw*5;??} zVGj6B?^WXaIhLS||PP%vo%F7yydv8W83s~a;Z3YCF!CaEN0q9va?Xr`Qw--ktTFENOr z?`frQ!jBhHwX3B0C1s(HYJzQ4^HQ7Qf3yD9|6f(GzFQlMOA*bWpxRREG72jy}?7|MRZK>V`G1Tfj^#+{=Ao;fSy;eRxMIuS`{pv z)iw)dQGafu_*KN{hJt9){MPcO*!({)?$itvV^g&mEZ+UsF@VHn`$e!VU&(=^FG+9tuv@_Ie)bL%Mu_d^0^>4NN4Ve>?; zC7YD^#&dIN>15O*N#AM5qO|Qq=5E5<<6kEe`Sy4_5t&=giY7T-D|Dax#v8SGYZ8YW zN67s|RTlIK>pSgS*Rh2ytk>-vez$*fu!<*IlCM_GzZ;qF?wMUSH^Tbb+z4c^S{zXc zlYYrAiKxC+>7sZcp7;F+sVvMj9N!&pRsa-_#w`3je)Rh8`0KZK$6vd=JN@$A-RW1a z?#_&xyXt*c$-n?NP9Q>7+0=lD%R+pyZWtZl5%6glwg))$LZ{_d>j=HX+jW1#5&;L} zw`Eu)5MUf{<8eKIn*Y0anSZhG-|gGo+kJAmPtNzrt9_F8dxSsmb;zW-7M(#fXxkdL z1|}sVQziRS(h!vLdRjDcxUaFLnWF6Dptrp4mfb&B?Vhvk z?2sJnMzufYy&rZ+h)vjDe$2O@>h74^}PA9y7zoc0y1*X@gK|4N!! za(Drl3VZ;*6IQ|^r+w=G*9Gm&lYZ+@phlkne2POU;OG?P3>R=+H65qksWP~! z2Ifkm+7)fcTWU0|l7MqShn!OzJZG&8x`+|n{t`0m{)}?#OEiCt*6ZX^{KVoPi;q=8 zOO}oBEajgY9+S*5)Bf;e{%y;%wlDDS>9*z7tnC@aNx1@hTtUuKA02LiS5ykJR8=zo zp^5$leDPNLr|>PBcuuyC8#!bv1ae$a-T9DKCX>!-XrGI^dBg?vuV}Z~lcRkC4nxVg ze=at?3pL&4frWpeeh8_Z8k^9aSs$9unL)M7Ww+P9qW+ETw4Ez(BDU2+KYVCu?n=8_ zo*8Yw(U<{Pf;enXw)}IwwJw(nBiwQv=K<;z-Q8^ujo2#+=5ZEf$Z|O;0k&I?({{GE zuP7g(ZGV>xecw#twxs{u+O=)HNh5HKD3*0?9OpS-abkaD!68@F52+oii3N9hwnMhS$e-C>$%M(4*8buBL$@}h zc7kr)R15C3SmMY=G*O%mB}2y%PpU3x=g7WTkM@Qu)#s3cO~JG}l&M z1e}4c8c3JIs{t2(oU4Y0WC-n4n-FeULxUhjVy7Cao$4lO5A{xUlho@sN#pi59#2?S zh(qRbsmNBW>W%C5xN&<+bN>Fk6MGNRRzkd5wKv-tyGe+p^H|cn7nac?%}tNDY2rufY(N%%#4Q=N!w$12WI-awTYz0FKS2XoNq5ro z7Y-a^l9iecoU=jShpCdj%UXUkwFiBNjQakfC3~@4E?VUMzW;9Dwm?nMA#X+);M%vw zu)W(ituShTcwwiKgNpqWjx2H`GQYdDhg+=P#$!&%m1#6U1%SM4*6S*Xpzx(c9QIN= z%j8=Y&nx#WnMmGfsq%QZTpowLbXy(y?Qv_YD`RO+`X;0JW@J-vQvV*t1p8F4r?Nd< zpjU7u2&0JKb3K4tRG}N)(v~d2o~KeagHz*ji5xqBes}J{uBD82JTXNg8BCr^<-jYD zl*%|h7(Duq0Y+Q8T?mmA+i%$vtg1nF2X|-*;e{BR+r4Q>6AFP;*d}Kij%E@>)5yrx zFPYy|Q*9tZhk7kxIr1)3*7%Tv*kq8Co8VK%_5fML>g6V1N$T_IbcBUW%Oc2t=RWtd z3KsHz)NrNsx-nd7C-EPUEPCn67;s1>2E39)&K80}#s-?gxz|m5L?lu(xbhNo)tjzV z;|tyR4)Y6dD`|vqXH>5jcwpXY66P1udo2;MKKOc18t_i+T~x1cC4w2!>qr+XaX9j0 zZN=8JoOXm(ayzmZKlW@{jvp&2MC1^iLO>&b;1X6qLz4#fv0hIas_UhIuH683jc;ul zw;iGNJ}eL@qC|UfVh?>)nVuE_>WE*CjGhR!xrtF(li7mhl4A&GcKMi!v8_aLaujw# z@#&edgP;f!zXXXB!CZ?e^BE5!>4RdBc$ry{-Tgh6T?hBjEw5$qNIn`Os@yceH=C7z zi3mPha@xwd#01Nku0$3=c<18`N-X5~B*G$rM%fbFvqYlxD7&_mw+9eh!eM9?qB)b&8GM=!qN=Jtk0#aKw*mIvQYjZd|y)bjbeZ|0@rGPpJawl zAc=3ddz0pxzUGh8nQom9yA%2bsG39nZ)9*bHot&9es z&`yEF4&c^Ix!0AD&KksaB)OJIy7Z`5}vDD{4(!vfy?lO~ZR`OG24CdkDQg(Rzq|5=#i<+YHlmP2y?mFO?DBJ__&JS4ht z--j*BR(U`|75ZU=Yjc+l(PeJ{k->y){j_IC4vGUD@&VeAlLVZ9qu-9~oP?D&vx)8K z9)UG#fwl!nNKQ(^JiQMU4Y&~k-w^QIpkM@npLHae&;wsegYL8q;bM&QNsNr@OJk@V z1K%V9&MKSYxN44AweZb%wNPDBtk;bSXTjJm<#yVS*L?Xc_=eouzYV=qdFVfdDjMSn zyh{=3nt*g?d|b$X8p8cM(c^Y)2$a*0+ASD57j-*kA-dTlE|n%$w8C>P1=8^!1c^kK z6yoH&Wt@D|E#KFL!^F!y1w2$VQWl|i3f=mVfjZ*;xv6iq6|B`MyaEsy?aDVKIS1?V zhHY*&6cWufHy7Ishc1+bPkg*NZNxeL+PNy!>7-tv#HF2oiOutQ_+i#}ULLHpBI|?t zWNl%kwnW!PB9QCE`o>STlsxd$!8vzE5OpGPTyR?^FMw|?(3lf7QW^(5z0>KY3wmfe zib39!FqHc;;Hrv>wS5Lx=(5>Xy?#03*7wbHggIIX-~uXP#EkE<9&RX!NY%|r*z=P% z-OL2k>p41q0R)hxcKpzkwe+jRyA%7Nmv-}>AFe<=5bFk|oIhDEZSmzltekezrJMN{ zs5;DZt0SI?Vk`KwPi&-R8td0%d$vBdFs5R8){xcJhL>Pu`ZOK!$7cPaJ0Yx*1C&g* zTh2|p*NE1 ztGBNZGURpM_2Dj(bVW6U^~Bh8>7$2MJRRVvRWVSVyx-N$?k$%(;KiE0k+f%QJMY&? zW05T$Mt!>=v9k(M$ANK~3*B&%)tJjj&V9LLwsjPwDJyCdFh*bdgx&g7Q!TKFArLoz zxGaK_GA|)q#^uTJ5Nguaqavb})8>K+E@<`XnjD9qWYIs~-?siE#KyVUIBuNGHJX;M zR_ojY@+wlNjkR+n2H)jcY+Q}4qN3-HbE)12U)#A-mIGEaSNE)oJ~{0>4#6EVG3Ms8 zI{hjs)K==mzHpIy3R+$GNFT2~ixsDTYRI@L-#1pnGQ96d%_V@grTZe1x0N#@R;5Ux zAGU1PO?pm8RQoQXX%P=%cubFk0%BL*=0;&$JNI*$`;5Z+G!EU;3T3V@)I${YprP`u zEjy{#Lq)D+@ebi)r6BRPS}QMH#eU97eGCmQoO*q0=TEM)DK#V??LhHZ>x)%?yTl8q z+KtVTZx`~gKa%c`w0w#%r(=e@tmm;6()4dSf|jMV$5N(N)Q>s|vL$$I;y7rG3xROdnmE>Dtod&31J~xhtCO@>Z)+ zvWT&Lyz7@eRw0I`h}>ND)=~3+LvRx)y2n86dp_-^Jn8FVjh_Hx%h3c0*5{H;? zOw%c-!UDu_r=WZpbotr7#~&gY<@KZ?>8^DT&#|o@noRCm!4y4!TR<@2CLu;Ey6`a` zs`e3!yQ6^*L`A%c2*CK9EN(?U_e;JguPPke=RS$mo-QfpJ9rBIDuH`g;j_-_tVMeK z8n%-4Z81(ZW`hWCt0@tG=QWUb&_K}>|Jsyyz6LD7FAIqbUfXL*A&HWZnQWz?G$7nv z3QYtnNVr-6aN9vr=*&Su?^`7Y~m5vLN1am88793*3PY;DR$dTd}Fray~Li zhgG+>>9*Cj+VfT;PE(q_y*fMf3w2R62SxXJAztPw$C#)Lcf=z4s!+=NZVRZw!Si|@ z1n5>y$Fa3sroQhNovoHAm9n`yN~SxCxUEOwv$3nlYQ6H8%qa&$`QvWXSuW?2x1MA_(eB0 z4s{7Sfy|+-#CecKFp@(q2Haq`=cvmw>INHq+F8U`C0*a)HsWjye*ACz!nN61`pd~i;hq)AMv;jfhkFya`7JodY^D9KvW z7ti?uXtkDq%k*VyuU=1I?Cs+B?jZzPhfN3_9PBRA8J|NVbi9zUhwT7bohW;RONy1&I6Jp=D_xw?N1 zwEYPx^Q)*oYs(sFxZ`^PbF<#~%;Yrlub7P&@5rKmI%F;&A0rKW(&w=MaR2ec_D9!! z?ss0ffBfnGS8vB4T|b3pW4HL>7-X_Irm#h{cAqNe$^$I&W3=Otzx%q%lfWH7m2U-X z%iZzZ|LnS-9xAoXijB3=7w)d1U7bGGq$Y3C-mWp)H>Dy$`$)9gZOPQ4_2VYA4tU%P zI(##KpV-#NkCTTFt0nwgt}J3rEyp3Fgm*GG8-B&UptUG5Q#Cbx368=kKRTHBA_zPj z@r`8`=4BVOAteV*CnWT0!c3B2n#4)b@_ zSulCg+I{BibhrNFN!@Mx9rush!2SGR@>sNgsc(q`1St0(iPJ^P6`JrBs?UsTO>HMH z4i5KroArA5^5Af<^}JpWd8W=|6~Pq@^IMLW`?zfR@$Ru!wR61AKsrUmwiM&m2DV)2 zuNUu*UT1BsAaAyeK?w?cUNed{6X}eKUu@; zc(8YHA2Tpfp!NLWqAYEIIdeY(o#BY&rHb+mL&H0#E7Kh|8~lYKoE5+w*c-=x%q%uR zJTVgG3Qw+0Jo&M|B5zs-;5QR-ek>slIi zt2GIO9e}4lG>qa0wA<*{U0gsHMP8021SW>eOR0uNszTD)B>24O!ae=WcJL0l z=f_)mph)h^Nd9Dj9F@v{oR`Yn)M~kM-n7UaIV!jN&d3cbL|ooU_ z&(|7xY?sI-Uvj8U^0ju)3j3r)dQTj(u`~y{L=!&+K?zGl`NScA=SEYH?GqB{`xh!7 zefTq!Mxek-Bb#nQNPac#v z)7^7;#7FKZx#6vUIbS2a=Z})!^T$bVJh1Pm^Kj!=6Ke{5NBynl3M$1Vf9;2~ef%DfcDyJ1S7%S zHo-j>MoFlf5bCUCk^YGx@3x%>i8@_CzaAi&+%44uiLdT|tVS<>t7;S>ulH3JRSE$y zYl$%=kLBRbzZ%eN<2nrv6Y!in1+Uyrhquq$H`ZlG30yMF6CCDV2&i}bruW#vp@=D3$wdC91UaaE?KAN^8{~L>c z+0wJ}l^R~QI=h~fNDh(O8I`QaT!4(6tM+Bf9uhZl$Ysl(!z;90T!%+5TOBlak`C2W zdKMjn4ZJv4AmL$ih>tR>Rjx?*PnEXbPAq;7%JOUF;ekt+#m|stImQs^Mw*4M2wN}K zDAFu{E;kzlfKc?*sK+lSH0$zp7Bca>J@SocP_Nq)|9xyvNVh9iw9HD19_jMsXn=Z- zL)LOIk(P&YGiP3kO0Wc3z_4i-8z_35f)zaQDcj(7$fZzcy9LCsD7OG>NlLYZW&=tA z#_Rt^#P8S(f0t|+pLBhEQawK2!fz6kzQ=BVi==M6y+NQk#5^0H~+?h3X8ISv&0 zmx=ASkX5^sh}&e+8coJS=8E#rBU`De;tZ`k=Y05Mkz8=2g!w@(539*nL}ZB{Y1V(e z2V7!Sk(GfWoA3LE;Vn~4)|MJ>l0V;6>H?Y=Ib`T>EtX5r4BM^O*>aic4K{Mf+}~n< z%O&Hni&CuCDV}gT4<}I&@$T<-hTagg!IsOdIbfpL7mNgRZw`h!mdh=$FM-lPL+`%D zr$Rnp-1b4IT#$VHzU@y31Dc^Nnr&Zh^ndi#Mn5eW-s$U|{TFj*|D|8jM$%$T);2_I zck7}xRj_eflQ51EB@x!07Eo1PAQ)H)`ahln8rBm1)$hN#YxE|W? zr1l-Ck>8cXa5^)j+TU9)@%KRd9xj)2UTEyMNNC?V)gr=55eLsI4S@4r%5{t4}aJ(n|^$DFgURs)gg;5ZntRd7~PBg5}^x=kSS_oYE;%Ev&DI?yD zgE+DNn}&E^>H~z3os=kKqR!!-q|HY@kMGRAVS9}L-Z|t*X+1*mf;bGP{=+25K&lU_ zE{ZPvWYUG7zWBx?F)t|+r-R3THC7hQ+lk{bJ1L3i6NA=El$v1)@W)FbxoG8+egDA} z_%SqFGa(3p0N=Y-!*-u-XU%#W$8ftuZsrk-0_k}2Ud$Np6p}0 z?b3^;^bn*8>-|?6PJi#q4LnqiE%Gp!nE0KU_?;bo+j=!btL1VHoTCWYfNzPO(n=b( zd6mEI!`ikVs}v0mKUTwkqiHurbCfa($lLWHya1!({D`MUF*Zjr-i&5kM$>%snrrxl z;nj7P+k~&=$7+%DmiT*c zFF()7!X7#$?x5W2XU~?)A?mj@zoGeW={$&N7>|SGd%IB0`VC!wI3&6McOSTiZ|m|? zlKa05sy&7vT38b*giNB_+<&~yqEK~&@_m1v)$3d5BRj4NsOApvWWz7F@}I`I$CKb9 zl)FxD51o3QZEwFwOa^R->>jjEF4WuBW`2YnuEYi6(vP{7c>c1rwze7SzbHC{WN$Lg{`3a&2R zI*`_rH@UYhat>E)P>Z=h$|bx@2h6J8L~Z# zck|B|fJ1(_Hsp>6&3k@Wuk#6(^JvWQymWv3c)x}c1vZW<_z3#$Q9UY;>e0V#R7Gos z-!=kAr3>YM>kX_PBLOAHPHFWVmr{ON@#*>z#itANx=$CEfGI^|DV>&i++@3j%jvY# zp3^Uk-i8%YoyvsGVu^?aH{#sN>zymwt}beSKF@LvuwdL<;(K z^+kFA6f9pBd5x^q@)p>>O;K7 z!qy6Zeu=v5Eg_5Mdh){+Ne(x2L{pQQ%D;UmCL2ydB~&Gwj^hW8Jt=j0BCA?_3qSS$ zviElDZ5z$H;PI@hps=#J(2Gr4RAfm84RcwNohVi$MY5eZG(VXFniN6^U=TLT|E{io{;RHfYf-N+)|{mvrhJxK*hi^X{%$vO zQQA1MM8>)|6FIOUq?_gL1eQy=@W*>Jqak!+!+IIG6Jkk|%cUd7PByHU&?IZUoPgli zya7%smrH~GP=Ueb_kV`XALYYYL9X2(duhKvk(Mrc{=Z|WI{|4@trvw!r2wQaO!Z3~Nggq}ZivRN|5 zw}_E=g5&MREn`&JGK?aTex6PCNl_NHX^hfF1FrWl>!R+akt#;vUW1j#mGg;rRv>{- z_H68L;lU(k!`#oiy;iH0gB7HIfIJ5GV1BQ^0@NA-Y6doF`72-L60@VBGe{kD4R(rN z+63CEfL0g;0n5O4Gi3AH7pIXo97H}H)cC}%@#l5`mjbt|ki>mLoek%U7Tx79i#fU~ zpqKYOoxN8oshKy5nVW4SM2&yBS}*+3X05nUubm@f`O*C}OQ)a*ZDc`zth`mGjE1#O zfNpoKX_P!KZ0K=I78wyQe^s51w?HH;kt8aSgg%9CVCD%<^O#G#QfmlW8w_&yPt|V} zO;*~_+O#Q40Bfm0aBu+F%BS~90G{785~9(B$(lB#bln(eqx*XOU8b3pMu_z;+quH= zUBd}$%ci>LoIeDLRC=!%3!@zMuDqEr6T>Bdbf22n2qMv z0IPHY#o7hv>LO3T@>C3~*GnVm`#!8>FSkXUks6{1Dz}%WQ$bmabF$>S^_*dMDlt1{ zfW;;2IB?_Q(tJwAI9yh}#?|CqW-1+TcMR-GL_ZJ)5U@8G;07Ik)AHR&BBpOb69hFN zaNM*23w~RNEMO};OMfc}$}@rUnpS>ZXw7Zj6a*CLkc2}7pf%J2vDOkolX^X>HbAJ? zMWSY#*oW0DKgrphy0o~Vk40$H7-hu;uaz%D#b;qJUUTj}j0edW%8DEQ4%wJV$Y&iw z6tYxx;w>X_hm?wcQ3i_80}**0WhV)Gb`$U$=fhLcU2uyLHyUX95?Six1bc6Yjq7#S z#!bxW-JZLkfc?e?flfg8LK!=A@+ln+K5ns}s8f0;7E0%t7`d3xZ0%Jl+eBlM6?EZU zFi_^Drh1^u`HTw9>S*f6x+huj1|)SU@G{kbsv=V}*sNE7=m-b?Ld=pBRA&PP`w+YF9Z?x$#e<$ymJtWdVX{=4=WQgbj zU(qG20Hch%?vsij-mw6cp8LnL?{io|1dm_(ftU-&y?LJ=_vVdupU_iq`U3enp|Hg~ zrKgQceZn_TcTVXJy(CEHQ16_`zDV;>(R|W7YJ{BbeP8@+_vul?7eAXKw%Jw+%u+{i z&wr5esy_^Qf7Q+2iZ0g3jAy1S>km_sfm9(A2WYH?abwU49o30X%M4FCW4dAlHqF8x+mx(;|{UQRZs4%DfuOki3i$P}1Kl z+Vt!41m3_`5y1j406-}$gQmuTYxZYk!Rx#Hr$oOMe{$cV$X6(5DuKDO6-7nI&eJ8r zTYvh&tzQ4zgvm_nDY#Yb0pn(x(7d2Q_)Y=D>%*xA^bMEGjc_Fk{VZzuJ0H21`V|co zytSAs(nFf+pRqZlmug63&=>G7)gWC8krn*kZKKxsC1@yJ8nQ5>f=vGmzE)Y_%)PW# zFbZ8VeQ|%`R{vb9#&)a5)va2uxe$PE$t5Y$6g2SRjZ@Ro=nf=!vC}>B z#+LF#afn4pVsu_uW(k*sj)@1YHB}!}30vT)(ryuqZUBuTkM~IY$}B2lFBdN-^*R=} zU1-jt(V%CX47=@OT(c?TD%ftj$aZB?9ecJVf4)ts-EG~)N>ZH>4~Sz-(vQ|8mD4lR zN~HlP177ax?SB*!0#(yOIDbLaa#~xmSTbUiA~p2ZI%mjkj=Zuc+DX)Vk}6W++#8eSGH5h_sOva>fwn{i^O)t*4gIZ8PynH0&Q5c0RI~vxnxHD)kqszX z5H%X;+ZRneGGxJ5Z-L?ldyL=8CHCMhifjW1WTK$6b!W5m811nouL(>EH&Dp9>*EJG&GvnPROa zW9t7Ll{LsW9G(&RfZt);6x_p8Z46oJ!K=0V_`9h)F~S8(GdVntut-Jt%Fa8c|lih zTxm{N+RgMHTNn(~o)$Yh7f6LlxP==QA0k8haij53V2)oP#xKQNb+Yza{vCHSVvXPX zD36&_3aJ4d zZylLS%hA_bZxzce*ifx8$Ks}zYQ)QhK)h{5^;hF+T;Ix_`4 zTA2xyV7@i=PKx?C#vp*e8lNPR=FY3Aj2vSpO`4Fj41+NLo;TaXK|n}4fT2yAyQ1Xg z&bqP*FFb0rAy;?D+a)xCmyFT@9e)z=5eOd9UJJAln-COn8Rx}!>! zgjmYLTobBm#`(421{@2q?gWK($GW%__zDho3tJhgs;ve^LPNJ|Jq64S)PF9JcZe6g zfuy2!oSb$GCTb!1anJ8LgD?cIC+H<2j{;oH2Y}zA9ZSK!}?`K8RwdDvSnkI>H~gOGJObU)AZyeM~2GEHlp3Ev=>dsS9_ zKoO4hD7KgPsdsE_FsFMfdcNkM{?EM?Jy{cXVQx&v6*{*3buVu=0dWhJREwBpwt}o` zDCuaWE)J2E7d8@0EUeUUIS7+ZEmrzo-e_-%SIQU79lB$$*QXoqCQ>|HDQ7y=6W%?v z4yA_`rw-L|S_%{ThJSkY9}+q;kFL7BLs(EO>BS)mj3In?zYre7_3w;F?iDkopc-u& z`gs?`JDs|Rq>!*0VEmBKW7q%=t+TS-(B0>yI(I-SdWa&KQrvS2GQFefR;2!Jxw&@r zxfQ7jJmE?;IV!e*e1Xbc_;5xrtjBG#bTL20W2az<;yryJp@|vC1=}$$yRZ z$50aZ$WeO_X9@lvWYhsbDCvqK9ZX zbx4T!L|?(NSLLhlArly!#WyZxa_llGMV|D9d#>dfk7ATaYSsk8Zc-fle54l!0 zvc;hg%;NtSg^EJ|kOpf=^bt+m%ti#o{)DCC5YTPRcz?C(aC=C2TMCPYX^?lu*7i`h zzA(Ah_FPQ*E)GDU)-$u#acR_zg@Y3X#-YH?pB|eISEUJXRoy*K8=Ul&V^A*Wy3D8b zIzr49vL?CM0JnL3k@W%@zr*8q3-RB{{gX!*x6uo+or&n;mh*w3fc%i)rPufOI1b*v zDM~1v0)J3};(U+ZF|Y!~g|JAfb-8C^-Lx8UT`)j3k0-pclRYpSy~D~Zse{8$I5Zj0`ar4KPQ}BL5Z&)ek4Kwyvjcqkrw|Umg zO>>l!GE?yi%)K9!*o2Pm!ISr>ine|zQu)QVkAG6;;<2eLrop;dRxa_o4MV+-d+(~a z_dBD|y8=gAg`kk&&=js-{28sBzi5CN3(Lm8Vuh?7-fk3brca)n>mG=-%zSju{Td39 z!=aX4=65j{v8h|(Tk}@<_L1vT<$LKx^#!SDCMUy67o>{|{u zK%>XApH_5dxwHyW?LVz<|EYEH*hHPugc&fS__x)?7tl%8^!BaQtJq9eHCa9uhxKEL zIq{xSl`&ve&cLiobtYi4H=puyDR61Pc3UU-r?udW$z@576Q0*ENOuhJg?^1o#eWu8 zGdSs9h#T64z6+M>{bUT)cw_Iz<=%a&=-h?49u_N~S66;zll`j13dObZ+GMV)jV-=H z)D)}yYqox7Y>WSGJpvc<&L~(NZe16pJL3gNq%IJoMYZgDW6PiucF6m}vD|KDTB=TH z&R+_)(KAxDV8*Iv-mC!mvW_0QuYc9_G3m07Yd7@qvdqFy;e6!4!dhY1Vx9!8^MV~L zfi5rXE5SwOiPA1->4Q}lBoUa>y>Hvk!35buLM2R^V(M=sIL46=yx z-|Jg>{)<5s<_$cr7~W34-!mQ>els2#-tDdEhdsj^-p9Qa{l*OVZf`|@H-7_uHcyk^ z8*dECVkt44^X6qAFOe@E2f)frIOKv55XY&<1n32NDR;ow$qGJ&2cuFbblLD@SyfVZ z*W2ktrcVHH<(#fo1;@>Xcd~k271+;x-;f;{?z34bHE3V$=zSzRM9m`1IX6&?5SQ!d9%tB0Yge8PRs-3;OZ zU`>H-C0G<{>pY#3JVYAr1RIf-M-c3>{6tzdghFwE6-_!-sa;Mt&xx>+mK&0|=+1-( zGaF5WI}@2m6@{4VoIRnO%*ej5vX?!g#Eq!2Q{@#UGoQUyD|PwuwSQ_uXsI?#K!Je< z;Cdra4nX2Yn?e=e5+b8?{snjKA;Q1XwFs2DOS9&;+`WiVII2@X*_8JJ|FyNI#l)&s zmY0Fo-%>H5@O~1ZB9xj%fi>0MZ2~MV=0U!z%^TQR0WaI8lWo?sJO0r=xap-u&Y5Nx&gQyYR-hJS5_A}*z}wA)6ecHu~9 z7sdo2fnp1dC@vt~?6$iAe&eC0C_Ip>ifld6icqa3x;|2AB~~Qi8ln{uQbqjR9b&aT ztL|oq-CV2%T}rH&5026w_|L<;hQR{L7T9>kMQs>h8F7?gUw=A`3Qr-MTMbbri zl-$3{FU1IXgnu4cb`nn)vj&xV+HkOOrW#>h|2D`aLLOC3{ko;@YqYgZb=6AkR<6jZ zT{C{IYU;EK@P4`hbv!mlsF*BW?>t8Fz3Qs9d7^j!E3;$H@53fE0PgxBt=H3a9)B6q3WNT%D=|}=Y=~UgVQZ0A5qVQtZM0!6B!L$d& zAL|-f8SWpc1@5Ti?uyWg_L1_!VBuPEgEB>p=L(MF6PUtj!!PLmmZ7XB(!Y4T{8Eg> zSAPa$Gs|UKr5PGhs$Oq2vX|}cZqe_ZS7=oBat9P=){wosX-LMx+?>0yGp2vB-4kt< zwb6!LW!!9+S4y>}f&O|%Qz}5CdrP4(@fN0E?|nL_1>ZPh;}BdZ9j&_tF~VYntDrPO|_s{r8uw=S#9rbD9AO)*PcZjXs!B>zODENAl|$ex4$0{r&Gb)_sQ{YU~Mkw1i{8pHU z!EC?!9SRLltrw`$gO7d&s=~i~0GEv40SOe*bTGN)(DR9ST3T0By{@c!651x0sow!C z3fsQi4Wv4Ff!VH?-`@cme}5$;!-R9gqVzU0;ta76457GtvEH-5hBV6LYfT!>Nh7}^ ziOopXi^(R_5j&b={*WAqB)4rMR6GWVF%Hl@6D=GkGZ(6&Y6RWi=9mwo;s0PL{WV`{|S= z02j&|FAo|?L6Z9y+VpsH9%+df=;6LXxdgG`r(Kt<=u<}wF}0Z0t1?ZN;g3f6_I*d&&Td9^t@R_rDcT7+kexq-Nyzi57Lwn{8{5{h@^ zr@aLi*C5&{xKNc))bp34qDsx!ANA&_PqU?gT9-@xTfW^ye=3BiD&=z9HWA4`8Dn0{ ztx*JQfBx#;Y|GO{qmjN;N;4f2YsWQ7FYVXsN#RKy$%I>c3T`(*k^o7TOEck?Oo-Mv zTfQHoPntyUpnlRz4WBfL-i~)@+Kij%gC+)7G;zrlO_yO;yu|4B-M)>(uHeqK0W~TU zB)+dg%Fc*ve@{CAxzR3n-qeN?31C84_{kBG0z0{PXFzifrH5^c4Zz)rpY~#80;nh* z!rq|fHljIa{zP_TVrpY&%{D;*!43grwXE6fpn?!rHG(>o))4GB%^ia83!ate;O^3sbCqs9{|;FtjfUDPBzp0_|FFB`{XC_+-{Rlr z`1b(+f4+df&$sb!8~&?ze`{2c)#zi=p3tgf8O^psAgCE{}%s$`>pqBkMiWZJKhgN zD8BdW4L#iJ(-&>;*F7kucfCh{-|f>E-+Hfaq0%3QP-y_LXZ!vGEVN$lhanyA_gD0d z3fbMEFP`^T^h$;FSM++{7b7;DGWJg<(f0W(^~E%uuxY+*ocFWx7erRUg}-#(=ZM0?^u$1qkRfTAMb|$m#P2Qr&>A78}Cy-qrKTa zg?-nX@2}|9;_fDmTY1VK+usjH^x+SXmwL{RX?PzqXGM(t{HX9oWKXPSTTY;DK4X%4 zJ;4dHT(;$$!7%Ua=U;y>_c7`D10emSPKe5k)et*meMTs4J zd?ZwCpWZ3Ox#7L#4=V#k)!`pg(g*zw(sCs z?~b8z+YWVXw&(pmZ;fgGBbIubnV@9OOxKKa(@d?q@0sMAOSbYOL7P9B-?S5ghNitM z#;=P06%(9FKe?9059j|Y#<$W)e{2*lv*PLMn)x96;nC&>d(Dg;77Donr|3K#jjU}6 zyxh2UkiSh-rM~Z9F}Oopd9L-Tq2w4*x(v z40LKVQ0NGBLbwP{+J=7xN0#Qd-5swD^aFDc-0H)0Y!7<>PezR zGJId2F**U2RvtVv*d{Itgt^^MI$g*Dq|iSPGq=SaDAbEAAOI zk%pf*(as2jvn2C-a9J0l8_}5bapUBUHkfjpmJ%Ahg6pjGumS$8I%3xk^rqiMKUdPv zO+>t`B^*tzf8tl$01zRpn^IZlHjOpRQD(5YC3#S$RFOP3r&<Y8f55z6Pl-^IVx7!)YBE*GhJ*V!_o^YYVzNs8e;8#AAhQA@W{PZ4>$fyYT*q{m z^4btrN51y3nt5+`-QM*+>(hej&=g65^}Z#P6-4MNyP)a5L7 zKi&|2KTF+UpHO&`fO~|vjiv7GKH(R#)Sc{;RRvJEsnlx;G4-4F(LOLG_&GOb$B)=G z9R4kIe=pQA{3#3bzqbFZ_j&ts{yF(P?LRXdom}pVV=nXCGzYX44l-V;-H8Tkl8T{Z zy$MC1VRYQ#9^>*x1&4L@_0uIDI1EHfMFw}UshcgALj2YWF$lnS>B!#}O#(@vdAi*O zan#=Po&6W}`l!*MiIU!pUdEdJqoMn{-HHcUf6kAEBRcV4w_9w^0(8vcPhhXa0^LGqyK|Mpu*&@kp&?S+TjvGCEWYQUn{ ze`U{h$$#!rS@RC*utwwMqNCc|CT0n3T}5-3Q0BJ_vor#xVSalx0(WC8?PeZ2X!I{v zUpbMrqkJ30=Ym_4>W-ij>o>!0xTZF@yJ&ql_)I@vcpe~GuIY!VLw=AQ4<<9+kKr=%XRL=q+1MkCZB z?|u8+@Ly}jwn3DPxKuma92@H3K+dHPyWoG13!ve?L}=V??|JQQwQ`-^x(Qh><+S}} z*MC^GZU@=bs!D*bDArzy4ZIU;{LerzJ?zyxqi7sfB3_L z6lu-G30aCOejlD5bVQ=mR5f4xRyk6}7o)C(7 zxjzA3t{21Eo)(4r*TlfgQ7!LkM0RBBmhN+hoLw{uXK!i%g5zJ0A zUH~k1UhICKn*m^_4ONEXKb9G%GgM{72&L}e#cn<&bOVyUm(fap2UfUYoaALxj*=I< zZ>EG6Fwni&{Vg{_rMO*&3zQ}=c0Xb_P3$LVytGEQ5}4galUy%$ze`}df4pI_mr-Nj zrEZKDyBCT6Bg0l z?x-+rKG~Bn1b0+w28Qk>E6l%7eENMdCmT?0(^MO|a|2+{KlzKSzzly8_?7nM`%I~D5PvBES9-7`k$T7?-eC}hCSew6bP8feeT5H7Ht zUw7S{%1gjqILPuh)=hvY{ECNFrK6v@z`8hLQXOrInBl=~6o*rmz{8O!W1Rb7l^u+| z-U;h-jfgA$-G}?ee=r((x!4i|zFbTwzq#_I2wpuFoOhEIJzw znVz%WyS!giL6l|`%Vl-EVX&afuyh2^h5z_>!vDCGs3tzdpq%qv?x*gY(OE*@<=km2 zZ)>rcI&a=nb@e;H2m{kuLhV+PdVQAk?65vhYYVR)iVrkAe@jWNbw$TH(UlN9*bC7Z zj*)a+xox!7O3h(=MxNWZpK1`Q$&$S~GsUESZlod6`DsSoIpBl9j=9}R_JNJZ< z@1h>K(7H_e{@FfZjr3@ProioXzxAn+(s9)nCcVABj&94$x!Plm4H0FO{h_i zWk>ocfAObNr8cuw(NslKJ4*BF6B8K2y*{bnp)y@8D*(b(epbl9Tc48U@@V?#oUcb+#I zbeuODz@;1cQ8LAJssMOI0*8v$dRuLwKU8Wfe=l0No`(fe%#F^^Fm^^`WkdNh;B;DU zDLqo2(vzvbwY^fHQga3e07Qxr0c>vbk`$pb1_;yHxx--1f|(rzAIEUihseCslLMsaG-Ysn&b4RZ#EFS}M1 ze`f5N{OuE~kix+}GDyYFc)}`1$-LjZC>p}(oaeo5rNTJpkIePDd&0WbaLpOTl*Dt-bNkp}$BWnf{C1ko zY9L#EGo7YW*Et#_Nt)Nt?y2_0X^5&Cf6kYh>ok7JBrwpN86LLgjN2YG9OBiY{D1w= z|0}CODd~hI4(W_@H3B%p`OyMquW7lcH9>COB6TeLM8Q6Aj%~5|1f20CqqosC%TLW6 z52lQvkOJcgQo5&PmE!~lluewv)9*+=@`cB6d__G;gerQ$_6eA_y)78`(e{^^e{!sz z{pO-!{7~=0y!9WMQ_#|Qkc|8}t5C7|vnAfKd|;J$W(^O#ZXDs=8r02}OHLJjCviJn|_Jx1p6$GHjdWlaQ2v~U&QPba79U}r5nU!NYwo$_-5i)4j|As{Ggv%8z`@4QKUhO_u?Nek+E_{ZjGN0CDgt&}!5fj`)IFv(cx zSgM)-pC3T!+7=J*ZT^7pI)gIPb#w<-@Lv9Y+L% z%}=CGx4rGP_ss6De`&uE1bgSXe|8HqQ9KsCdwe+FK-Zpe<#^2g6YkDheqypY1~ z-uaqSElid*v^RK|{Vnx?r{FkGJ*V+#jlbkdP$2|grk`a~^riEBlN9s7q1>Tx9p`31 zaE<1_gpHMBWdFWYWt4!h_XrEA$dFHq84M5%s;8ajIO2sDf2+pTg#Wys#&(f!N=0gY zo$5~iUh1^c<-rS!x4kCgZ)FN2OrLqTDwmD^petzcT`8%6buaan4&XK8K8{T>V5!^O zwRAaibziQgh8)olCp%QWgNya2_<|wi%iKnc3!UdoihI{oc9B$9#%0%4rLbVkLSfEU zS>#VqvHyk|N>+82UrQtt+FEPUUzgJ3ov&&ve<~-nfE-N?*HLN!4B#Urvn+I#0hf5N zWJ00b?8$CHe_E+a=VkvBIjBq9_8}4JVi6JX{<)8rlJx;Ef7j*Xj}-h_5MNfxx$TD# zD2u8xOugea?==@Yv+f4L^BTFcDRwrc&VyPQ4RK@4YbZjmxnG<{E(@VGvyj{;!9%8#@*_$#$!c@)Q@E@Lw*!J~AfUj=^+I_xPZcZ0!rj)w$f%6C6@cy*_- z&ssx)nT9w7(sUXrjY1DUrfdk`4L5kz?mA7U z5qR)!1XDyJhhMTG;4BZO(cqY6!8E#I;o-vTIkI_8?iQ%O+B_S@kYogArRJ1S_Ghxb zKiU0c{S*tU*0o8I|4dx6cTeU0TUeX$jQ@8}fsdjIfG((*dJv;#YcgWf^gjDOzl%$4 zf1HBH5Kd3A);_Uj15v}?PFhgymv7dtFA%aB6(mC8#EM&~k zEb-e*TlWdKKdh>%=d3iGV%kfqF+vN~e`HeZ5P7MkLShyjt*ll5y8`n)VWoyj;S(hKnH*6d-!=>ZfeT^*3L^xT`Z1%ZVsi za=d=RM0X==qNa}=KjI>%#>O1^xj0Qle{$##frG^$);93+OSL{hu_p#2`&qrsf6Kj1 zL~nDux6#vkn>^CnsMy=ce7LJ#luS+%GhRQj%NKTdU#~gDTvn4Ow7+zovewn)tGn>a z|9h;BTbjk&|4ke{SjW#BR{o7#{cCZYWmhy?|2MGeU>&1w=*Gse{;7nxs+Q%{Pr5&V zg`M}qE?=g>6c|+WwfS}45;7amf5S}Em<{ zPIu%lJa@F=kiO|!L?g8|jKK1?`1lOQ>PFzEZKUR`tO5#YE7p)?a-Z9c6M|Ks&rLda zlypncOd@23faYW?Dbm=wWaWcw+yu#4GXl|Be}zaUh6E^h85Jc!K>ncLe{s|0GT8F{ z?d5Wzlo(|DF^qR%3kQHGbN)4#M?#-4jNmV`n4^&(@F2Jf2b zav4@MHT>G9K7JsLBWuVEp=0*7htVJQwUgK_#Im&Is18_LgR(71I+x2W?MIf`N+^*U zRsa@gG@_Rpax(XS*maX$f7DmTw#GZj54)zcMpmUR&fRWF8{<`5=Vk|m^=w7;TF{YD znAk8i_qm>-?L0i7x#^sGC5TOJCG)sJ%`B5*$H6^Rytjy%BIKmJ;V)eQ$ z;56g(1R{1O9burQ7iJhi1XW`O3UIZosS-NU_2ufZ%_naO*?QMWe`CjU?&I~VIEt8} zHL|6R8&ZDo#M{v1Zx;k4)m0bl#tQPYudZeK)DBAV{@>ios8GM!C?h|Sh$u7_`(vcp zdu=;&cz=3K+}MlyoEuatwSLuhA54-{1s#G`{zbE{O%1q7!^L9_NU(Yu_;0Mb+kHYq z|9saB%PZ8qNGGV*e_>V6N@{We3Zlc6*@Z_BayjgnkF`1dGZScVP&^7H-9+_B=*TkN=j8BG<|9U=x?^ICyKs+*Myi~^&$`m^%qtNN7u--oP;@Wsi>f+fu=L@z`(wVAAFU;De{7O=Zr_AJ`g^1`Q}_fJ zX{qao9)X7_lf)2&&~-4rM{;}@8^(9~*x*jq4DO;hxF$k&`qafE5C5)(gI9Ig_FvHR zHsB6xjgA9zrxw>MMrh(2S3z+YNI239p2xS9?4<1qTxB<{kW?-l#hjSnW8N4Y#moX6P>h6F8MT^G-8oFaFl0e>HkJVKs@`t zf7xCx;s1HNL%^3^IV~=5)di&h_EK3?+HBIQ)ba$CzKXg`fn}uxUU;>6(ZQp)`S`$9 z7i0|i`kn3msmyu%_0K|U*fKcqAM~4-_jaG+VaEHge~-ul-Z%R~(a-yRU)%kEzfXr+ zoGh>qe%q%)k-~eokGA{X$9-yx2fgn^i1#x<{YqsK@6$e7>+5vC@Br`EeQmq{6aVe8 z1H(W+KcH~F@R9>*5kEbkxYT-)_;JMI*h|I7WjgZ)2OPaBJ1CgW-yhICy^Fo0fG!67 zzk#H1e-x-*ADf{D1l>{Kk_SOrwgc9`i*0|K{*=K3z_!0bf4ud-rI)wI@t$ek2R z`0-Y^3m(Uh0m~6iL!yP3x1xpzfxi&YVLTXRUbym=!zG^el8xTXCvHGP$j*yDzcm{O zf2n>POxCrMKk2w3q0rD`OHWEgU9o%(!3JSiNw0iI_6FT21@w+(8KPp8dUM?F1%21( zN@9%QVKf3$Fij_ihd4S3SDer(byB=z9ipj}V-x>Fl;j8E3PyxA70pC8Sb9E2Y$la7 z9s~kJu<}cJJH;`@oZy%kT-p}<8TWZle>9NAik`N8=D!LL)R-~iZWH=T3>a(=628qF z9<1P~^e|?N%)32U2_s+nF@x+{=RrJa;uf~0C+2dw#pH{}Vw>iDb(T9h|lB^^SrQ3pFgM?*fFM`VM`Cp4eL5#ZcZvqX;HY;3qqD;b#U!AQupD!m2KEttHVz z4_0WRZC>H!PN@0ZC$MeR^0byo#m}142uWvw0%f!8`$WJws%e?JJ+=e4!<7yQ=mJC3 z0v-*a6WslZT0}yizv6=KyfjC}^d=srG}i>UW|XHip{d&oDF7PJ`@8|-e-6;=(~++m zW~_&Ez8)?FaG*Qa3*ty`AlUxGXcU$+tlAuszFL+Jg}9oJ2RD`#A3Slf5Io$S4|y4I z6wT{(Cr^L4IzIz|=uEOy41z2Sc8QchKHAya10WUP_H4ZmT;RdpyiW)L*($dp2P)8* zhxd!*(7Pd40S!30Ixls)f4nO$#I86p_7N=-W)gdt2d`(zCU8GT;8oYAEX95O}w1O zX_hU#1U41DFd?w5^q_R$Zn^n#nGm|Ql~cJpMFcK>(Te8V>IjIKf3Eja79PcE!p!Bd zD-M5oz3w_tGcaDhTsl}ievs~I+r8jgjl?UItg7~jg{8+DJl7o&3F+Giq4onyI&fy>!!wRW1A8Jm+1s9?XHy4U9xo6!| z7uMDLZ`W7?p@>W=fA0*?$RnO&xx70-v3gaSQ?7I|I>4)QF@=bAq&iRL;H|bY*?Ug+ z<8WW(e095ld21r0E4^jL{Zj$q|Df2-E`8VSk4nejH;XpCR_ zXC3lzWIP@ot$lC82Scp#R5Dd!uutM8Zb-*OqHCY}Cc5cqm000)Md;BZ(`uc5SguvV z<_&i&X2Q6n5(Zp{c^p&=L#FUV6CZbumD{sCK)cXLqA zDFN=Ke-lhn0?K)V$So|a$=1*s&q!= zi+i-FDNyIBKekblt3f1NtyU@pwYr?xuv38^_ToO5PI5!JRJ&{JXm^d;fY_H%1Ogd< zmhVILvO;-n+%nN^kj@Xfnw%6Ieabte@!DfLX+kDg=U$SgGHPqf9Ur)WKW1VPB-B`l z#RGdclCMXp!kRZ#Lgxp{9LYo^7@Pet6p~7;3HO=B1l`?8^zP{nY@BZn5E}g!01cxZ z-^s?&#L*o(zEziW?le>GbGbP;RE^1hmUX0!{DdZcPQ295)dR=k9-bUBKk7EyUb_>x zOhQcd`ouK`B5*@Gp$q*W7OHweLy#JJ4=ZCZ1|~%=S9)o<Lx$$7#LE2oZe%9 zx@UE%PcX>cOVT-e1&E7hFH5OA zdcW6ky)!&89QlEJ3LPgdp+~BJ8`9H6=S+_05jY^%6A3=Ovqq+5ik4B>l{sNb>$Dnv zW)#?ab3j?&6B|W$y5sejVQBNmePM>sm}Xx!OQdgCn)x}X<`ECPRKQy^JnIpPb1 z3YM+Op%ua=y@`QsF!C2&t(v7?v_s-qfGf4k4e1CTQ^k^y-Gc7B*a+$OkitbW zUoNlV=A#7Z>Q}6OW$0)$H~>2_5Yr`L%%&M!K^_Ki>`kb8$nwq(sI6`4Mbx}+cqz5d z`Cd$|%Lf1y%No3lrU~5j#U;s0C?2=G_XiY?4c?Cj>fVpzs166hy|MOrtBsnawjD}0 zOOHp6Ual&0cOx&Qb9wiFS**YNbVex8oRj5pgaXew4iuq5$MoY&M+(iFNgcKOp>+{~YX`+~-!7w0_ z+6&GRfqSQQhe0h2ZW!^;w){y5-I!CJmx*lKnSW$z5_Qh%^|MkvXS$v<^}1cGraLmO zK;B5{dZ@*9WV*qB5-WqoY=B1qts6ACaV^xPvjfxS89kXWI4Qx!#Z+2GNQzt7ol$(k zKaWKJP*lQ$w9vyi4j7kRf{z3xswL_?5SiHEnQt+Qr7#4B$m`$oqjh(cY_&=%?!a-; zq&YEE8fm_b&W7y9p!-_2F{kR_m)bh$YEGr&oeAAd$iXQLkck44F6-&V%Wi$cz_wlS#c8aVZj zW}u05oEGVuR1UnYB~FhPVc74twF2@s(2BZCkpEhvPBZ8d$WtjGN`wt0G63K+y2AY1)0nAYxx^g~;JZ{X)}t&4{4hql~_?}wcaZNJ@t z#7yE-ql<1JMe%Hwms7|GF`D?5w-g%5@Da<6vbg2wBu}R_)k2a<(R-6M5=Jp+ZqX*)EC|NkBc9gq3c$=mp@JOjc zz=xWD@u^9rZslv@sS{-v^5cBy;3i)vdTUXyZ;kAh&2d7`K`5bhTLk&$ubg{lxl~P` z6EM8dlz{7Q0xw&Mkcle)>`oV>F@ip@1i+h6CkiBc=#b-Hee;>JXeEq$ok8E;B63B% zNskxP_jM9DjN8qC^RL|kf=WlCu3TLcNmp7J91R4}IehOCEr1 z%1Dh|j4{?n;Iq+d!5(1lG=!f_2$$=?|yA^Aj zye)4T+B3klhD23a!{o-YW6Ua$cZ!pLeeSC<0R_*(!nRyi=g*0(5Y8!AsC=Oo!S{m^ zmkk4=Je~3g1N!+uLF6%zjOG5r^gi(tdY^%baHlNoONzo!Wu2_y)aA>s#Y&Z0 zAtfO%NOZwDWKHrIMWA!10Z;zumXKoowosteY8i3?)@w9kVW|f#@@=MB$t$sd2i&vqMU#iLi8P-Zm=TBehzsOuNigXSK{y+y|S7 zOjgjzr;CSBYQ=NuE^1QXla3*Ooxyu`y`G4UF0?`dn9D7fF3RBYTR6>25hs3HJ1h@E zMNluSMM;(qk^s^o!MZ63cw$g7H2v6QFcJZINR~?%eW&I!h?q_sGV+k}fyZz; zH-n7fvT<=h!)S^>|9JpdKlt+W83+?+{f4!0!gp=A7ytVM1 z&eSW1WoCCDI}?+K#7`U2$*Ay>@)GV`F?y$kNX3rP%k8e?c-lKN*LH`7Nkv1);lL%( zfxy=w)t$f6piDO?!vIV`v%dz>e*opMRpsPXP|$3WZh+Y%KP8?E_NX-SGre2DH6!Xq ze&$6YouVSOda5rm2V&E}L8HtV3rU3*Cj+`zO7u8JoD9uoQbC5qAVUO(n29|q%cTp> zU-dcc<#v~qfy#IHyUsJmb6SoE1JlN{VP&FcdMLN4b%DYTYL?7(4-!s4f3TcrwrT8d z(~S2iH#`Fc1xtpUOXGC+GH`{St5KG!6{`eCAcKKUAf8anwAW2vZg*2Z_hMnS8T)PG zrJM<1ztSp%FSnP=1pdT)Zp54h;2$<=iXTy?xpP!q9I_{hF;jx^dd zBOa_}W&kXu&9vF>e~3DHF?`b2>+!2*yBjwe9!${+u^qB{;q<*}A9XGU`E6$hmw@3j^0t$thE+HxOxjs}$` z6w$!sf{%LLnFuejDqIG@@4-DzQgIG7u*wb!^e`m8j4Lf)f1bkXpq!auL$$s^;n?y1 zmerieAiu3Rd%&?}F=2v+pnNe%)IkhlKU|&~l>PD_4S;HNq)G?hW_{<)<>mFs*~izX zC&$;XFTcBf|LNk5L-Q5xZnDoH0zm&1{pV2`{DBR`s(QW zH$P&CQmWmM71wu&LjlU%hu|w9+d)j1aV_vk5X4xT1Vq_;w#%NSMJ5T*;qr z8z8HN1UtALN51dxZ!eedzrDR_2d%;8(8Z)Yk1>kx5+2~~*kaW4I_q*E3a zfH*+19}bI;d$Au5dr`lGibe3)!qtl81x9@G#SL$Ly}0kg9;&F_1)~T^*Z1HrS#jkV zs58YvNjTJxrzsKcs(NYPPkL#etVS%a$&qr4KtfpT4-no^PzeStm0rcm2me)O7MGKU z+o78qtU&0n7@+G22HYx`j|t}x0Ca-WFEmpGJ++lGd37UIEP42*Etd}x0y2Lh@_pee z2VyGJW>J1y`_lTt<7;gdALr2|28iK2bI5A*P|MlF?i!rP zgwqu=7(_MZXwUr6YAHxS%mr^X@E485euG7VP|lIKMj68k%6#>@RUXP-1nWVo{`~CV z)d(tC!;IysX@bk7oFd}mDbs(W;hGf)(hUa@qHUWGQRads;?!O+9OMJM#HXt$8KzIk zDsOcnuT(5t*~zS}YdkGvR{cXvHhhgxK@G=q8a$AuEjViEI6(XmO>Wx@wUWxED`m8m z+x){d67+4)@ysC=v1&|pAYy2vTY20Kr>^qJNm_h^bai4=C^;NzkWYU&W2!k7fD5Wg z)ucild^M%;M2%ZuEsR3kOLoSovF7RPi9IY&f$hs)GCXMJ~IDO;<9 z74H0%4{oU+^;qA08dQJ$kqQ}WOx25v)&vkQ1DX9Y5QAPoyrMrJl(pK#`R4SthN$IX zR{QS2I_i*z?0&+gt`nq_g`Dqj#)@L5eC1-`1xHOcg9dp%2yWR_A5Ma*oWm(&|73V; z1tEY4OaI9dD;dl)^pC90LX&cp3U423g{O=aG~BhX4yU(Or)7Vu#TDKLtawm;$y@gs zR=3tEC~P>TOme00@+Vg0=b{!nc*RH4)t+BPc4L9-Sb+OdK2~6b%>&X)KzJYZ zi^EbZg+o&HH zn5K6TlCfjF-@t>6k`VK*qg6c8C?K5(RaHV77|N^gj?N$Om&>iua=A5^TnPG*9jOpM zqDOvY$bH6iPCCCHfZpF+?6ybVTwJ33BMMea@V4AB)hj^(6|3oH?t*x67oMoSMD}e2 z%7TIIg`opBQQWXx=AA`1fKkl4V8rc3vP$5L6?T7?kD*3`+G^9(8bwKvnA28daVV-} z!9+|;&3)TOun#|=5&E9j9ElZGQ8eOxqp81Xz-7aLOH`o_3_BmIl&cED+J);~vToOt z^{CYzR#(_9?qPMYHw&QyVj$^SW@I6UewM;WBaO5ovlXdw(uGOmW$++W@S|S!SbmSd zvo(J%4Z#DcI0!n{+1?z4i77sv6k0n0vz%gcCq{Flp<7h_l6%zxh&t-~lMdHZ7Y~yF z`SWLcQany)QhUsGCh`3!N-_uaZb0Efd#J%6vfi%Ftg3c%pVy{RggZ!tz;ZHW8Ov#H zb{hq^;8YJ_@uD`U-9=eG7&A{K%HV%!6TvpRs0HaA6IXZ9zAv>w64tWxj^)uE z%S5DX>KBUj!w}VO2U#s)QGUy&4yb%gTA~Pm-IE#cekMPBegM+}MndzwKLDz6)Dx~0 zGSOUH8ccCcS-hMS4~HWkw7NR5;;)`Cpo1EKeSwr(^keO)hFk?rs;$tuabg`AK(l{% z#u@ebnPx&B=Ke^$jz&Xw?)&h_DG+3O6JW-L!OIKd83-_9#*pBs?=L!1jXvtl`@MF* zlia#zq+U;M-7&YY;$%LfV*&tt^S#x2C8VV7QsKf{d zQQ)!)u6DpW>3t42JJ=2_65jot<;#Ed)v94+)KH#V(wEgBj#YV}(^TUNWle0e>9qAZ zIhwLT4mWAYuOe#0Xqx4;cC)}z;7%LmUhN)ycBw=z`GOBHY=B5xI^s*qZ0qk*9d`?r z*Ob~H6KK?%U(gf>gFzpyjWmhOI=)vxiC9nN-5$vR#z5IEObAd;Zq+_;FJymsTNiyX z%RLJWo3&pKxS<*pJJ_+FYkqm!{awgoL|#ObJY z7drH}Hr&DAwb9b7;JXFE+1Y>0fA$X-Yw9keX^?f7+(YMbem9wrZ=OXopZUE9a7^Yo zgDk%g+YDO2PuX4i2W#Hkk4B3o03qpyi31_Oig zUn~UrXU#zpHUW*^2y-a-AH|S2!UV%EoQ3asy1!gL3q=i?u0p5&8)|>if#r%=77Qlr z&yrcl0!6WSZ{@}OyC_E_5LPT`WLe4$!(?RTyeV@+H!)i?PdAqaz?ts{(-F(9oMpwOpw)hZM! z59AxAsM+{d#qg4?GI)Ow79b6bcy{?!$Xsf0MQhbv>;$k&KTIQ$V-SYT`)RDAc}@4z z_`M1#mvI^;8 z!1DH3wTRArbd0Qsvs_MRbnJgRa0yD44)^*0kpH_ppcDQp07PoO4|dAdeugP!WP*vE zq7ui92$?Y#3%iV-!VT^Yj(3jJ;2#b5w3Vk_=brD+@L_JzavIj6+#!u04jQMeD5SPL z?aUtwQOyz83Os-I$1Qn{KBGs(Fn)56{ihqoX)u@6u#wXcJ7{LrJvM9>j{$J&k9+qR zpk`(_?lFCMI~|OmBJcN>{7E(IcrlGeNjhaG$sdC_3a|17H!t}un|r^ve#_?YE8{;i z3TDCY0CRI|8-K&nhxR6#)i2#s`j!s(Nle4{fXlRceV~7j3~#mhy%bz6x&y<>MK(co zs<*UFJR`jKwqFUjB&<^C+m2PQG6Cfp{EUN%FnFI&hj6#Bc_bau7%U0w&{NY`GuC=V zAAK{X7qdPMn4b>eSH}&Q(ytGUXQxwXi;74N#RB}flwp44&cb3wPXQ)mW3T+YyOmc} z0aBjIaPEKPWGiO%dQvqepgzj{twfF{G~}`I8v$OE9H%v73CqT*;ZbJIh|{+wqQkVX zd`hqb#{(5kGob5mZ&oVc-yQBrG&1xCm5c;0kIN;fNP>oB{wiHA6ZMgB?MVI+EthQ8 z0`MKzR1Jn>x`{acv0dqW%fbZ8pL`WjibD?RXkdT##Bg6Aw8ycT9CzJ{r)fhfN@-nn z!(L5@HD9LF78%MRT5tLJ^TWy|z0duBeAD>PZmSQj(LR6vrtT1(s>uWAJJv-g z-Lc&`q<|&kCScV?T70PjH^E?;I+#h{Hklzh$X>=ubbzQDVwngm8=8XXLCYN$Mfu0r zU-W-QeR|~2yYr$u2^iGW9n+)kk)runDjCqZ#AE5qeJ-q!6NQUDN}>=2Dq-Tj(77M? zG7x{zG5^)>3nvf~9q-s5m4QGeoHUmpy*&kk=(*OP^|vwZtv~0k1193toRi_L>idBC zZ+U?94T-kNy~55Z}U*Y4HC?YXkr2f7H6Q z6$|^vl_%xnAMu^5Yj$M78g0)#+VXu^noieR6pJ2;nR^5)+KNzH4EB+9PZboxN8)`{ z1%8B6$bdE88h#fp!Qy~!&08GbU1(*ONGe~;F1zm0OW6uu#8FD_Uff6 zo2eon)a@ekdUvxvWl7Ga-b0Mmjqz(ys29v=A?OWAH@ur+G5q^HpHu|@Jezqp!xd%o zC@&X3SyTM&tg`rUR#p6PX71Gb9jp(P15|t2>D~UqGquzqu8JSjsiWSz)7YH{QUXZ6N>5^6bX>$ zxGEWpvRo55yP$tb4=wlZr!fG2T8) zHTSSOH$u0VIV>HC7H%Cq{nLLy0OkIQDLUokqC)T4Gl#2F;L>;{!PPA}6 ze2@lls582E2wlwjefq!;MqbY`RvGQw89-aLM1Ll+%&0PJb{u{g*x_2r&;Buc7V)z# z>TiO@SweHLqA^B+(5*jaZbC@W3Ow^~W^U4mI6g2K_ZL$H{wDU08+ijE0XCwH@-!=r zHvq=}UBLb>r-i6h-wS^XAnBP#bQ6t>cev-imt`8mAqfAfsfAR*nYwzqLYa%r-M z^8NpE8p&f#$3)fw@W-s^{xObnQ2Uh!>@7Dp%(n8Lwjc!~yJy#;u~r0GF1Mcb2F-uI zZvOf?>{lisR$)8q3fpTHAn`iu}#1DsC zx%9{(QE2)d<{Nh{;S@_~x8=Yfk#W~?h@(|8@EYd0;@T98Z%D!YZi#DM`YP@a)-a6D zA~3DgR%rNHtcAvnMn|nIoouZFHJRF+n+XRJOx1gd1hv#%@BE%R4|tg{cQ% zkZ>d2-6yMTr!{|JZ0!Fy)5nFGv6G79!Dx8*w(Rd{JfoEnh#D=ts=Tq%906?KQ7K5I zeLs`uh3fjQpHSabxxUxy>N~Rz5c5Y45IuYQ?HLC$apa=T)L0)4IKhVsXHl243ddR5 zj@hE|nhm@HJ&t9z*6{;{gp$NffXz? z!p~`9fED^bM{e~_OKh}u-J0tSyXVm(6w>?E8UoowdB1@uXaBz<%Igh8$#(QUWN2sN zTlxFxAQ-bLU^DYoENy`(+I4#WP)l6~-v5xA6?6n6aTTrEUHb`a$Rd*#%po@cbh+%Xy6i>xv(;-M)n;Gzw1XmW1XHMFuR=n z?F2A^I!y}jiT~f!d$Z>rxUn#>`^ui1x4VE}@1O~@a6Y?hmcciiBtMN3mI25gx9VZj zw=rCAsnI4v&u2tXUz7EGCdFumtaH1|JQr>7oLYaaR&z*0;zXa!>|1KcvOG$V5U2by zX9r#mX4VQy9Xt{J1l|bc$LmQ7uKgyrU}|_T_O3%??>aUX@n8VUHT3aK8fW020Nocl z>e36h_c6;kB&m0yb{{+ZpVlsM`p7QvFMF3THi^Pz?PO-{5W=ctXbYfN7k?I z|J;ADJlsENc>ok%$>bJ294Q`^tog5hY5vdP+@l!lLi8P+;#4zzCYk>GN$`mQ??=nVFu8{tOPuaNR_T|zl)+@i8@VC*3HE_T-{{3d^@AyUX`L% z26#WAp#+1R_}g@0$eR`hW`zP(?Ew&($Z&rt6SLkziv3l%o^(hIm`~reQQMA~FsWx4N;;*)XRCja zhQyi!@y3RyV@#)tfDGbw)N$w~+&Rwx&hkaqJ@!G8atVKlcZOEi2VD&0U+vv`-R053 zNlS^6Tr%ADejMbtt-Hb8ZPTzf>Nk(&zJ95OOH%u|;WQn1j$nh_Jtd9HPWAX8%A72x zaoV_4&~&e$3LKG9-#-N)cK&}3|KESEGAXBRQ+N!Zf%!cqoEf!$8_kzlnJ;08>9G$( z+%af=xrW@Q{#kF-hX+I<=m#G$Ox*J~vti79q?~(*;mwu76k=Y)9WZOFk8x zB_g^^2A9<(0wV#Fm-!_E9e>%$%zf|fxqW;rq8p8Vt?sU_uDZ$xcedhvy_#6Iw@-Tr zbbu(j6#8z3?Fzagi;Avjl&bgSApG)hq^_uEM=~X(8d{LpTUN32OY@@|1?q|sS zf+obd^!2Rn-D-_TZO5|gu6ilO=!X>b@h=~BTSQ~c<`DUo)s=UfDu3ylPL+Qy!xNWj z9Npc~FJqs7N4S7{d;PsV1Zn#2ImN(=(AoJOx+gtpF$$z$hyKj}%FS_RHx>+8YX(MbCh#c~DsGwyVzld7IuIagpxIsK>Ngl4(>wWyo$L;NZ-T@O2=Qu}8Cu-0h)P}hb zSOqQOXKdczymB9h!@qpgRWG8_m&-a9_ZkmvAEWnsB zL+c~DCHwfedb<2tF+Qg4`f6N_xJK z7^!L|4H5f*+=r7yq69ra1-?lIDkp2y1quT}(0>+cy22&WaHm-t&Ubdcd##qwc^caN zx~I2tP10gy{K5!I!kwq#dL>_erjjo$8-YHn3I~bK7nj6ZNHQqIg4wcGDs=@M%@tIx2k+hzgC0z!Is8YJ_*GV&3IX-AebG0X9_ zOcqoiR=HG{3FN=f^Y`V2EcluW=JkoKAPrO>Ntf+--p;{Ca}Y+5*P;b&m`)dG0;N* zH|822{!_(fEm=l-2%`FPN0!mCL4QNL@&p}>>wh!(+odA^DRW^sF0c~l#6!N4qIY6@ zH;JXgb+I(;Kay`-8T|1Qcx2$f8g-iH7RV5rBn$z@Sa|$uDi(QRqRbA#*@zj399^R$ z@l&&&^^Fu%#m0>jZ}36B0PNeU)(VkB)&LfjGc6n1FI3IGsaYDQYW9lk0DpcwQjJsYzAq20Ny8cqs0;Shs*duuef~?AZk5^w>ahN~E?>+EcAo3Vtx-+8*d zoiv+H08mzZ%n}{$`%+fArMRb*k&!W& z0>fI3OZJvrryf-KkPau5&(1Y>op9#j(gUSYDkEWJe*^u?5r6z?w3?$vSE%WV1Y5Lk zym(o)bJ(jTC*43J-Q~Zt$5t*t;OGFLlXlnmz_k@a+fAhQiOoG3Woi$TOkLwM2O_?X zOw2tkD+%vpF0iNJUZWP2$x3Guus%BHCe~o010a z5;BuX$iAFO*neDEViNp^nS|`mF$r0TN$?FO!QaRvWMw8H`*J2>V@qQ!-^ky#BttC4 zH;d8{l1~b75f56bt6M%JU2R6*{xP<})b;=OxdxMIc!-9-at~j?J-CZ%#m7$@r1{tL zFl6}#QDj3rG`~b98RTsCkU)}(S`$ZpY;c)mvsxw@FF^E~V>{3Y+syVIK$`6nN|JWP zZ#}X4R@3V8=0&1;k!W7@Cz}^-Xdc$%Gi`*5fQPIH(BnSx&Z^aUSK4R~`B9e>F9K@; zfVY1y0-FJUKC6!B>FRh(Ika$YNVEfR9X(P@7JUKknMpTHR?oH@lWzT{55}n*s@<4q z#sL09drjh|?j&`SsbyRaYF8lH2F5!uuDnRAkg^{y04HU@FNqv!b)c>s9cdqr z5|W@KhIFhT3WJCpA{ZeLyd^hNmDa|NHS-rgWef0s)ghnrWcu~*nmEEuC%x;NVY%&Lr9A-E) zO*SlRRP%pFUx0FWVDX{^U!W^+$B&RR?Fe~SafD13rQ2xU{hz&yZhXCs{({@+#((*J z^lY(z@jhCkM@_J~Hfd^7QczV7DWKTz!%+MEJ@CjxG?4|WdS zXl0j8_mtzpRuEl6uHex?*`_bb_Iwo|a6e?qe0))DJt!Lefc@azc(eWB-#ZUvd&t7@ zA**w&68U;IOsMzQnuEpzxiT>s4(o$|;jkFl?TK@-uoWoO7n@!asAuj}H|t_ym_?vm zsj&LFj^cD>(ghk4M!mcxoBp1RZ@|e6MZOObb^z1?o*)dnLfAEF zs(vL0-pZm?SO^YA&gIq6TDUy4YlT)g5vrleAv+O9FBXNx~l z7_gdUK?IkXN!f|0N6)&B{a zd=Z!BfnF~jgqW)*v(!X?0ry$Y6Cykm3xkS=0Jk>KAOVud0Fp##he$gQqxi(`My({y zkpF~csApjnE~7}o)QZtVgMk6Omd1<2`7CNQoDE+$8ctMt)8k;uZk7d;s164Gr4)c0 zwW{9g4;EFIq1rGxp|mrtAcgJiczZjp=x-_{slF)P-saLP zq4ENYj37#0QP>HBjTew1KT*oX(d(vo*ilN^aw(L5$rmNj|oTn?h?Idv3ZTR$( zd&aVnY4iB4S|$bnBk2G*lmjRm4)E;`ELIv=)%ikcXx^iTOc2Xt-2tF8I)Jd`0Dy*r zjFfYUAM>-%yG8F;K96r$6ntXcS9q|<1DAYl-0W)aUYqZZ^>-i4cNhA*N9MZ|y}aMe zcUQZ;`v~#V@9ypnB!r1f{(j*kDZ35gdDgw$T{(v`eZs*ED?a+9eRQ_Fa(;G~<~9N{ zfBoz_S=eYe88{vP_WOPo_MU&+#n2vr5@ca-?>l>c<_?}ch1B@;t$cbep7vzice|tB z;Ahu4+#Nx%v-8XdfBzi?ulVcEo(d~Wm9^tGdKyC;UMG~->^ieIk(Q^yv_1hCxr1F2 zw*{~g@Lscv;?=YswAd~BeeFl!VzADXf8f`^6tzJ0qxu!1JEAJa#B_%x>rumc^kMGD z5uRx#w&U5|37DWiE5-ZZ=x}22k18Yh;@R^)UqtFB3uoyhHq;#7d|MRoyZ3ETEK)2I zy*!umUmE!*TK-1L7{2v&Zcil)aiKd+>sp6GLvTIa)I%FygC|_*J8HLPe^@>Ae?r?% zccZ9euz-8;{de2j@Z0I^6~)%&7Y%;)=LY|myE}YK&zm|B(RaHszWdbDYzIxv(`9|i&=o<5~fe)^36 ze$RjR==WKN|L*c%o({S?!`aGzdHUx&{FmpuCw{->)U^9tADi3>ZBZ3DHW`ggX7_Wx zFBlEkOitLkwU%og`lx)RqB#Q&>Yj}Dc+YseXFT2`9`6y4_lU=P#N$2U@gDJic#n9z zCp_K*9`6CINuCYPf3bk?OYtGa?>#NW#}p4;?dkpIQNMZ6Z@5r)G=)UjdQn-asBiby8%!UY(HYN3!}ob`yuu2;DTD#n9XzXh{S|&)!)^zgzt(X0NqM0 z6^8YSJ>$ha2q=^(0SKxRgYko< z+ofXc30c`~Z*K*yt7|uO0g8h>X)9-~LF;NVo{3$F>!SbShBwSViSihKlXrv+q=XUp za%X#c3uGc}pe}=BNZ9_Q#+~ou{THGS@(OLE|h#+zj~G(T{CziXlN zPy6_<3kS?r!drG3cYg+#q&os%0ST7`JOV>kv%!N%V=*#qgW(0Zo3<<9 zR;1IG(aZ+xaE0Gv~&s?w4PtLC!KzAGPc#d2`=(e$fJMktUFie~`W8Yqx4oU| z=Yxj|f~32nGWEmffPJhqAfEL>iqnluQO@6_5@Xb>Gx8z}8e1VlKvs29pu@H4JwctK zLve^c2*YkHU5-VxE+BLR&^ul2-gkDNcY~FGLmK24i5qqAQI~WOj8=|R3nvoG#JkS! z^O3G37uPe#qpJIw#hI_6!^3Jd@Xf;%beC@&zKk|qu;1rsM9yYOoVw{!```*;o+L^K zm6-4&T{cK$LBQ3*XUQKehd#K96EJiW#j3+gQ$(5SC_V9^kuq6JY2O=7uda~g81>$N zkVx98^D|6|E;bxkj7n`dwrH)kSI(refJ-!o`Ai|3k7PzK+AC+#8_46#((&}e4dun- z=!j!PQL>5Sjli{oCpaKa=$U3MMdU4@^PI1`i;XKCe@Qh$MNCb z(00n{2BHp=QYHQMO0Gpg!$z2A<|NI3j^j6z##8XR;P;Y-#Ch8l(nX+coL1(H`onXl zZju~|Dyks3LzvBuH04c5GfmvwI!g-EddcxmOv|U z?2MY|vyvMGK=L)1ec|5FOd$7vDdFxTdbzW2N6lUb?*D-!$@%;eACi!*p$9u7RWO8i zAuMWOez#xB)z(t(NC9yYCw1Y0e&4=_2QFszN2l{Y_IGT08QGr z5S#&_p|tLz^7i8wOz$Tg42%JxGVWJ7@I^eLGivle@WO+;BkAv3npUoV;zqBtqD;x{ zG}dUOa~1hs3FossJo&nBp%^b!5NZM|8JjZDSYLKX7nzPN7V2(7Mc5$M<)R{;)Dq`B zYu^gl7$E1-iiI5c)Ud!!B~Ygg^5z%hF5SfR0=eky2ZM0b!#^PGh0^<$M-A&*X)u}Q z)@rr69p>fdTZ=4kT-sINyj`gloH-ip5BfVv;rtOohLaL#^Bzh)k9OFlB1v>Eq~Qj za$jQ=~#E#p%LEJq4o}W{s07OYzuFocDR*>3rE3i;Fk`!S`8}F zVIM7Z5i2n;&ZPdU;mdbY(Pu>=JKKZSD0M@g33Rlft??_Dg+l^Te=A8dj*wtPuVjXJ z@ZTVTGrJp_GxAKYgKGcLI)NS;-yN0e4iXrm;Ptw$?J5t{cRlSnb;Pr8c>(80Xo@ar z9=niLxpC)f9o*kU2CsuK#Ntjd`%*@yl^JPmIHejvFJx=W$9f_E$^LmwwEa+VuT8pp zT|MS5TOV}S><0{%Sytcvp`cZLJm4ZE-s<(jK6d4%D$12rz2iGUUNsJ(UVdb}`g!Bl zLS=YnPF__@{v*q*$JTqJUa_4}1GWD5D*AtItv+MsSe~&zvsh#6H>}n-tb3-b65Y42 zL;?o{f1i)9PR?GRoSj_$`cN4N3~|e9s0PAnlyA+=Y6$0J+43GC*KiS&&`{Mjw zbpX!?(Wti!(Gozx^-KXoLt6=2tyWBOfKbaPG~J5lgK*SxlO$Zy{cBw1SX?a|Q)Ahf zf}=}yZ>IyjwQ_fI=FWh-XS$`SZex`j-xBvCJ;TgkHlj}#u*9b2#Sz#lYD)A24$tkx zH$}mDBYmAqvH-|B#LaLq%#drQtv9Z zC6t*yx9dp)!T|&R<)fFdO9F5N<6k~@w{iUjf=MZ4@!if&BMF3dB~{)Z*2CZ z7M3KBTqlosUx++&P9DtO4|`cZ;}jun(YNC5?X7ToJKNsg%B9j13@hFmDm5rhFCb5a zwIJ(<4Uk*&e7d17*9ysMT+1xrOIuA}+m86OKyhGsRnsIwtt5>TmgWIi1l_Opv%4Vo zph|z7C9>~5H)C=G?qW#ovlNDHwaYz1!M?kXu~B@T%IGM*OWlOOfizvowZe-FNV zSIZ*zc?tPc`yh&c!B2Re-CSFV`FCuQyD4+^T2b!O3}639Q`{)AuBGm=ThqYzndb87 z#^5+L4jk%^8@v`X)nFr!H-5;OukG!s@q zW9zX4w=WYngsda7+@%g-tyc$!f4w+8xcKqv_~P*3{qbn!@SH4_)8o3;u!h6ou)Y}V zH1`gguifT&bicbQ=6d*<^ZlwA>@#?Qk(X!wU;rQ(6 z>f+$-@zue_)vMz-CuemrLiK)EmVOTIRO|L|SR38%IGyJ!dsuH{mhGWker2^+Hk{p! z-r@`IdT&{SMsw7!_q%}HxYOJl)%)GYeSET?42P|%XuDqwhpqK!NPEmim}7s{M{9AL zQx`jfzb{6-aNfJ}JrCPY>iw>&;>J{VUsZT>qQU!QQ_Z1(*Z5KAdGU0&c>b*D?0#?e zi~6hA#r4?kJTC^P;i#}JMLL+(aClwscXtM_u1ABNj>xnl+xK9NiX9R69fSs*s%m|h zL)__UzX-#=T|jv7k58k1p~ZhfWWRuDc!FAvoRo#^7O)<=J7~C}k;;t0hi*p)8Aq@D%yMxP}I3tTvB)Hm=>dX%s}08Ux8q z4V2GnN$O5!Ztdzl{RU%57mP7ES2(j)&W5$iYyoQp(Uhe@UQe8gMPh$tSH+4pzGyw^ zzih7@(4{`SdUg8l@UK^V4ZV8%?g$oyg}GGG37kGRmp|4zIfC!bkIf3g-CA`zSXQzp z!(rWW4toO&?_ym%#=}!Qh9^7?yODpze#cY&4F2ZvEby*8C?V9s@oWaSLn0i-Y~eAK zCBWJa=l>w546lPchGT!XaI=HkC;a3d_`|^l!*6`$-np2k7hmyB6`z*@3w?Z?B_aMs zyeQn3t~@u(i_VS+vs*Wm&x<4#&mbG6X8@XE=m=x)HVB6deo9?|Vl%>UBpfmkZZp*+~#T^a)Zhf>71EKNyplcd2d zNqqp{6=j|9JiiVD4`fAb9P*w_*(AtF29q*W9ZgcE4?)U)p9d*Qo8%Z~!6$|kmgV3h z1A zzQfPv5nSkEkI4s|1hUhuk#hv=K(ypj_KK+G+D+2{$M-sj+;n-Bv6-7pW0ZAW2YJSF zF%{W@a`n;l^a}TUs_c5?6&2*gbC`V-BUc}6mBe(p;uCTly7|@2O~fOWJD%qx76b?) ztRHqskduFjYkI|npaHGDS)MW%rs+S4N}%WXvIC;ZtI6C=eVk%p9#CQUNQKSjIW|!& zsPI_IH*viicf$NDxI4r!N;8u*o&_17S#cWV%d6k#3>ztaQ|8UnEVyM?q8JsE2Q%i& z_{@tFh6H|t;zk_9shcwYN_Gbii!>CH8Bnm4D@A|Opd-XPO+Of%+W;4rQd`aMWrxDl z5rdY`laSAm-{(wJhli-BdBnZ>@xj%?Y7k11)Mgb)Va{X16Z2?&yJPMRFXv9|zT$~m zHFC~@>aS_uxYxJ?dNKaMbqVf!S7NGwqYB*PT=BDuJ69OS9!j(WoXRZ4sf=kol4?_y zUYvhTiRyE{-O%2|B9wQtg*SC0bRb8%7O;olM8JBD!?B~urlyeLmYeDZ@yKr@JAzFR>S0(vYyVZrnS z2Sj`lJ5rF%lZ2;Rh-^Y?O6UT62Jb?jsz`s`6vh)=_h;@6yW*1vpNTxp(A79juVB}@ zO9R{i@SUJJ3=pYdh>y{X%{-L)!^14$56O}q_(YHWG5*}*l*Ro+c5yP#`C85UI%Ohn z(aS_7SrFld70FLC_4GUKck+wer@urxS?*I7xlg2$<*Db2hiS?mu=? zL&hH>`CTN_NAiQ@I_2|RWh?W@Xf3$IDM%!ryV$0jr896yk8?b}!m$jOG`;dzkVhU1 z0=QKadK+X&2)8Ombj!$b8P`KQ6TE)+(BVM5-GO!X;q+8vqkyG}iPw?L4Q|{c=XxH5 z`Be~g2Nvh^t&xMP($zG8bLi2^37K2LT?+{ZSfC`dDQK*;cJ{y=y5}p&h;!L>8;&aN zdVfo>kv0W`F@G4xDSskvh6j*0u-lHt!{@Cqj&EGJud27}LVi)$z_Hicw!LzG?hQf) zM&28b4}hXRay*w4RstmqahWpJL`FMaJ@iIP^lWO@&~p3WM(| z3|^{OXSa%VwyZ!smjbujLa@XIUuz1{`aFcW%nTd@RR0&ZjJmSbo9MTTCG$4#R+)2cHJN&jbH35?y9FesLaO z$2rf`a;!HOZ-0)LI#>c90$+lcTUY`i2Cwr-e*Np0f>;8^0RfjuSppn?KeLp3sHSYr znd=+5WGwQJ;8fs-@8dLoPt1N4gT^h3-bJS%z?o8a!_o4mc$wvack^nV=RAXo!t|6E zQfB1A#5oGGBy^YO2&Ldy6P8PYzsTKuo|RLc0tka+;pyTnD6wV?1Q{6#1>I#S4p*p4 zkSWaq;hlTJ@(U2G0v;WI-mFr$uXXi=;=-$Jy&R4CA}QI2s2NA&^Ozu<-rDPgNx{;U&`I^&!gf$eXge z0J2YU_gV5Z05$`%MlO?p#VGb}gjE-%Ifdc4U{QuE2#fqu4+`pkQJ&F#mb>2cM3||Y zl|CmSgqxSsu*gf7xJzAI zf*5}-h5wER%z4JnS;*WB&$e*Wdd3zxpXzWpJY)G?oZgVKo@@`2AdRIc3#soSVeSnP z0A&3h7ER1K$PRFShdPS6EP0->qtu;D80p>-y~mT73IZs2nhMaDzW*K=Lng$$vp8pR z(ob0aHui&Yz*6}3hA+gtheVI?AjtC%mApNnb@wDn@Q{mO0E)_RIWcFp)*Cb{iOf;e zFokpk}GnT)93i5X;kt=qwmQpA88Ek=> z7z{sk?+)WxLN^(Fqy*7~%uO%6l+g(tH-h6^+By*CZCDY`TLWze-Bh_Rtj7OGsdMCEbGC=d;JqZv~8ifPw9h_xv<6F$_+VyTG=)9i#?eB*|f7CZ&`L8nIMj3i$65V^w zGP;Jk)tECD`H~C?I)hYHVYi8SJhtnni+5)fFT2Inwgf)jSif-)`D!)iOCuLVqB#za z7hQ$3ObrzF*)l^}apGdf;PjigIlFV0C-3Yibnpf881R7}po<7l~Ae--@GinjZYwV*yI?NV-aCAPc_EM)51ec^9O9f>Z*6 zMF2R$@0fQ(pWQ_9wN9?Ya$N`cAxvvn@e<$7*~vM@idqX-oucE;-@JmOWiXk(;zt7F zviQ`wNQeY+AgGU{_##h9WJ?r>J%(3URP}5+&%x>ifAV#s2`3fW7Lbo+kuR!4q|N22 z0Uo!o2-*AC;&V2UjsNghEFu`^@8-1n{p^N#$%K(75z0eD=sZPOVi6A`x?pibtaK7Y zV(?jh^7i;dZ9y~|oJe`+Dt7w;;Zeex9_oxJKmp}`hun;jRzmPBxPWS zg98&pfB9w1`&teFuU`~{`84lxBq3tTNIpuVLF|a^RQ&3awpq3aGF)h{1$|8!U%#(| z{DS6hka}V=au!Ss0u@;K7?T)`B5azz|TppPb7ZLyWis!o$%Q_yCK#5bVk33e+nqN&~hwsEg$e1`Rd~0`-vqji=Ofses$!J#;H_Eg%Oye29Ung-oW|q@naFP4@ z#RVRWsBTPXBuEK2UDS{SpCtcu1x5E3Gk1Yq%vW0?NV*EJMw+Dh=;Y1G<-zII+vAIi ze}gy2aC|#Ce|vCwaCLEcescB(&Ua@Ar|;fqDbA1IzdOIY;-9JrN(f|d_Wr}=)%y>p zryqdWFODy--k+SIgbUwZ9-p5bobLYoE!a9-{CaVD{Pybn_~IB=sMF)4t5+XRPA_qY zMcU5K9ZGuq_p6J`gY!%69UM;oE+3DMe_tPbIEDA`K3u;4a4D(-@_>^!KSBxbez8U? zC+eYwAlUr2gmh+K8q=a$=m4Ms$pdz))gs5#BPVF_?&I%yh?4LCq!Pjso=gNOod`if zI`;N*{@_F{kp1@~4}^f4>f_-Cf7M5{ z-;m->t*;|PHmh#a*TnhS4aB3t@NVdT^TP;#uSQSW z4(Lt}hQqtz&>z|G{Agrb_R0xHD`86!n;MSuYuGtedvt;AXEYeYo>GZY8lkol&DDP^ zh8f7(zG*#q)E+r_nccnL?Mic$e-+xUh;DlEqS+s2PwM^N@UCIE37#P9-s|;vK$(#~ z!rJraDlM(iYE{w-Evaf*JP+&P4saqT9)W8akw|EpOZ9%<5AEU5YH!jkcCnTKOVn2ID1pSC)JVe$NHoD1y>y}EPG{t)P(+> zofTl)KXjSX9^MJM#+xwsgLh*0?akLIIze8ZT%I0N3-8ym1pU?0-4ZMesP{9)9Z0Bp zg%~*wxDtfaOpCW3&fxRX3Koc;OHOhl98BqMfivz}`p7I9v;f?mIzK zK~O<`*aq#$)>RWOwAw4@6cmYsKAmrBwQ>D7up2NG3+Ti$VR=la!6}Vjawj;?I0~1B z+7@tnxN{?C4NcF82Spu?86CCe5nQ5=oY$mPh&qMWv?1f_Oj;2xe-~tX^0*JTro@vm zZyMv?%#SHk7NXRW<${eQO#wPw&fH{7-7asGQ+4XcOA4raJWo-2l+AHKgo4U4VO*O8 zv0||PRBODA9^iQw&KXes32({`a$MI-8qx%rWXTiJK*JF{ouK2pjE6nq$nh~3Pd)>i z%WLrsv_zyw8$}Tie^!+6oVybdO$uu8geE5>ZAj@S68669tUvp190QIujoGZ_b zkq#%CVwoUH&VN%oCybc+X^?Tj2M81-=Oz);# zxOg{ZDOcD-Uj7sGEs@tJFfzrWKaF#!(I<+Y#XcvHPk4Of)9F)^qK@OIB!_-(?1DQ+fTWd01pX7=S@8lV@)S1kr$1iHzm~{FOg=mV`VZMq+T4xIQn$r(z*88 z%{Umx_z+P*49`$|4F;hA77Q_tT%m(NeXSMlF=e|E+59CdPNffN;`cR8-yxr^&N zX`JwM^GG%Vw2n~y60M>=zJm&dQ4sQ@Zmv%q=m#l(@@dq&jd{!pjcf9bw)|KG8p`Y*AZTDR<}G@_FH>oqqJ zz%jXh>O5Z!uc0+x&8+^jW>&Jl>pRzjozCM?1J$|&1KI4`!|VEB=szAc>cbX3pn(-V zHtc#m42#p#A}mhr`n>o6W=HU&MfZzw@w&J!URC70Gg$l`rrh|T`5NqoKndfq9wsZQ zf0h>Yq?F_dm{5VSQSyE?Xg0WoRKxDqv2d!?deK84%NqKkh@t<4B4mc_m@5+CkafPvtFwBWLhzQ8j%a%C80LL(Ofj;u` zN~0FCJC_ev^XPZ72U#~}QI;jbiloA3f8^>w+Gf!sW`1>m>nJ?EVG&E|@hq10xwEGbG&IqSPl)RmW-8ojg{xC@qm`>yWDs-p zD#+<3=IT&dmt37_b{HNd(X8%XZL#}-NJl_LA>ScJeQA@e>FZ* z#*~0^;wctdl7*gTv2J8jHbWEb;zmHqibV;NQ3_B)89DQR5~RHEvK*;%p_Yl?-_yL7@lE zHExhlLj z`}p}6>daSu@???X?y6;E- zy{hl}(b+gNA30oT2%3=lnzd!JHC|oTxWY=lbenzr4D~<2{{w^!M&H7~p!uACsD_ zlz3<|g?Yq}-Rgi$GK-A4A;}lOgN_?o;4pEZNnrH-UTyI)N%HMl(Ok>)OC(Fg&FhjM zT{(suqZ^_eA;AI34W0_d!Gig^>I@^}$C407mnUpTrmRV77CfBhQcK`S=Pvg#JxnQU2hy;Te&`$^xjtYJszX)GKVb#UT{qSBth^&UIHr%bOnHS!Sz$Wd%Qh(5jfBn*u^p12Eqm^@{wfoF$ z_n*u325e0i(k!w!ICBo2rNdeW@84Z;lW3Y)to64K?=FvxIDgXxk#nPEd4yW`e;`ZE zPPK|>E)DmH+i;%i=gGwRBMljWot~JPFBJ2)S(PD(YOv?6kOh3*)49L;3(L68ssmf; zz#lW-e)7`pf7ge@!EiXT`xc5AcolxIHr<<*4h!EHX9Z*-zR^zunWAtwr0KW(no@+& zir9GY(+S&;z1*!*GoXyB&Vkl&@$_)3*({pPm)a8eqTQEWy&9qJ1&sLV$=R_v;G5`R z!yT>l4bS$G^&h&Jg}bCUg2eoZ-_#=C0Om)lm2>X+e`~S{QgzXoEBz#?AMN!6eAaay z)S|)dv&Ol6$8%2qP_KEjnCgkL(NcXis+M92XH+Q6LI#n`vdlU48JW^+ygx^2Qu$6DY z{&@WN!*{3e&Sidr1s2*Z^*To*IB32GBd}*Hf4eZ_c2_pV+h88%y#8=@czN>f?CR*` z;{Czp;g4Ot>!nyp6fhXxHNoVlH>|@j9b8_XpS=12yYcbq@!{n=LOGDtqr*jU7v8~Xn)*sNfS!kZQ;n-#Zr#qG4Xodr>G ze>-y*g*Pi^vm%I!Brg)LNEU_9CdDLm{Q~|2br)GN%ZeXA6*oVAD*PlU{KT6Tev&MJ z6EFxHD){rk`N_en(_^D=jV@_62M5i6xXn*jBmS_{+`DQ5cEZYb&iCMiiV~6Qlz;;_ zx(0SRy6N=>H^J8eugr6!TRiM)j)OTue;i>nkr^}#ITTZXXIXz7N!P*3l`ulZj^QB4 zZ_f>1!qUUA?o{~C_!K60!U0M_Qc{u#W)nO$6VPFRM)fz>bpL+M*1Zo4pTjO*PnoWV zg{%ujYjKi`a%0gF^h+Fg{HF7Fykw`&&0M+yVH< za=M%kR4%Zb(D|Wb9lm!WG_fOU9Fx)=bAe@NHS?2S9mvcQdAg>x!HVAGA+Tb7EaI8CN;7*9}$ z#@>y`oEF(d!n}FtrbdaETqwgX=jru4Zyw61@|I>4iape@RBp$sWQq zo$rCk^t-*0bF@cFHqB{#%Ml=#O+*l*P?j(+Z>DaJs{`U0h~MVKoy~I@BFDHe>2Cv% zHA`ReSc$jB!vgD!!_75kw1$l1)MI{A!Y549z}L~*L}i2MmZiC#au!E1=wl&$$Dqju z^1(RL00f)YT(&7Aq)wY^e|VAt!fi$wHv#-&#*FAFPu(onqEhD1q0+NhO#9LcRPO?q zyT~KKrw+dC(JV)Q?_5Hja6AIhN3$T3{WWtJO+8-r47}ihQ45d|>fnF)7BIBxwsVT3 z?d1f&2t&=S{;xxqFy^F4)FW8qlIkHGn0yW40?iXJB1QUUGeXxOf2X8QH1`^Svqw zqErY(=JI0U8v|jt*&}l6<TZMNq319fc>ipmqonhgiPc{-ZV~|T(8rOf|*##pCLZl11Z3XqOsDORy4hUbOOsaf(}xsFX-&Eamwj7}x2t`XB@T(#SbMQ}MI9vBzV zM|vcTkcKtne1DYl6kDo_7l)xsET)RlYXRK}|I39?|3f#F-$*Leg3Wl& zb*_zAMPqs_I=Nqq6h|E+w$vKQq8Z_GlX$E_Z7Oba>T-{2Zz}6irZ_!zLn)OsXS~}~ zQq>owf3%z-(L&`~U|cCLj4S0-9&)k!>%nr~6j{ogB1^f0yqTj#^9XEt6%~s9s-nYE zQFOFYfpYyIs&cNui|UC6Z`MmGH%A5KtY0*7L+8&v2)SvK5;`Clt!>2+9t+S3dTa^_ z0z|-YXZ$%%ebDzX-qRU}2NKs4Wn@#=N3{nGe*~X+w?i$OlE=c3WUnX~L|i1qMKMho zOqm*b0Y=StH~4AtqbokAb8cv%9~j9V2Q?{3aI_xFcjZjJ1=PMk<<(3XDECrpN{2Hq z4(Bt)o$w&Xr{64(iUIpw%gKwbwMZ zf7tcc@@TjIN-PnZKHd02RS@5Bq;sd7ISQiQkKtTrve0-EZF($3`6Hc2n+8vZf=4wqIGrHx z8^b!4FGY~kiDVf0e8stX?PftH0($i@e}X$K5VKl7XVa9o5L9?ScS zhf*nE#vt+BB9I0%^O^`Ae^eO|m<3uuE9oXv_e${7ltKnAB4j-k5tn2>_*6K=vlj*!U!?A7|<&xr8XGS1{}j?_4xF{XGsKE9HmQRk({~twCS^i zS1q4P;S2>h>4D_mU?sJy01BA1x_pITerwz*Xe6Rut8W*k>??9_f1BL#5j=`}A$aB_ z1&@NB;F&Oy38_5-Q14LQt2W1hGAPjdGZq(h^w1<>sS5i+mU96hkKxA?5h`cZOk*yt zRJ@iTMYtn$aWRhPx$uHRPjb}|VO236AqGfjIVYQ@IODm-4v#bW&qOU7`n zQid0LV95i-Ho_A*f01imv#EO<$U2l_jp~%&8C591Tx^IZ<#V5_!Sg3sj1lGwiWw)C z5*aJjKM|oW_l$^AW~C)iM00`_K|!=4Osinx@_UvE-!|lNMvIcC8ZAy9wG(jq)fCi* z?%2eymKTInHY`QCc5qt?sU-;SH?*CS!eXDK5Z@*F#b*<3e@!yeL7EgJOX8F<#`zDe zX3M-KYC51*GF^Gn`jIm3Miv<4vBGdw%~Q8dd{P4MMoeBhzg|jC1p``TGtcF|I?L3| zW8};+!jj-BVmYN$y%FS6dM8NGjFyC06kSQzuQof*%RftT8cLMB8%Kal9C>mttaC_6QqDnDMcM)V2 z(OjxR(bujL$E%phoU`RBN^BO{)J-@~*#u~dY)UQ}C|(gY27xuAg@wZU0U=9>uf^Xc z9&)nO;~FZeP7qN0pmeTB4^40_Bt5yPD7p3VIA}L_e-@%5xQ!BznZd21@CU-<%H_&V z$lYtffuPz`Gjx~SnE_kmav)WpNf)VFL`aw66eHF%40-YF8pdg!2+F}|CBj{gk=<2} z=af;7&Tf#Utcq@uIB4w^=N-!N`84ouB2wnVcWjnSxg^R6n^lyt+KOquFbV|LEs{OK z4^|>*f1X4sN^vw8*69u-q)?hdhM)+e$bTm3`C$Y>IT(CZDZbnk&i2S>!Ir8B*+wxn zrA|Yf2%joVSIF^v!S!k_$QJQ2<(+&FjDqgsR32yhSfXNGc$bf8BPIjmIKP(jGa>0Io(~292;-1kFex zNHAmh!bp*-B{EuLG0^F5^V$tLXHQ|Re<(r6=eb1MKE*j6)fASbCX4Vi9)>BKpb1-$%ArYdMp;hk$PSh%? zkekSgA|nub2qk+%L%Y%5Y~LZj))9HZ1f^v)GV`;jwLzD@I>RJm=x%M@imO|<8mD{8 z7a$ALDpSyw%{p63RjZQW{oGPB_xX$>nL>pc^cNlmj-!I0h^70 z2EW;(Dohl^?1cJq&9FjYO&B0NKeTN!Od8zpk4!5nW&79OV520_BPiogGZxK0Th9&M zr2IR<5`Y9Q?YbRF2;HMSWhzt!e`?AbW3r4ClwT^ zdxUyjwT7B26zHStc9?{zGTPAPWWl3**W{o#d5Z=4l%@6dZ$JD{Ycz*`!x{RGT4UId zk3am-cG|xUpTNJGil&I+lSlBj*=%aBn$2eOQQIy7d~=VkRUT_aB~D_fJ<;k^84BOfqcnC_fmFYt7~)$T?lMA8C$&;ET?;t4Hnf2*H`ma47oQ zu+i4O*9}ks_S8t)+>mlT(=C-!ALI|(Acwjyn=)Rm>pv`48fvr zKA9>^jyYmLOybSZstS&Se{cpld?WcDr(cPWAuUBJz1VP@4(Sc1g#1GAnB)$Sp2>%b zP?a4^XDegrYbA*C@975}vIG$&2qJo|pj~*vxkxz|N#`Oh2YpdL1eTW0BWAemHXN=x zRuID^vZ5d~A=Ng*;GWg{VFeNGm zX_9mN-78Cfxo%ZT!5B{9a{QU%`EeUHv0BnEq&$=jCSqWz9YZ zuP|JN)vx+m$bNz@!Vvk}uV3|tzWpTVx2r8TWrn3Z8uj|HRezuifHPk&$L@cor1%}? zL}=m+z_KcKvFheYD|3(jHt9Pdpl2A>=zgKWUd{7T|pT?bMiF zzrzYpbv&r>n9-?#0~i3#En3k5dbN-p&#$=w4jr6n85Kr?f2d~WDjlR-R$8(J@G^*? zT>{c}CtMGTV`s(DL(3b&^B7(%S?foD8!4>}Rg>H}iEv|9Ll1#d=b$yhE!!mBst=^; z?iLtV54*$Oa4;MV?}x>3HM;L?%0r z?qy`cRSvT!f8DlS0&SG^I%p5}UpyYQoi@6iXzOJA`v$VDS}Ap=cpcxe4l}-C_Xk6N z)VLaeJC8Z`>9=DK9bX&sCKJCv;*bb0UZ1gPLUR z_uA!yf6{9MJd9NgezQjQ6RYhkd+o=A-yV;iJZ?Lap5(^SRV`p8N!p$`1S~Tgwh&sz znK+jHf5cj&_#U7+{PxW-8?|lQTuv|Fy*+hmZkWZjld}_jRn6nsre#%O?_8K|H_uxJ zEJb@bXviMZ$bI3gnX}`?PS(8_jy3cWJkh_S6>KWEZdzj?aU!1J3HJhGId8q*NAQC3 z=$>UO`=fL2Q6i3Dz?1S2d=N~&3>NGoAan6-f9v%}_iw$G4WL}$_=G~X+|+rp!T7a* zgnmyx%4UCD&qXDJ{OS}O4zKOLt$nwaxH3=qvH6x*tNNq+b8lq>C|bdv`_xzGSbf1p zJy3lDN8|2SWVhRc@DvkT6{S+{B88dN4&?z%*m@S2U~v74yv0Dmj*-rsVoLIPx?5*X zf9^y!$26)%y04G8R&TuVuIEnBdnr;jJGQMB{2wjk9dTs)@rj2U3MIHbYSX2RAjLQcF;%9AVWsf5>KY z!Y4=M$~7O{1`5m%RwJqcf77}3GIA|S$V_CvFM?S1H{jrGf`Kzm38i!LPwuU&Z#w_X z68)I$tF7k6uw8t!3B7DMd@>w9u^ekSeA2c|H0g?7xoiCJfWTdVn~ARF8}_JSAv~GL zh2~1_pYE-Tr(f^})zZ32P9{~*f9~oJf}1IrMgbfJPk)qWT74<*o(WM*D8%wiQ^4FW z0Bbh#vjV_l(X1mYB6cU?iK8!vCYDF!2zAt94z?Z6%Zsn%9xzZi_VxZ2EI)SJiEL4- z07vfC`&&JG4Ii1#ZclZt_uH<5u3f2O+5|-Q$D5;#!1CY5Vyi~8uY%GVf4~NVJ_ix+ z%W_!j!`adC`NiS8^W!Ut9tcbz$6GR5w7c${Q=fd*!s~x)kx0Ij7K6!u3{)$aIEjSS zawk2D&*(>2+}9irJh7aAqWOL%AdzwN!X4GzNY(C1yDy0iwQF|VSfq4TY@T5W|G8qP zf#S|rLaC9O|K9cObX#%D}phzMF0U7`$DgyJ{@7jCU z1Q3*!oYTGf-s|OMu_hD>g+dK`c%F>qST#`;Wh}o5r;N z(R|7+Z#inY*b2f&oDlAK7?bf-koa>hS48ZY>pV$mL!GfK3zJwS$jvYZ4tMQ0KXnwt z!|Pr9-LC!Tf39Xd^4pp-P-DRC2ej4Qd(W0uXP=jO*8zRT)`0c4-4=RUGD287OqrDF% zCt{~mQ%I~%?bWRP0xu70H$wSkxKdGb5o1bjC6>AAsQ*-abPaq%@vYf7z<;;XA zCjd##eaDo72aOmPqz9Y5AAiCn9hUOHDbKklLCj2-=FBR>oB)I-s{)sMOy*Z}@r|tD z5wVHntc`2r9e@b0%d5I2c~!htMAF>hg?H8!e@iZ1>@B7Esx1atTXCI;!V$)`ijg?6XW=l?CwgiLpz@wtv4ft%M9x?idvkzAr9ziR~mRi#z91?@*QYSSVuYDw`~ zOM3;zlOLVfwZMp~tj-l6=QSt(+LH~U%RkX@(w^8y_Jw_{u#h*L0L1MhSBLmN6>I*( zqbv*ls?bPWcxo*zz94x_+eDn(>NXazf7p3!g=Vb*K+pB(#z*exF$?(BK0GF|TCMqs z;Lh58Betv((+Rl<-dG3+LLqhFi>64IqKeQq8>X!>@U_}@XrYS6a=EDEF`)toDP-Oc zr9458zyeK@Qf=Q;?g3g9WT{&vU{hvb&H#=PQ11XfLzkpo4He!jz;rn2dj z*aN2QDs9s;EiQw(W1R;v#&mkhK5M|sd%dd;JX`|dwx-^!F<_njPa6*dw&d*apDfD` zSJa(ILb*Crlz65zVc$r~jq;bO*Q$NeAb-@QMh(|$TNar)hAXp;k_LmGemvX|A_J?zDTe(&e==P`;oDq>u=nup@apjGb^qw{;^y$|;(9M)_R;y_^>zR1 zR}Kr{jj5`Wfj7J>ES>g3JIzu&DItyMWO;=GRJ!jvQQQ91X@ zk+ELJzHU1SLQo|uhv~^`+u-C!)=U4r8y8o%7PQycB;rR`&RKcJq=moOoTSC#Kwhr= z12SI5OyWHZnR{7le?^#rRb*I2&EafyV#>5T<7J|=Y89C7(*E6EWyc94W`sk7^lpUzHeduIt2dBVE!VcBzB8|i|y&ssNYEOkP-MuP!to?=!kTe!&~ z7ixYim30LL@cN39#J;|SJm`#uvKMot3tpYP6Feg7_MLxA8K%t69p~C4`>DLne(F-% zPvvMob@nCuf2n`}^!Ma#Ik$YeEZI+8R_&+q$L*)|TApk_b?JYx&M(Gluin5SWGrxP*9Mr){Jx9EL0~`2E2_8WceW|HDln3@?roG7v}C zKZDE`48fLbgz+$37{ags*a3~fy`jyh2A*pSAi_zRZ*Uqn?BXzv8RTc+y<)&g0VfTE z*nOcJe-?o0Vt77r(JD{G*Y`OyuECJVI2wDYaTe#sc@mEdu0l0XePtY@iIMRk2?OIj zJlBQ5T@*7A&VB~YeFn~a%Zhz-#Z?gOL{OVu2tBmg--Q`)LrJmc8hW&n%{JM0!vX<* z$!m+PS*Q<8@KSz+M|7hk<0{)Nl^h2vM$J`me;_TOwXIlZ(!4SYmhF*IR%p7GP6`^? ziCbrdypFE4*6wH3u=(ki+t>nk`Ew%Wd9B$z8#gi`_wygIM{9KV0I0VR2iyYjRSSNY zEmG#Q@Rp?oZn7bX0)lfKH&A=rU~>HtvuiFZbiP%qxca^NlP%@cmMp9Dwpv=YW=l8g zfBgF=`@T~#OjA~AUo@3lTA;Pk2Fn^b_nNWyg=fJwE;=Kjj$KG!g5Hj+QMHa-d$cBN z;o5v#^J@a|W%|Zu0vhN^UOFD<|22EPryJ$fZ-Vaa*anmF;K@Ey{4?t{++t&u7_t5Y zmg^_+)uFZaT!9j-m7A=yt<)+}X9IV=e=el`iVuSQcIkAD>)a!BlK;TpE4LrFx0F8o z9ouQwy1uJxM(bW>MFqL;g~Q9v?A0m#-qno!s+Ns;>V;AoRr|hN>W^!Zr>_|QP1do! z8d$Z-pMTGJfPqzYe3E-|u1GMHaphz5!+Zm!c2sg3A~svv#=~6NYsS!%M$qHJe~7h% zsZ?Fh-g8mFN1gG?Y1r0-lj$9zeFaiG3gRU@aF=%W2yg52XYF~Gwg+L{X7R0}{uN#O z>3@D&YDR$n`ITGqE4Su@S@tfrDveznImqv?n(5jgx2lD|+R z`HLqy$=|4>+)^8V60FP8Y|DM+PH|s$P4&qE?Qdad2MzOuY1u*Jh55p=^Hriwfm=~# zf`qn7-^JBFB`+a#^pD*N(AaWm=ZcJSWX~F!3!!;%&^5ObduQ9KW@zu(#7t$}Z~oQ& z)cg3!-Ryoct;b0<6LKxu<89n~+=~gCs>m{{9wsnC!ORYSO0D=I2rs}z$A1(|Oup9) zP)a(hSPhK4uGYE)t!2%S1)!A^@K%n>p-u0M=u6fWPAC_Xug3v2a=FRegYp;A(G@69B)wH^T z0yitN`3zBiZ3+P{TfiED6LA5j>H?M!tQL$w>A^|>hrOR4mabX;kd;S|{;lgDcoop_ z5t1VC?M$>cEioYerJQD=2w;ws8@6=KHH}#;m-ba>6$=GE^w#seZyl$%?}~39A5h*a z;^Y0Hjqdank3Q{Y(jlq$s`FyQbU!^prtsuBZUVD^g?#g1(+F?m3lq#kX{hEkLvau1Uoi!>1tC3AJT#5YHd~R+<gwhx*j(?7wKgYTzK(76FC+l9V<_rsMQFn~u{nfn&#UT!^@) zqA1U?kK-B`AUo;k?#0a}pwDgP@RDqE!$kLBm1%APxQgbD9lB*cFss5oem*a8l+2=e zo|F(=-SOMMwVJ_f*4?teT0RmkB5e-0Kgt37}@U&}84N`L$ z5;Tj&#yrl$$ly!`3M8Wc#^|41d@%YSPOh%cE-#FJANRRNzaO(Z5rn<4(eL|S6hVyT z9v=UVem_rW?_>1)5sQs}f8-nez83^Wzn{$qPeB`AAfa(a)5XnG8aKIE9viKH$ReQpj7p9pJC0>Akusj5K zBMIr;h}i-FK%nRhp9v+WM(W+M&vP$=06>s23Tv)GXu^chMzKU)lKIyC{GP! zWq@a$&u+bF&Uiw`@^O+(%AGLciKPZ+ET<6y7%dFUaO8uGSRTg11mdFhktjW4IavB< z0HY#$JWLFILsfRx;e#|8|bfQ=vmiDM%R; zPsUha3{$w$XJKT35l?EQUSN2G%*Z~cIShuO;bmcLOyPeL7(@6IC8M2f!^^#R7qC^J zZj2cTfUbnO>`De;FW6VmD?o044JT?NWz*!A85s6KJ`2wWGlr2e{9y~`&jIs>BhbuJ zCI@U34)B*zl%<*BgW3#ywuj-!z;MPG3@1L&;47QY06qpjz|k;_1Ni<83?Sr{0oywx zm``WMh!Xgb)Bv_Am`J8xZj7L!D7ix%8a`y;b>Wvdf-qijYTH3N1P1b>?J+@y)T=vEzFn5;uL^K&day$6uRoD6G zY&n~V=vILVDnDIYl2?6y0dOXFtH3`s^s!EZT-*Uxdlf$VK`nTQ{sZ3;1)xpzt?_!aqa>67KXJa4;QDy@_*?b6iPhbm)AG{X?Jjn|Ja7qjp2JG65 zyD$oTFNN>bi}2I-1{wa+8h7WR>5P$$Sri#0;WUPkcbla0^9zT6u;9Z^0oNE{EHQ94 z#hKi!fXCscNWA-xkGyuo9q#RIb=_{qGOO4}Ivhg#qtkl#%jL!Idy-tXZV%M%IW6Z- z7Y3Ph86Z_x;E^g6p&xr6oy^_R>ersB3fTJAn=rChR$H?v*|tmsp}V`Lfp;Zg75BD; z%@!LPg0091hY$^ahcW%I&k$yx`Ti^pa65*w6r(8unFvUXF~Wy6<}nWlY>rzrlK2w6 zbB+i$i5=OQj+>ZG)^HIENe*L0jsz^7PBER8C<7J(3~ zPBe{AaG3?-sz$4DoY>mzI?m3g&7R?IYN&LLzj(R7O6C54s+RloNfH@+`m_a^8(QvP zfBnUC|N85%mE2FQt?noNp54yJhVk&-vT&T-vTD#rO)Z~S+<>-iRB`fFV8RCT2nn}j z7aIS=-E8zWx0-IzaGYQZ)+vjK1hy>2&AwF?A8K?qADteO2y8j-UVHtS?-59~5|qcI zC@$s0Fge+O3IHDk+p>z)LbS+&T}MO5*lGbh2Hdacc_T9m8(XGhm|Hj}EV{l!)k2bbk?+WdyfalO~&1xvj`wzQIU?|BfK5ijFr!n8G z7QmGDDkZ8?_db3{WJKw0I+?3#)Na^^+`g9Eud;&JqmJ74&UE<%0Gutf%==2_&$9|b z-@>wg?<+|oB9WSOda)e{`9^PM-`gX5;kJfJdgA%x##Lwk;YtC%si637CKgKZCnAoN z1g-@UEA}HwTpwm+#YmEUvE9mP|WDK(ZD1VE(Cp+vw3T1&J4m z=LiJ*uw(zQES~rNz0fJ3unVl8%K*yrwP%4UG8D>r4?h$~hsDuparCA*IxCLOi=(&2 z(M54|Q@s7Pc>At+d-H4Y_U62}xGt`bE{f}4i}z>6Z@(7*KC&F=ZVLfH+q}sKu4;RK zA7*;L^H4|Y)hmaf8fdJrKdK?=lqOjqFX3QHU;?KI0^R_K@@ucott@&U0e?JbQg8;S~g!(H-X;O=kC! z+>|3S(eL1%=MC;t?Cd0FcW58c@5@^HeO^z$FKpQQeXyX0sm#kr$9|yLw-0o7?gL5B zeLyUx2by8U0a>FR2;$QN!H#;MxKP?ejqtd4>&H<0m205+%}?*a~pL*8U}cc2=;S@vn10}hQ58lY8s~aiSmF# zS!tendddWN9^rlTFOrxOs`fADQN)KehWqmvj8ZwaquK9E!F?a2@2*a6Zho((O0^}% zN5WGj69L)Ed7ODe)}P1Pv&6E0HL+?;yxJB_LZ?LWAqtI;;JxMMeizsMi?f@v!}GKMb#g^{?=KEN9G;zn?v)a*PX6=#+11H)|N8wqQRwRA=Ka;hb^q+* z;^eCT?(*#7Mr55`fDVMhm70!E-d&v>!EGsia`FBxg)ZMgcU@ED`nR)x)0@?H9A00) ze|yruxV*S{e}3M7ee&k)qBiH`;#jou^z7o~xPN+iMV0FHUmU(YDa9{ue!05*jkokY zh?6M%%i+cGxp=yhi{n!K!{Pb+6Y33~aCvsjgkliYcGxY+gy?Ji{{XRTH2V0MELgvAv!d#<}LK3BpYp zjNypJpbCuXlZ|Ma3J~cUW<9tDn*jD$bUi0y9?%5x`ttG|=hb2xz6n6g28B|fS-0q` z)PY3y#EF5AWSr&_f)+ZE;GJYCRTWzhRv1{1AC>k=Q}Vs=>8B}wf48=zQwCfF@e==x zv=n&NHcKL)zoqtD6=|-@FKfh{Ps6%CsDYoZY_@6zjumvu5Obm70-%hqild}# zN8rY@h}3D}(BSeN6!b{Z8wuhN{e6~tqp4Ss|Bzg#o=M38EbR(BtrB2ANfKNUP*)_V zuUeY`Xh4_0-F@|#rB3|Ye{|?2Ar6gT#oZIMc+ah~TyY+X4XeEl3psciX0hCow$>qZ zm(NCO!|Sh+eD#2q05(&;&t?P!<)I;|M|ek|<$!_lX{S$Z6+6L>nhGjpK%q_Uv4$O(C8<_4uBCDqzJ} z?$eKW3=EFQ1sZ~}e>bHdJlx--+|MtRhOtf*6!ttaKSh)m5^WMl7E(a7Y(9 zvr#(Zp)^cVGMdF$k|%ltHAr3`?4onD3mcBChp=!w?4-Ki3Ik|&z2RU&>n1wTfu$+;H zmhi|Dzv$+rf9U>95Ws+g7H%;f2j{A(?5YsAN0a|Hyev~jYYf0 zQ{K(lcq-4_<9W^nRT+8CL<~O*sF5zi02q{Dgya-6?(_@+N)UqptPjxZhh8in*aE9B z%jW^13lHdThA6?={gl7q>>m0&q7Waa*?lG&exHdqai8S~@SNRe;#qRW{c)m*UQ!;C zBL>q^e~H5>dJN{|88HWm9y%}h_(y!ibM7>b8z%hai{?D$EhY08+MM$~2Qwtbe#kp{Mf*l%xS<(+lc*KJ)@T*$jS9@PS$|+&P2dIoxA!&$-*O zHVfy$x!ZD#ME4qV?p>};~N)$Fe=+C!P*79!wFwaDPMD>$<#f3rcw`|g(2EoE4hD~Zk?{`B3=YTiPV zvZ`QB!9j1kxdTOZcxHu^;=WVQqZ7FHsMBjd1B!_@{3+m1w{5pyv~4hpXg_be_M+1O z>&q)Je%vvw&OYDfo_DS~9ZKw8sTX`@e;->vS9y1J0^U8-zChMp`xrQ5_XLNY({`=R zZreVuO}(;1YiUn-TddChKdKvQv5z{vz^>D3Yh>~tJI)<2zxwmMZLb=_FD<*4ct?$| z`x<@0=icYLMGEw0oMT04v2?2%yS{u$JN1O}rK#WvhqdX8h_N^ri{|m=|U0zbWzfA8U6cWwRy{#}|@Y0BDNKCSKT#K2ak+HqPKZ*=!0bulJ%t44$Nai>@2mgcHA z*S_lX%zppq{P6mkt7nhTu6XG5?EFLo%PQQX%eU{&&QGrTf4>}_-}Em}Pp?mIcou2| zDERi|=9kN39svd8^}E9(k$H1 zi_b^`Puxme4>usH+KMDzXpG7BLYM?-LiU(xW+9qfJz#$1rDcN+l9vnBpc0icl9fw7 zl|Wt0;()}}e-sc>cF-bm3AwQGU~wQVShzWf5SAmSA41VI@zNEq6xNIXS2ACT1& z=qCwjNhGUr14 z{7cj>ihzrVdxV*XZsj43JTwazuzJ_!j$(>XCNje8e}mH{nq(j{=O`8=JG#}?x1zJT z;0Kiru6^F=nJ1vg!-qJWfl&7>CpKnCNJr?+d<59YC!DL~Fb(tZl;xqXQc~}}bXn@% zOOB^QbXg0NSQLNl1*d!ia^ywP$pSP`WPbD73*O>pvYK%O?ltj9y9ty4Z>>}GN6<5p z=`4(xf4CdAoxiqY1VK(xy|Kq3;CLkQ>-+!3(nMT!oB*$y-hvYt@7i@(vQ$6|pN0`5 zDWi79N=x5F$v}&q#X|r+Wu+t}sS)o%zlA8Jl|D!1fnMkw;EVJU$td*o0&nM09?l~D zo)^g#3b(bS%Xxk|d=22#nSL=QUjd3pCAIXFf0rD3M=wM23a!vPaI+Vkzf$ED1HkR@ zmQ_cZ%vc>CTGpC@B1_Z_3z@n8vd7Zu@~&WWU6*l_vT|jWav&0(mOA8`u?dW6tsbV2@hl!DDkeWoj4xPl$d{h$S@a7-<}Z%F-&J^x(Y1rABWZ}& zgW0^(fLbnCUE9c;ON_%z}$f#=-P1;-3 z8))v~z$v$QZx0%~w14kBtom@2;Vz^|#$%dfXu|BwzF_Oj&5rB~`-8i*f3I2j2I6JI zX}UC+hcOIbA?Lxt48!}XI=qWVM}1iOVm4PF99fgqOOia_;>N8oz6E$j-3sh?clq=0 z6>p1cCeV@ktOK3#rLM5=SZ76p_uT2B?6KQcV$;NqEK4UN=h7a0ALQy+N>>(fVm>_9 z%o%BWPbW3E*0fEykGbuCe_9ovv0JTHpSer=wbzQOXapqkbBIuOD@-U?@kVym48S|~ z1j&?CFPrKQ+Kiv%?%PC5ox&W?LeqXw?M#YNmXILKlY-$@6*mbbsCQK zv(|rRC0}7D@!JTgDRw<;y#?eIGz11|($z_w%NUpkW|aYvDK(nAe*_lwst{3498wv* z0lpfj_sr~T9Jg`?#sLiy^{j1&wd=>aiSmQIvy!{7%qto;Kx~xZ0oou7ui6|ch^&oo z9H-}+k3fdO0TS#aYYh^m+y|6U$MYrerj!^;XB_j08|xiG(%_)H$_0es&| zpD{#zM#+Bs-7oJ9MW(Xq(Y-o{zjC^b*?3lTtgcwMakkJef1QnIb&4ID5BWfcIS~MK z>BHb^;(=z{UClHHn%Y@rK3&|eDU!<4e2j?-E&Fe21fEo-Te3MG*Sv+u3uA+rkCmc| zGa=`T+;nYD*?c(96CgzPaY9j*&eo`^Vs>X-vC+w5)&ShO*~7ojj{DcJJ}mpLh#zrD z=Mq+{IqYnZe_LEiR|!@L1LR=gaSJOYT+MV>oRZ{pNT$Xp93?p?7nLr!X|*Q6B*A4x zTq)#~38$h_e9zNC;0ZY+`4FM~XR))btTM^u5}TEJA-7KCf*b zyxs!se|V=!dJh1SL~o*ObCr0xWSNVmxHA+FD>GZ^I4D4dQ#MPLNh=!Aa`rh6lrA4% zkko+vCJUr2LB1SrhKX_va#^*_d_E(DX|m|OCFZC5|16`u;*GDHMwW43YkZX2&)eu;uav9V=TT#?I%A_cbCurwy<311vFN2&6rwjeqp{pV*^MgukNNo(3l7f)3wAQ`ttpi@z3_w{<=>Kti|iuvl7F z7j)JbEM%hNv_M(^2_z#M51|Gs`l*5>e=Icg`sR1#Q`2r&G&UY4OUsffa7ym|@gHut z(d}$J94)O+vbR1_Y0TA9p*FPRRZmrHZ#DYeZEMr&MdxcO6E*{Q)sAaz^`i6j1(Qo& z{SRL%x%jte2)7k@w7hy?mG4$Djpf;1NP9E^)N{mA+-qB_4vl`z#$FUxL-4tne}4zg z>-H+59kk}EQwM~ny;v<*P3&lz4v5v2Eq+-R_{4Ltzj_P~>GVeS z13wnx)AU4n8`Nc-w#*mY~I}Mw5KlEtfSrf0U2=`}<%J zXt>}_t3IcXwA0eE$`3JGZ{WBG=0_a!vT$0xmH@!CWDq9QO{;+VpE&_Hf8>JYHknDO zscwsWD|l?PH+#n|n}V$nTo+c#!~L~FR=0N^x=<4ArqFE4X*t0r*i5xn%%xO$u&r$O zI?rD1@421ADV(DC_S<4_e{Zi_ylNJ&UNwv6{(jNiFPvh(=ry-Cop$%x#<#`ZtIqzx z$G_ad*>pDBXh|s^t=F*Jw%zVM0QP^|>7mhZujjZJb=r2@Zd+wH{VTH$I#59YNJ%C~ zWhV4y#2eO=6!ZglW^8Og1dqa?HsvyMT+b~UHSl}Sap9O;8atk=aUe*FpON*Oj zQ5;E^A((o11k~#xf6yXF|hMr@cv$Xm zh^p3wS&PhXFeC>nycCr#UQn3HwG6`meHyBl7G|xfm!~1#Jof1Cl+iC%3h$JD>0O3d zi%## zrwoC#r)(;mMNqC_WbAX_1H2Io0U~q8K0}8ZfJ%aY!XaWx<514Rcoq^IHONl_FAeya z-M}++m;+KYj6t%AL&$3!`MC#I%0}cVAEW56Y0P69>hoBZcz>TUGCDIB`7oVUFh-^4 z)pPJY+h~7T&DB0&3lGL#R$lP*t*e@Ao%rMP7lbN%AeN&o3+CDQW25`*`3uyxm(?sv zzaj1{7(8%)YSzHPKI$1s?T*zR>yva*`{X#Twe<-!8&wUNz3%blW3*KnxuFreI7chx zqENbK=TfgAq+*G`9)zhusOGZ;-R9w^^)|`#iz8A!lWM{Q2Pt;3WT6%IA9qQd8CRrk^NYxdEmz+>zqZhY@qx8rtUm-&}_P`r9o?C%$y zzlu($(=A?g_KV)DeYXQX0Gp1pS@fFit#3YlyZP6r!Yv%92OK6{%Pj4(LB4^r1a|{+ zw=C$xx_@Nu|5Nfc3hlP=;BKQ^P>sSJ1<{(zfaVBra&dHd3^Js?>Du*Jjl#Z;V>K!R z4}p6}Y9{hqu}uMYwnnj}f!8Fp_%t9XM_e=(b8|(~DRKA_P!S-NhmPPO)kG5TLPNOl z=Uiiy-j_5-gdR`>VLSMAJ~kCgt>dYHN4PZk$t;cf=+|I zhprWa-EH4PV@`ZJjDrQb!Px`4ufCqGyI$)Fq1X?UKaeT}}q@M&R`B|dh%WU$@W^KiVsJPHQ0$)4bhT}NF!;bxOX|3igdWX$Z zuQ}w%xVv>_=FUU6TW0l#n@AA$w{bJT=@)pHiR+f8Tj19K}i zI8MVajvg=3b*}61I9|Q#HUDzEFS=DbRG7`d^zb(evEDbG7$hS+2%z}crnBP=&`4Dq zjO%wjSfh}QQTPj^m@FPprBD^VgHmBt(Qxz+R0UE(-XOgMC=0p`U0pC)T19Gg!+);h z3_ewS_=01%(u$M{R-OIOb+Ea^3+}8hD7*)yE?`~3vYe};5Lk1OQZUGT@_$?fJpuE; z3e z?hCg{-dO!&ibnTY$5IQ<7j#qlTP%$+g^XE)Jwm(46=)H$xvWQ zpA{=X=YxHrMYy&6?;WKfL4Q=3RTYcuwhuKLBB`p8WG?2G+1u~B`{tHeK-h8ap1WI) z(;(y-3q88=Pj7$I<)tvQX>~bki}ur)kz+St9R+UhO6vCNg`s=-uhxvO-V z95M|60fg>hKDgI)nJFSRvMIo7qbXFtL;0A_O6fqy#>ljx10)osk$-mm%gO7*i#Lr2 za=_j*Q@GjElJDtvOHw~^gRDt!(XrW&M1Y}ZaZr4ucbQ5j|6wR-1^8d?#J z)@6VSr)Lq89H#{y=!$Syz|ph7DHuF)?4-O2L*zAddC5QqFc00vkHyaS#mn8|`|paK z-5(2pn74{y15+%3xqtBqbXGdIXLa_L%uf*TZjdbWQ}U|Ey3bAf$V>e`)=taZpIq<} zh*O*NCyy(q<|zWE;JJzj0iq-!`#V2PH1kWcD?}ncBuao|HK(oOM@l;2YM7G3ft!Q` z*DJnwoR>kG%}ICeIg{p_+=foF7zp|XLYe0X-$Zx79lyggpMQJ&l#b{8@-f}DgU-sC z_L};g+uU7F$i>-o;gD(uX)-g=v<~eAuIVg)&F^|m@9;RqUOPnx==d%6&uM%~26Z>d zu(-<(aC2OzC;s7OrhXmc7+{!m{GN7F*^94k|FJ}ge=hgvxV#d z%If~!s0%uY1Ai8jN_&0|&!c=b45`AzLOAvK$sMz?{Vj&n};rHKR%I=S3p^ID?bBN5b)ErCAvDBPO&3UOIW(_fm>|QxazJSNa zY3ousu777VFy+Acz=e{wvH@W?L(h+O|pNzml8)`%Bun;HJdHh0p70# zLI09(mmrb;q8|a@m<+n6LSW~pM2`@#y^sRs5?fBA`|O3K>66x?=eGylwDEk?GJzxy z!~v$W19~)FwlV0|S}aPZp|O{?Smt{(%~Hu}yniU1yJF$oJ?t)vxpN1scki9MhaZ>R z>csv6W~X-A9W;8|JAlGzZe`_jaMB>q&6=!5Ly2ta3IJ&y!1N%_rv~(?LsDYQ*HBvhBOX-hd(rxc<)ZmuNHd-1Kp)us!o%n; z-GB1XwbVN$NoB8y>chDGSe@$d_Akv$O{)_=;J?w;>MSlQXms~*$56nQr=Z)a)4I)=4)85DbbQ?O~l7KJ}9QZ_Aa?~2=TaXSs;;&$pS3V(lE zOs7Q{7qh&W`NeEe1Z-4{QZFdrKjMsL#WX8^`Lme(@@Emu!XlXY(;}G77H~Ew%RsJy zkmu?QID<-kTk4W#Q!=(HsKP!_XfBtQeK&#A26`wBP#qKmDQU1^i|ONT*vn|=?#+V` zwvz+ow@Ca6=c_XZ2?CA}=6R0Gxqt7)w;s7Gi51=_RBg_S6$H{3ECA{!bmj^0z!Bi- zAiykSf0zo=2lBTWh9gz$FdPZxUcz0K5@hEX@4z3}gN$FoAJ~VC{{?@(M>iy{{2Gta zWR8wW`4rDKc-Bb?;69v=N`6X6N-zjewKl-EM{GoU=P1N27wHu}eH zffUiUP~0~WPlAyDp`X*^NGyLu$)6Hvz2sh#u=vS5y+~s zl_OR=TjXBK;N9m#Vj0N^GK+BF1(rKlw1QEES<7osE+y8KG|? zyBrSrSY+2T=FcN9E#2{k4v|B4GfxNeh`UO`sliL%q+Spb1`UL~NPkPna-x6{C_SCu;~H5$onQ|*g*rypotaZAWYG(P+lz>8^*T`9P(7jX%Z*E ztb^k_Nkhr0MRE#2PihJpLYW{L7S)3T6FW;Ys<@f?V>V@_WU!sfaxE!k!5kVrO~hwk z$v~n%L@Dyg%pgmmx%lpV`pS{1n8wXj4l;b1{SlY}u6BSsBG!@`t&=d?A z8slZkM;2$R!i^HHJ1NnkP(Zy6fZl4VK9#>=7Lt7n34i2jMrR*K=a4=?fUDz!YQSJd z!!gPKHi1Z$?-F1jMS^~kM>!Q#3P!JKlX2uTxs1)cw9+GT>E)kel;qsccLpf$9Va}$RjB$A=MqvGFf_XY`xo{@;$4T1coIG9}PQ_Fv z!%d$rn}5}G@C?5%ryz*hO1XKuTIn#1A~6-KRFtgno6hO6WL~w6O`OdIYmG`4W3rOc zq`g8l6%`|;blA9vKGGB64JrH=)0@qhmvWc$%1R^#R#UpbLkb|8&HkNmfuILE<-GOr3^7nRuk**Kv}i- zpsY1_sVQGAndCXj<4f_CR!}M|BTd*M;v!eh!&!G-X0;`hT4{=(M3F~ZOqHUy0=g6a zSATCp{SVzx&5@&2i#C&aPCMpOsv2W@s(?`_CyJv^DYf!6a*I}qS09O|mPVUOTVJ}` zqgqaFJE|j2PhE4D+A&wkTX(0rD=KCAhD3{1TY+hnx-hL$Q}vaL)nAWR>!!p?<&;>d z9e|l=cPkR`$?Ha;=&!nQSh*D)tyHAiK7UB6{nj9({zHSD)w@(5M@8jVzi8ru&cgxo z6TW5&H7+J=OEJWUgytfGO(8*m1bE#U4<~5=`rA@IopE@CGpa!_8+!rbpI{{XlXp9` zqAB-S7?ay8iiR;432{+OQ$|y!UcDbNFXd-T{?p_~R}80f4ppNc81Zm5OCqu3sDDZ2 zt8ylDXK6AFRb9=L0SYcXP33UrC((SWb|*YKB(6yw8=;hdM^qz4tf@6vrip=|r-@h3 z)A-4o$I6)kEQv%&JOodjLRb6E(h|FxEsu7qSz?Mec0VPnb|1bDJT&WH4wo6A{R5oq%3EkW zi8g%}q5P4~qs`Ku4h4_u)8OL-_r3|tQ}|FA<>N%dOv3LKztw?9fIS$|n}-P?UJ&6? zs>0bgrSl@j?oyIn2`DCzV9Ak4F|9?s%7lo{lQ?gxp1|nPn}(5!F%OQ@(0}D1S;$2C zgdvHwhzfs>re@B88E~+K1Fq^QNS%3j<`Pk%8W!QE7yL2LWCOEllH}uZOsg!*Q_}0lw*8Pr>BTL7^d-O zfMBYsPLn(RLWq{GQlNk&TMrPK(B z3NSB|>4=i%V?BJgT@q=Ey|!8sMT#a{45v&SOTmdd1Ywnsmy{Fa-le>-gd7bHf5rQy zh&TaiG2Y>Nis*AnlUsQ`TDq#T$qqBMWq=_Y4(~PDibxUC6%pt16*bKXsU1D4+KNV8 zzAtB7yt&w~VMM+|Eq~U`awPf%*K#?js7O=E^<|Rw#cHX1b)9`VJy2=kMdqB zC5c`ViDECx!I<{K zF3wOEoO=1V8L$~|T0WM-8H(_s2a^BNR#H<1P^A1-S5*k*YfCx> zZHTCBb?u@o`-;8WCa1)PpwJ6Jm=7rkitz+tJj}#o5*dTNUL^IZ&0(lu9`*iAgrbfD zO%j%>co0g=Nq?Tgk0&Blt*V(OTwJO4T4E@Ij?lx&IGN}Cd*UHEYlygN7*7x{NW?|Q z3H6h%iBeCoR#}C|D>+%L>ab)A-&M-^LXX_@2yGi7l=*2AMa=;ld$*x%LnXD;oN8XF z3DwNSn)G{=#|Ht;#XKZ$Q;PEmrJN?F5}7LIKarr6dw->bN@bNPP$csKE23i1ia5=J zGmq<8Mts?j#~Ce2o@%r-dDNbHA?&pIXq$PY+Qnb3E{LgWSc-COQsrqeH3gx5Lt7~+ zF4jqk@l}#5KAY^8vxI4%hZG~r!6{Qp=RY)kt?n(+(gAUKN##lNN6NCC|H$XVhbf-?0?;8lLy`ha;Jh3t+AQ!;m5W*nt6)+a!PSI;A&F&NU6&a!=;iZhG02a4#c|T z+M84cWjT`%&2pmXwsI;BQ=X5w#JhzYEtd;=ip(V!-jO*hL6Qod@VkJQl zCZUZCxAVgU0N+T<`>INM^-=h?M~20=RDUJNXB3~Nv}wSPK2PTUxakQwUa3hU+$7&D z6_<;yN|LvMlu)mY7Ef($(o%)uUQbdwdM#DMGuKjAb>W!Fg&Z8rv^fZaP?%~=lYrhT z#-st_Em`2Q;ZVd$+9SU*Kx5aZ!BSjIg5^XZNGPZBiIEc3lgMOki*cKP(b&L?_P$(rt#X-6G^<}s4gh<}sZAVZ)sqD!s!H8{nRV@gx^P-(iG;Mcms9bb0! zxjwqSHtkhbCY_z9(=J}PPx92MGk4Eh74z=0uZ=J+2Z4le&L~+=Z$0MNF0SU_1=97T z^=K7&YmLo&edRic4tTYgTa$@`-LC87`g&#=n}c(xbi+5d!cslvQ!m`Jx_=FGE3RJb z*MmB%7q{{*+$DcM!n)nATDK!wXXWvaRv*9Y_k$lLev<{b(H3`2I&2i&YOs*=7pwy+ z1{Tkw)SHb%znB9%BMeLmHPdA3#XRgqv$0pmRi{wKoOzKavjX%P1+K~k-hfs!$bbOt zg^PU9OrwIfE@Y=ps$8B#hJWrnDU(ltjTezQgPdG4jG$dV`w3Z?BRG6tXq zIV&Y~nciGH? zXOHE>#3BG5$i*}af`}DU7KHO@5#?zSrOkX$zk_2)=2n*-o0v`ca zBS77nkH!>X&^IU_W@!P(jAFtVoEV~l?}myn_oC1*==cm1IS2@ zwqTriq>vp=aIr;zc8~=K)9Aa@g1m8B0*Ti36d0Ij!2pSVAq<8KZc|(Y%nxBO!0CYn zg`Xx_Hul1_5DO~=M4QNXz5oqL0p^x0Eqrf=FW^sBkVpmJ0Dl+4L{Y&%yg`zWive1C z;!i;x58EKA?<@u^O704wLM}iATX@l(cb^sT82mf(2B=zL^v&`w7u3wR$R4|r=WSz0BKagwZ=ZPY7wjx zg=&p;;RqH^S(cMC+O3!J8gISyo(?FtUdo@3a5KDsXybe`a+2EF zfMvdpGQxd7xX=apy9i8!bEf%;Mx7RcBL&%ubALM<&Q3?1YA6;KLMQy~d!vyIPrXIo z_sBqw4vQRIMVv6m*)b--mFbk_)HB${`Q3{A;rL@ zK$>Q={Ej?Xo-jHFWEeBJ=L~TChkS5K&*OWU?*%-A5L2mN0l=RBaJx7);`)SxULAju z6@Ng>W+?r|*(Dr?`Z)EYgzlGk5zBlg-?|8KKQm#s0em$fw-XCOY3vurqFf9JIAN?= zwQ(c4TcRNJ31h`i(v<(OYzB34vp*`GjfF)S{@!|Ne@0$*80OBEe#&@5VRM0IWdpJ< zo3egDq-e_7iHG{WmwQn%(o*=xDx3>UkPhVJNiW0&cl1EdT*H!dRi=hpprzP+Y zQ;&)d^l9~1+SMN>XblW(G!mdLh_X&EaR6qixZ z0vHu1GD3lbVS>2jUMIE`B^S(}6VKuzmwwO!7Jtl`{aY`hwM#PyOciw|xpyK}qc>%I zBn?x>6t5=u)8UU40}khWog=R;NxV5q$<9rTUi_u+41;9YbW15*=`c>vp%zhlXl^3H z&Nv8t4jjY{i!c=PB`Xom*4mT|v2^VUm(h!tVQ$c*7Qc`{kWkjh6$^i4N!%Cfgo@Bw zfq!xNhA<{qd@ZIG{ye7_juGK(ts?zd3d#$57)%x~FY-p*n{dtzfxMLeYOfe3QKDmv z$Tb^&%Jxd(gz=v~?zOn}5VbLig$JVe6DzjD0iinxYESWISpg_9$ExqSOAMkrh{`MvkbzBMR`b)VSV!g#*p

0)7zmOl^-6KWDjG3lpEFNS|f0%Mo@=*>;c%CarR0 zL=p=RO`V2Rw;`Exm+R32B7b7#(J(@A@4TD>Q!1r|1OuL=6-B=ml#f5-;{|riEWr1u zZ=M0%oMwOc9%!orSmi7rPu^KR)OlB_nc#7nX`55Q@1kMH4H>-#EkYkh%0_A5yB#gG z@Vy9&Jpo2=w&0UBCZxL}ltrZXp~zyPg0iaYo`MncIIS>8vy!-W6n~%=64sz*yZ}mE z+*Uyl=U1Ao)8tNlY}|Y_oyXi#G4pPjUrS%F|W4m z&+fCFO=Uprw|I>Amc-V8-wgR|Fc#C_vEm4*IY)_uVxO;;GA?IET)JQ*%(v~jrcQ?C zbd&K$KvYB`N%p^IkfBB z@VWzhWoz!*3+NIQQyAh?qjQnL&o_KiWlY$;Isoz*Xn*ec&KBinvI;K0_hBAtil*uUm8bZQHF)rsrTXmTQ%9udLU3=2Gw~r8o zXa*oef!m_fZzE!z{b%+E-nSH-^7E%tl+9T1Y2)azFrE@CdjMoW^WR;x9!Y=H?~ zj}8AIF#;#crLxyJhbQIw0b_mj5o=AoS>r=z|3fQfXuaDYSVPlsvMtlH;QoWQ#nA*$ zJ8O97&Krv|-nwdv#~Kl{ALK4B+xuX{^1J7=Wq;>kDa8I4tm89*M*gAmp(uK;&D%F& zo&5;}bAQ}uJB1T$ZM3bT_@G|wgjwAQ+q+;G`n2&dVasCs)82$x%d(o}KhGbx6hV5{ z;uX-()G3&pLC?8!TDpqKV~r#G0iOxR8xUFHsqfXxr6s{*BZ6@B6{zEAS%Z;E^M>fu z%zs+ir}mY7VTX2trYU9BU9G7~Y#-~om5p7cDl%A?;I0kU=?MKvm+i3@`K}O-ti1G1 zrd>LZ8)l>1F>Pj>mT9R{^B9Jbb`N}Kac6&Hw-H+wSGnm|ye+GHdZNRcI}(qcH#6^a z1Q_18-{wSk0ce~_+Uax%jI7&Wt!xJFIDZXQkG*5rxwZG8^}|_OY9>%AG_WBpa7v%2 ztZSNkW_QoDw&)2$e1cFufiO^^cE2A3u;x6bkBZrC=We6j%2}2-nAL?u;NfXo_7!0H zyi1cjAw;Gdc*8E<;r-aM6Ii=IvL)cmRn{e3c^Kadgmc$k3<+y1NtF&G%Mw5Ia(@=^ zC8q-Y-u5yF$dA43S)1!G21%MGcbfCQF8wFbjo?j#XPpbr?QA|38&w4t5AyXmb_5DZ;_)+bzcNPmvk7#sp#U>hC*`w{}sn_T?Z#+_R1Me61vMT#*q z>2rwB8z-?A1D;t4k}uKm=WvL{{*IGa#o8bUQg;$!A-OP>P`fl3eb1k8pOPfY@qvWk zCH9%v0Rg@oKI=+%mU-o9EVqeuPAeo>=%iDc4*JEAeJ(2l`0KKO8vOnN!GG6jSpE_i zQr1SKc6K{Y@A35j8hTqt`R9~@?Kzx9S>QAP;2#>SQV8@PAA+Yz1j{>aUs=KY2UE;z z+0|u27B7c|Q9&v;_5SiVD z8GGw3O3{#uD0OwD9#yTJ@dm=4^<02%!sl@!kK-BpJm*s=%!=e@aYtSbci}Ma&*MR| z09uA?h<`EyYGIJt|t^Z~>e#O1((^Ki)c*#eT^ zW7)TJ-1NVb@Rcg7oPQ>Jm`g?u!BIVm?oU7}L=)JbAsk6Q?{?18M!VPUIL)nRo8P?J z|F-z}SJ!RZ?VdB=-ag!E;?EB%@lsD$tF?`!;f3V)A(fWN1r z*4^!$@0<9eQh-AbepJd!N_lxAQ@-25lz+XHDgXKxrN}ZnKY!s>FHcWJ%D=WT<%b{s zB~pI)={rbyt;(EGnUfzx%FFF-xYx_=?UyR$M@-q>mMJf{-L_@_f_?gVm!AG5R<-@| zdr{TS@e!o#9IM-HAJe1Blv8TJ>B+WA*`<`-Q@F=u*qO(0k6(6ebJMi!KlqCji~{oQ z{D^-K@$WJI#eX;OdKX{p>wn?jA1HFDUfB_)oWNgf@AmdFzQ*GpF>;ELQyvkm+}@$5 z-93eyy!-(l^!p!9;qQ;ofjd8LALHLs?1I;_Q|N(rPPId@7C<3iJJwByZy91Gv zmlXN#JNf3n`vFrxd+4Pk6k<6 zrr~t#Lw`aWN$=V+^&Hnu1V+R;8f+)QXapx!C{J5Ym}|jK{48zITqytsVLNKV@zrWu zr1baF0A@5v9>VvVM_v?h@di(Mi+afOB$?o*h^dJTG~(a`9*Ai2CvLyq{CXCnXoFXD z_WDq)lxMGx8EgP#PPOFaJg1%C+3RaInzKWtlYfpvaTx~-kCQkV`~iEUi|fM~oO*C2 zeM`9)*FYphO}n^$>&4y(x(+%Eu7Ylwd!k2pJ@7-%OZqo*0QxQUxFn8m;Ie*}Cn*d1 zqC_7y!I{)vy`G00^At#ogpLFHih}RYR+~GI6*RD>C_r;wSnJM&S0`)|`tukJCqUsF zaDR4U)3x8hAPFz8p|>ve9y!qt3scX>68O8m%n|Gy?z6lcTT}iH4oX-n53MPe-(n&7 zSyO@rt*Znkf0jwr8?YwPzNc08v1$zj~Sr?dpMcwcX~F7uM3A0Hx}}1r}UBN}BtXL2^;5U7Zrw zP~?A7HIt#F3H<~dxsKB+HXeT2U0Pe8pqGw2y(9a=wXcAq4y2|1O>4`v&v}&ddr;!( z-(l#L_XLLCE6LD%wZhPgDV&{GU9`PuY&@LH%g*Vt*}F$BYfEk*Y%da#Xjw=oBy-%Q z?XeCFMbor^Rb`dkd*xy91v+i}kKAtewjF;bk5lu73vaKcz}$Jz2~u|KAD7Ou4I|UC zqc7aK=5|hQ0K?N>&qm1S^Yd}IoKOf*PJn+RMRq7S4sZQ22VYP(n>P`6ox$QGxis!#70HJ>i z$iLwE7b5>cO&pLOuf@aE&Gyv&7_68119yEi)^%Qq~vZ}iXUCo^!w88 zvkpv_dK)+@*cP^ttmBZCXCTCv(Nt zIFgDlcEbs_+DkjmA1(LEaW*y{((UEer={)Zo!$+zub3U@u04?VZZ{5!=Z$%6buI7K z3vr9swyWw7;q|cMOj34Ul$wNhlXB-C+o$$~x!7s%**f(K^uvKz1kNXx{U(2F;PTYq zG*lK)hQ8~Rxt&Va&OK{szv=V_%zn@8cdk8^Px>Zn2+rw>nyKDHMAuP3?Ty0^#nEAL zbXpv}DUQyHqx0hEZE!XX}`q$$9S@GMi#lMd% z+VF3IL(aN;9QzfwRH1w8K68Jon<6=Px8PrWCVl|f*0;E&dXqJd{mPo`F74Oy%X`IN z?w1XKTnxt6>iPu*|_oA;@rZcTpQiE9nUuR)ZQZ7!`)B0 z;lsRBJFIF|f_mdPL+U8Ev6**FwiGvaH2NUq=hS>?ydC9_K0Oy-86gT^lAC7q2uNq?!l{Ftm(|!4!qz!C%R4b z81$7r%-ssGQYH6GqtyFz#hLq0l(yrJ4+ikT(t=uF&SEaha6CAX_VWG+|CtY zcMlhy_BU;JY5zOdcKCmTMt4t2)(bet!I#vfCT+qJ^Qz%mYdQEHy=mL&72mX%_C~C3 zD0ZFVktf41J|dhgev$8~*RE(PL`}#36Kc|$J7pRJD2&%L{0yRzOT%E!nYqce#JGKgX)y8l1dP=xhqC zwUDYbVmL&Nq8yR@b8Ne`%wN~uuf%Lv|LH_IeKl3@RbYj$y+G=~Ro#LKl?7@sSYowq zWybz_n4}GwVgJaDWN1uSOKdqTwHNv$SM%~PjDtpycJ_m`gDJOJD|khQ_k64JaEckU(qwy7QXZ|IY0^wNFy}b8K(K z_AfcV6e8+{i}FEX-e+Q06i0!*ku(Vz=8 zN2G@px-`_zM)y+XM%MV6x(W5eQfqjdAn+&GZz&Zv=>>nom{1z=X54y{s|bafTPdhX zNho!Snr2(_f=fpulF|{^x5k~$RpqFR8gYiRGW)tzY)b7icd20u9>)(7&5Fqk*0n$E zuKJsDlPo}t5Tscb-UV`T4`CVySIReVoup|yRbBX6x(8khM!mPBsyimV?uD=ud>xL? z;u5>^6&rt@pvCARH`qG`FQS*brRewHiRf9J|0u%m!)oLx@}@HuXoYva*CMJ`D6O&} zrt+wb>Xxs=5fo4fWMx1Ee!nqqLU9OD8kT!KACi5-_1H^!lVsWIMiGBzDV@EU)E*C^DO zC$~zadB!aZMEnZ;A7n%*HATYpeVls>5d(Fh2*1BNqo8a54)@Afex`h&GM3+jQ$}{q z$jhK*Vq9f&39%f`=%>bAsj^cTmuP&L(NLXvY4-mR_jb!|BU_``y`KWYc+-H{015sJ zG{k>Tk|kU7*|J8KN17oy5=1sBj3|HuG%d*jPo2wLo#ZN2sY+F-~smL%|kYHs1Kzu6I<6y8)YK(JSNlanlifqm&(!!mzp z8Um%CzWBNeed2&qs zlVLZQ81LXnexs);J`mk}`a*54HTn^?0p(;YF~g2tDjA_NnI}s#O;?MU-5F)t9Ti9V zUe>6csW}@bCNzIXy*;UR_{Si5?Nxuw=X(%LjMV~Vf!FyJy;Qmr1NA(w`g}#pB{v18zt>2{!^S6uI7mB96%HJeMu{ zG{o8B+1k%#gHL7gW%xZ2fMCLAWnT^kPg#ujr9usk6iRLg(g4|>-GlhVSE7(!aLu7# zNk1wF;n>}BzRQFJ;X=W8$HRZ!46YdaN>igG`ESwEh=)u+R$7|=_1|Pn=-ucq9M{{8 zoMO=d+4hzGcl9sHheAae-!$JXx#+Gks0>~YUK_nFWa&x~-ZT6U=`pY8{hPqQ!?LOW z0JmfujZ_bCd1-wU<1GWe0_4#L)V9V+uFAK}9nZ6_<5vCs1!d^}0C;~5mYKdZ^%WLM zu_AF2A?JG#4$VPQS-JWFlm8iA=#3K3;gTGBKN#1Y+>(8AOy1^hz}%h28BRA&E%xn+zgV}=4M{jcUttRxg3Hl zXcNQi*_u~Zy$BPV+MR#VC5VTf)pa6#jVzVVW%yjkznXciMlAArrHH9|u`0||ZuxMDT!MdhHMskq*Fa1G@-!hK zTqN|8I}y~$iVGF51Vrw5luZ{gHBJwOMg$!l*%2Z3ddpkV#euReH^WCcVD z2*Kk&2J)i-llO|t5HeHYhfB8AzcdTK4p&_A{_KDCKMURc+!x)S+*TO1t?_yEC#Z2L z$A;h`xKDp4yQjM!KVR;DIzHPyI{xg=DB1t8`}yA~T_>A~^-fqP7s`Hlo(bpPe$ zuJvtLV6amA^ZSFn-Q#zL#9GM>y~cp-+-%&{LsXs4A*P911LU8i5I(Ha2;`V%!gKHv zD4EznSzQL*)mL^&goS?;?mg&?l+SsJ{7twUA(ZF^SslkD zkU`_@&&|Z*2LkQ9JGeMmcOsl`b>6Fxt@>gVM52JujTERPE!M7#8v*`VauS%JB;Rw% zYoJQNzl6J`l8gWE5H9E^P}WJFlQplY<~^;AF|zR&yA#WlQ zllky12TL;b3^*iDb0+Wy@3<(7Jp$IKap>(FX5!el$cFvk0Z-fHXIMz@HI!T!`p6G4 z$N&`9Ah?0XB2Cz_>t=tWY3akLOX|(jUBV zMU;H(?Ht>9+5;~d&wqQ9+uLXwN;ow2eN-R{zJHrNcNLuLyo8kv7Xslo zIPxlk@smSp%R%9xv8;d?2A?ePLH>dWreX}nsnf4IwV`RagK^n>s*>7Ldu{A~Oiw-n zjhodB-|@i|#m9vq%%p+l8d;KG(SFiiE50FSBr-`wd;JqCDsRj?`kU{`-+W(ftE?m? zWRx|}weo*lhksm>hS%VydVD#2Z5RU^pUTDoN*?%&*)!8(g4sVkeEV-(KMi}8oaj^Q z`O@ouKKd{;)BZn7LsQLFdfy+6cdbKcxtGUv}Z% zp;(4gUcGu%A)xm0^dpSn%9}x@m+AZ=YxO~?4T*28Y=4u}ZH4?s-ShMh^Yq{&QavWt zRlFkp8YByUrtSGjFw*SfgEMuvf4v58*c<$GWL@7Cd9!OBr%jfdNC1C4=62DshD~Vx zbQ;qsc*lX@KpLdL?N{@K2wnqoGmT#mwQKl4rW5V@@vA@6z?U*!r34|2J<=eas~=PD z9!HakB!Ajyk+A4a6GyiuG*2{8D#7N*!JQ_Z&}!odl`1vvBUS?i7mZtQYcw2daK39a zhL+KQTf@+b8hr@h4;*m;9)_Sx1LV0jfp}VD4YWFO-F)@D&O*Y2ox7vSld`DA7Uq!#+ZSp^>aA$?$Q*f|GY7)-=g)(lVM& z<9`I57W+X$`T9r-O^Ye4c1$7Ahy+J4pSz=)I3BsNJa|jecwDmteq%b;;L@gXMIsHN zL~9UEPvbX>MJ#DeEm>S^0LWkgsWJbSY7i_&1DAq=2f!kY#gCc@+odhQk(q{CM;f@W z(a_I<#;<($;Eor+faiMn!-IKh5K)Tan16&iR7NerR)c6R(L{qVP<}Y}XGx@?DwM`u zYJdegKp{}W@cVE#%hx+dv>-u+CoM>TxC1{S3#x&yT7V)1jcCH50Rl|LS`_0(Veuo@ zEUd&HqF5Wjokkm7KQgL~z!{Y`0%u^_XnZ}>Mq@hCMnTB5(JY*5qv-liZItl5$$w20 zGi{VWdctCDw7`4|%(%d>I(O(JuT2^+2yI{!!lAV$q5=pf#M&$hry?ML#@pge!Tito zDuz>SA+s+qzid!a=cpH^@kt6J@Jl$=b559EsicF!Q#u$dQ*`){PNQUToeGSd-qUcL z{xKN5)YR*8NCGk@bMi>0K%hQD%zq1;qMx{?rHs9j!u+QkQKdBGN#Qt^eyX|KhcuX^ zF`WW;0A1z?ZNv@Onxjxr`3e@~Nq`Isl>^@5l_RVdZqN_I!X^17(ujgK?B=v$E|4|4 z6;kPfnQWJ>sPS`7iKKs36U<7ti{~s_TEZ1uIGK^g$k%wsjxSOD@jA{NAb%oUy7rD81PrPPbKG+&1^Gf_hBi1AkW289Mo%Qg)d;@CMJt!U1%a zSoDvR`-kRH+v#0|s+9b}o^pLl@g_rjXI`37aV*4=ng3(Nw;SEzDqd)nP9{nxrQ_Gk zyDFvGH|b7QI>w0Mh{b%vTz^5TUomPU5`O&~IdRtvH9hhdhAV8W!%?p2n>}B+~gt9c90kRhj0S05E=U(l8ku)rbCIER|^>J%1GyJ@ftTP*yDMG7A-(pzd=WgntwK4gJvQx$d)# z@yhVhHqIw5sMpe<)kr7xS~_VpOi-<`iy?au7*;P9>jou;s3H|rnm96}=la)4>W{}M zM7;tBuIXs@E#){-M4Y(nCyU{r5*R(VUQ1gIGX>{ZaL-Lg>{~jfcM+q53RFH0zQU-W zfZ0pwU*CFML4PdkfPYSaOV)Hsm5fuSyv)=uK-CyM*XX&O)M=&lnwfo3%iQ@7>0J<} zcZ*pXQXhh4ry-s8(kM>n(R>u5-GJz4y_Uw5E62x_4J!RfFwDmm(>ts+jU(2d#ltkY znWf21%oEXdfYO^@3f@OkIF!(hu3kuvm!g4;X(Q`Q>N--9xu!R8`+IP2lW>5{AoFwR^Z(TwJ=jup3g8q7jO_5)b z`-Az@hvnt4vLMWsgsV&jD-+uqzIH1MldPmcQsCu@LTPgWs7~c-2u7~Fb8;ntZd&>Y zmFV#4q3>H0{(n|k=Iq@HXyE*fNtVd@Q?JjhV|?qI!-H5B*(PwKO}v~@Kig<70rth7 z%FSPs%WYuTEWryypq?}=+J`1(yNUzvD6H3wCS(BH#=8w5^-TralgGk`+xX>Xcr$ps26omHH4tD;x|TP z2%m>(fA|w(D?B7){0pDwGvO;L0cs-oI(_DrFRmx2urZ3Vs%J^lH(LnAxxr$AJ+Ig$ zkue#OWV3npztWCcbpn+OL7GZZKGgTOEUQ8)7Px45QoFV^v#z|^8ua(y?w;)`eHsi` zhO#8l27mgT=kD!PHo(T4LGR&i_Uwd=6(D#sz%48N-*QMODhR5~B8*3k44!ny++Zpy z0EdUIR*~;~SPU5VW>CRQIMz&)vsw^v_E9O6pqvvFk)H1r@{WI?Tb#%Icm2X#-|yI; z{#QTqRLHOP*M9s%o!tRnvpbb%bL3~jn`e}0LVu#Dmccmj-yi`!8tJ5=(%X3sxOq@) zA=ocqBry~7?|^{PJMby+O#do7sMGULwx2^~(W^-j7cPspyZ42oRlO-c(nJd*;nzC! z^!{LQ|5LZpbN{X6m0uG3KfQisnEkhWJ-=4Rc^u;8X@x zRex>3hUrUFVTtM+GwaXfezh_8t72{*!%$rqdN&8BqE>#z*uWWoQj}8%OQq3&s9LK$ z=HaHJNL`8_47e!$CIElLmWDbjCiltTHuk+!PvO{q2Bh_U9qkAux;b@_Y-VV08kGF_ zYE=!He6Le*xRR%3AD~OJ)i>SMa%~C-h<^?k_^GipEPUeT4JZZD%ev7&W7=TAhWWRi z;i?&HOS1`fo7({l2K`JYy%5j+`xU(P`oXhoE&UJ6vOve=rM`01e#ai5KYtjc zJ3|AFPkL~ZHG6#xbXY<|E1Ug4v>`-64SC97P~&gq!659JuNBRiH~;AW!?T7x8S3^Q z*0A@F9lfaA6E7f|jmv==5(52KVJHy&Qfo&l|lQmi3+Q*#`Vw73<#%GhTXD zL+GXconqkT>_-b;FJ@@6D~ei(JAa30a8Q^kYQG26P%s==Q9NM^?I5|ZY`I+FlomI! z;;x-5K3%W03u=Qil7JsAbOn<+KR>nS(cCb}Dd#zkOmdq`@lU0c{I}hY&$zPBiStLG zKJq>CR$6WUdFws-@puu0k$bsKASAE5zfDH&*)|!?=kD<~8G+zo?(V`<;(zXKlW{Pa zxQE+h8o6J#iLjb+Uv85i4BWrA3D7|9d;ZI5A@_WnKp+?Qw{3D8u^A|nY?B#H64$>a zvxV<-Md>7yVF|&yUBfo--NkeZ%G0}Ji8?r zAlW^>CCOhiclVZ{Z$9_iEwcK>m=15r*Xxin_scE0_meyKmPrsw&<-HJ|pW4QzJL^FS9) ztCmWJOnO?H1Z;)*n@Hs;L4x+0M`*e^_O_g5!HV-~W2UM?qK4`^=W#bml_}9n{@uK* zn|bOROr5bp;tnQGi-|fW{>@xEyFU>%LtnD%=&cdO$%iw2ek_#$uQUtcn z$=nZjbTaeKy`8PCbFNNB63;j%Puw}ZyUz`zgNbo&^BW4#BSsb$)74FLZFZHfc>qmI zNAwl&)A?K(##xPk7wSV>UX`e1%K3z~dxd|rBFqc;_lNUT{gt5yA3r#ev z*k#L?Px_^9bB9R=3(JnVyfTMM7H7 zd|Bj*taf8*b}C1=Z|^#6D$D5S4zklVGZ+nVA0%`1LCB@+B#;-Ha3JI z0&E|?JW$sKRr}8?DXtj8!dhQ<9CK|%oND1kcV{ywofsSkbOnfHm3GbUl0{$!myZ$bSJXU7fK!StWkpE>{DCrmkVmT zu(8BU^HU<5Wtc(bYla7R!C@_QA!~7$nE+VIG+aT%jxyeD+USBOI!}AQ?{SBjl`kv-?-ZMN?&80X~AEiq5$~J-iqsf`v z9)Dbfk$C&GfQDdjo({2nBtY)-rHUD9i081bLiLh{WByruQ~+yvdFhYGpBL8>$;-8t zhZr2@02SQ}S}Ga4Ola(0Wd{VH_T`o6H*{&2nM)P{^E-!CAcjT=qJ{C9wm@P_RKp6s zt>Uwl%|Qx;!Biy-WTd8Z1t`zm^UaZ(9)FaQ$0APC`AVFo+toZS&+}T%wEs+|o|@?x z+)%1C3>=e$TDcH>Z< z3)ODMePv*)j>C#WB($~|GoO4`_J88Ai(b2y{6+D*QYB3IvoT2=7Qe|aWyW&nF($#c zesu`9g24cA3F|w<9#jqQWw9>Yy_JC<%`yv3*MKZon%4mp7&^>)xhx4Y&1Xw9n~^I6 zi8}L2zz5(W;abiEP2s%?Si0*XFvf5&xPPsKHMpLI%HT=26uNdj7>vunoqvO0v!|2a zJl(K*;J#WoEa5s2D2GXnVgXRXkVUy`Y^yx%@tRB>Y?HZjlxuKc4K5-ssi$wHTVA^5 zdFemWe|TvrEtS%emktK6;U7Gd;h&e1)JiQct$1lAb<=M8=1uzMjhDjzspokq{GaaZ z?7$x{#h;xN{!a%3=vH4SpnnRB$yxv5$@>x02y&>swv!Qk_7gYB{C%?bnYmEicj(pMOlIe$;{jB<10%{9Ez zUttaK``sEwip$e$!%uy%7^SHn)AZ=oT^c0eE0&HU{+Ht0J-%HpSo-H8VQF#`EoS2s zbU)Z*nt+P%V@g8^z?M#0`8je=#)pn1U8(dKbkM zwK>4sa&}z=oJcOH5`PkMmwEY0uK1ol$ARzDq6?f0YIb*5RrX{JD;`k#=o{TL&OW{U zlpY>kd^|{xKb;+J|6vkkiOsD|M&Fb;=*)IA!7Z}{FU$eM=^7(dTQw4 z(#)4p;*tox0Lob2=qHV<-eF7-hCxuz*(O-K2yg=rLRF@^u7_2O5Yy#`tGo2^z9)qRQnoOTLxkiuS%2(6TN+b+i#xZF5f3bKI$zkZm7Ly^8HrKHf7J``%KrJE3fz} z&z!Dr_P5@j29;F<7k@`9T|9m!-|>C3{E5yQjaYQAyn?TS+}VfGHQ;b@!T*O`dw(6SBcmkNX93tF`YD zg{kiTMh=eOUVh&FcyPJ<`Eu{z-O;g|Bb1C#D_-!=$&-rn`G56*<9uN9*_ZeQQN6cM z`_6EQX9OtSen0n?$MbW#t$1LVOR^?paNiLrynq$|xO=w${_^)3={_mJuUsN>55Ssg`V=6a6Y!CBqHNALG07cV* zZK884@LHFcIytv5FRv+t*T#dudl{1R+^ul_n5HX@e=_4y3ot8%UTGY5yioeU=<(L;}y;>_bMXl){aO|m%^uJWd# zWcjs`AAiC~uBo<2oW|6YPB@WG};sow80v+wg;Sf3-hbE_m@^LUIMtsJRj^Q)UXLz+_z$#=m zi#|Z)*L^>s2B=H6FR1t#Y!y%u}z_t<+{lFJyxI8VaKu8lT}2Hc!} ziM>qxq-2;yQ{(D<_w?A+P!u9^EXf1Sx9;yTh+xi_73B9{4b#*plz&ShSO7#;2CL3! zf>qfX&ux;`;@(!(T#_T|W&1o|uv?C12!B{6xfhaIvdg?`PQIV_FX#|h6;DfGh&*ve z^pKA+&*?Ju1{jW`P1a+~lEOBGCvReUe-f)JeD26IH&$|VKf75^#%|6PK-_kT8+ z6HU!aG$kkEbhLkE!8Mu43%W^GiNq87R-p-CDJ~a-x@h9{ zha}Im%RGm_&Z7jQSbJ3v*PJe9)L@KIWi^7zcP<9 z#Yc{D784qn-VP0Rs-@CdoZW>?BG;v`4!`EAe;SlY(Mgs3%FGoXER>Nb%*1=UY@_gd zF;gK2PN35ja2iF6ka^X$NmJHePUDoak9;a{GNrIttrkwng1CA0V?MR8y)J0CAUlWd z3g`KZ<+Pm9{}l#1glS~91anLSwRzx%3D6@O8wGHtY>5W^K{)+9^21&B*}tPFVn?jX zfAF(WZ%J`ZFsAYDY*qknIWH^wP8kZ{Hkdxdjsg~gJ2Yo$2w^HNLAjiRX%YO?X)m^I zb}4%vh=H=<;HZMFsN(sVD3!L1Usr48_iK*8%FY@*if-ZLWfQ3^f0RjnWy-O~LcrmVXmKpo|@ zzFb|(Bpm{4>jEz1UIlO|m;4sMXnYrhU}ty`jzPYa6<_^Go_KL!RxPotxXG`U63H^Z zc!e^I<-?0Zd3t5Q<1<)qs3F|ovMnu2@z$2-Uc22Z z9|2IjO?61xX45w;^eao#3Wgn4vaGcHIaXiEI{Cmcn8k5{~6kbfenPkgnKn0abf8s$g2jEo##hBG1GeZS59A}jw6^+PCgn5LlA8;1pQeLpVf+?=n#Tbe=! z$q;n66qZO>D4E5YE0wlje>aY9==j$x$yiPD@&ZNRP?$2Hp_^MJp}r5WK$aYX7OLy5 zCb&4R9#$c&_~a-cF&hxpnsK`_{b^p*bkCO{E&7>yy5rdJzb=#_<2Y0N!gb!HtIGAi z_N9U$7%g~cLVOgWW`7ilSXB5K%!j=TDwQ`MLoX!X0xuxHQ;!EMe>3zj5}aZ(4ZJpa z3_MlXlc^AF;0uz?+RdP7-XG%Gz(P1BcpY)*>H8q>&b>e5p5idwZ~`y zj<&d(rv`o(Dq>40;47S8^>kzq3AZ-83l@zj_>2PCFt_M90UHK%*rXTa+Fij;dg9WN zFCxCx7$y_Ko#3Whe@hLc_rdgL7EEv0DRnD<7-3}fZe)`AvJw#4!?)NG$TdK{Iiu%7 zve~nY{@~skmO)%E9OiNrUsu1G{h3%g7>9gD_bgEi?$^Qp8F(I*HFLlvm&mt*h)j=) zvwO;nLq!Xd*@Z6Kj3$^N-*dn60x%tO<{i>}kC$~#@6K5ne-b%&88qyAg*RI<)zQUL zpCA`?bsjh=Ip?(PNk;zcF>g0EPWatX^m>n_sqsr}#9^*${Bg1)PXtT2V(-I z=K0i_uJJ$(-KpAby9!3ovbzT4AP5#jgL5&SCZ=}dC)zcop*GIgwYF{&wTi2g83mtk z;7K>Hjn^a9Ezz2V`eXaW4P*1ftyHD~yIEYrF;>Y6fBl){&-R~5B}kG5O)9loyM1NC zI1VY;*G79fD3JrS3 z7l2ywOr5LQdG)KQa;WagH)Kje(959MG>l#pG088h#P2(>-Qt4&CzQRrztH6pGAhu2 zBqouJ1q>W9?+_ZnTRfz~jy~cn+jq1tb)<&!$eD%64nr{Osu_%g!9tQ*xMC(*Dj7%2 z+j7jMs2PEaHpgGq{3l|GGcU!>@Hm>f7hW}(*~uQ0mEnDuit zwc$`P#JmmRdOQowU}D??PwF;=r8;6JIbthQnwdg=ieUH4SsF8xUG`uc33H7xD@cNY zkAq)K0(gjQjhQMLE|s=66@dIEIgE`lBY;|pg?d#t!y5`U;XgA`b5=Ncq)oto5%azP z$|vtK5ZpS-7Wa1<({&blzfeHV61SB~%8T5WI5s>)$=X-bbe}JdyH1o314H=+12^4S zSe)KjmJ^Fe9Wh{JC5+!q)gf(O+P9G*7xo=QG|UtuknrdxY|HbaxSQa4 zB3e$E9O64_FG6l$D-!{WWZbZXc{?h`hDkztGY(J*Ba674onyi(6q+lQIE{n^3+!Zo zI4(4otb16CDrAU*HTg=Ci(CMql}d4tNi`$|S}KhhCxxCwD*(|bgpyf*GHSK#m}HAd zN>dZKfiuvqs^EoLD#>Yxh&1En(;)>jlrHDD9p(23I9=dAbHwaxAKCq097voaZ;s9q z2j(S!0#}A7XW)nh~&ZkAtv>93R_L2eI8aZaBVy8<(>^nm%YyGH<6zte0S$nV@Z2Dw6V z&BL;H_2P;+P|HEE%4$GffwBUBUwBBKjQ`}qlpj() zyS(yq-}@~zqL_!m{vEF`zfcdD8RxzXZ~|Ts$eB3B;dl}hD+>K!cVl5BGRU{ktCC6V zRmnl%RSBGkPM5`x0Q_K-i)GKETN)bP$xwm4=GGOe4PNQHxFg5ro5Lg|YNAfv3^kesTwp0hY#G7RJxFIIath@_ms7DNS|#BEA(03$fm87`5~CI>{6Ir(V+6VB9Z+X zaPE%zGzZXsOjCt>nWU+zwhFeS8IzNDpL4nFNic)9cVkmN-Ur;s(T}~Iv)J%sa!ARR zW18JPmYu^V8At0O^!AvYL@_fU?-wR?s{J@8G`2kN3sdM0Nq;M8mYToI%g-{><%_~x zBOU6ounGnlO(A*doPTV5%bjJ;UphAZkqCm=+YM+e1)Q}o+6u`3UFzz>}~(HU%{Y*U@EC3XsIT@uRd ztOOl@n+3gHT~q-vGQI}UBAGpEN@iO-i}H8PpZcIfItk*0X^Rm230-Yy;)-*^CBG+_ z7>A0F0%>s!S{{-+U66t(s=@bu92)wk5aWY@Jt8usIlCP~ns98WEo5;G%G=u64aTF; zd@VcZXOCJ!V;G^9(9t3e*rOH)$t|$2i>>{CGc4)*;5Mkp`qpYXyMYJ@e11_qwAfE~ zwE$8CAoN}n45<>@IR58n1lGe2!jQ&TTNlMJT?Nc-!14Ql>O%&s^%K^MLSv692iplF zMNtA;8@OKskXCfn`n`vcJ5+cE1U>zcCOY9;?gZM)H#8LCr-=8YutVS&XKAQN=DuRcT>#Z|kOVxw;0f@84Jb zDf|E)>3_HZ)UuC&g3Mh`06R2i@KMo!E}#Ud7GQ%rvd7QNXGrFQFqSiwccV1D55jSD z4;b|TBG>?Wr6G+C{q3iZ0$QPu5Oa~Ad;~fmogyr=HvU@B_)!Wxq0@}niU>5y9X23%85(R->9wrC({~qg}fPT{QgrQ_UO%3tn0iCc<`-gPsoxmw($Nv4>tPilo zuR+j=a@})~nAyVLV5u|-jr=v=@hEbI3u`_j*BoMnE);K0VR6kFU?r8B0mmZz0QEfY zkd{iBF%qOH9zP;u)$UVLLwP2DezB6-4yk7x!c|l^i!vcG4QaYh`xp4AG&Dh3YH^Hn zw#@wY92RxS52*(gAX<_za>AW#^iNi7e}>+?u)XhBXisr3%*e)4X=~2xHN^tOe22@; z4{3kxaG{aJ9YtvVWg@!nS?|k>>FqZE$V5}(1;W-0$0Xkd4&M=XG&gieA!D9EG z!I83e&wb82x4HHo=$it$Gn`94rDT_Jb=-oQ!o%u%2?oNa39j257!;O+ekFt|_M8z% zb=8wm`58fXNWrtgnl2b1j6K#rVZ&01|H9oz?JxCJSl=w8$9d4eUjxS{5TFdg&9NY+ zvqx~;_tmM{)fEA~zDYoTFUdfT$_b}YmunW-S$;PPNq!0}Smb^UhS zh`Uex@R1t^ewv&GcQkQtmkO3Xgt%tf1v90`F@~pxqC`s#o@RERF~z@N;GmQI9D9#} z@sve~)P9cbklr6a316j1!|)8`VB-B8lb>TdTgjh4ixd%D5#A0asJiy+tp|=*CEw@P z8!wDsZ%s0q0&na5))Q(>Iic}e!GHX_mvtBeJ%9X`yIisr05!%NG&I-Qg;#ocT-Gx+ z&AfBE2#ChF8F-D5jE#%if_B(L=HcYlQz&+0PM+3C|NOE!UgC}Q)>e2l>;26&^YZ&7 zUW%^XD>teuoji^FMMB*|b;~?-@omm+0e^7h*xScDhFs(0jE47ab8`Mzv53A^ZV&sa zYk$=-ndfU}%wf??WCl`}rjl{`;8_m2^{Ql_3(?Evhc{DcDJqK}d(-~IaKlO-o2Hbi zixIu`LgN@>4Vm4yIFM3_{}y;MQnzXK)6Ojj61~-AvhTgk7?=BLD(nSd!gQiI(!ZuNzKFs($=6`x4V?|X2$o(RA#@w8X4Ac9Fvxf?eDeG<;d+l#zRNWVy3V?_Y+H>VI;% z*_G-ra~Ub@!wM=!G7{t8cvd9=#3~ov>4&RaBydC+luEg*54CW60cSsyJ?4fhu2#i3 zxwbB61_RNqXxp$=*OkH(9T1fNlm|;jGTC78x8saFhO?G+w;T3?&3&hm$H4rl2yrm? zL!F4IwgCiJ&UI${A$Ss#3mPp=cYn27+qNUc|7=FS^2H&6#}t3cMZI^3_$^VwsF#!# zugN3R4JD+hUWUub(uC3e&3w$fe`&PAk`#OeM=N8Ut^3WL72W2pImT67g0A`#)pe=( z{`+k~eEmx?Qe2w;ev5J9k5@c1Ym-_nWlcEso%x`Pbvd`9=9 z8G3*euOB0f+m*ea#eSGfq8LhyX2JZ#XHfHm?#(FoC&_F0=(9L@5bt;MD5N3dWx=oa zr8~uioCP8E<9D$?4!D1#PI&TXNbhGAEO=dV3xu#6QGlF5?0O1rTcUq#cu~nyav#85n%aQf z&K~EmIYD^pxT5+S(Hf@KW$GBVF;DBrv`L<}ot2&{rJbrw9vv7m%~{PVq-7#mnFo>K zsc_d=nK=50c~m~~fw58q9PYy*p=8406mzL(7N4D8;Ti66O2Nr`!7L)c-<7>ty(6NY z3sOQ~PLF^w?0J9Q=ZT$so%Uu2ZgfB*nFR3HqnmFXZkA4FYW`9wDp2q}D(QC@BH2-3 zvF|b#2iFTmc{oIqj3kO4D=2!bP&1=`(jO0pE5!{+L|-}G%xg$QMua}%)k#$YcwC(X zTuY2YL;`sdMM~?1>$0S-f36u}KNJ3D`z?{7bSJn`d%Dn3=`D~oZ5 zz${87aKX&`d{;QbG94^##3)n&jC`V%6EW8yL`sBDzmR22f}$*glj+>27=56z7*xjI zjaeZDyc??O)oPN0pkd`9yq6H2UUM6i4}h1;p&>;o(IT(z7pK8eK+z% zE;8fU2;c+B7U%_KPX~WT;13ygW^eB4|M8#y551ffv79D}On07DQYDdz^*(lIWv7`j zE_#nymKlM6FJ>95a+P$G1Z8ht9+w|c&0?w;fCbq$oC=>mhysMKpQ)Win(+QYwkC@J zos((4kP(Ttgm>7Pd@!VB+?!dENv7&@GBRB*)ykAMu-W_zj(2*HtIMcRnQLiVcbGcO^%&}B>V^}wqb5L;zvR;J`yEu~a1SM2M50nx3O&Z@fMlH4D7PkKprMHpD3 z@4z#}c_sQAc!W5xS>g!E!5*`(qw!5&8UrTbismql=`h$mR)fzn1s>9a?glnt()EmC;nTDH|eyYB1 zcGA*M57pOoKP|z`g=QrXJ}mR0_QHN`2+}Nn7sOtbKV)bz@msaF%LB_EzJ4hLFd=vy z5=TYfS@m(EU&eIig9{XWNe_x6C?JkD=u=rihGAAq9U zo}b{k>jGyd*}w5)f5d3K7d^NK2LyN7D4HF{(cPDL=3X8w$>jt+Ud?CzBj~BQf_9UC zj4Cn~VYNXsPCupLn8sisd4sADyc}AM67Bvb7~N>|1xwHw*J>u@8)-%oX@fx!f?48N z8w`@g9R{74`SFwz)Sspg{@r{=O@diQG-NbZsssk<4b|qLc@5WSmS`yF3PNh@WHbw0 z-5L|kvP8oc^iE%rXu$xH^YJ~=aB=5<_H`f!cf;{VpWVPcM(5WV1X~EqzAO$ObOgak zA(h>UZx{32ix_ZVZm7l^%KCB$eSw>Y&$Ktt_>R6LGx}E;{m@)zb1bG4OUlWd(FwC4 z{8umu=vZU^G%;;`NhXl%pRay0TO>CcR0@^gzWc;|z-_3D-%YlvVwenu8AcX=ma?MI zqWnA=&4RZZ+QqMlW(0gS5ix1g;42Na>qpJvr7cX(d`R&Lj-z{U)#YpXsReG^>6~j~ z-_TH_A-ad9F=h;TV=yFqz&o)Vkc`(R$D{u=8da;+s$9JhDRxGCgFid#1KSmY;5CY| zh~bIle-%%!Ujf_fRUA*J*VnIqG$V|(pG;xK#8oJ?aS_URt-#CYQx@X{(lN{ve7Be= z$RX||jz%;|qS&S0j{cKgrb}|?kK!niLlzuDF~B<*AZF0sjOk>FuH-8NQPK{uK{xLvlS#3RLbA< zC4oISlDNJvkv&-6f$X43f8Haa(a^ttMi+?wY=(w}dH*0(B_2vwEgU#xIU>-gP-{|@#f*c+mvK9lYFz z-m2uS2mWVVrpVI6H8HdXuGtoM;HXA$k3-Hh&idx&@^(lT#a*dW$WsMSNcL})r;1}* zq&_Vrz5EqMg%qS0i%cSah4?FS*D?~Skj#(rV$#YHUHG_q zz^w4}0LiUzvlaT=xQpt$CNE9>4ma!EH{V#E}@#sg8vQ-sd_*tjirT?N}!Fon7tRPfk8}S1Q@gzU# z@Z=2=i7{SbMX!JZXM|;4>!Z+jsGB4(yVw$*vOUpnVf6k+U+AS$(PaK#X&g#!XxnuE zn>44F4IOOEM$t43zR|JvN-sxc9Xn8Lf4N+LB=j>@vIo>pkh86ScvVeM3(eP+t9Cv` za_OYL%gR#swj6iELie_+uCf}&|F_El=Gm_C|MQvi1#9o_Be6J6GIPozuUfz%@dKVDyeih!U*?XK2NB#cNhxh!u zzaTaB`%wVz`S&Oxb@ls64DVHQNRAJ@zP}(x0pWk6gq+0u?~v>sc>SY*WdEPU+5bcG z@xbfH33=&nDtUU~^C)~4%+wa$!Ez;a} zhNRzabx5bZJsd8H$UAzBgV`(?DcPNBtwEexA)6oH(J=7UTAg~WL!3sfu4KE8{hPZ& zWwm;>K^nDYQ^_@@Q9LbH)_^{LHtJ1B?b@HWYIC((r$d}tvr!+CzO%jE>X1ei21EL- zX01~tt!BO5Qp&^ib}>^MX;kYq(rnfo=*=*vZ{H$)D~T5fcgfx)H(3zNa@ouX8!x%( zGcc~Ic7r#e&FhvqgK}A(Q1X+|@T5Cl)UOUpCHlsTfMuOgywKOB5&;c=ciIK&_$4=a z6eVwm%En|G^L8#qnIRJBVJU{RkLx448091KY9{a1#i-ACx8SLq^(=D*vgKfpexG@r zD*3QkJfwfzZW6t0v}(07D~D$J>cz;a*=5MMvXYt7m3rZlX|;D#P6tznd;chwnLn1gIG1EK5BJv zER7m*>ez=`r%7s6Y_3t;elcoo!{-K+z|T(GZWD)RsMVYov#TUd?bw)OmY5_ShaeS zG&(IJANX>>xUFph-QOnd7bB|$D+JT;5DOZvCzi9_fN?-C#8YCmnxxSt{O8+?k5^@* zQ*S_TbWSGRh+gV{vX}ae#go42OLFoY(9~LWgqo@&phnEO(;@Y3yG@!k;yBwcM)d}1 z!_>4ow$o}8r`f>?t=Uadt9M5AhF#|wnvUIpBJc=bomv%20~A(a>`n0G00VASF?S80 z+u4eA8p?{`WVK;MU~(Mdz_L1GMbrgxs!q+W!*a9%%$m4=81ivs)uFc?Y`D`PP8;C7 z-XgV5ePqE1s~uu>w)x092nLSh;LvKYi)wbAG+I!)Ubj1>&9??XR@)g_4Y{h&1z6Zx zowO5NOR=QRi^OuO+r+9hp=$9ovf6x0wmO)IJFE%W0UBz}+Q`DW>Hu63hqvD#wH9`X zIBo0&?4DYGOMFQzc+CGL?G~|`+l2q%TdPX=4+kQr3M>DEQ1tCt)YGX08S7Lf>cQ2l z+fH3wOk%YVXb{{T9JIE+xn96x)@>)TI&~Wc1>aSl~A4v|BiD zoo%x1)QgMJ#>Hrm_6uB%DlP_8Au*d($D9sA8BFf40>z_#Yt)j^yJoy7&g{8e78`eIb$4b=Efvs!groNXt; zw<=GpCbepZv}-jYeqW55En+$CCgLibfciFoLk;#xtJNBHa1LsSC!Hn$t<~ssY^VY8 zWvkhLv70YOO+En#&>fB`TP?dowrd=zZrja&0ZjIu1CvIr0m!CS-|pmR6s`tt&3Xq9 zpmt4eKdb39Mphd)TaE7#I9i-K-=S5JMI2asw>S-)jv8rfb7*LDtll9`69?Obecglu zuuZnJZ!LrvI7D6~RucfHN!lZ;ApoU=ClEh>K;d5^ezW&BR!}42H)$gXIz;@wNNUw> z0`!vjRBr(U)|w3%LVJ7Uw6^(yhfCVz;9PCDF%#5Z?T8bsVb^N;7W?iFd$EEp z8+PkOVs)U2dK-_^TDxYqh~uq+S*0 zNR88Q@V8OpXos{K3FfKuto79csXz}$Z}dxht;fE+xVnewe5y~U9Z=zTBB;UYwej;b84E^YScBW)6|@H^~K0)0OqRK z0d>{uRvW8u>YC-$XO>fkpLW}FnoSLVTBBA=b6^M6>zY&F)*8+>6nRl|wzXEv(PlNL z0}tPD60Tr!->)xNyHSG~{cC+jL(if0*I}2>tgcJ(0u$l|5J`XZgHU&WBk#k35t83$ zW>>hymfwLn?m&iOj$pEF6x$J(}|0eW`Y+OHf~-`BRCZ`yMoglOeI{=5GzuLFR(s=aqQoh+-4cb{*cx2e=7axG_cFnM&4Yf z0sYwgLDuo#{}1X|YS>(}sdTx9W7+sY*72YJM`bL0>gv^e{%bgYr_X%A)oipo*vFca z&v&~63$giwzG}$F;stszMTYyhq1w5(d^h2}J@X4|f7wWihSXo!Z} zS`Ci>y4?Xp*@W6P$8KA7tzp-!y4JRviK8{`j)u=KM)+B4*)V{n#>;4}js}FA=F~cN zO>4C6js|y@rnc?aO|1zXZq)6zwq3V7+IGvXX-=(SH($ViBtb5_u5~o0i8a8Vj^@;F zS|g`s!8kfr1O7Nwt!mXQ=v~`tT29ld!5lVPa7uLAFDyQ>GYcpEroO%1u6Mp#jq3ZF z(;3xku&Uxe%+ncJa20{6ZZ++;)u>yJ)3#dMTFq|1uxeTh?n;_dwVGNTGCD1-U9;;} z9fw+NTdf9vEN};#>eQgAZ!hjTTJ>u~G+)KEyQ=1FH&zm_2nU-OxZe_Pc%QZUc5U_o z#_Lo^R=uIsTNVHhoLZfh=G5wN%xN79=HITvPfM$6FnzF~9j#W?IvRAc3iDaBaHauP z96WaGmZLT7js@`4hE!N7fF5g`FQS8kg;l9JZL8*gw6t2savB-{k5y}Goz4rZj*S7d zI2Np`Q^&&`*GH>c4ZEhn3EWI92l#FlzKq2BwO|&ZWh`X1Ixj4T18&EzSpYwH=WSX5 zKNi4`1@L15`~X2vvk-nP1Rr2uSsl3K)R7%wIZ$2w#sVU>romV`S_A$71ZfNtzrlGA^#6~Z`vivksJno&tDNp=>c8ph=kuq_0)`J zz#f7kKyn8o3|Gy1LUmT6D~YVkn#}BOT>akYnvBAI>U=dMuzCh^JDr;@noo zgTbp3jj3R~oK?C3+Nrn!%ZIFhc3^33WfV|h<*=g<-(;{CUW{=1oF<5XLh;u*XB*j& zp%8FpE6#Bcg3T%}bt9MnJSf<|jgatlzhb}7xqmaO)yl9L=UcufnK%_!A+T=o6G%UcVOl4`MI0ym` z#t1Y^%)kOqfnR~Vfx9S+WU4Et-{Rg>O2I$`bF)f$wuB|9;ES$AJAm7rDv zv*wMEw&GU7>dq?Nn93tplE{`DT#`nxiob69!$uFu(|6uvmD!laTyeYPVqqNYATW~R zi3imxRVm<8TvkXC(DbZ=X~DSy^KxwAxTJG~OM_GgCx+J8o|H7x1;}e!HQ-;Vge*o|(RIyQSbw!J7bid2E1d%-$(RPfo3HHo?VO5}KC$Qrt z({aTcS;;5TEv>1zcG$7tp1Ky?r=4G2>~D6vEiS$=7B(X6cwAdsafO8LC7#M&f;D1> zXBlS;foh-tuFoAv9hNwds|2zjw$Mwk2sFT}9h$;$3Sbii27=pt26bg6k(U8CLhfc7 z?iv#~8y*O{5}=S;Ji}an#m%x&qVf)THqy0Ku;jp7gm5S*DFgxs0{#X<2CQBK9Ffk! z53f06jWF2_9WZmVd3^X3*h!Ex`%$rzB4_p^VrPh)*=NGen+zxkJ_gO2q$oyGRPS4Y z#>?GPATkU5bBT5bgIc^pl^<|P0+EzxJn(QF@B@CB(f0~Rx%3Qw3uskwZQzKi!mKJu zr3g-~bNb=oKoS*IoK+k?T?#z#?bz z(7;g#JkM~rQ_^q2WP{jj)PXAmfp8!edIncB`ZdcIuF|08z!3~$hysPDTEdLsw9zPO z2;gKWem8g+1MY>X1ZxyS$pS0kf5VhkrPT=EhzLu|y4%RYJ}hl$jck-Qbu&l%uR{8dwX5cVZ0)m5uR@1%;At z8cVvKXPQ{F-q^;?mI6gxQE7pSWJSesv!IF!!oSp{1V~%41~wve*1!<18X;bz@d^-+ zXP`|>0aCCu{KIS3aI4}vdO`-U(u!*}ruN4~UU1b;!$_kJ#dq|7|8UcZFXG@a5Q8EKWR!UU@#6*U&t*n&59>I)) z=4wDB;A{zORN+AjOa)Lot2Ek|Q0oy$*&4KedERgjwrwC-VDG?%0gaX5qA?;M5M^Ol zV+r^1fV)|RXhdXIh+VKT3dS2#xdz!0+L6#=g9<({e8As;|SiVq|G9V~pLy~(C z$Kdx5XV((kM5v1;1I(+&u!U~~*9)-!xdo5EGqX@2TpKdC`0yJ-LHPw08|ngCz~BQE ze`gk_K<#nitJtzyxIR(@2ml%_af~g0_)@~XYsD(1ISf@&4gN>rcZ650@nWHXu2cif1(*a$PBQ=r$(9VSxCI>h3LF_0qCp`2 zerUj4%#FdH_-VETPR=R^f18TFn*&wi2EI^rLo@HugK@A}NDqp+1(?MZS|o{o9>VW^ zdVsQ#9wHd%K`_zM1O1-RgJqEhDur5qqz5ffAMMkFGYc1J0H8MfuB3$lHd-j$NIv`) z=wSx*036W;8i-J|085%2wFmSNTW;y0M-ZF|kY}`98YXd*ScW754s79f#C(N_2Q$M$ zRKlXoCxXG6XBe2PEovrMIsA@)(AOmAmEb@aifinzYJ2{0c5N+N&4 z7pAd@Q~docq1*xl9+)NlE+t!Vqzn*Fk^Cf7t}M6~(eDRpf_wT-&6x#ojl|yCmm-}9 zO#*frp8ZZ$7WF9TW7eQ}Ct-qcQUF5*MF8%KVBY|e;1B^-i_A++u(X1ISqab_Y_otE zu*UHUjUf_@vuODS8(Xl=qBbSa#0K9vC$lPE6Tr$sD~;k{9ka+iXO;8~+C8kqdvblS zFhavXOy$0@VAV2>s63vvzTy1P$i~Ad43j2_6DF-^jY7Ny8N$ z`2x+hyip`b;lZ*%W0-?~8O+ebqzUr^6L5o8LmhtsJ}5XEO9dy}p8&N4_^}}|8qP$R zys>a!fHxb`SAyxhz>fx*f#m1uU=pn1jSAW@N)AAkBm#mJ zssSq$*`Cn`)u0Uu9IQCdK%IUGj<>UP(AIB`!Z4v`FP}DvN#ILxG6Nu(Au#^SE(o$L^3Kadg0$Z8l za32I(EZWLIOk?zq0Gk-J8;oU3Kswj15gO>rHwrAV+`rC}9y^1jFe1}$)Arkhew$vu zWMpI&8jZv50@hbZ(hLAFYx9p7;6|YHZ=PIgTUh*gyRYX4!V+40iwC|e$=U_GoNLqUXilq z)r(jw`dA7cHZHjOQMS^YF9+vHcZ|>>0vz?AQ2KVWimq>#iZz$JL<1jx&*+XBPbR#N zTaaH5+#4Q#F&=EcC}5}T3igU2W^WcXLYVp9rcqi-MqeptQ95_6jNPdcQ3rS>$yzQI~(A-e(# z85m2z*Ry&XjgMr#{c!dv%7(65`&STbIC%P7KrUEWfHk6F3H~tlsu6P8kkJe7d-13T zdo`oGQJ`?Nuz&#)PBWAV@Fvyx3hWt-Z)~`s4u-D{ocl|x{rY+hg`Muoe760+h~LN0}B(8a)_(l3CJ;Z#D0D&&~Tj*Y%D!wim3ckr@e|5lt3 zgQmdQZMMKqWeS`nwm>fQ##a_EUMh)REE=5%n5;PLFSBHg3jPi#2ZAghVxu5yHwLd@ zOydn0#qc`Kazkz_PHlYSmq7A>Zv}c6s2@-ouHVVL2r`#{pP^R@{@N&U)hP7Pt+Yb# zi6EVYacvr3*SwHCmyMRJec-gWCAUB84S8 zuD9W4>5h}P?m4kUZ4Fn+q83vHT*Er8H2k4n45){wt~nK28x`ltXZJaZ`?u{yKo9h$p&XG8t!@cHKGknD^e*yn?Toi zg?HzFj4h0GAUsOgIjG2k9SHoB+=8LjYXlsU=)s5n7NqnW@dnsZ8R3%S{XYo9x{{3t zafNNTTd2mr#=%YqR&KD5IN)efyNWHjBIN`&qaamMh=b&+@n|DyK#c=iBP!dK-GHL9 z;An;hq$>1)fpZj$5svN*75D}Eg%PGfZyy7HOiKuI+cQGE<}{dIAy7_QQCSUq1PpQD zvj8d){SFB7=w;y1?gKZ1QiNY4P9@kcovE~c4LoDDuOtwSUI;LQ%lPXOX^ewZqBQ86 zxFe)}V*#ou&@gYnA1&NT%MJH%tG7_EK?I1Rdo=^0e1HswKbKOHa|P!pSKG=e6MiFq zKQIeY3>TF}n~12;4ul#Yulxp0GtBBGuiSze(=3b!+l5(ZuNwn@fxD#g;O@>BObc{^ zS)v)(!#z7!rZLRp9E1QoH`cn|pR=lwAOkgu!G(%hXbV;v6@QOdR1u4gu;>VjcA?OL zCH~sOqEtF!kx5NxRC0ty0Z6p3NeAM86<92!g`$8%`h}2KDduMtvJIv1XAgnG5(0(I zA<%lY1T%qSxlMJAfmhQ3RoG@IRsmo z0UrQ+AmX6GXwoKND>VC-#N!!%?6yGlU^nN%A%eyQ4pcB(xCENLE-+NXa=0oYXAXmr z8-flvJU3bBkm5mjm=W0VuB3)2BT)Hu2o`CShJUsH@SXbU`f_!3am4!~;VX8fHyVxv zu9xJ&plOX5XmDT|s3ZinK=5J1D-GueTypuVM!cb=s0@k;T*HTD1HP1hp02h0rqLw6 zp!)+Cz_VpH0<^_*$18RvHXM#ZZq3rrCeTPGd;>NK*m#uz1rO99Sof=jzX4%s*txER zfG_R-rAsxoffHC#_loK@oOk`pQU5}I#{S)B>0gce_n)nQsXlf8%d6G$y+8vsEFMw; zM40I%#9~6YnudQkTZRySbqckt8z6_87Pg)>5QzzK?GQ%S01a#!i0$_H6M`%S(FMf3 z`C4H)pp2RtFtvb)T35G}fg_8wQNz*_b|BEtEFowfg8MWDI|71gnq`fVw-{a6m4@Hl;EFrP?3si zAcA&K{<9Ctc7w8?a!?O1_7@L#PgWVwxvR$Lik!It-Lz1f239kAbI;Id zVZ_M%n>Arh*)@@TkeD7`>_5dNWxS)D{i$4128-$JPw9?-vXXxGr*%h(SCq5=S??$Z zSCk%faZhWfT2HmG?kf6LMfP#H+U<|2BeiC>CO^&^W8FiPGerF_Sq)Lo5N_<@Fo9b{ z2%pEVI;NOGkIq`Fnrl;Y$!hLREri2ZaEy|MM!}=A_BFS&rN9<-4JRRALgp+@|Dks$ z@K(&=ATV}++$;qp$$>KvBdmd?W5i5Nq4wzJS(Be9{JrQrMEFNYyAkq#tkey6OT5Oa@g55xAV)|l<$XPa z%d90OV!-+kijF-%yguaB31RP+WE0FU)P(gxPEf;VS7YXsnpU8O1%iwjX#K2}sNr_2 zhTE;$sG4?;p!5hp0j`GIt=fy4ZnuaPwHQbAMINWD?hvbkCukAyP8|{4o|+xS|Jl-e zLhiGF&q(hja-U7oyJt4iJCJ)It)``|_O}mbZ}zFZ=K>BJf%TT<1wjG-BeExo*AhrR zQheBj?Iw)B{QrXD6AZ%4JDLYjc1hlOR{#7hLGc@K)p)NL70%G2<*T=ys$Mt7RBtnk z-LBro&rdLeNc9B&xPsX1Cym22T@bR})lMsaVa9@3sf)kf4?2hD+Uksk08!wXuHiZg zz!DR3B^7%$3jIQw{WJuh>*|Siy6ppgU!rvY1Zz*f=vwfaYw(Yyy$D}Hi-Khb-hwT> zbGN}J@UJDsT*MS_^eg5>gP)Ylh=>M20AMx8@P9CvYK-|0BMWnZ->*~~22ZiO|DT(JAlX+Cp*o8@Y~`Sgqe+Jvn)yNx--f^m9jcQ3{oy3YFLuVB)X zoo#L|;JxqA+dFSMBWoWLTM5+DArlaPot)gpf60Fj|MDmeNFtlrvWap4YH^f6Ps0)4 z2+aNiDP-Wp2JiaXI8irBL*hOomXsri8`&n9lM2oxX=<%4NPq7@dSs_4T_9vOVl^Ju zhAjn{kx;m(C8ZrwwbBYwjX_30Fn6$e<`o=9%0eDBqw(-Gcs^k=HX3NmECk|z1&`7< zci|sMcBLSv64Y!wQLA~^tQw#+aZpFgYQevhw0qNQ_xPLA+%#wQz ziQ`l^-qm-kxiZKTnrofvm zX_q#%O=*`l;K&b?#584phSQ=o++Y#ftw_ES413yjG|3h2Rgx769UxItDo&w@FfrPw zZeWih0XDQrXANxA2KOlnxX6gQ#ECTEC1o+ge@LW7`Lp34zSE|P8NGVSpj# z=k>+=U*1u0z~J464>p60g^f3xBXl)iJnr7a2jh})%;&duboVUNdTgrS{8uSoIyJmK`}}qhzuoJnNN+p2 zzPPz_JPD7}0}VmVFJGK}H2>n8Z#7?8J%UHe=cgY@aryxEC;b0H==0iW_yB1RoTsl= z2qDgEV}<}@jIrmnv85WW1n`BXEE@qh=ls03eAJY+)}7aXR+Ip82$1mWd2Q9{g9V5F z5~x0}tuD1ynz|iL-7BS_snXP~&-NZVa;5n@K3Po^dUGQrXsxw5ubnE)$d#7gnG#@# z0~l^;`py>so)refnJvtL8#bU_9*NiLdclOo_Tf9Nxz^g(3*&`Qaj}ds=e1C*(1q|L zbZEr{kOZfHIu3hbHC{za1$0M?;Rl~>seB;qL3t0DS}+Ha)k5Qs{)%(P2*pY=ZR!Ps zQT3NmC>ps|-kE?QK!sCA7N+EsEA3>byemyLzFFVxrx>PJ2}b8iqBwlw8&eX^gWiu^JeEW)Ws!9k|r0UT7ht?}1zq1k{lM zEqK_@hV$DB_`&+&SHZGhd4K%YIu}|CEYBG+F?$hKqT3gUFrl-bCF{{$+lJ)XuZn0G z)H38FQm_dw4I-l@I2eM_5&(kfyEbVlofU0byXPUQFMVMte z;j>7zmKC{|Ws`i?S#AfEy->n)AP*}hn2{Ad+V8Dh+paLv{Yp99ejUh+K=CM+whTgU zB{jEwis-@fh!7DDhvY(Ncxz%|6-R{*Pn9r#D#f$)BMY*)8OQYzS$Z_1coxO2H`o9-4rs(lE`|l%G#xnH z$c7n_9OfNd-U6TIvDTSe;C31w1-A^HdY}j)ymmll@M8e)b?vzsPTk?rE3|eJ#5g^F z#H|C#a1!o%6sN5)L%5%YbCGRdBMjDEFl|k}u=rz8SKk6RlXT;N?M%YmByJt|({Le1 zM@|nYr+_ZU&wFq`4VP?sWRl+k7t@#p#yJH?J~n|_ZY-0)lAqSc`W09gWms1iUI08wl2G4&w#Ix9;jn6o26)rG^wWi<4* zL0bkD)l_GF5kwrhygpfc}`rz^LnA--$8eN3MyD`usoU~oEQAI0)P(bwDGVf;m4o{c?;Z3lDrr8 z7Py<_*&ffN2{z+su{mRGbRGxu13X$|iZF{} zT(M)$oa(PjCtZi5hBG|4%bOkwxLXP!X%rO%C}3_YV5fvRDBx}@fLqsp;ITGVGB~3n zd$IsvQaO0>YT@z6uz;8@fT#8pqC69~6d<9xq$)uHbz1>FJ(gX-9zqVoFeg0&?nV01B9?#exF%wgP}^#l7aB zy2FoQfyonBTWQ#B1^6w0U6|WuVQ!m+akmxlw-m5!YWqV0K2hLVJ%8sl%ztabH* zISD1L<qZmEd#z`O&K*Mja!SyIG%$=f29JGg^FoG=gXICcYn^0I5mD`$IGyHtXC zquU}*DJ^=Jyi~+WC+I?pplZ|)cgl*oy-5w%yElDveLA8b%H5Rx56XTN2Ga!x>t%>xGcz^_!e-nLq*6yOVhAeCnT$b!MZkcEm# zsGNk37Bm9%WHnQ&W+$tSP$IZt=DfzUmJ7pF4r;>XF!LVnIGi!b`j`>5N+VS7$mtO! zx28L6y6?_^N#FAW>Hvldt-6DyS3`2`y5po*!-#l@#2F>+aqc^3px(%whU3Z7U5R8O zg$gFSv!aJ}t(1!nL8H#J7Ls>|Lw}4;^^Sx-z_8wF&;s~^DGw*C%^*T>(eeKtKp5|d z>qm-6pcdLAhA*w-*}7ufsAIO{^9jK4j=v{!#iqu8NKhJx!I{j`0@_>-35XsyW_H|1 z-=N%FrSOvYb&ez$@tqQG7YV3k=NU?@D;T>EvD zps|?Yx=AU4;y52u2e;Twu+R6#$2s6Hy7Y7a3+mo^%>~O2JRI0u$qW8pp2=W$k>exI*uZv?pjjH0^22FG4q`kI~h>8+eDGsNYWvH zx~^5vip+7;N&jJwscv8(dv_AzxP~2!jK&aV-nB!YefD#Z3P+!L{A(|K)FffQS# zg=sr7G)gl9aEDFV5kNAnd$x%V;J8oYYzgkAF=X)=O|6~ zYQ)PCR;bcSlf62^G8d%Dw$1{K{*uluwCn|Gvd>q#wZidpriz{F!kL4@Wai*&JGJ8j z(|FaFD925Xuo8~MR-#m+0rD|_R%|Cvj+kj%P3FO2j8UHKAYj|$J$bSR(E$>6gH&E5 z$89T5W`@x}3%XTlhvBo}z&W`XVI5vtN3Tbs5puM+VcCV6oP{7SGcuEE)m|cz*vWK3_>giuMQ#FU zv@b(nnmoYAA`3Br7)di+I~z8w30$=MPT0`CJ9+85Dzu~X--RoDFGzHDllAz~A!m9^os1sb|pnSoVCz}^Hh1Kh+wquZB13F7a6;&5cu=u) zPxDcTAE&~}XhqPb6nb_9ET}OfG{$Na&CH0}BW9Hu%|D!1AgE=3x2RQ2ScIEWtD24Q z6J)Iztu@j&y4BKN<}qY@0bP^z^<;f>YyGT>>9lvbcKa3mbOFQIZ3Xx(1>|j`H?WJQ zODJ}034U7%ed{>1&8|YE0|UzJEiL%z7R;^lFhj{c*0n{s)mhdjYq&W;@3Dmuc&+Yi zMm}w;`hwBMwFlRKh=n`L^D1^=$_~$}DL9$>N~gNBLf!(;3Sl7{Dfb;IEL=!~pejwnh4^)#Gk&0Za4I10C&C=kN&3brrjgq?xTi|vYZcIsRabr2D1C`9~&KX|< zdB&#rAi8IN=D=~|dWu~yabX|a*m@i+hHDsKf>>6B@De15W8@3lSM5rU3;e85(#-W=(^CEV+97iQ~m)y}b1T4xXqXS2Rfm z@8lv5oV#R@>riJI-^MK%_<=>ocQv|sgO`7BSGI7Mbl^0(*aF+qgF84_Y^BNIfzbX% z+=6SzOi;z}GOV}z7EH<%*j~vkxC$P8+T-q|n>j}8js-u^P&O`>? zwB8GU#p!A<+|iAz+1zB}p!!#J3ogmb)I$9fG`YANL-{1MAH`NEA}1SAlTdwo^JcOE zI|)tAy~g-H^^EWTMny8$d>w%YVlcz~V?}QWCfz??us(U`?)3nDxHuf7d2!t$t7TKvR=U{le-DD)(&n0+tKg^vnYQg*MXzK z`aA};ijHLCrLoS6?ET=lzDV!eAk+(?B%-8<;=(9$5+k`l<<|#??R0)ljwKGm1ci%# z$sWC>mxmF`7hDIaKD&Iea!OO4{8J;;v+M#|hS!q)Ys3W)caKuz+0(x?1Vq!uiokSZ z8OwZc+l?*|y(2Ch;Wpt6UWq=p_34>p!-=IONk-`{kq)lc0h-<@0`D;Wj?-&JIPa3W zVsxGg9<*1LzgpxhZDGJ~(usBVi&7YWATQ}`kKGL}8L5nkXb(kf%I-MlN&hZ8d*SJ> zi1{qugDjBVJ9gke{d{21wgc3T?;tZ}ZpC177ulO2;}~wo)6+df4K0)lSBU*v$p!O< zVeLGXQ6@g zdU15p@M!D{TA7PO2xN8nn?!wOR2@CEb}3f;rFfBpdvVudMGM80AKZoG0iNYozu+;5ehkcPE<+HJ zY$b`2^we{8Xo|l>LdXrL(-xv^sji*nQHOrhj3HKG3ecfx9~>QWgtqzTeT<5f2M&vs zG^_Bzk77Ml4^dPubPtJV9T|ZIiN zKi5oKKfL+Y>F+=pCVV_OLb~{6v)0CO=QSQ42T8nW7<6Q|L#ib>J96xLIH&mI}H$^G?MQzg%FO7ELqd(y2995`Ryk9jR!` zK+B(zcCNIKlQ#0xB0T3PGjmD^3;aSmq5`x8KhqulR3!QKM9EQ1ixECFjPQ?AWn;ZP z&ALlobQZfLJwOPw@0Y4X6HBb{_4TRMkqc8Fel8X~Zb z^|L{a+`*%r8tq0p&|aG`mh#O?l`n^d{_8EvWn)HpF2xKg=C6S7cF;XN%1VqLo6x{J z54aF3lw{p>zaEtz6Wp-_jI!pxzLTpalzNaji`j_AubmJJ0`+|}PNTNsi!MZFA zbrodEm#M}zgd93fG%ILbSPkX=RvMr-RpJfSoXrr?`-H35cCQ(Pi1}wOCiDa_3qTMywq-QOl}{LAY=#`0iXuE2f?{N}JXigfJ{=Gx*l|?-~ItIsr{d2~R)E ztrx-?WxKj!yW)+aAWXH+S-akrjpeR@kL5F~jEVi_-fqMKa48-NZei$hMSlI68JuiO5>M)ud&e%<*CQg%h!rjzlG;5hCeig_>Ma?4UdX zt4MDNQLR4?zs>OHX~h9{N}n$+CH)tH|BW*RBgO`tTae^J+KmXk7JB6lHn}LA_-raz zud~NP%*_JX>vGeUy2h+E%(!UT3FFhyhQ>mS%>r2wEzN*uW1KpS!k=T_6XTJ`X@33U zs=Th-MUpNm{;O=^B_zLcyN8jmcJTQ>qGsP5eV`1b45x8W9jpfk4l^0cGHn0YhDb@E z;?|RMtMMDH9BOm?WHKg4@ZL2SIOtXu5-e1TVdJTZjrIQX!xi>A7)ty7$Ebqe40L() z0o(NWJ3B~dlU;`@MfXf(n~G$TS#YEk`&*_?3Bt1B!aAv;>dU`q5Cv)W5v(3&<1o)P z8Fop6rnXhY>t?2plwL>Ajb3KANb9RGtGVU5dm|jpO0L1v8RL*1&5tpRwvoJ z-gKq#Tw2ZNw!l&k(YB4~i8CbAChgZ~<1nVYQMkgy_hvNMU4g*6NRkw_cn^Dg!@REG zTil~LGmt1lY>5rTDIo@Q_bW7R8G8jMtoKsWe$lJ6GdU6vE-phOR8VOMN*<_5Zjl;o zHZ!JqugP2R1>$IBtps*QHwSRNQ7sa~j22xDBmJh2txSdh{eYUGZfi+ zFJFHrcp&e7COgiXGX4(xqbu!jEVp&Zd!1$-Xe0-1UbaAfvmlq*2)&L`Eo+!bvPdeG1IOfA^34=UcKe`5laz$V+*rbcVprm zk@p&r?U}z@GJiK_{;uoezGwM`s5~(s1^;R-nS+RN&2z4#1;am7HoH$(L z0^&0gX~&of^XiAMO1PB2nZB@lJX|UF8x~yvgHc6%(LL6zLeV{^-~Pv9|4EI^uBA=k zN?xuu7kK!>g zVnE}6qv8MQ1;APSM6#!hO=qgK7#;Wzm_i+M^4tz2*|ykN?ne>yTnQv?#h`K?9;1}qCf@qI7pUi)uL@CGj)X;=Z zhh0Vo(}ZG?g|tYw5r!NZ#R1>fSj~p)bR1cce_9tK>o8}OahRz#z)JdJ&e|D%aXoR| z3=S0^&;Qi_`x}k&#wfOgT5OfNjG;`}iLwAI)u$)5Aw%+6_y}C0o#4Iv!#-Dw0#--t zY<%-we5wJN5~Q`QA^m^U7ipAV#T?bgGBQGF8b=T4?uF14PB|<6z7_&oVQ1Hx4E5uM zUpSSz68O@}nf)D-TU7Fu!o15ALt@rx^`GWPdc759iqZ344S-RDv)qW%Evk8zZ)Iz9Uo@X0%rs9~>BOqc znNuwtx~+d#Iw7+PeTw~3do2>A;^xgWe##?SmW4j;Obu^*LJTA$NM3EuSa)LrL_~8W zp}2ef4kNBs_=0JM192j3JSzlIuifQxefrLsv)SFtUP-pxNAAs-Xc(ccvF9IRw`a(`cCBKGn(_ zTSwhECu;8;#sRaat!bOX);|7|?!C2y3p}~sGFUz6CjU@w4&#MYQLJ3<{-;z(o8QV^ zJLNxM;E^yuJ&HeyA5D^f(;)U)QMpS;%3E2A$%-zwott+on8bSVdcYjfphZHd&KNf6 zfQKVRo%!pTd&&>h*2@Ai7GZyviO1CE!v?si5=n-y9shtw&w_)eewEZlmNN9Fa97dpXhjRaKcjrp0E2Q73f2nHf|3ECB~c2^hE#3$80Fq>x@=? z8uUp46yj_tGym#Et}BaMNw0*%9TSGwY~k<7y2`MzI~G};DC53&GlWM+oI8l&Is13F z0f&Ah%v3<(f-Qc3f4hHu)E15eTIk#%PQPE=RNVDr^86+=nsE4)$KC`XLF45W#N!-k z)D};UKE@MVj7=eP=)%e!rmSM(c>kQkjCRy}35d!5Xa6v> zpTfSrpLJKUSNU?)StLJrr@UsS3?g-HLE@R*)dCN#l`D5<}ho46E(1z{iZ z0952Zlz&HEWue|(1ga(UMkM_;=@L}Zo!IX$Pm*VsA~W<) z$(lc5_&|(6e#eo$`mOB_1EmW&EZGmK9rpwa;3~o@)9!cXTO2d?Dd+8-@tHOu7gKo;qT5$F8Bq>Rzz7PSY{q?u;+j9M3CJVn%F z8liCIWQ>xg3Im4#Cu)Vo2_cDh@o5l|LGW>@lwnJ1J+ae(I%{3oXZLP5uaHHhGUopO z{&8)uEoXL`=!Od7D0!EVU7+$$;A_rgZni8{{V5oJ&trmnv_DWvOkGGUMlODa&*Hv=~5INy3$yjQL1%_7lUZ=785!ew@}=m%!V%RdpD_ zH%gvIph2mS5iFBs$wdi=z$T_3NH|r1?Y@cXQZK?hLl-iM(dkkn*xs;Qy#e8{`!C%= zDa9bQe=2*1J!p3&KFWSHv{}UYQ)g+KN661Sl~j+A*wygC$^YUXULFa0E`5OicIG=p zhmH=nbS*UQ(O&hDb(j2_1#;l zCVx>i1DBnuXF)7$^=-hs?xi`|DXTe8o`uBd3OIO|mg|^-Oax>M7$) z1b)GGC4N}yTdR8b=z>TZXL<@y8)1heX{cTD)wEBw-~;w6{@7LX@pD-B%(Z!0m=dSv^#k~Q&S1V zRsXU;L(Hv+x5j*bMTb^p15Ba_zy$c6=57q|)rJVzk_5&*4u8C+t=80lS#l8gdzrjS zdfPR9>#x6}NouO+cbM$qsM`xh@>0lhaxTU{0%K6}P++XhVkhdHq9b!{o(?_+qxeKW z;hYude9*Rb48q3iiOKu8n&GKJFepVJ@z>OH*y8Uo=kk`!a;&(CGmD7h=ar+TWsj|u zQF&XhGMggylry=9#3h&^VBlvRt6IphRnc@w!~J&*IAzZbg^g_YI)-1er`c_%!|5n6 z`94z!KV5oJP2vYc@4Cc2S~N;_!}u!2-<=FRY9m7EgLrV(Gf*%!((sGz=Z7jXVr<5wZW;#D7tLFXaki!u?Iax%ur3)0W9%AiznoG}tMPy~~W z-GwGCB^wV_j?AM;$vG$MfkjsE8JI*SYnQ%H*V?V^8u*X|lQ|%aGRsfFyHge`xx`gh z)hoj$At+NH$yr9j;2M+vmzAaBzWen@gGsae$3(}jZqD_;vp+ehXc=e+l{kXxUT31P z-24tOTdu-3m4Ivk*Gb$ctk(D4BwIeh;f1l9#CFGOD|?;>?VrRezGzZL?mWpTF1qo9h+cje_{s zvQwY`rKkH(sQ*Jgiu7Y*!O5_E|L5;JQQAY&f|X|QY-}?}T?Ns?or(gPjP-Vz~;|Rg%ktt(wd`%|TfMYoR=#c?q zv2RW0e!rs(-sq7wW3goo($|P2?25_M*h^&#m5Ktnns}0a8G3963x$dTo|^cHKAGds z3>JV)MFC}vLM5oC-Lg+c3zLCP+*%aRh>G)f5(lZ8lCOsJg&|c^@pxGBoQKvg`6ELK zADRN1F$?}#0^~tHbcRyyih^a6TFiT&)a20D%=Q_8VYR%HUDt%I*$#%u_XoJLSG z=Cw5xO${Y%f^l$g)cz@IC<+>yjtBj)qN)HYV(U{!=qW2oBLC)*^o-C;t!V1OA8{$l z!y7jJ0_6{2ody1&B{_w6L041bHnQx(Q{i5$FX|4LK^UdSXDru#1;zHecN2k&fQ8)XgDn`)KC2qZO}Y_} zGiVG?&S`DWweipMC(jUErHB%>aG7PG{xj?mzxc0OTE{6$5FIIU}|FGdymr!(=viF86+jv_*SLXKF_oe4`m<(+%&$DRJEcK z5(iyj*OL7%!pfw`nfzv5BfsiS@f*utgLA$5%YUa!q}@3Cp1W|e@aALtslf(CS#Y1V zdsqE5fA1x_eTu{CKakQ@`%6a(0$_MGsg_k-=w>|Rm>0g~QC%1=AMqn38r!1UIw5@i z!P*Od9{r^dX>E2Rv;VX`KFtvZkcGJg=bZL=uz!~u8#oYM9{V`;1IC9yN^e>08Ur@( zR*~wDsq!~HIL}06NsccPU9pk0`rP{z^dgnG1lL>B4da%)Xlk>Y*Uum7n*juW!f@@D z-04Up?;(L2T~dC=Zfk^n&-AnZaMeZR{w3^r_F;u~HMlR#nf}`SOrd0zrO}f05i19O9NF7I zVvx3I^BDK>U%a@Pq939wnVl(~9y_(al8aG?n5$yI5^E$geq8IMDk{FDuOgJ<0Vm8b zG9;g(X-L%86vx30o#6Y+8tw>2>Yzai2F;?<<1a_1>>OWY%O%Q0ASE}hr|uAGH_IP> zS+QvveJ}x;zUfHsNEeMXn{CR>?<>XKQW>kQf}C0smTFwUz)!r>ly4HIzvAL`QZ8fS zYv5(R_%sDy`^SfDgvAdiM-_08SaJ+eeEs3~71|<5e$21O$$H#1( zsZepl-#+19(K4y_0DUL}h7l5)H@#IZVusfuhJ=^n5e2-R}?ni8w?Z_RUhHo@H9Uo8+wI z48HBL{<+5g?RM2WDe$hjD}11?^S?o0eH24@<@LbfBlYVkKP84?(GP8#_}vY-0xPd- zvW*5{d-=D}Pr11Rr+retYkc-w!!A>aBoowfdsE7KaUTO>NSZ3sUuykY&BI*XsVP?S^ z`A-Fy8}S%6+jFlxgX!Xr`lwAWLY=u&$n1<@BIi2&Z?f&(3q|9(BtYjN&5<*?TYb}G zzHxrniiE%JXwQ{0U$V@B5{z^wOZGiA@TiO3TpSyGQRs?%Dtq1YAo>E|(%jJk@?j<- z%VXI(zxILqna9F8Vj-?+4)@>_0Nhvj8hhMN8#>hpA^^~H0R}p?gj)ls6Z>{WZYMn;cY}B-#TY!DJ23BgDC5VFowx1E0?65p)j!`# zcw^#2BbE?oqP_JtkpkhZ<`IWWxTRggJ@>=8C$stral_VBu)0AfF7b!O$UHym|3>>g zeB8ZZ`*GaeEUCrVLkGDva7l9iSOirMS%_TzH`RIh2;7n7;a1g=_I)Juk(v};{25FC zzUb=)TgfN$4(E|t;`^a13O}&a9{<7&4GR7x?YmN^GS?p&hPhi|pl5f2NY zxDW_VyH1DKmAljH$j|5e9ua@_cRz%ky7dbRdoTEQJBR+bvl;I2_}mHgFurK--EXhEqe)NdxgKcOETqbrTZowu zdfvUroB;Dw3qQRS-+(d}a$ZMgpKSGjYb&7mCc*MDPpEyx?!+3psy)Yg=c;9q> zEA)W8f@Wd>XTJCAcHg%cq;E` z&nG!K6C0TelkfdUuXj)}{p<Eb5-ZOpUgf1>IZi>fD_-l z0aT(%<$6K>K$ss|^nCPMJ#ljIe(vnc2~7?hy;?zUw3~?mP4NH`GH+avJARJZjlvkP z8{hi~RJ{v_naTIwqt`8H96YQ4e&yVm1Jwb`oBRQ^ z2|&IH^xhG^-Y?E?&U){c{I@%8JQE;a4F2aW<|%yuPx8K3zaR#{*uOkoZ;NRf-SKRy z?DTapR|TD%4NIQ5U9{isMIa8|B>j@D)4MN&qlhyHF7F<)xnMp4GCV0Aqpb2Z_ zYy|4LGK1SayCxP^j-RJTnKv|ZbcH=XfF>g5&W6PS43-AlRX zW1G*;`PY^JIev5pLk=BXoZ9);1Pu=EaIT`(V14zfS)1RSA3iKDdX*O!w;@9NLpZ2?v} zu4He4%t2oz2X5)8lQ_-p7eBXltmqc)4Dzt-Yb=m1)_Pb0^c-G2*G?h35FS zLee1TW(~q2!#1Fb$;ovMA&nfdoylL0IE~R1J}+-eusj}JkFVc?a@bx~8oF}$UO5{2 zavIedc&b6o%m1L-Zdz$)+*kdW-fVB@b}cCC`VL~o+)~2YbFC4lujV;do6j}8mWA)=nf`5AeeaVuZQ-khbBrlxYiP&bY19jE zNYUdP*zhXh{~xLBNy$V0`!^{RIB^m$paA|;;UIASfG*H=mM@91Jc%Z(5a%5U#<}z+ z@}b{;U4jn3mR?9I(%P>SeJ>VG*qo?a^`Y&4Q?qchoK@o%O*$@JM;4$ZPRBW1VwcdN zS(xM#^x$IH*ZHsa^2&YlFbu^$CUascnPv`L2D*a@#%T`3{tg=vLD&olj5XTx>n z5m5i9v;3wH=6(|&E~amnmab4PX`)C;g|;)MUc2-8>9k^KCXGg`&zP(1)KdP5Ec?#5 ztZW$}ah_+^i)jNqBm`5l(3|#BW~117=c^EDfdk4|sThvq#7qVMOn<6@T<@dpzm}{n z0{jDOOIEie(>sVU=aegL65hb$%GIS8ko55~7|$>R-3hYg`En>PK!>5QFu11*8a)|J zh)Ko^RV|di3KbH}9`?d`D35qJ-$E4II6JzE480KP6c)@*Glx922)By zSU3Nv3H}Qbo6gCFxY6iy+0LbC(4spkmKrYiLen7HZ`AZ zNy-)|3OfW-bc`51UGCeJq$EFtM0>jYEho_iInNwCf&g!GZ{*iq2*1q@4b6`6 z_EP>_+##b4PwW`p{vf3P-X3tO(YJ}YA$fsL+T26zff-L%b4!xkgUWjP`Z-!;r{$sy zYRqlI$9MPt!h`?b^=2p!?v_W_JezrpNceboj08JZJO0d08C9;z|#+<0H{cJ*Qos$3WL37caeeSCgqjRkx<-v7{Sx+p`f z!UnGY|LP9MD)kOmAk6~p&rGn$K#!ts}vCU{c2D$c31skGG|VI37zu0 zfcCekqGc86A<6s4V;CP^z@<$}%@Olaf)m7;G0T#1t5{ykUSh5pl*OnU3TeUkd*3J;t;FXok+{;gNJf?_B?2FTU zd|b@muvTdBGVV;_8x+db4RX4x-43e96waYPB76aJNXmpOY3~#U9?KdxPEM@!n zdeuHYt$BKx>y0a9&z)d=W-D1JiJ1{1Yy1$p5lFhJOBC~R*E={kw{@CtejE0$y^fS9 z|IKZFcy;J$vr9JU;BF8E+@k)ntsmQ>Mb%}tB>`2Pk5$KfK$T&d5@|ZPIP=@|`V23# z-IicJc^u&k3{F}2v}g-;>e<9Z?D@Jl=&Ws2Y*zCRCU|Kd-xf1fshdBzAtA^k)T*zn zNmn0KaC~P;9A1v6&Fn{Tvk4@&R6qbQGS;)&U$0#qG>F0Y5vxmRDF@wE3&Z=7Yb)N- zX3h=kI_K&u{bmh5&h4#DEnW{MZOzRsnl}IIhzArb297^L8>Mt@?|4F*Mp397?D~E> zhbZip3ch|)pY%QV8A#9Z$^m=5S$2$WdB3$ht?hJ}=$5*5O|8|b)4zIh0ek0=9U~Ww z9$sD@|Ae;cG5s*L=qG_d@Cq*6UsrRVhtoF)hr90aIZH1Xcm5WB;V49Writ^{^VjuP zV0WDlJ(`w_V3jYjsNc>ObBBq(dS@T*VR_-4Il8nQwf0MrgyyG6+L*Hyw9jjr^dYga|Xs+yZZL*jt~F9b1Q$)=YNyc_vKLh55G3y z7@QV&|GnnJ`E8&>STEw2%?k6Z#3IoZXJRv zLhIJ&cd<_%2=#iq0FLwWoMJ+A=c2Hd9XMe0wqc5`0jmd!vhZAdlQP;&Iq@5j~{LCmXUfaaIw<$sj0uxeYQ!$;KF z4|ZoEnU{4#x;55%7#kn+@onp$=Af&Z=i_pP93fqSq=>b*>GA;9`{S+Qd=K4aS>e0+ zEMvh?a$e8#QZ8Z0<$?O>&bAqt|GeuXr$(y$J)q zRNi2Qe$|@*NY1@1WSkj(Hm)|Ii~!*fQ+mXHZ0h*ozmknOBT)2Z^*M4)`#`b2)9YcE z=ilZ$FWR}(zn5K1;%D&|?u?In>pDc$v54rDgL5Q}Pz#eVf+V!C5eUCU?iibuO=-+$ zM*W0NWHIxZrZ|n9d&`zxe)F|p-K_6bM8Db_eMCJ4@bF-Ec6&&AD$fD-IP_gtnQ>ww zu6Gsh^E$h@J#%7!5dovR^{O}TG;t5*N#W4^mmIUvdsof>%C!+wLSwqR;dQjj?(Xf| z7n!@2;KVL#aeT#|ibC!@VfL=NoL`us}efBNTzTLe$cboHyZm>;1dQobiY9@%Fc?h&qpNBkgYt93=mOK?nww`T z*O^cVW{>TC`QE2neN1Qfb|U{y$RM1SJ_gwagwxWl0i5;#e&2jp_EpmdsXcdt?YzFg zf^V9P^oet~&o`pAS1(%Fc7MyhLF)VJm;|G`#w z;X`N$#XyNUnftrVJoN!%;bi>N{TlG?Z#9TSSfC<$XUXRQ)Cg}|3*UM3P}8-?1p5*U zME!MpXe(ZscJ2)Qd*%uEh-aAO%00ODi|5~V!e~VA+^6q1a8n7Qc2F5HAfmarBAfYF zFq7M0#_mgw_r(RNN!qtJ8Nqjy$0z2^Vs7;vyWGXm$D_RhjdyL@dm7;7!J?bHJi|oj z!^=JQCog`DOR`J5cR@Hk!S=5?C$)^2)rRkTsv)tSqJ39 zy}touk1B*4?}3dVj^17VJmUj~{*N2)y^Y0=3>yVF{<(=LKJUUbH`(~J!nQQR)5{<# z{8apvYOFQARic5x)ra?HVrA&K5{A%J?!5w*wzh2oysDc~H(o`w&5UCEaxj8z->~L_a9*fcgKav??I5BHQq^FR zz{d#6+aNK!IzCjx_Z`A|$+v6Qx@SIwtZ}SE;ln@r7h-R_h|X+wjXu2e&JToN4^Q&X ze+>bE$BjvYux)6R@5O@HxVRiTu`f**ldbK2+e7H)1r!2WxkiBFrJUGWVD$p>ox$Av z1=5S2dGX9&pPV)Vt!D%EkzeQRUh==z7aTCeuGEM;GW6(`%O4@n_`{q0hc^v`N5N&f zl^tInn6@vUC?u}!Vo)M$uGcOQ;9=p;uE>G+7`s>H^d;zqhI-LPmz@u@jlpUUbBwlB z#v^)`=!)bZq~v-7<+o2F)r$VU>t_S+JK_=Un+&x&=Z@#wS!5xRN>IgaRJ_41Z=M~4u3?pWYzcE;y&3StGrHna=0htZ|h4D;x}C~?BF68 ze_DMwTUC4OzHnPLU471Z-Na~B9dmh;Jv#-Vn<=oZ1LUO+Z7r=t-+YC?-K*0nJnsuf znfD)3qQieOU0G$7qw18%Fn#%68V1-+@2~k5F7|N)O@#g8lHf%b6{b7c_qfE%2%mz& z_s~{18%GLWbP4c=zW-4ji7OT-DGReBp>igbInQcRh?r;B-Z?T#rDrQjsvbwHh;tz9 zDXRQXZ39MS3`t{st!c+=a*m`@UxW9>6-Cc=lzktsAtV$6Lo=b!%*{|<29&oFia9o1 zM;UwYMwL}mRJf^`sTEoN_2v9QflnfMKp`+}O6Mzd^0e_5Zs4_Uc-1BT+}1q);fAQ@ zPyPX9bmLfa@cLsNheDn=$Sn}_X67gee(}G`QlkAPxSZe=i~O`^$tZa&_NQWjC(_6q z>^L8SrlPj>5|v3jhkHLYZ$JP^(riUoG?6Cm0E5BnH6NgbP2_vX2kn15+9~B`b$ET! z-&<0Jz4X#g#!Kb6p*E9zi)$vzsg*wTQgl%;+JTsGe8=>K**tKUHCeK_G8%L8eKXD< zv4e<@d_b73NxM-_Qa%q5#^$b9I+2(-Gg0Ftdr{WRY+zVAPy0xE$YsZ&+~wHdi2pr zPt|szdEU&yjxOW9VNYrcvV&M${rtMd4tJy2Y=-}z`%PKN|0#zn#QQyknjD7$d)2lHS#{XF>%8HBNY3UYcG`iW+i;Q^lVnD<=zYlud|P8sKui&%pS`3nfAxYPQ_+c*6L zRlH7&H_fe*bOfNCz8IcI(#^J>$BBGYNz3g~A3(+gipvc7}wpBB;wtR(Dylo{?Plw*8W)bxG;G z{bNFd#ek}$`t@5$yg7VF*w}mLST*Ha?*p9d6D6v~XY;M!s!-_Uop_zyY-(9YVU?N9 zQ|F$=&$K^cy$BW;u89j;`GsCB(XWD3ofjT}TNE>gV2UBoA?Y73dR6koamI;MhiwB7 zY=Pj|BQM34+lya2$jIeoNMw%1!K>A;Kcme`1DgdlV-jT`pL4r0aHf<92E8sy6_r`` zme(tPE+pp0&I;-w~9%=I+3TQ(pJWU*2ML?IV`5}raFrFumG zyZgh%uAFC2GP&2o`~3chBW-&+adJ$;!#bYU_)&A~!#czqhIFXHdfG0W&Iwmt6v+w2 zT7eA-Q5u{WHue)rY#=v@gGf%&PT>v+uyM1>?`)(BrVC|<1Rh)5iFA%JXRv`y%UqJ3 z$1(0a>byI_FN3^|d+Z0*<7t5hTA0A0c`%(K+TBLDQ{ER;3GzXlglz^(fh{mqr8dF6HEureim(~6_%1E7%JZY)riy- zY2((sK?pr_)I%F8 zi5Ew74`h3Yo+GeQPhx_5WCgEb+1GppuPugol-TLkKkJ=7w`paX475KZAM4hcSW~~3 zZZgRoKE$mV<~wkOM&yuDzty%U_DQ=UBNW!|>V2;O-osc@2y?#V_uc@$^vq&=jAW>^ z6E@kQZmEBGTF5T&-0uwbTKZ+TVIJjGgN3nrP8}yUMj4bPNazQ83&>HPJ|_0z-duYB zmO+U!e+%TXlqGQQh#nl{{VDw-s=Tz|Z%e@WW4+vAKc)^mp1%WPlb$12WS+;H^jN?Wd$=ZqZJ0^a4mHLF%6jaa$Xn6840@so_! zCjsi};QmaYyC_<+keqgwepS{)wfTp7`T?ZSpp3q@Z(K{SOcrKD7ZhK^$j1`}E7?%yJm8gsIHmj`0 z_&CV{?wiO=wA^dSn;%Xfx79r9E4i~VQiK+N)RaAX}sWk-&?D;8z6 zRxL_qoZ5i20z<<}9eqE@tiPs(ftgdwL5JBK!Z)s(jL?#h%skEeQ+QpaS zNTqwP5R(SyKu>+G+Dw%Lfb^!k@;*4vAQ6s{&GY;HqEcHo#+&%Ac2a47x@%BDF$W7+ zxmo-x46P2T&|?@2l-a@QT|5jCucc15clF&J@OmtJb-2Lz}gEb3+mw( zq!XR55=Q!wB**GWRZ|a}%G`{pU8n2)h|VVBGIX$YQ4$}O5$EF@cZ&xPJVFcI}k6l%MI9z4TOBf%v7HGl90EL;xv3E1|3E9FZ|C z%@dmfHag;7aFo%2 zG#%dPw#R=@jFBS}!HE^APXtoqN$2S;$r^F?*ZPSG2FCx$^)`t^lK#ty%0&S);g?)d4mW+tzT2`J(nWn#Vn2ys zL&1QjxRC@?v0wE^YF^78J4A3ZnLvt0(s66o-x*35G?KHiW&b>qG2%TD#wkubrF;L- zIQ;$4KPqW+T9aG$bvu>KN%eth&-O76IPRj93U~fPW~4-Y2@a2RoQN@%r#q3npf_$G z9DPnY!7!B;dJ4pjE)iC~$&paxk?(EC;rq}I%k@Eqc5|-4AzWDSM$GzD*4L$q<(R>& zr|NQRV?_H5E8E7JKsD`^j|l4ojH1m5q_eWaa{ue@^D;7f9gV>z_};qc`7b&j_|Jzl zjgFmwAQ?KFZ@yn#?5X*FM>v-&&el8|1V7o-96VE}Z=`6Mus`uzK2l3uEkSW6Pq$Kf zFg?qsYS#w|pJnX`HKCRvSTqfQgk|5_eR#1GW=Gi4$8^yUcz8p*>eQ`078i7S`t!;w zPa1>6{svm*OHdk^R4htPoN)k>R$G;Og4`lr3L|rNQrEa=8Z@2z*_0iSeR5;N=X^V3 z!_|1+TVlhVLUEPiA}z>p5Zeul-cY#KnY*?bQ?ZpC_66w-9^)jc+MlR-!ln!r`EP2G z7&cgY?#WVZwlKJ}W}U2T#y{#$&!@jW_TXJ!Br#mN847$2-qO)#K!*W}BGBm3!;2pV z^!yMa_rvVM%mA_MgJ)F797q8w!%e| z&kMKAeXmye_tK$cm@tgM3W*7p^I|b9oHH_0+qGXyQB=v3*tbKi0hG!!VfZD}YoxLF z0|WS_Y<|1wT^jG z4^GLSG?`;a5rj;Lvp|RldapcMXC$wd@z*DXsUr_##?)fUw1Dit) zjZ}8JVhdaj!)}m;Q@(cm%&#%`SA0r|RD}L1T^toS0^=J%=YRff=?%87TMn3eKN?6U zOi?k?U;P7VV1xC4@?6IfI?er#EIyt(2^=Y)vIzdj&MFo-D_@qkpcHl67c%Z3Y z>J*+v*nbm8A2X~#8-s++cBUxa+RYv+#7}2m*8H-wv9a!yYHZ}5&$xTR!ekRCw~GdDQMd*Eai12rCs3$DfM>&34kJE}e#j3+E@nZ#Nh3 zuQ|Xq+8IeE$HO!KTj}c5d1=Bvan5tGqgY#U4#&mgE-|E_d!s#mMd-}6g}E%DQ=(an z?#*EmWv5|98V>zW&R1c1YKs48obc^ejYJ|N8q}L-ClCDir1L&xWkCSKwkxr@r?6b! zom~a-yT`dJnk^n}ezQP24KkQiiN|;5cQ`-*62^`FyYHa8kI-y3%>Vd_RN?;tZ9tO0 zPGld$$E|yvPXdudfmWmTb7`&5T_Tlpfq!(oIPI`W*#E?EeBk<8{RMb)B0djrm?_Y( zNJU{{JR|7x8Ifh8IRAJ!AG5Pbn6YyAJsq$|I)i6fm<0-cCH?&@)Ur-ha3PwN;*1(8 zEr&$Ze-78Sw$@+VcCNOX>zj2uS>wExQ^!zADdoXPxv*rn>9!zPP^6fv4+wb~42uq3|X|K1C(Scg)&FeST1zOr# zZ{FBwA>N>fmqR+&T1|zwGw22!FMde8;Dj-vf0S}6w=OQ!_qLN0)RY7@dBHZ)vUy{B zN45}Z6Yx0TZ8LPptDt}$Clhes2p#PC8JqBEu@D(&)-1*R5sn}SF@`){U6o%A;Te%T zPc9}p5zazkGvk*8S#blcH*Pcb}}YD_s&jy z^vX^m2Zvq+)dE+mNyK)nskOQ)el*U!No-q3N#3g}Jg-%^EeOneg$F~q0&YjSlW9nZ zA;F7-8YuoDA>6eDc_iR&&mo~21nt;eUFE~9?NTImQAh%*Q;8EghHKhXl{RD0^d@pYqu|TeErUT8cc$;^UAAzIGOBMhXwff=1(|e8D;Zs9XAPKXBoYB; zi59=Zr-|A@FD?>2v@-@h4XD41pKh#2JRMWZB6GM;!zN~K<&;v+)2^=C-7%T>DC^h} zNgRiam7{&>ZT4td(L7J_HiqfzGn@(4NlxdW$^AjY>?WMIU+t%`*2_+y>5i(3IF1K2k`p< z5AJdi_87)NEX3ub(bfq(Hi>#_VMm?p`)c2~xLDa2@*hqBG~n4OI5a7EO3l2Q1E^;I z8Sx*O_zx~?+oBMsQ2244f5uFDa)xsac!02O!QnLWhU`9!)q;c_3VxCudK37QJBiIy ze1{dvqg|-q7Z;45nh<@Ie?WqEO7V2Sbtt1Z z)hfcMHttsY#Ey&BKMj$}t zjGZ<~NSh>~O(LQae^!Ph6g*<42O@0C@H;nS9yujd^ouY&H^FUpNK;9)5jp2(!Z{Tq z%~@nBq7TVrEFY4YT-SoUF{$8$FiHrlw3DVvm8?+tl^)0)dQ(8l1d=Q} zzTcSGFax?h5-XfZX(+>si()`k5x3cltsa-6i93pSJV_=2f1BJ+vMd?B#)A{R#_U9O zZDRCAizEvYbuR4%!7O|07 zC=pFaN?Yx8f5!-+C@!IEJa!_MjPQq6HlY5R9e2~UmPp3Fuj@%TquUXU>OKkam_O(H zxiw^y*uuEm!NQT5P>EcR=dv|!NH`)l!Ps8oc4gOa^B>h6bDBf!dznWDwAmh1tlNQ7 z=IQHp&}+}v=&X?mc9AiZel*;y5M7iv{tE>rKpostH7=9f&g((I&w- zqG83ve-FDU?|5GgKX$wjqWHXt?I_>TB4S*5Ngy|(#UiTHbrJjw*1I)^I4BuNrFu?# zEDvZV1Aj%i`e6?2@_;5}MiVlm3E88~_Fhi#o+S8)hTT0*@R2!Ronf6G@hO|nuYG#N z$tTD3sBs1kZP-KvQs;4+4)bIlu$sqnI{?~`e@S;t_IpGGYdLh<2Qrp*G6ve$qP5}L zzC6Z`L`*aA4Y;~$9~gnoAd%`2IXw4CSSo;#(F_=e%yH8=aLBMu_vLyN6f6c*MW-H+ zA>4@6c+JQV^#&fpQF#)DDGbWoX?v_RC^HdUi$Na`cAD7jkEqzddk*mnc)E{zPTMbI ze_9(Lsv{kKErk+7pkjUe#Sbl~=d>e6yBZ9adU}i;L0F$+y3cx!xHMK5>p4AM@6f0h zD2;O7Jz_b3yS}CQymB|`HhZT{e)&2z*Z39>*wQOZmE zJWq*_O8{BM0kpfVfDqh^o6wBUsz^0TUM^4i*6OOdYKED$8&OLWs<=9&%+pdu2?-rh zOUZtC@^6V5!wAfXk{dP)(R=e!~=N}>FYkA>2{QBUf{=~~LQ0Oispg$SZJInfZH1C%u$wQ8sr)&+7{ z&Nqlnxgt(4vFAnMVB(EWL%&cG^ix{VKI*CTvPIREXskcEw8nxmQ1(z||5SI3Sv1qT zc9g^kE@rJ{g+67mApJQ#f5LpXe`|p?Drsz9Z;{6J)@F}z`(-#QTW;6Ne^{jP;?{N2 z*nIIt4Q5@3S+_{5v3Y%+G`6=8$i$SRYBe^Rn>j8MnU3wn8;EQ{`V|VGE|;bDXnzi@ zddeC9ab$B10DN~6yPNjNIhV$}a1C(?ZjLTL&l0Ntww|iD&BW2M*YdpE>MG#|NH2Q$WA?7lIPfAH7A#2bJ|SJ$5tU^Ue56!RY)hH2*d_~ZU0a3}bA7gRPu zhoqU8g?{JB!0mTP`eXn+s@tEFY%3KXIov?|!L8fsC^LJ`3 zh##r+O+Eb`Po;0^>8Blf4w4-Ae3Xpcdoh7}Tt6Zs2nhC$M@ASgf9zhoAOL~D04KmJ zzyJfn0iI+m{xHOgfPS>VvIqzmlCk*1Vh!Uv-a@3EFUSakm^}-v7}snmxf~-TAV&DY z67Z#=!Os$!V#Nww1(F3B7dpAJD`m&Yep%Y=ym2~xW31|QPR|!4YQn!M{F}#Kn`97m zPTnE-Z8amYH)5$9e-m(hm%1T2_2M966E`8J*(ka_1uOrF8}T)g|4tW_5uYj~)6uQT zAa%or_#)n906Qln$xp%`YO>&!_|{~=Lm9d&D`BIQn!`anViN{Z@_aZfT`d=WD@EAu zO0$>;blx9BUwBbYubu*Npu@Ny&Um%_i-7l-FbkhBo~C}~e?7Gz0d2N}T+~pn2hQod z8+U_VkNRz3`Xl(S2@&sM-&|d_Q~o1$w5WhjA{ZZSZ1fR`3nLINC~HqL+Z@MkPYmR_ z9Fpcj5^6(u%EFDcbDB(}pcb{bi&Kmb@GV0K-1s3F^1f0vEw3=8W zjMrz4c#J;y3IAf~z8)DsA#OP?O9?O$7BGjLcj$Q#rsK$)yAQL*{wtjahqvCqAcqgL z#=|?WKP1Sqz|YR!@hcBxLNOWtU63boV?b#--wBXFf4&!p7V!N*(1m+~Y?Ig+6tQAs zkTlEf!NR}Rg0L#lX;FL3Usl6ETJ+xtF6}5;$SNcYEVeUX7Gf@s@lAnXE?i*|%p zuSTO02sMyM@g2~D^;x5E=)u{(q9SodLh=x!^rxu3n9~eIOmj7T((oNHr{5jZxN+(| z3F86qe@OP6%Mct_1I*^!@SYniC|KYq?0KgQUbZ9dhjK z>LrD?#ywtAV$>n~psm;|mvI*4JWc}U4i*H&e+#67$l%mIa>z0K-E+tx-Ctcj-rZ}c z;PlGuVRzi=?)SRKJ@-_8@bL0s-#(OMD^@fxMfM>Xm+Fre&T>58v6Px>tg`}SB)!Po}TFp!R> ze;sh1q->Ji>St`SYy7h3j%#O)Y0Slz_S_*rq_Y_BA_D5W(dw#RW6aJ_GRYBh7?^F^ zEObEjtkps70lhh+H_tPvDH_yUq)BGHvV~)(&H}qH8Qac+jOUd*vbnM^E{ud|UPw4; z0>+$d7Q;rxpX2RjeCQYCq*EA7X`MEne}fVNRM}E066PBS803f%{&< zlWC<9Ue|)`fJMMqu!+@^>gB9);-##^w=N;c)Z(YV?b)$}48c-{#U)u%*tb_Qe|Tk{l77w>W&_ZjuaA?}jr3FURB20@`ix(VO+h03X56m=~uB($Y6>ne>% zf)ci?G?M+D6!qp{K6f|CYW(=GCH>ko9>&R8EY#;<-Nk9=23#7q`>M~ya6(ka7YB5w z)guh$bUaB2tyQmvr8{M;^gp@sf0cjmm50#a1^@zn+={|M%*CM+7@(D6My_uz`%{GO z4Rk{JcBsiKTdS)Klw0zE$~=BJfl3-uy#ZnP30x5lLP*(!>ooCjaY)h*4FPD_ZS}N8 z2`C+Y@`t$iOQ(JptdyEwuTm~k#Hipjvnf0UDTFD|*V zE1)Yat=hE~5bxIw^5y2cy1G)zQ`=<#b^#TQWdCS79Bp(RXRvhPNML2WFwa-YXbu-7DS}2dHFFlc@wL{*F?5uIN(MTqPYpoYwym1Y# z$NZmrhgROkd?Sdt1&&#-bN}RR=4XxZB*_v0FWBQW;5KO&3X#|WmcZzMxY(23xaYL> zZqwJ1IAK9N%T1v)e=u$53Td+)?ij-%_pT84E+elS_B6KT)LUIGWY=3JK)y#Mw1}yZ zB_ODZV zyvcAnR%Y0A_A58$(ITF*LtvU=5^%WhL$S*O;{HLdUJ;Qa_X7>CeU&#Ib{H?}i+ zGgBk&G~Q0a*tUQx6?>g$?samXW3$ZMrywEAHwP(0H+Ki9g#)NEjy?xOp~7$|;;LH% zSR}P)ol0s^e?o8Wzru#!NJY4eWH)J}xwW;mwYr+^;OF*kp`7*c7r$b;0NL8!-bTRI z_V)D(xDOC+eSLi$;nvsJH!9%1Ea0wRzb@deU%ycWC#zmV(l{`FZ&i``@o)UG00#mE zG*DedYYq%l3Sc0;4E!5E(15RBzafFIU%y^SH3v4+fBAJqO`3eG6#34xOrspS{b?LY z`4wA<22Eat&k~EgCaUK@9fxEUGufnX*V-AuY`tmK&5XLWb4}#9S9Y-e8cs&*O5Loe ze#N?W^UAV?Eo(A=x>n6XyRSn=&!M5vd+upSWKX)M9_c`}{Kr{`#O!QebfeNOjphq- z7+&&#e<)TDh+^{rQQ`I1ih44zQeh3BIjgJOo>O~)u)3=M5~lK-WSd*cXYd#Z^Cesv z!g1s~rr3PP1jpGSYvgmvS&+9+t7XKxOc}(wC$1Yr=<0m1e`t_8eYYCXZiV{aU5c9{)b!zoay)O$ zGgB*jL??7P8VOwjycDk`Zm_X>m#`9ET&PD8!jMCCH(ZwW2R3c}1w}OxVet`5N!K7B zO<)gNZmGelh7$2c=84KQZ7vKbFz--s<sb79Th+X{900 zgvY0LL{gGCyC$5p@ZjfKE+IPZ!*vp-Sn*YaoC%}Qf)@n^4lvg~j(S_TfdfB9U6OL-Y85~n4AG})*EVp4$6o_&-X6c-y5 zs|FF#AoUIe(7ZmW5=EBvGzkqrZ`9NcJi^u0#HbDu%&xMN!r?EPORy!Rm_0GFxxjK^ zO1Ol0E7QeWLTSY*^rdDiD3X?%3A~{}POf;#mzGk3!g#2z#e zTo?%_7QV(c*Vs*1Vz$waiwh(~zH$zX1u-s;|JZDVJ-&?oahd-75nO6+JL&RmXHL_i zDgC^-Lg#9Q{*S-6_`_mt2^oD8f0{pI%aIa3v+09k6$4&{e(A;(Dl*j5Nv94()Q#kp zllQ=4AtaOmNJRrF`2$5?vW&5P`_z72O8J}x3F0zim#aRiIYNB*SY`sI8tjuS9NgHRsbGA9le;DsHQ#eNcShjo@W2JIp*eQmauv6*=69+c}z*~p8t z&`&MWO$o^4#}&kKP68eW!>aceNA!GBr~ z$#lybJA~I>j;1rcv+O0^tOM#VuJI2W$ut{Jm7|DJ&Wt}C@KOo9e|XI4=iU$t1-Sx4 zzE9kPVhOybkSn9=yPnT8G0yEU7BJ)X*w|NL};6rCYIvolYv+UH%T<~NI z-`vHm%iWH`PwqO`1wXpe{F58J%3u>N#-HHf(29w(Vgi)R<43p{GJ+Q^rOvA1xUYHF z`uJ-r#<~3r$6QrL0ILEwQWfPY9r}JFh8866Qhn^SK zSXY<~BKbYif4Fhu2H9F)C)VtqH{+)4y~2v_qH#`D31R%+D=pe*s;00>JWD;ZOX_Gf zHn(3aRZ<$qVzG@zceN&XQ}A2?WRfE@zaX68e8kz<#LC5 zldzcCFs+oLv$$c2hKZj`$H?5!p1%;^OUzszAK zVX0Jce^rv|Ljw9?axh6o5?cIHe^c>BugouX!QgP>og`0K^-ShjC*DM& zd9m5*Rq$6Dp7b=C=ySC&opKlrYd$#(NF|`6Fh|O>N|*~s7vy2*`5tWu(T3YvkfV}o z11TN}ZE2mo$+k#9vqt2duttI?pop-W0VjJuUS@)1WK%zkvV zvk93Sn`A~UPQa$+(eecBHLakyk_-Jue%2h6}nBb zzVRa2Xg0~lc9U$bH+jYoG8*ffq_Khjf4@x{o9jFasNQp2M7-Su~awOlE&K$C@-j^P!_oY^=+59z2 zdtf>&y(RVeJJLb*8`3z5nYECd{do$nH08w|A8~S@2K)T+#VCxqa~-!SD;zFzf5E^p z<^Ce}I^Cd0x(CE#Bw{^!NFLFMU2CxoaEovtW-~crWXKhdKFdY4Z_vZ;Mz4LuXxBOo z1GbiVA@CYQ@M4g=H04Fs9R%~DKHnGCxCuJZ63}oROXsYH&Nj3Sj2noNpV1N<{_t7M zL#~OR%Mpjck~)fS$iA?bOeW0Fe`;JV^2*2?uoNDy){;qWIu5*y)qHTWdCDfNmL)ae z2=)J{Rs7!<>OQMmH6{4Aa(liGqs?}Q?f9(SVNyHu7L$|epxXie5ob)zthZP`wKkie z`O)Uqqa|}IVwIC?$N+uas_{(-UMW9hv^cqrI8cvt&zO1#g)>5*k`GxvfB8HnNqV$s zKP)@dc;t{{6uUiS%4Li(u~8zWZmm6XY|L)G0&Z%8rIQ>Yk^t#~hIZMvRjM3QWg?ItmQRTVjezv}=bf6A#%guYb43R?5E8gR9b?sALg(7@65fidm&+SfkcJ(=}< zkibVTD0)DcjE!3b z<9i=`GJmVXUyTgklJ4 z`o!d9IXRpJ$zy?)e?)FS;DPV0i;K_PDH?4dh~w-sw0(fa+&wkD*uc_kGcXWPZy={* z(3kGoA%)T491^iSAI=Vth|v&}Fo(t@p`W{BGjx07$mGy*1%isQ^0g3)!T4J|v)r3_ zqcIEcwIzQb5jMh>IjzA z(e~-dz#co?bLIkR`C2cds(>bo1w}gOgmz` za%S|iUPH`&JA~IrGr8cniwOH%cNpdtoRJab{i`cn&T%JR1palS1t}-ai0Kc@?ON%R z!5VmP0f~d|e-+F1Z#H-3-r#$YZ^H&rTdQ&R9_v?qrw*}SbN^PZNM5*Tv%Bam!1+<` z_KobdjUEMuDZBO%8m#$3}I-84jD>cnN6}shr+iwJjfi>=}f0`A}7M!wcaBTry&#))_Tba{C z5C7IaT3xkg1=msnO@A6imO}reo8Za4&g46T3kwb)f9YC6kt;k*{0GqnKlJN7{*uM?%l(^g zn#viKq)T1ieDnTtpo(NEg%Uu9JS#7lOH4E(a=~}O6-P_C;&9C=I&E?zIG9{3_d3Y` zu?C#{VBH8|Ilu@uPr=O~7Zg3(Gq0!5u6bQS;y9Wfvhu%Z>fBKcQh8@P)MmZvNt9F)5hG{K{qd9+;#bz0g zZ2{Yhh$rBXs-X@q9p05zKE5l>UuJ+`MFDL-%BI>9ht^bo=4lm+G?xQwV1;F>#D34T z5YKT`+?dFo;7)cRC{At)Th%7skFma1LhPxe&{KtjNk1+^oh9Tc31#}xFWR!(e`u|{ z;H&$xQyv@!6W!6RtDKBqcH)PZvWZpfKvKmG#DWE^GL|yg0V$&IMm_#I!<y%a$53 zH#QJpp{=6XEu8xK0-BkNXI5Asf4Qz2e;yaIE7nv&n}%tIHY!glf+=l^q(oQamQuy4 zEvl))aU+F?cCdN%SwnY$Xya~|1arbKyxdhaGgLcGvhLtsH?wvvFuwEytDTq^s;F+Q zp-g=2sfgQc*5Fk0a06wEPEeRoO%aq+B&C;DChcX5J9Pj~&&|ufHw#m||JNH^S5>I1 z@nw)}8su7|)oQhE;U48?e`9?MI63V&C`~vXVrz{i(ltGoWoWXI!KFk>ah^_XozpPC z^rR_z?Z5PVNeFj;;*P6M4PRsiRhg;C9Egi1S48~BQ)ZlNbN$|0{oU#nqyhPH>X5>@ zh)(BbYf=vl;?~lcsIS551F>DSXd8tw9Mu+)i%kR|#c)Qj(m}#;!p4s^T!q&S!CFj!8rkYFgy!!+$yvo$7nwt zL9zrv0tkpf5_k&QwPVf4Lj$=XYBd|{AlM>C=ETe_G!T6sF2i>Tvexx7GVW)Q$5hJG z(AnbiS5U@|!^gt}RkppkwY6QsC*Gjfo3KH*@JU;|$XP|if3LHOGM<=}u%KT|A#Gui z`WB8d@rU$(iNAzHN%0HmuHaTQXQ@qtQE{I4iX~g2->xvtwMJGw6kNlWgS^YDF?f15 z=cHTK86-~OPBxzxt{qS2Vv*lzYK%Xt@@dL+tCn4+{F$3v9$CQ z?iy{gNRlv4@v*_#G*rl3ksFT>k03&A}E6LmW^h zKg~RUD5oLpyD7`^OFjH}u7*|2tyT{rFT(DWw!;<9K&eW1Q+lUqCytv^JcR|9C&=Z# zSfNJ}!7#)8PElNR=FQWWIT@YlxHCP0)Uh{VarR0oe^XinwSoL=ItJ^46a?^`D{+g& zXmc673JV&9LJsMOkPa?j@ByV83JHPitzFRdxe-aHC*{0Iyn=q#p-+({u!MQ)a0kz} zxbnF$9-ZhD0e(J%pze7lHLu~>W^}XXEWhy-&TixN4ZUAV&jgJM^ZUEYdZv zn2; z?RT`hcrB#&J6+b30bD{6z@^lWu9mF5dr(hPz*p9RG@^kQc0BM|7q&Baf{BG?*Ggo0 z=emM#FCfw?v=mNoL@0L~-cX@fHTqbdNU%{YAYMcX@$GP6acLe89kirov^f*);E-yg ze^Y0xQ7?{n0#}d7<}kZvTHN3T^IU3?$jpe3H7tFh~$AClz*= zn#mgVmk^A-HAYHhs2yYXDxj~Km>2oI;jok2s&sqYXpS`o(u_%3v-4Q#ha z9eMsFNnxV^Qv8V!M!oswghd`~1eN96e{JVnI9L{1j5{-aq(PHbT8P7Pw9+gNN_86; zrIxl28VB1^E=9N{*QyLQig+O|cAQQ}Ondh)vRMpKZnJKdJN(UGQa6=fV+?Fm^^{v# z@jz8zya>I5e*rgPjhM}{hslt|P9E$OEJYLA4SOnlrXDpj_duP&s_^O2z;;Bme*o#N zt{tE|!j)SCJB4S?j*o05?`58X))b+1kWPX+Z3zeHj|2!t7e)K7Ta$@5x7X37Cn(6D zVRiKuWZrE(E9i=C(Tj%F2;+dw?t{3H$ad9Xss~dt~f5f9Es0(joyJiZKlPHt(`w&vDG5ycee6W4h+mtooSS zlgileSyJntu^)_IyRZpkMjxYm~5x*Pp4ij;DYOT!5N<66E2=C=cH>^9c7K!cTu z@7%6i6a%aklQSVaM;L|z?-&H1Nw{`*lAPW3#;MDr8}x{uM5((Ee>&( zfqv54te;HSF^I$SmWXE5e_Ns-93kk>o+8%QMdvcevJmNq)dF<1kBL!$o-?4P+|v)%INTa2t${MIYm&yd5IGc# zofOPS;)J}yuPh%{#UyWdOXW!$}JyhMA~f8-}4G(kC#=bqB5 zHF;|-ELX<&BYtfC@x`xUV#{C?fyFPe=zn3z41 zb26u`NPVuUx@>;R8uVG(mtG7npO}?1PnI>OUR<7f;AC3J)%knwib-^-pBOQVwEAGl z>wT#BG)*Q0XTm4Af6dD9Oj>UeZbr++7x}>qI<^q6g%PzZ5t=2b2y*c_vlEDOk()-g zqdN7`MJt-k^2%O`GYGQ6gAj-5k#}SV#e-Eet9I(8wK$2_;^~M@LcivT&`t1y0^MZo zG%qfKo3zzxZMIsiC1}pV?6ek&pt<~!$T$fl7U^Vkhi-3ee{F2-jw#wlH$XqTCmdJW zf#cf28VA`X)gX3o^Jc5*)VH?QU)&~ku-a;_Z#ea>?TvMCo?aY6eR^`Xd%TcW;**kX zVSY7lKx6QA0mByu4)_r%SV5>696JkgRz&yurRYB4<)4dk0RgbEg0N7RCyLB`>T-Qy^01?eU0q zT@(=@YC*zadju^B_)M#b>V!RBVIU@^J%*?bq_^C_YIPO1Z(?yBuG4tpMN@X^IjQTE zjmX5gfBI0=!L?52VX0zRa31F!1kuA6Kg5985aO5vfU?zDcBn4n%nF{IO@a!XR2VZ1 z%RwrLD?%gB$elo!DHMBn|CFAC;c}Fma2GS`6PR<5{Rkl@rt$<*OsIqiSRc{B2az8} z3X>@_nKCP-L@YD$3tb4#_hK%lS_?9po60I7e?+V`wcT#7*UlS8iQ`rj|@I5W4Xfz*pd6 zR3|>RN5*+N=dbr>kkuZO*}3z`{{wYVr)d$$vd; zq)R-E4q|Q?DS}?eZ|Mh+MD+2gom^aC8UCg_D!=J|H5PsBG z@SR6^E8%Y?)Js?HWP(T!5^FRf(uo9(7UZ2O#}g!#!Z6v_d`Rt#SogdzUOQ%kX#}qo zBa(oA?zL0f=YC;R-7Wo{xgEd;;E-TJZg(n%x$xvgRRZ12r%GNEz%NK=et#*ow|QVu zjVkkk$NNZxHWpXJI(WkHdr(076RvKB$mMOh->)9*Hebv@d`l6&n^3HG4eD!IuJL#TaBydbYEp`9LhT~7L@BE8owM7-(kJ_}HHv*${< z*5z;{lKyBc(ax9e*cst3nSUqAG!E<*+qfD#{5^tnR1+ybHLo$){c>L%usfjkPCUr3 zBSgV<4d6_;`$~NG$wvII;x`*&!h{Odk-1e zb?N#d0Rdg=0OaR02mnFHbWSr8P!(8aSe*auOjy(jyPI7oVtxFzuUoFjdDYDM;zuec zx6Rpws*Ff^8eu|>oX7)Ybn{P(_C`l#~_s75e z6$_f^KNFQ$w207WVU**0-)Q?>;H}2TOBT>g3iN%L)A!T{SAWtsG3ooj=*GFgTaB+w zpRJ8F*8oave;@|syMo5=Eo(EDA~Sm9`j$ZIXd`VP{k}l@1A+91%LuJN`sSPGg;~iY z+CR5Q0DpfZ(Ef#}<}WMID{PCxbX3^>N?`kIf$eXe1zUt_KRGHt_*;#(XQydW!17-; znx2D}Q8ERS<$qJAFn#sMMS3pBWJ`i7Q2m5oW(~3~ZJe--c_X!bG6W}JM|4lVVcSF2 zknBVD7_*P*zRcbi*@uul$Ls@oD6{EYQF{hn5+M}rK{#W9z2Vd?-~>(>wI8zTyjOGAeQ~B( zB0rP-(0>Ul@;x)o^E_k=hdij_VqRsy2e%(cb_HuD4-9_A&s2f{J7MH4hPRM18}ne1 zONAuV{l1twjw<*Vtn{cpPwt8iuCHG`J#9<4SNist+$D*lg{2rVY9s8L$DC{H6(vRt ziWs$N!~_+@R1{wz4_dxN$I|)ZVrR&(tj72XxUS6M9^XRL(I3>g4|YY=2LDr zqiMw52Mw|wxouu<7G%FqZ#2napMr1W2mR%)2kGNGLPAe`IcqbL!eGhC=U?JBGA{7m zn@#w?npF=kXS9{y$f8pOIa>`MpfT$Kw1kzfLjf~-g%yAqzN5*e&Sv_^2m-3n(LN8w4@avnh zjz)898(_q)HmEq!Po1l96W^QDu4UtmOnT)}2rJ5y3qPT0K_*lYOY{D5? zB3DR{JjNTOxrI?p4qTiZ~yAml}v52_4UgWn`l0R^%zSCCSU!&Z$jq-8>OdsJ) z8!?$MVlq;MvtxvFQa$nzhGoYh5Py0cih)~IQAB=T6YjeFVH$ZU#_f!djsa}CpGmpn z>Bc%~HExjg#wOY5k(aAIyn+@9mf4QROJ_O5`6FvLqd6IZ54@QJ4>aJvurNGuPOC%o z`{o`ziMqJZlOFP?>0$%w%sFM?#E27DbRGV+p{qB4ruz<-NsN-Kg< z-Y~!$dAmcUYXRKvgz)BJ*68dVzw+QA{;OOJR0A!|hT~3U;wFZIlU8NwdS&WHW$Nal zM?U)>NIieLnJX$h-v}nR(vj`HP z7unRLRYk?dtssytxNH4nf`4-v9ntC$3%R@Q22AT)^9n(Cu7y^PP}j-Ensf*a^n!>p zYNtD)xN5AfhTXI$-T4ZtbvxBJQadA$IyQhg1g-jk8&Lj8e7sU|`&5Za>s;|NtIV)+ zU{09R4zAL1fdc|{b#OD5`v5>M)OVOaCt8qu%cP0F${i1%BezSN+<&f5-|o1j?Qqwp zM|0PwpY4dKi??tS^x-NbZEA~***lboe3<_F-+fFXOL=$vsc_^mRXz5`UUz$*7Zq-J275 z4#3@PVanakg4`Q$bF-6h5LxcI@?U_C7UW)pZvmG9taE^qaIh3&L!wv;u_`+)wtVfFIo z;GL#!0IPh*$7b-}*E&R)5~?u8^0?kDn#L)P6Ao!C1&1`+%JFkMcLjO{Y&G5G801;c zRCHnj(Iu3Dihm(@$!%2l7N)kUd`rr=9<{}&oh08d@_j)qf3?o@1)Z;4xkTk!REtin zt}5&N=knLuK^G9YVQW?UE8OAoWqfbY%{`{}@JJM^y^w{eXi4Sb&{ewyWepAk=WWB< zt#&WrxR1R?em2>n43W0Z!cjHRN)!zDbF&{oB zDkNg{Xn!km*)!5=Yp|mCSvjZ6#?z(8C35!yijsaUUIF%@o`0QkKDsXm~1y z1>iY^B)qWfN)4@8dbo)4g^RD=@#RWfu(MP6OuNJWC+@MlTFuLIyIv0)s;Yf{KSBGG zu);rr(_ULEf|C%8HJQ)*T@VbAc)fZIwyh;o>VINiE$d42ZHagx4Y^_?A{@$%#A@fi z_@gsdM9o>^z?e^ZWr|$%jzhYV_mm5tyEX@_d+=Ngj*$3c#aMr=7{etkOz(w35V75e zt~4QhNY30tw(z1Eg1FnvL0ww{p8u}Wtayz`%4yE5eDgDXCm-vCkNmurXlz$H*o?}W+a!F7J7(Nx#p&ed&ENSBza&Hi# zwR&yIWwl}7S}V?b53n!i0Ioyb@tn*@XFNmy~byk;TaH9e(5c-jVGYk ztj!&+bVy*O{P{1~-;XEEX91iJVuQ|Kvli%HdL&%NS0Sx9OEeD4iN$)5>ma7pynj_o zTjV{-Vw_{W5=JaTH(Ru0wTfw|=KbZq%{98jodwt@m3Cq69)>o*(x4-Mx6GlDW%K<|Wadlvg57!}t_DN6FY!$FY5m zPf*;(Hkrm@=33*-0-wTUY~*fiiEK8*?74Eyp(}mc2~Does}8mSl@9;wv9OH}>f^S2 z%L$$?!gz2fq6`hLt=9t_wpmdLIk=iYB&ApHK-kW`+(JWGxnc{+(78luj(@fwOBCm` zXH}d>^m3i~UXS|}FX=c3+|)Mj1hlibhQapaGbCr%zqlxQj^H5Z`WyWyg4hEtF@+FM z@gPE=V$Ml2%e|Bz(I(laO>#_|?Q18=EU()>&bgW5yn;}Mpo}G1Niz?2PF3J38`p9E z@pA%VA?`NaV~DzXWHjWmdVi}X{}LKDuO5?q-H4#EzO@$1z*??-ME%_c1g(JjNd%6o zwXlFT2#Al!K8zkle>_QMEV$@P;0u&g$qmwK=uV1K#<=0XK^hr*@9q%%P;k$ASpp2#Y{D`VJ|VTJd))W1fipB=R7jdG`?ZE4Knr<}JwU#WS>7amve6 zQ+iy>7d^OM+f{CjTz_jMR-KBBf`@Og!kII30Roc_(4&#>N3n~VNW$i zVfG2;er@?gUMN==rIVt{T~7{Vl@FJK_a3#V3*Kv>Uq7My9)8K_Jx7=s5?$`cC#~?~ zqv)=TMq@VFjZk)#7A}8yZxkbsy?$z3cPsj)zr1rco~d(wg`LE+TT8NinswKInLY$Z z-fF(f9`!$k9b_039ecqxV(vWL=kP-6M1eBb^81g z-Wn^z8(h*L;mIob<;R8}L!V@9TA$w7a2g6@!(*t->A4p7d*FY89z}Y;Zw6sQd|h0WUeLEHyi=*l7Q$n%65ZD8fpa|Xf`GP1{WiFl ziJIN4MDR zoFtPH8d5~zv_*gUQ|??zIh8mKycCn#y4z(Ubc7|SXQ?Z9AOwwh%Z#s4h>3~8)^%51 z1u^0|Xs}uQ1eM;4n(xK66IPq1+?6hd!PGr(fm$sehR>Y}wH?bjXBJox6ltyHsN-7Y zT@_jRfE^|Jo`0GS!!i1cy%h(yKriox()(zjmFu$qY}Uy+w5pM+5@x$60yf|;?5Q{!#iP>?oVK7I|b;tbTv7=aaY^iH1?(EH1)6nqR=#YKk zjGXQI4BZ}bPmT`ZU0_UUf_nfa?K$MYb}oN)@V|=g8lUdvUmXL_abvozDE7Toa~1p!XVIXbyir6)%4g zcFgG3)d8Bvm-+_mn@WjA`|+WAUZ);A;v)m6sArkc#`^W^H?G>lYemmD$oTEiCH3v? zTf3UNXG2%^#E#oqzd9gCbmQuP?9;5FPnoNGGLDcgK87nY$ti~cD*h< zDGeC}&yH9(>4B=)XFEfN@>twa0&3z-zk>s^wq9TEJ&Bw#oNVdEMYjiUxPcR5 zB+lS9Ha_{z8n83KB7nb*G#P)vbfGuz#tt`8!Lg0<@jPJ^=z)jXC!Er?w-|Xqu{>nl zm3#aHO6f{-_ijPi${uqM;pl)ic3ckDDuT(}QZk#^58(Ay(}C{4D?0B)^dh;Nzt>j5 z&9i0Cw(klRz^hD5X)r~KGd{sU^_T6O>C=qV>v8Uj9bROgF?f}JadCgXAh+pCvpv#I zU0mSF#A0uc($5MsgC#=fd(m3nLNQqFepXVn9ymm$R%y-V z)%mE!8)SJVdK#t{sE2=FWA@to+PbiY%jm({TdWnK|SLm?GF=KGEDc%FGEO)cGnJRB3hJ7OVU9jNX5Soe;K&*M-gc3o*Sd zSA6?*;av@qaOJ%&eXl{1SbDRiOYiketi2cFg|k@9vQMb>n`Ll1l38(<`bmUJ#WkxA zJ<7i?E-c@2dZf$!(aMgQWe%~E9e=I$90PRbc9ukmYT z{0Zyz`3#jpu17_8>~N>=k?tdKIIRpIl|589Adljnvmk$Obm+P7TIl+)W|0W(d21G# zxE7j^TMP1}$d?x)l9i3YvH%0xS)**SHi)L&DmS_ma1^bnGu#nIyC(CO>XE#B$^WH2 z5&#)v+H8+^dL$r~l%Ic~;t5YjW#x;u^)0RW_x_esGyrm`3p-X?wB8O|Vm79Mb(Bt;O&SCqYUU&Ao z2RQ;X_1LwCb^1o9%d>b=kL<%fTc=hGG0!ZpOoB)aMq#~n0nYQQ*_fK*V`|tJ<&NzTMf_0_~6GZ41H8m`*ka+{!e&mr@r)($g?j^A>%y$*IK{r665Nlh}A zSyeryAY~4jFUTwN!WoxIeNTnXW_9(gCb;X?&cF(AFB4`_az&^Jgi#L8=khoTGl+kx zb(F|h1`~451$ChoR8VSbqFveORaK zj<7X+oY}`QRRl0zx8#)zrZYwOU7 zAlO=_NdSE`i2J?T1VU|Fb=zW6`fSzxxqB z`kuI8QfmT7TeH2#!*#ZHkEj({?jA^`hO4XgKAhxXcTdG|H9~71$-tWXDy(L6u~1&Z z%iHi!tP#0FJXj5j_Q11EF}E}}5I?N|Kc2;00^dewLb4d|VlZmABGw(+D#Dw(kvGn# zi1H}`QE<^!h|~jFkw3}9&<1}H8A9oGtNJJ43nM_L!i38?5u!d_>%z)QxX)C=wH5_` zLa)ZdI5~@zCI~3u(GY7^JpjKHJSsQebdpM)z)Dl!JY?`c6>=I>)N=2N2+04{*5sJB zC3CM}L-E90ILiln! zs&i$i0k9e%7KkBY3ZZR7T4^S*(n1ALw`yL-3*54E4gNtIC*Cy-*GN z7*)V9%}5N*DY|6l(#wBDv-Lc#A;M=EFbW0bWHmZO@LC?E#f&YjlJ>3lx!Z#rqOmi1L|1fxeBxF`nB)^2C|I^c=#kG| zMVd6aOf7sJ(xe7&8JBq zU6(iH4l6S=w>K{8(d6fq&Z*`r`D@H1mnP(@rQ#8Bpz@qrohXD56VyGs3+GVM;9>X&!7gkF66l~9M ze4lodV`Yg~dQp%#aehi28YKr!c6{_^93RVeh>3Ehy#0QK^MZhszP0B%6Ti(~^`0IJ zRoKHXESz1bg^4C)pCPHnu|N!xd7TX1JtTkbOEgFJqdkC|0bxBe+qjfJYb({P zKrPWfbY`?u@fWX)^2A0^Lx=J5At$@ekvkus3I4G*GuuaJ`}Ivbe(*~8&?p*uqXcBV za#cGYRpPO;sa6fPI)U(9?O12~=nKgPin(A?0ckf(g-dJU$T6;p(x;>r)JRIG13L0k zRsVm0&t5a2Qg4Cw!Imhe`UIT+7_Cq-zJmgQ!t~Cz6=nw%x`o$VMMFH=m_hmHI*=Ru zQ0AKdfFBQR6lm7#Ucpe$t#l_$260MiEE8J=C+zl^$#yu6SD?vxW zk=4NjTy~x%mSDniIjJ^LwrieOc`?FgnOvF8BF5sF6Ij07M8g&2sXiLkKLvA3il~1w zVyUABLQKW>MyUqUsk|5XbGjFJX7Ekpi&)fdYK3UFpdj?$0B?bg#Vmg=*L*n$bjm8U3K*FjMw{|yhfq0=#Ob~U60ntr z@?1tL4dF9|*eR`rR#^AlyDcf7Tld{HW(BBffbH*7XD>cDf_2XbNh3XUUwIQCp|S#hksdT5q9xNEJGiO8rrcKfd;@@ zm(BMBF#%4O|MvqGOWjyow^j~ro9oumgWDxmU66Q9!>j^oc{a;*-P$(H^2im6`5Uvc z=JJ)NU^q?KAUapTRJ=`XE%8oYs-COWR}c1$yVe&MLC#}+mtXh;A1tRnK64>ME@pPV z%v{KXN1iE*jCthJ7PlQD9|$j4O3Rg&v@QtZ)K?Ey*O!;7_yZh&P6>Cyd!n|O7|hJs zvW6lu`EeQegQlq_QTeQrZico-Np{QTtb&%=EvZKzm8%Wfusg*}u%RYl;1z5`1|FD< z%Ugo4Dru?hfkDgyp^HW%6*j*1XigN~FuAEewLXs)%sHBm&Zth7cJKN~gPmn!lCzvs zgTQq=@bYPZ1B1wa@O-E0v-lQqoTV77!9d5gWk z_!GgiL0)JGMp}gKnp|ImvB=z-)7m4u$6!rnQiD;+^3D$H5VT-fiRu(%8qN`o2|MDt zKAJUEn6W;Sn?x>DlP@+r!=S`uvL&y0L-JrO>_3d{c$oz?y@#=O3Z8oLQ)qyYE_M=}W8S?yFj^YJxM) zjGwB<)xH<9ah@F*To%C2i3~_2hMh!1iN3LcjsSAsTYN&!whD9W z2T(kr1!(Ahg~s4q4|Y||kutdMgqu|&3{Cmc#1w0@uymw-u(q2qXxnJEd6cF?Y*UZr zgc7;vUz}i=3{E^%^u%hAd;uBpBq~0YxXiC>8Tyuv-bUn(K;;`rvtS(jk#OWWQ9lHB zUEzbG;zWai4f?+69t0DO9cJ51y>5w4(hMC>M4EAbqWr*APjPZ-;+8Vd1P5|sGoTq0 zHe)KMwdv4oGNCntXFPLAIg3TnjNGB{6M6e4-cUPWmrNOF#(=){vD>;SVl?Iv10P`o zHEh1gF@g0F$m_@A+4#$uN`qRnXk}*S%*bJb1X4CTi&$G4 z)|QI3zd4A6DVd4dxZvHP%2)E`;xRwjG$Tndar1hvZ?&_I=5HY4o`LH33LW}TU-Kl! zH%j1=UKmhO-rm6^91X3ITQCPSGlYAH`BO)K1Qo`{k;bIzk zdPhmmol!j0uR3Z5LuP<$xx?FpHx-QRV43}7V24|46EsZUw}#9n0i9U=I(y{0yCF4y zIB>C}TUOFu>99wE5;0p0*W@UKUdPi3k3kugOq_PxI@fP=Iofqr^G?d-{_%ZcOF^b8 z@)k$l2XkZm%;)?p(mB7v+$NG+^%X7`m+<`Un`H9^R%dsk%ibPuYS>|&GOjk~agEQ? zs9?}ao!((_dNWQ1wPN(cW=feNHg@=b@DL@Wd#z+@hGmxN3bT(-yWB*(srByS>8dn2 z$C_+N&wV%#94nB6d*5S6tcCH+vE#_G6YOt=#QC(ZVFVNCnZgxv=Gd;KsBMpj@wiLY zs$h9M9uL>>o-!T}^QU7713ikwFgZ%M1f|PRXaqA6j*_v*O^cqy#mg{E7Nv)O^7Eb- z!jmhcNy_|f^CKV%E8cI{V{QUFf&v|O*!1HEFMFf4sx$XGsppU41hQuLqI0eP%1fjz zZzTN4)tjYPOYBC8I(3mDh=P^E%}&lnc4iY8Y(+aa?@>J9c9`qUn!Kl;*RFW~?F#na zuExA}QD2-x%M1g{z9R#+l_Gzi&#Rrk|Ju2l3Fpc!(Kn@!U(B(L`|Ub#mj}*;ijCz=y0u_~c}5As(lSnL zgMxmqvJ5DEAh)y6Ht?7a`@9VclsfZP%Pq4$$r@uGdV*C(nzp9U4sOJMQRBtWGA%8f^GKNXGqz8~T%>CHWgitS~y!x%DKnjur8J{|>{3os_2 z<;M??#&P1g)hXZfo1&(4JPOPxHP*D|sFZ^cFu4kkosWdzCdLF~9;1PsJQ8wmyPmr~ zaH}FuV~(p5-JOzrhZTc=`$Xt$qF)}!2<>sqahk(2yc8+!azxuJei$cU?=vsI4fa zSL#J*A2wxK6$+9BkA&=qs52l{-hckwk>(b{{ET6F%wX_*La1J+a-Nl-HX6y2L9ot5 zQTW$cSArM0g>v?P1xw`~+;C)HxgBm;e&$gPQbPeoo%!e!=DL-M#H<{m0L4rTh|rYN z+ld#(Oak}5haWgs@kBtykHH!%SW^Jm%L+?{hFOJrHg&;JZsEo=qonDwJ(dNBeT_?J z#Ac#CL8{*jar&EX-FHhMQ zCsw01xl5Q_X-xz_>0870q$4mbCScg-XbYCLoY%90>nwC43BI(pyvY(CLeb5;QV*l$ zwux&e_7bV}Q3n_`yE|4NcIWIPEjALg@3TW2aqSS7WQerA zzmJP{CXox0dTMS)uz(j;Fs8xat5Qxu&bHtqE}&`xkoyx{Fq*MVQ1=G~w#HKAncF=) zcpp9n_FT+!tY0ZpTeXrwAQu4M4))#5F9>D54AD1#xh81jVKIzAc+Jq$E@}N<_0SXT zrCMjLa7q^~YMm%|CXBB;`iF4^=TEgXgqn>UmWT zR@?T{Go9&jY8qE z(bD~9P6m~Rxk1YB%>->4!eZxuq@bZK{I(re!{c&vhuijnD+9$fD7T`JTZxC3d)o_z zWeIBu+bV|_3$3yO@Fmv{5@I^mwz`Zlr!i)KWIfZXZfhQExThF1C`KhwVh18V*5Zpx zhB}6}ya^5=sN0+Yl&0K@#Owo}V{zLjjKuZ%eT<67T}Jk5 zgF5xVj+3)O6Ru7SgS)qxcd{r9UN~iJWe3MXyy~~Z4nU%oInj%R`@DcEL>Pms~@^^8u%0*<9Seh71p;+7?CDvU{6R|WH4qIlS zi<2YuIru2jU{b8tz+z%gkBYt_#5Cx91I&P0YWiv*Dp`_FR9!4vr=#LD`94 z-^bk>jNvL)rYzoysrLjp6I9%P+~qDBpFkdvccKgq{Bnv$0fq^!BlW{!?*3-0{M_sT zewIHWEL1xvNrM7Hqhoc3T9580nOjhWl>NvVh}C6`r{n_aEAv&VvmM_h#@t$=UfC>7GLxjx6? zyhfIq#3W0N`f5?mRCxaLYkbj|xm*Z$gD8KSUofQSk8C=@PWR4RQ2;J`4N+v6^qoB0 zbAjtX+yy_ts0XZ6Y*M&?gQF?|`v(Rw$(^tc_+>6r0>#5N*Cq{cx0vM}N!2^u*@>?@ zJK-0+Qe|qM*0g~frt@(Ss$5SMEtO?@(^4V&nZA4~<0lARz8QA@m7LVFDTV5}2vkd8 zfzDM91+|Ptydv$NVK6zV9L^j}jxWv}T1lyT_(X=i8TtByoDy1pp!VVN3-0}0bg0GY zYabFRyX(N`0CCenS##L;viw#TuVAa2RvL{EGl`11}JNzMO?C2QQydo6Bl-~5~ZR`F$DdBmSv zwbJ)}tRH1Q0l;Csi5C8iG6?}z{jyxVGW6k0^kG?{!)Z%`)%?RTMc;tByjvGrKm zi^3?-tC4s8ekEP6@oI6xR8m&Q6?JKWiN$7s4W`8BQ{#>4!A@c1o0~wA#CS>W=1?ijS1v= znSTA^;(}^W>CA)%X#B#!JG%ugQ|qEVGM%oGAgWTMB22U|ja1n|<2C8e2g7uJ%ONryrvnrjZS z?h?iYoHY{IqrSHUGVshm9)M^fSr-fLTW;M??7(*@&ZQVEGdmYA_p6f{wwu(d&Kg+; z%i#nh5nJxZr&yY~hA5d@+dR@)t2d4@J7$Tb8fZ_|Ce)SEabiRxEx?37Yn0VbCHr93 zhWgQchRY2 zvvvM1J{tIy`+2Z&&mGIKC!u&U;9cE5-=P89!NFZ;d0FdF7ULHUu-DJ`O;xv|wW2UQ z=M33}0Ol$lNKj}L$cy%XtFBmQxFDW!XnARWwpus6AcvSiqqUQhs*C%!gvRs&J!P2H zQ|N$!aVs&JECNnLVFOI4h)bd&aIjN`L0u!I+hd*eu+u1?%!<$vsN|jVLF*GH4pT{m ztA!kc@C1V>97Y_CzyDxqLxy*1B!R5g2i(kY~ zCz1hQ!hm=TdVssHJj-n4`28!M+xIZ&stCI5v4CqLz`={E-B+HI0b>k!Q|1!6QGy{d zlV})>V+@knM2SD}SlII-Y%I|}D}v&Gry+awycG>~xAnYDN@caCXc;RFP^u0OAVLdD z=Ei8FS5<3^+Ek^gR9!(Sb_MGJMs`MSMqx%)MoUIOMl?nzMj1woc{UCAb@eCB`$QTe ztVY(SF1Dl0#!r;H4`e$8l*!TtB?#xUxrq8~%w4^HdhEryBjEN}yPSv$7ot&r>80mQ zot3%OAN|NYt5MB4jcTfLnfqxNOzwYUlkewjiRb4o+BU;m5ATI0C-HyQAGp~ zcRlv)`fMI~+iY%Za^2%Cu8J6cFtmVGGlR%|x0Idsc}0h3GFJ*~z%ee2&Yw*3+g)_D^HhOk0c1A_#@$VeFek8BS^ofhY^s!zy zvaxZG--{D_G^^M=C5_f)=^`Stl{ibL0i4hd{Ma8118*(=CX zkv-g~*!t@usDW|+p_-mrxv}QvKJ@+zJugcr^twMs*&}=9<}WboBYQCPT=o!y%(P$^ zP!yHf28{M)_%D+yxLxhbu);!+6_H~+_}xH+4Q{#+8y zkwVf*%B5aHA?4GcAr}G#NX$+nbDInPvBUhHrRCRZMi$!kJ8FJvFwa}|z{kgbE7*N{ zx!2-wZy}+_N7Y9Jkqq|gKX-VPIuDj|MZTg;GD8TJi~8iFkw96jJ_;|FjKa(RBM z$1(~_McE32kZYF(%+*Ow#PkG-4*sSKO1=g5tkaDCI8ko3)I z1Zy@}q7uViSsgXOn~BhhjrO&F=KumTInqgSH7=d_F?(y%-np0j{V;%=TD@n80pj(6 zJK<8{HH1MPw>3j{?{d`Z#c+q^7bN$5@J#~HYJvLL@~;JOinaz zG!C?lDSnrXvl8W?y?lz4j7{}NH*iHmqeN#YWVUv4LzBjqG`+)R(e?3viEYwE1o4eE zT3iJ09JvwBltu#nC6q;G%T1RtF~M&b@$ia6jBcSXK%=~ZLjc!NP_Z?PKoD$&G?WpA zFM{1@}|MH zfk$yGI@qi?msbxK%om4$d&!2&R+p%C%xE<_5q7{o7I;POd(2l>)!5k~-2iOU(YVa| z>yTLjic4-0y0)_gCq?i~5`clxSqdAWQH)i!LJTQjH=9|+3d6tKYOFc4^NY|}&)j(G zly3l}5{om7+lch&50|T0R_>gLIhJkN_`CzfT{yz!%ut8REwAz zUdZipA(}AN9kEYtjXO$b3v!}FW*;Sk5tb!YJ{wc;>

Sh>@{~wu_hQjx}U{ZBcPFAqG6A zNHG!&>=Gjofm2y0e*etwoFNfKB=&GoJb7^1sS7*~g$czNCXi!B?CTWF?7}G7_qjMT zm#*Z!xG27VF?VSPhs)-ui24~?o}6#v=&TfSifh@F9?p%-><)Vk$rlmnMMT*#oV}HN zyl3z^2elb@Y`3hhlidQVJrT}7aqA=v=bpdwia1Y?H1!hv-p@?l{|>lh5@4Z)IS^1} zk^4M@mEZfec~g${Uq3$_n4jV!&gvk_l#6W73W+6~UEoV_+@gS;i*wHJh-)&yD)U>6O=E$S@ED?*>a)yfGxUEd|!AbtgRPH?_%le zdzX%?Yho9tjli|@))3m!G(|lew(S%VD<%h)xlUN&+VKKxk?O05hQj6Ip;NGYi?PQ~ zrDrTA+XtrK;8&fuG7Kiu;$l^Q`O>vT$|6u@q*aac+9qc}_30d1HHd`{2&06J?88Cz z(upPYS10s5b)2-FD z{SBI1D0FEucW>h-e!;@JqT&^gy9Nh_;&P7_-x@A14s{+nl)IjPGDg3%lS`qr93o0L zE;BL`+m*H0;+1ACfKM@F_4dX3#|pH^OyTedsf*~#UkwZtuNi70Ak zr>rEUC9)Mt6CBY52fXfRO>p*Vf>YjvTuxdOobo0(`)GpN4?~j9ke!NYzz&*n&xS6A z=3Ap;dZEn3h;Xidi3-4IQw2Q{b#7iD50xX8ETv_i%(qe352aO~`#1UK5hJ3&U1BRc zDP9h063arrUCnn0rwg)Hko@z8Hui>;(KEH`psOa}Bzf;kCF+}s6rs@Hl**9&nDg|+ znht!E@OIEaXbSOailXWAQ{7{uLK<9O>|sFc!I6u%m@>hC&GnF9(nCV*f_KHvj;Ol4 z5m~UTrzh9tu1e}RUAc6OKrgQj9zLC2$(<8uNrhl{ZwP;9fR(Pn%I5B z>?jX&v#i`T8`v~Susi$6HJH}a<;Zs{4RK#Q8W6_6M}zDRxyQreo(xM89+o6BjB{X_ zKBZ4*1$9Lx-7J5Tfy=`NE=g&4Nh13qcR$O0Jmg`pE28)HbFvy9Uahb#ISk1=`@mVH zXG*x!h2abLx@U0J=NORU>DjoC;YpBVVD60JdxPO7$&x_X7dP#pWJZ2M*7Te(x++>= zm@{Mjgy?S`nSNtQlJHVH=fEj%l#lzsl>Mu8tJ0gfPNaWXw<*0vv;I-Kg$jDyoHG{9 zkLPWe&n-UT7G2UPidyKMW{&pGeX1EcDdGd?$9Ydsym3Aa=j~zcE10E|!ZG^GH=bki zx=mT&&B&hb!>x@{T7hUq$69m5jtg2%GUtz3t*K~&ES%|OPt#D^uP)OJy{B*=Do$!Q z{qNCn(p^!%?N`Z(r<$SZbQw?}UQv(WLGxL`Y@b2Rn;EyV{M#F~bBqr~R(8+_lKmuwmYQvu4Cz#0T0f7tWt z$!M}34Z42l!9!RKlIhVtSczmBNV9Tno9x&-Yx?WT;)5uR_=cVF04VXPzU5hWv(c96 zz*i3SDcXS>t&!a?{+O6}R-oIFwK<{pO{^!|m5tHA{GZN~baaWiL$539q?8mZk08V@X6$ggJXkXkjmR3kgY zsF7-WVQFKdy|K|*+I{wo-DiKc^nbth!GHhc*`@#f;DZjSVHO7gai^WtfA!>=Lxj0}EfC$VUGVzewifKzi4(b=j?1!4a=C?tj1(%z zmni9j**`7QDfw4oHV@;4e?xO6O=B-vXK_jc1a>>=SYaA?+f_Eq6Dz9>4Xs?^nST_q zRTn<#)r06Mze-L&c!uEls^?cB6B^Rckb-r&Toq?VaBB(AqVVF6l0*xZmG^J-NXYDM zdv3)MPuR@QQ9A_BF|6QIeigp+m8dlh>6^=MAL;Du`0bU>&Q8^De=m2|*li7|Fp*-N zQV|I7HxCOzhD6GAKlv|D(wT(H1QsmIT57IUXJSfP>O-LDe*t{YcwurPISmRTBC&oq>!rJNxNR(5AOwG^eUfVSv+_A znHs4Vx&lXxE(J~6OKB9wd)E%&+1{<-h63+wT#r!(oMlw(eij^n<8t z2N`m!G7LN$54`A*14^#gamB85$9=x1Qb|VOMsC*<=3l2l5*d1B=P^Q1!*jjSq{k$!v{Q^3_>HDTALfA|zBvVu4Aq@IjCJ2??w0Bu%y z&8Ac2PO;feIyAF7ChWqnq;?OFNBgY`OY~sqc-6g2xCf=Bqh4|)DlY+;<9Ogfvg&ds zzRB`LwbPPJW6y~~_ttZ5mgegBLT@{P#dz;bSHMCl4?x>han)$aJH~M`9(ftfpc$GW zDF{H{e|79>C`4dwM*Ykze0fH_VIwW<`iM4E4o)@lPw!}^DdA~g`~TqR0W|KsZ~7J9 z%4T$vIh)Xs|D%0t+AtfmX*TJyxlC8g6?)h_OjpfSdc-_JkD5p6G4mKbZXTz$*=*1g z@TY4woAf07am?m2U57ue*<7Kg;ExAe#nbSoe`hvV=^3-RnzEM8?#^s%1C!(CwiEeW zcYqk7b&Td_fe_h{c<|N`TFoPSB9bQwhZ%=SqnX9v(_39paWh(#9_og`*J#Lo)}&%>wO0?+W*acp$FeypaTC(p6orVpj&X zGm0vTMZ&;}#8d|9)w&;_@_K&gxz&JafAlRbE+lgbo03{kBdOT^;Py4Dj64@}0e@;i zjkGGT$pte7Swdnk-8d*9EEJ$(iXyITdZBP*T){B(a#jI(p!dG6Z(zif2n~K zZXNH8s)wG~h!5gReQhH?c;}&64quru2{RERu!>UIYnslLoR?!%a*4r6Y0}&qk zU9S`m{u6iuFOF@{W%l?)$us(ym}Dzt6p2 zZvT!<#d7M25Wc{EniM1;IA-uPp|BG04u-D-;>)%%*{EWRkP|0($O#bwf7NlK;bg?M zCpCxGQy%Y+oWOIZPMS{nlJM8JV`nryDZ}5^!cWWaceL;`GW-=S{5~1}MJ@cS4F9qgeolt}RttYX zhJQy3zaYbZDZ_gsduzHbehe9QeCZMcO3f++1-b+I zfrGX5Box@gjg5Q4D3Qa4tgIYM4~RUxr%MK3mPZ&R7N*RCaOmy6fAj;TB23>6@?xFy z(hu|i%!eDhE-z^crlV7(D-Ll_vCY9aEv>+dFNq55zW5U78BhkV!8hj=W-%AF)1{_2 zNHZf<(?MQ$1$ecs^F zF!zW!LA7WoILq=%(@7<7k&P+KV>BlZH_IcKQ;%R?Jv=3kIrRwU)uWf?aoLT~&+&9) zCs-iu3TM zm&d^%73Z8iQ|P)mBo?M(J?`rmkNaxuL%G<8wAhP&Ki?V*QgI#`#(ofmIu;MqO4`g9 zwW-%6&d0}E{DT9ppQ{T7NgU9(NYD@$Y(e3sej1OvHkfuwr%rlDmmnnsDu01InCU#R zH;%UXRIoc;D_6kXZaXi4+| zs3FEBN#>xt-N}qH^7yXHoajuMdFa@KE|AOWQlsAVw$rINeOcPqUE0^(+Siiybx`{Sn^IEhNKEsfZWqOF-(xl!UhUC2J!1dX0z=v2rHa-kzH=%N!u zHSj`N;P%WqY|o+;4C1pg%om=vav@PM$S(^j4J!#mun-J&d0d8Vi_0LkFc&YWc~nGb zDDS*Zz-!GJ^%_+QcEjW5z5TVsx)+z5CIoT@^fj1&+=-VHCj=xCvm9M>d>>5{|HXD1 z+qSL7PQ%8wt&45jY1ptq8{2AZHMX6+dES3+X7?7e`&r!H?0n}p6VuL58cHA2i*Y=4 zAI)c?bQx+$b~>q>xG!r<-Awy&`-EdwOq^y1@M%(ZQQz~uZ-x_#H%Qt(R&M*M5w&01 z(tl7x5&_Psr2^g@nz$D9a`Y{n2-Yj5*I^<01`cCE=*8+X`)WEqY4JKpk<2I61wOKo z)3kT$4$3NIExzYfNF)*CcQdmls}rbM1D=FNzXJlI_$#?n`p-8p0&ag0)}hRyXij^9 zDDlpJ!MAaftD|{gwq=^@ST5_a$GmR{eqMkx3j(rN17L7s>_--BJ5TN;*v_aF^yN@m zr=yBx3Wl{CXSy3L8x?!W^V*IFS9x$CWj0yfvN)IN z`~-U$HuRNZ&(`HOh~2-|CvFh=Q~!CuHdot*pEVN^onLG9S@O|UY05l?c@AGT)!Iq^ zCI|@5X>MOS@}=U8Rc<xjxrDvEBAhOSu=^RHQP^UK{8!^n}v zSBp{962UJ|)R4et=V7R8@fN8k&gEszN%PGkMye~8kSx&I#IoZa3k)PR%+9{~VEcBB z?;W0D`gZAGnh!=$Ft>(7nJ0X@8tjoAR0RSXTNZlA7{90fLqd4X%%sBm?Qa=RwbRb@ zTNwVpqRjKnUI1@j)m&=Ez8hoWxBk@i0-O0QUmO9cxQrZAMw(`vIktSnp5WUpBldE2 zUv+~lf{tL$mtURvL6BhfH5sYJJE1*{9H(5XsiDlJlxDR{4d@elV&5IADtuC^f9U|s zt3J<19fJv+ZR3KHlgHK?(?_dx(i~ke<@QI?cX=B=_NEkVdsH<9n%R$6n7n26(_W3# zKLiQlx^0H-;*8QJUax!}wd`Ndi%&a-F2|FauBy^vaR)y!IJxi%UxJ?)^_`p&13 zvcK#9SXii@TgP7Bt^4;Za4Y!3%k-)7DfqI|ci3iS~n(f{{*^g*G zq{AB2sfWwnrHj&7{LR6wC3x1(P_za@(WCG)Ha`a}q|02SA6YN^>NOIL1=&*E2%*U5 zWG$C1>aD++C&Ya>=} zbwUk$mfZPN3IX0^6qpQ=~T+I?U}Jn?})xJpfwMU zHJ-XYQ!F9L{lAcn*C?Lv4mPcgHz+BusT^149J0`z-?Jnoe$0Nlwo8(O*l!BzMlh$w z#7TH|qr7Xy_Zg`+A6at!^Ku<+UAevXt&d?|?wH5;DHQ>2y?9W3G+j;&IM2lV`~Lbx z@XRq@HC(%5kM==6s7wQpd??hS* zN8~R=?&aaF9~XvTeF=s8O3TTUjL-y8J9NP!$ikQsI+!fVx+B4|HXb4?8GIBe-}~)L zH!f01>)D{B$N0J_Si(f8+t+S0jtrq1*x0x>Ul^q6rF5AbM}dg>u<9|16EshlO{RDO zvxk@0>v1<+hCizD$07YxfF@Y6Lzs-2Uw2uk1h-rzvSdyrv(XB@< zc*nWf$w+a|yrg-z@N|5LB5moU8u69@%Tpx`(Jo!ifK?ec^$tlZ1GybRLfVBb>! z6HT0P$LRYB5D}ibsK8tg72ReVfB9WP|H?PPI<%Z#mE~QH_x;D^v;Lr|=a^VjayvAOI22i@X7}M*B15nRycRzjprb9c&3pU~mjTFN!UUN8F?o-< zw3PH@As1*yKg)Cr$C(=mwX+pLFwL;ODHy<M#Bc2=rnH;I3n z{uyMR0jQ3YXvwrh=@^ZVd-76JYxM5L)(JT^>D4Ym=+rKTUrN=(QQS<8<4Lc#$ov>B zo!5yO^KmrX5Uwy^u|>s;R<3AsXT$FkSpG(wSC?K=XXm=!EzZt6G6$^_|3GJ!s^S=RiYSgF2(rKsKW(fXPV{erPb@S@$@Q~lZi&xQcc>KT63 zAfQ#B;x5W(tq~!j3_fDw#~IH6Umg<)S59%q?ht2xe`Bz*NgVO{op%*fzp+VLyl2gK z2?nkoGp^Z+jRBsOkTXL5?HMNZtYfhz6UCGeMN3)diTbtQgS7v0yE{B-HAZOLPR;Zw z)>hvpZ5%DH&-Z;DPdaysXRbXYFD8aMj|7rvceze?q56|4Bi7Gn%)y*GAg@{*Rmba> z@M@jr?6}|~1_qa;G|vXD{dLlR4b;aUtgv0nW;>r=*6#`E%gDtM^EMyvU&-P*pVqah z(zGna)inB+RHXCQNvU9_xydcw;Zc^G{p9UDC7=~$An!@DWuQxfrN(JBP7?`(x8a__^?2#Av2>SIe z7mmdNp3SWwj&u~}6Ufa5r8GMojjV8#+M0hX7mVwI1a^74R>PB`-`Oy1Wb;G<&*XF~ zvx$E^YT}q}(zb*jV#zlI(pD|sqc{;G)=)1NJ56GK-?H-|XAH3mb1RMlpsH*wCClc{ zjkReEJZZrW>RX$Y8I9At^ni*CyEWDU!vrIfF^g8YNm*I7*LX~R zVM3wauyMt*Ok;4mMiPxaATz}eUofSK8qSfsDJ#nlQ*fGe z7?32)&}e&@40M8`5m_2DlaYiwa$`D8uOSqdm=)Fkb;$co_4}GH1nvuB<&(IoVx=)8h|E~bD3w&(u7B$7*FBM zqMVmU9w@LRWgveVUlm|4J**nUP!A4^iezP3n2+N_NE@KZyYMr|`T}p}_fvCFX$O8K zLo9ihr(9Jf6~`Pp#^#`=WN*$I`XMkm|Ao{wNX^E*za#Oqg2&BpkB@PIH8uAr`n$ps z*?jQD!o(2rDPU&~VpV#gsYE<=R86cfX$(Rk^WRYqq7kExC_MOWs6K|0i=hH@lE2fl}b(2Qk!JK>?s!?);2A;0|;dTju5I~91dSZWS5XH`-y^#GGpY^vW z)48zH!%H{z_?0SN^3JxTF)fP51WyZ0Q)nEz)db49vpJ{jr=*S52y5#_cfpzKbaZvj zum_9#VaigHAU@xte}?B54w_Efz|5PdTvH9es5^MVNldRAKc(*1Ki6^%5t5|@HJiWY z=1mvV7{J;7->b>owVCv>H%zlbSKJ!WUH90pjGX*XA34f1H!7<2^y5nz98^PEpl=Kc zvwwx6mZN$iQqvCotg|uAw)XF#t$O6Mw3M2D56|jgWw@zp>bNncE_PJLs&I%#-Hy*) zSSIb~q+DAIfO3<#mB$`@$5GF-RH>aV7^)5N34vfAQd!QBHRHU^we7<(HEcceJ=BYP zxr&Bqd@P>- zARw$e-CFY53(}5bWTzCp7-d4Z$+wi8eMY$SGk*(znij3P)2A&v<)ViX^@QuWgdU_g zCjp9KTVuSx7|2r8NAOnr@u}!zj3m;aI^`4eZa5P&DN-cIjTD);7HGeZ7LA{Ee{PD$ zeN2sCs%X1=jb{yybR>Ub(GW!E#}%y|Mf_n|3rp*p#QqsnFSalsF5ctRhy>fP97)+s zTH{Lqr#j<kWj8LU(>GA# z%kbEoRMalJIAsV~0k=xyv~qGX;D#rS9#I;9t0Xh9a7+32{xK5x==ukHBb zbLP^vDAeV1w!Chm=~1h<+Ht>j-gMwm>u3J3V&PHSQRj0uux`VArvI{Y-t->%Iea`Sd$#0)efUd>PcAv9V?wSwJhZPU^TEB=#ivpDP2N?dT7%Eor@b;>q3Od4{im4&Y=^A8G3hO@ zaT1+aK8zr9NFF7X6^UzV22c_?&1sxu7LrF!r7e2sWt`Lp>lc<``lWCRJ()oC5Xv~I z7`8)N-WjnlJwhgq{PwHkRmAisD-hBtDGy%c|0!Zb2BfL4`{ok{>xdf8i<@y0V@Ri{ zJUYp(f}>H)bW3+Sg_r>qRV3;&7wZV>4MYnOIpPa4t4z$aM~@(YKlmP!VHz{76e+`G za;h02)69DMSwy^l@D@Jk9jk{sQs(`HF317wkrNVt-a<&|nSX#cKD3ZCX4cqSZP&$0U{ zs8OH40IVZ2AqCTFupQ#^&UmCWXZj%pzg|#-{!WrXmimEzM)-e5xWI~9p^!jA^2DQZ zyoiPr)T5Bzg0YT3mEguShoc3}+{51UB^;2{&RHeF=FVTD1*0)As|yk?KAkk?{4nYk(jl+wLD6t44aL(NC~4-$byQrng4AZD7GI|iM(4>L;dl&{6E z$kng`H&H<@#9{4)Av_ccN7TT}!mWcaN95a7Rml4p3`jukaMK^WJ38cjmlN{+NuUsT zVGWoi%Vjnh*E*n7x7MCucsc%4?vY%NYyE!A?XvG-j z@ev2V3IG2UKXjMKp>SOsv*A~k<+0DC@vw%^9{KDt*r>w3Ko12nf;U_NH|YJ+<-a1r zaol64Q4GMgpu|4zp^ZOl#+Kg!pN#b|06T;V=!Bfc|CEz4*;BfR8w`a0Mi78q_y>T7 zg}wO#$_sbh6p2vZwL$bxn?^x{4C;6h4A=lY_FaHrL!NL!(iteIz5jqukL4ue#KL`s zGzfB@K;Seml+^*|)oGqY^!NCW%yacAF*u~&9Uds3@|M^E1WI`%0*S!&gN?r7fX0uI zl5NFS-`_YvpqDa`$oj?`tB7iw81oIA@?h`l0828V6b(<2QFx0b*Scpu;OEiQVFV^oWBh z*GSNPw@;h_EV3J-ps{|_S`l6Y_K}3tN2Gv)pPw9T(EB?hC;=H(bd4M289dzu3{kMk zJ*b7S+=qjVgSb(9yDz{1*i=81V$wrSP=U$DN0r+g4-<5mH(j<{>A+XC)wXc6~Opz!073&P-2L2nkis9Lxa2Gx~$P&EG?3^e|M05W>t z1c$YL>6FRqxqpWO72LcSfWv0pQiBS*tEhMbUH!2@G#6f=fXPiM(9pdLs25X`A)|K@ z)Xll22@9~Iw@(;S^j_c~+I(7pK_(f~RbMEf8dvCBu>5$&4K7T@LG(R?fi?Up1VK^_ zc0u0Vy3U!LBcXV&om|~mfDLAB(_IAux9Mit@@0kq#R6dUuy2<)RJ;QV0qJ03Tn}FZ z3~4$|lV+uhus;WW2RO)mEguBl>x&EeM5~KK1*!%$aS?4mL9Nif*d)!+zW5>}<6|Ha zqxT?Ge26JF9E*n=#IS}s$c%LdW5tAn2L)$nt2_SWwXxB6Fp+8XO&E~m{Rs#}*@*yB z26qZ@76u2oc&}4DkhVPwfdbwzKnWNhup|_d*&r|{A&aC&=^RZk(2g(IXZ8M}sO~^- zHb`t8fCkMSV}A~LRAS3ZFHnJo0|cn)IafxgN7jFp;$b%izyOucUw{R4`b<$A)Wrxx zfK&iT5dIBzvOqV_LL&$)+-s8JfVIp6EC3#W18G6%gFsI+Hz2Tk3s8W&{{RJwF@F~# zh~kA0BvPJF8qjxR10rkl`a%k5+#lc~wsw8aZM}XGrV#CG&aG&~Fiuc1SZF6CNcpe| zw1pR7>6_x4*~25-Icb<7)CG>7DV7p7aWQEKpU*lm_Ibh-&eno1ep(T zk%Ajd5QxuYzncnx0$u)X0Tbzc8=;72n*=5(;#r~Z8AOIV^I%}v-UvZFg~AA^&ikJt z^O)c>LqHN7`cMH+6ek0lAjtK1CeZK&T2M_t&tcalM8I!ysU1VFTa=BT6!4pmq2;BM;N5o(}oM-;jc)(4S{&<*#Rm zwYDx$L%I#J{}n%cvS_|((;wcrh1j2IqY=eIP&&sMgbUVCIzxz+&tVXc8bjU}t3Jzs zNLy`#6|?`Gfq-8aRA)R1pA|CMz*sz-Mm{!3t4-F2`guenFg6& zcTR~pm*^?^drQ7X-k1I|hr(xk{*56Bf}KeR!r!EX<*TZv_Y~1kG9Ir8{ToeBysxn@ zK(8qtt1-N%_DHXz4$lkwJ3R)^Q7`K{WJTQ}te5$3)o;U4{jUc+l5xk4p+4XKEOJ(3 z67GC+2~u_2FGbsPYyAhQR1FWY=Nnigu&C`j0^-LaS0v6;*3=Zy)k8;|1FS$u{p&H+2?IIHS_z!9fge5WB>iLbgSB*J z8XYg`&j<{P*Bo^j^UuypdL;o_e76$folKS|D_jR#&bIZ{^7TF@|MN{c49VCtPul@h zL3C<1FuaIOHQke^@9r+2fw%-2&{?}CZ^HhgT?YDN(5U+08lVI zd%=9|oFemnj?T!)dq|!v4}JCN=ICOK2E)geQ_fjK{2QnG9C$f5FI)l*MMHLoW=vC4 zV)NaBToNUrEE5J#su&?187K1U8xZ>KD=JbI9#fmA{L!o~6Z!W25>A3=u34(LSl^te6otvTSOewcNQWe{^c>y=v;A(9GMrbk|cV_+^y# z<;RUp`pzZOJ3g_QhF200_?JW-C2=P1toREK19{J<-G^8#$m%^;h}@S70QBI9_`vAh z?E$BBTRBgzpI2|Gnq0DDMc;+psZ`-7DWl z5Jg53SI8|c#%I;l(qaP6%E|N9gE>sld1>=wqWTq!zzSvdg}(2CFY~vWrf?+rHdGe} zfh&#wyD#a#o{Ynzfl6Qg72e{@3lsMG#NRC2U=`{Ht<8#b;_f(**Z)8(KCNRZebl4d zoM}zRI+|2uNI2DA@=p~m!v&QZGR1gbx*j_aN+U%SGLl!H>C=2~pG)>D>ku`L=<6En zCtOHdpeESF4qCRUyzz{xRwEZwWR#A^G^K_zM^!3~8|=f-&eBGcJbVY4sXK6=zgXyY0)77C;j9aJHJ2>;De`YI ziI>US-PH?WOzac^@vh)LUo7`R=0h(^QbI!a9V>P^)h)5vW?pjKyBZtG;XjX>X2aZs z$2(ukVg1lf#=pXBW)RUYXHl8~e$F?)LfST*dXFm)_ae?btY4?1 zqQXih{P!Z@?gAWspBj4elq^z{cxSV|vx0_IOgVK^)<`k{Ppghm95+8yBw2V0t~QwE ze22`I1;^uS`l@B*vdfe#?c_27zmTiBD`Lmh=F-~x-LYb6Lg(U2~^ITbEqV;z9n;0phf$9(CvYY#Ec8n;> zy1mq${h1gVsCanJ_opP z@9SJ9rz&njVFK4r(%UUmwmJw{E2r=~Bz*|U9(O4qscv0{{h+_D6mCn&mQ&|S&z~NP zZ87f4!kNm)_Q-U!Q3a+W|GDyF)CCk6voZzGZR9|(%<#EO$ZZmW>V}eCy;BcU6 z7zM8ZL*|HJc=JrXqc~ARol-jr8T=Bf!e-rpsCDW9kq(m~qRpAiEJ%iu{-5}(kx!Bc zDm^NyaQk0BaFwJ8;)BrJr6dF6!d&JhP;tdsBXOJs2yK(l{2KA?FkG@W{SRiI6jki2 ze^caa4UbTO3SkD5$jv5fqzI#$$DW#bIIm;@*l40vwc$`Dng~L+o+1ti(?@Q*_<)$!oDX zKTJWXv6BA8H5OXoJI8$_?lUbGn|O?iC(FAetT)$p-;G=;|9t^NPg|g3T-2spXHb_0 z#)j?C;Bzw!Tn5Z?Dj5dPxnFa1nHbt|HOWJue^&Z`D)pT!Rbn0f-T08}v1wOt+0KNY zKxmvH=x?rSSm){y?0Fz?1U&cuv=%56Du@{1a%;-)N#G`<{~o>eP3H?QcEj!)agkiW_SzrEmEygOd)^!dm#I%I^@)G0>j*u7HLH z+zP)(79;RpoDp21{M4}{MD8_)+`WIINg0Rk!GPxF&?+nYem&whFJ-x}(iJXY#xyl8r{Np+?i*9M8AY zZA(cM@jG)@M?;0o=Lqd8$usjZ0IMmTADS@S-&pc2RWS-To@(AOZOTU?bUc_Z;29}F zD?dT#x0B;C@?ajWvak^D(stU>4H_x7=2cR6TLvw7zkb7{D_G?}130^y!-tixB}y18 zy8HSA=6A%^*T;9`Yv?E!W&K|+zX;nP-;k2d!*X#|p7Ec*E)FOecpGQA1BlbKt&K9y z9Q}Oz(mX^^Nw&{qzleY2kDES^+Cz>eGAoWkD3+@g8s5`o+~tTYSIO8ch|6v_h15Rz z5MBijv#QW%*!i+egeQ$kLC31OebgP)r_gVJ87wbHE^&To&QzD|s!Dd-r(KYaikoUr zSnS|Tcb$~~TUZp~epjZ-3gmqi#XF7m7BsL{Y=iC+D68kbG`aLU&KxkmICVs-Jf)MZ zXcNE)!~7s5wtN*QG1f3Cg=_pQ579$s%8TOpY z_Rd%RhhqoM6$N*)CbXwm_od2t--^Te`)#dZE+WP?bA}$|(z8WOJHQ0C^0{eWk2G3G zqj;e*QW@v=2tLhwlUft_Zy|N`PMY)U((`!hZ!3^BClO=+ zE+g7zcYOQFHVFC=rUX!^&mEA%z&XPn3XC0iiog|m_$_|!vq`2sNm_CfYopH;RT=|v zCJrI#ID)Gv0xb={Zkbb)fLrmqR0aNR>!&!uI8i0sxaD=Hj}hDaEcHYt&b=Q8@pb_6^|LgOJh|E_ioCig}z7WGs zp)e;pyB_~5B?k6qhL=;O)(V|keyKX&eB-yE_HUgNaE+GOwfWXbJ6=lf#>lIN5sX|o z^Wn}XY)cS4F&Vk{E6mu=INtQPTg881cAqCnm{!}-aNpl(w7Crf@;lAOHW`A-+nBoX zx+rDa>l!Xk87do4^NvYU2haV)Q%fCRkF$NS!qX9B-~eM*@~Oz@!jBj>h0!Q}<*Fpv zFO#r3**5(FxtT`OODo**iik3PcW2&%g8n3_HZK~*^IZr`tOMU24RmwuzoV|5Hcsddz594?o9 zeDgN`MFFI0aqc_(LjQF5u!u0ucszuij8?6R;Zg`^aEZqOR;Bh>gDyAm&Z7_w&Z3cB zGG5)fWPBYz?#@wYn&_H<-XW)<o3~ThC}AfahiOcTzXUy?;U? zS1w^2=`OE=*Au19;%M5hWGllfrF}@9!a&R8%@bz{BV*&YQAnpIy=Kvj#(4Q|_?I|6 zuqoAPw97R@Z9CS1Q~wcJf|>sFZ2$XP>KI&_rXw6HYPCE2D@8YQm?>Y6L{@>?>smgo zjyeFNoK-io+Ms<&cL;UMnt^ouFU_SE-@8heaeObG*EmtA)2rvujGnmkCt#vDM`x1N5)=4YPR;@=A6Z;aB-(!}4XCCPnMA5302I?mq}RY z)hW)nEQIpG-h859=(-Nz|6GlES%A?eSoQOah#L(29;k`0LAUy63xfY)fIpNLn|g+Z zD--InT>L7>Siis%-(2#}2(uW87Cz5@>d!tep42YdFUNm&ieVe*lTwz)6P!tQiSj!S zeU^}oYBsKF9NvGJmg@+*+N0kEB056oBeh`E!vy99&ZBB>eJ|^FjJgOeRDoEGO?l^xP zkiKLdWVB!LwK>8TL+LrHXge!28>8l>t{J1})l1_}J^43t zy}|{v@+Ki}MIkq3vmfkh$Q+p*XSZZbaDWFf>0IcFBDeMz!-c;SZkh9S$v+H7Wtu6_ z*sj-|Y7b&7WifFKjqUyci$Ub^7Zu?PL|Ak^{o|&GGe(4TmcO-sbkd+yYGd~c;}3S8 zoSlAas+`u2IJ6Zp$F0aMROe`e1d9H!>a4ZLG*P`%Ny*k&v3&(xkCJ>h^|MZz^n$b1U%n>UTLy zWHOY^q>7v4>>yGMTuWa1Zn}B2EWR{D*}-twVcBL`P#>*TXIsHn$D+bz!_B)f&04p} zTAch5*@uc7vKE+g*wlBBERiQQ>Kl48PQHZFe37#R}zn&THAW&jF<4CTH zSy_IJVz_Yu1arRgTP{r=zoU2T@HY0p~p#8^{aGX0l@oEuO!S>d=Sh-PTzZj2&9L z00e%jRHGEC;^*g6Ugl6wvDD3`LL_Oa&YBto@|IoR9D2T-aedQOlNHZ4?S{#0=(|8g zHz@#vgWm^fA{@TUPCgX+d$(7Jm>7@h&`=WeEB^qMs}oNMMn$OD5LlzX#}PtBAr^WN z@ZiehBx0D7ig)yH?&f-dbwzVqYx=BNb3M6kKMxuOdhlPMu!31C6+(>E2H@&)?_I9f&Xz?KqmMC<^-O?B9DmH5Ycp<~ z6HFD?w81K}pO1~#9-0?m6p*4TEnyZk5g-^n7)Atafo9=X(>tk00s<*ydt?T`dnY5(_$86t}(-0!c{uK(UsYFW0)Hz>Y);e4PVWPVNk7b*Nnq{)rXr|6K;x>!7#i^CW@Mvn-R`HhT zL8cJ%7?c+W!gK9jYT`b_3AS_mYodgmH&Yhp{0FexS0>Pk{VrY5YQBox%3e5 zy|=dp_JtO||MUOhL$ZF>8F-vYB)GgeV))NL2x-8aAsgA@+4Ul9Jpvnr4@P>~Rd=Bk zzL}UX1kzGy!kn{yaZCkhPT8!g$8oErv%5heV@PtSjZ;n9!|p>1)QrzJohEl(&FmWs z>zX|8H)!cR7|GT|D-M z6PqfHZr`W~jPpm-&%f?nK9#p%+^=f+m}z$#=t6G&Rvd@phL>H8_A zA>7`-e(_X(qmN>9-?>)hWVOFaOe$>+P5w|Vyxb_5?Dw*Oxb}XR_~+Eq+r~1x|1SE4 zh3Dk#`U13cIztXTIATr>?q1!F*d?s^Lvon%af_uo0haC!gtA`izatmgYpK_58IElU z#r;wU9U6_@%{DvN{Vk2|FSnJ6)mu6=&U_}8SV~PeG@tcCjjHZcR;pGEPEPdYCi|zj1hYUbJf0HX0{z>ti$sM!iV$k1LG!HfAheufRkj*treHe05 z_GiiQO8(&p9OZPt@N#un84=ic+GsdjhvVeFFWckLJ`!EM$Ifl8RT7!>W_*3Tz8j?3 z+lH~9&jy~~uL!;go)JB%D2)VThasS}heJW8e|Y(FS| zqL9#|`KVH0D;@W~tahmH1W|(dk7m~~cbc6f7vbe-2b)-+4LN$qtr)=3%12N$xI$Ev zrZ68{WHkrWk!1{o{-dTYt&vp9Q}$p5h>&Z~*Sf#hqOC8E*@32_aAb&b#jUB@T#+Z9J}RS)7Thts-j#5*fMP+bG< zcS;u?tnH2bup$}q=ET}ANn00AMW?7;I@?HJHmbRyQjk70>Ad4)q3yZGdsjZvV}8f1 zpJw5W3b4$%(FGv$a23bd50dN<*Zd(wp&GHR$(2d<8sm_Iu~!h^GygY9B*byy`}&;b zk+avk+kqVq7XRVMd9G^u;vEWDMjd*k{64A=T^*yVaOED2JMuP!t!U9n&9(@ScJ;<0 z&-q{oh)^4*sdu_LZS>VUw&3GkeB=q`;{SGM>X=chD)?fEit>K&@woYceZXR%hh*4K z+~xleQ9IK`%C96l_scA-=2`qb%vY7d#r3w`{<-7CmaPioG05I7bOIbWVN!TVN*hndyr6;6>UC9>zINW>`)_zC zmBx1C!nVcGH*HA0k>3sO?7t~3j4h}u-_OO_$6hrc&JBO}vZJQ>FzF_%F@P&gxhqv1tX1li=B9wU0 zrW8w<>+AM(Wuofo$Fh`xz+3GiN=@U>nst4tTA~7)*O|llV0VBl{^P3h)$YXd&hqx3 z`qQdTKyG|q$(6_H@yO#m;+v-kQP*9oJ+wf(B(eWe92I}>%Y7N{tzY|%hDEN2{!e%P zH>`M1!S{}*)u4}6qDRJdgHAdd&*V)ZFLp5+#FOjKW_h~_R2qSG6AqnEMwZ3tjHl8Dmxg0na<@%32sE zUseOD)hW%BeO}+QUoRsk+fA&$rU$onX0>&V&YzHQ80OjF?9{(0A+aLE>^=Ezt%MNB zX#cpe_4NC#YicII^RrU@_3>(80O~g?1@nG9Z|N;-C*YLCgmnc%p?wgT$L^Y9-UV|j z?-WLs#7Rf@Qr;C5+ks;eWR&jul5k?HhKSh~L z5ds}qvgWIZyA9sy(i4eNjK*+_Q|P}1yTg8;h!|OrX!5~9=*)G;0|&mXtb~Tk8t+)x zp1@KluoE6QB;73*e<|&mv-Ls3#^^KMKsam2UitgHs}p+i9|bd0lJUJ*KBDmF=IS8O zS_Ty5Cq;?vxa_gyp`y_|I)G5-wv@}%UV7>Z>xce>8qE|YS{WfYxttl7~O7# zG+&U!U)u&*=OW*D3G^jKXahxFmBbA6Pz%_?oCuo!GKwUG-b_$UI=K-fH4sn;xgQ4| z#lnOg#Q6i$HOln z?MtsTU{Ot-eA1m(0wP&d$H>QH&k63}qL zHfbo~-YmD>wsz6*&?pPDl*o>>QS*9_2*LW0QN!M~{(;4G;6cdK>(53_1u7&Ep~XN% zADaL~aKZi5QT&fHF*6X3eI&f{jMswbj;#cc=GH{u>(#yf1=$1F0O+)}GSH*1k}u>X zFOVO6m>qQo?Up(cGZ4m*x(-sMA3a|JToNH4>y#)cR~gS!A_NqLDiX*8!R!>Q^5f_& zBb;1oO-Nb91f>V%<{s@)f`(WGWgYBauDcg18f>bFgwR^9Z(9v18hB?&PTLZTWqtkK;i2tKX;jHcN}djl zMPuAJ8hg?o#68)b|0{~_`Dlvm0ZxoBH!oNpsRs;YY1lFXWknJoRx2A?z8}N&rif1s zpm%!i5L}v8;q8P~3`XdzXcBA*ZRwBBtCCQomowL6w|?1N>Q9&V+s6>;`96e!j#b_d z(|#Xm7sNLKT&Q)wu&4yISBknyB?R!gJ%%7vyX*5`on&l z6eoTq{Zzy^^2;mopiGpJ2Uwq$k@E1g5FjU&yVn@7b>70E5E$O~##1CF1{0@+_>!gc z%_S+1s#J>5UL?#JG|L=c=p{W*`6NBl%8c$L*FxF?$)ES)rTX?>{I?QmZn0y~C<*f; zIuMZxj4(kSuHQLp?Jnd_>)WP|`xr|er`_^!ZKu--IL* z`opEB)u}{^OTB zCZAUT5S#>9c|I?a^JiWc?5}T>2oJ6kw6G}Y(~U61fa$hf@&=(+gbV-OJqo^v;g60M z_18mm7AONaM>(j+OnO@8x+H0yZjFbE_Yy!jm24(P&3p zl9+li(#YY9W6nf2E%MZ;zR#J8iX_xCWsPh=&2ev^n~E=A8)SZucFrF!XeX^1XF0$; z2wJ4%;`E5&=kX>&9RhFYB@eR#zSGf#aFu@!ubpe;3J&U7wzJz+e zZ#qLi;rU&eTYi>uRQ#osQ1u4CCzxQaYt^c$T%^})XS(Bab z70dh4=6KX)zD47IFKE6H#9_R}iO#|%ZgKdN zztmDketf^__vpBEyXjCgE4J={q1Kc)+i1Qlbj0+%h&wya@89W&0+%fWz5^VZ@9ME_ z?Z7IgGM4=ER&+R*OK>Yx_n)~suWj*&EmPvy6Dd9Z^6y9|b)h5P;Dv5!3nK2*fV*v; zC#wD^M#?#2p@oFUo#^ z)9e;(+~US5J8Cy$y<^%`=^LR#%-EBm9vli-vqlnA%BJkmo~-mWCL0>KB7^gwR@dxVf5xtm z_VZ~YZme_tT5>r3Q&?iY9_|JcUFcpbsh7OmGj?a1sR6+u>RLPK0j|EQc5Qzmh;duO zfBZGC+x>K@YlES7bbrV>jLYk)4bT3vIhuUpQOJH!IOB*;2HOmPRlB6s0YvlgqFHiO z%b0GHz@b7kgv#pEBpj_0M+-JY(989pKZ18qQ(zp=frWO{)nDqq=scWcp_)~aLzzO9!rbpcCSYuqjX*F~8Z z-YQ2YXXhl(Di0!NPx8ggoL+zB(y1}dr`^s5FIQU6VrSr9l6?##sq#+)a;Pa}9~SPiN;s{W6)Tq=j%?*IGm*M!4%_|VX4K=U4j(=& zhlktD^YH7wlgFtioeO_Yc22#0r+W5?xdZ3112+9;DUNSt^WJytXIo=GgR!6OjQz`- z$8OHm9u#Z0mi>Ha-?;^LTQ9e}<~F0-pyJ52fdOXm_LZ{htMPyRui&czm;6C*RPCW# z4{a~A2tgRX3dk~99;75G?oAPKmL#Z?zA$DE^l-Me4n+DOoSto_AaqhvjMqjQs*o3>o$Aem|HgFe*<2S!bmk?+AB_=p6hX{=_e*Abk%gEQ|OfS;&f@HdSyt>L~=}bR9 zr-TQDKR#J67Ih){8bS*H>Q@+%r<(nFU44|zo)(2Oc8-6JZMm)bLMO8;IO)?;J7c_S zd~Ti$uYcvh+xfbRogs)$N6z@eUVFwr1|PYmvM!1{Ro>LYOBMx;;H9o$L7cI3k}OJn z@T*^ydT~PX%*GdW#!gaRWm9K}e$m*t9pBFE9Ov<4XW2v@uCG;T>CEy>|H^S}fHE^p zX!#;Z_bz|PK~wqKNoTIJx~fj|%yld~>^e<+8v{*ed7=*N$prkIopU37; z9l947?W5TdJ$A+pJ^0n;;8$-@_B*B4vG1#L{MFSH2uAnUG^;`r$r*m`>~fG; z!-%y1b~VO>cuL>C9Zo66v;HbCARyphUp#-aIurxFzrJ{et^IzHPliW}!{ctAn8#ly z=ZVb%y%~*0qs_rG1JpP~Wx#N>-%a^%SHs#Zu{<0@*c}*5r!*U59Ks^WX8CeB`YqET zq?m3JN_|y6d^r57gp@@PwFWj6yH~_8R{MQe;uPEMYdmedX$=TwZWoO4%B_*^YBPVH zn$5$c0<^zKI)l(0SEK8y@x3*fEz05S$*dD=JYEqvBrysh2nuN*1=3lZxd!*(7&I6g zr|uTm4W<}@2 zudYU$(ZLG!H_vM{{djD4nacKLs<8lV`t zp~CUV-F+Dd77`xqxOP9m%&Ira#*$*zlkHx3z}`5g=8q9#c$bU8yR{+MmIo;mZkG0J zUtGUlmsNUl@{3!Wy~qO+wx+J%_iL8^I6=g}P!wdmthSnLVnNP~?aG+e9nO z>EC9m;kHC=zlth_{I}^0Zux(uC-7P>?NaWUNJoEjH>FJ3XAqj; zY`-#QFEHx190UuBDyI3u0dWPa_iYlqZJ2JuaPOrjWJMUp097}wIOXocuz@JLW10175O$~S;iR_==Vg_z1|5OME?$4nFTz0@Yh6BOEbzO~8N4vr z%}lEBO;3XS{@7~Kbe0rno=v-J@U(P+k^XbGxWXg|9r2sJjZY89x{2aZx!E2YgPZr> zI^08UgS22Sq~Q>90^)y-s1LLStkE*bW|&{D)Rl=e)+Em`FCE}D0i#CN`LF8+Y|3M| ztOKN!#2)pXuN_rO9fG3~^tZ81>e?v1ImxH%ayZ&Fr_4o~&GHM9%w|vF+VdTX3loI}fGRqk}5lj3Q&%)~!2`}oFva5AnH zB29FEjXW|i!?x>1K1Go>0pe=oi;+?%#q{*^q)3+blC&I*o;=z2;G2OB?NgpY){d7t zn+-dbT(@-BZ|r|GDY3DQnb~l3{ka$(4VD9U@Xg?O)MfZ)hsbR=Ayk$bIY}3lE`ER2 zS`T3yOboX^K00=DocU+T3e86>?mPL3qeqxdXIPSS=9V>2v={@|Z9+iiIM31;o*Yae zK2a0a-(dX*<1sxL)ipbhlM2!@K6&CbVR?qrK5i(s~Ktu7o%;ft67uk16B!ku_jTH?RPPLCWf%$ z)NnL-_0KQzm6ws} zBGp-C9vy!SKhSUAroU5k^zb2UaOqO#YeVbpTM+zr755v3MMN-!AFmd z;bquO;ikKXYd<%5jQ8^A2f&gq)C7KHXGt-Mh$|sM;%;LX6D9&1FO^A z=<$DWc&5i^mV&pA)H6+%<-DEofw@3c!$S!8JJs%$`|;@Q+fF|)o7Hf%ai7_#Ott%5 zqZR?uHNn8sSu+&VTsdr3)}3lg{H6n=(|13%qqgsXX53yXaUq+g&8v56-smZA?f^{W zK3MF5>w^>0Jpi7(7>?p;ea5?h!Dd(AxJQ3eQs|Qg>I*%7{_x>&_~lKj{bjw_AB@M( zNjiISgge;teLL9E@%G+DJ4T-$(9Ln&29UZ{?}=BP#XN=^PUsjuwMa4(8p}HplrGC- zqjAFv(KKI_-Vr+~UelA}7~kv{vyf`((i|uAk zvZP8adrQ(15<}(?&HL@!sRrKr&}6~oJgQ5O8%O+j_z-{seg8>c?R)$|=lOrbhq%`j zs5ALkpCs!=H5|DwhOloTE|+r;Gg=Sa)>06hlI{8MUa6?u%|?V8dHqByY3?vwOi#0L)Ws<;o~MKlLIobL}7xEZZj07)ZY7g~c++@Q)=ug`z%Ur5lqOf&O0 zxqOYG@5{llyMp&-S6Kq@zQ#1c;IvW~=t~o77n-?Rf#oMQaam1YuZnzVm!D&F{A{G2i*&JYCfcb5eW2Q1=#$EU^dMIw+ggcl*_Q?zIR1Z55ApPac+dTTXnaL) z;K*_BKI{M5GwYYhas`RHZku*p$N6Th!5_+uw|)_ay83~K`j=DlDw{%O;& z9m1cp7y6sQ=DlE+9qz8(N3(PAXqfF;I$`>quFEP1bG|wvc-)cO>CSDimNVd$W@s)p zCy@PoICwnM%N(p&!4%O*oxUz1XZQ&J4o9HmHKMm+2CRQtM*x$e0H(PCkCqupR&ba9 ztej(IkjlL~&@!Cv?P1F@9aXi!fSAdi|+jA%RfJNj2oOY{3I`&WnO?;!b$ErXp`iCf5pJ5%}n20G(TD7 zlgG=XtaN|zSYP6fQ9dqpajuKUFP?t%@n=uTvZvoN&zjuhjScQ#gc#JtnD;}@zwROI zrq%1a-$h@ZA$#<31m%oAPwOfIeE0=De}oeFH3*dG4>Kp^1Xm-^)8vdcmjOJBB}Hk zLOD&=b8nfJQ*yQJDhda`U(cN(J{TFDk>MtQC?F1->CE|jp_5WO$)e1ia;2x~$(1wM zZ>D{rC+qaF?m`jU%`J~c7GwhH!lL>@y&N`gUH=)S6I1_FR7_aq4Uyrg*Z*r)R|ZTfTOX`zkfm0TYRV zV>a&^N!seJlj)?u!T|=lw>8Osv-dFE4fdD^!A{i4cRxQIUc(Lz87Q*DMPBMQtwx<> zB6G{+Tp#u`PVCyA(0!&SuxXntTjr}t(~zEAK~@q-w}7Xm)xuzZ#d0|KsL;40plN?2 ze7Q)n3YHbPcQB`fMXD-vYldt$)8PeNeOBYn**hQK+;=(Z9;%mM+cvMY8fWAA(Iq$_ ze|a*z$d4|MNAC0SpPKI@CA06md_JNx%c+%2Y)!hzb`sc z<#!DtTQ7ByPTNtyY{$=AIF!Q9j<1UCX0N&BY|AzGfaP-=nn9k z2Ug32erkmqgrOwvQtsZgcJ)mcZ&fj`R?AW>R6z{ml{g}Y~K|y%D8QpIBm0y22;B*3sz`QFX z(_dZo?bpnr8jiU;*JjUu1GyVK+>Kz5o!3sGbky5+&JSICD*j1d#dDDQ)|NF#`0Kbe z1iKny$;k3bbAo?2?%Al<1MBy2_0AqTC&YVq#`~R3ZtU!X55Dt+#Ath`KHs@pbWNYE z*T|eScNQ=QGO2u@bV`5Y&iCQ^t023_g<)#0#%X4;}y)dDTED3yP18Uvkr3Gu zJ+FWha-Wsso|VUT8UbumKYjaFe=kEjaQ^CQsNPrwPwYInzQ~XC@!Pk?ZEW*qciuVJ zMdjPJqCL&U`TMM$^{5+vB98R7HhKRgEx*OT6)zf2dr;!x{0@KW{(5d*Q&;c0#N~bR z*#mo2lR5a=LIHnV0X$%5j5_>A&)~@&ZE0`tt-le??&3(_vV|=_h6}uljCtb?`few9 zWCU(L_uuUmpZC;uRtwqjLD{2P|IX;X0CZpMLH8cUPrk=)@ji;|8+MNV5=>9^^lXw} z_JjN6^SKF#p{Hkhb~8oW3ZvUBjhBlaJPZ!>7Ube^*q?vsa3yx*QoLj8vm&{GFe~%e zd^0zrZFA;7?Kb?i+LadmY3g3vWNx@GG`sd7L>u&=xjr#019yc$$CDoq%ez(i9{vdiXsXdg5t6I}#yAd__ zty9m4@|=H*`inJ%L2k8WVGT2kb^#q*#GR%6vNNFJH_8T7u)pVr?zULRI8%W03@+Y( zom)S&7kOR{Z#0KQ3(XN^Ap<|*q1Bye0vzbH`V9Ojw-M&TD&R*4zk6()*FHo~z29L( zAE(d@VDGLLx;)jo8aUG;FH1;&nr0vV){Q6nBrkuo8z(1~F0MVVy09BPctcDfM9^jI z(1U#su9^pE5d|{dPUW_&tyknArD}ebXZoV^2L5a-m(yf{4YKax(kS(+t_W-yDSJk>MrTV3RJnUxcHC+U1$=Ur;-;8>tGCM*$rXC>_e)nvn(Y*t*kaBmj_YpjH=6s6=6<8O-)MfMjb?|hndE<$ z&^DlUEo9Fx5A0WiueAPY8Y%NdI;)Eee*Ri39q7qsAJ0^~wmE~r=CoQa+!#c**C77z zPSM8%0!trw7kYA*R-Tc6rQd2VnSoe0W|aQsz+2|udQ~zp%ODTDy!)&36>qmE$ueDB z#p~27ldSYgU8Ehp$cm2}rR2ONrIW4Yj2?ULR_wAD!T43#JV9HjPn`AO6;)_#V=;416%DIn6 z*F7-I^fWgC-Qp%Xk2A}wO3x17Rbg|=+*9tJ3injHr^-F`uT9K8fKxpzEV<}UBDwAl zeY>N&-UI^@W%KK9BNG$`Y%sja*<|K|xg7+FzTUzMaC3$oUAr$o`7F7fBVXimLCFVLp^ppIJVB^b68q%-gew03RmO#r z_*H?9{u#W+h?g*n?vGXa{1gR4rrG6!$0FP~R$CosOfvUZo(H(^^G*Q3Q4yBKTNv+L46%hy%^S-qR# zDD3pnA@+FR>|yOuv36ibZ`q>QuPZPSS<;)+?~UsmFHn`UA=} z+mzM3UeD5UwMedF@Df51@dkyi->maWyR(Hmo4GqXrh8hrh8ur%%`R5s-^M5qb#=W5 zBNyP2GrMjJ@`bxzxFDdt#@@{_4l~?`(8ivj^Lj(8g+{WffxI!uX|SUsifxE{_tACD z3o%PrzmBGfNKWde_P27E1}riiVZcL6Sl1?IY;0r3ZsOXMynbNyRhx7sgw+=}CW)EVZkb6FqZtjFi-X5&^iq!WTc~T5L?<83);E^|BZhp3y zo(@g6zgvOqg<~r>KGbY4(-jv)TDpEfh)pjbd5tZC%9KmVQew- zGY6gaSF!VJ88M9)owjruJHMXE*`!}w;+06UY0xX3nwwH|GVK;8G2Bw7CzDyPbc)4{ z(q3^EJHO@=AtmjV;xK~QOv{7b`k4+-`lFlh*a7C(*M6^9>@SA4*$IzCZ!&3y)jAA1o29&F=5phCU}52&I(3wH4!VAGQwJYW&!jP1dLV50Wm0o8Q6g9nM=V!DF| ziJ#0SlN~%@GcI_tg9j{Pn#&zLUE$rZ%ueEVB3r2vv0S>+- zArwP0E?6&E>`;<`Vcp0H1qlJ2VO0Z00%(JC=0IxT@W4Z)4& zL-IINo@D9L@ZY97S7oo%oaYYVPz?4O$Xa?aLFh_crm6z<#5FzRVfPqDPW&~^WH8v><^Jo7*nh%60s9; zBAFjS8^05+)*s z&B4+JlK3(ROby@RKIGt5kuNPCb43LA@i|i=A|c>yp#0FHHW!5Z)YK3m4IHinVU+sV z2Vgp9gvroSi?47HGz=n9WBQGQ*ld3~jP4jWVp7plK>fXGomf-uAyRI<=fK4h8qQOu3phageN z&GWz)q2n_mWyDc(iM$9SD)fJmN1}enn4<)dk}Aq;UgXv>^fsSn!Mpnb-`x*_LND*S z@7LtExbH{yB8gJu4>btrkk4Ygo{|4m{+kL_nl8ayx zR$K8J&Eb4E2d~3JMxyB7m>&uw!{KlyINTJG!-4%qgd{>-l73TO#!i1oi7@<=bOK6( zdh-!LAc%TlQw}g>ga-)NpoKx~FirvlLC^s~_v(7r&AG+LcWlr1h@?Z79#&v7+`l0rBPPB%p!xO`mQFosE_U=J-UzZ*j=~I8hL-F8|OV1 z|9fL`PePFwD=^snxpAF%FXO}t%B2DkOGH==C_!j|XA)&JQVf4(=zbVd)zz*<0(k+> zgD7&C4?KJrejft33;u#2%$}E=hwZ3QJ|=vlT~IPm{??!o$0Of}(3prK*&8$5gg6ml z6dC?TQ04F-BsAcN;D};y4K=b#8mcfv6pNT*LhVPfTU+drJK<^{!85*}!0Z+e2;(N_Xz&Z-MPHVs?&?)T<=!|qAZ)j8HTcpz~_uPJcMIK`82Zl%t8|Q zzG1zoMDDv&V`UT>QgUWiDWU=e^^`^;6s-WvS5hFh37EI40RMr;Z>DUp+w;D z&?kxpkss8k9*{^FsL_x`j-ZSP6$)9aqB$cWXGjYni)6S3_;)pddz%j&2Hw>;4+QRS zppBDrNX*W5^ZM=lF?4G!3Y z$W5uwrErMiJoI@Z%?qSauq&>>a6v_cM$)ID-BWIPADB2`{unZGTuG%I#Yv=uB1HBUOBN=kihH|5*!LUBJD);T#azCpo_b{sRJ=C;@ z?%t`Sy(hqX0=y@{_aeZaB<^aWi`6Rm#VRVV?M z>-yo3t%PG?V+W_gwp+`)~rrgIj#VcJ4T5V-$SVJ z)9egCMJNoj`$&SM#c9+mXbf;df&!c8b~O=Zh!UIye!G;$tili&Shx8R5v->E+Ax1F zh&uSiZg|im3c5`Ybe9pNCOP2$F}!6@f>7zk;=j4!99R3mPZ|;s)yb3h#f`Hd;ioHhfcXbwk4_ zbJw*<%9Sv-wleY=Iz)s#4DtT=6LNozz%y6^ll#wf_VW;VfPfuL;LQDBi6D^ubw^~4 z0&6nIqpd~9$J{rLE`bcG?60|Hpr}GJkU~iA2o-|yTqqhzJ0(1zX2}Cixg&&6A_Y$O zEtdPC#GQ%z0m!Dh*aiB_fCy?v5i%y21D*Pk`#%{M7l0N8cYt6Q;ycEA-^qW3wWY(m zoJ#iuc~6k{1o__sl(3%0$NN|oWFCk#}&p-h3FE7#$h&0AI5){mS!x=}!kAxAHOFw}9$|pe( zNoM6KQjrK>lwk|5q~X{(7Y=_DKIqeB5bZJD;lM^SWA^CNum@C>=SvaYhgKY2=qgBi%D%o)oj41ER-``9w-Qb1CJNF;JDVD3Gj{QU(!;L};u{ zSm^hj|CpizOJTmTTj88V0Saj;_?y&1UdF)I-LP>qgecYuSU<|b;NO4Q2bB|vI2VMD zAffPuUz12{7zFeX5*ZoXV>AlY*80X^xXuGZQA+YC=r6CK2&PnI_CV0QhY_gpRpas% zfmEKEKmeTiqK{teT@{Sz2VxHx{PTSY6$Gb#c-OPQlE7VVCik>(PYeH=w9wu2AxwH% z-Fzuv0VtA;BVa%y_#J;Srh>4x$ngoM!d76sU*X4=4u^sSzI9_{J_v|Aom->ONEssK z0Sy9&`5_TJV2$RFNze%z0fPu5hlPp+Ot7fo&V2BtH_I4=6zG@{;Y;6|p8Tu&q28S+ zzy>BX0$`;e_J9mYq{4{Tyi}JoAofgPG`&KJP;IR^>SBG4S89L8BB|KcipP$i#E;CK zTSVa9+FNk6SPe-isX_g43+49QjqkemTT*!UCE%VW?rGv*ohG_xK=RExx%mW;iU{dj zyFeeTGJQr^$o!zz4oXHMVN^sk@D=H0;+X%>|ZO13%6J

&)AmkBF zFfp1yKQykYoJxPGP-mo+Y?Mn33>;19FfJtJ4wpa*)Nez~{8Uofz=)fFIk)Ig5#R6< zOv(sCbcMv1s`ko|lrSzM>dvVmQG=BssEUj-(RY|H36l*z2E;ceemub2e~XBU1co0J zXiON~U5$og@f(5j?p>5KB9--#RX&Otco0S^*z2Lo!4Q*$y$}$zE4FOE_aEIklp{A3 z31S6~chY|#67`jo_S^&`QhN<_))==2CXCv!yTGUmMkOp0s1nGAq+;UQt6i0CivX1hXB z0S&ofpd^HS>@KIVy$O87q(vY_?ye?slAIcqrcZgewU!7{ArUcXAiy8HyOR5;^!8{E z0!>tzU)$k>UlfN^UxdcF>IXJcvS|NL477KT|M&R+BjA5e`k!P~QeOQGBbkH&G1nLY zx)^`12(9m@5ha0%Gb^}sxCB918{wfgvY;5$FleHgDWZJiVM!W*y~?1w{YXVPf{X|zI2d*J!4o?Q%%qJZEbv26=k>vZ%A=5B!~q&bksobC$4Jv6 zyn1sLpj5a6oUI$^z8!$?zkc$4(R|*6!){6AU7Pqljoj17FN{WdajX}~^oGPWE2C`> zgx@nJ?v*4vCD~5N4~Xay5z6TPCFg&XISdpCKHPlngy?s?n+ZYuAc{IDw^fqtlmLeH z@QXo$q`us5ORP9CY?_bvxHd9nzug}S3BkZPLZbml(*aW1{>r@%65iDv^1I6bzpK-+ zLFU_)nfF+IkJUdDtnTRr*O%+!>gJ8OSX^ATdmGa?rHCy_U>u#ZnW3SiCHAm_2L%uPT(D=H3y~&$RG%aq7bgDUJ(hYoQ$tXgcd*D z8umI)BW^RZ`WRc%2jY@~w@&tOV5yd49n)rXsp#M_#5+3@Huat2TO5ZeMNID>r8P^yIrnG3e3q5v7IwFi3 z;{fi5GIAIu;s_Y}rq{lbzBJZpl7X2Ohh%otWN7-AcbQZ~`X;?pC}^++OcN!Ba^ra) zDo9V&gQW==6B!#bE+Jz|50sb%MM`CML|_T~kVNN5*cEu~zyyDG7d@mj0YL+sO^^l= z6Aq(3QPOS}K!p%e%^VSt00Y22Dj+`5C^A-Pkz)QfCSdGrB`{wL6B6=pZ6N~MfpcC5 zMHLX+yT*e%C-SuPc?Z|ydqTY@)O$kx!3nh&QE+*6_8pYrdj!LP$Xsg%+k%IRXsxsLzA2Bg^J85H^OkA3?ZGu;gZ4S%{>DQr4rUdCk#xs92!8X1rP;O z9r+`u=Ipl)l{GMth~%B`qoVbB_M zSc-oUl3IxaqyguO5gCX`unxP%jI@9%8G`g1`6$VnXakUy(u6+~LP-dwLV}1XWE!mD zqpXvaa|kJ{FbEr$EzNMExRDF<5UtiFh1UG2btjdOGEQ2`WDxMiCo1w;D@{;DM%M3D zFv%daD8z~kf)33HZgLCp2sH)~h_e+9eg=P-&qzRpZ!BCWh=Y5OK;DZ+)+slTfcrf_ z#EuXCSSU>JgMj;eFraMT3R0NhTR^b!%!GoFHyzaHfW|gNrvmKgg9Qex0;Y9QNFxukr0@@mY*75~hg8-!8f`Y5M3PdaYww|$4hTzD~6%B*16)Y$t z4OQUp!#Rn39x2vJ6}WHo$tax%ccBi|*x;H{Nr%IQQgtNV_L%}^PK89Zj9ds;Wn-2E zIzUQPN0bm5aHn)=R zr)tuth3y(5Zxz}MHyxy>CY&YbL3d4Y8#52U&$wMs#2btV%^{{hvzo@=1Es@}=q+WR zA>NPIH#oF?Pya(CfV+s+?kVA(68<$Qp=W!imwA@V^zD0oNR6Q<#?$0B2c>^S`7km@ zHVh0rn`ajHZFptbK3SP4n6)1kgi;aG_Gk zeq%Hwen;QQ4YAQkI-J{J0X5Qho-q{=G>mMli#pFJ7>P7k-)~cq@Ry*|RH60Tg8bYy zK#(^HOgtCA0}e!QlUEx}c58oFV8ApZst;O7Skf^;nL9XwI!FK=0yXG@r~jY>o2>^e zV78h9sNC5Wt|o!I?fUn$a8C>WdbH5p^D{u`TSCu5<|rGHKA>Q558=y|41?O>1>za_ z*0fS`8Ew@toCs6qssFyDV?4i9`12^rB& zRxugaoYRU+=?FzZt{OC{NWfk-G$LCX;7s6hMx#)6O-&=4LDE-XMjtZtjtZK@!C-hJ zO+aZLKv1j@LIBfNri;2$E1r}T!bl2;6o*l5egU>FLYZ(opoNY4U_!MYWi}j7D9{k4 z3nldQ7Cyq6L2~hM3rK%>>5ioJ67fSwLfr)tUb|@%S`|Xl$v!gG%133B(u5RTk3~SZ zVwg&8+eI7AK&5f65L`jf&OXf1K!;0i3|k_l!am52#x+D$Hwz%1Ujy;Go;ijD?-26# zo)+(E@tzjH8!h%M%<_{Ht#8r9P*^k@eN)0^lEc+K^cw?p2)}AymVBN+<6D<+Pl(TqVXOCS+O81s4HcO%salfsLK5XvD8 zNWj5Xu@kXQ`X+xj8&^KiJ0oGhOd#f8fI&?Z`Nlp|gnSE-u_GjrR*D{>{c9JHu_HMF zC8EipX2j54IN~m*mLbL2yyIsf_Q1Ie58ORK|WDJQW^H-@w{$D24K7&f;V65eCHDx z8nHZ!BGfV?ccg+}LLvspfh3GE4D*pE>O#tDeUl^G79VKOo$>D zds(p+LQWgG9`_;Xh+#oPNe%U}ATY9;$vcN(H444@{vwtzcfvZn*(6IO~56 z`^l69J%f^{F2MDu6Wj@_8)tCW|QfRYw|Qf;GX7 zBNA~V#$Y~pRv?cjBd+|`wSw`$YL6i66sQCO!IX#^CsV>BYjZ#$Zy`jV3Bpi@s*Nsr zTQAet5u8Y2bEEJGdok1|{Ej~dH9++_=5p_2MvUQL%&r^A5LNd*#IQ!z#y)>8jJS|n zcwv$>8Otwf40T(q=-b)K_XtHPzh}5sq{};85AO-|oLTp&Sa zPi;~>V0r{3APQ*5u2<2(#==yBIh+KX3fZ!zT$v+U2qD0lHUd{Og#v(@H)YHw5eX>R z@KPU0ops`-K^4wx5e)>~l<0rA-AzHon6N&Kk4^3KfDbQA0LO>v2iG~jZTR%ghp$UIh*9WdmLts=A zAvDnkt$l7(_aW%rw;*d{_nECK2qC+02d>*j#`~xzygT*0kG;JmtUFu)?Y-_L6m#dnb$hA};cagJhA;Go(SCRq@F} zUwUPg6qS9DR(e^+Qw@I_iG#1#WtE;>^$X|8Dn^k+r7tTF->M&3tg}NrPl{o0)-H-N zMs+niPJ^8nHOl6zWSUl25HQ%5;BH`BJ4qIcNise2Cfv=>7Sq$ADWCAsWAkmNrO{|~ zuu5jLG@Hk@Kg5NHgYWdUzmsXdlWC`uZUfMT=OHM($vk%6Jyw6#d&66%m%DGV{#(qv z1<&TVsJ(@$n;LEYW2t9p;tZE*W|!FUC0$*Pt}U``A;mObte2TVl}ayHi==`*8Vbu8 zsyWLvTPBx7`0lckVzh-927Nl%fCZZkNzRihDadM(O!aBLnCarfS$f_eE8yO3SXbky zDc%qyhUV=wTcm%P_TI0NgQ`ffGR19lRpcvOR97)Bqr1JXZWsGpoZU=M^TO=Grt~CT zEZnjx@-yvPRB$KxrB|LNv;4xv&Q9`T2^A|{xZZ`HoTZfqea(t|<(cw-@~D2+qq)nP zRp~O#=HAIVo5JsSx}Ky{Z=%0VbulE;r7qzv8{PR8y6=CD+1gAihI98+!>v26eX%gN zN11-BV-f~fg1EU=%Mhct%)j+c(yGQxTih+NyOp=lCsj<8FW~LEu)sF!(pz_2pcmK5 zg#Q`4t|^}P8sEQr-L~nq9n5zc|DkvBX7fDF&vg;w=jfU&l8Ih8iTQo~JCSUlhqIhL86+oNj+5<|=cIhGo>JP~4(cNdX&2??Dgq zs(L$i8p<6xkDVKdx4#Fm-3{ml4)7g2p2hr;k3zPfa%Q_Lqr3Z@>;V~|VsYMSBdS^tfu zk8Xb@Sg?9BaZ{s!mCjF2N?p|x#jbu$DWxN~U+=aG=Go}~XYW0iR7aLZ(O2Q_6BBEC z(=?Jmu2;;&6~YORuwrK7guH|-G6?AC@Av(EAMrlP4H-bf_cG(oT08bMs;bQLSy@?W zK3l#QEt5yb<02pYvGtxJO_$E8%V%RmYzj|;>av_J3K3=fZU6O0^!d8~d`qZFL0pTb z=SJ%|26tCJ>xge@auhGe^fCVmyuSvD=!ic@Sk**=@|(A{#a{qVK(N0eISFTN{&;$N zzFsOMBi0|6)29SC7~ba`CMrpIdVk?rCmhvyXTCb3&W0z}=a(X=1Y>^}U$sIUFRH=H zVnOnCJtJQ@%YEfU$#TRCbF)YLpZ|Q`6;x9QhXQz1O7I<7kv+;66u#D*=hO6G{~KSN z<2j3ECa++8ZFa0+pKyln*LW&&sv+<9d?cDFSOtq)io6$I244A)v;2ITaE>gs%+7~< zKgJO7otUi`USUs!O_zVg299WptoJkYoQMv9NN(^H)0t!hqct7#2tq+9H)ynr~{{4y!PQ6SiI+A1ssFjBy0l?{PgG=PVt>k(zByjW}otp zO1ne2{q_=V3laI50s-_}3s+Xo5`!j#bU|lU{EyGM!H9w!H zWXi^i@YS@EU!O0hW?&B8NC@Pm#H9bxhJTBMTNy#v4M7?xWq@<>Z$)H<@aR0tkSr@a zGSA5lbOB35BrcJt3uTk-aslyvVT+~^C+(Lw5S?+C5$)GNW+bEby!wv*^MArO)Ys^{ z=&t4b#?YUCD)E2rTFym_;g?UPPq}!JL_ar%Zo0pGQasOoEf#|ycnL}`mLnG-002S^ zpMENdKb3f!^^{yFUi#%z6#4K|PWa`Mkppt2Tqjpb{tM*DTqzeGz~S%Gr(z5nqWP&L z{^#lQKdP~6_)u0zZn^Q46~agQ_^DcTesMhgkK8|Lp1=&l0FXH->NXf&75<)=jj9AQ zJU>?2pVin46amLy@So1e!9QV&{*^>K+#>NU`7JAUEEIS}yYyUmS>O>Bm*hv^2@j7C zDXuH|xFYZT5|{j{1btIqTI1VSt-t+*ALV5?8*&twU2o7+=ANFKjDrm0{d|y8izvMm zZVwxjz90hjCcQR}2yxu=-_s&GqncT;u`3YHwl(CLI{LPols{}~_ zHJ28w1V0t$LxUxCBMO&zuxK+|_ABrR82~@umujp8Uu!=%tNCRx^0{Rtkt&1%puX8ft;1l%+qYWM0bFU;5@s=C>-4GwR^3 zZ`+pRxw)TXOu5)J(NH%(t7gv9be4OLo(AIoK0Upq`hM1q3&3BfwfGL41D~F9IPg`ZV$*(o`dO)e( zwZkV1PV@p%G*48E_gc&mCCuoDfL|vyPMeoJ^!j_W+yYO{wxAPpR`{;v>BEX2>3#G) z%|(+Dj?Ws?H@MK>fAV3KsiqlDtF4*^&i6dae2$A34B=6TH-mL#HM+{b77~T=C?UB! zXVOwqjVS&7WofDAUDNjcgV%#YA7kB{i9{a}qNhDi8!Pyx75c6z`dwEG{+z*;%}W|^ ztU011uafGxUV&FsJyFSd)!9SBa}4v}2a2W;|8+e|weWghe{L1}s$T5da}=R})0SO8 zd$o;w&Hc+QO>(a-$(!xiw>E)#csLL1{A`X-+InH8v+Yea^jOvT3O$VKtwI;Z@{88s z`54}+^=u3uY~s^b&UaQ)5FebD^K+z@^WPL76c5~?wi7HVH1|+M$*J~VMHw+T4{!~q z=G>3@ddQsee~E2>(okK^`TEm^*S0`WJ+TmR3$LaXRhB(35+QZ^0MF!Htn7@NyU9?- z+}+djT_ksT#U&m50r9HlhO@(%Ej($jwxTu?j5`|9JuHGRl&4E zOL*N1MB%G>%75-$NxoUP&kg`M@iS8YV+iT3bAh&dmXYcr+t2;Z&wGV5LH%i6(M-ki0yQGAN3e| ze}gkh`h22(7<20}SA9{i_Klr>eKTS0sjIS!H@!=Dz)+e9y?nqQVOGTZ&ywe+F~xSE zVQ)ez5q3Z{y`RglAj;3F_T*sTr)S=G!rx?MA_W6KJw2bofu#uWwBRYGObd<TtoB|zi_yig{Me>s}Ovc1Wb?ag!YH=Td#&0@dge=~gDQ*B-R zV^0fWwXN{1W*w1zyX6rNyyXAn-@m5bUlNu6(?i9Rb?#xi+d@HMO+gov*T5XO4~vve z4rkogSuc874`)BdkuLI-`){rQMwC8kRh+l?;2j3>yF}zmf2w`&9S5V?r}or_e>$JD zOzy+-4>+UH{CdQi;xv9NI@d9StPU+}vHBt*6M1UHA-~ETM2^Jmn%)(r4gS zW|DL^l~0%S2QO+4{`++XlP>3yciI~-lJS~9sCU|qD2a|+a74l9MWJ9=(R#?4Rw2$3 zOPJq`?a08 zgYLp)_&hJV?$=I*pB+5^t0&_g!tJrNzpa=x-3qLZmsbiTG9cM&E>eS9D%!n3_#q2PL3U3`tS01ujfi9g3s z@^`_$F?(cw2UYFc{*x~ef9D64*KZv351jAjUZ46{<~w+!yxsXTMi)$&nAl3-?Zkn0 z;-dPRx9olPU*HrweL+ffAicM~!q(yK@f-L9kCd6q+&_1O6;t)JjRtjUbmaA+x<}8=bZH86|>oj*^hPlocsMim;IAI`yZDIF@O_h` z;RgCIJQzMoG4I>Be`W3skJ!3hzG!@anb*XvMyW;|j;74qg$~6oC>?*(C)0-+ z@IxdODAn$Sust7Aev3l5|FA_iV{NDY}%vvPumv4 z!oW$#;21cy1VW`|Pjn2`WW6tnD){&k{N4-c@>0Qp)9J=eM+Lj2UQ+l{moK;kN&>&? zmwUJb838YsmAC{f2Yt@}SY7`Mm&mvTCx7RQu^A6b%GQ1NTd%Oc-^iEZ0e*0N^Z&U$ zXeTbkLv8M-$DNixe;uBLKt*<}cm2ZOKzTCfCtg6_Z#!|njyIc_Xu4mk?8N(il6iI@ zo<)54hS3i)*Ru;s#%F5ENiesIns^tY5u7>HKW)^(e)H6^{sRSXIoN*${1)KH`~&TO z0he*P1S2{oCc@X?y*f+jli~tP0iSXd5cH2{DXuS{>!;||)$tXVgJ>B)?)uex!7&gcYH^=ZFrSx#20&i zkGF@ex?ak?QrOS%CAwg%ye2xKjl9Z-3ixjs;qbw8R^e34{V^?!R1cohAIai6l@ovd zf=WLBY4_!Hmr_^x!?Wkp-nrHBSu`$u@}06gxa8yFJHH-$k=XiH1(|rlQ-Yt=GI(k;*?ZZ;j6!6(NU&xDJl5ckXX0LB{_;z3aR^9yd z{$|+2ABBnG5y6^5ctIBnYgYHgx9BZ7a*dI7F7@ldCDXanrnFNz;KZREaM3wX0SsyC zJ-m^*cIfEm?GlcZLE3-bZG>kg=4kI}pIs5fS6xewMGEmR@%=@&*zgu|(O~IPNRiL1 zYlm^~Bhkbwm@DSMtflsCbFuRP;fc4F%;%pH8vOHzqN$Rp0Hsj*7@;)$9eYkV#2sC$ zVo`>`{h&I+@t`nI=n9C_|1PWpFrqSj^D!YID?S%Mv1OEt9;%bA5(`gC(R`P6?bg3xE=GYEYSA+rMrVaR9T z1xCTtFao#6F@zu4;o5e8`SemMh0m%8^88XNeGb2(-w5&yUL5f2yXtS=nH(oZGy-@5 z-FrQQfPZGspvyh`h6Mb_Hu$j({_)%3ct#k)TUsu27VSkeY;XJY_UC9EgL`MbDJNC? zhk;NCcJ6<3`;`+V%MmYV@gy>}M3Lb}Jolu;sTTeB@9R<1>2KrU`%&*+7o3TugrTv+ z?;rkIfB0wp2L4&2u0}4$IaUvyn0@1bRLN#|&cs4`i60LCgqmKzbxo>!4Bveo<0)&% z_RRxIeWStrippNULuDTr($g?wUl>j6BUbPquq~Hmy#yoyYnO_>1U!E&b;d6bdZO0) zB}0EHR7Wx!XJZ(v4?d;m0SG;N^O|oLY{fFg+)FZJckCm}A$yyi&d{Ox0Tt(1fg6Uh z%zhU<%L*NH=aS%c=SDjT#whegFRRak87>=&G|^8x5bvUWqNwYt?W*pj z7&_lokt7pZFA6?!41<57d(H>PaDPEbEz$G4N>o*{tt|UB)GEF3e!ky|=rscuI31S$ z9~=hbhRb#6rVrY~0SUY(@9Q{??LWys_Cl*I0uQu@J#e}T!-7dXbhF-u`k zfsj!UR93g-g!?*#9f+>AQJL5Mt>lU>Y|1b+1`pk|X4T@4XmfwZ=ruP(8Jw3(0wA1c z#3w{5{er$Yu$M0UYjs%DK`4HWCiAzLy8rR?KiSVjnKufgx+yqf7*9hCy|ZLalt^(Y zmNNYJ=wbB)>gg$SE@fb1N9?)64d9bzC&&43g6jO@c={jDhhMq>K7CQ2z_fktVR44` zxrcyK@9848W66Jx=(>fy*3~s*-OXzO486W3p);*Lw7d7Z~TpZst!?hlsbG-I;;YJ#1gMMk;nFr&W9~ze$ zpT9+WE-j+-i`$K$Nsl&k?;a^UFjA2UzEd=quZm-|XV24ZNUgU^OcwKVA&S&e2&LY7 zWQVm2&u)JJh*|q2bsq!d1&N35DkVEx-g}?Lsm&DaiX?zcl6$N>R?j|gE}8nD;;0qp zFc+8Uzhz0>o7G3B4*q*iUFNLzLJ`E7&$skyeF32`z~+gQw@W25&fZc5fY9Sx67lLH z>-bNwii+ExvV2N9beizP_GxeXbTVR>Y?V&N@Omndzkxl|S$A&t=P2cu|A7Dbk z?Gi~l8_8RR{sxrqLZ|Na=WbX2^j@S0g=dA{Tdz2}eZh~)Q@~~awW(;hX)2e50Q{nG zec&O4;9n&_OZI)-hA2ka+fCcaL)nA0E&jgaurqaLTpy?lWAuGA70##c(4DiK%W^No zFAL3h7`zM4F2A`pM(X&v+Nxp>x*4GG24DgD;?RZ{hCe$Xf8(a_8+=PZ`rVh%#1A2q_E#8d$;8?(*?Vo0c;P$MsZR+n(r~#T|%XLn7qz zj-~4bPE^>nYB@>B^aop%sIZ>+VtcwIObunJDW>WBRB!|BMK5v>j_75)5rVxoUSS)L zoZ=aJ?5z4??N4cdM5`~J0WKtlz86w#*JGnfL$5Qj>zTFacq-58ag_F>wjFPO>AqoJ zDw#|BtU0>DiT&cFVt_nfg8lDJkQC~-YcW57E73lHuaAdvueZmRo;v}$(%V}g7kYc4 z?ZDMm178~#<=A4ya@3t=dgtrFv4YD5zhoJtpucD#T%YbQ8J6n{<$jMJT?&~Ve{eBe zU;4L}$>qtuzw|?;Oi#6M1PjQ2_?Rjb5`e?gUMz(MV!3oAj7~0TNx4v$`#Bf!hKJg& zn-E@dcC`x~7mAzLqxs(98o2l9`#Jsf8X1ZH9GV#yJk>s{_W9+|HtW!oOY}c}oem2{ zWISSq!pG2*=5=E7nC)_Mt3K4bcQ*D=RToj+syjYkqqsodFiKFtv>YRUT>R^t9Y zI~e9aM@lA7Dg59$@)-A{$MmT;)_yeUooMlBmKzCe97#8uNCutnCTGYYq#?MK|A>`nRwvF3uw3#*f44MBEd1PXTcW^`!Z=o0X;hV)ZRg?^Y$f}u~lYxCdBL$GK z56QqK$*Gh>j}P~Mu<+Kg@BTq?;e6^**l@1^X9>VP*jRbsqctW_AFYaYe?EyWw>O!- zRpg^>i*(@jSL;8XGCh30Q!4IJN*bETD~~!R*n$SI%Xfcp&OG~)<^on%3AhJLK(Xd54; zpLh5D#c+-dP96mJ?RAcboTHgelu>@WG)~?TAaV`{?~}{vf-%H>7Nm^S=Sf`TdWWl6j(G zNB7?ZFqssQAWB7r#(CDjF5xEM15T2XP*M3IIt~ zEQ-ivmJgY(XsBnfNSGb9AlfULNQU23qkO+nqBPNQEawB2h)$*w(YaGeNPeA&aXl4` zMg9d3+3zp=O$z~UDYq1d#xd`w1C$;kUlEKf_ks~p@Agb)0oMl+C6JbVG4o|DW_ zU$JninD_!F5A=ybVl*D2kVSZZl$WOpo#VY+EUkQb6zzvQIlcp@Er@f8zG$}Gtpai0 z(|%B*J$5oMp{utX3`DJjPP-4mSNneo_jl;th2Dh?jG9`2&*wJ8irJeS72mYdg{LObZi+HodBljx~asF5IL;trIKWa-A32w)av{g3s9f`LO4NuASe*dim0EQciP%g;kztAD%MOirryW!=> z(Lz6rIR$)9eL(4tQu?Eu{(#dTNa`bUbGrYq(jU6|#_IZObm{O?_1lj4mgaqJM~vJY zpUSzjV(yf8hS1Ae;>9(8YyWku;-aW#ud5$knPcWSha#B0d3t*ZeNz^!Pz7V1ep!lc zgnl9l&Rr=5Cd%A~YN}@!#C=9$z;?2uLw7pJ6%2Ot3w?-Lh6P}5+Lgh=J`T`6&H&^=4&?3tXBh)YlJ1p1VG3{@zXx&Sz|J(}4nL4C`xrZrhPciRWZQ6+ z8AumP(^(8-MP7V=03yQ*@G#Et;61QwoItwFHAWzV=PDm?aR$`8!+Dhp$h7PN6CeSY zP`tbXx^hnDR{y6o?p7@Uq`{{u%h$x66cPYrp zd5n82zL7<}~oF!!t85*ACBr>|i{2Y0kFb+ToccYmZ!+uMnU|Z_F8L z^zel_Lqs0FFQ=hCc7C4hqz7-y3Dm#gvV4a<2)p=C;tn3aQD0>ZuH2|^@C8?H)c@x& z1=sHuH<^O#H|l>kQ*iwvbe}1BY^Z-Q1-Gtt--s!AY%0DdQ}EbA`Z!Z?Kkz(jzoUBDvai%1vx{r^t1vw1P2r`=}9;{1OT-O=Y14NMH9PqdLg zyAJ>AJ9^}E>NU(P29_j2pF_nVr2H%Zc%v&56(aoCfLwwoM?dq6@EaTxM2)zA^{-+O zgrX0rE@AN1N{u3|{*t3Qq(IalLLF9>lGRZNi>Q!9U2PIgO|5n15zo~z#)LXdch$_7 zx+QC4?6|tYj!WPi zT`$1CqzJg8QBuA#)DXgJG>UAVHoO|XvF4mGs?&JAzo`_bbEBhGu~KzaE7xnAS#Kqv zNY-gJyObEe^0 zC2TZ7#qvZe6;TLXc3c--Qi?+kHHO{RDuhxU)hLFV)ig>Bbc#~jB%y9Moq8o;^rdCa zJw!$OzO@1!78F>0&CLwIS>$lA=!yJnwJh2zmd}F>X-PdwqlZ*wjv}_V!hDV23A;e< zl9SoYvV^YI7%obZhnU-c_9~#zwpjN?usCW?JhmUK0(30bVb2s=q{WajQOqOIs#}s7 zi;|f>Yv2$D>-b#SXw9PPm$0V8(e-v~WOZB1N^42VwNXA_?2ns0q*|%?;TUsPvn zs!En%6m01gIHwt9J@=pBq_Dn5(i?r5_f6eR=`#*Vtu z0Y-uHo`ZJ~K$dHKS=5SaZ%{&KL#wp`7oe@66-ooi@Hwq z$GXxprCrTL>w2pNVJ&O4=C%e_hZ~AGTrzESD9KBIL286l6*REruS(R21IFMusC8JE z22162+71gVf#zLW5^~s>UWO{Z-M%lSV$}7%Y zwMuPr+%cSg(oj}R*M->zsdj5zI!t z7HN(b7$WxJNmoL4exI$>YZqgYSgChFI}c-sITdCt+(uz8 zKU0c}l{@vlQq#d#AiMnI3 z-NF_6#T{;RCcO&n6W#f6AkK+SKToem9S5tkwqy$%yt(a`QOZXAZnvQ^s?ywR;Pt}k6H&nMK@Yxdp*&b z6vWvLzoW6b*F@DhOL>bThrmjor&g3VMtFg#O-2i9v1H40R+y?3Wl@dFFreycO|drU z%o^6#psg@Pk3*1=iUwQQhOovu-*J0unV$$_b8a-vSy@yoGz={4j$5iXRZTH}5t{CPK74D8LhfnMVMFlVPFAez0$P;cqmWG6LF``W>mkf@ikAb%Co5}dedMcv2Ae! z86%+G@cNWK^*oNw3Dc`n#n=EcKYn4|=4$x(3VeZ)dakbX)=M{ykt!sw74gyXP zTNYiUM_pu8?Y9M1ZVsVfkU+Mf>Z|1LRgHgv46bP$9`LXHS{G>(t+>}K|yRPO|Tq>|;YaHog z7D6`ym0L6xHIVMNS+5I@{5DO|5^ImDl&Y8Q3S<-RmY;EBq&_Sy*O2UQ*o{@J0A+p< z!2ST(p(ehYVYOn9CR(C@RGlFUs=b{o>dPwSjzGT~XqE*BW34*rfHn(^W;#`Dsr{1L zVw?VAr^_x4w5f?*tsA{Yy*PxbUBjUlA_%~gRhO&2Jp@$Jujj4)I^QSfq&Ms^#mc-r zwluFlai}pfZPrL!tRq0D-JI5^4Z2!%w|$p1G1(_avlg^%43(jOOja9|--paHKZ0mlQJG_J)Fwk7mtY9bL~% zt1;p#Bb}Xp%|Q?F8~vT&70J$~<1i{wn;O-&;Z9s~wG!3E6s(Mv&D#oE}e8%36f?RuqaT9v8{!MiR)k6OKn&(-t#DhO1lwCODSI@1uw$a+rC zTk9Pn>kX8*>=B4|8WR8?Z2V$tX%6elVIFp9S8TI?%lQ~w^;K8z*vzae&^|*nxt{J# z`pbbX&ezpVfb(mP*bIAK9^c88C52X}_L?xd>ZletUd`Zy0ZKZXy0HRvMGcTT<&3sP z6O$3J!V>nVF{WU3ijS6?R)uaXw=PoRdL^|~wuewF=yB_%w+hhtZU}SGhQ?Hqsv2gu zPC!zBe}q>%qZ!^Bjdrc^5*CN+KA#`-`F5Vd9e?CIK)u3~i+*`C8R<-bSo4vpk>!q5 zlksM&(e+@OTw9>0s=d+3W_!e@V<=k<4f=GoqJb(+IkTX$BDMtVx|%=s2w79<^>{%R zJHW6Ft_Wo|WMFDVY{)jPRTp+cH)L4em_1^DL9`nqs8v&iIZ?N&Lu!OzGK01oz>J45 zr2>_fO>rc%#hS=_LqOFoav2*7S!oTrn$iy{WU?_g6%d|y@Pf0ldi59+s?QHXdc3Ks65cs zaA>!#39E)YMc~0&c2^;;(zOI^L2LIvxZJAfuZuKV;ff$f{K!fw3U1#r0ja zJes)HXxJ_zx{SG6Em#OOtRt>wP1AyZwk8E_-9=?R^&7x$%Zlu_>g&DEbir%hWXsXj z>5i|E0%^Njyj8WA9Ex?PlGhzpntX3LHyC5k?N9JJZkyC-02^$P=&>C}+Nnj|fMM9c z``)}qiZg8kv%6KlHFn3q7FCxR3FBzD*VXHKyT8=brlc~!vfhD!fi%O%iy>-%2^*(j zqSk6)7smi+n&Y)UMEzVXOe!n^UJv;yW*bGJoY9GuvKF@j3MdRJs5zqwlo%#SO{$#nscV@XZ!^gtJYV3AvW;bo9mfKfsph^;+4og$c zm8RT49KtKJZxF(?y>+JUf~m}Zl=XJwvJ_1+TcIW^z6T76e%=wbv<1Kb&sTBT6MSGg z7r>!sZkuyzL}`70RqxkT(b3AY zdPO7V-6kv#TSx=#jGO(L)a?!yL(&}vT4k>8M7u$@Dz$N)q~YCeV1eyPSpkbMVytTQ zWeH^z1Dz6Gwp5bvL5YR)i=BrkuV)iJGswtMhsjkHQ2E|*YgfYJJz z6#d1nJ)4;T3D|T{pA6N1#+LI4V~ouBrB&MD+nU03ZNsEoZ#rQ}i(WMiERX|H+LqTF zaz+y~VPd&ezvI}Rx8@bQOsyD;MmD&zm1PA0CdAksRD`s{BbmT0j5I8suzT z>3NN*?_wRmpDZNPtpf^gKokr?Z9KpLPG4&3x6O@d({rL!vToYqR7$rGk8GR<0}vEt>KTG|cgLW!()0SB64#6k_W zY^OtXx|4AYTTEzwzmo6tMXa(Kk+UEm{C1rgYU~y##>RX?d63*0EiF`TEZ37^n;?3l z#Q<70Wx8uJn%7^fwrh7d*$g!rgC>&Ip{u2i>5b)%CCS;b~`NOxQj9e3QBX5b{gGd-&6Bc%??Pxg54}5vtCdXK>!*6T@wZ|GbHb%0cl_j#+Yhl8sGAE%b4l z-Zk=n`GMMCq`=8qTA}0>Ji1cGHY;3r-x(ZrC+zpeHDflh#`t)rXRo? zsmkj9m z9ipTRHWIkB1IgPmYpBY0%hd+8>hwC4R-RB}f`FFtu1?cNTUamH<&FHLbp2^QWUpr_=IZ$6K_fv1Ba|A8?PlH^3bD7&XVL7Xm>L=yW?4ZcR1h7 z=E|aoAu=hlLZ!&Hr;XY|*k}?6V$C{P6L6ykGCH>ohEz>oYb?FqpxBsnk)p^N@Dv|1 zdWSLR<)(nzlijAs80E671ME(3uX(qvz?L;!;m8n}G|{;+sdvg)XE-JE`MlYqyS4QK zY_yqGe$@cErMHs0O2gmaeI4U}c0{pfX$!7_jw@}Dr8<>-zv5{uQKm?IN85vmW)&x- zMK`J|)~S~yr`lJ1#L-rjl_zSWrOSFWK3$JJJ+E*9#$)w-YoKF09bU~QTe{XTqy>r0 zs)7nHNWy~vqfx0CNV`mJIssRi>~O8vYL%+(a$aa*!`igjHX7CvUKFQ)_435%l9d^& z1pN``bo%4odHZ ziu9Is%IdT`SaJ&H8dTX`H4s{2)He=r`NTnGas~NrsfC@sZxdKrX_Mt&79{OOnrxc4u|7>6_PA^sK zE@rYr%o}c<6-aa|J&h&Hb!I{dlQlq(>rIYm8h+KORsfk6Dv-3T`%az~*W^H9tePzL zxhXvE!_LO56xYtKOmGu#>CkJXAy`Frty$%E2a$=j9?X#Gng$0vwd0A3HP9;cE$OhW zErR%#hFT6;uMgFK-DZprCQZIF0)?PFrE2A>GZ5MNqSUL8YfQ6(uj+PwOoPojJIHSb zetTw6)}k*N8p4e>8@yL7j!N95pLSW9;BcpW*xS@>0wb^cLrTp>nfYW8g^>Sh%t^2^y_p z3sxCc2gca4-38Z^)?xzG)JBWif+#BT%fWI0sM3`3M%I`QR#r<5_ZXtp%e8uY$6?h` zeZX6+GHwrQ3ZPYorpR@JrBSO4i{wVv@pfkKAG0mRA<+ppmn1s>5KdX$T#tB!w(^bp2^;ZkR4ZGm^%l3@<~+F)W0eXv(@ zgNoIW6nIr_`>Xb_T9dIFqZZ|wXD%nht_hCR=FnZ*wx_JhEpAAfs8OH6a}aL3jyvaO7q1x`SKFFI|Z%{6- z?H$2Qs=7-UW0Tg`bzwLw4#&t)hWZ|VzVJ1uOt#meSEfZ-A5e^|*6ooGbolY4rc`U7 z&cP10^PrYoBptO4SX;5Urt6JCwQ2yaDx=b%y(P;8Ri-?@YJi|>n$olc0kURqk+oET zO<<|AZG917fY@Y(#lq0BnpO+i<5hJjSKw}SGbCiai&sce5mv5NP(Rqz29^%4s@2i5;&iN)&-YkLk@91FGa2K7 zQqPli6YbJ7iRo2{S_6i+Mqd^c0mao_uxlvJ&aD%2iJfhyjw??QT?(*~26yyIYhk)5 z-`W-#7FO5bfSW5`8>1_OG1_Z?k9ELkO{pd{o@dY!VzcnJ;YCTw*Rvt!C zsO*f;wX~DUO&w|)gQB#V%qoDt81T#f8mQw9x7Qu?nJvD813joxHKsv-&DvGPu5^3$ zvNn--wPhffc}#GeakE_(Mmw=53}@8|)-vt+WL6*BN?BBAu0k4sgR6|zp#s0##zq#) z2P8d!{Cc^n^m^mD#IuYia6G~EJ32C{E*lk+;hYXt@5|znTlzHF0vUtzx@DraV#S$O z^en7S7fbUU(%nw2@wPU9LP4I}uIzBMR~i4GIcB2!&p&B}T<-i)S` zYL&6;5~_~J%XP1@WSc8TgjY+r+>smZbV&dv<{9;VyW~ULF;}vf>8@>eOLc!4pmWF` zwFVr`@T@v)TZRt|cI-@V8-OzFPlBSowwnf6o>w-#8sygd0t$(L&=i0b8D!Avz+CXA zM^|-E>bmpBTt-R`P5XKY@oPJN=+szdVu0ML(d#&*!#MT+S`h%B=L`%lcSyuwcO_#| zs}1L}xS6@mtX-U`-e_e@`OdH>&!GB{BaVBk^-8Tf;@UAJ z3lm3PRf7=JEmh`!&B<~?F>8^~*8BiCK*zt(6_#9}@_9;!L}=C`raO}(ma0Sh8eze$ znuOH?uE}U9JDAm_W|N=JVHgx!8tm-qVu_!vNtW++7Q8GGn}Hqp0A_SmOcYm}QQ0RU zjvLOomOXQ{>3Bm+WY3f2A^a7Sn3GrE5gb zTNG7^p39WRkO(D%JzQj#)LL9Dsl~#S%9VA$wj{|}ZMIpj@tG^QU87A-w_OS}1|z_S z=@E#`tU7Ks?HS*yf@7YULLT4k(+j8%L3(4Ole&ii4PgOx*~W;-lN)M-VKb*$gGa%X zV~y5et2EZtDbgEGf60O7PlxTcK4{e4Ds8W%iHK^=e6`wfIWTYMr-RD4>3hqm)P@Ef zv{@Msi<&ccv8CGZ+pc5CJKF#)iynWzw!Qq!Q3IPKco>5||PRIVz0s*QJ) z*4!6~5<8Gu6P(A@@))XH2HIHanq==bt;KYU?r=fJ^1cn{hwDzOQJXZ|9aG+pa9zpwsRsExl3g%RxsTGh=j_F9k)Wf-ovilRHAN`c`vglUkXu%gs7w z=MiE)pcXbbut#8iO>bCnZ0I81wQB1I2Xv}RPa89e22^`6KrLFHHyu;Zq7H91?5^70 zZBURQ+l@&dVQLs<&0BCA=C*T;T8erfZY%SR%WcJ}e{IbI-74e6ZpqZJ4maJ}lus!BXS(7` zD!%L&fh}Cq*gA*TM_7L{*_Nzn(_41MMqT8LN{=Wn*wSRT(&uH?tQqZ^q^vp%D!(Fh zpg8S-f3V!^m*%5wgY(q^}0&}Iu`>@@m2*PZSv#d)6V z(G`3);&D|+wX!u|HOkJsGe9O=Y(+OZ>lwMDjOj)YWZ$*>hTg071gFOvZgJey$StmM z1G3b>oNY-WVYln-%$^xcA=6*)@P#$V6NEJbe}N|2Dv>h=0HsyWtSEh!mfHHdxI++f zyBf30`T|(aXu=e%E-sZvHMl#R4KSc@b~SNqT9YOw=fQ#43)m?QHZKy?7f=x=sgy*%L(!d(Ku=M^uZ7&B36!}{ zd=0^Y`;c_s`y-brbq+rUgxv1sg`h;Pe{c5NZS37C2ob=NLK`vDVpYc-&Mlbzyc!=g zWg7Gx@pln~^{#`m-|xo(cCdu{BB}eAwWg$f4ltJT7xCMTZf1fzJ|Be~WZmv~;l`Gk zFi40pssJ*ix$dcbe@Lzi}p9muF~)AFsOi(7IYJ#t|wZ~G=WHOaMeuy9tJilu&O1~V2Q{V zB=CGWV}DIFGEvv5JRi9~T6`Cjj`Ltc0pWz%9hW_onon`q@1s!p-% z%l=@5CO)GG@m7$K-^6>OOh8(uJEKg9{H z>fPMoi^w4=^xjp{1KHp$p(!MrTQSgyF&~_;RM@P(yf2Iz`oE`Cw z1H`Pw_mPl@v+$0HCqN(>`EUiUaW#9@$=;zqhetT4;~o#N=}qbL-C21M*wJMJ)keot)csLQJxs zW06sv3g!ew_Fs(n;xZG7f68x7MW~%HZ<(s1>meW~K9~Q~^_cnnocXCKnoH%xZx=po z2Ga&6ouR;B4wh264DlzTudzWyS%_k?ePMuK_iA#%8KSYDa+Q5rF?{XvGn*c7e(>St zV?*;))W+okY+U0g-mI~N4;);b#p;;f0a>>}7d+0^6wuE7Rb2H;f6-S3N;JRt*+?{x zA~g%0kHm}TMengzZMt07>)YWr`m~-6p@)B*0#DdjUL?w;sI1ld%pX!2dw4tqNS0c|WJMx}cM4!SvfneGRP7{uHC>tVr=p`dHII1jUjc-t#&_5{x zJaUOstsEqIe<98wf4{d-t?fH&V@)k6W(PO=!jPuje$dG6L%tk=`;KWG%W!ylfMrVb z2|qH>6vbpldqsuh)@~V6GQS(H*x^?UB;7uX;ga-ubdz((yqXHF2>hC5sjuv|JBaG7 z8cor{RS1S(>ti^NI@pft%8O4IvtemkA&Xx~JGFwb2nSCpe;$2{SqAu^9&C?MYqVEs z&YI>2ROsr5tv|HH;IyG4k6Xt}bx8^RGlZRP>%onJNj{RD!wSj@J4(AW{j*KY5uMqpey6o1F%`cYzEBZgxe#x)7& zi(y!wLnda%Dd9M5=yd!Qj*2Tix1$5pyEe+VSQA%@R0iM(>-Ih8Z1z9Uf{wvAtqUNe z&K~T#7c9spI}vvGg!KoYsba!BHU_jQRkKNwf8D9lG#}3Smr4=FI0&k+P@s>fkB zf8>)|W^2H>1u0KpU`^(+>EI`e9+W}uM@tXcx;RYQd@no%0BE=v0-E4XprIlnSEc>j z22iOWseq?OZ^Xc%f!IT|VdK+gPzATq30z}h>qymjklL_viMoS_|<{7c=Hh`-9Op6QIxG5ASz8T32qq+eC ztv?4@%eO4qTBzeVrWUQa*99b5-@Qd@_{N11;Uzc=a=q4gb0Cr)40Cydl73i8}~!l9LcsS5yuSif6lbB+nJZ%7uGf(uegxb>?D4Qdd|NajpbYtfp4m8 ziB>Yl)3s|tH^n2TC#7qlbXAyD3lMq2G^c^}&?o(sg|-)R?w8Jf{&Z@ao^?d(#cQD| z#giPIK00jC18^4*rsfbwQ0kMk`#as4e#*$k-mOu$`xLmvmwrd zQ0W@c$-|i(sW?NP%Hh=lT^(Fd+>K%kLxGMZ=`64ngmlUwOAjJ6>Lk{iz;Xl+L@WAi zo6%P%LEQJw6ChNDj`4D|$vHTMM$JE#1Gzx2qgs}M`!x~^Yj{)#D2w7BhVlF&+q1OX z-1>gVTPh)e2jzGxnHl(-e*o4^!6JAK5>96A*Z2-GhoIH@ETV_s11|W;2TJ{1(Ce(blo;CNxrOxoQ_P zR%}<6B|)vOfP4O2it6;7=32tJA7WEO2hMMcYwhVu@R2v;&AUCPf2badDlZ(V$CGIN}$GW3SQGsYgDQWilIQ{i%KNR(ZV-gi7kJmdBaX?A*Ohv}_ z)1?|CNsZY+1X(Y~-JFWhQ4WeBL6L9d!Apa9sm~0Qhp`g2k_?%&9}O8=FS%k;90vWn zi+v4{8zn5d!)Hg{e@>TqzANU0duwhU!D-~9q|nr$*Z+SXSc{2!n0u{l>P?SVQH3!m z+Sr)sai>ITS?0R?A(ctur;gd4XX;wt!@?MV(>b%UjcOQT{gJ$ky9z~TaJT0IL0p?H zs$ub2-nKL#FUHXI_KPOl?&hHEZo$PEA|mOadoj3t@SFX@dojIf1%jRo13# z{0eumAaUERkh?dq2>}F-Y=ez-POfH(9}^i??jzan%k5S^heu9Yk6Zb*a~Xs)>}S-}P7&^23~#DMd!C z=lsy>L8{_mng){D1}6w9&kX^~5@)vX0;Yi%nl$(Spi->!a4GtUFXJxwcsX@*Lnly` zsTOPz#i5>sv?6UeeQ%&vclg1k1f~r9R&O!EBtj06e>g${=h9XR3=bRX1;7_lEACTx zB?I^sL3;)na6^-5$ z{IA=-e`?%OU|rQd#3Z&evH`Ql-;$(}?u|u-JbAUhzsT5%R9+7cX7G0-QD2&>A6uJe zvnp==o1vW04`j|@s%ebVblI$@6a+_;1X}1F_EA#+to1z?X(^)_luGhL_xm8}ZVf8f z4#p^jBivK-JG4+CqzJ$w!V~-#Xp@={D2q%zMsJONq6$DL ze<5bVhv&DC{kq2&2rn5RqQ;m%I;0-^p@#RTZx-kJg%NEaMH37anL>R?_ZB}Y7{u;QBxP_tB8-!ow2>wF6G58Lsy578s&K?j(;u2^XX`M zCa5R|$K}#{UhdBT^UQ;;)UC1u*vQy$fA2Tf(6vH});y%hhqk&t>O%}raB=(ZQs(Xf z&^G^8u6jldMQ8|8CnRyK{UpX+5%S6YS&PrVUEO^hX6zZ)>e0 zgfamysDvs_={X~M-$sq4!a(JTArSa`atE1E))d6>(|5_*MA<)9q4>NN$Yf4Ml> zp(xmR2dW5BI@PuO)30GX&dCV_>j5P9yL{v7G{b7lAFJ}sqH*Pvtp>Q)R!X2l5OPE+O|?(#uy5v5Og3pBPRTTQ zf^BOhkj3`ZbKVs>1@H#}0ZlsOe@++3@1U;mXPp%Qmz_q~3pw@ULPhH?gC*Vs@VAN) zR^*-dO;y+%F?lE(Q>}n)G8lMmHJJ6DNE(Yw8^MolYau-9CtDE^ttwut5q!nac4mm# zPJqV#hRMOmux~q+&~sE}tS2V+pvO+hsN06xN%gp0YcPDSoqjxt-hvB~f6Q^nAOx<~ z0s7P$i%I{{h}rn@TxJrFBMPu?=F5S)LjGtM0yk*}dSwcEmjjTo2`qF`LE2!@jijlD zwPe7r73S9|ax2H8yVyVX(McXQV-4?C6l;4@t^KoowAmING{__k`Mso+I*SM1XdLe+ zarDv2_cR;tR3zU!(#I=pf8m`DI#5G49Mn&;k>iGI<%ve;LNwm5jxNg7C41LF$NX+) z>WDc_Ertw#f7ci$t_qqDs(A3T2u$H;aTGO0(!rNy}G8-VOQjZMS<2HwbF@XbHeb zV0owld*w)j`od*esFhgmi?0S!(}PYFeeXU|OdX}h2rl2JUrk#He*>{ka2Y(j-$yTr zUeQE^%5)Ij*+3Hg+ku$EUtNo=d#h0&H|xGRtZcdbfHMv$WiR%}X21asbaZ<;IjlbW zgF+ZBXr%@!bu+-5_a*Vhw$=NJeX^>!$<^Ps4sY*__#8{l3KH11uAUF4c*{g#t9wWU zR*Y=&PM&mvWm_|Le}s`KGyK5kT)lHEBM0&iPx~!(+aR=8Yga~^NGSOU?OJhyWz&y4 zf2CYM=d=$TRsQwiZri|Su`(k(DV z#3$rqo&`mOtgUv9r)yftP$C_g4!d2s6I=Y480~SocLMxPe?fe98W-Ux{Q`C*UU#T; zk5bDD90rq{DnX7QIYqi1jh&E2gNwu6!+{C(`_0QxTFU)$p8QTGc^2OzfGQGXjgYL%lH&Dz=by zADQ|l58>3?vH25=)}P;9O|?lgXIovn+ANHhF3Atf4F?f?XzPgMQSJB<}7woVy*`MZ>E~j zjDX~BT%i)@&psl2#Kj+R`^fp0ZqRkC9F#b{JsItV`5<9CNFjK4XNwLv8$T$|haaI& zh{_z`C@zz;f0oORqyn1fCm}F+n40OZ()rve(k^G&FCgN~n6}v}7v=B)JF&Nn5Oo#W64wxIa)M^jqwe7Zj%nl5iPt zrHW(K*1QT?ApGt+xK7Bkk#|xD*V;qFe_ix@;0G0g=K7-{;p;9=}6nwMIqd@8jG3s&fa2N*~R>>pK{BAUy zD>(Wa@IVV%5xHpp_dB))eSEo}*bxfoB{s3wR;IWS{Btu6{>*|YXL*;Be|t|0U7@Ei zioU17NTb$~p}e+{zlw^!)9A~vjNW6sD++90)9arrWW(U(Wn(d~(Jk6hIJWSYsD z^zn4rvuodX63(T-pu~b*KE-b z0(4kebO?`voW6Q0K1FNHQ{jh`jbZ_Hxi7Z(*`V9ha8nvp0y*Hb)I!7QF(}kr=o4O? zI~_&_2?AySU1qT#LoO5j9-GsT>Ctf|C0xx&$HKid^z2UrE^C)*Tyfbj4R)*X%5fOh zsW$x}7?~hio06JaF`X!%8RDD23=D{HQ@TMUIWpDxe}$>7yR|nJa`QHwELK;%R>qdz zB>omKSjldWBEgL)|Go2wf$?CuH37%Vdgaki9)wS0mKKgd1K4IC85BCm1=S~w912s^ zR6iRz9l~|2Z7sVU-O1(gu8*^xWRjV-C}3Ogy;n5YT2su$AKh!|r0@_e-}7$$#ca6` zHNoIif8{GN1KZ{=ZrYSV9>KI&xsFPrWqxz^c^4}yBKw)LN&bvOO73@qDG1hAKMF{} zf-?Bc*1U2G)*zfp={qN)DY_SV@h~D>3WNv{;BU7%!w25qEv-&oPa6PHo^yvKSIua5 zesE#+kLh%?Nu45S$545D>_`H4SP}C!b{l!+(d4r~gsl%Ao11xaojfVO9`2ZCS%7%^U0G^DswyHiQ--h1*J z2-J8`VlX?1yRpH^-C)IYgaOdwqx7@i^Q0{@sdk{WG=CHPL@E&2+~ENrmn;@s^}+5i zfAIv9emp9@FNxzuQ8vcThHJIdd5f^a_=SAQMhg~|x>_%V9kM_YKS!e(*3M7D+2XQ1 zlsTuoepHl8ZPYEj3ecU0FSTDpFscPy!pd&xf7X>O{Q{j!U^)#tj||Eljt6hc2{i97v&Hc8 z@7Lmwg*V%k0};5(u<4*-gv^ssrd^OszzqB;NkMmhuRy}CGQQhr%Y%eZhJu)=9)XY` zz7FmccoVkCo1SODn=@|lAVv^1flm@JJL zoe>>9T&&V*(5L6a8v=jGhW>WBp1qdX3{ZnB*dWd_fV;~Q*y zZ{ZCaWDkY4ZxaRABiHv^%Em;mAfs>ykwk9`igU&C$_p8TbLjY)f51qe6vvgGBGki7 zBDq(Hs#W|KycC?L##-z}ocxLHe;l{@GzA!el@QA^$1J)@&Y0i5Tj^CqA}pJw2#z>A z4?Sl$KEnqlM31@?_Ntw^k)h=*f%%KxjSngbQ|izf4B;C`EC;(<^kXZ8CP8J-v+R$^E8v_aX4s|xrClS2=B63DOs{yYq$sdiL_Z2eZ!fLyhm(IcrZs&CXWi}gF~5Tn)7UiijgonI zLqq^-5pcYq^~SZ4H;$4x1>}C&6A^2-xmMjtn@#19c&gW%e{NhDe;SN-7r1vWDZC{{ zNf6yaS){et#OdeMjOp|HXU#ur%*O-+EUW0}L9N0pWX_4p8hXF28@qVy>o>o1B@Xf5 z(!yvLMkD_E+8eQXSsODvp91Md!Kcegi@7626IS?{u#nJ-94u5vQS?NIfLU&-0wawH z#+ba-kXouJ55W=)e@{z)9^!0rM-0SnN}6`Wq3wi+|vs~htT)M7h z{DgR|EsWmr%$?SqDWz;mykcmaJ{9!t>rA83m-6{mw3t@w^yUL%f6O4es+?#i_g0zh zNl&lRh@yUb3;|_hb_cd6wm8l&U?g+e}8F&X#3AfVr?hcc!O(hCGr>a4X@dvJ3^9Ce=0q>Qo~p)@b40}sS-i1 z+8s|}Hp;6&D|IQvLe;#>xWyF-{r3rvs{&NdIdnLpSk!5Bn`=r_xxsNvm1)l{N`XMv z2Jjd_AnrKXeQ~RmK;gaG17rdRH|1UUIrCzpKZ;;;f7lz?{hw!}Bw>2FDl==wv)2Q( z)%_?~1Nb2l9GDn6J{m8DpoJ$CF1s~;n4bmrd%&E@jH;b1UHAN!o5y3Ce+g9{TaNm9TO2%&Cu)tqJo()$G+BvOr(@!T#O64t>4#s#+kpM6aJwyR zggjrJe~d<{;yGtAzZN8=A?`~6dh_tiL-c6p;+*Ni%plHWQLfNbl^NZ*cg3RulAM& zBc`Yqjs(Nr2K=b11{ViQ4~{g-A05No@H)@AoRTCFX*uf>DbY6v1NhjtBA**YBR_)il~QOy0Z5NVd&d7Mu#E0;ER!#zk<-ZGPwCR5W>4TZeCbfSDet^R)pQ>@MY%|cbsLY7tR|G~EfY?pHaw5*M^5V6)-&E`RkHqG>z6*) z7%AR-Y|N015WKlJW=jF}t6#mH6OkdlWgHBB#n~xr2qaE~CCQk0V4=d0CDhmZ20@POoFIH*Rtt6vu9gWAMdTEv**>U=`oe%j9EVm_o+_EHE3R1;3(Z<~KTXog8JXiJw)Y@=A7zm{B7G>hn4%qLeo&`K7d|h|05#+h6 zWug@Esrocg);Q=pryK)|^HqZb{ThfcFsAm~fb7Xyn`kPz5w)jR8qKS3zSp}zB~6}w zrg0+j=diYb#5_x&R*2Dh6qibz!FOS3*ly}iHiPOSf5@j}phO0Pw#C~|6}Hk+=Z5&c z(ebV(f9EQ<#(>h9W&@mo#g87<$cyCXoe>y6E8=`PyFCVXVq0?DQ^u&t-sEQ3lQ402RQblc3(b;YYN^XbxtR9nJPR?Xv z*{(cjT@GGBbaRG5d*IQSMY;~Gh|jcNB;v&E^b#)|iN=Gt7E`%Ao+^#G^`pkr0v=dHjB#tz2kVIx52x(JWmzdz* zfA+0ssN6VkeWTKEDgGjKoka@Q#+LH5eK7j-vs{5nZ=ubORKd+l>g-yDn#m4l0stlQ z{uU1ybMG>^^+uL4)JKUr>W(mNNd=a|(b-XUMOQw*-lYePiD#$q+gaO+n9Qu+P*IjBVFXWL^Al)t=JuVy&$XUm zd+G^I+C^5&+X9i(Ty&knFXW~W?kNch6=bkVzi6dTGA*`*(!VyCe|(O%Q+Jh_Z;?huJO$aT8@o-isrWN(ki8<525|Z; z3c8$A_&}+Us@nc=S(1Up!t!-$iM9slRd~EIP?4n$j^XFqn$8ihwmG+nFi>iU=}5i5 z=e?q{Zdp$rVacmLaolPr_FXI&@V4O2>H&p=JgcuAN99)m#JPfK=T(ele<}8O0u=#q z{Vp}@%uL!l1tn zKBWgPbP0DxA=+vpsbe}cyV4RbP-QEBd~J4lUd#mYF9L_ujKWeY5foqJ4e>Y6xt~pV z5oh|v5dY-3n`Wd?I%^y*-xDt?*)cFS05Qi> zAmr(>Er|y3N}kQby!NTR1BUxR<4IbsPr2ZjbYHR=$8>V$3CJhMne_**MJvRSHt!of zjsxd`JwC#9Kc~#5{O6^69c;FGn0^9TXYJ>>U(^_7zq{38&6=@YeFGr)s?ozK6f0#kUQ?QE6pml#@X@1kvfk~?}XTFKWY|5yf%%LL2dRC|z63n^& ztJvMPBY7cQfR7TC?Z{1CQ+`{FSTxdKAt}|R{b)NXv~|_Jia@Gr;^gcXGx;K1@DH!d zbkA2EdS{%8U^mPmBCNsX$6DUXK3&YpB*ZR~i{CP?Gw=PQv7E&@awxc!{Y8ZkkVIpZpz}6SZ&TNs$J7bX0tX z7OI<;{^}>_$P4m~(yEVd+_^I))CIWr0<6EU7m@}qH= zhrL~sV|n-ue+oW$774?;a&HXSeKTyMZhj?VHX$5a9`eqgc{iPhrj}M@L zi$|}EBGA}dUQ4L!joy3;x@UXQmps|I?dk0f;WP9%f009W%#a4PP%3b`o`Z07?YiJb z#oB0B*iN1j&@z-fH{LB|_W!6;vZ3z-(EMYIS318r1iEJ%Bb7t1}Ivj2?ScRHLoMY3yW$-LO^9 zc-Hf#7SqL8aZ?pz+YBoeor=a#F_8|1dqGCg6Y&YqPHgUijRXtnU>?;!7oFBFe@~Ma{1CLE45o@FgO3OoMXAP984vq>(RWL7P%%b>K z1|i8T04psZa}fM+<##7)Lp7>!8d&-=2iR<1lB*SVmWmI2N)bw##i#$^seT1oq=DNf#{&T%>3Da}kF6 z0nk?|R$&94;4o^4o6nxCuyF$9M^u9_Q6Upapd{Au95{yN^dX+Qo9B-Je_*IrYlI6e zP<)htkTCMA$5fBFrM$f3W(d%l@;y){YUGsjJ2GhrfUfUe}|3NlSApMEwc3<(pL48U*wfVavn)CFFAz-#!btDl!jhv z_|4cXZ1&??Zkm$u)j+oDfrPGce(&bukHgP)Ku(g$UYfusrS+r;=}0_KpGa7^pfKMh zr5#t{p5K+2g6T!>E)EYtW^(>YdpltEN>1c&{U~17-U(rR@jDGaf8_%Tl}?K&OUV`= z629*QZtz!ymlV;WTMR|q9`APPm^DB35Xf7k#|`+&G<>k-wc8eYN5~i}JZoX_JlpyT z_m1K#L=DzM8-{&=eBMwW;NX085<`seT7!Wcch;3Y(>4wdtx;-o!?vgF;E%8C*x8un zj<*tJm0ynJ-V$ZuP2PL=&b{}?_3ESd%RX2_2*dcm0%4dlC*?gRl27$xKZs)H++Kle zdSGRED@Fje+v&xf~tDFyuqe|OZcyBbqly`@!l!=Li2V*R(Q8mma=sLj&O zkgK-1dMQr32>>vkEIu>^8t~F-6(2691-?dwB!0MV?&^5d%gN}4Z=Hs%+n|w`P$r*} z5fEJcViEP+b2J;3C>W84i(|i={|N(u(U;9iX)bZLk+!*`_53n?9))yBRrP$fe{V5O z3$z+fIeD2+)!^c-$d9)FodS&OPx|KPWS|k&o-Kjr80MB@w?UH3!O5rAVs3OI#j-Nz z$b^hqzs;rk##xgFSxxV7gS&JZVOi>DChh`hFHK>=;o3U4DqO9M7J;TeN7rlOiIus? zd z-p!Zzo*TtjO@We^WlVF18r2EaO9esr^7AB8EnJ|~8S-w}Mx2)^ua;K zM?l|0b7ptpC>guS{ZH~vqk6aXrrVW-Whf-)WW77m(^rOc7?IR^(OwcLc&#lrR|XsN zH(Vse|59@-g5e-n=@qtubN1^PHVuu-(}sc@vxz#3d|&qwWxTq;bzJxpW#1Z zV`79RMbe`4*_*MbNq-QE@tb}Im&s!y$4#a(n`YAsHLu`>k7uovaA;Neg50(4gl^ZqWzEH5syBX9m3fttH@@*h6 z%6@i|g#Ll$;H)xNe+3Dl{|WG*B&W>3LfsVTYr|yamzD(Fp^&wVJiNm^WcJawm6ch6&WsyLeBYcV8OusFw`?*af0NpcX=LDw;R z;om}#MFICP`GW?j6j0P*=?>r~&5%Bbvs9|MSaah;e4n1(f9-w_D$D4FRuO(!u?S!G~w_PGc287S!HN{IiPH}UOPq4|oMcg2+h-eN( zHna&u_=oubeijn^^YiR!_M+8%I<^YZ5MYn-c0cZ#M3A-7w11a;qZ|92b?dhims@W) zh1eKwFbfcmf0Cah&$DZrODkWF_Wa&6#f$C4f`7YzM=Wj%Jc=V^0{O71*tV)Go#M1v zIo?rmQ2Z=IV#zUiPe#y@@uMe4hV7StNgi&VQT2oGD85x_jXk*my8LQtt>rK_K6=_a z98=_rofB(hi8Ua3a7kzByU(kq31R{bj82QI+l<#UfB77N&6u(huOwRd*oF0@Qbw(; zY8>$tqt(^W0645q6d-Gk#3GMuRuwK&YR(qVupHgO)z5;`<{kk`A@XN|J#~nY3<+U( zTr$Oxk4xjJCez9W?=2#zNBl$CF_FKK!LP`J5@tgm1#_WNCXzbA*-6{7(Z@&ey*6Sd zy1db{f6Co(Oy@WH88Mx`pAVFEfbW~7GZ9KQr5ni`u{ObX<=EK??uGX-b5NHL8&}CA z{1n;n;)PmA*oP+ZS2cu4c|-lG330rR=pBlZeEVLlv0BH|^rTJz0=oQK5W=H-RvFn0 ze39aDpFy0|NSFFj)S2z;3JFhWAwE&UIguQwf5>e)(5ErmVjz$*W&T?X!!Un^J(B>= z{W!qW9S<=X1R!|MExjL)kcz;GUFEW}7$19yLNxWE$Dg#4+vW?w1*N@6F^dexlw6E< zdL8U(;0FLe-Qf^eS6$?EeowPQj+z*?utIxivS!z-8cc5RtSE`FPU8!WmyV?I8-nS} ze|jzX85tNdB9Ow+>Ju2Ux0M-KKSwhoavA%VYSob9Zz zZirsDe7^JM2bN(Y25l6$!e%(60@V$@)go!_=kJyoDZlb-^yN1`ui3SD}2w;!gv7pdNmXAx*TE1(yLEyM%Ude3@n-4del?2qH^|Td)6Mmg z=L!lqIis>6D|8*Z5_KFONKHC{6TcJ&xfk0+fA)v9YfO{1 zcv{0#mLZWzKN|1YSg+37%tDnxWs#hglyO^CgP22#V(rbVY+$+hfM~l4*rui@w<_0VZRSJ} zrYq*<1cLXzwU;uxVqI6D_ZM~*UC z6h9?|R70Ur&)PaTtF-$~L73Dv3MmVeXm1RT5rl;EK$1Y;=h0$>r*08+Mvo`TZh&IG z(CPpL)*=R>6-gYnWf;(4RJZGu=Y#0J#6i|P`|y^po2MFHfm`Wve{3cm7*2Bdn>+F~ ziEYQ*w6AM;O&JE6VJU#k9fqz#Oc5$(^K#|Bizs89&UkWxai}=yh2x-PMq@4}$lGG> z8NpFGmyK!G=iLS3I=6utTVXUB{lJP8Y=+^ng4)3dci3f3I9O%mh;wY55tQiQw}C+~ z^lReH!8_Gd%-Rz3e}w_)g_dGP*MNO42GyyToYxMxR6khR~l9dn7P7? zAEti43zK%S-IT_>H`|nds?=V;uy!$s0k+Odhk%DWmn#?s+InX_rd2d+>RtPpGe{L@K>s|5^U26%7Kk%FTKVjrWVZc zrb!G8F*`gWV}N(|id5gLThr00PQtT4Eb6697*q1lcdsNV#&T7;&zSAgTB7$B7Wc$O z5vIpYkYByBe_FD&{Ftg1fp#+ITJgm!5o+X<^9|V7bTgwx4AIZ7!#3JwC*{lqYxo#2 zoPXhf*X-|>4Ii%Em&54x(KM_6yVY_0$$Hz)LVSNNst4f!GeFG0s$IuRe)7YSQ~A+{ zgeKrCjw{BJb-8|qz(+UUCujNqg9{^w)U_LA#@7R+{(k$0G=Blr`qZm)+4yOMe`nHI zG(5i=H%Yv^r;82V|G>cfLR^4md2|C4>vo?PjnwFGjYB!$Y;GrINwCrvsJKPLiwRRw z9FgnIe1aE*%c|9RxlSK{%J__d0f)s1i|0{LJNk4S9@~cRPYes0Op6(E!svXv_nN_gMGhgzs z+?tNX6Y_1&N7d$3k`Zx?K$$!%L9`%Pp8M?-Dq;yPowKan%tcK7cuR@kwq?NRtd*`} z67xlE&Pg*F0QQc2B;4zCNJU6Usqx0r-~?D}-Plyt^napB(!ACESZx710z#346s}BY8FXbGHczoC1imnk0s^^WX6b^WmSlIDPJNwSJfU2(WodRtSrHwZ7o=d)W zQhQl_bjohHC1{SJ*a!DWrgh3EZVR=&lsDSU4uoXamrTkNOFSH|cgqQBi_uhPvTlpN z+dT<|b$?{nlF9t1O7ALu*?{vRPYT~Q)W&rq+AWKI3vtdl!Y--wblH@iNi>dmWbP>d zHklr$azMlgV8eh9{aT3;ge8^|S;UxC{D1M`$@hrMkrrw^j&eWvu4IGW=ZpPryj|CDr4AH(P zG2;|l{wa;3Y%cjq;qn^t4T?GpdCt}Fm47CsQF0bdB2u#B*@6K|ysO@Je~r|dZhYlj zm927F=F!4n^}Ko21S&g!CaNspu7n(4WRlqjhv2b8OzmKnO$^R@`R`1MHvSW1(+LV- zYqP!OvSM6D;5pDbw$3@yyIra(p^|SiF(?t0I=pqNYPW__8|wumN3ScV?)o^|ntua` zkm5eLJF2MW1Cg5qdrQi0*-=(<_0+v2ZG!ANnd=931$eRvtRPJ)7Y9T)>jHBU?*IAM zztzh=WxxOU_j{>pF87l8z)Rq-+cHANz*C_S`>R|3DchMIJo; zD+Uh3NTkLvyhE(xfaJd=jl_Xr*w_AR$tQm~62B_XMD3DB0>jA8KOpk`2Y>1a?%mj# z1VevD74zSyq(4)5;I|LTryeZY%i@=>KL+`9CAsvmxVr z5`U?~Fm?M^Dg8kUg@}-~k={9KLJY%#eOoEMh$X~`OCK{t3V;(tPIjU2?p7X&uFg()a6RZmZ-o0izsO8 z0+=rWhNPJ&Gf(I$SN99YP5yOg0$#)uTMj%GCcCNLJ_aNiWHcHPkbgzM(;g=3S+VV4 zkcvl>YMu{uoQ;-+o7P&o(t;Bwo2l;jP|5NI-A7!ZH0Bzy8)gteu=Zjuz~pG>70>NB z{M++2LQxXj4n^DeD6cUJ)UGs#hQ3PDm1yMwFWw#{so;Bc}vCRmcpp z`S|N)l05&djBbhd_kZ|I%@|D4y(sb;4u3^4*Lax6s>0ARoX4x~U7$jl@g!W~WP{>) zq?;5=b2r=OhK*7NbR8}`(*(FJG81CxTZunAFk1ai(tR2W<8>zyFzZY>#-yttf(*%@ zQNTGdlaLOh=$wacP}>NJojiF1FG8WvIuXDg^9^&lUQnXVaDN#sEVJy93`Xz_KXEMyL1*Z+U_pphY1Qh_H#m$( zwdu1>fq}4C&l28#XK1>dac#HpFv+g^{V)1S8fw(B>trPJp&#Y@BMnu-=zT?@9Lae~ z*LR#ytbbnJImZ>rt13mV`r~-zc%}|(SWD)qX!&KLXZ8s4nexsHKB&>SWpV1rB1el_ z>NG3<(#94+-YO05{3sQ3w$I-~???zmg*5|5ha1|kr)PlY`StSRR!CPgFVDPkWM*B< z51&vP;_H~XwGl_Zt?wZ-gjjFXos|TEA8gAaGJm0&K`*s-0wDgIWs>}W;|iHE#^O&5 zxV-nKmOC+)>jByQc^L-=d$}u~IzC(bOEY5t1%m0pVkDLZ4%>ucWP_-MXX0-oWfR(bCqD%Joth1ozg;R z|9`Q9u;o8|ICmmo0+EaWQ+u#*GEJNy!+=UH+?}M%K}dOD@T82>z32S6P#1P&vS3?O zr!_)B{aH7bjX#vGuR{J!aWje=d3_)5H8jmgEWMXmkGfQl-KKp>wJ&016$EZeUTEGU ztx>!&VnWxU)Ab3pq%;5AYVV>5>1Yzz1b>+%XL4ODqsY=uhb#9tddKLMSxRYQDZk&9 zvQa|wePD{KNYp&ffjW&8!ZxYG`I>G1^TS`OV!jY1feR>)3+ghyh2F|oVl2M^wKKLG zB9V4>NUyLjXSnQ}rIT)WP!xQ@42?s+uaJh0d0<(Br`Qz%M>VMEP zxqr@|3`U)6$$OK!Q7K4w$Fy}Q*IgWM!I&#j> ztEpK!fIKtt74-Q=gb#(JXxZ3EsTTZ!UGh&k`V|!Z;+a_=;Im^HP^>DqE(GZ+e2YpRLvLq%aG6BCYjhyLrt?p4! z=$IyHV!J;lGxAn4=2kdifmHn|Aosba*WU#!W>4K#XcgygZPX0+(Q%m<5l1mgOz)A3t-)q(yG-Dey}4EP;s^iV6S-w`Y{A2nDd;d`G19^`s&$k z1-s_fMkvos>tXhWv2dT8IdFEF9Z7M%1x=HZrAg9 z&l()`(|p9I+d;($dpdC(Kt?LdYC=?5jKA&V5y3{wB<;DCu2fB6PXpl)6LM;W+?!kj zZmzoCKSW`cuhvg2a6F2>*nhOCH~`~ZuSC0F@d=G;Y11AivfV%71>suYJ~ zkq&!XCFWUl3Y%KM`%?LO;f+Lx)Gv&{D90#@BpJ3l91n6`H2@1SOFgPmg63F>!0K6&D*3j23lm?s zxPoT`2x2*7%`l6*5inaO5&Kx)en3V8XiON!$q5NGR#nFoPJhQ^oSp^TH;oRYpNmYp zv!ZUta>Ph+Y082)VVKYYtS!W)Ampz#I@C;95MFONjJ@{k*lzXvG^% z=?Fv9sn&)lH#AzeVYs;{vHa2%fW{0l3!~S~Zd+Rek{oi2JX-7Cv2ayj-Oc*ax9Cnn zgrB^GOTv~_7=Okdy3H6oRdkpO+^4MZ?x(;ir++?;!&CbPJEIYp;01ldI7n4_eDs1X zn!euF6@^Bf8$ShZ#TnEJ+yrOlssXx*z}gN(A0PbQQhobu=;LvSI6ZHsChLO|qHh*w z`d#%kdtNQW&^s4#5`^IXiAiw~C8^JWDxacN&v3CRa1-m178v30y4O+u+c3oUgVXbo zCm+$puORXA&z^0aMTez_M}S%^#zqwDjWS zet-3^15(-&9M7ZwWGy;?M>iO#^T(`>EZ@R8rq_K5tPVPjIig1OayTHpG$^ZWaddy0 z-}eij{!DU`@Z9``GdfRc@a29!GfXo}SJ+~ebZyN?O^L&EJ(bU&Imji5+6XQ=ix?mm zaA6_kUR)18J36QQl`bKrx074K1Q1bkF(!R5B%WtPP zS@V&7;|?q&V85EODi+>ew?>@RY{%JFwiJ?#NJ4q&1N9Ua=Jehx8sc^qNcGSjhQ8S2 z!81>A#x2YzhbW3GxbvIt3}3+A`pkjY7%r5>+`{YAVsx1u#*v^Ln66XiRXKt@zkk&! zvfr2(IJ-{@6rb{Q>Cu66NHF2#+N1gw4fMG-NSJz|Jf8@Jk=pLfm4J0vWiE-8xq|?( zip>yARMc@8PorRCRL#8YII?+`obBFFgY?WeLlAgl^;<_V5xe{-?LK0&uuvMUsr12o zGe(=OWf--x)a5JDecqLe2MxDLsef+zdxqr6GJG6?+ zQIdRDBLb-4Pl(e|7KSCJi080|GfIYTabN>UVM;w;8q=eQ(Yi!Xs1SuF#W_nVRFd$0 zB04s^$6TTT^V&XF@>hqt#UI-XUD$X*jQWez<0?`ni+5zP!_=@KzJJsLH@;-?4_Mr0 z>9_OgFe`9xZf#)c$oX)k+<#vo<^d^R)k@eBM4rG11|(@nlsZc;l>Ls?a-I|J;HTuF z77r8>d#S?mcLc7xy7Lz{0X_g{R{{{Fd_u;NqyG&J3Wy;=iD_l`HTmHVAxIfLm!L-r+8@lq1ik9>YmXG1eX(}hlUOjW>c_P9 z3?t%FOP*fmZeiG}^emVjL_d=NmG@V${?dLYXV36!?1nZ7fni}q;+_Gv`lnfThQqZnHw#9U+JCHMnll4-VJEwrF9DscO5cFE!T+5SY%8$b>`cbTnJO6 z&ksOuTtGy3d=q)9RD<~{3$7*>bv~7=bx>FjqfcX{%g|NgCV!AaU!Z^kWXAQN*~Kq} zz!tGJV~>w`PCf09(Aj2Jbaic5UUY2c(zty&B$QZ151({n>J=YOnnHuL8yc_IBh@q@ z0C>|1L!5r7rpa{b|RNpsvx~Cv%JqU zJsv076o{E4QGbZNOZP|~|TBQk}vrem2O2@NWko7W&?=s>>=U>^)BXr@F&2KHXMr2FT2D8 z{`TO-MYuoUjtTY|t^55UkTuvBL!HGrHE zid#jKmXmbl3)_ZuWcq(OZA@mU#s>0z~iWO8KmJ^zCzPjCSFBVn7uB zDp-54MJY5T(3^;I%oD=#%p2fl!zfX#?&%j{S^}+5>7(1@s6Eb^C`3H|1th(t;JqAh z6dxp1s3#iDM!Wy764A}>WgVEp`y;9#Phzw$dv$GO`nJV1eM^7f!ONtUpp4jlwZk!? z#GcQ?^2y7-*7mC+y7{PE7xHFoNM#U`6}Z-pX?p|q0=Co@DAuXZNMs3S77&&V2ecHD z(et-qY8Q<2+q6P=#LJruOynTJ3Iqe~tfmcFt@~~p0hS0N;MH|^G`HW@tWYZ}nIX7E z-kWHLMUN)p>Y0BG{(Pa_9X9aFJn9xsJZY}W$bIPicQq2GkfbKSlMXL^Q8w?M8N#Z+ zd>n*EM4i+)w`(F)0PWyAzbq}upcH3OAju9r?~;_Mfn$rQ*_xSmyYOc)ajwxuGW~If zb1q_<6SbhIN}+I(nzjX6UkL=YAK9n#d0??d?QUDDk(Ph0Q&wm7a!jbW*H zb{b>Zxd+7Oj+|-9G~137bBkSZ=Ff%uMDT2f{S+!zpu>dzC>2Y5ZWd}Cw_pRb7^ zEE&pi8<>9vLI3*`=5ATHWK;lI?m=~L^@4{GMQ}ur(x$macWorXN6H_ zXr^)6_w!u0?`M3ccwoNywNBH>^B+NGM@4N3%D#{8 z^iF?$5f0}T!D|cf82O*K8H5$GpMv@`HbTJT>PfpW8UgS*R#B#N}p02K!C9-a58E3op*I!;OFWR?fG-e{{g_go(PK9m1=%;D<_FVbr`Z zo3A-)9T$%>o-+j+9@@chR2}{d^Bm z5K9zDbKQ#fF6fy*i{L7vpZ#-#di|;@xKm&Q=s{EqiiX7|<60NS+!0OAR>4Rly{UiI zdz({2rWqN|>q@_1CR*d^3kTjWe#e95$qWB^1tk#_$4osDgsZWfA$*{Ztq)7K6fQo* zl%aO6B?9v{Yk>Wel-nY67`q{6KM~#u1ZytV_>|%e;)w8XY-basQ zN&$pc)*OLmK88?}XihX?pk}T~_QqaE%^X1Xo%RZ!rDqDIqjS`a4*^@`Ey1-74lz*pKQNh4o~}hzk^SzzYHC{*Sra-2@Oj(bHj1@g zA%(k@ux?$>Z<(7%??(!QDh4WZy!B;sw$SRKsf_tFvmmQK|A-tp36O!*dZ;eX-WfnU28=#t^+8 zxo@m`Qo80)hI|pufc<-b z5L$&m!fz<}L8+wd74Nd9NY+Bn*CGDqXoSBojZ&6&#=nvI<;*CcmMO4rZ=BvXb7xPW z=pMGQyvXTBB%*)16csQ*BQBim6P~S)0>Hw18dIdxGWhrzhBw)kzSjVwBc|?#%ax*8U%(sX9om=(d@y!7Ep6saEr|FXQXnvsyp`?|-nMi}9J02-; za41Fm20?28ao4cVSa0sS8$pEy?_nsx273V*mL8Y)Dhhwuu2;!|Mpq&xbJ)K`3a1+L zE>Hdv{YbD4(ML7dhfdX(ABkK~Q4IVe3NJev!-FHrBTM^aBP+ zw?NnCF`9o%DS@d<|aGUZ>{Q)1|?+}>3@1eP{)b3MhX zQPaZME{&a+JpEUX236OUGdq%F)YO3iWvQ9=QO066C1|6WgNmFXEF0o0&^|1maRt;}7)SbuLFF1<<5n>J-3 z2sKd@S;qJmiwJd1c{)>snPlhXbsC6F_!YTC5&++YLvUrA@nKQ~l>z%tD-h@mi`N<9 z3;=&HO#)jht1%K%a$Tpy>Ie{xn`hk-GK-0OlFA26wUxQ6l#s={;q`xcxD1AfBq7eF z%;}%pWLBN{)xFn1>B)7Kh@!!C5wnozrG~x&P@r4U{>t7?97&q6vKYw-k*+e{r!*KV zt%QktMNpu8pDZhjYe{X{X3l#ySTj6j@)Uo8ijx$V1o@>D&=mAvBV69(bVkwE=xF7u zolo*cf`}jmNyaF2h`wo=P0RDiV+#3}Es>^chUp3O5-QRlOsc9{4pvuX;?M7H{9MPW z?t?80!*{VCJs0k(qSr?zUY4h z#wSjS+(?h5n!ksU^hl$-Co4tok7;Y(o3BZ8nFKvc%9KecF21#9?if6NX}l(<>q6*gA^y=v5&e*Arg zNmP$Qy#Sgz$f$*RQq20vZR=a^8@GSx8KNEqzxAuHOy#Nxm~&0utKUX=JB_~W`MXLg zEzS!8$Ej+o1+orh-xgwsE=~yFyNjoVXKDCKEL#p~-1S?oH@BOqI=B32D9}~AGg_~JvlmPmJmY`hRRIGL zJ|Sf-Um2-((q{0B0PC+~z4P~z+i8zq&T=eqE@yfUn82o4pknj>9mXRKQD&whhUjdi zHV;%N_7Tctt8rRB6@#B=$Y7F*lDl1oJ|Oboo`jD3%eLspUVzOMVo0e4T=1>u$A?m2 ztWC_pA{-BKrqqUEKb&f8X_tR3{Sa*^*B}eV&(`&RTeERfkwILT3TUZQ+foCmuhH|s z6h@94&~prTaIn9&vF}E&apU)7gD6Ae(73+J^t0~@#Zf#J)Bq!-rmN_6n+w04dtxj` zF+j9V(#PNsaA}-Drda$s09hMAYtrcHpo!x|k+_I=<8+O17Y|m|n;M0_h$$ z^nmTYIajUJkP&@|l6&X&80MsQg1JZhB6Nd)ZKQ)IR8!?zP`|3A0zvT*z@5sx^5zTY z8W)!qv6=53jm1fEMRT)Xrq+?Cb8QH>#yT7wFt2mu*NM-lRu_yoRo{;)7W zHmFtp*B@lC5HmL(hUu1L}o=}SBBM(1)-Z< z;K*O?w>Yi4dalhqA}Vomv~7|PbdVcq;uv*!RxK-YzWslcTq`fMu zjS7`(HZCXSpz(n3)oYYI7XljM*-Q!o_Wrid@Um9sxQYJpaog|hqMEd7890)_&=Ey} z2&XF`sKhl5K_F{{fp~$L(^TiI{mQay005Y_qwbpZ4034Q1HUyE`U{^ za=17L4Jm+Y!}n95HW3UuVx)hpgF(NtzLv4sRo{f0$UV_1l$8t3>W!jt(3Vh7#o7y? z5W4U53H??M0EzYOLdPV)Aq8-$w>K#j1`-R_b?oUCtVK9DV>dG_#XEq8DH&F!tz!Q! zTY-Pj2=SSe1JyzmG{(v1DNiiRnhA;zIV~QU;KKJ}`qc^sMrg(XG;nz6oKkipU$Zd5 z9b1s&;vb1feyTuzRqiXz7@(-d<+$Rct-~@6ukWlnfard4b)1z8cfka8WcdHijLZ8n zPBEOsLh{#s1;p&X>i=rQB8-0>$6THe_9A~XhaK$Tv?*waikg&{dN#*4GEQ$eE?qcR zJ^Vb>J*GPB7eMwB`d8;R1bxqF8wE?uGzQ&syx*c8-Qk!^Ms-yc0$SbA=%!|Sob8IA zxCzX-dYyY~a)ZwSMgv2&rV5>SKka}1adJXFAH=>y0Vz4#e~8K3uj5;^*D8eI9BO}F zGB=pQG^EU_X(nvxKCKoFaC=xF>4_{sL)Tdvew3Q+=KG`1Qok0;--#X7MH;W8v*D4d zQ5%upiv0j{J*$eoo*`Vu%}8mC1jowjHHPTwV81b9LsUWn65VbbI+AusTRftEUsN1h zbM{F~&%!}0a5pV;Mz!*z#t~kgdfI>1Png^+SN;fogZibdZ-dIx2yc7J-?Bv~fZ?V7 z4KIsuqB(;iD}VQeUtcS5P;1wPk6jRWfkiu6wc<^CER)kQp?2eXv=IF;J~FR`TucSL zY_0cg4}Ex{t3KKdK8#Y1O>=qeu3Bjw8TFzHId4Cf$pZhUvh2zOo8yopx`2OiJ3@XS zba{uc60F)n)!mqSYB1a4T~$FvqHl@R*geShNG|2aK>&9%oHcj^^C+AyDZuZ+x2`dR zBu1l#BdW{o`UUDVmQ*HlAL~A-b9D{?_OM=~x205E=k^U)m~F-c)0w61HA_%-Y~eU{e9P#lEsVdr>z=XQ@qyk)6+@%jL&*cyLvhy&TnD5G2e z6sb)OOH%3KxXg!#@~>O6N({B2CAOOiyTT;a(lReQje*9E=n zL5K(lZ&^CcfK+cr_IvW>uJ4ZwHznfvVoxt#R#>6Ji>+Ea~=^sOruRZ!uR-=(VaV7)+{f5`i@L8m|3lII5>$JAImsz{O&4ceg4%g;)6iecK1hu~ zpO>&`)TUlb502-0gx5r?Um&NIRK%mUhNd4F0=8$WGqfI9b_ahftth{kS`Oaa9d8V1 zN0I#Xm$v7lz4vwjd)SaAAS4M=V#f%?mm5<^i+i%+r1u?9O%qyF-D}0S!w<|~pZCiL zmjco#BgVlDx6Br?8Ammr=avvXLaFXbpIh77TdVPq@gx#^PqK!!VpF+?)37DMcKu7Yp3@Y6# zjax)AYCPvj!!@ujIC2gtE>YnjUF~lnR1Q_rU(&J@2}!@^FNdk`tesv<%_BZ=r2IPg zY@R9})kCaQpFla&;jdcgEmPGYeL;u|+k37kBJB58c<+DRC!q1;lyqjcXJOkV71G9R z!WD_GZ=OV8ons?WTQ8c6)L4ywp>4Jo`ocqIjzD&0m!xV#BZux%1m9tLu~pW;+`VzH zJECY2R9zGBj%{scE5oVxBnxN_+_v+iE)^s(C|Yi~*VatrVD$I~an|~{iuepk2ZeE! zL>Z5Q66t?C+N3DbniMf8qG&Nxd1b21oExlf@t1-?sFF=*0ZeLo!GW~ihS>(h@T^Xckj0c#2#XHVe4n z18N3c{-hM>J8Xo4S_~r0tGp@8uAlX^nU&ywTk$`(6%#67X)n)YqU>|j>l@h`XyO%~ z{SJSNFW{QrF~WBTMPg7Lcvjm0q{_;ZrD`ZnH-#g8%k`ZQypi6P0f>lVJMnK z0mJeqOzB}xF0Jl+@;w+%<!&>1_9?=Ts6iOq$7K*-xsnjf^=7iKvVBO$qFp z#OkiP3Rykc*VMy6RlpOkk1)cPfZw-hqJ}Lq6{~DJ)v!?eKzG2g(`9zml5Zxq-dlgU zRpX7P4%hSdymxX&9MlI4rdrba6P$mffMG!jrARVr;!uSt| zJEbD_u_-~RkOzoD0pe33VbBYe6&|^W26O&Ed{|5=g$TsU^^%z9wLtZ+!UH-j%j_k0 z7i%Dp21sVT;o;28lw5NNTyf-eyJ8KfzW>^XZG=c{*)EG7ibQY8p{y)|ir9Z7So~8> z|Fj{2!lH(BtB-Zy4(;u8IJ)cHR0ns{iz|6EmPeRQH5U6GwTAc?KAv7=rp?Mq7(jU# z7JfZhAT}S(hXgV2A#n_WHQfh8_K2Ws)uQW&Wjq?|5Z$7J4(73gjO4Wu1tyP@D56LX zW#hJrkXL4vIKNi<{S>4JEvkP$elO>`+djXQ5J;EQ|33*38&qI@?UuRa?csAJtpdzD zf5NFW3ZABkeDqH9F}f-QD?-2!dsjfe4F9tQ-~CN6Y~((0pM*WoZTgSX`-u8&cL{`M1t1osY>0vwbE_xD>3#P z0_)etrl~_U>gjNNEmIe3sm_h5mKkfzw{UtwK<+==qv)^aAKx05{VxXtc7rJ^4h#Jb z2lnOfe#V1jaL?HiWRHJ6UDt7M!Vl*Z(rmQnAwpoEi;XZB<6baa@GKbpcPeiLEG$M} zxEcC2z)rkh$Luv!;CwPigsFV=VDH9)8!Uj3iRUMq6m=lVgy{H`jFfJA@&WG>lS(bT zdrWD-jRlvA)`O?Ui0jL*FbA8RX7vo+unP}n5( zem#2+Q7+LRrHg+w){!oQ(z=c}IyJDp;Wm}?NpggVE&a`5fMZ)d?Ud?=&Fq%$zu8k} zwGQ1$w~?%6?xn=wquo*0ZJgm+Fr#D=$N!3f`vsH}WUlZsBOhy-xW>)O-assg8ZZ?q!Zux%^#!8`C`_SfKyJ1NO*VWe#EBLR|i#B=I6WDSD zeMNR2kQ>53(InJP6&uHjFMgQ(_ zc|Ki4QOkc-P5)w;vevbKBlV@^tg5RvER2f65C@E`35 zG2pSfwy2-3eqa4-WuX25BjM{G`e()IMb&KSY!!-c1aYzo&ZUc$lcp)j=_82o=!Hq_GmgtAN>Ue;OVxN5#f(lGqu z*K3I^i9@)plRxx%AIFgk^=cXgSllDrK+T@EYS}#UcWPIc(h?C6JpQe0H1EgJ8{~_f zW!V^edZ_$?j1=PqF_O3TS3^NAc@p7i`J|f|$=iNi7YJo=VZfB%9d8n`g(M6=iHA^0 zjUay}YeSFu6}Vs&p_ZR+GxBi!_S-X#AaA7YZ#Rur>G`M@Fe}98KF%x$*qgrXnzVfN)#Qfp9%GKh)2XHW6!asT0v^45ys(&SNLI z?I^McLgir&N%&cuN|SaK!G5=Qkfw9c4V-`TYt@EXT}~@a4fmp@Eb;1lnyjyF?Wv4w z3MNbDv~5so+DjNwq%f~!ZZijxM*?lq;5MpnYAdCTUSBQuKM!qxr|w8$@c>31Q?jZ` zR`(@W2UBl$_*KS{*2;@gn85ouH}FM>h}g|2R+edo6*^?qcfu!E?OwsW03fT*<^lf1r z@7YtbXCs^6X55HAPfwD*G6eT= zO&F}BK2po9f@nEcex>EGtQS#ac)jW=fX=dA9o;ti$WH&kD}uA5Ed(*>R4g}e~0mLT|7e~KTS05M%9itgMcvf?dAEJuGzs97is1SARwFOp16Ib?V;h3;M9i2&l|nb~6)CL7+f z;}JnUmR}Wip_Q4_8qnGWP*&n#!9H6Ujvq^(C0g4O14v2OZLctVDr_t{N+e*XmPvi5 zV!bBzs*P^hSWx!z@0sR5;Z}yoA2zD;u?eUi<5SEg$LU$h1Asm6xk!J$HQl{>!(62P zEq1n%&n#;x4dMzo`URwX{}SJehT&$G)o-Bs(5c1~@;gnV5K{RPHB1?x`x43{$l19Y zFNyRyWqLv3xE`zuY9j(d1LI{8JkZoL)fsv0T8kgJ1o$0 z$_Ygq5tIUI8*mQW>agc?kQAh$iT^~)qLskGkAk`pw=15Z61ab7wF;>u05s*sk~W}v z@hm%v?`oFk8P`ntoT4H{+A~XDa3 zwTjLkPWsGc>Q{iibryc}l($-bnE5V|(E($7^- zZs7M_N{5%~0r;dFck9p@W~=>pjVBqoO>)h-6xHT$&X{Lm1z3f^!24_Yssn zW6;JA*W7rXKUEuKL3+--WOZ>wg?y(0%qR3?q*ouX@@; zRo^+2TL{~I?)zp&QmX;|$yI?ta(ec#n@slthzk+QZ3~hG=L+3iM#DN)Nj;d17Slx( zuy5`}w8=AwpSm#UC(sM>rF`2$8q8Q#vz2g+0#JY9a6n&&LfaCdL!Pc}D!u3vudwTm zSve1$-{MZB9J*hN3~sXb@)%CVdZ(cSUNCXiC43GAo{%nHEkXMo1cyRxAWv|3K^BZh>jiKJ{UWH4ic9MEaNvD2bB7tPzJ{~H0m~py-EU~phAW2Q zvzveVi)Vf^6XHABm7lUTdDhJ}jb(eq)$ugdZF^6z{z>$zWy_iY_Zi3B;5c_Jo$-k9 z8UubK2i@fHPIvLEET1UL%6gken(-_sHG=_R!ZIRKUa?x<8|XFsZmz5HvlVWfT9EXR ze0e`ax`jKb&z}rjaMhpU`o7C`c73{7xF>)8#OjljNS_$fDm}FwFyvXZ6oxm3Qaw~| z2(dG=`mir77J%ao-#t%SfyHe-$E7U2ZOGc7N zgCUR;M7i&mBd1<)HG;*ISKV6c>#2V`{OGR?xUSja#_cd?Qm$)`QzDZTDSO+m?x-8~ zXl|Th7s=5B_=T5`uHU!$6gZCA#KXTMF>vfs;(1XLZRu3uihUsP&{pZf0&FS26y^ER zL*Et#C%0^O7#NEnk+LQ(1S|bSTr)Y4kMgX)nBq@Ox>R=rNagu)&fMLI= zpR&?|vIdQ9fn8d%(sBqP|79 zE$tVth|NI_yth!nOGKXh)zp6)jK>Mm5%keLAgyQwkE!iqeemop^7jcvRkM?7Kx^Ms zE1I|5Mj<%q`n>;{Q|S~b^~3dWTZbU{&f!ww=ZuUz*6p~(GXAxmDqQOX3xBwMdeC-7 z(m*q>x{8q6U=kB7iQgxamRL>c1Yg--fia4OM7;(atwLX}dZ0H282*18>;`-8t9$(P zcMczdeOB<)$%M!CA)hjr%=|a8glCGrOe2+u5c#lyVQ_PZJBn2>yr8T|91#!lW?S+1 zEb(}01&vic)FQg1W#pjTOmN|7X@s6z5=#Apd61~+ui42B|N9Sx)*A|2bUI}w*^>dP zj%UIjcf;eop}8ExF13FZ88EtsAdhmP;q{;XdSk%7Q;<6MaECW5KI5^9&#^Xjyo!Tr!e7Gj3ITuLvzOoH#%zh=ia@in zpeygGJ!MMmi2WUxz{Y_AGyO()7{00K++Qsfi{SUmr3(3u*!6KT^tC8ZFI{J=b`o_9 z?Yd0!pkW$VLxgp<8K!r{R}OES5ijo1&Gk^%e=%RRUVEjJIKmy%pL;B z3fF8+xM-!H=8%8;Vsn&HSwf(UEoik`-#a`Aa#GZBl!`m3oHq(1rk~7?l(Knd(~q5H z4UaMNT3_t&1^(=mLsIdDt21IChA<^vg_xLahIkM3WH~&pP!@$R-beM_c*E$_H(q_D zFn%7F#0?#w$s5#Rfw*wVVfRDOb?GNA<;9s%GsoI*t1o|S4o(Hogn!!v_CP1hwp{Vc z)(<5ns?#j^imr6a!mTN4XDC>3DR3qBELL6D6V1Us5bH3Ks9xiV1^6-1=9si-5~p6r za2kMQ$Z|qO6aKjGS~ut&;!!Mj_+#^BkwRcdz0j+&+hYPiVLX zfw#Zrh**D`Lc%ExGXn0>Y=6IB(pbn?&L89gm273t_`Bix!@E#0b=Cqzvrx(aDx5&` zSE3pqgx{PZ-JhkKGke*gqEy?Vk9K%A|CJqhC3VlxYVT=U7ClvF{g$;xgXPdO%dZV} zo#sU~DfGq8#->ke_4x1-tI^0H@)LfVp;%?Cx6glIDRab^mm5#`6r`gZs&GWQR&6&p zTFd&6ohJo{EF8Gw?!|r0RP*5ksAUgVG<$-dIdX55?FA+wO(IZ2qjFyxayL(5PnadH zRQ}2*M`(~r+jiL=s9^|r@E0>1ey$8e$QdP@Uz2lzm-e;-$oFUDuK(`O)A9c9%zf+?_!!jRQnB?c_OszATmYPZx`Xwd-WLZ-Tw@Af4=< zI#bg`>C&YfL1he{?jhjB;-9q2AF?!u(GwJhJ^?r2B~qLNB@GGX#8W?2f+L^YfRkO8 zPhEG_(*95r$KO<%9m>aThQL`&r@@x3m@L1njK?%6V9WLnZ*ePHhVV!{<-fBj#4q5p zFeNE})xJ?4+>P>plldR+n!m>7A83h=EIs`-YsYwS_Do|*^YOt_C1oUoR>uz(D3K%S zPDu4kHNUtIcV26t>0JX2u>Yd7n>>;|R(_r4PA#qtGj)v7g}nPRPUro|RsPedD#Js# zi_>zuB&a8>cWnk#6Q`ae0X@m}O{2W-X7l4>x@d715PK3Wd9qc&;g9g8H<4ZjVs||T2R4xL4 zO)l$l#8tJY50R-?!)00xf7-+6tjUq_D0N{Gfbu&3PzP~YiIb5n|I8~99$}BO3{=Jt zd)bj`Z%*NTbTdow3yb*b-F*A(LH_k>6^NVs4nGfKfpc4Bobkp z*`-t=@F$66aWTKidthrUtdlVflAim5|A^*qMTbS=g1gI{s$S2KrlZ z23zbf?jb&UE>h7*w7$`Oh}vWZ}TM`xYAjl~b4c8?aiY3h}=Hle3_ zi_djj)6Cc}9$yWwCB4l?iaZy8&SchNac7y74FChwG#I|OA^y^HjKq2LMI}cM3HOHE z)9{Z4&Dl>9Lrl`Mfei=*L-j`RC&8N}7ha$jf1$?tl{IV>@MnJH3e1S1Xx4h7WoMES_!y~wsES5>x@NK4 zDC@#jh_5s}iC#EFCbO{AV37AZf>auJ)Dg)<(WEC+fw!bbY8;X8rZ(n{K7(r9j3Xq)>qDjfq}O8wx360A#?5&Tg8vA#lV zT%cY1585-%?ag>89ZZpb3qA<8)inkOqr~QtAYxZBm4E`hf2@dVmR`{YajYs)9i~x% zG0gd@!zi4i^Rq4jb?_#N#jlB)BIQf~SiA(F_fh*-P0uhO;}^FGWPz6q8;N_g9BeiC zcjkaI7T@MzP5K+W`dZ3mC`)`RWw%!p3R%c!RVRBAKE?R9e@r%isCkA@XErDExJ zUg5vUHmU60MwvTn+1ZDXNTPStO*Ev3QQ*Q^2NuPU7AYU;?O5Ul*u@G6U5aztUsDe> zHOkC(5Kw9S(_E~f3a?e<5woK*iBqg0yD`C3eRADq7u^pfSbqQ&1IO;?d? zI}St_ltGw=VP?FUg_$|4|9Wb5Np>y3mTa9mprhxDr(o0*(c2`Cq*g0G7}E$EtMRM{ z{3~e~zJV-%++B+vl9*nSe(V>ns}Lm|c~f9$)C>z0&p}wo{cBqJYaD=a`<7ZcrSQ#E zmJH1#Ys{p{gHFIcgEcu8!$&1s9kbiDIW{ z2D;wC*F+VWC3@V50dTWKF-UNytdaVoG3rD`g>0A<aF?-=6r zhps0e+0KJHh6;27$A*m}H{(#79jQ7P zZU}c*19+l~EzGu@{H$p#?kk>(m63yuIM2ctVQeHeQc zH+Gq>W3$l!TySg3dLl)5vIqM~s+gH9llibqo|GrO2eS6v2grwClv8`Z&1!?vXgctQ zpe$A!y^PWN7llGkKo@I;m+9X7m!}z9^XhP!G=p|W5ntA9I)Q$27U@59h1EAh3@4m> zq~C=kGe0LvGQ{l}Cj+?W2ku3G6fTq&u?&v)NBUw+gLYwe$#ymWG~BO}1|t{?Z9-2g zYJwQUHxUS_^&}y=zf(gbH-Kwv9S`)Ng-{+VwR|?~!$S-=$%tqVz0!A22kCCnxnR|~ zK{e7*arB?ye}hOkwZF1W zgA4WvO}>dz7BS?PGXr7v8#ApeB2X9uPq12~_>>z%XZ~(8Fp;h}PhT4(jE4i0C!NE$ z89uI^oWonJQ)K@RC+(yO?TSWZ+0PZTAuW&A`fDdL1Y*NX=TG^8L;oXJK+c1{4u3&| zm)dZwl$EJBfp4+wrSXR_EzFeVBB>P8 zSmUdC?yG~shL6x&fY3dKV|RA%@X9B_$I&9xf(g#c6LH1gAgjQ>^Ss3F60oqKgJ^?o zfZ%N?%^4_+;)UAX5Ic4HSW)V;MHgfEI;44eJIeyoOo2AUd*D=mzo=!~T=1rwR@_jw zLlB~lgBz%8(m9A5}n*LA5Mj9_*IyvW{-q5ef%rm8p=2zW>JMxeO7x^`mSgHAB zFy1kwF^-wz`wn=a^5S7LbK(%jSzFb0B1g!U{52i$t&0Laj{P~a)|>|I6Tc4b{={tQ zVuV!BGiB)>SjcSrlrcUV(`FdB(s5P0qoOw8Se6ZN$}Tv6vX2G|Xt<)jcrX4i+gKH7 zxe4D-bqjTR3Hm5Yls{H#6BK@~i;##yrLt3qy4eX!MzK77#EpPbp{In2fKu>cKWwTW zLQB8OA;i_tztX~fRj+OrTwt|XRmX49bpozFOyP2&;?T*v@1$^St1zpLA8j$c&J-Y* zDih;fI&(omZ}oH&$Ylw)(I@^pW52i>kP(u8z~(|Pvx%Eg1QZc^!oQROuo#3fY1f)$f5Pji){51sSWQoh)oz_Wci}j`tJn7Wf ziOfm~-d06f=<*!O?$fC5QBczAEasgY-se1^r>$2tOX}44?x23Eyx+y3erlKUpjJ;1 zrf24dZh@D6XFSHe_khw?#Y?ni9}*ogHE*V=)3k)H0lChm!5bu3JF0e_sXKEMPET}X zDO%-!lBmd)3lOw1;&-LM<%|0lm1WvuxCbgb4)00h)YI8Rcrp#eEd8^1DN2D)$m*S= zaJN#6cq9jc&;rhUaAGQPOi~jZpb7&_BS}5MrRq0hCQWltBN}5J6E~{ig?4fPcTK`8 zsO}!5p}}C*Ojc{sZBTOZZc5sM`hH3d0#@!Z%{Rubd2H<*GAm|I zE`n|(dRiTK=|SDfg_Z<+nnf0Nbs#!~q|!Cz>f0ifVI_h7a`msk#e6}nGe@alil+t$Dxe&h+Rx-6FWpYu|&tWNLC>v zGBUhaAv)?Sf`LOSg6q<(LIb^c687_CC|Lrfpz+&|%Z02toLX0&MPeDC&X)ng8oTq_PIoK(QKi(NZp#(bihEZKu3=Mukif6NQ!@qM zQ>hCVV1EawE8eM5m+(row>2zRQVKO z(;l96%r_bC7SWVvIK4N2m8*ZD8Ci_z(%5MtNba)FP*QzPTT{$?QWB?dl3c|3de51o z_Hfqi3k3af&W5F{5M1icO{xtQW}h?X#ng4b&= zqZoHgs|GKNqei?`Q*MOCxAXw-FI+_kHd z_ZLBnxZtE!W-2_nob!&WUQI^6MMF>3*8OBSXXMb2!3lt8D}?vhB=F1zmi`RfeD)Pd zkSf`_s5mOLIjx?5US(+5L?3xl_%UDvg(7D<6mU0^=#spc+k~VmSD7goq5ulOcO~1# zK99!OO)mWwI_E~vuB^bhy0Kq)71z>cwAU6|8v^OprIavg0W?8l)s7%xn0l*s#FbP} zl>4;&F(CN?7;!1k4S=_w;sfrN@&{5?Y-Ro$;hATKyGTcWGa7BF=FVLuiE(0oJ$7V? zovf5*xgc@AwnBi%^D`gz1MqAik2@0Fqs(5tN0)-!$TH2m$oS)l=kVoWVdu|A^mXp1 zolDtV5rLVeK+T5bZOr(KX-cwy0$m|(&COF`k`+afT?ZMY3^z#0u|XCfkR($3^e$h5 zJC<9^{)fAN=^lxA@^mCM5lIuK_%9EfSX?1Nc<;NJ{N9$l5J}nQS}U@ipOo0(+>hPY z4WLY0brrX9b(Y--bjm2VD#i`{-yoZk-hCNAX}diVnNL30YSP>;{@D}(r3WC!8WXw*Jom?P7hUz{ESg=UO zlUbCyvgsP}Hr0)VPVN{k&Hh63eS7-c!cb#Qc@!i-JI{D`VnY)|(7bKhuKAi#*uGx_ zP|I%YCVfd2eEbz{{v&QnAC5x4@i4^FrYC3a`%8gaE?)&Z8|~8a z_w(%Js*N0pDzL~4q~0<%IBoJfzH2X@4*QO`LXibv$lKW{GZLxBhBAJFR!#P<&e?L8 zrp7ERc4CS2%^&*9B6H1~BC;Ibk_M(zLksGEvf6At$P=nxN|Lb)K^m!?hi))NVbO&^ z1OD`>c05p$-rf$R8;NP8y@Glb2$>(J;=n%24-4H4>U3IMtQQar;a%0y%Qd3^)VOfcq;sI57HYYV*|?3Tp_&AL z{;bS4PYYnKlQh&Yyu2A|uAXS2aZHX{g*eJ&K8KEeMA`R0hKU?ECYa~7Ar_!%L?9@I ztt^wKN$Br2Ed*4x@q?7>AA1^@gEu@%X!7gx#j7*|c>1;iLwGzQF~)i)7KgZmKW4M^Z#@HCzW$GlLv|twbDy z{c$zZ1a!xW=VcWC0y?lQNCYWSQR1R8^MRG&ahuVaKHx-v&gepueU!H)XZ4?@bBk>5 zZD5n+(lKcqg8R;yjj(O&I)JfISbd)m#P#4mQsvc!BK>FyL4qCF7@#@#@T&9T+sS`C zHdIVcKDt+#%-$jFc+kqujB})a%sIu;N4q!0dHVd@_%XLXHq)t?75FW%ce%Sa=jI_E1L3SSNt?*2o{()GN50;pnIg_s7G zWOHmo$C4jp9TzBvT3jOu21Q2vKdzPEZJ;?Qm*B(xJqQbct&hehw|!e)Dfc-{(t=!_-0V9Gh!9EPL=usxLDSs_+Ed7mElgVN=u?Jz2^9Tg04$z+ud4@dvGs6@X!BoS@Vq*sdj5bib86StFuXt)Zv@gX_J zelR?~4jA1k?nN*2qjE-`KAD7SgSe%UoD5cG!sheCeguw(jH{eJ&VEhb8i1r6z%Q6{#=wkJSHI^VhNBIyX4k(R`T5P*!!8Y7vF$0@UJ zoFQgH608VJLC&_=p$JJUnVgoItv~H z7bs%ZCefAWh!zC1q-(utLYIydVg%dU!4He+S6d80Y_j;Uwzy5>`Ac2a4r4K;K%Xn$ zSi#nvn&p6heCAN3j0~>)0i_HpI<@M=+!*2KuQQ)3|N6d>m}AB~Y!dF=b+nsO0UBXB zNRzK_OFUu_gpcXi7=L~Ar9Zo_sBZwjJ#q;>U*WjSSK&8(A-#p}1je!J<%@2zabd9u zi;g;eIVZ#fn22iNT&kEy+3rzmH z{z^Rvi%lcOL2@ccPASBX5$4)+J+J)%%P+@kW*QMx1LK*dt|spNhx(O04CW7qk()}A zbsi;Chw0mGWF5BCdUMJ;$-;FcQ(K58bssyg6>YT@oa|<3BWSH`InHu7+Rb81Wahx{dAM zzOERwXMVU^-k(4KD9-_w73n{U(VJnP6q8ba&Kg&S}6{`!jHVKi@K>|g-n6k^Ek>3_?hWIsp2 z1kNH0_5}RF^+6>)BJl0J{s&NTQ?_}<3c;q618TnJdp|}CS=LvOXx{;)p^DgGpkH4{Ymbt=K3s{`U1GAo z%IU{K0!@m%>Gm5d$sIo;pSqr&kYT`o$bpf54>KPou6fHWTsNjNh54Oo@5(P-65T$@ zDosBS@GBz~hTLd_1bELVYEXLSC?;S~`aTmIV%9g%7VQ|LfPtI6u-V{3>>OxQ#AQ&5 zT}Pd|fmg>@D*R^;e;d@7a;}5nOtqrP*{iPP>g}f9l?T-p66QFQKZR zH625Y>7i;+2oO7+WJ7#Zt?d5~de! zmjN!IG=8M~7-NZ>K#U*cQ8h7@X_0y-)lIL=l}TwuwKAdsO^nhfYWZv6fg)?i(;dbm zK|VmeLh&Ut8**%o#Dkamt|WxOr}&)jgBGQ@!b|cTgq(q85CY{zmx8u`lD(Hb0L`s5 zm|Pwv8F4a(cbr@sl-S8w^$d|3$?6_*U(fxO|Hj9esO|C{ZZyz z&vvoh=JSVSryyw8Uke*ch=1}VaEw$bfCJV}bwcKyi^_T~r_Q#uxe|)w4TfBW%kmd4 z8tdlnNhVCbfwE-QM9)vG@86;^4vk464CpAc0119egyExW0hu;`%93b$83|vPoH|Rt zAt(;XL(DE_!@DDl?*!}%O>I2@ltESZ0RaeIm|Hg~QGQaP(Cw)s&YZPr9&OndzQcnI z;%Q?mIm|pXvUy8Jx;Kgm-|gVelY6$S~ zk-9;YJ`9OYA>TA0a%5~o=2O;%t8QozeWU{FN6aqXu81*zu)g_L0t6oPQHcBg_*RAI zuxQTZt!c+Aa8}8#w)WxhK8a#+8?|dik0?3vU6@Gw{=%!duB)XI*?rV!d*H0#dd^vi zZ@VP6U*TwxvX6D3Y)pE(V0P7<~T1;O|VR7VYEY+49^*`Bn$*mk8m;U>~&dTlYZI5A3h zK9z{x9^$7(Y6Osw%-0Tbd|tnHbJGTZ3pj7Z2xku%=ay0q4NfQYv+qdWh|`h;sm>9y zv$?B4(Cu-XZ)-|4>ptk%f_uK!&)9bdcFr_Bc1P>J&ck$tk{!n~EroQqwCFMx$Pp46 zLa<+d{dtB$cjL&CEh(TEyd8UASR=P@sL^qh19&%23XHpf?m|=w9 zVwiyp3RgwzwhT50B6t<5I55r!0<|EP$YLG>9j946!*O+``ck7hT;t9}12c{~i@S!0@xP#EQ6>z+O-&1J|*2u3&M5#$7xy(dr;{qD*jmWKx9U&O6~Br483OBnU77fXVX={9PqzHpc((8 zaPXGZ_jsxovtKHho_8%MyJ%X7fYrpqd_6ok|gGg+p7M=A1o#Hnxo8@oN9ljhXnS8ysPf>_kGZ) zfLWCuG{6B7#|9@Xg2MxtnrWcR?0uQDl{F8mgf>@!sU z?(ub6Ro6}A_YO1K>nKPQFy*xPq}O5b25I6OYx?=!OIIy5NU*Im7n+NjI#uP)r%ZI7 z?)t-GZg1VgYmUcGqDs#gM8BG-0yjh{caeA30T@@BmJ2Vk|NG49=NGw0jG$ciQ4QW$F zomyv0I0vT)tc{|JYl`phCSZ`?K*=9=&aPk#)p%PFKf-G5?$PfW{yin{kV8SsP{qo% zc^@NQpxxDzND(U&i1}iFlrSi|$`Gd1(;$!PV^+Y1_La&Jr?~K(TWNP!{M@go^Q^pi|$t!W0=NiY!ilg3xn2wVD z9V0gYoJv*PEMUg$pw>+^xQ%eDr}*?YQ@4CcMEja_x%euPBz$Opo{u~BXuv5-{uTDK znrPcr+6a2QTM9?&Z#rN40~$~e<5&#GiE-o7d?(-wdP5R?Y=7EEcp1ThVeZ7rl3!_**bB%-aX42fUt2kZO8ri{ zt%!wuHh?nVcovQ7ft<;dh{%8 z$GAf0qF>Q}syK!RBsL!R#~%|Y;dMKPM2Tl;PtslrS;Z24$kw8R%VI|zev+JYmu12< zxM$uL6HX~qg&6BU!Dbn>=<~B+;&5z!OJot1UcC59xI}4^?qxgSNo36yga#@%+n$EA zXIh*ff+84sY~8a}Q&MfQi}Uuyk$k(5bq4-kgaMw~B)I<=E2kqf5hr-&0CJ4C4X5CV}-uXt{-nLk{5T z@)Md0w~7`y0?E;L*r78ZR_^V$WokRxWvtTh_Ag=6sXVK>EhCBjlLQlNGt zaJPql*<*7ZI%p{ksGxLq!(}-5aT*zL8Tz%T9*N6LHvtmFI%J5L*ZMHMjcQSJJVpqi zp1^9aY(-a^h<{6(P_zLL8N^qt_w0#^NYvE7Svy} z5|ryO`|bs6YLqx#+`&>N)*5Q&7hbN6yr(qlVx<6ZjJq4s2fbZTj~GmJ!-LiF*n}_v zRm$7AG?d_}fgCEVIXo7n;#1*F>?;ygR+uS;efUVdy1=ZnDL9(2+yOKx_&{j*xZPrZ zzp(eg7iK`lFcE_BC&U}|iLJ4wkHtHsz9Od3Th0`|Q(*rCTrj$uU#5U5NO9MADDMXt zv7Q!#pVhe?4{Xum*6wc*r1h?EmmOFw>48_){F~&5FESXQJe7u(bt^_>IW)C3egO!I zL1}jS2hN`;Gg*jE=y8I|2dt(r%3m9Q!2a4qjRA6WI;-xp{7k}dJAw5WC2+S^7(F5> zd7&aqMk5}(5?w&>7-J~i3_Tpcd>$>XxoWEzH(||d8U_E_%V|4~24s?E*-@m_Z7K!L z#~+$a2a;mfUHhOJ3c0H1Y~nC38w6eVRYsPN6r7M?UzDR{9Wn!PAn-sQX6~+k-r1SZ z5p9*KL1E`@s=%5LqMnt&@F`$mGD!tbNl2}0+q+4`@seX6bMPybzKs!&=Nmm*x;zUQ zdl!lN5E|m46Ml&5|hVfOpUCQq>-L>$Z3Z$yu zxMOx~C+u@gCYJ5yyj^}Bb;STJVguPeTNoXQL`D#9m92Z%7Px99;X_H#7JO;ikQfr~#2HO4+N0mq zidzln%0=V^IA}A^W!rq&Mze5tBBDkb#2j|(^w@VNKmrTL#_Z*TmDP}>NFUPJR7mg9 z5~5Om@FNE?Z)`HR7ZF>YplGH`I##+5OsgW3XH)S_^Y&i^4g6$(M2B>&pPdFQ`GQ^r zI~CgBpRvS+Kt~M-h@ZZ$HEY66z@+!o>PWb4A;IChQ5w8M+_}zCl@QynpB4iSzH4ux z`zG;|+DqeC#%|;dfP5tjcy8$q-vb=Y>PFXYrU9ssDwLI(I;U+bq?hw#0ZlZ1$GTKQ zn1qq&$$GKyDqVVihYeP=Uy;idUt9JiCh6xVh!=E_^wrk61Sw~Sh#!whZOP4&2{{O= zUmuqW9`)ibjt^61oMq^HuiTl?VIOvOo9XFpOed)Hwtyl?ioJC-aoA0dC&~8nx+qmg zr)MMCy~+#K0jf$*kLc>mRrZ1)`sDF3oopQI)wlK&cH*wZg06mhY%pt=4`9eqt;qUQRGD8z-$7uoV zOz|Ksas1_yhR~P(k`U%;RU|DmJy%#d9of26&N>#G;2Rzt=lPvYdO~QRzop;QoDIBx z@)i!nFrlM==!*@eTC>5A&cnU{-{1!kGdqz0&-OMiOoNTx*w-YmiG6MT2KgHtXib{M zRy{2P-V9~zqiqwGTr`W)^!s5C^Z5ZR_lWRN3qGh;I$mG4Nb5=Gv_I)LlL7gNqOer( zUkVG(p&$C-cWj%pnrh4Se3OkxlJ#aaWO9i^Ib&mgll&q*L4bua3+TF!>bBsuYVT%o zS{VR!)3CuY#6<*h4(>9*vfPR9Jko`YDr4EDtxVdvzy~M_=u|-dqy1Dza0}C5Oy~5oO}ge6-{Nz1 zAazi;{Hg_x27kYD{+7e+4kI!D75BGE3}^mzrN#{A|6&zJs2T6CZ;T``4a0r4b+K|U z8q6_POL!N!LuZ)GBH5te|w)S%30es?6Sb*1Vr={|e7VI=Su3 zt)V0t3aXg}DMdDF;uSo_@0*_|sY;o*eyh#Ob$8)wEeb&OHs>>HTo1ID*y_ei+*|A_ z1 zL=1kuYk!{C4Cj6#UYd+5f9+2LjL-Sr&N)r>2g`ian@x_?U6z^ej<$YvSu%rB->_Q6 zPQ?7SSx8NS-(A}RL!BQC+9d->9WL8T4!j-t+7AWBJg_^g$>ajxJ8sCjy(Bq#!HK;B zIup)ag8p;hg&!fic3D`61MhOJMbx3Fb(=z}f`6=Wze6s9^7i;dIfcIReyWNI3+o+& zo(LD_E1{`EsP1>BHTM+c(~l#BEFV~~D-Hq~M55aa(i-HrOTda9w73_GDj6(^cfooS zyrQ3r78G)L5Qp9sI%4R;u^kR(bc=Z(0^)BRdM=8?T8qqpFCaG5Vi5@@KJdyO zK!5Ki74`g&wRdY;R(ZYzzsf{kb%z}xDxPMpvWfx+Ll)=WV|{++x@DKLb`JgVP#P*yjhZ zPr6lqgq2gW}HnogKC|Ta40jb_eShvoL9P4RY7h-(~D_Qi(Sbr1O z(5y4OOeX9;nuYps=^l?W=`U{X2d$Fv^&r{<-u8~ij7eJz>rr#{wPyU75A@c=>0$Rt ztf@SonrPSCoheU$F8{Y{Z6}5iwvyS={BrKM(Q)ZkC)!`bUJEh?$x$`()#b< z)4b=F|9QO}d0_Xx^pk8|?IZx$=Up;HMPi{|Kkadv+@tyHJ?PKUv6qniv4k7AiwPQN zaP;oy#9B<+C`V$uMq+ne?0@*o0FC*XwVg@JpbZNFp)#>NeZdy>826ZcUtF)u5Yjao zu8b7>8R7M*+0FJ6gXZ;Z!=smL@hVIt65!zDI9Nzf?^m>gWqKooBOh(+=Wa9^j^C&8 zEqPr>=f!^e8cx+MslDm2-fw2(YG4og=zxdRKavx_i#c6u#n_(gaDUIGiTj#}x9__5 z3C8REvdXJ|CT8=yKa}iXl!yA{LtNshxbN%u%AS1X+?RML>cuWzy`1Y%3-k#mr_wRC zEJ~82hW$h*G3%9B%ZfK= zzR}}gT*xRM#XMyqI#wcDpR~HbtPUbpH*QtO+sc?PDr0V?a(}4F+o#C#$8NvDj}aOc z%lZW@e)#cXv0B_#bYnlo$*~(ge7aRVPn?~e4r+84^S#mZKcDv1+nn56f8HNN(|8t* z2HR6G?k!Gz%K9_gIo2OTw z>Gig(+e=byQ-5P^xXBV;H+>nV7sr#doZ3dVzrOpItA9U_#*e{xznW}=-Yz??(X75_ z)9qy~qM*M<{oweC-y>~t9R$PEda~iW{$@UVO{S@DkK6fl?GJj7kF%OAKG}BGp2FSy zyc%Si)ZN#?5j_9EfP$_SZVD=XAJyu0m<%;_YOu8mJm2^DRB8tmug>lmWlswg1~cKcgM=q4(cS5am@K8v=vEY`*NYMyxi z-h5Vc{eP@>GBZ4_Sk29MT+N^D@;Z-n!zi+3vb%cOWWa8h*(YFYaTq<#WLRHc)>}W6 z?n+w@*XPLCIZ0LZ4Wk^0Ry!>4nGFZ$mH$2&*Vp!dYJVsE{$5s#c+-BOx;zip*<|f| zeS9)TNpJh|+v~o9;rLaoAKu5dmaCD6fZVQ8uz&K3$zuxR`qFFavGyo>1iflx3j4p59d1G*V$J849D4fR!?5j{e+(M$-=bHJy*|x4lfpD znMO98qZeHL128+6k5UwLPK$betUk8?zT}@#c!!#Y%S+}?!sJ>yBhB8QAt<-c?R(RD z{eR1N5w^><6~%;j?7dxVle?A-KYL>l`8&Lk+cnr?_!#@%-P0%AO`bS?4C}!tz~g7y z&R_SiLfM9&TQryju4m#s`qULcXw|-4Mjko%f{fs0c;Yf5`O{v72eGm;e{-^y*(0T! z)uf=iHkS3ixKBjZGfY-G!^^PF(t`2Glz&rN);o8y^cHMGoujQ6E+xTAi#_)qa%XO5 z#pYNx&Gkj;)_T+Wats>#xjP&DWSjc}>rU2d`H743gXq(CUmgbf@G*0O-8(ihEZ29w zbw-?~TxTTv;7_G!yO`SGY${T?wJF<7W(v*Gbg-OuQq9y`i496m}G9?%%g*I zoZh#WDVJnNt32#^$zyf4$bR?Sm0_M{<6}EshFLnZo%nq&vb-rbvNs#?aWGt6%=2*- z7pG=aJtphJc+aqI>A z)bUk(u3K&$56%hgGnhP|a5T>@FHVhqBOzoqwCGwo;rL zeTVOxK-kU2J+_Rr;i}x{7iaGqtj+JU?UJ3}WzRQbh!*eb^}I|9f1ZEl25kFV@0gGM zps3O))91J1;RN$jyQO3^_FVrK-sd-bpWH6dd{e$Z=UduyvSK*cxN`Ffd$`|j*JHD7 zT30lK>_kw@KaZeip5IA&a(|1RH%`*q;Wtce6(_jnDgRWEzmS=3%+dchRBTE)Q9_FX#Ii z=C|hZ9QLO-$RgA?o5X$T%Z-_0Wbb?^0HRdQN zwDYrY=A-SE2z-qDPIP?KcWCyR%e%FB*ep1no^QXn z`l*bUi_cmIkFM|2d)nZ2zT01Jv}*3B6&o(Y%P8HgZsE)QYt9l0`hdzwUZ1&hdRWjWyUW<1(@f&mZ5| zBRP*y^3E=reE}z&W1h}Otv869wVO{(-`NMx=Qz-`I?+oy+JCx5Fkd^zy@mJpfp~0Yrvgu(muJ%tJ`?+VCgpZBUre5)Nza6au^IXNqH1t8{X*h1SWVM6$v>NWq;naIide=qb#gFu{ z_4lU}KD}Y~$bTLP;rq}@=o$*Q5KzhzC2erXZ)C>^F`VlSRwie3AKvemRUQ^AbG9y8 z_iTHbxbDi<_q1`bo{(noTzE#c;(0mT2f5XKw@^kCM%S0snx85YifCGu{V+;T}`eIOPK4t zeSeumW79>es>qif-{zIK|Crun&J5PF-aM4Ede$2?7iKL7{OI2=pXT{otye2Eu?I;V zhL88V#8Y#?v|gE;`T8@kui$T+wceXKyxvBd7L7xwc7N84KGCN%y?C^FuV#8QMx%Cb9rw@PN8cFE z{IR z^T_%%eEYjxBbz-QK4<+1q(lPQ1zZf%3V|w3l$#{i80&vgnzkho6dwTOmc5iyjg9<$eTyhE{&U&*zhyJEb<6e?x z94Ns7%feFXc9lXX z)nC`q{;}A;Y0jR`q$eJ3Se1{5`mxFTeK$NmCWlXAWyRid9^30U+pSlxWca#Gdqz24 zw)y0}f9uEFWqS0V<7US5Dt`eFz#dmZQR;kJJxWxiRg9`&voPK|p$ z+8^HSuIlxn41GJg%Sqas;(Pen>|f0!?e$UFocztRp3Hhv<38z`a*>`s{pk$rAqnEh ztZ`i2cfn=5+!SUIt}f;(9vt|$}x88EnthIh!C)xaXo}9eHO`AX4 zD7-!&+#lR)ebNNk`e97+dSamSN=CA}_imH)@O*Ew{%tUdQhT5et`S{Nt#bWZ<}zVx zYZDGJI%`&I=2F+(mzinHiD7?=n({3kY^$L|=8LtD?zb&k4nL-D`^6HySv(n8WOlk$yLU!UGrg>iKmzZ& zFwZN`bvBJIy1)PG#gno+;(0PZ)Rqr9sK2k#V>%Q(nIYQj+1l2TuX!inN-wtieF^^m z_`go&^SXQe_x~o#)9JtckN^38UjF-G_uBp6r}Fmdzh0;RXTN`|juZWV{wU4MPB4~8 zL<9C=#SX$;1Mo5>RzPBTuViCP>qXN77|Ewgso;FLae*e(egVLw>SSxVy`kHFP2asy z1>i~bHE?3?#L0A1_)xe&p@lE(<<-T6^i1>g>a|JB&dHFq$;gZN&1u*QiPSO!zBcP*V~WH^IlVb~GV;#p&})+s zIj2LjPC}BUYMYj2$<2hcW#?FP8l5zz239Qkw+T zb`b;9n z7>U$2>qMRNl?kb()(JKA=A2{AYiZ-1S+_nu(4i0X>KT0CxYtx}8 z;--tEoSn@eU7N7fK6PT?Ym$?5b;nZE0$F0dKoSRpX*vjQk)bUR=XB9% zhQIQAJ$r6jF-w4#_#r7mKsj&N$JMG8k~|1o=2%&5`DC_4KoCf&J_ANF4IqRezj!Rx z=!gPYVX0o%7X7EaGP5!;fG;`@npXswyGFp{9Jzx z$Q^k1c_S}4_Xcgv48ah_gY!B{kWpmWRsBL(aJfU5tP(48cv{#6+MwAh@?Scu51XUKGT`1~q}Fr(2a6NKlMmC7e1&0`!f+XSNOI zs$Ufd8|59>#3A^wu^McybN0GkozxH%SwYMV5H5mv8(-KKaqb{dJ=Z}R;J|-*#&~o` zfyuRF<^or-5UhsDcFUx0LjX4A@CQ#T;l99U?(RALT6nJzuO%l)2*H%g=6o%r+U(hOX59BmlL4%vyGJnj|1VesZl_3)>Jh zHWf8If7cwf3R~5>eRI*0%cpSO|NiTJPXBiQ-+TSAZ z4{!vx3=lTT^q|`}BS$1sYIxkhQ87k9F$hYQQK9k|(RdtSp<7N(zwa-|Vxhs?4Xaj~ zHOGiLv2>pxhzWtSSdH-^Dfk8nwLYAx`~XqL5R$)gk&_q;+U24rG*5rwvf)qP{r8Mv zSzw%j3w(x%Kh=-hSpXSRf0JoP@u28WKNP3MrFwazVQ5jHHoiO>O@axaO1slnc%1^K zKy|oCd;pk(;WXQV_S}3kZWbajzx6cf>Lv~0M$csM4@^F%nr9?4gcHpRldIFv+{xKb z4#DOpUcUL))ljuf($#-WB-2%gwdYu4?Jou=XL{z}ZR&W_)ra$+-0yPS6hxy9i_I+V zVo7~I*3!Wr%op*1*6G@mY@p()`wTT3sT#wi)3-UL@R^?bubBZnVZc)tg%Ez%nF_>9 z8@OD6q$P;d_{rr3h%E90skxjpB#E~}Ge$&3R2<-%t_j{cSYdxEmt+y~)Ey+LEecF{ z?DntPz~c!9k*I&wF%5`|G0s>ZKn9j%rit7)3TFu}&VV1nK!PI#gkaDO;B-MG5uWBm zk?9#;P7sk52F^SzWA(2Mmz8NkOt36WoR1C6IY#wZKgg|Zhp@&x++PKE76w^Ty2Ln( zZ4lgEAelLcQb>Op2B{mwpVs{kaCILf<#01{BO`LlXSNZ7IEuF*v;{oY$dL*!h>b-oi0CjNq!7ER z&SS~2EvOBWAjJ6+%Y;P{Pgz(H15$)FhamO&d1->6W)sDOt~Oi?OYH+Z?yzckPE1mp zeI$~|6tRCr0^2l=Y9@$!;MBvF#N{M3OFJr;sv?hD}uRgfM83g zjo$1ul>`*mEn_UmavZ5xU>L&c1Dtx~GhM|zwoP6bAVr``jq~Zh;VrQQ6oJKHC(s;V z0Qi=@cGp0xG)j;`pAgZ1t=k=!{|hsc6+!8mwN!uMhI(GDw~7T-^Ct);EmQG;S&r&% zt-mq##hor@yp}Zz6QIMW-};|y%$DCAK~&ven{{d+kv1LXwDIVRJ>=Z&R}=-7Ize2m zXKWcfUaebAx3VCn$l74^IvIi^$Y5&lYKcX0aw2#vn&gCe;hDL<1Wq@pzFqQ*^DyLg zv%Y^P0mI3gekRM6h!LuL$Bju`6btm0`o;)Yg0vEWq@J0Jg^E9ehrS8IggZ?!lmrEr zFs=&|A;kSqP^x|i(4TH>AP1yXJ=7{*lDgj*VM}tJBQB_eTd|{bwWG~&mt4sL2h{IB z`|g&L&46swQ~KRL$W2zUx` zUz2cqF4+_~YuuWtJXE=)*2VhvV{!YNw?Db9{qVevsp^j6YPG~Did1_~M`XHG^|9gFMkZBfdI$`lp{T|4ohIS1*4| z0%D|^m&gB#ADVNHzxBsRQW)1W2e@a5IH8DQI8g#M+kb0ImI_v z?AXKv35k#_&7b`d;w-XijtySnn_XQH{71E`75lE%Y+pzagoL&F=i@IAF_cgsstDP& zex!7PMXE3%QG5n*jUnjpBl8L`avp!ChGnVh1v5H4;skT;2V+7wZM2Oqii;uCHvbzq zR7=AUjQ2lwZ-zkKS2qfQK&J9jejooQuLB08g3cVPx)V#(0_213B`JZ^e~q^s<#dU7 zYLNgown!SqEzFj7J?Fp15<4f2FXEAx5$-g)rQ+gG?Lq0#YuMVXd2D8qsMvq}={r>{ zshH9{G&9NsE(o+G3JSxJv5O!z5U?-zl-xj)yNJ&WlT!~%H#;@S&$?G8$eD=+x7BvQ zRg5qKG9_5V{%;RC62#uiv^yRe&g|~E4|Q&1&j60P{bXz**!L#==>rLFA?vJk5(EGcde{AR_~Ah<`i>Ai?fW z6Ep_*1qO$RqZHsVT3rzK6i;ox8o)ntT&i9m{e$@(hdg)N#2rTe_JEwd;oHaGoGjJ4 zzM|{jVy^I>{nT($GtE24hKJMq2Mb!Sezw0wlH)&3={C@w^}z% znC(oisXoM^ihm&`*fW1e<-rl~5}X-4WXHeq(s}dDF8UVl_?O1Io+?#%r!e@p|NReh z;me`;&FR=+#Va_%mT=W|#e~SX!(9`){7U}~u5yA&4yKn&(glqHg_qgV1xf*B zmn72#EdiOAQqu)Vf6L3MWr_k3PC_I<^Lxw?%V;P0bo^`*<0nT~umzj}M;#9@d!D%6 zhxFRT^n6E&n-X0_A zR^J+J_ZHl~e;;g4K&m`VD9-p2M<3q*%s#;X@Gc8UZh};L5#17zsN)#@*SE?cArfQ-0Y^wuf<(IL3ub~muaP`g7=&loQE!x#iOp#Y zr)l1QQ;Q&uEW~W!>irU8^@k@ch}G%5K*9XQm^Mk=Y43`+{Vk3hPP;fEHQQQfmVzXLBckL;{^8Lck*8kkPdx;w;$z5?WCg4B zdQw**5tYJUf))v4_ky9%bk~1>M}D;Hn+5(zhyfR{6v_)f09+(*r1fe^E2E=64&_d#n9!S98MP^@wh@Y38Qr ze;jcl@fyNpX`V$Hg+;bRd}5mV9`jk&e;g<%-5ik15M?1T_;nkhx7wzavT88uGKGJA}M3UXkCf~$X)UW!>m1KmhZbrRI0+7K< zEjyqCS?#UMCApeBHjtTi=dT-rUQ+zzFaxvto9^JflEHWz1?4}_-?vXx8o-yr)&(bj zV=E2ycjolfAHKTj9wy-(D^qHC*VaviAY@Tv8EXt0(_Pa;qPmff7fedm|E>t+e6Y)Q=T zoQnPQ6F)pf>iB2fnx9$zPj3Hk548@z8uWkB`~2Iv)~^o0-k@|L++o0g7@aPVn}Kea z=x6R6{LHrv){rLoGk^Zksx;>tZ^N#CxBcp1=PvGQenuialR9m?Sj{Tq%}4|y3${?A z1-c-YI8O%DK)mCu#4OT>lYyFNos*FN9U~4W{P*66eS4qgr}tTQGcw%i2s|AxRa$@N zXr}7pS}CVqQ!of#>iS_D;ZztxAF*LUOdtmnm|+E)fu_7>P?fCFx;eDc7mPxGpI~%Y zxPjOrNvy=t6(+fch)v*X9!k)4d(e-PM_5usIy@odzvjoE`Ki;@6AMGNG=ntwND`oK zF7_|aA1;>C+94|+G@IC&q(>K36&sRvvrHJxj^M!~xTE1=f#m)HCHn4yrbKoDm+kU^*W*2-f-O*>#Ge3ocwYF&S^S6axSn&V7H@Q?@u zb6}9I5eNhE#OFf5bC%0eR|IWAn#nbT3-PjEH>YKyK#%z+CXK9_ybI#QTFbG|p@N#fv@ARS+S!>wedGwtua|*g)TsVYFb#;k>v|Hk24u76MXP!gNGb)o*nmz?b`ku27LZyP z_TB)-e#8rebC=@r6_x5|>&8;RP@lY=P|OKFu@ ze7!_Vu5l25e3`cTlN#YtX)Ajq4K|$NU|qcJ7+N!v9wegVOil~v4_Cko3}Jk{Mrj6Nv1Ebn8 z#-E(}=3%GxQf#UjdY&3TGu*DmfuLO-{LurtnePwgE#b?mf|cj+cG?0wKwN;#lcgKkNMMlU43UDnC;rs@aYxKBZ;* z@0yjNAp^PRiDig&1#44)OdSj<+O2hy1hI`}h5Nv5Whyd2XGp3(i@4TGGoXY5q4;Ph z?Y(v(?3uvJYD{ldOVbowdGC}>#YlL|H*a(VD6t7cJA*J`^+`(LVNV#|4BIaU$ zUU^9tTdKzn8~d`bI}TGk%(jaK)qmwq=OImpe{x;ToI77g8vgVI zt3UpxAHS*6iod;EYXAC$yO;~O{kx6#TThxUe$IcxpN*NZ7`5z|>kYh+efeE~okSj1 zy+OwT)@B`#eDymR`A@I8D%IEfDII@hN5n==bp#<#NdDl*2e2Vy;rY_3Jn9_(#WABClZ^c23Y-h=t+-5z!P6a8XSJxvmx} zx1cVzME+xrthjwZkn$IQo52yJP1Jl)dE0cSv^)Nha+5Q=q^W_BTmwJ%W}ulD zI^STn@tKMth=mcP7sScO7NrTF35B6oV4w^ukCB>>0;T)4G_W*(w~+Ib()moq&Tmrb zd>?gvKp;`r>3kP+5fm_DAvj9s`{(QlI&1(V?dtz82K?~E&Rd3L0!XIc*^=5e>U6vq z)ykii4oOQxa!$`mr-nLhoG2N3A&tA^wG8he{+%z)hrk023>`Te2U(5QkQ#=R7FKmx z?SG=G*FpV~8?f4cu8sKq!gAB&qTJgME6-WrN~`(6{c7^{=wYx{4cIV{49RLAd>BRi z6ny7P)F#03R_WBDc38muc)$o!=a45J^oL=`>BOrDGJy)7AT!{q2jcymhNvI|qoIzP zD@@hgAks19#KIZ;rK@J8()*8KGv)WE-#L;wJ8ptcg7wXR_k62aQ0LQ9c=}tfe{eUo z{<=;h9r?gi|N0k}{%Uas{mmEk+F$%4`|)?xUw-zlJv7whM{`vk4Q{EP3DvyQ-$1HUH$45}q!wh}?7BH7J;&vR4r;#9y0cls0K?-Bwc{dakD_j$oeDl=8 zQZr#tykNck;2?H^bMJ@CTd)R1B~46W`lt1H7D=*ywiI6oQnZ!SHhJarOJW_bF@k`F zr$EGn3)K}b6t3#ZQ$%ApOQ5f0VtX7g%d4!@IT|+j@%avDQyL4w2TrGbvG&_j{ov@A z+YNqjPx(U!OvOq|N34#!95qpTglvYHx*ns$$n@+*Z2k9bP{zMpQY5zO1I>tiAJ=?| zu<{Ro{PCT(tXBSzpPF2!LlBIgHWkMWnPTvvx!Y(wH=@MLFl8VZLEQ5~ zX}&;;rPZ()hvUt%;115{4Fijj^6fRO&$0eNNJf~8Fc~Ba$+>Sy?{EcXB1m|6N%r1t z&;#XHP_ZSLjkz8eO!Zv4zo@rtk!uXgin)@184E!cD892_zdKSskZiVI2daL?Ugz1F zs(e=4QoInv137@~C@LfasdLD60~+0Qc(_&itHBj7tar9l_v4?rf$4^lQ;_Tp z+eEE@L^99m=2QMWfHA{0ha(UIXMxWh4%~0OdbF&6 zTXzfn`bdI*)4zYQpwmgyD}>2Ue5u--j7X?|`B)qO^?u*?vdzEVzncYkXZkmf>0i28 z*W)Yi`S*QWGx%N$q`d7^{`8d*`|%oBo)wlg-s}b}&|Son-P~PQrx9$Ch~v<3Sp&t3 ztOB&Q6OK3y>at=qCoCOk2b+{Wh&bSX>wT{$-aRbEl1OzuS7LUgKVwUw>+40V$3#-< zH-(AnSz>^`Fc24tPgx+gCd~E-F=JEH&L^_q%4bdpu8BqD1kpwkCZ(E9D~!-N;#(3h z{E-PV^dR#MeQ%@qZQpNkI$W>pTwhsr-L8#sWC~(a{5WiCu_p~%=9QYiQ!mVaM7AEW zC9T=svQ~lk40MpJ?65{yK9U{oln9c$9aMrql6f#A2*n>SW}M`ji$v^LVdWsD6B9-^ zqIxKw$$C(_3gcWbM=VU8uV1dj8?!`+CjpUw0YCc_73VJjYp$*!RKCBez_RoGXa9xJN|ENWd-NR)7mLMxZ>``;CzWl=8xajO^`JP}-({zA5AZo??me znQP*i!VPmPg+oKKR%^T{FASqoZx)v^4e2~auvwkwrXdq-Xxv7H32q+m&05qTsJ)db zg^-Ad6O%%W;G^RJv8QQlNmy4W7wL~}6z411Q3l!X$Jmwb5KJA=(^J>|eAlUEKiUkt zI^XlIZmKU+*9d+4F2##~qrZ5=SKCRxzJiXge{&rDr6+W=mG4^2@XxxbwGF+t^9%4+a8 zXWc`h5LlF^SqgyDehmcy13X{p3+Txl5JYJRjbpF|p)RL?&ot*b!YucqXgHMr%jZfR z{|t9^CHay7R&mZYz`NuB;z@@I6vw|D_NzzW-+6S0ALK6&Y*+J&UPd@I$J8I9G>qmU z@DlZLg|Xspbs0Dz5(9g=l@?gY+B!y5R*!@C?cx*6=xW2NK8 z)~4g<(bC7 zcj-mpz2fAOq7r)>7l^AqCp-0L@ZzWvC{SND!+z#}D@djzU$9?1|9#H-x#shiPqgfh z?}Pq$0n#s?$UmCq=r1n@{c;AiuOK^U5^RK;g}Er5IN7|UzBP01g608Je$k9Kw(@D# z5F>>4UXuhSQrftsZ!X%_Y!U|*reW9d5rw9MfrX|}7xP*#KVWV|)V03x-fVRL$diR7 zqyvqAeDD!xm?x(4SRkI->RKGx{=1Y{bmx+78RYa&uX;qD&1So*_lxwng_% zxdy-;e2y6?ePq`vz5ZvGt~hh})j85D{HcpG7vXq`*0Y@j96ve1RFwZOYhTu$x{h@H zgSfCsoW!{-$+9g^AP<1KBPPgTz)74$z_0&*o{|m8K7G319=?n2lN}()8cH>+wTecP zOZsh%h!O!&g2D@6J$lEW)ZY_NjWKLH`YFj9*7o@i<^p;1m}8m3FZ<1Q#g^RiCpJxx z6Np^zf4n1RJ;$F9ejeI-9TN_Dmmx;PI8fJ5(eI1TSB^O>t$%96E9C-BdFGL_rJ@Ue zHP`R-$6c|8`R5y`DyAyAt4_cW!$K~feqwTmW>yA5M!v%)pvnXnD$XFMTYJyQ=^+Js zF;RR{$U9$3^};|+psI)~c!r66BF!dJ$_F{z8)AKdEtg2dB7*O^$){Y&03<$>SU8tt z6sP5c?#v5SVnVY|pNW2*qp9%@?*)B-SEL01td|^how~0B&s|j&*LvfS@4WwAYrf^h zPWs-R{qo(fHtWXayVgB_aks8(eZ7-!J`zz&aG-EF=I-|UC2}LEYF|Sk$!cGCo05X2{Pr}i> z-MV})=7_I@wP!vbsR|~s$zS{o_Q_$LhBel}7-}oLg5r7|wEo6M&`+uJkr`siSgy4Y zQpAX8?$ep2+_ok*v@ul`rO!@(>PU;!-$3%Z^&zhWh^OZj6ak28MHJhh%`=S1c*1%f zqJR~E`^UV;1IMb`-MDbLrghchnofOe#7^v57wb9pf(uZ^DVC-C_|*_NeP?EBu8HIk z1B^uqep$1%4;UAAOCl|XxG0eM=;4I9>B6PdzJQ44cS1}Y!94%WvZ#iCIZ-m9wFnWH z>2Fz}IT9+yycNbSRQomG#uSz<9VtK+6ym~w%Nsc8!P!DSz~Gl_dvSYSmfh}D^B1;D zbJ$_oKDb0Mikx{7BPN*o8Rx`b-Z27g0^D19Z`D%SYly}xo=f_QTp)@^ zl3~tJV>Af4a@DGylbXd>or4|+FyP(YQ%PSbED@s)veg(M=TsrT$tSTf^;6RW*$Jw- zn7h^ePSGuCcLc`qHcf?}OXcU4jHSvaCyRg~2&wIwZ7ty<*Il%Knt$%}{`=7wwKSCx z0jI$m7I~C&)LPLs?;no#9~|$&=f3|{x4Y?k@UxE94@p)z^rI9k=O6Wf14Z6U975hFHgDPG5OWfTz6uT+^KuE_k

VW*l?reX2I9#WTr~)Re1(BWWo*N%ju*pwS`yU>A-e8U=Oe>eFxbj8JN%mPd@}Yng zn-Ix8s+wTxu%OomX)-Db3bF{|Tm^_t$9jh|7vB^QBl_uoPK)EOe~tggKd;5LCjN_C zrTv@%w?EkT^I+3=g?Ry1d59gyajEAb#M%>4Yf%6EdAw7T!i&IMS+b;d;L7HE2_{1f z=|6iO|01?D?Unlz1qM6;`4lih*z^F@clGl)Ls+R=TYK)q3mf%8%C5_l9N&l+$7m{xGoNr zEoF{5Jyn9y?>45cn`(}aSUN0=GpIpL)0k1o7`+pUg5vRkM+8_vQa*}t*AYP=Wsy5p z3lOoV_n#wSswN884^ykjMcnheuRGw`A?Q!bs=ls&Pds49d>Qxr_r8X)9oF`qi|cxq z$XUDZYnO5YtN1Qd7S@eJDtZ6>lASdf{Uj$u6mKPzi~w`#-hL;dn0QdFqmwi0F6BMu zK`AIQt)E<_ieMy7zbK#p^y|$yWA>N zuEX9Xmlp=7a=}RVLGH~UC*2o2bw?`L0kV#Pz`HshjW9>;5J=Jldoc zFxK~3-P@I4amP;o`!cLy2FF12p|#(7U3o&B zJYy1Bf*t*a4$(Z^4Qhd-%{>^xoJFL5F)ipt;IBs8h^>7c*-QM^4 ziTln>|9e=U@$WU@+0Pm@^e<~*m|wJiP%lv(zXYlYf>W1A^=C{)l>Iw%9-?Ah7W+ls z`P6|i$80)w>~VI+SWOx1YzT!G-5B^;L#%UVKUkEcn`iR(J-cDq?r%=S?M%7@L9fqpEApLH0Ur7_vr+v$$YebQk)X=glF&WGX#P!qxY z0BN}x?yleGq-K3Ccg)Y0ljP!Bv!DH1lRcM%wnJZQ>T3pPwp%GqJu!~F^tx8o8&T$z z~Fuv^xo7%+5QZIRY zWBGQBFd8Dd6Ezh0h2<>n@fE&9po*OJ;_xcq2oah?Y1?@Y(;szI2UTu$a z(YEav_dbY6&T(%u`n=n5h{4cz+zvR>8>WSyH6~@j&($&5?T;a~n0JwX{)SdZ0y2cl zDW$vW0UwMmyVA=`1*+wQR>uuGwMu;XP`-=1xQnECnZTN*Lb!Yz?*nEsDjT*J;T%*G z5@t3*7H^42-L(Jg!c|2#+C99{CLU2LkAPV4e4Nu=^%$Zy&O^8I^75o{E!(Q4&VxgD zRX9CY_Da1uQzI+?zI2c}Qx+s0APG8Dph zQtA5+2q;Xv-^nNZ5BG6%``j~A0&B(5(m|mtc`h*zA{v+NEX7Xoa?89Y_~a;Y0u1?$rymgW44E$;isF9XORHKbHw)qt4hp$=IPlK&dmNGvPzhczJDnqdiS;vW6l{VK0og4TIabC z2lQN#i-2dm7q58Kq>tnJdY;yIEN?#ZO-FF6b0R~`RQ;-540@EfwoN<(d5QKZY~5qM zdZ}YVFRssI%baWIn<$9BT#*k`J6sVWzwZY~TySJ{12K;6_kxBZIK)+4A~>+FmV5oOS} zy_@OiXIKpDy7;BFaLe)iXea8p?>_mh>o@2jYJcEb`#cGMIIH-%Z@~Xa_vAw-9oe1& z)zNT&YaSi37cUs{$meNDA9#^-wKyX0j724@5~6Ge>Uhi>k`)$tdOyKoRd+t9;)N>A z&2_FT<2(T|f!b~faK_G6hQA!F|q2;$k5{F?aLz+u1*6?P`anI_sr`;* z6M(euDr#P;$2ji&erg*Ri>$G(OLV-J#-}0YaI-hn%?^GO;fGB(bJ5?;+l~tEda}@5 zvrjlhfmEJ*szP_$1V$T;B=X+ziv+xX85av4;PyTD@renoD~KAB*H{;GWV`C_EUE%z zu_A&8k<->7*(TU^^j%&?`Sc;y*a~nBSXe=rmZ5v9xR+Ibd7et&V||^DrN8Xt)y}6Z zvQYa2bAH`q%P6GwVRA;ziSXTljv1{#lA?KYWxH^dTn3je#{9@9+Glz+rqXGD;G;yz z(_6j<@5SLhM4zO$TPG&ICFV~H>O%FNor*aKUQf0vfOpIj=cCMch#l4)9KaHGI}b%e zHqteapPyh3?+NHZ!7J#5-rvSKa`0d3Up^7(m6;mJ65 zjswbpvLLm%`qJiSDTTYB&B8r@!7`LBsljtWS|OR@XWAf4&55hRmu15!6v6iDUk~@= z5L^4t%ei_26+bK$-zC(F0BL+sQo6MsIQrL8=Z&2X!WZmyD)c=65i7~TF4ljEjgGjF zh|}NgL!6F2_BL?_Ey{krdL6=y9fR%WC&!?$Wo5xobip`&k2dfVGc6T=r%M^6?y1^N zGt|}0?Nz?bi^;AH)QIQ(rNqwfRkj=RsLADxXHL-w#imSe^&$i<#vIRc|k$;6dmX zAw7S3>}*k2m9HGUF_m3N6c6KqG0gLlbV*{-qQ*@^6qB)NO}GSwCrfYQ6^xBrlgM&4 zbGVa0d*b-K?$nsd=o8Obg4(wNc>q;<%_}B(aW~lSf#0pM`NcJVj$#6F+LbfdnB!HK z1cVh**#&glJaG_YS#V0|N%+EA7u0m8UPnl3J%zFjDAw^rYfkIJC9-m-I}vuuB94e- znpLliR0}@0TvD_jjKD`G-;SX^M`lYm7t;}J0;M3eXilDN^}KpJ7X00MGRzxhJ$J3o z5e_^Ce8wKbv>SAPF6$usy?+slEAEE8aLs9TNR;i+ExeLilBiC+P<7#gN=mbOMrHSr z*6|nAmFI7r(W~+B`u+O#8dQJS*iji#DI>ZQBh-i^X)#KmB}8izwC|IX<-)wPNjCm-jVU~Z zsJksp;%qK=G-oT27+~bOyQoYcB$UN@B|_>+i-c7Xa`>d)o6QV z7~JWQpGOLR4X*vC_9`##6RFp3Hrz))xnIm1?g~?>c;TvUrtbpf8v5RwKEy`T#S7#m zy%gkAoYuU5-#molNNyep0Yyf*erIMy&`^@LF4V$LCLP{CmlAc(GjXq9LoSfcOKl8W zP833wJ%Dc=XFKub#&JFs{hs=@-ykn)t>Fs4a=6!jT;R5%_NiKE-ShKo;!GbRqq*KE zKsKs*bH(>_3f>rAL~m>R`?hsuw{TUQrnI8~Jc7D#^Gx|Ja5O}eX3FB5$I89h(oEe4 zz$o33iEfQ8!7`q0T!$IzSS|2ju(o)Pf5y1I@wpGWmVU#c{f_&uxRVn3|CgJ_G2)Uh z@hhNzPUSVPur2!-k;c0p-QwyJOFy5;D5bl2M^X_Hm4>@>`$c8x(ZgyT$-<*2;qpry zLM}EOxaXwtBSN7Z)q2~qTLq6_9UzWb8KXnSY669Dn2D^6#q$$Tuud*DxXG^MpVV zt8=-}f%6~(2Wm84nvb)^`8`QK$36~~pqSfqlZ()}{OcS?@`fQB@+vz2bAR8VcX!qSvpF(e&D&-{#``@vPzNSDh7qj&-kj zRdZ;CL(GjS+6z}jK~?EyZ_q|3mM-7SFXY9$Q0!v$F}e_2f|OGx_!E>Nb>w5XJr&*i zV>f})zMh>^3b(UE=F0SQ(+sJSPHIXroTGq1e~wUB#cR_SiuGPNxruy>a=~$ZgkA$> zMpUrNQZkoZ&&|pDIX8mD#shPIItX@H?*&?TMAuM;IU9UeT}QbLI45N}?&&a0RFMOJZJ1wB^RPra zLdj)}_}D%^e_+l*;q%&#JDXmh5PJmIuVN_PmcWp{k^`u^)+I z{$=bSyMko) zp&Cy1>3oLwC72Y>_JxY(jb7|1-mOI_8V*~lGi4v!z{))elsG(p@N4~tW^aho3X)v7 zAxMSsn;V`@mt=7`5p5SSb4Nr$un;}eOHspAH)JA{7;ymKm~Wl_f52l9N8IzY#52&K z{lR<=uS}n&Y@+jwY4t`_Oc$nI;@%;3JzXg7kaM%Z9E(p(heGi&18Xt^H+Ljcjxt6j zc)x$H9a$pJIiR_JI$bdiOg38Eo#xD@qCvelN?_bXN|uy93r|+u1L+U{#ePx+FkqJQ z#sC$D>;?eb-Nu~>JO}vPb&&sWIFA`vBnEqGd;)3I?Pp5We~15gN?RW>Ddg1u_aae0 z7oWA6)9kr%1af)Hh>FIJ#V{`^zv4nR3FQHUKCk(xJR3}Z3lhXboXS2Yd8vN5);+gp z`_u2I$9~62lcu``_U{Aj`FZc~?)qPW-}7 zndI*N9{X>wjj?QrZ&~a7#4?cJ;JOYa*FV93is8_geOLCgu1uFHY zuHaLUBD|DjQgipKSu@A^Y^FQZ36&m6`NCI!JX!B!q3;zBLucQC7o2DG%ES$q%LmJ;O%PLa??f3OS0ODvm-5uH315!Ra&;e07XE$zqXW=k(3Jd5nk5J zy@+=PJcDg~ARg4jtxKn#kEviX+1U=MTi?OlL(MV}pEChvDrVIo61E4T6ilMVz^8P8 zxlS5$f2b{;7np%_fE6Q-1pg0vITI7eVd@x9;%c8TFFM(bcF1oqBm_Wg{2Qj`1%k#+ zf%a4L-qx_Jx3^4`+u+1%n8oXm*sttVyc%ES1hSS;*?uwFc z&5))sB{{##igAPY$ja;*|E!)VxF1>5+=W<^>Tv)7Zyo&D~QjCI6Fcz zHix%%YP=``vZ50x@qS7OxZbw#M>cY5e?^zLScg|8C9l^D#PO?gI1g zhSA~oo&`wMaRthH9%06M`y}BvUjc~f>q#-jK~SQ!WLxu_{4zwu#+Ch zf@9Ay<3qV1-X=`=OWc1WBA@;H_H|pfSRD2p5zwD_JU3unI4`DIAU zZ!9PwJfT4Dsphz5S|eIWm&kdie_VIwJPGpJ(s6L12+McXJ z2Rh#s7I~|IINuLTjjtbF3?kVZd|41;vZD8~FfWYz<@a54f3+EITK&mGd9Zy)v;pMU ze8)yDWsdrGC=*OYi?N?MkyStycSJM&aFj`WU@wvCmJZSm0ddU5^EC35e;}w`3nd9;}h$VAEnm% z=wBo6qKrtA=L9;B>c<>GEDQFE}Q@twSlf9fFqxX^YnD>Q%j z@7{x)4iwUS%p{|aP)20>G&a5qa~0h^>*vf4-rGglAX!X-d&FEv#@BQ&pzt(?MzJ{Y zwxe$$A|i{o`n|Fz$fF18ySujl(x*K(aRst5K4j>?x&kTy9f;xHf5CksFc0YT{HU8g z(f-%Z+W-})Pi@K1@tHrV7k(B?kH!*~YU`bvE$$pxb}PP(&-^?cMKpf#GFE$O+f3?i z$Ln^$Hmd-7JWY-Ge}%aX=$Ysm#N`jYfZor@#gFdCAGnQybNzEi-p?2?D&?zxX)X~_ z-(k_dxdLOBILsR0Zd!8x(wb3veNEzP(YyKZNr5@!3@xwOYMR=w^wc`EehXJ%p{~CS zX`FgHbK}eQ1$2{aS&J~??&@S*vKn2*sl@1FPl!*{yvf8U|!ovy15eiZ5n@I$_D zQKu!OY5SH@w&Bh*@N?Dh+U_R8LVX7h?VIl~BcL&PxeqcA``QTg8RquXxyT2W za^c9DDeLjN_sF0(rC*?4I{>?#J}x)|#A**{s4C*#m--daa;N-xTv&su_Y z4b({d%X>pl#Q`_hyPx;=^%Ye@OT$xNf~Pk}nNZo)J17E)$UYP03ioJ2e9*XMor zIfUJ4L*Z$T98O`MBZ(ZEJx>S^XdN>~s$jV-GncT_&uarO=`N{X)LF#0p_oau>RULX5Fk0-_j4+f)Q#6Od|>T~z4S%^b1HQFfm1 zrA65E@p)1|O)tuv+so6eziG?< zf10!M7j{+r#={Lc6^jn~XMg+JAN@IhYZ&)7jjGhmf~}o8mvp|^G2h8ff!6<0rcaA! z$Ehuz%$BdQsFH{^6Oe^(l-)mljfmKxV}6KEY#NW(>sSu2VgC75v?Z+yP|YcUf&{7Q zIT99fFItGpJQaPhgGWQ}@EN zVv}xByR+Yqu|S{MVl;0)T*9aW;v<&9b3_|&*sq)t#T<7H|M_QMp0{{rAoLlqe+e{V zGnp|#g%w405qhE_UTT|uYQ^sM1dfoz12^e^Z{E>*`i^Mc32P_xY|Uuh1dukHgqW`m zDLVnC?lqLBEr+yOC*O~#Yld|I8Ig0mfP;UsxpdZp}?ETbN|1!Vi93Nb? zD}S!bb+6-j+5KAI9rC0;)OtG2f7es;ZcnLPZcB{uc2(x>sx`ydq!NO~A^}<6v$`Am zHP-ycyHb{+^JqtDZPs~aretMLVAdt%QoC7>ahSrKNQMnULelJ*sN7E3E~x|XLAG8N zr$oi5*X8G1HW6vNB}4qK{iIigep)xw(JoiL!uX2Gx@p)B`xEmuHEIm9e`~$@FoDN! zV%pD%X?q{a4#^?dVSJdJ_$oP>Irf=65oCR$UVQA}o#aUO#SxX27L7@i850qEtbvMB zx#TD2lb3i(P%mCp%n_B3m&s&0DlorN3iK}pwuHs=1n5KF^cC}dUu&FER?aZz(sSCX zI*_QL91pQ2Cn77Q$i#+Ef1h$k2G5Rq8maFm$_UU6F!TqeMj1~)GzZRDTJ8Mra z<|gp%gs3W^N|L{hb?gl9L1zvOPleb$4r3?obPs_oU-Wq;+-pVs*SRIR z_{DWaf8Ex8Z!F9He*b==Mu+IMz#eMy&r(@5+wvDTwLQqtpXN+|e`Y)JaD0yFIEeYh zQHX_!ebiVaPsIyQ+l+u(@^h@6I_G<|g722jf9Cu9I^Wm$;a|=--qAlX71oO8(DJQU zH#>fbo*QM+jj$(17mZZ?m;KS$jaN5Yua2jT#4C{E%+I3qvVZvuD~uJ~+ICTB%>TX5 z=Qi-(pU!{g8~Y;Yf4qZ#yOYj+4h3r(y$?~cgjPFJiv{Mqan7k+TCbuiJy<`yLK*7u z){Y`4SrP*Ttk`FMHa~Nv_xNx|n{{5J7I_)RdjiPsj5x`Cxdrr%cK$Y7+;@n$&3;(Ctx1YWu8AihPp96wJX z8hv3bE|eb%f0i3x7X45<{}xeT+K%4vCwONlk{e@2KV@{n!yJeKm(<6!cpo#=eH?Uu ze10yPt45KI$*(oGy1L!JE>!d&N{5Qhhgmebe@xXDm?RKqWu6=M+9lY71Ck{vedMxSf#!Penv=?U}bv&c%UUZgI+?*Qg)%-Z&13k|Yr96>uMX zKfM}X;4v99$Q;Xfl|BetkK1^`d^P$2fy2P2_p(g3WrK`n7uo}dnq$6=ufJ{In4$lR z`V^aBf4Ij(NA&l2VLW{dWY>E+$i#7|H(-t8UiSn?Uhi(;M{9f> zzSpKdn5r1@I{liW9>QSLrix>&;@@At7h8O%``QZ+Z%ztiy|unkj|J=D{2avjaOZq4 zC+)kirbgR-p;iV3g|8<$nDj~Ugh5FHYd=W}e~%MUx=e%cVMWg%gCrN6KJyPXpdf$>0efH+8d|x#Q1NSc4A}<1yo_xofQ{3Nyqvz&v`{qbv zf3)3NXVT~EcDxzKj)-1~ybHDh*=7aRr6lZ8RMvNf+SmBJ`nlmh zShR+FVC zn)P*~>KD(F|Ej_FoD$tDsNc_b*EoRvfAnMZfBLnoTc)HRJ8?o0hb9V~ROz7}yW-JW zAl~a?Of^OUmRyTuRU4kQu|u!#XN^n$_WeabytKNDT`HMBS2ms0;uSvIoDlWpK)3a3 z6qW8mHx(~b2*)YE?g>%CQ<=3@CicXAaY=z#r`RRs{`Y6R6t17K2qV)byv#l3Y+wQ2>!<`zj2=Aane_z+B776M5cWRyfYn;^Ehg;nBr~cFL z^$^=0pN>70kWY#A3o&mVo@O8`AC!9r5g34aiU-t>t3)PBL1a)b&T0~aShVjkV$OZK zd9UwKz{d^s*J};@ zs++}rI81S$YrJrMhr3euP{#NG`ykNu+b|c#<7AkNl*sy>iuTBu)OW$Ue^H&NU#DLn z941%0V21PS&8d(flj1x(GYWfix+^OBAzwyk=^jB81rd`k5Ru*MxRnt+3zv}C=MsA{ zmhOZmi<+LCnd z#nzH(o#^snj;7_~6tQts%qQ)xFx$W;-)c0KnipGOZz~@fw*pyBe`tw)pv#y-$>6Pq zxnNIIutt= z-~;i=%rQfHieqnAN`F_D!MQD2)>xS1bjHjkH9v=EI489{hP6z?95u6}6-ZMM$&TB? z8jBKh@fZz$1fi8jl=t|e_IBsaEuLQH6|fdhi9fTz7W6NIk4ljmQ}#i6g7X^eEF9L*C!ngI zeuAnhhW>KLzGvkXU*BVNKBb&EO3aO~{ZXD`f0wF`xeO&gPML>&mrvZA%uel8Ry;Vr zmI;a7ms&&=c$DPxhjrSxw6;cBk`z2;sTY-B=qEU$^+8q3*KIZSAkN9|t(tDj81Qb( z-M4W2zHwzMvb-EmA$Evr3x_6lEQdJe4#yTSpq$g|t ze?#1OkG=J7zQMIO+&)eyVGhmeEu^g-A>Tp839FLtF)6VR66${?{@oHP&~xw=$~(ej z&DI;nJSa*cMB;5e-xGmay!=d|eVJYeHs+$Z$8@dkWZoYM_So)xH`~kPEEfG-97wns z2ZSW6)ECH+=S1MW#KP5X88}MyJfu@ce@z)xh{M79PVgu9s9#r&hrQ|JYu~ugVOcPW@9H%sO>0@!n{|rtl}V>m`VAaJw2~U zT^(L2_{Q?Tyf;2zL2G!9?y3^|QIS*W>)zd7^*t4K45!(2;;N$uoHOV2=}jLBe^1rZ z7lC}k^3|)mZ>N@?Q>br#CdrFZ#5={GgYAssGyBWDwlEjEZg%w@*`WP`+~~U>9!Yup zsoUSq**TH_+dbT(7Pt~e<7@d;f=wxO1*Ol?_t^BjZD4VMocuBbDUY*_8>P_BjaB(y z-Z%b~D(oT0z3lSVGH4h1rLKaLf235{PcK_Ob!F_bi}}4gVcy4o&dGmz2QipU?Q_b>Xy32L?cVWoxzqiTdW_gbiDY>w7_qG>73{RiPHBbt1MH0=D+={Xa0cNc?+4`k@l&$N#9}E0fjHi6 zUbmw_2mz`InA1EhDBkyoMUqZH$$KcL`tvaGm1Eh*zxkj05_^7Led%5!`PwgUd$oVI zGuZn9r*p&5EOaluAKxA0e@NX|Hs1a0!JTWT`Z-bm^Tni&#XphH07?o6%KbgqE+Lal zoXT4FbVFW)s@f@!mozRZf!24V?@x<$LhF(^s>8lx6i(bZSCSE>1S}XK+2@i|5mAXe zfTC{mmQiR}FpF3DmZL5sqlDR!S!{{4NsaxD_1s3ISH$BhNubYre-%TV(g17pId*r~ zyjFR|A>uwV;CdEscm(V#fVq}i?QDnrz%}>s#?`F;8b|IR8}J(SwKqw1W(PT03H9Z$ z&l7{fW#zi`_WQy+V?UNEa1?(`@^b3jpW3CGytiOZ+mLfd#YaMBs;X4YUYS~?bzN*E=Db~4pgn|b0vdO`vrtn5 zdjt~AD{?0}U=G6!sc^{xxbMhA0y269Q6KaAjd{EE%0->>*WxGs6Bz#qSc)F!GT6Sm zKd^dr@mJkvoY#Ljhrw>Zc=TL`9y_kJ&90i$@s0?cr@0Jwe_H?hy_g(U@iHbB$#m~9 z1WJ`I*~?d|inMmiWpeH$k8{^i_mfNgXAJ1bhiCs}lf~YeI`jHio~pAa+g${&)n0k` zuzy%X7L;OKEszgTKuH&*^$9c+_+Spj+~FZuVl84a7u#qXFJ`_-25S^UI- z$bHC6?3ar*f0u?lIkhzv$o(;0)Sa}R5u3t~lbKqjyUpuOqwLRK(m(t8s#U|TTB3jCGOO$R>OZiQy>`_x*k^U6Sni*zvlnzzXE|z_ zV16;T5)drRvBr~%X8UF*&ck6%?&eFTiri7mk1v8Df1VO^jeMI}uV7A%ui_Fbf!LfF zPAunl4_}UOSL+AV2SUDoqw&Cyrk@4K7f(u~Ox09lP(_0^I6_;pc8!PJ5!H935ThVH zP{qbqEWmnsbViJx;<`sn284<}pXiEk3`Ye^MU!jY{AL?(dc=L+Y;tK47jWzOHuPk_ z_6Ir!f3>a){e^d|bB>px9(nB_e8V&qYP39x_5P5?E=eb=B?{^bJ2|Rh9zm*eqadcn0=oA4ceB36r8!hR&L6ea3z)#c^eIT+SjH0uL1wHJ8KJhi zfBJYHqn0*mK?>jO>$^weLn0rkL{iBY<{Jue#l!zS@9vi-aUSo6-2R^&iTs9r{^2=q zgP%KVkecZi%sXr$7ffmA1+x5@LPJ45f4xrj{BcUFj|uQG6KpD)&oSylmF9lRlnC61 zK=%J`9*vi;?sdM*+;mMo{QaS*i-i!#cf`Nj#-3F8ck!&+p;9Y{EjQssK83Wirigp~ z>pb%ETKD_S_x>AwGTmqKM~m#tul?-jaE%Q$pAqMq-)YSg%<-#VwSg;dK~>kDe@|De zFn(^}3u_)B>I>ZejwyR=Yrm$Dwx7va@@4ZdcLg>$UB;_Z*+g0Tx+P4nOY-#g@WGhy zUH9<%uGb^UYcU~mc?Z~YK=C;U={N+r&oSx=SAO0(L{y0@S{k11nY)d5Q^i9UtN>Ju zjd>mHy$xwxG0iC}2^?&nfSfYbfBgPT0J|au@l3P(VxE!ZE-UPuK(lBL(EmICc>a1k zJuLTp=9=DZO<`dP7m|6fEe}D8eL`7ERl5(Nsc(eR8Jie?Cvnf3?r*zyKF7CX;SR1vWPwe-hCUn483(J~hTi zcl}=XkM9Ni-U|6Ei2u0++bY%C9BH{=qV(a+7Lb>y-;*W^i00rZf8=ayO`jw7QsDXe zXI!G<6socDYPulit51WKct=4IIxg)t9?oMa(wDl7YlI<)clK-C4*ed#Ay;-Q79;!J z8;$DS^PP%`V?2GqcFx_blZd;9wrQ(V8Evhqk5AM;taJ5u&qJ~T*R}bBU%@Ku%k2{t z7@DVRJQ2w9=TAJPf0Q>~iaqE5ac)BhYGY}P`Kep-j4eGi;Qaym{%fdf=O8AMABWx} zGp+9hRqZ0jQItNP*&3TnrHY91o*l`A(FpQQG#|=wl*WP15eGU}h)sE4_3y=c20miA zLcJ~8^X$-`B4-k9^MYY~aN}DI_x)h8?|)YA5yx~8|8z{xEBKioqtm>8+}p+(_k`|)1PYwu zHooMyR)4Q^avj&@Ppasn!u&-QNSY@Y7QIeQj| zatVSh;KfX+yQZK&Q+-Finm|gD@GJr%kRuio;VA0h{{LKep3|*CG*c?q+fuQBgSX0k zthcfS^4|11zqPo|#t-kUSR~iIm&a#t{^Yy;U~EbsZ9?Ctvh>~lXY7;ZlkG2#A@0W9pwEF~09|m=`#XMYRA2`Z_Wp4s5-BN9fFU zfBX}2lKc2{jCQ&1@P0sja#v%vvs+DS;3ij;QOM}UZQer&4e{M0pe~6wov>}eG)HP1R+hxI&KMKIT z*P1qn)4bPp9~*hAo&+>1`+s8|T;1R4+}~U9fp+?3WKEx5G3P=A?{I%QqH5vWiZ=e` z3BK>=+EzEQ0P)~E($iQxiaM@}`2_gX*x2NLVv@<_^By+8Z9h1;UTe(H9u-P>f2{Db z0q5VCuRk~(>pfM-^A$9KW^%15gWQZBKgu`k#XP#dj9%c1wPceCaxS)oFuqHdNHZiN z47R|w^O`n32_G7t6uR> zA6s1q-qTt9=nCsTlX4RcO`COw$KWeG z*!y*(3J{yo1U<%eF?CcFQ}NW=vJvRl(~WZ^N}?YYJ{Fg8OIySSn^bfGMg5?G_k9~D zG>CJCtA9D*mv0=*t0oToe^EQt*zSAexQ#V0&g$5dF@k+-k28}9D<;w}`mq)v1?58)k2H>rC)&wq2(f3)sXHfY)Je&ITQ z`&0wY`hXJT;Rb^?aU=2 z)Avyj0~Ns?yqXXI9Dnd5gRV|LuW54YL-K1b2d()aua0mmV=9WR{!WRU%qAkD42k+Z z7iqil>Xu9&+>cPLe~*!YPpRTAfuPmz#|sqdb~WF(l-_lke@G;04$#oG^gn=>5Mq5? z{B^81zv8NS_M4Bae+C_AS9C39Lb4!PdHZ+Fw`kq7yLG$ruy(3#gR?s9+dSh~duW3T znR#IUo>xlb^0@NqBay@BuWMpFoM9%9(YVBTY>fT%Oc{>#{Xycxur~=1#1{Y7d-(?} zF=|j+piGUae}TLuiQvSM6(_L5i3Xc58HCD!Znpv2%E>w`1pV9y;*nS zO40@TgZzLn*x<*=K_WFlN(e08jIdKeUdN(02`gG|X-B6z^y0!zdNS^zt9t^%Ii_yMjsxH~+~)%JSB-kq9xD0;F2VkdexR$p z^mO6GyyKizM#r_WebIBQ!76wqI_^s3z>(cfe_8N}lOH?M@M56Wr(mp|K2MIC`guSG zLu<>GaD&Ryj07Zq}56;P@dDIU%(JO1_LbDs%EAFPG@A$2cLI8~}AFj<3U2Isz?LL}xZ?xPq#f zfBXK`QaNWl4}u`DYAQv@qCV?In((7wELrUP%i!!^Yo)l~HFjt`ry#H7f8k#vCN}fZ zx6W;HNu%yKlknO9nO}{js`U^bgH1Rtvw?b7pSnNVcziGQFee7?UkIQtvk`fJP?9^@8YqeX;ge|}b`-*KEwKy$$WeFfbfA$%1`j0-u(WxXi6 zmm))5u0%FG<2MeAEC1SskM%sJ{>q1D(qLV8?&G;GTz2O%+3Yp(nX6#5o>_z0tQ?~r z%z7=wr(0g@?k{{^;fhASeN-EH&RuyjIsfkQj++bl4bA!4_(iav<;i49HUrcGqEUbSp5$>( z`J7z(gxY)z9yiY)R#p(@j2+~;f0BFfPDphq2Ie}-D+dx{%id^kavY`vacj8^mipwy zrE|*45la$c^W$_c!D-B-SL0B--pZF7}H{eZVRmosRuuu8=DsE4>qc z35xVU`X`sk&gi7R*f`m&eD&(r&>!vIEa8pSlPBN5chVaDQWp)srE-^W02nFMYsU7y zXwr)Yr*J9%I5^o$QQOOle@5BQwnxnRnU`+aU|ssuma_o!4@&Q%br-oyu|IoM;y%Xk zJ=?HsZwJxFCykD@%{~dn>%0+NJ1f|mURVdu6gWwzB4mAoLD-3?`n3JJ zX6pI4Ji*W(ckNa#hD;3(gjP9^GH>6@!i$~b5%pVgjg*eY=5O%Pe>7{(_Zn{d3nd36 z!q~?2rez3dA1H^d{dQDBCSYj|i7wCa>ykR~UJT+*>O|?gM>k`l9=wPrHdU1kkr)xB z8ocFq-}x@ecX13NZ>+!0_K(YYFf(>zA}+Dee~KqxyS4nZPgkBmD_lvJsEoK@bcFlJ z*1q@9g){~&i5TU&e;D-?al7wck$nt`as!GFS^C8zgO9;;e1nB42C0dY|EB7}qh;Td z4Ik#W9KXO)pvVsShmc*bP=5HMzjn|En}^3;A@Y{Cs$B^&X72ffgl^ zk^$<-qCE+@J`&uQB`*Ew?ONYmYBc5wX%h$-Qz>hGAg3EAe-YYKW$CsK-w!QiwCJb{ z2?revi5k7Mv|tb`Nwu_BUF1^law7T8^jX;zM2#MvTD=wWK>lyiG=7>^&*Cj9x`-1F z6^YJqi&j|6j;}5@sC$fobuh3mtOlWUq7W51O> zICTd8TU%rxf72nx4myf|fKF^OLfhJg!x)e!dH``SWT3~b!{DKX`+P{L{q_0Kb5cf> z?GR}ZHU1;OfDH+&a97?q5V!*}Qc%`JxAaPoI8kwL*%Q9`hm9?e>Drx+Qw*T;< zyaGBYI2u-$STY7Ee`D;psImG$C4X2)8_tMuTMOiF3~;|2fckvtLZAAGXU@>3Z3zP6 zlVIF>s}n;WP%49m?~yKA-%I3@U<(idMKDig_l9ai3H0N(6r3Z*N56_ziW(uPfuII^ zRU!u!Vuw5;M-#5PHbl8dWqXwXw;TNS(*tY;TPVbDbE9+0e{bb8ed%)EbK|CL{B(aV zN$%^f!EZEYukvEQb7?e2y8;v&Zsw##>Qbme_((40B20^KJfoa+mxdtFM&QDC*HxNB zhQ8ul@YVt;V>Se&MQj-sh*xVvVDKt!sRh+@!ln}ma{jWxI3II%c`ra(1FBN0$6?`V zVYA$yfpdxBj`I|>qD){W2Q%sFa&1{YpZ#s4`5q8;4g^Ffa(<<`@a zKy3w@>VJ-ZfVp<5{@R9RIbskbqtj&gDl5CSD*plgVPp2wxF+r~{*nYu{KHM3@fN## zG-r*~-L-8muQNPG#I~UTqlg`Y~oD;xj-pM6%JGW)$9kxXB=k$GcY_$QxgZ8i_R~&hFk3JC64Em zUe6bIe?GJZr}^jj@U^IeMhE`yVtbi*4C8EIujKF}+J6rqUzTyksoHS&c~#hw_IIPG&1LWYKc?4N+#B?Y96`zP6 zYod4wsdyvE2U-Ti#+Y4$e+cFz7@!R*pfEjof2-;MirY}Uyb8iICY2{n={hPB#u1}< zsuLlTger{L3rfMeTiYQY8hJf`ivb=iP?M4xJ;W>Svj7MAl?>0giGlh(*8B(7=0~43 zdds=E#9JqTvC_*Q_}tss`Y11F<-u=Q(Y73ZB>k~Z62|7zx{E~*UE6sg>g_m=VML*F zf4K@AcABmNnbj4jO@r60(G1_m` zQHa#s)I1i`N%32J@LA9MVROTOF8^)(e<%4BSCU=yim$qqjn36o=gj)a#fW~3JA6fh znZMwH{%w4yuV11Oq_G3)Q9}RGmtEJEyo*3dzm}7L@UIYgdi9gMOV?%uV|z=%n92sS z`gTd8&-fMksQwMkjJVt2Po(gN&|W4-JhM)|f~9a>tp{hU&z$o1o}sqWNrh>SfAQYP zp{V%VGN@^s1w^OenD&X-@%=_`i%8SDVqU1rF*8z35*fU zf34FF`jOZQ?$Ra5KkQJcf+5ihTK7wP43Rfh^j~07U!tPHRDr6xY#NSj$tPr5)`2QN z8Diy=>Lj8{jHlM6Z3_D&-asNIe>&O*r)&Elh6sMC_YF9G#5>+$7qMQ&ITrNZt`tnq zaR9gV{wMe3MF*e=$2aX|anm?%pY16Z&0n17+Q0ZRF1CbDsJurTK{*D8MM7Dh*99vK z`*!k4LAZ5bBc8EGufFN-kKroK;ZXBWZ$yv-o%`_yP5&HUk7Nt!P% z7)FOwKv(GnT`BJ^aL{poPro_;RX*vf%izOS2GBBf6|VE)=p4pR|I1a9gNd1 zGsTQkF?C7xJ#;YE2)IpjDHthjrUDcd zMPBw~SSAB8Kp~(WA3KlnfD81+=5{-K2>18g| z_Kg>`;r3QnzLiEV=vO`##6L9aXB3V#8!6OKLtlTFp*`e&Y>Bd)Mz6{IM)U9+%IPwQ~6~ zJ(avW;0j~Gl=3DoPcC)sMVxr=XoERE3BSx-i1=}(yEaEVYtEm&N#EsD*X63wm0RMu zf8%x2@cy59Gwyf+a6fdq!*!my>nmo<%5`eD$fuL>sLpH@!{e&E%S-EB)bc&~5@f2B z%{7nx*e!;2n^2|Dp5!=u_Q7;wbEk`TSJ7&7_JPYXsleArLJs8SSN!GOcbaCXS7iKS zc$RE9(=IV;XfH9G4?z>-ajx5BVCJLve+O>l90zBf|9RPmT=#`XKq}~vOp>Q$k{qWh4pghz~$v49m3&ZRW6bbHhS!hPbFnCa(tP_ z$a7`Nr@BLu=Le6;BV(HmrPjkVnM?vuG1mr9NC=1_le5>T&MfDchQm`Pd}h&Oehwbw`Yy}=XC&>AzlLs|4mHa23+k&lhZsu`!P z))wX!$q<)=UT-{dhi|e|76AYTY$u zx%mH_K1{o&99R5P4!>$KodWG7RN@}ol!H>=I~kqI=l3;C`iPO;whrl*NWm`}l7`B7 z)PI1s#&#Ls>q(!H^Lr<{!HJ)RGcU!TZu{Dh;N=tjx%Rl#1NQ}%edFS=vHua# zxtuR~j{BO$l67q&Aq~-2qBgkX%hKKiZ)i#f(u{y83e^2ZovPyQ)R`CLEJeZk0XCa<~~6C2}xOkZ3i z4DA$MbbTc*lPMKZf9hg>TZhrZv*b2w?8f)cnci@|NdmaP!M)2dutY5;JkL7eN{&EP zBh?ke=y!dO^X}J|!20<8`C z`4Z|IER8kzV+_7-c;7xy$9E6t7h9Y0*~AdvbXW@H-M`KQfARSoQuC#xLJKld)d)H@ zLAw;$*nQA)NFeMhnQ{W1)8oK>1UdM6mV+;-&>rctS}T_z{b&D#TD}yjQ9!hbF`U3_ z=OI<`X|Fk%gCO(o=Pi!IVDx}YV)P*4Wb%pqngtmDIkp=O)`{K3|Gg7}v5PkjRLncS zaV=tfk$Kx_e_uJ!?vnMV`ZDodgQu=tq%o60NMQ2x{`vKU{7SE%s~7YkgX+g0+VMeh z>v}Q8v-`U@=ZVF~%Bw%G1eeHpOX7r#?hoGPNsi&X&^Hxf6^Z?txE{ z|L5$Fyl}kU{3;ha(uEh{Y{Ne9lVf+B(i14bQCEKOe`bXwm7uE7Q~iL*a1g1$wsV~k z)h#@O^$$pIfI>TFl}tyE6UnoPK{ufg_TP=3pB~vHeMZg>DkT~2@bmXr1suC(uM4+@m?&DNjoR8Dxa)li7 z!Up~OZ}{pdWGW>UlCQ0UkP!apiYJU)r$jdEf9~D5!*%z}*>u5(WBjDiGBRr;SNwtM zKKl?HF&&P2)eK_^nsFyN`v~X}?c~+SM-Cez%fSgLh?StauLtJrdQ8a2CC=cfPdy#% zF=Ul%sBioSS;NgezK`2);t6;Y_jEoVh|9W)9Ij1YU1o%-fx)%33N3%c@sq$|?~)Rk ze{za-Ed7$n)Fs-c^Lp|)Cb-rGbE?nSkzIO?>2FPhi@x7Sjr>zM1~*QtNIVmFlkUF8 zqp9+g(W$2QxR?PL-uQo)$I|${3CWtd<2=8@SLvA8Uv0;&{X>}v6p=&j*(<&Wxs|9t zdTc=+ILKJfbz*}`37#`9b1{5w{td46e{R8*BNWD-NXdwl7oN&9KI-N1%*XPIaclpE zU!Yl^fAVfag6C4;E1Zvhj}uzHJVQn!l`_<8yz#JG9G0BE$-m)RFE^Acefi#K^ig&0 zZ+p}S@*fcQJCtO8UT3C;15wG#d9%E(X@!h~Tl_itFV6A0KVs)EXRVIS#2xQ-e|pgb z`uu^nFuFLXW8=*^l!D38OCN2Ukoa8AQFlE@u3ONe7RTd9ROr(v5&btC=WDe4xkXfL zPVi~|GIiI|#_b<|*vdtBaf+dp-wQF2j=A4YeLAh8s6b>%aN3JuEYZXQl zokC)jUd$yo_~VamSQgQ_ts^p-VrBCH$Y&b9*yDE{P$o3wrR- z{pig}#C;bA+tvrCQ}Vy9NxzSe=qrG?xXAh zPDVrIpJ_Zqn}gqQ8{z&W7bB2}vG!}rG3P2g)1dZ}PE=oY>HELq-;9J$t1}l{f13s*m({^Hd)E2X*uB_$B#2u9Lr=-{Q?bs|$=X z-tg~i3^HX7e1c|{_e({vWbk~3OZ+weB7RiygT zi_;*~;zc@nLI40=K%&3XDwVi(#8GihiIs3(X3)eflxRDJ7|+ zbjsW~`!PQdD4%$-*-el~Oe%MWc2||Wn)%>p;qw*Q1P;{6=Xs8}Z}(^9Em87c@c~2q z6T@_m_e&lxd_#X|*H_ea`+u8x_$T*Jo;ZzP()g~e+q`ei?EhrmE}}@vz)t5#KTA|K zU`f+n-ZHwiD^Ew9mTSf`I!egh+#T{uf8jY_f>It)M;Bz^`IG1}=MTcAYWfP3+n9h0 z(tjno9K%DOneqAh^$d{+i8{rdkCU0dO?xSC4SM5JSnrXej6^@%6n~VOUfOKL_?YO; zm_plr1?dhWvGv->UvSq~YFr#E#x83JQRjYAU%0Qmg38KoePjcWA3fx}rE>goGm;OPAE%`NG_dI`|^*Re#CR)2hiWcb1BY)o=hXB@(Kt1m#P!(zF zriW|AJL)_|7rnN5HH1=8B9%|2|MH^608|yzQ3r9rs5>1))Mahtty>_j5jEIHF<|T| z#+PP5$;V^)ml|Ww0?!E^K?pSnn-M|V5_M0sJzJmSP`|JH`xxu%m-u8{GaK85g4bq@ z|EoQRHzO(#41cbBd_grQMt2%9Zb#t%@>~P%GoaB*r-m~D$xjU)qvFk8|L@K-`g$Jg z;C_h%92Q#RmKOk^I-%NQXc2 zibu(hn15W_U@GxxE#5lzsV--;EjJ%9Ffja13>f2QX}cu2KO{N?H4$<%ES+ep+Hm)F>32HE%$ z@9d8{58StAX;ge|r6u7tJ^0A23@U^oD8~G(r+?Rb9B;jH$Xd^yzZ4FUDlqpG9l|@> za*=04aGc3T$7%{v($n%5=WNT&cTRm&8sV)}{5Kk3=6u;v&G|AwaX$A~9EH(xC~S7) z$iCN^cJKU2;RW-Yi~mlS_9*{LUE9mp(c7&yz5ttyMf)Qb0h4M(WVK5c5u2RQcD1p? z)qjk&ZN8&ASPwu36AN-TdU1sA?mfuHL6IFni-0NV4&y4}QUrunHGEo?CsVwCAo+i) zf1A=59r)zPHy;J+uuFpbnP9CpTq@5gMKwMmju6M2#gSi*m;FlSqnFi9Bl=*=O{3p@ zl8b@TQ=bFIs^D4otm`}vf9m8KJ>0+6$$!1tdAiNMmufUp1wZ2pjE1vy(Z#(#pP)bC z;;X%BD#-tD`^BRY*e7<7j%|7F6BIdRa12+w;UQiHGWDpIj}yzEByoj2L=XS(_lbE; zbryMp?g7y0hW%k$Za9wnQdGm(LQR6Hi388wk2|-WEG37#PxpaR!I-h?Y>IK~PEOS$|59t7jE8~f80_mMepY2WJ_6*L^kjjicdef3Kp>D=P# zf74&T#&KVG2h6;J9C}xtkL$eD>>>1x%cSw0EjARaz}sqvAJJcw)ZwdMGSCCA1K&IC z%C9kZsH%9lTF~|c`;(+2Gk<@W{;)~%E*+sg4|Fc%=36W;edfs@!Dc#QxIq&3fa?=E z2DN*_cr-mEaKr$Xx!Vptn6?s(twkL^$kU1zR(YAHXqi|vPkKX@#0cygGx;CSMXhZY zqwCInV$vsw&yy1>CH4o@a({w})Kr|*hYK5wZn|l_FLRGV)wUz+&40}`;@ifg*ry~4 zu)i#@j|NHLLlJl`>37fl@Vs7;q*a%VUMcNDsNmszKIsgVc>Iqz?*DYApZrH$8@IE& z>e2{#n&uTbb^a;0#>YdH|2(@lc-T?P!}Iw!2VX+_4)^#gANWlh;mT2CVl1t*U$^1Z zsMGjn!$_~1#ec~;|9`9^keL!};T!Imvg(MVPF%75U>Ljcd)aA6Vwt{LuTk#U?MF&Y zJs^Jqa#*Y3z!49eh%skRIpRv17|G9n~-ET7O^idxRWV8g1d?%Uii) zCR>iZQq!+(8p8Wa=TqUC1F?DB@Q&C{TT)WJQTB4J)3ct3r)=xBl?Nd?bz7+SgZ3xF zKsP!Gc!n@~36GH$lxl_Zhz<C{)Ot)Ag^EB?Zj)6Ivi=*#dS6VI-TlY=|8uFdh>Wa@u4hmk!iCZ?ra1N7v!k`yeI*NE4!hTu4r~$T7DLS2nj2 zuh?Kr`F}thb;wEJ5*3X3)A7YcjZhs&|E_>_d+UQx4byX*ALExAXoAON51OIf_ zUt22sBRXyGU0qJ8a%1Fc8Ee}$F|kdo3&IwiPk%S17@mpxitq-@Nv+V=Pk;?tE$|bV zi?cq(MH4w2gr_dMj?43R7oF)l9QAcSc4?Dl{NL@{PZwUMtT?wV&IQC#!5WmcN4$?m zCyw3AhE9~0RE12k;GIC41bN>F6-XPToSw2Zk7NdFLk{-S!ilmql`|rb{mDQjc`Oy< zd4Fm%2gq)0jcIudsZMO?m>pQBbm(ytP#IKS8CncVoEsUF$pB+jSdn%{l(&bB+eF#j z5wk+a=#OlwVC;GW(8#?gcw9YQ)}v?Ltj7Z&?mzS^n|e9hTPElI$K=A(aM$14VSMI) z9VfZ58LGf&&?eu#kFNn05CezfUEB)t^nVnm7Y%*n&<&771vC#iv>y8E@;7kMUku;9 zqv5oby@$T+{(GA9WAv%&JsgDxI&r*04;=OWW^~ePy(3V0P+OJZjQ+y4`-k1{70HVK zP5<(LR!6JAMV&8K@^_od%opyFT)tw5y$~of*gH9d^o$!TL8n>o!xO!S&S1V_00dE;|rs zjy;}jLE2UZnNN{ZI1gSpbKeGvkC=>F?{W9=m8PGDpL1J(JNeGp(5_9?#yKtElSYQO zHbkz1SlN97721gVQ4EvR?i<{Wvww0|UcC8UO#*J>CdL~&0;hLCei$vH_mi`OZ|575 zY-DsXPg$p0!pu`$%4DC9*xJdB{TY2F869)$rgQx%=EIyzrn+ti1Zg-JP)~!C=@M<` z$bFp{AGDpQH6!4xly`y&&h3d{`t_l+Jb19W@LeO0Dn%UC6rE&byhFv2h7aKmQkc)~^$bOyX+Cq#6+A zZYJx4W3U7eu2bMoSf@}Y>wnkDhQCQI337GsQ}U@&hsi8#?c2@{Y>f4XxYdhI{VFms z^~s5!nKyAP)e}~1F@`?{~$7g-^ z#|=5iga+zA(_LRmQBUnc<2C<&UIS9rn%{P5zG~;7lM%?O9eVTw-a3v`(BTvOXWHuq z?&cM%#9VvRj}35-DMcaS+@}^JQc1!~`tkY{#QHk#JmS2QDbhQ~k>2C^5PQ(Mmv~bK zE`P@%5vX6_km!B<_w&bAuF(Iaoz0*wZK5l|luclYJa)KeuN=umC+vK^fW|iZu*Tr6 z@**>H)z0g`In|r;mC=9?EtCUAZRRXJ?1SPOIiql&Du2j>+DP#{)Pcj{Xu&(ik6!fPk9cN8e4bCV zTRW#s#GyNcZ=euK3Q;YFy5Ye&U^|{PylL`s<-uv@`n9;6udeg|6{q=l;Y~qKo8W5C zJ=gzFUX|Co&o}+cU(aDzoUY+oI2n9s9gs85Zh5bJ8Jrdg+olCM2FR<%2Jqs{B7f;O zXSMt|m!+hTP;v}>dQ1X__$?Xug!Dh=9Xed0H_MumWEE{uC_^T(hHxz*!XkjCt z7S{=C8}$inJVLXLh(kA4gLyC((;$av+J!OES-hBlGgsX=Vs#@=Z9Gt!?Kl%Iitf^} zYqVEPg+^^aAxU1&K;j-cwQ-$!_0Kou03e4a1%)hj^BFY*$TPE8oq_t zvMB6veL8DrmWvHT5=X>~>^6;E%h_CX8=nDrWiRvE1zT|5f7@s7&t`1i+Vp&;`f+al z_45yV#+f_y>dzjZH8$(uj02b@o7aHwWB)VAi#gMeA6vo+Y1;?G|5NIYh<~j4+Y}0+ zrA=}{!Z=wIs7AACxSB~TM5e~gvkckX(V`GGk;#AoZ|_xiXhk;R?sm@PvPhx=T*_6tM>^!q}F8j-b&!_9$P*$!3USReVm`OnV zlS}$)wwP${W#vX=Tqa{2HMh@pKlMqLs)N`-P<`)13wvZIs0}&&P<;-wOFK0hsE*23 z5#qd+8hC0mB?eEAdtcWdOXQecoV8Lb7Z$B|P-(>BY)6^szzQUet0*i>V`quSiHGPn zT<;F5Ij3dt*ZjlR_Lm=51`h$!moHZa7k|^^rgTXELkE1svYj`2_}rrA=HMvDI1ni4 zn2%M%BNJo1W+_R&wCy33<{APk*v~`HOqo)udyMNP=rn zcGd}M>`=iNY~+9?akahh$50(iT>V5ZW68un95)01lsH7<{8w&4appwFVw6M7kALuR z(`Ie#SBT3ax{1$zZQ2KUZ0m3-{^i_fpMu|Fkgx6bhxO`*PHn@>*^IZdO?cwjCy0U% z_axG_zSs;G(Olu2{S|~YU)U$>=u^vXY=MG^!cHb&jcQKd%abG4dY<~Rb_n7#6fZ>b z^jiE$|a4b2fAOe+AugDs^poQ%w zM<1)~wc%Os4`7$j49+17pT+g*OI36A^gq4DG0&r5f_ z-9LG_<6=@R!nisGFTHovMDt>anu(m}Rc4dK^lMQBQAC^_Q$0thG$a~vk$)KHSJ*HJ1kz9p5WXe98m302GoY^%Tv7J$jiUh4svjx&{$UaL~%;d zmZxk9l2`tC=Kk)ZKs&ezB?XWY8B3k!3ox=O4j8>)5ULx%lYg57-KIVs444$5R!=cI!szozLq6TagUI{jkJb3?D7SW0 zP7ZncVy|Jki{_Di?7qH6AF1-o(^_0-amRT-HI1S?hC83 zXxsK@UDd6}fNiv5%nPJGPMPx=d}Q_Lrr+?LUxTv0A*={#{RK3PnJ`F}+*+aeT)uUs- z^~xS9NkYgtD`hkxQRz4`TTr_cE&2IGzqy~v_7_`QBNdT0e@)`=X!vFO*0hN7h4Ycj z^XRl6ERuH}vza*X@*_@Nx68jsykGe<{@opmN$6S~f4mHbn}4!zVFWOr5S9$vk-g%* zWNJtG%GoB_Q}o;xqojC>Uv0OSe9hwTwx1`pJm>f3NP7~sUq_QwIMsGm zP9E%Zy(;PSbuv3nHpz6mm8 z0GECY1n~k;z&b$0s|Qrz^CT1+2GW}Yh30yX`57UZDt}(b=6%|m14*fmBN}gke*mah zr9t8EJtZ<=;0``oCmLDja;5;I`4aOt#OE_kxP}0E$^(p}p&Fmpg|F{cUpf6&OJIIG z@Yx4f4=mt-hqh;|giwz*z!ZDN^yWHx1*Z5!6I0Ja%oGjmxrC?Y{=h;$ix+?nUUP}} zyIB6N$$t+xIg+AmBz(aehi|;5B|g7%dtb(+B;)2fBwz=>+&B3tuJ7min!h0=j~Fo? zDq`AW2%^ID=KXIL%uxf|mU#Vy`FX|mV5Yu1Sl=Py-B<=4MdlFATRm`4$CbPwZ^0MF zpJRJu`1?*%9{m07I@H(emY?B$xUY}PeSb2>>jdxHL38~fAy#+SdIRzG{@w4X z{=@I@?wj&9_a|_DUh>QPpy@A|pYmEicjY;k&s_q`n)>RL=DO7{Y1(SJut&W_KW_EQ zKbD8s*||MR96uc#525DM^Ycx9OHA*Y{I_rKSM&kM7{1l{dFS$cRe< ztq0xohp!3(Y3@U!$r%5}=C8b7Pr&d%H~Wu=?}tE=HyU*w+uyALjmzfyU;m9tKy5`- z)+Km8}Htww3CSx3zNs6m$*UIr3>Q$==2I`}2&0Tm`)?K`kXP4m$LTA;$1k3^j1 zLBYEs;dx1&Sj?*Yvv8Dn3-Vpz9K}4#54F=0naof1U@jI`-lc;=C|;0Qgd!J(O~q8? zR^b++EtXax3)c~)D&oQ&iB(WcYBv+>vY1o%BtEl3#hx#|pjg*lF1DLuS9@)Lu~Wr2 z^=(mQMVtC_ao|NS4wi19R*WXnjpWLqliGdbtE!$px^ru#>8t0eEo47;YhH_S zn#edVhSo!0I=U47G6aI?*t#M#)b;p@47{(AR5t!*A} zC*CgE9^;+gg}Guwn6_H2R=3q+t=_2h(R%5%I-^$b-Rrj|y_fjIDqH-!KU}oho#W|) zEPB1p_wv0TJ{O(s;C;P+PCH*8@2mausI%$47yD-*&8 zY0@cs@A3Z0?Uc)RxqpIx&T{nbFS^@K@$v5NyQ@y&zl%lpqmzHX^Lzg~1yXVF@87M)q=?KOkN;o-6OJwJqp_G7y<55h`4 z9(!|VQN2Fu-dj8T96Zjyr~SpO)m{&#)0f$}y?mW|FX^Wkc2$pmvdwvX{r~@N^Ogpx zRRv@+<&&A91UGfVBsplZ3g4zaSWXW%(E%ps7+yRY!yw1n#3DaHWjj_o)B9N@!=RivZ_F?lS!@;vd@#%Di!TApT?*< z6Q&jLol`qn+U2$F{5cGKA6!`2Y~mcS@k5SEMmrkL%3q&*<14+F1w z)HK$rDd!VQdy0c2lzFaCAhlQP3hZUd<8wJTRi(-y@X(gUCa*9?h&TO<*7YC8g02K5 zw(x;?^-u#JYCInO<^_zxipbj7*?QWLo93uG$ zs#qg}l`mxx>?{n(NX(&hKDj*EUh}`iJ6cbuSK;6XC5T)23m^(qR2M75`vHobW*{Grj^NPS$ z;Fy^5+jWHfm$1}7Kckp5-%r+>S>XW7A3ahH>)-p1@L_1i9gqV1A-0Wl$jtP^M`|#C zq{50dBts4C8BtB-+eB};RE->dcV*8W3H*2JIKYANWmhKV&7v5n)l@+n5Z zlgqfArhK1FX`uWakSxMErrcN5stjtvaZR08bf&Psh#C$R*tJvhd>TKZa1K<^-&GQ5 zG>OuR;0}%06?CyaLV^8*BkMKRO_Dmmf#Z6#`BGIRh1a9LcdQEYthhb`1uS2e9AyR+ z5b}M_oBLS5NpruhkEAg7HZ zgS|T+#<+YCr{Wj_pLhk|S1hdr(<-X}zWg|Mq+_+OSXTLY*7xVo#Wuov{@Zl|jbx9G zh9q1L>7yQEA5t)NV>k%@uEDH_Vil)bFRH@<&eu~%%*{F)hqPKM0#TA+yAF$519{W7 zS#Kq-iL2%}&XED373`V!IQ4)0B2a?Wo_SyW`)>q&8VD;};@*Pb+#xl{9r&`YkS$N* zK1SMhT4||B4D(9MHHp4vvNY?aXVzvbTSj%53u~q-m@!;O|Io)2uFaQy4B?@g(V-?u zn9s0&A!2ECz-wjJv^mf})O{{$e2!cTYnBg5wXEQaMLVGG96Wi>M@oOGCgt#t^KFXr z3Fe~*4$kcmA2f+4d1dw$?w|S$ch|hx=QwhS>tyOga!UDJ)b{L0e(Z8zF+Vue=od@1H1jO$ z|IBw8D8D?ny+biNZ~ezQtZ#i#QoR)P$-#ae?ci0mbLq6**_1+m8b2Qc;0exuGX5=H z?6k@6>59L59_H^2%k5NT!KSvDC-y28>^SpxTtg&x{+ux1hi8AU{xvPk&EE27TJHPM z^ST=>j~OqfZvItyu&>q6I>0%eCruqT<8OUw_jC?{u*A8aIGS3)PSnsBw7VSo*Y`WV z&pq6a+&$yJ8^iqlZ^jCak4hDrFWiT7`AAC916w}YW)Detm^xF_j(cz!VJV2XRfZgc;jNf{g} zhU+A8&3K8)pxL+LKATT$g7?#{KPTS5xbNe;A(G>YWq2LW_6rQDGa`{TH-YXIWl=Y-;CyZIc~HOfnx&jo*eK7Ze&AK>RnlfL=fq;Ebq z>6_1WeKO|Dy4qK^sw%Vt@&L;}^ZkGT5QgcdMSViv)h1nq=>QUwF7y2WK-wnVYmDiD zH0iokg`BBPx(^@EDjfH4M48Wt`JDT{Am{H3^SSwc*`!zaxopy#&#NZA`Mhq@o6nyn zA3!+fE=!O&*^aA34O1= z%^Ml+&zRm--1MCzw*816w8VYtKr~|%f72t>q{sd6_?q6rJ>M<;c}z=wO}{Ac`PS}k z?&z3z#0M=gPXp7hU%Vb6nEpL((pOwy+Gq3pYubORswBUr{lL7BuW4V*bMME@po`Cb z@OlJHddxfP?kdiA{+vRS{-9-1?JL3Y+1IoR^F6+%WhNijwB|h3y06FFS(E-;CxyxPHLYu&eNDT;eB9F3bxn11?yMl4SExB}P0%)jYt~E_2Th%+hB}h( z{%wEn=u5O!F_El_BS>dBoddb7d700FSYD0&%-fmICk6?j#r6oWpw%&8AUE4ef{?on zQf3o?3a?$c+rZ`efZQNJcoj=h;JnJInEEARY|F^!8%XnL>MH>vPcLbw(c$Q7M6m()DqK&HtT;wt9~6#9EobDh9kktO~`7HHsQb4|NI|IvFnlH%XJ7oBXx z$MmTr2TUg=zK5f}<_a7O-X-4W=yzIyt>9H7eYS=Es2KAzLupq>urJlW!i-y z;O_$PP#FaDIT3hOb)dQ5*$e=zw`{GEe|vB6+jFVL@Bg;nn)2AP z>8F2{KMGimN35T>^LR)V{x#+5VEP5the_AuOG*-*e`f*PV87X%$A`viU&A-PFV$E- zCH5x^ul-9t;jUe2AIt3^N%@;eCo%5!8Nc^6rVlXP7m|H1fShoAHS>R+zVE-^!239V z1h@Dh)>8TsKlI0cguppo3FmE!3m&Y2X2yk#eElAp&Zvd;_>JQ+kWk~e4gvWTQq&FU z0xMytp!fd}_Fg-(Gs}YBs|a&72ONQ(ES`Y?kwx#lyAkNU_Z;Zmh5q+my$&D&>MNqY zjz|-cnbquwNSeK9(_(*&pZ|yV)mNd7iJ`BbU+R7~Kc4=nQ-5*N&wbe6ic@LpCqI-k zesT*rnzL#9r$!3r(YD9%6&L<{(Y#Ij660OGNBbV*SA0nO6%$x| zLi-yN(tJ+)7!zK6MF$)cReVba786@PQHL3m(ELmX)6ZL(jSe6tt@x7;X(qGyhYmF+ zr+JtTBPPFij1GT0rl@%G=MX6^9-zYyQ_(-+vJz8WKhEkgbtQ;DPTN?5LPy)rCZf|3 z#)REf)MXMd0So=Qd~kTSgZ&I)~Gtdo)&q z|HY+S)+hfo?rg{^7p@R><`yjPL4wPNzr_ZgBbb_{F4n7nhpJ!h}Iuj z+_L`2_TN9UhUfj8^UQDGJnpw`%hi6lRsG>S`^!D+m)n2zd(!pGTka44>c8?5`pdh& z{I7nQ|I7c3ooBUQuD!qf{cWH9t6%%8|E+$x@!Nkto%!XE^vkX5mvevl@BFa#uYUVi z-sXPgIs141yx;NL?jJ5)+Mo7Y8u|+Uo{2V|AwKl)-+i<{e$q;}{zq5KA*{$eDaxGL zXwNBv*D3AS>5y;nuRiJi)zSX8_h{EI|Jz3E{c^qPp+xYXIEc?|`2GCVt^7>_cus{s zzAb+}8uZ&Y=5C2`KZO{C2@7I6sxc!&&fB7GI`R_XKxL?1q-0GMA&R@s> z>bHL9UuJ&wcRc@>|LXt#&R-|}&Y#s->^-3L0EWf1$!1-Q&-h-g z{ljh6pljD|>7O?fm-esc_rE+p|K%C_*Ym&h>2CI?zp(yFay;}~D)?PAP#DuyKl>i4 zT{)BueI29zpS1^DzisSdC_%HvnHaedDE3f^P|8s_Qf6lO;*I4>r-19E&-@a+P^!0x){r_y({Fnao{lEN)m;C%6|M9<- z(>f&kfB$d$dKCLJ7ys=){_p>@oW}isFO#hJU;lm@n*RAuC2@=J|N16*8ace$)5b@4+O8NERp}2CR;&Z|09Pe zL96~1;`;NK+RrPfpC9e#{wQYI|Nlbvn1QBky4t?+rTvtO-Ck#AEe@d@i!Fa%X8l|| z9d!kRR9fX^Dfi60Jq0@b(BBwOD-b%qdKXD}4IsYlO$C>dK1iJJ$O=yO|%T4=VYfhnb@D?L@wTB;*0iD zFGB!H9nsSxqij+I+LFdcO-W2>GNWiSIZ-8pQg&ANGWeVg1D3*nZ<2 zA%(P6m=^wid5K05OZN*Fhh;nWuWJ6_QI3){og0eNTln`2E%4cZi-Pnhj0>8ImQT9l zbgm_c1gY~po!U6X_VJX;2*`;HK_EocfRy*OFlk}DrpIa7^u~Wit&Ef-I0C1Oo`*Bp z7Tv?IJ-xJlY{}<{drd^#!DUr2zjXDn<^FS0fFp9TuL&)x8jG@B>J@m_LRKx-_+ zWt8n-WSf9`nZ`;R#(F7tYTZLOvDd>}EGcerel^<#%_-V`2y)Vxj|Ri?#iqr=pS-0B z?Y+q_{$Vh>basCV#Q8c1Cy%Udw^1}1Sv8*eerTRMWBP#c_%9JsE8YdBa^aNwgJg%I#*hmJ|$FbDlEesO!kA@sbvt({7eF*ECCZ2IXs1yyn)W^!ujzlzqIz^iIm1%~=Qp=`HyW)N zJsx5;KF%O+BfSAM;z2NB`tldeh?XW{T;lk6&HR)@3bSUaWjOgH(bCb97Et7;IO$wL zrwdtT9TmNTSR!j*LcME|zQYK>^XhWoQkmg;@t>4^kqd-}B`%h<4Q8=KpFG%}5t6$; zaFXuzp;CW}vZoR0=N{tktM{iRh)k#7c{R0D+*@d5S$BogB+Ft~Wl`AIVUWI$oLPO6 zBAcDvvFCI=GPz5v8vb^)P1m%9KHRfZ8!srcpsjWJGI-xx?qd1lA7ca_=IN9YudLZRo~=hLEq0k>^vhEp9rSHAj^O z8(-uLR9W-9Nk_L!)?+3Y5$r+h{?zJ?yH?Y;RbVG*mdr~pWI*$Z9HWQrS7@tJoJ^ff zI*dK+we!tKY~mT3rks|t_oK6SNK}S2xazmE9gO}YZ2Bd=6%@2ju4|G$PAc6@TlSet zUU7dgcR+OHWD~w>Y|JBS4adBWuV1NZ@T_O4I0|rD8TsQI_w>z$sc0LDUraJH@7Bkf z+;l5Yx+k1!S&NrwO%lC$6X)FPvQ%nC7IHVoRam7MkE`mL$d#y$_O>#tX(&{s$yL3C zY>+K|TO^h2<@T{6e;B!aaCSsey!|ZkO0IGka z?XyN&MuB!+?pmppyx9HsT@j)P2?*qxyd;V_N{DB}Rfig@ulz>JW75XAtGOx1RdvD%I0c zm91uqBwHR88t6ivQbkGyJB_ucyXm13EiDa?K!^xLfc}o~j7E_U10>#lXU+_lp(ZbouP{F6qMI z2g^*yUTlP}=1~>enAS+&ZxgzPHNvLL2uEIVy|>-9s_gXg_U4?@>J4FbJ`K8<&&Gog? zPMyQMY{ZvgBvq2OPTy%>34sx-0g*FY#neDE5;0uSrBK?ic}%!0oGV#Ux0jk~yKvXo zw^l`2atKRLhO~|=KaxkbG8S0$eWmGPwZni`d*>tF5r-;~6;Y#?c65JuA4LU+-hDuX zG^;}C)ah`_GQkdUb9mvl`|kDS1xKy|6ue!8p$K6H6=$qAu@g$PxWW%VID*K4)-lMH z@|@nbRFErm(D=+-i*ox-;Ia>MwIwV}A_|^5?dFhJ^lp41L4}Yd>r#&uY5hti)6#kY zG$UGIYwHR$-^zrGQVD-LcD^4)n?gASm8)VBkB_p@%0oPwK3t;a>9lDPom|g-SM}Pe z{3_#J(4osh1*|?QLWLh~Dg%zn-vhFJUnY<}nxHxe9r>0WNIeUSu+y}n%C65O&_$Q& zv%Yud*rDz1+J#$~x)#~wH~)qb1K`~FBv+iprF=3iVOGJVew=^4#Ztn+Pn3RrRP?!P zQhAcJmOfnSDgkk4wW)}m-X#Hdx{PSrT@uVf{AFFZEw5-pDlqkpe4aDM%DNbLBjd-P zWRe-m(qkE;(pGjt*<~TN5yOJzh3#5~Zx1Jk@T;y*vZks1bkcr9zIi+r8r}Ja&PnW* zA03dT;dB~}iZ*}rqNH5^;&N1(_nSRB<^Y!-5MGKouHdYpN!U!XS@pBF{qu?U|h-tNAB#82uZzYpxDKdj4`nr$19~Jn*k=7(GE!3vib9K`zm|br7PokKO@&nHJFsxcT znLXXrZtc!SH}&I7#$?gxgFUjoj>`9YFP zmbkGaob4wXgBhJ2Scj_8^pWmZj701Cv?C)1{_0b{TJLS0-^Q^wQt|gys;@OfFB?Uc znXtgx!^!1Wx8B1nAV)sXD0{B;LkSt?B7IZ!wh4Uq^7w@6CY7K&pQw z(qb^qB=bhgRa9&0T#CA=r<)Vf(AePR$j1#T^C~842YTuI`w>bJ3}ER)u0eSe0<%!D zzUA!LZ>2A`qe!2_`|RoWfcx=oQ-@U;zqmnSGwhiunneqHGM%`u<0r*Y*O{363Z@Br__IjzL|wqc@Q00QPBqBN@uYr$v6=RIW8UcSnveXK)#*M zx7+~T)imuXbhA(%$V`W9y>w$knq_Cd!IiWimmbaCR6dqw=*?DOQ|jib6Vz>V9xJQN zPB%dJvNQnA73}KRG?M7KFcP_D{l|j6D;Jc{!^9(6DfzQWVe+vS(9C}oWwGh^0oS1R zbg^ZaY}J+Y7MZ{8mW6}sR`SsP25ZytB!W$?}5O5!Xz$#U;D#- z@=`)d_VYQdtN*kd(BFT4fGa1Um8f@u%1Ay#)NRGw65%m$uLii8It8KdB`?r{-Lzv% zxwGs3VzcS`pzrVk1S$KEDw`a6oNZQC*rz$gZb`hf;Uibx+D$-;W-*WiD@n$GUo3OE zP9WzTg{EOa)14-J^@VvD-X%EbUAq~DxT7(lx|Yw9-zv^hjHiDzxfYN14Agu0MKPx( zaWF!Jz*yS>HfKyS5<_11lO&J?!$75(UGo_jPEgvHDCv^OaOPPB4nzaA@5#DlG2Q&j zQX{-GHq056hi-oXcG=DmwPJi00G&3Wjy^|j9B+HTcbs=;7tP-DeW}+|?r>?P@tY}j zaB$Klp6pN_+-`sQ&(F@4?iKop^0zDz@f0&G<&2E$JwF!FLnmB^S5$0Lr)H-JqN_Z< z;w76|8I|G_e0LNwc5EU&YMop#;q!^shOXGpw^6}Wr7C~WyYh|VvZz- z)@?2jiRO6xzBspT9~@2q3_FsTXWhvKTW#vk=AqXG^?6~S;SL`~QCg4Rw+uhi1D#6P zEP?oqpJ#tny+8_C6_^AZ4I+T=-3Yt`G*U-M+@?1i6zoe}1)7A{_Ld;|6_K&Q3QZmX z+)XwnZ+HJ(IBWOk0|?J88S)eJ^+Wp306ivA2Cw4KYm9Cia+&ScNW znhx!3nPW~tOxTL8FO4T$=D26{h5%4jg(0Z=DoWj(`HtfdE2V9E+|vd`MG()GKs>1gep~CziXJ{+4s1 zdg{7z$fr9ZJ7zrHsbB9T@&x_r>0&=q$zq&Efs2)G%me!caM(NaQb6zdk_7s7nsKKO zMI|$WdLZsSZg$}Cpg}mdNQHvq=65^AhbMn4AfWR3mO;c1)AA6fClUQDuBcD}&Da4c zQc)u>TplIoK4oyg9!o*w;(p;aqp>^brw6_v`$<86+Bk?W& z!>Qxx_z8FkntVG4K>!t8x9Hp}191)1fuhvC4hJh+S7RSp+7wSAQ~8y{B-(nhA8+sE~<)XHtS5EX>ot<6hu+iJ`Ygvq*jc2Rw11J6;hwt7BNfXul)td zp;xjehtg#&05U+$zj)KL0n4QT#%H>NXXSHy`Zg;c89ae(c&rZ9>H$hD_Z>H>oR$od z>~>z4F*}Xqz9XY6=H8RB6`XX*i^0tQ80?B8vB&d3xE1R?^wS-{_e@ZYe?l|CcLG>{ zw3JtvNtmM%V0b2moi@1mA~^{GxkC}jlT)3mdYRgG*+%Gqnb9t-=2g_G+o6R@2xPE# zfH{*&0DF~k8@SqfpCmy5?BjWt^li}#q{(^?F5UkY+pcvma==nJWFq9qA6zD3%P8 z#;c<+YA%51D9k5UQ@JE8BJlyV(2o9_T!`6ALs3j;Del`#!e(OI4Xcwto9La36q^0% z-LmNde?dME{PYS%Ksg`(dCAl9@U=EtHf@9WyMNg zW+es-uG701;^M?h;!YR3(R&I!%`@`wN%93QYAb~D3oe3BEK2dBe&!@@DHP~XZC}t) z#_(EF#6x|=-gIIJr4E+w6PKD%}Q5gJ;>(Z+$542=q3yDbiM{%JUIVNoIH;pJvu^tBCsQ!JVrb4I&hM zii4uby&635hXU_jCG4ZmlAq_iuI>wB30}zYIcUE+~Z54!PQTO??Qrrk#Lm* zSXAc@%P9qY0pwlcMBqV&b>Q69Za~8q-HO%dVm`=sH+1diDlK<>=5lBCrj~1Vx`Dm7 zOk%+?Epn_HPhOxemp;@yqi1ddPmt@{Hcq$=ETLGh+&RGb3PGlND zj3x@8h9%n`Xa)6jOC*Y`c8dN=OMLzsEBj`^FdA#gnwy? zBP4a2pijWMl@_4~b(FJ-zh9I}1iAIkj2yklP*4b6eSMvlq(3DfyTiUynsP|K010mI zUKLf|J|TjGLivoRw=s!`t`}2+f!ki6PLukgl12W8H4@B!a_H6}taC8;DFFhOZ!=Q> z!-xUmYntH%vtB5F)|K;R4Q0<0{`%kHJe|l zHq=kYz^PVB7i|fd<*Q+@}96bAWS_r7}P?k@>Ta8+o=1>-GdJA8Sy2(^O?rBb(M5WKq1c~aXy_T2o(9XgoRqm<@ z#9gqQWbd*0%A&B&rS-T55Ds^N1v_bI-x#rfec&5sP^#EKIR0w2!lGLDN=MJKV;DZE z+P#hwvfrGd1fn4%MANMo3^z(U;-hKX91a5CrQcWg*cx9MDPsK*^JO~{`5EYA`x(o$&fl!5%Y^W#o zuzZjKxAOdq>ETV8LqVD4LKm_vPe_~Puu(!3zoQE?71?*k^fg|wdADnV5PtexL2n;U zw~%4lqJ!&OGV%}|+mnYjCooiXQ+7c_2FVbfII)TrUavCJ>ZQHigI=35;9Ki|v7Rt& zI3A*aJ=B=Xt%k}9D>_(D*+i0aWrrxab0bGqKUyrmeGh*ZV>ev zh?y5bxn)@YH8A>b_|ZRMLI3SP{>T42Sm?jT3faH0LRj}3E5y_P-{t&&{zeIz-t+lM zi}>gEF8ZHbIi|8F{!tKpkA7-uGtNIbbNau^MbOQ~Z4nwnKZzCMKeYgVGH-s;la`X! z^&u1TpWq_zM=7)Z6St&A4M`SAUT5e({mcH=^82R@tQ8|S9#{YF1tb2HLyr4}YzY&d z;(-6Dm+YqhWO4jl2}>h?G0(G2P)~8d{geXyM~V28-utyA=rYbkfT7~2yTm`^`6*9G zmwX%32)?C={&PLCtNxejw1E&8&;ZxP&*e&trv24N(~m_2>A>%?MbjetPl4w4M{*pz zy!ygdit0bb2XpuzeLr1cCL>#36J?(GpZPofX{G%q%`oq*1<4A3eeMg|{~%CFO}4?THGun&xwtX&TGUzE#W1)q;&B9p1y$kEzR^%}m#Xf{(gt z1S5tj(4&@r;Xp-*yBBXermcN()KIITJOiH?6%RpddR13GQNWn0kS0E<#*OK&aq>cI zU1m4`5GzJ1DWd2A3u@MqR+>0f)xr!tHR;*Jzj8%t@S&cguF&=nONyIJL=A!9BeIclVNe9W)LyJ{cqn5G1NLru)5tR=sjP>_T=n@)|dHBje*ueGPSjj?SY$1a* z(IIMZ&)MMD#UQUl+YAH-c)Ym27`eVMBB0=nC6-_qLt^6rs>nPgb&%g3X^aEVtL{V6 zKubD*H;>nh(AT^_Nu$8l?ve0A3d960VV;DM_;eCCYL+uOw7c0EM$5XkGjj4KpNw%rtFp zTz4RsyA}L`%!NX48;3}yBl$>|xvOw7>ZX~02F#Ba!wZ;{EvESVT(M}%&AM+p_HlMScWLj%t4Hu&?MRXPqx&8tD{ zV3e#@W-##jZb8pUSVSN3%`f)o!Z8Ppaw{b8y7Nugd>2<=-ZzKq8W6^k4L}WEyGtH_ z;-cBWj}7O~>!UVDD>xeNYi~-HN<%w&VVft>m>LkT6uQiJ-l~+FI2ePS57kB*CKYz{ zwAmD07^MSyCpUfj^Y#gf;)OJ$1(Pxx7I>EhR7;fe0v_I*&B|y~G>iQ@DUa-V578-D#foy+*12z8dTS!4PJPM_`qM&Tw9r6o3NB zRv9=PsmvG7#kb-+fBNno_oSAtW&jb5di$lLeymP6;%LnS>md}>@r56JZoc?E0X#v| z!ncwKQ4oU6e#kG^MPy!EKKnuTJ9a|^y=c%9^-tf(g@{eI{>keTu?e%=Y~+rA^FktD z$nXj_hFErxoN2~V_O*%`9({Vx04)uC%%e_*9QjqP^H*^3DMe%J-J3i#eR-1WynW@N z?>1~uWHIeWP**5NL>pj%%%+l2`N(g0sVJQ6g|=cNkG`}9H_^J&#gt0b9yMD}D9=|X zsn5PsJnVN-HYouhNCO>WcG}0^Q_4XTUoH3M?@xKdNzqf&{J(nN*>971f^K_5cjEj zFB1YqTapjHI6C7i)Y$#KgoYe*0lCW0T^MDw+SkMs;{tEq#)$wP@0l)-CpiK@E-Dz~ z=r8?U^tc{Paf+M+m19Ar+5Y+L?9^bs?q6UzUMf9f=nPp>^>t2B@Hs`r5jXyq{uoO#Aqmfw8wAba7JcU6kfJs@ZfCf0P=TQgw%<8ot%l1cV4FFP2qM~deJ6rm3%H8V`P>R3Y=`SuTxDn*($J3% z}29(rRt-uHTH{qEjORP_LcOOIg%eL=bP9=4_MI?uF}tgHKJF@!@?=w{zNKSon$?K zu(P>)T1YW`2*eLpwhQ~Rc|oaS!+)o?hCJyg-az<}x`k4%MZ@~hFp7fnPBn zIJp6^@UCxPIkZV^b$TQXBUo-$56YsdpaxvXTi$Sgb2N^49K?{?3w|*UY_$EV7MA~A zHy1+kE3M#<`3c0`esp4?)Yy;YMKd0<>w~P-RBF>O8sop)XG^EdLj4%0EYP$zHx4w& zu_qZYW9VLs?!I3h=dhg1v~QVWt?KmXVAxUGL7F=~*O3O#=6TeaMYnHwSjG(OO4=Z{ zXaI(PkuS2}U}-SM2Qw+1zH0SY#}7(U@WCQkci+{cCVuV;jMLIEMLDBj#3+9bgRWU@ zqrN;D?#23UXGwiT`XamhOCmc9=HNXJhCVA}!fm31@~ReE%;ol7v2{=|Aix}yx)xa! zsh4RibNmOWWuqc5&$j$4pbR0S!x>sR8fTe*4KEgR0O*a?;?=|UzBmiVV;`)TKHkb? z-S(yBukH!X1=VxTQ4`?bJ8YF-!$VD)pynH_f}wXSWGhE7kq2@WKG z%C-8*h^`pII0zwk8GeV)T)1)^uKX7JYAj{}g9S2-D%R52L&;E>%$@gcPyz^yzu6wx zx=S5jok@Dy2clv-46F!Xi714J_a< zx4R_x70;pU`0IlhgD%|~fB0fcVT}ZT#UwEwW4qp1&R`3LqXtWcssEBbY#4ZHQe+|* z1c2C!hLfs*cOIz+NR*~5_#$q8ulaP;VHP)%m@~;vpa;HU2-b`!M36{L?+l(fuMO&n zT7r>8!2ElHbxjp1x8&leV#c|KpX#Ruk5u$iE~&`l#g-82k#f>Z5Vl8Q;AU@s=O0%J zQQ*50MHN5ENEcWXUEH`;Hz+WND|~BYTM#FT5Ap`X(_WBvSa(r-)72dFknB5jJpeIF zOiwgNl zGw@@I@AS=(?J&wA(#Sk+5=D}K05y9$v-TqGCqz-O5$I2GuCJBWkdK)>44w0Cp~fQ) zLRA{DlmNuMk()#7HG#!pTYtbaWea;OGz*6)#vNEY-LjB2zDwX^v~i@GQWXrKx3_`y zA3Ws13_L#ba+MKorWgfOA+YL&0_j12_OmM3+iPnB?9OL`=ZN z!H!W{&04P9$e#h15jAJc#81i7Krf?3dth9jJ`rcWTYHcD2!89yoH)x@3d8M4zG)gG zbVGaXI%|=#(Z$gt?FiSRZGDEJ)AuDE524N3pEs~uE zFA7t_@oVSBnGNHARG^|$ymv7XgjjH{@9PlIy^)&QMqXx8wq5Ap zvuo>-BSpM_nPMYa-tlGAr}|zivO-BEQ*nU47{wE892ToHe}n zS%;Z~A22PX26>NhqC`?^gBy*z-~}pBR@fbjcpRhzP(bPHQ3Xezi#zC%uKoE)1*zlE z27b1Eytfc}%+8%KHicyEEvFGTTC|PBRnYCG++1LPhhj5oNM^iC5kLCeli!idFtTVY z>@#+;Gq_f-aiSyU08}V}gD$-eJK)N4Hg`=_Av%fZ+>%AxLgYZ6uuK5LM>0*QN9`nz z%+$`Kq}#(M;lV_CUhX^-7mW;7WswF~Qlv=w~ex%p|bNZG^J`GlH(-E5Isx zV%P6~3LtN&pE>~VMvAAQ` zr^xcvACr;eX$p_UWz@1n?E<&675yA@`(o06rN{(Qgd-Eli-4_C?dwiP&8Y9x^xe@V z)0DAyM}G{gr5!TzN5LD!XGrCpHC|whIQqgX)W*S18FhyWJWps1?wsW8iNsovuxohk z8uuU%9LwN=kR0YmOknjazDb8tY@S{MSr?TORypFpufEk{CU9VZn~HLPK=P3=hkvzy z1qpMRPk9k=&Mov35{-kyG0X41=pu?G)t#6s6#zTKK(|ET6BN4mRXVFMf0cz1DDpkX z*EDvg_`DtMylRd;Dui(rK0uwV{Xe|DZLhjM*C_a1=H0|hM8&83WRh7e1t$cME3?I_XJ|^EQ1nbMxWpjU;z>G~?`9%j*V1@n+tyPXJs3w&pu(Deh z)PmLAk(dhMW2ok5K&%mm_`0a?JbV7r*{y~sAX>@=IR2n+9lZr55n@s1)0=WB$hkK0 z@LN!X_A$eQqVAt24d!u(>TguRq$m%Q*+t+(22;^J7q=WogQS-Wh;3vL@#o#kq{-UZ5NAqKI83LiO|$?e>JX+pvMm_!njQHce7hqCD5x2VGWyF9jp+Q|)ytu;s|3;wj`=@8CL4WQLH=(I(k<=q$~RwNJy zNEz)Y#7|llNEWqK=B=z!b}Nu;>Ssq=>@QN2+QU+8#|%_lUy%)A;S>b zoXqmc5jTSCiW_O!5Zt0rQExMDKwN#WBv!%Egha7nzq>>wMKT1R#6^IjynG^V1xnRs^sy2Ex5q^kn2D z#^W1vNj9QG^PMEkQ@L0@qtLUjKahPxBz~0}C()G(j@r8kwlZHCtj>o7MQ|XG-EGi8D1+2F}LfXtjv7& zdY)Qn^V{+UL7NZ50Edx_VXRB}X0dbTot@coD!8~SQGb8(b9{Xxiq-|30NP#jwr(-* z)wN*^_$<9a?R8UGOU$5 z`=<)p+XB4JK&QX_F#;@}vH%XqbhJw!*s{Ls9KmC_^EUFMnk0sWBy|qf@w4*XUI~Cl zX{hZuBk(2xad}Xj=fYx((Wb!h|G(szxfr(f? zXguQy-=3B8umKwOb~Vc?9;!5IXlgw`{)%VBPGo=g_eeVi7ciR@adZejt^NHl%GV1a zYakgOi8fME^fXo&{n2zA^@F89r8u?$K?tcT)cJ0{J`6v^%zXN5l2 z7XN>S9{xaj*8^36xR^)ad|dKI-jMNWWU5@YwJ&?g?Z z`Lln7cq&&-v*o*QO#WIK`-=8~0Vo~u4Of%8*DEzVKOjhggK^~Kmzz|{(a^f(QB5?& zxGAI;B>8#&i+BITx*tW|M9$n`$+$qGN6~*l#Dj1VL5&ZI>5J^Fk6O;SQSivX3;}ti zJw#-}y%SV2!2zLOSMcDUHI%34x0GXBEn@H_OywJNJL4 zYl>)iWU0|CA&?APNg&+Z9-EroOG8=q@!EAs2c;DDqCtf2G7eG}96;RRsy@Ix8{iYl zSf+tITJLr97yHUl7RfnqVFVMd4?EI2c$cHPu~-7G^txFH!^g)0CV;rz?%m$_=0dwq zxYbDtn*9+>B>{Fz&NMj`)MUlt#WpaBs(vI?Rpq@)E)lLw<7tqQFh|(DhK}LNY z^f5R=cby3GCQD?NwcBpf^xurZ7#V!hLShy&-~3Y z?|8OPb4sPQ@KtJ`;m;{BwFai(RKA6{8$AYmU!`cUV-+w%&~HOZ-@mvwP@pv+8%L-F zC`n!n5Py2#nZRV+FA!J_=Tm<=IuwTHYN5NZXB|XnQ{MSy0KkQDA);gUR^7Gi7nV6_N=;rI1Y4=M45~mJEw;3L4MBM^pfnjgE!7!w-LRUP(YaZ=9NW z+0ekq@>q#bR=Sc_AnmUVwZ;~SZd8b?^f;W}+T>TI*@znvj!h^5>(ut0r6$U`7fY#U z>|o?$8D#uwp#o`9YhyrO$`Ms?fYSzf>$p1^hiTIxjx90^O^96VFrg}ZEGc2C{zWIb zaI`vey(v1@WDm8U`(uCc#`Uo>&1At=Im^CRQDGbMRMl~t@d{Jp(V04;?Tma2RCDR2 zd&LQaPwS#7R#I0$y`{bWSRMRk2hFdpU&Ivmx`TOruJwuIwW? zJD470MBsX3|^Xa64!|K!Rg;9_Up7Lh{1{|#vD^^`#Av{sd8ITbJ1R8#nv>T4#!`E`>b*^;$6@6vzA*{TJ zlWWpOk&@*qW_x31+7H#vG-xfqt|0vFT-KZ-oRb|dJFSWbSQapj=l;E>WMIy8r3a(k zoMcHcoV}$A$v1zJCOOH%;l;eT2gm+IR&os#L-V5n5JfNP;8`vR-=)Zz%>ZQH4Soa! zl^6Td(o#RLtH5XX84NZ{_^XX|Lc49}0Le+AV7*qVsx0Dee_;mcsY8#>@<}-A`}uDD z#ZsA#RR4v5MFNCT=D=miaG^J)W?ureM_~Q-qZ6u6rYnD;SOw%yjE_cd70O0tsRnoC zv6*SrU3KR~b@{+_|~12b7X`1C)4jJtZyXUzZ(qj=6L(~M{{p4lfRd; znlI}iKYiy;*ljGLEc1ora5l@?qGbO!V1i7}K;(ZZwioPSzy9u~hCDn|yu{bRSXfxg zr#kI1<%{bmp)~lyV`@K@pEBV zu6a1RGO^A^Cop`h7c>f4R*^P%^yj--EN93MqN`sqJ}9X+$r`Hi#uBUNP@_qgoNv!@ z1*(4w`ld2=Rqo+iJ5hgxw4-T4lZVZEx1tN+yuO2cnwAkkv`^gpZ2+w}@wrZa)ZHBf zOp?mb&DhZLnEpGUzkgL*=EnVxi7f9ZwHYb9p9I|O8@pRdw+Ml$zm(8%{JwN{I=x4j zE4@Cf@@b1rlo4P-9!c)kBonr z-`hgeo!WHTdC=(wCwvyxK=_4!pwF+}ArxY{?}XD`0W^t&jEOyR6em4HleXcD5aKa`PEdE=!b@=`NPK_n$R8HK?PlHKaMSASVOnAfmqcd~b2CT;} zpSw(~;REN2pS;-gOM$ga=K&Lj&I6C@=BKGhUg8tU-ZNb~gF|waRihYEmG4a)i z{#l2e`6fobWxudRCXqm)UYt<`l{1FC&Va!X9hq?hAZ55KA~1UC`c;3R^G7BK)Rx=0 z;3Z`ihnMOe+@(mKR2(yFdxHo&(I^FOs(7L-3i4^g8ghdYJuAg*D$RU}<-i(l##iLVMl>6T76DTby*=bj z!?mg6&uTJ6SJ5YI2S$IT4e;>MQ9U>u8*8i&{8!bhbl_cnL>2M1U}<{IvB5O%ibi3c zA+Et5e7{VsxH2V`D9TxQ|Asf+8_+v4(~vKeB7_#`T?_j6DgGV#^Ot!XJ@JB5QHTux z0D3GZIZp!dEp)pEm>EiyR8y0PJ9RlFi;4cRY1GdJ2OOCRV&{K&5it=}C$}G@2~d-# zg5C*WL-|-j+EuPDWevQ-ds)@6C$(J3)LeDBmVsd}3)eH6PJaflcmxG6RiJ+^OgEAw2nMSHKIwkv} z_<;M9nw%0CgTc^z|90Xs>*KLj)`;5Qlt8>k7@^fj{J@+wSVL4lmGq=;_Gha7&WQCx z3pH09Ll}P)4%cyxIiSXe;u7_9Dc+-O-Rco|CMQtiH@#wNMZ}}Z=Ku;T#6U&H3KQC|EO%Fqxuy;2D9@VqHZFhCUku{ZmUuBLQ8Bk)uRTZ3W`Ani zJqp)uyw1FZuNQB7=1kwSVwn3)%m`f=eY6JEhD%V_hs}Ak9N&bHdnR9hl#M1I+Cow z!kl(NcZ6u&r4YG!F#u$QWWrmWI3tK-ogq?Ix140gRfeE+E#Q zSUg2FKWaje-fnmY1h9;nj}L^FM$xg<%@;SfCsvMZ!)GH2Z?w7Qj0EHHy)5&q+`;}A zGL~7aj-6nvgzt$a#VTrrR=4tHm1rIhW0?C)dHZ|Cc9X8`p=| zA^ML(g<;*~pVH(XnZfy|R7ubj`hyHe^nYw4|Eb_#V@8IjLMs1)MM3{Bvh;_A#*CyR z$qM-|r3#BS|DE^xx9g+h&yq&sUvAj{H$MMwDoB5VrhJC{ zQ>^?g;h!=^{!5W@j{G=Amq{>G{Y@A7$0rz3<4cs&e;HO8hFz>;`)u@2 z!SdHU|M3|5zph&OPsvi5YZ-qrVIon#>^pkvnhpxdTVCWwC+fQ3I$r|CNnc)T;2{9= z`X8GIjA#bGe-B186*nB@5?iphy&XIDiDed{Izn zD!dJ+dA;sTNPduAV`=9;;4T-UpXx*J6=11 zOh9ACTJ!-*UlQ;609^rGYqnfYU|u@SIh1j_D;!M#0Lsebm4|2}*i=X(NwA?m=E3HA zFkhUpp-~RUl2pP^{d~3KwR1+7_hY=>Jp;{H!o`tUK?$spB;3&SE2)-;Om;EfuO|+S}5rP`Qd^HXG$nzWUat7V~A4Xd<&qYFQ;o_)a>{s z_ha}%v0Wl}R(_>M;m6#|QM3*!)f4(mxVDJE+2yos-6 zagK*>0LaiTYS(`V6oVhZva&RLJ=b%hz-)W8Rh6`@QIIexgw@qG}=yIWazEFodjj`$gZxIxtnG1 z8;u{=wh;H1(?II?C^>AnDwmTO9hB0Q-)0Dmm!H(ZFr*;)$Y=vToqp956?0Wp;Z`su zIB^Ce=p%nVH0f0QisNV3d&i~)E3l?(_~QXOLc@qip#|pF&OAXe(lY-@$VI;Zu_5S# zlRi`qp^U3M+ePI}`<4l!NX9$Z@|7j^0SuM~8Mf>F@(0d(w9ZW3d$D+wCfv8DH_NxO z`;GT*(ufCD#9qrsZ?^5g<%1f*&#*kugSc1Iqs@Qpqc6BJhzo7DOZPBnBhgalde!T_ zxfQ}FP~)s=i@bv>N97uowfrF|f!zntNa*xgXP;s^+VFcRA&j;IYtUK9Evu&p;r(0bNX1s28a_h{o9J{#@XbTU+ z)C;UT9ZHObOUd}IU$L7c5Pa2u1%6L|H~~N65sGTdMV{M<4IFWWcu8z_LMN~n z5eo;QhY)Tl}#3e@CzyoMJ`Y&!`t5+WQB4xO=3VtSCZ z-Qs!!J4|Q{(dn3lV$7{~5FU0vkCt~g67lO)q=Szb8)|UH>{oftst;i|Q|5oXD8;e` ziES;Tl{vb({vfzS4>yu$1Xl7a$51VNkCG^^o$e`mcO-j;hCrj#a8aIsh-PXXdH%ud3=()^}3Es;uM3*@tmJw+zDHprO z-d&*Fy{x2i9LzJ2f3FnZ+X3Q^Ih1?0$~&319ed!;*Ky{7&!&?0x~_kbBnThD^oRf+q^XPFQkszMZ^X{9%8@`Dcr4MXb1jJQ3tkAG zbnsV-4j^_WenLR#f(_5xB=f*t3Uhjjs3C^x@bM8gEVsd0xMq)71`_F{*xoga?`aEg zDNlr^eu`>csFIUcEYN=;&*0BjHxpY)Ddo1GT*{_9+A6@4OR5!@%s;FHrhFX;AI2vQ zPQyRcCL<9`%{D`+{N6T*qtfGj*Xr{Cx5#}i8?Selec-Z_r$fTG`wKH5- z4XDQ-@t+h-I2=&qc-T)NS$_6z!};A6lGjR$!3yoIL%zZf$eVv9hp_9*{Q$G_r72Km zu`J=nBU}iCLU!nv-*h*7u%F0qHN&qU-8{IXGs}84H;5~wt?+)Trj?3=7&&^$CMS*b z!rVs+qLq^aBI#vpyCL;nS}|tLMcz0xV|#Tbd&J6%5}QPS^ia>4;u4T3B3iZTSD?HP zT!3u^aI!;}es6yl8K5t}<+}o?ecFT&Pu4AE{IzbmSaj1bb@NP_7;L#h3j4rug3BOo z%!zH|o_Hw-&W%s_691u~B1z_JJjQtR5yuq*EFMTPi;^dGZ_{vk1EtuH`Jp%jJd@59 z(Y-EfI9mQM)vDa8-(0NJVPUv@K}EyiGE%+)6Zx>y>Q$d+3p)& z{-&CTp*PhY_2?u`ZKN$09s%+CZ?@~V6+D4|P(?m6WpqoV&ymIiC1d??Vm~VWGPVbH zoHJ{bgnrFmEt;Ak4I8-nSF#fZX_Hyx;4VKx;9|Mj!H6e=>*{}~uj(T5E+=&}3 zFC)FIx`o01)O+P%FOiLtLVu8*855r|3yzWi=z4R-y>ATG(fWKNO@B0`7StpCK1XCW z&c#yh08#nJjXJgbRfjfGXG`xn?=7pcDc~ zde2LGtKwmPlU_ZGVvf-3H~~WEG?MQ)wS{*>XT2UD| zpg(^WKps1L(@d)F#FHfLt%+tmaE@VF2WQ^^=>1G8qy^(e%&uptGlGx_aVM0>45X*; zG$B}QH{iu#u2-_0gRJ6(u!NMhh9{ubK=?_<#DkM1;y8`An0zD6Q`;OesZ&`+PT?*@ z;A-M@5Gpm?)ZI*d@{7O&ZUi)T*Llp)ugHH7y1P5cIc_GE1Y&$}-N!l*0FTj6j=%fh zvt8;S@og|XdtmPzMR5J=pm#%KNh_11&iGQHYNw(#_%!HSslzE5xs zcefp*9{?z0**$ptQegOupU8~*;Z*QOTwjYeKSWK{+(Ig*)`6b7k2mUx@5#fo{TY8c zcWT#6>qV`F51oup{H1~R<^#Xid0S>~#7q=s-SqXO_S2hW){i;MD?im)GcqOfd?bLA zQU`nm5@EwZipVw(=VG`;MEZ>+nWjcXAeGhWBG!mI!Qs42iR_Z0q-!*4ol@i0!#%y- zJNJb6(gsKqzU()WLF0A+NycaKmt}v__q7NJkSn`R_`T|cT%S3BftB5M`;rex?2B0- z4K8#MTOaoivulToo%Xi3K;~RNDDf|6Vd}4|d#n zc{gqrpR5~oef8KPhHF=~nVXxsW{;*7=!JQ=UUwB*%4Oy$iDNLg1{+0`m#=@riePns z)ui3$rYNtX6eg-W4IgJBiY=wez2(l_yN$QH13EZVltlq00IFNSZ<0 zc0Z!LBng#?Gl(i8RZ?@Gg=Z2?rmd=eQ9oQwUD;C^YcC=Po?$2lf5djoG|g*fcwH8b z$k~6l>YjO?_NBBa0u1iBW_y2#X>u^gBj8A~D(pnq9WuRx&1ApWCrVw4ND;rBT->MKDb!%o`IP#8C-bLotg zL+zW7O1f)H&4<2q93fQPcr8sk3wTle;ptz>#L!pUL_}T20$HFcI&FWD52>@D_VF5g z_TY8uEm53-4&&H?w8CQ>&>Ef>eUlv{zg49d$o7bC8YoP`AGr8{kYmIT+u$fCzKej` zCnAg5pmQ-@+l)%la0jtIPlmnXH@K7lc=<_YhWZn_xn^wi+&UZ^0fJV7yBOXt5&2A9 zRKV-{7*-~<+VZF`%5#5y;apWjJi%x}`FJ?KKNH>Z+ZHTr@5>vZ2{!0Us~!p->txTP z`Vhh5Ou^!d+NR67KrcaXyfs}*hSVMAxwC@n$Eqz2O`6zsr~#Mus0RSHuF`Fa64Lay zy`xxWFH<)Y{NjBqgTkbJnAze<*wJQGjJ_ zvdz^)QC^8d6u}Ce6B3oDhG>Y1NEUz3T@{oV&p8AkV5JM2%KiL9ZNSGFFT<`|*u}_0 zjJQNo8soz8XF3RcKbbL(Tb>k|{oQup+7?h!AG4#hrKg4(MPS$LZ%YS`~QtzRIyPK7x zp5i+^x5`hkBzo2L5eE*g**;LWe*43|KzhaCbBp`52|i83*`)`6cP=t}oGdM~1r2g> zJK+|ASn)(cGXaVgc*B}8sYjXw5-8Tm8=ibdH=Be1m$KYh2s3Q6`c1O$9a7FWA_O&a zdTNtGGE~EGfqZ$aVQ(dW7iJzqRA=+E!7FT%>gfd_ z^9C#Y!i5aOIZ;}bI_0yyl5f;SH-<8P%XBE{`oMH4i?4R3Z9x3@>NV?+8a%#ZIwnX& z_7mn-RO{C-T3xdHBMpLB;0$(fsw5}IlEie!BK{K;Uv$5JS||cI@|Ee{VlM1H^to2p z)+F6B>hIZ&oI1CVp=DWCT5!WW8Rm~FJ22X!FZDw*z`~Wb< zwSJ!^C%P?v|FT^W%zz%=SU&=OuBN@FMAp}sU+K*mr&{Zti=I92%Q~7y#5>yDnjvJe zPsVK|eh}I-=+p+YTBq)jNjU}&S=qA!t$Z_%Wh1C54d9@L+i7Wu%?n}r$_nk!XeMkv z&t1`Fol9SRp4{r+Jw6WlpYP zd5Q6hi=taa=9Z|}eckd}Q+L6$6>H#IS2GMn@d94{41g8b>)lv^ughkLyd zklVjs5HWs>(EkBx4Kc~V0*zWRXTgV#y|GduKzG=(r6HA0jm9kDUo{J$ctu^;`gvcUgCsr*l*wcKZXvdqjPu%c0d=xH8r zNqBaue2BhJ%t@Q2ILud~3|X>;X?>CaNn?^u}wDVnM3gm zWZ12gQn$n_ui-H)yCjl{>4yH`9AL<+K1~MS+#1guiI`(KB{+}wX^;jox5?Dc{d#GC zxR@&IBoZTo7DNAI3;bt1w)w;DDr~|I%hOCt%rZjzF`TIQi_mtH%z_}@jGd*>HTZy( z4H6i;Ee^vFprIdepY38ESLbQ)EwWu<6^dHMOhvz&Nu@zFq(7vCpmj{X8ZxL=3DECu z-!ha2)`$W~qYtV|0_hnVit*n3%s_&dn=BC<0hO1xED{sMrxaV4^*wNI+kSMgeh zd~uOW;)%>7-QE3?Gj1&Tb6lVs=^|aX@0&0agOJ&K*eM;0DLD4l~v ziiD_4p~s~f`>(#|n?~#@_bGfL8S~0haqIKv>BsVO99G2#DD+cnyKbvU>*J_OZM9n_ zFB}KEtVmve842NQePc7aYNaG$6;aJC>CQJfp+bpDqDrT)B>x%JULimy16KFA7}#A( z!|!OkQhq)>NR3tsO!)16Z}%Q#^{dOfQx4Ue(rtvg?$mXESMa4;vLyZA8r^h-8# zB3!MUbijrY;aasQb@TWdSFYue#40ZcPhq{95p`GJd!P7naSNzi#|mL7^4=Bxx7zLVN6+S{r6MqE+fKBCZ>_YAPsPfnl0J|L z6WAJD!b7iv65(9t)VSPn1+&kBZCM$KWuaJq&jstQ|0y&s5#s8QRlLEg1JsKzB1$XB zV5Lr1hF>3PWAd~TvLTzW%t{p>fIlp;&Ay?&;7W3AB4z)~3Cj{eircHhzJQZ*!HdGW z{ec;N*qw%DN%m#@i91~(zv@pmsWsK8?)hzm&p@NA4pmT&shgl_$ePM24NQCyO2X<@2Rs?*n z$DyrG@2y(#01OFz$8gl{lGHrI_Yle8*Ze3Xyxf91-o-eR_+bi2Cx*IB$VrypRqCrq zR$uaK0~7Uhw2C%&GozLXpkSClef!$$-7@3msm4AzC!cD|(3UWnTQ&$stf6n0X)qBZ ze}4deE5Thq#l-CuKRz+j;9DmJnc|ncZcQKj1;WzRO2RG|5imhGHNX{Z`Rr7>d62|JCMZetUK?}4$!o^BAG?GICBp%olm&1F za1Pz{wkUhdv%hX{+d~yCBF`!#QD7~)V3U!)Q4=1V_kr}&CWXEueMw6ecevo^e+;r+ zQePPwW4Bz7|3*+o06}RQk)?4a;;lj*cJcHy6H)qUq_Fuc08FVdU{^ZN4Zf46oBA8NBU`3Q>MGuhbMk~NOAm0iS%jmoVGqZt|# z8Bm<8kJPg)^QP-9x&hBg(3YiPGOVIoi>vW2&sQqyv>D%5TB&%oMn}A#LrR07^M-Xu znsYFlpO;qv6SYWW|7!Z?}NO@8-f6sIHXZw#V(>hZF``JZ=kTBB z*NJT~$yt(#of64!SzU=4UlP8eYgo%=5#cs!=8+ZYZluD!xni9XpY1|LdLwr#B*SWQ zJ{gLTn3N+)13synOMB24F?u}(Yr08V;Mvm9B}%7sz%Sal4VnV-$DMQ;bV}-aUx0An z(G9!^dDk@zygm?#F%9hU?WH4~Oo@T+hJg0H+weBU=0Y^Em(B(eRj%76&yByg{QSZw{&923K+igIWkicegvG0J+;*|c($Re` z$M41Qjk2m;DCc_yy?>%h)T89=<>&0M$Ta2kK0rd!=Tu*2QEuI5!E&rc1CyZvlM zmeXfCCUAYwl$UJ7?T+(I=|+~-pJugM>|=HvPhNglb?d`pJ%5F#&0aTisqeOL(A-&i z;MV!>)JE>EokCwMjf%dT^7v9_&hF?c_g5ARUE-QNzA90N#Tvu8vZjyN;M zV-D7T?gjkYR)4_9^|HK@55)BOitPMfW?g2>cq4;;JLnf@70Udz7+t_c-V=z zHN?(5IPAm|@J2#@z{;B@$ngZzyHr$3!(?=TXN4{Z;2SkNEF zimW=^PJhK=iV*;>!zqWEf5ag5yDDi4* z?ZZlv5zloDWQ-u#s4UfWkd|Jlgn&5^ziDs`H`5?bS6aFys3#Gi6o0@o$KwUFQqW?s zrvgN;Jqi##5c~jV8vqVJV7Gq&i~j-}PG>(r3McNH{s4~~K=!`COjASq(0%0c2bl2} zh<_Z*1pxBJ96(M7@aYfeY4{Dc>y(h;PW2VfMvR1hdw^Y}XdQh!9rhiRdRm`)Y9Ox> zU=WpZ>cCb2vj;j?0)uBIjg8-Lj|@_86Dk{>(Dnt&sU1TATD?l4?$e3W%INu z#lb+4^rBG*+^_Zc0*VN63g!aLPowD{;NBm8egnUL$>TS0_b>48U!cZ<$KRl{pMSA4 z1qZy=R4WTGd4&*PHeq7;G&MnyIe_IC)#n)oxeWoabs%fH-@fdKPwk1h4l2jB7p;wo zhZ+TuTvS(d*1^;V0>IN5K?rUEtTO}z>UY{&KK8WEiOpHjX7PiEK0S!TCWp*WVdY2I9YF$7beVkh1WFIsInFMV7$gRP*KDw)Pz2)X+40({sveiU%-~X z^y4$uKY+krAf$Q!2PozbUHAn|ENtYLUd<$hnd#L8oukE_W9Xpe*arsE+${mp-BOiQ z?c~-hI%Y`RV2a?yniqCL`#dP&6#$yFzs!1??H~QDwTI8l{3V1OXc#zhb(uX zv;~}7Y9XtDO2@b-sa;(?3}qX^497T^&7iCOk9Q2lBcQGY@TlepFjrZn1DNYiko5(q z(%$(11V>-szryZU@EiDhur29EW8WZ`&M$!JpJVGAr2Pvlbn3oko+Q)ut!#fxorDT8z z5XftrPzR%)T9Bq?Y=55@sBd;a{GtK9$0ap4Kp-hF;UDUF09ozmkA9|qf>oE!08;z$ z3620v|Jd+%I)UGy!oSAi7YM5W)BOR2{<|!a$N-L;8Z|Bqvm^_`s0~Awszr0N>$ua) zy}gD2s=iVq&@O$>HaMUKW|0>D`smko=Y zxAVo5KR~`cP=7?x4_W*KW1R>ez==>W1y!-;3ukz`dlPL)J)2*5f7N0tIus=woS?^$l9IhN47T8L|ERSGE#A}dcv-) zHITY?L8yl48TN9mQi=>nL!)5W1STSVepgw20K7J^uYZiAfOx8}!91o=$G4RVlr5mG zs+-=v(y6IYu9_eUQP*mK$)<)H2^8RtE+_z;LS1_>m*-Tce73nL8^ACW0IaBJEkrTI zq!ot@i})8Y_H7?rpb0SgJ{olOOJ`?4z-xtUevkJmeSh(Mh%E&;JuIEy@zD=}Tc;8> z*AcS(_po`5$` z5MvtZq#ARYX79%VRPNf$a?9?Bv3@ISt;_cYRSp&_G$AKInS(k>RY62ZCEzVF1e6+x zOBv)Owbjx{!ft0#(VobHkgZYIV#ISZ=vfK@)_)ugy>G~Z>KCe}~OlzW&NT&``DX|bt6(2^o1fM&C8H)G?gg=0lKm8o|;=aEDm<8Z;YLMR`$=VQi zhxGhyA0bOW^O=tin*PmKe*uoWDE;<*B7Y7XbP~|k0tL`j*xnr|L!^qfGzeAB-2{>g zA>Cm}j_^zjMD;!54a9)!pzkXOMy}RFX42NNs@8-h2r;AKaROZMrY}na8VJDs6}7F= zNrl0zF<|o1)Avn7d=C`(fM^5_;%$#-(}%qF0Of%Oq7E>3f@r-~sRGfo<{FF~6MygP zrXw2Zfv<6eps;X(4;?SB=YTU|OQgb=eNZU5UdSR2VB=APDsDq@*g3pAV-pNF47{%y0P2!6#2;Fx2#nEZ%rV844IT@9Z3-*j-#b6;8fGNq5@h|ozeNA z2a6vYQjm`o8Ryhi#i91_?#AE~H%-6%^yY_-g@Ydb^8aMe!Oz%re#VgBynhqG>l0xw zzj*r5-7nB}vro@v|HnB0WgEW1*niDQ-zmHl0{5P3q@teMMH;B(@Gu4k0jaBk$c4S6 zL(u35rdzd$T&;}ukTzSDODbTP5K1&yq~=wUi-hGEGDcuuZ!v0IHK21hU?)NyqF1>N z3YkD2X}HfFgj|jKwT=pcUVogbfVaozl5}Qua<$ZfMnEBid7w(*F@BvjzauV$z)^i- zqt4L*Ra^_oJJ_|_#gwoaa4*%h&iYPM0 z+`fqPU-BAcmjBM1R1)(baS+HcETXf@@|eUp6VAVHk?MvG34<}=^w6ccqy zp#gSuCa~3j4G(AE4$jkw<=2U^nQsX=HPL1jVH!ciBOUvgU(d`Wq=Tm4?2pkE``YQD zLVOPjY8ygyg=u*)4~E?^VqT@>XeQ=WSQ7d+ALdn^m=EUCkOQi|VxHoAZuP^rvAq1U z1JmF9$j{_^UVrtm3j;rK;fz2Poqxq&Jk?EFTYzGu-QgV+^4%=k1GwZXlM&PPS+>5; z*BYZp?^zZbQI31MDt!s*Cw4`=d1y4Z)kG-)r7B?Vk9oSWLm-6^q{vU00VpP3Uw4qt zRq9KztI8^5B8KDvpc(f1sBjRL(P%l#J3w_2W3p5u6n}?+Du!8bp~JuD>=Zy?*yAI0 zLZMHIi2o&0e#QUiZ$I=yS48Zo|0Rd>m;Bk6&-!hTf9T$ydCc!QuZZ)1<${=1W!fVT zqbNm4I15#dZUSTl z-klfXf7eeDmwx8#KXmiUw|(-C>CgE8&z!Kxg@0Dj2vm7LjQP)eYRFG&^zShoE}Y3f z=`h}$0_S8#@syox>bUC?-?hp?pu;=M$Ei_}BLXn%|h{suVmP#6cp`Q z<1_@4mZ^Z0ZkBEUBZm~G+OHUH#d0oO8{#=t zXSfDIL;$6g#fXS6GcixVA`Wf{;sP`XY~L};hk3%c&-q~w2fOf1M zcc)7=wQDrMQ;6U3i3Pa}3iB4gyhDs2e0Dzz4OY(%33amU6m&va}Cx~~LK0um|6$);(D8EZ&ChkuX_ zpmybW$yG{iSAgokyAyuLX}{vJ&pb2+zxnmW0~rl`=m*~FKkdbjybPLGV2gb~y^e3P zsoCje6uPu^RnBS+kcB7a)YPnYR9b7mF>If@2q-&aMrLT#qFYZ)wV)wksRpP$qVdEw z3xW}kyr^bm@gxK=2|*Awm`8@{D}OrlL2Z^Df?O;Y8RO-6T!T9n^U6f%iwmM5pZTRD zf6SYH&l}Spb4T*Y-F;{xt;jz%l75Zpp&qLLp;Kl0WBub-PCUbrvskOqPVJqQ%A3lB zc^TNwSYy&2g664#8bh8EtU4==QH(%yP&=S)d>Uv){Q^Aewo)|5K=u)$7k|__W1A@^ z;!2`H3Hg8vuyct>X%mrVK>~7jVagYVbTc)gVOS%;X=L>K#Y%HEmFE~z*)#f7*I{Pd zD8_JZY8G!0V9nSr@r_>L)$Zg7gVxnPdC_dFw5S$&4|TAQSHK4QD1GJPKDtvmKm7Ax z@1!4dM&svN*_Y0I#%nKf=zlfW-|ao24FVMJ@boDr*$)CqqH^lkRf@H%=46_bK!zQ_ z>a|xWS4>F?b4J0F2h+&i@UD<=N3|M|E6e1BrKivGp#4tbt$ zyCQNE2>cPpMUu^k;6rC5_^uZs&P+|jvqutPG564HU;GRxN_?t7(8;sBYq8O_jz$$7 z7Qk@<$+#}3Do-KcY>t_uLi1i}Osp9)7gOorm23?zbwCW4=4W343=3T{>+JDDz|jRd zqd`+67#Tfgv<$cAQGfNp-VDhDQKIV_4X_oC##d*Jn5+ffZWFLnA3rW5rE8(al|YYI z^G#J8V)e6=dN>KD&&7DWit*@iHrVeUYknU-4YoRsweLJmNWXMQtj+uyv&PqY!*{-? z*M80yMV>g2(T~{ZGdE#>u0IU*2Ve3i9q1#{V2+)%Y7V$dEq^HT>I_3WBMvC)wbx0R z-Z_*+rKwY9K`LD!Wk(aV1kko{>Lyk@C4!&4A%wH}69JL~&Q!=-u61>8`Axk}r3V7c zG61mwGtZ`2&jZfYDS&UmZiGbqR&;xvQW3_j_Vu<|xb^ozp>}MYLEAReGw*^1QM~6H5{#tANl^-_y{O`G> zKk_8M;(y6u&CTKR_L{b-gB05Y#hmC-suuN?N>XkHjs-jEpm&as0t>7e4q-phd>F+x zkV7qg)H;)2Dj+qc4MJt(>&B{wfum#-`qlz$!BW{m#ywCFww|;WsrKhBQVx0%St>lZ zg+M4|Y@>2oY&%7)OW9hd#JGzP;`vEMD+5w&i+|j?Msc7glZyc`(paiuvA#|t9a0Bs zvuh~;b`fw|%XbRMi3u2&EiCpTG&W-V3p`Mq@FY%#B40nR82+b#3_k?fsvkP=b*NMD z`DF;QK}ccAjRiQ#Xu{nXj2fKrFoX%#B%V#+D>pX$7$dDED|YHp&Mcqba@5K8gL0cn3sXQAI^%{N#g4C&_{0OtUp zjA{ywfGI!)jtU?_7=Qz46@Zx;0>ssj0w0vdSZ3B4AO(#WIDk6=1Oy#`Ty@vu6X-bb zm?Z;9R7ZeTP6R*9gQ$-P@ZO6(UUKsUV1KR70LVaF16OROfeM<4JoBeZqQ9zkTy;Qe zaC;n=<0rs5{o=!4{tN-Jdl!hYKpPN)nKkVDseomHyO4_^z+WNwD@ddep{G#by&peL zIHOq|*J4`~5eSjJI`#nEf-VA;*zH4B2k;TKSl9n>mj2R*!Q*Jd|6yNMBnq9clYh&; zJzmMV5?4NoXVY&;&1J9aV!7ptZ-bHyOeK9~T^)|UX8JK)f%$AM<1Er9m$F4DP*on5d5hv9uVx6>V?K=-(c-RcP< zke*IirfXRX#6gmE)dENpJ%7%U0M;$cR3e#~y#tz!phj$f%|xq-4GcPgX$$g--T@&J z*^xlhp)q?0bo+~^pl?}bVY{&ySPe_v<7O;Hc{)bo2R0M!iK!p8vz33#+X9>zbB zthY)z08&g7gmUdK&=v2EzpV|_dZe>seFuu@Ztp;5+gSaSH*HJlr@Zew=nv(&ZDajYE`PNx&4+TM>^lU9@}q5Ib0|k` zEBO2^{}yB{nD}B&|7?M_7yd(xTO098=*;)H)Xvb3HC7R{;9!FX8)NUl!$uBkAuI=; zF2Ok!`hpy0-vP{vwjgo+Xb}*I%WG?i0jT3Bu|Ys2!u%cZ1V2xqYb>X(9fQkN*vhT#Yt-S-4ZDT}`=pSuc>I~&((033Gcf-9Vd|JZ6`5$;6IBbYHN>@)qwj)bmQEAITnQ&Bp zwIdPwo2mb3Z>OO8w8P`D7PcMVTM!9*E#eCFPdWopR(#?dSd%4C#eWey>Ah1Ve}A;6 z5VlCMZ2+`8*a+Jq#vKe&W`E1`&4u2qe$up{sb~op!D;0$McdrMLsPykd!egb>s{0W z=Op~fM^-uqnxEPwP(-XLp8qZryp2x6XMNMcGT|_Y&5vKIiI@|lQ>XgYt$#hvohRu5 zTa!i>e&61K@t^I!sszoyWDiBG!hf>+Kk?A{t%De}!p8f9o^QD(xoHF+K@u~z*6L>) z0;mc5hwnG2cl(Yx@TuSz5iyC^x5c1~Hqb>( z=-Hog(RTX|xOg`R#CVCAzkeL!RuN}I&8MM#2UDw@@}T_Rl#6cz9N{IHiThBvG4L{eoZ5jT0PQ>-sbLT_f zi);F!AFiVh{cxRr=!fgPZN>Z@JL21^Z9BspL-aoh`nQK=ddr4a2_l`p;^<3s<&6T(E@zi3O*?(i3_HMHXwZQHi2{meUUpzBM50dqBx zn-M-MiO9VL#i<2O9U0~C*fEfR$W`?s=Rq;7PG=6jeclXp0PIma?AO12t~tkmHn)}n zgWjD0Fl@Qla-nBsvOHE6kOl+sdn3AH3zup7b&qUrP**P8r*t6~oj^%_yGVK6^xaDyT z8N@#Lw3xS8-MbI7!4==f9DT}dk7E#&F^v~ z5E{DQ(SWEKW%Rx&m?l=fEIfOW{TA zN7)a_O9oO7uA(Il16;i6QSbmN5nSuUJrR>MJp#E{x6pI2V}OZ$>gjMFfgOXD_-~8) zaQ(SQGJp8L`|hxw_v5=veAgEADTU8ZK2>}NgABkE&?Ru7q8x)*+-nSMxcD9hJMd{| z&R_uy0ZSq79S||`UJmvYwt_BaP#5jnF(?6?f*pswD9v_D*ZGGp1`+jf z(ci7;CzAfDy~!X42H)a4JO1H|L%~z<&*OjrgMvMPXFSoC;D1IzE1m~m_cr^~zXosyj2!GacnZ0uL%SK+ z4uA4vz-I6z+LduI0;vA~pjZFYu2%4|5PVGkPX8R*!{7y|GO$EHK?C+;d>B-~qOdQ0 zcz?-nD*k5!-_nm94Jz8@4f;#rY2efUmA;7a1}s58$lGv4eZTelBaau*Q=uQTFMS%m zvlZ<^;A`ACA9)$W|5`lX5Aqt?(GT~7o_|N8{!onfaOlS%kCWifU%wmvt=^9w#Nzt~ z`LZj@$F8VY=tcKsKccVx{fqViHt4y~6EPkJ`4I7c=+__o*YT%a;`iC#?*Xt7a@qnY z3A$X|BL&!8$jK4>xeEKS2jctkS3CJneSC?2`wTiC{S);J_TnY#$%X!Z^pg+u3x7Q_ z9C6=@`o8Ur&EZ<;LHJ`mZ!A0C>v{je6)9iqX+-$H8lVW63|s~!px_3ONX3DN_>Rp# z)}8B z=l}WN#EAp{<+#lc|KscGT%P*>uz$AUVe{X$`?h{O@BeiuX&?RR@E;dZ#p&~eA4*m{ z{$nXg|Fym6|M7P`qv=mTBj+2?2zAjnpi$_F``7AgIqqL`=?nm)l&rxyS0N_oxd+m$ zg)SpCkWB^2Vr7vsm=+@_6Q&Y;(WfWq2ktG0G6|`%e@cF|yJYL}T{%VY+<)(1P*UA! z2kO!?OV)%J&$&MDSOUSVpU-4?(2s>*dQ6GK_k8)P?&}~Ew^p7mR?df znQxVFKfQPdx7LH{BRVYSv#}S1JChdsWwV*mk@|?YPs!V_^ZNnpo82qvDyrLCv8__c zQghqZ-IGa+BJz)Er*zw5)PKCkVb$Np<@UZ(jK_4tkz0q(=O}*fFi;s@ev~FYIqVXT+-R+`| zyF(mF`|HD^%GIzlRqC(1nlGcm40id_E)u>4(|1u^b}MYm#xN(5UVrQ!s~*^%{A$j- z`DheTN$K&-+_%wEC;hHn?`GE~xL>U%pGL_z-xO!_=BHQmvX#zgd^%0bdUP8f;`Ex$ zr7k0n>%?gHi8N!>w8^39R4R|h$BAi=)^lqXYWM34kbq&2xMun~U-i|ao6qy*Rtopa zez!hvAFixSTy&H$hY&PhN)@z613d%8z`JUcsbxH#Xr z%KFD*lhm%fKd)HCXh_!C>+;@3c)olrDi$mENWph@ycT`$C{k5jZwI)U8uF@o9^WJW z-p)J^9HvKmZ#(v>9*?SXoGT9t?A37`p2$9!hx<+Xw%k{=x^zy?9%uVJK2RtH#?ib@JzjTS_fBhzKJNig$*HJQU2^I= zC?2CbW&OxJtuGr%J-d0uchOWb!jV25g%1VUHziRo%YVBwH|)8kUb@tKRe2uD*+!$|>LH(RO>5>UFUJeA zD1qV37k_cb_RH(Wdw1L2e%wu(bAGI*QS%C)`~0ThgS80eZPM92Y0mG~90yM-5_IDeBV)8&jT*4e9y=h}HBX_$94ep~g$ z8~cW@_b)gaV}WcKkP2<~k9;|fZvM4!7+m*3A1~(DcH8hi-%3%(ChRV^OtJDseRp0D zbawT*o7^73f||;#Sj^H!xliBS*uN(sE!NB0f1MW(Ntv&M*D`>9SU>gkIflvA@*Mt{ zb$?U)fc1tK*RWQLThG#>JnfaqCSFGI!=8*U>)^0gZFSTcxt1!$RXTy$g88j|)|8H! zc`MIx`lPSxE?iB?O^zIOHKb;t>nVw(m3OJj7Z_YQB>yznv3bJrVp4L)8<_6a(brSF9GE)nmRa)$$C21Ct z?>2RjEsDUlwYXXO@R4ky*L1E=Yj?%&Q>)#t>iP&Y7rSV6P2&6P8MUS_7M$bV9Dl$3 zN4t>2(K4`4$yWQ_w!H)$(@Q_`R*0mw36) zZC2~kbCO;4=(1N?_RxEyxTKlmJLS`4B`p2RVROG}7uvM_20vqDzAto=uYcSm582*u zw)7Om$@#e%zXG@QD*Fu0;;gl2SRYSa!TbJk*iPn6zO6RJY%*y;Qa4FZK3==EIv*(! z$?#e?dvdlt$?%I=g`=wV;rlc(;^pRAtt4NKeEq(z=XpALOwTa%lhbN`&7Ofg&)9Nz zzfZzcd8|kKpqe$grv{Q;Nq+|ert^(ot@o9)wu=Qz+u4D>Rp__2pP_7=Zq3W0n%mOj zjNmk;`d-gsejI+GYNV# z$4}4l!D4+iSj~H9emTW}_suE3oq5nqV*U0G(re`1#$jqw$2vs1)HPhXZGA5wZKk}i z>73lR=2Vcd?4(n#)qk$guk^)nO?iI6N7#!119G-N1bjvC@#s_WdYO7cdX(@M9 zo#|_)9vXM;9*clYmB|R#WaOma)v@!q43+0;Gz%v9p)RfaVJ)-SrG6WGBhujOUUrLR z7)E%$wfrXU&HKZif%`3o3)61arCX0K+WIuVRJ`0TCsU@3uYXFVaBK@b{Z|$YgtfF}da4TI`oFW2VUF~&iWCQT@ia8BL zPN6s;=LpX+sDDvhdg<9^eN7`4xuDIhgX&N7&1oMFho$HjC&M@2lOH4cJiNT|{5@}u z6+hQkNDgLSOPj)k+r8aK<*j>;E|bSrJ7}>&vTHIgGJFGFbb7z{hIT;Qc2)K2mwn)l zGyTxsk>8u^lW86bt%#b@oNAhcXL^t(D+g6)bJJV#4S(3c+X5-1`J}n47f?Mr$BNV5 zn{Dz$S2EnimxC!uqs7(fR=bvuz3euhZx8bUZkg`maPtB z>s{VkqPd9HwYd&Vzd-q6b6hXSGTa>G&E8mMt>ChnZ91OcAecxf8T3uxS2_Z z$TqrUKkdeuI(`$3I{#HfIXTSyXeF((cQc9>H%U{K!??eyi+FVMnx$_*?e?5y@m}*% zB{`m#Ub!M_u)SPd1FE;^HhLK3LS}sBwzui3ntx}Bzn-lj59%?A^mXmNyiAe~akD;i2S>CMUQk%`jhZTrN z^V&1h;OHgoLh=o;Z39&Sq4faf@)f@(M|tY|W-^^Li`-rov0j*l*F`xpwy}jS^AqUr zY=5z7FX4^r^DTZ-XS#ijJZS-TQgYvqGW~WrUABb-Ux#~lP0_W$;pjMh?v)_3-p)pf z(&k{UiutMU9BJHdXV7Hx>=+e$t&NM(GB_r0mTx5@MY!9>b8W31Q+czS)H9VO8wo8B zSJkO0EDUc=uGhB#Vm!&%Up>g4|FQ6+muanp-k8niMx zW(G8^a$il?>-G8~wPh0H(X&-2WAhp)sm7ei44!XE;+*g6bEna=-C$*l>kAE}1 zLyIzCU@?lf2p;O>5=Ax-ru}3WKaUUco=m%2TW0TxwG1{6)l+U{%(}<($TsBjbP>Iv zvFk_7*Io%6cKP0R9CiSJwj)rY5?L#zL2ePBd{%y6IIL~nHw2@pq z%UN^IPTg#mY%j4rj&G;rS zw>zKjf)Sa?7WIYQ&k=#o^^>s-glt}qYzbhK`5mh?;88+UrP zzO8lKJ}+llytYkU`;*gZ6MwIE$J=4jhoQO2!&5VTA2#u2_AquR*=ZP|`SR@d=H014 zR!ez$T-;-{L*5K6%E{(6r?Y1en44Q-#AAQaP}#W^<9dJIz_IpnmUh0scO6qE=OXO- z11{I@uH8LGuNgdS-^xstVwGZ3ao_bLd0a2A@`+#X?9Qx9F>d=yD}QaDa__xQI}1`* zJ?=--{p9G39^loqP2AQ?@4Yt^qn^p_>hU<1K_2xwA3M|JdMPKXTyYcaJle0>;b6zr zLS(%veHKjBtAElxmWFxTNwxutx9{epjWEuS|pIJ!LUmgl`S|8m5K=or1zeYaa|Cz8G98)a^8 zP_13A`(-yxa{n~3eSf^T+0)Y88S#F4o?Mca@7^Y#JK@FcZb#QkWq7KStUVg1?ds*^ zJnhiy{2pC0{}GtSx!)HMGCG@QVRNdCFma0A_%`=hI$NC%l{>8iX~lofPp3GtZl(LF zZYfh3F4}p49HNY$1%iR6&9=wscHc^cn%cZOG(~}L@R}ZMAHUdqKHsI)42|c8`zRx2 zHgU{pu)IDj%Cmb1Hjw&h?RCh}~YZf=Vc zpIU%hdQ@h+Ub~5%?9YFxTee15P>)KUyIpz*XR?|l@e8I4HXjE_E#zd`GW}A)go6UB9!5h54-RtG^dC#soChpt8jgP|P zEuN>(eswLJz1&|WWwm>&8U}&XOvaX_UB{L)Sp-`&9?9!?W3YdDKYq-~#xL^MigUNv zM=3l9#61Rea6YeIrn*Gy(U{7Y<91T|?6%upPAa{sJ8<$3_3?gtZ`a{<>dBitLGNR^ z8AW>vdt=g`qs4T`?AmpvQW!X1AzwXrz6r8JG#aZ5C63Ek4R^RWjQXIH;3>t;d2&ze z#X=^NyES`An(Tis&pWS-F|H=Eg6LGaU$3%{G_O2i-(u@>^(^i5!QITyT3?6S3XP|u zzEAsQC~fEA^dVJVbZ+9ElZP+v&l;sNkYg-o_Q@L`+rx_1_uHm>*^fuUX0My|h#&IH z?8hW|&b66oUxW7LkcLbk-QcWYPSh+g^F@3-u^mg41NY;0fqvRs8DFCnYVWxs3;5=nWShi{o} zkpJ{ETMxP`#dbI;`BXn`aE^_*GP+Z97&p~jtIvP+#))q7q`*1};wZ)2t5-VHT-x~O zt;V-exz?n0v{*ZL|5B>6NZz_nyY#)7q4}-v0lHc{V}1`O??vPs@Pob=k?S@S?D=}U ze{R9)f3bDm%Gv_Wdc7b5ybBcIy?ODF;l1}4SqqEoO^w!_kA*~D0O&|$rcp+g?q`sW~ZZ$<) z!G4xfheqaD8`a=e#6a;NdA*1D;l+`oJD>Z>v|wI zZC;?1Uq*k>(W@-qAIOoqTkqTlM7_k|1hm1?T6#XOytx^9kO^s^9)6E5kLbh}plhJN zI8Pr+g#cGjpE$Eh6bY6-8z+8}Tb!nYOnK=w++n1#sGv07%ZZ)^X!fl3^d>x;Xc6|b9S2bkBEXg_&XwVwYD_y}cz@>WjRsT%3;`BQAhg*I zv>TATMySre{@&JGseT7Ex5Lzi8D$vEheL(es#5<^trss%CXaZsY&R@t%F2HNKNHC! zTJWt}aS2lKX7Fl~=21OpBf=5=69efB_RF5%Mq3QK`k@UmCHHjxunJ)HU z7CH27A0NS&xbvhP0?SQr6oh}L#~plui4>KKhBApj)*d^{xgCy$5S>F(Ny84D?((*k z)WrG(b2I*6pRW^#Rq7FOABM%6=}y~4s0q7QqFtX$&Pk;GtLH^{+{`@D{7llP!WJLm z==C^z49x$otChhvzc0T-AA7QGjE8A-@h$<$fZN!Q`@FF5CRs z*oKaJxx}WUHmGIaaR;`L1uZSn+(F)(-Z;*PsZ+dL$L+Vs9k@;FY|}cvcu#2rz?JF97-!s(VVyA9vMaNjw(F^?pY}E)%V~@~lx2}& zuCqe|nW3L&3SWOvLd%ll!*s|s9~ehg@ad7)UGIjbSG0J`&cSw5>MF?TYo5am()0>A z^RtnnZ3&1`uG_gJat*8ddP<7&w@8dpNEb*^d|B@>(FEKJh4{8`gH^a?fTm9|U6JRq z85^KnHhd4DhvV;D09744vBq|2k8%X`Th>tGE(J}A5T0>#N>iR zMqV<4@?%5fv<}kg+BtTHBYiD52b*i}6La+m;43ZEcFr;O>4Q_6^9>)dJS=b*N2F7a z;C6pn>=^~>+{^S83|DqNkOK}b?X6%}5-*NT9*S^8O7X%d9;dx2ow~BPE;WQ@AM&DH z2gzxdJ=7vdi2MaTA_=~&1ww57l01m3o|QR+$eZOmE*nfu9mKx$B4s2EQb`<%#+Ecl zuaV@imAPeYo&B(uv}G1x# za9_IUJa3ub1S-~9io3Ct;SU~0W)bD3=Tfr=pyxaU(7(Hg9h+=iXR9TjkO*$SYbYN` zIis~jSVEH+A1|L~1=2h>>j0NiLS(h_VjpA^>gt>a6i**%sEPtq1u@P7-^+EWv-pTPuXI z7oG2>=cWne<<#Rc16po-=7%^`5X?0eMwF8xsmh*E&>kF( zxh0c??DmqdId>eQUo&c)KMQ}Dwg-oU2V5ou+zGj-G44HRZimuzHBps0QDtz|l!OtF zx%e975w-l!E9@#a-aM+S)=wQ!+|#Lm*U8;7!0M3V<=>e`1#o)7F^f~YdLzEIbWj7o zqEArUe29Shfz*b!IGAqZ1CpCz2x4^)ph2h9`^VEOF?C0bd`P)QK2CoPCn(~4@KLd? zC#v9nkm!#><4%h$Fm$%9pq)5CW5|brgt`K)dq&<8g2jX#Uh``dop&5D{QN$Cd3hAs z^F78q-T(-&qI&VIsCaMnG8uEhF@VB15WpV=_3hMEp-LT0ScG`_180n2~`EbfpZpzfB1L@nFjT z{GJ}9?6JchpReC}s?Q|2_xQ{PadV*U2})Q+fvWUA6=9Z4^)Amso+a-vJoP^B|VE(6EhD2jSUAQMe* z4L_52YSK*dRWLpT7K;RH9{a0mwfbS|`KYQCJiPOxbQwhZHW`va?&%-Mz=Swga^g(ZiU`eea>idWhdHAH-ESq(U|Ara&1u9Dyi0_&?6jO$1=MH$%b=chskene2M5zI&`U_lBvp3^MRL6JhN4kCB>JPm4dWQON8lL zF3pl^<8f`F{MtZqMTS;O6D-(h82YY|`2O`!#k{kv8U0lByCIo3jWjHw-}#8VA?ac2 z^*=O<&hwUTU&*`@U)3am>xV#G6zGCUQ=W>?*@RGL+MuA2(OSK%K%3$nA%MQ&?xZh{ z3dw(hQ3Gr9E_trJvc7T{efh=2W!7o^GzBP9i*1cXCp+wX__@{|?I=9hb)S(Z8)vb~ zhi_3_cY?*9;*qCL@nHUJ&|Ejg4j83>^-FaDn+zyoch8H}89FE8;7D1UVKuY#$S6Ux zYsO#Ncr|Y~EUgb;B!~skxR(WVRS*#=m_2`7E9-xe>*PPGGec%2aZ2t`GM(l@oqFAk z`xUHkDqlR=bK=WG0AGSj&4Qm~U4+v~#)G`Y91^+uH5t;xKNS5Cl|tOC-W<}o-e0+F z`tsUZxnyea(sKsrlzbK)-BcEGXfEY%918jkkOC~>NXx|vj2r{ zR)QyQTk8TqIZ(o?U`-!*yZEhO9vbw0nK6%*ocLV$VIqa9%8(w$3C}#Qq>NAkvW*vr zdR@gS)(XnWtPV**j;c9el8L{O^nQO#AnI|(MjsS&p?}ufBF!3fWwC_FIBfoE)GqW< zQx|5#Vg|L#rNc>_j3N?EP38jVS0=D=`c6S+_M3enup~qYYdgczrjDS52E(_K#FI7$ z0We(}9?TqS1{90EJnPZOB zc8#oSd~cm>6x`%Uu@-gZL&0HjWdK=fYz3Vt>3-XLR*;Mwx(0l~+RtEb&mLliaCv+i zKAfUAADPs`sx0)xjA0G9MT>uO%2c=L!}`(`%D;H|Ee@EMu0%8v#8a6%AHW3GArxFs zasfIw5?!9p?i({e4z54%*Ljao^_`6F&wlLvrID-@ieMf8~)PO|`K^ za>c{Qsb5Q-9<=U+94nRyAtxDN;erpb#0ZKP*t;~;J&oW7#>hp=q*`ZKAGC6Mz2a-hg;2ghyR+tBlAX^ZrU1x=^K5Vp>fI%~aLBTex+< zre>yl_!BBiVt2BNF~$;`C@2}teDsZuuaJMj-p*@^C-px5O_<*5%Wi;J!~zUv8#)l+&Bl>5J3e5B9wDKM!jBdWLz1DLbU^{h_H1H84o=+*7G zCxN^x+jf~{NZ?39MvTVvhMjdD^dNPA#N^agwaO*&^O;4NuGFj7X*1WyyK$Gj5?Hjm zZ!Y@drOt9L)q_8JdncNMD1itAwZgT+y4AiU;_HhPVcI8BF_?c^FP-(xDyN^~XP8~? zs62l~xik}rQe&Ja&JXHf;z2qggs$T?vr5ljxv&rp=?<5T3FlU}VTMh3O(t3bsN?CK zZBf}8&^&7+sCCaVI}okGtmcyky7^~?y!3YDjiHeSbyn7%MJPCJ+Q)t^REn;JKyy(? zb)Br>;YvAsr$>Jh7z+}Pyb#fMD)MqJ-?|mpkWzm5GGxOvU!Z3eH3iYw?X9l+X@A6$cCu+IoZ|(QI}D`(9yn)z*9>wc3gqs zrw1zCc*{E0>I)~LzgStNa$=nK>(0b zf`Z1S|A>G5%Q3bJzdW?mX~&8udCIArq)_~3fSV1_kiZy?UW=kkGTBzWs$+@AfZ)2T zwQ5x6;8eyK3+xFF`&{lIr~GuhkN$At->S?XIvgR4G#;MC%W1e3WAJjth~x(rmbf!x zNbj{qkZCvO@WTc=CXd5WiHqF7r%IU)Ey{BC65)S*nJld{1bu8vj#&~1Haj-*Xrq+H z+30nByLseF3r_m64BARsioyZ6>Q?j zSBKEiHbX5YU5bpj2-tP6{Pbd$I15QiS-(}i2S|~*(FA(z#E2Qelwsk{iQof2099b0 z<%)l{(=7R2AbF{-|14pI>K{I=QCT@D9btdC%jK2#hsUtvs%u%_!0U`e>M*!mYekQ3?!0w=p|t&K1zQ*f2@l=KwF_KGWvJ)j!U!jXS+=C157P)L9LwD^Nk~` z9K!tGqG8lMyRz4EX~B5e&QCkL<;f>9>HUbJ7PK9rBlCL+4ufDe<%@w&BL6(IK*E2b zX4zcZ;%V~8(DI&Z7vl5+Vi?0t;VfX%iZF@uB3dY>YrI0*)e}H5wSeDzZP<7sf5XDj z_5nj_QE_acc}bbQzh(DvCTIh0i)t%+w4@yUFA1vl7nI6 z=<+4_6Y;dSu+8zG<3quct;ug3&Ch?Hf;ORmQ9At)EMUk*w+-VWn32B$#8115Qr=z% zEC%Lnr}vKjc)*qA-^w@ zZA1zAy>P#UgB7lb;u{%DX3NoPPI7W0bwD$*UA}pRyHoMuf%lv3yx)0n!|#9QZ%@!= zE$UPdZN_yYXY^N>Myi1|pOBCTstS?j^XYw)3|7lTo6_ZH`xZEO8Sam`A!xiFnm98V zEU90CQ98>#G#v1Wp`tH+1ojv_9E#zA$lNe8lA-p}@jB$2=*OA5O8#Y|V~$6~-COcQ zF_`0Ewwy=#f$zY#go@1%PbPoBdP8Umx*F~wYO*Mpyl<9mne4AxJ zGMqbQ&xgh3D+g_2UA-WkbbWkZwhOBT)Z4gB0wFX)7X5}04P^_{z1D+t76!L6sti6+ zF7K8sT=4OU`O(dAAjnLy7~jM`h|?hosT28PQU-U2Q^g)jEtCZLkfkmpXuu#wS$T-E*9h!Gmc>2Q70&V{a&B*yJF?PG{avW*&>ej>MpK zx7OTlF1)^;_H*cpLn-KjUc{w1z@KQq0z9rZ7d=PsUG@lsk-**f(w+ph2&gw;>}7Do z)dk#WmdH^d_F7VZPHLU@S8{i|myX>LHW3L{ej^#T`L!Ih|N4JKALq1D=#F;kA|t}K z;&7fQ$|idQg(J|hx;mI49@Qau^1Dnun%+{7w`>N$=J)z4#3$7p&v|4NkB6Q(k5VdT zJs~sA%1uk@4b}@-Isk;HHl#*cQL;7ARAKNlfao?nfNrF&6w*DxP>^VNa4OR78MZ*R zO}kI34f`6aPcVOSymp(gM$A3@B+?xaG@MMA2!2R58gfz~Wr9y%t0SyIh42lee(|gG z-o?H_Dy9TNhIvW;3MqdW(kU=5(h0>JRxVaZf-=6R zqR8^51rJw+9_}!q)%ajr7bhY1*DWnaQL8>a>9_a1fElw51yUPsCq4AZ6Lv=8d2&2c zD%t4#oz;>#n?A{_JFDztm8~?Yr8fD6yMz##*jk}v>olAyi{vl|48N8`38^aZ%S(-l z4C4W1Nx^@Ck!&lGUon&`j1*g7>rB9!9>!Y05j*;_Z-M0P(GxAbVV0*4+RxC-<_5X8 zj30zEm_^?ly|_iv>2XAZI!um?+|`L1kk)Amr8=$!OFBl-$HPP?g=hSch4h!l&;%|7 zr?{PsWo*0)9@5ZmJ}znAXB@@Yl-RJvw7kFbVnTmOJUsoZT--Wmh}~5-*7TBMG5WX( zUOAfsQNA5Vi$S{)^Eev*Du7i0bHU$YA-yR~ zP?`^K;-WEG{Pit`Hq=#f^RLdCM$x1?4*3RSgTl$Cvk^a0?t#Q9yY3EUIg+)!Fv*W5;Wkxqqh1QFc zy(&5Chc2gQf#goST8_eP720%s!5xEW{DeRX%F>wXG?qX^RHbyloTBCvPV<-M+FTw8 zmYLJ5_En!Y+79-EvtNU=u|ca9WF2LlOkaN<9^2vqB~`TCA^}Yk=I;{q@RIy!k<}v6 zIjYu74J1u&ct%gln`2TlYo`Z`2wn6?k^jdA{fUkhVOqD z$1$2ff2ED^UxvB;M<`|xi0R-lmx@0u@A{_)XSYA|IA+8xNtVce>BRZ}u!6b&!t4HZ zm>>_H)c>$F48w&8!!Tde@SCOoDa6H|VVLQkiJbrIk3nL<D#1|sN9SPwnii(NRw_$)_z{2KGsJBE>iyG~8~>qqn0>>$;eT}F43|;=4F={I zmS>1N8^z=A`HlGRCQ|oTv>K(Oa)LWU&A<2a{VS!{|3K1_;kyL-ryu_-|DUG2ze`#% zyZf4#<{FfBP$O`Q948Uz&M*k-l=RDRJTZBw3dut}1G%1f_-Arj4^o!7MgD)N-5_t2 zH~y3|pnb+vnzR+QA^s>=%`#{HNzfFFUbPuXhVx2FLQ9zvcZ(1YTh;5KUJCn57abDS z!1Go3&XbUgP7+BAV1iOQI`hruRR-59FK;~lDf%Q6{j zP$PgCb1p2bx9|uc^{y#K@z#Hm2@Z=b5EHB$kSo~w+K42I!x|2IXPdbo9`^65Bu6oJ z&GYl+ynwp)>)6UIzSuAtXA|V4w8?%H)?COiF;!6Pt@nXDu(?5|8x;?ZLn|hGi|-O# z8;#)0UG@QzN8ozF+Ts|1S6L7!iD_*jYLe_HcKPJ4Hi&eY2&7}#0nkR`h`9{EKGd_GOG8*%-YvEnv)uDab{m#j7EK3U13Bocp}~bz zNKr#r5!oL+YgvEmO*P5kD;adx}5D`tG7oXH3e0@Qf8s5AA8)QYB(C=u5~}B ztPr{SlfRl*@vTm+T*xVJf)okc>B(PcyCeU5yezbGoMlH&jcXszA;? z)Nwo_=TeH1zB7IYAuL==0sIXY!=KWOhV>>vF&U%NG}wO*rGwz0+Q*)O@JH8R{1pwZ zCl`VXgJ==wGwJ~%Ko8*3P(*qG@7p)~v@F?L^~UVB!-@(dz-QwB0)UCd3ke8%-okbO z^!RB68l=d?5JAO656x6|cJuSp=bVdy!E3f(tN+a`5c0!E1C=cc3^DlmLD!~1IU*&3 zhz2C~)j5CfW6CHn4Fyn<8z6c@2!3V%IoxbClv&0MHMBJ|m-qeCkmx-JwTP=n7DaoB z2g8(624Fw@@eN|>LO9+o3K-4jD3q(s2l5+0d&e0`Nl>jLKe26BA=}i0BiM>VN47j% zYD6ge5LX8g7O|D{wO&uoP=guYHB{QJ!n@J#MgY-@dc7++>Z*A@&=8~gk2m{Vq{ zpRQedhpR2_m3eMj{p9OW%0u{q`%!-|S?qtgqzamMahX}}eA?mJ$REl0)`W7FBY_=` zT(%CCI1A`p97U|qiEE-*2VRlh_w-mgSPPPM1`wYXL_F;E7IndKr(6XfZ$;D_4i z>omlpEk(4LV0%o|jw|#)u+vt;NbE5dl1}>y@t!XeRk`BJ)3#U*W`CCilzz< z^pWP4(-7TN2s;$x^tE_ZUO-{P1POntQf-J*%80g80|DQq=J(v|2WO$KAy)KUg*81- z2m@y(hVeZ~DRlGDVSQ8! zxQA*FfS;3Gvrj+DlBpS@v9FY)*mQAIWo&0&4YkUM*+NwgSssC_BYaz6f8FSO0SYh|<&^f3qojnG#YS@iJc@MU0hSR+Liy9&_41QYPY+0( z068_$iNlg^cm~U+Swd~xCLJI=cKXEVGge%;y`>_)0xvn35ukvslvOb*rs(mgkUHc^ zA-DXfd(EYA?aaR2@`-=O+YO=6*)w)4I7Z5YUPfa4&{NmF7x4(ZHP!Ux7+{Bz%k4vwX{X>CFKe8c-^+ zV4-?BonT3R;=n!Cs=?1s?j@B)_WqX$JkST6w#zZ@b1#2$pgbI(eKDlK0P2EPB0L=e zKrX?2W`Y$66nPZ9iH$B00E%w41Ky~YQ#VJRnBT5MM}6iaJEoE++)v`IRTF_^WAoM- zzOGU7dn`2iR@iK$*-~%h9BDDB_!Lm>k_o1=FR}~cz#4*UyYKR#x1s^p7~lGoRUM)o zCIq0(;MIRmr;f8-9Tcxj0J+;F^ZfCR)E{%>T6S;6-D2fqpsLQs4uQ4U;18sy8P_Ka zU#>#R4Eegp)n%?oc67!hH;UbRiHsd&6SyQ_ajG&rGh~pcd~xtP^&Aqws{(B{3g969 z;HRG)sg|t#K#!~6#3oG;=E591woL_CV^Omq3B7-D$Z}o9TuN|Nk{>O#a=X;+;NrsR zE>$qy{Xk;cwli|{?27qAe6+mXA1L?8gmgDy`4ENmfl$qeaLzT=AI>@njol^x`oG|C{)9iosZ^ zrUgbxM865}pmKoDr~KT}BSjzTd&Ubvvb=wY#2U0&^TQNRB=}f;+BP>RmuJ^a7wF#5 zrB--F$6qA0Q=pS%TR==muV4uhn!-0cZ~{X5%W6oMV^hFtBlALrI+H!3Hw(+3JSjW; z6#nIL2XC_qcJ@o~{F|yvP|D{-tG|p9!r+({O^`y^&zswAYH<)2-SF8W@ZEu9<|Tjh z;2YB902G@JpPhOSW!E3lxR`T9)R=^h}S4cnr;{ml0!UV?B0~mQ5&@F#V3_VRacwC)# zh{(%EAUqr36pdVR{@jf4PWpBW3J!|k8}J0>zbx$gxIDpdtQvN@{g`C5zGHue(RSZP zyK{zxxwSV%M`gV~^B&MC?Yj&D-%40sX@>_;PHzMlduT>AEJc#DTjo;qyQz9YD7R0!;4J?1fj&U! zuap-tR*@Fb2Dyl42wN~qEQ#}=I@QC7rwqm5^-+^R#iS=hpu7P!q zNW{Cq>Z)3B1KxxoFW}GL=8aroOy`>IPIF|O7~IOhznHzS`5d?uO}Y(5hsQP?0F(b{ z2r;76tw2sNk##7w(?sYQvpFmYjZK!zkGsRQx(@C`sO+fnV7?`|)0KaZcJvyu;h%TG zedv>>Y^fMB`YJq6MZ4k{h_c-`xFKaaiEIm%BKJ63p-4kP)<&=U6 zNm><8`mIFSWIYn&LxUU!7a%Ath8JTQ`F5mUG@`0jcvH$9ghVxc-2VE4OZ4{cD zy7$gUoP@^D>UmMN0&4c)0U4f?EH|L{q;Co_03f3=m)PeKhnEn#lycagV}ZJ3+;?>Xe-htCv@t)jaP+O85* z^VrjHMSwL|p}C&2*X3S0a#W24?9VhT7g^be->h){8z9heJwjJBC zDLLi}7BZpAi<-x;Bb}K_4>lB@WkW4E#rUteLoZxLIFmr?ZO@;WSTNx3nbtK?d>jz} zkrsI`4nDd=<@X^{7r#@f#A5~(;o85Bs8%)oQ{c= zrffJn;-xyOMP9a9gN=QGhFJm}+g{$E2$segjn8N0nmKwmf6nNz-BLQj+D~h4hEY0; zgs00p!}x)Rs`VO$cy<`WmyEhPxOAJJaTQOQA)SAVmM{c=pmV_4or(csh1XB_Kp-0& zz8fW3FpL6WqwC`a?TON6UR-kungzxajVcbS>#heg#6t zd?sc`b9{26>ekxjf&H1n?g0_L5gXuQTjwj*aJ*g!xx13XVhK#r36?`j`4$^8^Jno{CC5&E1qqD!(!aoEp2*d(q+hj@`$6T&6t zMv@o+@l5Qhak{jE`pYx{9)#qY@>YsOCLWXbBOrIqyCE5>ct>WPZKH3IKJ`x=`+_DjCO$XEc*03dx-Z9=fy82V( zE}CP-LQ8rvkwQP68gkdSj*`=$lm35zn^jKykb&RFCIWav4+0d1zMD3ZO}I~QEUoVR zf!6O_IY${l5^Lc!igaS$l@WkG9?o5AWiqsCT z#x1bdZ z01AqQQeMnIKY+e50I-m{n=^k$jtO8gibph59xqwh1J#DDz!HVzVyLY^>~*~^XE$tHRTkhT=vva&r@*) zO=Oq|uhAIQ4Pv0c^uK@U`mQZkm1WBh(gPmBW8%H{Z#dyWfDq>Ed!V)UKIiuBu9vDo zm5R*Bh&g8jWTeva98M#THOP0o@KbGVw;ta!Br4>`aMT(Sf8cZ|60t<5BM#-c$3$8& z$Q)ppvgCdb8^{Mg&T^oeY4T^{F~w-4W-ZluAqveOaxJIz{bYZQh0WmNdZU{z5!;GF zqMsz>>+IT^fIprnHN!I}ON!@hRjr>dF`T>43U{CEp}4a@e#q%5WK&ncwr?)kGb}SA z6I~7C0QAyNG^1jNroyYio;A8~2~%s%*KDbZX?Kr-OMQO?gL!2Y4);Z&#xt+&{^+2^!cxjqnD?8y4-AoL#S#>%8gjYkD3DaP|bn4C? z8*lG=l{RWerBS6kg+8KIN};~?h&9AFohZwpn~H_5MU?;6HzqQuM%bkdf{+e#YrFincx1 z>BR#$3avJcu%g<78^1M_yMu#Pr;(629>EyHY32ZZ*+Xip)Z15YFETaS_(^M%)o?Pmh^Ad6apV@ zQWrdN?L~j%WwJY?;TaGy8*3qRg3uoIgZ)Y%n;i)TeJM4FTDim#z=qmweZsnJ;LTq# z%jt4?#{>$GvqFs{ct{8Td=1%Wi-s&RJn48)2xDEfO`4Iq@*jtVw?!?j)pM-ZG+7dw zODsDmsA;{)$t$=eNEYYuPzOx1^RYW=gOk(Zd7XbQf2tOjL1$hJ-jQKD6s6P*kK!u% zE|;`DaWz?H_99z{sO95&;aCe*_#KxI(v#XnAsKw;RVT07$QbW!QJnnfHlkRu+D}ag z{q$1T<7D+#kqj|4ef>mT4gBEwi?SQtU7 z?+kGM)8d6r0yKezhQWQNs z3Xi7vuSh^NmtvAZyC+SK;iqUYynOf%+Mj<3=R5~RU+wz|(JOpx&%40{C0Qhi@cH9m z=hoc6I`hHpxzoH7azn3R0&4j5WqD;INPc`EjAJ7R#VEXlbe1j0PPwp4+GFY*MjhuP zVFG>F+X3r;qwaH(ERVsvtnD;yE3Rx)JDHy|;hN ztg%m$J68|O+{X|EV<4AcoMMvf{SADom*v6>B2%CGehWYyeVp`Ef% z)fct9Z^N4y65+u4Ic&i~M*xx{GGJd_qRpp2cuA|rB|=P zPVN^=-RHA+%aTC)&fOuUC+vu?+MV05p2`Ksc=`B7Hvx3Io9rU#MhTkpc@QRIZtMCo zx~J6HTzIY0M=%am#e6tb9khSf6B8bm^C?yRK(F7B)AA_?-?@zqIYaPex;(4vgk*oE z@5YW8!=)w}WlK&qJ>w}UD zT}85Yi7QW0(hv^h@Z4-}J{ohLb-iFMDD?o;cP?SD1F1QTca4C2@63Np#(_+(lqfPH6-!Veqw&YCjqy=F*Yf*cCRaJ*xvks7t-JVCU z6L!etK<(JQZG6uNlcVHfH}V8N(iTD(jr}4M_!=fM+6OgHRp(rZnkvph(R?+R%QT7G zL|GC}h<&Vd9cMn;zKnlXKriiM1olZ}e)>sYX*M-UjeJZZJ%m-OhU8OlDLv*UcR9yv zoP+twLk&hE^0Sb%-M)HunTd`T%kTC`yJESl?5nNz*Z|h-xs-o*$8(pt10UascF1PV zBmwP!)M+Lw&g`<%R2ytllA3MV`qj`DA{M}NBwGG!6Z(jO)g^xbA%uAP(^#<3T#tfl z6IfYr$mX;7;L65pq?cJkFY?jn(F1toV8{r zmulvre!VD5-k?9A)9ecogQU-E8<$l6LSeEEMI?Q)At#c-sYNsH80niqu<~WTAAwTb za~bG}AP=`!?6ZG<+53Dn0pyN;+xBP5>1m`9tqG=&|4|ET;LG=WCi{LYvhpn&Z4kXO z{i*4cW8%YrWgh{hSc;inno}C+O3==?!So!CT|P6O@BZw`*ZYHwL(S;4mz0jRlvZKU z?Sde(IV@^;LJO&$s7U)S<2zE)Vvo-}T*f|~=_`e*?HPa3<%AfVe!kP!nx-`}09jM0 z*Fk5moR+L0=bt{gU2lfy@8bcqZ2r0ew2R*(q9pSpkno~h??G?xR}hn(of(u@GejfM z?-PAyQJ>~Mf?Hulnp|(pJ(AW4D6I`9-=o{N9OcO=Ff^ZY5jPm2C%x?`GSJ-^LY%Go^Hm>PdFaF=XVCi9Of8631&p4*!x zW7ae3=T7Z{Hoy11(NrPl92&EWN7IEU_7t6{!zm{oT^4ehS6@BHC|X?Mo0p5#F{0IO zFNEy0j5N}ai}bK!rz^}J%PT5f`lQ3BsU#`CA47li^j6MAaL`o!ov@st~mrw+!e^`4{Kl;=|}lZQoVLLdNareTIZ~@lr==j_08;o#ss% z#NbsMSL`H1_%jZyXU*{l4FPV0#Ir4Ft}=hU#4B(S$LKFz3J!*UrB;NV08ci{9GxQ` zti_6~>b0t`FRVvMk{8VP;hm-~9(?e!)w6II0t*x8ZMQ3=*LdJApU83#^#pr~_@u$T z<@e~vLhb}TC8*C4K1?wPwiH$%8rly`ov}d+$PF!HnLR4MvEeBX=YeM3vPQr3Q;~mo zB7*ml*giA?mcpgga~hp+L!7=)>r6vNWP^Z4312EbU1jFw3QM$6-PSQ2U*cs!RD=#N zVABrtO3{MsK^xD)B)4x_lhg39v?LWSjU*4$B-ju^H_OjMe9KzrX?@cD5gQ3*NQm?C zfN_9%UrE+Uo%XT#9g?_|T9s9etVl zUJ6`ggi$pBM?kp0vk|t0q8u5QB2niBOV%&nsv{)(7H}E(s&Fi7Ih7TcIq{Lcrnx-D zi%?S4QYmZdJ$*xD2_O)ZuGDc?X5>Sv(%d4MgLYVY0ei7oAy3ha9$E zs&AlB5CD48%AHzVWcFIx4f804s5U{FBiO9$qC-B@!*hIrlC3fCj8q;|c`dRUV6PUo zrasws)hM1K?ik(ySqCML>sZ@aB*aI5lf9o?k{LoeFO?L~HKK8yRRhf-ty6CznB5^R zH)3vB8!=pnhDp$p@rbW#(!RJ`bn>)@KDUUm5eBFR@8`9&93_CvF3E%h<$^7Bk4R2+ z0?}1-KDqRfU~_|!!udJ3NpE>&rhLop?X3*3qwX;osrRy~#azafUJf+#!nEOk!tZKz z>(`0He2v;~xI(_Hz|a0OwPIhphLiG(Y<_DVZR7aC!Sri!gQtuWS@On_P4vPvs11tg zgGX9A!S+mOsdH8WXISDwLO#NK^qiZ? zqFn=I?QzM$aiBi*gVV`E45WU4MA9?@EmjJi_%N_HNG?57Ry2V5Im-OK_usZnSlKhs z8YjUJc^;+(NMd}9iaft-^LWp?0u6doR{AR-^D(p+XU3kAE0 zV;o66+Yg*YcDUI2_#r*n%EUDNehBr7jzazvi1IWJvrh;d?y#xct{leecwUe&n^>lM zU`UZe>$Z`;sxh=?9Y>;ntzTEcWhglwAktg&?2ginpQwjCvm$Z#M{*7cBV6qBYjt7 zSvIlS$g=J5Oo@udn@4rVc8-9|9LL z2w^8K8lBvHnM=A5!)ZP!O1=4^_%9z3z=B89$N|GmfuxpUp&zfIb};_V8{4>XhB zn#SaRWj;5aJrsf51Ail!m2#Y(I}YQpiiR(Eicox?d-1fAo#^=J=(&kim?odfHy_J! zc9RTVjBtbOXD5?FE2?3Qq2HY|015v&llI=?B2b|D9UBY&-C838*7)s<0l5C3?LIQ# zE8yPW&HGqM!v;^A_kcGsIS>ty@yuRHAt-TwRsxC)#-06CS_6)li)@1iPn>($cERvC zpX~0+*ptxN(R3f2JMMYJ;Fa{;Yrrac*8Avy=0QUyjqVi{Y*-KO3|Mq4PXCSXyf*LlB_PLpX-ONSDJf3#BNpux_Z`)RT%6d><*qot0pZUK>6~UaoPhsqTRT>^cvC4eOjq38Fz#NzAep=S2dDC1#4GQQuYQgyZSml=^tMvB*s= zs*B|JDHZ^o99d*|n=#i~K>=WJv_2Kb=~$a#h1YSIo3@F&IxsK}uGFo68By4WQ#2td z>QqMN?M~8Tin#_>apcW)%=+NHWWM5j({i={@_en?@y2G1vzklOC%dV-9`m5-Lh(2n zfgk?0KzO9pm1Z0j4+`2Yw|m=~)TWD&MdjzCb2U0|K^X{Y3WmI0jJQ-`8rF`djbg9p zd8%lql*NNS7i*W`#5a3??15HVrzp=y@(FZVKX+~fFt64?4$;Nh#YG+^3)5 z9c@BQv-X`dVx5-UeY%WKjCjv+Czdd>a}S+32&FU@5vV6jr83=5W#3+}ecWl(8ZwU} zh9@w#ED`25JC3$5F56}lqJsl3G3RYJ^W?RzWud;U8f3X`N&uf)U{RykMopY>vTK zb(R&@auxj*E!uHC-|<4jv#tAl`;OhM8c1J{40m>#6HSlrFqpecd4p!+F}^ zqM=%d#)9rAe4$lOi9yBS#=Ln~>XqgY-~;>GE8$bX_LgFQI+UgU0^$Icsl9mdZtK{? zo>3VC>>z)-Eu&5qhcm{m*;lL-hddvMKgNPHc?fp2Q#!H&QodPnMRYwR)N+zv)#0gZ z@h;XlQaGMJa`RO6u-V$yy@ZpE>W9KM3?CE%=gxk|z8pLeuvuT; zJ1iYXNPvxhjbAG)!Lm%&rb(fsRFua2W&LpJ8N1^8*%#dcW+h%Wc@Z|f=8bI^;ie+Z zdl%pN7IEg{SuWP+OOHmcCc=Y_0!&wSGsG30M15w-3a1K(1S0_j=}5=nbAhRR{r z%^y7ms*6&OO%uag)_*Trj_TJY!0anvWVdr)KITS$8yO}gtTSSx$=$aN)jmFI5uMSQ zpGLWi)`!{D&lW9y!K1S=a=ZU@^iM@ zBLMvQe7~n_P&g=FAbS^Is25F}2Pc}uyNjaZa?Hib@VP!Zw zvk}(7hy8AVwE3#Hb58Z0qblpe+S%|jb#)5jv1a_B$18eQi)bFRb=Y%BkWsui>|I`e zkH{PFp~F(q8)JVUtq=m8LcLS^`yh7N-(N>QuuIAHM+7QjZbUzoy@J!s;X*E-pPJWm zK|DJSK8^1eAH6Q%#E?mXeHCa@%V6xS)5qpDwe2rj5M%%p$Ozac*%Ow1HYl>`T z?N4{vwBjHn0;UxSM~vK@Z%>|z03**2vbhfSL>x9G5t6iD4>qrhW1*7Sg+&k=kSNm2 zOCNJqdZ(;&F;jq265`Wwkv}#c$>5!yCwYlD=G!cFy()r9W@)L%GNOhXVgT`f>Q(+A zIPZ~M>LAm5s7*=15H{<7?M|`^>M8=gZ>H_=&Ja$?hhG?aXsWD%x^Mm!G|_NW16g=U zC^OktI(vHhpmnycn>KPEV|TY^Qr^V_e^w$wY&h6RBBj1?_Na0EP{japA2Y_5j`kh+ z&?4o`2>F=)eDFmYTIxm7Naw_VPXRdg!-%~C^!nfkVWn+PV<3&~%C_&jLAgL3)xkqC zY#*PMrA}#O+jG%`tWYWJTL!AGf>4K}_nhAd*lN8Y&&z&_9j|e^xr(a2g4xT8rnS>9%>l_3b@fVx;*)lNE_`H*j-tz< zIh=wv(X}{ILeHaqV3VE_mbbBW8)?Sb7-IveafOm?P@{z-4a$I6)0K1x-@Zn3G`RI+ zYfei@%y-Irn!8j4F(u*PIFziI=PNauOxsYadw3WO0fB_(`Z`wLqfL;MU$)5t^ZW{h zI4(4E(T|!qC^9(?%B~)Nj|se9ek!@&;kc9_6kZtgD7M<_E?o3U7}+ufehv_ln;Dh_ z*z?{&fA6gld8CAF zPE5ncpjC2k2-||RNiNGiMHUXIa4iQ|u5_TP+-aRR*}=$g~U$zM#{B zRIP7XGzGbuaH9Kcx|H~b$w&rXl&=Ooq%q$TYSldAB0G*i0H%4GY{|kBgZZ)l@{M)} zv06Dm9=HtpQX4#crQwBJ5YwHFD;);QP=NP`L)g&J@$-~ z1p>iKu?O~?lh!JKW4ZS}0{$G5+83p+(v(C@_P9uy)o7d}n&b6o_i4Z(rnWk962j9J z@v51mDTG#k-PcRMjH_t6OQ)%aMxFVi372;`AeMznLOkEEJ0V^o`-Ux(O`-JCkl8xy zzvL;^cL}-AgzIC;Z*&wLWBKW&Xs+oml>hchm_d$H7FyYVzW1FeM0er9A-pW}iPzSV z6#42u1$F*}QRc1n`=pAk;e>yI#a!G$pR@3U;=ZoQjRMASRQ-!8P+v|Bo{jps%dF?qx zYFCDB*)m^$N^4c30co>Y+mZC#OL=7;xLdR%P39O;*U~v4i1YH4hs@9N5J$s|amJbS zs260|D^!cBU00P*3*Otl)AH@XUFYx(f}T?D`NVMNm_86asNW)K#4d|p%BFt5Vfmw013#*~*&S#D9hM#ok;&|{Kx?Xmtu{#xijBtT<;yF--aERnU zz0WS6g>Z}}eb9;-CTaHVKJ)VXT43>Frc(p=8{ztJG4FG+ll775;gjjmy%`Gom(Ist$Y==<|(em8r$JI#O-$idxu zh1mhn5B}oa{Ni2x><9bQD>;D5^#mCYK*j731i%xhCjdF%cOZXkN@XgU)tePHxcC%m zTa;^^V|INoJR8+7ITWP3h+=mJ@6Y^y33%y%L!q`|g|%S?$TT#;EQqK$6>VjJ#Wltf8+DHN|uB=j*Qlal}2y)Gai{L z4g@B?^h!q!&mQDc76r#S@f=Y zIh?-xNCGq-;3^}sncEd*AAh2M&>qAlA3&%dV86Rw>~0Z&x4?FpXlJb98AsqVmEM6f zbaz~L;l{Z5T)^ttI6u1PT@MsHzk&y^p0qOkr4(WdV7dEs?*Tp#VW3d#X9p?(>BZbZ z@n>D&-@F}LV$doC_2ekPe}T+nL}s#OyQ;R^-Zm>jaLk0e)&Rx^%%ZD*z~)bN_XDN* z$9^#;*f=jZb|h3n*0Xyc96{)x-BUVbUU1cwt0OJOA7U&dxKQh^7X(=74RNZ*zj3B7 zS^$2`Vmt8fak7B_jni;qM*z9VR5^0r$7p@TVDk+OV<9q*A9;Y}SwCWrjs)xKD-aQP zXdk3Cuz7c#<4;~Sq5S=S#VFu`0we}F6Z@?|ao>mA|JXymVvMQs@%9Pj%0uC@M}3ba zqs=3x%0pq#2jxg99dwMv;IZjWZ=>EDZSQurvSE=S1nJBnnej1DNsQUbi|kghUI#%Fbh;h9#&4BLgIZt2B#) zjwkEL-0RnlN(Sl+LO|^^^leY31FYsdf86a-t-Ju|&%EVR{^6al$9F8>Sn&XN%vU@S zV*`~5#iwiVO$!-O_4JS#V}`DK(hp6t^pSb0sVHm>uB`J-WoZ8On=iOKZ?OuH-hi3z z`RbtJPxa78#f}5Pu`j)iFXQlQ!_!Zp)lZ}Kf0&}6JPTZ4Yqsw*;`hPVauhR_f5rni zS9Xb$_L3_!I|MdA>p(ZyRnRof7jAoLzxG%*L+;@9^XKa4&$aC4*grVNocRO%#W-e< zMP|=M!MQOpk3c;>gvw;<1|HO)ay2iAu!9hj1pOwj|TSry2^Iz%A?+6HDke7#8L(GQ?ezV+ff1f?7Y8{Bu;M`rqP!5E zpw?5?A{Aigf$Zg-)e@O=b!=|Sk;>6Qz^qmq=GtDpixY_2|dUSfa%yyL6y?cwdGm*ZIif`;%1 z_XOI>x=OHGVlrz-R9u>88$~v6|79@9vgIC>6%Z=W-s>kjyUpI~cl!U}8D_NkjqUfC zMEUYj!iAIMOsC6KqsXrr3RSf9pI7r!^)kI;6OrBIm00 zn)<1A@~1QHTmS9yU;XFmb?h^ZJQ$4r2lwx> za!GBm(`>PCS!&irs^LZ5rSj$iwY?tK%?Ia7snxH^Aqt&2fPd?s1M$|E&g%R>{{QU@ z;~%|w`*9xre`2f3rI`D}GeT$UPE~z*C%1Sv-c4I)u=CHopg8&e@g5797`%b}1p$2X zo^;!vscaU2U;Na@$xAm+gC?RWdu6$5`TLz;^rbs*UQ!@v0%^`}x4*Ib%}WJ;rzoww z7&q?y@h#Y>b?FaJ{}YS3#MHho9R8o{SN`8RG@2cJe^C#-P|wxy8b?k%gN?eARh&aK z&Kh?-{H<`dU;b-TvCtfT(|EtJufjG_yOfkY>i}FX8^~{O0@+nKo|1}<|lJ7B(e`4YnE2!}=_vEK=#Tf795wpjRz~s@b$#2~S zy4NJJe+474NGz<)p2FSNuwlty)0~HW#AxuTzk;OVF2)YJAysix-z(nqe1RSPij&Sv zYW!ZUMUas-NZqO9YF*XN-EsZSQd!?IE<9J+wYNh-kHoq3LC=5`d2^nAKpVjup&tlU zVe`9wuQLukcy8a-&z-h<6_ek2M$pl!<4)|Ie`hH6FP|9xwZj$v@xdl;R{|LiBDeu_Fl>S{$Nn&EAZ7R^6;p#LST1$XNmS(s;IZZ0pAKbLveJQI{;!c zcw+@Tt({<099au^T@}S#;DkUl5oZDo)r5V}u*XiymmP?}u+2el?Hayyq69qvi`r@r zxbDXH9qd1zjEweA9w*qp*cg=khhevWf0^K?)qmJ_+tdWV82M=$OxXYAd4J+1MuFEQ z%~XYEMTK!Q`5Gg;YE#vMpRd|s4_0#@1l_qwnDXjjz>FGym8E{=C*@T@al{}GQ#(#n zHZP>A`|_ilTh8vvHL$p9ddRHFiaRbd9IdLvI}EB0713Pv2ek}^)pGw-BTDF@e{&U3 zKY;t`!$GqD%<~uDe_}oSGlzftNN<=umXzynj`|ndnR{;c@2dFcTff}avu_SyMkjt4 zURs;$y21)*=-z49RHnAvvzHzV9Kwn) z{$#z}aQf;q2!`)m?~%cY*hd4Sf5z%>VfIziar1PygkpHV^)4qJMbpUthSd|MK@U|M9b6D}!_? ziE8T|3v!in&0T16imN-(;x$mCK{;GIJ`KXD`oyK8O)h!BmMW70l`Zj3fBaO~n7O>F z=D0itbtkg^3Ud{8N}Rs94(#*QL0H(xoAqa85uqMM!!o(uwVGpNj&%agTNjS*RXMjm z0|!E37OT`12)7P4c>d2iV?v!vf8)3WKW_R*PyGD&zCQkomA6maIPizR2j#@2pB_8^ z`MV!K{PmY#j`}A)fQa^uf804>H$?~J%4siJfZA?Xt6aS%0=x4WU!{uEE$X!&N@EvN zr59{H>3mg5F|1YwLX~1295F*Xs>duM)9ak1cY0UX!2Mt%tW&(hTe2G;?y){HSdgqb za3sdBgd?YACNArG?~SIb97ldCtjx;*e(}!k3;VPN2d!2I6BSHsLJKTR^AgBG9hc@^ zEUOq5dib0NR2Z-o@BolMUUf)2#s2cZE5`ojZIN;1gL3QU`d=HI|8V~8BU&Z@@RmRS zaMbOWSe*x^fAJHml5Z_^&vmw$ir|*(G9qJaS`uP^P-p0f_;6pdyqaNO}8YlH#MMN%M%mc`Z zV+Ycq>x?a094jNbOF*3$k9?!|npZVa{V|*CmP(a=={HyZ;f~wa{&LO_m)M>MjDN$=KlPANY4Qy;w~mR~1SJlo2}JF{525B= z@e(4PdBkd?J^C(Y{L9`g5EzNmd-@xByAIW}d#s8rXxMOPV<$&s1|*6DXX1{3?e;xD zz5HNYU2i-2tK}3T)xpcLd)2vAS`SW9tO=krdcr%({kQ}O7BqkpX6K6GfdBxj1%JBZ zR_d>gW8d2TuQvG06Mxq9r;l!4X_PC!{?So?Yn1=sTlmpss+IZ=KO|fA~ja z?s~&dZ|3Gb$G>qP{;j=4|4Xy~Z1sne1v~#+Gyhwk_-Pj32O)LXd+K?p$bZU*u8Zfv zX;KIe6wpYnoWa8=Feo}B6=@%CIfVjBBem?~xA>NH@-0yh8DTq>^Y^>E<4H{31~ zUE7!|+0lPfwR0#n2mJ6*{6a6r0PIq+%8iHV zzxhi3i|0Rn{#)O_esF87JAYof8z+Btz&{)vi~r+q&iujN&oTaLd9Bj_@t;4n)aq}} z^H+2J(fGeS^k-Y?j>lWqRCnC@zEs`unDfWeu$g8j_^J}6#!hUWmz{JvWd`?nh8M+Z z3Rs6%c>r511MlLo^Q`0VvI83ah|PdrKbqSu+-_IQv8$p@MS{K0I)9_vPBmq!Z!ZcL z(`MUI8lxx~lYd%RVAHxEeZE2~aWA-P$`z&yMIIj%o9EesF9lXlJrVgnJqIYeZLs{b zST2lzvANeAU~+q2ut%)$AXdPxzvWSI?2`d*gPUBw>&oUTUt)mH0-I-;mhpcgpJj5`M%0c+g#j1Y+Aai!@I#ibqM9<5 zGuCY;c}b^ZO5S>RrKIfuhNY_GMf;~CJm}=P_0HG$^*+Hv{)CF9?)=TS!i|}R`;Qj? zi=Vf~`WG8-?7&&({4eeG)As-1f&SBB+-+axZ(eujHHYo+PX4bx{ZoU@H-`I*yOn=L zHT9Yq4I{V}1xcJu2#4*+u>H}S6*rlkl3Cfd9K4n0WVXXR?Z$ZGtI1>?D&dYvl3kmW zgxymI{OJh8S9kVEjPyx9fz8&F!ZvZEwhzOv2>Wv+F^}x|y}dy}1Y2*Wf^SzysH;_D zZZD4C+-Vn@lhZ96lP^HYdP5D&p<91^!&f{TP|`VNRzQs&kRfQQAl}(M>CL|c&o4w< zX4#4=1b6+a`!7${AQI<#TMPf@Pb{C^Oy%zn=1J1ws z(VsQAdHp}neo^E4)p$$B$xCqj>hSEv`Oi(>yJZz(E;0dw2uB`sOL zUy!fZFDWqKD;O{!n^XDkDH+C`bM0?^Yv<%t@|5v1Y}A%pt-h(dC7Xx}j^bLSr1aE7 z4_6L&^ekX%A3W9nM||1(a;F~^3IYewlmHL2QleP0eW05Bmf4qYN0uZ|qXRjG1O~VK zr3alI0e82W!v|Ob0iUBmkC(#92QUIZH@Ec32QLC0m;d$u{NMRk zV*a<&EalP@@JRQb;7?x2)yI)O=C2Lhn6 zV09{&@q*i;Bu=;Ba0sacn{K7s)^h*1_ffg+2fBAjOSZ+xO3+v30ilYa&}8s0tZiB#Q^DjwZ53-#agA58zgP#{#BiwIT4= zzg-jh3Fjn1HdpR0DkWqZ`O<_rX$b%U;3AFff}Op8c=_RZ#3G?AUFV{|ab6#<3kh7AG3nIQ{m_2GkBEYO4Mf(QVk2vU zs>STQ!RAUU?s!+=-y=pP9z1+?E)(o9^sgM_F4&mP#u9i zFIZB4T)}ZS&z{(386m$!FmXt6y!6e$#_m?h#+U{H*WauGx9p_BxmAlp2?fzsj#kLl zGw#u>3Inu^P$YK$w#BdNBz$9qEwm+v(Xr_?`l}-Ex?lQH%rC{U7e>$Mo1Mnue*#YN z_SZBC)fiG_*!(?e4_`s0BOi(vULh{>xGN0AHB=`-jd=r!DW5qkfjwzChB)EOsXZ?~F`@sOG4<+qCb*#04Dgo|qAFdJfr_aR_i zbXwv~hw6Sv6veCDfqEdRYvc0;XUc$h!Md0iQ5SpMi8Q1@UEDu!A#UBN9uVaW@kCvJ z(F|h=(^cHLnTIub5dx~G5t%;(Z48NP7$0xV#FBBP7|kWf=vA8b`1(ZM$y{VO6_XO# z&s(l)(zsd<&W1Af`Uiz%XB=|pNJoKvI`aiSmsp90jfvMFhCJ)HG9?0v$RG7oj)F5> zv`C%iF@?xT!q@+R=h2;eiE`_go-Uhzd$d-n1aZHk-$v}&@$z;$-@*P)P0RBcaug2t zBY&!}#7i77&M#w)1~r?24H}G`kj|nFjvb5LrNa6RDWx!itC&0|j$M7MP6i^Q@$d6s`(T0GVu9{wLj2@ zlzVN{h{}U5kPL4a9G8Gc!y}pH@Sep<4N%M#nQJmIB!FkfmG&&-n7*BVdMTuCz+5Ed zo8SP-$F(82)f{mZ3D5agiI;(71L)(jv&zFDoFn&sp``~|+-VKh@?d3SpcGK<1Hh~0 zpu~uZz?$CGP=%u?$6w^Y65uyQsJ}9K3Ix{$l;BME@WNmSIv#zJ-RAdRo;Yru6o`y_ zbi#xv%0x8uM1YTw^2g184Bi_X_NK&dZw(Tu5xN@pm--xvx5wg;W$iRdaX*AK)z%&L z)(o<@Nb8>jLV&p?i~iNxm8k5;uj)c>^dg&-xayS`zA_oHV zpxuTmLl6dM>tK+7E@L%}13o`b#)Z?PCwM;pI^44bp)Nx!uyJ*j!6a@M!c^!(E{+f4 zO+##8C4F10_&f>$C?OKQ*+9E4r~Ij+kIX<=2^clZQMn7ZZY(J!zM|nR`yFey(yLvj zQmWx(Smwq}qNk`9 zo17NrqbiXQNKrq4>Vmjr<7J>|#hRy`mueV;g32g?W2aPK7i|;spiRRtIMQ@baI>>8 z-!@(-PoI-UUm7H*w$(+qA_F%%sBC+I)QmQ408p}OZ7Ml<`w)Ylj-R&%y}yZF?6JVq zHb!M1x7IU%2SQ(Y@7R-6XATkK8?jO{S=oH>Df1;HrUvR0=l|tV%W-8E#syy zkR>V5HS)Esf+liie38V~vk1cxwm{d={n_9~hqyI=pgyOWQ;sTqMcKd(ghnGZhZByC z-F2HhobKs|Q(uud$>KXVy{iq<28YY{T`(RBbns;OEjpo>MT{ z^DDQ17YaXUrfK-ev!zojhbUz<1j35O9xaE@#>T;%)RCGdL-O0wFueGmtKmn6tvXKF zkqzTPTPS1N0`~cX7I4MoRr0=2(A{*rA@1l%818>GX7+dK84G{QlQxqVaaPGh4|9av z<`!;bP(DfU71BKB9uWD@zEjvSl>>RuX+vy(J{qBE0Sw5#ys6U3t%$_qh*n4W)6USy z=PN_oNgYFdM`Z!)p-V_ZKxD$8gzBw7;%Lv{V#iN{t$n7RBRr-E+7705lQQ*CkEu{E z8#$Pt90BD`{hl7B;Sl{T0}j(3^seiamHY@!`*fTSmO)q)(R9lg%j=Mpj=(Z!JyW4I`68iFKdJ za8y;IEN^&ek)=DDz@jrV2pjw`d^Hb$_hoxD>sF~`GCpllbk9o=ex7lX)=MVreyFo` zg8w+=H!t;CVmda7g_dDx0k$gxf%cj9CcEcp$%@(RMBg^60G0aGxldHPus;Rbt7z$( z0N^$&+=PjYk%l@GerUAclb<|o;08tMOVJ*f>xbKPiBPcgAzFqzp_8Jt-Gg6$_T%;v zWwt>ACB9u(UsUr!VGd>-K>%G^T-X5=NoB4)a(<7$p}@M$buDn8vh@M1N~hjTug#Xr zDgBm}*ZjRp17M5QRX~%B^#%tHj;Uf4EJRn(PuZ`?5k!viK#(Vd9X(eC!D<4B0QUOi z=--xw9kgnkKi^K-;dCkJfLLCCKvI{L(_MoFZRm$z$Wt1dNmeKt!c^IBe{spX)|%hx zNA!d?tklZFnOksnS$$$=>Q7&8&>ZAy;u`9(yhY&`fz`qUj?*F!cip(SglO@g)A0k$ z0_c)2lVjb3=+i!8o{UwdCbvQ*s-g6|wAXZxe8C7^IQ(|gJ+UDTlw%2hD$3A7J!XWS zb(peb=`^%0>uuCbT^m)LEV_5fvT|R06V+Jr8KE+v(?JVI949!JcrvpBfwiANz+wTq zc=J%>Nt`AR`tc`1YciWr4G6RO63(L)v>&G~VC|1rAa>5(Gd>5_RUAsU%(#02z{;*_ zais}g+)ExOD7A{OEzIhF3jgHP0XpTT>`{McR;oZa?_lH-J8u^w(SLSOfh)lo0IF?m zg}25qM@Gluw&g5=soo|aasu~+2V@X`C80d&2xH5lgE6So(8f= zBk`S4p5*FsQHsx$>&oW)V0#KkERLLI3WNJE2OxSb%UByrcbog-P~EIpie#_+7^{2Y zQHVHYNt3S{L!{Y%sTmrV9ux%`@MXFkeuesk##}rS0Tn|xFsjC6Sq|BXcnYW5KGZbj zoPKMTn!0*NCueJP>HY))qHc_qH-(mbO9auY2PeXA@|M^ZE3H?`-Q;BIB5r*=w5H}+ zRcSmuw{z>ILUtRR9LS08>Zks0!fZ{BbO;ql$EP|F_=pI9ngFfBx*gm77L#xFM?bK9 z4`TYS6B)!8_)t+=MDg$p19H-MJ5F23c0Z9?)g^DPSIBL~#d}I3CKJ*-DxU2kQvB8${V}rW#s~i+ zadqNCBZ4Q@>$9Xx6<~dHZhAH0JG8|3nf>v>AVTFdx1(ad7WY{-F2MxFP4q1c(gMsj zh0hp&=m@uUiyt-nlLPep%m$C(XPCcdg!_P(c)EE}?4qqcDVv+CAIH)^KVU_?a&pgN zTUcBkDr=j1s@y7{=!DBJ=BZYM{;9>+eb}_-YF00hKohvPgB2nn&4|FMM}_4?(Vdf5 zd72k<)M7`408&e$NvG_8(Mk&UN}>n!SUhe$zV2zFJ`+N|0d=?D z##e};dt$}B1uT6|J$@4FJSP-25#n?^cs$m{x_7g2sqexkTw7W~k5_>#zsA7tf!O?# zxDc=({Vd=J{+Iy!rf9@how|T=*=Z3&-W9pcMe$>(rJ~WT!AVlF$P=OSKwptCtE{?z zbeqL;vL8%I1%6sU*ZmUNKz`@|-|{qzG~a4P%{v9-*upR|W!#}wG8MbcBpdo@dP(Et z<+`;}4nb0O#)MCOw@_3dDAg1Q+ZTwe9Qqr+8NI3m&aD11r?SRwGS-^AP2gssUcV`5 z@+~7Z&j91}V`ZPCNjf7*dNJK}G~h3P0OjkyY}~~@EV3ov+i78oBfa53Zc_(P>2Z*=W%}d@{ZV#<3>j4BnP&3@ z*7nHa8u8sSyR(YdQkxs7=}i>nH#6C5cOhVVh@^8^)iu>_Xxv`6!Qah03x=LV9bw?{c01z8MRE1RT91(mOMbVe* zz5#({;b{muCJ|d{VE5h6zTW#+K2V?#;zT{eK_x%lrOn8ZY6j0i%YL$MGkq;<=ti1U zmy&I8+!vg8POn$ahhp4dz3-BylsMifaVw=4aafeI-X5KTGX``->U4HkC4GFL$5fYdnvChg{yHSPR*pO zOn<8c{~e~!C2Ki~Xk=x|vyi;(9jbUYWms|PCk2|ujTzDe@86ZO{sJRQ1~DmN>9j5q3$PH<|j!F$ulS2m>YY;{@WGo z1D+c zgDmKq=4WzUFYwK^2Vhm1UbQ!R#2LQef{6hSJf#P;(LBBbkxxwi{7s#6*rm3k?rMY8 ztioY`H5mL{u$#w~#Wv(Xlk(k_d_nPBn0(~ib*+n&a{*{Q&V)(2?E5RxRq(x1K^on zXd^BsSj>}eOuwpjZEoK0ZFJ&CJro)fcY6C&IRpUNm{L&DtDAVDP zvT4<(tvHH6J_<$VrdW`jzJD^NbFGgs8(VT}8lKH7w24q?a&D}-IcAMmTr!0YGg(iIF<`0nmUUX_54 zbV9BPxrir~H5uJRBf6fa6dJ9c*LUUs)12&go?~6waPiVTCUPVveu2-Le8=8LgRB+9 zn;>){2YReB_oJx(?z`gnw2AN*7d&f!M}Yf%gHBeH!5;jIsg&gWCOZXy`N4|q*v?f< zIT-9T!uw#9yw^mT=qg;c;SNC($~{AR$>qW|bZmkk{1*ja6Vn@0&IO8_cgr=tj9Mlu z=q%*17=&-7CuDqnF7I74`#{MIEGf03iW*zjEDMRwgnBA_bG5Rx4kt2=Tjz3rUrt!< zu&00p6y5eo^HLvRu4mFpjKpAWUNu{7;bSfBwV>N>IavD=lub(uIUb!U9QiQOFkC2}otzge*q#cN+DV{_pR zeqrsnj_)dU&2*{kxcI!ZDq$~JC!k!CS`m|L1+KHIfu8BRy6ylo^V*4uRZFT;*JTs6 z5;KNF;Wb%(N_YAzMXzx^dQ%xjNm<1UqMPu$Y+Pu*BlJ#jV)w^%3)eV*lv5YR^B1`^ zHAKWwrf1)vnq(A&jkooOhdmi#VhH~oB!<2gnAIA?)RD3PF+k40rdvwjaf&3wc=aXm zLrxU>*;(M3J@=ujh`7nyg|&as8Bn`%5{Gy|(-^^b@IsDCz;6I3LBh@ipQQ-RPwW_; zNu3GqJ}G0(_NL3Lf+7DXNNkjxe{Gn3{t4d{D=;%Rti;TSnFV}q&3!Pk4>#QkmNT>C zxLSUXEV@x^2e_aOb{jIc_922d&BzNP?BHt4LGgUFJ5Em1{*_+>GSP-?blTh)G6X2@+n9ekaWEFKzW!?adWc-=&mGHnu{t6RNWO8>He{E)N{x9?d z3u91|!(T=y$>7z}#du0Yq2#BK=;DbjeI~~(d+8+pY6h)nk##x|(SrJ8^e~2dDKxO2 zPwVqQ5tDU0*RmJ~6Jz~=w83EwAm>i|f!E<)$6wKotcfIBjcF4`>|eV;OD$X2LYMQm z41jE&ki`9J$iEj|2Y{y~f7&Zh3z9opNK&~ve>5#+>f=x+FH#XQAPb8jOrW1Jms_Xk zEv)qpwIRuLFUFpz4jaI!Eg3vnNb-Jysehi?k7QlNWNJ}SrgM#wIj*6RthXckZ;UqI zqNBlU&6IqrLFBF_HJd~1es_W(*4aDZhNd)K=hD^Q8VC0W5wb?ze`eZL*n`;*ZbMk^ zOhMrZ5pJH(d>~hSr)#+y@?8|F+Y*&Su6wBT+JwN%Y~{U7pf836YJ%dy9iy3wS-Bf3n8(n=FShgsV+E&QB(E3M z#Lv&dfez^*XP{3cb?P;kn_F>wgf-FvoxAtQyz@!^C|iHHf4g0UMhOMi_ktH{v^0>m zY+>p=r{ty<_+I==-u>0HRx57TL-qtdRKB~R*;3~dyKLpU_wB}%%b;JLmVr~DW{&vpgpA%gCr@N`$0yoCc zJmTn&wH^Ke*U=w|Ex+?60$7YD(4TV1|{AEFb?N%$|1e(L|t?Vqj~iQj~K z!)8uG;ZIDx|1*pHy^wQc*D?E4f}xUO7?t`fDs=xQrGqm6gZB(E|DuV;e~u#KZx%ck zSh(Vhe{fU&X^y#c|3CWTzd^FRL(JJI9*M&+#y^L*^jA;lB?Yam<<3y^FE`>v{+`$K zzgz2GBMTw3Z}PW(Jo}GH@BW6O74vUj^U_?+vJM$+=)xkvIiY17rW13aB!6qx&m`Fe?TTcq`06kK8hX`=_g<4WZQOG0^-B? z+K*AFF7(}x6I_*Ruds>QDa9${bDdfqA`sA-saB)Sc7b2jhd`&dOz#SoxpnFwK)&|+ z>+Ey4D!;0xx%n}WURf%nV7|Xma#r0V3^Cv8x)eL>H>eW8X=HEkdi(uaoQ$hR_R-oc ze_fq0s5ZF}%iLGy=Fj)Hxp{q6eVJ((^_p_uX!7q#OZ(mGcIic3?w&KK=d??s>WaSo z5Eru!jtF9F+MX<%kif3OIKL02o}C0xx4SJ&D+F_bzLAV}m09YC}sAOMYe~el! z-I8EWQBWvnwHOxpa@+4a-RL9t58W4&rULcDxOy;-mC9Tm9 zXqkUUbF;6`sjiwesF6fGEqbfMR$!I5Si9PAk=`=fph6bmNg`6oU95wpdhJ?~#>mzw#%`YTu zKEJp5I{jFmZZUKfFK(MK!+eB$2nnWY2pdfJYdl}r4|KL#U|v9`qf*AiZC+>+{JpAu z5s^_M+L)7-^2ax{N`EOTe^VED6b^@KKuh4Ki$%sex4{Nc0~|@%xf7o`p)jh*{5Fi; zaQgQ!d;#;bNlaj+lzR!l@md6~kKo^@gD^NzE`gC4!l4*``d=Iv>s zn<^r|eJY6R{1F1cU5|c@JPIQ3p^p?r8hQz5=ypwNgYx`UhX619e;}Usjap9mk_FM2 za9>>yt!KuhZRi(=`z7^NKX1i&zhmOh-157I#nxVEWAVd`O1c~>Y02aZ)wqW+=C15A z?#QaSHM@c0oS}rOh9BXIltMCdS1{J!jJjAqcqI`GC&5f4Pns*hcPQbBF*+_et8(%SKf3u*5Ki|fH^6))Ue>x?fxV$00Jx|fdCVZxP09sy~xAoL;efDE~sTs3Pm?j!X5a1WUD68(HU&j;1mOj z+iUfK(i^`I+v#~Af(a*lIr`$WLI|{Q_bb0I*&sm>$R+ z0?_NaLGBJU!8L&l&zEZkGysbnHOh9;RbH6`ovpp|5x*KiYh?;POj8xg+GwO4hr^H~ zuot>aIF(wU| zS zq#@Rsf0!WA#7Oef3u-NwM-Z`l?Q7#>AawYo!Z~9}e#= zu$>Y(oB^d#KR@hmdw%lE5{R8opIuyy=;=NbMS19mt`w{=nggail)-M4B<_^)Gy9}f zIJ3O?aN>r=>jWvaLdikyOmH?8YKsOCe*l-qf27M9=m|>~LqYSbvr2?=B0W%xgCeLj=XG=mYI>aSp6aClEJbp}~9OE*&TS2hN zf+SX)Hj6|L7B|Hr*30+h)0rq{JB(4xYAN=47@S4oH?AqLqLGKoGlY47n7Li`L>^Kt ze>TQ;gf!plOo-YQ1*ejQOgSLu@ssgqXqXt}QJ+No9Uleclk$Pd+uB!eL_VIC&zj># z{1_+C+_p0(DpH=G0-M>J=@6s^?{W)Bf`F2TH`7gOc;FI%yP;@ z*X%HQ(c07f*c$z>|e}6F&@dvN<)qv}+)h`)<5h zA)}ic|5fvKqb7;Zgu0-4m}?IV;Qh}C_mnqVa1+*Pee=m1vc}}nEJ4&pX#yi0f5|YW zbk>DRX_XTUn|r#cFn550(qWQrvX7=Bu|7oc4w83<74_GSd8pJdP z?=UXCrDBC3q7c6+<+;H;9>+I~e@cVKH}xm8f!S9UOj5@HuGi?d5|`6{H-h#DJ?4}_ z&yf9!96}MFPNAO~NR>IY@5Ha7KpjWNb+suW6d? z3~{7P;NCPNU3bVlZGt23V2_jD1}pjdhN)Gk(8iUr%HP>YXlOfQzWZP!EpcuozSGsI zH~pDP|K7jba|+CFnUCO3lkGc69(k2~!}F^Xc|c9`Tj_Wgkv)cyqOx`7=GJAKI?3*~YelYNt4pH{~ zXk^pZM}c8ZB52v(^Q;>T1rvPv@B@|_n~=KJ5&o?B%+8-%;z(6+rl0R>DZFQ-xZC1RcB%3+>RH7>PWbo~J^#8Iq3Q zkbZu`s3Q^o0x8!oew=@T<`{m26E21?dx^p-aL7*A3Y@ zXhz)#;&Xr;Pd-|xdc%W?=v;7FRY`dIwI&;b3Zjrq#d|h9qd8?NPRL)Z*C~ji-wO$$ zYq)gLJ~t(zQQ&D*m(`$X@V;lMgi60fjaMjIRp~{0#?|m*e|@tswKP>T2t8);5G8@a}ycvOL`1T0$(G+I_`^fBR0@ZUSx^@c`sEJ#7FDKHs$@3GU(YMkDlL`Hb=9{q1%AM0Rl;H z>OC#I9P_P%26)*nU88;^A%ty?{L}F*_u$|pq{q`9(cd_2iogR^^s>t^Pl}U3R+iOz z@qis)DXsNj>*33G+?r`6|R9dXoIA} zdvJUzAXi%Rfb;Y5+QSY<0&Tc^zA52nIA%U1kCV}5#=04EvTgkwZhZ{H7dgO6>H zn@&gme~v7dTsE^{Pwlr5Qn#t``7#$oramUt#HC&tC58yj|=QD_I_p6+CLqn6o8crS$W ze`2WD9Xn)x(vyW<`*)7&6dp1-n?~7?8j|Aj{JG$V{H9Qvl%YPFwVHXUf%TpO`T_jJpp%Zf}MGv+)-{& zR9fuXq(WIxX8JiC@QOWT7m%rMomeAXxIRAGx9>?z{hN?Jo#=r{5IuU@bD5{*2E;ZP zeX$1Q$E~PYmw5r%F*36abY|)9671%U=XgbTNYN@;v8-qKT1DyZKrm zmdEnYI1M1RCThbRJPO`z7k5awe+5z@wj zBjzr5zrE>gwEo8jo8m~3OGGz`0b>i}NlD*unOfmKI%tqW%aaBur`dBB#JMNXVhajh zmN>^W*~E{D@M75X30Q44uo;-!3^@frK3h3WJI+Y+e(p|BD{LoMi`-vC`rU zl0ckI`xYN#kh+4>z4qLk?O{{j3SX3T=`d;nd}0p~zEG}OXweNpe_qq`Q4G&dB5{Cu zqiwJ}4J{sf^35xd`kfucYe4n=l28Fz91s^nf(yt z4w|{(LT%cAkyhi~4{zqPyPN!lo^Y2KmP_6=dU+%+c}+2XNt(F(^6Q-`{Zz(c?qcWS zV%8eOStW%4BvM75f8vq+9*o~}m)GgC0wXYesJ9wn=k-f(l2;gq4&m0{*j8zal3WL? z;d%7^JrziK!Yd*Aq)$WPjq{F5JI<)9i4J}c8`b=q2>hrAI+C0GrH|H}5O*+)HhQdT zGss0YMa>wO+mAJZViw-*(yG-orz6;tT>+4JjAXgJS``>Z;j`{JeBRvu+s_>2oWl{7XpJb-2N*6eHfiyQe%CDTSJ6k0FQB}7Ksia3y_j#xhyO?^c2Z6xq7V_vzo z&^gkkl^QV+e-XX9NYeSDDBBLXS}+zh zX7IT$Gc9*&f1o^~09Kh72b=RSb`06~C{UF;q|yY)-p45J{_x62jWoP@JC;1rm_0Yv zsft*&?<4wLRv9GD?r%2W%f)chwvvQQtdWX$7lhyP4Zhs2H4o;9R~2x=*IaN&4D(bn%^}WlzA@9;yvWwE1PY!HO${BV|FG zfmTT*iF}7hF4z_r$XP9uN=*!EPlGI{*V6B4-g zWqEf$i6CW|GqEjbJOE{m$R;*70{Sp?fwaIMZ+dKpp*?`882&e9X~bf1{>tq1;Wj6p z9|uC2P|~&WE6u4LDEkN_Ir66m*h-Icl3$*14Wh#*K6yg)?j13NOekDugraPfS9Ig9 ze`tNflD2OeCN)1Hiesty-;}+{)}u<3E%-nh00JZ!)WDs-@7o!D4-yE`JpElL;+*{R zX4a}zqloxIAGn)s+qTV+8?~$*?;%Q4EZPb5GiwNyZPi{@H;?e+g{gxoPCPyQPFSv7|-Kno|rd>^Ab(<}#*(n6VMH%;z zXy6qq?!+Q`COQqZ4>QoiRObvU=u^q4H{Y#75rc7;AXMH6cURBPSS=SJ8Y8nellweB zkidcT0KTz{FK6?%^LH*GfvYi<+qI8v6Vunn55p=f9#Dxf%u5i2rHE&SKTRoAf4-B1 z?|pAIuSpxhE4J>j6bR-Nvc2ti{^UI8cR%9eGbEeHz|efQ89gcrOD;wHrtFVp{(gEP z^kK%l;QFnLb+m(32 zxNLdzbQUmDC3RyLk2RymhHtu#f3W#ao4OG?&%el`^H`{MPrrs8hUGEhHTNn^U-m9N zgZ)7hmGufk&zt7YmwmVh(}ast(@$r8&N|X6=@)rc_z-T0`Y`ug@IPNQLj2>KC9eGG zXC*;V4_6+~I5ci>Te=L1Ts0jZh=?n)_UbiB7;vso*U1cZ_Rn=S8%#@Wp#>5rgTqTeOOGsE$^S!;2fI zqeJxZWV6ofxvJDKS5XVh#co7Fm&NNj7jUJ)( zMyfB_j-ztFt!FS6ocD=Vf7DiHer_FiL23PkY|UH8UW9_K!tPPz+Ts2(eeG6#J`5O{ zpMmn{$T;)PDbs#WhyBDzU*uoTe|Ij~-EE#==696NsjGBqY=Bc``2+)_hYj0ptb$`E!Hoi z@IKb$+o)jOKEi(Vh&FJHs0p#iCxX0~Es~0w$OPa1)jqe(oN1Ke-1(j(@fSmhK8lLR zS>kiLb@#)Br2ox?e|kLk9m9`&v$vOg`#ca@pbKUcC*kSDfui>ew_?HFPC{E5PsEtc zBQvqdYLnWZ@1Gt!P6bc-2D|$bV%s>mxs~f7R`L17Pv`cPt-^@Kqj(Vab|SW&5Q8M% zalZ19+A75d%j!7&A)RA=qX(JB$i(J6^)mOo!KiR!dU+MPe+@G?c_SDjQ{dvaOR0@s zprGDNQ(0^%d#A7W2lEw7&gbduaA<=fn>94XjNwYO-GSB0TTgd$qJB9ysU68uF@g$x zX`TGriL7P0Jv!4M_SyJ6U$`_-FC;T?h3(%jOMSQKyQe*sKd9F$*Fx04aXx~K1)MMp`7Y8wFK}lD>+`T-5o5iU><#HA-AlYWoTI6> zCm#j6TU4F>^ZVL(`F(uM8zE41EVnT-m){f362GLm(d5}$X5_QJ0pbS422{4TsheEe z(pg%Vf22AqSTe*fT2ZwTgMHcBbKp$(DYMS-{kqw(ztbo77O;KyOwaahR}~s>?@v zKSu#RFJwy&T_G~-5EWmH+IUu~zjG)1u{0oMe})ee&De*gqXZHRqbW{y)S|+25}@3g z`6=abf#y#>m;g+ejFUbVui6!@_F{^Wc9#?#J4Ayh;}FF_9;Hw&M{;#3ec+D{1bJkf z0TMP}H<){0`6xUSq2=C5p8?T6Ncnm=aBoRu-*^F)sD@jPcz45l7r%G~e>+-l+iFQM ze>Lxi>drMD;~>$_bYNZY74E0^r-99at8d<6t2ah?9C)>fxq;^o#2%hr`E|RGa8Ol@ zrtJ;oC#<{Y3p+I&i^uFMrudK;bw#yJcYIO(gc_x14j$Xx`WjD+V`A{o54~gR*U5dO z&dGOQ=eS1`A4p&Q37f8K@C%1u_nrNt}Ep?G*=S>ZB%?B-kgieBd- z-mW?FDL%7h(bQf*6WzI;CF>w&?-<2*3Z87M6twxu{G79f;^}S9*Q$0~Up)^_lCDx*5o#4%GDqk{ltJKnJEZMxq2vO0Cj)?DDs0 zFxcUH<%)O{&#AsqJ#M7R1-{<*FD?*WWor5C1nd6zte(sr^(JE2*uJ8=rTMK1Y>!Tr z4h{Wh{H5~O?W^B3Z{}2XnA8|Ue;&SDy&hxHo;8Mj$4muti1E3-qImXl7Yzi7(&ZDp zErmdznlDmWe1ZgSELWD^n|n=d_{bX|KfUc4wa+y*#lCqyROg|6?8jCk$gQ^bNARTj zkkspTc^V>Nx*pGYGAKib$|E()CH-+f_V&foi9wYMP`ErDaYuSl^vu1Ce+sR1hDH4J zHGWKPPr+k@{}c={uyMD%6Evav2_ijb91KF^ReACcA@5*s}f z(ondBUzgYCS-rmd+hOCAe(v8rlJ>8kx)1R-Q*sWCx$mPA{eH7!dRDWXmJv8rn|@R7 z!cpg{*o(Z{C*96A_Slj4f6UfAe&=Y@M51$i&V>fPUwce%f{Bh4{7RD%QxjX-l!`Tv zxm=PU5jxO2${qRAkD7UW!qxXDc0`b<58{tGXN|BqY5(*yy^dZitl~GR9~IGtoPX^5 zFlOJN1YAFA-4^5MD1U&zJ-EU1^*GwG#{*G<>eP_b{VYO!yJ^KXe{4+)TX~sjqrS-> z7oRI4d}SomOMM)Pr(S2Duy2myjy{?16yI0leBEmKRkPfhRyZ|4f(t)27w}ARa?I|( zW&Fiw_Z^m|uv;H}0!+%pnrufDJZv{?nv|%)-lx2Kc$?Fw?Ss2Re}CGevYtFC2*o|_ zGA0YCX?5R}QW)>&f2?%Jt8CK!@fPjuacpH}A@OL6gp5}-&CXIKW?fM+p4%q9Lq2-V za5^qksfnJ-c5^ha7kE90FkfNiwsA5)r_Rj-;&yvJpQtVMD?bo~W_-q%qd)EK+^77b z5TQWS@4&6NXH|c~FqH2iip1a5dr@g9WM zh60KT0BbtY9?(_pSK=noA4iqO(Q`LX{K}ZxsoO)%HJnn>)JPd#2K&3vwjmo??)i#_ z?L%X@jp*n;TI|hiOTSd`eJ>Xi1nuLtaGOJ>HFf1oB-hAVC~o6`2cJv{7lpEeLDQwXI^RQi!*@15EBvvFth>k#>{l#5we_o2;6 zV?{^xE`%gY2URAxdq$Lyb51(XnNMB_Ssm5U?-}=lyfrmFWb!t7byn< zN^@Dgov3}Rp7Vtpkr1PXcKCpXjYb^k%|!~UV6|pa#=1Y&Iz&RG!C5#yO(om1K$VMu ze~@O|xXm$zJ<3y~k=xhUWj8xTWzz~~^7j|}LK>A2h@)z`lQH}zc$rA~Ry#DM)!+#$ zv?gX}qg8q;=}GtXJo6G|$Aj#1U%nL}C=_-bYu8;yz+4QvnCe=C+^WPHt=Hp(jg%DJ zch|$axTpr|9!j*<%sPm26`JCPKl=sNfB60w7$T^eRitZ`zjbEZVnoQi%--MGfE-|@ z1|&xwoBz_-S>UkD0Y}0(jcOwrIqw_sPL0E?iZ7fU$Ooo@e3fLq?FB}U?>zFtl2lE( zqfeNg=vf-K-rb5~B}$d0+e*^hW~J&g*=54$t2DV~;!H|RxC3dZh&`De*e>#Wf5K8P zdavBd7D~xH?Nr}bIdi3K4*W*TH;+1<+;svge7v-S5kx2HFwL&(wbtPL-~t&eEzb=f z=Annxb9TFSZL6NdnLMsuTRSU8)7-SW8%>q%oHJphs2jDHw6#D%YdE}HT@Gqv!8r5n zYwvq`qg8hpEy%onby`|z;Ua(8_9>p?z zw4XgX7dNDqZ%%eSf19s!&_Q<}%#j(4 zHy)H~z`~nO-(KXfZdyS2<{_B}FXLacd~logaEC5TrlUtGI}xgC@%NCbT(RB1WRhmV!PYGqt_9U!RsvB9wEqzL)SRa8g%!2UX?vM17qB}4$C&OtKuBDp#Yb5JZDAI)Q|C{ik({PfL!8># z_>IRZp&EFds^YzR^>|Z{!tFXr-fZHIOCKuH((m&b6Zf=0f33z3NlRcUkPXvAJbUDZ z@~XV3nZcq7^QoJg4!?Fm*U#wd+B98LwP&!b_2vKCODurZI?A|?0!;tQbC09w{ zHpGM+TzMv*<{V}B#CiYWyhE&?(A0hjkrN>OUPB1}J?a80&|XJK z;AXJ@bqL1+P`Y05GVbtf9w&N%?=z+psWTm6{lJjWpgAKc=l@-ORmnG1S%7^O0H1r6Gu=kWm^O2u?Ib05P?=F1;0|EQ5P&vXyMB)RRIu@W7khX^&$jzhu~N7_^-Ww=a*is>cBmHxpiMTg+e^a|FRc1pGrw;rx;OS?^Q#Dz!cUzGb z&x}oII3vRk0{tD#=dCIP?bsX-6x(_%woXlP;^PbC-(%e*nWM1JL^s_hXB4B@An#8b z!fb@@xC_|sDw{)6ew_JE#;NOzAQfWL$(erL&TvugiX^DW^i5GqzZYjzBA}Qj0;Q-~ zf7-}*fj&`pqAclNKtk--z4_ecyFKTAM&vz%z4w(qH*1WEVrb;&tmB89+>7_vu=Lo6 zSpB)9&7L}9#7T{qey6^*v-`o%Dk*n#MBely%N6`n-?7hVXv$2npD)xmYM~K-S;nY% zhlNfHc#rj;OPJwLQ|V`sAw&jmTSJ+3f74jv$lm1%Cg(Jr*g~tqjoBXGGImZ;iSzFH zw6SLzSN=k7h$l**&|hsN*i|IXkLO9Vy!7>uqeQ}|C;`;BEK36GB|0id+y}jiUMFuz z0_-CJzN62J-89zPZd^Q`AktcCmjEZ!RfJc|_7yvo*otmVqAQ3dQ;0|ZTYop3f8}Y} zvAsxulY29Squ1`?nVHW`oQr2yQ#JxVH$_g0E0s_hxA*gB@I0b6U>fk`pVmaIm zl|d11?wsG)UE@+{wc^|T9GlMPT3Vco81EU*;_r1q9bf<6?0Xp!ey(?~xcq*{ZuE=d z79cY?zcHgQuK;i$lC>74hk_DC_a(hP@hU+?HH&fZak2hHsX#EZpmQz1cb23^SAiP7&+8e|LA^^TLio_*BKwdAo!q;2dYDZC^hu_^I&YQ3c;{vIhY; z&y~Aq2ibl4 zX0+RcUbK%!PjU(S2o!8GchtSZ+gD%6)ksM&4RcN%KBsxV_sS;lIb&YuN4N>~D_ZE+e71#}F+yT!Xf1faqr32a5PHp3fvu3rI z(XAxHuoc+$bCb4SNe*0ZwP&4!5wjfb#f_Mb_RqB+!Ve4VsX1hoz?xT}MfT_YdDn`3 zi2c*-NQFPQtal!UBdAJUvd=SKg#p5zNT*f9Kt1eS--kqs+}D&PnpGHtZwh;GgbgaB`e;w*1a^$F8*Ba8&Io)D6W}=#blRdg< zowA)LZ!2+gBKm|DvC?S;&^j5{{*>fIeyu^)oJvo7XGAJtzQ2N8-2z*i==Kzzpi^TF zH61T6K9-{D{)@iNT}l>9W9NGD%+s#RZ?tFi>!(;d5vfKg%o0RMx(}@Qw%qG)f}b)D zf9gk2b=j8i`4`f@8Hk}d_j*8UX7XkfY}-=eO++bUyX$ZtTY=G?e*U-?Tu7(Z3JXPs z?07Y1#y1Zkv;+09@T$@_1lZC%ePI{=H1&Cv8X4PNM1+p?l&NK z?$oPH)O-q_s@jm-)6KYfdNWB_t_A@=e?__ibzOt6N(;zT7@AY}CHj4Twy|@bWX-p? zMmYUiFw?Y9c5*^Q#LQzZFm{Xc%r@X~OLpbX_9y1Kp7GkI%l$b&qECf6ujAFxADWVH z1be)NiQ|+p>Z>|pthWQ0mZ|tw=kH2pPn%_FLy|$y2V4D7hQSYH#rDEIakaOWf8h=j zzgvyM-N8;LS6Y_^HTD-|wv(M<)2Tq|?sznz!b=!|QQ=V}-s><8Uff3u9;g0!Qx0k! z`P5Sn)V`O=>(ufGE37X1tfmM%&TML9;(H8|zD`(_kDg67@yC5=_gJ?joIGEr^;A6u zjc)dqK18{+sN<8n^`2Q9y-uLOfA4KH9M4ZnnjKA>8!Wz$DWg)zA#Npv^EM5byUcrN zzaEH{A>~wLr7>h#$w}t-4MZFhYIG)-?C||Mq!DCHCzsT2us1$>Ivlw{e(8`M=)0fa zHl6D>o&#c==vlY3lMFa1taU;n-fs@o(r6Gm5+0%*2-P|f7AUqX;p_!re^%{&$OsG` zgWLK#@`Pw%($7AahU`SOy87@XKZ~yqBZY8|tH3M18dONeo0JU^`vQ87=>XOi%U~>; z_IV$pv6$J9;PPO2*lXO)Cz#NXE!#U{>Nmw4&O9OW(yKY(JB*-DQz4Ud<`Mrb$~$;BIzM z`Kc_;b)!Wo8$5S6?&BdI(&2dyhLDNsvs|@<*sR4sUVXbCpLcRMf4un>_Swe3yE{l1uM!XUFmP1F1#5Dc~bNX$;wIdfy#ec z-Fw!ZF1;TwJBV^L=a&gH?=r|g32$`5$9XKA(T5p8V|^7{B6w zgO!VEU#$xZSMC6pku3-mf7^VtDmIDW6{Gg}8%RRMBOKE}T5tFQ09*d9%?f{~mi*?Z zKrQ|KwH{Nk9~uVr;q|N6h$&Vh62Y%yK^v^)y1(~pN$A6GO3803$u+s;*9>CpDGhDn zOJ%xh0igey7y?KIx{Z8ybvOwC*8Xj7`!ki~H-!WMfXct$hiepIe|^*z;86i!&%Y*; zl?n8jtCIdMDge08uN|U{*Ax=?n>qpjY2=D0nkb_cVvU^TG#!2-pWq-h%lYi4!-t0xMd9wyoQCIAH#^3x5)*z1{b-S zBqvgs)R7E=dcM^Tea{SV zXpMYif^vzrQgH1RR*}fp*`!s5)O~3{jKkaMp%2^bvs9Iwu|x(N^bf|@`3$I94AB6-FCSyr{%`7RGS(=+gN2y zuu3q=9pSTqe@{woVBM5vi9|>V+4+#5;~^%aJt4c{W*L`^Og- z7A{}p5m!k*yAzyH3Fu~5I$KdSTfZN_Zl5S3w#9wGfBNfw3d=S@;7T*Jy+Lk%cE$BJ z^I^?-Vx>XV%Uupio)X?9u!B@jvMeEYtn78>cYQuZtc~-)3IK2Y z*Y&y63pm>{AjmMiy?o`N@Ixl~Hj{3s7^4uXc zrKxW;f0vej_kkPS0Q&{A1;pQdDDb<_q22W@66G_(q7itIUQ=_LM3>f*T7EnXy{#kTL;o9e+}_bZ&(NDPKG) zOSFm7+yDg36Zups54s63+qIsBzt#gBPLym;f8-nvmuwCP(FP4Gyq!#YJE{z-M+JGN zy`feH^`?Rvhybu4iyQ>NDn_~bvvAfF0JuZhvr%s^yZtx(3)g{g){8`Ww_4q09k$Ol zp@uGFbFLUtgZ!H?#R0Wq7qCQ-zv2%hC#Zh-=_NO-!NSsMl`&nhO+Wi`I6_&h3&iZ= ze<@+vBIeqU-dguRoFnV$SP)4nSQZtLfW%fh@$1xA;DsO>Jb?HCV7c&f@tp=OaPb}FFYWXm7hyTR% z70GqonI(69%Qa#zDKWad%@pMspmCgu$*_~GaKdcKI@!!aFK z5OjnFk!J)PF(bB2%dTraVn4tZ1{@?J)r4%{sSqUiE&j&FZ-STwz6e^uf7KEd0Ex>! z#xEW>PLv$U=k4s@SiEwv{3rkXm)~{{zU=mf*qm&lvXSsL<+$2|Wv!1p_^jD>2Um%T z2aXNeWa)O;TiLsq7ktg^Eu7i-Uw&3gaV~(XG55V!rY*l_GDxCS?sO`Rk5b%C=QIZA}e-pWb3}vW`NEIBts08r1)xl&<|t>7 zBAFmNH;59w1%1xH)K6TM66!4u;3I)wP-9SB-l<|~LkGQ!V&(kAdH>R7mpCNb*F#UX z#0sOMbMYOfJFObTe}UIscuuc)!+;oEI!oGs^l`Y1v!)Bry}U)O{9A8!*Y_y=qp_}d z`*F}eYrf+D@+XY9I5Pb>)sFaX*Q=b}9jxv;O6+Jyb!fq4$pLbeko7RrjtGaMcEAIMcoe3KR?O8;}9iE}Ypc$e0Ufe?R`euzuOJlK$w;+Pbva zA=OnVIb0|((!bi1Z-_=}hz3O3DmBt7{o(lyK_#$kqI~IE@fXkDKLA-kroXz@`A=QT z9e;J*@y8zj##sygy$9T=FY8D$vk{KKP>oRWqK z+e>0a(YjNm{PD-WcmopE&MJWev)yDb=cp_1 z@?h?Od*#1UjME=PpqUX z-hLdcGuaA$`@25#4<`xM(rmK^?ko%`OWHCES`sVOlCExvCD~8a?5(%C!wnvjBimle zTh$zO-P^W_R)6fgjpKmII8?~(mLr53<^&luAJ(h^-UFUjL;+C6Y6}K_w_}*ef!$dd zYd8H+O$_Go5}-0}??UFRU(T@H6^j}r26YuK3Yo)NSU~C8F{RuQgGZwVf+5O`% zO9$(Zs((RW_B>2=?j))*iMe*K z_5nl=Of|=PZUHxw-FVJ-8Q(=OlRE_qNun-;QsuI!PfsHt42Xog5SM#t>C3GcoG6IOg8Dh26X6w&cn- zNfKBh>#9%vmu3!&T4B~#tF5TRdc0GJ)_>rE<($)@HeS!Rn;KMq>cj_7Ab1nt`c}Yl zX7eBZ`Q^1M=Uug%50C6G&ac|%PfT4r%3eGJ|J3>aw{P_ye`UcRzVZZkL~z}$oU!vk za(33(Dv^g$Fxa3`Yp#MU&j>sWn-i<`ewt;{_k8A+It>(>dJ~w9iUe+1gi1c)5q|}O zjqx^1tfJKoO68d`%q&<_uHL|#SaT-uY+sXo%Re_yWhcq!jVoPc}{1lDv-9w z{PXRbFA`hw=dy!;*#Xt`pB>yR;I}_}|L+>+@+ZMa$B$?J&4)jm{iDyDXjH50r*r=D z(fv=3`NhJI)30924_koWil)<*y?@L2mJj(PZ;i7PVYNwclM>1DEp6`-wnl65#3X5- za7&9ZoBy5s;{n^?);0FjT`m$ zf8~&WIrpl+z?<0w=<}*u=R*rxE2KuB*+b`m8`FA;c2bM%Zh{TDyI5xd(ovRtdIdNk z6t43fess5k9SCshRZA|0N`LFs&PRiLWvfl@&Dbk956g14<)hl{~A+QEOx ziUU2PGWtUVlm$@es^snl&PHY@>BGM2)k}w$2!Y@WQ(H<3I1RvAhw`r;_kOk9Pm8$L zf8~}c68{@s&Itc&hbzu*kj=thoc&X8ozlPBRI2z*gVMwwlFe9(;(zp9#?*btx>YSY zo+fZot}>_(?jdh-mcUe%$?TKD22`mH(XMm-l6G0D;MAbpKR|Z0uj_gvulu#_dIRgo zFXpr+wpkFpo@V8GJNGnEVAE8_wj}nxp_)rfskhzq09xZ6Vt-~yp9pD5{uN(50CxZy zOgkon*`&YxnyrxekAFSyzdC6&E9v5&-@Y#m^Dl0;fBEer{^rACML1k3*F{s zx8s(ykzD0VX3!#VrP4~aMknzgB4(EuhqE-xqMmUG-g>2!9Ouojj#So4YzEgS-Dc@} z2cFq1thEuj%uYASWgHP~fyYduab|0C$eVHZHUL!Q`K2*XO@CLubO2jjI(C&U?VrBw zf5yHKt8DoBxYEXdbmu>M;&1QoKXT5WTypj4q(kP}eUEjU6l{CzEirqE!PF{ly+A)Y zguvvX>gHP!8})`YF;c79e?SGqBM-5wTu52v|^2Qk5?Pk8_ zM0vMG%UWQ_ZIA;W!8)Ue^Ycw~vXS42Kt13?+$_v$2}e4(K+n9ItM;a>h)HD7JM?VP zMLF7>yj*O+0i&p3>L_5{DiQ-Uz4Vo2B0ZQceFeEoUrCnrx8HR6SsMRxz>Q(E^FO%o z{`c9!uYd2vTUqm`cKI*u&+pNn+Uzf1g3+@$mm{=5AZ56VNyeFG5;YeD#g1E8p&@_$S}~@DTpd zQvanf|Jm1{e`n(#y_NlX<-dLZa`WGM=%-OG-n@D)9;!2cG57~#|IsJ=>C~%k`j1+W zV1Bmy#p|y(&i<(vr9X8-CG;Xzt#(ZeNz3JuwPXUcVvC_Frn)LrsGEbN`b`8}_mRwnH` zYXB;dEA;ANo7&DT-jQwI?y;clCJoa*cGDe-)tQn*A@#6SJQMlc9lOn}uWHT&#U`3W zUzX(8J?xvs;x=-U>w54#D-dmpl$<5vynkv^7L4d!@68F*gNIY>qI9D8g{Z(O)edgv zodprWVyJecaMtt|n8;O6&6XP7)V4oQGsO~V!AV@WE-zdQ)eqP37sEfU`wvaH{jvFv z6R&!W&940Rm+SuV*_2tMA6_AZzc#z{y6~?(tE&e=u>bJ2T1jx> zqg<=Afq_7&woGCzUx-{S2*~{Oxs8`=tz_M;;}~j@S&ju&#WI_u18XW+>wjYy&=(>` z>%{BmYcCy+Bi-F&@+6nmKyR`9o;jk!DQvZ6OvOQ9MXS?W7~0)c2}l;)GK^!1g=M1_ zR)oTV%z@B?>@hj~)}i4csoH^7qa$U;;l&Kd3Jy?*f7c%FeIxko*Y^^T!((Njpa6Ct zxt?#hUdC71Pp@*WRzUWzhJXLtH~E7#>9WZW!{WsU|1(FAJk0*!(7E`85HJ1)=fy>T z`1$_>s|j#)MLaa0Wu$M^fI?_J7StcXYOGd~az?)1rDadL0KC=fK7<1nFRvEa-@Vj; zmCV_k&Vk8Xl@_ov^kcsGV{`Ye5LskGaZ7}K7y$hrJySRU+^3M z!SC$}HtND}df_+dxL^MJ|AA3atN#ygZ`Q0jtgVavm%OE@lnt2f+?Nc=a^E1gn3oh7 zkc|xn47f?6>R?i9CkeAt}cOkt} z);Y=O)@=3M)S?T{Lk1aWs1NBnZ)_*GRhd;3wK`FNtj$c{;D5k7>8$Q8A0PDM;9#2E zNniE!x?S&Owt+U?D72n)KSgHAuT1ojew4FhtG4xi-7Y`z>`^GY%zW_`?T*KD{W`$k zyq$ASIUm1x9v^+y8rN^kA{xf?6IWExu%1YNZS7~R`@8<=@tgRy$Gok9Kmr&FnHn8?-e1Mt4zfcV^H?s_!Yc z0-2kpS`%dZ2*jt?#ZOfUBceuQM5wn#;+afWt{R7-<9}CF$ylYxRaagQE4N+V%&rNx zt@K#8jV0$|%;5xP#xA5=_EAYc`9$vDy(z9KLW6$J0R736ao_AG7W?Dx{lrZqZ5VqV zSIU3pm62>+ceQ%$O)M8ICM{E6L9 z`G0yqqk8zOyBbYPv-)J+$wg_GW{8T~h@-1ZZ&5^7r7kTKL#b>;ZvsK}9zkg9UV6}G zNIGQ^n60vdT)M0~Q#A8zqI)o+zTen>b4ul#ueV6)ZfJ+Ir|sgU)-mzz%X+ND^^mX8 zc>KK|*13C7D!$HbwaixdI@h2){`p<6bAOw6rcmu z2KL$Xvb3cg4jIbe6bJV8iqczzwvik}KoQc+Q<-EW&34{@r7xr*ud1Ac0-+@z*A3tr zjttDFD8BW!cA?>U4KmgL&OvbhjO*;5SUZs4HQzTkseQ*yCH*Id`CZ2hn2Nt*+Ul1t zRn+O~Uu(~a)ARV0J|5(D(Cz;Nj%j}759vaUZdoWvVS&U4Oz)S zi43JKEj+^*g@xZsWzmJTmaoR^Tt9-JmTrrXBcpH=9Fi;yk8Ypt{tN%cAM_yi&+Cxu z|6zS^G}V9;`{a^SZYQ~}m%nzFuVuM@s8W8 z2X(6JUKmo0mZ=Iimz?d@145=u^%qglE9+@3d(6|D)?K5er+@jR^*a{CByCMb#_Ptr zt+V=s^Bd#-F#%mMaW>vjUG;)we!3p^xJz!jSFI@$d&3j^aLis2X`&CF@+>fpZ+nG+ z2OMIme)H&_(uO%|No+S{bC@0;4n9awGe=nKd>thxjHlu?K@dhV7!saQFh00W`oVSf zVdu2uqW9tSaDP2WM(;fijk%CLrU-_ls33lKaccyQOFV6Oh_34JR;h(rK#{pLdiBNJdQH9XB=xIP@MR&3YRW53YXN${J#bk{bwS6COG)^5g zH!dC%dw&@WOUGei%V@S4)7o#JMzxyQh)hSLgW?&1Bxwb^5hM{lFaBCML!2U!+f~^0 z7{2Sw?k5j5Z&FoI5U>@q7Kjcx9e6|!@&a;Ao-HMFxjve3sgP;$`> zBn8u(jYtNmN#|>)DETK-F_66=QKg21&H3=i`A|ctH}GL=bN$PGw~dk|8=K=u+Ryy* z@qbf37++EEjIhdQ`Z;6@6yX6pTlQJB4Hd{yIP;ye_=c=P`%5^trF14!(Q+CQgP&3_ zP)swgmr+TUZpZPN1Agr&v^w&@yaLv_9~~gd@wc|dZNLiMcZfWXR>VK6$SCFRL5DD(CBFh*KJNMuR0k-wDb*R^6 zDTLvhblC|bz2trxSufv^rfAG@qB#8G-|CBw!~|20f8)7|UkrEro8$bn>Bo+122Hfz zTydX@&JRAmO{hI~bObAT(>*mGk`2)Cupwz8E6=OmkVT?8)Wdpp-hgp9S3dH5S$`L^ zt%sd5=_E<>phEdvSClX&?RJ!kp&p@iOUDs){BZA%O_CtkTTw{|%rFoL*2_8UZCq2h z4*p7;ayi*J-#Gf$j%tJM`O39(wucxP`Ue|*Dky4Bh*qu9krfKIzSGpIjIuF8MA8|# z?){sI+Iqfwu&JW9dk@>YtfTE%9DiH!62FQG!GXdoM`_0UY~~o*q3{;BeH7_D&fyB9 z8p&XufunmlZqJjp1Z@1#*F#%Ac&RM?#58{JYfUo!=p{~5JN7v}tCYD)tDjMSAP%#M z;$7CRS7az}Op_|D>=&Zp8A^()hl|oZ>8TjVQM@UZ2|iX!rAt&3n&BI~f`4!l93~UE z6ONDl$dOkCz_=WJ+cAvikKe{|vRuyjxY^o(mA_~5CWDFQ(GWpJ6qYE^ita8;?}+J& z%cN@U)pb}jzUfKf{;~}Ztz@gz4&-R*8EbQAgRxpe)4KkE+f9T~f_9Dh0-SPw)t|mt zk!BdL&$^tiA^bs(u1+gqC4U%ltRKAoCvVcD&W@BU9YLOcc$Q-ysdAKkQ{6RN6$g?= zC^b-2RYXn;M5Fl{9pX(hKj{I{Ei^Aok-g2fe97%>ziGw8(_b(AhFV4|A}`}#J7iO& z*mDb2o49jD-M5K?v~Haj%tR4(1{^TTgi(VO%Hv%2fNO<+!0Dv%uYZmF@+r`e(oDoM zl6-~T!ETsJJc1ugoFxg`K9Rmbp6>0$jy~^ghls@Wk0y)~yrSBkn|JiEFYwVn114sW zZLYt)!Cq{B?Oi9SI$wKzV;Nrz9{1s+?{7c*2R?JZ%Y=1bdbX0B5lYq?vh5RNJ#K7- z5jZN;JMhDlKP&*zA+fA#dRZtU!|n z9$YVIjQp-+P!?`9MnBlD-+m=m)TCV3aen^F-H?-ul?A5ud4H)RAO>QEfvB!)O)EGJ zQ_S-dAxNvJP4>8CW>V*0~~BEqm_j1 zIm?xn+HnSlkAGzeGf)q3Bp_4Rb9q#l#G&h49xvZK;;+wQ3^6t?3qQJeS9)ma2N%Qf znlL^xq7u|lSAP*N=v<|4zGFEa>kdeQeRWPjCI$M%;fC`@H=<~eVZWIzbR5^O?*{oB z@G<0}0MF3Zn@_&r8|(HZmD`OkmOlIky*jK5?a2P!W0k7+h+{C59quI;F-gai5{PV< zKh!t$VaK>^U)6@@M zcs;3Kcw?jciA?N-+`^1RxB(3EThKwxxrS4Q^PTds+J5ejG0_GIKC&IMkAJ zSmM6(w|_wQR8Pjx4qrS6Sr8tO)O;UDHt=g}|M2T4?(q{#YzN%YuiWa*Afs2#*UyW1 zRhxPbL2W5H^cu~Kn3}#BNGb+mC^)gVKIE{OZMZw(C((r2TF7CWqG`hy47=W-1#j2%FKqb_?tf>*^H1D=`Ro%I{J{^ew%@qf(AMvI ztVRnJgG|(F67=M1vx4ZEo#~!l>SKbStwfS37sX4|i$ZKUt6r#Vd4O6{MXJ7vZQ?W^ITyUYO>E9 z6Sk@Y>>_(JM`=^O*1yVrg^5Ql@?^;rAb-1U#0WH6h%QGJNogqeMGFO76`2Ku=?CH^ z#7~wdL$+N7Zi&H`mf5^cn*bJcY`7ugafDuo-D8xt*K#BoqP-y`?&W54nr z{qhF;>e41vm7n-DaaEPdw0|;k z4uNhJFBSDmdQ}SJrp4hQ&BZoIr$}$>+E>WUIg5MT4E|Ovj3|EcxPlyl&;5r`pO}sK z$t!)^D6@}_`fG0zUo#AH{{8(I1N*y{4F=t?(HV!Gd~Nq<-^kTqYJNV?05w#`4fSFF zB$P45�mLlz_jobX1$rkSYX;T7RW}bHsAZ_e{!Jh=JKS%*Bx`14`Aj(DFmBbj8l3 zb%#`X*Oi&?ok=feJe@2Sj(oG6tc19eR$3!Ii0@E@ib&}xbA5K!JzJG&K_RS|(N(2? zXf9SZ_z9*UQj{`n3-aKPv>N;M6ZpFF`WJq`cRsfAM}9Xv_V>-#PyYDXe1C5nADa{G zJ=)Ti+`uOk(QvAG6t5dM(q+2x0|ipwK53B3T2m0c(H)~OG(;6C_TlU{><%8?sF)mS zD9sU&oLq{+I2SJyGX?u>ziydew97Vr_hDyNp&0WlJ{%FzdUO8 z!y5Kq^J$a(<>P+%Q)g+2et-5^+D_Oy1%j^)k9p;2ly>6J4Z*?2kg`!u24D}M_! zp(~Kfv<+YYi|6}&mMLtozY51oeDcg($A*7$Y~2tuA8H7oWAdN(xw=qP6qtPj zCZ6S-+KOz4iqBBq)G`X2v)rtn8WoRxqjjy_XgC#j|B9WUi(=qfA*<- zbi)x?dj0k1|HMboy|GQQ_{2keZ&?_fvX+rF+l;F+^HUGC_J5ADrH;-zdUlZ+T;O>Y zNoV}1fSCT%a8^6m5B5(Nt^L)FU``z`lB$%y5`)n`4TaK7Gc021I7tAk|;i zmE3V_r=*!}Tz_2G)uoXFIh|=i$r4;FKKq6iBcV|Oj>8-dTkt~d4|dYl^cJ+CZ5}y0 zz5O$P8|AD|A#ya$CwqgSu^gv<((3I7R%=Y^r{<>wgN-j93iNilbK>GGT*ds=9OImk znO;^e=TaRI*q``JBk}mmjmJR%*)n3O%jb>fjGPZHk$=CR<7oIe4Ss;0^V~<;qZKH? zn#gQZtT4WhG{GAWIcjH*tTRMNw6AWTkOMSmBP>iGC%3Q@31pt*G9qdHqjGcIV_73n zp9Dy(9bJ;ndh@ZfCR3Mi820tO06_W4pEw^3*)WHwC!+)LJ(VG$&4WY_u9uP%u4=lRorM}2Y z6wYj!-t28;S4N-Cj(C{o!{FzEfw`Xtc;L0a+oAIJ>g{*$L!+d5Bq6>x$EQYso7tFL z`1Wy0T}R$6%ufq_LK_RbaEI4aky$?Af!4wWn9lw7kA4`|*7Rqu#IWA^=!>t9!R=}{ zoPRCw!JM?;cwnQJs9D}+pQ&ac>ME*^K~^M+0$Y>m9`Td)zJn0F+c{<-fdf?!*Hp_| zy4H(Ac5~`F$PS3C>;KxHBd_%(?DlLpwG|S;n8yV_fBNSqo#zA~m09z9a_ex)?dP=H z1$X|&jogA4qBSTTBun&YG=Kv|3(!L-7=J!ea}2LgRvKpm(^PJn^$1~`MAUmBiz-GF zzYr_bE$lVaOWK_=3#*_%S<&GwF=_KKu?UvPT2FQphc=T!KBevvmKRy7Dap{A=!{kf zi)2ka*HcQeOTB5UZ^(uDcVy+Ht%4nL*F+$pi@dvH72&cCYh_bMH*fP&&U$>!_J41s zuQseb%E~<4bS-mLQY6PVAFJzTYiDJmZkwM|Zimxk*1i^%tr)!cy?;C0C;Z;z_ttPP^81qCo1jeq)qFEU2Mz-pED_`c z?HD|vmOx>S2r|K23JbpCQyX&wN`e!9Zb(ajGK&fV!zPa-=VxKT%#>UD3j>Bw1#05} zAW?QXecu3L_5z@m*8qFyMR3790Ae_M3m52@pivqF3US}S9z+5vGj;$q?|+{J-kzVc zz?l-Q_&G0(K*Ur92t&&T9M}@@n4$ub(1_0vDp0QN;x$q7~&r#`A4fDZG~uGrAK-fiP$E^gS2pKm8#F zprZ>@DGb|gO%onNxYmZM;`{iANUco?{?AWx7F)Cc3s?N}qk=A8OgS4fsYzvhy_7(y z@Qf0~rT;vZ_yN~w)>zxGMnJ!nee^^KANvV_1~B}<{|e*3*u1qlKYtB-O+f&qh7Ch= zFoCH@TN8xkrzV_y%6V#@qHHL4Z4d-OkOYAVD?ymPg@dr_g~@v~mSpJ=|Fq-3`m+Em zj2lp*#ES~1la~ZA^ZW&<;rN!+GTqlW+a_wl@-j+l(92|B3=70$VxN<&&h$M%ya%A2 zLu#%ciNk04TlyfFZ_lg!P^JsO zbnHpG1-8zwMJL-@Kte6=fa#b{5U#OOYFKTcaO9dv5rlfGTYzcj;5xGr5qTTj1{9QR zqf5vAe*T=b2Vz+un+u5On4NXs&T6!kY4<~0>~)p0+D5bndVh%@dIfYWd$FU`uvKFI z&Mw-9