From de8b5a3624499bdb9fa6e99840259998638f093f Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Sat, 22 Feb 2025 06:33:29 +0100 Subject: [PATCH 001/188] llama.swiftui : add "Done" dismiss button to help view (#11998) The commit updates the help view in the llama.swiftui example to use a NavigationView and a Done button to dismiss the help view. The motivation for this is that without this change there is now way to dimiss the help view. --- .../llama.swiftui/UI/ContentView.swift | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift b/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift index 30c2dc4310..1c3cd9d2ef 100644 --- a/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift +++ b/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift @@ -124,15 +124,26 @@ struct ContentView: View { } } }.sheet(isPresented: $showingHelp) { // Sheet for help modal - VStack(alignment: .leading) { + NavigationView { VStack(alignment: .leading) { - Text("1. Make sure the model is in GGUF Format") - .padding() - Text("2. Copy the download link of the quantized model") - .padding() + VStack(alignment: .leading) { + Text("1. Make sure the model is in GGUF Format") + .padding() + Text("2. Copy the download link of the quantized model") + .padding() + } + Spacer() } - Spacer() - } + .navigationTitle("Help") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .navigationBarTrailing) { + Button("Done") { + showingHelp = false + } + } + } + } } } } From d70908421ff22b013e8209f2d12e5c750663c620 Mon Sep 17 00:00:00 2001 From: Gian-Carlo Pascutto Date: Sat, 22 Feb 2025 09:43:24 +0100 Subject: [PATCH 002/188] cuda: Add Q5_1, Q5_0, Q4_1 and Q4_0 to F32 conversion support. (#12000) --- ggml/src/ggml-cuda/cpy.cu | 97 +++++++++++++++++++++++++++++++-- ggml/src/ggml-cuda/ggml-cuda.cu | 12 ++++ 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu index 54c0f66d2d..cca2bee0b2 100644 --- a/ggml/src/ggml-cuda/cpy.cu +++ b/ggml/src/ggml-cuda/cpy.cu @@ -1,4 +1,5 @@ #include "cpy.cuh" +#include "dequantize.cuh" typedef void (*cpy_kernel_t)(const char * cx, char * cdst); @@ -82,13 +83,14 @@ static __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) { } static __device__ void cpy_blck_q8_0_f32(const char * cxi, char * cdsti) { - const block_q8_0 * xi = (const block_q8_0 *) cxi; - float * dsti = (float *) cdsti; + float * cdstf = (float *)(cdsti); - const float d = (float)xi->d; - - for (int j = 0; j < QK8_0; j++) { - dsti[j] = xi->qs[j] * d; +#pragma unroll + for (int j = 0; j < QK8_0; j += 2) { + dfloat2 dq; + dequantize_q8_0(cxi, 0, j, dq); + *(cdstf + j) = dq.x; + *(cdstf + j + 1) = dq.y; } } @@ -225,6 +227,18 @@ static __device__ void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) { memcpy(dsti->qh, &qh, sizeof(qh)); } +template +static __device__ void cpy_blck_q_f32(const char * cxi, char * cdsti) { + float * cdstf = (float *)(cdsti); + +#pragma unroll + for (int j = 0; j < qk/2; j++) { + dfloat2 dq; + dequant(cxi, 0, j, dq); + *(cdstf + j) = dq.x; + *(cdstf + j + qk/2) = dq.y; + } +} static __device__ __forceinline__ int best_index_int8(int n, const int8_t * val, float x) { if (x <= val[0]) return 0; @@ -387,6 +401,19 @@ static void ggml_cpy_f32_q4_0_cuda( (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } +static void ggml_cpy_q4_0_f32_cuda( + const char * cx, char * cdst, const int ne, + const int ne00, const int ne01, const int ne02, + const int nb00, const int nb01, const int nb02, + const int nb03, const int ne10, const int ne11, const int ne12, + const int nb10, const int nb11, const int nb12, const int nb13, + cudaStream_t stream) { + const int num_blocks = ne; + cpy_q_f32, QK4_0><<>>( + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13); +} + static void ggml_cpy_f32_q4_1_cuda( const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, @@ -398,6 +425,19 @@ static void ggml_cpy_f32_q4_1_cuda( (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } +static void ggml_cpy_q4_1_f32_cuda( + const char * cx, char * cdst, const int ne, + const int ne00, const int ne01, const int ne02, + const int nb00, const int nb01, const int nb02, + const int nb03, const int ne10, const int ne11, const int ne12, + const int nb10, const int nb11, const int nb12, const int nb13, + cudaStream_t stream) { + const int num_blocks = ne; + cpy_q_f32, QK4_1><<>>( + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13); +} + static void ggml_cpy_f32_q5_0_cuda( const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, @@ -409,6 +449,19 @@ static void ggml_cpy_f32_q5_0_cuda( (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } +static void ggml_cpy_q5_0_f32_cuda( + const char * cx, char * cdst, const int ne, + const int ne00, const int ne01, const int ne02, + const int nb00, const int nb01, const int nb02, + const int nb03, const int ne10, const int ne11, const int ne12, + const int nb10, const int nb11, const int nb12, const int nb13, + cudaStream_t stream) { + const int num_blocks = ne; + cpy_q_f32, QK5_0><<>>( + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13); +} + static void ggml_cpy_f32_q5_1_cuda( const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, @@ -420,6 +473,19 @@ static void ggml_cpy_f32_q5_1_cuda( (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } +static void ggml_cpy_q5_1_f32_cuda( + const char * cx, char * cdst, const int ne, + const int ne00, const int ne01, const int ne02, + const int nb00, const int nb01, const int nb02, + const int nb03, const int ne10, const int ne11, const int ne12, + const int nb10, const int nb11, const int nb12, const int nb13, + cudaStream_t stream) { + const int num_blocks = ne; + cpy_q_f32, QK5_1><<>>( + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13); +} + static void ggml_cpy_f32_iq4_nl_cuda( const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, @@ -488,14 +554,25 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg ggml_cpy_q8_0_f32_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) { ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q4_0 && src1->type == GGML_TYPE_F32) { + ggml_cpy_q4_0_f32_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) { ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q4_1 && src1->type == GGML_TYPE_F32) { + ggml_cpy_q4_1_f32_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) { ggml_cpy_f32_q5_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) { + ggml_cpy_q5_0_f32_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) { ggml_cpy_f32_iq4_nl_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) { ggml_cpy_f32_q5_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q5_1 && src1->type == GGML_TYPE_F32) { + ggml_cpy_q5_1_f32_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) { @@ -524,14 +601,22 @@ void* ggml_cuda_cpy_fn(const ggml_tensor * src0, ggml_tensor * src1) { return (void*) cpy_q_f32; } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) { return (void*) cpy_f32_q; + } else if (src0->type == GGML_TYPE_Q4_0 && src1->type == GGML_TYPE_F32) { + return (void*) cpy_q_f32, QK4_0>; } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) { return (void*) cpy_f32_q; + } else if (src0->type == GGML_TYPE_Q4_1 && src1->type == GGML_TYPE_F32) { + return (void*) cpy_q_f32, QK4_1>; } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) { return (void*) cpy_f32_q; + } else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) { + return (void*) cpy_q_f32, QK5_0>; } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) { return (void*) cpy_f32_q; } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) { return (void*) cpy_f32_q; + } else if (src0->type == GGML_TYPE_Q5_1 && src1->type == GGML_TYPE_F32) { + return (void*) cpy_q_f32, QK5_1>; } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { return (void*) cpy_f32_f16; } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) { diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index cc772801e0..f685423215 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3075,15 +3075,27 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_0) { return true; } + if (src0_type == GGML_TYPE_Q4_0 && src1_type == GGML_TYPE_F32) { + return true; + } if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_1) { return true; } + if (src0_type == GGML_TYPE_Q4_1 && src1_type == GGML_TYPE_F32) { + return true; + } if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_0) { return true; } + if (src0_type == GGML_TYPE_Q5_0 && src1_type == GGML_TYPE_F32) { + return true; + } if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_1) { return true; } + if (src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_F32) { + return true; + } if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_IQ4_NL) { return true; } From cf756d6e0a314e0fe25ff944341d79ea8c94cc96 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 22 Feb 2025 12:46:31 +0200 Subject: [PATCH 003/188] server : disable Nagle's algorithm (#12020) --- examples/server/utils.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 6f8ab2b93a..4db4c783af 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -7,6 +7,8 @@ // increase max payload length to allow use of larger context size #define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 1048576 +// disable Nagle's algorithm +#define CPPHTTPLIB_TCP_NODELAY true #include "httplib.h" // Change JSON_ASSERT from assert() to GGML_ASSERT: From 335eb04a91f481f37c0c9b302ee31b449b04c3e9 Mon Sep 17 00:00:00 2001 From: Rohanjames1997 Date: Sat, 22 Feb 2025 04:48:57 -0600 Subject: [PATCH 004/188] ci : Build on Github-hosted arm64 runners (#12009) --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6555eb479..859d538c77 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -173,7 +173,10 @@ jobs: name: llama-bin-macos-x64.zip ubuntu-cpu-cmake: - runs-on: ubuntu-22.04 + strategy: + matrix: + os: [ubuntu-22.04, ubuntu-22.04-arm] + runs-on: ${{ matrix.os }} steps: - name: Clone From 5fa07c2f93c73161bf09ef0b23b5d2686f9a073e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 22 Feb 2025 12:20:17 +0100 Subject: [PATCH 005/188] CUDA: optimize FA for GQA + large batches (#12014) --- ggml/src/ggml-cuda/cp-async.cuh | 2 +- ggml/src/ggml-cuda/fattn-common.cuh | 120 ++- ggml/src/ggml-cuda/fattn-mma-f16.cuh | 810 ++++++++++++------ ggml/src/ggml-cuda/fattn-tile-f16.cu | 4 +- ggml/src/ggml-cuda/fattn-tile-f32.cu | 4 +- ggml/src/ggml-cuda/fattn-vec-f16.cuh | 2 +- ggml/src/ggml-cuda/fattn-vec-f32.cuh | 2 +- ggml/src/ggml-cuda/fattn-wmma-f16.cu | 6 +- ggml/src/ggml-cuda/fattn.cu | 73 +- ggml/src/ggml-cuda/mma.cuh | 75 ++ .../fattn-mma-f16-instance-cpb16.cu | 10 - .../fattn-mma-f16-instance-cpb32.cu | 10 - .../fattn-mma-f16-instance-cpb64.cu | 10 - .../fattn-mma-f16-instance-cpb8.cu | 10 - ...attn-mma-f16-instance-ncols1_1-ncols2_8.cu | 10 + ...ttn-mma-f16-instance-ncols1_16-ncols2_1.cu | 10 + ...ttn-mma-f16-instance-ncols1_16-ncols2_2.cu | 10 + ...ttn-mma-f16-instance-ncols1_16-ncols2_4.cu | 10 + ...attn-mma-f16-instance-ncols1_2-ncols2_4.cu | 10 + ...attn-mma-f16-instance-ncols1_2-ncols2_8.cu | 10 + ...ttn-mma-f16-instance-ncols1_32-ncols2_1.cu | 10 + ...ttn-mma-f16-instance-ncols1_32-ncols2_2.cu | 10 + ...attn-mma-f16-instance-ncols1_4-ncols2_2.cu | 10 + ...attn-mma-f16-instance-ncols1_4-ncols2_4.cu | 10 + ...attn-mma-f16-instance-ncols1_4-ncols2_8.cu | 10 + ...ttn-mma-f16-instance-ncols1_64-ncols2_1.cu | 10 + ...attn-mma-f16-instance-ncols1_8-ncols2_1.cu | 10 + ...attn-mma-f16-instance-ncols1_8-ncols2_2.cu | 10 + ...attn-mma-f16-instance-ncols1_8-ncols2_4.cu | 10 + ...attn-mma-f16-instance-ncols1_8-ncols2_8.cu | 10 + .../template-instances/generate_cu_files.py | 18 +- tests/test-backend-ops.cpp | 35 +- 32 files changed, 940 insertions(+), 411 deletions(-) delete mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb16.cu delete mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb32.cu delete mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb64.cu delete mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb8.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu diff --git a/ggml/src/ggml-cuda/cp-async.cuh b/ggml/src/ggml-cuda/cp-async.cuh index 51aa41e7e6..ecb659997b 100644 --- a/ggml/src/ggml-cuda/cp-async.cuh +++ b/ggml/src/ggml-cuda/cp-async.cuh @@ -24,7 +24,7 @@ static __device__ __forceinline__ void cp_async_cg_16(const unsigned int dst, co } else #endif // CUDART_VERSION >= 11040 { - asm volatile("cp.async.cg.shared.global.L2 [%0], [%1], 16;" + asm volatile("cp.async.cg.shared.global [%0], [%1], 16;" : : "r"(dst), "l"(src)); } #else diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index fefbd319ba..7b9566fb4b 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -516,27 +516,25 @@ constexpr __device__ dequantize_1_f32_t get_dequantize_1_f32(ggml_type type_V) { nullptr; } -// The HIP compiler for some reason complains that it can't unroll a loop because of the jt*ncols + j >= ne01 conditional. -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wpass-failed" -#endif // __clang__ - -template // D == head size -#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) +template // D == head size __launch_bounds__(D, 1) -#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) static __global__ void flash_attn_stream_k_fixup( float * __restrict__ dst, const float2 * __restrict__ dst_fixup, const int ne01, const int ne02, const int ne11) { - const float * dst_fixup_data = ((const float *) dst_fixup) + gridDim.x*(2*2*ncols); - - const int iter_k = ne11 / KQ_stride; - const int iter_j = (ne01 + (ncols - 1)) / ncols; + constexpr int ncols = ncols1*ncols2; const int bidx0 = blockIdx.x; + const int j = blockIdx.y; + const int c = blockIdx.z; + const int jc = j*ncols2 + c; + const int tid = threadIdx.x; - const int kbc0 = (bidx0 + 0)*iter_k*iter_j*ne02 / gridDim.x; - const int kbc0_stop = (bidx0 + 1)*iter_k*iter_j*ne02 / gridDim.x; + const float * dst_fixup_data = ((const float *) dst_fixup) + gridDim.x*(2*2*ncols); + + const int iter_k = ne11 / FATTN_KQ_STRIDE; + const int iter_j = (ne01 + (ncols1 - 1)) / ncols1; + + const int kbc0 = (bidx0 + 0)*iter_k*iter_j*(ne02/ncols2) / gridDim.x; + const int kbc0_stop = (bidx0 + 1)*iter_k*iter_j*(ne02/ncols2) / gridDim.x; const bool did_not_have_any_data = kbc0 == kbc0_stop; const bool wrote_beginning_of_tile = kbc0 % iter_k == 0; @@ -548,22 +546,22 @@ static __global__ void flash_attn_stream_k_fixup( const int channel = kbc0 / (iter_k*iter_j); const int jt = (kbc0 - channel*iter_k*iter_j) / iter_k; - dst += jt*ncols*ne02*D + channel*D; + if (jt*ncols1 + j >= ne01) { + return; + } + + dst += jt*ne02*(ncols1*D) + channel*(ncols2*D) + (j*ne02 + c)*D + tid; // Load the partial result that needs a fixup: - float dst_val[ncols] = {0.0f}; - float max_val[ncols] = {0.0f}; - float rowsum[ncols] = {0.0f}; -#pragma unroll - for (int j = 0; j < ncols; ++j) { - if (jt*ncols + j >= ne01) { - break; - } - dst_val[j] = dst[j*ne02*D + threadIdx.x]; + float dst_val = 0.0f; + float max_val = 0.0f; + float rowsum = 0.0f; + { + dst_val = *dst; - const float2 tmp = dst_fixup[bidx0*ncols + j]; - max_val[j] = tmp.x; - rowsum[j] = tmp.y; + const float2 tmp = dst_fixup[bidx0*ncols + jc]; + max_val = tmp.x; + rowsum = tmp.y; } // Iterate over previous blocks and compute the combined results. @@ -571,36 +569,30 @@ static __global__ void flash_attn_stream_k_fixup( int bidx = bidx0 - 1; int kbc_stop = kbc0; while(true) { - const int kbc = bidx*iter_k*iter_j*ne02 / gridDim.x; + const int kbc = bidx*iter_k*iter_j*(ne02/ncols2) / gridDim.x; if (kbc == kbc_stop) { // Did not have any data. bidx--; kbc_stop = kbc; continue; } -#pragma unroll - for (int j = 0; j < ncols; ++j) { - if (jt*ncols + j >= ne01) { - break; - } - const float dst_add = dst_fixup_data[bidx*ncols*D + j*D + threadIdx.x]; + const float dst_add = dst_fixup_data[bidx*ncols*D + jc*D + tid]; - const float2 tmp = dst_fixup[(gridDim.x + bidx)*ncols + j]; + const float2 tmp = dst_fixup[(gridDim.x + bidx)*ncols + jc]; - // Scale the current and new value accumulators depending on the max. values. - const float max_val_new = fmaxf(max_val[j], tmp.x); + // Scale the current and new value accumulators depending on the max. values. + const float max_val_new = fmaxf(max_val, tmp.x); - const float diff_val = max_val[j] - max_val_new; - const float diff_add = tmp.x - max_val_new; + const float diff_val = max_val - max_val_new; + const float diff_add = tmp.x - max_val_new; - const float scale_val = diff_val >= SOFTMAX_FTZ_THRESHOLD ? expf(diff_val) : 0.0f; - const float scale_add = diff_add >= SOFTMAX_FTZ_THRESHOLD ? expf(diff_add) : 0.0f; + const float scale_val = diff_val >= SOFTMAX_FTZ_THRESHOLD ? expf(diff_val) : 0.0f; + const float scale_add = diff_add >= SOFTMAX_FTZ_THRESHOLD ? expf(diff_add) : 0.0f; - dst_val[j] = scale_val*dst_val[j] + scale_add*dst_add; - rowsum[j] = scale_val*rowsum[j] + scale_add*tmp.y; + dst_val = scale_val*dst_val + scale_add*dst_add; + rowsum = scale_val*rowsum + scale_add*tmp.y; - max_val[j] = max_val_new; - } + max_val = max_val_new; // If this block started in a previous tile we are done and don't need to combine additional partial results. if (kbc % iter_k == 0 || kbc/iter_k < kbc0/iter_k) { @@ -611,19 +603,9 @@ static __global__ void flash_attn_stream_k_fixup( } // Write back final result: -#pragma unroll - for (int j = 0; j < ncols; ++j) { - if (jt*ncols + j >= ne01) { - return; - } - dst[j*ne02*D + threadIdx.x] = dst_val[j] / rowsum[j]; - } + *dst = dst_val / rowsum; } -#ifdef __clang__ -#pragma clang diagnostic pop -#endif // __clang__ - template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(D, 1) @@ -690,11 +672,13 @@ static void on_no_fattn_vec_case(const int D) { } // parallel_blocks == 0 is stream-k decomposition -template +template void launch_fattn( ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared, const bool need_f16_K, const bool need_f16_V ) { + constexpr int ncols = ncols1 * ncols2; + const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; @@ -763,25 +747,26 @@ void launch_fattn( nb23 = nb23*bs*sizeof(half)/ts; } - const int ntiles_x = ((Q->ne[1] + cols_per_block - 1) / cols_per_block); - const int ntiles_total = ntiles_x*Q->ne[2]*Q->ne[3]; + const int ntiles_x = ((Q->ne[1] + ncols1 - 1) / ncols1); + const int ntiles_total = ntiles_x * (Q->ne[2] / ncols2) * Q->ne[3]; const dim3 block_dim(WARP_SIZE, nwarps, 1); dim3 blocks_num; if (parallel_blocks == 0) { // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup. - const int tiles_nwaves = (ntiles_total + 2*nsm - 1) / (2*nsm); - const int tiles_efficiency_percent = 100 * ntiles_total / (2*nsm*tiles_nwaves); + const int max_blocks = 2*nsm; + const int tiles_nwaves = (ntiles_total + max_blocks - 1) / max_blocks; + const int tiles_efficiency_percent = 100 * ntiles_total / (max_blocks*tiles_nwaves); - const int nblocks_stream_k = 2*nsm; + const int nblocks_stream_k = max_blocks; - const bool use_stream_k = tiles_efficiency_percent < 75 || cc >= GGML_CUDA_CC_ADA_LOVELACE; + const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || tiles_efficiency_percent < 75; blocks_num.x = use_stream_k ? nblocks_stream_k : ntiles_total; blocks_num.y = 1; blocks_num.z = 1; - dst_tmp_meta.alloc(blocks_num.x*cols_per_block * (2*2 + D) * sizeof(float)); + dst_tmp_meta.alloc(blocks_num.x*ncols * (2*2 + D) * sizeof(float)); } else { blocks_num.x = parallel_blocks*ntiles_x; blocks_num.y = Q->ne[2]; @@ -793,7 +778,6 @@ void launch_fattn( } } - float scale = 1.0f; float max_bias = 0.0f; float logit_softcap = 0.0f; @@ -832,9 +816,9 @@ void launch_fattn( if constexpr (parallel_blocks == 0) { if (ntiles_total % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles. const dim3 block_dim_combine(D, 1, 1); - const dim3 blocks_num_combine = blocks_num; + const dim3 blocks_num_combine = {blocks_num.x, ncols1, ncols2}; - flash_attn_stream_k_fixup + flash_attn_stream_k_fixup <<>> ((float *) KQV->data, dst_tmp_meta.ptr, Q->ne[1], Q->ne[2], K->ne[1]); } diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index d777f5413e..b2e0db9a2c 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -5,12 +5,15 @@ using namespace ggml_cuda_mma; -typedef tile<16, 8, half2> tile_A; -typedef tile< 8, 8, half2> tile_B; -typedef tile<16, 8, float> tile_C_KQ; -typedef tile<16, 4, half2> tile_C_VKQ; +typedef tile<16, 8, half2> tile_A; +typedef tile< 8, 8, half2> tile_B; +typedef tile<16, 8, half2> tile_B_16; +typedef tile<16, 8, float> tile_C_KQ; +typedef tile<16, 16, float> tile_C_KQ_16; +typedef tile<16, 4, half2> tile_C_VKQ; +typedef tile<16, 8, half2> tile_C_VKQ_16; -template +template static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int stride_KV) { constexpr int D2_padded = D/2 + 4; // Size of D in half2, padded to avoid shared memory bank conflicts. @@ -27,7 +30,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( constexpr int chunks_per_row = k0_sync_start / h2_per_chunk; constexpr int stride_i = WARP_SIZE / chunks_per_row; #pragma unroll - for (int i0 = 0; i0 < KQ_stride; i0 += nwarps*stride_i) { + for (int i0 = 0; i0 < KQ_per_iter; i0 += nwarps*stride_i) { const int i = i0 + threadIdx.y*stride_i + (chunks_per_row == WARP_SIZE ? 0 : threadIdx.x / chunks_per_row); const int k = (chunks_per_row == WARP_SIZE ? threadIdx.x : threadIdx.x % chunks_per_row)*h2_per_chunk; @@ -40,7 +43,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( // If D is not a power of 2, the rest is loaded synchronously. // K/V data is loaded with decreasing granularity for D for better memory bandwidth. - static_assert(KQ_stride % (4*nwarps) == 0, "out of bounds"); + static_assert(KQ_per_iter % (4*nwarps) == 0, "out of bounds"); #pragma unroll for (int stride_k : {WARP_SIZE, WARP_SIZE/2, WARP_SIZE/4}) { const int k0_start = stride_k == WARP_SIZE ? k0_sync_start : D/2 - (D/2) % (2*stride_k); @@ -52,7 +55,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( } #pragma unroll - for (int i0 = 0; i0 < KQ_stride; i0 += nwarps*stride_i) { + for (int i0 = 0; i0 < KQ_per_iter; i0 += nwarps*stride_i) { const int i = i0 + threadIdx.y*stride_i + (stride_k == WARP_SIZE ? 0 : threadIdx.x / stride_k); #pragma unroll @@ -65,12 +68,54 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( } } -template +template +static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( + const half2 * const __restrict__ mask_h2, half2 * const __restrict__ tile_mask, const int stride_mask) { + static_assert(KQ_per_iter == 2*WARP_SIZE || KQ_per_iter == WARP_SIZE, "bad KQ_per_iter"); +#ifdef CP_ASYNC_AVAILABLE + constexpr int preload = KQ_per_iter * sizeof(half); + constexpr int cols_per_warp = 8*WARP_SIZE/KQ_per_iter; + constexpr int stride_j = nwarps * cols_per_warp; + + const unsigned int tile_mask_32 = __cvta_generic_to_shared(tile_mask); + +#pragma unroll + for (int j0 = 0; j0 < ncols1; j0 += stride_j) { + const int j = j0 + threadIdx.y*cols_per_warp + + (KQ_per_iter == 2*WARP_SIZE ? threadIdx.x / (WARP_SIZE/4) : threadIdx.x / (WARP_SIZE/8)); + + if (j0 + stride_j > ncols1 && j >= ncols1) { + break; + } + + const int i = 4 * (KQ_per_iter == 2*WARP_SIZE ? threadIdx.x % (WARP_SIZE/4) : threadIdx.x % (WARP_SIZE/8)); + + cp_async_cg_16(tile_mask_32 + j*(KQ_per_iter*sizeof(half) + 16) + i*sizeof(half2), mask_h2 + j*stride_mask + i); + } +#else + constexpr int cols_per_warp = 2*WARP_SIZE/KQ_per_iter; + constexpr int stride_j = nwarps * cols_per_warp; +#pragma unroll + for (int j0 = 0; j0 < ncols1; j0 += stride_j) { + const int j = j0 + threadIdx.y*cols_per_warp + (KQ_per_iter == 2*WARP_SIZE ? 0 : threadIdx.x / (WARP_SIZE/2)); + + if (j0 + stride_j > ncols1 && j >= ncols1) { + break; + } + + const int i = KQ_per_iter == 2*WARP_SIZE ? threadIdx.x : threadIdx.x % (WARP_SIZE/2); + + tile_mask[j*(KQ_per_iter/2 + 4) + i] = mask_h2[j*stride_mask + i]; + } +#endif // CP_ASYNC_AVAILABLE +} + +template static __device__ __forceinline__ void flash_attn_ext_f16_iter( const float2 * const __restrict__ Q_f2, const half2 * const __restrict__ K_h2, const half2 * const __restrict__ V_h2, - const half * const __restrict__ maskh, + const half2 * const __restrict__ mask_h2, float2 * const __restrict__ dstk, float2 * const __restrict__ dstk_fixup, const float scale, @@ -78,42 +123,60 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const float logit_softcap, const int ne01, const int ne02, - const int stride_Q, const int stride_KV, const int stride_mask, const int jt, half2 * const __restrict__ tile_K, half2 * const __restrict__ tile_V, + half2 * const __restrict__ tile_mask, const tile_B * const __restrict__ Q_B, tile_C_VKQ * const __restrict__ VKQ_C, - float2 & KQ_max, - float2 & KQ_rowsum, + float * const __restrict__ KQ_max, + float * const __restrict__ KQ_rowsum, const int kb0) { #ifdef NEW_MMA_AVAILABLE - constexpr int np = nwarps*tile_B::I / ncols; // Number of parallel CUDA warps per Q column. - constexpr int D2_padded = D/2 + 4; // Size of D in half2, padded to avoid shared memory bank conflicts. + constexpr int cols_per_warp = ntiles * tile_B::I; + constexpr int cols_per_thread = ntiles == 1 ? 2 : ntiles; + constexpr int np = nwarps * (cols_per_warp/ncols2) / ncols1; // Number of parallel CUDA warps per Q column. + constexpr int D2_padded = D/2 + 4; // Size of D in half2, padded to avoid shared memory bank conflicts. - const int k_VKQ_0 = kb0*KQ_stride; - tile_C_KQ KQ_C[KQ_stride/(np*tile_C_KQ::I)]; + const int k_VKQ_0 = kb0 * KQ_per_iter; + tile_C_KQ KQ_C[KQ_per_iter/(np*tile_C_KQ::I) * ntiles]; + + // Use wide variants of tiles if ntiles >= 2. + tile_B_16 * Q_B_16 = (tile_B_16 *) Q_B; + tile_C_VKQ_16 * VKQ_C_16 = (tile_C_VKQ_16 *) VKQ_C; + tile_C_KQ_16 * KQ_C_16 = (tile_C_KQ_16 *) KQ_C; #ifdef CP_ASYNC_AVAILABLE cp_async_wait_all(); __syncthreads(); - flash_attn_ext_f16_load_tile(V_h2 + k_VKQ_0*stride_KV, tile_V, stride_KV); + flash_attn_ext_f16_load_tile(V_h2 + k_VKQ_0*stride_KV, tile_V, stride_KV); #else - flash_attn_ext_f16_load_tile(K_h2 + k_VKQ_0*stride_KV, tile_K, stride_KV); + if (ncols2 > 1 || mask_h2) { + flash_attn_ext_f16_load_mask(mask_h2 + k_VKQ_0/2, tile_mask, stride_mask); + } + flash_attn_ext_f16_load_tile(K_h2 + k_VKQ_0*stride_KV, tile_K, stride_KV); __syncthreads(); #endif // CP_ASYNC_AVAILABLE // Calculate tile of KQ: #pragma unroll - for (int i_KQ_00 = 0; i_KQ_00 < KQ_stride; i_KQ_00 += np*tile_A::I) { + for (int i_KQ_00 = 0; i_KQ_00 < KQ_per_iter; i_KQ_00 += np*tile_A::I) { const int i_KQ_0 = i_KQ_00 + (threadIdx.y % np)*tile_A::I; #pragma unroll for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += tile_A::J) { tile_A K_A; load_ldmatrix(K_A, tile_K + i_KQ_0*D2_padded + k_KQ_0, D2_padded); - mma(KQ_C[i_KQ_00/(np*tile_A::I)], K_A, ((tile_B *) Q_B)[k_KQ_0/tile_A::J]); + if (ntiles == 1) { + mma(KQ_C[i_KQ_00/(np*tile_A::I)], K_A, Q_B[k_KQ_0/tile_A::J]); + } else { +#pragma unroll + for (int t = 0; t < ntiles/2; ++t) { + // Wide version of KQ_C is column-major => swap A and B. + mma(KQ_C_16[i_KQ_00/(np*tile_A::I) * ntiles/2 + t], Q_B_16[k_KQ_0/tile_A::J * ntiles/2 + t], K_A); + } + } } } @@ -122,9 +185,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( #endif // CP_ASYNC_AVAILABLE if (use_logit_softcap) { - static_assert(KQ_stride % (np*tile_C_KQ::I) == 0, "bad loop size"); + static_assert(KQ_per_iter % (np*tile_C_KQ::I) == 0, "bad loop size"); #pragma unroll - for (int i = 0; i < KQ_stride/(np*tile_C_KQ::I); ++i) { + for (int i = 0; i < KQ_per_iter/(np*tile_C_KQ::I) * ntiles; ++i) { #pragma unroll for (int l = 0; l < tile_C_KQ::ne; ++l) { KQ_C[i].x[l] = logit_softcap*tanhf(KQ_C[i].x[l]); @@ -132,109 +195,209 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } } - if (maskh) { - static_assert(KQ_stride % (np *tile_C_KQ::I) == 0, "bad loop size"); - static_assert(ncols % (nwarps/np*tile_C_KQ::J) == 0, "bad loop size"); + float KQ_max_new[cols_per_thread]; #pragma unroll - for (int i00 = 0; i00 < KQ_stride; i00 += np*tile_C_KQ::I) { - const int i0 = i00 + (threadIdx.y % np)*tile_C_KQ::I; -#pragma unroll - for (int l = 0; l < tile_C_KQ::ne; ++l) { - const int i = i0 + tile_C_KQ::get_i(l); - const int j = (threadIdx.y / np)*tile_C_KQ::J + tile_C_KQ::get_j(l); + for (int col = 0; col < cols_per_thread; ++col) { + KQ_max_new[col] = KQ_max[col]; + } + float KQ_rowsum_add[cols_per_thread] = {0.0f}; - KQ_C[i00/(np*tile_C_KQ::I)].x[l] += slope*__half2float(maskh[j*stride_mask + k_VKQ_0 + i]); + if (ntiles == 1) { + if (ncols2 > 1 || mask_h2) { +#pragma unroll + for (int i00 = 0; i00 < KQ_per_iter; i00 += np*tile_C_KQ::I) { + const int i0 = i00 + (threadIdx.y % np)*tile_C_KQ::I; +#pragma unroll + for (int l = 0; l < tile_C_KQ::ne; ++l) { + const int i = i0 + tile_C_KQ::get_i(l); + const int j = ((threadIdx.y / np)*tile_C_KQ::J + tile_C_KQ::get_j(l)) / ncols2; + + KQ_C[i00/(np*tile_C_KQ::I)].x[l] += slope * + __half2float(((const half *) tile_mask)[j*(KQ_per_iter + 8) + i]); + } } } - } - // Calculate softmax for each KQ column using the current max. value. - // The divisor is stored in KQ_rowsum and will be applied at the end. - float2 KQ_max_new = KQ_max; - static_assert(KQ_stride % (np*tile_C_KQ::I) == 0, "bad loop size"); + // Calculate softmax for each KQ column using the current max. value. + // The divisor is stored in KQ_rowsum and will be applied at the end. + static_assert(KQ_per_iter % (np*tile_C_KQ::I) == 0, "bad loop size"); #pragma unroll - for (int k = 0; k < KQ_stride/(np*tile_C_KQ::I); ++k) { + for (int k = 0; k < KQ_per_iter/(np*tile_C_KQ::I); ++k) { #pragma unroll - for (int l0 = 0; l0 < tile_C_KQ::ne; l0 += 2) { - KQ_max_new.x = fmaxf(KQ_max_new.x, KQ_C[k].x[l0 + 0]); - KQ_max_new.y = fmaxf(KQ_max_new.y, KQ_C[k].x[l0 + 1]); + for (int l = 0; l < tile_C_KQ::ne; ++l) { + KQ_max_new[l % 2] = fmaxf(KQ_max_new[l % 2], KQ_C[k].x[l]); + } } - } - // Values per KQ column are spread across 8 threads, does not need full warp reduce: + // Values per KQ column are spread across 8 threads, does not need full warp reduce: #pragma unroll - for (int offset = 16; offset > 2; offset >>= 1) { - KQ_max_new.x = fmaxf(KQ_max_new.x, __shfl_xor_sync(0xFFFFFFFF, KQ_max_new.x, offset, WARP_SIZE)); - KQ_max_new.y = fmaxf(KQ_max_new.y, __shfl_xor_sync(0xFFFFFFFF, KQ_max_new.y, offset, WARP_SIZE)); - } + for (int col = 0; col < cols_per_thread; ++col) { +#pragma unroll + for (int offset = 16; offset >= 4; offset >>= 1) { + KQ_max_new[col] = fmaxf(KQ_max_new[col], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[col], offset, WARP_SIZE)); + } + } - float2 KQ_rowsum_add = make_float2(0.0f, 0.0f); - static_assert(KQ_stride % (np*tile_C_KQ::I) == 0, "bad loop size"); -#pragma unroll - for (int k = 0; k < KQ_stride/(np*tile_C_KQ::I); ++k) { -#pragma unroll - for (int l = 0; l < tile_C_KQ::ne; ++l) { - const float KQ_max_l = l % 2 == 0 ? KQ_max_new.x : KQ_max_new.y; - const float diff = KQ_C[k].x[l] - KQ_max_l; - KQ_C[k].x[l] = expf(diff); + static_assert(KQ_per_iter % (np*tile_C_KQ::I) == 0, "bad loop size"); - if (l % 2 == 0) { - KQ_rowsum_add.x += KQ_C[k].x[l]; - } else { - KQ_rowsum_add.y += KQ_C[k].x[l]; +#pragma unroll + for (int k = 0; k < KQ_per_iter/(np*tile_C_KQ::I); ++k) { +#pragma unroll + for (int l = 0; l < tile_C_KQ::ne; ++l) { + KQ_C[k].x[l] = expf(KQ_C[k].x[l] - KQ_max_new[l % 2]); + + KQ_rowsum_add[l % 2] += KQ_C[k].x[l]; + } + } + } else { // ntiles > 1 + if (ncols2 > 1 || mask_h2) { +#pragma unroll + for (int i00 = 0; i00 < KQ_per_iter; i00 += np*tile_C_KQ_16::J) { + const int i0 = i00 + (threadIdx.y % np)*tile_C_KQ_16::J; +#pragma unroll + for (int t = 0; t < ntiles/2; ++t) { +#pragma unroll + for (int l0 = 0; l0 < tile_C_KQ_16::ne; l0 += 2) { + const int i = (i0 + tile_C_KQ_16::get_j(l0)) / 2; + const int j = ((threadIdx.y / np)*cols_per_warp + t*tile_C_KQ_16::I + tile_C_KQ_16::get_i(l0)) / ncols2; + + const float2 tmp = __half22float2(tile_mask[j*(KQ_per_iter/2 + 4) + i]); + const int KQ_index = i00/(np*tile_C_KQ_16::J) * ntiles/2 + t; + KQ_C_16[KQ_index].x[l0 + 0] += slope*tmp.x; + KQ_C_16[KQ_index].x[l0 + 1] += slope*tmp.y; + } + } + } + } + + // Calculate softmax for each KQ column using the current max. value. + // The divisor is stored in KQ_rowsum and will be applied at the end. + static_assert(KQ_per_iter % (np*tile_C_KQ::I) == 0, "bad loop size"); +#pragma unroll + for (int k = 0; k < KQ_per_iter/(np*tile_C_KQ_16::J); ++k) { +#pragma unroll + for (int t = 0; t < ntiles/2; ++t) { +#pragma unroll + for (int l = 0; l < tile_C_KQ_16::ne; ++l) { + const int KQ_index = 2*t + (l/2) % 2; + KQ_max_new[KQ_index] = fmaxf(KQ_max_new[KQ_index], KQ_C_16[k*ntiles/2 + t].x[l]); + } + } + } + + // Values per KQ column are spread across 4 threads, does not need full warp reduce: +#pragma unroll + for (int col = 0; col < cols_per_thread; ++col) { +#pragma unroll + for (int offset = 2; offset >= 1; offset >>= 1) { + KQ_max_new[col] = fmaxf(KQ_max_new[col], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[col], offset, WARP_SIZE)); + } + } + + static_assert(KQ_per_iter % (np*tile_C_KQ_16::J) == 0, "bad loop size"); +#pragma unroll + for (int k = 0; k < KQ_per_iter/(np*tile_C_KQ_16::J); ++k) { +#pragma unroll + for (int t = 0; t < ntiles/2; ++t) { +#pragma unroll + for (int l = 0; l < tile_C_KQ_16::ne; ++l) { + const int KQ_index = 2*t + (l/2) % 2; + + KQ_C_16[k*ntiles/2 + t].x[l] = expf(KQ_C_16[k*ntiles/2 + t].x[l] - KQ_max_new[KQ_index]); + + KQ_rowsum_add[KQ_index] += KQ_C_16[k*ntiles/2 + t].x[l]; + } } } } { - const float2 diff = make_float2(KQ_max.x - KQ_max_new.x, KQ_max.y - KQ_max_new.y); - const float2 KQ_max_scale = make_float2(expf(diff.x), expf(diff.y)); - KQ_max = KQ_max_new; - - // Scale previous KQ_rowsum to account for a potential increase in KQ_max: - KQ_rowsum.x = KQ_max_scale.x*KQ_rowsum.x + KQ_rowsum_add.x; - KQ_rowsum.y = KQ_max_scale.y*KQ_rowsum.y + KQ_rowsum_add.y; - - const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale.x, KQ_max_scale.y); + float KQ_max_scale[cols_per_thread]; #pragma unroll - for (int i = 0; i < D/tile_C_VKQ::I; ++i) { + for (int col = 0; col < cols_per_thread; ++col) { + KQ_max_scale[col] = expf(KQ_max[col] - KQ_max_new[col]); + KQ_max[col] = KQ_max_new[col]; + + // Scale previous KQ_rowsum to account for a potential increase in KQ_max: + KQ_rowsum[col] = KQ_max_scale[col]*KQ_rowsum[col] + KQ_rowsum_add[col]; + } + + if (ntiles == 1) { + const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[1]); #pragma unroll - for (int l = 0; l < tile_C_VKQ::ne; ++l) { - VKQ_C[i].x[l] *= KQ_max_scale_h2; + for (int i = 0; i < D/tile_C_VKQ::I; ++i) { +#pragma unroll + for (int l = 0; l < tile_C_VKQ::ne; ++l) { + VKQ_C[i].x[l] *= KQ_max_scale_h2; + } + } + } else { +#pragma unroll + for (int col = 0; col < cols_per_thread; ++col) { + const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]); +#pragma unroll + for (int i = 0; i < D/tile_C_VKQ_16::J; ++i) { +#pragma unroll + for (int l0 = 0; l0 < tile_C_VKQ_16::ne; l0 += 2) { + VKQ_C_16[i*ntiles/2 + col/2].x[l0 + col % 2] *= KQ_max_scale_h2; + } + } } } } // Convert KQ C tiles into B tiles for VKQ calculation: - tile_B B[KQ_stride/(np*2*tile_B::J)]; - static_assert(KQ_stride % (np*2*tile_B::J) == 0, "bad loop size"); + tile_B B[KQ_per_iter/(np*2*tile_B::J) * ntiles]; + tile_B_16 * B_16 = (tile_B_16 *) B; + static_assert(KQ_per_iter % (np*2*tile_B::J) == 0, "bad loop size"); + if (ntiles == 1) { #pragma unroll - for (int k = 0; k < KQ_stride/(np*2*tile_B::J); ++k) { - B[k] = get_transposed(get_half2(KQ_C[k])); + for (int k = 0; k < KQ_per_iter/(np*2*tile_B::J); ++k) { + B[k] = get_transposed(get_half2(KQ_C[k])); + } + } else { + for (int k = 0; k < KQ_per_iter/(np*2*tile_B_16::J); ++k) { +#pragma unroll + for (int t = 0; t < ntiles/2; ++t) { + B_16[k*ntiles/2 + t] = get_half2(KQ_C_16[k*ntiles/2 + t]); + } + } } #ifdef CP_ASYNC_AVAILABLE + // Preload K tile for next iteration: cp_async_wait_all(); __syncthreads(); if (!last_iter) { - flash_attn_ext_f16_load_tile(K_h2 + (k_VKQ_0 + KQ_stride)*stride_KV, tile_K, stride_KV); + if (ncols2 > 1 || mask_h2) { + flash_attn_ext_f16_load_mask(mask_h2 + (k_VKQ_0 + KQ_per_iter)/2, tile_mask, stride_mask); + } + flash_attn_ext_f16_load_tile(K_h2 + (k_VKQ_0 + KQ_per_iter)*stride_KV, tile_K, stride_KV); } #else - flash_attn_ext_f16_load_tile(V_h2 + k_VKQ_0*stride_KV, tile_V, stride_KV); + flash_attn_ext_f16_load_tile(V_h2 + k_VKQ_0*stride_KV, tile_V, stride_KV); __syncthreads(); #endif // CP_ASYNC_AVAILABLE // Calculate VKQ tile: #pragma unroll for (int i_VKQ_0 = 0; i_VKQ_0 < D; i_VKQ_0 += tile_C_VKQ::I) { - static_assert((KQ_stride/2) % (np*tile_A::J) == 0, "bad loop size"); + static_assert((KQ_per_iter/2) % (np*tile_A::J) == 0, "bad loop size"); #pragma unroll - for (int k00 = 0; k00 < KQ_stride/2; k00 += np*tile_A::J) { + for (int k00 = 0; k00 < KQ_per_iter/2; k00 += np*tile_A::J) { const int k0 = k00 + (threadIdx.y % np)*tile_A::J; tile_A A; load_ldmatrix_trans(A, tile_V + 2*k0*D2_padded + i_VKQ_0/2, D2_padded); - mma(VKQ_C[i_VKQ_0/tile_C_VKQ::I], A, B[k00/(np*tile_A::J)]); + if (ntiles == 1) { + mma(VKQ_C[i_VKQ_0/tile_C_VKQ::I], A, B[k00/(np*tile_A::J)]); + } else { +#pragma unroll + for (int t = 0; t < ntiles/2; ++t) { + // Wide version of VKQ_C is column-major => swap A and B. + mma(VKQ_C_16[i_VKQ_0/tile_C_VKQ::I * ntiles/2 + t], B_16[k00/(np*tile_A::J) * ntiles/2 + t], A); + } + } } } @@ -247,12 +410,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( #endif // NEW_MMA_AVAILABLE } -template +template static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const float2 * const __restrict__ Q_f2, const half2 * const __restrict__ K_h2, const half2 * const __restrict__ V_h2, - const half * const __restrict__ maskh, + const half2 * const __restrict__ mask_h2, float2 * const __restrict__ dstk, float2 * const __restrict__ dstk_fixup, const float scale, @@ -260,7 +423,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const float logit_softcap, const int ne01, const int ne02, - const int stride_Q, + const int stride_Q1, + const int stride_Q2, const int stride_KV, const int stride_mask, const int jt, @@ -269,63 +433,78 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( #ifdef NEW_MMA_AVAILABLE //In this kernel Q, K, V are matrices while i, j, k are matrix indices. - static_assert(nwarps*tile_B::I % ncols == 0, "bad nwarps"); - constexpr int np = nwarps*tile_B::I / ncols; // Number of parallel CUDA warps per Q column. + constexpr int ncols = ncols1 * ncols2; + constexpr int cols_per_warp = ntiles * tile_B::I; + constexpr int cols_per_thread = ntiles == 1 ? 2 : ntiles; + constexpr int np = nwarps * (cols_per_warp/ncols2) / ncols1; // Number of parallel CUDA warps per Q column. - static_assert(D % nwarps == 0, "bad D"); - static_assert(KQ_stride % nwarps == 0, "bad KQ_stride"); + static_assert(nwarps * (cols_per_warp/ncols2) % ncols1 == 0, "bad nwarps"); + + static_assert(D % nwarps == 0, "bad D"); + static_assert(KQ_per_iter % nwarps == 0, "bad KQ_per_iter"); constexpr int D2_padded = D/2 + 4; // Size of D in half2, padded to avoid shared memory bank conflicts. - // Temporary shared buffer for loading K/V data with KQ_stride*D logical elements: + // Temporary shared buffer for loading K/V data with KQ_per_iter*D logical elements: extern __shared__ half2 tile_K[]; #ifdef CP_ASYNC_AVAILABLE - half2 * tile_V = tile_K + KQ_stride*D2_padded; + half2 * tile_V = tile_K + KQ_per_iter*D2_padded; #else - half2 * tile_V = tile_K; + half2 * tile_V = tile_K; #endif // CP_ASYNC_AVAILABLE + half2 * tile_mask = tile_V + KQ_per_iter*D2_padded; - tile_B Q_B[D/(2*tile_B::J)]; - tile_C_VKQ VKQ_C[D/tile_C_VKQ::I]; + tile_B Q_B[D/(2*tile_B::J) * ntiles]; + tile_C_VKQ VKQ_C[D/tile_C_VKQ::I * ntiles]; - float2 KQ_rowsum = {0.0f, 0.0f}; - float2 KQ_max = {-FLT_MAX/2.0f, -FLT_MAX/2.0f}; + tile_B_16 * Q_B_16 = (tile_B_16 *) Q_B; + tile_C_VKQ_16 * VKQ_C_16 = (tile_C_VKQ_16 *) VKQ_C; + + float KQ_rowsum[cols_per_thread] = {0.0f}; + float KQ_max[cols_per_thread]; +#pragma unroll + for (int col = 0; col < cols_per_thread; ++col) { + KQ_max[col] = -FLT_MAX/2.0f; + } // Temporarily load Q data into tile_K, will be loaded into registers afterwards. // The loading is done with decreasing granularity for D for better memory bandwidth. const half2 scale_h2 = make_half2(scale, scale); #pragma unroll for (int stride_k : {WARP_SIZE, WARP_SIZE/2, WARP_SIZE/4}) { - const int k0_start = stride_k == WARP_SIZE ? 0 : D/2 - (D/2) % (2*stride_k); - const int k0_stop = D/2 - (D/2) % (1*stride_k); - const int stride_j = WARP_SIZE / stride_k; + const int k0_start = stride_k == WARP_SIZE ? 0 : D/2 - (D/2) % (2*stride_k); + const int k0_stop = D/2 - (D/2) % (1*stride_k); + const int stride_jc = WARP_SIZE / stride_k; if (k0_start == k0_stop) { continue; } - if (nwarps*stride_j > ncols && threadIdx.y*stride_j >= ncols) { - break; - } - #pragma unroll - for (int j0 = 0; j0 < ncols; j0 += nwarps*stride_j) { - const int j = j0 + threadIdx.y*stride_j + (stride_k == WARP_SIZE ? 0 : threadIdx.x / stride_k); + for (int jc0 = 0; jc0 < ncols; jc0 += nwarps*stride_jc) { + const int jc = jc0 + threadIdx.y*stride_jc + (stride_k == WARP_SIZE ? 0 : threadIdx.x / stride_k); - if (jt*ncols + j < ne01) { + if (jc0 + nwarps*stride_jc > ncols && jc >= ncols) { + break; + } + + const int j = jc / ncols2; + const int c = jc % ncols2; + + if (jt*ncols1 + j < ne01) { #pragma unroll for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == WARP_SIZE ? threadIdx.x : threadIdx.x % stride_k); - const float2 tmp = Q_f2[(jt*ncols + j)*stride_Q + k]; - tile_K[j*D2_padded + k] = scale_h2 * make_half2(tmp.x, tmp.y); + const float2 tmp = Q_f2[(jt*ncols1 + j)*stride_Q1 + c*stride_Q2 + k]; + tile_K[jc*D2_padded + k] = scale_h2 * make_half2(tmp.x, tmp.y); } } else { #pragma unroll for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == WARP_SIZE ? threadIdx.x : threadIdx.x % stride_k); - tile_K[j*D2_padded + k] = make_half2(0.0f, 0.0f); + tile_K[jc*D2_padded + k] = make_half2(0.0f, 0.0f); } } } @@ -334,128 +513,217 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( __syncthreads(); { - const int j0 = (threadIdx.y / np) * tile_B::I; + const int j0 = (threadIdx.y / np) * cols_per_warp; #pragma unroll for (int k0 = 0; k0 < D/2; k0 += tile_B::J) { - load_ldmatrix(Q_B[k0/tile_B::J], tile_K + j0*D2_padded + k0, D2_padded); + if (ntiles == 1) { + load_ldmatrix(Q_B[k0/tile_B::J], tile_K + j0*D2_padded + k0, D2_padded); + } else { +#pragma unroll + for (int t = 0; t < ntiles/2; ++t) { + load_ldmatrix(Q_B_16[k0/tile_B_16::J * ntiles/2 + t], + tile_K + (j0 + t*tile_B_16::I)*D2_padded + k0, D2_padded); + } + } } } __syncthreads(); - // Preload K data for first iteration when using cp_async: + // Preload mask and K data for first iteration when using cp_async: #ifdef CP_ASYNC_AVAILABLE - flash_attn_ext_f16_load_tile(K_h2 + kb0_start*KQ_stride*stride_KV, tile_K, stride_KV); + if (ncols2 > 1 || mask_h2) { + flash_attn_ext_f16_load_mask(mask_h2 + kb0_start*KQ_per_iter/2, tile_mask, stride_mask); + } + flash_attn_ext_f16_load_tile(K_h2 + kb0_start*KQ_per_iter*stride_KV, tile_K, stride_KV); #endif // CP_ASYNC_AVAILABLE // Iterate over ne11 == previous tokens: for (int kb0 = kb0_start; kb0 < kb0_stop-1; ++kb0) { constexpr bool last_iter = false; - flash_attn_ext_f16_iter - (Q_f2, K_h2, V_h2, maskh, dstk, dstk_fixup, scale, slope, logit_softcap, - ne01, ne02, stride_Q, stride_KV, stride_mask, jt, tile_K, tile_V, Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0); + flash_attn_ext_f16_iter + (Q_f2, K_h2, V_h2, mask_h2, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_KV, stride_mask, jt, tile_K, tile_V, tile_mask, Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0); } { // kb0_start is always < kb0_stop so the last iter can be executed unconditionally. constexpr bool last_iter = true; - flash_attn_ext_f16_iter - (Q_f2, K_h2, V_h2, maskh, dstk, dstk_fixup, scale, slope, logit_softcap, - ne01, ne02, stride_Q, stride_KV, stride_mask, jt, tile_K, tile_V, Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0_stop-1); + flash_attn_ext_f16_iter + (Q_f2, K_h2, V_h2, mask_h2, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_KV, stride_mask, jt, tile_K, tile_V, tile_mask, Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0_stop-1); } // With cp_async there is no __syncthreads at the end of the iter, // there can be a race condition on shared memory access for combining/writing back results. #ifdef CP_ASYNC_AVAILABLE - if (nwarps*tile_B::I > KQ_stride) { + if (nwarps*cols_per_warp > KQ_per_iter) { __syncthreads(); } #endif // CP_ASYNC_AVAILABLE // Finally, sum up partial KQ rowsums. - // The partial sums are spread across 8 threads each, does not need full reduce. + // The partial sums are spread across 8/4 threads each, does not need full reduce. + { + constexpr int offset_first = ntiles == 1 ? 16 : 2; + constexpr int offset_last = ntiles == 1 ? 4 : 1; #pragma unroll - for (int offset = 16; offset > 2; offset >>= 1) { - KQ_rowsum.x += __shfl_xor_sync(0xFFFFFFFF, KQ_rowsum.x, offset, WARP_SIZE); - KQ_rowsum.y += __shfl_xor_sync(0xFFFFFFFF, KQ_rowsum.y, offset, WARP_SIZE); + for (int col = 0; col < cols_per_thread; ++col) { +#pragma unroll + for (int offset = offset_first; offset >= offset_last; offset >>= 1) { + KQ_rowsum[col] += __shfl_xor_sync(0xFFFFFFFF, KQ_rowsum[col], offset, WARP_SIZE); + } + } } // Write VKQ accumulators to shared memory in column-major format. // It's faster to do small writes to shared memory, then large write to VRAM than to do small writes to VRAM. // Also for np > 1 the combination is done via these values in shared memory. - const int j_cwd = threadIdx.y*tile_B::I + tile_B::get_i(-1); // j combine write data + if (ntiles == 1) { + const int jc_cwd = threadIdx.y*tile_B::I + tile_B::get_i(-1); // jc combine write data #pragma unroll - for (int k0 = 0; k0 < D/2; k0 += tile_B::J) { - const tile_B B = get_transposed(VKQ_C[k0/tile_B::J]); // Conversion of C to B matrix puts it in column-major format. + for (int k0 = 0; k0 < D/2; k0 += tile_B::J) { + const tile_B B = get_transposed(VKQ_C[k0/tile_B::J]); // Conversion of C to B matrix puts it in column-major format. #pragma unroll - for (int l = 0; l < tile_B::ne; ++l) { - const int k = k0 + tile_B::get_j(l); + for (int l = 0; l < tile_B::ne; ++l) { + const int k = k0 + tile_B::get_j(l); - tile_K[j_cwd*D2_padded + k] = B.x[l]; + tile_K[jc_cwd*D2_padded + k] = B.x[l]; + } + } + } else { +#pragma unroll + for (int t = 0; t < ntiles/2; ++t) { + const int j0 = threadIdx.y*cols_per_warp + t*tile_C_VKQ_16::I; +#pragma unroll + for (int k0 = 0; k0 < D/2; k0 += tile_C_VKQ_16::J) { +#pragma unroll + for (int l = 0; l < tile_C_VKQ_16::ne; ++l) { + const int j = j0 + tile_C_VKQ_16::get_i(l); + const int k = k0 + tile_C_VKQ_16::get_j(l); + + tile_K[j*D2_padded + k] = VKQ_C_16[k0/tile_C_VKQ_16::J * ntiles/2 + t].x[l]; + } + } } } - const int j_cwmo = (threadIdx.x % (2*tile_C_VKQ::J)) / tile_C_VKQ::J; // j combine write meta offset - const int j_cwm = threadIdx.y*(2*tile_C_VKQ::J) + 2*tile_C_VKQ::get_j(-1) + j_cwmo; // j combine write meta - const float2 KQ_cmr = make_float2(((const float *) &KQ_max)[j_cwmo], ((const float *) &KQ_rowsum)[j_cwmo]); // KQ combine max rowsum + if constexpr (ntiles == 1) { + const int jc_cwmo = (threadIdx.x % (2*tile_C_VKQ::J)) / tile_C_VKQ::J; // jc combine write meta offset + const int jc_cwm = threadIdx.y*(2*tile_C_VKQ::J) + 2*tile_C_VKQ::get_j(-1) + jc_cwmo; // jc combine write meta + const float2 KQ_cmr = make_float2(KQ_max[jc_cwmo], KQ_rowsum[jc_cwmo]); // KQ combine max rowsum - if (((!needs_fixup && !is_fixup) || np > 1) && threadIdx.x < 2*tile_C_VKQ::J) { - // Use the 16 bytes of padding in each row to store the meta data: KQ max, KQ rowsum, KQ max scale. - ((float2 *) tile_K)[j_cwm*(D2_padded/2) + D/4] = KQ_cmr; + if (((!needs_fixup && !is_fixup) || np > 1) && threadIdx.x < 2*tile_C_VKQ::J) { + // Use the 16 bytes of padding in each row to store the meta data: KQ max, KQ rowsum, KQ max scale. + ((float2 *) tile_K)[jc_cwm*(D2_padded/2) + D/4] = KQ_cmr; + } + + __syncthreads(); + + if (np == 1) { + // No combination is needed, the meta data can be directly written from registers to VRAM. + if (needs_fixup && threadIdx.x < tile_B::I) { + float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols; + dstk_fixup_meta[jc_cwm] = KQ_cmr; + } + if (is_fixup && threadIdx.x < tile_B::I) { + float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols; + dstk_fixup_meta[jc_cwm] = KQ_cmr; + } + } + } else { + static_assert(ntiles == 2 || ntiles == 4, "bad ntiles"); + const int jc_cwm = threadIdx.y*cols_per_warp // jc combine write meta + + (ntiles == 4 ? ((threadIdx.x % 4) / 2) * tile_C_VKQ_16::I : 0) + + tile_C_VKQ_16::get_i(threadIdx.x % 4); + const float2 KQ_cmr = make_float2(KQ_max[threadIdx.x % cols_per_thread], KQ_rowsum[threadIdx.x % cols_per_thread]); // KQ combine max rowsum + + if (((!needs_fixup && !is_fixup) || np > 1) && (ntiles == 4 || threadIdx.x % 4 < cols_per_thread)) { + // Use the 16 bytes of padding in each row to store the meta data: KQ max, KQ rowsum, KQ max scale. + ((float2 *) tile_K)[jc_cwm*(D2_padded/2) + D/4] = KQ_cmr; + } + + __syncthreads(); + + if (np == 1) { + // No combination is needed, the meta data can be directly written from registers to VRAM. + if (needs_fixup && (ntiles == 4 || threadIdx.x % 4 < ntiles)) { + float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols; + dstk_fixup_meta[jc_cwm] = KQ_cmr; + } + if (is_fixup && (ntiles == 4 || threadIdx.x % 4 < ntiles)) { + float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols; + dstk_fixup_meta[jc_cwm] = KQ_cmr; + } + } } - __syncthreads(); - - static_assert(np == 1 || np == 2 || np == 4, "bad np"); - if (np == 1) { - // No combination is needed, the meta data can be directly written from registers to VRAM. - if (needs_fixup && threadIdx.x < tile_B::I) { - float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols; - dstk_fixup_meta[j_cwm] = KQ_cmr; - } - if (is_fixup && threadIdx.x < tile_B::I) { - float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols; - dstk_fixup_meta[j_cwm] = KQ_cmr; - } - } else if (threadIdx.y % np == 0) { + static_assert(np == 1 || ntiles == 1 || ntiles == 2, "bad ntiles"); + if (np > 1 && threadIdx.y % np == 0) { // Combine the meta data for parallel warps via shared memory. // Warps with threadIdx.y % np != 0 must NOT return early. // All threads must return simultaneously to avoid race conditions with work on the next tile. - float * meta_j = (float *) tile_K + (threadIdx.y*tile_B::I + threadIdx.x)*D2_padded + D/2; + constexpr int nmeta = np*cols_per_warp >= WARP_SIZE ? np*cols_per_warp/WARP_SIZE : 1; - float KQ_cm = -FLT_MAX/2; // KQ combine max per parallel warp. - if (np*tile_B::I == WARP_SIZE || threadIdx.x < np*tile_B::I) { - KQ_cm = meta_j[0]; + const int jc_meta = threadIdx.y*cols_per_warp + (np*cols_per_warp < WARP_SIZE ? threadIdx.x % (np*cols_per_warp) : threadIdx.x); + float2 * const meta_ptr = ((float2 *) tile_K) + jc_meta*(D2_padded/2) + D/4; + float2 meta[nmeta]; +#pragma unroll + for (int imeta = 0; imeta < nmeta; ++imeta) { + meta[imeta] = meta_ptr[imeta * WARP_SIZE * D2_padded/2]; } - float KQ_cmn = KQ_cm; // KQ combine max new, max between all parallel warps. + float KQ_cmn = meta[0].x; // KQ combine max new, max between all parallel warps. #pragma unroll - for (int offset = np*tile_B::I/2; offset >= tile_B::I; offset >>= 1) { + for (int imeta = 1; imeta < nmeta; ++imeta) { + KQ_cmn = fmaxf(KQ_cmn, meta[imeta].x); + } +#pragma unroll + for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { + if (offset >= WARP_SIZE) { + continue; + } KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, WARP_SIZE)); } - const float KQ_cms = expf(KQ_cm - KQ_cmn); // KQ combine max scale per warp. - float KQ_crs = 0.0f; // KQ combine rowsum, scaled sum of all parallel warps. - if (np*tile_B::I == WARP_SIZE || threadIdx.x < np*tile_B::I) { - KQ_crs = KQ_cms*meta_j[1]; + float KQ_cms[nmeta]; // KQ combine max scale per warp. +#pragma unroll + for (int imeta = 0; imeta < nmeta; ++imeta) { + KQ_cms[imeta] = expf(meta[imeta].x - KQ_cmn); + } + + float KQ_crs = KQ_cms[0]*meta[0].y; // KQ combine rowsum, scaled sum of all parallel warps. +#pragma unroll + for (int imeta = 1; imeta < nmeta; ++imeta) { + KQ_crs += KQ_cms[imeta]*meta[imeta].y; } #pragma unroll - for (int offset = np*tile_B::I/2; offset >= tile_B::I; offset >>= 1) { + for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { + if (offset >= WARP_SIZE) { + continue; + } KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, WARP_SIZE); } // Write back combined meta data: - if (np*tile_B::I == WARP_SIZE || threadIdx.x < np*tile_B::I) { - *((float2 *) meta_j) = make_float2(KQ_cms, KQ_crs); // Combined KQ max scale + rowsum. +#pragma unroll + for (int imeta = 0; imeta < nmeta; ++imeta) { + if (np*cols_per_warp >= WARP_SIZE || threadIdx.x < np*cols_per_warp) { + // Combined KQ max scale + rowsum. + meta_ptr[imeta * WARP_SIZE * D2_padded/2] = make_float2(KQ_cms[imeta], KQ_crs); + } } - if (needs_fixup && threadIdx.x < tile_B::I) { + + // Combined KQ max + rowsum. + static_assert(cols_per_warp <= WARP_SIZE); + if (needs_fixup && (cols_per_warp == WARP_SIZE || threadIdx.x < cols_per_warp)) { float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols; - dstk_fixup_meta[(threadIdx.y/np)*tile_B::I + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); + dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); } - if (is_fixup && threadIdx.x < tile_B::I) { + if (is_fixup && (cols_per_warp == WARP_SIZE || threadIdx.x < cols_per_warp)) { float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols; - dstk_fixup_meta[(threadIdx.y/np)*tile_B::I + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); + dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); } } @@ -470,27 +738,32 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( #pragma unroll for (int stride_k : {WARP_SIZE, WARP_SIZE/2, WARP_SIZE/4}) { - const int k0_start = stride_k == WARP_SIZE ? 0 : D/2 - (D/2) % (2*stride_k); - const int k0_stop = D/2 - (D/2) % (1*stride_k); - const int stride_j = WARP_SIZE / stride_k; + const int k0_start = stride_k == WARP_SIZE ? 0 : D/2 - (D/2) % (2*stride_k); + const int k0_stop = D/2 - (D/2) % (1*stride_k); + const int stride_jc = WARP_SIZE / stride_k; if (k0_start == k0_stop) { continue; } - if (nwarps*stride_j > ncols && threadIdx.y*stride_j >= ncols) { - break; - } - #pragma unroll - for (int j0_dst = 0; j0_dst < ncols; j0_dst += (nwarps/np)*stride_j) { - const int j_dst = j0_dst + (threadIdx.y/np)*stride_j + (stride_k == WARP_SIZE ? 0 : threadIdx.x / stride_k); - const int j_tile_K = (j_dst/tile_B::I)*(np*tile_B::I) + j_dst % tile_B::I; + for (int jc0_dst = 0; jc0_dst < ncols; jc0_dst += (nwarps/np)*stride_jc) { + const int jc_dst = jc0_dst + (threadIdx.y/np)*stride_jc + (stride_k == WARP_SIZE ? 0 : threadIdx.x / stride_k); - if (!is_fixup && jt*ncols + j_dst >= ne01) { + if (jc0_dst + (nwarps/np)*stride_jc > ncols && jc_dst >= ncols) { + break; + } + + const int jc_tile_K = (jc_dst/cols_per_warp)*(np*cols_per_warp) + jc_dst % cols_per_warp; + + const int j_dst = jc_dst / ncols2; + const int c_dst = jc_dst % ncols2; + + if (!is_fixup && jt*ncols1 + j_dst >= ne01) { continue; } - const float * meta_j = (const float *) tile_K + j_tile_K*D2_padded + D/2; + + const float * meta_j = (const float *) tile_K + jc_tile_K*D2_padded + D/2; #pragma unroll for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == WARP_SIZE ? threadIdx.x : threadIdx.x % stride_k); @@ -498,8 +771,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( float2 dstk_val = make_float2(0.0f, 0.0f); #pragma unroll for (int ip = 0; ip < np; ++ip) { - const float KQ_crs = np == 1 ? 1.0f : meta_j[ip*tile_B::I*D2_padded + 0]; - const float2 dstk_val_add = __half22float2(tile_K[(j_tile_K + ip*tile_B::I)*D2_padded + k]); + const float KQ_crs = np == 1 ? 1.0f : meta_j[ip*cols_per_warp * D2_padded + 0]; + const float2 dstk_val_add = __half22float2(tile_K[(jc_tile_K + ip*cols_per_warp) * D2_padded + k]); dstk_val.x += dstk_val_add.x*KQ_crs; dstk_val.y += dstk_val_add.y*KQ_crs; } @@ -511,9 +784,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } if (is_fixup) { - dstk_fixup_data[j_dst*(D/2) + k] = dstk_val; + dstk_fixup_data[jc_dst*(D/2) + k] = dstk_val; } else { - dstk[(jt*ncols + j_dst)*ne02*(D/2) + k] = dstk_val; + dstk[((jt*ncols1 + j_dst)*ne02 + c_dst)*(D/2) + k] = dstk_val; } } } @@ -528,10 +801,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( #endif // NEW_MMA_AVAILABLE } -template -#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) +template __launch_bounds__(nwarps*WARP_SIZE, 2) -#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) static __global__ void flash_attn_ext_f16( const char * __restrict__ Q, const char * __restrict__ K, @@ -579,20 +850,23 @@ static __global__ void flash_attn_ext_f16( return; } - static_assert(FATTN_KQ_STRIDE % KQ_stride == 0, "bad KQ_stride"); + static_assert(FATTN_KQ_STRIDE % KQ_per_iter == 0, "bad KQ_per_iter"); const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const int stride_Q = nb01 / sizeof(float2); + const int stride_Q1 = nb01 / sizeof(float2); + const int stride_Q2 = nb02 / sizeof(float2); const int stride_KV = nb11 / sizeof(half2); - const int stride_mask = nb31 / sizeof(half); + const int stride_mask = nb31 / sizeof(half2); - const int iter_k = ne11 / KQ_stride; - const int iter_j = (ne01 + (ncols - 1)) / ncols; + const int iter_k = ne11 / FATTN_KQ_STRIDE; + const int iter_j = (ne01 + (ncols1 - 1)) / ncols1; + + constexpr int kb_niter = FATTN_KQ_STRIDE / KQ_per_iter; // Number of kernel iterations per assigned KQ slice. // kbc == k block continuous, current index in continuous ijk space. - int kbc = (blockIdx.x + 0)*iter_k*iter_j*ne02 / gridDim.x; - const int kbc_stop = (blockIdx.x + 1)*iter_k*iter_j*ne02 / gridDim.x; + int kbc = (blockIdx.x + 0)*iter_k*iter_j*(ne02/ncols2) / gridDim.x; + const int kbc_stop = (blockIdx.x + 1)*iter_k*iter_j*(ne02/ncols2) / gridDim.x; // If the seams of 2 CUDA blocks fall within an output tile their results need to be combined. // For this we need to track both the block that starts the tile (needs_fixup) and the block that finishes the tile (is_fixup). @@ -605,25 +879,28 @@ static __global__ void flash_attn_ext_f16( const int channel = kbc / (iter_k*iter_j); const int jt = (kbc - channel*iter_k*iter_j) / iter_k; // j index of current tile. - const float2 * Q_f2 = (const float2 *) (Q + nb02* channel); - const half2 * K_h2 = (const half2 *) (K + nb12*(channel / gqa_ratio)); - const half2 * V_h2 = (const half2 *) (V + nb12*(channel / gqa_ratio)); // K and V have same shape - const half * maskh = mask ? (const half *) mask + (nb31/sizeof(half))*jt*ncols : nullptr; - float2 * dstk = ((float2 *) dst) + channel*(D/2); + const float2 * Q_f2 = (const float2 *) (Q + nb02* channel*ncols2); + const half2 * K_h2 = (const half2 *) (K + nb12*(channel*ncols2 / gqa_ratio)); + const half2 * V_h2 = (const half2 *) (V + nb12*(channel*ncols2 / gqa_ratio)); // K and V have same shape + const half2 * mask_h2 = ncols2 > 1 || mask ? (const half2 *) mask + (nb31/sizeof(half2))*jt*ncols1 : nullptr; + float2 * dstk = ((float2 *) dst) + channel*(ncols2 * D/2); - const float slope = get_alibi_slope(max_bias, channel, n_head_log2, m0, m1); + const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, channel, n_head_log2, m0, m1) : 1.0f; + + const int kb0_start_kernel = kb0_start * kb_niter; + const int kb0_stop_kernel = kb0_stop * kb_niter; constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. if (kb0_start == 0) { constexpr bool needs_fixup = false; // CUDA block is working on an entire tile. - flash_attn_ext_f16_process_tile - (Q_f2, K_h2, V_h2, maskh, dstk, dst_meta, scale, slope, logit_softcap, - ne01, ne02, stride_Q, stride_KV, stride_mask, jt, kb0_start, kb0_stop); + flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h2, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, stride_Q1, stride_Q2, stride_KV, stride_mask, jt, kb0_start_kernel, kb0_stop_kernel); } else { constexpr bool needs_fixup = true; // CUDA block is working on the beginning of a tile. - flash_attn_ext_f16_process_tile - (Q_f2, K_h2, V_h2, maskh, dstk, dst_meta, scale, slope, logit_softcap, - ne01, ne02, stride_Q, stride_KV, stride_mask, jt, kb0_start, kb0_stop); + flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h2, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, stride_Q1, stride_Q2, stride_KV, stride_mask, jt, kb0_start_kernel, kb0_stop_kernel); } kbc += iter_k; @@ -640,39 +917,46 @@ static __global__ void flash_attn_ext_f16( const int channel = kbc / (iter_k*iter_j); const int jt = (kbc - channel*iter_k*iter_j) / iter_k; // j index of current tile. - const float2 * Q_f2 = (const float2 *) (Q + nb02* channel); - const half2 * K_h2 = (const half2 *) (K + nb12*(channel / gqa_ratio)); - const half2 * V_h2 = (const half2 *) (V + nb12*(channel / gqa_ratio)); // K and V have same shape - const half * maskh = mask ? (const half *) mask + (nb31/sizeof(half))*jt*ncols : nullptr; - float2 * dstk = ((float2 *) dst) + channel*(D/2); + const float2 * Q_f2 = (const float2 *) (Q + nb02* channel*ncols2); + const half2 * K_h2 = (const half2 *) (K + nb12*(channel*ncols2 / gqa_ratio)); + const half2 * V_h2 = (const half2 *) (V + nb12*(channel*ncols2 / gqa_ratio)); // K and V have same shape + const half2 * mask_h2 = ncols2 > 1 || mask ? (const half2 *) mask + (nb31/sizeof(half2))*jt*ncols1 : nullptr; + float2 * dstk = ((float2 *) dst) + channel*(ncols2 * D/2); - const float slope = get_alibi_slope(max_bias, channel, n_head_log2, m0, m1); + const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, channel, n_head_log2, m0, m1) : 1.0f; + + const int kb0_start_kernel = kb0_start * kb_niter; + const int kb0_stop_kernel = kb0_stop * kb_niter; constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. constexpr bool needs_fixup = false; - flash_attn_ext_f16_process_tile - (Q_f2, K_h2, V_h2, maskh, dstk, dst_meta, scale, slope, logit_softcap, - ne01, ne02, stride_Q, stride_KV, stride_mask, jt, kb0_start, kb0_stop); + flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h2, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, stride_Q1, stride_Q2, stride_KV, stride_mask, jt, kb0_start_kernel, kb0_stop_kernel); } -template +template void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - typedef tile<16, 8, half2> tile_A; - typedef tile< 8, 8, half2> tile_B; + constexpr int ncols = ncols1 * ncols2; + constexpr int KQ_per_iter = D <= 128 && ncols1 <= 64 ? 64 : 32; + constexpr int nwarps = (KQ_per_iter == 32 && ncols <= 16) ? 2 : 4; + constexpr int ntiles = ncols <= 8 ? 1 : (ncols <= 64 ? 2 : 4); + constexpr int cols_per_warp = ntiles * tile_B::I; - static_assert(D % tile_B::J == 0, "bad D"); - static_assert(cols_per_block % tile_B::I == 0, "bad cols_per_block"); + static_assert(D % tile_B::J == 0, "bad D"); + static_assert(ncols % cols_per_warp == 0, "bad ncols"); const ggml_tensor * KQV = dst; - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; - constexpr int KQ_stride = D <= 128 ? 64 : 32; - constexpr int nwarps = (KQ_stride == 32 && cols_per_block <= 16) ? - cols_per_block/tile_B::J * KQ_stride/tile_A::I : (cols_per_block <= 8 ? 4 : 8); + const int KQ_shared_rows = cp_async_available(cc) ? 2*KQ_per_iter : KQ_per_iter; - const int nrows_KQ = cp_async_available(cc) ? 2*KQ_stride : KQ_stride; - const int nrows_combine = nwarps*tile_B::J; - const size_t nbytes_shared = std::max(nrows_KQ, nrows_combine) * (D + 8) * sizeof(half); + const size_t nbytes_shared_KV = KQ_shared_rows * (D + 8) * sizeof(half); + const size_t nbytes_shared_mask = ncols1 * (KQ_per_iter + 8) * sizeof(half); + const size_t nbytes_shared_combine = nwarps*cols_per_warp * (D + 8) * sizeof(half); + + const size_t nbytes_shared_total = std::max(nbytes_shared_KV + nbytes_shared_mask, nbytes_shared_combine); float logit_softcap; memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); @@ -680,42 +964,58 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml fattn_kernel_t fattn_kernel; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - fattn_kernel = flash_attn_ext_f16; + fattn_kernel = flash_attn_ext_f16; } else { constexpr bool use_logit_softcap = true; - fattn_kernel = flash_attn_ext_f16; + fattn_kernel = flash_attn_ext_f16; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, true, true); } -#define DECL_FATTN_MMA_F16_CASE(D, cols_per_block) \ + +#define DECL_FATTN_MMA_F16_CASE(D, ncols1, ncols2) \ template void ggml_cuda_flash_attn_ext_mma_f16_case \ - (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \ + (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \ -extern DECL_FATTN_MMA_F16_CASE( 64, 8); -extern DECL_FATTN_MMA_F16_CASE( 80, 8); -extern DECL_FATTN_MMA_F16_CASE( 96, 8); -extern DECL_FATTN_MMA_F16_CASE(112, 8); -extern DECL_FATTN_MMA_F16_CASE(128, 8); -extern DECL_FATTN_MMA_F16_CASE(256, 8); +#define DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(D, ncols) \ + extern DECL_FATTN_MMA_F16_CASE(D, (ncols)/1, 1); \ + extern DECL_FATTN_MMA_F16_CASE(D, (ncols)/2, 2); \ + extern DECL_FATTN_MMA_F16_CASE(D, (ncols)/4, 4); \ + extern DECL_FATTN_MMA_F16_CASE(D, (ncols)/8, 8); \ -extern DECL_FATTN_MMA_F16_CASE( 64, 16); -extern DECL_FATTN_MMA_F16_CASE( 80, 16); -extern DECL_FATTN_MMA_F16_CASE( 96, 16); -extern DECL_FATTN_MMA_F16_CASE(112, 16); -extern DECL_FATTN_MMA_F16_CASE(128, 16); -extern DECL_FATTN_MMA_F16_CASE(256, 16); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 8); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 8); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 8); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 8); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 8); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 8); -extern DECL_FATTN_MMA_F16_CASE( 64, 32); -extern DECL_FATTN_MMA_F16_CASE( 80, 32); -extern DECL_FATTN_MMA_F16_CASE( 96, 32); -extern DECL_FATTN_MMA_F16_CASE(112, 32); -extern DECL_FATTN_MMA_F16_CASE(128, 32); -extern DECL_FATTN_MMA_F16_CASE(256, 32); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 16); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 16); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 16); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 16); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 16); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 16); -extern DECL_FATTN_MMA_F16_CASE( 64, 64); -extern DECL_FATTN_MMA_F16_CASE( 80, 64); -extern DECL_FATTN_MMA_F16_CASE( 96, 64); -extern DECL_FATTN_MMA_F16_CASE(112, 64); -extern DECL_FATTN_MMA_F16_CASE(128, 64); -extern DECL_FATTN_MMA_F16_CASE(256, 64); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 32); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 32); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 32); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 32); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 32); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 32); + +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 64); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 64); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 64); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 64); +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 64); + +// Kernels with ncols == 128 are only 4% faster due to register pressure. +// DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 128); +// DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 128); +// DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 128); +// DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 128); +// DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128); +// DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 128); // Needs too much shared memory. diff --git a/ggml/src/ggml-cuda/fattn-tile-f16.cu b/ggml/src/ggml-cuda/fattn-tile-f16.cu index d4edbad07f..b8b415effb 100644 --- a/ggml/src/ggml-cuda/fattn-tile-f16.cu +++ b/ggml/src/ggml-cuda/fattn-tile-f16.cu @@ -302,14 +302,14 @@ void launch_fattn_tile_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); } break; case 128: { constexpr int D = 128; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); } break; default: { GGML_ABORT("FlashAttention without tensor cores only supports head sizes 64 and 128."); diff --git a/ggml/src/ggml-cuda/fattn-tile-f32.cu b/ggml/src/ggml-cuda/fattn-tile-f32.cu index 0d274f3325..4352a28446 100644 --- a/ggml/src/ggml-cuda/fattn-tile-f32.cu +++ b/ggml/src/ggml-cuda/fattn-tile-f32.cu @@ -296,14 +296,14 @@ void launch_fattn_tile_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); } break; case 128: { constexpr int D = 128; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); } break; default: { GGML_ABORT("FlashAttention without tensor cores only supports head sizes 64 and 128."); diff --git a/ggml/src/ggml-cuda/fattn-vec-f16.cuh b/ggml/src/ggml-cuda/fattn-vec-f16.cuh index d9ac442460..e758a0f6ec 100644 --- a/ggml/src/ggml-cuda/fattn-vec-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-vec-f16.cuh @@ -310,7 +310,7 @@ void ggml_cuda_flash_attn_ext_vec_f16_case_impl(ggml_backend_cuda_context & ctx, constexpr bool need_f16_K = D != 128; constexpr bool need_f16_V = D != 128 && D != 64; constexpr size_t nbytes_shared = 0; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, need_f16_K, need_f16_V); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, need_f16_K, need_f16_V); } template diff --git a/ggml/src/ggml-cuda/fattn-vec-f32.cuh b/ggml/src/ggml-cuda/fattn-vec-f32.cuh index 6ef8f9dcc2..134144a383 100644 --- a/ggml/src/ggml-cuda/fattn-vec-f32.cuh +++ b/ggml/src/ggml-cuda/fattn-vec-f32.cuh @@ -290,7 +290,7 @@ void ggml_cuda_flash_attn_ext_vec_f32_case_impl(ggml_backend_cuda_context & ctx, constexpr bool need_f16_K = D != 128; constexpr bool need_f16_V = D != 128 && D != 64; constexpr size_t nbytes_shared = 0; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, need_f16_K, need_f16_V); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, need_f16_K, need_f16_V); } template diff --git a/ggml/src/ggml-cuda/fattn-wmma-f16.cu b/ggml/src/ggml-cuda/fattn-wmma-f16.cu index 45702ad651..de38470abe 100644 --- a/ggml/src/ggml-cuda/fattn-wmma-f16.cu +++ b/ggml/src/ggml-cuda/fattn-wmma-f16.cu @@ -478,7 +478,7 @@ void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggm fattn_kernel = flash_attn_ext_f16< D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true); return; } if (2*blocks_num_pb1 < 2*nsm) { @@ -493,7 +493,7 @@ void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggm fattn_kernel = flash_attn_ext_f16< D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true); return; } constexpr int parallel_blocks = 1; @@ -507,7 +507,7 @@ void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggm fattn_kernel = flash_attn_ext_f16< D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true); } void ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index b0cf152f52..b1becccb4d 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -8,28 +8,50 @@ #include "fattn-wmma-f16.cuh" #include "fattn.cuh" -template +template +static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + + if (Q->ne[1] <= 8/ncols2) { + ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); + return; + } + + if (Q->ne[1] <= 16/ncols2) { + ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); + return; + } + + if (Q->ne[1] <= 32/ncols2) { + ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); + return; + } + + ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); +} + +template static void ggml_cuda_flash_attn_ext_mma_f16_switch_hs(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * Q = dst->src[0]; switch (Q->ne[0]) { case 64: - ggml_cuda_flash_attn_ext_mma_f16_case< 64, cols_per_block>(ctx, dst); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1< 64, ncols2>(ctx, dst); break; case 80: - ggml_cuda_flash_attn_ext_mma_f16_case< 80, cols_per_block>(ctx, dst); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1< 80, ncols2>(ctx, dst); break; case 96: - ggml_cuda_flash_attn_ext_mma_f16_case< 96, cols_per_block>(ctx, dst); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1< 96, ncols2>(ctx, dst); break; case 112: - ggml_cuda_flash_attn_ext_mma_f16_case<112, cols_per_block>(ctx, dst); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<112, ncols2>(ctx, dst); break; case 128: - ggml_cuda_flash_attn_ext_mma_f16_case<128, cols_per_block>(ctx, dst); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<128, ncols2>(ctx, dst); break; case 256: - ggml_cuda_flash_attn_ext_mma_f16_case<256, cols_per_block>(ctx, dst); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<256, ncols2>(ctx, dst); break; default: GGML_ABORT("fatal error"); @@ -38,24 +60,35 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_hs(ggml_backend_cuda_context } static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * KQV = dst; + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * mask = dst->src[3]; - if (Q->ne[1] <= 8) { + float max_bias = 0.0f; + memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); + + const float use_gqa_opt = mask && max_bias == 0.0f; + + GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); + const int gqa_ratio = Q->ne[2] / K->ne[2]; + + if (use_gqa_opt && gqa_ratio % 8 == 0) { ggml_cuda_flash_attn_ext_mma_f16_switch_hs<8>(ctx, dst); return; } - if (Q->ne[1] <= 16) { - ggml_cuda_flash_attn_ext_mma_f16_switch_hs<16>(ctx, dst); + if (use_gqa_opt && gqa_ratio == 4) { + ggml_cuda_flash_attn_ext_mma_f16_switch_hs<4>(ctx, dst); return; } - if (Q->ne[1] <= 32) { - ggml_cuda_flash_attn_ext_mma_f16_switch_hs<32>(ctx, dst); + if (use_gqa_opt && gqa_ratio == 2) { + ggml_cuda_flash_attn_ext_mma_f16_switch_hs<2>(ctx, dst); return; } - ggml_cuda_flash_attn_ext_mma_f16_switch_hs<64>(ctx, dst); + ggml_cuda_flash_attn_ext_mma_f16_switch_hs<1>(ctx, dst); } #define FATTN_VEC_F16_CASE(D, type_K, type_V) \ @@ -209,8 +242,11 @@ static void ggml_cuda_flash_attn_ext_vec_f32(ggml_backend_cuda_context & ctx, gg } void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * KQV = dst; - const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * KQV = dst; + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; ggml_cuda_set_device(ctx.device); const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; @@ -252,7 +288,10 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst return; } - if (Q->ne[1] == 1 && Q->ne[0] % (2*WARP_SIZE) == 0) { + const int gqa_ratio = Q->ne[2] / K->ne[2]; + const bool mma_fast_for_bs1 = fp16_mma_available(cc) && gqa_ratio % 2 == 0 && + K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && mask; + if (Q->ne[1] == 1 && Q->ne[0] % (2*WARP_SIZE) == 0 && !mma_fast_for_bs1) { if (prec == GGML_PREC_DEFAULT) { ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); return; diff --git a/ggml/src/ggml-cuda/mma.cuh b/ggml/src/ggml-cuda/mma.cuh index 0a5656e4cb..9206bfeba3 100644 --- a/ggml/src/ggml-cuda/mma.cuh +++ b/ggml/src/ggml-cuda/mma.cuh @@ -73,6 +73,8 @@ namespace ggml_cuda_mma { return threadIdx.x / 4; } else if constexpr (I == 16 && J == 8) { return (l / 2) * 8 + threadIdx.x / 4; + } else if constexpr (I == 16 && J == 16) { + return ((l / 2) % 2) * 8 + threadIdx.x / 4; } else { static_assert(I == -1 && J == -1, "template specialization not implemented"); } @@ -85,6 +87,8 @@ namespace ggml_cuda_mma { return 4 * l + threadIdx.x % 4; } else if constexpr (I == 16 && J == 8) { return 2 * (threadIdx.x % 4) + l % 2; + } else if constexpr (I == 16 && J == 16) { + return 8 * (l / 4) + 2 * (threadIdx.x % 4) + l % 2; } else { static_assert(I == -1 && J == -1, "template specialization not implemented"); } @@ -289,6 +293,42 @@ namespace ggml_cuda_mma { #endif // NEW_MMA_AVAILABLE } + static __device__ __forceinline__ void mma( + tile<16, 8, half2> & D, const tile<16, 8, half2> & A, const tile<16, 8, half2> & B) { +#ifdef NEW_MMA_AVAILABLE + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; +#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};" + : "+r"(Dxi[0]), "+r"(Dxi[1]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[2])); + asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};" + : "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]), "r"(Bxi[3])); +#else + // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead: + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[0]), "+r"(Dxi[1]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0])); + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[0]), "+r"(Dxi[1]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2])); + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[1])); + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3])); +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#else + GGML_UNUSED(D); + GGML_UNUSED(A); + GGML_UNUSED(B); + NO_DEVICE_CODE; +#endif // NEW_MMA_AVAILABLE + } + static __device__ __forceinline__ void mma( tile<16, 8, float> & D, const tile<16, 8, half2> & A, const tile<8, 8, half2> & B) { #ifdef NEW_MMA_AVAILABLE @@ -316,4 +356,39 @@ namespace ggml_cuda_mma { #endif // NEW_MMA_AVAILABLE } + static __device__ __forceinline__ void mma( + tile<16, 16, float> & D, const tile<16, 8, half2> & A, const tile<16, 8, half2> & B) { +#ifdef NEW_MMA_AVAILABLE + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; +#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[2])); + asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};" + : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]), "r"(Bxi[3])); +#else + // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead: + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0])); + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2])); + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[1])); + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3])); +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#else + GGML_UNUSED(D); + GGML_UNUSED(A); + GGML_UNUSED(B); + NO_DEVICE_CODE; +#endif // NEW_MMA_AVAILABLE + } } diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb16.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb16.cu deleted file mode 100644 index f09bdeff79..0000000000 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb16.cu +++ /dev/null @@ -1,10 +0,0 @@ -// This file has been autogenerated by generate_cu_files.py, do not edit manually. - -#include "../fattn-mma-f16.cuh" - -DECL_FATTN_MMA_F16_CASE(64, 16); -DECL_FATTN_MMA_F16_CASE(80, 16); -DECL_FATTN_MMA_F16_CASE(96, 16); -DECL_FATTN_MMA_F16_CASE(112, 16); -DECL_FATTN_MMA_F16_CASE(128, 16); -DECL_FATTN_MMA_F16_CASE(256, 16); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb32.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb32.cu deleted file mode 100644 index 221108873a..0000000000 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb32.cu +++ /dev/null @@ -1,10 +0,0 @@ -// This file has been autogenerated by generate_cu_files.py, do not edit manually. - -#include "../fattn-mma-f16.cuh" - -DECL_FATTN_MMA_F16_CASE(64, 32); -DECL_FATTN_MMA_F16_CASE(80, 32); -DECL_FATTN_MMA_F16_CASE(96, 32); -DECL_FATTN_MMA_F16_CASE(112, 32); -DECL_FATTN_MMA_F16_CASE(128, 32); -DECL_FATTN_MMA_F16_CASE(256, 32); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb64.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb64.cu deleted file mode 100644 index d24b085758..0000000000 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb64.cu +++ /dev/null @@ -1,10 +0,0 @@ -// This file has been autogenerated by generate_cu_files.py, do not edit manually. - -#include "../fattn-mma-f16.cuh" - -DECL_FATTN_MMA_F16_CASE(64, 64); -DECL_FATTN_MMA_F16_CASE(80, 64); -DECL_FATTN_MMA_F16_CASE(96, 64); -DECL_FATTN_MMA_F16_CASE(112, 64); -DECL_FATTN_MMA_F16_CASE(128, 64); -DECL_FATTN_MMA_F16_CASE(256, 64); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb8.cu deleted file mode 100644 index bdf86c0eab..0000000000 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-cpb8.cu +++ /dev/null @@ -1,10 +0,0 @@ -// This file has been autogenerated by generate_cu_files.py, do not edit manually. - -#include "../fattn-mma-f16.cuh" - -DECL_FATTN_MMA_F16_CASE(64, 8); -DECL_FATTN_MMA_F16_CASE(80, 8); -DECL_FATTN_MMA_F16_CASE(96, 8); -DECL_FATTN_MMA_F16_CASE(112, 8); -DECL_FATTN_MMA_F16_CASE(128, 8); -DECL_FATTN_MMA_F16_CASE(256, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu new file mode 100644 index 0000000000..80108615ac --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 1, 8); +DECL_FATTN_MMA_F16_CASE(80, 1, 8); +DECL_FATTN_MMA_F16_CASE(96, 1, 8); +DECL_FATTN_MMA_F16_CASE(112, 1, 8); +DECL_FATTN_MMA_F16_CASE(128, 1, 8); +DECL_FATTN_MMA_F16_CASE(256, 1, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu new file mode 100644 index 0000000000..66161c0abe --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 16, 1); +DECL_FATTN_MMA_F16_CASE(80, 16, 1); +DECL_FATTN_MMA_F16_CASE(96, 16, 1); +DECL_FATTN_MMA_F16_CASE(112, 16, 1); +DECL_FATTN_MMA_F16_CASE(128, 16, 1); +DECL_FATTN_MMA_F16_CASE(256, 16, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu new file mode 100644 index 0000000000..ee88c72aa0 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 16, 2); +DECL_FATTN_MMA_F16_CASE(80, 16, 2); +DECL_FATTN_MMA_F16_CASE(96, 16, 2); +DECL_FATTN_MMA_F16_CASE(112, 16, 2); +DECL_FATTN_MMA_F16_CASE(128, 16, 2); +DECL_FATTN_MMA_F16_CASE(256, 16, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu new file mode 100644 index 0000000000..d888a5a423 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 16, 4); +DECL_FATTN_MMA_F16_CASE(80, 16, 4); +DECL_FATTN_MMA_F16_CASE(96, 16, 4); +DECL_FATTN_MMA_F16_CASE(112, 16, 4); +DECL_FATTN_MMA_F16_CASE(128, 16, 4); +DECL_FATTN_MMA_F16_CASE(256, 16, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu new file mode 100644 index 0000000000..d93a2d08ed --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 2, 4); +DECL_FATTN_MMA_F16_CASE(80, 2, 4); +DECL_FATTN_MMA_F16_CASE(96, 2, 4); +DECL_FATTN_MMA_F16_CASE(112, 2, 4); +DECL_FATTN_MMA_F16_CASE(128, 2, 4); +DECL_FATTN_MMA_F16_CASE(256, 2, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu new file mode 100644 index 0000000000..617464c945 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 2, 8); +DECL_FATTN_MMA_F16_CASE(80, 2, 8); +DECL_FATTN_MMA_F16_CASE(96, 2, 8); +DECL_FATTN_MMA_F16_CASE(112, 2, 8); +DECL_FATTN_MMA_F16_CASE(128, 2, 8); +DECL_FATTN_MMA_F16_CASE(256, 2, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu new file mode 100644 index 0000000000..970d2b6869 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 32, 1); +DECL_FATTN_MMA_F16_CASE(80, 32, 1); +DECL_FATTN_MMA_F16_CASE(96, 32, 1); +DECL_FATTN_MMA_F16_CASE(112, 32, 1); +DECL_FATTN_MMA_F16_CASE(128, 32, 1); +DECL_FATTN_MMA_F16_CASE(256, 32, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu new file mode 100644 index 0000000000..65cd377c39 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 32, 2); +DECL_FATTN_MMA_F16_CASE(80, 32, 2); +DECL_FATTN_MMA_F16_CASE(96, 32, 2); +DECL_FATTN_MMA_F16_CASE(112, 32, 2); +DECL_FATTN_MMA_F16_CASE(128, 32, 2); +DECL_FATTN_MMA_F16_CASE(256, 32, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu new file mode 100644 index 0000000000..f4a8bf3489 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 4, 2); +DECL_FATTN_MMA_F16_CASE(80, 4, 2); +DECL_FATTN_MMA_F16_CASE(96, 4, 2); +DECL_FATTN_MMA_F16_CASE(112, 4, 2); +DECL_FATTN_MMA_F16_CASE(128, 4, 2); +DECL_FATTN_MMA_F16_CASE(256, 4, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu new file mode 100644 index 0000000000..de191a8ab6 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 4, 4); +DECL_FATTN_MMA_F16_CASE(80, 4, 4); +DECL_FATTN_MMA_F16_CASE(96, 4, 4); +DECL_FATTN_MMA_F16_CASE(112, 4, 4); +DECL_FATTN_MMA_F16_CASE(128, 4, 4); +DECL_FATTN_MMA_F16_CASE(256, 4, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu new file mode 100644 index 0000000000..e8cb0e1b31 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 4, 8); +DECL_FATTN_MMA_F16_CASE(80, 4, 8); +DECL_FATTN_MMA_F16_CASE(96, 4, 8); +DECL_FATTN_MMA_F16_CASE(112, 4, 8); +DECL_FATTN_MMA_F16_CASE(128, 4, 8); +DECL_FATTN_MMA_F16_CASE(256, 4, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu new file mode 100644 index 0000000000..a532e96296 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 64, 1); +DECL_FATTN_MMA_F16_CASE(80, 64, 1); +DECL_FATTN_MMA_F16_CASE(96, 64, 1); +DECL_FATTN_MMA_F16_CASE(112, 64, 1); +DECL_FATTN_MMA_F16_CASE(128, 64, 1); +DECL_FATTN_MMA_F16_CASE(256, 64, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu new file mode 100644 index 0000000000..bf25181aa7 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 8, 1); +DECL_FATTN_MMA_F16_CASE(80, 8, 1); +DECL_FATTN_MMA_F16_CASE(96, 8, 1); +DECL_FATTN_MMA_F16_CASE(112, 8, 1); +DECL_FATTN_MMA_F16_CASE(128, 8, 1); +DECL_FATTN_MMA_F16_CASE(256, 8, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu new file mode 100644 index 0000000000..378c132e65 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 8, 2); +DECL_FATTN_MMA_F16_CASE(80, 8, 2); +DECL_FATTN_MMA_F16_CASE(96, 8, 2); +DECL_FATTN_MMA_F16_CASE(112, 8, 2); +DECL_FATTN_MMA_F16_CASE(128, 8, 2); +DECL_FATTN_MMA_F16_CASE(256, 8, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu new file mode 100644 index 0000000000..372641be9a --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 8, 4); +DECL_FATTN_MMA_F16_CASE(80, 8, 4); +DECL_FATTN_MMA_F16_CASE(96, 8, 4); +DECL_FATTN_MMA_F16_CASE(112, 8, 4); +DECL_FATTN_MMA_F16_CASE(128, 8, 4); +DECL_FATTN_MMA_F16_CASE(256, 8, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu new file mode 100644 index 0000000000..9ff5968b6a --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu @@ -0,0 +1,10 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_CASE(64, 8, 8); +DECL_FATTN_MMA_F16_CASE(80, 8, 8); +DECL_FATTN_MMA_F16_CASE(96, 8, 8); +DECL_FATTN_MMA_F16_CASE(112, 8, 8); +DECL_FATTN_MMA_F16_CASE(128, 8, 8); +DECL_FATTN_MMA_F16_CASE(256, 8, 8); diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index a2628f16e5..dd373a09d2 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -18,7 +18,7 @@ SOURCE_FATTN_MMA_START = """// This file has been autogenerated by generate_cu_f """ -SOURCE_FATTN_MMA_CASE = "DECL_FATTN_MMA_F16_CASE({head_size}, {cols_per_block});\n" +SOURCE_FATTN_MMA_CASE = "DECL_FATTN_MMA_F16_CASE({head_size}, {ncols1}, {ncols2});\n" TYPES_MMQ = [ "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", @@ -57,12 +57,18 @@ for vkq_size in [16, 32]: with open(f"fattn-vec-f{vkq_size}-instance-hs{head_size}-{get_short_name(type_k)}-{get_short_name(type_v)}.cu", "w") as f: f.write(SOURCE_FATTN_VEC.format(vkq_size=vkq_size, head_size=head_size, type_k=type_k, type_v=type_v)) -for cols_per_block in [8, 16, 32, 64]: - with open(f"fattn-mma-f16-instance-cpb{cols_per_block}.cu", "w") as f: - f.write(SOURCE_FATTN_MMA_START) +for ncols in [8, 16, 32, 64, 128]: + for ncols2 in [1, 2, 4, 8]: + ncols1 = ncols // ncols2 + if ncols == 128: + continue # Too much register pressure. + with open(f"fattn-mma-f16-instance-ncols1_{ncols1}-ncols2_{ncols2}.cu", "w") as f: + f.write(SOURCE_FATTN_MMA_START) - for head_size in [64, 80, 96, 112, 128, 256]: - f.write(SOURCE_FATTN_MMA_CASE.format(cols_per_block=cols_per_block, head_size=head_size)) + for head_size in [64, 80, 96, 112, 128, 256]: + if ncols == 128 and head_size == 256: + continue # Needs too much shared memory. + f.write(SOURCE_FATTN_MMA_CASE.format(ncols1=ncols1, ncols2=ncols2, head_size=head_size)) for type in TYPES_MMQ: with open(f"mmq-instance-{get_short_name(type)}.cu", "w") as f: diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c9ab6c135f..e1f7e6758b 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -3119,6 +3119,7 @@ struct test_leaky_relu : public test_case { struct test_flash_attn_ext : public test_case { const int64_t hs; // head size const int64_t nh; // num heads + const int64_t nr; // repeat in Q, tests for grouped-query attention const int64_t kv; // kv size const int64_t nb; // batch size @@ -3131,7 +3132,7 @@ struct test_flash_attn_ext : public test_case { std::array permute; std::string vars() override { - return VARS_TO_STR9(hs, nh, kv, nb, mask, max_bias, logit_softcap, type_KV, permute); + return VARS_TO_STR10(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, type_KV, permute); } double max_nmse_err() override { @@ -3142,13 +3143,13 @@ struct test_flash_attn_ext : public test_case { GGML_UNUSED(t); // Just counting matmul costs: // Q*K^T is nb x hs x kv, P*V is nb x kv x hs, per head - return 2 * 2 * nh * nb * hs * kv; + return 2 * 2 * nh*nr * nb * hs * kv; } - test_flash_attn_ext(int64_t hs = 128, int64_t nh = 32, int64_t kv = 96, int64_t nb = 8, + test_flash_attn_ext(int64_t hs = 128, int64_t nh = 32, int64_t nr = 1, int64_t kv = 96, int64_t nb = 8, bool mask = true, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_type type_KV = GGML_TYPE_F16, std::array permute = {0, 1, 2, 3}) - : hs(hs), nh(nh), kv(kv), nb(nb), mask(mask), max_bias(max_bias), logit_softcap(logit_softcap), type_KV(type_KV), permute(permute) {} + : hs(hs), nh(nh), nr(nr), kv(kv), nb(nb), mask(mask), max_bias(max_bias), logit_softcap(logit_softcap), type_KV(type_KV), permute(permute) {} ggml_tensor * build_graph(ggml_context * ctx) override { const int64_t hs_padded = GGML_PAD(hs, ggml_blck_size(type_KV)); @@ -3166,13 +3167,13 @@ struct test_flash_attn_ext : public test_case { return t; }; - ggml_tensor * q = create_permuted(GGML_TYPE_F32, hs_padded, nb, nh, 1); + ggml_tensor * q = create_permuted(GGML_TYPE_F32, hs_padded, nb, nh*nr, 1); ggml_set_name(q, "q"); - ggml_tensor * k = create_permuted(type_KV, hs_padded, kv, nh, 1); + ggml_tensor * k = create_permuted(type_KV, hs_padded, kv, nh, 1); ggml_set_name(k, "k"); - ggml_tensor * v = create_permuted(type_KV, hs_padded, kv, nh, 1); + ggml_tensor * v = create_permuted(type_KV, hs_padded, kv, nh, 1); ggml_set_name(v, "v"); ggml_tensor * m = nullptr; @@ -4278,14 +4279,18 @@ static std::vector> make_test_cases_eval() { if (!mask && max_bias > 0.0f) continue; for (float logit_softcap : {0.0f, 10.0f}) { if (hs != 128 && logit_softcap != 0.0f) continue; - for (int nh : { 32, }) { - for (int kv : { 512, 1024, }) { - for (int nb : { 1, 3, 32, 35, }) { - for (ggml_type type_KV : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { - test_cases.emplace_back(new test_flash_attn_ext(hs, nh, kv, nb, mask, max_bias, logit_softcap, type_KV)); - // run fewer test cases permuted - if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) { - test_cases.emplace_back(new test_flash_attn_ext(hs, nh, kv, nb, mask, max_bias, logit_softcap, type_KV, {0, 2, 1, 3})); + for (int nh : { 4, }) { + for (int nr : { 1, 4, 16 }) { + if (nr == 16 && hs != 128) continue; + for (int kv : { 512, 1024, }) { + if (nr != 1 && kv != 512) continue; + for (int nb : { 1, 3, 32, 35, }) { + for (ggml_type type_KV : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { + test_cases.emplace_back(new test_flash_attn_ext(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, type_KV)); + // run fewer test cases permuted + if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) { + test_cases.emplace_back(new test_flash_attn_ext(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, type_KV, {0, 2, 1, 3})); + } } } } From f3e64859edb0d55d4223ead78672597cd1a218df Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 22 Feb 2025 15:03:00 +0200 Subject: [PATCH 006/188] ci : fix arm upload artifacts (#12024) * ci : fix arm upload artifacts * cont : fix archive name to use matrix --- .github/workflows/build.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 859d538c77..b96e1f50ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -175,7 +175,12 @@ jobs: ubuntu-cpu-cmake: strategy: matrix: - os: [ubuntu-22.04, ubuntu-22.04-arm] + include: + - build: 'x64' + os: ubuntu-22.04 + - build: 'arm64' + os: ubuntu-22.04-arm + runs-on: ${{ matrix.os }} steps: @@ -242,14 +247,14 @@ jobs: run: | cp LICENSE ./build/bin/ cp examples/run/linenoise.cpp/LICENSE ./build/bin/LICENSE.linenoise.cpp - zip -r llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.zip ./build/bin/* + zip -r llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.zip ./build/bin/* - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} uses: actions/upload-artifact@v4 with: - path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.zip - name: llama-bin-ubuntu-x64.zip + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.zip + name: llama-bin-ubuntu-${{ matrix.build }}.zip ubuntu-latest-cmake-sanitizer: runs-on: ubuntu-latest From 36c258ee921dbb5c96bdc57c0872e4a9a129bef6 Mon Sep 17 00:00:00 2001 From: Ting Lou Date: Sat, 22 Feb 2025 22:28:28 +0800 Subject: [PATCH 007/188] llava: build clip image from pixels (#11999) * llava: export function `clip_build_img_from_pixels` to build image from pixels decoded by other libraries instead of stb_image.h for better performance * Apply suggestions from code review --------- Co-authored-by: Xuan-Son Nguyen --- examples/llava/clip.cpp | 8 ++++---- examples/llava/clip.h | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index bf70e351c9..e2150c06d3 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -1729,11 +1729,11 @@ void clip_image_f32_batch_free(struct clip_image_f32_batch * batch) { } } -static void build_clip_img_from_data(const stbi_uc * data, int nx, int ny, clip_image_u8 * img) { +void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, clip_image_u8 * img) { img->nx = nx; img->ny = ny; img->buf.resize(3 * nx * ny); - memcpy(img->buf.data(), data, img->buf.size()); + memcpy(img->buf.data(), rgb_pixels, img->buf.size()); } bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) { @@ -1743,7 +1743,7 @@ bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) { LOG_ERR("%s: failed to load image '%s'\n", __func__, fname); return false; } - build_clip_img_from_data(data, nx, ny, img); + clip_build_img_from_pixels(data, nx, ny, img); stbi_image_free(data); return true; } @@ -1755,7 +1755,7 @@ bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length LOG_ERR("%s: failed to decode image bytes\n", __func__); return false; } - build_clip_img_from_data(data, nx, ny, img); + clip_build_img_from_pixels(data, nx, ny, img); stbi_image_free(data); return true; } diff --git a/examples/llava/clip.h b/examples/llava/clip.h index 841b4f6f90..f557122008 100644 --- a/examples/llava/clip.h +++ b/examples/llava/clip.h @@ -73,6 +73,9 @@ CLIP_API void clip_image_f32_free(struct clip_image_f32 * img); CLIP_API void clip_image_u8_batch_free (struct clip_image_u8_batch * batch); CLIP_API void clip_image_f32_batch_free(struct clip_image_f32_batch * batch); +/** build image from pixels decoded by other libraries instead of stb_image.h for better performance. The memory layout is RGBRGBRGB..., input buffer length must be 3*nx*ny bytes */ +CLIP_API void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, clip_image_u8 * img); + CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img); /** interpret bytes as an image file with length bytes_length, and use the result to populate img */ From a28e0d5eb18c18e6a4598286158f427269b1444e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 22 Feb 2025 20:44:34 +0100 Subject: [PATCH 008/188] CUDA: app option to compile without FlashAttention (#12025) --- Makefile | 12 ++++++++++++ ggml/CMakeLists.txt | 1 + ggml/src/ggml-cuda/CMakeLists.txt | 4 ++++ ggml/src/ggml-cuda/common.cuh | 4 ++-- ggml/src/ggml-cuda/fattn-mma-f16.cuh | 8 ++++---- ggml/src/ggml-cuda/fattn-tile-f16.cu | 9 ++------- ggml/src/ggml-cuda/fattn-tile-f32.cu | 8 ++++---- ggml/src/ggml-cuda/fattn-vec-f16.cuh | 9 ++------- ggml/src/ggml-cuda/fattn-vec-f32.cuh | 8 ++++---- ggml/src/ggml-cuda/fattn-wmma-f16.cu | 4 ++-- ggml/src/ggml-cuda/ggml-cuda.cu | 2 +- ggml/src/ggml-hip/CMakeLists.txt | 4 ++++ ggml/src/ggml-musa/CMakeLists.txt | 4 ++++ 13 files changed, 46 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 69cab9c667..5339d490b4 100644 --- a/Makefile +++ b/Makefile @@ -680,6 +680,10 @@ ifdef GGML_CUDA_CCBIN MK_NVCCFLAGS += -ccbin $(GGML_CUDA_CCBIN) endif # GGML_CUDA_CCBIN +ifdef GGML_CUDA_NO_FA + MK_NVCCFLAGS += -DGGML_CUDA_NO_FA +endif # GGML_CUDA_NO_FA + ifdef GGML_CUDA_FA_ALL_QUANTS MK_NVCCFLAGS += -DGGML_CUDA_FA_ALL_QUANTS endif # GGML_CUDA_FA_ALL_QUANTS @@ -800,6 +804,10 @@ ifdef GGML_CUDA_NO_PEER_COPY HIPFLAGS += -DGGML_CUDA_NO_PEER_COPY endif # GGML_CUDA_NO_PEER_COPY +ifdef GGML_CUDA_NO_FA + HIPFLAGS += -DGGML_CUDA_NO_FA +endif # GGML_CUDA_NO_FA + OBJ_GGML_EXT += ggml/src/ggml-cuda/ggml-cuda.o OBJ_GGML_EXT += $(patsubst %.cu,%.o,$(wildcard ggml/src/ggml-cuda/*.cu)) OBJ_GGML_EXT += $(OBJ_CUDA_TMPL) @@ -876,6 +884,10 @@ ifdef GGML_CUDA_NO_PEER_COPY MUSAFLAGS += -DGGML_CUDA_NO_PEER_COPY endif # GGML_CUDA_NO_PEER_COPY +ifdef GGML_CUDA_NO_FA + MUSAFLAGS += -DGGML_CUDA_NO_FA +endif # GGML_CUDA_NO_FA + ifdef GGML_CUDA_FA_ALL_QUANTS MUSAFLAGS += -DGGML_CUDA_FA_ALL_QUANTS endif # GGML_CUDA_FA_ALL_QUANTS diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index fc5eac151b..12afe0f25a 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -151,6 +151,7 @@ set (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING "ggml: max. batch size for using peer access") option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copies" OFF) option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF) +option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON) option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF) option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT}) diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt index e63ede2fbe..96bd5a0be2 100644 --- a/ggml/src/ggml-cuda/CMakeLists.txt +++ b/ggml/src/ggml-cuda/CMakeLists.txt @@ -69,6 +69,10 @@ if (CUDAToolkit_FOUND) add_compile_definitions(GGML_CUDA_NO_VMM) endif() + if (NOT GGML_CUDA_FA) + add_compile_definitions(GGML_CUDA_NO_FA) + endif() + if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16) add_compile_definitions(GGML_CUDA_F16) endif() diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 7e99838c09..adf0d3ecb5 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -204,9 +204,9 @@ typedef float2 dfloat2; #define CP_ASYNC_AVAILABLE #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE -#if !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ <= GGML_CUDA_CC_QY1) +#if !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ <= GGML_CUDA_CC_QY1) #define FLASH_ATTN_AVAILABLE -#endif // !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ <= GGML_CUDA_CC_QY1) +#endif // !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ <= GGML_CUDA_CC_QY1) static bool fp16_available(const int cc) { return ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_PASCAL; diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index b2e0db9a2c..718ee5402d 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -839,10 +839,7 @@ static __global__ void flash_attn_ext_f16( const int ne1, const int ne2, const int ne3) { -#ifndef NEW_MMA_AVAILABLE - NO_DEVICE_CODE; - return; -#endif // NEW_MMA_AVAILABLE +#if defined(FLASH_ATTN_AVAILABLE) && defined(NEW_MMA_AVAILABLE) // Skip unused kernel variants for faster compilation: if (use_logit_softcap && !(D == 128 || D == 256)) { @@ -933,6 +930,9 @@ static __global__ void flash_attn_ext_f16( flash_attn_ext_f16_process_tile (Q_f2, K_h2, V_h2, mask_h2, dstk, dst_meta, scale, slope, logit_softcap, ne01, ne02, stride_Q1, stride_Q2, stride_KV, stride_mask, jt, kb0_start_kernel, kb0_stop_kernel); +#else + NO_DEVICE_CODE; +#endif // defined(FLASH_ATTN_AVAILABLE) && defined(NEW_MMA_AVAILABLE) } template diff --git a/ggml/src/ggml-cuda/fattn-tile-f16.cu b/ggml/src/ggml-cuda/fattn-tile-f16.cu index b8b415effb..ef3569fab2 100644 --- a/ggml/src/ggml-cuda/fattn-tile-f16.cu +++ b/ggml/src/ggml-cuda/fattn-tile-f16.cu @@ -44,12 +44,7 @@ static __global__ void flash_attn_tile_ext_f16( const int ne1, const int ne2, const int ne3) { -#ifdef FP16_AVAILABLE - -#ifndef FLASH_ATTN_AVAILABLE - NO_DEVICE_CODE; - return; -#endif // FLASH_ATTN_AVAILABLE +#if defined(FLASH_ATTN_AVAILABLE) && defined(FP16_AVAILABLE) // Skip unused kernel variants for faster compilation: #ifdef FP16_MMA_AVAILABLE @@ -290,7 +285,7 @@ static __global__ void flash_attn_tile_ext_f16( } #else NO_DEVICE_CODE; -#endif // FP16_AVAILABLE +#endif // defined(FLASH_ATTN_AVAILABLE) && defined(FP16_AVAILABLE) } template diff --git a/ggml/src/ggml-cuda/fattn-tile-f32.cu b/ggml/src/ggml-cuda/fattn-tile-f32.cu index 4352a28446..04b69c83be 100644 --- a/ggml/src/ggml-cuda/fattn-tile-f32.cu +++ b/ggml/src/ggml-cuda/fattn-tile-f32.cu @@ -44,10 +44,7 @@ static __global__ void flash_attn_tile_ext_f32( const int ne1, const int ne2, const int ne3) { -#ifndef FLASH_ATTN_AVAILABLE - NO_DEVICE_CODE; - return; -#endif // FLASH_ATTN_AVAILABLE +#ifdef FLASH_ATTN_AVAILABLE // Skip unused kernel variants for faster compilation: #ifdef FP16_MMA_AVAILABLE @@ -285,6 +282,9 @@ static __global__ void flash_attn_tile_ext_f32( dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); } } +#else + NO_DEVICE_CODE; +#endif // FLASH_ATTN_AVAILABLE } template diff --git a/ggml/src/ggml-cuda/fattn-vec-f16.cuh b/ggml/src/ggml-cuda/fattn-vec-f16.cuh index e758a0f6ec..b7686c1ec3 100644 --- a/ggml/src/ggml-cuda/fattn-vec-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-vec-f16.cuh @@ -41,12 +41,7 @@ static __global__ void flash_attn_vec_ext_f16( const int ne1, const int ne2, const int ne3) { -#ifdef FP16_AVAILABLE - -#ifndef FLASH_ATTN_AVAILABLE - NO_DEVICE_CODE; - return; -#endif // FLASH_ATTN_AVAILABLE +#if defined(FLASH_ATTN_AVAILABLE) && defined(FP16_AVAILABLE) // Skip unused kernel variants for faster compilation: if (use_logit_softcap && !(D == 128 || D == 256)) { @@ -300,7 +295,7 @@ static __global__ void flash_attn_vec_ext_f16( } #else NO_DEVICE_CODE; -#endif // FP16_AVAILABLE +#endif // defined(FLASH_ATTN_AVAILABLE) && defined(FP16_AVAILABLE) } template diff --git a/ggml/src/ggml-cuda/fattn-vec-f32.cuh b/ggml/src/ggml-cuda/fattn-vec-f32.cuh index 134144a383..c1d2dd8d19 100644 --- a/ggml/src/ggml-cuda/fattn-vec-f32.cuh +++ b/ggml/src/ggml-cuda/fattn-vec-f32.cuh @@ -41,10 +41,7 @@ static __global__ void flash_attn_vec_ext_f32( const int ne1, const int ne2, const int ne3) { -#ifndef FLASH_ATTN_AVAILABLE - NO_DEVICE_CODE; - return; -#endif // FLASH_ATTN_AVAILABLE +#ifdef FLASH_ATTN_AVAILABLE // Skip unused kernel variants for faster compilation: if (use_logit_softcap && !(D == 128 || D == 256)) { @@ -281,6 +278,9 @@ static __global__ void flash_attn_vec_ext_f32( if (parallel_blocks != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); } +#else + NO_DEVICE_CODE; +#endif // FLASH_ATTN_AVAILABLE } template diff --git a/ggml/src/ggml-cuda/fattn-wmma-f16.cu b/ggml/src/ggml-cuda/fattn-wmma-f16.cu index de38470abe..8828652fb5 100644 --- a/ggml/src/ggml-cuda/fattn-wmma-f16.cu +++ b/ggml/src/ggml-cuda/fattn-wmma-f16.cu @@ -51,7 +51,7 @@ static __global__ void flash_attn_ext_f16( const int ne1, const int ne2, const int ne3) { -#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#if defined(FLASH_ATTN_AVAILABLE) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA // Skip unused kernel variants for faster compilation: if (use_logit_softcap && !(D == 128 || D == 256)) { NO_DEVICE_CODE; @@ -425,7 +425,7 @@ static __global__ void flash_attn_ext_f16( } #else NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#endif // defined(FLASH_ATTN_AVAILABLE) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA } constexpr int get_max_power_of_2(int x) { diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index f685423215..ebb2ccae04 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3203,7 +3203,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_FLASH_ATTN_EXT: { #ifndef FLASH_ATTN_AVAILABLE return false; -#endif +#endif // FLASH_ATTN_AVAILABLE if (op->src[1]->type == GGML_TYPE_BF16 || op->src[2]->type == GGML_TYPE_BF16) { return false; } diff --git a/ggml/src/ggml-hip/CMakeLists.txt b/ggml/src/ggml-hip/CMakeLists.txt index f4a4683639..4a0384dd47 100644 --- a/ggml/src/ggml-hip/CMakeLists.txt +++ b/ggml/src/ggml-hip/CMakeLists.txt @@ -107,6 +107,10 @@ if (GGML_HIP_NO_VMM) add_compile_definitions(GGML_HIP_NO_VMM) endif() +if (NOT GGML_CUDA_FA) + add_compile_definitions(GGML_CUDA_NO_FA) +endif() + if (CXX_IS_HIPCC) set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE CXX) target_link_libraries(ggml-hip PRIVATE hip::device) diff --git a/ggml/src/ggml-musa/CMakeLists.txt b/ggml/src/ggml-musa/CMakeLists.txt index 1bfc07c5d7..2c75abf61d 100644 --- a/ggml/src/ggml-musa/CMakeLists.txt +++ b/ggml/src/ggml-musa/CMakeLists.txt @@ -83,6 +83,10 @@ if (MUSAToolkit_FOUND) add_compile_definitions(GGML_CUDA_NO_VMM) endif() + if (NOT GGML_CUDA_FA) + add_compile_definitions(GGML_CUDA_NO_FA) + endif() + if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16) add_compile_definitions(GGML_CUDA_F16) endif() From af7747c95ae71a5db4184947f837179e82c70b77 Mon Sep 17 00:00:00 2001 From: Aaron Teo <57927438+taronaeo@users.noreply.github.com> Date: Sun, 23 Feb 2025 05:39:24 +0800 Subject: [PATCH 009/188] ggml-cpu: Support s390x SIMD Instruction Set (#12019) * ggml: add s390x ARCH_FLAGS for compilation Signed-off-by: Aaron Teo * ggml: add SIMD for s390x using vector intrinsics SIMD is activated for: * ggml_vec_dot_f32 * ggml_vec_dot_f16 * ggml_vec_mad_f32 * ggml_vec_mad_f16 * ggml_vec_mad_f32_unroll * ggml_vec_scale_f32 * ggml_vec_scale_f16 SIMD is NOT activated for: * ggml_vec_dot_f16_unroll (pending bugfix) Signed-off-by: Aaron Teo * ggml: fix missing escape character in GGML_F32x4_REDUCE Signed-off-by: Aaron Teo * ggml: add temporary patch for GGML_F32_ARR and GGML_F16_ARR Signed-off-by: Aaron Teo * ggml: fix s390x GGML_F32x4_REDUCE Signed-off-by: Aaron Teo * ggml: full SIMD activation for F32,F16 s390x Signed-off-by: Aaron Teo * ggml: add option to disable s390x VXE/VXE2 Signed-off-by: Aaron Teo * ggml: change vecintrin.h include to ggml-cpu-impl * add __VXE__ and __VXE2__ macros Signed-off-by: Aaron Teo * cmake: add s390x target detection for VX/VXE/VXE2 Signed-off-by: Aaron Teo * ggml: move s390x vector intrinsics to ggml-cpu-impl.h Signed-off-by: Aaron Teo * ggml: s390x Q8_0 SIMD Signed-off-by: Aaron Teo * ggml: correct documentation for Q8_0 Signed-off-by: Aaron Teo * ggml: s390x reduce code complexity Q8_0 Signed-off-by: Aaron Teo * ggml: s390x bugfix typo Q8_0 Signed-off-by: Aaron Teo * ggml: s390x SIMD activated for Q4_1 Signed-off-by: Aaron Teo * ggml: s390x inline vec_reve Signed-off-by: Aaron Teo * ggml: s390x SIMD activation for Q4_0 Signed-off-by: Aaron Teo * ggml: add VXE backend feature Signed-off-by: Aaron Teo * ggml: remove test.py Signed-off-by: Aaron Teo * ggml: s390x SIMD activation for quantize_row_q8_0 Signed-off-by: Aaron Teo * ggml: s390x SIMD activation for quantize_row_q8_1 Signed-off-by: Aaron Teo * ggml: s390x SIMD activation for iq4_xs Signed-off-by: Aaron Teo * ggml: bugfix iq4_xs Signed-off-by: Aaron Teo * ggml: s390x SIMD activation for iq4_nl Signed-off-by: Aaron Teo * ggml: add float, double, and long vector data type Signed-off-by: Aaron Teo * ggml: clean up iq4_xs SIMD Signed-off-by: Aaron Teo * ggml: fix improper use of restrict keyword Signed-off-by: Aaron Teo * ggml: update warning message for ggml_vec_tbl Signed-off-by: Aaron Teo * ggml: untested implementation of ggml_vec_dot_iq2_xxs_q8_K Signed-off-by: Aaron Teo * ggml: update ggml_vec_dot_q4_1_q8_1 to use typedefs Signed-off-by: Aaron Teo * ggml: switch to restrict for iq4_nl Signed-off-by: Aaron Teo * ggml: slight dot product speed improvement for q4_1_q8_1 Signed-off-by: Aaron Teo * ggml: s390x SIMD activation for q6_K Signed-off-by: Aaron Teo * ggml: add missing `_t` to ggml_int8x16x4_t Signed-off-by: Aaron Teo * ggml: fix missing `_t` for ggml_vec_xl_s8x4 Signed-off-by: Aaron Teo * ggml: fix more missing `_t` Signed-off-by: Aaron Teo * ggml: add unroll and prefetch to Q8_0 increase of 3.86% for prompt processing and 32.22% for token generation Signed-off-by: Aaron Teo * ggml: patch Q8_0 to use proper vector sizes Signed-off-by: Aaron Teo * ggml: optimise Q8_0 dot prod compute kernel further Signed-off-by: Aaron Teo * ggml: add unroll and prefetch to Q4_1 Signed-off-by: Aaron Teo * ggml: refactor Q6_K variable naming for readability Signed-off-by: Aaron Teo * ggml: fix Q6_K typos Signed-off-by: Aaron Teo * ggml: s390x SIMD activation for Q5_K Signed-off-by: Aaron Teo * ggml: fix wrong char*x16_t naming Signed-off-by: Aaron Teo * ggml: Q5_K y0 wrong signness Signed-off-by: Aaron Teo * ggml: fix Q5_K invalid uchar type Signed-off-by: Aaron Teo * ggml: fix Q5_K invalid uchar type Signed-off-by: Aaron Teo * ggml: s390x SIMD activation for Q4_K Signed-off-by: Aaron Teo * ggml: fix Q4_K invalid vector intrinsics Signed-off-by: Aaron Teo * ggml: simplify ggml_padd_s16 compute kernel Signed-off-by: Aaron Teo * ggml: correct ggml-cpu vxe wording Signed-off-by: Aaron Teo * ggml: change ggml_aligned_malloc alignment to 256 256 is the cache line size for s390x platforms Signed-off-by: Aaron Teo * ggml: resolve pr merge via cherry-pick 225bbbf Signed-off-by: Aaron Teo * ggml : fix LoongArch compile error with 128-bit SIMD (#11701) * ggml: resolve pr merge via cherry-pick 4571953 Signed-off-by: Aaron Teo * ggml: cmake remove fork when determining s390x machine type thank you @ericcurtin Signed-off-by: Aaron Teo --------- Signed-off-by: Aaron Teo Co-authored-by: Jinyang He Co-authored-by: junchao-zhao <68935141+junchao-loongson@users.noreply.github.com> --- ggml/CMakeLists.txt | 1 + ggml/include/ggml-cpu.h | 1 + ggml/src/ggml-cpu/CMakeLists.txt | 21 ++ ggml/src/ggml-cpu/ggml-cpu-impl.h | 151 ++++++++ ggml/src/ggml-cpu/ggml-cpu-quants.c | 555 +++++++++++++++++++++++++++- ggml/src/ggml-cpu/ggml-cpu.c | 91 +++++ ggml/src/ggml-cpu/ggml-cpu.cpp | 3 + ggml/src/ggml.c | 4 + 8 files changed, 826 insertions(+), 1 deletion(-) diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 12afe0f25a..68b3f148ea 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -122,6 +122,7 @@ endif() option(GGML_LASX "ggml: enable lasx" ON) option(GGML_LSX "ggml: enable lsx" ON) option(GGML_RVV "ggml: enable rvv" ON) +option(GGML_VXE "ggml: enable vxe" ON) option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF) set(GGML_CPU_ARM_ARCH "" CACHE STRING "ggml: CPU architecture for ARM") diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.h index 9b8a697546..b48cc560e5 100644 --- a/ggml/include/ggml-cpu.h +++ b/ggml/include/ggml-cpu.h @@ -99,6 +99,7 @@ extern "C" { // other GGML_BACKEND_API int ggml_cpu_has_riscv_v (void); GGML_BACKEND_API int ggml_cpu_has_vsx (void); + GGML_BACKEND_API int ggml_cpu_has_vxe (void); GGML_BACKEND_API int ggml_cpu_has_wasm_simd (void); GGML_BACKEND_API int ggml_cpu_has_llamafile (void); diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 686ecb0a38..aa5ad5d8d9 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -310,6 +310,27 @@ function(ggml_add_cpu_backend_variant_impl tag_name) if (GGML_RVV) list(APPEND ARCH_FLAGS -march=rv64gcv -mabi=lp64d) endif() + elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") + message(STATUS "s390x detected") + file(READ "/proc/cpuinfo" CPUINFO_CONTENTS) + string(REGEX REPLACE "machine[ \t\r\n]*=[ \t\r\n]*([0-9]+)" "\\1" S390X_M ${CPUINFO_CONTENTS}) + + # TODO: Separation to determine activation of VX/VXE/VXE2 + if (${S390X_M} MATCHES "8561|8562") + message(STATUS "z15 target") + list(APPEND ARCH_FLAGS -march=z15 -mtune=z15) + elseif (${S390X_M} MATCHES "3931") + message(STATUS "z16 target") + list(APPEND ARCH_FLAGS -march=z16 -mtune=z16) + else() + message(STATUS "Unknown target") + message(WARNING "Unknown target. If you are compiling for z14 and earlier, you might have to add -DGGML_VXE=OFF.") + list(APPEND ARCH_FLAGS -march=native -mtune=native) + endif() + + if (GGML_VXE) + list(APPEND ARCH_FLAGS -mvx -mzvector) + endif() else() message(STATUS "Unknown architecture") endif() diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 9ddd972a5c..7f7d210cbe 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -59,6 +59,15 @@ struct ggml_compute_params { #endif #endif +#if defined(__s390x__) && defined(__VEC__) +#ifndef __VXE__ +#define __VXE__ +#endif +#ifndef __VXE2__ +#define __VXE2__ +#endif +#endif + #if defined(__ARM_FEATURE_SVE) #include #include @@ -359,6 +368,148 @@ inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) #endif #endif +#if defined(__VXE__) || defined(__VXE2__) +#include + +#define vec_neg(a) (-(a)) // Vector Negate +#define vec_add(a, b) ((a) + (b)) // Vector Add +#define vec_sub(a, b) ((a) - (b)) // Vector Subtract +#define vec_mul(a, b) ((a) * (b)) // Vector Multiply +#define vec_div(a, b) ((a) / (b)) // Vector Divide +#define vec_sl(a, b) ((a) << (b)) // Vector Shift Left +#define vec_sra(a, b) ((a) >> (b)) // Vector Shift Right +#define vec_sr(a, b) ((a) >> (b)) // Vector Shift Right Algebraic +#define vec_slo(a, b) vec_slb(a, (b) << 64) // Vector Shift Left by Octet +#define vec_sro(a, b) vec_srb(a, (b) << 64) // Vector Shift Right by Octet + +#ifndef vec_and +#define vec_and(a, b) ((a) & (b)) // Vector AND +#endif + +#ifndef vec_or +#define vec_or(a, b) ((a) | (b)) // Vector OR +#endif + +#ifndef vec_xor +#define vec_xor(a, b) ((a) ^ (b)) // Vector XOR +#endif + +typedef signed char char8x16_t __attribute__((vector_size(16))); +typedef unsigned char uchar8x16_t __attribute__((vector_size(16))); + +typedef int8_t int8x16_t __attribute__((vector_size(16))); +typedef int16_t int16x8_t __attribute__((vector_size(16))); +typedef int32_t int32x4_t __attribute__((vector_size(16))); + +typedef uint8_t uint8x16_t __attribute__((vector_size(16))); +typedef uint16_t uint16x8_t __attribute__((vector_size(16))); +typedef uint32_t uint32x4_t __attribute__((vector_size(16))); + +typedef float float32x4_t __attribute__((vector_size(16))); +typedef double double64x2_t __attribute((vector_size(16))); + +typedef signed long long long64x2_t __attribute((vector_size(16))); +typedef unsigned long long ulong64x2_t __attribute__((vector_size(16))); + +typedef struct ggml_uint8x16x2_t { + uint8x16_t val[2]; +} ggml_uint8x16x2_t; + +inline static ggml_uint8x16x2_t ggml_vec_xl_u8x2(const uint8_t * ptr) { + ggml_uint8x16x2_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + + return res; +} + +typedef struct ggml_uint8x16x4_t { + uint8x16_t val[4]; +} ggml_uint8x16x4_t; + +inline static ggml_uint8x16x4_t ggml_vec_xl_u8x4(const uint8_t * ptr) { + ggml_uint8x16x4_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + res.val[2] = vec_xl(32, ptr); + res.val[3] = vec_xl(48, ptr); + + return res; +} + +typedef struct ggml_int8x16x4_t { + int8x16_t val[4]; +} ggml_int8x16x4_t; + +inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) { + ggml_int8x16x4_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + res.val[2] = vec_xl(32, ptr); + res.val[3] = vec_xl(48, ptr); + + return res; +} + +typedef struct ggml_int16x8x2_t { + int16x8_t val[2]; +} ggml_int16x8x2_t; + +inline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) { + ggml_int16x8x2_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + + return res; +} + +/* + ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs + ! or iq4_nl for example implementation. +*/ +inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { + int8x16_t res; + + res[ 0] = a[b[ 0]]; + res[ 1] = a[b[ 1]]; + res[ 2] = a[b[ 2]]; + res[ 3] = a[b[ 3]]; + res[ 4] = a[b[ 4]]; + res[ 5] = a[b[ 5]]; + res[ 6] = a[b[ 6]]; + res[ 7] = a[b[ 7]]; + res[ 8] = a[b[ 8]]; + res[ 9] = a[b[ 9]]; + res[10] = a[b[10]]; + res[11] = a[b[11]]; + res[12] = a[b[12]]; + res[13] = a[b[13]]; + res[14] = a[b[14]]; + res[15] = a[b[15]]; + + return res; +} + +inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) { + const uchar8x16_t v_maske = { 0, 1, 4, 5, 8, 9, 12, 13, + 16, 17, 20, 21, 24, 25, 28, 29 }; + + const int16x8_t v_abo = vec_pack((int32x4_t)a, (int32x4_t)b); + const int16x8_t v_abe = vec_perm(a, b, v_maske); + return v_abo + v_abe; +} + +inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) { + const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b); + return acc + (vec_unpackh(p) + vec_unpackl(p)); +} + +#endif + #if defined(__loongarch_asx) /* float type data load instructions */ static __m128 __lsx_vreplfr2vr_s(const float val) { diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 14ba288fe1..d0c407bd6e 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -1011,6 +1011,38 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0); } +#elif defined(__VXE__) || defined(__VXE2__) + for (int i = 0; i < nb; i++) { + __vector float srcv [8]; + __vector float asrcv[8]; + __vector float amaxv[8]; + + for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j); + for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]); + for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]); + for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]); + for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]); + + const float amax = MAX(MAX(vec_extract(amaxv[0], 0), + vec_extract(amaxv[0], 1)), + MAX(vec_extract(amaxv[0], 2), + vec_extract(amaxv[0], 3))); + + const float d = amax / ((1 << 7) - 1); + const float id = d ? 1.0f / d : 0.0f; + + y[i].d = GGML_FP32_TO_FP16(d); + + for (int j = 0; j < 8; j++) { + const __vector float v = vec_mul(srcv[j], vec_splats(id)); + const __vector int32_t vi = vec_signed(v); + + y[i].qs[4*j + 0] = vec_extract(vi, 0); + y[i].qs[4*j + 1] = vec_extract(vi, 1); + y[i].qs[4*j + 2] = vec_extract(vi, 2); + y[i].qs[4*j + 3] = vec_extract(vi, 3); + } + } #else GGML_UNUSED(nb); // scalar @@ -1337,6 +1369,44 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k) __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0); __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0); } +#elif defined(__VXE__) || defined(__VXE2__) + for (int i = 0; i < nb; i++) { + __vector float srcv [8]; + __vector float asrcv[8]; + __vector float amaxv[8]; + + for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j); + for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]); + for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]); + for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]); + for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]); + + const float amax = MAX(MAX(vec_extract(amaxv[0], 0), + vec_extract(amaxv[0], 1)), + MAX(vec_extract(amaxv[0], 2), + vec_extract(amaxv[0], 3))); + + const float d = amax / ((1 << 7) - 1); + const float id = d ? 1.0f / d : 0.0f; + + y[i].d = GGML_FP32_TO_FP16(d); + + __vector int32_t acc = vec_splats(0); + + for (int j = 0; j < 8; j++) { + const __vector float v = vec_mul(srcv[j], vec_splats(id)); + const __vector int32_t vi = vec_signed(v); + + y[i].qs[4*j + 0] = vec_extract(vi, 0); + y[i].qs[4*j + 1] = vec_extract(vi, 1); + y[i].qs[4*j + 2] = vec_extract(vi, 2); + y[i].qs[4*j + 3] = vec_extract(vi, 3); + + acc = vec_add(acc, vi); + } + + y[i].s = GGML_FP32_TO_FP16(d * (acc[0] + acc[1] + acc[2] + acc[3])); + } #else GGML_UNUSED(nb); // scalar @@ -2488,6 +2558,37 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r } sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3); +#elif defined(__VXE__) || defined(__VXE2__) + __vector float acc = vec_splats(0.0f); + + const __vector uint8_t v_m = vec_splats((const uint8_t)0x0F); + const __vector int8_t v_s = vec_splats( (const int8_t)0x08); + + for (; ib < nb; ++ib) { + const __vector uint8_t v_x = vec_xl(0, x[ib].qs); + const __vector int8_t v_xl = (const __vector int8_t)(v_x & v_m); + const __vector int8_t v_xh = (const __vector int8_t)(v_x >> 4); + + const __vector int8_t v_xls = vec_sub(v_xl, v_s); + const __vector int8_t v_xhs = vec_sub(v_xh, v_s); + + const __vector int8_t v_yl = vec_xl(0 , y[ib].qs); + const __vector int8_t v_yh = vec_xl(QK8_0/2, y[ib].qs); + + const __vector int16_t v_xylso = vec_mulo(v_xls, v_yl); + const __vector int16_t v_xylse = vec_mule(v_xls, v_yl); + const __vector int16_t v_xyhso = vec_mulo(v_xhs, v_yh); + const __vector int16_t v_xyhse = vec_mule(v_xhs, v_yh); + + __vector int16_t v_xy_ = v_xylso + v_xylse + v_xyhso + v_xyhse; v_xy_ += vec_reve(v_xy_); + + const __vector float v_xy = vec_float(vec_unpackh(v_xy_)); + const __vector float v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); + + acc = vec_madd(v_xy, v_d, acc); + } + + sumf = acc[0] + acc[1] + acc[2] + acc[3]; #endif for (; ib < nb; ++ib) { int sumi0 = 0; @@ -2781,6 +2882,35 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r } sumf = hsum_float_8(acc) + summs; +#elif defined(__VXE__) || defined(__VXE2__) + float summs = 0; + float32x4_t acc = vec_splats(0.0f); + + const uint8x16_t v_m = vec_splat_u8(0x0F); + +#pragma GCC unroll 4 + for (; ib < nb; ++ib) { + __builtin_prefetch(x[ib].qs, 0, 1); + __builtin_prefetch(y[ib].qs, 0, 1); + + summs += GGML_FP16_TO_FP32(x[ib].m) * GGML_FP16_TO_FP32(y[ib].s); + + const uint8x16_t v_x = vec_xl(0, x[ib].qs); + const int8x16_t v_xl = (const int8x16_t)(v_x & v_m); + const int8x16_t v_xh = (const int8x16_t)(v_x >> 4); + + const int8x16_t v_yl = vec_xl(0 , y[ib].qs); + const int8x16_t v_yh = vec_xl(QK8_1/2, y[ib].qs); + + const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const float32x4_t v_xy = vec_float(v_xy_); + + const float32x4_t v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); + + acc = vec_madd(v_xy, v_d, acc); + } + + sumf = acc[0] + acc[1] + acc[2] + acc[3] + summs; #endif for (; ib < nb; ++ib) { int sumi0 = 0; @@ -3915,6 +4045,27 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r } sumf = hsum_float_8(acc); +#elif defined(__VXE__) || defined(__VXE2__) + __vector float acc = vec_splats(0.0f); + +#pragma GCC unroll 8 + for (; ib < nb; ++ib) { + __builtin_prefetch(x[ib].qs, 0, 1); + __builtin_prefetch(y[ib].qs, 0, 1); + + const int8x16_t v_xl = vec_xl(0 , x[ib].qs); + const int8x16_t v_xh = vec_xl(QK8_0/2, x[ib].qs); + const int8x16_t v_yl = vec_xl(0 , y[ib].qs); + const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs); + + const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const float32x4_t v_xy = vec_float(v_xy_); + const float32x4_t v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); + + acc = vec_madd(v_xy, v_d, acc); + } + + sumf = acc[0] + acc[1] + acc[2] + acc[3]; #endif for (; ib < nb; ++ib) { int sumi = 0; @@ -6797,6 +6948,77 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = hsum_float_8(acc) + ((v4f32)acc_m)[0]; +#elif defined(__VXE__) || defined(__VXE2__) + const uint8x16_t v_lm = vec_splat_u8(0x0F); + const int32x4_t v_z = vec_splat_s32(0); + + uint8x16_t v_x[2]; + int8x16_t v_xl[2]; + int8x16_t v_y[2]; + + float sumf = 0; + + for (int i = 0; i < nb; ++i) { + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + + const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums); + const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums); + const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh); + + memcpy(utmp, x[i].scales, 12); + + uint32x4_t v_mins8 = { 0 }; + v_mins8 = vec_insert(utmp[1] & kmask1, v_mins8, 0); + v_mins8 = vec_insert(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), v_mins8, 1); + + utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); + utmp[0] &= kmask1; + + const int16x8_t v_minsh = (int16x8_t)vec_unpackh((uint8x16_t)v_mins8); + + const int32x4_t v_minso = vec_mulo(v_ysums, v_minsh); + const int32x4_t v_minse = vec_mule(v_ysums, v_minsh); + const int32x4_t v_mins = v_minso + v_minse; + sumf -= dmin * (v_mins[0] + v_mins[1] + v_mins[2] + v_mins[3]); + + const uint8_t * scales = (const uint8_t *)utmp; + const uint8_t * restrict x0 = x[i].qs; + const int8_t * restrict y0 = y[i].qs; + + int32_t sumi1 = 0; + int32_t sumi2 = 0; + + for (int j = 0; j < QK_K/64; ++j) { + v_x[0] = vec_xl(0 , x0); + v_x[1] = vec_xl(16, x0); + x0 += 32; + + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + y0 += 32; + + v_xl[0] = (int8x16_t)vec_and(v_x[0], v_lm); + v_xl[1] = (int8x16_t)vec_and(v_x[1], v_lm); + + const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); + sumi1 += (p1[0] + p1[1] + p1[2] + p1[3]) * scales[2*j+0]; + + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + y0 += 32; + + v_xl[0] = (int8x16_t)vec_sr(v_x[0], 4); + v_xl[1] = (int8x16_t)vec_sr(v_x[1], 4); + + const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); + sumi2 += (p2[0] + p2[1] + p2[2] + p2[3]) * scales[2*j+1]; + } + + sumf += d * (sumi1 + sumi2); + } + + *s = sumf; #else const uint8_t * scales = (const uint8_t*)&utmp[0]; @@ -7526,7 +7748,94 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vbsrl_v(acc_m, 4)); *s = hsum_float_8(acc) + ((v4f32)acc_m)[0]; +#elif defined(__VXE__) || defined(__VXE2__) + const uint8x16_t v_lm = vec_splat_u8(0x0F); + const uint8x16_t v_1m = vec_splat_u8(0x01); + const uint8x16_t v_2m = vec_splat_u8(0x02); + const int32x4_t v_z = vec_splat_s32(0); + + const uchar8x16_t v_minsm = { + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + }; + + int8x16_t q5b[4]; + uint8x16_t q5h[4]; + + uint8x16_t v_xl[2]; + uint8x16_t v_xh[2]; + int8x16_t v_y[4]; + + float sumf = 0; + + for (int i = 0; i < nb; ++i) { + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + + const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums); + const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums); + const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh); + + memcpy(utmp, x[i].scales, 12); + utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4); + const uint32_t uaux = utmp[1] & kmask1; + utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); + utmp[2] = uaux; + utmp[0] &= kmask1; + + const uint8x16_t v_mins16 = vec_xl(0, (const uint8_t *)utmp); + const uint8x16_t v_mins8 = vec_perm(v_mins16, v_mins16, v_minsm); + const int16x8_t v_minsh = (int16x8_t)vec_unpackh(v_mins8); + + const int32x4_t v_minsho = vec_mulo(v_ysums, v_minsh); + const int32x4_t v_minshe = vec_mule(v_ysums, v_minsh); + const int32x4_t v_mins = vec_add(v_minsho, v_minshe); + const int32_t mins = v_mins[0] + v_mins[1] + v_mins[2] + v_mins[3]; + + const uint8_t * scales = (const uint8_t *)utmp; + const uint8_t * restrict x0l = x[i].qs; + const uint8_t * restrict x0h = x[i].qh; + const int8_t * restrict y0 = y[i].qs; + + v_xh[0] = vec_xl(0 , x0h); + v_xh[1] = vec_xl(16, x0h); + + int32_t sumi = 0; + for (int j = 0; j < QK_K/64; ++j) { + v_xl[0] = vec_xl(0 , x0l); + v_xl[1] = vec_xl(16, x0l); + x0l += 32; + + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + v_y[2] = vec_xl(32, y0); + v_y[3] = vec_xl(48, y0); + y0 += 64; + + q5h[0] = vec_sl(vec_and(v_1m, v_xh[0]), 4); + q5h[1] = vec_sl(vec_and(v_1m, v_xh[1]), 4); + q5h[2] = vec_sl(vec_and(v_2m, v_xh[0]), 3); + q5h[3] = vec_sl(vec_and(v_2m, v_xh[1]), 3); + v_xh[0] = vec_sr(v_xh[0], 2); + v_xh[1] = vec_sr(v_xh[1], 2); + + q5b[0] = (int8x16_t)vec_or(vec_and(v_xl[0], v_lm), q5h[0]); + q5b[1] = (int8x16_t)vec_or(vec_and(v_xl[1], v_lm), q5h[1]); + q5b[2] = (int8x16_t)vec_or(vec_sr(v_xl[0], 4), q5h[2]); + q5b[3] = (int8x16_t)vec_or(vec_sr(v_xl[1], 4), q5h[3]); + + int32x4_t sumi0 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]); + int32x4_t sumi1 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]); + + sumi += (sumi0[0] + sumi0[1] + sumi0[2] + sumi0[3]) * *scales++; + sumi += (sumi1[0] + sumi1[1] + sumi1[2] + sumi1[3]) * *scales++; + } + + sumf += d * sumi - dmin * mins; + } + + *s = sumf; #else const uint8_t * scales = (const uint8_t*)&utmp[0]; @@ -8243,7 +8552,130 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r } *s = hsum_float_8(acc); +#elif defined(__VXE__) || defined(__VXE2__) + float sum = 0; + // Lower 4-bit and upper 2-bit masks + const uint8x16_t v_lm = vec_splat_u8(0x0F); + const uint8x16_t v_um = vec_splat_u8(0x03); + + const int32x4_t v_z = vec_splat_s32(0); + + int8x16_t q6b[4]; + uint8x16_t q6h[4]; + + uint8x16_t v_xl[4]; + uint8x16_t v_xh[2]; + int8x16_t v_y[4]; + + for (int i = 0; i < nb; ++i) { + const float d_all = GGML_FP16_TO_FP32(x[i].d); + + const uint8_t * restrict x0l = x[i].ql; + const uint8_t * restrict x0h = x[i].qh; + const int8_t * restrict y0 = y[i].qs; + + const int8_t * restrict scale = x[i].scales; + + const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums); + const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums); + + const int8x16_t v_scale = vec_xl(0, scale); + const int16x8_t v_scalel = vec_unpackh(v_scale); + const int16x8_t v_scaleh = vec_unpackl(v_scale); + + const int32x4_t v_minslo = vec_mulo(v_ysumsl, v_scalel); + const int32x4_t v_minsle = vec_mule(v_ysumsl, v_scalel); + const int32x4_t v_minsho = vec_mulo(v_ysumsh, v_scaleh); + const int32x4_t v_minshe = vec_mule(v_ysumsh, v_scaleh); + const int32x4_t v_mins = v_minslo + v_minsle + v_minsho + v_minshe; + + const int32_t mins = v_mins[0] + v_mins[1] + v_mins[2] + v_mins[3]; + + int32_t isum = 0; + for (int j = 0; j < QK_K/128; ++j) { + // Load model upper 2 bits + v_xh[0] = vec_xl(0 , x0h); + v_xh[1] = vec_xl(16, x0h); + x0h += 32; + + // Load model lower 4 bits + v_xl[0] = vec_xl(0 , x0l); + v_xl[1] = vec_xl(16, x0l); + v_xl[2] = vec_xl(32, x0l); + v_xl[3] = vec_xl(48, x0l); + x0l += 64; + + // Load activation quants + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + v_y[2] = vec_xl(32, y0); + v_y[3] = vec_xl(48, y0); + y0 += 64; + + q6h[0] = vec_sl(vec_and(v_um, v_xh[0]), 4); + q6h[1] = vec_sl(vec_and(v_um, v_xh[1]), 4); + uint8x16_t shifted = vec_sr(v_xh[0], 2); + q6h[2] = vec_sl(vec_and(v_um, shifted), 4); + shifted = vec_sr(v_xh[1], 2); + q6h[3] = vec_sl(vec_and(v_um, shifted), 4); + + q6b[0] = (int8x16_t)(vec_or(vec_and(v_xl[0], v_lm), q6h[0])); + q6b[1] = (int8x16_t)(vec_or(vec_and(v_xl[1], v_lm), q6h[1])); + q6b[2] = (int8x16_t)(vec_or(vec_and(v_xl[2], v_lm), q6h[2])); + q6b[3] = (int8x16_t)(vec_or(vec_and(v_xl[3], v_lm), q6h[3])); + + int32x4_t summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]); + int32x4_t summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]); + int32x4_t summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]); + int32x4_t summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]); + + isum += (summs0[0] + summs0[1] + summs0[2] + summs0[3]) * scale[0] + + (summs1[0] + summs1[1] + summs1[2] + summs1[3]) * scale[1] + + (summs2[0] + summs2[1] + summs2[2] + summs2[3]) * scale[2] + + (summs3[0] + summs3[1] + summs3[2] + summs3[3]) * scale[3]; + + scale += 4; + + + // Load activation quants + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + v_y[2] = vec_xl(32, y0); + v_y[3] = vec_xl(48, y0); + y0 += 64; + + shifted = vec_sr(v_xh[0], 4); + q6h[0] = vec_sl(vec_and(v_um, shifted), 4); + shifted = vec_sr(v_xh[1], 4); + q6h[1] = vec_sl(vec_and(v_um, shifted), 4); + shifted = vec_sr(v_xh[0], 6); + q6h[2] = vec_sl(vec_and(v_um, shifted), 4); + shifted = vec_sr(v_xh[1], 6); + q6h[3] = vec_sl(vec_and(v_um, shifted), 4); + + q6b[0] = (int8x16_t)(vec_or(vec_sr(v_xl[0], 4), q6h[0])); + q6b[1] = (int8x16_t)(vec_or(vec_sr(v_xl[1], 4), q6h[1])); + q6b[2] = (int8x16_t)(vec_or(vec_sr(v_xl[2], 4), q6h[2])); + q6b[3] = (int8x16_t)(vec_or(vec_sr(v_xl[3], 4), q6h[3])); + + summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]); + summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]); + summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]); + summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]); + + isum += (summs0[0] + summs0[1] + summs0[2] + summs0[3]) * scale[0] + + (summs1[0] + summs1[1] + summs1[2] + summs1[3]) * scale[1] + + (summs2[0] + summs2[1] + summs2[2] + summs2[3]) * scale[2] + + (summs3[0] + summs3[1] + summs3[2] + summs3[3]) * scale[3]; + + scale += 4; + } + + sum += d_all * y[i].d * (isum - 32 * mins); + } + + *s = sum; #else int8_t aux8[QK_K]; @@ -8604,7 +9036,57 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, size_t bs, const void } *s = 0.125f * hsum_float_8(accumf); - +//#elif defined(__VXE__) || defined(__VXE2__) +// const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; +// +// uint32_t aux32[4]; +// const uint8_t * aux8 = (const uint8_t *)aux32; +// +// float sumf = 0; +// +// for (int i = 0; i < nb; ++i) { +// const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; +// const uint16_t * restrict q2 = x[i].qs; +// const int8_t * restrict q8 = y[i].qs; +// +// float sumf1 = 0, sumf2 = 0; +// +// for (int ib32 = 0; ib32 < QK_K/32; ib += 2) { +// int8x16_t q8b0 = vec_xl( 0, q8); +// int8x16_t qb81 = vec_xl(16, q8); +// int8x16_t q8b2 = vec_xl(32, q8); +// int8x16_t q8b3 = vec_xl(48, q8); +// q8 += 64; +// +// memcpy(aux32, q2, 4 * sizeof(uint32_t)); +// q2 += 8; +// +// int8x16_t q2u0 = { *(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1]) }; +// int8x16_t q2u1 = { *(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3]) }; +// int8x16_t q2u2 = { *(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9]) }; +// int8x16_t q2u3 = { *(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11]) }; +// +// int8x16_t q2s0 = { *(const int64_t *)(signs64 + ((aux32[1] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 7) & 127)) }; +// int8x16_t q2s1 = { *(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127)) }; +// int8x16_t q2s2 = { *(const int64_t *)(signs64 + ((aux32[3] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 7) & 127)) }; +// int8x16_t q2s3 = { *(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127)) }; +// +// q2u0 = vec_mul(q2u0, q2s0); +// q2u1 = vec_mul(q2u1, q2s1); +// q2u2 = vec_mul(q2u2, q2s2); +// q2u3 = vec_mul(q2u3, q2s3); +// +// const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1); +// const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3); +// +// sumf1 += (p1[0] + p1[1] + p1[2] + p1[3]) * (0.5f + (aux32[1] >> 28)); +// sumf2 += (p2[0] + p2[1] + p2[2] + p2[3]) * (0.5f + (aux32[3] >> 28)); +// } +// +// sumf += d * (sumf1 + sumf2); +// } +// +// *s = 0.25f * sumf; #else uint32_t aux32[2]; @@ -11365,6 +11847,27 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * sumf = hsum_float_8(__lasx_xvfadd_s(accum1, accum2)); +#elif defined(__VXE__) || defined(__VXE2__) + const int8x16_t v_k = vec_xl(0, kvalues_iq4nl); + const uint8x16_t v_m = vec_splat_u8(0x0F); + + for (; ib < nb; ++ib) { + const block_iq4_nl * restrict x0 = &x[ib]; + const block_q8_0 * restrict y0 = &y[ib]; + + const uint8x16_t v_x = vec_xl(0, x0->qs); + int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m); + int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4); + + v_xl = vec_perm(v_k, v_k, (uchar8x16_t)v_xl); + v_xh = vec_perm(v_k, v_k, (uchar8x16_t)v_xh); + + const int8x16_t v_yl = vec_xl(0 , y0->qs); + const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs); + const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + + sumf += GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d) * (v_xy[0] + v_xy[1] + v_xy[2] + v_xy[3]); + } #endif for (; ib < nb; ++ib) { const float d = GGML_FP16_TO_FP32(y[ib].d)*GGML_FP16_TO_FP32(x[ib].d); @@ -11643,6 +12146,56 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * } *s = hsum_float_8(accum); +#elif defined(__VXE__) || defined(__VXE2__) + const int8x16_t v_k = vec_xl(0, kvalues_iq4nl); + const uint8x16_t v_m = vec_splat_u8(0x0F); + + float sumf = 0; + + for (int ibl = 0; ibl < nb; ++ibl) { + const uint8_t * restrict q4 = x[ibl].qs; + const int8_t * restrict q8 = y[ibl].qs; + + uint16_t h = x[ibl].scales_h; + + int sumi1 = 0, sumi2 = 0; + for (int ib = 0; ib < QK_K/64; ++ib) { + const uint8x16_t v_x0 = vec_xl(0 , q4); + const uint8x16_t v_x1 = vec_xl(QK4_NL/2, q4); + q4 += 32; + + int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m); + int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4); + int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m); + int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4); + + v_x0l = vec_perm(v_k, v_k, (uchar8x16_t)v_x0l); + v_x0h = vec_perm(v_k, v_k, (uchar8x16_t)v_x0h); + v_x1l = vec_perm(v_k, v_k, (uchar8x16_t)v_x1l); + v_x1h = vec_perm(v_k, v_k, (uchar8x16_t)v_x1h); + + const int8x16_t v_y0 = vec_xl( 0, q8); + const int8x16_t v_y1 = vec_xl(16, q8); + const int8x16_t v_y2 = vec_xl(32, q8); + const int8x16_t v_y3 = vec_xl(48, q8); + q8 += 64; + + int32x4_t vsumi0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1); + int32x4_t vsumi1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3); + + int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32; + int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32; + + h >>= 4; + + sumi1 += (vsumi0[0] + vsumi0[1] + vsumi0[2] + vsumi0[3]) * ls1; + sumi2 += (vsumi1[0] + vsumi1[1] + vsumi1[2] + vsumi1[3]) * ls2; + } + + sumf += GGML_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2); + } + + *s = sumf; #else float sumf = 0; diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index f27b981715..723253495a 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -237,6 +237,8 @@ typedef pthread_t ggml_thread_t; #else #if defined(__POWER9_VECTOR__) #define CACHE_LINE_SIZE 128 +#elif defined(__VXE__) || defined(__VXE2__) +#define CACHE_LINE_SIZE 256 #else #define CACHE_LINE_SIZE 64 #endif @@ -1211,6 +1213,87 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE +#elif defined(__VXE__) || defined(__VXE2__) + +#define GGML_SIMD + +// F32 s390x + +#define GGML_F32_STEP 32 +#define GGML_F32_EPR 4 + +#define GGML_F32x4 __vector float +#define GGML_F32x4_ZERO vec_splats(0.0f) +#define GGML_F32x4_SET1 vec_splats +#define GGML_F32x4_LOAD(p) vec_xl(0, p) +#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p) +#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a) +#define GGML_F32x4_ADD vec_add +#define GGML_F32x4_MUL vec_mul +#define GGML_F32x4_REDUCE(res, x) \ +{ \ + int offset = GGML_F32_ARR >> 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = vec_add(x[i], x[offset + i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = vec_add(x[i], x[offset + i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = vec_add(x[i], x[offset + i]); \ + } \ + res = vec_extract(x[0], 0) + \ + vec_extract(x[0], 1) + \ + vec_extract(x[0], 2) + \ + vec_extract(x[0], 3); \ +} + +#define GGML_F32_VEC GGML_F32x4 +#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO +#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 +#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD +#define GGML_F32_VEC_STORE GGML_F32x4_STORE +#define GGML_F32_VEC_FMA GGML_F32x4_FMA +#define GGML_F32_VEC_ADD GGML_F32x4_ADD +#define GGML_F32_VEC_MUL GGML_F32x4_MUL +#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE + +// F16 s390x +#define GGML_F16_STEP GGML_F32_STEP +#define GGML_F16_EPR GGML_F32_EPR + +static inline __vector float __lzs_f16cx4_load(const ggml_fp16_t * x) { + float tmp[4]; + + for (int i = 0; i < 4; i++) { + tmp[i] = GGML_FP16_TO_FP32(x[i]); + } + + return vec_xl(0, tmp); +} + +static inline void __lzs_f16cx4_store(ggml_fp16_t * x, __vector float y) { + float arr[4]; + + vec_xst(y, 0, arr); + + for (int i = 0; i < 4; i++) { + x[i] = GGML_FP32_TO_FP16(arr[i]); + } +} + +#define GGML_F16_VEC GGML_F32x4 +#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO +#define GGML_F16_VEC_SET1 GGML_F32x4_SET1 +#define GGML_F16_VEC_LOAD(p, i) __lzs_f16cx4_load(p) +#define GGML_F16_VEC_STORE(p, r, i) __lzs_f16cx4_store(p, r[i]) +#define GGML_F16_VEC_FMA GGML_F32x4_FMA +#define GGML_F16_VEC_ADD GGML_F32x4_ADD +#define GGML_F16_VEC_MUL GGML_F32x4_MUL +#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE + #endif // GGML_F32_ARR / GGML_F16_ARR @@ -14419,6 +14502,14 @@ int ggml_cpu_has_vsx(void) { #endif } +int ggml_cpu_has_vxe(void) { +#if defined(__VXE__) || defined(__VXE2__) + return 1; +#else + return 0; +#endif +} + int ggml_cpu_has_neon(void) { #if defined(__ARM_ARCH) && defined(__ARM_NEON) return ggml_arm_arch_features.has_neon; diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index d0ae10ee37..a84203f29f 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -557,6 +557,9 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r if (ggml_cpu_has_vsx()) { features.push_back({ "VSX", "1" }); } + if (ggml_cpu_has_vxe()) { + features.push_back({ "VXE", "1" }); + } if (ggml_cpu_has_wasm_simd()) { features.push_back({ "WASM_SIMD", "1" }); } diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index e9f3420c29..7fc06724eb 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -240,7 +240,11 @@ void ggml_log_callback_default(enum ggml_log_level level, const char * text, voi void * ggml_aligned_malloc(size_t size) { +#if defined(__s390x__) + const int alignment = 256; +#else const int alignment = 64; +#endif #if defined(_MSC_VER) || defined(__MINGW32__) return _aligned_malloc(size, alignment); From f777a73e18c218848bca0748581c043987348a5d Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Sun, 23 Feb 2025 13:14:32 +0000 Subject: [PATCH 010/188] Some llama-run cleanups (#11973) Use consolidated open function call from File class. Change read_all to to_string(). Remove exclusive locking, the intent for that lock is to avoid multiple processes writing to the same file, it's not an issue for readers, although we may want to consider adding a shared lock. Remove passing nullptr as reference, references are never supposed to be null. clang-format the code for consistent styling. Signed-off-by: Eric Curtin --- examples/run/run.cpp | 91 ++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/examples/run/run.cpp b/examples/run/run.cpp index 4da1e50251..de736c7d5a 100644 --- a/examples/run/run.cpp +++ b/examples/run/run.cpp @@ -323,25 +323,17 @@ class File { return 0; } - std::string read_all(const std::string & filename){ - open(filename, "r"); - lock(); - if (!file) { - printe("Error opening file '%s': %s", filename.c_str(), strerror(errno)); - return ""; - } - + std::string to_string() { fseek(file, 0, SEEK_END); - size_t size = ftell(file); + const size_t size = ftell(file); fseek(file, 0, SEEK_SET); - std::string out; out.resize(size); - size_t read_size = fread(&out[0], 1, size, file); + const size_t read_size = fread(&out[0], 1, size, file); if (read_size != size) { - printe("Error reading file '%s': %s", filename.c_str(), strerror(errno)); - return ""; + printe("Error reading file: %s", strerror(errno)); } + return out; } @@ -1098,59 +1090,66 @@ static int get_user_input(std::string & user_input, const std::string & user) { // Reads a chat template file to be used static std::string read_chat_template_file(const std::string & chat_template_file) { - if(chat_template_file.empty()){ - return ""; - } - File file; - std::string chat_template = ""; - chat_template = file.read_all(chat_template_file); - if(chat_template.empty()){ + if (!file.open(chat_template_file, "r")) { printe("Error opening chat template file '%s': %s", chat_template_file.c_str(), strerror(errno)); return ""; } - return chat_template; + + return file.to_string(); +} + +static int process_user_message(const Opt & opt, const std::string & user_input, LlamaData & llama_data, + const common_chat_templates_ptr & chat_templates, int & prev_len, + const bool stdout_a_terminal) { + add_message("user", opt.user.empty() ? user_input : opt.user, llama_data); + int new_len; + if (apply_chat_template_with_error_handling(chat_templates.get(), llama_data, true, new_len, opt.use_jinja) < 0) { + return 1; + } + + std::string prompt(llama_data.fmtted.begin() + prev_len, llama_data.fmtted.begin() + new_len); + std::string response; + if (generate_response(llama_data, prompt, response, stdout_a_terminal)) { + return 1; + } + + if (!opt.user.empty()) { + return 2; + } + + add_message("assistant", response, llama_data); + if (apply_chat_template_with_error_handling(chat_templates.get(), llama_data, false, prev_len, opt.use_jinja) < 0) { + return 1; + } + + return 0; } // Main chat loop function -static int chat_loop(LlamaData & llama_data, const std::string & user, const std::string & chat_template_file, bool use_jinja) { +static int chat_loop(LlamaData & llama_data, const Opt & opt) { int prev_len = 0; llama_data.fmtted.resize(llama_n_ctx(llama_data.context.get())); - - std::string chat_template = ""; - if(!chat_template_file.empty()){ - chat_template = read_chat_template_file(chat_template_file); + std::string chat_template; + if (!opt.chat_template_file.empty()) { + chat_template = read_chat_template_file(opt.chat_template_file); } - auto chat_templates = common_chat_templates_init(llama_data.model.get(), chat_template.empty() ? nullptr : chat_template); + common_chat_templates_ptr chat_templates = common_chat_templates_init(llama_data.model.get(), chat_template); static const bool stdout_a_terminal = is_stdout_a_terminal(); while (true) { // Get user input std::string user_input; - if (get_user_input(user_input, user) == 1) { + if (get_user_input(user_input, opt.user) == 1) { return 0; } - add_message("user", user.empty() ? user_input : user, llama_data); - int new_len; - if (apply_chat_template_with_error_handling(chat_templates.get(), llama_data, true, new_len, use_jinja) < 0) { + const int ret = process_user_message(opt, user_input, llama_data, chat_templates, prev_len, stdout_a_terminal); + if (ret == 1) { return 1; - } - - std::string prompt(llama_data.fmtted.begin() + prev_len, llama_data.fmtted.begin() + new_len); - std::string response; - if (generate_response(llama_data, prompt, response, stdout_a_terminal)) { - return 1; - } - - if (!user.empty()) { + } else if (ret == 2) { break; } - - add_message("assistant", response, llama_data); - if (apply_chat_template_with_error_handling(chat_templates.get(), llama_data, false, prev_len, use_jinja) < 0) { - return 1; - } } return 0; @@ -1208,7 +1207,7 @@ int main(int argc, const char ** argv) { return 1; } - if (chat_loop(llama_data, opt.user, opt.chat_template_file, opt.use_jinja)) { + if (chat_loop(llama_data, opt)) { return 1; } From 7ad0779f5de84a68143b2c00ab5dc94a948925d3 Mon Sep 17 00:00:00 2001 From: Florent BENOIT Date: Sun, 23 Feb 2025 18:15:51 +0100 Subject: [PATCH 011/188] run: allow to customize prompt by env var LLAMA_PROMPT_PREFIX (#12041) Signed-off-by: Florent Benoit --- examples/run/run.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/run/run.cpp b/examples/run/run.cpp index de736c7d5a..38407d5190 100644 --- a/examples/run/run.cpp +++ b/examples/run/run.cpp @@ -977,7 +977,8 @@ static int generate(LlamaData & llama_data, const std::string & prompt, std::str } static int read_user_input(std::string & user_input) { - static const char * prompt_prefix = "> "; + static const char * prompt_prefix_env = std::getenv("LLAMA_PROMPT_PREFIX"); + static const char * prompt_prefix = prompt_prefix_env ? prompt_prefix_env : "> "; #ifdef WIN32 printf("\r" LOG_CLR_TO_EOL LOG_COL_DEFAULT "%s", prompt_prefix); From 8303e8b0fb2c1e26a8edd58071f9120a5bd6930a Mon Sep 17 00:00:00 2001 From: Akarshan Biswas Date: Mon, 24 Feb 2025 15:48:25 +0530 Subject: [PATCH 012/188] SYCL: Fix GGML_SYCL_DEBUG macro (#11995) --- ggml/src/ggml-sycl/common.hpp | 2 +- ggml/src/ggml-sycl/ggml-sycl.cpp | 3 ++- ggml/src/ggml-sycl/softmax.cpp | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index abad847ca8..a5cab5065f 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -35,7 +35,7 @@ void* ggml_sycl_host_malloc(size_t size); void ggml_sycl_host_free(void* ptr); -static int g_ggml_sycl_debug = 0; +extern int g_ggml_sycl_debug; #define GGML_SYCL_DEBUG(...) \ do { \ if (g_ggml_sycl_debug) \ diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 3d24d21654..d4c97ad17b 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -41,6 +41,7 @@ #include "ggml-sycl/gemm.hpp" static bool g_sycl_loaded = false; +int g_ggml_sycl_debug = 0; static ggml_sycl_device_info ggml_sycl_init() { ggml_sycl_device_info info = {}; @@ -157,8 +158,8 @@ static void ggml_check_sycl() try { static bool initialized = false; if (!initialized) { - GGML_SYCL_DEBUG("[SYCL] call ggml_check_sycl\n"); g_ggml_sycl_debug = get_sycl_env("GGML_SYCL_DEBUG", 0); + GGML_SYCL_DEBUG("[SYCL] call ggml_check_sycl\n"); GGML_LOG_INFO("GGML_SYCL_DEBUG: %d\n", g_ggml_sycl_debug); #if defined(GGML_SYCL_FORCE_MMQ) GGML_LOG_INFO("GGML_SYCL_FORCE_MMQ: yes\n"); diff --git a/ggml/src/ggml-sycl/softmax.cpp b/ggml/src/ggml-sycl/softmax.cpp index 563e0655f5..eb20bd251e 100644 --- a/ggml/src/ggml-sycl/softmax.cpp +++ b/ggml/src/ggml-sycl/softmax.cpp @@ -249,13 +249,16 @@ void ggml_sycl_op_soft_max(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { if (dst->src[1] && dst->src[1]->type == GGML_TYPE_F16) { const sycl::half * src1_dd = static_cast(dst->src[1]->data); + GGML_SYCL_DEBUG("%s: F16 mask\n", __func__); soft_max_f32_sycl(src0_dd, src1_dd, dst_dd, ne00, nrows_x, nrows_y, scale, max_bias, main_stream, ctx.device); } else if (dst->src[1] && dst->src[1]->type == GGML_TYPE_F32) { const float * src1_dd = static_cast(dst->src[1]->data); + GGML_SYCL_DEBUG("%s: F32 mask\n", __func__); soft_max_f32_sycl(src0_dd, src1_dd, dst_dd, ne00, nrows_x, nrows_y, scale, max_bias, main_stream, ctx.device); } else { /* mask unavailable */ + GGML_SYCL_DEBUG("%s: No mask\n", __func__); soft_max_f32_sycl(src0_dd, nullptr, dst_dd, ne00, nrows_x, nrows_y, scale, max_bias, main_stream, ctx.device); } } From 651adf4b6675339465179b81b194d98cc14704d6 Mon Sep 17 00:00:00 2001 From: Aleksei Nikiforov <103434461+AlekseiNikiforovIBM@users.noreply.github.com> Date: Mon, 24 Feb 2025 12:27:01 +0100 Subject: [PATCH 013/188] gguf_convert_endian.py: implement byteswapping for q4_k and q6_k (#11349) --- gguf-py/gguf/scripts/gguf_convert_endian.py | 55 +++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/gguf-py/gguf/scripts/gguf_convert_endian.py b/gguf-py/gguf/scripts/gguf_convert_endian.py index f97e91bd4c..837831799b 100755 --- a/gguf-py/gguf/scripts/gguf_convert_endian.py +++ b/gguf-py/gguf/scripts/gguf_convert_endian.py @@ -43,6 +43,8 @@ def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16, gguf.GGMLQuantizationType.Q8_0, + gguf.GGMLQuantizationType.Q4_K, + gguf.GGMLQuantizationType.Q6_K, ): raise ValueError(f"Cannot handle type {tensor.tensor_type.name} for tensor {repr(tensor.name)}") logger.info(f"* Preparing to convert from {file_endian.upper()} to {order.upper()}") @@ -96,6 +98,59 @@ def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None if block_num % 100000 == 0: inner_pbar.set_description(f"Byte-swapping Blocks [{(n_blocks - block_num) // n_blocks}]") + elif tensor.tensor_type == gguf.GGMLQuantizationType.Q4_K: + # Handle Q4_K tensor blocks (block_q4_k) + # Specific handling of block_q4_k is required. + # Each block_q4_k consists of 2 f16 values followed by 140 int8 values. + + # first flatten structure + newshape = 1 + for i in tensor.data.shape: + newshape *= i + + tensor.data.resize(newshape) + + block_size = 144 + n_blocks = len(tensor.data) // block_size + for block_num in (inner_pbar := tqdm(range(n_blocks), desc="Byte-swapping Blocks", leave=False)): + block_offs = block_num * block_size + + # Byte-Swap f16 sized fields + delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16) + delta.byteswap(inplace=True) + + delta = tensor.data[block_offs + 2:block_offs + 4].view(dtype=np.uint16) + delta.byteswap(inplace=True) + + # Byte-Swap + if block_num % 100000 == 0: + inner_pbar.set_description(f"Byte-swapping Blocks [{(n_blocks - block_num) // n_blocks}]") + + elif tensor.tensor_type == gguf.GGMLQuantizationType.Q6_K: + # Handle Q6_K tensor blocks (block_q6_k) + # Specific handling of block_q6_k is required. + # Each block_q6_k consists of 208 int8 values followed by 1 f16 value. + + # first flatten structure + newshape = 1 + for i in tensor.data.shape: + newshape *= i + + tensor.data.resize(newshape) + + block_size = 210 + n_blocks = len(tensor.data) // block_size + for block_num in (inner_pbar := tqdm(range(n_blocks), desc="Byte-swapping Blocks", leave=False)): + block_offs = block_num * block_size + + # Byte-Swap f16 sized field + delta = tensor.data[block_offs + 208:block_offs + 210].view(dtype=np.uint16) + delta.byteswap(inplace=True) + + # Byte-Swap + if block_num % 100000 == 0: + inner_pbar.set_description(f"Byte-swapping Blocks [{(n_blocks - block_num) // n_blocks}]") + else: # Handle other tensor types tensor.data.byteswap(inplace=True) From 08d5986290cc42d2c52739e046642b8252f97e4b Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Mon, 24 Feb 2025 22:33:23 +0800 Subject: [PATCH 014/188] [SYCL] Optimize mul_mat for Q4_0 on Intel GPU (#12035) * opt performance by reorder for Intel GPU * detect hw type and save opt feature, and print opt feature * correct name * support optimize graph once when compute graph, record the opt status in tensor->extra, make CI passed * add env variable GGML_SYCL_DISABLE_OPT for debug * use syclex::architecture replace the custom hw define, update the guide for GGML_SYCL_DISABLE_OPT * add performance data * mv getrows functions to separeted files * fix global variables --------- Co-authored-by: arthw <14088817+arthw@users.noreply.github.com> --- docs/backend/SYCL.md | 16 +- examples/sycl/run-llama2.sh | 4 +- ggml/src/ggml-sycl/CMakeLists.txt | 2 + ggml/src/ggml-sycl/common.cpp | 17 ++ ggml/src/ggml-sycl/common.hpp | 58 ++++- ggml/src/ggml-sycl/convert.cpp | 37 ++- ggml/src/ggml-sycl/convert.hpp | 4 +- ggml/src/ggml-sycl/dequantize.hpp | 55 +++++ ggml/src/ggml-sycl/dmmv.cpp | 140 +++++++++++- ggml/src/ggml-sycl/getrows.cpp | 308 +++++++++++++++++++++++++ ggml/src/ggml-sycl/getrows.hpp | 23 ++ ggml/src/ggml-sycl/ggml-sycl.cpp | 369 ++++++++++-------------------- ggml/src/ggml-sycl/sycl_hw.cpp | 13 ++ ggml/src/ggml-sycl/sycl_hw.hpp | 23 ++ 14 files changed, 803 insertions(+), 266 deletions(-) create mode 100644 ggml/src/ggml-sycl/getrows.cpp create mode 100644 ggml/src/ggml-sycl/getrows.hpp create mode 100644 ggml/src/ggml-sycl/sycl_hw.cpp create mode 100644 ggml/src/ggml-sycl/sycl_hw.hpp diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 0cb39e7927..5da439e94e 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -42,6 +42,16 @@ The following release is verified with good quality: ## News +- 2025.2 + - Optimize MUL_MAT Q4_0 on Intel GPU for all dGPUs and built-in GPUs since MTL. Increase the performance of LLM (llama-2-7b.Q4_0.gguf) 21%-87% on Intel GPUs (MTL, ARL-H, Arc, Flex, PVC). + |GPU|Base tokens/s|Increased tokens/s|Percent| + |-|-|-|-| + |PVC 1550|39|73|+87%| + |Flex 170|39|50|+28%| + |Arc770|42|55|+30%| + |MTL|13|16|+23%| + |ARL-H|14|17|+21%| + - 2024.11 - Use syclcompat to improve the performance on some platforms. This requires to use oneAPI 2025.0 or newer. @@ -97,8 +107,8 @@ SYCL backend supports Intel GPU Family: | Intel Data Center Max Series | Support | Max 1550, 1100 | | Intel Data Center Flex Series | Support | Flex 170 | | Intel Arc Series | Support | Arc 770, 730M, Arc A750 | -| Intel built-in Arc GPU | Support | built-in Arc GPU in Meteor Lake | -| Intel iGPU | Support | iGPU in 13700k, i5-1250P, i7-1260P, i7-1165G7 | +| Intel built-in Arc GPU | Support | built-in Arc GPU in Meteor Lake, Arrow Lake | +| Intel iGPU | Support | iGPU in 13700k,iGPU in 13400, i5-1250P, i7-1260P, i7-1165G7 | *Notes:* @@ -660,8 +670,10 @@ use 1 SYCL GPUs: [0] with Max compute units:512 | Name | Value | Function | |-------------------|------------------|---------------------------------------------------------------------------------------------------------------------------| | GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG | +| GGML_SYCL_DISABLE_OPT | 0 (default) or 1 | Disable optimize features based on Intel GPU type, to compare the performance increase | | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
Recommended to use when --split-mode = layer | + ## Known Issues - `Split-mode:[row]` is not supported. diff --git a/examples/sycl/run-llama2.sh b/examples/sycl/run-llama2.sh index 3b9ba3b2da..8ce71e3700 100755 --- a/examples/sycl/run-llama2.sh +++ b/examples/sycl/run-llama2.sh @@ -3,7 +3,7 @@ # MIT license # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: MIT - +export ONEAPI_DEVICE_SELECTOR="level_zero:0" source /opt/intel/oneapi/setvars.sh #export GGML_SYCL_DEBUG=1 @@ -13,7 +13,7 @@ source /opt/intel/oneapi/setvars.sh INPUT_PROMPT="Building a website can be done in 10 simple steps:\nStep 1:" MODEL_FILE=models/llama-2-7b.Q4_0.gguf NGL=33 -CONEXT=8192 +CONEXT=4096 if [ $# -gt 0 ]; then GGML_SYCL_DEVICE=$1 diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index 3579a311aa..3ad044432a 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -1,3 +1,5 @@ +message(STATUS "GGML_SYCL_TARGET=${GGML_SYCL_TARGET}") + if (NOT GGML_SYCL_TARGET MATCHES "^(INTEL|NVIDIA|AMD)$") message(FATAL_ERROR "Invalid backend chosen, supported options are INTEL, NVIDIA, or AMD") endif() diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cpp index 022e7b7637..9069c47865 100644 --- a/ggml/src/ggml-sycl/common.cpp +++ b/ggml/src/ggml-sycl/common.cpp @@ -99,3 +99,20 @@ catch (sycl::exception const &exc) { << ", line:" << __LINE__ << std::endl; std::exit(1); } + + +void release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector streams) { + for (int i = 0; i < ggml_sycl_info().device_count; ++i) { + for (int64_t is = 0; is < GGML_SYCL_MAX_STREAMS; ++is) { + if (extra->events[i][is] != nullptr) { + SYCL_CHECK(CHECK_TRY_ERROR(dpct::destroy_event(extra->events[i][is]))); + } + } + if (extra->data_device[i] != nullptr && streams.size()>0) { + ggml_sycl_set_device(i); + SYCL_CHECK( + CHECK_TRY_ERROR(sycl::free(extra->data_device[i], *(streams[i])))); + } + } + delete extra; +} diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index a5cab5065f..7c503a1b10 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -19,6 +19,9 @@ #include "dpct/helper.hpp" #include "ggml-sycl.h" #include "presets.hpp" +#include "sycl_hw.hpp" + + #if GGML_SYCL_DNNL #include "dnnl.hpp" #include "dnnl_sycl.hpp" @@ -35,7 +38,10 @@ void* ggml_sycl_host_malloc(size_t size); void ggml_sycl_host_free(void* ptr); + extern int g_ggml_sycl_debug; +extern int g_ggml_sycl_disable_optimize; + #define GGML_SYCL_DEBUG(...) \ do { \ if (g_ggml_sycl_debug) \ @@ -182,18 +188,24 @@ inline dpct::err0 ggml_sycl_set_device(const int device) try { } ////////////////////// +struct optimize_feature { + bool reorder=false; +}; + +struct sycl_device_info { + int cc; // compute capability + // int nsm; // number of streaming multiprocessors + // size_t smpb; // max. shared memory per block + bool vmm; // virtual memory support + size_t total_vram; + sycl_hw_info hw_info; + optimize_feature opt_feature; +}; + struct ggml_sycl_device_info { int device_count; - struct sycl_device_info { - int cc; // compute capability - // int nsm; // number of streaming multiprocessors - // size_t smpb; // max. shared memory per block - bool vmm; // virtual memory support - size_t total_vram; - }; - sycl_device_info devices[GGML_SYCL_MAX_DEVICES] = {}; std::array default_tensor_split = {}; @@ -260,17 +272,46 @@ struct ggml_tensor_extra_gpu { // tensors dpct::event_ptr events[GGML_SYCL_MAX_DEVICES] [GGML_SYCL_MAX_STREAMS]; // events for synchronizing multiple GPUs + optimize_feature optimized_feature; }; +void release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector streams={}); + +inline optimize_feature check_gpu_optimize_feature(syclex::architecture &arch) { + optimize_feature opt; + + opt.reorder = + (arch == syclex::architecture::intel_gpu_dg1 || + arch == syclex::architecture::intel_gpu_acm_g10 || + arch == syclex::architecture::intel_gpu_acm_g11 || + arch == syclex::architecture::intel_gpu_acm_g12 || + arch == syclex::architecture::intel_gpu_pvc || + arch == syclex::architecture::intel_gpu_pvc_vg || + arch == syclex::architecture::intel_gpu_mtl_u || + arch == syclex::architecture::intel_gpu_mtl_s || + arch == syclex::architecture::intel_gpu_mtl_h || + arch == syclex::architecture::intel_gpu_arl_u || + arch == syclex::architecture::intel_gpu_arl_s || + arch == syclex::architecture::intel_gpu_arl_h || + arch == syclex::architecture::intel_gpu_bmg_g21 || + arch == syclex::architecture::intel_gpu_lnl_m + ); + + return opt; +} + struct ggml_backend_sycl_context { int device; std::string name; + optimize_feature opt_feature; + bool optimized_graph=false; queue_ptr qptrs[GGML_SYCL_MAX_DEVICES][GGML_SYCL_MAX_STREAMS] = { { nullptr } }; explicit ggml_backend_sycl_context(int device) : device(device), name(GGML_SYCL_NAME + std::to_string(device)) { + opt_feature = ggml_sycl_info().devices[device].opt_feature; } queue_ptr stream(int device, int stream) { @@ -680,5 +721,4 @@ bool gpu_has_xmx(sycl::device &dev); void ggml_sycl_op_flatten(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const ggml_sycl_op_flatten_t op); - #endif // GGML_SYCL_COMMON_HPP diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index 05b01db2d8..86b200e070 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -125,6 +125,25 @@ static void dequantize_row_q4_0_sycl(const void *vx, dst_t *y, const int64_t k, } } +template +static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int64_t k, + dpct::queue_ptr stream) { + + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + int constexpr WARP_K = WARP_SIZE * QK4_0; + const int n_warp = (k + WARP_K - 1) / WARP_K; + GGML_ASSERT(k % 2 == 0); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * + sycl::range<3>(1, 1, WARP_SIZE), + sycl::range<3>(1, 1, WARP_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]]{ + dequantize_block_q4_0_reorder(vx, y, k, item_ct1); + }); + +} + template static void dequantize_row_q4_1_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -452,10 +471,15 @@ static void convert_unary_sycl(const void *__restrict__ vx, } } -to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type) { +to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor *dst) { switch (type) { case GGML_TYPE_Q4_0: - return dequantize_block_sycl; + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_q4_0_sycl_reorder; + } else { + return dequantize_block_sycl; + } case GGML_TYPE_Q4_1: return dequantize_block_sycl; case GGML_TYPE_Q5_0: @@ -499,10 +523,15 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type) { } } -to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type) { +to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { switch (type) { case GGML_TYPE_Q4_0: - return dequantize_row_q4_0_sycl; + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_q4_0_sycl_reorder; + } else { + return dequantize_row_q4_0_sycl; + } case GGML_TYPE_Q4_1: return dequantize_row_q4_1_sycl; case GGML_TYPE_Q5_0: diff --git a/ggml/src/ggml-sycl/convert.hpp b/ggml/src/ggml-sycl/convert.hpp index 0ce2874aaa..355dae22b4 100644 --- a/ggml/src/ggml-sycl/convert.hpp +++ b/ggml/src/ggml-sycl/convert.hpp @@ -21,7 +21,7 @@ using to_t_sycl_t = void (*)(const void *__restrict__ x, T *__restrict__ y, typedef to_t_sycl_t to_fp32_sycl_t; typedef to_t_sycl_t to_fp16_sycl_t; -to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type); -to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type); +to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor *dst); +to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst); #endif // GGML_SYCL_CONVERT_HPP diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index b8304c3a27..651c2160d2 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -16,6 +16,8 @@ #include "common.hpp" typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, dfloat2 & v); +typedef void (*dequantize_kernel_t_reorder)(const void *d, const int64_t ib, const void *qs, + const int iqs, dfloat2 &v); static __dpct_inline__ void dequantize_q4_0(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { @@ -40,6 +42,29 @@ static __dpct_inline__ void dequantize_q4_0(const void *vx, const int64_t ib, #endif // GGML_SYCL_F16 } +static __dpct_inline__ void dequantize_q4_0_reorder(const void *d_ptr, const int64_t ib, const void *qs, + const int iqs, dfloat2 &v) { + // const block_q4_0 * x = (const block_q4_0 *) vx; + + const dfloat d = (const dfloat)*((const sycl::half*)d_ptr+ib); + + const int vui = *((const uint8_t *)qs+iqs); + + v.x() = vui & 0xF; + v.y() = vui >> 4; + +#ifdef GGML_SYCL_F16 + // v = v - {8.0f, 8.0f}; + // v = v * {d, d}; + v.s0() = (v.s0() - 8.0f) * d; + v.s1() = (v.s1() - 8.0f) * d; + +#else + v.x() = (v.x() - 8.0f) * d; + v.y() = (v.y() - 8.0f) * d; +#endif // GGML_SYCL_F16 +} + static __dpct_inline__ void dequantize_q4_1(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { const block_q4_1 * x = (const block_q4_1 *) vx; @@ -167,6 +192,36 @@ static void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restri } } +template +static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32, + const sycl::nd_item<3> &item_ct1) { + + const int64_t i = item_ct1.get_group(2); + auto k=nb32; + // assume 32 threads + const int64_t tid = item_ct1.get_local_id(2); + const int lane_ib = i * WARP_SIZE + tid; + + if (lane_ib >= k / QK4_0) { + return; + } + + dst_t * y_ptr = yy + lane_ib * QK4_0; + + auto qs = (const uint8_t*)vx + lane_ib * QK4_0 / 2; + auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k / 2) + lane_ib; + + const float d = float(*s_ptr); + +#pragma unroll + for (int l = 0; l < QK4_0 / 2; ++l) { + int vq = qs[l]; + y_ptr[l + 0] = d * ((vq & 0xF) - 8); + y_ptr[l + 16] = d * ((vq >> 4) - 8); + } + +} + template static void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32, const sycl::nd_item<3> &item_ct1) { diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index 0d097357ce..99d3859de8 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -3,7 +3,6 @@ #include "dequantize.hpp" #include "presets.hpp" - static void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){ const sycl::half *x = (const sycl::half *)vx; @@ -91,6 +90,112 @@ static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * } } +template +static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> &item_ct1) { + // qk = quantized weights per x block + // qr = number of quantized weights per data value in x block + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + + if (row >= nrows) { + return; + } + + const int tid = item_ct1.get_local_id(2); + + + const int ncols_left = ncols % (QK4_0*WARP_SIZE); + const int ncols_align = ncols - ncols_left; + const int iter_stride = 8*2*GGML_SYCL_DMMV_X; + const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter //64/16=4, 512/16/2= 16 + const int y_offset = qr == 1 ? 1 : qk/2; + +// partial sum for each thread +#ifdef GGML_SYCL_F16 + sycl::half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics +#else + float tmp = 0.0f; +#endif // GGML_SYCL_F16 + const char *d_ptr = (const char*)vx+ncols*nrows/2; + int i=0; + for (i = 0; i < ncols_align; i += iter_stride) { + const int col = i + vals_per_iter*tid; + const int ib = (row*ncols + col)/qk; // x block index + const int iqs = (col%qk)/qr; // x quant index + const int iybs = col - col%qk; // y block start index + +// processing >2 values per i iter is faster for fast GPUs +#pragma unroll + for (int j = 0; j < vals_per_iter; j += 2) { + // process 2 vals per j iter + + // dequantize + // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val + dfloat2 v; + dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v); + + // matrix multiplication + // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2 +#ifdef GGML_SYCL_F16 + dfloat2 t1{y[iybs + iqs + j / qr + 0], + y[iybs + iqs + j / qr + y_offset]}; + + tmp += v * t1; +#else + tmp += v.x() * y[iybs + iqs + j / qr + 0]; + tmp += v.y() * y[iybs + iqs + j / qr + y_offset]; +#endif // GGML_SYCL_F16 + } + } + + for (; i < ncols; i += iter_stride) { + if (tid>=ncols_left/QK4_0) continue; + const int col = i + vals_per_iter*tid; + const int ib = (row*ncols + col)/qk; // x block index + const int iqs = (col%qk)/qr; // x quant index + const int iybs = col - col%qk; // y block start index + +// processing >2 values per i iter is faster for fast GPUs +#pragma unroll + for (int j = 0; j < vals_per_iter; j += 2) { + // process 2 vals per j iter + + // dequantize + // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val + dfloat2 v; + dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v); + + // matrix multiplication + // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2 +#ifdef GGML_SYCL_F16 + dfloat2 t1{y[iybs + iqs + j / qr + 0], + y[iybs + iqs + j / qr + y_offset]}; + + tmp += v * t1; +#else + tmp += v.x() * y[iybs + iqs + j / qr + 0]; + tmp += v.y() * y[iybs + iqs + j / qr + y_offset]; +#endif // GGML_SYCL_F16 + } + } + + // sum up partial sums and write back result + const int mask_start = ncols > GGML_SYCL_DMMV_X ? WARP_SIZE >> 1 : WARP_SIZE >> 2; + for (int mask = mask_start; mask > 0; mask >>= 1) { + tmp += + dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + } + + if (tid == 0) { +#ifdef GGML_SYCL_F16 + dst[row] = tmp.x() + tmp.y(); +#else + dst[row] = tmp; +#endif // GGML_SYCL_F16 + } +} + static void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y, float *dst, const int ncols, const int nrows, @@ -759,6 +864,28 @@ static void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const floa } } +static void dequantize_mul_mat_vec_q4_0_sycl_reorder(const void *vx, const dfloat *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec_reorder( + vx, y, dst, ncols, nrows, item_ct1); + }); + } +} + static void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y, float *dst, const int ncols, @@ -953,7 +1080,6 @@ void ggml_sycl_op_dequantize_mul_mat_vec( const int64_t ne00 = src0->ne[0]; const int64_t row_diff = row_high - row_low; - GGML_ASSERT(src1->type == GGML_TYPE_F32); // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics #ifdef GGML_SYCL_F16 @@ -967,7 +1093,7 @@ void ggml_sycl_op_dequantize_mul_mat_vec( if (src1_convert_f16) { src1_dfloat = src1_dfloat_a.alloc(ne00); - const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type); + const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst); GGML_ASSERT(to_fp16_sycl != nullptr); to_fp16_sycl(src1_ddf_i, src1_dfloat, ne00, stream); } @@ -977,7 +1103,12 @@ void ggml_sycl_op_dequantize_mul_mat_vec( switch (src0->type) { case GGML_TYPE_Q4_0: - dequantize_mul_mat_vec_q4_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + if ((ggml_tensor_extra_gpu*)dst->src[0]->extra && + ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { + dequantize_mul_mat_vec_q4_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } else { + dequantize_mul_mat_vec_q4_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } break; case GGML_TYPE_Q4_1: dequantize_mul_mat_vec_q4_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); @@ -1020,4 +1151,5 @@ void ggml_sycl_op_dequantize_mul_mat_vec( GGML_UNUSED(src1_ddq_i); GGML_UNUSED(src1_ncols); GGML_UNUSED(src1_padded_row_size); + GGML_UNUSED(ctx); } diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp new file mode 100644 index 0000000000..51c19f6b3b --- /dev/null +++ b/ggml/src/ggml-sycl/getrows.cpp @@ -0,0 +1,308 @@ +// +// MIT license +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: MIT +// + +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// + +#include "ggml-impl.h" +#include "common.hpp" +#include "dequantize.hpp" +#include "getrows.hpp" + + +template +static void k_get_rows( + const void * src0, const int32_t * src1, dst_t * dst, + int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/ + /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/ + /*size_t s0,*/ size_t s1, size_t s2, size_t s3, + /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03, + size_t s10, size_t s11, size_t s12, + const sycl::nd_item<3> &item_ct1/*, size_t s13*/) { + + const int i00 = (item_ct1.get_group(2) * item_ct1.get_local_range(2) + + item_ct1.get_local_id(2)) * + 2; + const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) + + item_ct1.get_local_id(1); + const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) / + ne12; + const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) % + ne12; + + if (i00 >= ne00) { + return; + } + + const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; + + dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; + const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03; + + const int ib = i00/qk; // block index + const int iqs = (i00%qk)/qr; // quant index + const int iybs = i00 - i00%qk; // dst block start index + const int y_offset = qr == 1 ? 1 : qk/2; + + // dequantize + dfloat2 v; + dequantize_kernel(src0_row, ib, iqs, v); + + dst_row[iybs + iqs + 0] = v.x(); + dst_row[iybs + iqs + y_offset] = v.y(); +} + +template +static void k_get_rows_reorder( + const void * src0, const void *src0_dq, const int32_t * src1, dst_t * dst, + int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/ + /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/ + /*size_t s0,*/ size_t s1, size_t s2, size_t s3, + /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03, + size_t s10, size_t s11, size_t s12, + const sycl::nd_item<3> &item_ct1/*, size_t s13*/) { + + const int i00 = (item_ct1.get_group(2) * item_ct1.get_local_range(2) + + item_ct1.get_local_id(2)) * + 2; + const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) + + item_ct1.get_local_id(1); + const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) / + ne12; + const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) % + ne12; + + if (i00 >= ne00) { + return; + } + auto ncols = ne00; + const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; + + dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; + + const int src0_off = i01 * ncols + i00; + const int ib = src0_off / QK4_0; // block index + const int iqs = (i00%qk)/qr; // x quant index + const int iybs = i00 - i00%qk; // dst block start index + const int y_offset = qr == 1 ? 1 : qk/2; + + // dequantize + dfloat2 v; + dequantize_kernel_recorder((const void *)src0_dq, ib, (const void *)src0, src0_off/2, v); + + dst_row[iybs + iqs + 0] = v.x(); + dst_row[iybs + iqs + y_offset] = v.y(); + + GGML_UNUSED(nb01); + GGML_UNUSED(nb02); + GGML_UNUSED(nb03); +} + +template +static void k_get_rows_float( + const src0_t * src0, const int32_t * src1, dst_t * dst, + int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/ + /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/ + /*size_t s0,*/ size_t s1, size_t s2, size_t s3, + /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03, + size_t s10, size_t s11, size_t s12, + const sycl::nd_item<3> &item_ct1/*, size_t s13*/) { + + const int i00 = item_ct1.get_group(2) * item_ct1.get_local_range(2) + + item_ct1.get_local_id(2); + const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) + + item_ct1.get_local_id(1); + const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) / + ne12; + const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) % + ne12; + + if (i00 >= ne00) { + return; + } + + const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; + + dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; + const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03); + + dst_row[i00] = src0_row[i00]; +} + +template +static void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, + ggml_tensor *dst, const void *src0_dd, + const int32_t *src1_dd, float *dst_dd, + queue_ptr stream) { + + GGML_TENSOR_BINARY_OP_LOCALS + + const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); + const int block_num_x = (ne00 + 2*SYCL_GET_ROWS_BLOCK_SIZE - 1) / (2*SYCL_GET_ROWS_BLOCK_SIZE); + const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x); + + // strides in elements + //const size_t s0 = nb0 / ggml_element_size(dst); + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + //const size_t s13 = nb13 / ggml_element_size(src1); + + GGML_ASSERT(ne00 % 2 == 0); + + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_get_rows( + src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2, + s3, nb01, nb02, nb03, s10, s11, s12, item_ct1); + }); + + GGML_UNUSED(dst); + GGML_UNUSED(ctx); +} + +template +static void get_rows_sycl_reorder(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, + ggml_tensor *dst, const void *src0_dd, + const int32_t *src1_dd, float *dst_dd, + queue_ptr stream) { + + GGML_TENSOR_BINARY_OP_LOCALS + + const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); + const int block_num_x = (ne00 + 2*SYCL_GET_ROWS_BLOCK_SIZE - 1) / (2*SYCL_GET_ROWS_BLOCK_SIZE); + const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x); + + // strides in elements + //const size_t s0 = nb0 / ggml_element_size(dst); + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + //const size_t s13 = nb13 / ggml_element_size(src1); + + GGML_ASSERT(ne00 % 2 == 0); + + const uint8_t* src0_q = (const uint8_t*)src0_dd; + const size_t ncols = ne00; + const size_t nrows = ne01; + const sycl::half* src0_dq = (const sycl::half*)(src0_q + nrows * ncols / 2); + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]]{ + k_get_rows_reorder( + src0_dd, src0_dq, src1_dd, dst_dd, ne00, ne12, s1, s2, + s3, nb01, nb02, nb03, s10, s11, s12, item_ct1); + }); + + GGML_UNUSED(dst); + GGML_UNUSED(ctx); +} + + +template +static void get_rows_sycl_float(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, + const ggml_tensor *src1, ggml_tensor *dst, + const src0_t *src0_dd, const int32_t *src1_dd, + float *dst_dd, queue_ptr stream) { + + GGML_TENSOR_BINARY_OP_LOCALS + + const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); + const int block_num_x = (ne00 + SYCL_GET_ROWS_BLOCK_SIZE - 1) / SYCL_GET_ROWS_BLOCK_SIZE; + const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x); + + // strides in elements + //const size_t s0 = nb0 / ggml_element_size(dst); + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + //const size_t s13 = nb13 / ggml_element_size(src1); + + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_get_rows_float(src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2, + s3, nb01, nb02, nb03, s10, s11, s12, item_ct1); + }); + } + + GGML_UNUSED(dst); + GGML_UNUSED(ctx); +} + +void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, + const ggml_tensor *src1, ggml_tensor *dst, + const float *src0_d, const float *src1_d, + float *dst_d, const queue_ptr &stream) { + + GGML_ASSERT(src1->type == GGML_TYPE_I32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); + GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type)); + GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type)); + + const int32_t * src1_i32 = (const int32_t *) src1_d; + + switch (src0->type) { + case GGML_TYPE_F16: + get_rows_sycl_float(ctx, src0, src1, dst, (const sycl::half *)src0_d, + src1_i32, dst_d, stream); + break; + case GGML_TYPE_F32: + get_rows_sycl_float(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); + break; + case GGML_TYPE_Q4_0: + if (ctx.opt_feature.reorder && dst->op == GGML_OP_MUL_MAT) { + get_rows_sycl_reorder(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); + } else { + get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); + } + break; + case GGML_TYPE_Q4_1: + get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); + break; + case GGML_TYPE_Q5_0: + get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); + break; + case GGML_TYPE_Q5_1: + get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); + break; + case GGML_TYPE_Q8_0: + get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); + break; + default: + // TODO: k-quants + GGML_LOG_ERROR("%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type)); + GGML_ABORT("fatal error"); + break; + } +} + diff --git a/ggml/src/ggml-sycl/getrows.hpp b/ggml/src/ggml-sycl/getrows.hpp new file mode 100644 index 0000000000..cdbe6c2f41 --- /dev/null +++ b/ggml/src/ggml-sycl/getrows.hpp @@ -0,0 +1,23 @@ +// +// MIT license +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: MIT +// + +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// + +#ifndef GGML_SYCL_GETROWS_HPP +#define GGML_SYCL_GETROWS_HPP + +#include "common.hpp" + +void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, + const ggml_tensor *src1, ggml_tensor *dst, + const float *src0_d, const float *src1_d, + float *dst_d, const queue_ptr &stream); + +#endif // GGML_SYCL_GETROWS_HPP diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index d4c97ad17b..792e0569ca 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -39,9 +39,12 @@ #include "ggml-sycl/backend.hpp" #include "ggml-sycl/presets.hpp" #include "ggml-sycl/gemm.hpp" +#include "ggml-sycl/sycl_hw.hpp" +#include "ggml-sycl/getrows.hpp" static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; +int g_ggml_sycl_disable_optimize = 0; static ggml_sycl_device_info ggml_sycl_init() { ggml_sycl_device_info info = {}; @@ -64,14 +67,18 @@ static ggml_sycl_device_info ggml_sycl_init() { for (int i = 0; i < info.device_count; ++i) { info.devices[i].vmm = 0; dpct::device_info prop; + sycl::device device = dpct::dev_mgr::instance().get_device(i); + SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( - prop, dpct::dev_mgr::instance().get_device(i)))); + prop, device))); info.default_tensor_split[i] = total_vram; total_vram += prop.get_global_mem_size(); info.devices[i].cc = 100 * prop.get_major_version() + 10 * prop.get_minor_version(); + info.devices[i].hw_info = get_device_hw_info(&device); + info.devices[i].opt_feature = check_gpu_optimize_feature(info.devices[i].hw_info.arch); info.max_work_group_sizes[i] = prop.get_max_work_group_size(); } @@ -110,6 +117,27 @@ void print_device_detail(int id, sycl::device &device, std::string device_type) global_mem_size, device.get_info().c_str()); } +void print_device_opt_feature(int device_count) { + GGML_LOG_INFO("SYCL Optimization Feature:\n"); + GGML_LOG_INFO( + "|ID| Device Type|Reorder|\n"); + GGML_LOG_INFO( + "|--|-------------------|-------|\n"); + std::map DeviceNums; + for (int id = 0; id < device_count; ++id) { + sycl::device device = dpct::dev_mgr::instance().get_device(id); + std::string backend_type = get_device_backend_and_type(device); + int type_id = DeviceNums[backend_type]++; + std::stringstream device_type; + device_type << "[" << backend_type << ":" << std::to_string(type_id) + << "]"; + std::string device_type_s = device_type.str(); + device_type_s = std::regex_replace(device_type_s, std::regex("ext_oneapi_"), ""); + GGML_LOG_INFO("|%2d|%19s|%7s|\n", id, device_type_s.c_str(), + ggml_sycl_info().devices[id].opt_feature.reorder ? "Y": "N"); + } + +} void ggml_backend_sycl_print_sycl_devices() { GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_print_sycl_devices\n"); int device_count = dpct::dev_mgr::instance().device_count(); @@ -138,6 +166,8 @@ void ggml_backend_sycl_print_sycl_devices() { << "]"; print_device_detail(id, device, device_type.str()); } + + print_device_opt_feature(device_count); } static inline int get_sycl_env(const char *env_name, int default_val) { @@ -159,17 +189,21 @@ static void ggml_check_sycl() try { if (!initialized) { g_ggml_sycl_debug = get_sycl_env("GGML_SYCL_DEBUG", 0); + g_ggml_sycl_disable_optimize= get_sycl_env("GGML_SYCL_DISABLE_OPT", 0); GGML_SYCL_DEBUG("[SYCL] call ggml_check_sycl\n"); - GGML_LOG_INFO("GGML_SYCL_DEBUG: %d\n", g_ggml_sycl_debug); + GGML_LOG_INFO("Running with Environment Variables:\n"); + GGML_LOG_INFO(" GGML_SYCL_DEBUG: %d\n", g_ggml_sycl_debug); + GGML_LOG_INFO(" GGML_SYCL_DISABLE_OPT: %d\n", g_ggml_sycl_disable_optimize); + GGML_LOG_INFO("Build with Macros:\n"); #if defined(GGML_SYCL_FORCE_MMQ) - GGML_LOG_INFO("GGML_SYCL_FORCE_MMQ: yes\n"); + GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: yes\n"); #else - GGML_LOG_INFO("GGML_SYCL_FORCE_MMQ: no\n"); + GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: no\n"); #endif #if defined(GGML_SYCL_F16) - GGML_LOG_INFO("GGML_SYCL_F16: yes\n"); + GGML_LOG_INFO(" GGML_SYCL_F16: yes\n"); #else - GGML_LOG_INFO("GGML_SYCL_F16: no\n"); + GGML_LOG_INFO(" GGML_SYCL_F16: no\n"); #endif /* NOT REMOVE, keep it for next optimize for XMX. @@ -241,19 +275,27 @@ struct ggml_backend_sycl_buffer_context { void * dev_ptr = nullptr; queue_ptr stream; std::string name; + optimize_feature opt_feature; + std::vector tensor_extras; - ggml_backend_sycl_buffer_context(int device, void * dev_ptr, queue_ptr stream) : + ggml_backend_sycl_buffer_context(int device, void * dev_ptr, queue_ptr stream) : device(device), dev_ptr(dev_ptr), stream(stream) { check_allow_gpu_index(device); name = (GGML_SYCL_NAME + std::to_string(device)); + opt_feature = ggml_sycl_info().devices[device].opt_feature; } - ~ggml_backend_sycl_buffer_context() { if (dev_ptr != nullptr) { ggml_sycl_set_device(device); SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(dev_ptr, *stream))); } + + //release extra used by tensors + for (ggml_tensor_extra_gpu * extra : tensor_extras) { + release_extra_gpu(extra); + } + } }; @@ -291,6 +333,9 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, return; } + ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; + tensor->extra = extra; + ctx->tensor_extras.push_back(extra); //used to release it when destroy ctx. if (ggml_is_quantized(tensor->type)) { // initialize padding to 0 to avoid possible NaN values @@ -316,7 +361,6 @@ static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, size_t size) try { ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; - ggml_sycl_set_device(ctx->device); auto stream = &(dpct::dev_mgr::instance().get_device(ctx->device).default_queue()); SYCL_CHECK( @@ -660,32 +704,7 @@ struct ggml_backend_sycl_split_buffer_type_context { struct ggml_backend_sycl_split_buffer_context { ~ggml_backend_sycl_split_buffer_context() try { for (ggml_tensor_extra_gpu * extra : tensor_extras) { - for (int i = 0; i < ggml_sycl_info().device_count; ++i) { - for (int64_t is = 0; is < GGML_SYCL_MAX_STREAMS; ++is) { - if (extra->events[i][is] != nullptr) { - /* - DPCT1009:206: SYCL uses exceptions to report errors and - does not use the error codes. The original code was - commented out and a warning string was inserted. You - need to rewrite this code. - */ - SYCL_CHECK(CHECK_TRY_ERROR( - dpct::destroy_event(extra->events[i][is]))); - } - } - if (extra->data_device[i] != nullptr) { - /* - DPCT1009:207: SYCL uses exceptions to report errors and does - not use the error codes. The original code was commented out - and a warning string was inserted. You need to rewrite this - code. - */ - ggml_sycl_set_device(i); - SYCL_CHECK(CHECK_TRY_ERROR(sycl::free( - extra->data_device[i], *(streams[i])))); - } - } - delete extra; + release_extra_gpu(extra, streams); } } catch (sycl::exception const &exc) { @@ -723,7 +742,7 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; ctx->tensor_extras.push_back(extra); - ctx->streams.push_back(&(dpct::get_current_device().default_queue())); + ctx->streams.push_back(&(dpct::get_current_device().default_queue())); for (int i = 0; i < ggml_sycl_info().device_count; ++i) { int64_t row_low, row_high; @@ -1337,83 +1356,6 @@ static void quantize_q8_1(const float * __restrict__ x, void * __restrict__ vy, reinterpret_cast(y[ib].ds.y()) = sum; } -template -static void k_get_rows( - const void * src0, const int32_t * src1, dst_t * dst, - int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/ - /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/ - /*size_t s0,*/ size_t s1, size_t s2, size_t s3, - /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03, - size_t s10, size_t s11, size_t s12, - const sycl::nd_item<3> &item_ct1/*, size_t s13*/) { - - const int i00 = (item_ct1.get_group(2) * item_ct1.get_local_range(2) + - item_ct1.get_local_id(2)) * - 2; - const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) + - item_ct1.get_local_id(1); - const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + - item_ct1.get_local_id(0)) / - ne12; - const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + - item_ct1.get_local_id(0)) % - ne12; - - if (i00 >= ne00) { - return; - } - - const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; - - dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; - const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03; - - const int ib = i00/qk; // block index - const int iqs = (i00%qk)/qr; // quant index - const int iybs = i00 - i00%qk; // dst block start index - const int y_offset = qr == 1 ? 1 : qk/2; - - // dequantize - dfloat2 v; - dequantize_kernel(src0_row, ib, iqs, v); - - dst_row[iybs + iqs + 0] = v.x(); - dst_row[iybs + iqs + y_offset] = v.y(); -} - -template -static void k_get_rows_float( - const src0_t * src0, const int32_t * src1, dst_t * dst, - int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/ - /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/ - /*size_t s0,*/ size_t s1, size_t s2, size_t s3, - /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03, - size_t s10, size_t s11, size_t s12, - const sycl::nd_item<3> &item_ct1/*, size_t s13*/) { - - const int i00 = item_ct1.get_group(2) * item_ct1.get_local_range(2) + - item_ct1.get_local_id(2); - const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) + - item_ct1.get_local_id(1); - const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + - item_ct1.get_local_id(0)) / - ne12; - const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + - item_ct1.get_local_id(0)) % - ne12; - - if (i00 >= ne00) { - return; - } - - const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; - - dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; - const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03); - - dst_row[i00] = src0_row[i00]; -} - static void mul_mat_p021_f16_f32( const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int nchannels_x, const int nchannels_y, @@ -1896,81 +1838,6 @@ static void pool2d_nchw_kernel( o_ptr[cur_oh * ow + cur_ow] = res; } -template -static void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, - ggml_tensor *dst, const void *src0_dd, - const int32_t *src1_dd, float *dst_dd, - queue_ptr stream) { - - GGML_TENSOR_BINARY_OP_LOCALS - - const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); - const int block_num_x = (ne00 + 2*SYCL_GET_ROWS_BLOCK_SIZE - 1) / (2*SYCL_GET_ROWS_BLOCK_SIZE); - const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x); - - // strides in elements - //const size_t s0 = nb0 / ggml_element_size(dst); - const size_t s1 = nb1 / ggml_element_size(dst); - const size_t s2 = nb2 / ggml_element_size(dst); - const size_t s3 = nb3 / ggml_element_size(dst); - - const size_t s10 = nb10 / ggml_element_size(src1); - const size_t s11 = nb11 / ggml_element_size(src1); - const size_t s12 = nb12 / ggml_element_size(src1); - //const size_t s13 = nb13 / ggml_element_size(src1); - - GGML_ASSERT(ne00 % 2 == 0); - - stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - k_get_rows( - src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2, - s3, nb01, nb02, nb03, s10, s11, s12, item_ct1); - }); - - GGML_UNUSED(dst); - GGML_UNUSED(ctx); -} - -template -static void get_rows_sycl_float(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, - const ggml_tensor *src1, ggml_tensor *dst, - const src0_t *src0_dd, const int32_t *src1_dd, - float *dst_dd, queue_ptr stream) { - - GGML_TENSOR_BINARY_OP_LOCALS - - const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); - const int block_num_x = (ne00 + SYCL_GET_ROWS_BLOCK_SIZE - 1) / SYCL_GET_ROWS_BLOCK_SIZE; - const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x); - - // strides in elements - //const size_t s0 = nb0 / ggml_element_size(dst); - const size_t s1 = nb1 / ggml_element_size(dst); - const size_t s2 = nb2 / ggml_element_size(dst); - const size_t s3 = nb3 / ggml_element_size(dst); - - const size_t s10 = nb10 / ggml_element_size(src1); - const size_t s11 = nb11 / ggml_element_size(src1); - const size_t s12 = nb12 / ggml_element_size(src1); - //const size_t s13 = nb13 / ggml_element_size(src1); - - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - k_get_rows_float(src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2, - s3, nb01, nb02, nb03, s10, s11, s12, item_ct1); - }); - } - - GGML_UNUSED(dst); - GGML_UNUSED(ctx); -} - static void quantize_row_q8_1_sycl(const float *x, void *vy, const int kx, const int ky, const int kx_padded, queue_ptr stream) { @@ -2494,52 +2361,6 @@ catch (sycl::exception const &exc) { std::exit(1); } -static void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, - const ggml_tensor *src1, ggml_tensor *dst, - const float *src0_d, const float *src1_d, - float *dst_d, const queue_ptr &stream) { - - GGML_ASSERT(src1->type == GGML_TYPE_I32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); - - GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); - GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type)); - GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type)); - - const int32_t * src1_i32 = (const int32_t *) src1_d; - - switch (src0->type) { - case GGML_TYPE_F16: - get_rows_sycl_float(ctx, src0, src1, dst, (const sycl::half *)src0_d, - src1_i32, dst_d, stream); - break; - case GGML_TYPE_F32: - get_rows_sycl_float(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); - break; - case GGML_TYPE_Q4_0: - get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); - break; - case GGML_TYPE_Q4_1: - get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); - break; - case GGML_TYPE_Q5_0: - get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); - break; - case GGML_TYPE_Q5_1: - get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); - break; - case GGML_TYPE_Q8_0: - get_rows_sycl(ctx, src0, src1, dst, src0_d, src1_i32, dst_d, stream); - break; - default: - // TODO: k-quants - GGML_LOG_ERROR("%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type)); - GGML_ABORT("fatal error"); - break; - } -} - - static void ggml_sycl_op_repeat(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const float *src0_d, const float *src1_d, @@ -2589,11 +2410,10 @@ inline void ggml_sycl_op_mul_mat_sycl( if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && use_fp16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) { - // GGML_SYCL_DEBUG("ggml_sycl_op_mul_mat_sycl - fp16 path\n"); ggml_sycl_pool_alloc src0_as_f16(ctx.pool()); if (src0->type != GGML_TYPE_F16) { - const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src0->type); + const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src0->type, dst); GGML_ASSERT(to_fp16_sycl != nullptr); size_t ne = row_diff*ne00; src0_as_f16.alloc(ne); @@ -2605,7 +2425,7 @@ inline void ggml_sycl_op_mul_mat_sycl( ggml_sycl_pool_alloc src1_as_f16(ctx.pool()); if (src1->type != GGML_TYPE_F16) { - const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type); + const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst); GGML_ASSERT(to_fp16_sycl != nullptr); size_t ne = src1_ncols*ne10; src1_as_f16.alloc(ne); @@ -2626,13 +2446,13 @@ inline void ggml_sycl_op_mul_mat_sycl( src1_ptr, dpct::library_data_t::real_half, ne10, &beta_f16, dst_f16.get(), dpct::library_data_t::real_half, ldc, dpct::library_data_t::real_half))); - const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16); + const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16, dst); to_fp32_sycl(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream); #else auto dnnl_stream = ctx.stream_dnnl(stream); DnnlGemmWrapper::row_gemm(dnnl_stream, false, true, src1_ncols, row_diff, ne10, src1_ptr, DnnlGemmWrapper::to_dt(), src0_ptr, DnnlGemmWrapper::to_dt(), dst_f16.get(), DnnlGemmWrapper::to_dt()); - const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16); + const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16, dst); to_fp32_sycl(dst_f16.get(), dst_dd_i, row_diff* src1_ncols, stream); #endif } @@ -2641,13 +2461,13 @@ inline void ggml_sycl_op_mul_mat_sycl( ggml_sycl_pool_alloc src0_ddq_as_f32(ctx.pool()); ggml_sycl_pool_alloc src1_ddq_as_f32(ctx.pool()); if (src0->type != GGML_TYPE_F32) { - const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src0->type); + const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src0->type, dst); GGML_ASSERT(to_fp32_sycl != nullptr); src0_ddq_as_f32.alloc(row_diff*ne00); to_fp32_sycl(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream); } if (src1->type != GGML_TYPE_F32) { - const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src1->type); + const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src1->type, dst); GGML_ASSERT(to_fp32_sycl != nullptr); src1_ddq_as_f32.alloc(src1_ncols*ne10); to_fp32_sycl(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream); @@ -3085,7 +2905,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) { const int64_t is = split ? (src1_col_0/src1_col_stride) % GGML_SYCL_MAX_STREAMS : 0; const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride; - for (int i = 0; i < ggml_sycl_info().device_count; ++i) { if ((!split && i != ctx.device) || dev[i].row_low == dev[i].row_high) { continue; @@ -3393,7 +3212,7 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, // convert src1 to fp16 ggml_sycl_pool_alloc src1_f16_alloc(ctx.pool()); if (src1->type != GGML_TYPE_F16) { - const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type); + const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst); const int64_t ne_src1 = ggml_nelements(src1); src1_f16_alloc.alloc(ne_src1); GGML_ASSERT(to_fp16_sycl != nullptr); @@ -3509,6 +3328,7 @@ bool ggml_sycl_supports_dmmv(enum ggml_type type) { } static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const bool split = ggml_backend_buffer_is_sycl_split(src0->buffer); int64_t min_compute_capability = INT_MAX; @@ -3570,6 +3390,7 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor ggml_sycl_mul_mat_batched_sycl(ctx, src0, src1, dst); } else if (use_dequantize_mul_mat_vec) { ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_dequantize_mul_mat_vec, false); + // save_tensor_txt("1/dst_1.txt", (float*) dst->data, src0->ne[1], sizeof(float), ctx.stream()); } else if (use_mul_mat_vec_q) { ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_vec_q, true); } else if (use_mul_mat_q) { @@ -4251,10 +4072,72 @@ catch (sycl::exception const &exc) { std::exit(1); } +void reorder_qw(char *data_device, const int ncols, const int nrows, + size_t size, size_t offset, dpct::queue_ptr stream) { + auto tmp_buf = sycl::malloc_shared(size, *stream); + SYCL_CHECK( + CHECK_TRY_ERROR((*stream).memcpy(tmp_buf, data_device, size) + .wait())); + GGML_ASSERT((size % sizeof(block_q4_0) == 0)); + GGML_ASSERT((offset % sizeof(block_q4_0) == 0)); + int offset_blks = offset / sizeof(block_q4_0); + auto qs_ptr = (uint8_t*)data_device + offset_blks * QK4_0 / 2;; + auto d_ptr = (sycl::half*)(qs_ptr + ncols * nrows / 2) + offset_blks; + + stream->parallel_for( + size / sizeof(block_q4_0), + [=](auto i) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + const block_q4_0* x = (const block_q4_0*)tmp_buf; + const int ib = i; + + for (int j = 0; j < QK4_0/2; j ++) + { + *(qs_ptr + ib * QK4_0 / 2 + j) = x[ib].qs[j]; + } + *(d_ptr + ib) = x[ib].d; + }); + + sycl::free(tmp_buf, *stream); +} + +void reorder_qw(ggml_tensor * src0, dpct::queue_ptr stream) { + char*data_device = (char*)src0->data; + size_t ncols = src0->ne[0]; + size_t nrows = src0->ne[1]; + size_t size = ggml_nbytes(src0); + + reorder_qw(data_device, ncols, nrows, size, 0, stream); +} + +void opt_for_reorder(ggml_tensor * dst, dpct::queue_ptr stream) { + ggml_tensor *src0 = dst->src[0]; + ggml_tensor *src1 = dst->src[1]; + + if (dst->op == GGML_OP_MUL_MAT && src0->type == GGML_TYPE_Q4_0 && + src1->ne[2]==1 && src1->ne[3]==1) { + reorder_qw(src0, stream); + ggml_tensor_extra_gpu* extra = (ggml_tensor_extra_gpu*)src0->extra; + GGML_ASSERT(extra); + extra->optimized_feature.reorder = true; //used to decode/dequan in next steps. + } +} + +void optimize_graph_once(ggml_cgraph * cgraph, ggml_backend_sycl_context * ctx) { + dpct::queue_ptr stream = ctx->stream(); + if (ctx->optimized_graph) { + return; + } + ctx->optimized_graph = true; + + for (int i = 0; i < cgraph->n_nodes; i++) { + if (ctx->opt_feature.reorder) opt_for_reorder(cgraph->nodes[i], stream); + } +} static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; ggml_sycl_set_main_device(sycl_ctx->device); + if (!g_ggml_sycl_disable_optimize) optimize_graph_once(cgraph, sycl_ctx); for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; diff --git a/ggml/src/ggml-sycl/sycl_hw.cpp b/ggml/src/ggml-sycl/sycl_hw.cpp new file mode 100644 index 0000000000..da121ffc26 --- /dev/null +++ b/ggml/src/ggml-sycl/sycl_hw.cpp @@ -0,0 +1,13 @@ +#include "sycl_hw.hpp" + + +sycl_hw_info get_device_hw_info(sycl::device *device_ptr) { + sycl_hw_info res; + int32_t id = device_ptr->get_info(); + res.device_id = id; + + syclex::architecture arch = device_ptr->get_info(); + res.arch = arch; + + return res; +} diff --git a/ggml/src/ggml-sycl/sycl_hw.hpp b/ggml/src/ggml-sycl/sycl_hw.hpp new file mode 100644 index 0000000000..bf689450ce --- /dev/null +++ b/ggml/src/ggml-sycl/sycl_hw.hpp @@ -0,0 +1,23 @@ +#ifndef SYCL_HW_HPP +#define SYCL_HW_HPP + +#include +#include +#include +#include + +#include + +namespace syclex = sycl::ext::oneapi::experimental; + +struct sycl_hw_info { + syclex::architecture arch; + int32_t device_id; +}; + +bool is_in_vector(std::vector &vec, int item); + +sycl_hw_info get_device_hw_info(sycl::device *device_ptr); + + +#endif // SYCL_HW_HPP From 7a2c913e66353362d7f28d612fd3c9d51a831eda Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Mon, 24 Feb 2025 09:09:51 -0700 Subject: [PATCH 015/188] llava : Add Granite Vision Support (#11794) * Add super wip scripts for multimodal granite gguf Signed-off-by: Alex-Brooks * Add example for converting mmgranite to gguf Signed-off-by: Alex-Brooks * remove hardcoded path Signed-off-by: Alex-Brooks * Add vision feature layer to gguf params Signed-off-by: Alex-Brooks * Clean up llava surgery and remove name substitution hacks Signed-off-by: Alex-Brooks * Add transformers llava next tensor name mapping Signed-off-by: Alex-Brooks * Make siglip / openclip mutuall exclusive Signed-off-by: Alex-Brooks * Fix projector linear substitution Signed-off-by: Alex-Brooks * Fix linear 2 substitution index Signed-off-by: Alex-Brooks * Increase max flattened gridpoints to 64 Signed-off-by: Alex-Brooks * Fix hardcoded concat for multiple feature layers Signed-off-by: Alex-Brooks * Pull vision feature layers out of gguf keys Signed-off-by: Alex-Brooks * fix num gridpoints and use all layers Signed-off-by: Alex-Brooks * Avoid dropping last image encoder layer in llava models Signed-off-by: Alex-Brooks * Use 10 for max number of patches Signed-off-by: Alex-Brooks * Standardize vision feature layers Signed-off-by: Alex-Brooks * Cleanup logs Signed-off-by: Alex-Brooks * Update comment for vision feature layer init Signed-off-by: Alex-Brooks * Update notes for alternative to legacy llm conversion script Signed-off-by: Alex-Brooks * Fix notes rendering Signed-off-by: Alex-Brooks * Add v prefix to vision feature layer log Signed-off-by: Alex-Brooks * Use current defaults for feature layer Signed-off-by: Alex-Brooks * Use constant for max gridpoints / feat layers, style fixes Signed-off-by: Alex-Brooks * clarify non-negative feature layers Signed-off-by: Alex-Brooks * Remove CLIP_API from func signature Signed-off-by: Alex-Brooks * USE MAX_IMAGE_FEATURE_LAYERS const in layer calc Signed-off-by: Alex-Brooks * Clarify feature layers are non negative ints and not uint Signed-off-by: Alex-Brooks * Fix condition for reading feature layers Signed-off-by: Alex-Brooks * pop last llava layer when feature layers are unset Signed-off-by: Alex-Brooks * Fix unset vision layer 0 Signed-off-by: Alex-Brooks * Update examples/llava/clip.cpp Co-authored-by: Xuan-Son Nguyen * Reenable assertion for out of bounds get_rows Signed-off-by: Alex-Brooks * Use std vector for gridpoints and feature layers Signed-off-by: Alex-Brooks * Caculate max feature layer at load time Signed-off-by: Alex-Brooks * Include base patch for granite vision allocation Signed-off-by: Alex-Brooks * Fix trailing whitespace Signed-off-by: Alex-Brooks * Add max num patches = 10 back for minicpmv Signed-off-by: Alex-Brooks * Use unordered set to store feature layers Co-authored-by: Xuan-Son Nguyen Signed-off-by: Alex-Brooks * Use max feature layer for postnorm Signed-off-by: Alex-Brooks * Apply suggestions from code review --------- Signed-off-by: Alex-Brooks Co-authored-by: Xuan-Son Nguyen --- examples/llava/README.md | 19 +++ examples/llava/clip.cpp | 108 +++++++++++++++--- examples/llava/clip.h | 5 +- .../llava/convert_image_encoder_to_gguf.py | 98 ++++++++++++++-- examples/llava/llava.cpp | 6 +- examples/llava/llava_surgery_v2.py | 41 +++++-- 6 files changed, 235 insertions(+), 42 deletions(-) diff --git a/examples/llava/README.md b/examples/llava/README.md index 0124513617..0e3c320320 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -101,8 +101,27 @@ python ./examples/convert_legacy_llama.py ../llava-v1.6-vicuna-7b/ --skip-unknow ``` **note** llava-1.6 needs more context than llava-1.5, at least 3000 is needed (just run it at -c 4096) + **note** llava-1.6 greatly benefits from batched prompt processing (defaults work) +**note** if the language model in step `6)` is incompatible with the legacy conversion script, the easiest way handle the LLM model conversion is to load the model in transformers, and export only the LLM from the llava next model. + +```python +import os +import transformers + +model_path = ... +llm_export_path = ... + +tokenizer = transformers.AutoTokenizer.from_pretrained(model_path) +model = transformers.AutoModelForImageTextToText.from_pretrained(model_path) + +tokenizer.save_pretrained(llm_export_path) +model.language_model.save_pretrained(llm_export_path) +``` + +Then, you can convert the LLM using the `convert_hf_to_gguf.py` script, which handles more LLM architectures. + ## llava-cli templating and llava-1.6 prompting llava-1.5 models all use the same vicuna prompt, here you can just add your image question like `-p "Provide a full description."` diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index e2150c06d3..76d4a78520 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -120,6 +121,7 @@ static std::string format(const char * fmt, ...) { #define KEY_IMAGE_MEAN "clip.vision.image_mean" #define KEY_IMAGE_STD "clip.vision.image_std" #define KEY_PROJ_TYPE "clip.projector_type" +#define KEY_FEATURE_LAYER "clip.vision.feature_layer" #define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type" #define KEY_IMAGE_GRID_PINPOINTS "clip.vision.image_grid_pinpoints" @@ -444,8 +446,9 @@ struct clip_hparams { char mm_patch_merge_type[32] = "flat"; // spatial_unpad or flat (default) - int32_t image_grid_pinpoints[32]; + std::vector image_grid_pinpoints; int32_t image_crop_resolution; + std::unordered_set vision_feature_layer; }; struct clip_layer { @@ -585,6 +588,7 @@ struct clip_ctx { struct clip_vision_model vision_model; projector_type proj_type = PROJECTOR_TYPE_MLP; + int32_t max_feature_layer; float image_mean[3]; float image_std[3]; bool use_gelu = false; @@ -651,7 +655,6 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 const int hidden_size = hparams.hidden_size; const int n_head = hparams.n_head; const int d_head = hidden_size / n_head; - int n_layer = hparams.n_layer; const float eps = hparams.eps; int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; @@ -752,13 +755,19 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.pre_ln_w), model.pre_ln_b); } + std::vector embedding_stack; + const auto & vision_feature_layer = hparams.vision_feature_layer; + // loop over layers - if (ctx->has_minicpmv_projector || ctx->has_glm_projector || ctx->has_qwen2vl_merger) { - n_layer += 1; - } - for (int il = 0; il < n_layer - 1; il++) { + for (int il = 0; il < ctx->max_feature_layer; il++) { struct ggml_tensor * cur = embeddings; // embeddings = residual, cur = hidden_states + // If this is an embedding feature layer, save the output. + // NOTE: 0 index here refers to the input to the encoder. + if (vision_feature_layer.find(il) != vision_feature_layer.end()) { + embedding_stack.push_back(embeddings); + } + //const size_t nb_q_w = model.layers[il].q_w->nb[0]; // layernorm1 @@ -846,7 +855,6 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 cur = ggml_add(ctx0, embeddings, cur); embeddings = cur; - } // post-layernorm @@ -857,6 +865,19 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.post_ln_w), model.post_ln_b); } + // final layer is a vision feature layer + if (vision_feature_layer.find(ctx->max_feature_layer) != vision_feature_layer.end()) { + embedding_stack.push_back(embeddings); + } + + // If feature layers are explicitly set, stack them (if we have multiple) + if (!embedding_stack.empty()) { + embeddings = embedding_stack[0]; + for (size_t i = 1; i < embedding_stack.size(); i++) { + embeddings = ggml_concat(ctx0, embeddings, embedding_stack[i], 0); + } + } + // llava projector if (ctx->has_llava_projector) { embeddings = ggml_reshape_2d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1]); @@ -1443,14 +1464,26 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { int idx = get_key_idx(ctx, KEY_IMAGE_GRID_PINPOINTS); int n = gguf_get_arr_n(ctx, idx); const int32_t * pinpoints = (const int32_t *)gguf_get_arr_data(ctx, idx); - for (int i = 0; i < 32 && i < n && pinpoints[i] != 0; ++i) { - hparams.image_grid_pinpoints[i] = pinpoints[i]; + for (int i = 0; i < n; ++i) { + hparams.image_grid_pinpoints.push_back(pinpoints[i]); } - if (n < 32) - hparams.image_grid_pinpoints[n] = 0; - } catch (std::runtime_error & /*e*/) { - hparams.image_grid_pinpoints[0]=0; - } + } catch (std::runtime_error & /*e*/) { } + + // Load the vision feature layer indices if they are explicitly provided; + // if multiple vision feature layers are present, the values will be concatenated + // to form the final visual features. + // NOTE: gguf conversions should standardize the values of the vision feature layer to + // be non-negative, since we use -1 to mark values as unset here. + try { + int idx = get_key_idx(ctx, KEY_FEATURE_LAYER); + int n = gguf_get_arr_n(ctx, idx); + + const int32_t * vision_feature_layer = (const int32_t *)gguf_get_arr_data(ctx, idx); + + for (int i = 0; i < n; ++i) { + hparams.vision_feature_layer.insert(vision_feature_layer[i]); + } + } catch (std::runtime_error & /*e*/) { } try { int idx = get_key_idx(ctx, KEY_MM_PATCH_MERGE_TYPE); @@ -1476,6 +1509,9 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { new_clip->image_std[i] = std_data[i]; } + // Calculate the deepest feature layer based on hparams and projector type + new_clip->max_feature_layer = get_deepest_feature_layer(new_clip); + if (verbosity >= 2) { LOG_INF("\n%s: vision model hparams\n", __func__); LOG_INF("image_size %d\n", hparams.image_size); @@ -1489,8 +1525,13 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { LOG_INF("v_image_mean %f %f %f\n", new_clip->image_mean[0], new_clip->image_mean[1], new_clip->image_mean[2]); LOG_INF("v_image_std %f %f %f\n", new_clip->image_std[0], new_clip->image_std[1], new_clip->image_std[2]); LOG_INF("v_image_grid_pinpoints: "); - for (int i = 0; i < 32 && (hparams.image_grid_pinpoints[i] != 0); ++i) { - LOG_INF("%d ", hparams.image_grid_pinpoints[i]); + for (const auto & pp : hparams.image_grid_pinpoints) { + LOG_INF("%d ", pp); + } + LOG_INF("\n"); + LOG_INF("v_vision_feature_layer: "); + for (const auto & feature_layer: hparams.vision_feature_layer) { + LOG_INF("%d ", feature_layer); } LOG_INF("\n"); LOG_INF("v_mm_patch_merge_type: %s\n", hparams.mm_patch_merge_type); @@ -2235,10 +2276,10 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli } } } else { - if (params.image_grid_pinpoints[0] != 0) { + if (!params.image_grid_pinpoints.empty()) { // "spatial_unpad" with "anyres" processing for llava-1.6 std::vector> possible_resolutions; - for (int i = 0; i < 32 && params.image_grid_pinpoints[i] != 0; i+=2) { + for (size_t i = 0; i < params.image_grid_pinpoints.size(); i+=2) { possible_resolutions.push_back({params.image_grid_pinpoints[i], params.image_grid_pinpoints[i+1]}); } std::pair best_resolution = select_best_resolution({img->nx, img->ny}, possible_resolutions); @@ -2404,7 +2445,14 @@ const char * clip_patch_merge_type(const struct clip_ctx * ctx) { } const int32_t * clip_image_grid(const struct clip_ctx * ctx) { - return ctx->vision_model.hparams.image_grid_pinpoints; + if (ctx->vision_model.hparams.image_grid_pinpoints.size()) { + return &ctx->vision_model.hparams.image_grid_pinpoints.front(); + } + return nullptr; +} + +size_t get_clip_image_grid_size(const struct clip_ctx * ctx) { + return ctx->vision_model.hparams.image_grid_pinpoints.size(); } int clip_n_patches(const struct clip_ctx * ctx) { @@ -2929,6 +2977,28 @@ bool clip_is_qwen2vl(const struct clip_ctx * ctx) { return ctx->has_qwen2vl_merger; } +// Determine the number of encoder layers to iterate over +int get_deepest_feature_layer(const struct clip_ctx * ctx) { + // Get the index of the second to last layer; this is the + // default for models that have a llava projector + const auto & hparams = ctx->vision_model.hparams; + int n_layer = hparams.n_layer - 1; + int deepest_feature_layer = -1; + + // Handle other projectors; incrementing here indicates that we + // should use the last encoder layer for the vision features. + if (ctx->has_minicpmv_projector || ctx->has_glm_projector || ctx->has_qwen2vl_merger) { + n_layer += 1; + } + + // If we set explicit vision feature layers, only go up to the deepest one + for (const auto & feature_layer : hparams.vision_feature_layer) { + if (feature_layer > deepest_feature_layer) { + deepest_feature_layer = feature_layer; + } + } + return deepest_feature_layer < 0 ? n_layer : deepest_feature_layer; +} bool clip_encode_float_image (struct clip_ctx * ctx, int n_threads, float * img, int h, int w, float * vec) { clip_image_f32 clip_img; diff --git a/examples/llava/clip.h b/examples/llava/clip.h index f557122008..ce6f61944c 100644 --- a/examples/llava/clip.h +++ b/examples/llava/clip.h @@ -55,6 +55,7 @@ CLIP_API int32_t clip_hidden_size(const struct clip_ctx * ctx); CLIP_API const char * clip_patch_merge_type(const struct clip_ctx * ctx); CLIP_API const int32_t * clip_image_grid(const struct clip_ctx * ctx); +CLIP_API size_t get_clip_image_grid_size(const struct clip_ctx * ctx); CLIP_API int clip_n_patches (const struct clip_ctx * ctx); CLIP_API int clip_n_patches_by_img (const struct clip_ctx * ctx, struct clip_image_f32 * img); @@ -92,11 +93,13 @@ CLIP_API bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, cons CLIP_API bool clip_model_quantize(const char * fname_inp, const char * fname_out, int itype); CLIP_API int clip_is_minicpmv(const struct clip_ctx * ctx); +CLIP_API bool clip_is_glm(const struct clip_ctx * ctx); CLIP_API bool clip_is_qwen2vl(const struct clip_ctx * ctx); +CLIP_API int get_deepest_feature_layer(const struct clip_ctx * ctx); + CLIP_API bool clip_encode_float_image (struct clip_ctx * ctx, int n_threads, float * img, int h, int w, float * vec); -CLIP_API bool clip_is_glm(const struct clip_ctx * ctx); #ifdef __cplusplus } diff --git a/examples/llava/convert_image_encoder_to_gguf.py b/examples/llava/convert_image_encoder_to_gguf.py index 4fa1d6ceae..de29687ec9 100644 --- a/examples/llava/convert_image_encoder_to_gguf.py +++ b/examples/llava/convert_image_encoder_to_gguf.py @@ -6,7 +6,7 @@ import re import torch import numpy as np from gguf import * -from transformers import CLIPModel, CLIPProcessor, CLIPVisionModel +from transformers import CLIPModel, CLIPProcessor, CLIPVisionModel, SiglipVisionModel TEXT = "clip.text" VISION = "clip.vision" @@ -37,6 +37,18 @@ def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: b def get_tensor_name(name: str) -> str: + # Standardize the transformers llava next keys for + # image newline / mm projector with the classes in haotian-liu LLaVA + if name == "image_newline": + return "model.image_newline" + if name.startswith("multi_modal_projector"): + name = name.replace("multi_modal_projector", "mm") + if "linear_1" in name: + name = name.replace("linear_1", "0") + if "linear_2" in name: + name = name.replace("linear_2", "2") + return name + if "projection" in name: return name if "mm_projector" in name: @@ -83,8 +95,14 @@ ap.add_argument("--vision-only", action="store_true", required=False, help="Save a vision-only model. It can't be used to encode texts") ap.add_argument("--clip-model-is-vision", action="store_true", required=False, help="The clip model is a pure vision model (ShareGPT4V vision extract for example)") -ap.add_argument("--clip-model-is-openclip", action="store_true", required=False, + +# Selectable visual encoders that are compatible with this script +encoder_group = ap.add_mutually_exclusive_group() +encoder_group.add_argument("--clip-model-is-openclip", action="store_true", required=False, help="The clip model is from openclip (for ViT-SO400M type))") +encoder_group.add_argument("--clip-model-is-siglip", action="store_true", required=False, + help="the visual encoder is Siglip.") + ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.") ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp") ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None) @@ -109,7 +127,12 @@ if args.use_f32: # output in the same directory as the model if output_dir is None dir_model = args.model_dir -if args.clip_model_is_vision or not os.path.exists(dir_model + "/vocab.json") or args.clip_model_is_openclip: +if ( + args.clip_model_is_vision or + not os.path.exists(dir_model + "/vocab.json") or + args.clip_model_is_openclip or + args.clip_model_is_siglip +): vocab = None tokens = None else: @@ -137,7 +160,10 @@ ftype = 1 if args.use_f32: ftype = 0 -if args.clip_model_is_vision or args.clip_model_is_openclip: +if args.clip_model_is_siglip: + model = SiglipVisionModel.from_pretrained(dir_model) + processor = None +elif args.clip_model_is_vision or args.clip_model_is_openclip: model = CLIPVisionModel.from_pretrained(dir_model) processor = None else: @@ -187,26 +213,71 @@ else: if has_text_encoder: assert t_hparams is not None assert tokens is not None + if args.clip_model_is_siglip: + text_projection_dim = 0 + else: + text_projection_dim = t_hparams.get("projection_dim", config["projection_dim"]) # text_model hparams fout.add_uint32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"]) fout.add_uint32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"]) fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"]) - fout.add_uint32("clip.text.projection_dim", t_hparams.get("projection_dim", config["projection_dim"])) + fout.add_uint32("clip.text.projection_dim", text_projection_dim) fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"]) fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, TEXT), t_hparams["layer_norm_eps"]) fout.add_uint32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"]) fout.add_token_list(tokens) + + +def get_non_negative_vision_feature_layers(v_hparams): + """ + Determine the vision feature layer(s) for the llava model, which are indices into the + hidden states of the visual encoder. Note that the hidden states array generally takes the + form: + + [, , ... ] + + so feature indices should be offset as n+1 to get the output of encoder block n. + We convert all vision feature layers to non-negative so that -1 can be used in + the model as an unset value. If no vision feature layer is found, we leave it unset. + """ + num_hidden_layers = v_hparams["num_hidden_layers"] + to_non_negative = lambda layer_idx: layer_idx if layer_idx >= 0 else num_hidden_layers + layer_idx + 1 + feature_layers_key = None + # Key used for llava models in transformers + if "vision_feature_layer" in config: + feature_layers_key = "vision_feature_layer" + # Key used for llava models in the original format + elif "mm_vision_select_layer" in config: + feature_layers_key = "mm_vision_select_layer" + if feature_layers_key is not None: + feature_layers = config[feature_layers_key] + if isinstance(feature_layers, int): + feature_layers = [feature_layers] + return [to_non_negative(feature_layer) for feature_layer in feature_layers] + +# Determine if we have explicitly specified vision feature layers in our config +feature_layers = get_non_negative_vision_feature_layers(v_hparams) + if has_vision_encoder: - # vision_model hparams + # Siglip does not have a visual projector; set projection dim to 0 + if args.clip_model_is_siglip: + visual_projection_dim = 0 + else: + visual_projection_dim = v_hparams.get("projection_dim", config["projection_dim"]) + + # set vision_model hparams fout.add_uint32("clip.vision.image_size", v_hparams["image_size"]) fout.add_uint32("clip.vision.patch_size", v_hparams["patch_size"]) fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"]) fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"]) - fout.add_uint32("clip.vision.projection_dim", v_hparams.get("projection_dim", config["projection_dim"])) + fout.add_uint32("clip.vision.projection_dim", visual_projection_dim) fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"]) fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), v_hparams["layer_norm_eps"]) - block_count = v_hparams["num_hidden_layers"] - 1 if has_llava_projector else v_hparams["num_hidden_layers"] + if feature_layers: + block_count = max(feature_layers) + else: + block_count = v_hparams["num_hidden_layers"] - 1 if has_llava_projector else v_hparams["num_hidden_layers"] fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), block_count) # /** # "image_grid_pinpoints": [ @@ -258,7 +329,8 @@ if has_vision_encoder: fout.add_string("clip.vision.mm_patch_merge_type", v_hparams["mm_patch_merge_type"]) if "mm_projector_type" in v_hparams: fout.add_string("clip.vision.mm_projector_type", v_hparams["mm_projector_type"]) - + if feature_layers: + fout.add_array("clip.vision.feature_layer", feature_layers) if processor is not None: image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean # pyright: ignore[reportAttributeAccessIssue] @@ -274,7 +346,13 @@ fout.add_bool("clip.use_gelu", use_gelu) if has_llava_projector: - model.vision_model.encoder.layers.pop(-1) + # By default, we drop the last layer for llava projector + # models unless we have explicitly set vision feature layers + if feature_layers is None: + model.vision_model.encoder.layers.pop(-1) + else: + model.vision_model.encoder.layers = model.vision_model.encoder.layers[:max(feature_layers)] + projector = torch.load(args.llava_projector) for name, data in projector.items(): name = get_tensor_name(name) diff --git a/examples/llava/llava.cpp b/examples/llava/llava.cpp index 3007140459..518aad3f1f 100644 --- a/examples/llava/llava.cpp +++ b/examples/llava/llava.cpp @@ -353,9 +353,10 @@ static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const cli LOG_INF("%s: %d segments encoded in %8.2f ms\n", __func__, (int)img_res_v.size, (t_img_enc_batch_us - t_img_enc_start_us) / 1000.0); const int32_t * image_grid = clip_image_grid(ctx_clip); + const size_t num_gridpoints = get_clip_image_grid_size(ctx_clip); std::vector> grid_pinpoints; - for (int i = 0; i < 32 && image_grid[i] != 0; i += 2) { + for (size_t i = 0; i < num_gridpoints; i += 2) { grid_pinpoints.push_back({image_grid[i], image_grid[i+1]}); } @@ -405,7 +406,8 @@ bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * } bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) { - int num_max_patches = 6; + // Granite vision uses up to 10 patches + base patch + int num_max_patches = 11; if (clip_is_minicpmv(ctx_clip)) { num_max_patches = 10; } diff --git a/examples/llava/llava_surgery_v2.py b/examples/llava/llava_surgery_v2.py index 2d5b32fe6b..b07c3e323c 100644 --- a/examples/llava/llava_surgery_v2.py +++ b/examples/llava/llava_surgery_v2.py @@ -33,6 +33,33 @@ def save_model(model, file_path, file_type): else: torch.save(model, file_path) +# Helpers to match weight names from specific components or +# determine if a saved shard contains that component +def is_vision_tower(weight_name): + return ( + weight_name.startswith("model.vision_tower") or + weight_name.startswith("vit.") or + weight_name.startswith("vision_tower") + ) + +def is_newline(weight_name): + return ( + weight_name.startswith("model.image_newline") or + weight_name.startswith("image_newline") + ) + +def is_mm_projector(weight_name): + return ( + weight_name.startswith("model.mm_projector") or + weight_name.startswith("vision_proj.") or + weight_name.startswith("multi_modal_projector") + ) + +def newline_criteria(checkpoint): + return any(is_newline(k) for k in checkpoint.keys()) + +def proj_criteria(checkpoint): + return any(is_mm_projector(k) for k in checkpoint.keys()) # Adapted function to clean vision tower from checkpoint def clean_vision_tower_from_checkpoint(checkpoint_path): @@ -40,7 +67,7 @@ def clean_vision_tower_from_checkpoint(checkpoint_path): # file_type = 'pytorch' model_path = os.path.dirname(checkpoint_path) print(f"Searching for vision tower tensors in {checkpoint_path}") - clip_tensors = [k for k, v in checkpoint.items() if (k.startswith("model.vision_tower") or k.startswith("vit."))] + clip_tensors = [k for k, v in checkpoint.items() if is_vision_tower(k)] if len(clip_tensors) > 0: print(f"Found {len(clip_tensors)} tensors to extract from {checkpoint_path}") @@ -84,12 +111,6 @@ def find_relevant_checkpoints(checkpoint_paths, newline_criteria, projector): return newline_checkpoint_path, projector_checkpoint_path -def newline_criteria(checkpoint): - return any(k.startswith("model.image_newline") for k in checkpoint.keys()) - -def proj_criteria(checkpoint): - return any(k.startswith("model.mm_projector") or k.startswith("vision_proj.") for k in checkpoint.keys()) - # Command-line interface setup ap = argparse.ArgumentParser() @@ -123,14 +144,14 @@ first_checkpoint = None if newline_checkpoint_path is not None: print(f"Taking newline from {newline_checkpoint_path}") first_checkpoint, file_type = load_model(newline_checkpoint_path) - first_mm_tensors = [k for k, v in first_checkpoint.items() if k.startswith("model.image_newline")] + first_mm_tensors = [k for k, v in first_checkpoint.items() if is_newline(k)] # Load the checkpoint mm_tensors = [] last_checkpoint = None if projector_checkpoint_path is not None: last_checkpoint, file_type = load_model(projector_checkpoint_path) - mm_tensors = [k for k, v in last_checkpoint.items() if k.startswith("model.mm_projector") or k.startswith("vision_proj.")] + mm_tensors = [k for k, v in last_checkpoint.items() if is_mm_projector(k)] if len(mm_tensors) == 0: if last_checkpoint is not None: @@ -155,5 +176,5 @@ if len(projector) > 0: save_model(projector, f"{args.model}/llava.projector", 'pytorch') print("Done!") -print(f"Now you can convert {args.model} to a a regular LLaMA GGUF file.") +print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.") print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.") From 34a846b5847a18d133b360074f1fb485b2632b2d Mon Sep 17 00:00:00 2001 From: lhez Date: Mon, 24 Feb 2025 13:47:07 -0800 Subject: [PATCH 016/188] opencl: fix for small models (#11950) * opencl: fix small shape gemv, remove unused extensions * opencl: fix `transpose_16`, `dump_tensor`, enforce subgroup size * opencl: fix for token length < 4 * opencl: use wave size of 64 for all Adreno GPUs --------- Co-authored-by: Shawn Gu Co-authored-by: Skyler Szot --- ggml/src/ggml-opencl/ggml-opencl.cpp | 54 +++++++++---------- ggml/src/ggml-opencl/kernels/ggml-opencl.cl | 3 ++ .../kernels/ggml-opencl_gemv_noshuffle.cl | 13 +++-- .../ggml-opencl_gemv_noshuffle_general.cl | 13 +++-- .../kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl | 11 +++- .../kernels/ggml-opencl_transpose_16.cl | 32 +++++------ 6 files changed, 67 insertions(+), 59 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 7a0f94cf24..f590624608 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -444,19 +444,8 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { backend_ctx->gpu_family = GPU_FAMILY::ADRENO; backend_ctx->adreno_gen = get_adreno_gpu_gen(default_device->name); - // Default wave size is 128, A8x uses 64. - if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X) { - backend_ctx->adreno_wave_size = 64; - } else if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X || - backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) { - backend_ctx->adreno_wave_size = 128; - } else { - backend_ctx->adreno_wave_size = 128; - GGML_LOG_WARN("ggml_opencl: Unsupported Adreno GPU: %s, " - "using wave size %d, " - "may not work as expected\n", - backend_ctx->device_name.c_str(), backend_ctx->adreno_wave_size); - } + // Use wave size of 64 for all Adreno GPUs. + backend_ctx->adreno_wave_size = 64; } else if (strstr(default_device->name, "Intel")) { backend_ctx->gpu_family = GPU_FAMILY::INTEL; } else { @@ -1376,6 +1365,11 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, int M = tensor->ne[1]; // ne01 int K = tensor->ne[0]; // ne00 + //For matrix-vector multiplication kernel, we assume K is a multiple of 32 + GGML_ASSERT(K % 32 == 0); + //For transpose kernels, we assume K is a multiple of 4 (satisfied by prior assert), and M is a multiple of 4 + GGML_ASSERT(M % 4 == 0); + // transpose is out of place, so we need to allocate transposed buffers // <----------------------------------------------------------------------------------> // // use sub_buffer of max buffer size instead @@ -1416,36 +1410,36 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, cl_mem qT_d_image1D; cl_mem dT_d_image1D; - cl_image_format img_fmt_1d = { CL_RGBA, CL_FLOAT }; + cl_image_format img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT }; cl_image_desc img_desc_1d; memset(&img_desc_1d, 0, sizeof(img_desc_1d)); img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = M * K / 8 / 4; + img_desc_1d.image_width = M * K / 4 / 4; img_desc_1d.buffer = extra->q; q_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err); CL_CHECK(err); - img_fmt_1d = { CL_RGBA, CL_FLOAT }; + img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT }; memset(&img_desc_1d, 0, sizeof(img_desc_1d)); img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = M * K / 8 / 4; + img_desc_1d.image_width = M * K / 4 / 4; img_desc_1d.buffer = qT_d; qT_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err); CL_CHECK(err); - img_fmt_1d = { CL_RGBA, CL_FLOAT }; + img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT }; memset(&img_desc_1d, 0, sizeof(img_desc_1d)); img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = M * K / 32 / 4 / 2; + img_desc_1d.image_width = M * K / 32 / 4; img_desc_1d.buffer = extra->d; d_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err); CL_CHECK(err); - img_fmt_1d = { CL_RGBA, CL_FLOAT }; + img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT }; memset(&img_desc_1d, 0, sizeof(img_desc_1d)); img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = M * K / 32 / 4 / 2; + img_desc_1d.image_width = M * K / 32 / 4; img_desc_1d.buffer = dT_d; dT_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err); CL_CHECK(err); @@ -1454,8 +1448,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, // set up and call the transpose kernels // <----------------------------------------------------------------------------------> // // weights - int height_q = M / 8; - int width_q = K / 8 / 4; + int height_q = M / 4; + int width_q = K / 4 / 4; kernel = backend_ctx->kernel_transpose_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_d_image1D)); @@ -1469,8 +1463,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clWaitForEvents(1, &evt)); // scales - int height_s = M / 8; - int width_s = K / 32 / 8; + int height_s = M / 4; + int width_s = K / 32 / 4; kernel = backend_ctx->kernel_transpose_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &d_d_image1D)); @@ -1864,7 +1858,6 @@ static void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tenso void * buf_d; #endif -#ifdef GGML_USE_OPENCL // Make sure everything is done. CL_CHECK(clFinish(queue)); @@ -1900,7 +1893,6 @@ static void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tenso extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); CL_CHECK(clFinish(queue)); #endif // GGML_OPENCL_SOA_Q -#endif // GGML_USE_OPENCL // Open file and dump. char fname[512]; @@ -2865,6 +2857,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(status); int height_B = N/4; + if (height_B == 0) { + height_B = 1; + } int width_B = K/4; int padded_height_B = (N + padding)/4; @@ -3013,11 +3008,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co } if (N == 1) { - local_work_size[0] = backend_ctx->adreno_wave_size; // localsize + size_t wavesize = backend_ctx->adreno_wave_size; + local_work_size[0] = wavesize; // localsize local_work_size[1] = 4; // reduce factor local_work_size[2] = 1; - global_work_size[0] = M / 2; + global_work_size[0] = (((M / 2) + wavesize - 1) / wavesize) * wavesize; global_work_size[1] = 4; // reduce factor global_work_size[2] = 1; } diff --git a/ggml/src/ggml-opencl/kernels/ggml-opencl.cl b/ggml/src/ggml-opencl/kernels/ggml-opencl.cl index d3cfb2f91e..8882a8c9c6 100644 --- a/ggml/src/ggml-opencl/kernels/ggml-opencl.cl +++ b/ggml/src/ggml-opencl/kernels/ggml-opencl.cl @@ -1797,6 +1797,9 @@ kernel void kernel_mul_mat_f16_f16( //------------------------------------------------------------------------------ // mul_mat_f16_f32_1row //------------------------------------------------------------------------------ +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif kernel void kernel_mul_mat_f16_f32_1row( global char * src0, ulong offset0, diff --git a/ggml/src/ggml-opencl/kernels/ggml-opencl_gemv_noshuffle.cl b/ggml/src/ggml-opencl/kernels/ggml-opencl_gemv_noshuffle.cl index 5e195411d6..ee5c79f000 100644 --- a/ggml/src/ggml-opencl/kernels/ggml-opencl_gemv_noshuffle.cl +++ b/ggml/src/ggml-opencl/kernels/ggml-opencl_gemv_noshuffle.cl @@ -1,9 +1,11 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable #pragma OPENCL EXTENSION cl_khr_subgroups : enable -#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable -#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable -#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#ifdef cl_qcom_reqd_sub_group_size #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif // assume #define QK4_0 32 @@ -186,8 +188,9 @@ total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \ total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \ - -__attribute__((qcom_reqd_sub_group_size("full"))) +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif __kernel void kernel_gemv_noshuffle( __read_only image1d_buffer_t src0_q, // quantized A global half2 * src0_d, // A scales diff --git a/ggml/src/ggml-opencl/kernels/ggml-opencl_gemv_noshuffle_general.cl b/ggml/src/ggml-opencl/kernels/ggml-opencl_gemv_noshuffle_general.cl index 5bdd4d0676..469d3edef0 100644 --- a/ggml/src/ggml-opencl/kernels/ggml-opencl_gemv_noshuffle_general.cl +++ b/ggml/src/ggml-opencl/kernels/ggml-opencl_gemv_noshuffle_general.cl @@ -1,9 +1,11 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable #pragma OPENCL EXTENSION cl_khr_subgroups : enable -#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable -#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable -#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#ifdef cl_qcom_reqd_sub_group_size #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif // assume #define QK4_0 32 @@ -186,8 +188,9 @@ total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \ total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \ - -__attribute__((qcom_reqd_sub_group_size("full"))) +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif __kernel void kernel_gemv_noshuffle( __read_only image1d_buffer_t src0_q, // quantized A global half2 * src0_d, // A scales diff --git a/ggml/src/ggml-opencl/kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl b/ggml/src/ggml-opencl/kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl index 57768c8033..ecb577b993 100644 --- a/ggml/src/ggml-opencl/kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl +++ b/ggml/src/ggml-opencl/kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl @@ -7,7 +7,16 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable -__attribute__((qcom_reqd_sub_group_size("full"))) +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif + kernel void kernel_mul_mat_Ab_Bi_8x4( global const ushort * src0_q, // quantized A global const half * src0_d, // A scales diff --git a/ggml/src/ggml-opencl/kernels/ggml-opencl_transpose_16.cl b/ggml/src/ggml-opencl/kernels/ggml-opencl_transpose_16.cl index d59a0c05dd..cd4e0afbad 100644 --- a/ggml/src/ggml-opencl/kernels/ggml-opencl_transpose_16.cl +++ b/ggml/src/ggml-opencl/kernels/ggml-opencl_transpose_16.cl @@ -1,4 +1,6 @@ -// 16-bit transpose, loading/storing an 8x8 tile of elements +// 16-bit transpose, loading/storing a 4x4 tile of elements + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable kernel void kernel_transpose_16( __read_only image1d_buffer_t input, @@ -9,24 +11,16 @@ kernel void kernel_transpose_16( const int i = get_global_id(0); const int j = get_global_id(1); - const int i_3 = i<<3; - const int j_3 = j<<3; + const int i_2 = i<<2; + const int j_2 = j<<2; - ushort8 temp0 = as_ushort8(read_imagef(input, (j_3+0)*cols+i)); - ushort8 temp1 = as_ushort8(read_imagef(input, (j_3+1)*cols+i)); - ushort8 temp2 = as_ushort8(read_imagef(input, (j_3+2)*cols+i)); - ushort8 temp3 = as_ushort8(read_imagef(input, (j_3+3)*cols+i)); - ushort8 temp4 = as_ushort8(read_imagef(input, (j_3+4)*cols+i)); - ushort8 temp5 = as_ushort8(read_imagef(input, (j_3+5)*cols+i)); - ushort8 temp6 = as_ushort8(read_imagef(input, (j_3+6)*cols+i)); - ushort8 temp7 = as_ushort8(read_imagef(input, (j_3+7)*cols+i)); + half4 temp0 = read_imageh(input, (j_2+0)*cols+i); + half4 temp1 = read_imageh(input, (j_2+1)*cols+i); + half4 temp2 = read_imageh(input, (j_2+2)*cols+i); + half4 temp3 = read_imageh(input, (j_2+3)*cols+i); - write_imagef(output, (i_3+0)*rows+j, as_float4((ushort8)(temp0.s0, temp1.s0, temp2.s0, temp3.s0, temp4.s0, temp5.s0, temp6.s0, temp7.s0))); - write_imagef(output, (i_3+1)*rows+j, as_float4((ushort8)(temp0.s1, temp1.s1, temp2.s1, temp3.s1, temp4.s1, temp5.s1, temp6.s1, temp7.s1))); - write_imagef(output, (i_3+2)*rows+j, as_float4((ushort8)(temp0.s2, temp1.s2, temp2.s2, temp3.s2, temp4.s2, temp5.s2, temp6.s2, temp7.s2))); - write_imagef(output, (i_3+3)*rows+j, as_float4((ushort8)(temp0.s3, temp1.s3, temp2.s3, temp3.s3, temp4.s3, temp5.s3, temp6.s3, temp7.s3))); - write_imagef(output, (i_3+4)*rows+j, as_float4((ushort8)(temp0.s4, temp1.s4, temp2.s4, temp3.s4, temp4.s4, temp5.s4, temp6.s4, temp7.s4))); - write_imagef(output, (i_3+5)*rows+j, as_float4((ushort8)(temp0.s5, temp1.s5, temp2.s5, temp3.s5, temp4.s5, temp5.s5, temp6.s5, temp7.s5))); - write_imagef(output, (i_3+6)*rows+j, as_float4((ushort8)(temp0.s6, temp1.s6, temp2.s6, temp3.s6, temp4.s6, temp5.s6, temp6.s6, temp7.s6))); - write_imagef(output, (i_3+7)*rows+j, as_float4((ushort8)(temp0.s7, temp1.s7, temp2.s7, temp3.s7, temp4.s7, temp5.s7, temp6.s7, temp7.s7))); + write_imageh(output, (i_2+0)*rows+j, (half4)(temp0.s0, temp1.s0, temp2.s0, temp3.s0)); + write_imageh(output, (i_2+1)*rows+j, (half4)(temp0.s1, temp1.s1, temp2.s1, temp3.s1)); + write_imageh(output, (i_2+2)*rows+j, (half4)(temp0.s2, temp1.s2, temp2.s2, temp3.s2)); + write_imageh(output, (i_2+3)*rows+j, (half4)(temp0.s3, temp1.s3, temp2.s3, temp3.s3)); } From 58d07a8043a1395177cf77b3e4f388e34182ae64 Mon Sep 17 00:00:00 2001 From: Gian-Carlo Pascutto Date: Tue, 25 Feb 2025 10:27:58 +0100 Subject: [PATCH 017/188] metal : copy kernels for quant to F32/F16 conversions (#12017) metal: use dequantize_q templates --------- Co-authored-by: Georgi Gerganov --- ggml/src/ggml-metal/ggml-metal.m | 82 ++++++++++++++++++++++++++-- ggml/src/ggml-metal/ggml-metal.metal | 43 +++++++++++++++ 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m index 087e7f5814..c550142a7d 100644 --- a/ggml/src/ggml-metal/ggml-metal.m +++ b/ggml/src/ggml-metal/ggml-metal.m @@ -407,6 +407,16 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, + GGML_METAL_KERNEL_TYPE_CPY_Q4_0_F32, + GGML_METAL_KERNEL_TYPE_CPY_Q4_0_F16, + GGML_METAL_KERNEL_TYPE_CPY_Q4_1_F32, + GGML_METAL_KERNEL_TYPE_CPY_Q4_1_F16, + GGML_METAL_KERNEL_TYPE_CPY_Q5_0_F32, + GGML_METAL_KERNEL_TYPE_CPY_Q5_0_F16, + GGML_METAL_KERNEL_TYPE_CPY_Q5_1_F32, + GGML_METAL_KERNEL_TYPE_CPY_Q5_1_F16, + GGML_METAL_KERNEL_TYPE_CPY_Q8_0_F32, + GGML_METAL_KERNEL_TYPE_CPY_Q8_0_F16, GGML_METAL_KERNEL_TYPE_CONCAT, GGML_METAL_KERNEL_TYPE_SQR, GGML_METAL_KERNEL_TYPE_SQRT, @@ -1012,6 +1022,16 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, cpy_f32_iq4_nl, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q4_0_F32, cpy_q4_0_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q4_0_F16, cpy_q4_0_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q4_1_F32, cpy_q4_1_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q4_1_F16, cpy_q4_1_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q5_0_F32, cpy_q5_0_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q5_0_F16, cpy_q5_0_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q5_1_F32, cpy_q5_1_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q5_1_F16, cpy_q5_1_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q8_0_F32, cpy_q8_0_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_Q8_0_F16, cpy_q8_0_f16, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQRT, sqrt, true); @@ -1287,6 +1307,18 @@ static bool ggml_metal_supports_op(const struct ggml_backend_metal_device_contex default: return false; } + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + switch (op->type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + return true; + default: + return false; + } default: return false; }; @@ -3899,10 +3931,6 @@ static void ggml_metal_encode_node( case GGML_OP_CPY: case GGML_OP_CONT: { - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - - int nth = MIN(1024, ne00/ggml_blck_size(src0->type)); - id pipeline = nil; switch (src0t) { @@ -3936,7 +3964,47 @@ static void ggml_metal_encode_node( switch (dstt) { case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_BF16_F32].pipeline; break; case GGML_TYPE_BF16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_BF16_BF16].pipeline; break; - default: GGML_ASSERT(false && "not implemented"); + default: GGML_ABORT("not implemented"); + }; + } break; + case GGML_TYPE_Q4_0: + { + switch (dstt) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q4_0_F32].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q4_0_F16].pipeline; break; + default: GGML_ABORT("not implemented"); + }; + } break; + case GGML_TYPE_Q4_1: + { + switch (dstt) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q4_1_F32].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q4_1_F16].pipeline; break; + default: GGML_ABORT("not implemented"); + }; + } break; + case GGML_TYPE_Q5_0: + { + switch (dstt) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q5_0_F32].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q5_0_F16].pipeline; break; + default: GGML_ABORT("not implemented"); + }; + } break; + case GGML_TYPE_Q5_1: + { + switch (dstt) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q5_1_F32].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q5_1_F16].pipeline; break; + default: GGML_ABORT("not implemented"); + }; + } break; + case GGML_TYPE_Q8_0: + { + switch (dstt) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q8_0_F32].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_Q8_0_F16].pipeline; break; + default: GGML_ABORT("not implemented"); }; } break; default: GGML_ABORT("not implemented"); @@ -3966,7 +4034,11 @@ static void ggml_metal_encode_node( [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + int nth = MIN(1024, ne00/ggml_blck_size(src0->type)); + [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; case GGML_OP_SET: { diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 83e7ac9f41..d092a16906 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -4341,6 +4341,49 @@ kernel void kernel_cpy_f32_iq4_nl( } } +template +kernel void kernel_cpy_q_f32( + constant ggml_metal_kargs_cpy & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int i03 = tgpig[2]; + const int i02 = tgpig[1]; + const int i01 = tgpig[0]; + + const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00; + + const int64_t i3 = n/(args.ne2*args.ne1*args.ne0); + const int64_t i2 = (n - i3*args.ne2*args.ne1*args.ne0)/(args.ne1*args.ne0); + const int64_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0; + const int64_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0); + + device const block_q * src_data = (device const block_q *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); + device T4x4 * dst_data = (device T4x4 *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + for (int64_t i00 = tpitg.x; i00 < args.ne00/16; i00 += ntg.x) { + T4x4 temp; + dequantize_func(src_data + i00/nl, i00%nl, temp); + dst_data[i00] = temp; + } +} + +typedef decltype(kernel_cpy_q_f32) cpy_q_f_t; + +template [[host_name("kernel_cpy_q4_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q4_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q5_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q5_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q8_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; + +template [[host_name("kernel_cpy_q4_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q4_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q5_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q5_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q8_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; + kernel void kernel_concat( constant ggml_metal_kargs_concat & args, device const char * src0, From 3e9a2860e996657fc10db8393cf65adc40703082 Mon Sep 17 00:00:00 2001 From: Vitali Lovich Date: Tue, 25 Feb 2025 01:29:33 -0800 Subject: [PATCH 018/188] llama : expose llama_model_n_head_kv in the API (#11997) It's useful to be able to have this from the library layer as it's a key parameter of the model (e.g. to figure out how much KV cache memory is needed). --- include/llama.h | 1 + src/llama-model.cpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/include/llama.h b/include/llama.h index b0726cbe63..479196026b 100644 --- a/include/llama.h +++ b/include/llama.h @@ -477,6 +477,7 @@ extern "C" { LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model); LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model); LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model); // Get the model's RoPE frequency scaling factor LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index f64c3afa02..36a0a009c4 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -3838,6 +3838,10 @@ int32_t llama_model_n_head(const struct llama_model * model) { return model->hparams.n_head(); } +int32_t llama_model_n_head_kv(const struct llama_model * model) { + return model->hparams.n_head_kv(); +} + // deprecated int32_t llama_n_ctx_train(const struct llama_model * model) { return llama_model_n_ctx_train(model); From 4d1051a40ffc2fdff80bc532eea5b9568b85c156 Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Tue, 25 Feb 2025 02:46:05 -0700 Subject: [PATCH 019/188] Add Doc for Converting Granite Vision -> GGUF (#12006) * Add example docs for granite vision Signed-off-by: Alex-Brooks --- examples/llava/README-granitevision.md | 183 +++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 examples/llava/README-granitevision.md diff --git a/examples/llava/README-granitevision.md b/examples/llava/README-granitevision.md new file mode 100644 index 0000000000..d2426dc695 --- /dev/null +++ b/examples/llava/README-granitevision.md @@ -0,0 +1,183 @@ +# Granite Vision + +Download the model and point your `GRANITE_MODEL` environment variable to the path. + +```bash +$ git clone https://huggingface.co/ibm-granite/granite-vision-3.1-2b-preview +$ export GRANITE_MODEL=./granite-vision-3.1-2b-preview +``` + + +### 1. Running llava surgery v2. +First, we need to run the llava surgery script as shown below: + +`python llava_surgery_v2.py -C -m $GRANITE_MODEL` + +You should see two new files (`llava.clip` and `llava.projector`) written into your model's directory, as shown below. + +```bash +$ ls $GRANITE_MODEL | grep -i llava +llava.clip +llava.projector +``` + +We should see that the projector and visual encoder get split out into the llava files. Quick check to make sure they aren't empty: +```python +import os +import torch + +MODEL_PATH = os.getenv("GRANITE_MODEL") +if not MODEL_PATH: + raise ValueError("env var GRANITE_MODEL is unset!") + +encoder_tensors = torch.load(os.path.join(MODEL_PATH, "llava.clip")) +projector_tensors = torch.load(os.path.join(MODEL_PATH, "llava.projector")) + +assert len(encoder_tensors) > 0 +assert len(projector_tensors) > 0 +``` + +If you actually inspect the `.keys()` of the loaded tensors, you should see a lot of `vision_model` tensors in the `encoder_tensors`, and 5 tensors (`'multi_modal_projector.linear_1.bias'`, `'multi_modal_projector.linear_1.weight'`, `'multi_modal_projector.linear_2.bias'`, `'multi_modal_projector.linear_2.weight'`, `'image_newline'`) in the multimodal `projector_tensors`. + + +### 2. Creating the Visual Component GGUF +To create the GGUF for the visual components, we need to write a config for the visual encoder; make sure the config contains the correct `image_grid_pinpoints` + + +Note: we refer to this file as `$VISION_CONFIG` later on. +```json +{ + "_name_or_path": "siglip-model", + "architectures": [ + "SiglipVisionModel" + ], + "image_grid_pinpoints": [ + [384,768], + [384,1152], + [384,1536], + [384,1920], + [384,2304], + [384,2688], + [384,3072], + [384,3456], + [384,3840], + [768,384], + [768,768], + [768,1152], + [768,1536], + [768,1920], + [1152,384], + [1152,768], + [1152,1152], + [1536,384], + [1536,768], + [1920,384], + [1920,768], + [2304,384], + [2688,384], + [3072,384], + [3456,384], + [3840,384] + ], + "mm_patch_merge_type": "spatial_unpad", + "hidden_size": 1152, + "image_size": 384, + "intermediate_size": 4304, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_hidden_layers": 27, + "patch_size": 14, + "layer_norm_eps": 1e-6, + "hidden_act": "gelu_pytorch_tanh", + "projection_dim": 0, + "vision_feature_layer": [-24, -20, -12, -1] +} +``` + +Create a new directory to hold the visual components, and copy the llava.clip/projector files, as well as the vision config into it. + +```bash +$ ENCODER_PATH=$PWD/visual_encoder +$ mkdir $ENCODER_PATH + +$ cp $GRANITE_MODEL/llava.clip $ENCODER_PATH/pytorch_model.bin +$ cp $GRANITE_MODEL/llava.projector $ENCODER_PATH/ +$ cp $VISION_CONFIG $ENCODER_PATH/config.json +``` + +At which point you should have something like this: +```bash +$ ls $ENCODER_PATH +config.json llava.projector pytorch_model.bin +``` + +Now convert the components to GGUF; Note that we also override the image mean/std dev to `[.5,.5,.5]` since we use the siglip visual encoder - in the transformers model, you can find these numbers in the [preprocessor_config.json](https://huggingface.co/ibm-granite/granite-vision-3.1-2b-preview/blob/main/preprocessor_config.json). +```bash +$ python convert_image_encoder_to_gguf.py \ + -m $ENCODER_PATH \ + --llava-projector $ENCODER_PATH/llava.projector \ + --output-dir $ENCODER_PATH \ + --clip-model-is-vision \ + --clip-model-is-siglip \ + --image-mean 0.5 0.5 0.5 --image-std 0.5 0.5 0.5 +``` + +this will create the first GGUF file at `$ENCODER_PATH/mmproj-model-f16.gguf`; we will refer to the abs path of this file as the `$VISUAL_GGUF_PATH.` + + +### 3. Creating the LLM GGUF. +The granite vision model contains a granite LLM as its language model. For now, the easiest way to get the GGUF for LLM is by loading the composite model in `transformers` and exporting the LLM so that it can be directly converted with the normal conversion path. + +First, set the `LLM_EXPORT_PATH` to the path to export the `transformers` LLM to. +``` +$ export LLM_EXPORT_PATH=$PWD/granite_vision_llm +``` + +```python +import os +import transformers + +MODEL_PATH = os.getenv("GRANITE_MODEL") +if not MODEL_PATH: + raise ValueError("env var GRANITE_MODEL is unset!") + +LLM_EXPORT_PATH = os.getenv("LLM_EXPORT_PATH") +if not MODEL_PATH: + raise ValueError("env var LLM_EXPORT_PATH is unset!") + +tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_PATH) + +# NOTE: granite vision support was added to transformers very recently (4.49); +# if you get size mismatches, your version is too old. +# If you are running with an older version, set `ignore_mismatched_sizes=True` +# as shown below; it won't be loaded correctly, but the LLM part of the model that +# we are exporting will be loaded correctly. +model = transformers.AutoModelForImageTextToText.from_pretrained(MODEL_PATH, ignore_mismatched_sizes=True) + +tokenizer.save_pretrained(LLM_EXPORT_PATH) +model.language_model.save_pretrained(LLM_EXPORT_PATH) +``` + +Now you can convert the exported LLM to GGUF with the normal converter in the root of the llama cpp project. +```bash +$ LLM_GGUF_PATH=$LLM_EXPORT_PATH/granite_llm.gguf +... +$ python convert_hf_to_gguf.py --outfile $LLM_GGUF_PATH $LLM_EXPORT_PATH +``` + + +### 4. Running the Model in Llama cpp +Build llama cpp normally; you should have a target binary named `llama-llava-cli`, which you can pass two binaries to. Sample usage: + +Note - the test image shown below can be found [here](https://github-production-user-asset-6210df.s3.amazonaws.com/10740300/415512792-d90d5562-8844-4f34-a0a5-77f62d5a58b5.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20250221%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20250221T054145Z&X-Amz-Expires=300&X-Amz-Signature=86c60be490aa49ef7d53f25d6c973580a8273904fed11ed2453d0a38240ee40a&X-Amz-SignedHeaders=host). + +```bash +$ ./build/bin/llama-llava-cli -m $LLM_GGUF_PATH \ + --mmproj $VISUAL_GGUF_PATH \ + --image cherry_blossom.jpg \ + -c 16384 \ + -p "<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n<|user|>\n\\nWhat type of flowers are in this picture?\n<|assistant|>\n" \ + --temp 0 +``` + +Sample response: `The flowers in the picture are cherry blossoms, which are known for their delicate pink petals and are often associated with the beauty of spring.` From 0b52745649062c04b546aab662cfdb220f4f0621 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Tue, 25 Feb 2025 10:40:22 +0000 Subject: [PATCH 020/188] server: support add_generation_prompt query param (#12062) --- examples/server/utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 4db4c783af..4867b1582f 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -598,7 +598,7 @@ static json oaicompat_completion_params_parse( inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(json_value(body, "tool_choice", std::string("auto"))); inputs.json_schema = json_schema.is_null() ? "" : json_schema.dump(); inputs.grammar = grammar; - inputs.add_generation_prompt = true; + inputs.add_generation_prompt = json_value(body, "add_generation_prompt", true); inputs.use_jinja = use_jinja; inputs.parallel_tool_calls = json_value(body, "parallel_tool_calls", false); inputs.extract_reasoning = reasoning_format != COMMON_REASONING_FORMAT_NONE; From 61d4f39dfeaf3bbcf4e1535ac03953a5d766680b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20O?= Date: Tue, 25 Feb 2025 12:04:45 +0100 Subject: [PATCH 021/188] vulkan: implement more backpropagation operators (#11914) * vulkan: implement GGML_OP_ROPE_BACK * vulkan: implement GGML_OP_RMS_NORM_BACK * vulkan: implement GGML_OP_SILU_BACK * vulkan: implement GGML_OP_SOFTMAX_BACK --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 101 ++++++++++++++++-- .../vulkan-shaders/rms_norm_back.comp | 55 ++++++++++ .../ggml-vulkan/vulkan-shaders/rope_head.comp | 5 + .../ggml-vulkan/vulkan-shaders/silu_back.comp | 26 +++++ .../vulkan-shaders/soft_max_back.comp | 50 +++++++++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 3 + 6 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 131ee1ea04..daef8b89d4 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -241,15 +241,18 @@ struct vk_device_struct { vk_pipeline pipeline_norm_f32; vk_pipeline pipeline_group_norm_f32; vk_pipeline pipeline_rms_norm_f32; + vk_pipeline pipeline_rms_norm_back_f32; vk_pipeline pipeline_gelu_f32; vk_pipeline pipeline_gelu_quick_f32; vk_pipeline pipeline_silu_f32; + vk_pipeline pipeline_silu_back_f32; vk_pipeline pipeline_relu_f32; vk_pipeline pipeline_leaky_relu_f32; vk_pipeline pipeline_tanh_f32; vk_pipeline pipeline_diag_mask_inf_f32; vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16; vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512; + vk_pipeline pipeline_soft_max_back_f32; vk_pipeline pipeline_rope_norm_f32, pipeline_rope_norm_f16; vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16; vk_pipeline pipeline_rope_multi_f32, pipeline_rope_multi_f16; @@ -504,6 +507,7 @@ struct vk_op_rope_push_constants { uint32_t s1; uint32_t s2; int32_t sections[4]; + uint32_t is_back; }; struct vk_op_soft_max_push_constants { @@ -2121,6 +2125,7 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); @@ -2180,6 +2185,7 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_gelu_f32, "gelu_f32", gelu_f32_len, gelu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_gelu_quick_f32, "gelu_quick_f32", gelu_quick_f32_len, gelu_quick_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_silu_f32, "silu_f32", silu_f32_len, silu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_relu_f32, "relu_f32", relu_f32_len, relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_leaky_relu_f32, "leaky_relu_f32", leaky_relu_f32_len, leaky_relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_tanh_f32, "tanh_f32", tanh_f32_len, tanh_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); @@ -2190,6 +2196,7 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_wg512, "soft_max_f32_wg512", soft_max_f32_len, soft_max_f32_data, "main", 3, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 512 }, 1); ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_f16, "soft_max_f32_f16", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 3, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_f16_wg512, "soft_max_f32_f16_wg512", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 3, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 512 }, 1); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_back_f32, "soft_max_back_f32", soft_max_back_f32_len, soft_max_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); ggml_vk_create_pipeline(device, device->pipeline_rope_norm_f32, "rope_norm_f32", rope_norm_f32_len, rope_norm_f32_data, "main", 4, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_rope_neox_f32, "rope_neox_f32", rope_neox_f32_len, rope_neox_f32_data, "main", 4, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); @@ -5283,6 +5290,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const case GGML_OP_CONT: case GGML_OP_DUP: return ggml_vk_get_cpy_pipeline(ctx, src0, dst, dst->type); + case GGML_OP_SILU_BACK: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_silu_back_f32; + } + return nullptr; case GGML_OP_NORM: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_norm_f32; @@ -5298,6 +5310,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return ctx->device->pipeline_rms_norm_f32; } return nullptr; + case GGML_OP_RMS_NORM_BACK: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rms_norm_back_f32; + } + return nullptr; case GGML_OP_UNARY: switch (ggml_get_unary_op(dst)) { case GGML_UNARY_OP_SILU: @@ -5344,7 +5361,13 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return src0->ne[0] > 1024 ? ctx->device->pipeline_soft_max_f32_f16_wg512 : ctx->device->pipeline_soft_max_f32_f16; } return nullptr; + case GGML_OP_SOFT_MAX_BACK: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_soft_max_back_f32; + } + return nullptr; case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: { const int mode = ((const int32_t *) dst->op_params)[2]; const bool is_neox = mode & GGML_ROPE_TYPE_NEOX; @@ -5672,7 +5695,9 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co switch (op) { case GGML_OP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_RMS_NORM_BACK: case GGML_OP_SOFT_MAX: + case GGML_OP_SOFT_MAX_BACK: case GGML_OP_SUM_ROWS: case GGML_OP_ARGMAX: { @@ -5696,6 +5721,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co } break; case GGML_OP_DIAG_MASK_INF: case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: elements = { (uint32_t)ggml_nrows(src0), (uint32_t)ne00, 1 }; break; case GGML_OP_GET_ROWS: @@ -5791,7 +5817,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { vk_subbuffer{ d_X, x_buf_offset, x_sz }, subbuf_y, vk_subbuffer{ d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); - } else if (op == GGML_OP_ROPE) { + } else if (op == GGML_OP_ROPE || op == GGML_OP_ROPE_BACK) { // Empty src2 is possible in rope, but the shader needs a buffer vk_subbuffer subbuf_z; if (use_src2) { @@ -6313,6 +6339,10 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const }, dryrun); } +static void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool dryrun = false) { + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_SILU_BACK, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }, dryrun); +} + static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst, bool dryrun = false) { float * op_params = (float *)dst->op_params; @@ -6335,6 +6365,11 @@ static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }, dryrun); } +static void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool dryrun = false) { + float * op_params = (float *)dst->op_params; + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }, dryrun); +} + static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst, bool dryrun = false) { ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }, dryrun); } @@ -6370,7 +6405,12 @@ static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, }, dryrun); } -static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, bool dryrun = false) { +static void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool dryrun = false) { + float * op_params = (float *)dst->op_params; + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_SOFT_MAX_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], op_params[1] }, dryrun); +} + +static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, bool backprop, bool dryrun = false) { const int n_dims = ((int32_t *) dst->op_params)[1]; const int mode = ((int32_t *) dst->op_params)[2]; // const int n_ctx = ((int32_t *) dst->op_params)[3]; @@ -6398,7 +6438,7 @@ static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, cons (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1]}, theta_scale, src2 != nullptr, (uint32_t)src0->ne[2], s1, s2, - sections[0], sections[1], sections[2], sections[3], + sections[0], sections[1], sections[2], sections[3], backprop }, dryrun); } @@ -7319,12 +7359,16 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_CPY: case GGML_OP_CONT: case GGML_OP_DUP: + case GGML_OP_SILU_BACK: case GGML_OP_NORM: case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_RMS_NORM_BACK: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: + case GGML_OP_SOFT_MAX_BACK: case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: case GGML_OP_ARGSORT: @@ -7377,13 +7421,17 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_CPY: case GGML_OP_CONT: case GGML_OP_DUP: + case GGML_OP_SILU_BACK: case GGML_OP_NORM: case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_RMS_NORM_BACK: case GGML_OP_UNARY: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: + case GGML_OP_SOFT_MAX_BACK: case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: case GGML_OP_ARGSORT: case GGML_OP_SUM: case GGML_OP_SUM_ROWS: @@ -7475,6 +7523,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_DUP: ggml_vk_cpy(ctx, compute_ctx, src0, node, dryrun); + break; + case GGML_OP_SILU_BACK: + ggml_vk_silu_back(ctx, compute_ctx, src0, src1, node, dryrun); + break; case GGML_OP_NORM: ggml_vk_norm(ctx, compute_ctx, src0, node, dryrun); @@ -7487,6 +7539,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_RMS_NORM: ggml_vk_rms_norm(ctx, compute_ctx, src0, node, dryrun); + break; + case GGML_OP_RMS_NORM_BACK: + ggml_vk_rms_norm_back(ctx, compute_ctx, src0, src1, node, dryrun); + break; case GGML_OP_UNARY: switch (ggml_get_unary_op(node)) { @@ -7508,9 +7564,17 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_SOFT_MAX: ggml_vk_soft_max(ctx, compute_ctx, src0, src1, node, dryrun); + break; + case GGML_OP_SOFT_MAX_BACK: + ggml_vk_soft_max_back(ctx, compute_ctx, src0, src1, node, dryrun); + break; case GGML_OP_ROPE: - ggml_vk_rope(ctx, compute_ctx, src0, src1, src2, node, dryrun); + ggml_vk_rope(ctx, compute_ctx, src0, src1, src2, node, false, dryrun); + + break; + case GGML_OP_ROPE_BACK: + ggml_vk_rope(ctx, compute_ctx, src0, src1, src2, node, true, dryrun); break; case GGML_OP_ARGSORT: @@ -7636,12 +7700,16 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_tensor * case GGML_OP_CPY: case GGML_OP_CONT: case GGML_OP_DUP: + case GGML_OP_SILU_BACK: case GGML_OP_NORM: case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_RMS_NORM_BACK: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: + case GGML_OP_SOFT_MAX_BACK: case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: case GGML_OP_RESHAPE: case GGML_OP_VIEW: case GGML_OP_PERMUTE: @@ -8560,6 +8628,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_REPEAT_BACK: return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: case GGML_OP_NONE: case GGML_OP_RESHAPE: case GGML_OP_VIEW: @@ -8576,6 +8645,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_MUL: case GGML_OP_DIV: case GGML_OP_CONCAT: + case GGML_OP_SILU_BACK: + case GGML_OP_RMS_NORM_BACK: case GGML_OP_UPSCALE: case GGML_OP_SCALE: case GGML_OP_SQR: @@ -8585,6 +8656,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_PAD: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: + case GGML_OP_SOFT_MAX_BACK: case GGML_OP_ARGSORT: case GGML_OP_SUM: case GGML_OP_SUM_ROWS: @@ -8976,15 +9048,22 @@ static void ggml_vk_check_results_0(ggml_tensor * tensor) { tensor_clone = ggml_group_norm(ggml_ctx, src_clone[0], *(int *)tensor->op_params, ((float *)tensor->op_params)[1]); } else if (tensor->op == GGML_OP_RMS_NORM) { tensor_clone = ggml_rms_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params); + } else if (tensor->op == GGML_OP_RMS_NORM_BACK) { + const float eps = ((float *) tensor->op_params)[0]; + tensor_clone = ggml_rms_norm_back(ggml_ctx, src_clone[0], src_clone[1], eps); + } else if (tensor->op == GGML_OP_SILU_BACK) { + tensor_clone = ggml_silu_back(ggml_ctx, src_clone[0], src_clone[1]); } else if (tensor->op == GGML_OP_SOFT_MAX) { if (src1 != nullptr) { tensor_clone = ggml_soft_max_ext(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); } else { tensor_clone = ggml_soft_max(ggml_ctx, src_clone[0]); } + } else if (tensor->op == GGML_OP_SOFT_MAX_BACK) { + tensor_clone = ggml_soft_max_ext_back(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); } else if (tensor->op == GGML_OP_DIAG_MASK_INF) { tensor_clone = ggml_diag_mask_inf(ggml_ctx, src_clone[0], *(int *)tensor->op_params); - } else if (tensor->op == GGML_OP_ROPE) { + } else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) { const int n_dims = ((int32_t *) tensor->op_params)[1]; const int mode = ((int32_t *) tensor->op_params)[2]; //const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3]; @@ -8997,9 +9076,17 @@ static void ggml_vk_check_results_0(ggml_tensor * tensor) { const float beta_slow = ((float *) tensor->op_params)[10]; if (mode & GGML_ROPE_TYPE_MROPE) { int32_t *sections = ((int32_t *) tensor->op_params) + 11; - tensor_clone = ggml_rope_multi(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + if (tensor->op == GGML_OP_ROPE) { + tensor_clone = ggml_rope_multi(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } else { + tensor_clone = ggml_rope_multi_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } } else { - tensor_clone = ggml_rope_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + if (tensor->op == GGML_OP_ROPE) { + tensor_clone = ggml_rope_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } else { + tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } } } else if (tensor->op == GGML_OP_UNARY) { switch (ggml_get_unary_op(tensor)) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp new file mode 100644 index 0000000000..76009f3df6 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp @@ -0,0 +1,55 @@ +#version 450 + +#include "generic_head.comp" +#include "types.comp" + +#extension GL_EXT_control_flow_attributes : enable +#define BLOCK_SIZE 512 + +layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer G {A_TYPE data_a[];}; +layout (binding = 1) readonly buffer X {B_TYPE data_b[];}; +layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; + +shared FLOAT_TYPE sum_xx[BLOCK_SIZE]; +shared FLOAT_TYPE sum_xg[BLOCK_SIZE]; + +void main() { + const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; + const uint tid = gl_LocalInvocationID.x; + + // Compute derivative of x[i]/norm(x) = g[i]/norm(x) - x[i] dot(x,g)/KX / norm(x)^1.5 + + // partial sums for thread in warp + sum_xx[tid] = FLOAT_TYPE(0.0f); + sum_xg[tid] = FLOAT_TYPE(0.0f); + + [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + const FLOAT_TYPE gi = FLOAT_TYPE(data_a[row*p.KX + col]); + const FLOAT_TYPE xi = FLOAT_TYPE(data_b[row*p.KX + col]); + sum_xx[tid] += xi * xi; + sum_xg[tid] += xi * gi; + } + + // sum up partial sums and write back result + barrier(); + [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) { + sum_xx[tid] += sum_xx[tid + s]; + sum_xg[tid] += sum_xg[tid + s]; + } + barrier(); + } + + const FLOAT_TYPE eps = FLOAT_TYPE(p.param1); + const FLOAT_TYPE mean = sum_xx[0] / FLOAT_TYPE(p.KX); + const FLOAT_TYPE scale_g = inversesqrt(mean + eps); + const FLOAT_TYPE scale_x = -scale_g * sum_xg[0] / (sum_xx[0] + FLOAT_TYPE(p.KX) * eps); + + [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + data_d[row*p.KX + col] = D_TYPE( + scale_g * FLOAT_TYPE(data_a[row*p.KX + col]) + + scale_x * FLOAT_TYPE(data_b[row*p.KX + col])); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp index 38075b7555..96c9c4cbd3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp @@ -29,6 +29,7 @@ layout (push_constant) uniform parameter { uint s1; uint s2; int sections[4]; + uint is_back; } p; float rope_yarn_ramp(const float low, const float high, const uint i0) { @@ -48,6 +49,10 @@ void rope_yarn(const float theta_extrap, const uint i0, out float cos_theta, out // Get n-d magnitude scaling corrected for interpolation mscale *= 1.0f + 0.1f * log(1.0f / p.freq_scale); } + // Backprogagation uses inverted rotation + if (p.is_back != 0) { + theta = -theta; + } cos_theta = cos(theta) * mscale; sin_theta = sin(theta) * mscale; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp b/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp new file mode 100644 index 0000000000..f9afa9b13c --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp @@ -0,0 +1,26 @@ +#version 450 + +#include "generic_head.comp" +#include "types.comp" + +#extension GL_EXT_control_flow_attributes : enable + +layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer G {A_TYPE data_g[];}; +layout (binding = 1) readonly buffer X {B_TYPE data_x[];}; +layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; + +void main() { + const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x; + + if (i >= p.KX) { + return; + } + + // Compute derivative of SiLU(x): 1/(1+exp(-x)) - x*exp(-x)/(1+exp(-x))^2 + + const float xi = float(data_x[i]); + const float s = 1.0f / (1.0f + exp(-xi)); + data_d[i] = D_TYPE(data_g[i] * (s + xi * s * (1 - s))); +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp b/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp new file mode 100644 index 0000000000..29bd77d7e1 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp @@ -0,0 +1,50 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable + +#include "generic_head.comp" +#include "types.comp" + +layout(constant_id = 0) const uint BLOCK_SIZE = 32; +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +// In this shader Y = softmax(X) and X is not provided as input. + +layout (binding = 0) readonly buffer G {A_TYPE data_g[];}; +layout (binding = 1) readonly buffer Y {B_TYPE data_y[];}; +layout (binding = 2) buffer D {D_TYPE data_d[];}; + +shared FLOAT_TYPE sum_yg[BLOCK_SIZE]; + +void main() { + const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; + const uint tid = gl_LocalInvocationID.x; + + FLOAT_TYPE scale = p.param1; + + // partial sums for thread in warp + sum_yg[tid] = FLOAT_TYPE(0.0f); + + [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + const FLOAT_TYPE gi = FLOAT_TYPE(data_g[row*p.KX + col]); + const FLOAT_TYPE yi = FLOAT_TYPE(data_y[row*p.KX + col]); + sum_yg[tid] += yi * gi; + } + + // sum up partial sums and write back result + barrier(); + [[unroll]] for (uint s = BLOCK_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) { + sum_yg[tid] += sum_yg[tid + s]; + } + barrier(); + } + + const FLOAT_TYPE dot_yg = sum_yg[0]; + + [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + data_d[row*p.KX + col] = D_TYPE(scale + * (FLOAT_TYPE(data_g[row*p.KX + col]) - dot_yg) + * FLOAT_TYPE(data_y[row*p.KX + col])); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 3128c3d507..fc8bbb9460 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -427,6 +427,7 @@ void process_shaders() { string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); + string_to_spv("rms_norm_back_f32", "rms_norm_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("cpy_f32_f32", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("cpy_f32_f16", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float16_t"}}); @@ -477,6 +478,7 @@ void process_shaders() { string_to_spv("gelu_f32", "gelu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("gelu_quick_f32", "gelu_quick.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("silu_f32", "silu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); + string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("relu_f32", "relu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("leaky_relu_f32", "leaky_relu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("tanh_f32", "tanh.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); @@ -485,6 +487,7 @@ void process_shaders() { string_to_spv("soft_max_f32", "soft_max.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("soft_max_f32_f16", "soft_max.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float"}})); + string_to_spv("soft_max_back_f32", "soft_max_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rope_norm_f32", "rope_norm.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("rope_norm_f16", "rope_norm.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); From 393fca629e6ec391b76edf778cb3f7a8a3bc56f9 Mon Sep 17 00:00:00 2001 From: Molly Sophia Date: Tue, 25 Feb 2025 19:28:22 +0800 Subject: [PATCH 022/188] ggml-cpu: Fix build with sve (#12059) * ggml-cpu: Fix build with sve Signed-off-by: Molly Sophia * ggml-cpu: Remove unused variable in sve q3_k vec dot Signed-off-by: Molly Sophia --------- Signed-off-by: Molly Sophia --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index d0c407bd6e..8d5e3e20bb 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -5265,6 +5265,7 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r #if defined(__ARM_FEATURE_SVE) + uint32_t aux[3]; uint32_t utmp[4]; const int8_t m32 = 32; @@ -5276,7 +5277,6 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r const svuint8_t m1_sv = svlsl_n_u8_x(svptrue_b8(), m0_sv, 1); const svuint8_t m2_sv = svlsl_n_u8_x(svptrue_b8(), m0_sv, 2); const svuint8_t m3_sv = svlsl_n_u8_x(svptrue_b8(), m0_sv, 3); - svbool_t pred_s32 = svnot_b_z (svptrue_b32(), svptrue_pat_b32(SV_VL4)); float sum = 0; @@ -5289,7 +5289,7 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r const int8_t * restrict q8_sv = y[i].qs; // Set up scales - uint32_t * aux = &x[i].scales; + memcpy(aux, x[i].scales, 12); utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4); utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4); utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4); From c132239bfbc1aae3a96dfee3350afcb491f64ade Mon Sep 17 00:00:00 2001 From: Judd Date: Tue, 25 Feb 2025 19:32:20 +0800 Subject: [PATCH 023/188] add OP sigmoid (#12056) Co-authored-by: Judd --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 14 +++++++++++++ .../ggml-vulkan/vulkan-shaders/sigmoid.comp | 20 +++++++++++++++++++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 1 + 3 files changed, 35 insertions(+) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index daef8b89d4..5864e9819d 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -249,6 +249,7 @@ struct vk_device_struct { vk_pipeline pipeline_relu_f32; vk_pipeline pipeline_leaky_relu_f32; vk_pipeline pipeline_tanh_f32; + vk_pipeline pipeline_sigmoid_f32; vk_pipeline pipeline_diag_mask_inf_f32; vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16; vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512; @@ -2189,6 +2190,7 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_relu_f32, "relu_f32", relu_f32_len, relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_leaky_relu_f32, "leaky_relu_f32", leaky_relu_f32_len, leaky_relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_tanh_f32, "tanh_f32", tanh_f32_len, tanh_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_sigmoid_f32, "sigmoid_f32", sigmoid_f32_len, sigmoid_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {1, 512, 1}, {}, 1, true); @@ -5342,6 +5344,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return ctx->device->pipeline_tanh_f32; } break; + case GGML_UNARY_OP_SIGMOID: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_sigmoid_f32; + } + break; default: break; } @@ -7335,6 +7342,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_UNARY_OP_GELU_QUICK: case GGML_UNARY_OP_RELU: case GGML_UNARY_OP_TANH: + case GGML_UNARY_OP_SIGMOID: break; default: return false; @@ -7551,6 +7559,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_UNARY_OP_GELU_QUICK: case GGML_UNARY_OP_RELU: case GGML_UNARY_OP_TANH: + case GGML_UNARY_OP_SIGMOID: ggml_vk_unary(ctx, compute_ctx, src0, node, dryrun); break; default: @@ -7738,6 +7747,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_tensor * case GGML_UNARY_OP_GELU_QUICK: case GGML_UNARY_OP_RELU: case GGML_UNARY_OP_TANH: + case GGML_UNARY_OP_SIGMOID: buf = tensor->buffer; break; default: @@ -8439,6 +8449,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_RELU: case GGML_UNARY_OP_TANH: + case GGML_UNARY_OP_SIGMOID: return ggml_is_contiguous(op->src[0]); default: return false; @@ -9105,6 +9116,9 @@ static void ggml_vk_check_results_0(ggml_tensor * tensor) { case GGML_UNARY_OP_TANH: tensor_clone = ggml_tanh(ggml_ctx, src_clone[0]); break; + case GGML_UNARY_OP_SIGMOID: + tensor_clone = ggml_sigmoid(ggml_ctx, src_clone[0]); + break; default: std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl; GGML_ABORT("fatal error"); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp b/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp new file mode 100644 index 0000000000..776581e2c4 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp @@ -0,0 +1,20 @@ +#version 450 + +#include "generic_head.comp" +#include "types.comp" + +#extension GL_EXT_control_flow_attributes : enable + +layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; + +void main() { + const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x; + + if (i >= p.KX) { + return; + } + data_d[i] = D_TYPE(1. / (1 + exp(-1. *data_a[i]))); +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index fc8bbb9460..c5e0bba82b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -482,6 +482,7 @@ void process_shaders() { string_to_spv("relu_f32", "relu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("leaky_relu_f32", "leaky_relu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("tanh_f32", "tanh.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); + string_to_spv("sigmoid_f32", "sigmoid.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("diag_mask_inf_f32", "diag_mask_inf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); From 401af80b546005a06854827b732e3b46979ae028 Mon Sep 17 00:00:00 2001 From: rhjdvsgsgks <26178113+rhjdvsgsgks@users.noreply.github.com> Date: Tue, 25 Feb 2025 11:52:52 +0000 Subject: [PATCH 024/188] server: handle echo=false on /v1/completions (#12060) --- examples/server/utils.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 4867b1582f..6830c2e1a6 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -521,8 +521,13 @@ static json oaicompat_completion_params_parse(const json & body) { throw std::runtime_error("Only one completion choice is allowed"); } + // Handle "echo" field + if (json_value(body, "echo", false)) { + throw std::runtime_error("Only no echo is supported"); + } + // Params supported by OAI but unsupported by llama.cpp - static const std::vector unsupported_params { "best_of", "echo", "suffix" }; + static const std::vector unsupported_params { "best_of", "suffix" }; for (const auto & param : unsupported_params) { if (body.contains(param)) { throw std::runtime_error("Unsupported param: " + param); From a82c9e7c23ef6db48cebfa194dc9cebbc4ac3552 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Tue, 25 Feb 2025 09:30:21 -0600 Subject: [PATCH 025/188] vulkan: fix assertion when qy_needs_dequant (#12068) Looks like a copy/paste bug from qx_needs_dequant. --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 5864e9819d..abe3e7908c 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -4192,7 +4192,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub } if (qy_needs_dequant) { d_Y = ctx->prealloc_y; - GGML_ASSERT(d_Y->size >= y_sz * ne02 * ne03); + GGML_ASSERT(d_Y->size >= y_sz * ne12 * ne13); } else { d_Y = d_Qy; y_buf_offset = qy_buf_offset; @@ -4769,7 +4769,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& } if (qy_needs_dequant) { d_Y = ctx->prealloc_y; - GGML_ASSERT(d_Y->size >= y_sz * ne02 * ne03); + GGML_ASSERT(d_Y->size >= y_sz * ne12 * ne13); } else { d_Y = d_Qy; y_buf_offset = qy_buf_offset; From d7cfe1ffe0f435d0048a6058d529daf76e072d9c Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Tue, 25 Feb 2025 18:52:56 +0000 Subject: [PATCH 026/188] docs: add docs/function-calling.md to lighten server/README.md's plight (#12069) --- docs/function-calling.md | 390 ++++++++++++++++++++++++++++++++++++++ examples/server/README.md | 377 +----------------------------------- 2 files changed, 393 insertions(+), 374 deletions(-) create mode 100644 docs/function-calling.md diff --git a/docs/function-calling.md b/docs/function-calling.md new file mode 100644 index 0000000000..92cb6531ab --- /dev/null +++ b/docs/function-calling.md @@ -0,0 +1,390 @@ +# Function Calling + +[chat.h](../common/chat.h) (https://github.com/ggml-org/llama.cpp/pull/9639) adds support for [OpenAI-style function calling](https://platform.openai.com/docs/guides/function-calling) and is used in: +- `llama-server` when started w/ `--jinja` flag +- `llama-cli` (WIP: https://github.com/ggml-org/llama.cpp/pull/11556) + +## Universal support w/ Native & Generic handlers + +Function calling is supported for all models (see https://github.com/ggml-org/llama.cpp/pull/9639): + +- Native tool call formats supported: + - Llama 3.1 / 3.3 (including builtin tools support - tool names for `wolfram_alpha`, `web_search` / `brave_search`, `code_interpreter`), Llama 3.2 + - Functionary v3.1 / v3.2 + - Hermes 2/3, Qwen 2.5 + - Qwen 2.5 Coder (WIP: https://github.com/ggml-org/llama.cpp/pull/12034) + - Mistral Nemo + - Firefunction v2 + - Command R7B + - DeepSeek R1 (WIP / seems reluctant to call any tools?) + +- Generic tool call is supported when the template isn't recognized by native format handlers (you'll see `Chat format: Generic` in the logs). + - Use `--chat-template-file` to override the template when appropriate (see examples below) + - Generic support may consume more tokens and be less efficient than a model's native format. + +
+Show some common templates and which format handler they use + +| Template | Format | +|----------|--------| +| Almawave-Velvet-14B.jinja | Hermes 2 Pro | +| AtlaAI-Selene-1-Mini-Llama-3.1-8B.jinja | Llama 3.x | +| CohereForAI-aya-expanse-8b.jinja | Generic | +| CohereForAI-c4ai-command-r-plus-default.jinja | Generic | +| CohereForAI-c4ai-command-r-plus-rag.jinja | Generic | +| CohereForAI-c4ai-command-r-plus-tool_use.jinja | Generic | +| CohereForAI-c4ai-command-r7b-12-2024-default.jinja | Command R7B (extract reasoning) | +| CohereForAI-c4ai-command-r7b-12-2024-rag.jinja | Command R7B (extract reasoning) | +| CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja | Command R7B (extract reasoning) | +| CohereForAI-c4ai-command-r7b-12-2024.jinja | Generic | +| DavieLion-Llama-3.2-1B-SPIN-iter3.jinja | Generic | +| Delta-Vector-Rei-12B.jinja | Mistral Nemo | +| EpistemeAI-Mistral-Nemo-Instruct-12B-Philosophy-Math.jinja | Mistral Nemo | +| FlofloB-83k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit.jinja | Hermes 2 Pro | +| FlofloB-test_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit.jinja | Generic | +| HelpingAI-HAI-SER.jinja | Generic | +| HuggingFaceTB-SmolLM2-1.7B-Instruct.jinja | Generic | +| HuggingFaceTB-SmolLM2-135M-Instruct.jinja | Generic | +| HuggingFaceTB-SmolLM2-360M-Instruct.jinja | Generic | +| INSAIT-Institute-BgGPT-Gemma-2-27B-IT-v1.0.jinja | Generic | +| Ihor-Text2Graph-R1-Qwen2.5-0.5b.jinja | Hermes 2 Pro | +| Infinigence-Megrez-3B-Instruct.jinja | Generic | +| Josephgflowers-TinyLlama_v1.1_math_code-world-test-1.jinja | Generic | +| LGAI-EXAONE-EXAONE-3.5-2.4B-Instruct.jinja | Generic | +| LGAI-EXAONE-EXAONE-3.5-7.8B-Instruct.jinja | Generic | +| LatitudeGames-Wayfarer-12B.jinja | Generic | +| Magpie-Align-Llama-3-8B-Magpie-Align-v0.1.jinja | Generic | +| Magpie-Align-Llama-3.1-8B-Magpie-Align-v0.1.jinja | Generic | +| MaziyarPanahi-calme-3.2-instruct-78b.jinja | Generic | +| MiniMaxAI-MiniMax-Text-01.jinja | Generic | +| MiniMaxAI-MiniMax-VL-01.jinja | Generic | +| NaniDAO-deepseek-r1-qwen-2.5-32B-ablated.jinja | DeepSeek R1 (extract reasoning) | +| NexaAIDev-Octopus-v2.jinja | Generic | +| NousResearch-Hermes-2-Pro-Llama-3-8B-default.jinja | Generic | +| NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja | Hermes 2 Pro | +| NousResearch-Hermes-2-Pro-Mistral-7B-default.jinja | Generic | +| NousResearch-Hermes-2-Pro-Mistral-7B-tool_use.jinja | Hermes 2 Pro | +| NousResearch-Hermes-3-Llama-3.1-70B-default.jinja | Generic | +| NousResearch-Hermes-3-Llama-3.1-70B-tool_use.jinja | Hermes 2 Pro | +| NovaSky-AI-Sky-T1-32B-Flash.jinja | Hermes 2 Pro | +| NovaSky-AI-Sky-T1-32B-Preview.jinja | Hermes 2 Pro | +| OnlyCheeini-greesychat-turbo.jinja | Generic | +| Orenguteng-Llama-3.1-8B-Lexi-Uncensored-V2.jinja | Llama 3.x | +| OrionStarAI-Orion-14B-Chat.jinja | Generic | +| PowerInfer-SmallThinker-3B-Preview.jinja | Generic | +| PrimeIntellect-INTELLECT-1-Instruct.jinja | Generic | +| Qwen-QVQ-72B-Preview.jinja | Generic | +| Qwen-QwQ-32B-Preview.jinja | Hermes 2 Pro | +| Qwen-Qwen1.5-7B-Chat.jinja | Generic | +| Qwen-Qwen2-7B-Instruct.jinja | Generic | +| Qwen-Qwen2-VL-72B-Instruct.jinja | Generic | +| Qwen-Qwen2-VL-7B-Instruct.jinja | Generic | +| Qwen-Qwen2.5-0.5B.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-1.5B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-14B-Instruct-1M.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-14B.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-32B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-32B.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-3B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-72B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-7B-Instruct-1M.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-7B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-7B.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-Coder-32B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-Coder-7B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-Math-1.5B.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-Math-7B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-VL-3B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-VL-72B-Instruct.jinja | Hermes 2 Pro | +| Qwen-Qwen2.5-VL-7B-Instruct.jinja | Hermes 2 Pro | +| RWKV-Red-Team-ARWKV-7B-Preview-0.1.jinja | Hermes 2 Pro | +| SakanaAI-TinySwallow-1.5B-Instruct.jinja | Hermes 2 Pro | +| SakanaAI-TinySwallow-1.5B.jinja | Hermes 2 Pro | +| Sao10K-70B-L3.3-Cirrus-x1.jinja | Llama 3.x | +| SentientAGI-Dobby-Mini-Leashed-Llama-3.1-8B.jinja | Llama 3.x | +| SentientAGI-Dobby-Mini-Unhinged-Llama-3.1-8B.jinja | Llama 3.x | +| Steelskull-L3.3-Damascus-R1.jinja | Llama 3.x | +| Steelskull-L3.3-MS-Nevoria-70b.jinja | Llama 3.x | +| Steelskull-L3.3-Nevoria-R1-70b.jinja | Llama 3.x | +| THUDM-glm-4-9b-chat.jinja | Generic | +| THUDM-glm-edge-1.5b-chat.jinja | Generic | +| Tarek07-Progenitor-V1.1-LLaMa-70B.jinja | Llama 3.x | +| TheBloke-FusionNet_34Bx2_MoE-AWQ.jinja | Generic | +| TinyLlama-TinyLlama-1.1B-Chat-v1.0.jinja | Generic | +| UCLA-AGI-Mistral7B-PairRM-SPPO-Iter3.jinja | Generic | +| ValiantLabs-Llama3.1-8B-Enigma.jinja | Llama 3.x | +| abacusai-Fewshot-Metamath-OrcaVicuna-Mistral.jinja | Generic | +| ai21labs-AI21-Jamba-1.5-Large.jinja | Generic | +| allenai-Llama-3.1-Tulu-3-405B-SFT.jinja | Generic | +| allenai-Llama-3.1-Tulu-3-405B.jinja | Generic | +| allenai-Llama-3.1-Tulu-3-8B.jinja | Generic | +| arcee-ai-Virtuoso-Lite.jinja | Hermes 2 Pro | +| arcee-ai-Virtuoso-Medium-v2.jinja | Hermes 2 Pro | +| arcee-ai-Virtuoso-Small-v2.jinja | Hermes 2 Pro | +| avemio-GRAG-NEMO-12B-ORPO-HESSIAN-AI.jinja | Generic | +| bespokelabs-Bespoke-Stratos-7B.jinja | Hermes 2 Pro | +| bfuzzy1-acheron-m1a-llama.jinja | Generic | +| bofenghuang-vigogne-2-70b-chat.jinja | Generic | +| bytedance-research-UI-TARS-72B-DPO.jinja | Generic | +| bytedance-research-UI-TARS-7B-DPO.jinja | Generic | +| bytedance-research-UI-TARS-7B-SFT.jinja | Generic | +| carsenk-phi3.5_mini_exp_825_uncensored.jinja | Generic | +| cyberagent-DeepSeek-R1-Distill-Qwen-14B-Japanese.jinja | DeepSeek R1 (extract reasoning) | +| cyberagent-DeepSeek-R1-Distill-Qwen-32B-Japanese.jinja | DeepSeek R1 (extract reasoning) | +| databricks-dbrx-instruct.jinja | Generic | +| deepseek-ai-DeepSeek-Coder-V2-Instruct.jinja | Generic | +| deepseek-ai-DeepSeek-Coder-V2-Lite-Base.jinja | Generic | +| deepseek-ai-DeepSeek-Coder-V2-Lite-Instruct.jinja | Generic | +| deepseek-ai-DeepSeek-R1-Distill-Llama-70B.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-DeepSeek-R1-Distill-Qwen-1.5B.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-DeepSeek-R1-Distill-Qwen-14B.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-DeepSeek-R1-Distill-Qwen-7B.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-DeepSeek-R1-Zero.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-DeepSeek-R1.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-DeepSeek-V2-Lite.jinja | Generic | +| deepseek-ai-DeepSeek-V2.5.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-DeepSeek-V3.jinja | DeepSeek R1 (extract reasoning) | +| deepseek-ai-deepseek-coder-33b-instruct.jinja | Generic | +| deepseek-ai-deepseek-coder-6.7b-instruct.jinja | Generic | +| deepseek-ai-deepseek-coder-7b-instruct-v1.5.jinja | Generic | +| deepseek-ai-deepseek-llm-67b-chat.jinja | Generic | +| deepseek-ai-deepseek-llm-7b-chat.jinja | Generic | +| dicta-il-dictalm2.0-instruct.jinja | Generic | +| ehristoforu-Falcon3-8B-Franken-Basestruct.jinja | Hermes 2 Pro | +| fireworks-ai-llama-3-firefunction-v2.jinja | FireFunction v2 | +| godlikehhd-alpaca_data_sampled_ifd_new_5200.jinja | Hermes 2 Pro | +| godlikehhd-alpaca_data_score_max_0.7_2600.jinja | Hermes 2 Pro | +| google-gemma-2-27b-it.jinja | Generic | +| google-gemma-2-2b-it.jinja | Generic | +| google-gemma-2-2b-jpn-it.jinja | Generic | +| google-gemma-7b-it.jinja | Generic | +| huihui-ai-DeepSeek-R1-Distill-Llama-70B-abliterated.jinja | DeepSeek R1 (extract reasoning) | +| huihui-ai-DeepSeek-R1-Distill-Llama-8B-abliterated.jinja | DeepSeek R1 (extract reasoning) | +| huihui-ai-DeepSeek-R1-Distill-Qwen-14B-abliterated-v2.jinja | DeepSeek R1 (extract reasoning) | +| huihui-ai-DeepSeek-R1-Distill-Qwen-32B-abliterated.jinja | DeepSeek R1 (extract reasoning) | +| huihui-ai-DeepSeek-R1-Distill-Qwen-7B-abliterated-v2.jinja | DeepSeek R1 (extract reasoning) | +| huihui-ai-Qwen2.5-14B-Instruct-1M-abliterated.jinja | Hermes 2 Pro | +| ibm-granite-granite-3.1-8b-instruct.jinja | Generic | +| indischepartij-MiniCPM-3B-OpenHermes-2.5-v2.jinja | Generic | +| inflatebot-MN-12B-Mag-Mell-R1.jinja | Generic | +| jinaai-ReaderLM-v2.jinja | Generic | +| kms7530-chemeng_qwen-math-7b_24_1_100_1_nonmath.jinja | Hermes 2 Pro | +| knifeayumu-Cydonia-v1.3-Magnum-v4-22B.jinja | Mistral Nemo | +| langgptai-qwen1.5-7b-chat-sa-v0.1.jinja | Generic | +| lightblue-DeepSeek-R1-Distill-Qwen-7B-Japanese.jinja | DeepSeek R1 (extract reasoning) | +| mattshumer-Reflection-Llama-3.1-70B.jinja | Generic | +| meetkai-functionary-medium-v3.1.jinja | Functionary v3.1 Llama 3.1 | +| meetkai-functionary-medium-v3.2.jinja | Functionary v3.2 | +| meta-llama-Llama-2-7b-chat-hf.jinja | Generic | +| meta-llama-Llama-3.1-8B-Instruct.jinja | Llama 3.x | +| meta-llama-Llama-3.2-11B-Vision-Instruct.jinja | Llama 3.x | +| meta-llama-Llama-3.2-1B-Instruct.jinja | Llama 3.x | +| meta-llama-Llama-3.2-3B-Instruct.jinja | Llama 3.x | +| meta-llama-Llama-3.3-70B-Instruct.jinja | Llama 3.x | +| meta-llama-Meta-Llama-3-8B-Instruct.jinja | Generic | +| meta-llama-Meta-Llama-3.1-8B-Instruct.jinja | Llama 3.x | +| microsoft-Phi-3-medium-4k-instruct.jinja | Generic | +| microsoft-Phi-3-mini-4k-instruct.jinja | Generic | +| microsoft-Phi-3-small-8k-instruct.jinja | Generic | +| microsoft-Phi-3.5-mini-instruct.jinja | Generic | +| microsoft-Phi-3.5-vision-instruct.jinja | Generic | +| microsoft-phi-4.jinja | Generic | +| migtissera-Tess-3-Mistral-Nemo-12B.jinja | Generic | +| ministral-Ministral-3b-instruct.jinja | Generic | +| mistralai-Codestral-22B-v0.1.jinja | Generic | +| mistralai-Mistral-7B-Instruct-v0.1.jinja | Generic | +| mistralai-Mistral-7B-Instruct-v0.2.jinja | Generic | +| mistralai-Mistral-7B-Instruct-v0.3.jinja | Mistral Nemo | +| mistralai-Mistral-Large-Instruct-2407.jinja | Mistral Nemo | +| mistralai-Mistral-Large-Instruct-2411.jinja | Generic | +| mistralai-Mistral-Nemo-Instruct-2407.jinja | Mistral Nemo | +| mistralai-Mistral-Small-24B-Instruct-2501.jinja | Generic | +| mistralai-Mixtral-8x7B-Instruct-v0.1.jinja | Generic | +| mkurman-Qwen2.5-14B-DeepSeek-R1-1M.jinja | Hermes 2 Pro | +| mlabonne-AlphaMonarch-7B.jinja | Generic | +| mlx-community-Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1-float32.jinja | Hermes 2 Pro | +| mlx-community-Qwen2.5-VL-7B-Instruct-8bit.jinja | Hermes 2 Pro | +| mobiuslabsgmbh-DeepSeek-R1-ReDistill-Qwen-1.5B-v1.1.jinja | DeepSeek R1 (extract reasoning) | +| netcat420-MFANNv0.20.jinja | Generic | +| netcat420-MFANNv0.24.jinja | Generic | +| netease-youdao-Confucius-o1-14B.jinja | Hermes 2 Pro | +| nvidia-AceMath-7B-RM.jinja | Hermes 2 Pro | +| nvidia-Eagle2-1B.jinja | Hermes 2 Pro | +| nvidia-Eagle2-9B.jinja | Hermes 2 Pro | +| nvidia-Llama-3.1-Nemotron-70B-Instruct-HF.jinja | Llama 3.x | +| onnx-community-DeepSeek-R1-Distill-Qwen-1.5B-ONNX.jinja | DeepSeek R1 (extract reasoning) | +| open-thoughts-OpenThinker-7B.jinja | Hermes 2 Pro | +| openchat-openchat-3.5-0106.jinja | Generic | +| pankajmathur-orca_mini_v6_8b.jinja | Generic | +| princeton-nlp-Mistral-7B-Base-SFT-RDPO.jinja | Generic | +| princeton-nlp-Mistral-7B-Instruct-DPO.jinja | Generic | +| princeton-nlp-Mistral-7B-Instruct-RDPO.jinja | Generic | +| prithivMLmods-Bellatrix-Tiny-1.5B-R1.jinja | Hermes 2 Pro | +| prithivMLmods-Bellatrix-Tiny-1B-R1.jinja | Llama 3.x | +| prithivMLmods-Bellatrix-Tiny-1B-v3.jinja | Generic | +| prithivMLmods-Bellatrix-Tiny-3B-R1.jinja | Llama 3.x | +| prithivMLmods-Blaze-14B-xElite.jinja | Generic | +| prithivMLmods-Calcium-Opus-14B-Elite2-R1.jinja | Hermes 2 Pro | +| prithivMLmods-Calme-Ties-78B.jinja | Generic | +| prithivMLmods-Calme-Ties2-78B.jinja | Generic | +| prithivMLmods-Calme-Ties3-78B.jinja | Generic | +| prithivMLmods-ChemQwen2-vL.jinja | Generic | +| prithivMLmods-GWQ2b.jinja | Generic | +| prithivMLmods-LatexMind-2B-Codec.jinja | Generic | +| prithivMLmods-Llama-3.2-6B-AlgoCode.jinja | Llama 3.x | +| prithivMLmods-Megatron-Opus-14B-Exp.jinja | Hermes 2 Pro | +| prithivMLmods-Megatron-Opus-14B-Stock.jinja | Hermes 2 Pro | +| prithivMLmods-Megatron-Opus-7B-Exp.jinja | Hermes 2 Pro | +| prithivMLmods-Omni-Reasoner-Merged.jinja | Hermes 2 Pro | +| prithivMLmods-Omni-Reasoner4-Merged.jinja | Hermes 2 Pro | +| prithivMLmods-Primal-Opus-14B-Optimus-v1.jinja | Hermes 2 Pro | +| prithivMLmods-QwQ-Math-IO-500M.jinja | Hermes 2 Pro | +| prithivMLmods-Qwen-7B-Distill-Reasoner.jinja | DeepSeek R1 (extract reasoning) | +| prithivMLmods-Qwen2.5-1.5B-DeepSeek-R1-Instruct.jinja | Hermes 2 Pro | +| prithivMLmods-Qwen2.5-14B-DeepSeek-R1-1M.jinja | Hermes 2 Pro | +| prithivMLmods-Qwen2.5-32B-DeepSeek-R1-Instruct.jinja | Hermes 2 Pro | +| prithivMLmods-Qwen2.5-7B-DeepSeek-R1-1M.jinja | Hermes 2 Pro | +| prithivMLmods-Triangulum-v2-10B.jinja | Hermes 2 Pro | +| qingy2024-Falcon3-2x10B-MoE-Instruct.jinja | Hermes 2 Pro | +| rubenroy-Zurich-14B-GCv2-5m.jinja | Hermes 2 Pro | +| rubenroy-Zurich-7B-GCv2-5m.jinja | Hermes 2 Pro | +| silma-ai-SILMA-Kashif-2B-Instruct-v1.0.jinja | Generic | +| simplescaling-s1-32B.jinja | Hermes 2 Pro | +| sometimesanotion-Lamarck-14B-v0.7.jinja | Hermes 2 Pro | +| sonthenguyen-zephyr-sft-bnb-4bit-DPO-mtbr-180steps.jinja | Generic | +| sthenno-tempesthenno-icy-0130.jinja | Generic | +| sumink-qwft.jinja | Hermes 2 Pro | +| teknium-OpenHermes-2.5-Mistral-7B.jinja | Generic | +| thirdeyeai-elevate360m.jinja | Generic | +| tiiuae-Falcon3-10B-Instruct.jinja | Hermes 2 Pro | +| unsloth-DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit.jinja | DeepSeek R1 (extract reasoning) | +| unsloth-DeepSeek-R1-Distill-Llama-8B.jinja | DeepSeek R1 (extract reasoning) | +| unsloth-DeepSeek-R1.jinja | DeepSeek R1 (extract reasoning) | +| unsloth-Mistral-Small-24B-Instruct-2501-unsloth-bnb-4bit.jinja | Generic | +| upstage-solar-pro-preview-instruct.jinja | Generic | +| whyhow-ai-PatientSeek.jinja | Generic | +| xwen-team-Xwen-72B-Chat.jinja | Hermes 2 Pro | +| xwen-team-Xwen-7B-Chat.jinja | Hermes 2 Pro | + +This table can be generated with: + +```bash +./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null +``` + +
+ +# Usage - need tool-aware Jinja template + +First, start a server with any model, but make sure it has a tools-enabled template: you can verify this by inspecting the `chat_template` or `chat_template_tool_use` properties in `http://localhost:8080/props`). + +Here are some models known to work (w/ chat template override when needed): + +```shell +# Native support: + +llama-server --jinja -fa -hf bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M +llama-server --jinja -fa -hf bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q6_K_L +llama-server --jinja -fa -hf bartowski/functionary-small-v3.2-GGUF:Q4_K_M +llama-server --jinja -fa -hf bartowski/Llama-3.3-70B-Instruct-GGUF:Q4_K_M + +# Native support for DeepSeek R1 works best w/ our own template (official template buggy) + +llama-server --jinja -fa -hf bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q6_K_L \ +--chat-template-file models/templates/llama-cpp-deepseek-r1.jinja + +llama-server --jinja -fa -hf bartowski/DeepSeek-R1-Distill-Qwen-32B-GGUF:Q4_K_M \ +--chat-template-file models/templates/llama-cpp-deepseek-r1.jinja + +# Native support requires the right template for these GGUFs: + +llama-server --jinja -fa -hf bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M \ +--chat-template-file <( python scripts/get_chat_template.py NousResearch/Hermes-2-Pro-Llama-3-8B tool_use ) + +llama-server --jinja -fa -hf bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M \ +--chat-template-file <( python scripts/get_chat_template.py NousResearch/Hermes-3-Llama-3.1-8B tool_use ) + +llama-server --jinja -fa -hf bartowski/firefunction-v2-GGUF -hff firefunction-v2-IQ1_M.gguf \ +--chat-template-file <( python scripts/get_chat_template.py fireworks-ai/llama-3-firefunction-v2 tool_use ) + +llama-server --jinja -fa -hf bartowski/c4ai-command-r7b-12-2024-GGUF:Q6_K_L \ +--chat-template-file <( python scripts/get_chat_template.py CohereForAI/c4ai-command-r7b-12-2024 tool_use ) + +# Generic format support +llama-server --jinja -fa -hf bartowski/phi-4-GGUF:Q4_0 +llama-server --jinja -fa -hf bartowski/gemma-2-2b-it-GGUF:Q8_0 +llama-server --jinja -fa -hf bartowski/c4ai-command-r-v01-GGUF:Q2_K +``` + +> [!TIP] +> If there is no official `tool_use` Jinja template, you may want to set `--chat-template chatml` to use a default that works with many models (YMMV!), or write your own (e.g. we provide a custom [llama-cpp-deepseek-r1.jinja](../models/templates/llama-cpp-deepseek-r1.jinja) for DeepSeek R1 distills) + +Test in CLI (or with any library / software that can use OpenAI-compatible API backends): + +```bash +curl http://localhost:8080/v1/chat/completions -d '{ +"model": "gpt-3.5-turbo", +"tools": [ + { + "type":"function", + "function":{ + "name":"python", + "description":"Runs code in an ipython interpreter and returns the result of the execution after 60 seconds.", + "parameters":{ + "type":"object", + "properties":{ + "code":{ + "type":"string", + "description":"The code to run in the ipython interpreter." + } + }, + "required":["code"] + } + } + } +], +"messages": [ + { + "role": "user", + "content": "Print a hello world message with python." + } +] +}' +``` + +
+Show output + +```json +{ +"choices": [ + { + "finish_reason": "tool", + "index": 0, + "message": { + "content": null, + "tool_calls": [ + { + "name": "python", + "arguments": "{\"code\":\" \\nprint(\\\"Hello, World!\\\")\"}" + } + ], + "role": "assistant" + } + } +], +"created": 1727287211, +"model": "gpt-3.5-turbo", +"object": "chat.completion", +"usage": { + "completion_tokens": 16, + "prompt_tokens": 44, + "total_tokens": 60 +}, +"id": "chatcmpl-Htbgh9feMmGM0LEH2hmQvwsCxq3c6Ni8" +} +``` + +
diff --git a/examples/server/README.md b/examples/server/README.md index a2ae614d7c..a2a0903261 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -13,6 +13,7 @@ Set of LLM REST APIs and a simple web front end to interact with llama.cpp. * Multimodal (wip) * Monitoring endpoints * Schema-constrained JSON response format + * [Function calling](../../docs/function-calling.md) / tool use for ~any model The project is under active development, and we are [looking for feedback and contributors](https://github.com/ggml-org/llama.cpp/issues/4216). @@ -1120,381 +1121,9 @@ curl http://localhost:8080/v1/chat/completions \ *Tool call support* -[Function calling](https://platform.openai.com/docs/guides/function-calling) is supported for all models (see https://github.com/ggml-org/llama.cpp/pull/9639): +[OpenAI-style function calling](https://platform.openai.com/docs/guides/function-calling) is supported with the `--jinja` flag (and may require a `--chat-template-file` override to get the right tool-use compatible Jinja template; worst case, `--chat-template chatml` may also work). -- Requires `--jinja` flag -- Native tool call formats supported: - - Llama 3.1 / 3.3 (including builtin tools support - tool names for `wolfram_alpha`, `web_search` / `brave_search`, `code_interpreter`), Llama 3.2 - - Functionary v3.1 / v3.2 - - Hermes 2/3, Qwen 2.5 - - Mistral Nemo - - Firefunction v2 - - Command R7B - - DeepSeek R1 (WIP / seems reluctant to call any tools?) - -
- Show some common templates and which format handler they use - - | Template | Format | - |----------|--------| - | Almawave-Velvet-14B.jinja | Hermes 2 Pro | - | AtlaAI-Selene-1-Mini-Llama-3.1-8B.jinja | Llama 3.x | - | CohereForAI-aya-expanse-8b.jinja | Generic | - | CohereForAI-c4ai-command-r-plus-default.jinja | Generic | - | CohereForAI-c4ai-command-r-plus-rag.jinja | Generic | - | CohereForAI-c4ai-command-r-plus-tool_use.jinja | Generic | - | CohereForAI-c4ai-command-r7b-12-2024-default.jinja | Command R7B (extract reasoning) | - | CohereForAI-c4ai-command-r7b-12-2024-rag.jinja | Command R7B (extract reasoning) | - | CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja | Command R7B (extract reasoning) | - | CohereForAI-c4ai-command-r7b-12-2024.jinja | Generic | - | DavieLion-Llama-3.2-1B-SPIN-iter3.jinja | Generic | - | Delta-Vector-Rei-12B.jinja | Mistral Nemo | - | EpistemeAI-Mistral-Nemo-Instruct-12B-Philosophy-Math.jinja | Mistral Nemo | - | FlofloB-83k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit.jinja | Hermes 2 Pro | - | FlofloB-test_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit.jinja | Generic | - | HelpingAI-HAI-SER.jinja | Generic | - | HuggingFaceTB-SmolLM2-1.7B-Instruct.jinja | Generic | - | HuggingFaceTB-SmolLM2-135M-Instruct.jinja | Generic | - | HuggingFaceTB-SmolLM2-360M-Instruct.jinja | Generic | - | INSAIT-Institute-BgGPT-Gemma-2-27B-IT-v1.0.jinja | Generic | - | Ihor-Text2Graph-R1-Qwen2.5-0.5b.jinja | Hermes 2 Pro | - | Infinigence-Megrez-3B-Instruct.jinja | Generic | - | Josephgflowers-TinyLlama_v1.1_math_code-world-test-1.jinja | Generic | - | LGAI-EXAONE-EXAONE-3.5-2.4B-Instruct.jinja | Generic | - | LGAI-EXAONE-EXAONE-3.5-7.8B-Instruct.jinja | Generic | - | LatitudeGames-Wayfarer-12B.jinja | Generic | - | Magpie-Align-Llama-3-8B-Magpie-Align-v0.1.jinja | Generic | - | Magpie-Align-Llama-3.1-8B-Magpie-Align-v0.1.jinja | Generic | - | MaziyarPanahi-calme-3.2-instruct-78b.jinja | Generic | - | MiniMaxAI-MiniMax-Text-01.jinja | Generic | - | MiniMaxAI-MiniMax-VL-01.jinja | Generic | - | NaniDAO-deepseek-r1-qwen-2.5-32B-ablated.jinja | DeepSeek R1 (extract reasoning) | - | NexaAIDev-Octopus-v2.jinja | Generic | - | NousResearch-Hermes-2-Pro-Llama-3-8B-default.jinja | Generic | - | NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja | Hermes 2 Pro | - | NousResearch-Hermes-2-Pro-Mistral-7B-default.jinja | Generic | - | NousResearch-Hermes-2-Pro-Mistral-7B-tool_use.jinja | Hermes 2 Pro | - | NousResearch-Hermes-3-Llama-3.1-70B-default.jinja | Generic | - | NousResearch-Hermes-3-Llama-3.1-70B-tool_use.jinja | Hermes 2 Pro | - | NovaSky-AI-Sky-T1-32B-Flash.jinja | Hermes 2 Pro | - | NovaSky-AI-Sky-T1-32B-Preview.jinja | Hermes 2 Pro | - | OnlyCheeini-greesychat-turbo.jinja | Generic | - | Orenguteng-Llama-3.1-8B-Lexi-Uncensored-V2.jinja | Llama 3.x | - | OrionStarAI-Orion-14B-Chat.jinja | Generic | - | PowerInfer-SmallThinker-3B-Preview.jinja | Generic | - | PrimeIntellect-INTELLECT-1-Instruct.jinja | Generic | - | Qwen-QVQ-72B-Preview.jinja | Generic | - | Qwen-QwQ-32B-Preview.jinja | Hermes 2 Pro | - | Qwen-Qwen1.5-7B-Chat.jinja | Generic | - | Qwen-Qwen2-7B-Instruct.jinja | Generic | - | Qwen-Qwen2-VL-72B-Instruct.jinja | Generic | - | Qwen-Qwen2-VL-7B-Instruct.jinja | Generic | - | Qwen-Qwen2.5-0.5B.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-1.5B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-14B-Instruct-1M.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-14B.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-32B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-32B.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-3B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-72B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-7B-Instruct-1M.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-7B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-7B.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-Coder-32B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-Coder-7B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-Math-1.5B.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-Math-7B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-VL-3B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-VL-72B-Instruct.jinja | Hermes 2 Pro | - | Qwen-Qwen2.5-VL-7B-Instruct.jinja | Hermes 2 Pro | - | RWKV-Red-Team-ARWKV-7B-Preview-0.1.jinja | Hermes 2 Pro | - | SakanaAI-TinySwallow-1.5B-Instruct.jinja | Hermes 2 Pro | - | SakanaAI-TinySwallow-1.5B.jinja | Hermes 2 Pro | - | Sao10K-70B-L3.3-Cirrus-x1.jinja | Llama 3.x | - | SentientAGI-Dobby-Mini-Leashed-Llama-3.1-8B.jinja | Llama 3.x | - | SentientAGI-Dobby-Mini-Unhinged-Llama-3.1-8B.jinja | Llama 3.x | - | Steelskull-L3.3-Damascus-R1.jinja | Llama 3.x | - | Steelskull-L3.3-MS-Nevoria-70b.jinja | Llama 3.x | - | Steelskull-L3.3-Nevoria-R1-70b.jinja | Llama 3.x | - | THUDM-glm-4-9b-chat.jinja | Generic | - | THUDM-glm-edge-1.5b-chat.jinja | Generic | - | Tarek07-Progenitor-V1.1-LLaMa-70B.jinja | Llama 3.x | - | TheBloke-FusionNet_34Bx2_MoE-AWQ.jinja | Generic | - | TinyLlama-TinyLlama-1.1B-Chat-v1.0.jinja | Generic | - | UCLA-AGI-Mistral7B-PairRM-SPPO-Iter3.jinja | Generic | - | ValiantLabs-Llama3.1-8B-Enigma.jinja | Llama 3.x | - | abacusai-Fewshot-Metamath-OrcaVicuna-Mistral.jinja | Generic | - | ai21labs-AI21-Jamba-1.5-Large.jinja | Generic | - | allenai-Llama-3.1-Tulu-3-405B-SFT.jinja | Generic | - | allenai-Llama-3.1-Tulu-3-405B.jinja | Generic | - | allenai-Llama-3.1-Tulu-3-8B.jinja | Generic | - | arcee-ai-Virtuoso-Lite.jinja | Hermes 2 Pro | - | arcee-ai-Virtuoso-Medium-v2.jinja | Hermes 2 Pro | - | arcee-ai-Virtuoso-Small-v2.jinja | Hermes 2 Pro | - | avemio-GRAG-NEMO-12B-ORPO-HESSIAN-AI.jinja | Generic | - | bespokelabs-Bespoke-Stratos-7B.jinja | Hermes 2 Pro | - | bfuzzy1-acheron-m1a-llama.jinja | Generic | - | bofenghuang-vigogne-2-70b-chat.jinja | Generic | - | bytedance-research-UI-TARS-72B-DPO.jinja | Generic | - | bytedance-research-UI-TARS-7B-DPO.jinja | Generic | - | bytedance-research-UI-TARS-7B-SFT.jinja | Generic | - | carsenk-phi3.5_mini_exp_825_uncensored.jinja | Generic | - | cyberagent-DeepSeek-R1-Distill-Qwen-14B-Japanese.jinja | DeepSeek R1 (extract reasoning) | - | cyberagent-DeepSeek-R1-Distill-Qwen-32B-Japanese.jinja | DeepSeek R1 (extract reasoning) | - | databricks-dbrx-instruct.jinja | Generic | - | deepseek-ai-DeepSeek-Coder-V2-Instruct.jinja | Generic | - | deepseek-ai-DeepSeek-Coder-V2-Lite-Base.jinja | Generic | - | deepseek-ai-DeepSeek-Coder-V2-Lite-Instruct.jinja | Generic | - | deepseek-ai-DeepSeek-R1-Distill-Llama-70B.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-DeepSeek-R1-Distill-Qwen-1.5B.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-DeepSeek-R1-Distill-Qwen-14B.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-DeepSeek-R1-Distill-Qwen-7B.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-DeepSeek-R1-Zero.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-DeepSeek-R1.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-DeepSeek-V2-Lite.jinja | Generic | - | deepseek-ai-DeepSeek-V2.5.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-DeepSeek-V3.jinja | DeepSeek R1 (extract reasoning) | - | deepseek-ai-deepseek-coder-33b-instruct.jinja | Generic | - | deepseek-ai-deepseek-coder-6.7b-instruct.jinja | Generic | - | deepseek-ai-deepseek-coder-7b-instruct-v1.5.jinja | Generic | - | deepseek-ai-deepseek-llm-67b-chat.jinja | Generic | - | deepseek-ai-deepseek-llm-7b-chat.jinja | Generic | - | dicta-il-dictalm2.0-instruct.jinja | Generic | - | ehristoforu-Falcon3-8B-Franken-Basestruct.jinja | Hermes 2 Pro | - | fireworks-ai-llama-3-firefunction-v2.jinja | FireFunction v2 | - | godlikehhd-alpaca_data_sampled_ifd_new_5200.jinja | Hermes 2 Pro | - | godlikehhd-alpaca_data_score_max_0.7_2600.jinja | Hermes 2 Pro | - | google-gemma-2-27b-it.jinja | Generic | - | google-gemma-2-2b-it.jinja | Generic | - | google-gemma-2-2b-jpn-it.jinja | Generic | - | google-gemma-7b-it.jinja | Generic | - | huihui-ai-DeepSeek-R1-Distill-Llama-70B-abliterated.jinja | DeepSeek R1 (extract reasoning) | - | huihui-ai-DeepSeek-R1-Distill-Llama-8B-abliterated.jinja | DeepSeek R1 (extract reasoning) | - | huihui-ai-DeepSeek-R1-Distill-Qwen-14B-abliterated-v2.jinja | DeepSeek R1 (extract reasoning) | - | huihui-ai-DeepSeek-R1-Distill-Qwen-32B-abliterated.jinja | DeepSeek R1 (extract reasoning) | - | huihui-ai-DeepSeek-R1-Distill-Qwen-7B-abliterated-v2.jinja | DeepSeek R1 (extract reasoning) | - | huihui-ai-Qwen2.5-14B-Instruct-1M-abliterated.jinja | Hermes 2 Pro | - | ibm-granite-granite-3.1-8b-instruct.jinja | Generic | - | indischepartij-MiniCPM-3B-OpenHermes-2.5-v2.jinja | Generic | - | inflatebot-MN-12B-Mag-Mell-R1.jinja | Generic | - | jinaai-ReaderLM-v2.jinja | Generic | - | kms7530-chemeng_qwen-math-7b_24_1_100_1_nonmath.jinja | Hermes 2 Pro | - | knifeayumu-Cydonia-v1.3-Magnum-v4-22B.jinja | Mistral Nemo | - | langgptai-qwen1.5-7b-chat-sa-v0.1.jinja | Generic | - | lightblue-DeepSeek-R1-Distill-Qwen-7B-Japanese.jinja | DeepSeek R1 (extract reasoning) | - | mattshumer-Reflection-Llama-3.1-70B.jinja | Generic | - | meetkai-functionary-medium-v3.1.jinja | Functionary v3.1 Llama 3.1 | - | meetkai-functionary-medium-v3.2.jinja | Functionary v3.2 | - | meta-llama-Llama-2-7b-chat-hf.jinja | Generic | - | meta-llama-Llama-3.1-8B-Instruct.jinja | Llama 3.x | - | meta-llama-Llama-3.2-11B-Vision-Instruct.jinja | Llama 3.x | - | meta-llama-Llama-3.2-1B-Instruct.jinja | Llama 3.x | - | meta-llama-Llama-3.2-3B-Instruct.jinja | Llama 3.x | - | meta-llama-Llama-3.3-70B-Instruct.jinja | Llama 3.x | - | meta-llama-Meta-Llama-3-8B-Instruct.jinja | Generic | - | meta-llama-Meta-Llama-3.1-8B-Instruct.jinja | Llama 3.x | - | microsoft-Phi-3-medium-4k-instruct.jinja | Generic | - | microsoft-Phi-3-mini-4k-instruct.jinja | Generic | - | microsoft-Phi-3-small-8k-instruct.jinja | Generic | - | microsoft-Phi-3.5-mini-instruct.jinja | Generic | - | microsoft-Phi-3.5-vision-instruct.jinja | Generic | - | microsoft-phi-4.jinja | Generic | - | migtissera-Tess-3-Mistral-Nemo-12B.jinja | Generic | - | ministral-Ministral-3b-instruct.jinja | Generic | - | mistralai-Codestral-22B-v0.1.jinja | Generic | - | mistralai-Mistral-7B-Instruct-v0.1.jinja | Generic | - | mistralai-Mistral-7B-Instruct-v0.2.jinja | Generic | - | mistralai-Mistral-7B-Instruct-v0.3.jinja | Mistral Nemo | - | mistralai-Mistral-Large-Instruct-2407.jinja | Mistral Nemo | - | mistralai-Mistral-Large-Instruct-2411.jinja | Generic | - | mistralai-Mistral-Nemo-Instruct-2407.jinja | Mistral Nemo | - | mistralai-Mistral-Small-24B-Instruct-2501.jinja | Generic | - | mistralai-Mixtral-8x7B-Instruct-v0.1.jinja | Generic | - | mkurman-Qwen2.5-14B-DeepSeek-R1-1M.jinja | Hermes 2 Pro | - | mlabonne-AlphaMonarch-7B.jinja | Generic | - | mlx-community-Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1-float32.jinja | Hermes 2 Pro | - | mlx-community-Qwen2.5-VL-7B-Instruct-8bit.jinja | Hermes 2 Pro | - | mobiuslabsgmbh-DeepSeek-R1-ReDistill-Qwen-1.5B-v1.1.jinja | DeepSeek R1 (extract reasoning) | - | netcat420-MFANNv0.20.jinja | Generic | - | netcat420-MFANNv0.24.jinja | Generic | - | netease-youdao-Confucius-o1-14B.jinja | Hermes 2 Pro | - | nvidia-AceMath-7B-RM.jinja | Hermes 2 Pro | - | nvidia-Eagle2-1B.jinja | Hermes 2 Pro | - | nvidia-Eagle2-9B.jinja | Hermes 2 Pro | - | nvidia-Llama-3.1-Nemotron-70B-Instruct-HF.jinja | Llama 3.x | - | onnx-community-DeepSeek-R1-Distill-Qwen-1.5B-ONNX.jinja | DeepSeek R1 (extract reasoning) | - | open-thoughts-OpenThinker-7B.jinja | Hermes 2 Pro | - | openchat-openchat-3.5-0106.jinja | Generic | - | pankajmathur-orca_mini_v6_8b.jinja | Generic | - | princeton-nlp-Mistral-7B-Base-SFT-RDPO.jinja | Generic | - | princeton-nlp-Mistral-7B-Instruct-DPO.jinja | Generic | - | princeton-nlp-Mistral-7B-Instruct-RDPO.jinja | Generic | - | prithivMLmods-Bellatrix-Tiny-1.5B-R1.jinja | Hermes 2 Pro | - | prithivMLmods-Bellatrix-Tiny-1B-R1.jinja | Llama 3.x | - | prithivMLmods-Bellatrix-Tiny-1B-v3.jinja | Generic | - | prithivMLmods-Bellatrix-Tiny-3B-R1.jinja | Llama 3.x | - | prithivMLmods-Blaze-14B-xElite.jinja | Generic | - | prithivMLmods-Calcium-Opus-14B-Elite2-R1.jinja | Hermes 2 Pro | - | prithivMLmods-Calme-Ties-78B.jinja | Generic | - | prithivMLmods-Calme-Ties2-78B.jinja | Generic | - | prithivMLmods-Calme-Ties3-78B.jinja | Generic | - | prithivMLmods-ChemQwen2-vL.jinja | Generic | - | prithivMLmods-GWQ2b.jinja | Generic | - | prithivMLmods-LatexMind-2B-Codec.jinja | Generic | - | prithivMLmods-Llama-3.2-6B-AlgoCode.jinja | Llama 3.x | - | prithivMLmods-Megatron-Opus-14B-Exp.jinja | Hermes 2 Pro | - | prithivMLmods-Megatron-Opus-14B-Stock.jinja | Hermes 2 Pro | - | prithivMLmods-Megatron-Opus-7B-Exp.jinja | Hermes 2 Pro | - | prithivMLmods-Omni-Reasoner-Merged.jinja | Hermes 2 Pro | - | prithivMLmods-Omni-Reasoner4-Merged.jinja | Hermes 2 Pro | - | prithivMLmods-Primal-Opus-14B-Optimus-v1.jinja | Hermes 2 Pro | - | prithivMLmods-QwQ-Math-IO-500M.jinja | Hermes 2 Pro | - | prithivMLmods-Qwen-7B-Distill-Reasoner.jinja | DeepSeek R1 (extract reasoning) | - | prithivMLmods-Qwen2.5-1.5B-DeepSeek-R1-Instruct.jinja | Hermes 2 Pro | - | prithivMLmods-Qwen2.5-14B-DeepSeek-R1-1M.jinja | Hermes 2 Pro | - | prithivMLmods-Qwen2.5-32B-DeepSeek-R1-Instruct.jinja | Hermes 2 Pro | - | prithivMLmods-Qwen2.5-7B-DeepSeek-R1-1M.jinja | Hermes 2 Pro | - | prithivMLmods-Triangulum-v2-10B.jinja | Hermes 2 Pro | - | qingy2024-Falcon3-2x10B-MoE-Instruct.jinja | Hermes 2 Pro | - | rubenroy-Zurich-14B-GCv2-5m.jinja | Hermes 2 Pro | - | rubenroy-Zurich-7B-GCv2-5m.jinja | Hermes 2 Pro | - | silma-ai-SILMA-Kashif-2B-Instruct-v1.0.jinja | Generic | - | simplescaling-s1-32B.jinja | Hermes 2 Pro | - | sometimesanotion-Lamarck-14B-v0.7.jinja | Hermes 2 Pro | - | sonthenguyen-zephyr-sft-bnb-4bit-DPO-mtbr-180steps.jinja | Generic | - | sthenno-tempesthenno-icy-0130.jinja | Generic | - | sumink-qwft.jinja | Hermes 2 Pro | - | teknium-OpenHermes-2.5-Mistral-7B.jinja | Generic | - | thirdeyeai-elevate360m.jinja | Generic | - | tiiuae-Falcon3-10B-Instruct.jinja | Hermes 2 Pro | - | unsloth-DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit.jinja | DeepSeek R1 (extract reasoning) | - | unsloth-DeepSeek-R1-Distill-Llama-8B.jinja | DeepSeek R1 (extract reasoning) | - | unsloth-DeepSeek-R1.jinja | DeepSeek R1 (extract reasoning) | - | unsloth-Mistral-Small-24B-Instruct-2501-unsloth-bnb-4bit.jinja | Generic | - | upstage-solar-pro-preview-instruct.jinja | Generic | - | whyhow-ai-PatientSeek.jinja | Generic | - | xwen-team-Xwen-72B-Chat.jinja | Hermes 2 Pro | - | xwen-team-Xwen-7B-Chat.jinja | Hermes 2 Pro | - - This table can be generated with: - - ```bash - ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null - ``` - -
- -- Generic tool call is supported when the template isn't recognized by native format handlers (you'll see `Chat format: Generic` in the logs). - - Use `--chat-template-file` to override the template when appropriate (see examples below) - - Generic support may consume more tokens and be less efficient than a model's native format. - -- Run with: - - ```shell - # Native support: - - llama-server --jinja -fa -hf bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M - llama-server --jinja -fa -hf bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q6_K_L - llama-server --jinja -fa -hf bartowski/functionary-small-v3.2-GGUF:Q4_K_M - llama-server --jinja -fa -hf bartowski/Llama-3.3-70B-Instruct-GGUF:Q4_K_M - - # Native support for DeepSeek R1 works best w/ our own template (official template buggy) - - llama-server --jinja -fa -hf bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q6_K_L \ - --chat-template-file models/templates/llama-cpp-deepseek-r1.jinja - - llama-server --jinja -fa -hf bartowski/DeepSeek-R1-Distill-Qwen-32B-GGUF:Q4_K_M \ - --chat-template-file models/templates/llama-cpp-deepseek-r1.jinja - - # Native support requires the right template for these GGUFs: - - llama-server --jinja -fa -hf bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M \ - --chat-template-file <( python scripts/get_chat_template.py NousResearch/Hermes-2-Pro-Llama-3-8B tool_use ) - - llama-server --jinja -fa -hf bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M \ - --chat-template-file <( python scripts/get_chat_template.py NousResearch/Hermes-3-Llama-3.1-8B tool_use ) - - llama-server --jinja -fa -hf bartowski/firefunction-v2-GGUF -hff firefunction-v2-IQ1_M.gguf \ - --chat-template-file <( python scripts/get_chat_template.py fireworks-ai/llama-3-firefunction-v2 tool_use ) - - llama-server --jinja -fa -hf bartowski/c4ai-command-r7b-12-2024-GGUF:Q6_K_L \ - --chat-template-file <( python scripts/get_chat_template.py CohereForAI/c4ai-command-r7b-12-2024 tool_use ) - - # Generic format support - llama-server --jinja -fa -hf bartowski/phi-4-GGUF:Q4_0 - llama-server --jinja -fa -hf bartowski/gemma-2-2b-it-GGUF:Q8_0 - llama-server --jinja -fa -hf bartowski/c4ai-command-r-v01-GGUF:Q2_K - ``` - -- Test in CLI: - - ```bash - curl http://localhost:8080/v1/chat/completions -d '{ - "model": "gpt-3.5-turbo", - "tools": [ - { - "type":"function", - "function":{ - "name":"python", - "description":"Runs code in an ipython interpreter and returns the result of the execution after 60 seconds.", - "parameters":{ - "type":"object", - "properties":{ - "code":{ - "type":"string", - "description":"The code to run in the ipython interpreter." - } - }, - "required":["code"] - } - } - } - ], - "messages": [ - { - "role": "user", - "content": "Print a hello world message with python." - } - ] - }' - ``` - -
- Show output - - ```json - { - "choices": [ - { - "finish_reason": "tool", - "index": 0, - "message": { - "content": null, - "tool_calls": [ - { - "name": "python", - "arguments": "{\"code\":\" \\nprint(\\\"Hello, World!\\\")\"}" - } - ], - "role": "assistant" - } - } - ], - "created": 1727287211, - "model": "gpt-3.5-turbo", - "object": "chat.completion", - "usage": { - "completion_tokens": 16, - "prompt_tokens": 44, - "total_tokens": 60 - }, - "id": "chatcmpl-Htbgh9feMmGM0LEH2hmQvwsCxq3c6Ni8" - } - ``` - -
+**See our [Function calling](../../docs/function-calling.md) docs** for more details, supported native tool call styles (generic tool call style is used as fallback) / examples of use. ### POST `/v1/embeddings`: OpenAI-compatible embeddings API From 53e4db10126e277486eccb94c6728f4146c75741 Mon Sep 17 00:00:00 2001 From: Kante Yin Date: Wed, 26 Feb 2025 15:49:36 +0800 Subject: [PATCH 027/188] readme : update infra list (#9096) Signed-off-by: kerthcet --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f70c8ae1ed..f7f1a521e4 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ Instructions for adding support for new models: [HOWTO-add-model.md](docs/develo - [llama_cpp_canister](https://github.com/onicai/llama_cpp_canister) - llama.cpp as a smart contract on the Internet Computer, using WebAssembly - [llama-swap](https://github.com/mostlygeek/llama-swap) - transparent proxy that adds automatic model switching with llama-server - [Kalavai](https://github.com/kalavai-net/kalavai-client) - Crowdsource end to end LLM deployment at any scale - +- [llmaz](https://github.com/InftyAI/llmaz) - ☸️ Easy, advanced inference platform for large language models on Kubernetes.
From 3567ee3a94d57ba72b6d023ca507d11e75767edb Mon Sep 17 00:00:00 2001 From: Aleksei Nikiforov <103434461+AlekseiNikiforovIBM@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:39:27 +0100 Subject: [PATCH 028/188] gguf-py: enable reading non-native endian files (#12081) Currently self.byte_order is never used. Actually use it to byteswap read data to allow reading big endian files on little endian systems and vice versa. Now it's possible to convert little-endian model into a big-endian model and back on a little-endian system. --- gguf-py/gguf/gguf_reader.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index e17a4e8314..3568ea0dfc 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -146,9 +146,11 @@ class GGUFReader: itemsize = int(np.empty([], dtype = dtype).itemsize) end_offs = offset + itemsize * count arr = self.data[offset:end_offs].view(dtype=dtype)[:count] - if override_order is None: - return arr - return arr.view(arr.dtype.newbyteorder(override_order)) + if override_order is not None: + return arr.view(arr.dtype.newbyteorder(override_order)) + if self.byte_order == 'S': + return arr.view(arr.dtype.newbyteorder(self.byte_order)) + return arr def _push_field(self, field: ReaderField, skip_sum: bool = False) -> int: if field.name in self.fields: From 69050a11be0ae3e01329f11371ecb6850bdaded5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 26 Feb 2025 14:04:48 +0100 Subject: [PATCH 029/188] Refactor gguf scripts to improve metadata handling (#11909) * Refactor gguf scripts to improve metadata handling Added contents method to ReaderField class Added endianess property to GGUFReader class * update scripts * fix import * remove unused import * attempt to work around flake and pyright errors * second attempt * give up, ignore type * bump version * apply newbyteorder fixes --- gguf-py/examples/reader.py | 4 +- gguf-py/gguf/gguf_reader.py | 63 ++++++++++++++++++--- gguf-py/gguf/scripts/gguf_convert_endian.py | 23 +++----- gguf-py/gguf/scripts/gguf_dump.py | 34 +++++------ gguf-py/gguf/scripts/gguf_new_metadata.py | 43 +------------- gguf-py/pyproject.toml | 2 +- 6 files changed, 88 insertions(+), 81 deletions(-) diff --git a/gguf-py/examples/reader.py b/gguf-py/examples/reader.py index d841048c6d..703b782b5f 100644 --- a/gguf-py/examples/reader.py +++ b/gguf-py/examples/reader.py @@ -2,12 +2,14 @@ import logging import sys from pathlib import Path -from gguf.gguf_reader import GGUFReader logger = logging.getLogger("reader") +# Necessary to load the local gguf package sys.path.insert(0, str(Path(__file__).parent.parent)) +from gguf.gguf_reader import GGUFReader + def read_gguf_file(gguf_file_path): """ diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index 3568ea0dfc..5991cdb76b 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -6,6 +6,7 @@ from __future__ import annotations import logging import os +import sys from collections import OrderedDict from typing import Any, Literal, NamedTuple, TypeVar, Union @@ -15,7 +16,6 @@ import numpy.typing as npt from .quants import quant_shape_to_byte_shape if __name__ == "__main__": - import sys from pathlib import Path # Allow running file in package as a script. @@ -28,6 +28,7 @@ from gguf.constants import ( GGUF_VERSION, GGMLQuantizationType, GGUFValueType, + GGUFEndian, ) logger = logging.getLogger(__name__) @@ -53,6 +54,48 @@ class ReaderField(NamedTuple): types: list[GGUFValueType] = [] + def contents(self, index_or_slice: int | slice = slice(None)) -> Any: + if self.types: + to_string = lambda x: str(x.tobytes(), encoding='utf-8') # noqa: E731 + main_type = self.types[0] + + if main_type == GGUFValueType.ARRAY: + sub_type = self.types[-1] + + if sub_type == GGUFValueType.STRING: + indices = self.data[index_or_slice] + + if isinstance(index_or_slice, int): + return to_string(self.parts[indices]) # type: ignore + else: + return [to_string(self.parts[idx]) for idx in indices] # type: ignore + else: + # FIXME: When/if _get_field_parts() support multi-dimensional arrays, this must do so too + + # Check if it's unsafe to perform slice optimization on data + # if any(True for idx in self.data if len(self.parts[idx]) != 1): + # optim_slice = slice(None) + # else: + # optim_slice = index_or_slice + # index_or_slice = slice(None) + + # if isinstance(optim_slice, int): + # return self.parts[self.data[optim_slice]].tolist()[0] + # else: + # return [pv for idx in self.data[optim_slice] for pv in self.parts[idx].tolist()][index_or_slice] + + if isinstance(index_or_slice, int): + return self.parts[self.data[index_or_slice]].tolist()[0] + else: + return [pv for idx in self.data[index_or_slice] for pv in self.parts[idx].tolist()] + + if main_type == GGUFValueType.STRING: + return to_string(self.parts[-1]) + else: + return self.parts[-1].tolist()[0] + + return None + class ReaderTensor(NamedTuple): name: str @@ -101,10 +144,19 @@ class GGUFReader: # If we get 0 here that means it's (probably) a GGUF file created for # the opposite byte order of the machine this script is running on. self.byte_order = 'S' - temp_version = temp_version.newbyteorder(self.byte_order) + temp_version = temp_version.view(temp_version.dtype.newbyteorder(self.byte_order)) version = temp_version[0] if version not in READER_SUPPORTED_VERSIONS: raise ValueError(f'Sorry, file appears to be version {version} which we cannot handle') + if sys.byteorder == "little": + # Host is little endian + host_endian = GGUFEndian.LITTLE + swapped_endian = GGUFEndian.BIG + else: + # Sorry PDP or other weird systems that don't use BE or LE. + host_endian = GGUFEndian.BIG + swapped_endian = GGUFEndian.LITTLE + self.endianess = swapped_endian if self.byte_order == "S" else host_endian self.fields: OrderedDict[str, ReaderField] = OrderedDict() self.tensors: list[ReaderTensor] = [] offs += self._push_field(ReaderField(offs, 'GGUF.version', [temp_version], [0], [GGUFValueType.UINT32])) @@ -146,11 +198,7 @@ class GGUFReader: itemsize = int(np.empty([], dtype = dtype).itemsize) end_offs = offset + itemsize * count arr = self.data[offset:end_offs].view(dtype=dtype)[:count] - if override_order is not None: - return arr.view(arr.dtype.newbyteorder(override_order)) - if self.byte_order == 'S': - return arr.view(arr.dtype.newbyteorder(self.byte_order)) - return arr + return arr.view(arr.dtype.newbyteorder(self.byte_order if override_order is None else override_order)) def _push_field(self, field: ReaderField, skip_sum: bool = False) -> int: if field.name in self.fields: @@ -192,6 +240,7 @@ class GGUFReader: offs += int(alen.nbytes) aparts: list[npt.NDArray[Any]] = [raw_itype, alen] data_idxs: list[int] = [] + # FIXME: Handle multi-dimensional arrays properly instead of flattening for idx in range(alen[0]): curr_size, curr_parts, curr_idxs, curr_types = self._get_field_parts(offs, raw_itype[0]) if idx == 0: diff --git a/gguf-py/gguf/scripts/gguf_convert_endian.py b/gguf-py/gguf/scripts/gguf_convert_endian.py index 837831799b..0e0febaa79 100755 --- a/gguf-py/gguf/scripts/gguf_convert_endian.py +++ b/gguf-py/gguf/scripts/gguf_convert_endian.py @@ -20,22 +20,15 @@ logger = logging.getLogger("gguf-convert-endian") def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None: - if np.uint32(1) == np.uint32(1).newbyteorder("<"): - # Host is little endian - host_endian = "little" - swapped_endian = "big" + file_endian = reader.endianess.name + if reader.byte_order == 'S': + host_endian = 'BIG' if file_endian == 'LITTLE' else 'LITTLE' else: - # Sorry PDP or other weird systems that don't use BE or LE. - host_endian = "big" - swapped_endian = "little" - if reader.byte_order == "S": - file_endian = swapped_endian - else: - file_endian = host_endian - order = host_endian if args.order == "native" else args.order - logger.info(f"* Host is {host_endian.upper()} endian, GGUF file seems to be {file_endian.upper()} endian") + host_endian = file_endian + order = host_endian if args.order == "native" else args.order.upper() + logger.info(f"* Host is {host_endian} endian, GGUF file seems to be {file_endian} endian") if file_endian == order: - logger.info(f"* File is already {order.upper()} endian. Nothing to do.") + logger.info(f"* File is already {order} endian. Nothing to do.") sys.exit(0) logger.info("* Checking tensors for conversion compatibility") for tensor in reader.tensors: @@ -47,7 +40,7 @@ def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None gguf.GGMLQuantizationType.Q6_K, ): raise ValueError(f"Cannot handle type {tensor.tensor_type.name} for tensor {repr(tensor.name)}") - logger.info(f"* Preparing to convert from {file_endian.upper()} to {order.upper()}") + logger.info(f"* Preparing to convert from {file_endian} to {order}") if args.dry_run: return logger.warning("*** Warning *** Warning *** Warning **") diff --git a/gguf-py/gguf/scripts/gguf_dump.py b/gguf-py/gguf/scripts/gguf_dump.py index 20f23d729f..e282892d64 100755 --- a/gguf-py/gguf/scripts/gguf_dump.py +++ b/gguf-py/gguf/scripts/gguf_dump.py @@ -9,8 +9,6 @@ import sys from pathlib import Path from typing import Any -import numpy as np - # Necessary to load the local gguf package if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent.parent / 'gguf-py').exists(): sys.path.insert(0, str(Path(__file__).parent.parent.parent)) @@ -21,11 +19,11 @@ logger = logging.getLogger("gguf-dump") def get_file_host_endian(reader: GGUFReader) -> tuple[str, str]: - host_endian = 'LITTLE' if np.uint32(1) == np.uint32(1).newbyteorder("<") else 'BIG' + file_endian = reader.endianess.name if reader.byte_order == 'S': - file_endian = 'BIG' if host_endian == 'LITTLE' else 'LITTLE' + host_endian = 'BIG' if file_endian == 'LITTLE' else 'LITTLE' else: - file_endian = host_endian + host_endian = file_endian return (host_endian, file_endian) @@ -45,12 +43,20 @@ def dump_metadata(reader: GGUFReader, args: argparse.Namespace) -> None: pretty_type = str(field.types[-1].name) log_message = f' {n:5}: {pretty_type:10} | {len(field.data):8} | {field.name}' - if len(field.types) == 1: + if field.types: curr_type = field.types[0] if curr_type == GGUFValueType.STRING: - log_message += ' = {0}'.format(repr(str(bytes(field.parts[-1]), encoding='utf-8')[:60])) - elif field.types[0] in reader.gguf_scalar_to_np: - log_message += ' = {0}'.format(field.parts[-1][0]) + content = field.contents() + if len(content) > 60: + content = content[:57] + '...' + log_message += ' = {0}'.format(repr(content)) + elif curr_type in reader.gguf_scalar_to_np: + log_message += ' = {0}'.format(field.contents()) + else: + content = repr(field.contents(slice(6))) + if len(field.data) > 6: + content = content[:-1] + ', ...]' + log_message += ' = {0}'.format(content) print(log_message) # noqa: NP100 if args.no_tensors: return @@ -82,15 +88,9 @@ def dump_metadata_json(reader: GGUFReader, args: argparse.Namespace) -> None: curr["array_types"] = [t.name for t in field.types][1:] if not args.json_array: continue - itype = field.types[-1] - if itype == GGUFValueType.STRING: - curr["value"] = [str(bytes(field.parts[idx]), encoding="utf-8") for idx in field.data] - else: - curr["value"] = [pv for idx in field.data for pv in field.parts[idx].tolist()] - elif field.types[0] == GGUFValueType.STRING: - curr["value"] = str(bytes(field.parts[-1]), encoding="utf-8") + curr["value"] = field.contents() else: - curr["value"] = field.parts[-1].tolist()[0] + curr["value"] = field.contents() if not args.no_tensors: for idx, tensor in enumerate(reader.tensors): tensors[tensor.name] = { diff --git a/gguf-py/gguf/scripts/gguf_new_metadata.py b/gguf-py/gguf/scripts/gguf_new_metadata.py index a8cfc9d581..7aff6c9259 100755 --- a/gguf-py/gguf/scripts/gguf_new_metadata.py +++ b/gguf-py/gguf/scripts/gguf_new_metadata.py @@ -8,7 +8,6 @@ import sys import json from pathlib import Path -import numpy as np from tqdm import tqdm from typing import Any, Sequence, NamedTuple @@ -27,45 +26,10 @@ class MetadataDetails(NamedTuple): description: str = '' -def get_byteorder(reader: gguf.GGUFReader) -> gguf.GGUFEndian: - if np.uint32(1) == np.uint32(1).newbyteorder("<"): - # Host is little endian - host_endian = gguf.GGUFEndian.LITTLE - swapped_endian = gguf.GGUFEndian.BIG - else: - # Sorry PDP or other weird systems that don't use BE or LE. - host_endian = gguf.GGUFEndian.BIG - swapped_endian = gguf.GGUFEndian.LITTLE - - if reader.byte_order == "S": - return swapped_endian - else: - return host_endian - - -def decode_field(field: gguf.ReaderField | None) -> Any: - if field and field.types: - main_type = field.types[0] - - if main_type == gguf.GGUFValueType.ARRAY: - sub_type = field.types[-1] - - if sub_type == gguf.GGUFValueType.STRING: - return [str(bytes(field.parts[idx]), encoding='utf-8') for idx in field.data] - else: - return [pv for idx in field.data for pv in field.parts[idx].tolist()] - if main_type == gguf.GGUFValueType.STRING: - return str(bytes(field.parts[-1]), encoding='utf-8') - else: - return field.parts[-1][0] - - return None - - def get_field_data(reader: gguf.GGUFReader, key: str) -> Any: field = reader.get_field(key) - return decode_field(field) + return field.contents() if field else None def find_token(token_list: Sequence[int], token: str) -> Sequence[int]: @@ -93,7 +57,7 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new logger.debug(f'Removing {field.name}') continue - old_val = MetadataDetails(field.types[0], decode_field(field)) + old_val = MetadataDetails(field.types[0], field.contents()) val = new_metadata.get(field.name, old_val) if field.name in new_metadata: @@ -192,7 +156,6 @@ def main() -> None: reader = gguf.GGUFReader(args.input, 'r') arch = get_field_data(reader, gguf.Keys.General.ARCHITECTURE) - endianess = get_byteorder(reader) token_list = get_field_data(reader, gguf.Keys.Tokenizer.LIST) or [] @@ -230,7 +193,7 @@ def main() -> None: sys.exit(0) logger.info(f'* Writing: {args.output}') - writer = gguf.GGUFWriter(args.output, arch=arch, endianess=endianess) + writer = gguf.GGUFWriter(args.output, arch=arch, endianess=reader.endianess) alignment = get_field_data(reader, gguf.Keys.General.ALIGNMENT) if alignment is not None: diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml index b4a47333dd..d214e87201 100644 --- a/gguf-py/pyproject.toml +++ b/gguf-py/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gguf" -version = "0.15.0" +version = "0.16.0" description = "Read and write ML models in GGUF for GGML" authors = ["GGML "] packages = [ From a800ae46da2ed7dac236aa6bf2b595da6b6294b5 Mon Sep 17 00:00:00 2001 From: Ting Lou Date: Wed, 26 Feb 2025 22:26:52 +0800 Subject: [PATCH 030/188] llava : add struct for FFI bindgen (#12079) * add struct for FFI bindgen * Apply suggestions from code review --------- Co-authored-by: Xuan-Son Nguyen --- examples/llava/clip.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/llava/clip.h b/examples/llava/clip.h index ce6f61944c..002c419653 100644 --- a/examples/llava/clip.h +++ b/examples/llava/clip.h @@ -74,8 +74,11 @@ CLIP_API void clip_image_f32_free(struct clip_image_f32 * img); CLIP_API void clip_image_u8_batch_free (struct clip_image_u8_batch * batch); CLIP_API void clip_image_f32_batch_free(struct clip_image_f32_batch * batch); -/** build image from pixels decoded by other libraries instead of stb_image.h for better performance. The memory layout is RGBRGBRGB..., input buffer length must be 3*nx*ny bytes */ -CLIP_API void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, clip_image_u8 * img); +/** + * Build image from pixels decoded by other libraries instead of stb_image.h for better performance. + * The memory layout is RGBRGBRGB..., input buffer length must be 3*nx*ny bytes + */ +CLIP_API void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, struct clip_image_u8 * img); CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img); From b95c8af37ccf169b0a3216b7ed691af0534e5091 Mon Sep 17 00:00:00 2001 From: Vladimir Vuksanovic <109677816+vvuksanovic@users.noreply.github.com> Date: Thu, 27 Feb 2025 08:42:48 +0100 Subject: [PATCH 031/188] cmake: Fix ggml backend dependencies and installation (#11818) * Fix dependencies between ggml and backends ggml backends link only to ggml-base and ggml links to all backends. * Fix installation of ggml backends Set up GNUInstallDirs before setting the installation directory of ggml backends --- ggml/CMakeLists.txt | 3 ++- ggml/cmake/ggml-config.cmake.in | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 68b3f148ea..610010da8d 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -212,6 +212,8 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) +include(GNUInstallDirs) + # # build the library # @@ -235,7 +237,6 @@ endif () # install # -include(GNUInstallDirs) include(CMakePackageConfigHelpers) # all public headers diff --git a/ggml/cmake/ggml-config.cmake.in b/ggml/cmake/ggml-config.cmake.in index bf39f9c007..823eb797b7 100644 --- a/ggml/cmake/ggml-config.cmake.in +++ b/ggml/cmake/ggml-config.cmake.in @@ -112,7 +112,7 @@ foreach(_ggml_backend ${GGML_AVAILABLE_BACKENDS}) string(REGEX MATCH "^ggml-cpu" is_cpu_variant "${_ggml_backend}") if(is_cpu_variant) - list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES "ggml::ggml" "ggml::ggml-base") + list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES "ggml::ggml-base") set_target_properties(ggml::${_ggml_backend} PROPERTIES INTERFACE_LINK_LIBRARIES "${GGML_CPU_INTERFACE_LINK_LIBRARIES}") @@ -124,7 +124,7 @@ foreach(_ggml_backend ${GGML_AVAILABLE_BACKENDS}) endif() else() - list(APPEND ${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES "ggml::ggml" "ggml::ggml-base") + list(APPEND ${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES "ggml::ggml-base") set_target_properties(ggml::${_ggml_backend} PROPERTIES INTERFACE_LINK_LIBRARIES "${${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES}") @@ -139,6 +139,11 @@ foreach(_ggml_backend ${GGML_AVAILABLE_BACKENDS}) list(APPEND _ggml_all_targets ggml::${_ggml_backend}) endforeach() +list(APPEND GGML_INTERFACE_LINK_LIBRARIES ggml::ggml-base "${_ggml_all_targets}") +set_target_properties(ggml::ggml + PROPERTIES + INTERFACE_LINK_LIBRARIES "${GGML_INTERFACE_LINK_LIBRARIES}") + add_library(ggml::all INTERFACE IMPORTED) set_target_properties(ggml::all PROPERTIES From 581650b7cacec2872982fde381bd3bcda0f78699 Mon Sep 17 00:00:00 2001 From: Daniele <57776841+daniandtheweb@users.noreply.github.com> Date: Fri, 28 Feb 2025 06:52:51 +0000 Subject: [PATCH 032/188] vulkan: improve im2col (#11826) * vulkan: improve im2col performance --- .../ggml-vulkan/vulkan-shaders/im2col.comp | 53 ++++++++++++------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp b/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp index 122b1e93fb..09aa849e88 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp @@ -40,6 +40,20 @@ void main() { const uint batch = gl_GlobalInvocationID.z / p.IC; const uint ic = gl_GlobalInvocationID.z % p.IC; + const uint src_base = ic * p.offset_delta + batch * p.batch_offset; + const uint dst_base = ((batch * p.OH + oh) * p.OW) * p.CHW + ic * (p.KW * p.KH); + const int oh_s1 = int(oh) * p.s1; + const uint ksize = p.OW * (p.KH > 1 ? p.KW : 1); + + const uint base_linear_idx = gidx * NUM_ITER; + + const uint max_ky = ksize / p.OW; + + uint current_kx = base_linear_idx / ksize; + const uint rem = base_linear_idx - (current_kx * ksize); + uint current_ky = rem / p.OW; + uint current_ix = rem % p.OW; + A_TYPE values[NUM_ITER]; uint offset_dst[NUM_ITER]; [[unroll]] for (uint idx = 0; idx < NUM_ITER; ++idx) { @@ -48,36 +62,35 @@ void main() { [[unroll]] for (uint idx = 0; idx < NUM_ITER; ++idx) { - const uint i = gidx * NUM_ITER + idx; + const uint linear_idx = base_linear_idx + idx; - const uint ksize = p.OW * (p.KH > 1 ? p.KW : 1); - const uint kx = i / ksize; - const uint kd = kx * ksize; - const uint ky = (i - kd) / p.OW; - const uint ix = i % p.OW; - - const uint iiw = ix * p.s0 + kx * p.d0 - p.p0; - const uint iih = oh * p.s1 + ky * p.d1 - p.p1; - - offset_dst[idx] = - ((batch * p.OH + oh) * p.OW + ix) * p.CHW + - (ic * (p.KW * p.KH) + ky * p.KW + kx); - - if (i >= p.pelements) { + if (linear_idx >= p.pelements) { continue; } - if (iih < p.IH && iiw < p.IW) { - const uint offset_src = ic * p.offset_delta + batch * p.batch_offset; - values[idx] = data_a[offset_src + iih * p.IW + iiw]; + const uint iiw = current_ix * p.s0 + current_kx * p.d0 - p.p0; + const uint iih = oh_s1 + current_ky * p.d1 - p.p1; + + offset_dst[idx] = dst_base + current_ix * p.CHW + current_ky * p.KW + current_kx; + + if ((iih < p.IH) && (iiw < p.IW)) { + values[idx] = data_a[src_base + iih * p.IW + iiw]; + } + + if (++current_ix == p.OW) { + current_ix = 0; + if (++current_ky == max_ky) { + current_ky = 0; + current_kx++; + } } } [[unroll]] for (uint idx = 0; idx < NUM_ITER; ++idx) { - const uint i = gidx * NUM_ITER + idx; + const uint linear_idx = base_linear_idx + idx; - if (i >= p.pelements) { + if (linear_idx >= p.pelements) { continue; } From fbeda9002d4b8b79a4f9288a7ff0d34ef4fb23d5 Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Fri, 28 Feb 2025 07:20:08 +0000 Subject: [PATCH 033/188] vulkan: matmul dequantization improvements (#12015) * faster dequant for old quants * dont use unpack for iq4_nl * vec2 unpack for q8 --- .../vulkan-shaders/dequant_funcs.comp | 6 +- .../vulkan-shaders/dequant_funcs_cm2.comp | 2 +- .../ggml-vulkan/vulkan-shaders/mul_mm.comp | 126 +++++++++++------- .../src/ggml-vulkan/vulkan-shaders/types.comp | 2 +- .../vulkan-shaders/vulkan-shaders-gen.cpp | 10 +- 5 files changed, 93 insertions(+), 53 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp index 10318e8766..8835c442ec 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp @@ -82,9 +82,9 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { return vec2(int(data_a[a_offset + ib].qs[iqs]), int(data_a[a_offset + ib].qs[iqs + 1])); } vec4 dequantize4(uint ib, uint iqs, uint a_offset) { - uint32_t v0 = data_a_packed16[a_offset + ib].qs[iqs/2]; - uint32_t v1 = data_a_packed16[a_offset + ib].qs[iqs/2 + 1]; - return vec4(int8_t(v0 & 0xFF), int8_t(v0 >> 8), int8_t(v1 & 0xFF), int8_t(v1 >> 8)); + const i8vec2 v0 = unpack8(data_a_packed16[a_offset + ib].qs[iqs/2]); + const i8vec2 v1 = unpack8(data_a_packed16[a_offset + ib].qs[iqs/2 + 1]); + return vec4(v0.x, v0.y, v1.x, v1.y); } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp index 4770469edd..4ccbe613af 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp @@ -92,7 +92,7 @@ float16_t dequantFuncQ8_0(const in decodeBufQ8_0 bl, const in uint blockCoords[2 const uint iqs = idx; // Load 16b and select the byte for this element - int32_t qs = unpack8(int32_t(bl.block.qs[(iqs & 0x1E) >> 1]))[iqs & 1]; + int32_t qs = unpack8(bl.block.qs[(iqs & 0x1E) >> 1])[iqs & 1]; float16_t ret = float16_t(qs) * d; return ret; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 39657195cf..a8fd93fdea 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -32,6 +32,13 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; +#if defined(A_TYPE_PACKED16) +layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];}; +#endif +#if defined(A_TYPE_PACKED32) +layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];}; +#endif + layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; @@ -243,74 +250,100 @@ void main() { #endif #elif defined(DATA_A_Q4_0) const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; - const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + loadr_a; + const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + 4 * loadr_a; - const uint ib = idx / 16; - const uint iqs = idx & 0xF; + const uint ib = idx / 4; + const uint iqs = idx & 0x03; - const float d = float(data_a[ib].d); - const uint vui = uint(data_a[ib].qs[iqs]); - const vec2 v = (vec2(vui & 0xF, vui >> 4) - 8.0f) * d; + const float d = float(data_a_packed16[ib].d); + const uint vui = uint(data_a_packed16[ib].qs[2*iqs]) | (uint(data_a_packed16[ib].qs[2*iqs + 1]) << 16); + const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d; + const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d; - buf_a[buf_idx ] = FLOAT_TYPE(v.x); - buf_a[buf_idx + 16] = FLOAT_TYPE(v.y); + buf_a[buf_idx ] = FLOAT_TYPE(v0.x); + buf_a[buf_idx + 1 ] = FLOAT_TYPE(v0.y); + buf_a[buf_idx + 2 ] = FLOAT_TYPE(v0.z); + buf_a[buf_idx + 3 ] = FLOAT_TYPE(v0.w); + buf_a[buf_idx + 16] = FLOAT_TYPE(v1.x); + buf_a[buf_idx + 17] = FLOAT_TYPE(v1.y); + buf_a[buf_idx + 18] = FLOAT_TYPE(v1.z); + buf_a[buf_idx + 19] = FLOAT_TYPE(v1.w); #elif defined(DATA_A_Q4_1) const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; - const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + loadr_a; + const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + 4 * loadr_a; - const uint ib = idx / 16; - const uint iqs = idx & 0xF; + const uint ib = idx / 4; + const uint iqs = idx & 0x03; - const float d = float(data_a[ib].d); - const float m = float(data_a[ib].m); - const uint vui = uint(data_a[ib].qs[iqs]); - const vec2 v = vec2(vui & 0xF, vui >> 4) * d + m; + const float d = float(data_a_packed16[ib].d); + const float m = float(data_a_packed16[ib].m); + const uint vui = uint(data_a_packed16[ib].qs[2*iqs]) | (uint(data_a_packed16[ib].qs[2*iqs + 1]) << 16); + const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * d + m; + const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * d + m; - buf_a[buf_idx ] = FLOAT_TYPE(v.x); - buf_a[buf_idx + 16] = FLOAT_TYPE(v.y); + buf_a[buf_idx ] = FLOAT_TYPE(v0.x); + buf_a[buf_idx + 1 ] = FLOAT_TYPE(v0.y); + buf_a[buf_idx + 2 ] = FLOAT_TYPE(v0.z); + buf_a[buf_idx + 3 ] = FLOAT_TYPE(v0.w); + buf_a[buf_idx + 16] = FLOAT_TYPE(v1.x); + buf_a[buf_idx + 17] = FLOAT_TYPE(v1.y); + buf_a[buf_idx + 18] = FLOAT_TYPE(v1.z); + buf_a[buf_idx + 19] = FLOAT_TYPE(v1.w); #elif defined(DATA_A_Q5_0) const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; - const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + loadr_a; + const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + 2 * loadr_a; - const uint ib = idx / 16; - const uint iqs = idx & 0xF; + const uint ib = idx / 8; + const uint iqs = idx & 0x07; - const float d = float(data_a[ib].d); - const uint uint_qh = uint(data_a[ib].qh[1]) << 16 | data_a[ib].qh[0]; - const ivec2 qh = ivec2(((uint_qh >> iqs) << 4) & 0x10, (uint_qh >> (iqs + 12)) & 0x10); - const uint vui = uint(data_a[ib].qs[iqs]); - const vec2 v = (vec2((vui & 0xF) | qh.x, (vui >> 4) | qh.y) - 16.0f) * d; + const float d = float(data_a_packed16[ib].d); + const uint uint_qh = uint(data_a_packed16[ib].qh[1]) << 16 | uint(data_a_packed16[ib].qh[0]); + const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); + const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); + + const uint vui = uint(data_a_packed16[ib].qs[iqs]); + const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d; buf_a[buf_idx ] = FLOAT_TYPE(v.x); + buf_a[buf_idx + 1 ] = FLOAT_TYPE(v.z); buf_a[buf_idx + 16] = FLOAT_TYPE(v.y); + buf_a[buf_idx + 17] = FLOAT_TYPE(v.w); #elif defined(DATA_A_Q5_1) const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; - const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + loadr_a; + const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + 2 * loadr_a; - const uint ib = idx / 16; - const uint iqs = idx & 0xF; + const uint ib = idx / 8; + const uint iqs = idx & 0x07; - const float d = float(data_a[ib].d); - const float m = float(data_a[ib].m); - const uint uint_qh = data_a[ib].qh; - const ivec2 qh = ivec2(((uint_qh >> iqs) << 4) & 0x10, (uint_qh >> (iqs + 12)) & 0x10); - const uint vui = uint(data_a[ib].qs[iqs]); - const vec2 v = vec2((vui & 0xF) | qh.x, (vui >> 4) | qh.y) * d + m; + const float d = float(data_a_packed16[ib].d); + const float m = float(data_a_packed16[ib].m); + const uint uint_qh = data_a_packed16[ib].qh; + const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); + const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); + + const uint vui = uint(data_a_packed16[ib].qs[iqs]); + const vec4 v = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) * d + m; buf_a[buf_idx ] = FLOAT_TYPE(v.x); + buf_a[buf_idx + 1 ] = FLOAT_TYPE(v.z); buf_a[buf_idx + 16] = FLOAT_TYPE(v.y); + buf_a[buf_idx + 17] = FLOAT_TYPE(v.w); #elif defined(DATA_A_Q8_0) const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + loadr_a * LOAD_VEC_A; - const uint ib = idx / 16; - const uint iqs = (idx & 0xF) * 2; + const uint ib = idx / 8; + const uint iqs = idx & 0x07; - const float d = float(data_a[ib].d); - const vec2 v = vec2(int(data_a[ib].qs[iqs]), int(data_a[ib].qs[iqs + 1])) * d; + const float d = float(data_a_packed16[ib].d); + const i8vec2 v0 = unpack8(data_a_packed16[ib].qs[2*iqs]); + const i8vec2 v1 = unpack8(data_a_packed16[ib].qs[2*iqs + 1]); + const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; buf_a[buf_idx ] = FLOAT_TYPE(v.x); buf_a[buf_idx + 1] = FLOAT_TYPE(v.y); + buf_a[buf_idx + 2] = FLOAT_TYPE(v.z); + buf_a[buf_idx + 3] = FLOAT_TYPE(v.w); #elif defined(DATA_A_Q2_K) const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + loadr_a * LOAD_VEC_A; @@ -623,17 +656,18 @@ void main() { buf_a[buf_idx + 1] = FLOAT_TYPE(v.y); #elif defined(DATA_A_IQ4_NL) const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; - const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + loadr_a; + const uint buf_idx = (loadc_a + l) * SHMEM_STRIDE + 2 * loadr_a; - const uint ib = idx / 16; - const uint iqs = idx & 0xF; + const uint ib = idx / 8; + const uint iqs = idx & 0x07; - const float d = float(data_a[ib].d); - const uint vui = uint(data_a[ib].qs[iqs]); - const vec2 v = vec2(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[vui >> 4]) * d; + const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d); + const uint vui = uint(data_a_packed16[ib].qs[iqs]); - buf_a[buf_idx ] = FLOAT_TYPE(v.x); - buf_a[buf_idx + 16] = FLOAT_TYPE(v.y); + buf_a[buf_idx ] = FLOAT_TYPE(kvalues_iq4nl[vui & 0xF]) * d; + buf_a[buf_idx + 1 ] = FLOAT_TYPE(kvalues_iq4nl[bitfieldExtract(vui, 8, 4)]) * d; + buf_a[buf_idx + 16] = FLOAT_TYPE(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)]) * d; + buf_a[buf_idx + 17] = FLOAT_TYPE(kvalues_iq4nl[vui >> 12]) * d; #endif } [[unroll]] for (uint l = 0; l < BN; l += loadstride_b) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp index dfa16cda51..907067d7fa 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp @@ -139,7 +139,7 @@ struct block_q8_0 struct block_q8_0_packed16 { float16_t d; - uint16_t qs[32/2]; + int16_t qs[32/2]; }; #if defined(DATA_A_Q8_0) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index c5e0bba82b..4a81505565 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -325,11 +325,17 @@ void matmul_shaders(bool fp16, bool matmul_id, bool coopmat, bool coopmat2, bool string_to_spv(shader_name + "_f16", source_name, merge_maps(base_dict, {{"DATA_A_F16", "1"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); for (const auto& tname : type_names) { + std::string load_vec_quant = "2"; + if ((tname == "q4_0") || (tname == "q4_1")) + load_vec_quant = "8"; + else if ((tname == "q5_0") || (tname == "q5_1") || (tname == "q8_0") || (tname == "iq4_nl")) + load_vec_quant = "4"; + std::string data_a_key = "DATA_A_" + to_uppercase(tname); // For unaligned, load one at a time for f32/f16, or two at a time for quants - std::string load_vec_a_unaligned = (coopmat2 || tname == "f32" || tname == "f16") ? "1" : "2"; + std::string load_vec_a_unaligned = (coopmat2 || tname == "f32" || tname == "f16") ? "1" : load_vec_quant; // For aligned matmul loads - std::string load_vec_a = (coopmat2 || tname == "f32" || tname == "f16") ? load_vec : "2"; + std::string load_vec_a = (coopmat2 || tname == "f32" || tname == "f16") ? load_vec : load_vec_quant; // don't generate f32 variants for coopmat2 if (!coopmat2) { From 673cfef9aa8daad09966208909f346eb996628a4 Mon Sep 17 00:00:00 2001 From: hipudding Date: Fri, 28 Feb 2025 15:23:47 +0800 Subject: [PATCH 034/188] CANN: Fix build error with GCC 13 (#11990) Remove unused header file that causes compilation failure on ARM platform with GCC 13. --- ggml/src/ggml-cann/kernels/dup.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-cann/kernels/dup.cpp b/ggml/src/ggml-cann/kernels/dup.cpp index c7ba38d10a..d9b9574494 100644 --- a/ggml/src/ggml-cann/kernels/dup.cpp +++ b/ggml/src/ggml-cann/kernels/dup.cpp @@ -1,7 +1,5 @@ #include "kernel_operator.h" -#include - using namespace AscendC; #define BUFFER_NUM 2 @@ -183,7 +181,7 @@ extern "C" __global__ __aicore__ void ascendc_dup_by_rows_fp32( copy_to_ub(output_ne_gm, output_ne_ub, 32); copy_to_ub(output_nb_gm, output_nb_ub, 32); - DupByRows op; + DupByRows op; op.init(src_gm, dst_gm, input_ne_ub, input_nb_ub); op.dup(); } @@ -206,7 +204,7 @@ extern "C" __global__ __aicore__ void ascendc_dup_by_rows_fp32_to_fp16( copy_to_ub(output_ne_gm, output_ne_ub, 32); copy_to_ub(output_nb_gm, output_nb_ub, 32); - DupByRows op; + DupByRows op; op.init(src_gm, dst_gm, input_ne_ub, input_nb_ub); op.dup_with_cast(); } @@ -230,7 +228,7 @@ extern "C" __global__ __aicore__ void ascendc_dup_by_rows_fp16_to_fp32( copy_to_ub(output_ne_gm, output_ne_ub, 32); copy_to_ub(output_nb_gm, output_nb_ub, 32); - DupByRows op; + DupByRows op; op.init(src_gm, dst_gm, input_ne_ub, input_nb_ub); op.dup_with_cast(); } From 05e6f5aad0a4bb48fb2775f2a78505540fdbc47d Mon Sep 17 00:00:00 2001 From: Prashant Vithule <119530321+Vithulep@users.noreply.github.com> Date: Fri, 28 Feb 2025 13:06:12 +0530 Subject: [PATCH 035/188] ggml: aarch64: implement SVE kernels for q2_k_q8_k vector dot (#12064) * Added SVE Support for Q2_K Quantized Models * Use 4-space indentation in the switch cases * removed comments lines * Remove the loop Retain the curly bracess for better understanding of code * Remove the comment like added for q3_k_q8_k kernel --------- Co-authored-by: vithulep --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 247 +++++++++++++++++++++++++++- 1 file changed, 246 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 8d5e3e20bb..2679b71ffd 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -4587,7 +4587,252 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r const int nb = n / QK_K; -#ifdef __ARM_NEON +#ifdef __ARM_FEATURE_SVE + const int vector_length = svcntb()*8; + const svuint8_t m3s = svdup_n_u8(0x3); + const svuint32_t m4s = svdup_n_u32(0xF); + const svint32_t vzero_sv = svdup_n_s32(0); + svfloat32_t acc_sum = svdup_n_f32(0); + svbool_t pred_s32 = svptrue_pat_b32(SV_VL4); + + switch (vector_length) { + case 128: + for (int i = 0; i < nb; ++i) { + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + svfloat32_t d_broad = svdup_n_f32((float32_t)d); + const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + svfloat32_t dmin_broad = svdup_n_f32((float32_t)dmin); + + const uint8_t * restrict q2 = x[i].qs; + const int8_t * restrict q8_sv = y[i].qs; + const uint8_t * restrict sc = x[i].scales; + + svuint32_t mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc); + const svint32_t mins_sv_1 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4)); + + mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc+4); + const svint32_t mins_sv_2 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4)); + + svint32_t q8sums_sv_1 = svld1sh_s32(svptrue_b32(), y[i].bsums); + svint32_t q8sums_sv_2 = svld1sh_s32(svptrue_b32(), y[i].bsums+4); + + const svint32_t s0 = svadd_s32_x(svptrue_b32(), svmul_s32_x(svptrue_b32(), mins_sv_1, q8sums_sv_1), svmul_s32_x(svptrue_b32(), mins_sv_2, q8sums_sv_2)); + + mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc+8); + const svint32_t mins_sv_3 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4)); + + mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc+12); + const svint32_t mins_sv_4 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4)); + + q8sums_sv_1 = svld1sh_s32(svptrue_b32(), y[i].bsums+8); + q8sums_sv_2 = svld1sh_s32(svptrue_b32(), y[i].bsums+12); + + svint32_t s1 = svadd_s32_x(svptrue_b32(), svmul_s32_x(svptrue_b32(), mins_sv_3, q8sums_sv_1), svmul_s32_x(svptrue_b32(), mins_sv_4, q8sums_sv_2)); + + svfloat32_t temp = svcvt_f32_s32_x(svptrue_b32(), svadd_s32_x(svptrue_b32(), s0, s1)); + + acc_sum = svmla_f32_m(svptrue_b32(), acc_sum, temp, dmin_broad); + + svint32_t sumi1 = svdup_n_s32(0); + + { + const svuint8_t q2bits_1 = svld1_u8(svptrue_b8(), q2); + svint8_t q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_1, m3s)); + svint8_t q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + const svint32_t scales_sv = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc), m4s)); + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 0)); + + const svuint8_t q2bits_3 = svld1_u8(svptrue_b8(), q2+16); + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_3, m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 1)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_1, 2), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 2)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_3, 2), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 3)); + + + const svint32_t scales_sv_1 = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc+4), m4s)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_1, 4), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 0)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_3, 4), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 1)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_1, 6), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 2)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_3, 6), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 3)); + + //------------------------------- + + q2 += 32; + const svint32_t scales_sv_2 = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc+8), m4s)); + const svuint8_t q2bits_2 = svld1_u8(svptrue_b8(), q2); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_2, m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 0)); + + const svuint8_t q2bits_4 = svld1_u8(svptrue_b8(), q2+16); + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_4, m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 1)); + + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_2, 2), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 2)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_4, 2), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 3)); + + + const svint32_t scales_sv_3 = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc+12), m4s)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_2, 4), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 0)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_4, 4), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 1)); + + + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_2, 6), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 2)); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_4, 6), m3s)); + q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16; + + sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 3)); + } + acc_sum = svmla_f32_m(svptrue_b32(), acc_sum, svcvt_f32_s32_x(svptrue_b32(), sumi1), d_broad); + } + *s = svaddv_f32(svptrue_b32(), acc_sum); + break; + + case 256: + case 512: + for (int i = 0; i < nb; ++i) { + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + svfloat32_t d_broad = svdup_n_f32((float32_t)d); + const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + svfloat32_t dmin_broad = svdup_n_f32((float32_t)dmin); + + const uint8_t * restrict q2 = x[i].qs; + const int8_t * restrict q8_sv = y[i].qs; + const uint8_t * restrict sc = x[i].scales; + + const svuint32_t mins_and_scales_sve = svld1ub_u32(svptrue_pat_b32(SV_VL8), sc); sc += 8; + const svint32_t scales_sv = svreinterpret_s32_u32(svand_u32_m(svptrue_pat_b32(SV_VL8), mins_and_scales_sve, m4s)); + const svint32_t mins_sv_1 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_pat_b32(SV_VL8), mins_and_scales_sve, 4)); + svint32_t q8sums_sv_1 = svld1sh_s32(svptrue_pat_b32(SV_VL8), y[i].bsums); + + const svuint32_t mins_and_scales_sve_1 = svld1ub_u32(svptrue_pat_b32(SV_VL8), sc); + const svint32_t scales_sv_1 = svreinterpret_s32_u32(svand_u32_m(svptrue_pat_b32(SV_VL8), mins_and_scales_sve_1, m4s)); + const svint32_t mins_sv_2 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_pat_b32(SV_VL8), mins_and_scales_sve_1, 4)); + + svint32_t q8sums_sv_2 = svld1sh_s32(svptrue_pat_b32(SV_VL8), y[i].bsums+8); + + svfloat32_t temp = svcvt_f32_s32_x(svptrue_pat_b32(SV_VL8), svadd_s32_x(svptrue_pat_b32(SV_VL8), svmul_s32_x(svptrue_pat_b32(SV_VL8), mins_sv_1, q8sums_sv_1), svmul_s32_x(svptrue_pat_b32(SV_VL8), mins_sv_2, q8sums_sv_2))); + + acc_sum = svmla_f32_m(svptrue_pat_b32(SV_VL8), acc_sum, temp, dmin_broad); + + svint32_t sumi1 = svdup_n_s32(0); + + { + const svuint8_t q2bits_1 = svld1_u8(svptrue_pat_b8(SV_VL32), q2); + svint8_t q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), q2bits_1, m3s)); + svint8_t q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32; + + svint32_t scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv, 0), svdup_lane_s32(scales_sv, 1)); + sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_1, 2), m3s)); + q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32; + + svint32_t scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv, 2), svdup_lane_s32(scales_sv, 3)); + sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(svdup_n_s32(0), q2bytes_sv, q8bytes_sv), scale_2); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_1, 4), m3s)); + q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32; + + scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv, 4), svdup_lane_s32(scales_sv, 5)); + sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_1, 6), m3s)); + q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32; + + scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv, 6), svdup_lane_s32(scales_sv, 7)); + sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_2); + + q2 += 32; + + const svuint8_t q2bits_2 = svld1_u8(svptrue_pat_b8(SV_VL32), q2); + q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), q2bits_2, m3s)); + q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32; + + scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 0), svdup_lane_s32(scales_sv_1, 1)); + sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_2, 2), m3s)); + q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32; + + scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 2), svdup_lane_s32(scales_sv_1, 3)); + sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_2); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_2, 4), m3s)); + q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32; + + scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 4), svdup_lane_s32(scales_sv_1, 5)); + sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1); + + q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_2, 6), m3s)); + q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32; + + scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 6), svdup_lane_s32(scales_sv_1, 7)); + sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_2); + } + acc_sum = svmla_f32_m(svptrue_pat_b32(SV_VL8), acc_sum, svcvt_f32_s32_x(svptrue_pat_b32(SV_VL8), sumi1), d_broad); + } + *s = svaddv_f32(svptrue_pat_b32(SV_VL8), acc_sum); + break; + + default: + assert(false && "Unsupported vector length"); + break; + } + +#elif __ARM_NEON const uint8x16_t m3 = vdupq_n_u8(0x3); const uint8x16_t m4 = vdupq_n_u8(0xF); From 9c42b1718ca8299f9afeabdc122badeab64c9690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Fri, 28 Feb 2025 09:26:43 +0100 Subject: [PATCH 036/188] CUDA: fix logic for V100 + GGML_CUDA_FORCE_MMQ (#12098) --- ggml/src/ggml-cuda/mmq.cuh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 0451c65f30..f2aca1f201 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -109,9 +109,9 @@ static constexpr __device__ int get_mmq_x_max_device() { #if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA #ifdef GGML_CUDA_FORCE_MMQ - return MMQ_DP4A_MAX_BATCH_SIZE; -#else // GGML_CUDA_FORCE_MMQ return 128; +#else // GGML_CUDA_FORCE_MMQ + return MMQ_DP4A_MAX_BATCH_SIZE; #endif // GGML_CUDA_FORCE_MMQ #else // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA From 438a83926afcff3643ffef5543db67545ceffe39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20O?= Date: Fri, 28 Feb 2025 09:42:52 +0100 Subject: [PATCH 037/188] vulkan: add specific MMV kernels for IQ2 and IQ3 quants + optimizations (#11595) * vulkan: implement specialized MMV kernels for IQ2 quantizations * vulkan: add MMV kernels for IQ3 quants * vulkan: Increase MMV batch size and unroll IQ LUT setup * vulkan: fix init_iq_shmem for WG sizes larger than tables * vulkan: common batch size for all I-quants --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 55 ++++++------ .../vulkan-shaders/get_rows_quant.comp | 2 + .../vulkan-shaders/mul_mat_vec_iq2_s.comp | 90 +++++++++++++++++++ .../vulkan-shaders/mul_mat_vec_iq2_xs.comp | 87 ++++++++++++++++++ .../vulkan-shaders/mul_mat_vec_iq2_xxs.comp | 87 ++++++++++++++++++ .../vulkan-shaders/mul_mat_vec_iq3_s.comp | 90 +++++++++++++++++++ .../vulkan-shaders/mul_mat_vec_iq3_xxs.comp | 88 ++++++++++++++++++ .../src/ggml-vulkan/vulkan-shaders/types.comp | 50 ++++++++--- .../vulkan-shaders/vulkan-shaders-gen.cpp | 2 +- 9 files changed, 509 insertions(+), 42 deletions(-) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index abe3e7908c..ce15f620f0 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1992,6 +1992,7 @@ static void ggml_vk_load_shaders(vk_device& device) { } } else if (device->vendor_id == VK_VENDOR_ID_INTEL) rm_stdq = 2; + uint32_t rm_iq = 2 * rm_kq; for (uint32_t i = 0; i < mul_mat_vec_max_cols; ++i) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f32_f32_"+std::to_string(i+1), mul_mat_vec_f32_f32_f32_len, mul_mat_vec_f32_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {device->subgroup_size, 2, i+1}, 1); @@ -2006,15 +2007,15 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f32_f32_"+std::to_string(i+1), mul_mat_vec_q4_k_f32_f32_len, mul_mat_vec_q4_k_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f32_f32_"+std::to_string(i+1), mul_mat_vec_q5_k_f32_f32_len, mul_mat_vec_q5_k_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f32_f32_"+std::to_string(i+1), mul_mat_vec_q6_k_f32_f32_len, mul_mat_vec_q6_k_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq1_s_f32_f32_len, mul_mat_vec_iq1_s_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq1_m_f32_f32_len, mul_mat_vec_iq1_m_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq2_xxs_f32_f32_len, mul_mat_vec_iq2_xxs_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq2_xs_f32_f32_len, mul_mat_vec_iq2_xs_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq2_s_f32_f32_len, mul_mat_vec_iq2_s_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq3_xxs_f32_f32_len, mul_mat_vec_iq3_xxs_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq3_s_f32_f32_len, mul_mat_vec_iq3_s_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq4_xs_f32_f32_len, mul_mat_vec_iq4_xs_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq4_nl_f32_f32_len, mul_mat_vec_iq4_nl_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {subgroup_size_16, 2*rm_stdq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq1_s_f32_f32_len, mul_mat_vec_iq1_s_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq1_m_f32_f32_len, mul_mat_vec_iq1_m_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq2_xxs_f32_f32_len, mul_mat_vec_iq2_xxs_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq2_xs_f32_f32_len, mul_mat_vec_iq2_xs_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq2_s_f32_f32_len, mul_mat_vec_iq2_s_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq3_xxs_f32_f32_len, mul_mat_vec_iq3_xxs_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq3_s_f32_f32_len, mul_mat_vec_iq3_s_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq4_xs_f32_f32_len, mul_mat_vec_iq4_xs_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f32_f32_"+std::to_string(i+1), mul_mat_vec_iq4_nl_f32_f32_len, mul_mat_vec_iq4_nl_f32_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f16_f32_"+std::to_string(i+1), mul_mat_vec_f32_f16_f32_len, mul_mat_vec_f32_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {device->subgroup_size, 2, i+1}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f16_f32_"+std::to_string(i+1), mul_mat_vec_f16_f16_f32_len, mul_mat_vec_f16_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {device->subgroup_size, 2, i+1}, 1); @@ -2028,15 +2029,15 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f16_f32_"+std::to_string(i+1), mul_mat_vec_q4_k_f16_f32_len, mul_mat_vec_q4_k_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f16_f32_"+std::to_string(i+1), mul_mat_vec_q5_k_f16_f32_len, mul_mat_vec_q5_k_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f16_f32_"+std::to_string(i+1), mul_mat_vec_q6_k_f16_f32_len, mul_mat_vec_q6_k_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq1_s_f16_f32_len, mul_mat_vec_iq1_s_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq1_m_f16_f32_len, mul_mat_vec_iq1_m_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq2_xxs_f16_f32_len, mul_mat_vec_iq2_xxs_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq2_xs_f16_f32_len, mul_mat_vec_iq2_xs_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq2_s_f16_f32_len, mul_mat_vec_iq2_s_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq3_xxs_f16_f32_len, mul_mat_vec_iq3_xxs_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq3_s_f16_f32_len, mul_mat_vec_iq3_s_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq4_xs_f16_f32_len, mul_mat_vec_iq4_xs_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq, i+1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq4_nl_f16_f32_len, mul_mat_vec_iq4_nl_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {subgroup_size_16, 2*rm_stdq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq1_s_f16_f32_len, mul_mat_vec_iq1_s_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq1_m_f16_f32_len, mul_mat_vec_iq1_m_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq2_xxs_f16_f32_len, mul_mat_vec_iq2_xxs_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq2_xs_f16_f32_len, mul_mat_vec_iq2_xs_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq2_s_f16_f32_len, mul_mat_vec_iq2_s_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq3_xxs_f16_f32_len, mul_mat_vec_iq3_xxs_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq3_s_f16_f32_len, mul_mat_vec_iq3_s_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq4_xs_f16_f32_len, mul_mat_vec_iq4_xs_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f16_f32_"+std::to_string(i+1), mul_mat_vec_iq4_nl_f16_f32_len, mul_mat_vec_iq4_nl_f16_f32_data, "main", 3, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq, i+1}, 1, true); } ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_F32 ], "mul_mat_vec_id_f32_f32", mul_mat_vec_id_f32_f32_len, mul_mat_vec_id_f32_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {device->subgroup_size, 2}, 1); @@ -2051,15 +2052,15 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_f32", mul_mat_vec_id_q4_k_f32_len, mul_mat_vec_id_q4_k_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_f32", mul_mat_vec_id_q5_k_f32_len, mul_mat_vec_id_q5_k_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_f32", mul_mat_vec_id_q6_k_f32_len, mul_mat_vec_id_q6_k_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", mul_mat_vec_id_iq1_s_f32_len, mul_mat_vec_id_iq1_s_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", mul_mat_vec_id_iq1_m_f32_len, mul_mat_vec_id_iq1_m_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", mul_mat_vec_id_iq2_xxs_f32_len, mul_mat_vec_id_iq2_xxs_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ2_XS], "mul_mat_vec_id_iq2_xs_f32", mul_mat_vec_id_iq2_xs_f32_len, mul_mat_vec_id_iq2_xs_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ2_S], "mul_mat_vec_id_iq2_s_f32", mul_mat_vec_id_iq2_s_f32_len, mul_mat_vec_id_iq2_s_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ3_XXS], "mul_mat_vec_id_iq3_xxs_f32", mul_mat_vec_id_iq3_xxs_f32_len, mul_mat_vec_id_iq3_xxs_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ3_S], "mul_mat_vec_id_iq3_s_f32", mul_mat_vec_id_iq3_s_f32_len, mul_mat_vec_id_iq3_s_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ4_XS], "mul_mat_vec_id_iq4_xs_f32", mul_mat_vec_id_iq4_xs_f32_len, mul_mat_vec_id_iq4_xs_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {subgroup_size_16, rm_kq}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ4_NL], "mul_mat_vec_id_iq4_nl_f32", mul_mat_vec_id_iq4_nl_f32_len, mul_mat_vec_id_iq4_nl_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {subgroup_size_16, 2*rm_stdq}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", mul_mat_vec_id_iq1_s_f32_len, mul_mat_vec_id_iq1_s_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", mul_mat_vec_id_iq1_m_f32_len, mul_mat_vec_id_iq1_m_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", mul_mat_vec_id_iq2_xxs_f32_len, mul_mat_vec_id_iq2_xxs_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ2_XS], "mul_mat_vec_id_iq2_xs_f32", mul_mat_vec_id_iq2_xs_f32_len, mul_mat_vec_id_iq2_xs_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ2_S], "mul_mat_vec_id_iq2_s_f32", mul_mat_vec_id_iq2_s_f32_len, mul_mat_vec_id_iq2_s_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ3_XXS], "mul_mat_vec_id_iq3_xxs_f32", mul_mat_vec_id_iq3_xxs_f32_len, mul_mat_vec_id_iq3_xxs_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ3_S], "mul_mat_vec_id_iq3_s_f32", mul_mat_vec_id_iq3_s_f32_len, mul_mat_vec_id_iq3_s_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ4_XS], "mul_mat_vec_id_iq4_xs_f32", mul_mat_vec_id_iq4_xs_f32_len, mul_mat_vec_id_iq4_xs_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_IQ4_NL], "mul_mat_vec_id_iq4_nl_f32", mul_mat_vec_id_iq4_nl_f32_len, mul_mat_vec_id_iq4_nl_f32_data, "main", 4, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {subgroup_size_16, rm_iq}, 1, true); // dequant shaders ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", dequant_f32_len, dequant_f32_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp b/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp index c9f855687d..cfd645a38a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp @@ -1,5 +1,7 @@ #version 450 +#extension GL_EXT_control_flow_attributes : enable + #include "types.comp" #include "generic_binary_head.comp" #include "dequant_funcs.comp" diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp new file mode 100644 index 0000000000..9718a05e5a --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp @@ -0,0 +1,90 @@ +#version 450 +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require + +#include "mul_mat_vec_base.comp" + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + +void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 16 * itid; + const uint nibble_shift = 4 * (itid & 1); + const uint ib32 = itid / 2; // 0..7 + + uint ibi = a_offset / QUANT_K + first_row * num_blocks_per_row + i; + [[unroll]] for (uint n = 0; n < num_rows; ++n) { + const float d = float(data_a[ibi].d); + const uint scale = (data_a[ibi].scales[ib32] >> nibble_shift) & 0xF; + const float db = d * (0.5 + scale) * 0.25; + + const uint qh = data_a[ibi].qh[ib32]; + const u8vec2 qs16 = unpack8(data_a_packed16[ibi].qs[itid]); + const u8vec2 sign16 = unpack8(data_a_packed16[ibi].qs[QUANT_K / 16 + itid]); + [[unroll]] for (uint l = 0; l < 2; ++l) { + const uint8_t sign = sign16[l]; + const uint qs = qs16[l] | ((qh << (8 - nibble_shift - 2 * l)) & 0x300); + const uvec2 grid = iq2s_grid[qs]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); + vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]); + + FLOAT_TYPE sum = + fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x), + fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y), + fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z), + fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w), + fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x), + fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y), + fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z), + fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign & 128) != 0 ? -grid1.w : grid1.w), + FLOAT_TYPE(0.0))))))))); + temp[j][n] = fma(db, sum, temp[j][n]); + } + } + ibi += num_blocks_per_row; + } +} + +void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { + uint a_offset, b_offset, d_offset; + get_offsets(a_offset, b_offset, d_offset); + + const uint num_blocks_per_row = p.ncols / QUANT_K; + + // 16 threads are used to process each block + const uint blocks_per_wg = gl_WorkGroupSize.x/16; + const uint tid = gl_LocalInvocationID.x; + const uint itid = tid % 16; // 0...15 + const uint ix = tid / 16; + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { + temp[j][i] = FLOAT_TYPE(0); + } + } + + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg) + calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows); + + reduce_result(temp, d_offset, first_row, num_rows, tid); +} + +void main() { + const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); + + init_iq_shmem(gl_WorkGroupSize); + + // do NUM_ROWS at a time, unless there aren't enough remaining rows + if (first_row + NUM_ROWS <= p.stride_d) { + compute_outputs(first_row, NUM_ROWS); + } else { + if (first_row >= p.stride_d) { + return; + } + compute_outputs(first_row, p.stride_d - first_row); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp new file mode 100644 index 0000000000..c496043241 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp @@ -0,0 +1,87 @@ +#version 450 +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require + +#include "mul_mat_vec_base.comp" + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + +void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 16 * itid; + const uint nibble_shift = 4 * (itid & 1); + const uint ib32 = itid / 2; // 0..7 + + uint ibi = a_offset / QUANT_K + first_row * num_blocks_per_row + i; + [[unroll]] for (uint n = 0; n < num_rows; ++n) { + const float d = float(data_a[ibi].d); + const uint scale = (data_a[ibi].scales[ib32] >> nibble_shift) & 0xF; + const float db = d * (0.5 + scale) * 0.25; + + [[unroll]] for (uint l = 0; l < 2; ++l) { + const uint qs = data_a[ibi].qs[2 * itid + l]; + const uint sign = qs >> 9; + const uint sign7 = bitCount(sign); + const vec4 grid0 = vec4(unpack8(iq2xs_grid[qs & 511].x)); + const vec4 grid1 = vec4(unpack8(iq2xs_grid[qs & 511].y)); + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); + vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]); + + FLOAT_TYPE sum = + fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x), + fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y), + fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z), + fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w), + fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x), + fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y), + fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z), + fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign7 & 1) != 0 ? -grid1.w : grid1.w), + FLOAT_TYPE(0.0))))))))); + temp[j][n] = fma(db, sum, temp[j][n]); + } + } + ibi += num_blocks_per_row; + } +} + +void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { + uint a_offset, b_offset, d_offset; + get_offsets(a_offset, b_offset, d_offset); + + const uint num_blocks_per_row = p.ncols / QUANT_K; + + // 16 threads are used to process each block + const uint blocks_per_wg = gl_WorkGroupSize.x/16; + const uint tid = gl_LocalInvocationID.x; + const uint itid = tid % 16; // 0...15 + const uint ix = tid / 16; + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { + temp[j][i] = FLOAT_TYPE(0); + } + } + + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg) + calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows); + + reduce_result(temp, d_offset, first_row, num_rows, tid); +} + +void main() { + const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); + + init_iq_shmem(gl_WorkGroupSize); + + // do NUM_ROWS at a time, unless there aren't enough remaining rows + if (first_row + NUM_ROWS <= p.stride_d) { + compute_outputs(first_row, NUM_ROWS); + } else { + if (first_row >= p.stride_d) { + return; + } + compute_outputs(first_row, p.stride_d - first_row); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp new file mode 100644 index 0000000000..94d4b92e1e --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp @@ -0,0 +1,87 @@ +#version 450 +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require + +#include "mul_mat_vec_base.comp" + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + +void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 16 * itid; + const uint ib32 = itid / 2; // 0..7 + + uint ibi = a_offset / QUANT_K + first_row * num_blocks_per_row + i; + [[unroll]] for (uint n = 0; n < num_rows; ++n) { + const float d = float(data_a[ibi].d); + const uint signscale = pack32(u16vec2( + data_a_packed16[ibi].qs[4 * ib32 + 2], + data_a_packed16[ibi].qs[4 * ib32 + 3])); + const float db = d * 0.25 * (0.5 + (signscale >> 28)); + [[unroll]] for (uint l = 0; l < 2; ++l) { + const uint qs = data_a[ibi].qs[8 * ib32 + 2 * (itid & 1) + l]; + const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7); + const uint sign7 = bitCount(sign); + const vec4 grid0 = vec4(unpack8(iq2xxs_grid[qs].x)); + const vec4 grid1 = vec4(unpack8(iq2xxs_grid[qs].y)); + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); + const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]); + + FLOAT_TYPE sum = + fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x), + fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y), + fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z), + fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w), + fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x), + fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y), + fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z), + fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign7 & 1) != 0 ? -grid1.w : grid1.w), + FLOAT_TYPE(0.0))))))))); + temp[j][n] = fma(db, sum, temp[j][n]); + } + } + ibi += num_blocks_per_row; + } +} + +void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { + uint a_offset, b_offset, d_offset; + get_offsets(a_offset, b_offset, d_offset); + + const uint num_blocks_per_row = p.ncols / QUANT_K; + + // 16 threads are used to process each block + const uint blocks_per_wg = gl_WorkGroupSize.x/16; + const uint tid = gl_LocalInvocationID.x; + const uint itid = tid % 16; // 0...15 + const uint ix = tid / 16; + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { + temp[j][i] = FLOAT_TYPE(0); + } + } + + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg) + calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows); + + reduce_result(temp, d_offset, first_row, num_rows, tid); +} + +void main() { + const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); + + init_iq_shmem(gl_WorkGroupSize); + + // do NUM_ROWS at a time, unless there aren't enough remaining rows + if (first_row + NUM_ROWS <= p.stride_d) { + compute_outputs(first_row, NUM_ROWS); + } else { + if (first_row >= p.stride_d) { + return; + } + compute_outputs(first_row, p.stride_d - first_row); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp new file mode 100644 index 0000000000..af48f32902 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp @@ -0,0 +1,90 @@ +#version 450 +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require + +#include "mul_mat_vec_base.comp" + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + +void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 32 * ib32; + + uint ibi = a_offset / QUANT_K + first_row * num_blocks_per_row + i; + [[unroll]] for (uint n = 0; n < num_rows; ++n) { + const float d = float(data_a[ibi].d); + const uint scale = (data_a[ibi].scales[ib32/2] >> (4 * (ib32 & 1))) & 0xF; + const float dscale = d * (1 + 2 * scale); + const uint qh = data_a[ibi].qh[ib32]; + FLOAT_TYPE sum[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + sum[j] = 0.0; + } + [[unroll]] for (uint l = 0; l < 4; ++l) { + const u8vec2 qs = unpack8(data_a_packed16[ibi].qs[4 * ib32 + l]); + const uint sign = data_a[ibi].signs[4 * ib32 + l]; + const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)])); + const vec4 grid1 = vec4(unpack8(iq3s_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)])); + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); + const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]); + + sum[j] = + fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x), + fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y), + fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z), + fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w), + fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x), + fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y), + fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z), + fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign & 128) != 0 ? -grid1.w : grid1.w), + sum[j])))))))); + } + } + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + temp[j][n] = fma(dscale, sum[j], temp[j][n]); + } + ibi += num_blocks_per_row; + } +} + +void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { + uint a_offset, b_offset, d_offset; + get_offsets(a_offset, b_offset, d_offset); + + const uint num_blocks_per_row = p.ncols / QUANT_K; + + // 8 threads are used to process each block + const uint blocks_per_wg = gl_WorkGroupSize.x/8; + const uint tid = gl_LocalInvocationID.x; + const uint itid = tid % 8; // 0...7 + const uint ix = tid / 8; + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { + temp[j][i] = FLOAT_TYPE(0); + } + } + + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg) + calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows); + + reduce_result(temp, d_offset, first_row, num_rows, tid); +} + +void main() { + const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); + + init_iq_shmem(gl_WorkGroupSize); + + // do NUM_ROWS at a time, unless there aren't enough remaining rows + if (first_row + NUM_ROWS <= p.stride_d) { + compute_outputs(first_row, NUM_ROWS); + } else { + if (first_row >= p.stride_d) { + return; + } + compute_outputs(first_row, p.stride_d - first_row); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp new file mode 100644 index 0000000000..3fe9dc3a41 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp @@ -0,0 +1,88 @@ +#version 450 +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require + +#include "mul_mat_vec_base.comp" + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + +void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 16 * itid; + const uint ib32 = itid / 2; // 0..7 + + uint ibi = a_offset / QUANT_K + first_row * num_blocks_per_row + i; + [[unroll]] for (uint n = 0; n < num_rows; ++n) { + const float d = float(data_a[ibi].d); + const uint signscale = pack32(u16vec2( + data_a_packed16[ibi].qs[QUANT_K / 8 + 2 * ib32], + data_a_packed16[ibi].qs[QUANT_K / 8 + 2 * ib32 + 1])); + const float db = d * 0.5 * (0.5 + (signscale >> 28)); + [[unroll]] for (uint l = 0; l < 2; ++l) { + const uint qs0 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l]; + const uint qs1 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l + 1]; + const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7); + const uint sign7 = bitCount(sign); + const vec4 grid0 = vec4(unpack8(iq3xxs_grid[qs0])); + const vec4 grid1 = vec4(unpack8(iq3xxs_grid[qs1])); + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); + const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]); + + FLOAT_TYPE sum = + fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x), + fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y), + fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z), + fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w), + fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x), + fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y), + fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z), + fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign7 & 1) != 0 ? -grid1.w : grid1.w), + FLOAT_TYPE(0.0))))))))); + temp[j][n] = fma(db, sum, temp[j][n]); + } + } + ibi += num_blocks_per_row; + } +} + +void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { + uint a_offset, b_offset, d_offset; + get_offsets(a_offset, b_offset, d_offset); + + const uint num_blocks_per_row = p.ncols / QUANT_K; + + // 16 threads are used to process each block + const uint blocks_per_wg = gl_WorkGroupSize.x/16; + const uint tid = gl_LocalInvocationID.x; + const uint itid = tid % 16; // 0...15 + const uint ix = tid / 16; + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { + temp[j][i] = FLOAT_TYPE(0); + } + } + + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg) + calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows); + + reduce_result(temp, d_offset, first_row, num_rows, tid); +} + +void main() { + const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); + + init_iq_shmem(gl_WorkGroupSize); + + // do NUM_ROWS at a time, unless there aren't enough remaining rows + if (first_row + NUM_ROWS <= p.stride_d) { + compute_outputs(first_row, NUM_ROWS); + } else { + if (first_row >= p.stride_d) { + return; + } + compute_outputs(first_row, p.stride_d - first_row); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp index 907067d7fa..f01179326e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp @@ -466,10 +466,13 @@ shared uint16_t iq1s_grid[2048]; void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - for (uint i = gl_LocalInvocationIndex.x; i < iq1s_grid_const.length(); i += wgsize.x) { - u16vec2 g = unpack16(iq1s_grid_const[i]); - iq1s_grid[2*i+0] = g.x; - iq1s_grid[2*i+1] = g.y; + [[unroll]] for (uint i = 0; i < iq1s_grid_const.length(); i += wgsize.x) { + uint idx = i + gl_LocalInvocationIndex.x; + if (iq1s_grid_const.length() % wgsize.x == 0 || idx < iq1s_grid_const.length()) { + u16vec2 g = unpack16(iq1s_grid_const[idx]); + iq1s_grid[2*idx+0] = g.x; + iq1s_grid[2*idx+1] = g.y; + } } barrier(); } @@ -565,8 +568,10 @@ shared uvec2 iq2xxs_grid[256]; void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - for (uint i = gl_LocalInvocationIndex.x; i < iq2xxs_grid.length(); i += wgsize.x) { - iq2xxs_grid[i] = iq2xxs_grid_const[i]; + [[unroll]] for (uint i = 0; i < iq2xxs_grid.length(); i += wgsize.x) { + if (iq2xxs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xxs_grid_const.length()) { + iq2xxs_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x]; + } } barrier(); } @@ -733,8 +738,10 @@ shared uvec2 iq2xs_grid[512]; void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - for (uint i = gl_LocalInvocationIndex.x; i < iq2xs_grid.length(); i += wgsize.x) { - iq2xs_grid[i] = iq2xs_grid_const[i]; + [[unroll]] for (uint i = 0; i < iq2xs_grid.length(); i += wgsize.x) { + if (iq2xs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) { + iq2xs_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x]; + } } barrier(); } @@ -756,6 +763,14 @@ struct block_iq2_s uint8_t scales[QUANT_K_IQ2_S/32]; }; +struct block_iq2_s_packed16 +{ + float16_t d; + uint16_t qs[QUANT_K_IQ2_S/8]; + uint16_t qh[QUANT_K_IQ2_S/64]; + uint16_t scales[QUANT_K_IQ2_S/64]; +}; + #if defined(DATA_A_IQ2_S) const uvec2 iq2s_grid_const[1024] = { @@ -1023,8 +1038,10 @@ shared uvec2 iq2s_grid[1024]; void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - for (uint i = gl_LocalInvocationIndex.x; i < iq2s_grid.length(); i += wgsize.x) { - iq2s_grid[i] = iq2s_grid_const[i]; + [[unroll]] for (uint i = 0; i < iq2s_grid.length(); i += wgsize.x) { + if (iq2s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) { + iq2s_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x]; + } } barrier(); } @@ -1032,6 +1049,7 @@ void init_iq_shmem(uvec3 wgsize) #define QUANT_K QUANT_K_IQ2_S #define QUANT_R QUANT_R_IQ2_S #define A_TYPE block_iq2_s +#define A_TYPE_PACKED16 block_iq2_s_packed16 #endif #define QUANT_K_IQ3_XXS 256 @@ -1092,8 +1110,10 @@ shared uint32_t iq3xxs_grid[256]; void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - for (uint i = gl_LocalInvocationIndex.x; i < iq3xxs_grid.length(); i += wgsize.x) { - iq3xxs_grid[i] = iq3xxs_grid_const[i]; + [[unroll]] for (uint i = 0; i < iq3xxs_grid.length(); i += wgsize.x) { + if (iq3xxs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3xxs_grid.length()) { + iq3xxs_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x]; + } } barrier(); } @@ -1200,8 +1220,10 @@ shared uint32_t iq3s_grid[512]; void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - for (uint i = gl_LocalInvocationIndex.x; i < iq3s_grid.length(); i += wgsize.x) { - iq3s_grid[i] = iq3s_grid_const[i]; + [[unroll]] for (uint i = 0; i < iq3s_grid.length(); i += wgsize.x) { + if (iq3s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3s_grid.length()) { + iq3s_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x]; + } } barrier(); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 4a81505565..ee1fec4e11 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -402,7 +402,7 @@ void process_shaders() { for (const auto& tname : type_names) { // mul mat vec std::string data_a_key = "DATA_A_" + to_uppercase(tname); - std::string shader = (string_ends_with(tname, "_k") || string_starts_with(tname, "iq1_")) ? "mul_mat_vec_" + tname + ".comp" : "mul_mat_vec.comp"; + std::string shader = (string_ends_with(tname, "_k") || string_starts_with(tname, "iq1_") || string_starts_with(tname, "iq2_") || string_starts_with(tname, "iq3_")) ? "mul_mat_vec_" + tname + ".comp" : "mul_mat_vec.comp"; string_to_spv("mul_mat_vec_" + tname + "_f32_f32", shader, merge_maps(base_dict, {{data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPE_VEC2", "vec2"}, {"B_TYPE_VEC4", "vec4"}, {"D_TYPE", "float"}})); string_to_spv("mul_mat_vec_" + tname + "_f16_f32", shader, merge_maps(base_dict, {{data_a_key, "1"}, {"B_TYPE", "float16_t"}, {"B_TYPE_VEC2", "f16vec2"}, {"B_TYPE_VEC4", "f16vec4"}, {"D_TYPE", "float"}})); From 84d5f4bc195b9540fcb902d869015fba7ef6baa4 Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Fri, 28 Feb 2025 04:31:47 -0700 Subject: [PATCH 038/188] Update granite vision docs for 3.2 model (#12105) Signed-off-by: Alex-Brooks --- examples/llava/README-granitevision.md | 61 ++++++++++++++------------ 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/examples/llava/README-granitevision.md b/examples/llava/README-granitevision.md index d2426dc695..f08a21cc17 100644 --- a/examples/llava/README-granitevision.md +++ b/examples/llava/README-granitevision.md @@ -3,8 +3,8 @@ Download the model and point your `GRANITE_MODEL` environment variable to the path. ```bash -$ git clone https://huggingface.co/ibm-granite/granite-vision-3.1-2b-preview -$ export GRANITE_MODEL=./granite-vision-3.1-2b-preview +$ git clone https://huggingface.co/ibm-granite/granite-vision-3.2-2b +$ export GRANITE_MODEL=./granite-vision-3.2-2b ``` @@ -41,10 +41,18 @@ If you actually inspect the `.keys()` of the loaded tensors, you should see a lo ### 2. Creating the Visual Component GGUF -To create the GGUF for the visual components, we need to write a config for the visual encoder; make sure the config contains the correct `image_grid_pinpoints` +Next, create a new directory to hold the visual components, and copy the llava.clip/projector files, as shown below. +```bash +$ ENCODER_PATH=$PWD/visual_encoder +$ mkdir $ENCODER_PATH + +$ cp $GRANITE_MODEL/llava.clip $ENCODER_PATH/pytorch_model.bin +$ cp $GRANITE_MODEL/llava.projector $ENCODER_PATH/ +``` + +Now, we need to write a config for the visual encoder. In order to convert the model, be sure to use the correct `image_grid_pinpoints`, as these may vary based on the model. You can find the `image_grid_pinpoints` in `$GRANITE_MODEL/config.json`. -Note: we refer to this file as `$VISION_CONFIG` later on. ```json { "_name_or_path": "siglip-model", @@ -52,6 +60,7 @@ Note: we refer to this file as `$VISION_CONFIG` later on. "SiglipVisionModel" ], "image_grid_pinpoints": [ + [384,384], [384,768], [384,1152], [384,1536], @@ -94,24 +103,13 @@ Note: we refer to this file as `$VISION_CONFIG` later on. } ``` -Create a new directory to hold the visual components, and copy the llava.clip/projector files, as well as the vision config into it. - -```bash -$ ENCODER_PATH=$PWD/visual_encoder -$ mkdir $ENCODER_PATH - -$ cp $GRANITE_MODEL/llava.clip $ENCODER_PATH/pytorch_model.bin -$ cp $GRANITE_MODEL/llava.projector $ENCODER_PATH/ -$ cp $VISION_CONFIG $ENCODER_PATH/config.json -``` - -At which point you should have something like this: +At this point you should have something like this: ```bash $ ls $ENCODER_PATH config.json llava.projector pytorch_model.bin ``` -Now convert the components to GGUF; Note that we also override the image mean/std dev to `[.5,.5,.5]` since we use the siglip visual encoder - in the transformers model, you can find these numbers in the [preprocessor_config.json](https://huggingface.co/ibm-granite/granite-vision-3.1-2b-preview/blob/main/preprocessor_config.json). +Now convert the components to GGUF; Note that we also override the image mean/std dev to `[.5,.5,.5]` since we use the SigLIP visual encoder - in the transformers model, you can find these numbers in the `preprocessor_config.json`. ```bash $ python convert_image_encoder_to_gguf.py \ -m $ENCODER_PATH \ @@ -119,17 +117,18 @@ $ python convert_image_encoder_to_gguf.py \ --output-dir $ENCODER_PATH \ --clip-model-is-vision \ --clip-model-is-siglip \ - --image-mean 0.5 0.5 0.5 --image-std 0.5 0.5 0.5 + --image-mean 0.5 0.5 0.5 \ + --image-std 0.5 0.5 0.5 ``` -this will create the first GGUF file at `$ENCODER_PATH/mmproj-model-f16.gguf`; we will refer to the abs path of this file as the `$VISUAL_GGUF_PATH.` +This will create the first GGUF file at `$ENCODER_PATH/mmproj-model-f16.gguf`; we will refer to the absolute path of this file as the `$VISUAL_GGUF_PATH.` ### 3. Creating the LLM GGUF. The granite vision model contains a granite LLM as its language model. For now, the easiest way to get the GGUF for LLM is by loading the composite model in `transformers` and exporting the LLM so that it can be directly converted with the normal conversion path. First, set the `LLM_EXPORT_PATH` to the path to export the `transformers` LLM to. -``` +```bash $ export LLM_EXPORT_PATH=$PWD/granite_vision_llm ``` @@ -142,7 +141,7 @@ if not MODEL_PATH: raise ValueError("env var GRANITE_MODEL is unset!") LLM_EXPORT_PATH = os.getenv("LLM_EXPORT_PATH") -if not MODEL_PATH: +if not LLM_EXPORT_PATH: raise ValueError("env var LLM_EXPORT_PATH is unset!") tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_PATH) @@ -166,18 +165,26 @@ $ python convert_hf_to_gguf.py --outfile $LLM_GGUF_PATH $LLM_EXPORT_PATH ``` -### 4. Running the Model in Llama cpp -Build llama cpp normally; you should have a target binary named `llama-llava-cli`, which you can pass two binaries to. Sample usage: +### 4. Quantization +If you want to quantize the LLM, you can do so with `llama-quantize` as you would any other LLM. For example: +```bash +$ ./build/bin/llama-quantize $LLM_EXPORT_PATH/granite_llm.gguf $LLM_EXPORT_PATH/granite_llm_q4_k_m.gguf Q4_K_M +$ LLM_GGUF_PATH=$LLM_EXPORT_PATH/granite_llm_q4_k_m.gguf +``` -Note - the test image shown below can be found [here](https://github-production-user-asset-6210df.s3.amazonaws.com/10740300/415512792-d90d5562-8844-4f34-a0a5-77f62d5a58b5.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20250221%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20250221T054145Z&X-Amz-Expires=300&X-Amz-Signature=86c60be490aa49ef7d53f25d6c973580a8273904fed11ed2453d0a38240ee40a&X-Amz-SignedHeaders=host). +Note that currently you cannot quantize the visual encoder because granite vision models use SigLIP as the visual encoder, which has tensor dimensions that are not divisible by 32. + + +### 5. Running the Model in Llama cpp +Build llama cpp normally; you should have a target binary named `llama-llava-cli`, which you can pass two binaries to. As an example, we pass the the llama.cpp banner. ```bash $ ./build/bin/llama-llava-cli -m $LLM_GGUF_PATH \ --mmproj $VISUAL_GGUF_PATH \ - --image cherry_blossom.jpg \ + --image ./media/llama0-banner.png \ -c 16384 \ - -p "<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n<|user|>\n\\nWhat type of flowers are in this picture?\n<|assistant|>\n" \ + -p "<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n<|user|>\n\\nWhat does the text in this image say?\n<|assistant|>\n" \ --temp 0 ``` -Sample response: `The flowers in the picture are cherry blossoms, which are known for their delicate pink petals and are often associated with the beauty of spring.` +Sample output: `The text in the image reads "LLAMA C++ Can it run DOOM Llama?"` From c43a3e7996e585e2addde1e44057a4f3cdbadef8 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 28 Feb 2025 12:44:11 +0100 Subject: [PATCH 039/188] llama : add Phi-4-mini support (supersede #12099) (#12108) * Added Phi-4-mini-instruct support * Update regex per ngxson * Change the vocab base to Xenova/gpt-4o * fix conversion update script * no need to check longrope * minor style fix * fix python style --------- Co-authored-by: Nicholas Sparks --- convert_hf_to_gguf.py | 11 ++- convert_hf_to_gguf_update.py | 5 ++ include/llama.h | 1 + models/ggml-vocab-gpt-4o.gguf.inp | 112 ++++++++++++++++++++++++++++++ models/ggml-vocab-gpt-4o.gguf.out | 46 ++++++++++++ src/llama-model.cpp | 13 ++-- src/llama-vocab.cpp | 11 +++ 7 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 models/ggml-vocab-gpt-4o.gguf.inp create mode 100644 models/ggml-vocab-gpt-4o.gguf.out diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 8b7c75d85a..6358a94e9b 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -699,6 +699,9 @@ class Model: if chkhsh == "b3f499bb4255f8ca19fccd664443283318f2fd2414d5e0b040fbdd0cc195d6c5": # ref: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B res = "deepseek-r1-qwen" + if chkhsh == "ccc2ef013c104be7bae2965776d611e1d7a8a2a9c547dd93a682c9a9fc80352e": + # ref: https://huggingface.co/Xenova/gpt-4o + res = "gpt-4o" if res is None: logger.warning("\n") @@ -2512,7 +2515,8 @@ class Phi3MiniModel(Model): rms_eps = self.find_hparam(["rms_norm_eps"]) max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"]) orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"]) - rope_dims = n_embd // n_head + rot_pct = self.hparams.get("partial_rotary_factor", 1.0) + rope_dims = int(rot_pct * n_embd) // n_head self.gguf_writer.add_context_length(max_pos_embds) self.gguf_writer.add_rope_scaling_orig_ctx_len(orig_max_pos_embds) @@ -2536,7 +2540,8 @@ class Phi3MiniModel(Model): n_head = self.find_hparam(["num_attention_heads", "n_head"]) max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"]) orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"]) - rope_dims = n_embd // n_head + rot_pct = self.hparams.get("partial_rotary_factor", 1.0) + rope_dims = int(rot_pct * n_embd) // n_head # write rope scaling for long context (128k) model rope_scaling = self.find_hparam(['rope_scaling'], True) @@ -2565,7 +2570,7 @@ class Phi3MiniModel(Model): raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor') if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2: - raise ValueError(f'The length of rope long and short factors must be {rope_dims / 2}') + raise ValueError(f'The length of rope long and short factors must be {rope_dims / 2}. long_factors = {len(long_factors)}, short_factors = {len(short_factors)}.') yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG), torch.tensor(long_factors, dtype=torch.float32)) yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT), torch.tensor(short_factors, dtype=torch.float32)) diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index fa4989a80c..07d3ce0e4e 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -109,6 +109,7 @@ models = [ {"name": "megrez", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Infinigence/Megrez-3B-Instruct"}, {"name": "deepseek-v3", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-V3"}, {"name": "deepseek-r1-qwen", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"}, + {"name": "gpt-4o", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Xenova/gpt-4o", }, ] @@ -131,6 +132,10 @@ def download_model(model): files = ["config.json", "tokenizer.json", "tokenizer_config.json"] + if name == "gpt-4o": + # Xenova/gpt-4o is tokenizer-only, it does not contain config.json + files = ["tokenizer.json", "tokenizer_config.json"] + if tokt == TOKENIZER_TYPE.SPM: files.append("tokenizer.model") diff --git a/include/llama.h b/include/llama.h index 479196026b..ee6e73915f 100644 --- a/include/llama.h +++ b/include/llama.h @@ -105,6 +105,7 @@ extern "C" { LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, + LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, }; enum llama_rope_type { diff --git a/models/ggml-vocab-gpt-4o.gguf.inp b/models/ggml-vocab-gpt-4o.gguf.inp new file mode 100644 index 0000000000..9baf7d77ae --- /dev/null +++ b/models/ggml-vocab-gpt-4o.gguf.inp @@ -0,0 +1,112 @@ +ied 4 ½ months +__ggml_vocab_test__ +Führer +__ggml_vocab_test__ + +__ggml_vocab_test__ + +__ggml_vocab_test__ + +__ggml_vocab_test__ + +__ggml_vocab_test__ + +__ggml_vocab_test__ + + +__ggml_vocab_test__ + + + +__ggml_vocab_test__ + + + + +__ggml_vocab_test__ + + +__ggml_vocab_test__ +Hello world +__ggml_vocab_test__ + Hello world +__ggml_vocab_test__ +Hello World +__ggml_vocab_test__ + Hello World +__ggml_vocab_test__ + Hello World! +__ggml_vocab_test__ +Hello, world! +__ggml_vocab_test__ + Hello, world! +__ggml_vocab_test__ + this is 🦙.cpp +__ggml_vocab_test__ +w048 7tuijk dsdfhu +__ggml_vocab_test__ +нещо на Български +__ggml_vocab_test__ +កាន់តែពិសេសអាចខលចេញ +__ggml_vocab_test__ +🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token) +__ggml_vocab_test__ +Hello +__ggml_vocab_test__ + Hello +__ggml_vocab_test__ + Hello +__ggml_vocab_test__ + Hello +__ggml_vocab_test__ + Hello +__ggml_vocab_test__ + Hello + Hello +__ggml_vocab_test__ + ( +__ggml_vocab_test__ + + = +__ggml_vocab_test__ +' era +__ggml_vocab_test__ +Hello, y'all! How are you 😁 ?我想在apple工作1314151天～ +__ggml_vocab_test__ +!!!!!! +__ggml_vocab_test__ +3 +__ggml_vocab_test__ +33 +__ggml_vocab_test__ +333 +__ggml_vocab_test__ +3333 +__ggml_vocab_test__ +33333 +__ggml_vocab_test__ +333333 +__ggml_vocab_test__ +3333333 +__ggml_vocab_test__ +33333333 +__ggml_vocab_test__ +333333333 +__ggml_vocab_test__ +Cửa Việt +__ggml_vocab_test__ + discards +__ggml_vocab_test__ + + + + + + + + + + + +🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````""""......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL +__ggml_vocab_test__ diff --git a/models/ggml-vocab-gpt-4o.gguf.out b/models/ggml-vocab-gpt-4o.gguf.out new file mode 100644 index 0000000000..478df726fa --- /dev/null +++ b/models/ggml-vocab-gpt-4o.gguf.out @@ -0,0 +1,46 @@ + 1165 220 19 220 27124 5503 + 37 19194 259 + + 220 + 256 + 271 + 197 + 198 + 279 + 2499 + 2775 + 13225 2375 + 32949 2375 + 13225 5922 + 32949 5922 + 32949 5922 0 + 13225 11 2375 0 + 32949 11 2375 0 + 495 382 9552 99 247 13 17159 + 86 45404 220 22 10191 2852 22924 4750 6916 + 3907 53641 1235 185386 8118 + 11400 107516 15867 20804 22851 134178 77431 32010 104312 37984 16329 27751 89335 + 112927 222 350 14559 8 22861 114 2524 64364 104 15148 350 76466 166700 121942 780 8 91349 350 7393 74471 484 853 1617 2316 6602 8 + 13225 + 32949 + 220 32949 + 256 32949 + 271 32949 + 271 32949 198 271 32949 + 350 + 198 314 + 6 6837 + 13225 11 342 70653 0 3253 553 481 22861 223 1423 7522 18165 2178 34058 22369 16412 32999 16 867 8208 + 147475 + 18 + 2546 + 15517 + 15517 18 + 15517 2546 + 15517 15517 + 15517 15517 18 + 15517 15517 2546 + 15517 15517 15517 + 34 60213 53904 + 2960 3098 + 126470 25980 160432 16609 2775 4066 172261 19432 112927 222 350 14559 8 22861 114 2524 64364 104 15148 350 76466 166700 121942 780 8 91349 9552 99 247 4103 99 247 220 18 220 2546 220 15517 220 15517 18 220 15517 2546 220 15517 15517 220 15517 15517 18 220 15517 15517 2546 220 18 13 18 220 18 485 18 220 18 1008 18 44735 107516 15867 20804 22851 134178 77431 32010 104312 156437 1423 7522 18165 2178 34058 22369 16412 32999 16 867 8208 105024 106657 1967 53641 1235 185386 8118 22434 39336 26178 26178 168394 194663 27271 147475 25883 6961 9790 1339 461 83 1280 19016 1354 11 461 1099 481 3239 30 461 44 625 3239 17291 1520 480 11 461 35 481 1299 1236 17966 30 1416 6 27493 261 54602 43 diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 36a0a009c4..1da4eae7e6 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2202,13 +2202,16 @@ bool llama_model::load_tensors(llama_model_loader & ml) { } break; case LLM_ARCH_PHI3: { - const int64_t n_embd_head = n_embd / n_head; - tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; @@ -2223,8 +2226,8 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0); layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, 2 * n_ff }, 0); - layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), { n_embd_head/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); - layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), { n_embd_head/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); + layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), { n_rot/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); + layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), { n_rot/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); } } break; case LLM_ARCH_PHIMOE: diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index ad9ffe66aa..163ff64f77 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -392,6 +392,13 @@ struct llm_tokenizer_bpe : llm_tokenizer { "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)", }; break; + case LLAMA_VOCAB_PRE_TYPE_GPT4O: + regex_exprs = { + // original regex from tokenizer.json + // "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + break; default: // default regex for BPE tokenization pre-processing regex_exprs = { @@ -1592,6 +1599,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "megrez") { pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2; + } else if ( + tokenizer_pre == "gpt-4o") { + pre_type = LLAMA_VOCAB_PRE_TYPE_GPT4O; + clean_spaces = false; } else { throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); } From 70680c48e5f77d2d3138712a6582bd8c1e548922 Mon Sep 17 00:00:00 2001 From: William Tambellini Date: Fri, 28 Feb 2025 05:41:47 -0800 Subject: [PATCH 040/188] ggml : upgrade init_tensor API to return a ggml_status (#11854) * Upgrade init_tensor API to return a ggml_status To prepare for an 'abort-free' ggml (ggml not to abort on OOMs but return a OOM status), as agreeed with Diego in the ggml repo, upgrade the init_tensor() and view_init() APIs to return a ggml_status. * misc fixes --------- Co-authored-by: slaren --- .gitignore | 2 + CONTRIBUTING.md | 2 +- ggml/include/ggml-alloc.h | 2 +- ggml/include/ggml-backend.h | 6 +- ggml/src/ggml-alloc.c | 61 ++++++++------ ggml/src/ggml-backend-impl.h | 2 +- ggml/src/ggml-backend.cpp | 17 ++-- ggml/src/ggml-cann/ggml-cann.cpp | 5 +- ggml/src/ggml-cpu/amx/amx.cpp | 3 +- ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp | 3 +- ggml/src/ggml-cuda/ggml-cuda.cu | 8 +- ggml/src/ggml-opencl/ggml-opencl.cpp | 3 +- ggml/src/ggml-rpc/ggml-rpc.cpp | 3 +- ggml/src/ggml-sycl/ggml-sycl.cpp | 8 +- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 3 +- tests/test-backend-ops.cpp | 105 ++++++++++++++----------- 16 files changed, 136 insertions(+), 97 deletions(-) diff --git a/.gitignore b/.gitignore index 56b5ac2c18..2c67ad7f7c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,8 @@ lcov-report/ tags .build/ build* +release +debug !build-info.cmake !build-info.cpp.in !build-info.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8fab0de6f1..e68ff92445 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,7 +39,7 @@ _(NOTE: this guideline is yet to be applied to the `llama.cpp` codebase. New code should follow this guideline.)_ -- Try to follow the existing patterns in the code (indentation, spaces, etc.). In case of doubt use `clang-format` to format the added code +- Try to follow the existing patterns in the code (indentation, spaces, etc.). In case of doubt use `clang-format` (from clang-tools v15+) to format the added code - For anything not covered in the current guidelines, refer to the [C++ Core Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines) - Tensors store data in row-major order. We refer to dimension 0 as columns, 1 as rows, 2 as matrices - Matrix multiplication is unconventional: [`C = ggml_mul_mat(ctx, A, B)`](https://github.com/ggml-org/llama.cpp/blob/880e352277fc017df4d5794f0c21c44e1eae2b84/ggml.h#L1058-L1064) means $C^T = A B^T \Leftrightarrow C = B A^T.$ diff --git a/ggml/include/ggml-alloc.h b/ggml/include/ggml-alloc.h index 23600eea99..2cb150fd2a 100644 --- a/ggml/include/ggml-alloc.h +++ b/ggml/include/ggml-alloc.h @@ -19,7 +19,7 @@ struct ggml_tallocr { }; GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer); -GGML_API void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor); +GGML_API enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor); // Graph allocator /* diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index fc9571c82c..64671495b3 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -56,7 +56,7 @@ extern "C" { GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); - GGML_API void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer); GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); @@ -342,8 +342,8 @@ extern "C" { GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data); // Tensor initialization - GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr); - GGML_API void ggml_backend_view_init(struct ggml_tensor * tensor); + GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr); + GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor); // CPU buffer types are always available GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c index 7244a9cbb0..a3d3f69013 100644 --- a/ggml/src/ggml-alloc.c +++ b/ggml/src/ggml-alloc.c @@ -89,7 +89,7 @@ struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer) { return talloc; } -void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) { +enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) { size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor); size = GGML_PAD(size, talloc->alignment); @@ -104,7 +104,7 @@ void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tenso assert(((uintptr_t)addr % talloc->alignment) == 0); - ggml_backend_tensor_alloc(talloc->buffer, tensor, addr); + return ggml_backend_tensor_alloc(talloc->buffer, tensor, addr); } // dynamic tensor allocator @@ -933,42 +933,51 @@ size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) { // utils +static void free_buffers(ggml_backend_buffer_t ** buffers, const size_t * n_buffers) { + for (size_t i = 0; i < *n_buffers; i++) { + ggml_backend_buffer_free((*buffers)[i]); + } + free(*buffers); +} + static bool alloc_tensor_range(struct ggml_context * ctx, struct ggml_tensor * first, struct ggml_tensor * last, ggml_backend_buffer_type_t buft, size_t size, ggml_backend_buffer_t ** buffers, size_t * n_buffers) { + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size); if (buffer == NULL) { -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), size); -#endif - for (size_t i = 0; i < *n_buffers; i++) { - ggml_backend_buffer_free((*buffers)[i]); - } - free(*buffers); + GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), size); + free_buffers(buffers, n_buffers); return false; } - struct ggml_tallocr tallocr = ggml_tallocr_new(buffer); - - for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) { - if (t->data == NULL) { - if (t->view_src == NULL) { - ggml_tallocr_alloc(&tallocr, t); - } else if (t->buffer == NULL) { - ggml_backend_view_init(t); - } - } else { - if (t->view_src != NULL && t->buffer == NULL) { - // view of a pre-allocated tensor - ggml_backend_view_init(t); - } - } - } - *buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1)); (*buffers)[(*n_buffers)++] = buffer; + struct ggml_tallocr tallocr = ggml_tallocr_new(buffer); + + for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) { + enum ggml_status status = GGML_STATUS_SUCCESS; + if (t->data == NULL) { + if (t->view_src == NULL) { + status = ggml_tallocr_alloc(&tallocr, t); + } else if (t->buffer == NULL) { + status = ggml_backend_view_init(t); + } + } else { + if (t->view_src != NULL && t->buffer == NULL) { + // view of a pre-allocated tensor + status = ggml_backend_view_init(t); + } + } + if (status != GGML_STATUS_SUCCESS) { + GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t->name); + free_buffers(buffers, n_buffers); + return false; + } + } + return true; } diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index d1c2d76d89..c36c12d657 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -44,7 +44,7 @@ extern "C" { // base address of the buffer void * (*get_base) (ggml_backend_buffer_t buffer); // (optional) initialize a tensor in the buffer (eg. add tensor extras) - void (*init_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + enum ggml_status (*init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); // tensor data access void (*memset_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); void (*set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index dba7be33b8..184f99af5f 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -126,11 +126,12 @@ void * ggml_backend_buffer_get_base(ggml_backend_buffer_t buffer) { return base; } -void ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { +enum ggml_status ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { // init_tensor is optional if (buffer->iface.init_tensor) { - buffer->iface.init_tensor(buffer, tensor); + return buffer->iface.init_tensor(buffer, tensor); } + return GGML_STATUS_SUCCESS; } void ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { @@ -1641,7 +1642,7 @@ ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, // utils -void ggml_backend_view_init(struct ggml_tensor * tensor) { +enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor) { GGML_ASSERT(tensor->buffer == NULL); GGML_ASSERT(tensor->view_src != NULL); GGML_ASSERT(tensor->view_src->buffer != NULL); @@ -1649,10 +1650,10 @@ void ggml_backend_view_init(struct ggml_tensor * tensor) { tensor->buffer = tensor->view_src->buffer; tensor->data = (char *)tensor->view_src->data + tensor->view_offs; - ggml_backend_buffer_init_tensor(tensor->buffer, tensor); + return ggml_backend_buffer_init_tensor(tensor->buffer, tensor); } -void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr) { +enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr) { GGML_ASSERT(tensor->buffer == NULL); GGML_ASSERT(tensor->data == NULL); GGML_ASSERT(tensor->view_src == NULL); @@ -1662,7 +1663,7 @@ void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor tensor->buffer = buffer; tensor->data = addr; - ggml_backend_buffer_init_tensor(buffer, tensor); + return ggml_backend_buffer_init_tensor(buffer, tensor); } static struct ggml_tensor * graph_copy_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies, @@ -1708,7 +1709,8 @@ static void graph_copy_init_tensor(struct ggml_hash_set * hash_set, struct ggml_ struct ggml_tensor * dst = node_copies[id]; if (dst->view_src != NULL) { graph_copy_init_tensor(hash_set, node_copies, node_init, src->view_src); - ggml_backend_view_init(dst); + enum ggml_status status = ggml_backend_view_init(dst); + GGML_ASSERT(status == GGML_STATUS_SUCCESS); } else { ggml_backend_tensor_copy(src, dst); @@ -1823,7 +1825,6 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t assert(g1->n_nodes == g2->n_nodes); for (int i = 0; i < g1->n_nodes; i++) { - //printf("eval %d/%d\n", i, g1->n_nodes); struct ggml_tensor * t1 = g1->nodes[i]; struct ggml_tensor * t2 = g2->nodes[i]; diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index d410c02445..b8d272cda6 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -796,11 +796,11 @@ static bool need_transform(ggml_type type) { * @param buffer The CANN buffer from which to initialize the tensor. * @param tensor Pointer to the tensor to be initialized. */ -static void ggml_backend_cann_buffer_init_tensor( +static enum ggml_status ggml_backend_cann_buffer_init_tensor( ggml_backend_buffer_t buffer, ggml_tensor* tensor) { if (tensor->view_src != NULL && tensor->view_offs == 0) { GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); - return; + return GGML_STATUS_SUCCESS; } // TODO: can backend doesn't support quantized yet. Just leave the code @@ -817,6 +817,7 @@ static void ggml_backend_cann_buffer_init_tensor( memset_size, 0, memset_size)); } } + return GGML_STATUS_SUCCESS; } // TODO: need handle tensor which has paddings. diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp index 5ec5263ceb..0f067137df 100644 --- a/ggml/src/ggml-cpu/amx/amx.cpp +++ b/ggml/src/ggml-cpu/amx/amx.cpp @@ -50,10 +50,11 @@ static void * ggml_backend_amx_buffer_get_base(ggml_backend_buffer_t buffer) { return (void *) (buffer->context); } -static void ggml_backend_amx_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { +static enum ggml_status ggml_backend_amx_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { tensor->extra = (void *) ggml::cpu::amx::get_tensor_traits(buffer, tensor); GGML_UNUSED(buffer); + return GGML_STATUS_SUCCESS; } static void ggml_backend_amx_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, diff --git a/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp b/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp index b311a5b1c4..c24fd56e20 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp @@ -4135,10 +4135,11 @@ static const ggml::cpu::tensor_traits * ggml_aarch64_get_optimal_repack_type(con return nullptr; } -static void ggml_backend_cpu_aarch64_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { +static enum ggml_status ggml_backend_cpu_aarch64_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { tensor->extra = (void *) const_cast(ggml_aarch64_get_optimal_repack_type(tensor)); GGML_UNUSED(buffer); + return GGML_STATUS_SUCCESS; } static void ggml_backend_cpu_aarch64_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index ebb2ccae04..d23686d161 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -540,12 +540,12 @@ static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) { return ctx->dev_ptr; } -static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { +static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; if (tensor->view_src != NULL) { assert(tensor->view_src->buffer->buft == buffer->buft); - return; + return GGML_STATUS_SUCCESS; } if (ggml_is_quantized(tensor->type) && tensor->view_src == nullptr && ggml_backend_buffer_get_usage(buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) { @@ -558,6 +558,7 @@ static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, g CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size)); } } + return GGML_STATUS_SUCCESS; } static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { @@ -792,7 +793,7 @@ static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buff GGML_UNUSED(buffer); } -static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { +static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; @@ -838,6 +839,7 @@ static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buf } } tensor->extra = extra; + return GGML_STATUS_SUCCESS; } static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index f590624608..dc9a718f71 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1211,7 +1211,7 @@ static void * ggml_backend_opencl_buffer_get_base(ggml_backend_buffer_t buffer) GGML_UNUSED(buffer); } -static void ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { +static enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_cl2_init(buffer->buft->device); @@ -1251,6 +1251,7 @@ static void ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, tensor->extra = extra; } } + return GGML_STATUS_SUCCESS; } // The optimized gemm and gemv kernels are used for large matrices without batch. diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 97873acc77..6c3b80b088 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -464,7 +464,7 @@ static rpc_tensor serialize_tensor(const ggml_tensor * tensor) { return result; } -static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { +static enum ggml_status ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; // CUDA backend on the server pads everything to 512 due to CUDA limitations. @@ -478,6 +478,7 @@ static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, gg bool status = send_rpc_cmd(ctx->sock, RPC_CMD_INIT_TENSOR, &request, sizeof(request), nullptr, 0); GGML_ASSERT(status); } + return GGML_STATUS_SUCCESS; } static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 792e0569ca..d804e66061 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -323,14 +323,14 @@ static void * ggml_backend_sycl_buffer_get_base(ggml_backend_buffer_t buffer) { return ctx->dev_ptr; } -static void +static enum ggml_status ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor *tensor) try { ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *)buffer->context; if (tensor->view_src != NULL) { assert(tensor->view_src->buffer->buft == buffer->buft); - return; + return GGML_STATUS_SUCCESS; } ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; @@ -348,6 +348,7 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, padded_size - original_size).wait())); } } + return GGML_STATUS_SUCCESS; } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ @@ -729,7 +730,7 @@ static void * ggml_backend_sycl_split_buffer_get_base(ggml_backend_buffer_t buff GGML_UNUSED(buffer); } -static void +static enum ggml_status ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor *tensor) try { GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported @@ -804,6 +805,7 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, } } tensor->extra = extra; + return GGML_STATUS_SUCCESS; } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index ce15f620f0..a413441ebd 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -7923,11 +7923,12 @@ static void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer) { UNUSED(buffer); } -static void ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { +static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { VK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")"); if (tensor->view_src != nullptr) { GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); } + return GGML_STATUS_SUCCESS; } static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index e1f7e6758b..1dc2cdda30 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -467,6 +468,7 @@ struct test_case { // allocate ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend1); + if (buf == NULL) { printf("failed to allocate tensors [%s] ", ggml_backend_name(backend1)); ggml_free(ctx); @@ -588,14 +590,13 @@ struct test_case { /* .mem_base = */ NULL, /* .no_alloc = */ true, }; - ggml_context * ctx = ggml_init(params); + ggml_context_ptr ctx(ggml_init(params)); // smart ptr GGML_ASSERT(ctx); - ggml_tensor * out = build_graph(ctx); + ggml_tensor * out = build_graph(ctx.get()); if (op_name != nullptr && op_desc(out) != op_name) { //printf(" %s: skipping\n", op_desc(out).c_str()); - ggml_free(ctx); return true; } @@ -605,7 +606,6 @@ struct test_case { // check if backends support op if (!ggml_backend_supports_op(backend, out)) { printf("not supported\n"); - ggml_free(ctx); return true; } @@ -618,22 +618,26 @@ struct test_case { printf("%*s", last - len, ""); // allocate - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr + if (buf == NULL) { printf("failed to allocate tensors\n"); - ggml_free(ctx); return false; } // randomize tensors - initialize_tensors(ctx); + initialize_tensors(ctx.get()); // build graph - ggml_cgraph * gf = ggml_new_graph_custom(ctx, graph_nodes, false); + ggml_cgraph * gf = ggml_new_graph_custom(ctx.get(), graph_nodes, false); ggml_build_forward_expand(gf, out); // warmup run - ggml_backend_graph_compute(backend, gf); + ggml_status status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } // determine number of runs int n_runs; @@ -684,7 +688,11 @@ struct test_case { int total_runs = 0; do { int64_t start_time = ggml_time_us(); - ggml_backend_graph_compute(backend, gf); + ggml_status status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } int64_t end_time = ggml_time_us(); total_time_us += end_time - start_time; @@ -722,10 +730,6 @@ struct test_case { } printf("\n"); - ggml_backend_buffer_free(buf); - - ggml_free(ctx); - return true; } @@ -738,17 +742,16 @@ struct test_case { /* .mem_base = */ NULL, /* .no_alloc = */ true, }; - ggml_context * ctx = ggml_init(params); + ggml_context_ptr ctx(ggml_init(params)); // smart ptr GGML_ASSERT(ctx); - gf = ggml_new_graph_custom(ctx, GGML_DEFAULT_GRAPH_SIZE, true); - gb = ggml_new_graph_custom(ctx, GGML_DEFAULT_GRAPH_SIZE, true); + gf = ggml_new_graph_custom(ctx.get(), GGML_DEFAULT_GRAPH_SIZE, true); + gb = ggml_new_graph_custom(ctx.get(), GGML_DEFAULT_GRAPH_SIZE, true); - ggml_tensor * out = build_graph(ctx); + ggml_tensor * out = build_graph(ctx.get()); if ((op_name != nullptr && op_desc(out) != op_name) || out->op == GGML_OP_OPT_STEP_ADAMW) { //printf(" %s: skipping\n", op_desc(out).c_str()); - ggml_free(ctx); return true; } @@ -756,7 +759,6 @@ struct test_case { fflush(stdout); if (out->type != GGML_TYPE_F32) { - ggml_free(ctx); printf("not supported [%s->type != FP32]\n", out->name); return true; } @@ -764,7 +766,7 @@ struct test_case { // check if the backend supports the ops bool supported = true; bool any_params = false; - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { printf("not supported [%s] ", ggml_backend_name(backend)); supported = false; @@ -785,40 +787,38 @@ struct test_case { } if (!supported) { printf("\n"); - ggml_free(ctx); return true; } int64_t ngrads = 0; - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (t->flags & GGML_TENSOR_FLAG_PARAM) { ngrads += ggml_nelements(t); } } if (ngrads > grad_nmax()) { printf("skipping large tensors for speed \n"); - ggml_free(ctx); return true; } if (!ggml_is_scalar(out)) { - out = ggml_sum(ctx, out); + out = ggml_sum(ctx.get(), out); ggml_set_name(out, "sum_of_out"); } ggml_set_loss(out); ggml_build_forward_expand(gf, out); ggml_graph_cpy(gf, gb); - ggml_build_backward_expand(ctx, ctx, gb, false); + ggml_build_backward_expand(ctx.get(), ctx.get(), gb, false); if (expect.size() != 1 || expect[0] != 0.0f) { GGML_ASSERT(ggml_graph_n_nodes(gb) > ggml_graph_n_nodes(gf)); - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { GGML_ASSERT(!(t->flags & GGML_TENSOR_FLAG_PARAM) || ggml_graph_get_grad(gb, t)->op != GGML_OP_NONE); } } - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { printf("not supported [%s] ", ggml_backend_name(backend)); supported = false; @@ -832,27 +832,32 @@ struct test_case { } if (!supported) { printf("\n"); - ggml_free(ctx); return true; } // allocate - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr if (buf == NULL) { printf("failed to allocate tensors [%s] ", ggml_backend_name(backend)); - ggml_free(ctx); return false; } - - initialize_tensors(ctx); // Randomizes all tensors (including gradients). + initialize_tensors(ctx.get()); // Randomizes all tensors (including gradients). ggml_graph_reset(gb); // Sets gradients to 1 if loss, 0 otherwise. - ggml_backend_graph_compute(backend, gf); - ggml_backend_graph_compute(backend, gb); + ggml_status status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } + status = ggml_backend_graph_compute(backend, gb); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } bool ok = true; - for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + for (struct ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { if (!(t->flags & GGML_TENSOR_FLAG_PARAM)) { continue; } @@ -897,20 +902,36 @@ struct test_case { float fu, fuh, fdh, fd; // output values for xiu, xiuh, xid, xidh ggml_backend_tensor_set(t, &xiu, i*sizeof(float), sizeof(float)); - ggml_backend_graph_compute(backend, gf); + status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } ggml_backend_tensor_get(out, &fu, 0, ggml_nbytes(out)); ggml_backend_tensor_set(t, &xid, i*sizeof(float), sizeof(float)); - ggml_backend_graph_compute(backend, gf); + status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } ggml_backend_tensor_get(out, &fd, 0, ggml_nbytes(out)); if (grad_precise()) { ggml_backend_tensor_set(t, &xiuh, i*sizeof(float), sizeof(float)); - ggml_backend_graph_compute(backend, gf); + status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } ggml_backend_tensor_get(out, &fuh, 0, ggml_nbytes(out)); ggml_backend_tensor_set(t, &xidh, i*sizeof(float), sizeof(float)); - ggml_backend_graph_compute(backend, gf); + status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } ggml_backend_tensor_get(out, &fdh, 0, ggml_nbytes(out)); gn[i] = (8.0*(double)fuh + (double)fd - (8.0*(double)fdh + (double)fu)) / (6.0*(double)eps); @@ -936,10 +957,6 @@ struct test_case { printf("compare failed "); } - ggml_backend_buffer_free(buf); - - ggml_free(ctx); - if (ok) { printf("\033[1;32mOK\033[0m\n"); return true; From 06c2b1561d8b882bc018554591f8c35eb04ad30e Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 28 Feb 2025 17:44:46 +0100 Subject: [PATCH 041/188] convert : fix Norway problem when parsing YAML (#12114) * convert : fix Norway problem when parsing YAML * Update gguf-py/gguf/metadata.py * add newline at correct place --- gguf-py/gguf/metadata.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/gguf-py/gguf/metadata.py b/gguf-py/gguf/metadata.py index 962c27b204..e807f43468 100644 --- a/gguf-py/gguf/metadata.py +++ b/gguf-py/gguf/metadata.py @@ -121,19 +121,39 @@ class Metadata: if not model_card_path.is_file(): return {} - # The model card metadata is assumed to always be in YAML + # The model card metadata is assumed to always be in YAML (frontmatter) # ref: https://github.com/huggingface/transformers/blob/a5c642fe7a1f25d3bdcd76991443ba6ff7ee34b2/src/transformers/modelcard.py#L468-L473 + yaml_content: str = "" with open(model_card_path, "r", encoding="utf-8") as f: - if f.readline() == "---\n": - raw = f.read().partition("---\n")[0] - data = yaml.safe_load(raw) - if isinstance(data, dict): - return data - else: - logger.error(f"while reading YAML model card frontmatter, data is {type(data)} instead of dict") - return {} - else: + content = f.read() + lines = content.splitlines() + lines_yaml = [] + if len(lines) == 0: + # Empty file return {} + if len(lines) > 0 and lines[0] != "---": + # No frontmatter + return {} + for line in lines[1:]: + if line == "---": + break # End of frontmatter + else: + lines_yaml.append(line) + yaml_content = "\n".join(lines_yaml) + "\n" + + # Quick hack to fix the Norway problem + # https://hitchdev.com/strictyaml/why/implicit-typing-removed/ + yaml_content = yaml_content.replace("- no\n", "- \"no\"\n") + + if yaml_content: + data = yaml.safe_load(yaml_content) + if isinstance(data, dict): + return data + else: + logger.error(f"while reading YAML model card frontmatter, data is {type(data)} instead of dict") + return {} + else: + return {} @staticmethod def load_hf_parameters(model_path: Optional[Path] = None) -> dict[str, Any]: From 2cc4a5e44a3a9ab94426816770d611b33fae8f77 Mon Sep 17 00:00:00 2001 From: Vivian Date: Sat, 1 Mar 2025 15:45:09 +0530 Subject: [PATCH 042/188] webui : minor typo fixes (#12116) * fix typos and improve menu text clarity * rename variable trimedValue to trimmedValue * add updated index.html.gz * rebuild --------- Co-authored-by: Xuan Son Nguyen --- examples/server/public/index.html.gz | Bin 1260358 -> 1260365 bytes .../webui/src/components/SettingDialog.tsx | 14 +++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/server/public/index.html.gz b/examples/server/public/index.html.gz index e6a22a4e3770f7ffd9851c7991a6a514f7856061..dc8a4fc6f2d843a6c08664211c0680795bf22e49 100644 GIT binary patch delta 889960 zcmV((K;XZ|_Ds$8On`&|gaU*Ev;-7)f5yjkX?$Fr$(1uRT-Lcb)wwfoPcqHkQZ?(j z&xN=}uFg1EBV%VE?7cElo`d99)fH@2fV0Dw8nhBn%W~$N6s7Zt*2`B~pLs7?gMZp| zY^U(Y{H6ZKVEsWb(}%lj_wno;oYiN0mQI*)r>hF&df|nV(F|K`G zROx%H?r@=7;WaqKMR)$&t3SSQe@ujvGyJqDokdZC#rdRg9dtKwz;%A$G#-Q>taY7a z#pKB%scKz5(U-_Eswb5$&vp6a<+G1J`TQAK^h_7#URzeWb*~(Z5QDlH^I^#O+dYK+ z>~g8g6vxRv6hZG0gRtB>2~51O=cn5p-Z{~@ z>S>)_ksCc!8!yq%PM{#R0~O|0D72!3Mp;ZUy{KHrn8KapI|0|K)N_6D6y%I7sr4AL z^h{QBZ;@70a<%B%fCj%^&7C3M7#VYbktTpBprDT#%=seINu`}6s|u%D>S=m%%sCPbe|Z%E(>4W^%ukbnKRvmEf)kKQA9qV@Ho)=9#c=R(sgWa~*Bbn| zNb(w%6{L6Yl!Ha8Ym8WhLMGGU1sD~UE>&vyXSV46| z?U?c3woWU?Yb8DD))znP_^;UWq5FLNVzrp)a`+O5Osf|OR8xER?!oU)@ZXo6Bjo!g zk*^lIOsAV!z}JqSw{tl(AM}U2e9)vQGM(hxU)1=%pnuQs?Vt0r9Ai+dXbBty`#RHC z*x%dHI)y5re^5!(y&BKQA6=sV2gD**?$P`>zFhA}HM2CyiusPRZBfjyDxN#QUmiH5 z4*FU1Z`ALYHzviUbKx=egmJ5+6PNqE15U+zoE*Tm?hkrLdV{iqyD6DKS-6DFTgu)S#_fAYSXt-U{!SqVyUw%%^_7fe@55jLAeS0vyC3xjP~`^#AMCg z%Z=CYUhQW;Y_w4-qOQ=aRbG^f1oWa(!jjoQ0C**tf}Uv*LkXnItyq9;csX+Sg-p%~ z?xAbIy=M%561h0f6Z?k?7@R?{Xt^1&8g3K&K6y-78?*S zC+(R9d^}8*)~B7;xbu5>{k~NPVfC!T0JhFIO`MUW$fnyU#YNpa_q044PdXq5fq3jp zz=2Gc{{hk7(T^Xa>H`aAhhL0nJQ81vTJDPrg0^+j={kKesB5kc#>yWd?v$)94| z4e+?Tc8d>LyT%RZ8*!w!&8GVgX~iv}e~Wm~a5ftyZqDxsmF=(R)-iSYzC&EmC!amA zM{AFRe>YG7FK-XT8DsaoG3ax0M>okE!lrLTv%ADdZ^XilJcbLri;P7Z==c8kW)uW& z-uK@h@Lco^OxBpz2`|`VNB-96o&maNd(gdy@$>J%Ek49%djsd_FTwOwPtPXBe`P;! zOEI6DTnBo3re`-(w0&T7n-l3G>%qg|KyN{=4u|~@9iEEalmqYi^jVo)Kn9I@Y<`)O z(YB}R?{_=?z1r0l{$c7~+d5tLsdKas?yM#}2=R>Z;0a{gc@l#z`Lo%;?Z5q}xv?&~ z+i(5)rYFq~GwSa%=bKOp&}lS#f42#yc-3vG_mpnCU61t#(bGxop-f!WhAuaZs6B6; zdPbG!Tr@wd*9S_g8xhtB!)V^_ILYlS?bn?N4Ie5SD{rq)58Z9Gj`5R+xCmUl|2nrJ zMlXw^9^UAuhpvMoC@=*fphN35tqDk=)B1A=mD@&`D{Cqq9sK5riA(wje}k}ogUMQ) zQfDCEU1hpD)w&)y)3T^4sBD?$AN|^mC;Fr)wHqfVwJxtcufDL19vt1Lkp1N{cId&r z8&~ZObo~JxZ|C7n+vd5l04-H}S)J*N&Kr2SdAOP;8Ft9J*H`T|_~Bp1&gSB(y-4%v zsrOA$ESeSrRj&Ze|ItF_xmNKly)Au z;ohyyW^_nqv2}Zrt;**6CKw4|U};)VP)(}31D7kroTlM)rCU5SEh*UB+7~L&Fn8`& zdYLB81qcM~wCxm<=@~Zn=G8XtOt8JRJ=6Ba&Xn6byK`@E^*^F*wg<1B_U*6S+~|HT zOc9x4`i8xoo`1wn%bf72eeKJ!Mxa155>(hFXxiP40uR;Cconr6< z>V zc}x$w>zC%If4WTT^=Wx+TOc0*yl?L`&;q|q15>uVTql!B8DE_0QV)+%&wV_)?tx*Z zr-jMY6}O33oLN!VdUo)>2iB*|J>~AHa8IRss@zlm+N5IxI5pG4mW%#JB-j0^Zx7Tw zH^E#p`TV-u$pn=FYs{Q+HkrBLZU^~)ueb05B3OZ=e{1*Er=J&j;l0%JRhE?Q=Q_^{ z_wyo8riFWm_QRxdpQjU>w9J90-NRy4rWzu1F5I@O?}P-KwPAX(Mn{9rdWG~?>uNP| ztJSj88mM3O(10GayI!d?bd5e}5>k2zW~2n|47>upT^rZ%7|JBUn5!iQCV2#|YOv8? zWe^p*@xWHb_YEmc0&5yNFJ-MjGaEoP)d3oJRXL?$@jRLa=mmXH^tYh!)o`vk) zH!BI8V-Je$qa8@2QEvpRwz>bYE%hyv83u;X<}=NAnFT|l1>JBA3aH*fwB!aB3UlLl zwC5JO7wo-LVQ$$DrGncX_&iN@Y&txEjbx7Mf45yO(!H53AQS=T7$bwlz)_GEu8UO- zixh(jz}{S?u)!r+Ix~BQ*@5xxrS-eKz6smnxS1`cPsKEEXJJ2I zC*}0?c;DB<+M{9Zz|P)^MX|@1mz^n?E;=k_UuR}j+ILRUOna*(c;Ga1yLtTq2LIN^fT-*1Js6om z_{r>g(@yEfmxGGCcQyx+Y4@rwj!KIMGYZ{Gj7HfK=(B-m=+Jih+Vg`TB}GhCv#bLI8G zjd>jQA_Cml#)CkGd=DN_MST|R;z7WsCp&n+BFY)tg9pJx?ZE@8>2wDVe-gpPbO#R- zKbcJ?J9xlmT<~NE4_L%BmpgdC6x`UsgWbOW=!0|SK-Jgqz(b@DBNB4?D~C{@Mcm<7xbDD1 zv}KA2#o?atLkCKP`V6NujUB^#aU$I-1Z;bSaBHtRw%OQWrCxN~>O6Lc+G>RKsWH?A zGwn&9E{yzb9_Om+wVM6hAsm{)Ujs!;&#F{N1mD%D8=wPbG-Aobf3|j8K&!W^Hgj6W zj*JL{)esa42R92<#f~4DH5WW`1e8C<_G#=yp}9dB^*48t*a>_C4HnAHy5_MXIMkq5 z@CuiK?{K9^K$(vOo*QtkmKA@8x=R|Ii`Z6SGJz{9r?SrUZ+@`FAiGa^D}5SamzaU}^85yS3a zX#+`o83g7D-{C%#S5}cPEgo}41o!biQz0TD;BBD%(4saMe}wzgJRw3FI9v(BDD`m+ zz;w8O~!ocTduj15q zf{^$u45D_&gel3;tR$C#BY=>?NJZffMF~F&(l1epU;%!57)cdvEnW;u5eb9Bl!5zD zZ}DO$hyaoCzVxZ!{k20RRDr|+e4h)|UpheKC?G)?f8rBVvd~dJ6cG1O&5hEBAW_K8 z{lFKY<1-><#8GpJvIrw8^if8le#n@k1d);|${Sha);aW;PrKkExe7C*8ebvRX_sJ<&Jq7UoO3M~%>Lu6{@g!S8QT4zb-QTp_s#CSkP| zZ_pg}e|NL-Iy__~ivEq|p)fKM4rhYHNf9L+;5Q;95#o~c+x9AULP~^@pQIB|5;V+5 z0D&Otg>5^)kP#jrV1o?|Vux`OAP9mE2)aktyE5kuKE8)N-y`NdVty}(+2w|M0r?MS zKQUhTPs5n+0PVmhzRz(rgpxuy40bm~DXP5?f9!&Y&}`w7`Yd#q&k2{5@yM#+7!j1? zA&zp;R&t*a9!V}lBOq<`VhJvP5O9TCdK82LM&OhRUsytIABF>;B8nNb6PyOj;XWr( z7^<)lo}!RQAG@QJ1=M%A3`i)#h_gLvx#|15+@e0ZkM!t1!ee)3oelE-L>cEj7XN!= ze{oMkQItz?*!-zUop>(e#45_A0u@U{SOX|QXn=bXWi(O@b?AN=Qq{GtL;__2&Vwj& zm=7|17=9N5xC{P*BFyfWoQIoPqkc^I#=4+np!}^#BaTPDQK2yrMY1<%I0*duqs)jon}d_T!=ZuIE6Y5d_? z@CL7cqD=T6!S50Lz1+FF%InjM5Q_`VUn zsYL0!^Tg5^GNk0ptWrb;D(WeXL?|{2FkeZ5*d~1La2^m5a!Ffs42Vw+Dl-lmf0uy& z3{FY`C!A3f0aT=z+*%k^Zt;Lg#ORPwwzV*^6NZFJ6gm8MRdTdZD1}E=MuifA!$Y4a z8bp53pn5#h9WZbQtLGxb%adVLi8Zf{arUC7789 zV}V&I&Uw)AtJ}Lk9xF&d4X8L1A_8l_FCwmjhRfa3$6a@o4L#m&t-L49d&0aY%$14fmI~`iNyL#Bq@a@Hfhi1W#M&=roG3$+LPQ~Vn2bOT7N7)ze;>pI5zG&* z5;2sH3W;E$kj-pC^KXnrA_~Eq2ak_G7Co4waOnF6MI{Au6(_zgeU!3*YjD6GL~crb zE`>uB=b_JAZC)Uafn9M0jteRxG?G3IEl;`SeGuY6_+u!<6ep1ql82V#-10gQ zSV$xbjjbh8fcy$@Oj6boe;6mqH}7-Bm;ikxXMteNIXPm)7l8qRR7wKe6;sA|&ylow zVC+3I0(^=X2?8k)B~ii$j{?LKUqy<;`=RoEv`9^E{{o;+C5ev<$eH5cq0mRO*zq|D zDEjs>KZ@86n#GRq6NT;t4D3q0k7OuG8_JEM7Q_0|s@%u3<$h6Bf9?@f7|es+r{4g`q^r9I~RGH@7`L^M`xw7Z0;t55#H1LUNo$5fJ>o?xdoJu2sS+_c2^|%6D?ieLfd6M?|oO`Wwf*AnM>3 zyWvKUDCjmp&|PAnTZC|Dd-R4r?x4}TCzN|a`PmUlceglCG7YuCZ?oIL5my;USS~{$ z9U%fjeI7;|LXbQ(p$I(k`wv*J=ztqFw4i}fpl+bPukie5e|@7xBpb&!1xGh@j523k zi=u&xNAZv{S+ZYL-0UlsiKBBvKG`zrk`p zl*pO5AAoMEe~VpUzYK_=W)>l1f;lj#FS-AtNpS&KQIG=!yAa=T*85g2tQ$JK%dT`! zkoN?6Pmte^AiKNYsVUH_^IPm6Z`JAx{L zT374E9a=YF8VSxZGlX51cq8uoyS`g}=(Kt)}rfPaDl;;tLg zhBWSAqPnM%dm8yU(MWfXm?!0IXM^ZTV?L1*_gqRjWeiNEED9uRw3I zhxs89JYcQmk4dlz8U=$0BZq~G1Wd4~mCk&Kr8mnMgcRtQ5#dYUx}N;2#-ZMwD8L6M ze>4hUr6Bi!3`wNIh&QrSmoy;vOkga%LWxjqtvK3ZeU3+J#v-ZM){4iDpu~^NnOj5< z-P&7l^jHl^D5*jHa0})3oQ?0w`z zu>|^|3037(N`*EfrDSVdV&LFtQipLNDR;O8QlS1O#mtYTr45X_`R8+r4i)i@EWxCV zAVpV5e5snK3`q&&GBSa!-1kvg>r)6WvlKvyADR$X8U|o_?SU5)4oGg|94O~}e+#-8 ziAH^HB(8wcs1IBq02z`%p@xO=K#8c2Vn3R)HbN@?Kn)ZU;QRNTdrG{g#CuBoc9hu5 z-CCTaw>d>KMIsXb0Iu$wDiXC=8G@=GF|x`>e>DRS!bk;sBXl`9VzRK80)lqcmYd)GTX#<7$PGn; zT7l!4G>AlVB&EGD!H6_b1D!QS*1&`@8+I2MZNX@SWkPk97Xuf3B#(um= zOvKN-I1k?=>OG?VAc)%Kmdm87b@oq4$k)$Sv$URl%((VZKMEp8FrS2he`rxCfPKa2 zWTQ|-4oE7Yz&wW*@k42ZAI7Ny97H*U`_0J~Ts?f_>dj;Tq6-fR3q?eKZYSF%stRbx zjQ}Mf?PGU2jqgq18zC(MEpm4?QIh1;m^6LL!>zSMkP3;2K?4Et*xi-fN2RxCdl2ZN z%KXL;AL61o?D`@!!ByY0f18p;`+s7hy?gw>$NwJy|9jg1B(Ia|>L-}VBn*f-#t5** za7AcyK8+{|OrBZ6rNbpC!ZsNm8Yc^iNezQGo0%fYHxZ^(MQ|!;Q;7c?+q*|HF3 zI%^z5GRMdqzIbPWV<%{QBB;j8R6EX7$yOT2p1v@)juUl=I}Jh z&L%}t&fICSNT&{zf7*5yl{?tf+#Xm5u@Eww%g#Y{t0mcK$?G6u3znO<3TEV9x7<2aQJ|!;AwoiXuPShK|vuMR@e) zDnPAp2RK_N(0v?$?>~O>ec60IK*HXT#`~D~J&oMc$j^*Me|mYWvt)Wh;hLqfwg!vFC7ZOe;Kql#PsX z5`z0r#}Y1-(M1Wx13afHKa80FGdKcnnVABjY&9fbe?pzRK3HN$_+b4qIX77#Sr014 zH5cvUR0WXxUO_=ZMOhnF2q}$CCYCVr1;b1%KJyV`DC9^`8`{Vc5&8}f6$v33OVr78 z#|$T-67t*$8WL`+u_HZ4B#dlz4H2>s9}wX4(YSF05KR)8j5fmeeax^E(I#dHjaJmh z@_>x9e;^3AF$)DYW&<=M9)iT$M=ay!#{=WH<|{6@@XBaDY_hY!7b@7G)tyb9hUDJA zUEI^&J?-7o-rZ=gR}-^LrmO50_fC)XeOhmsmHiNI$X5;%sTf1r`tR3h=)QllaF(ZffDGHSfaAVE!N zA?GWwp$Cd6E@{x_ZwC99vWM`{hkB)qlTiAm6GPGgk<5g~NN}Y^&s`X?L(&mp)Hnxl zKa`QfunHgCl_}@Fsu>{4RP(X%d14wwfRfA|@P0 zeWIjg7C?oNQq61;kpKt4J}MwS(I_%rXpv(6HYQ;1Y$dQ>3lkFRaBU$1`hj!aBt;dF z+q=PoI~Ve_^m&KS<9kBAC)9gF{oV<+f0t2kadq}BjNy9})FAX3L5G8gz;P*s;A-P# z9Y%(lB~{>ut0E#q#6z)hvks*)-U$IDMP^hHLH!7evf9J}LaCsq4FQBVp@|esVjSet zP%igFr4Tv7|gJ7*XK zdif3}_+C!C)b;q>C_R`qIee``Js5MY9!M;m}3v_~Ky=J-77Qb`y>_=nw?ImQDL zB`6CzR3e0-5^xX$=N1zBD)c***!@s3i|$bbfgu5peAG(V2%Ru6)pBS6r4~RHOnJzn z4#kAhs#uH*1T`y-RbrbU9xKB!--Infbh(Q&eF8naQMREh*NP@Y!2R@9j5x&WEDgqJ6 z9`rDM0BFDZhE_i!oTD=hjRNp4-GV0u4@HqdnvgQc8qtR*8d_;+!jpU%(P#^t=$B*i znMr0~dFL28Lrizc%yCaufA>`N?@m?yI@wj4>#Dj%3CM{r!yd_SN+Kz@vYLg##-YPf zjF8l79H0z1SB%I&M1pn1HD_68St*Y@=1+zOg5xb{@ioI#gqmYf2>@4hKrr(RAB;e+q;-6%w@( zQ$8)+tRc!)p{;P!L3(P^S#ln9*A&^9xdCy;n*~L@!Hm!xatgGoY2rOlI~-vCz=7y(>T09QZUYNUm}XS&6B! zTU`NE?(7TKkigxz{yi<+)55RG59Wi!$T6ddj$eYuih&^Ww6J_Fyn zR!T0Tf2}8sB*K(AszF6!)qkRKi3zAphZ&&J3Tjg_*~ypb8!rkfnC~-(2R?~}jA*B- zm<(*qX~m^7ePn8kkIELM2`Pjgi-2&& zu$0<%h&H-`N)ucmxPqjeeVC(z4wu|GwnR#WeUKZAYlx<97C=6~7UFl^a|{XIA?58o ze=XkA;yo>XJ6h~{m=z}{THj)cp|EH+`=*4$B!{DW=(i5)kbYA#ZuP)W_^SIvbFz;j zLq;<26-HnwUqoUPX&0LNR6%n32#V84*`x^yLvZpl!XhYtg!^DU?S@wvv4e3e{}nHI=jb#jSSptbL!kEtkzniH}m=az*gj5b; zKmrcFik*yg(l@o)xblJC83_Ys5;6Y_9BQJ-H~yI-qlWIn5xJORCTTRJ(G-0!BAJ23)c8_ysB*N0j#ER+wvt4|gGfU31LKjfe}yD4 z8baC{uD%E%_PUP==%)%m#d&fGX{q{%U@Y^>FbqjUwv7iU6AKoal?plEV8NX|*$iFY z!F+R1p!WoNPoUo)f%dcIno_K-%=7Au2_!%M0fY55ZX{l0s}s z-bAKQs6e4a2t`=Hr0CF0WG$PFe}nxC`u7zx!2)gEf{pfL0Rs~lgUZE<`y1iWC)AF^ zlxz!EihxBOq6vkeD;hCB05p;T6|7h2)AAUO(l&*-HXi*PyhOow1zkXu>Oc^QK`G2{ z#eAr$vjxEzq5&g;9E(K^;!64uG&S99dPH9+7WM(SF9AtFq;JDWS%f+Ye~38b%Df;< z#G!BGeC9*VH;xp}1iBOfc}WPVz_z0JiQEX7cqWLz#cRrD! zQOmO^LMt;$M=JO^6k>oHNWwV7Fdt>2E~Kp0SD82i6;as-l!?ycf2K@=5I)4hci?1V z;RC^#Ewd5Yyf%>xcpJvTNkq*P0_Ngyupp~Q@J34_C<#?yD>-qS8;8kAMqzWD2~ovj z4=eV8kkeML$9*U|VnonTQbT<#2#l^~^4@7!ivl$vqfBxt{cVIZs~Om`7Mz7U_-3B) z(Q}$f&V|2)ZpOzOf4v?sS}@ok+nqyk)5RSUQQTALJ%!#===V&aJ*!|*&TcsYGT7^l z_{o$6JA;yFF2MC@3fu{68)tCVgm1lEu@aAfTqlD1-P|xVvQrI08Gr=@Vw^?Se{-o#LBr5dmI%pAo{%5W zcS%J=3d4|s0OF_nYl=Y(zL5(d^)OWI=`E@89rdM6!iLtL?vmP6V`jyLYBG&6!P?}; z5sA1_V=x~gD^SLh5m){uw1V-#T92UW6qp18!IX&FAXCC4>vKS%ZXsl!3Bpi@YLi{^ zwo#_BBRG-5f7V9f5sqT0P57O74r+kvbFAgw$BY=$!5FR^$Pi8UJ;bn1*2X_BjJS|n zcwve(8P6|j9CbHX(f6~JACQVte#dmJNSAjw9^Mn^J%Qd6==VvW-QBTT<&~~)IY5HW zp4y^#AoK_*Koro9U$3Hp&4sB1b2te&6>=k*a%Hw?e<6ebZ`ue#%@isC8rhUFTSO$F z;KNIOAa&Nsn+A_?-l%9G;ikm6?QRJw#)OSwjC=;NmIRxB&~fQv*3{1wB+-J}9H7>Y zeq_I13BewbF=k_7@Xmswc3lbMjuiwxtj1&khzj09r z=yofrfBN}OD>vyAwp!5!wbME^O8C%}C6@wyr6nW;@|9@iK=Ac5BpRAzb$#I4G6cpX z5kePz(AxXPbRUAXcu!&X6n0Nx z9SS2!rpx+zmR8Fwxr*myIy<;XXZ2|ey50l0fA*49U3e#1aS@lr#X*v#^BieV=XHFN z=}WJwld`rq(poR7c&fo7aq#u3s?(FJe&ak@#;B60^=0khTg@$tb#{v9NjdC&wTr5Z zQS+Fcr@_vP24#z7GEM6%NEmEekQ>GE=PZINXQDW*lXTI2>*YQ0!yNew(28mkx{ za~5g7NG^x)+hr%^XbUe4`gE`X8`f)*f1D?EQj%qsO!aAz&2;(EEIn_L6>#r1tZQ)8 zG_MH~L;H4`XKAjz59{QhE|a`Uk&Q0PVyVmeD#m4Wx7XF}VxNm&H`CLiG(6a}o}^jk zR&`mNY1g8HJ1H)`>NJ@Z7cLHVQj`mLu-2vPUFgYKT6-|otSpwEY5ynB>L)#$f4i(% zoi5UR?wzdiDO|_X)g+yI6a7u9%OR02bqROb=+3XueQ(TG=CfkhcTY8(y5lAm3u}8+ z={GtiVSp`&n_EvAV)PcpH{MBFH<)RgyDgSmd6_<`W1@TkZ#Ru4_Sv-F%5i~ST&olQ zXK-E9JnwbBpL^ZL^fm*`cRK&Rf4O+eJWq>rUB>u#bWO5kqBAEk*VjMH$Yi1Om6N3( z{W{KzdbrA5Z1L)}&a^wzb&_V4OD1(bij!%bo@@65ULo7B*t;K9tHmNIuWXms-vVTL z?;G>)0bqH5+49?#Z4IBdcT=#`hhzeKVfWgIJ*Q=wpT+cGx+<%pjF$!ae_ zPD{BX=ZSM8@%DKT+hssENPzF*gnEJEBiV4x3vRJDdc8u;r zuwc!{#O)I$taNd5Qt7(+P#o%)lu|l!`_J9>fVnq1C{{J1$z9iDuN?eg_}*bn`}V2* zdm}O%Za<+bG|o<7%kT_GNu*QR|bm&2q?6A!o8(Lr;k1(a(1XH--9#Hfj?%B-p^mz9-<6D7+LFO1C|?Z5u@byrYLAtVLx zsFdIbvLbtwFDQJiH_xZ(zy3SEILBia%S>Lu_}c7P!9L*(->>md3zF)(fw&C&H%7VgpCCMb`TndQLg3+9gd4%(n{UY!z`!%^f?UOjm^DHAtjhj^9e2&w_8@mIiTfFw+8(6&NVg($7 z-6U)S4*c}!98U3_P|~xbSZ1H{m*ct9G(TJXEazIffA5K3lW_v}jCj)9xbMlwSwB@% z5gpY_?;QIx%V1qCW-Y@^RyDt#r)0{;lknBFl3!mhr)FRd-AFdbONmMUqXqvK3AZwW zup5FjP|5)3;@^tM3gOXtmLXYIcx0Y$JJ1Cz(I#-3sK#E{5petk|LJTy_$v(2-*VFqmq>g|e#?p-3k6=$ ze=a>2UKV&n#U=UCcf!NtLyGH4KCZ|+KO+?QsLb;C`f&4^;f4t8+B3t6tmY-$-z;5c zZC8w>m^~$ zrunxY@T0u!W~yyq^;(HHp$w;WpWz^aT;HH|e=?M3~Epx?hva zX_%Z7SywDrzIP^?&g4EO%}=(4NEz$c!~a;eXr`w{_-|N)bi3T=D7cf6iiv zZ$s%ddWs=Z9lJP=2VJq}4`PUi_^4|3kv7Jp`1Ne?Pl$7?zzuTU1iJ`ui!lH5E!eU| z`k$M#J=rqTGx3V@zSW^~Hn-u;tnWeZkd-Z+Mnqb+(wIRYdVB^pFepME*r9cISa ztHQB#_{tpBpJ*~YJzao5=gJSO>9|!CEJu}9^VPHLL`@>pq%Umty7Y&XZJct6kjJ-@ zeNthQQgk?`Zl9iVj^$-w;_s)YoU4h!F**DyY?5T=Bt66vwQ8Qqe~8`q*!x+ z;XI>Yvg=>}{jyj7iV-~@bBXjQr?*t! z&)RVT_>)?T&pGhvDTf0;CG}9R}p&{yF-X8?MT@fJUv~S zBex-9W$+hSXI=L&{i3hq?112(zU~vxrX~DStfDC^mh0`?enr{(!rZt(fGnze^YVP* zy&Sq#NLc6Sf4A+Zk=Fi44HE2Z4fah_h|kUa8|{2#J)AZ;+urDD>m$?tW9Z(q3ckl} z+&taDv&z1=_PQUrfkQuP1OM#4cSSq+AZ||+?~<1ANx{FwjQJ=tk-cJqCl^fcFMYi{ zJi;YrMlsKqGDnkTBodoAW@C(P)FfZrxHPMeoJ^!j_W+yYO{wxAPpR`{;v z>BEX2f9ZAfBh5t<3dd)S=^I?=Z}~9GRMQNH)mF^{=X;)IKF7rihVUrFo54D=8eQdI z3yH#bl#tw=GifQQMwI^kvb0q5u4()J!RtZN$5{7fBGE^L=xNW>#tOb^g}!Twe%IB4 zzh-b{^O6P}YmWAjS4nkTufQv+o~Y!!>g*xme>sNv_X9;!i2uAErCNBsFSiPPQ!n=I zIf~G~Ys)U6z1qgT=KkfDCb`$9MTbn>VOwPkRKO5tdwq6+NYX?UyFGa5eF#HVi@@2sRCJ~%Ds=SVH*-zh#QCfuR66D%n-_fSO1srFw* ze;F}24{!~q=G>3@ddQseiEV$LA$9ry&*WUJ z?2Mbc$xz1J-P7}3BzJknw<|h4X`eG6@kMmB8^53rlyXp$(hYjb#qMj4Dc{QO9lSIg zxaa*;!EeN2f24km2O%qT&m?BmfuqJ@N7=lqDWTw;X@z98X0G(9dQ=wMKpdc;3_ z!~bhP;qM(*L{mGy!aeG8ErAIa!=oUcWq-K*V{Q7O?tu;--{LdlAGL>n)E>Tr->;N6 zl*pF}0F*<&CJ=0Iaxh?Tas?2we>bPvb+-ltl>FMp%pe808$2tYs_dL-F zm69V1o#?^fwnY`k|4C+0IF92-N!K50^W=mIv2Pc99kMW|eSV~GR|!*ye<~}uiV!?% z#CAHok9rKf!5JleK2bl6x%HTTtoB|zi_yig{MIU2;Wy~&mB&2#cMoqy`hVn5~o zWB9tK+Pe7relLjCw!*WTbwu{n^oQE_-f=LR zeQHlFsPi$)2fZ4r@iqa8PEBHdZ+D(lIXYvM-+Ts6bgnF z&4-+672+(>iE}yQ7_yHAUZZSp8;MGow=YF>SNN zZ|%3MT9*DGRm;+cR4uP*)P1z?f3A|$Sub3Xw@l&N`{8CPe|EpMA9v85_k-m_0p~Y% zg3Z**erqT0pt~>_KF^D;`>jLaXNl*3^I+UVc-|A$l&o*;hMGMZRr6cBaTnovHv-l% zRa5@Po&?#65Pb8*x{L6#C`Ju9l2vi%Z_~!+W|D{<_4vqyj#C%s%TAf0F8tNo;Oa@Yx_BH>5yk{%x-? z-5=S#PdD8US9Uf?-`&9HF|Pday4`Hu?uY6==cFI6n9Wwqeyr2y-0ugv?4R`6|4`kh z{OaQst0yOd+?Afrb7cIC16y1Mn(4=5XF;3J=fS5%Sr{tV9> zn^GJ9KwC3)owf9=CAa!{O-cS3h~#uuq)gB0%2rF^6K!N5jI(^iL^d-%ZT*_viaY9_y0^_qf{dfM^k3( zLWg1(l#ajalj*|@_#u)Clxlafu{|GB{u71xNs@o@9AEtX?ZzYa*W)Wn1QQznJ@|}w zxnq%5(0>Cu1BG*?0R;Z(IgKQTz%Mv{L2uw36g>q+uYe}iX3CaNR1V6(t!gu%B_LI= z6eWbP*|bOTpSCT8g@Kcf!7*@Z3WQ3{p6D2=$$FoPD){&m{LvHX@>Icr)8WQWse)Zn zFDZN$5ICPArPDDzO@PtUfkJ=EunPyOgxtafg@2a9_7%1_@9@>!LU_`6)|Fh~{XMZ0 zIXcu%xoSPl+nfL1jpOrSd+cj9J!oj0_1^nu9Si$LL>5k2hu_(TrJW*Q_n`bRNW!jW z%l}E8FR+ed1;3tlKaMST9)l=5ZT5kP(s7865tvd6{;#=09HZ-~+W(r1Hp0K3^FLPC z|9?cCY7lm>+pA^0DLjJHqGTwJcKP*UY{tWsvUT76 z-ZSjKkMgCMzz@x^Dyx z>pxKNmV^CAz;6M5%-{Yeu-@~#e;Av+R|)Wh?jH{gaXJ_eowO(8@==(m)MJqlZXx)V zmFrMNzQuAib;XU{_*VWDrCI}!Z%pE?hreQ4$!&VSth z@BjIK|Mie#_S47-XLsxH0US9+mEq@4eX{2w72JJd;^OA>Dlnh$7W>VPVgJ^C)?)ue zx!7&g4}3?tZFrSx#HYQ#$IC-kT`%QcDePzX5?!!WUK5?rMqcGZ1^l;+aQNUkt8gmj z{+JdMk7RP4%89>zqLQzF*nfRF-KEr3{_yPiw0CZGd=`xhpM0k*4=(w*_`$CS zpAuW&svtA1AEssb4lQ5fU`J6@8pYD1Se;qwUM=x@w?xY6=vk4s4@*r`z-M&6kQcus z-|YI$Uf=BS?Y{n3b@SKzi(wCc6efm81ZxW61zjwxS=|@kqPOJ8HAdFC)PJuBmrUnQ zo6=6@fD?yuz(wak1u)!G@8ONiwL?cgZ32ADI zj^F(91%$rj_-*{x`Q;1#0)I&YpoXKjZ&u+LRW(Tn2a{**U%o`6l;DluSj6$Z_8jYZ zj>`F-*kcXxmoJBAeaS`a@t2r``jV3z%Luy)LbQ8G;S}d@x!a5GS{gR6osQb$o;2DI0uVpCp=Sxl#w`0|a zvcCc^WfcAjCQSfe;P26h_^3~{;%fm!z7}Boo9iyd1Dkxv$=@h0ZSBz@y8q14f9B|) zH%AX=N#ttP4CN~SY=5x-Ox}Ms*nj5eKXdf^Zm@#ox!-Sh{g$bknu+DiL@a%|NlrdB ziXikA`3gc`L&z)vAq@Enyuc`!A|r5%jv@TW4(GP}%a@l@DSTE%kmr|D>1+5E{YH>y z@FKylud4s@&g3{T+9QA$(7o3)2>5IE47%L2@3?{gvkd-O27iC~WpF$r4B;&;mpO~} zA{w^0eR}(2G>*Z&GvAbxs{O-2CGWUY;QLYUUKgB+rG%le!|#9mv;O0s^*i`yjk+4S9Oqa)bYk|M15zc6;W-lv=_P(R z`~zxw{oXaH?td|S_j!z`tRdTX4=DAWi1`hbz5am8K0wmbFk_#Lru7jk_z&2Ye*7sv zq2F=RrHOc$fDs+`M0p(Kz!PL!R~5wGHRy4&i{k*-amG!rS)-wQr)tbgFT3iHQLoA0SJewpZrTI**> ze<@T)G8$(zjMWF9(&GSxp1pa^w+psnnPToG8L>O|k>!xRO%G@2(ENakbF9D(Ls@3O z3!Y_#j=6J5@VaxOodjbPdL!~lo+LO2hqRZ~=fMn@4Mm#hryYn_(LPbsb=7uN_fia< zAF4=_34g5@1)n&ELD4PcTFk%;KJX)Cj5>=`HXY%4?^u}c`I05%7a5{Rvo$*8HS%W9)v#C z`j3LecIT&+mp)bZ2caUH_CIILGVx6IOMmcq33PQe3bUp72KQFjJMu~SR|Im%2iuu| zh3snrcMR;3nn{-$6>25Nh5w`mozlheO*5S9;W@`^Zx?Q)kv8aOmO?1?)_)^A ztX+6^13=8$C#m}wATLNvx~n_c+4SD~EKV(^XjLQuWRl!t-LZQ1fpf{!{}e~9IET47 zP5&-a;@+q}I&|=V=g?)&YA+N)ocVl9&(;?Z3IlAOIC;BNBIE2WRR9P*z9kW_F0zh) z1*@pI{VB_*q(i3(|JXk5ZJ$mkc7MrM>4b*Yi~JYZGo5wkc2Dkf;!kV;KDyWQS-7u5 zO1Mq@{&EOW=lvX|{PHjGU%x-SUB2Dyg_-efI^4D2kJs6KEWN8?&w!tD`H02867($2 zM=s>2H#1igbulzHzI=;)1!3T)D-Ne#!!!B)7~||4jY9Wm{<^!}fe^>-?SIgz^Q3?F zy#1bcMz>2_+8HHp75XpO`7U(oUVrX(=;5D2clAls&ggu4%|(Bn-u{_4|IC})^Ck`@;u?D)I&|WP z5K8+ijJ0H9?U(GmHh)Px^E)4-j}(#8PKx77S3c{Adp*%GW#c}*-H?gyb>4w%@UHQc z=|{o_J_KtB2iVd{=;ynzx_p2wA7RTEU}Ia52its6%_D_y@1ZR8;+XYAcB003yDz9x%^v&W9M6lr|wg7KRM@p@IZ4R$2V0b= zu%7s0d%7e{4P~h*rs>C2a0Bf{FLDo3^fKNE!Co7$u#G9Fc!nN3tG-zKQyL)A>WfE! z3pYdG3wLbSW1~t#uQRdhnYHM6D$nY1l=h>x9dGHrVO}bkOZ%)jy1|M4;-q4LJYRzS z_f3!#>bGk#KYxHL(LR80kB4%vx5t*AI{~`V+gl(PdV8Vmz|~d*-x?R?*kZ+U)SYE| z=kvg^g3ARzGYwMEpPC5Qhx;>Rxjs?u_xRDNkm>OUC&Trre`}gt9_;&5KUB)}RQpD- zfQ*l+LLmV-Jnh9&XdsqLN5bgjl9rSUb-AB&5pQ^??SHxn;U#BRyU=l=xOqLA?;Wmz zdyl?f(_gQVk?7B%nQ_5W?Xzm1Uk+`v4o$hV|HrS>VWDUnkC>tGF*K!lotQjkyPVvr z5B2VyjXhM=MO3%yj?dR9F3@+35>zlP#|S6?IjY3lWx@`I`OlG($x{kHc#b^A{pc}$ z>WwuKDSrr}#QM5-$D`)?p8SY^xQNCl%8)=tq5t3(?s4R2PU1pD_kF{iZX-X+zuiab z`MkfoGvkU_``6ALY{vE)yici1!&igdB!b8pz*YPR7)g* z2`BcWAf9Ds$^+?)Oy!4xG*m&*#p`2Hn zKQc0(8T}tMW8M!V9fEx8QvN?^CI2Y{58N1G;KOWS%Jab9WoL%rd(FLo*uiKj$;gDY zzZqLTjV-L{3go$Y=m{*V>KFOc7XXP^ouiFH3gYQH!nV}L0edA=ASRizGO@!uI3kIc&A>x;*1tbxdiZ>& zRNSMKG&GS{9(7Ezt0a3-KegXJy*yudO3qHKwC9Cuzdv!?Pn|O7SN;z*`da($b>@QH zAl<0@!h^kr%_TwgA>?E#bD#G9vfsYWH?;ScJqvLu?pFMe@OJXB@cxo;C&1hO{(l-$ z)Gh#T$<;CZ`0YQaX#Dnjl!S&dgC~+ke2{em@p^e{rnHY|tz~ zZ?dBbg)nAO_??4t#a#U3{lzGNZ+rh(iEzZ&6v`QiK_u$P2o1n}!H~Ld_wD!JUkvV0 z(SGI}>f;K~Y1Sd9e3Aq1Gmz7aBYzBbk_E*X`mvg$ZG4n|-re^X!#OrMc@W&U*Eu3` zj%GShM)~a$oxCGJk|#C7og67fa_XrrgN zq;fLXqO4Lj3rl4o%Z+`ndRUk?UXPOnnZ!qDnaDR#go`pmpeilVB z+Ov2|cf*}ca{f0UIluiL?_{26*wKAAe#~=ULCMJh(jF-E#!x7o6n_m9Ut@2GKbM{j z)il`6&%iS%IpK9EYS^KLC$B{N%)RmTg+lwRq%dz?eRm8WLfk@pr+kYpn9E8&eK1Ue-hVRAD&Jr1kY~#16oKa& zMe$0ox2dB0bN}mwfSvCzcF5+O=Im>9QfM;xy%bO!8yVhbV3~|~yN@rP@&0fo^-8iM z{OQ~%;r%7L6dRUly+ywy)=>5BtFK0B(eis9)w`By6=+fRb=Gb4TUGSbzGE3rjzfmhY&dlk17W^8r=Tmhj9`e}gx`8}C_$O$u&wwoUKb1zYxcDtmg+891y6-$4 zTgF8vfGN6eDly!M9M<(xNwULkqxYdyn&ON5SE*y7YKF=NsRWnxLWy*`j4RqIoBEE# z+lPjyxjZP7MKc)1t`U-*O5bE8i__h#vbv{DQ?lg4vDbEO*O=nv;7<03eLu(EB4GQACbC90rt1 zD0UalWDW6dd%|E=cMF-m!HLe}L!sU$>&myh2U@ompLwjj?>&dYFaW+k+!#Kz*MIpS z$z3np->1vg;4J-av2|$l8@jN6m?{rL=&GqbK78pp3YSqF5Z)hISdUj= zyP_{x9~=|TS9R}ytM`{!?3MjKNj%YE^$*S}&ewI(d%nNr35V4RmtYnF-aoK4#k22& zBOrthNmR?+gYf_)K;r$)MG(|97=I)oQn`E=Of+nTb=AE9B2uJKJyuuwN7j~Q%4foe z8(;$KXmQSubEf%m{@{iX^D+J)?ACW>4`PHL;tpbv409k!Qid}Cd5{CSJHT1SK$4_; z3@+_=}F1KoPc2O53Hhx(PxkT$A;q8$)6$2wSzxPjLf}U712fiSpAd1}GK~Bzl6yeJ_a1ZKZ$NkANx^&v_1va)L zN}}Ut;9qmm!T;pOc#6XB7F++;5|AGcJ1&?m$`O$X7F4+JTH!bdViOkR}arj%RY32 zzSxonFVGp-tA}U0)b+zNgX`+ynJ#tfEWAjMM**%Ko)<~=*v zAn<|`Y}qc`TvZuIblIkSm8d|ysOeeC=^SxFDxmJ_JI z!)5sndk}W$d|K5|m$Ov-@i z|56Rxay-`bQfe2XaZG-H(O5517>@&SCN?=hs>B5y3lM4<3sZ;Zf$O<5gEd7uNRG-!i)=+ zO=0pyQ>K0SYvvwz*V%>2!sLsUMH!b3jl!(!_B3JcsRayp#{vcnEnp%*)-GU?@kOKy z{r-O^+S$CA(SOr!vtx1ozlrYXYibWn45Lr9k-xeQ|LQw>COn24Hm*lNbM1KZ^>jr%g+Scru+eK-aC6~iK z0W@)MBNdA#EKD{YVYuy~rS=`Z*u;n1R?Tx&s4*q&U0tG;#j3+_)%Msk0euv>lYcz^%o;JK1 zzOm+VLB;Y!D-}@)U3Od-T~dle4mF0| z)+&Tj9n~m?n$+AEgNgA8d&JxZg8RAr7LwztB3jo%5oK<<*0+03$puGSbXN|J||+x9A;(6(6j zMX)$(PCT|BtO9f_*I~~TTBOC0GEvMU(5hRK8HX)#l!_oD2 zYky>QTgysoNy@cRK40vQn?0mjsrcfwMloZkG27At(e$=WZ{o5dMX!YseQ!QddUSM^BGzJde+OgOM zbBZ0iA*&l>+;Zyj9Bvdbg%CO@xvpb%(ytT{G3*#LS&84(drDy|&G-)NoLv`5uRwL5O>*hF5Ar5@iy z<>_eVEw*}jqUarrXDU96;_hg(9uy@66UL6Z(g8++@}7ft5I~k|d|A|rYj03OXMaPh zwE-8Pt)LZ5f#wyZ1O;80mHb_4UDnLe(4W`TveM9fv0Q7{A&(mt6_o8x`POo=hB#oU z)#Uox0lb=^Aa$G)N<~u|gGCgR>#ZW)gp0aP^vAl=Go@Y4MC*F11z{~~wC1)3R)-sk zI9xJqbtuV8e?e-5R24L^<*!QAh<^jd;5evtSeN6_0XXvnxyl(}AFS_K-Ev&UhtQz1 zC5Z+@5|tIztCGe-p<4CHpuJ>zc1K3pxkbt=&Rw-iZF1Z(oYGKMdbL?sSQX0%hu4MK z2B~?AU9V*dJE+C?S49l9Mk~Zp-Jz$i_@ScKWJVObQ{Kc;UeQYmFtoJYcz>Z#EKj!q zS8Cgpi47pVO`Vwb=AJ0QMX9q`=F1(azEF)(Tdj1rl&IwU4P;QBZ#2uIs?IdjJBh(TXVbthU^qH&nkd6q$y#^yF%>o24u0!V0!Keb3wlo5QlF%M3 ze6YyLIHIq0qBhd%dK<4z`gV6Y-Dqxgm7jYMOdyNxVuTvvY}d|@$>mNbb-m2B?5^5v z&*E2+;eY-748x#~DHwUc z!6a~%FLLOzm&YXpSd4ko!H98VF>&x&kx>!MM!gnkju#jr_TfoaLUw+it<-FLUf;@F zygptvC!VNt5bCI->R73FKsyg(h&dHzE!;+7Ei2EK>QR$U~4LdQA`k1$VyN0F6pEH0SWc!$-J?KHc*+Kn1n%Aswf(Cq20n2`o$e?bbltj3hfi!`EVf4iB3OH zuSXpRtFyLb3md$-?Uqr>M*MEKp)sn`+?hT(pr{(c4T+J~nKUM?;Y4G0G+7ag#D=!2 zdS#J^1)ywUl*V}5+E%B9dXHKMKt(rNWP3f)niRy@4ZowYy4OV2IZJtqB8R|ApQl!o zH%54YsZB-;YJah0%X3zksuX2Wjmj{f>S|4~Ht5V6*4Ch{Fh!3;kdcZ8TiAxM#ya0| zduy4W2xD_@G|gFAR4X(LEbNY3sy9_lF%g;>cc(ryoKA%%y&0{#T1A*w`C(uIWxdk1 z0(dA-$`f&?&Sq4SHP2wwo+& zGi#MsM-I?sX<_cz{c*L{@aGkUtF3E>ybc0R5L*^qq(@z3RPDC~R&EZVVC8IVqD>;( zZEMsJYJd4YJ1%+UdetPwUB4M5ge$M)k=9g+L2u@-YJ*Y7 z5)=rlLHV)i+x(~GkORjdGIeh|R^0N9}>zMEmSVvi3}v1jAlAjY^nW{+G3mjVyDY44Ya9=U9B6vM!h(Ms$IjO z7k?rMz?4;&tG+!1RMM~Kt^PXSC+DO$>@da3ygjxwuRn39F*9w}NL;KVK&RcD)~5}+ zT6DL4mozciCr7guv~3KPp-fgAl;4_ROKOD4%MDzY%xzN{Q#IMv88mMpKr*bT2Dj7Bb^D?CGYC(de3GCb{jeXmIq zki3Mq4b9_p%q!=YW2-f3$p|_0O%7eVqTUob`ba>T1w4YM_RP@O4O#QtZmURDc7JoX zTY(m}8nbLKAVsT|tB74&fYM9_7iO&l*C?yv;jKYH(o~Iu`n1u6g5ubo`({zDmGje; zWLfpvsJ7I#S*41ayPoQ-n{o*r$x5@MJ6x$Wnyz?6Eb-g1Rc^>qg~JClU99-_dQL0q zqTU@VB~v44cY&A8CO-z49czu?m4ArQbs1UA^tp+NV^x75Fj&BX*Ref`Bdq~7Fp)go z?sT_sf3`7vQ8aL*Hnf)%GTiotf{%}8&0ZZ{&rGW^;wmGZoy|cH@EiS|;1$WvrsFUw zQJWgow&6}(a&sytc4${@v&;DyT=i8~@7T<&E6_eeG`XJcP5R4$F3#80O@Q-jj@S%)ULN1cl_iB% zr}mmKy6UJFI9|=*gaJxAn}52o0(C_VkUHgzwnY<@5wOA%_NXzYV0DU*mYY_EZY;Mh zQsR0gwN$o;P%G$h>!r5}(D`l%bI^vyRFbM1X17j2Qh$V3JEIxi8jW_X@e&q?>pq_! z^!aw4!X1C)J3zg{lZ$?NGa2bjfLQa9s*&Z6RFm;$tI_pfnp|6;r+=!w(a2_d#HM2? zTMZ5RbhV;^Dor`Fpt2&i1njz+KlTV&Q|a}1K^8l}unn#VWj17BYDH|wHmy|`c0)I0 zSl*aDVnMVUBdAqVg*j2TszYjoU^0WY8^DZ*Fr@;OmQ8Uav&EXodqfEgYHi;@0c!>} zIKo|G-9{N5x^-ZJB7fAj+U`-E9_1?7QdzO|XxW+6^UAE#@TX3@MfF^&B*HD1zyq>L zbosU=!-29@g^DnQTb$PPhcsLw1Vw6R=tCD^OLTw7~9Uc185$?b*Mbh)o^IHt_iD#Jb&cs)*vR zd$c;J@p&FZp(`7w(9G>&UC?R-ek+s)#;9}kOFDDTf9}ZmmG?9r;^tlR+@Zo zIX4(%(Cts~I)84P)Mx-3Y?0`(9Y)%zMcsg5*ueYVyhn;NZ3DBrRlhZM$G{dw3Gt)YPV=GQhIlfq;QD!^evuY6%;sVWQS*U>C;#XPV=+KScd%6Y7Bq2g1AY zqO=0$QggROTpXO(f-!^2+Wi!8v(!(IDc0f%z3^zg2xs*UNaJekDUHw zxx;s6xi-Vc!o+Iqc57xgZK#&pS8Sk45}poAQ_hv9+&~<{E3~wWQ7%lp$qMIBnLggj3 zwya(SBPmM8tnQGI8_EQ&BECdveO2$*RngJPvwB4%=G`VN4_inB?TnlKnbhqL7DLh< z23lpV?nJvmwkox8ouuL2ZeW4!Nm&7lFk-A~^?zjvWfTLQ5?!`blJG%^h4YJ@jjX(t z$##Tg4WH{!tKro#t%0_C@Mw*+N+2$mP-%eC`kEB|#jZV@nE(mcbWooR)y9_d2xE-Q z_@!0a;oF+RbZx_=TyHvINQ+)I4J?oYQQDT*8*)YyGht%6Rlnodp10-|yG*SZi$*rM zvVWCj1pp?**d0`Z;t-@7*wpM!nBjEfETVC znl&p(ztUPRC>0?jt7pS5fO#}MuTOyr;Tq&@TSe;_@a|9>cas*nv3OBQ-ARhI<`T%SEE20pcM|>^d?P$VIZZmD(;-t zN}kCRrY|zhTBEVz<(XRA4d+6MtabqhnqkC34Yq8jLv*^6aSdBcXup#0^hK<)8j-Ug zApCZn8EWhnC&tEnLV1wf87(bTZYAHjO&o{KhaV69S5WLAOEB3FFCRmX(i>DVQ$#_0rE8d=3=bx3zy z5gm8jnP%W5zB4_oQ35N|234Y~ZGU#7n}cqvyrx`7!3419FR)shrz);jt*izu6y-ZL zf+$%)kE@bJpt41%(L%<6Nx#dp(+$`#Mr}d~wjA$Icjd8E9hF9OS*7S^7gCG;>QL*I z^9%|>i#`JetG;3z#l}P+dmCx#_x;_t?5~tkZB318!y&oOLlYKutnFwx%YP4idE6$) zSg*rxYy-8&8aOsavZ0kFve;{3!lp7Op(^DL#QaLA_gtW~-g!-{rYa-`0=T;y%ngl1 zcMX0U5G#n$OWquW{9zRt2-~eDZLLau3iZLhk5UJDc-x&YG}@9~BWSAm?VR2<^7(<I4=QIoi(fUvV)HLWUs7Dj3zj3G(@US(|Xzye#kn$Ta7w3#W96k^x66GMh8Bn+ie;uBz#FN};Ba*_MSzA%E)--`!5Ga@R12BY zSWPpPsWBF~>H^+XF^sB%%FdWq22*6FbTLekI%Q{P*1YNfsWB5A4=Q|qw4&%qWhT+% z%Eo2;mNk%BdOgzHJAZXqTXdG_hH(uN5QhTWHa(HIO~{^X%&ECkD{5IclzF4WDU_7= zmsL+H2@a&Gx~(o}#u`{{C2UshFS}L0Gps|{%HDQZqr5CpVz5(&&6$GtpjBhkLWhbk zjC{VqZ7R-Y7_jQjkt*7a~sLgwWJ+QGo$a;6+rV-F2-|?ayU?Ghb;|yJ`9GocDVZs8=Pu2{|Uj5(HzAXa%N$%XU#^Egx=n8bYfCPgPT^l_x^CI~h_Gw`}-?YXTE* zN*4o%tSB3=B_Z=yQ%_dvA)-a?6iOi~k3NJ{)g8-vZsTfGROl>*=SDEZ^ zt=MXns_k-KXko+JwAnTq))HP6r}gs0=#rHgs|5WK=XCnx-h2xT%My(Y7`d(Q>YUpK zhVyl6Bx>VL8Bs{nB^$FE#mw++r`%ARU8^!HZ-0vPmUYVNv^!XG3g#MA*8&VUZ1W`6v7=YnE7&3*4RE~^5St-{QVGV6(?3<5*N zVSgFwjK}(lQ|3jSrpHh=Nn9YhYpA1d=;06!A>t#>n>)pL(Cg)ofSxQD?N=R%XMZ#36nKI zkLyj2Xc~Uisa61)7Alamt@}=%7T4rJVSlWeEcUr6JnqBJ#;X+9&aO;w6L0CzYo#Gr zMRu)O<#q>=iM1Zgkm;HR2RyaoiHbGQD)lYtu&phE_?CuR4q2}c)!k-{4<=2%G6IF5 zJf&*osxuJT`J&XTk84b`g0Jd!eoTYSIy=a32Y!2IP}ZU^85+WkHXFQGEsjd^5`Tat zyCRS-F4JwaYn7+PE@M{~T@}EsHL)Znt&T|z>kg;rIzVjYVZ#zgc8b-<*wh>~D@tSR z;7Xt2`}6Wr%oFq$=b>`5T4Ufz6Ii&iH3=H6VhdIoRtLt|vfTyOlh$Ga)YL|c+JY!5 z^2@<;0I1TG@*oE=}N>A!_p`$l-%&i+99W1(_ zMg@Tjp=5npscXDk(D0RyzaQd3>=oueOch!a-KcQN1nory@UUfq)t4<=#^6+nO9` zPOGC@s@oW@DpCg%XEfQNHWocb=)Qyb)?(Z2OsEpbH(jBj6-V|gFoRaG=~jqVm-W^o zRpfV7xzkZx!1m1n-o+bTwSPXw^&$WdaMCIjIgp(bTYAlJd3QdU*49*K*PozuqbKQ$ z`~=81@ovMKEZh0D)WuP(W{-{HxU*nu%%G!};4-<{T3)-cbq#Uk5AA%*U*oG5q6O`s zf?$BTEEcEmghuR|ynzQaNXaPW;H0n^(F8@bRfL&om98;Z3lo0>R)0BM07i>4P?6Ps zf2^A$I4{ycOXp;yHZgo==yolgCkIyF$r~~#dTxCPavQ8#o98isV47Zqm zkz)kY>+}%Dg5gqUXl;Ra+LB=o{Muk*4Sldza)XN1kra4UZTqYCuv(L`8lx8FnrAL2 z!>$RA)aKA#+P0^x%6~0xNSdfopTTnwZo7^<^R?kNAC!mF(wZYxjy8l=gM$?Y^Q#TG zL0R>kK59*RF4t>#*k&sbiZQ%cCfc2b zk5A+xp)7}Ok(>@q5=2R^qL*X`8hGTAqPNR3wR0`pR-|kr2D-a?%1A)afSP0rEo{Z2EJQB@tGx zR!~3K)CQIguBz41vf^~CmCyHBN|Ewod@~v2fl|+tc7GG?(ld$aRft*xhPOsv78L=- z)m^Y_D9+BU6LN{2ZKsYaPZ3=Tu#pCL^h#@Cx+ve;78w>+*WiGgD_$F;D}yoGYmarn zXicdmG@fgnZnf8_=#{)u-uMPT&RdvgPz}bI+k;MBYL}aQi3QD4r^^qlDI<8D)}~on z&U@e{Uw>TND_`91Orz?|Dm}k7BHL;`SejNIMp3BjjL@~TlgdpUY8r#0w3*B*fWH{< z%l;au;|;gh9rT$kzJdcis8ThiLCxA##jbRF_OdpSceQ08n0ZWaoN=>V7DhXb6_Vkc4pr~V;*wkXG}!_fgY&v&qPAkinO5{HtWFn8^BvONPOb5_HbOz3+pzAW zEmjw^U^%K!iM}-`jfoBu&>~Y^V9m;UHr|Y;lWLW*>k_Jt$IErEv1FSoM}${PxZIH& z?tgSi04C-c^?tkLL)$S|vY6?vZFfs`e;J^2$R4!@9L?~oI&52p4-9tfOm7>2GV4!* zqP@1823Vd~HoY3;*82hqiO>{)6&YmE>cCv^rbkzGPwKk!##}~94o&-d3Gr(?e(2O# zW@3Qcs?qB>q{BG%{#p?LpXUq=FLy}9V1IWdV^XUP=d!q&xz4OzoT=VuWlQ_yA^fRZJ9Dn^D;(A&wi)xt2Y1wCQ+DaJAxUK~nS_Hc4cP*qF#b zJ*rJ<64V5$9hdBAYo#)G7Sn3GrE5gbTNG7^p39WRkO(D%JzQj#)LL9Dsl~#S%9VA$ zwj{|}ZMIpj@tG^QU87A-w_OS}27e>Khv^ZB%&a=$demtgJCnLS%XKxlw*z7V5>CN)hW^&PRW7hPlxTcK4{e4Ds8W% ziHK^=e6`wfIWTYMr-RD4>3hqm)P@Efv{@Msi<&ccv8CGZ+pc5CJKF#)i+>)COo9|v zjLL2e#%e3F?+~*Mh!M_E;~8Ped&mS^wA*}RmxoG=1+OTGSw@^uRbII@R3Zp++uOD5 z#i|O3<5_RZ_u*|9?6Td-uDzUgd`(|&G1*+{HcC5&I}31TKr60Z95iL446dNM6@WwA z>Wzw!Qq!Q3IPKco>5||PRDZ52eX5Ojl-ArAi4r@IS`(bd)$$msTL#)#>Y8NlHm${U zi|%ki$MU`n=ZEV~t5KUY+Z|Khj^t&lS;x^z4YUDon>U74pwsRsExl3g%RxsTGh=j_ zF9k)Wf-ovilRHAN`c`vglUkXu%gs7w=MiE)pcXbbut#8iO>bCnY=7t?-nDA$1_yMi zN>3XziUw4BFhDI@o;Mv+(4r1+Hteq2-fd8jA=`~fA7N@3WzAb~8|Joij9Q9%A8sr2 zjmvGtscp>y-74e6ZpqZJ4maJ}lus^}0&}Iu`>@@m2*PZSv#d)6V(G`3);&D|+wX!u|HGj&^yfZ*1TWm!)I_nv^ zqm1cB5M@9>2+$Ph=0HsyWtSEh!mfHHdxI++fyBf30`T|(aXu=e%E-sZvHMl#R4KSc@b~SNq zT9YOw=fQ#43xC)t4L6oXv0Pf@!46n=5kPDL4Gd(63Ae=1j^km&ncA9w1WQ*mcRJr~ z&4HumZ4Mo9Gnqi7!1Bu$Hf826-ye3nzKS$Q8@2CYuuAK;%hv~MM(*ap@qAFFT}h=S z5HXDIU{l{L`rUC*AFwvHg(?()ZLxWkoMQ88vyC;;t$!{oeV~q;^F?hXP+DmLaH9>g z)~6mk0h&SJZYYt(e4QkUz-YOx)u$q~-dPjXF|?66ok96Qb2fzftgvbuF31nS%5E&f zdZ%UTT^^dajUw1xE&+}t$!dN$wP#(s*>-oF+HZDtblwoD-Mr=mC8xiD=}B!qs{(^y zWUS$y!!7l}BGRBrt-Mv;ECwrnX(CnJpc>+$IO@=iq2wuIZDQzLeO__eTGj1pHqHK* zO7R36e>!{ddCM_x;_GRw`cmR%I!V@a+J)7HAU9Rb$6G?b8*q3TPHZXF{X5O}ir+XB zuFEsD!;jZp3ScF&vDH(o-d9h(oT`1DbLCkxBNbxic?4}uFN0WU2n)K2P}dVJXPQ8y zH@Iphe-8s26q<712ZT;L-p~TMSfxJ0a{+#KJ_C}R(j|>vfLj3TAy#QFKJ(!swnu# ze;sJM6o8{6=0#HnW6q9v#{pv2;`>NQ#94Sp#1o*9?QP$~Wbgt6KWLM#>pDlI)w;@> zhfuzsV`I!7hyo4_+i-ovCXO%Y5k8S0jAhqz;G&j*uTIYGIw2<6&#}m;P6cy3Yoke$M>V6wRe_;-Ft$8+}^OhS0-5PJt(EEH4t} zQdCy!4lU$g!Z5*t(;{#Vkg5?ltWR~pH`>rvv@4NkmK}M|ETT{0o#Rs?>{veZ|0+Z{ypR*j};;VJ~fuk|sUM;&ZOb>+pUi`lR=t&qhpf25sS!B~WY zCl!yr#ViASP!G1ps5RQFG-plo11fa&!`2^KVsP3}k;kp$pb+BaA4LwA8^ZFA3qX8o z^~`St{e6@juQrt+;TthCXu?s&ku&yUD_liJDz+YN>7>y+asu`fFtO_60&*=m-$f@u z`?{or{u#nfxAovgL8YD(f5UN@qVZc;z^Q?0Jq%sDoPGkqS}bPUK4|QS<7>BfRwFR3 z9E!i=a{VZ=uMxws4da@G^TjZ%&mj{t<8m5N25k17(txJ18y^BW(0SMXK>Xx=7K*oh zes_IVSLWoGUgqZpD1XSDTIY7S8q$1n|KFRJW}d} zNzQ}hN#2tx$H2UL;O+R_2PoUz*2cB4TVBYoYpHxKA)3=yIC!6TANT1)j&O^7>_-cY zN^1c%lns3D`aP!Xe{_Mmv{IBlA5dQ=kk*$+4wWup6(l-M$8R{^n|fdSr4NZ-s&)IG zb2fV_|5$RwxYj-0pUDU^8q@rijS!6k_pFEOJK~d6tX$kSZb{Z*DOCsIeXDoRQBf3W z>sGx)#H7EyY(dB1o7M#oQfCi#-3u1vlbr~=d&2qy&{Q#De;ykH+LWrOx;Lt$qA=h zd)#Yb2`A$xR`B<6n{#7IqIK^1-K8mx9H!lSu-Non>}&y|X+u(h#;3^ycTl4xktFks z*mfI0Req+$g>2ju3KQRqkBdyD^iX~9NLSXT~niiwqf`oqF zk&1!SxB<2XKSU?O&8fl3=$UXkLw=Isg!fLdMY4SDg2mcgyL@i0#p_|G`&}M&X>2SB z?nnz3pw{oXFq>BwN1}DBK6|6P?h3I4o)8(w&(%43kXwlh$ATVN!tCL?o2;rWMl8v zsM~!C+<#*7j9U|Tj+(>kD5v!ik$R|fjp*dzOpa8XAy4J-YJsi}E-3CsF@~W)$C7jw z*a|{A<&dQZ5gK(8>rG%ef(N1%eYVZ$tCJw^d*=xdszS$jIojkLoI<1KAIpJUpx03? z%fS5_iG?*hssofo@ejjzev$22T5fKAKjbZykbl5~a=ewy4E#+1>!x54yaoxUGnSlZ z6fOhCLa`%6{23h!{)S(MLaJFi6BE>n@=;P~ zYS8QdzYna%#68Tt);9H~$E&Eq7!+-6%=EZZBDE}Y-TjctB=J+nY|k@wt?yxB48ZA} zS=mN446*)5UdCO8qBFSLbAceP%@)(V|7s9yiVV{$_fg06)pZamZ1s{5TNTKrHs$+ac9>QIj^wW#Cs(JY zt#KxS6S0M`JioNT0?3>|U9l=_Q#O8uyI7F8?N-R$8`y*Z0!OyNMmi@~GsTaI3@i5% zm0EyOl=tS5`C+w?)Zw?U-f+|$Sby%5<#cqc8V zEs46+>p#^*O4RRqEDHHyPRo=cBi3_%X!Rgf@i0vT$!voYgp}uofMtm@TX+G}zza>9 z`+ra=)_J%T{lu4X7ks>&y1AhfsLE6ewus_T&q7*}ww%5-$d{sG?Osh13f2V+Z7*P#EdGut> zec`ULO@diyd;+EjazDE*0Aj_caKRuIAIC3@>y=L#w;feSrt*N@F3AubC1{$pXy8Op zbavMx$)pg-FQF#I4UvU^Xn&-NMsEuK*KJ=l?kKRXY9C?}+Zow_+2e0X(n$BlqC%d$ z+TUMfY(*-shX*tGyOF3bP1TRB&9hk*xBksgPUr_RXE48_Cqe%iS^bY%| zDFD{`o{O}U(F{r@`JwxLkaV{O6>JA%ltS{)m#N;zUam3;N~||05r1Dvq`)Hs3_}(n z)=vubRoU=>aqLAf*GaRdpW-IMDTtZ0G(N3N@}eiazO|5|#cscoSrd#lyv%E+kl#Go zKZ0@VLH+>{Uu)8h5w7LZ@2tq*T!p4c2Unn}L3Bug-<0pO@!~0SSg;Z9sremRs1Q;F zU=iU7{tL88O$d}lrhguzw?;ov1t63VGvUMY+sA(0;|qkB3=mOc%pV<6kNr@?`_ngz zbN#}IHjttThKfufA@8@Jg5}y&R;zZVPSN76V!(aM!`An&M>;hM9P_BD3-ndQ$LG%2 z-fNfgVwa(-!$*zsJQT;j7UTJJG(8hk6ocb(={+y^XMlO;L4Q~3R@nh;WNf(in``J= zAw_E*QshHhT_5!!1}M0={dXyI_W)>{e=Ao#qlO|h1gR5}IM#j=S)sjC1U!cf~dE(RuMv(fEQFkm8SHZ5xsAtM)f*xLDA@yeRC{4-PILM zp~yVU`Z%+-oZ( z&>;vpB9*4vCwAC3^C>2qv=66b8a%iDSNOBe z3V_Q_BkYBo`f;J6b(g^sZvyyR#Rx0%PW+}S?2VW_l#QuYz&05SytW$5dQT*cMW&75 z$F{W)9`%#02#8h{uhj^?VrV-v#B3)(V}HZsU}V_0ol59AsxsCS6MN8Or)1P^L+zw` z+^#hkK7ZFvKb}Ny!39a?IAjn4SL*_;MWTC>lC?_W6@pgpZn+}51X-ucPomuJ*n3I z**@BAiw+uOl7{?VQc9i018+2r_mepK=;V8vjemD4lJ6bq>!4$PH?!vXFv`3T%`01E`h!IxS}jo}XeNa+O`DrVLJy@` zbf=`{D?COyRGtMb^F5sE?a<-yBx9 zTzwMEyneFFolN7$V{m@-feXB0|p?->uyQ+(|v6hdRjDPGnEjRrkanskIr7VWGlSY)a^XWqC%%jAcPJ*@U zDVxW5y<{F~GPLTufh^kq=n*=aVLiYxbSgHXKX&pdBXmOwJjBMcbj?7zhTOv7c2k0w(Coz-7_3_I3x6%kziOzf zMI}bC%?8`<41i?KS=8$QgpvW!M3r1Pf-eJ=;2+*|Hq%DGoj;u0-=-IC6$d)4z#>7KGX0~MUYPjXifg}+1lZwI7Fp}h@2;lWq?xm= zu3c>zh-v1ZtbGjfBuJ&?GY*-^Z`^Pe&&!>kAfrw|`J)h-hOoYs9)IaU0B6olzb)W@SM2pX7^+MTeX?Pl%w^Joly>eQz6$6r7goC6l z*2Ur&mXdK9BWsikIEge zWG8aR>}Fine=b{XK-~QOvgmEfYRHVIPU?WA;+Y!?w^L7hmENwn`Ez90q>P6ZPM(6Ev_9Z310ohu)};>8{G|%Ar?6 z?5omrl8qO6Q{SA1a1+9o&wEz1Z+k!s6+)wNX1@sb|*lR0O z+z9@;nSTa4-nc;EnUb|B0y@-2p-6$_npn3e7( zUH@_z9QHVSegYKM%#vztTPO(65ZF@Q<+fg^A5ke=Bvh46dXtS_sy9hmYGT&L9gZ8Ka1e%LRHJNIGw)JDp=l)@gPfkMr6Dt{a%+*-9i6<%6mQew>L_2yUc6|5p1 z>6ps*1XUTDS+ayFPD~;JH;eAT1OB8*J8w7NS(8WxLE@y@Z{Gd9DuuI4He%O}>g?T{ z`fkEv+BRxD4~ZS;rIRT=i4p${I+kKU*>aVxLQ|og(<7D8u~2g8wKbxBKkSNJC(dg zXGU3o>b%SK1(}GAV?DK&aRvdM^{rL(o_~%-(>0YkK%V6W+4tKk^j#4f*0V8ZVYJM`!hpXp<22+}Xgn<7UlW!Y^cM2LdGYo~Sn|~4o za+z+4M&Y|yAVYiS?E@{e#$=ey0`>44$h7LN_MPHB-LjDy{|9d$s#iuOU<5!l$I`l; zAyUXS^V7**ku$2>E&pD+7`AJ+=m!BhtSmZ&M?p?sJr$p#HRh@C!^uXmfV$ilTl{R$ zZECnFjVf|c$@QW`jiIhiuYMiwFOGAbcpkjNiuiwE9 z@Cu~y^BaL2@L6i1;q(|3YA*B%FV39~BZCA1Gk`9$*pDHXiGGjG>BscwxRMgCW~5`` zUK)D#CjytX%QUXIY?ubS)p+GN4C_>zeh`dI5Uou~&8?VDl+O(D&0huvM1Qy`-5`=2 zndpjhqhQI@Y$9-Hz_$@_5(BSx+*_Oj{JNE%@Fm z8f>j8=Hid;wRBQ=h?ehpw}1X(w%muBU~sDPm6(BT^A|U5${>$mTC7}0CDAg!Is3ee zl@*cwOxYxV#vvv5yTKF$>#H9Hq+mfA{AO!jIR$GFPNnpn6VVjii@bOk5iSKn1PJiA z+nnJ8@9&mYC$FasfGE$o!;-6Jv^zhzu=>Yzy4j>o5wv5dygha#fqy%!h z<4h>Kg1f;N1?tHdT4E0ShEBpX(*5G*^mRT*K-6#Z^QQIF)gA{ng;+|@(7=Kuw+%qs zv)BVcFaeAhFDx2T+^OBEBzNyU`3(eWJSZ`k9mL((;N)(w;yJ!&(F|+nC*f>ySsu!qQ(iwR%B42ymR<$uPUxy$VE)a=YaVRx2s)21 z{q(W@fyI4v{md{w>yGA~Uw^;6c}cKt=unH40vjJD{*jm3FMlE!)dDVIWw&(eN|t_s z&LuFN2AxL+We>-Lx8($y_m|mXc=`8h@yEiO?aF}&+-2Bw&@e*g$tcqAB39}w4%*WEH z*raBiFQaW2@qaCFLQXe8Gh4@+RNKyojvg*n=``rm^WhDFKV(CHyIjv-aZ(OWk?ds% z4XV{Ubjnd42V6JV&WSREXUFjkw!OFTh7GcZ!rHfqg6om%`z>W-qF0bnID|-|w*|$y zVtM6-jKMi{{LDXKq)&?DN>35$VJ4B>D@4^Q{tI3T&VN&5E%qW#{={~U+kBb=jKE5W zWtn3Z-6Uts@7}HSDk2e<%~AwMoSlcBvm2k`0~4Z0-3fcu&fLh*a+bjSMeoK3m4qpE zXbpz&jU$$WT`l@CoU1xgb7s_k;s!M@z^ zy9k=za9o?&Tbb@MtJ$oVCD_>SfxM!SLLin4?I|n8ClEh;b3s#Vm+@JA>vi&?HY}W` zlaA0du1I-Zg1&|()`zd89&ZX{HK5`VIK7#!8Gp7+Q$n)*h`JiH+y0JR7>UeJ2u+Q=J6Nt^<5zwC*KwcA{)?xfA8 z@_$D>)$7eaH!chfM!O5#JC_vR5~C!DZlNsFT5RIzew^V_V#sp(b-hXOHEmf3&}!?HYHv$G)|uidiQmv(dbM0 zd@EW^t95$w0kJ=3kX=J?Ghez^g-i`h6syede?rGULnuO>>*mDK<&I_>6S>dK=5JlSs2T|%s+ZQLnmp)LS>-b$ z^ThW?ZUgG2i%rrDUGGE_V+zg=4}Y2`8P^@ukY=yHG(xofXC<+=lWe@fHMbJ^3;KrF zY|$Md$*4b-o?NM6tQGioiP}_&AXn{i7UhM%gfrFd!E`R)-d9l$S zMX)*S4eb8Uvr&>Tyg8MyS z&SXZ_PL{5Fe#_0{G0ne(Dhs&v=V2Vz9Ub_}t{CnEtKcn1{k$y>9>){4#$TTNZWfxX zM61&=aYABq9Mtr~ui5_W)L)DK@L!VmZGQI&}&#O1>$dwP1% zueZS$)MnL@;YnTuPQJX?6sqeIEy<$8vsMZnY|o*`tJOb<#s6jK`hSbC!}5FpW^e1? zw}*Rm-Oohl-lC!9srQLd#V0T$lR6X)9Bu?aK15o!9&?&~ZS z=B=VNv2L8N#hihCU83wg6A&hT$E(PLw7fz;zg71SAb%6a|9|S>o%{)GGT3^uCA~7?&=vX6DDS?vS-OU-{##|ih zd?LqylP-_|;PSW@+$nI0i!$DU9p~z&magRhqn017s*d^ZsZx(lG_%ffkFwoX919I?_;XQ zaeQu`d4FH(Pjv0PJ91g@_k5sSP=~AD=cilp{>b>a9E@yatc~7#F;Ev!x!anpB7OOr z6|{?-*KP>o1hNvB?*#NCcToY3LFqBh(h*rXf*P%(X6CyR>3_QfJCzR0ryVjKX!yE~$4FKaO7@ls zCsG@p$Mz#9b#3bz@3Sgdf3NjRpKFX1Z$36=NJa?W+#9o{fcn+1-p+~05Z^KmhQ8wL z6gC7BC&H3sOgykqVMuay5F#5bfuFeV*j=myglBZiEusnHISdvQ(5hC081rMew#wyf z7JoJ-DSkjywlkiqHgT6r3+Wp3QN<0Yj=rYER^J%arZP&$_WOem#A>5^9pI=)?UiB2 zGe&mC-9HeWxwu=XN~@uFElh3M0n7+7FT|-uC+;*ZHa2});PWup$sN&5y8$-cciW$_ zHsm}Sn4)wM(_`VO&Rc4SS1q=u@IB2wtV|mH z3!T#)j3;@FPeS}er5f{)i`|8(nX|DvPT>KGhsnc7c~E#j@-ZA#Bi+?6i}OKk;w&v< zOmTHSp>98I=W;P0QY(8YgafLHGW-p)kf=q8)dV;l5QqF}zn8VV@g%A$>k^jE5Pyjl zqygnVc1aBxar{q>aml1fZa0G*(_E?4KH@E7>U?o9fk@->`0CB#J}CgEfSw;X+)M$l zy!(0yRoptF!o|&16$xu6`Ce@m<`WNnSS11}xu1vlk5LW(Z#bMxOO*Sn8IeUZOB2*z zwWSj^aY%;JO6Wtn7O?%M^Ex*20)I`~;=Xe&amqB6TKs5Z?)j~{YFnNw`vPihxE~Az zQh19pacKwacT>*-o)f;VJJ$&E+|@EsiuhE0nkZ`=^qo_VfyMc%!GV4aL>L%T`)xq> zWUWm!mE4Hh(<_bU)i>YkU7(UCPe0Q*5&3giTR>u-B~UBGXg!KcCC=cxFn=^`H}xl* zL3I)2(=kvYgF)Nk?WYP`X{mEVeBbDJSChYUm0M##=}fZ$&cNbFk80#a@^eoh7U-e% z*Vo$Kks<#p>1q*b%<8!<(`-boD-ap2hB>T?;NPI#q~^(KV<|q+^lXrj8<0q7%8;%k zW2a_j7y!KR%x~pF;UAhPWPfmUswJeN1E4e=mA?&9xUrXySG&;ZtR;HlMFCXB2BIhh z`#GtiwyEfBHv}cOLw#0{$uB2oGO=t|9<(k8uOPZP!=OFzXv`vA2Uf&q+Ak7uVs?6o z7mh?@x7AbQ?$vCnAehQRT)Qe}+rlG^DnNL!BI&ko1wY# zUd@xrE;@pZaHcV2G3q?<-Ra`)*R1yg1l-;Iacg;r?tJ1W`kS!=$Eg`+JwWA9fN|pLG~M|fy#>U0 z3Anu$XO{SDeE-B~Du4I&#T4pGOmJ`e)-zOY9Jszw>9-Vr5xUMIg==F=dD=c0{rOq0 zK&7|PW=E>v<|TD@twPOYhcf|y5_x}%2aLIQ8Qgj!%NXjTL>+ZUn6{(>OX2A3D7&I7 zpI`6NgT}=LlFUzin_W1r_?QXA^IFMS+$w`0hrF4lwZ44cZ#SxRr&TWD zW_M~GHXs#r!GGO=iMdvInVC-QlXEe`1rJBwDK{G81JA#N5JX#VMZseEU-qrN)3p2@ zP8ABh$5RyF`Cb?iz6o>tPT=QS&#*o9geL7GtL1Hh$Z0ORPT?1FQwaBz1ceGRSfyXI z(kGb~S{%05=jZSkmx%?xnnw$Hn#GXmo}9NwUeozrvVZRU3n@nFUmHw5N872p%FMS& zBO{)IY}Sq4CfQW{nKsB?5lRC%eHH~>&MADLR7h2Af4D5kz+z$fI<-Vw1N169UKyy! zQU}NI^KDJ%2w2;k+e8>BHN3ar9jcWEdmGwdT4 z`U4~pfz2(!CBgTRluafvmIkbKEa# zjDND<-RiJr&DbsyyvCF-3S<>sT?zR%28BflXR+bUgkRihW_H(ld12h`OhOk$Y8`VB zVyA`j-a0x5b4oX(xVB+~-F1WtxfT@xZpS$)f#F3PaLTWqS}JU_G0=ttVv!hH&(g@1 zumCcq@M!4*KEU$qi-NfljXVc8@uSU+cz@BCqfujbsn-h3AmS-lMP|^tKe05wY3abE zRhcv2#9}sOR8Qtm5o0|o)C>vcT>n+93HK>e7C+ z9TnQT>Rv@4RW)&P_KTT(5iaq<`5Cq;PPWFZ)Kk@W@Qp$7k|mc zZy8sbJb!U~K1c3*+eH*-8t`v>S#Dp1bUrcw{FP)h=DZ9@BY))b&PhJ_X&gz35Ax?0?+$^md2v z8Ty;ZAvvv@2{UPYGxlN*?m8jpd03`>w1Vmre3Yr_RjV z%MjvDqyqm`D$o27(Zw$tLrWfw`;5FYfa5@YuatBYk|knEpHlK zU~X$kCOw<~_;3%4?Bs{XbFi87Y37utj`aIr+T zmYz8?>w6szHyNx#%_H(UEw_Na6uoEGIL$w+K2!uxiwL0lY&87qb$1d1WoM4}|6-0kO7YAH$4Uok(eM`=!zz7yq^!M3taex6a$L@+y1hxy=4eR8IX79({ZHdVW53;W~8Uje^Dn zy%58v+{pq#q$ks?C)t4|%$BOAh^G_^k5R1-ijhPU$JuEka7827Nf(J({UDqX!oLv? z^okg&THks0MXI5`V*zvzGN3EE|=#x5o}I zn_?T2Iu-kNPQ0F81GcgrWb@(-$(9qL+Dg5k-);wJQg8{uU=Kp%R+WIKjnePCX$FX z(gdaDp|i6F%2WqOg`+B%7cyp1d@F;HWEOyx7LYjzez@|x6SbilRX7bSeVGGnwlB%m z3Oh^12R@|;rO~`I(##leagiYg<>q?@`%S5S@vZWQ^nd+8Y<6=*WJ+f9g)H~19y-tN zGk!x4_+elmY{fO@)@Zumh+3Bziqp?xYbESicoujwqltp6VgPd-2H(FlDFK%%j2+`L zhas(`ebp2v?^5Ul5K}JFv;4UTL;V2gs}!rS0Z(ukHN?$lPgdAC0rDfNL71qJi6l@G z>v#?vLw|Gn5KrCB^G5(M)T=eZg%&72N2S?~>Avt8mZnN=(7@B6k;uhafXKf2F-0Fnc8@^0$5z zuYYUrgfPDNora(C0fkDZMUFnFGAeT92R@fD&5>!A(9K0rQis1I;(K01jZMtH5kK#n`> zN}p*PhlkcEwYg#2Q+AMWE)z8b3`C%Km&PuNOXO}rxp7dwGdHS-M397Ml_T4KC;ru$^482c>L0}a*yTuFv z7L?Nsp(>SnB)6#P08yPuy&#>AmOpn7D<{(|9mvHj{=zS~?cWra-O61V_ML)B>>{Nl zIGIJWVkH50JO)-oMIs~M4DDkc$q+|E+}hU3_BU z9zd=xPDGI_i`!+MY0U}}pP5x&O5W2Zv67aZ45VJ!qI9&nns;-?r@y@?pn2(3AaSqc zPY*7Hk*rJx8qWJ-|={`og`-_U82zYflVlBKp{`nNsd{NrWl4S zo+`CgV1KFv27+13V|G%g&^Vvl>gN;ru4&;~i&hI*K`C`nv+Ok@k{e<35?3O^)KAJ2 zii!-&mGVBgX4b%B>FM+_=R(@1+Wx*x0{qe^c3eowxpU2SfE!U$uEIcHd}xs@!=b|T$%~#PgkX+Rre7je|0hpUukp>fCHfVnUj1QtPmlrlbCjh3?EUV zA!_@?@WAkAzxZ~~Sb|=SW->M(Ay)-nE6q}zFpk-y>UD7hXV7s9RL@Z^zzm22;rfEOpj=m@itcC!UW<+&&FHs8zkfY7jM4>aw6A+cG>UG z)?hD9y||qcI_{RGHBPNbWan(O2GwvehqEp{e|+Cc2fH>*>Va?kCr*@jHVW_gBHlCa z$&(#1K4JaQblo?FzhgC;YPS%-_j7eAbC#igQW8qsxmRtq!XJ-5h72%J6*GX&0)KOxf%;I5 z3*{9+HZXM4!k*v8Az@X`mcw8iZ&-<~!$RuvBLHQ6kv~Jp%$L`^t|UYRB>;V!ehvC? zbdlwsZ-O0oh`?<%{B3}3gd}T+0z01XZYr0~Ck;i4z^Mf3jqT&twQpsFg4XxUORdNI z8V7N>X?y+RKzz|5`1SPsi+_?}v_N(RA`s=4VaYBHqMc+C;9+d&Q^p(9F4QM5Mj*~E zVw_9H-)Do$e+`yf9R79kR;+e|3A1E_Wf>K6geFX!(1|Zi)0x`j?dHM))FK4i{Mcyi z%pV>Kj0g3kb2A?U*`PZ2%WVp7wrGV5L}Ig+dk*|EBYUX<8(fHP-G7*F3QEmpgD_*r z0LtQh-CmzbsxlnvkD9aRn@vTw_HB7X2R>)LKxf{PBcx-ZAi2whq+#L|`{qgIvX)L8 zeeqEIsZO(L}>aMI(ggaC!a^me&2i+jg8Jb!IAp?fcx?1%x-^dR4POhw4Brs z*NQ~dPCzcZp@P)iI)7SRSDM+esny-}0Vcj-tX%8e1^c)!Er8O;afKQ*r>RoxM}sI^ zV1{U@S!b|3%1u2|23J*{W_k8)Ud#xA_^6dr_y7=-wjGnBZZh4FlJ+YvEPt6F67gBDYZfO)T*edf z0+R3G2_&NI*5owbMsVLT)F#R%RoWvF`-mRJ-*l1T(K&i3(^yB6`j{enV#HD9Wrm+ls2?|Vig~Vcm+qjIh!tuF(W%HEE*C?3#Zrw zkoQZmJ`YmO+kak#WM%U)J?1eDl&KJeJNmpDth&V=A%cfOxO5PvXa!APwo>^fAAhk;dIZ7<#Osk&g&eHt=Hmqi zFWHQBITo=>=wsZ`n#gGC&@`@gL1r2dE4Lp(CUU@G9BCpbPtvS7D8U8I@@%SzH4|$1-S4t#`X;2fRqZOl;=%$Hy6=Z1*`zPVpdap z#QZ3=7k@cfM@R}050l+e*0S3lIzDi>Ll^s-`h_NN!CisHbGi7`rxXTbEwjjh`Y7et zet&Nn9z|ubPxS4-nc_odB4kry84T^7Wf(uxx7q^_@KsR<3m^2Q(9dQKy=vf(%2lKj zLMG;3XYkh-MrG#i3t24)W#V$>{dzm1yLa6U5z!{HlnhKV0-5NkI~4m`0GN+ zEOj^KY2gj@#7!v#Yz3BCs?OcefEDTb@_+gkT}8R5fpkV&%>T z4=qUY*QekX{L6|1?}k_x;4IPF*@$vsXi1`I_o+jc9oXoVKxaF6k#g#3cF)gyDHz@g z+LhkQNNcU&IdQ}Iz!--Ic)rS9gyWXvi`ZgolkR*Y`DH+TJ;{lI<6T=0z&KgR%73VE zr}v8~o6;m>Mk0zDENCJXT1p#o*`8T4(e|z&5G!Jgk9{AfNhBX{h9}U~FfN!HjwXk^ z!IG!qkz3n9XLDQ*K{^<#Dt@tAtx!%;HlB1ggdD@(FtJ+pxv8v$E%Ux{<9tUsXJ z_V`tTE|(xLT-!4?Te=Gb@^l`{|tw{r@lfey2@9pewNCPN$~ zv89ubcxD4rzt=`m%*m(fJ~aNOq#EE;0k0KQZ#(Z%gF9j}3@!8nABZ0;JiiyX$3%o& z`XPdp+gxRZE1~IEF+(?I#i8I33ThXv8I$ADv!I?8{Yb!-4Hp=1Ab1z|vD}PS9_eMjaKqOFBV5(=szy4oFOg3f|rjjS(@l2+5FFwq$8Ym?{Mt z05n4B3p++!JpFYp^MBid1v<#eoL$$+wT5Q7N^L|6dF_S^2WkQbXTDhgT-BP`eI|Nr zJB{gNfq^Vyl5A{X7cF|N$m-={%kk{T1#mWq!Vj9&`%2uUj`#x*BMpOpSSsZ%ucE^c zjSizhZcHb4ii4mw3JG42H3#`1xfI`^I&O{qI1wT4Fe&}Ak$=w*^~~Ztn}lX0jw>zM zX3FzCH$onW_D#aY?9*>LxoWBbYixX&2_}NG{KX&f=v=So2AGql2oqnY{k$+Erg+e% z(rt-{E|GeKO~?-WIAN1E7)}ilAwfmKe&vfimPe!M4vL2gzxDLQC|N-`xHtM2wqL3~ z@Hs1G`D&2eZhz$Y9;p*@={U;MAiw4w5cbO;MRnHV&rSjXj#ebrSXkYej+LjOB0?!o zthGrX*-2;P%;I(f<++PBQ$i{}|NGaoe2{kaUn$>ttUBgyp_sptHa+wOJ|NpIPlS*v z!Inp8k8A|p=lN7Zw)j-=8$QOfL2q?5tAK-}S0(zx)_<3J>>8`h7o-t!CCh`RkQ4OW zlFCqF^mRrJcvk)rt901MmX!6UbwR(8(C%{S_nRw1*Us63&J17Y@h3OL>Z9e+eHU%r z7Z299ft{A!qaiB$L?}SKAB5?9^@BQ}<4K)B43lSU$4h6{6h|aKsG5G5xy^=WFelId=?H(+qyT0ia>z$_{RoH zG5La+y#v56|Bh;Cu&jp&R)+mwRT4jhEU(%(g`e>$o?fID)K11;6{zjS#p>@iYcJ5! zX;4~?UdzDiPPfA4zYyIe0{j%ev}748mChKy0Dl3=Aw4s(;zd4pta2VA9U_z#t!z@B zjUTghY{u|#Q+XAn%X0fpP5%n`R@&B{`jI_$kadEIFFG94uPDRoZGplOauXB8d&F$* zGDkJY<++ygo-grV`U4bPx-L}cf{JrE8DO=EJxe!H8fTnE_(1^^;+d08qDx6`y}@>I zV}G)+M(wmjVvE41)Z~^d9ccRTT2X;r6zQ|yr}Z1}D;oh2@k@23`lfcGG9qocFoxV~ zX+gS^h{wUSsXmTuhzE3TlY!Lsu{kNUh-4$>@b|orbO3pplnQCZCcK5M2FY5%t`2G#iyD7?FpIW51jK34a5D z(U;9iX)bZLk+!*`_53n?9))yBRrP$fZ!t{^v>H!2d6`eu;Nq>wkGB6=0mk(wee-iN z&cu2x2iK+~V2>wh)z#L8S`K7)?_`O+rh|MpXJ*NOdZ$ zz(*AyQE1_+Ml;?{Apo2Z3`8^cw7gW#bNgrS=1Y9fjbf~(K*`H8ra426>V)d0f*^eP zc@n7>E>P+Wc{gk$&dZcnOEj{N2gMMU^>{5dOS&4wUEnps?mV$v(c-AvyniY^ON$B< z&aNhH+`Y8&4V{EJ!2rfEoV)8Q-1==&^`WB>K9EOnjXzTbxnp42If+6j01f8-Cf(MS z2b*|onuQh6=eK%4@Vjn)ZVK9pzonMv%7GVjNRn^CwZq)z1w=z?MlKj z6q0kY-W}=bD?>VrNNT-kFMkOXyw;YRD}#;s8!nRKf2lbZ!ElhPbPR`(wUZa7G(_=+ zZY7|%0h!iSiEdev=wmhCwW~W7L6#RU8AdD;6Qji`vY)8m4JDyn-}k$d84TiAO{86? zHQ?UwvhLP+*w9i1W)jC*R6N;mGi8*|@E@@;F+!6fY0>%Y%~;f=KYs|t_)R~9%j7YU zavcq;IZ+Twp}g)ktA4x<$#XBDNn5rbeNX@~cr|^3iHJaV)-0>zCR3SBv+0GJSMb8e zvsOwtw5ohT?pk+3w`BX9m3fttH@@*h6%6@i|g#Ll$;H)xN1qq@55%8cSr_8@X z-4y6+!(`=`mIU0PkhP3Fyu&wb@F+f+%ogVEhMVRtbR%i<7nzg+%;O)c%W&yDb*EQ% z&tr9}IGW;XF(g;8ILBY?0ssz4auCx&*D-tH-$IZ@0rxQZgMS986j0P*=?>r~&5%Bb zvs9|MSaah;e4n1(?S2j_(vt|i;`uyIQkbzE15k=KBRVe*TKmxH)hFJsC1k1XgrVxU zT_QvVgwNwO#Y;O*adVGPu*tne+#s%qXbwa+vk0`^ zXdymP!a0!~sK{+O(5ErmVjz$*W&Ya?!!Un^J(B>={W!qW9S<=X1R!|MExjL)kcz;G zUFEW}7$19yLNxWE#~-zl+vW?w1*N@6F^dexlw6EJu2Ux0M-K zKSwhoavA%VYSob9ZzZirsDe7^JM2bN(Y25l6$!e%(6 z0@V$@)go!_=kJyoDZlb-^yN1`ui3SD}2w;*SG-pdNmXAx*TE1(yLE zyM%vsaD16&APwXJt_UJahg+}zVB_$D%+QI$-V^Ilv}+qf{Zojub)7O4Y0_LCg$aB~ ziDYgXtU-*1gx4!!0z?bVs6vL_8jKbPLi=H9nDZxkU$iJznD0azlQ{=Ax`lNIZZTJu zqjdmg=L!lqIis>6D|8*Z5P22#V(rbVY+$+hfM~l4*rtD`C$}osWo_m}5T+~U6D_ZM~*UC6h9?|R70Ur&)PaTtF-$~L73Dv z3MmVeXm1RT5rl;EK$1Y;=h0$>r*08+Mvo`TZh&IG(CPpL)*=R>6-gYnWf*_ZU{ts3 zmFI)#zQjS+Jp1sLubZbDUV&Tba%?6Z7*2Bdn>+F~iEYQ*w6AM;O&JE6VJU#k9fqz# zOc5$(^K#|Bizs89&UkWxai}=yh2x-PMq@4}$lGG>8NpFGmyK!G=iLS3I=6utTVXUB z{lJP8Y=+^ng4)3dci3f3I9PvWRLt5E^MwKEg_dGP z*MNO42GyyToYxMxR6khR~l9dn7P7?AEti43zK%S-IT_>H`|nds?=V; zuy!$s0k+Odhk%DWmn#?s+InXs|5^U26%7Kk%FTKVjrWVZcrb!G8F*`gWV}N(|id5gLThr00 zPQtT4Eb6697*q1lcdvgWDaLYDxzCvG(^{hU78du!MG>aQO^{!`vRbmW{Ftg1fp#+I zTJgm!5o+X<^9|V7bTgwx4AIZ7!#3JwC*{lqYxo#2oPXhf*X-|>4Ii%Em&54x(KM_6 zyVY_0$$Hz)LVSNNst4h!UB^p)^23o+`O$}jCg3ZME5?#_xqg3!z(+UUCujNqg9{^w z)U_LA#@7R+{(k$0Gy&E6)T?vZ_-TZHXVO?SJii(@NxZwKiw)lYz`*=MT!3bIbORIX zcApoG)aY-GLpk7VZYO0)u+kT(xJAQ@2~$!Wk?YNTf)|9#s?~Y9P9J~D_>6%8hs6ks z=TT5Q`g9x~+lGJdPYes0OpdU!+Wnt?c~9oDd!Rj|u5U-GcrnvTU2@@>vX)#g-^5pj$_nLH~&v>;fX z`|T7eVhJvtv#j0BMNIv8ONrpNWx(jHm9AnE^F?jWNi%;L0QQc2B;4zCNJU6Usqx0r z-~?D}-Plyt^rA`9yw&|!Z2>w0LXpTY+6dA>nqMKS4+Zis7h3$?wJ zH`>e&gk*o$mrTkNOFSH|cgqQBi_uhPvTlpN+dT<|b!69)$^55E?<#)Tfb$|x3g0%= z#&sjwEsK5&an3oyE~)f%*_571G>&;>?kNB^nI5NdK*R`O!+;O{T8R;aC6*Fd#F$n5 zfAQhT_lV1p7HT|>azFU4WP{%4i~VlAUF))aZI6HNEcCsWGfIDfRBpf9Ay{SrMMl|c zVw@HM=q=}wU6*()3QBy3OoTwGkWMTNQoZ~x*Y2KD>YeL4D~oevG9vPCx7>RQe_E4SYkk`gPOb%WD7*Jc&rrZ*m-+L+o4PAm%emK+ zVWNNa-B&a&F{aVgYA&z6aoVt~Uv4dFA)1-)J!iKo49j!oPc(MtsAWbS42ShjP0&^1X|M}Oy)yqC*zyJ96d#P*W-~aWW=RW>Nnf`2^)mY2>7CE;dj2OU=@~<3;s`; zvg8@lEgOGwKK6Twwhw&$5OX@4*PiO(HNL;+U1*AMhd$4r9QkwH`1;1ELC!IM`8H)0 zCvte&K(($-fRwz>9cd z%YmoDWH;5@$ABb*j7B2@vIuzE!$dtRwjB&o@n}-b^P!Hj(Xw#UT1!`2aN=Y$)g2!y zS-znAh%1!FTqAbF3?c~DUd#oU9PPZ~xgCdpd%i{}3glqdPFYmjH2xZGYAHTteZavh zBIJ+t>Lc6!)0fh0JlYELJWN? z@n;7{tKUhwPh(-c?j!smtArT8d2P0sx1_um0u3pw%*ZxF0|BXAB)*hm>kq9See30vQPq z2lpxIS%p7?ck6#Y*a!3zbdN$xa-l^7h7qpxzPp>HheK_kLvRqk6DCH028%B@W$F<; zt*f6py+XeI?LmkfI=Y4wsVr_h26%1$BW&`;`v@1q;V{UL1N$h_mU+tVm3jxKAkW2m0D2mDs zOY(h02wLOV1K#l84eS1%e+2WmAQBvdvjP5cSwPC!?}v%)AnTyThkqKd)vG(_xFUH~rN~u(9IqVD)L{*4 z$vhP;zfAPZ9zi}+-g&_XH5#`pP90g~Xi-a@X2oCH*doYVrNNyarDD$Z`FrRc38AR4 zX5i>>LmT$=4DdX^US8Y^>5As%nOBa?tZVt<6H0$Wd>u2lHsa{F^*v;U5bKS)vyve2 zgKb$vCNwkXrPfXW#DB9)k{@tfAv4BU{D}dV_ukZUC&qF;AiF;=Vo&MkuI1>&CM2htl;`$iFFWMsXvr@58-@ zrWuK)_cH5ImkP4mv@falMU1S1z-`G3&3k{OHHtSzOz1jvx;~+nbmpI1?OhZh9Zdq8 zAd}=wu4`o!S=#Av<^D$R7`-w}DNQWp_q$RyN@%_hOmP*7n&&xCr;$R~CRI3Jv(0~g z_-j?n7osF^0p)Q)UB1)ne_UNu9eO7B&-tUlsB~_6 z%L#ZpN)a8IV2jgY@G%(XyrwC~(GEaI&iQ#YHA@GOXC}UaKHrF3FYIFjiv6UHZhi-eirUF7fYhI;Gz-;MNMeL{$7Za%- zu9q?zf|Kf?uW3k@#N)soM&z;{2_Rn&Cb=F7skskPm|tJ)Ey$`=Y7N1CT7;F&%HPa<6Uyth;|wTDAJY z4|XI1D$X_m>^09oKZf80bDk45zmQa4J=?8d*WB6&<++J{0TkbsXc&R{B!|tZc)c-6aazy=QKp$PXDV5aidOq)2gM)sWkN9*us2E{SCyoQiNM%_~h$@Tm zx1Bs9*l3xgJ-5=8stN3AARK>ULQbuady{Lx%~jX?hbYYQ)%uAAjz`fKn>O`^@{RNy zv@ITAm1{2)H%)kXb^G<$j#7A3sgvO_wYsmr%^xCBLfrjbB}BXle}Gxvbc>qERlbwlDo*f0vaob zwZ|)rA7F8^3EFOvw-`i(Sh`Hk!hD4Y)Iop|8o@)$Aj0`mq06Gm;W4u+*-~x z2dZK-hx<5S4u}tMEuPCIM0&!0-dbj~;*F+sgrVtFYeSS98m-$f++37ce(4H8V}_W8 z(d%Znt*rq`4!M6t9<6onShy;%?q+@ITXZKO!cShpC1J}d3}X-7W(=MxI?M&`Q`UI* zQ(%?TKcB|oseOZ;(FjcNg1%uKq^dkVdchV=UvKM*LZi-&p8~hy3~B{#f-`f~0Nq4j zZ3m){4}NbcK*IynqhvUT@drYEEymrSQ?gsfBW~tnUekYZT`?o}GM@Wzu$c9Z5XG(7 z&U`e+zck&#qVv&!MK1T1OB%mkS2v@6`5c0GbFxlKT#rybC0&(dDS&oi*M&vBG{BKp zH|4=YfE^`!Xg&<7N^=t!Okk}~pcuyfrAY9bZ6|XkLKNT=MnJr>`wh>DJ38kBk42(1 zk}x(xACP~nEI9|YQ{{ox$OHmOnNZZpPskAYZSSjku}+Lm5UKLYv5#a9%O^(eQ1Z&G z{d@(zMx8u}j;PJ(<8g>MJ#VHa>w^-aZx(0zUG+74UM<7WI~Q>hgy8;(NpTP*sn3Bb zpQ2UIaIqpuORXA&z^0 zaMTez_+qK8co%2#XGMg#M(ir{td@I#2_j-TsyZHHE(!azfn+oe{s_AB~ z7R#r9h}(@L60ixvZ#v&T+u*I<*~gS>NSe)X>dQrZ$6&!hinEjoZlHyEh%$E=Mk z-@-Yj*L?}B4myoFqDJ*{I3T?=D64I8bbp%P_Y0r?OmdR&-28A6_kU zR#cM-kgh!XMs?W$LJ$N{^89(yzO!)4Z>Kg{^O1ey4lE>KznZct7T#XBMx51b$Jti4 z6q1ZcLV4%|^%NK8^xi8P;&v8D_0S%MzS!fzGf!~FEzBo}D2gk%^PBDrU%=h^%z=N{ z7%r5>+`{YAVsx1u#*v^Ln66XiRXKt@ztt(S- z^vpOz5O`ztTSqYwyZk8aK4P=5P#S-&sr12oGe(=OWfv9FO_OWatvL&q2%_gxTJqv_(TW=XS1z#`PU^Ejh_WBYr7EtLW(S9*ka74DwT zC=7QZ%Jy)9IRG?af7Ks_=^ka6e4qENcBk3U`mnkqa}@e9v|Wz6{$Sxv^O1kzG}W=J z>--x)a5JDecqLe2MxDLsef+zdxwgtXw2I47l6+Sq0;u4Rh|^IPh9#zm=dgw|N``K6 zU;{~ENZIySq9_OgFe`9xZf#)c$oX)k++QK)0V!YA zO4t%ap1=qOBxy*LI!i8;{f^ago)hlir{tj)4-^u6slxJS0@q#L`AeGsAAqwf0f+u=|Aqzy#E_uGw6gn}{BVa5ClI_i5G5_}U_9BJzX9=3y)JNl$NPWeoWXRVA^Vc4 zcqxhMM?OEOv!R)x=|ZPapiX9WJ6Z+xV_JKL5pk&{Pp@;gFl<$N7EBMKpGknq`zu&~ zX}^=RXZSUCLmPy^u&^R=&j4Hf(=5>}FT)#@j|zG)wlKArcku8r$}yMMjLG#73AXBa zJ2HNjckzM%FmH1)TsD8Dul)OZDm|cjLPRe2#FVgE!{OSPn+2mv?OzZTJW7VpV~?7& z5yW`4!914~!uumF+&gzh1fCQr$)F9D6-3dWY_sMdIL{|z0Y_}2Nsp-y?E(p-A?Q@^ z2DI|hx&?^44jH7DYeZ%&GNzk4^X*G6gsIWz2Ou{tAfh|Ii9CN*s=<7f1y>V`I-knb zIw-7%(WkM}W#}q#6Ud=2P{08)<9g8S;uk_-i`bg6$45M;p7!U^*=ARCb!}K)bZq9* zxP3V!lvqU%pLApD6(3HTLW8s$8n4$Q)ifXgc+(2w#K*O`j%~kcmRI|FPPPn_ei^%=$aSdbsq1#h6*GE*81$_7Ep3h_3C)m3yP@&ao_Qt%09PCs2tPTZi#tfN0+G2g&|*o z_&pLmxmb*zxdXmSus(WE9Ec;Z{+EGL1s{K4?ZFnM(3C)LBFZsO2*)#TfSV1YM6tT3 zUxaB1v_hqiZjYn(IA@{|@%R^z^p=A6a==l1kWitXXfzw`{=Z5@H@la0U<&V#sDeC+ z(Z1}}wUO!D7Sr@Cfd?;>T7oiS`_&G|gc5r`56dSn`&!$tisT*C74-2ST-EcQbD6QsdmNiBJKwgYW#Zv?PO4oJD~o zJM_FuQlpTWerMjOfW#~seOh-psLf}$#g!bNJ@7HEAX5Y&ETpU&rj z#TvD{ZK+0DwoX}{)ypxVhU1>H-!%hx=-FwEW#=9cpF47PRuQK#hHIU7w(h4 zIiOy&_+$#$e@UANYlJP|Ziu-w_;ry|v87&d#j-*!@mCtW;`*X74qd5eP78fArvA9f zP_k=DnI&*zKp?6=Pe>o&_2BZ2ku7|_CW5eJD93GJ8U+3CPnf%9-I7rOWVr{`y^%N3 z@O>N1V4D-ece48^ZOde`{hxn^onz%G>1)iBLs-uNWxH5t?KT2k|K1FP5@JW%&?wOj zj74>=?^sHhN{RY_2fz^iIa=(Vt~D_esc5|zx|wrbd+aH3Q!|lo>KE|4-cm*+_vuNb zXPcd@t7g?{zynd3t~n+xiN&d51)LQ|nU!ZOe{)Tkb^4*@>cp-ewS1$bJgy&)5h7 zkEqJ?hEXWz`7kBzv055vE zF+TUeGK_HIsTl7YHa=|xAi_>w+6pa}b1yVHSDezFKo0cYfm9jnH|2Wk*!nEIIL{6@ z=36=6{{GPczY`|vf_4b6)`A}@afMOy!fd|gsEwmGk!!aI!t{UbZA0uN+N3I7lQB=b zysL<6M5~!WwZ%0HY-~AFektEYe^&JKJw!n)Q6SBAE8e@HXZ|dLtB8K~&kgGJtE%8m zfeoMsQ7tGM7MqM~T^MsmG&x%ZBa!r`R_|?237KYOIIk=Hf|+QIr!O3MzxW*wmM1U# z=M|JhP#iP$L=b3>S7H%K4B*+Y!tiQ1JDvD@McqSFgF-mf(L6GP69+e;8U(KYP4Tx0bhs zt49|p5B;6`v(718=~TkpB*rjfvwvwq(=ogS@Voees354Bh4h+6Mf$T;lA`rRCXBwd z8@xv4Aj_wp0N7XP?R0U7Ve|H^8Nn(!azsO+z(xLC!j66=M0HGx^0eyAi#BllPAmf) z;G^Cp>X?6FJ{E1#(H9X*b7NZTe_Pq_u@v}SLH&ZaOf8^}DYs^&N z<_F1?uD@>S`C`~b8FkyyO$f?+3oj!jtLTTJM&C4H{I3xLOEPMP>Gumu@G3laoJ#09 z$+RmJ`k`I5$Nxzv_H7qD6dlDhL3;Z12688;hQo3Xj`g9;=B|}2NXx2 z(bli|#KXK!J8iSQn-+iuJ=$$MrPKzdjx{Mryu!_+he(IaklNR(c;jk|_lcT3K@hn)w()O``{1D}0uoDU^=RQ8zvWY>|%-rV}x&eg#bEh9Uh%Co>Q_+D#?yQ?BLsJRb)-*7{KRkd)p}1dWC-!?pDIObveIfZXUfKDGaI@sLb)!m(AHi ztB0mC=F`lAtp5BXa^xgH22$&x<}jmaO72Y3DeflJ?pSmCwYD6X_%WPzjPb+Ibn8Ny zts;#h`4vnmB-^&RzV^jhH)lEqFBn7gdXR$_1P7Nr3)Y7BQYp|1dX_GvQK!nJ_-N} z?`cetPRroqXBggOTl!uDjE4RgV9UY09r31`a9oR~7-9`bi?)r-eB z1K@kIqh_C`OV*?Lg(ifORsv@t4UX=3q`<+U6!9AbtpUVc!#-oZx$ABO6&AdQp#&T3 z1z=cuT;8iFWV>D^3mRRCn9O1S5-FT&%)31KOY|ebHbfuQU?b+$rd6qI_uPL`CLVJr zN`jx@7|JbuJFdQo zImM=lK%56bi6Mur3t{_3E_d5l+w#*77$DsOU7N>fPKBV7);Z(urSkCekM<#iR_J0) zg2X2n-piDKWlxErw{m+|@eqGl+RV-M6stx}3uC)9c3$%IUp*RBU02TRNRClc2L_b6 zPSZ9QW;nOzKbo&iVERenIw8X;BE65`agZ1hT3bnq#cE_BnSm)!%Maez33g7QmxmA0 z+tE2l#1C)PXpm3EXr+0PEm2Jj{NfA^A>_4qQpffC9XM{5Vz%&VLt*pjKNXd1b600LXG;W@CN60KD z?nx>iFx6J(u2Mo4?}mTZ|K;H_7$TB{IF~Y~e{z#qb>dg|UIV2k*Ht2l2Gd2%LY|iz z`U*gSZbkbmdpmI?X~N24BqKz+%6OmBV63zfChiqMf%1K_tSqi2wPl+*@7Z9@@RZ3@ z1S(EaToUA$Qb1GCe~oZ?lhYYRTce|uuXaAk8wnzU6eJm=&>?^Nre!uQ&nJ&56f5r*JpadBLBPPH_y$OasbKn|6BwU3DRLt{mTLYUM$#jV?w+hHeS2AZphMH= zaNnai#DD%;dKiCEaKPm4D|p2;XNmU-gS}y2^@NDb@g}T~UDt#!fhms&f_eZ^8Mpq_ zqCFL?;kW%UJ2+9|W{FqWTm|&1QEU3~_ZcQpJqq;#XzC!N7UoGY>nFFZZ@F*WqGyPD z6#Uk&zA}}oCScArd9QvO;q5f~w&(9EskAsR1RSTTtrmaCI+T4|h#|T-A$;#Ho)(^^ z;VZFhIizvdZ@J#wZl>zo^1r&TzzE=880`1ci3I|MO?CR0`U$`Q7chZb1wfXSHGUyA z?(di=cY2^eSMAPdy#mf&FlF$JgI5I%MEHc1wR~ly+DV(iF9NK;j`hypPj06@emTpr z#JQa5IbeSRn`VKE&HHy4k2FM?nTi;qvz6LBP@&jID3h(mY57zPex4zNNhV6}b{YDB z$b)+lI_@vqq91z!HdBZpr513(x1Jv#N`bL9F$arqJj9t&8;1RGswNgVy^dU;_o!euWliCU99`TFN z4gR%}4xUg=m1{x$s*(x>#X|shD)Y*lFPv*!Tw26tzIQYhC&d-v2L~0ExkRk+RC>=$}U*Z$^+WEu60NH<_T2XAx9pzwGWe-dZ1^aQG1_8f3t7GOt zc}{GV6Ai}lW|x?iQC9b5jh2Wzx_p{J3kh86_{IvI=m=bo&~4f92tg^K^tmv zwV6Q3Pws6xIkKi8(}*ukF5Y?BfN?H!uwVUh8k_^SDY$*;ssN|8arqiFm6u#eo}7AYwn08Hg$dc$3&&)RFt?z*GfKUWI+_ zrcd|l5Jc0H)fUwRd)YRq$0Rh&^Q$0*#-v{)hP_U=E{HY0Zp2x_wI+=-C&IX$X-oE* zmgfsgxx^dAH8zs}nM!PoN`QYaX*Zco$=ft4RIb^$oRovc1HM`Q{mS}U z#%5Q26K*2+M5jcv+4k%`^D99RxaEH6V#F6|2s1-@5?yFa1sm2U;7mhv;V69s}YMZ{&gI4c|zEW z%p7*GgVUy5Q1~4dCA;h3e%7>r>2>(rTes6G{Eg)futw01Pxth zY4}lUwwv#dK1+Z7S}1=fc2pN>ypGO>N2*3`M1Cvw1I+cTD*Aeca2Yovr7;p5E3elW zqN{`b#)u732?Z!qOi+5E86^XtjQe*cZ z+atM@9|wN{+|6*-;1SHDaJr-bzX#vC#tf1ejT(-qF1PC!sMAt_3XO8{V#9ZFMfdNj)}&8a;CX+EgRAuvfYS>+$h$iblIVZqaO-d3 z#V@9$YHKA85rIccOiL`yXhnG2S+CTRe$X{>i3K?nL3&u;=tyS|dYo5Wa_< zJsR$(O^vzmle6xDO_e z(S9LkpFeqNB$Q}GUN0yUTEuBdoTPLZOJ!?q51h2B;%3XyS%-uXNA)WT)z*YaV__3Y zI5hy@-DmE%6XbHz&!<-cEox6SHbyRqH?x0Z><63P3w6B)*>8m*V+U~`WA`oUzUurB zMe|Eg?e*m(r|30ZoN)*I(M6kM`c%1?*u%mVl5XNQoUI5MO_8 zOd&1q$%d2OcRV#sXi;^q72gg&Fn@jCFCSbANTZAx2Q%C)^9_s(4fnu~K~kdGP{)K*FfoGb9}p##It!JPJyr?`V^vNNZBWpopTyQ00}WGIMUQ zzQtb(_O7Z2hAG#JeX6t_cqo7L2RtMde#GLd`j2^mS-o5Rh*Eym3)0=dvpQd_4U+p; zaM50sCQ?Ywuxw6-U}eqGsk^77?`e`GYR7Y%r9DY(dNzjPiE5p!c4HIFFJ_l(&orA> zZ6=|VDxc!x;H7~jmTWI4*)I}1d(s}~5#byw z6Dx6{!||ojIp#tZQ&J%;I!N<<+P-R4WeE+a8FcxRQlRg!5ejNCh%m46rYyUD*3)KI zg8yyB|JYVcsC=cpJd=sC&rz>$WNV;_S9tb2EWUtiev=2+M5M#_c5E&D*9!$l6$ZdV z!Mb5nac6RH3T`N|nNELooDDWLPXdOaXch$w%bzf%hdH^ly6?&NU^tb#lU`N%h6V&n z4C=9TeI6cmSWNm;m@ZQUVSc3-JON<`#wmi}6EiPGa{9nRDN?G4Yh>ovHWehFCFzw4uJ$0Pj{o*0l>5#xbX&m?MsDLj7|*pJYwaGPj^?1HAV z-J_mUNz5>5CXZ!5smeAo<|HJdF5WjKuxk>lyXq=r^=MyH4+B*JPrN?D2wMVv-=c{c zw#-zlvh7sELhS?H0mn|4*;Pxvnb>-7rZh0kphMV zDU>3~sEN<|d7XcV*E$@JOkh)|YXo3nzBodsk0*S`dUm+$#qbeD)qqoIvKdsR)b+OC zb|cKE=h76krz<1RP_L;uwH=mfA}OwwxPSwTmezVe0k=oP+Q(%+JLY?Qq-zSDFIMXM zk|9))QYCiTJlJU&ycl`dXkjz>N(tjX9PX5g*vF;>r9yumAPNPDPlbd*FH}}|ffj}A{ne~Q;Gc!|i%^`5bk=N~tHK6+b zYag}|BC%z=EP5ysy(NdTvIr_-lVI^rHT~0u1PY59(ycz$fjhLf&*A8ZF$C6h9}L+eg05AI zt|ON5XsknYiwZiJ#||=**G3eWJW8U7A~}?e+bTj{nN{NaTIu&wkRG(C`uM$^>u&q} zRze_MQvd%XKx|Nf^|f2(mbZt`m9z>l@B9g;(kOp;nkMqmJI%-Fst~LQ0YmIv0sS@@ z;118uP?cY5;wvJAUh%Bpk{`eibHJynzA7A>)rrmkf1ir@4W|#T~EFU>{e*rt6!v4v;FTwZ%MFA?NXJmQEp3@J-A^SOO%XG@Sh_Hd^eG9%u%fJ;DOlhph5>^(%eM1Pbn(pX2j3`*-d-ssf8_J-S3 z&L_zcCbsl9hXIal^|Vu}A2zdFy8nM>Pnp#^bSK?LvX;4*5`&L+M_spZhHJr$l1Uu@ zD+cZtP)?A!!pn?&tYzXFH!FJsv7ibE>QyDOU=5rXknBiVsH?9m70)dz_a!#nXVlCU z))2bcL?XE5Gf38lxcC=*Wf~)X>1cyimv>mHLiMrw6lnFH^U236Qr%?n0fK*_KGhp5 zg=XzTn}h9!B^_K>Uq7tizfLdO(QrP&-v@94o%~Ve;eub*~87 zO4ITTXO3Qz5kvS{57@VH?x$QIkKpiP?PpagX^IFk^1F`*mF)l);4oQ+{{6NyHYCF#IGQLM1hVn5+#w=2zf?QG{B4y3NSL@!N0DID)*9 zw!hs-N^5=)CVWIb{`7yhsApP6WjsCt^*vt;qRPY8;By=qrN(hIZAazC^IMyW!0Z9S zZ8ZnN_1OGSKTFy~ti`2HM5{5Ja>_f8o#3{k$RY@phdCtSXK^Y`+EoPm-QGc(&OtYD z&aYJ)W_3BOG&S6dma@dF?`g8Wwza1+t|^!-nbWpGscA1^M3H~Oypp-i97rAsv`K^8 zsJ^MKlrnmKwcP(awEdmBBZb8S7k%RPsyH@bSf_K)&dqTFY}XwJy&wcBskMD zky@3Td0gGA`a4 zAx#%_iWh(KHe77re-pm)sjsjcDWPVeFc6R^ zAiPL2HRX`u$rQSGg(m`tmuF^=VVG=q&yGg~^;my?RoI1AW=?BBYZpLSiGv0EY+*Ql zEPa+}ZA%OwC1JO{!tklEvE(R`fSp<<^_`0Kn%Jv0x@BWQ*~`CYn*W4b86tn!sLIDC zpn8l?F`FExXDJT=_PpmJ`POvz>J4*|`nTBGMn1Ewr8I~u;OG~S^8HJEFB*oMSysP+ z>O+608c)dYG>t+?U4YkJhV~%-^^S;x=@&CI<^4%tN+uWx_b-ZDz`M zo+dkCxJ-32b9+C}0m`=ZeB}sXZV<`sCqsYxkeYo{%xm?es0v-V0~Avu=+6XK4FEz( zTDu?MQ))Sd@Y5Pbft9UuyIur9$VTk2K+7p76m3LM3aD+sIc%%Lp3gy2kcKAy6D^BY z0tY_|>PFnIc!o;gp4BR(k^s<@8%x@N>cz9{D88#%o@ZP$<#URP6lu>ad6kEd9&CSH zx#lcxtuA<&@_oiXG3gY%txuMN#Gy**JMz^kI)6CnGnc7bg?7yze@fk}^`7exE{$p( zTyKV`Rfh!(%;SKb8;sV%WP_0f%W*`y71iE!2htlHy`|{{tm*<4ci|qsV+QGp=?l-0 zt9`mv{2MJf)fp`Sh_5NoWBof`kx+jXG&dn(8^1ZB$lul)vF_?zY@U2Zy>D6l)}`l7 zh(UZXKxJHSzsd?jAM|!tj!$V=FAkU(Y(QgH(=9E)Y7RpIaPU*8>oo=6>@#W$<26v* zB2d$MrFI)B#)SpwR2n~I3T&}JDwwXtDCLS#sH3T13URS)ibQ(>-@Cb19KU~{-Sb6* zs}RlaK!f(5&K&S^oDC%Vk}g8%+T2P%S3S9*>!hu9XpSFgS&216p=S*%PZ1=hEn;A*12J|Oa1qR9K*~4x! z-3uTtL@2i{NEVzcbaNRE>r^H6U@}@v7g4~zxf9VQ&mey4!l0i(FUXhjZ3}5IV^z&o z!Z8X!g~I`T9SUtrgbsPSwyE@@PrSmeJ7(oPcz%mJk#gvMEi$;t-phYuI2G%ih7Nea z#95c{ITUz8x_Gq&?RO9yMx}mm-uX%uQ3Vo>Mx%8$xMjvWLJL5*5p|?*EE*x6<5d8 zRJZLtz4|B7tClTm2HbyV9CL%?+_iMZBf@J8_>CNNlgB&V#jmn_qAV-xZ60aHv!K)r z280RAh)8+GYI$#<*YLZ!uFB6=xN&Mh(nIp){SfIE?xa3{GH}6Fe~Rk>FF?@0`!3hn z_32{ap7ax|Pf{X%Voa;_)ONs-XVFp^-WW>tP`M$*&dBP+zOYyTjyHUNTVBG6UdBw7 z^zI|ve%s?3{t>Xblm{$uv{g+?vcj13Ez;@zX|`y!>Fd&sN*g1IFX++|Jc)|wUOo!L zLL%P}m>U+^gq=DSI=Aqg>bl$_9ZcqF?Qab1u*0Ahf&bD3-=`WD@`v|qd;HU~BE-a-X05qa`gQ)@6D zCrC%oNB4lVq7gi%wu|+_v$x3KClpo9PO1T|eOIk$-f|m%h2W&?^ZsW}rBkHT57)zO z9fIIHhf9T@Gcxj6x8oMe_}6->aIF(8{NeWLLE9Bc1I@hZDne?5NldULexFQQVl|}` zd}V(H#wZpN^%`)r3Vpfif!-8g_;auu?76S*@zdWqd`h;Zzych>6DpdPX?$uo(X^44UhMR=5h?X)K+A`=pKSR z%7up0+p31w*P=kZbe*l*Nz^U0>oU!QhG}385!TseoU(=^VMIOm z^tFfgGqeO=Ng1&hAVR+{^Ae9#c{r}@AS<^qdk7#aT(dRdqLqG{L+*>sQATA6fikwB z)oOk3@F2)ZQO8j#?x1qsD2$kXGCNYr=9x`@KX#TiJjTpxeX+wA__I?ENyQhg&WM2+ z!jyCsVq&%#;yuulq6xJR@7{eDSf zA!9jzkPB3@l|AF{hUX9OLc!Eo3lPnJLMa2Na01O=iE4lleshX+f0k~}>}7|FQf-Gm z+Tq#!S9ai))ICG1y{Bne^i-MkThpym$6dbZ};EuZ&_cc?0&4&}9 zmOWh2>Wj71FU7XHjtGwGU+}_{SpGw47NGK*V^dH?d+J z-~+|}eOyJp7@w_=!~}%nua#DSDwEat4;@s$s&U#v`2}xT7un@UgCE#k9{F*12DLN} z5ZSbo=lHTz+CN<^64tJh<-Q5_%7b*Wd+JP06QxU+as-tzc)Evx6N`U;(kg$*(i}!l zP#pRM+<=!zaSoI;B$N|R{Zt8#d~yR$c3D1k-BnBbLroliQ)zZ6AGa9-XEB`yTef1d z{IW70)1ZJY+dI6)t!Np-Bk`2~&ZZE*fX~8|q*VJxd2lz%15W0DxNH6zmw%uoI z)G~+V-(^)oCeLDL3KnKzSKjLQn?)PwZ^e?8iJxICxx7Li+-@C2Msq}oWNAfEKWvt04A=A#Z@JCTqg1waJ!B79aaAfu zC<@6Yf&m_#b@nzEKZM#nTIi;!SJv8up6V?=*KtiVW50NOHN2MeHXAAOTsV_ii^ZK~ zQZ@h#P}5-e-iG)~&oL6`(HE5*JtW*4ZcoEM7BpvnKS>NRNzVp0AP@}I8^NChZ<1Vi zfnNND8s}Hmuu;elwW8t?aY&QyD?%g($jS6;akBth6j-is3)#xyNdgB_BejUd%gv~} z)CnpuBZi_`>xq_~NlM^jq@pSs@#&hyZlkOVTOq#E@FaTS5Sh%vQiDO>>j+Y5+)+m) z6HS|c$esT`^_~d-)J=Gf{r9MRM6K(p03=&6z0mAaB*w5D=-mh#>AQyS1ZGPsX_HB# zxn-ek?$fAr40tK^gA+=yE?q|OL;1)03bAp4cI`iC&p5X?`C|(M}iwDBA>OnoX$?h+-5@)oDHs=*4K`YV9-*` z2*_yI@D4G`GOiD2tsBjmBS6(iA$0ybVDavv5@jn#HPFemSML72`j)GvfZNpU%_$EJ zfvM7`)-k30O+`VPB$Kxsbv+tZ;FOAgrQ3Oh|03I@vUeM0?yO~JA3`FD-cdKvkRC>X z3uhfz6hm61e5AKyi5p-SD9UZY@SChQI1lnil>7ise2++32z#w7}x%z8Xy*-#Kn5S47%9in*A!+qT<(AK`LR z{cDU;X+Gy`G3kA9{z}kGmfwmNe;RVWSo#ZlJbatv5tVA?27MZT`l>(c0mUT+K{t>E zyJJ#A5>ZRikNv`S6)XoMXYe$I7(szx`48fA@0wQb8V6w9zNMB=F?cftK2V!bsm+5* zYk~zjOsmiwkSY?yF#(ylxX8RlRq&`Lvwcl|=~_jZ=#AT!DXtrCey&t?dkygBi!W;K zfCj8d5&~@=G0wA|NoXknrF~f-v`kg9RghZsbXGBg;Af z&jD7pPIf%nB@O&b9Dc*nVx!eYTvSo){L1+P0MdTjoK@O`K93km-;EJ9_9B+hw_uJ_ z-S!lT&o|LCN;lX`UWZjAT#P$j^L=5B6Wn{f!68z1;c^>Tt!9!c0R`HNH0$07xrS@V zEz3CoDxc;1^l+|T>O~lTH-W3%AMXmeA<@bU++?dmRX?0^)FO;s9n;|oE-*I|#ZJ)- zbiISGi7GNn^tcfN;AV+pkl;>PBlSmP)QO4;*)S={M?$=W@laH$&3m%6k)Vq0c%vuI z^@8)x)9*;LVf<8B{}&j~NY+yCooW7z5hqa^J!ZJ||tJ zzGpNeeri<#0Qn>m7~xlsko&|gY-oS6Q%V>b;J4u*8Agmf z=Yo_&Mr8x>oclq4Dq^sINd6J86?dv`ifa5uU`JMuz5{Hq^N zl%|_m%Z4-zO;IDwT&N9KwOE<7Jyr2U15S_Q5Qv$4nP(;wBx#L_3%8U=*KFcJ01;4Y z-&&J^c3T&&71ym*c{C85h@Ot=UO*X_qBS*n69fe+GrBo{Er#ZbL9!wJ$E~0d9`)vf zcN9mR{o;MSD&M2hfa5B8H(F*8{v^I?}fDNlM2WbL~TkPp8or}loE)dr=}bl?p^ zS*$jC8Kd0n&C@o?c9Pf|x#h3}p*&V<`E1sQhZt^>5z!ucrSF~&(%qtS!K!nE zYNVs$=s&^#PIyD{^}=ZX;Dcd43#-6+@hgZ)>C1y3ivozFK0YTFxc1o7*w$))Wt#>U z>=T-Qd=sTCV#qIN2Eyz&W?ET9pfCoWV6{l`DL01B{M}|?B3*HwzBWi04+kbsI)`sF zd|W#@hqqX#$o?Hp+DQ}I6^+QUpDSiVS{|+S*G^;z#Dm%ry`$!R!clk-Zy3{fo9tRdFm3@Q&(@Kyi6>?Zy=RM6l}IRE!G7 z73D>~c-ELhuz3)nqX;j)sOe#Wupc2+BDN<-f5BJRnbHN%Qy70H~7)Y71QuE1R zykkgX95cuF9q>Zs#lvRi#378cwyNtyj*u<+YdYXt7X^GA`*UWkIStw;ejVKXf!Wf< z2&tZD%F;csklFYtV|+HI%`kAKfUfnLZz-qIqj^Co|1YCWX!sSB6p_6ytN#WR5VOASI+G2X0DL^h) zO4O3F=k@@DluRsDC3>H0fs?F%6Ij?kwY9C+8I1EbQaX&D%3Y}ie|4^&v)a4a7EE#U zOdCiS3H`f(XduY++f&=olzZDgV6+tpc)zHJR;H^ME~a#4;WMMuJg%0+w_^yUfWQr0 zd^NHsHfPwC0OESoPf55r5h1TT!Bd|INSkIL`oj5Z2+GM4m%lr$lh77_>rEec(y6f% znUxZ}t%|bHkt-J-jl|ur?ZFfWEzTD`e*S{lmefS)jLPw zZlxCSNDc&{1)TZd#8l##q$W5(6$Y3_l6rzm)o;d3n&zNJG{!n6ZdAhy?c@ONnuJ$S z-91P{gTb`<_Gi+AW0OjmQ9n;lXLL85QDQpVzTiSCqP{4gP>lD)>nVP|V{2=};f%A&-H<_fa;#Nnf1ani31H{AHKEit$;`fLF4|&F)&bwE# zMCr4dd1Nt67b354P7-EjK)yxNvJdWZc)&EnOu;2L5z214L7QKjSw6auv6m_uvWSNBy!* zQ-TJI#l)>_(|@Wy;~ugCrF74jtk$I4pycG;l(Yr){gfO8tlVRoZ;V~@*xET{R?M7S z1l>sVv^wt6gSwRqEeZBCi!AKwKy>onz(-%h>7_0}-DcZ*Op_yeJvsJ7V}-wE-+YN; zi`x?37PNJLGRR}tZu7jpg?Y!3E)Hsg3Lqz|c{hhDNIi*ZVU3WrJn*6#T=`Qkbt{@| zd`=O6O8Unb)S@3CRr4QH0Kz7*^;BYKT z0iPtU^d+ zWO%VcbktV_1BX-u*QHs7272)%?B~f)vII&&pku|RsSd3q_|TYY@fF77GKgQ4G)Un(V}TsOZuX}Mrx+? zk2oueATz%y<3nRy{k}Bw8tZb6m!=`gGmRs|&?NG?DKdL3ADF~#Q)Enyf1uh(8i zG47aF4PF*Ujd-c1CMT2>*rK(kTE}$~2L^AwsUs!xcY&cB_Q$@mK_BN61A$4p4zOB( z_~TAw(~1+#@zXt&bybKZT8Cd8YGO4-pRm+$;=ky(U9qJ?@KW zRkC$aaa3q?T0OnW(6EU<@}%%%zz7O|Mb306;BF+*C3!Ko2}xJ3GE*=_0Th7mO16!C z9*wb^T>33^&W)g5S%GtPW54h!uBFXruPwAT1k$ZbDPhzCXoAM79YMk{^;YkQE2*9+ z_i6cKK=K1H;!>a+0B=9V2iz~^52UKt%KSCLGtUfnk&b3G+EUG(yGjz{#Qu7J?8p*3 zSt-qOLE?OEg#eG|XFlu);MqbRcO&aO_<_;zdUeaafJloz3*o7dt35CBxRdxt;l+QQeuO1KXzX? zfHG~>RouqaS#~4PDWlw~7&r8PgKSEA_htN~{R)UI9`9@wS*LH0rId$f^rt&F5N}yA z^nt>-ng~-*dx|xX1X9*gT9@RF_JwuLpNYR#&;x=h|9K9MwZscd%3`a3aUZ;dW5k0P z57DfG%l`DGDT$vR;FSoeH|zKyRVkEUMU=F`Tj19y2uk;pF*{9ma)AgLs{15h!6F$? zW>M4|>7gnqk&L`$%DW%-$Z^d(jB@mI9@kGL&;I12g3!w^fGo}9VwPX%tdd=>0$v`fq1 z&$E-OHgY7Yz#=b@ddt}0w8`)IuDy6V>^t5HMHYY|Z)c;-NTeDY%J>OdHQBp5XUkoh z8ndw2i6zoEf9NlZ%r$R{$Z~i~8kkNEEvUdpnSBB&Lz}3hGrLWPY5A1N$gHkig@kl3G4>PUP49RhmcwS505Io+uiT zZ;+UDyzrFRU&!lQr&k?Or_mIdt?R%D(?GOysyR!91@Gu>egY0zoNk zWtlWhLVvGmA)u;_AEaFW*werqyx~zolecdv^~WSZJojjgPH@WAprrQt&cad%%OF#Y z=oSe{hhmY)sQMsReCsV>bR?^I)0*y_7*=t19uV-2>IANTjl`$2T^FO|^l*o|YJdDQ zN;Li+^fk5LaTg8D0_&H5*7W*QiB^4Qsrlso-Cp8%K4KDCqB#Z8KQ+?Szk|KJm;X06-8RTds;u!3Y ztC=RCJ61e@FQfPu(1C41B1nmf5*Lk`53CH2+lidKst_S~-Dz7dS=|@Wl670an0L{6FSDhE%PX6Pu zp<;US(Y?xK_6}jkgI0ECoFiq zZ%*~wLvR**YnI(sSZ#`_{ydm@@!s}XMj8RtIbZ2e_;RRl_g|$fUC%3kDyCJ4X<$h< z$2N3-Ecrp!ae;EE#Wj*(P-Mjao3--04KxSk5`5Ud2VnuQ6`ErcB`a0%a+~VSGj6?4 zXT!(#k?*AipB{pNXF8TgE*2@%XgD`*trD4cLsbdmP+q-_I>Hr`zGCf)rOlu)Iofh# z4d`-~CvD7^kJ7Ndwxx~x5$r`;kgJp2sdgKGNd)DaBczl3jN;?)-3eZ6+g#U;X5f7H zCkgXle1eDOdv48gCg>~!NUiaX#M8|pU_|gG*oR>j{k=AbjXD4Inxc>Iym7&mT}`PVL$nh8O7KjX)X;?u!>cRW7Mb zux!J3YA4{wU$bN6hkHOmQ3dTg8oS`6CxO>wD(f_%3g`Y-20p=s@(dWl3FovEygAY# zRz363rRL;oLCzDWZ5VtgvlflYN*nEE2kHGPeah1~t8eyAf>>`>J0An6o z-47o_1CW#h_yu#$7?_dj>h~PPaJ1po?D}`9C9j_wZr7Ja@#O^qka1aKB(w2bgef{=+cowj9_~^_+c^qYKtL=O%@;47Po0Uf2qsbVJxN;=yT;8 zE7-bIvmB7m9Ey~Y!IeLtlwn1Gr&gVq8zcPub>?&BU*9(pbIh2BO~ReKj&@TjKqD*% zY4X)=iAM~A@G%`5%<^$p;+M=qh~D;$^kD*UD|q_@zWz&Lije9=udE-W@- zyw>Wf$AR&Ej!m8F*7MXSdO0Kfi|lWpJ{*LP<;67IQk)@dcwyB53KjW(Sz#P;fyrOj zU#SOSv1!CONKOUGDTVkk!d!c<=e0jz`Q>=cOe2D7U_8^*)x^F3P`|Q=!TjMca#KmN z&ZA`NFnzm?tiyI%Z%$b!S-6g5Y75b%?qkQ1mLq3;^lNmk2n*vWT~8RVI7d zv82bYT;Vr!&1tqSCGH|D=^>5Wg|H+`y}X8_OM=9A{Ds0;)%M{{&=!~B(=%v#DmSf5Y1qE2w?b}k4*vKopQ2A-3o9R=ft)LdPh-arO zONk;zQK7(w#iJN%_jp#e%>=M#84nAa^#K*`5W)?b{RRmi^y}+r?NO4~hbz&#OHB4x zIsI5jph=N8-F{;wx#LIVQ`gfIG7K0wFw*Z~=EKA_Z<&RE>&8^3FuzmnUHPR;qT44~ zrRfI(er2S>kQ;4~0Ph(^4NA`(#RLpW-)CY&%=!k}q8(!tFmSUMHXB@soda!(xC~0M z>!?#V@ap(Vh5rmf)@-4pKb?SnK$n%*UIlQ5Jk0MHft6nZ@?55K3+cdp6;}{U)c?dJ z>=MzZQQYu<--xLNFlSgOFhzQrLu#CX$ZP6YfLCkwT~l|wV6_(<_Zscie#%+(btR>@ zDy*B*LJzjR-~zEw9&ItmR1=AdU0_1*Jv;I~f=e&7G@A~`X}8hrPu<(k`#6mEB~1Dl6~1}-(5niWewUw|L;&M|@s(C=aP-N|Ry2E%R z$OoubD86K7LyoPHc<@r+m4p!Z6rb~b(4rJqcuAgvkTb9hLZH0pQqWei_p%3|xs?W! z%flppBTmK;FWvnY%Cxun&kbTQny6nr+!0ZcCy458GenJOatJC?YztBo#6p$yyQXWu zZr}LOxM-n_VtiJ3`C`UZN(z1ZtsFko`7y3zOgzHP*E*1_erDTGap7gg1&-ejkDg=% z27t}uO!&mFx^AY2l5CQGw~5SbYEd?o_=x|17VDyVp_Y~L8k)+d$@RF%FHVrCKgyiz z*)F!*eEyK^6a?-1Yhhyv@lT!vj*%(_aKPHBPRP7-QCZLB)Y-N+S3+^T!H}zPS^mUD zW8K_6$%M%_P?pS^==q8D{aZA~p)o0h0Uc!)Ai-~mFnn|^Ak#)!5=}27;p>u9XX!V8 z1jQkFh}p$#cz1;Hoq&CzsjUZqGN|f4AON8YbL%E0%1qeZaKInX@*{qb>WwcX*IN zJZ)?xhna^)HgCyD_eL?{yB*wla?ci<>s2lu9gS6X?W6d?*s9lT5%ypJAK2oUE& z7R|Z5HSKr>&MMi});=8GCs8bJqjs(65hX{y3lnMIUwAdwb+uF?yN~*851bWT&p9jc zZI{Fr%zzd}lM6%*@l=1*3achVhQ&Xb6WHd<&okmATo6AGv@uHJ)XWP}F{N!*Ik$%d zt@lZO%NP!6B3)WH*ne_sFVS6p%Jp8gOb?nHQ<))d=1xxIFK>gj&5aLP&#)Z}M)8{4 z*Wy7RGHZFLI6pHErtjBTrVA@tY~93M?Dq7Jc6BOD75MD7Eg|4q*`_v;hg{e^S)X~g zzsnDpE35;4qQnX9GpFIrNy65kAQ)eP>ZqZNO$#C0leQPzu5>5dMEXpBuPufaCr0Vc zrxMZIL;SQzjQ|po`PxB_&+FH2ZrT8F0q3n4;p_q9+)~P+!Rds4_8rL^aaxif)j2|T zHg^>Wx;<|5ZB2=0-3J|8aL?EJ8T;}X+iO{>qGC?awOuQ}n3x`ml^x$k$?scBAE3x^kHtjEnexI??b+T+DdLkBWP;$= zRa|ubD7|pw8P_%_5i_8-&XTTIuVfteW{S-Al5)b25pfhfm!yzTyA}h>r=*){0hvV2 zk*0|(NX)%U*nOUG5`6sveJr$-M--MCY5(l=>iHiUka7`!nkUoUDvfKI++Xc_FEqA~ zPi@>=o5mDs$`tqJI8Ccz4=O!H#UE=Ph|GvpsU6;yq1Q}1^RY?xY`O}V1OAr?G~<61 z4&Ji*9#8dR_DcoR^R5MD7flOMaLC0bX&%ZE7gEUqk)oyzsOB@dYN`eflU`UrRX*SzK0t1; z5d~;a_hDJO`19&t)!1`nG$_PclEj>GTh*WVgQcWibCmg#Q|-_6kifo>ch!CVz7HA| z(2JusZvdUQ8GkO?b-P>>3+(i8vWUSNEFSH>>jxBRVHQAh2KJweTK?k z9ABqZb=^dM?=Yjij)F7+Q%;LddL0&TkS4ydrk~%vbk$OW1lvk;p}DB3Q&sMK%0%bs zu0Jg1_SQYT=6LKRs`QLO^s9*~a6^=G7s*IpE3COFGS5LPdjw+AjVfa}rK-vs1!<2I zqS@_#G3zt`6nyA}m>@~A$4W(90$U#T2%iVM%Vm3DW<&;5#8A13Q% zZajyWR)(nlYUxOlE!8c`V;F`{UrD`#1TGL5K-e%gWlWKmyb_mru5p~KIO;u!=_uLX zF>(XIsZ_T}C7&k7>x3IFD)7DIxEGxxo5LVnK{#qBU@sq=b zfi&i!>~d^qcK3$i_4&jg&#e)#ab>47l6FlXQTh7Z-}uo{axZNwjeY8Oy7q376~cSCb^2tA zq6%i#hv>VYJnyRH$}it}&dk0YP{%IjX2XODp5>Yb)@etVOfLNz*6nM2(bNyAN6*4` zj4O05`W3B;V|YMf<8godF@X|)UbkaNlz4{rB<-b;RV>kmY%MytEOyl4C&@{7Std+_ zd**F1;gmvEh_U_?Y?eWbK0ga44#(!VL>5u$#fz_mOOz(*UbYjSMAmFUXrOYl?P)lB zro{;&D1wp4);(J_CDj(YIB#DZ$+t_5@_v93 z>uE9gS)JSQz!oiT?fwQqTJQRH*@4xP9(ZNVze#@hB7*_SQ)yUPw_-$=LsMJh7l5D` zlxC-Y;QWa)lZEJn9w(@Lz-kJk{Ivn>uT9h#AV;UO>ORYV&m;`D6IhQ?0(WbL(Ib+Q z7b?PJG~%%<(FFvLF^1C3(8B@D=h5PttG0@96V|+@QSh(5oVMd=KqhIH9Ysprrc%&+ z{Gr)&ASrg;wGW!1kgIymCJy7WLC|$yWn}qC!3hcWML9~=Au|vM0uSV2=I-j9oe3S$ zR;d~kcHX9c3at4c>RAa4p8^IZlT`4Ogw(pWy_-ZFFFEEh2ftG3+Zge9zR{zl%d>#7 zcaf+Mp&=eh9^Nc}Mm_Ve0?MPQtUA-GRdmc~C`Xl87Acxetqv_`#&R`2vWqU*svkAZ=0nMwYGY5fHx?7{^LtN!@e3i;g%IQuGBex-2TBG!3A3v4 zwPb7+aV2QiY0U&E6)wmiKGSwSE@qF zePQo^+jC`d=38jLxAOxh4lrawZX4Qxe?>KAz!JhoO6C=1HBFi*YQc;oaTV0#T2*Rzhb?@4L0#~ghd?*Rpf-g-Q5<|kBIHSo$d-S_n zajOAcxrm$q2W{rLY@09JXcq2HMAS%wn8R+J9{cVDNMPaEn7w?kvKo>U=|dWu3h6ys zLR88Re&itLjZNnEB4W!E6wP!=$4d8sX;ozMY%0EK-u{cAfuD@%kdF1U(|{#k(5qm7 zr$YPtGnTjz=%@h!@zd9}W=*&WnDm}n9SOHBBshFGN`rTZJJ&g?5@P%H(_+BEckL~7 z-z0ugduja2*p0jakgsF`&n?~Idw`=^-RRoQGyoM+g|aeJ=d^8w^m3jopozxsSeI%D zlQ0rJSuYk|rAzOy!HV`Pa=GGb%f7^aB>nsZ@q!MLzS=sMAm!{3@#8V6ExB1TAqPSA z>*G?vqh8#_@nNcrvkZOjl{*tU?8DA(Gd(PD7ElC9vA2#U4!h~`B-wsm7p3ax z^lT)%S9zg2Kvn7K5nY|R%09E*G+!x_ow#c*=lIO~CRfjSY1!g!GPr4*(3bOmO6MdG zx?!Ks+Av&iK}JT3xP3&qQ|>m}76v)wCeFGFphxnQIYhWWUkC{y{5}3kW@sYqI4yvk zDITOHj=x;e5c<+z62d&Kill|6=L$=wBU_isS;t}%e8Z#TJioI^PY4b4xAdEuvw`%(>eWYldk#2xASMcE1l>TAF0b+)yaGq(iGW-f{?axx7(UH~oh;`w@sXG<8adA_5@xUaufz4R1GB zArSWmrm(n&GMgWN!I%bIK|voB8R!*f?vLz3x=Kc>vEAO19`u435yv*&M8sTjFhq|A zp;oM`)V*U%(O9J$)N*ZmfsN|iT=R;b76w&Gl&vwf$dkFPpg)`hX_}(^g6X%Z!|;&q zX!^j1ZY!_3jQeuQM+=d|@*8kaDoEmKSm!$vIoC#_#o{1;XITq`$>jt=T_h_3zwNt; z>V4~wcZwf+DFjOiTF&fJ9ORH$-d%(&CT3C7^;8~L@TS4*cGY4Ag1POLs`1+$CoG(p z=ck;!C=PH7jA>`}473-RgC~~&HNB(30*|8X(xnQuK z%tw8-{6TWC!}sy=F?53g53y}#9oN@3mP~^UA65o`h&*JSHh+^;Wny{tU-I17?^y~m zfTzTj7c(x2a!;If(Ae@qTcxv0arg1{dqh-DJpe@^OQ`iCG=1&jazeb)E#BK3FjwFr zNtJg6WdgS!=hJo6;g|+yeI@Tpk*haSlWe`mucmjH9KHB+MTs)?_RI(+YKckw2Mprt z+iOdIOQcxyy_i(3AWcygzp||Mn5RXWySYcQNCxGQl~;CcXo_kYfP3EDnRbvk z6s$M_5$)l7Lb+AhDOTtVKdFSsj1R&M;z46{L%qUNdL`*?otZnU9h!ofri$UL*>H33 z{cV{pD2=!hn`Kp1gxtCXR{2rcN%DY`pu*aJGJzB&80}EQvBF!@Eb>3A%=bnbH_yW> zmi|@}8&I~_%Pq6p7OtM%e21u67DKp$&5BHRZQ)|?QE9LxsKpvK9%fB8#LTiiPzOi% z{LK~a0qPR8nl+Bd(h}*%cN3u@YQz&3z}QOmrDUmsv`yp0degXFTtN5$ohIKpB`kM; z``QOq(>_FI3(#xS=8)k5LYhSj^HP3B0GrD@)xUj;VVGI-y*dr!`6kdd(F>T&?y|~) zD7=mr8wX4k0>xmHFED_S<+3=k^V^Y37|yGe1;67XoQD&YZN`(uMf>ndr@goB>xrid zyn|Nhr6LIW47S<{P88?wXbZ0jDc0tH&16j(1Q_>d*R^~HPkqyuai{|8ns9Ol5vhQ5 z^$n%oFGK#=<6YsghI+LWUg=yzvKtAa7VU>x-;U>L&MobWX_i>zd9^KtABN!8p*s4J zYRB6#oiqx)7R)YWKshJq0ZxM9F{Kl|vje}8Za-Ab6-_Fu6tJ$+^>^WMKc@_TNF2jh z0fEohYl^PnYU&P@LYm7b7+F;}n$uR{Y{u_xq+GT%^H>={3rC1-?yx4@|F={!cZ4 z6dn5Xlpp^-tsghiGOnWXfx~J`4P%wSREx_J#$GDNp=;$D{3>OHkJPhROe1+>ag4i~ zd8DJ1R?U8?vo8*c*1)MCuM>e{>yx2JMLxxx8BDM|Ve%y~V&uPvB$sCug3r~Ac26na z^HEg`3fgMi7tm_<43zbM7Sm`tV}G+m@UHI1Vj$+voWJcbyTeG#f5!bC62qB)Ua2vI z`Jb^0Bh-xd=QlTaK0KQrEsjjIxtS|ISRA{RCxZR2b^Ba@@~Nb2rdn#1w0Krf zy6Y@?E>&jlJZs)hwtt1^BAwj!<dhuc>MqO7cSl>lx-6N&sBc)UVkcsL+bpCe!SAkZfuYV12JMmoqz;$uB?sP)eC>w< zV;4We=eDlZtwl1m@?{%^@lMZ+KdR&U!q;*FK zDXs(*=i)EE8R6gP{flk02=e%*DB59c2B?|;XY`Q)hSM($u>YF^|E~WM07E9kH=QqY z*j7YwRR4_6KW^d8FP{Tz`2+LO|vi9*ne;p zF*%d91tV9O^jx*n49aOtNU$MlVAQYvicS37KFa>A#fOFc{CUr(9zyw$-_zGo;q7-1 z3OH1c|FHIUP0BJ|m*8KS=%eniBSgionWLRa0tqIPe5iP)s0pH?q9*X`Claf!`@Wy2 z-+sG)d*+~`ECsd5z1LoQ?Ufpub%vM8gxyE8P#-Sc<8dba#m)VoRWiOFM0>#7-tm|* zX^UY!YOcQ4j34uX-kLZ)>^_M#mFH6v?RvYDMLu&kY_CzaZmt_IBxS8hi;(K|se0Ud zHp}3)&kfg(U+Xm1`802KTdTNV4zC>d_qHZ~vf@@1hvK>r&LyIdj%s((j*f?*wCBhh z19v_+eaT2m4`aXaCX;vnL`NTM*d*rI6<5CBX;*5If%Q7whV{tky{hwZf2N5$e>P%d z!_-dhZXPZ|%bQ&ejW}JL$>}i?D7Rnl-eubhHgtvk^zQ8sTfOn_zA?7v;?rD4nw{l; zud6(~18lgX!D%x5B>sLI^)v0dKh<|*?oN12o<|sx@%v*zYz`~4|4L={=rMVs*6TW1 z93JNJ(wm-6NA1kL+V)Rc{~dgq_uTS7ua_ea?B17tlC7(q1OWTIONOXOEY$0#Jx-H* zG=IGZ{aHHp5|Tfba07QSK?4nr-u;|^Sc^#;y;Tox<Vg;34&YkL1Mf zVoujuF}5c=+;eH-z9!=ByY790@jAb(@~WSS+5GMgB|8}9p+5N#mpCf!`#Qb=Pe8E0 zvL|0T_az>Rda;XFFXuYc0)4{CsdP*&i<0E1VL#Cc%^h-Z?I$HYfMipZ5pRG@eDH!S)o4dy7+_vi{6=j`hbIFRFevj2hGFd%9h{ z&&M$Ib$gm#Mz3};oDyR%uB%@Bf7m#;MS8t0>-LgV+tgSaZnA{eO<#uT#qlI9r?!#p zukZfl>fg_!@nbOFuO{1|x66)eG^_8~bbDEgDCn`-9%&C;S^l>WOJJC;LVHMXM7#R zQnU+LrUvH^0`-3SK^%QfXLPgu32K#yWI)-SsDoV(L z-ToF5x{1o=Rg{^J&!R0Ze~Wc7zM3cAzc-&1T|cXx%nVN}R&(tRuAt1MF z6s){r@|eQ7zVw=UtUZbzL9ZGaxjs&g1?&@V_AbrmU_!Q;AR@k0`X0T@!?}+4b+*+% z!*TYW)sxqBKcOdmvM}v)&((9F!;1x3rjZTj=ml5*0L;$iqZ9?5)1saqtB>u!FZpK_ z-l68<@{+leFu9h_e@L_UX9&vebNk-3UjH&)gza)|MKK{Bdv6!p{tj>C zb`7=|KE}Rx_w>njlP69e!+J0Z@c5aw^VdDBP`2Uc77b>B>zR0uK6OP9TD32ikw*@` zAR~Agp16!i{jb*(r?h}#qe+-kA&hRp23+2+2$x|8)< ze&XW%Ao{f3mxqBqe9W9+_l`{r%k`aaoe`%g7dqeb@zT8=r~bt$_gdhyFyQ^0Blpgv z>J9tDbQs@@f6XL)JU$N+u*ps9U$;$R7Rft$3B;9geI2sTY=6i4aEFN7jKc%7d*(o| zg+4vy%n7g6t4-RazTwV1CYc*J^XT9lr}yn;$|c#+Di3>J@>rcMvfn*-WtgYg_}Gq@ zVV2HpCw`xcEN{w@N8BXiXMpkJD`xINMdvf9lVlSJ|V_?4CWnayhzSN1o_) zmX4ZB@oW}W?|ElNxFXkJ91K?%^L$*z#iP1#e9DBh&b$k_{>y}%`gL6Xr3?|Pf9L=-Ki&LW?$$7Xw z%obKke{ZttYmvStdgR}?edi{ttrVw5-{Jcv5O#BMk1Zo@xGMMg#o7A?YxDbTyJY8g z+4Id9qQ(1qJuj2OpXZ;s0o(r8JLY3QD5^Bd^!crLIKlkXZYkM}J=ed5_xTOqC$~#9 z-<0pq`Ih#atQZb9uH3xB9`3i>_1J8i))mble>)M>^3Nmandf(sp4?*Rjg$1Y_$*C6 zI?&Hyyt`ZFU`J3CgyDYnelD%4doNq?8|Y~cdT(dCd}OQqNX%fp$&C@*2gk?Z-RzM} z>Cogf%$jX4Sm?ffj9`DlA30w3eP6CEG*9h!aSc;uR|Nj`RH zn=9|pf+Xi*-VTb}@@_32HVclY=i4u?ek$YT;%=TR1a|$n}$1{gw_MMxLQv?9DUAe|fvwolcLj)!RI2v57YC(R%FIr_YKw`{(9) zM}6IC%#rn}ulEH{P%@e&`{8TW93(wnO2=kaw%N?l& z`5XtDRwsH%M_acD=4OTcoO=XqT|?m(0!mq;qzw-F zjqDg9hI75a%H)jh!~6ZR%EMx1&elcio^4MP*In8Ao;EJl6Vfc63(u%_oL5XvWOBz6 z-@Vt+_0S`@OQZCL7Mho<;ouQ0BJ*9f``w4U2)pf^)AB_Qx8Wmg?W3rlf5m*MyOx($ zBbN@3i`hLl_787uZwKU|LWc^m^$e)t1cwUY1{^RxfKdn$Mn_SWh>E`TGah zuX#Cis`oVMMPvUI&0zEFJ#62Sfv@T4GMTx~E1tTum9{&6G}|9d?3>-*Z?;`yV>%IN zJ`c)++uPghvPxF{Q-3_1e~@0}i*59b*5m3Hz4!f<9Dfd%&8ad1hoM0AqvwrN@eyVS zoBAd#fV0vF;}^-ZDDbv}6$baN=~e=K=NPHXr{SnZkGi1cjCc$Kh z`E3@oK94fIN6+TI+q%QWZ5%EG{=u{Ndwj0MG3HRMr;e=;?oTZne<$;6VP`=Lqj)=X zNb{IOwC*e0+h=e#KzZqH(oxU&*(hzWRXfA;iuFUDw46q#v#G zZ9x26-%hv*#wbT?`t6^tNaz+XlPkI=R+OOy-9TiKHWdOb12iO(ouJ-RONvPvNX+in zWQFoAwVHKsO)YONfA&UFq^rr*VF`1cw=a`uY`SPw75UQR+r0AjAJd!6nZa7tn}>2% z&w8Wg!mQ$N6^=f4%_8_Ul@bP|^cxn!q)+=)}Uw;OcoHfD=S8+O=1e?jx z6@f1-t2#OJj|Khb$UQA(olTN^d)_el@wr}2i{)?WSM5e0f3AMWz+xt`}dpobni8mQP zP(IiB_5>nc`OFXBPa_)(`Pqn>3(Lo$KMhmOdl=U1fBEfaD;cj4NC58H{a4R^PjBAH z?oE$*P@xBbOHN_LS#K8T(BHIa+)L7o10^_MSy)OPZ`~qvSQhH~3?+})HJ4dfY7=Ao z&~~1mX#$N#zEc&D9*x80@@hxJ;q}-DsZYiWIG@kY>l=B?<$JBMnLZg$b)>v}kus%MzqKBA1% z4}FMR^{WNzHIV--wn@? z$>Eb&S+TdA$M!nTcI(wE8NP1Qo>7jMZ9aMLf8YA?c9|ai=eU{iyb8k$MmnRi6U+vW z`ZS#GUdQ=+xb5FanQs=WN4=|tQ{$eG_J?=7t9pGXL*I_>a+3C@_#S>X`&Tnbdwo%f@NP;*rYaAE%U2xehH-#C5tBbjc2M0dz7HjRWi3W4f zf7m&xy@-vji&QtyIhzt{WLf*tuZzWDAK-nNZH!xv(-7N(*;0LI-Za?Or_*OOT++*} zx12OMAuWRT)&pNOxW7mghPzZf11^rxzsiHWoFuPVwj?)e2WL$YUq&pV(p{* zZHtz}kEz?Z?;qyJ&tNiq%5b+6d##26nvby9ib*YnZ8jgK$X zZ&yN=(sbK>^)~UoB-X+ZC(SZYuI^t>-5ai-fBtr-+wHa)PKG<%>eaIa3u%6;$IGL6 z&e*A5@9W&)r?*^=$IIbt^yqzVf3Np64oBYTc`)1eXgJ=*sGgkHvv=Ql?Y4*gVu{`? zo{TIqJKd_?JENzWURFmSfp=Y)=auI=n?@Ji|9|!3Nm(87JeePA%ZD7)|JUd-9SWYz z5N-BsZR^O_yc2Mx7u)^51pmMNU#IeU-M#+nf05Yzl;&k87)vCg0sF9G2Vt%Oc$pF_AhEnxvN5LhqGou)xe&pL2x=HIwShBkGz>?AEe&q>#{QIR83b$iv;= z-}fL>%RHywc*)q$vFZ1*HXV8*Zn`+i+1U)zwFyh@Qzr(#COJ7*cPuq6kR=BGdar{? zTLJY9AU2k9iJ`#2iD3q%1u|YBNtlq#gtcZPPBG$#e;4rBfFLOyE`3KzgtWOOHU&eR znl5GtBym8Pri0)X8QKDIP8W@4_$$BHv*)%IvjljFACe*jl=FstT&-Fm$%C+Ej+Moh zPi9L51c8+5Jzyl$074k@i^pP(jwp~7mg;eB(SO=2Gb;lF_yT0fu^>JK;1VbetUk^% zDA$Ygf5E|m9#8fQVylWl#?RG&+<|xRH}ZmWZ_w7v5DZ~FIIp7w8AX;|)i;DCN86TQ z0N(ef-2FhMhSO*TrUd*vq^n|J*m7pDyo!jy8m~=;T~23-t>S|=UGDzGh{|E6;-u;F z_M8knBWb#r3b=)2l6S{;KP*QwQTL6!@9mX|f6}#?edET9_{hs+>$~=+wG0aCZy>`` zI0k3-;(Db~VTL;~vsQ!9F+_|ABx+aN#R%xa5ZnY#Oa!_Ef_rO&moyOTML|4lP!o81 zx>bpR1jPte!l`2C;Ub8) zfANKF5$6sP)pH%B0S=sJj7N7Am|Qz%E^rkK!D^Unw@m6b1YlDRfAF*t?h7mlLf{>^ zNr$K9{k_UYuTLn5!qdn*S9+JLEmJwGu4i}sQO@$;>&4ncnJfNB`Pr?H*(M{}(3Lu$ z1fb53S<9|YlLQ3FPp)-qVH<+RrlN-DfA5;3R$;4Jw{I?5a`{v^2ZEsP_rKoe^l!KS zN00yg#~fQg4dCW5G!Lg>SRR+<102CE1B8t-J?PHO$PtN@8Xh-rRE!Z&41$tnRH*z# zG#&?7=$2E{@9PV)SZMHe!>W~L%`u`*EZrvvVnX07R%3if3cf)?tq-RvKR}c*e}v?( zT;wFif_Ay+3C&ZuZ1~f6|2<<^78qyX0-qt`Pxa$=7C^?-|76-xJSh6p55;M5sb1b_ z7+MsljW3TzlVAd<((d#XUZ;R5P#rE39{}cHIL)@8JvZNsn}tZsZ#_-Ax=BO0(K8wR z1C!6G<{8Nh;Y9Poo2d-b2kss>U$s^leTl ze5U9AYi0mX81U3ZA%x#`rULQO1}+yMX$c}VesXyMB8&V$YA)vtN#gC$e~b}P5fulx zrfY(?4px}TC0Rr~bq7gmivklKyZx&+@OXkjBWv2!XQgZf0r0%u?>RT3nViKQ3@%;Aa#TI)4Klwu5N=!)|e9yxOB{1 zNF=MM|2o75B0dP}kk52tM_`Y!z3DM021V&H_9R4=dZud+9o!n7|viiE}UF zD&N1jTPGW)@I~Ru-!bJ(RV^NRE!VuWB+mI4Q|Of{zGb9C`aOSTe?rIQuHX9n2jjas zuW(5n`?nY~Jj~7}N>*k@II6j2Mqe!Ij#XTcs~GBdp5gJX-gNI1c4ZjFK!H;WNAVVf zwt&YPIa1*Tv9X8+5gi7E6k=D^c`O;W1+_sEgg9SfnXoA0DGLi?K#H*D5TxEeFHI2C zY@&G3)rMlVc2yfTsZWH3_%pl1+iL#;uvkLzPQvU94|E7Pr57`;*(+56|0} zs_rPRR!fXBzWVlF9bT*zhocSsUDt-4zZ}FntK5xal*MruM=jm3-`etrFaD`tGdL$R z$P>*of8wi?r+@kh^WSO=zj|R35F^#RJpNbw(42Gptv^PR!nmF}z&%662}Kmci4v&U z{##or7dfwgKF+y8>MB88_Ye+d$0jC7NQ7i*{+y2xXOUHNZ14)-?COHxKdN1=*mt#N z`$B>sB&^jx|Nin2LkR_iPfW(d@6^+O>L$W(sH@8AEE#{mOUL1&It z-H9b?0rJ82l9a&dzs6gRa=Ju3wMYOPTO^I*7G_Jkp7UR0iJgU2-ufo1L2EXWc6kq(&gI8lC0B&%S zd72kTW?*;?K}H7L5dXLjK!V+$CTI-q3k(hsM=8K#w7MYdDW2MXHGqHQxKzDB`Umqn z4teg5i93w`?EyJ^!?%yWIa#W8eMQ&5#a!V%`>EliW}0`74G*XJ4;Hju%Lm?he`mpD zi5>p;{3E8-?#M9*3L?Fk^{gbKxVaf2>VHR2I9AQqh~EZYn;2aW%{654bYgThdNZII zSzCyx-f#Q2HNd!LX8$wo^&xpJ;tM4KUt*N(owFWdkuIpbn=?Tq7QzOhCN3oc!HGdy z*?$pY)Ug9~ZDHtm7$)G)Z?$fke=ysbTvL6BLlyr*O0Z{+%7Y`|B{(y9$c}&IrSs;Q zUGy#9@h^>aJyojkPGRtG|N9^2!k0twp%47}uQ=)b`#P@7Htm7pQ3}`peoPj}o&NHd z-qZC6KUjWq1pA-i>cweF{M$#OI_~N-X%a+5($xhsi&t=jE#a!`iV2Z%e}}s!borJ3 z8?JJKNe-sc&kU%Hn=OlPzKT7OjKI3uE(0BrP()Zdy`aSM&tKoY8X@5vYiqo4d#e0#s5J-f@qNn$#C#9ACf5$w2xjBZ67+g2a zUtK`N3>kEYikJGDd$V7b+E1T1^hDR^VOYrp%Zt)o0!7@%@YRJ80-^9%;RB+^0moV$ z@ixgIV2f^DcxC=or^^-kOq>2EaZRr3G+|yS$0oPRf9K`YGDU$1Cn1ub`8{TcWweug zI({~Z@spz~*aFUgf1{3vmpxD1?!$c^BgMl>+i~!XlPL^M6`uad->!}-E}nH7&9}CS zzqpkz_;(F65`d&%{w>+nL_f#>v(M`xqvK&i5`fZp@(y=^{BZm9M!)O3CRsZ4WVf~f zsMoJ~%hWL$2~}NyFuivDem%o_r@`4)(vlj%R@(%UAYM_@e{fszc4DyQmuq>GW2i8J z8b~r1vP2+o<|jc+%qy!0-LX3ZoIFCszZ^%gfkl91f?mAB=u#RmbP^aP@i#vHHUk7R2gwUZ7xp`Y)&w z6<02p6!2mdMiJw1rcbZsU#&eOLAR#=$YWT0My$C1e_u13f9AK*ae}TkP)wU7?zDHs z+x`|u4yRq5keY2RG)qAe+pe_@mQJ;*RbNf!Ac$?{G(=nkFc`R|?CL+aP^{Jkt80$L z<}m2$d9tH*v4E??EY2-BM~{PDEYR^JNI+vt&0ByVjHv-)w&<=i_j?Jaqr*er5m9m^ z|L|y!f5=m>^`{+y4^gQyu= z^Lvcyz14ZQt2trtdPKL{G;>q*KaMz&cnx8)e>BgcjKU&YB0e!qeUJGp>pu<@l~7_r5(_LtoGLBl3Yz58^}z%>(>oIFDZU8n3{b<;ge!aG)`)bOsYn+ie3#1F68eKAZ@B$FfQ^luv2BC#>F`yV}8 z=}7i6DwJM{LGYnBO`QcUr2*u|0ytY4f9d80cFqjM2N~fcv3b`kTI<2xSTto0o0X1G zSD*fso46*ZUa{jQSoH-alwWSrAM})tibO)O)K_M0^&ub@ff8e0vHpA7n@YLBoMI+KpSf-~ADEg%${OO!Enfx7gUV0f+AMiC&L-1$i%ebp|1WuZnm1|&5*;iRmqg_n zuME}DcYFBM`aU#ROQk0fC;`iEPT{`u%iVRe1XZ>q=66oTe)@?Yo+5Spf3t4Q&n*8Z zw|}^YT8Cc^`akJ?{(Y|Xs{^n%C|wA57%(74rwim}pc^LonL7tR^KFAQq)GnFpMSI} z&H2XLuy!3Rd zA8T{vFC_oYOAVzN(+*3%I(0XbRfi4iQQpJ8*A{5f1T?eZ28qsWcs7Ac6o4wnww6ZDlKKmT6Wm4W)wqGI*r)b8Ci6? z6DO9zHG>%Pxd7c^XU#SSThmA5#DrvF(TiAMarO{*FccK?B3Kv!LD>QOlJe`CzZkbN zn})!J*5!O5Vh$HNCRw_7QE_o|e#D#_80aI$Y`Z+u$Zkn5$_b;GWLR|Tuf!F6?D|@& zTE>;)6Kefee}ioh##@8(pLuys#2;UaOsqfa)v$p4ujnip34XY5H)sFlacYL{H4KrE z1ty9uLsEqcSJK93^?D1SYR~=&f$BRMF=T21kNb;AcYxECgOW}=z9;ndqa8C8@)Zc; zEC({^bl+NeOsHuG?1aw}ElsWKFZN0cIaG6;$p{`2e}P~Q46-!>VIZFPyeUpd^$M?< zWs36e+M@DIAmHU136epeo6AODnuwUVretjv1P%C3@#CQ&*Z!-lg1zQ|Ol`xUXd1;f z$%N7sc)zFNTIq>q0SAdqr7H;gQf-Xc|42{#dKBvKyx?bsta#YZ44S^tw-5Q%hrctU zZ?62+f6wO0?HBjI^X-4}G<G&3 zKNL&@V(+@1M6ChY?0M0u9tDz0fi55rDIH0y-?YDr6j(W_I^bTrtlvgGwfqIUD-V2Z?2S|t`=FVT`~90Xsct^TA&xK!H8 z9!Y}@XE<0FZ##z8&B{Ype+1tkpazyRmJ9t^a3M*m1U&e`mUp@)gkAUwvk$JD$~AtK$JwuCZU;0Xt2y@eIY=`MUG1Wa-Fj z`3>@Jw)EpIV-(AhTt?#D>4sd}5iaqX%&=9=Ld5k512vJ1eYma)5HocfVIT#OcCX)E zdtwqXwk+PWQT;9ra}qC=@Rqw)9$(9SEjHt zv9}~cIAEOzi6}Xf(*pX#74QN>7$2`unn74BS)ludq>Mm}`v_k>e1YvctX0eQ6zsgPX+*g?q@u%DeF=r@ndEX}uJiYKEStf5y)Y zx2tg=Xjcb+^nh;W`-6E)`10_`Y}4?&ZeI-b&i}H;b^NWLhK{@Z?&fl8t*GMrBS-#v z2otm8WnaHQNU}Iqy4KG+fBR&WyOGMz)QD=fBZ^OH+5Wp`WoXDi?s;MvVqL-76d+Rv zLyC55-6TP5V_D%oa9f#*4A2>pf2z+SuC>w(D4{?oJ{rpV2`M?-kah6_T4=+{TL51e zHQ0_!P=E6pAznW=f+7>e>r_-o={-4jFjpQ@?%lVDxtLd8lEs$lv5C+TmKs6Q0S6}m zlJU>_mj`^Bn*C~qFi?!8ZH(rcg%w7JG4h1=z(O^01cRK}$`go1j4{NXe|xoibDj_g z!G<+Bj5CB}>cfw&p}3hT%HhzK1OmNcLD1)7y9Zuw`>#`P*22dZrWr%xU@1t4Pog^E z{`AxT2n6RiwQ4N~zTMjy@exmm9rBf(Q^)A$8Om6_Kq~L|fcijZaQXU4!h*ih(V7h` z!5deS?@ZwvGaIw*VnOv^f4S3nNYmk;Tvs#a&KHt~KRvqH7Z||1czkcB^ z<^pd29^?JilctNG^WX4iV`eNyE&Jtq121G>epe@vhgEOTae%d1$0J{T4@UmeYpzQ5 z@qS9jU)d3{QBxg3$P(JilKX!0CN>`ej&QjNv@)n?*|)B6 zVsI7{e>}lzewYdYTo@)2cO;2Jq|zzl2y;SWS#LbD@97|IHD6YJ zis~~EuL2V6mP8aAf4`mC>L0X4^>a*R<8!s&$D#l(h0&eMF>rsL0k=$Z5!C1MfTdu| ze#4l?aL~=;C;?HdkXTW|uTI5Rz-)=iUr7ba=6EF9>YB1oqXf#Yw=KMi7)ZS8E0Q zhX*7!Zi-ELf4ejXSs(1!}P4G#uzWH8nH4Ex|S_)5p>-7)rrq*BkG}4g|O!cpSVd<|HXVBk# zQIGw_FR~whSN-K@|Jp-CO@1_2<X}f@e>?q+bQ(>^$Dm(${c?d~+wt|fu(7D{4Y=~@;Z$LsEGQNUvB>2JqFOsc*2JrV5>Zh@ zCd3-7ULamQ7c0zGn04PF*pJLCAs{r*7h4f?vNG3+ZwL|+bT2M z6QxJUW|*n{7#&8YXD?#wzaN7#{^gP)u~i>vM(q1{&6fx(|G*#LY0GNm5BaIdbvgvW z_-Rvd+>j{-9~+1$erqv>mP(w&>k&!zi| zddn8M#;~lIE19tnWP##4`}Mmcvf>&XY6&JovF%abu7gTL0pjhjFEoFZGoaf zGLSllTsNT6O^1hDrN0_n@xpp%e@k^c{+Sz?ZYVhg$=zNvSAKH(Z+Y5rr0Azd%er;9(65gq__zN3iv^udnqDDH ze&S2j-eg2V{maMN_^;RdzLstN_4?f`z&q2wc})M(&AJ|6dC$ME+nT|5e=m^owp01j zS4Ql|YhZa+Sk`#68?Zok5l?n=cU_%Eutg${L&Iea6fd$0(ArKo;xwqsiqV{~bf6t< zQu-j`fUozxo_P1L6iXu2ey+sqNPotbLf6-eSdWRM)NcwC)w9F^ePJLj6rZv{Y)zQ$ z5n{%srkzh@!IjUP5L^?Bf5-`xEyBw+X>6J+Q?<{SFnM)BLe-{N$* zUfH?6vg*2B8{^0n#HRRh*wkWA8n(bbxnPc1m^xp- zT!}Yki4ac$A^`(__9rUNUjo)#?I2XXzpB8p^ZjXG-L>;BeD?*Zn!&r6yLr6Rh<;}U z{O>XF8;jo>lWu-vjmp1qgx`1$xbl?Uz8)2({Akx7f79|W=64$*>(t#W0nSR zK@7FdT=z_oAU7-#fA9;Ad5SK$kX|9qK2&(o#(lHa6C8^t3S>a*%39#T0SWFAQ8E&6 z3%C{F!i*6p&-H#|WC5kTFd-v*yA_nSsc#B-fTviZeCC>Xrf|dDO5xCutkoJX$_vBj z)SJa+OhY=45o}iHxoOA*8ydG!VS<~-d$Sfb2x@O-N+Bd7f8xZX5F_~LI6&-a8e0<9 z)yYNrV;jZ!N_Lb%_WLn*r8@*u2lVvRzMt2=&zx0G|w(?zz8U9%}wYH(xcAmkTb18YuMQEc~`4~(LmJeJ7h4cv{ z$V6-&83zoqe|Qr?D6VB~2xb|9BCkY{y0|e>ehkiPA~J)?G^Y@Iwv-!=A)`6RN~6WR zJYwbT=LsPJTVQe7gAfdhlEg$JKT0%H&f3$NtPN4lnN)tV;u+M$Su`=X?F%vLbPDQI zRg)t&P{j?X5V{TK-F_ml`!nev92Q0-Aq3d9m{Y{^f68DvZ<*k~zPo}7-f1)#C^<`x zk;5?JI_Zsy?#B?$NB3q3WLkWtvjWNQCutVENI~bY5MWhxiXi$MrhjJ>!ukFCyI+EQ z_vih3_xbP4bQq5v( za8QpI{y3}~4gEasgkU>Lk|H%|EF=C6Gz*8|Vo}@#NWN?tHwJp|_Zp&LN7erJ<*E0E z7sVS64*=Mx*dbZh=T6|9G^~L?JiNRU7CE@&Pwf8`g=cw;M{W(_eyXzw*iU?QcBTl(grZOtZe zP+=N&9UoC>Iv7}J3bmQndieo!Bck^D#(T5T{Uc8nmXHoK^1(-(VV;=EV}W>TtGzhR zIi1lc))|j2*oc|WwH=m)<>s$zs86MEK&bzTZ;aOKu^?IEv)1DOm$h$eOI_KvmsaZqGJjf5 zX!_}K*{yRpwcg=9r|+6H0D$$9qp4GOb>O+9%lukz?DL)XziZ96yx7UyyR%=u`_*RM zxO~^T=P&Nob*-;=^36vgdI=12CNOum-!G9HL3Q&QNJUorvhnW)rL?Z8ED4-uVlE3J zWaKc!Gi$6@3CURdO4WEM2?UhPr+j;(M%V zDoOcPeFXdcIRUj^9P=c4gBj<^dBynAg9dY!=wB^S+W}Pc>YXGryxppJ&*zA*gSBfv zAF1{yku6@l6!!69odgxuz&L6vyn+^H9gP0QM$%7Z@{uWG$#`b85E8_Q7=P~5nI~dA znv9`|s4gjeW(iXxQhfu-s>Xw?lpvp;SCIQ4%N5algEr5yLhA|Zd58km0L~xt9`zin za(Cmx;hI)uhif`@u@OJ_v`L?F8Z0JA(s-O@DdR$)5LHExi|w=jG|`PPKnwy9|dNmd%4hB&W!k zml0xush0{tyyYDy(8M6zmHSpM^>|IO^P}ubk#ZNZ~#5t-8mKXmBJD+>Yy5f0b-{T`Ar^)tf`mSE~r*g!^PaK?suAQNwXs` zjJ8Q4y-aB@D^;X)Hh(#d2w0MkYFzQ{NIJ-MmyO|{v(A4%T7!lrDkQ*aykU_?IY+G( zQ}h1eX#c_S?tSk2Uv;~izK1xQSp5*EC8HmO2J>tmpJqWCTPan$I=ejemlLXtz97f=PujKoA`E za^{S-^q9W6vLPgq8QqeDQH6l+_KdxkDcA2(hKtIp1A#g}t4-i+_K7yy{QA5X1LaIBKGq^{|e2 z$ivG81K!e;3gP={!s&QTP@hN3_k|u$3tDrc?Sh=N=HSHfw#9W}R5g?{dwMD*r{7IP z9VamyAGu^ajZ&yUPm_pK#W}r`nt~S5L4*XjPZBYRaMuw@AmO1i)C-W2YxW;gFx3-{ z>xZe;_9*C*<+W1fmS{(D!$*!F9CFXVN-3*@ZbceP73fmL)D z=n>Y91FA&#{ED9q8U3UtMCNY=6r2Ee=zjc8WIl1BT!$w%XfMS*=0PcFGOeB*t#f~% zY%k9t2lVUhC=n+?IfDV0g%47O#ZEBL5kpQJ*?&1ZswJUHX4CCh<}=J!+dkRrZ_qg& zz!5kRYjGd50`1KGFTeeQ&k>!>UiL%^i!RncZHj(Rq>;hZUi}bpHg(v$`0~QxR4h1Y zKghl9XQch&r}juCKS0`Yka$;TgW=|BMbq+ypGPkl?4QZGeL>6?oM%T+{sETe-0k8F z1b?}FCYXB+bd8@!UInLVI~01IOmrzCmhW)i7nc!*{JOtM29IOXNEn*?tm^E_uef7p zzQ6Qqn8MLFd}!mhUe_M)utk4XhcSzoNaGJs{<|RMe|^Ty)5`kH`)7>9k>rPect+C9 z*8s|)h2m6Pa!sLp%4NBWF+T^~MXm{I5Pt=id`E`-iD;$c6yq|ut{W%Xtqer>&vmPQ zuUnUEFJ=Ot<=0t8?7WY6cPp7wmC`-89(;;lYgHfJQ}pH9+PBh?7o1w@oC@a8OA3dA z9PpmrM#Bh{HyLtIT=EZ^s(%>oa))(yqq-)^6vyo;x9acVxzqVRKXKogoA3Me8Grs> z1Cjo$K~4W!1IzxRxqb<&=*8Ey@1NQ%tUeQ-V$B{ zU6SFc-50~tLfUGVWs&Eq%^;BFhktv5g`=W9eA~}S4kMx^@n_G4lPn(PoxHBYZnv9- z`D`ENpgq_;S?I@t{8@*gU09Rdcst$kxkp;8C+)1~l6`1l0ksj#50Dg#{_gsHPAcBz za);tPVsR#~HT&7GHQ5W*Yddtcrmkji=DU>^)Rn`~O|EM-dL!DNk}QV2R)1CFB$DFu z>+V^tmrs~%n{(*YdywOFsugMGORSGbT9e>v6Rhv?giptWrHLCqzVU23#2Br6-o+AR zRzFhRoR;Ty#H4XJ9>9s52+s``u{(VVb52Z+>w@-OTycYoE3dZ0xft7aj5;60CFiKK z8C~A(*vDY#J8pX%=?&8&&VL4zvJmI;=zE(K;GXs*ZqI@NAgTUHKTGHqJx4anc%4tEO3J*nDAPzp5gG)dXBHuQF z${kiI&r8SB`iSmgjem7U6+x;)7g}nyBN?7-&js5f^liiRvkIj2SfV_SfrQ+~`<;Bk z|KUEaZ=ZYST4Jp@Trw2OQs)BmAhLG&Zlt;6prBM|{{hH1gsIubU3_rv<228?wA!i0 z6mti>ulw_aU+?K%?>FDc+34t;O6jFVu!BFeyy!38F8Y^_7k~X{Z7+XiYN@EnR4Es} z?Jyv95~_SX+46*dI86>E+Nx0?Tg0jmkg;n|pEPK!k0SS9Mj?w1AcsywS|5y`jodQ# z2|fi%oB&6Dr^iMyONW%dG`o7Va1mLd_RLK3ZjrF zDvMX41HZB*e1CGXVq*rU!WQQuhP1eZa2Z(Br*S_{2?^fsIDQ@mnc5kDk|0WBGJ?{E zDJnCWymoo`LxW3|-c47lI-f=RzFNb4e}| zp7CD1;!)!+j_c}qM&Gfx`OG&R!L7-O^f6QUt9H@rQR3Rx(G1ij+$XSgj@9a=iU_?p z9#;)#^GFk22ugpJlTdcgKiy%iw7r`@k> z`gL9W(ptFX_E^$?9eaIJlwq}QKS{M>&x;D5?FdC)4Rnv<_v8f@(& zlY9AsA&+{V2J}JXnb3TP%%F;fq9$oRB2VupIIODH1D(H6jk&qjaa5GW zASY0b8v=s!a~ZO9Jf|RE>)Ch`!h@Zki^7NTdOug*M6HYJEIm*;#x9(m#_#!jc{iIK z-d@l4xnzHsTdw_ZJHFZjj}G(CKHwi&k2bYs5OjttzjAC2zqrbMo%k1K;Tm%yh8Xew zKYI&nk-Ki5`&@MI%lnwq?sCz&$w3v#)4Q%GStfE&(Uj;RTOfv)Oh%hZFG|0eVE*uB z20lyMw?G$eupI?#3!z6w69@E$K{O}Gm(ykF>YRU()91Ti(Z!1_-`^K)?X$O?t6K#A zPlyzr5F~c6o18!9{O#E$m&RMXiB|%u2&L_itx9pu1K>HQ?LGQH9oDJ&4pr@g8sC-F zzLbwq)cO5%Y+XiVjdfjO;Y@S8|4sN0!?{%+Q^R0_vcx#609!YTBX zcAbAy893Y8x0-OEkoS&%NWz!!lC~VgIW(y*0E&_G=qO4 zCzZVV()MSeq_d#)!a2b*Pz|ZTbwG^*GR4m{evsG`M+Yy z{RBFCSZcA0=_mxG@j*f9cJv_7zg8x1%vuOuu-BaUT)Z-R*sx zjz0D_X1x|=H(#9&VakvGcJq^C(Aa;nGUq6|U>&~)8~BNtmRiuI@)PG&Zl@_~>&nkG z;R{C>0xJ7RHxG~2v7r>P;5cv4yt^`(kz0(|otbM)NDk2eW#FE`&!ePuL5r7JCMc^S z9-rF)R30<*@y;W=%QJ1P4>{3Lmt8#~`C~0#p~~lft+OMN&IXDuhA_T>*`t4#&L<#U z49S{q4RQQ_L@@OgQCivGRJtw{1*ht1@cRAmB*@5hfq!wk_@MgjC6K}ooIIfCPnVxZ z)X~MO0Cz}L8xSpmDCZpWyd+tYNH(Z(6BErwoW-bpE0G6KCf9$wVw~l7z5VX_-3FVVU*jk)5vN_U-o_lSx+EY+0adL} zxAl_%NtU^wgr1})N9&y0&eUxQNk&hBYJ7@yJlWXODtCw~o#{>ntsW63qll*ED<|bb z%tsE%n-5OlBUNw5z?>typ__}%B%eSbNhRx(s~R(}?vDHaw4NOEhG~DtUF&j$J&ysO zvBNO!dY#M4Pk--U#Nx`kJ}+DeS{`COW^@a$q*5fTVmHuju%L?4w3<=Xek4`&1y$*K zTQ+#L9-f$PI1~k`RIRaghU4@Y_}1}b@Tz10D~M97*y)3=JVho1l50oJJ1J+c8Opf37iwrvP=grA^r8 zaz``10)_Eavx;O*5PHlckFBZn17b_(R5nZf76d1df!zelvqFD6`0DH0*elBttVMnv z$ql&nU)rlIzmJt!yIFr9{p5ZzZ@9~Ct)qpb+nKowv}2ijZ~72fbsH^^m-Ld8PkCC4 z?tSwBjsvxMBm^`W;QF2UD1@4lq;a5{<##WbDqk3^BQu2OkQeZjnzaWRM`di zHgPtKE;o+zDer&w)UW*pSzZ|pSNN5~z2*Y9B^{s2h0#4f&n9g85L)&1J^`}P^_wHV zpA&G0@FF`~+ugT~quPa|qa>j%1rQ-rxs#>Zb3mXWqBPYbv3V?=t1V6Sy$_ty9hsQc z*a}A0^BC7*hB{UYd>E`Po}-^JZf|_qcVn(W^}i75J(F8 zkS0|g0cw92%4nsA+9yzR#`8N`f5xD61`TqS${6eT(m}pqj=F|1v7aXd@<^YHeFj4K zDKMzeco{y<7U%b*#2ouLl#*g@(}^$A;PS6?9I6|Jtk0{M{MR+FfcU_l+F~MIN0xbX z)rB=Rk`cYt==vI-$@yG;o_?jsD|1_ zE+c^xk;;iPDFxLEbN`QS8r4J<%G1arQB<7wAxO0mP`IMcSKnb>tIAPUa%Su`>*{ko z{|r#tM|0aw(pm2q;sJ!Sj5(FH3*L z9dbQ4r_s;3ktDJnIBUV*VZ9eF2EXT}>V3(&L=;h>=rXAy2)xHC^qjvqin% zTUJ;bMO=w(bgIRIS8^*o_5>34`~~U>x#FdCP6XPJOdqFK zGxoDfEudC1R;`D*SVD^?(jvS+*4}?+B0|_jjjx1j!Z2Be44NRjpypwLc7&445b?2H zeEz`MLF4m|F+Yy@m`)e2zvFK4N^d8kOPAa3@HE1l^Qnr4fBEbEJ((XSpy>nbrxUpw z+)tm%KBqQ^1aTiZbxtA|+lbA^GQ12+Ce9+~6~@J=%lq_heKp;kluF{8`8arba&An?~IM{<>$Oz>me)D>o;|u^>+#J>_gXr?34Kn z?@O>LoW~a`>o*@81ryn{A-8u#<^&7TeZ3SlOie>3GKmle@QwM_>Hh~j24V7EBn6&<8to6}b3|!- zG~pAIXH3dBq9eMn#|7>kQdQH17L1(h1?E^hVlxW)#|%c388{hJTr+=d4Qz0Ke_b(E zAkW#SnLb@H4qVkn+nr{7Ol6IFag@NY4z(&MeU`2&g$v5-|HXb%IdI@3?G6D-4B0IJ zxVw!zm3R*Dxhp^Wzu`P)Fd`9*r`9KsTHn5=RR8brA5Tf+Atr^Kx_>VU^>guA>p4xI zYbKG)TZB~B<4E@Nl8ApRE@Tr^;dAKnnvdEYgFS*6@erq?%Sm3!AFg%B?b-hH`{}S> zH}0Uk&V$isI~c6ks1`D<4^!1J(9MgJM1&s&XepWNcfCDv^W}^+Hudh!SO% zu7Mufg4OG**ImM`4S?-^Oc zmOMlJf{U79{kg{hM!WnU{KXy@Smk}JiugA4kI^5B24L`%(ZOiab@c`OzG6zC#?Qr- zV%OcF=qss7Rmg-(&N&gUuq#K%simWpgIeSc8PRxGPS~*^q8jG7&)aJ3aV5DDKmy6P zUj96J+k*L7se^x+t4f;MEn!5^twcMq>2Zm{iYDF9a37H6+QPi6t6nEwK2Eq2&i)?z zZ}5$^Y=}1+t+NwPK>@>cw`7<=T|BNMTKY0&0~OjlV4vgDJC~2JT+lV(^ zwgStv)hvTG#DXbNQC_jt%I@;@}o=f+p!TqN$(!#l11;QytVXG!t#nQ?4K8UtROxk;_L`X`5fMwsr8~IsFHt9 zpuqbnCg8Z++#8IMQ!Crp!8*LQDNzmGqV^8z|4K(5>yN=1a~GI@x2zWb?pcCD9ao^d z;}NF3vrh_s^A&*TuAUTO40O&9zUhHcUSe=%5Vylzw>mSQdqMXG2=2K;v97~ zrXuo9U#rc126}9AhYXy>%W3Sw_hSn45$2`lP^qZ{tpjJVGWlglif=3^Av}K}NA9WN zxTZ!US}KRAd8-{~#-1eoq;F;)%q{B!L_|Kc>E1av21_`=5=cUEb>68Z;!6 z?l;!|h5Cg9{_%;;T^M!y{f|C7aVDP$>H%T@A=HyaJNr6zu^*+;`siLG@3IIuEZ6@$5V)Bj$7E6(u|~Jvp!1h;x4@$$#f@OpMBZ zjf*GP!}1p{ewPn_Cc{x~;wG0hEs_@CZ`oDLMyd`uOm zk5GhU`ZTn@b9)uuJ)7rD5ANGVk3sQ>f^dnw4y~{0UPA5~42|Yd>~06&Qieq3Z`FG_ zo*<7Nl;`Z;d`O=5*u;Mo$j16m0fTi3R028>{k?MiaZShg#%jn3jc9fUM`aZ_HoNmEa%b|9&9So_D6M*85SYCm;^lzCoRqfF{jbO8G{x zXAtMI7S*_&$Pwy0xS&p6v~Auz#x0*lTk^=FWcr|hC0cp{{8CoKVy8|(aQ zdaBdlQYXB_>)s=S-d0|YdhG!GcJ?qoO1ix1znOP~FT8)oBnAr?pL^|vjzf0!CGP&< z(q9THbipXJ43g-;bHzA`Sa{wCL6ZAKM~v2QKuJEQY#me7xX=Yhy+D30k`yAzb@a!Y zfn5*&c>mg*Ifp2|2$N(Ex{aV}p1sp3bX63%^;^S~p!A}>`K%#W*FcTLzup_TI`W0R z-u=9{tIvM`l{jw_<8q8k`RrMA`er$8NBCl;xE3n(8?D!UQNo`gH>*XMqBIfUI{L*Z$T98Nht zhYC3~dl3`i(<TyCqz>VypVxX`(p^Mt6(oP)AafT<>#y8Sy{-r7F*( zUZ3L{zxL0F`{5eP;v5-2rjOgMv8mhhLGOG0^!rZV#rS4-G1gH!jfEdP)-Xas;6hTq z_j2{>kWm1;LO9UI6e5hpVvzYT+$Pe8F#)9~=|u-lRnHNt9H!^_UX2K!K0Z&Xr|Cu8 zle>S<=98XGzFD%gK4$f3%4VM{w$l^qyO1D9L|ouR>gFR%mB%mQ7^;G(Gpbot0aNWK z*LJ>8quL?JYZD=r5v9Wu@^Cc(#0vWEqFN zo`A|dt7!k_YeYtj4#gon@o6+%uOrpJhWY1L*%Y+QLD#1ka^fep%TiIWI+hf85M_VT zfizzeV$ZC7^iMy}L94R=9Xgh`-+}p=ll45JcPKW_IF{Udd;W**`+5FfHn7hPAN-54GpJQaPcRxpr}jmp`6k(-c4xOABZ)rq zh|{e8a0sUi7IXXCKSN>d=>t27y^RoN3 zzB}YeedzUcnyshgeLSUVxh*ip+m)%iE7u%jlTu0^$rw~|&#QLmRao;M?g~`|>~S0> zm0e}2osiXd0<$(Im&!>qjKehML{e-J5|DbwW$Cnf?2yU_4^-o(Q9^WtdR<<&(IZNEP;UsIvRAivg|4-tbBik8p=^0TM8o6 zLdjHa#Plg+%7141X`sKKC?`NSz|kL=T17MgSsw)FwRZ+$?M8cYu{XXqj)^W~sulTr ztV7nn2c0>vtb-08VP6A5Jry$Ua2PvzXL<;1#iGkA;asHZ}lV}A>p2b-fZu)PZVTG}RQ;i)I8uNdC@ALT>MCVUuKl6=! z5lr5}yWL6WE{B3Ojn0QCcudP3spJB4-ZPWyh0I}@m5TelPZV>5?1^( zJKLX`Hha83qxCwgP>Z|>yPu66nhYnNtQ~cLwKs=#C!*1h_tWTUEHS7v8P=;0yzP6@ ztKyn3yZXq^zuSLDun~js9KAT?1oPU>vugA~s>Y_u8w}vztb_9Fa5qj_Cd%;g0A)39 zV%3sB7IS~V|K%Al=heh95nD*y_X_9Vcocn(4Et1haxe#?S!`nrMTh)lmnIaB_?oa0S@j^*_niwV36harEyMLPmNPazz9VJt4S7f7C2 zUnBaVP5v#Uz>OWf;ZN|+&?K{loPMhCgooLeJuaz>Y4JYhsQcLK{&?b?*H?`q6O&(S zY*l%?e;p|4K^Bb4)`M9%xPMIa7Pul1rR4ibS)U_n`o)20trDhCN0M20PfU#_kN}k@ zn8MeXvIKuzNMHSIHzQlXo-2ZG1>&i6XnKEJ_9N9b)&#wCUn8f=_AqYGt>*+j3Obgs z!2Cg=ZgD#&J)Vll{Ms{bAD{CBzue-KL#I*S@4aE<6Rn6Z`77W)_52+1@qPD1Ne-?nBJ=*-WD}7njL5k0xExj`8Kis9(&dd{a@6l*!cZD9x&P6 zN(~YZd?b`n`3NLf!IpnFx<<)JUF6kT(D+j0xH zGy1v&Y3A@0smSD@{BuMY+8RYN>#Obed1-hc|LuPG|1y`$j7rpj%ZaDBM|3#2|9(=W z64)bE4W{=EInif#&Wra|9dmH*!Y%S5FzJ6OcA`E--5of1t{=B=Oj(2Nc626PzHTS# zVZ>zkO4OadmB=>BsVOC4kD{Wwv-D5iXJW~ZAQ@q)sE^?^i-dvCAq>``j*<^(1F9z+_tx#0D4V*;2P$?NJ zzU~Rp($(o`t8MIw`{Ixsu}+agiv54@&$tO(KQkhvl^&$`*UxZL2Ua{;em}VxQJT{G zL&dkQUvAr@Sr2DwMKZX97X5uwr&`3M>)&Z~`mb?PXCH2H*Pr@Nzt=- z)-U9|et4RJDm~E78KiFk>M0&jKdzLiEF@8WwK(fZ1ai^5M~FH1=;m<@^)r8#StZe1 zWM+mcFa7tI#)YgPOT&oF*@xoP!um`l*buM}ZMo%AyNH*Wojx}-9Yz_}W1yHG-rh^Z z{}3q1%j|O?E+cH|Av-%cKB1k|3P!*qqZ0#NAGzG%4tsoDUw^&Uz^}Sl{D;F7b-Bii zFn744Ob=y@53mmcUBC5nVLg9N`ngDms@|z=j+`rV7p!aD%IbCc1(I>K+W9k_Uw2NW z3b>Nz*_l(=+tXcM(hv1AI4kD}GS7*ee1QzdosL@(!n1S;i9DgO7h~Z}XuPQCiOqC( z2WAO>qp~4>B?Nqr9^F(;;X}PbEw1ul0Wm`|}IeSt3qw zd()xX8EoL4xX6yjbt&rTmUF(<|Gb;t6_hU1*iY8rYdW9q()Csn+NI>u4xv0BLY=!* zz0B@Nd&JLE7}3ctzcaN15}l{nc;!oHpp=Xe7f?%QhZ@YNr9l#2Q?2=0aibGmT`F@+2wZQkI8?x9!!!PV<@( zReMZw-(&3&5}8k!GI{8?CVFJm*rU;ZvJUy(y~=&VZA0y#*>Uip>j~9jR->P}Zs=w? z7W>thA&v{Qn90$@hRSB;o>09-xP*_NcPR;Yg?)7X%~~`rz@d&f6m$dO9F`-ODBEM3 zr*-nK$x?OrSXX~z{Gh=D68;bTQ{=AP^3A9d@RQrP*+lohx0(0_t3eDEX zzRA>Rk8j5E)QarR4nRe&S2Zu89u-6lqwpN5Y(9=4X|sPY9Zy^+;`9<#CJ!_nzhGYB z9$*2y=<-0j=!}inyLrnUaqa7pm3v=B|A)Ey4}2VL|8M8W|98I4{C_+*Sk<{$pu<;q zufgkh`TPDJyka}*Uv*Zrb#v0;Il4K#`it}zhIE@Bd|DiR6}}=a$yyX1RcZna#$-B# z`3x4Hr-y%|m(8@<4g@|qy=Ce1?Yi0T*k>zVc=<>jTG}2ClQ?oC@CC_pMKV=4P(S8CBvDdys#@=BctiTZhZGhS!hhu@uitH5_!# ztPSNACLwi;XQs5i5Wn3r*a=#x(&u`LnnpR1&f}}%DN}i5f8GzakjJC~4$q^}#2tC` zBxy%E<1moNkK<@NUbTw_y9eaa1GI ztQbxqVnjDNqwzRWeH?R#WAi!C*63i}a8Q4!OPKuX&2r`B|Lter`t222A2;4(Z@rsu zaP1AZk77zVqiMB;q#1|EchFJH%lLap3haY~`d_hkw}cYR9DD`pj&N1+^@ej7@`6Yi zyPMDVSfUoMI8$g|rWb;ZxhU>2Q>#1K_eX*~wp-6h_bNWiMK>1*5^RP(A@M5lByxY` z1(A3!@nE%E`b_JNhjePGt-=yuT_@H+_8L8+ZM&Zm|h5+-TO@x+^c{ zN4M|t4>gXP_5pkpIr2RyUC+jR$b603j`Aq&dp%1_0jiF;Qs3#*^O{uU;gy1CE&rGI z#s@5D1<&DKSztdZawT3ETk#AVMx>ftl zD&?|Ve~UATU$iFfDf;YfXBeINU-R0)T$;Ms)pz8*_6u^O?|yhB#qp-RASpDDb z;Rdz9wLDs1%clayltP71e&?ED*l)EjW>U#8hgkI zH@&D(|h z3)4&Q$9Km#()5*$c0YS?=Zq8coT&f#VpBE}pU7tbtt5kDe-C4akVz^}MP+)rA+JG~ z%~V878Wog4<1yuV(|jG%svu0a*msP=Nw9ONIMG_df)kQ{E(Mh#RmcO#t0rqWg_`@b zXq9aR>N0Xl_&BulEm41CQel5%Gq>U374i655$N(>c^{|Lz-x1ko!vFBRa|k1xQ{Hj zp2Zs;0s9JIuH{xc+hIR&!@az5HOs%_$Q`6TUZcA9CMnPSpe8G+zZm~KamXEBtP6L) z&)qZjV<~;6#bcZmQ+9u9mU{BufIV#jA()CPPfaHmJ&x~U_2PeP;SsGSuxma-aQ&^E zy(kiSL=WYJG!)8r>Z-CV6Fqz7dXZFBzEPO-c3^?_5VkQG-0^ONni|+6kQiQ(GsysR z7@b;@7!pZHH={3l^4JDf{z`|ke0>Q(vQy3aVT z|8@@jU61kTxD0Tu!htxI zA0UTAk+ceE-_ufgwpgb5Jj?QdW`ZsllMi8f$QbHe~Hs4;ho?cc>Agpj^=T#?#!#dU<$8 zjGh*zM@$NY$}XSi$Y2OZ4NFbqYu)^28*h5VUEY6ea%m73aO?Tj_hi5J2RizdsSEwZ zJ2pATOJ9$?_7A>cnsPl@9_4y}NFs+MlhqOh_2oD|>VKtiCjL8pvw!%R^WjMB_6PSV z^7GYO7@wokp=of`_7sw4hq)2J9zas8y`ytx(i#lT<&I%qB^BlCy^i648n`^z4DBMR z)@y&I)RmX&P#BfPZX|gX$?1WYOc$p2Fx}WKljSj1278BiZdaK3Z)0+98>U*HY)b{b z9(`_E3*6tXHdZ`Cte3IJ3GexB#Sw-PXW(
+<#4-VGzmqu=p2pLGDMacl`OCp5C zqVmy0y0T*6XE2PERXN4OrXrt(W_fUlOTU< zv4xyJrLF6$;$sRm1@-hg*^9?1Ek7n8hFtQgtUrgS4^`OvDc90>9(>h(Uq4zeU+wFB znL5due0cjqUgZlZk?)AVw~1V(@%Q3c9S2&k7+-FJi+T!ZJDMWy`R{oY#kKDDoA3P( z`edfh;*S>DSzPEEa$W!aj1Rj!0r^bR`+S~~THZB1b@ zk`Bake_I^<1p9>Ygz9D^4M!mV02$4hvID4!IaQ71$(T}QPaO~>IOQ|*IoV1MZQ$8r z?(8GfdXr4msQVbu3K9>;2T_0DSBBF>L5Y2GruzSUp4hd|YR?7k^8N6?JvU=!dV0&k zC22g`D;fQ-=QS%SNQwH%+-SBdjnEv_r9zzT3BU%9JOE3l+R89V?lt*>nGrTN zuOR;C;&02uXmcdRg3EuxgEL=1R-k@Q5^EragQJnNZ8Uw3*h@iV>z{GSQcx&|)~oG+ zoUcADUf>-CdB7anZd{zlLMAU&5mg97knjB0xEcK(z9Cn3D;IK^Y8(Dh$KT|0)DNOA0YkIam|7nGHQ97kDr zVm3C|WU6#XwEN5ymsTywH!*xD#yBT0t zqd%WB`yY+tsrcuwwR6NV9pt|p)AI^`=ErIcuOIieb;dnm`XGS@mfyw~|JLg7bWX10 zy8Ov?{=?5izZHG;nd5Zh_N$RkH1=B;B;MtQ$5uD5F7|(Ap2!~N&~&9mm0xa-~C_iQklJ9CoBoxs;L_8KXXBlE$dB$zzT4@~ii^(GqM z?7`;XsoH-ds>Xf>cd6@&&xuF?H4y z%xB8)(9;u0NF1D{Pb6~0A|jcl4EO)%x^-E*23b$3T5k)@eF5$&^RV8^7sz`v>-^T> zI$J-yw|o&__g)sAf&IyM`@z^09@>PiPi5gb-PeEEC(9$-UmQd1pQCDW=*G-3aYPIU z-ww%gp6-`<%&f;R4$7l?Xe z^X7l+OKyuF4#;&s{4rln_J_`s*Y_Cg_s58uyv$o8pXfe^d2Q^%UX>wJC0XvIIUA6w zhO3rBRtza%9>iUm^L#hFD|OLxM)=t$pPS-N#8y)>QQ1l;_nAr6&UYqLjaoJG_-L2P~BWiZci9E*Q)0Svl2G9e7M?!F~-HfG)lImumoIz+o% zwRk_EKDlEs+v%+))pL_e%BkR{e?nE#PwWGItJT^LY|R*Aa#qw*pX1H6rLG#04pZ2~ z*h>$4voY)5g7xJSvi51Md5_1t#bZ;z7Gb137(00paMN2so&@T*r9ztg+Fc$JU59@J zgiYn>iW{WFdV2rdVtf%kZD>uH5_>*XRQCVDJUFJm)4991-~;XS%V0Epdc~Xzk)p-@$z<8U_bA+WmnZnXpBr0UM-t?N z$CPWZcocP9HTMXJskIrC`-x2^m(PED*!;Hrz;M0Rn4djrl<-*LWdrO#n6E!L9P2&R z$n%vnhI(?XDTCS!9zTjV?8Q8|zYJdBinZjE334vBxwO6uhbTKBGVr&+x7A)|uo=~` zD4(o-`FpIU&H;|*o^}hMU0*QBuKm-0_hRbmgmLXuvA*6ddI8$eWF`8PHPvE_L3?Q!-h794YsLleTw=)J@5NAPN;to=L}c>vd1sq zIG9&W9QdPlYOvk+$Z;DQUfihVQ_czYtv$|cB1aKXUfzv04=89hlEQ%xtR;2W`1$esYF@=cy z9ICe!w^i4$e*Ln*{oy>nU@vYg;$-vcZ3R6iy?%Jr7n2J-c*!_*h)m6W)6HuV-};dJn#*2m-p|S-9E*s`d~5!uLQZBK5?KU9|DKDaS-Dk1rVq|X zpx3|0$ibsj3x|Jz-&>sYYM%Y( zBb%?m#MvcXD;1N}k5}&g9rG z>UH}Tq}!{m@T0CYO|b7RPaHy@s+&hJ*^w8t)h>i0@c8w^c{hi(g->FAPbU7Bo7!~V2!*$ z8{oslRlxHfeSjMe?iU~DmV5o--uL-f_~Eitz6w)@c4#;Lq(sgrBRnJaTW~7G#I`34 z1f8C3;gDQ}_A>C!>cdg$DccN>9Q+5&SywP3?wNmYA}){<9K@`~oLOAd`NrEhlp?=8 z%OBWDQs(|Y_TH>JaV6;j{Xu>}3^w>Na*#+(kP-rmHzVwn5EzWWiTL$*MG7#ks&l$e zcYpW3+*-Ak5lW<-hltpF4<9<{W7QVMawFg4r1enI9k>MhH~N8Y_R{l}6Z4L9Rv8`F z#`b?jFR=#e;DzY8Cy@h3_PS-kr%rzAO2dnRTAzZkcKS3uY3koCyOZv=d^zn zMtJ6DW%?Dz$rLmf4A585>l4D)fyB6wOI+5oq6aB5)a6QK!!!QP zVR7SMyYjJK#?)W=&`cVv>+XF#*OkleGA5h7CO&f=j5l*@IG>jj)PvbW$ea&B=dRNhr38JUNPKA&hVF<9X@u^yMWL!t7ggX?ThL z(&se?^odo1^xl0d4T&)dg3A-|zHKO4UY_Om>h&d7F%T0k@!Y~GSQ&M9AzYoC8m+xKB- z`}A&Q1yRn#L7ppl0Pl=chhk{1qr7q;A-3#~hiAuON)We}pTSC>y|{GFSvh7<sLV ziCBEZMRFwBmtgAYP!zpT)V)U!io;&^MxVW^qP2CPb?tv@_@zD+L`{4P@;I7!wF{o+ z7T?>Vz1hvbjR3^sJ%I?r_WLO!*8kF6&wM^3iXw_Ufup_OtB~vwq>FTQyjh0k!2k!2E;K`)Jcc?o#Z}9+kL{F?`Q9tlFQ$XzPxP(l=${G<}UgFm!b>OWS#@p11(l?K8CqzAX5l?KYDjOm(CQ3DU%ddg+Rg`by z6hz)c?=JR_>v}LZc4H#0vCw~tC*QcW{IJhAo`6VaMuu^Cj zXRLS%er=hl}Al3u=W2cjoU V$N_o?86}qivu?Nu*?eIBdQf_EcHAt>f22OBpRX>O#Un2P2}!&n+z& z#7a^v?R5{i)O(yrzBheV_5@MmhsRcbjXaS5n>3A|rq#E2ONt)ighNH5OWdLr*0SrX zs}1TNV<5TEyc;OVZ+NV0%skq9XcI5$PSTgVG;rnmaN~dEmL|s7Z)Fe9-J$>578%HN zgt3E;;_sjnn~c%6w&gGZl0R&sEoVfy zttE0d2Dsl1Kz%&-U_gDuGiPYiwgdt3K`?H;)s2544=9zv!`E0Bt*;ewNw6h|fFhWu zvil>or340XTMEt*e>>JsYB2q_VwEfZGjz`|$xj z1zRe_Z*!w_%D>BJ`qK5h=f+Lh`1$@^lHAu{gWqV*Uggz(=hA3`b_FQ5+{{VK)TK~` z@ScBM%SD(LUwB42>mCh3ppC$Vub!(khYWqix!|oOQpRivNQ>AqED*2OhQQ!e+ENRu z7lh5G66E4}i*Y{Y?DAHCw1!lrR8PXf)52!CKLdq$(Ui%&S46eMSivM#AL_7G7&j&rBlv|IF0=0h?XsZ7?{s9)+rF!=nmgSg1kc`jM(Tl9? z*1G%$_=m08Pve?+!1zlNH1QAL`i$SPyC-wjSl?aS_UeXnyx|{y+Bxpx)s@#7uAZH? zEzy`3fv(!GU)7wwca}`zb-7Iye;v~v`bHUxhHUXHXP~w=fh=Uv^F|KQ5R-lEi*0`h zi)w;cjqyskj2zq$@v^f@(7b8|Lp0oBY-|!B#n5o_mT{dhsoL>kPoyBh>bD32LBK&NH9bjR6t>N z_SV%A6t|&xei4LcOe#;D(@j()j3Y+zTqi;%2~`-g7nFkcK5d7*YvlF(Ee3yhxI|4# zYV;6qxX%I{$A0LDtMe&BPz&(kk(x+dg$8DBT=uXNep8OmCIG&u=8vkAdmY~LJ4S>DQ`Vy>kxhYpeycL zksm@vr^WX3Tn)x-xk^opw$6V9Qy8QDN*#qr&23F|@cCLmN%aKlvHR|> z(iK%I9MVM13v>=U0s7de8b2%%%dN~fK%503I-!sqHghR{ix0l&c|UBw@t>=I8~;gu z#g$}Nz2ciLWutR-(>b$#axtRc;tpTYVCJuQpnn@5>g(rd3~B6u>PUYWbbQ%!ZOMBG zl=N#k4G7^q}Sf2&u?Ep|fufGFBZKtye(;VZykwa1O*Huu{I17kQ!wDU%8eHkq>V_+| zd;bQml>Z;L!M}q$b@c`7lTtQ+2X;;mOqElZAejGFr(N_Tu{GSKOOSuqp;84SqL;KD zl=cK7Z=&eGz@$D$MT4mVRrS~`oY<02$*imcRemtU$|u!XM3tD#tZUm8_F24wL`-z_ z8JutJgBT+CrQWyT3=r>lhh4;a6_;4hd%IFFyTk!}ulGN>C$BmHML79?)?OCh8prL6 zJ>{zTi}PIj7hlHJme38A*Rc_lV{lj`l=Wp@u)=U)Cm$4q%ZL!Zb!@%J^u_&9is->O znP`XSN`xhh!@?TTC?O~|@LlKVTc}0}yGtiE+L>Gmx;G@&Z5)k2(28UpXIFgI1%D*7 zbY2ghnbyos2E7-WrJ$6596MI3snI65DmR zSPI07!RvCuD;k^=#WR>N8{29isVXVtgEhqZ zV^(iJVTmz0m$;L&zsdH+(h5X=?i{?_D6}Cj3Gf}>tBDS)7#MDm?K{0Wy5ylw@Gc{UeHu4kEGt3=u z3SzW{8$1XF-Y2bo%^=`OI|^ERL7`m{j6rlTPP@?L&9w@mDnZ}-BZ#u@5H%oVN?*z& zRgetniR{CF^E^Z!JD*!GBCjGQFZOJIz~A-aL1@n1xTv_DFj8rvjBX9_Z~H=SA@2(A zQ`Vx8tLr=&kbCYG+#jUhb!7I}KeQ#Ha{ z>1)T5IK;9}72*+7msDRP2V;$Z+eDXwk(gH6*LGy|XK_an;OT^;-6M!zF(h8UKpHSj&{k9M-lLBMN#H z!C*r%#`($tqWFYcgVkDbGCKD-(-xhzcjp3x&0#;5DwTKr_UMR#icdj2q$GTnwumvF zKJR&djoqLh!M?|YCQEQCSbpO}=~Xw8Xw*ATuXCZczj;9$Zf|wtTWR!ye&u69{6n*T zM&U%Wu|f?s^!4``+M|)X3JlkG7LAp_v)PN`7dGp1vVk&l*o!=bn}pyP-R!_2C(93m z1=?90Di{7aFCFBEMQkJmJ;xXxB7mbFpUrlEHX-SrLANwGxm*9=&Z6DC97fg?NtfBj z8Erbr`{;B!y0t1HhA}(y=hsKYgf55ly-y)e=K+Y2>ClY`k0<%lKWMkAzbumS731t9 zXKB}B!KLL8gSAqO7OvxKdo+3M5!!l+{bV~we5%mFXb6_Im|4+`>dHP@XGj{J_)wF7 zC-hUlDL}4{?){nln>QHkUB6@SC$bEATs}=V%H^x{T=L$KD~ttG%9}nvy41CoapJw9 z4d(PH{4#SP;-|Il*&OYxIe+q|1DDTSm#aosZiVNLmutKL2@tv6AB%eQ1F$V@4lYaaWtTa4;9p-Q1W$#MAno$18p zP7m#_qSfZ?9hYTNfv=H-9LTp{@t1esX_}#4k@1h=S+e0wyT+)Yy~J=n1Wk;`rEZg< znUCThxRFa7oO%Aobsuuu7oLpY_S|4jzV%+=Rmr5Z*XsgOK}TeoJSNlRG*fYZpsH2w zjbZq<=&vgTF0Yp92#ycya+$odai>2ymz2rK@nsq#&y^{k>WxUAA3Y{djBUG=S`V{i zIt@U@TpK(gAs|Lf&R?QBvz${Jj?S6znMF^L%*#$>$J6c<_{eAm^h_rVMfDpBkO~j$fo_2a*xha+I8>B#l$YAps%l zDYum%M5V20B6m(<1&9eG@HlSzHLqb|;@ZkqvKiES6b=v*o3QyqWo6z4=K zx50TP4EE^HN1b=?Bc~ZBs2QmEtEUor(L*GGat0WeV#}+(iD^f9eGiX+FZdrWS=AQ; znrAyi51id+#u#`O#to2o|L|CBZN!)(9~+ZZGfrErPgqnWLtGA)$Lv>mb56MDR?nd` zmKeVoZ+6<0r>+df3*TpNsZ2P1e>z`u>nUp8HD$T_|C|BLdZrvV{8J9UYB8My?Icv< z9^90JQr|lpoyw=T4NM1rh>`tl9n()D1;1)Y8Y<)Q-~rki+hy`?SGZ{m-E4*cLCeIQ z$$++k(oDD*+XCm59*pffS zy@GCZs#>1KiA&){9XFHwj*5+}A8utY;GmX^6fOwGGBL zU|bxZ7uyQ?e;%IR`zsxcyw|fX+R`?`SS2

xOGN233t!PY|Pj-}N=gd%K;$`uP3%VhL*a zQ0?~T!LRi|uvHPm5Uh=_^j^11dmW-Hr*&ixU4?eOg!%?cV-5cpgRdLjw|CU>JplT} zHfDS_F~r|GECurJ-{yh%at^8aN>ZT(8LMgx-I}0XifruxXgMMf4wOtef$q68bUPr2 zyC*r^L522zNT1bOxdiDy2WQmsrBICnqD_q96kfUysfy1B&B+1;S$sWx;y4V(56C1& zj}p!%pV+TifbpLbyU}2s+D-i58zC6G_|1WedFS6;i&$S|(Kgyw4zzn@^PzSozH9i{ zvx_ulG6)GwAKyN9kI1j|^09tKA2O&x{H`4zBuCzVL>v9LJxDONgA*NPQ*95Ilk@{) ztkI`Do37_GoW^wOL)0TPyW-|-DUF>w!T8}m_!RknF8;_X$LqIW`qd81|>M^$`4oFHto>*gcBYQzjd^+-!m1M_PgpD*rzNam%WPF zL!O#gyc;Js^7kr(scGNzXUw=U{Af*F`NDFUQa)N)Sjw<6Q%tiofg9!IZw-V2?8o{@rB391KrXwI&uguGwl3?2v6)4>5lR=I}yoBtqdxVb0yar;d?0dL}- zF6RSrU00FAwdt#?j4(AcxRzF-<&QXi5;*KlQX*4Mv5uu*GMTzW+jLP+I}?IyU9h0~ zf*skXx0wFcRJiE-?P%nm!ZEmYT1Dc2nYf#DznhF_%2P(Cn%?7LhG2N(|6LwSc zYvzv2{0h6$F|ohej$4O^G7~5whupI_d=GLfQGeWNK^{2BSkHB0gGvdWGp=(nd~E*> zuJvBQl_M0!o=C}vlvke03qI<(bKzro!ML@5!!OXR&p&y$A;EJg@HNgyzsCuGt)8DC zqmfD(>NS4zuv;8foW9Dx;aV@Zlq-Gx-e~kub?I*h)Cck(5cfNhWN}$%W`+Y%$?JKu zysc@4jDzp^bM#+a;&p$-&R@@3U7LwJ-s|+L3H0#;Z(($CP{+nwa3}?ne07F?&LJopdl=HKy4@_$?>e>=a$+kaLU7-ziYw|P%p)_EJpOmvGO`@#FlSNG#9 z`{@&zq(5}je#Sc(9<#q0zd!iWFvhv@@tQSV);A0d`cTOCc!ynog}{msT3roN2qLTK z7oOSG&~a@vzD@IrL^!AEEQ}>*w6H$I?{TdnHIQDM2B8+u(#aD7m{!r$k_l&I=4c-C z7`1e_4&xa2zCBF28Qa;5m32m-yRd4*bzXq4(q?+#!gG&wv~XA?h~cnB{FEFXTi*69 zYFnr!7Q<)CDEfhaUM(~$*!KCy7^jq^j?yV}D-IPE@K5fcJaHPo zr14!_-}AmXv;UKMyNDtw13O(H{VY+{kR?rfd8_EVU3ofx=Cs@}meFxS?&j{0U;7Ks z`3jWsh&s6-1J9pE*ExR}u2j=knEZ?hxFG!(qRR<9444_8FS{p*L`c**?!KSR{B7Dx zc^lAMm%`?N9AzZ>*`}b>?Am5y#wSE?CluQD3rKeqiBGSM`~`P?rN;FkSmN^qyfUJK zwcC7WNKKM|RuAdFdW`<6(IT>yyy-x=cl_7DZdy3bO(R}rMr-W4h7fh> zC-s&4+AFB6{MLIm1o_@a&RZ%c&nJV2&FJW>(__lDk+1Bidm|{5vC;j}8u4s>qD*`# zp@w^*9Janb@EPpM@LDe@bnGRl6tT6$(q$k~aMWUG>@))d)&OiBvw7{`0dILr_&r z$6dsK0i*783{j7@k+*J%xJJ}qAIE^Prx;(F0VVI9@-H>Uo&=r~Jc1Bv5H=%%wk7JG zXnVf7#G!uO_V+Q?*DvwOxMnuC3k5ID82?v$4sT9WAQ;^C_=0LqjP5jI+)lv%<+%pj zCqSc>P7P-QlAjtpM#Y=G{@#r+b02RJqMg7kceMgP`T%9HE)Yj)*ECvkA$ zJ}bL!{Bym(%O~YJMYr9A$Sorf*xzRs)G$`kF3NLaKiHFZ6RWhl#1#!Gx#YL6Vn_=H z$hD1}=-2a$#QvT6{G>MiWUy}O&PHspub;?&t&Md2Bd>Ur{D{e=4W<&G*5a+xfa-ESKWzz(q!&z( z>guA2WUvv$2r?F^PTg50F_2~ zD;58Z#+Nx?c2#q}3{afU{S`-Hv>XbXeRE{r>rA_M{-p4VdCtXur%QX1|D~?&b?oTR zPi=ewHkpX_dn^Jb)tJa?pDZIbJ)`YvYlrJOYukKJb+8$N45k+3ZuIO3-8*=YO@bmj zfffN%(jCQB!leiZt!ntNDo>_=c>hT9|5X1rrO!I>$%${@3)Eqk1otz+Ms2xNo>Piy zazY#-jyH=VzZ@?ImCVP_>u-(dqb;|Me)Cx_hf2?U4j8L~XWfgg^CJAIlWX*F|5_*a zX6NZO`(CQiNEQ5yFEAR;)>Rkx{(OS|gsZRiTT?;)f7>rQN?@PZLprg4<)u$hpqgr;RmOoA63VDbg{@?Eti<;^z@`k-5pwkWe!>rtL9QUQDMzMvO1Tzx{ zp1Wh8i`Q5Bf7%x)&Jzc?&J#5M#qVqUN+Or%Kdh4<7heAMFM1O3gRxMk>iBEcnqUm~qrsAIWJjW|1`Ep@eX17uxe1X6JHd6TH>{<{SICci>odJrDL;&orBkFC13|b>_cKT) zsL?%)v2WUpYO$h)d>;`}Q+?5z2LWO0$wGhpgMtcN3waZ6oZ%QN6s+Xx4|oua%WmvX zH{3_&z_opEYgEvGa3D9freF2duYIKZ9asOG{`xJB`^r0D<`v}7yYYP7=A~v2p})CI z8sFJ+OTh}fPwns%`iqh}e9e^=>{V{B3V}=_fVGp@JlT%Q;XN*VFBLXLX3}9Kf?ckkhE5X=W)Zv3X zt!QDD*LjMTiAD3IKT=7Iz`hBS|M6VZ`s`tJ-K9@V2L$nXa!RGd{(xHUk1&;*inDrm zVT;jC-x}|$+@nyn?Z|p{bB*}62`LUKi302|OYEaT68KOAo=XP3OFz7y z^jrDyT>4NT55S3ouP~_PziW5+du@Z&PhS3?JWA%=cYg-?&$w9#)Ak{pq?{O?GfusD z7bn^ppSJO4r>t>nC%?wXfu+$FE_UA9oiN#Q?6sQhKGP81p1U6k&m4%&Xov8=Ubg#^gKLeTQ99V2+6tELcJffKM@AH(MiBFgwadrL|Ra)70x3%Fq{|G@S>G_ zImbN-w4ZTvp3eT;ei+@((0}Bs{c(o-m78nj-rkY^y=KDS_TK?nOi=4FWfUp`MfsVm zgOCkgIo304Jth9R$9`+4e4g5$uNx-)=l_U*3+DZd`Uel=`}&4vF4$b=gs?=cLnZLM z7%*(hZAebdc@Xz#t%q~#NwE|18oIZ*#%%gia}(?Hq6K$OhCgu|%SFR$$~P_`1`z`? z%J!vCeYmK)}y!YD&*E~^5vwMuLy4Wn%R+m zS!1s@6+CZFe{fOV$9=Anw!z+hbLMTrNpjBt)x|Q5WfPYTvBLz$V}e3Xn~dbrftu%y z)+XxcI{bDY#B>O0LR63o$;l^j%x%K8&8@^Mwir`B6ek^W61YSK6aIL5c2Of#$5DBT zLf+2PMjJF6#LALfmkykQ$zr`-Uil_}_jvAJTSjNy#yu=CI=a_6OWka$bufbK-nzLq zdRAmp)k#|rIAz49d>ck3;Zh|WHqT$cKitjklgh!E&e{i8mouu|82MTz+ICG$Y!mB( zutn#?O(}+FqP`-$!fIM8^z{>9!&VFY1Qz0=PjS^m&W7Q+$8O{D^4(Qu`VL2becO*+ z+oT!)cl-A9m6s_iE^Ui*0dZ8Y0c9N!@8i*_WB0R>6Qw0pA(Je4BakLRKJY;W(grDK z=WN3xnSt7pqy4yaqHIIuoXFE)I#fv>OT~De+ROp6pI8%GoJ-7)2V?GfWPQFd?4tk5y~BikyNxZV&naxV%VSC7~A=t(#0 z@eqjn5B1F>hz4A2N_4jrdU-(}qNp5V0Dli(f$yfh3h$;BdN&TS1(fE>C3a;SjjA&1t(KwbX^4*HAXtA8?_wzB^)kiCCTbAEz8RegYelkh+%j#ucR zquySPPI|5P1S$_|t1_I?J6yYe*!^CRtoYybFaKwCv<_U<`En(HGdc%1+j77mX1&%M z;vfHc-E`I8_?f5TAJ$3Je_p_QfJX%jNlZLfppwb=8feb2Ca%+9bAH!H{*!jKbF(w! z_`JhTnK4-Zrf1z|N;J5Co;vA-#f|H-3vuSylldn|+sYvGDRK(u!82#>>rnAAlTqs} z?j3h&`eFDvKkF|i-#c5{vx(X|=Ouj5$ne&N$W;(4dqAK<8}T5DVVc?lgWGY|?%IpD zU+ZbWP29v}OUK~!kH`Flb-cRl~M8=JJe;MkHI_RVDh zVx}&8;#X{c=?~pU#E^A@^ObvQz-)5a zhc>=)JPW+sbl3~j2E6fiSbQaK_#k(3(P!^$$x$XWQ2&|k`bvs=W)~W-`Pb71kh0eN zwny`II|rSNL00Y1qwnx~nt=`<;Xl(}H*hzvSSRM%o4s#=drT<`3Fkhw7?Da6UefoM z#~{|Xd8dPa^G>En{}M-fkLN?|LH7ah3dT$ zs~34{>w(H_$C+|b^j3~tqrGA#G-?A1N%C?I68F%ljqA+A`}T6~cFegMH8pVUL3@Rp zNJ?=0)?3e3$Sv0JEyR{Z;ehMYMLV-xZW)p|B3@*_ZR}dk_Nv?X1js9Uo!73|g3JDY z+rDsrHe>UonDLG?yf zWE1XwE|~0WcjrJ)r`y+=$RZ+#M~HlWm_}K{SG!k;S08{=!ut=t+U(`Z4{@pMj>kR*`O3+uE+?l(m zSb@ZG6@_JK>@4vl@euun>)lZ`=d=v|nt%A#UcZc&U*@`B<~h_I`nExhHTp^~ zIbca#eO~!vs1ByCex%p2Wa1xA+o69>93pZ4D?dST=0vAroI}h{@bIn8+SsoUmq&CH zpZ(Ib5AxX7;Y$3=xz9d-2fxK2-`edD>(vjP+J=|28E+Sx@YJ)<5C!k zg~B;|7lbum+Gp$JQ_F5_fr5y_PNrauYffP2$uVm^&HPw91o0V)7b4ktOg>nW!>c$Q zvf?CcnC||y-B$7TMHkf=#oddB7OpXrC=pI12Ngu1QtB01V;8i4u-)YNeVx5DJnMrY z>=XJ95jMiCl7aAOsi!W zSLfil|Av}qUMx{Fk@LLDY?7FMEs7wDh_h#^?+BHKL?bQ|<2)-GN^bD)lTP^=pnkNH<~kyUZP=mmpNy#TKNFyc)i+DO12gP(GsXEpjF!_p2Q&B$I3 z$6B3epxwRP9_eT5P1ZR1!SMnITX?w_KWte9jwOTB~y6P5__pi|q zdV}-&bF{3UUf6y=>ZpF}tksyq;CODg#&7W8tR9Dd<6ZVpuln`)G-$oBhf0zVGR{gF zO-WQbj?9+SE=5azJkl@jr?S0c8*8j0(&jHo{1uIMwr|afC|^47$s&)=hv71L(=nTi zBQM|M)OCA&N8-cUpYyNYL`*~1>iUyaINFv23nPH}l(1yfj_ft>Co?Jw{J$ zF;0qq$N0r|`^jz|f3^KQsqIN-kE|(uOkSq(*HqfmsQogYuEUwOvvT@ir<-+2XD`$F zX}V2jpUXwi{(5)`+~v-Gf|tYN`*_`oQ)j8R4u^E|oF5!frrXoX?~KlKwk{u?Su`)A zjn!Fx&0F@?@WeF4y9Ns%E|cs<4+hT?#_*tj;7|Xa4~vIDmOG3iHbxWtfBE;I3~muo z?&0Mcar(f)Tc93`rC{@Y-J_|75$G+rD)4}Cl1oT%Q${piE4sn^eE=@~7zpA8pny$) zh*uA&z~@OQGz_J;01D0Z0rN9PGF7~e&HJ>s0FqK4M>O67{|HdAN`u1RdrD-$z#YDS zx6U-OF6B%C#)}o^Z-mchoNx^R@{|V{M?*C}uLrxY)<8LfR!d-hyYMjpR}U@VfQPmx ztb|aHx4;y8!t~}kdI6^RR1;IrBg_;H?1hBK=Kjz^K8t67E?#qq_j_3Wp2-h5Ig+Am zCG6mp!xvuD5})7weqY9DBpr+x4;3-(F#=Ixdh`BQ z3l^w>ZA-j<#QeNqdoWYqU99gA@op@Gt|AMF7Og%wsN+hWk+)!n@#okc8UDW8lm~zR zeI4rC_38V*x|N^deYmfW>wPl8>jdxHL38~sAy#+SdJFOG{@w4X{=@I@?wj&|HutA+ zdtUO(`=IGBn4j`iKX>K1l+RrO%bNP?mgc%OC~4Yixo|+eL_cl~%0HHe*x98$N*q63 z91o%9v&-{MeoIX6n*6uF->>KckO_RL%kwVf*;Rn}OU%zkH^0BU&kiN@$RufgZ;L2{_Q@mZoj|ehvf=?1h&tQ{%-nPh&~whX0Xw>`lA-U?;x)A|J$|6e;*mK zu)j+tC1@<)2uywd%>P|`+_m?2|GVoi$uIpU(SOxnf9nqu2>Yi0w;puUA9fW4(%gqc zlL`Kf&0l%Do`T_lZuTD!Uk`yKuQcj*+Fz|9jmzfyU;m9uKy5`-)+Kmi8vO>k4FLqFDYA+X`+hSjPZLwFy z7xisXWks9%3vuK{KMq!Ipw^5g(v9TWq0`!Zr8oX)#ppG zd+@gUb6tEsc~$y(9)a56Ss!lQ@wlhKCI6? z{+xRIghex+tuHNL~5xT4Ko4@RRc8m8n_t0B+3;#_ld+*)+>y00Jva4EeY}s?W zYWxOax7&SxUn~b<_pAHXT|PZ`KfCzrWmk2Vtz~!Fop)bfGFToTI{mN3F+8?A?d~E7 zE7dvm7tXSJ>FEAzJNy`S7GJZ$a^7lhhO^o8e9~UM%)ICHLyUT=Pucb|zW)Dz-}9CR zs#OJKI^)y1paeH{qa--E zJGW=#cmmmG(QIV?R$!}WHqs>CqOz%wHT8t})DRy}0VL}RG95B!Mt|RQfqGR*O$J1O#vjWFItmHlj832o>Bo$G$64UZi#2wA*`UJ{j6z z0aa`e!OB;%2o4nyRosr!N=9X5mt_U*wdkrtMS?1B+iBbmulE*WT9y?FFZX3tNr+{8 znpTmBCAMqgX&xWu6%dAFMDF>27>X&aBA{;KX^2wn56qn+*n@-YP^>>%Y`3Z?!n`8z zDR4~8`1f^${g<%RzdWOuHs4P-npxox%O7>9hRyGNNBA%@;|@rH{SezmI%ID8VTT$F zsjy-V$w&ixLR1sE^xvruc;B_BcA6^WFQSHH1$OP!JfFoK6wZMP2K!0^ zjiymr5!|5>yMi9pM<}p=aAdv4x=B(8IB;B#wmVftQh0spd#9=}&x-34P{8sPzR!uE z)uEz6RY{?+d@ij|P@DT}-{-u!kM)~0_v`vZ3Uhz!`>3ag_pyF|mAUT~^@+yj{t-n^ zus!_+eEF29isVZ0plRd2S$j3isP>C3uBF=d;mhm87tUXSnZGji_x%i3=|ru}T29Fj zBWhV0uFRM$s2R%@Cu%mswhEQ#@Sanf_Nnjs;EAcZjs42r!^zuo8=|Q_V`T2sE~Y*E zXb7l}|89EAN4D*Mrrd3O&ozBC`-^h)fq4DDDQ9yZ?EU31#^r-J7pD;T)GPR*VreCq zR#E-;<;S@rov1^_vdYi1fxmzrwh`9z->wsAB(rTp$L`j0}Ix1=nL!Lu!zF@MT>gTb{&yjI`~v(o&Ha7L}G867A-)H0!2k z)@CbPMRk}9YpyDoGh9di(8m<6&DVVl;h~z-ktRu)&#`_XVrg{5Yh~871<*g#eJ*Nz zj@$@qo{va>wW?spqCHS|0iL|%Bc)W6a`eafHpTe_^U((f=XQvXn#9w*GW!bmPkn~F z8{X`59J#`EGIb(3qkJK1d;TLoc6q3nt}1|8MY5GF%z9|M1spH!xf(O}Lz|>ZMN5M4iH8m7>_pol`8LnB^w2r#~ z+f6oRT{=nxEE0^4xb&^{Uyt*;w-OWv#Y(NrJj(_@^IZnYFE4HHP>jyo{IL$}?>;E0 zUJ3f>U_X!c@FLr}blUEGMqx0EpH2br1m`~)|CTOx+T{0i#os*-^Y@D7b}F)DGg~YY zdz}h@cAEQpt|5|pe?ge!M2eS^QxF_s2_m7&C!LeevP7>FQmzWHjeJk#>`P3$OKmGI< z#QPWbeS9}Wa#FDjuLIwQN1SUXG0vNF8}}1}zv4c+_np69Rl#;8_awyrIH~IzfH?Sn zIidL3Za&9#jq;M_bAg{P-#6)p_<7o-Z$3Bao6k-9=5t-2jrp>!4wbE{3hjVA!1B+1 zKOg{vVY*pSpOJU9NmpSyfW)NBd_Mq?wn_IAV>%#Bx?WWwXKIt~-N&;E$2}ZT=5t~` z=e{q<<@>^XZoXeN=@ou1oAl=Mx=C+;K5v@z=JSWi2M~_=c%@h_4iVO=Q(``Fea3X3 zrd;xpt|>oh(&?uBiAi^8%5R%=i>CaSbhz(?fmeUe8yW7;nBG_1^qmv7{g@uL#C_^e zG-DKh(C`QLj-U!tvwiDX@zKsuw@0?2jE z%X|*R@@nj7-rjsZGe`(6K2HEkTAcz0a{F0H5OTjo%4`Bq;k7IGTew~yksAaEuVP6G zoL4y&GrvTPZ5jD|3uzwBd?i5S={4;vI-XqZh-6wkQFIf?2nr`mn%@C`MD}`){K=Kk zFYJ9DVc5zSz*h@SfyZYulMe9c?+ukyfKadk8c?&v{j{jzx&Q38`pP>%o?H@^kSWX; z=SFMZGk%NfHNiNG?e~3(uFJ#seFj{B{=UzF3PQf`+rR~JzwbN0+z(+R$m{p@2$*Cw z(k(dy$bdY;P{6AsA1JR5};(IviYp%hu;7#Iv zj((>#*a}`W(q~)fkBTurbCh;B2CBrG_K6OKS41;8l?vh6-`fm->bWpymGy{o+sg zs<0k&tjF~C^hc8)MLvM_z{7IF7M9q*hZ0^fU$(f^k7>V-=?A9XzWbE`GQ#IBWYe$y zvL7k*9-=>f%afVE=67wpHsu-$ zyf*DHm(6&MW#hk`Xrh~O``dei-=0e~e*d@q)|AJVO+Wps{87Mibg+J2FXJIq_}7%H zi|LnGA0}OsFDXfM`JDx9i~VMM86O(2eGOmuzEorVl-Qpvy!Nm8gu8a711z_LB;~Ir zoy54?C;Z-j*O)%QbUP&bUIIDe_-f`mecyk-fDdr~2)^ToSWD?^{Lml&5dxQZC7ib@ zE_kp3ni&@|^7VUYI-?fW;}?#{Kthe;JB;rH)2(g76cmxr0r0@iPqJ-FrzZhM0$zLI=$PoAvxx9xO zBLX=Qim$NKxRP%m&WHzm2l1A8npY58#8>8N$w&;g3&|FC+V)K_fwV`E3`hcY1XDp0Z6`2I$W+-c zFjZuKChb?4R^$cjB}{w&4`J`MBRjJ!=)H{!>g$L!5t&&5yhJi+|`)V{)2@=`dpQi^u4&V~UC=e-4p<(&7O+{4f>$6D}(;)%D}79#dC>_~W#V zB`9>X{cIvS9brs+2^Jl5Ojil+&tmT_A)w>zXA_U!l$=;ZUWIBq(XnC<%FKby*K2?(8LKbtR3rxSBr!iP?O zA?Byq2FS`Z&L(r{(Nq8*1u(R zL8Egx9lA$jHTYj#x-~BP7jL;ozy8gCUG4eh?=OG<3&;L&o#o^xl%EvMw>XHQ@4!FV zK%v>tkBMmgk;N_Rk8J<_BWrlxzd6tR_RZsd+qPWoms`~z&a=PVvwpe#N53aszr5xC z@UQ+WFQLD@`^*38m-)Z^&)9iZ`{mmE%irJj*}wX=zxvzB!K5s_~YBsqd~uYgZ_?x!}{eq8~yTMKWlCNm;d^>1pk%y{>uOQnFRmZvGvej z?)`G)FOUB6w)*A2c5Y+#mv_J8Lw@bpX4fx&|M;PVYJd6rm;d%3{au4?{#XCs`9Xiz zL7NMIxWUqYIQDn^750~Rzx?m~)!$$KcmD9-b=vmKFZX`!j@y4Z|Cj%Nk(d9j^N#!V z8_TVJ`S1L7{I7oNcm8GOSAWOzfBCQe@9+F|((n9Pt@X?GU;WbW`2X$$^51=+lUW9_ z&s>1(T=?xD`#b-ghX2Y#zw+!~d7b^sf7@q&?bGSM_OXBMBXhs@>Hhk+&Hj#``(2xz zh5Y5&FE{@3-~QR$Z~wS|zy15WKI~un%KTsZc)#|s-M{?r_&w||?|%8O|8)MZ|JZ-+ z$N$>zmHv$H)!IMYW(~S_?Uw#|GjVDEdVc@Q^YdSxp?^L9OP}s$fBFmSuO!DqzomlT zMFWK~UG=l?q1u&0+0fT9>i=1Lu=U%v|8qwCizD{&%ez0e^V#2jwp#kn`H5=3dq?*C z;fD8T>@uVOIpaDG{WEsU&G|pvV1GIGXK%6G3jNEgU;bxpSZ>Yy@;_%l$6_0Soh})>v@f(|HVD;(*Et6 zwo70C(*F;~e>tar{c`)4GyGp3{o%Lv%m2N1{BZX#*MHmRu)n-8d z(l^zb%ypCf&wuE)KYj6^Q`P%_k`VMiw>A5J&tvWXd!FpcfBr!5n#vOS zpJ}oc1ol62h!V8wUm>nPztnzSLH+z`Klev5%l`itvd0WGZPV5EjW6w|RP6RTGiz}O z-B@h-^D^t_;_0X>7^KoFCri0!=Itra>4*Ntcv^wb@zuLX!fOEWZEq^Lq~+`Y=_BR} z^s~`nAK^@Y;>6y{(Cei~TV#};0~L1u6N24WPTm?4mkfBFvT>Sgx~*20yL? zBqwt5CKF$@mx}4)z91M21qk4##tKHf$ZB%I(&n3gb=CV_fI-M7B}sh80n#AW@BOeR zYz*rs_Qv)b=LjjJt-`eM_sdH(idec|usAH+xqnsj2aj@;r0LvHoZiB}Uuc2P23!=R zM`2vhRJ4539j9|GK_p0>=jqhODYlQNR7OBfWC#Kwss^OIuZ2kq<25}_%ceIrYGtGx z!4WurUGzMh$+qYoe(mX{{bNf$N8D>7;tnpWg88MZk1h9~lL8!(i+xRKQPo&p)DmYZ&(Tj-0I%P9gKtj;@8kw-<`C>)uT#j*`ZR@1rC%JeCrY8xk> zVT%@Yd`CNtg3+}1$#_k77S*FG${C)2 zA~?Uf&AZWP#pv-6tMPFLaU1Ckpb-y(3DcLqXhyU&3F8vS$7|-N98#DyQ!T^ECyADh zmb8E(KgCJs3OZfLGV7@56~q!*`x5G1i}W2v0G?Nu1DDDS*Ngw8?2B9=JS=gsq-`*Z zCHmyS_Kc9+^?{RguMd?{ls%0|Klc!Se_y>nEkR^D{m!eYo#Ng?Bg?ugoF-WoyD9)d zK)%0=!oCiJ^nK*a>XQ`N?Cg#`r{j^yU1HVnx1(*krX}>@o~7D&L9xzB#+NGdcJ{Mn zHQTqT^<5aZJxYm>I=^paDE9W@R3=p?M*SlrT0_hoJ`X0ab~u-N7m@5JnrTBf7Bqx> ze|3yJpPFrP)7h*!sx;X6B441&n&(YAx?Qp!Gr5Rh4_f!9R&U(3n!c?9J3+H#UV0$| znpfl)J#4>1Tb1Hu>TJ?s>|w8+Z$4rZ&(JjGw3NLcoxMY%GNi#(zm@G^^e17{FX^qI zpnY;(lk{;?>1Nup&s_3~gSi8uBPW~ie^p~+9#LyJ=5>7iN>zhrJxj$=fYZvzAK$pA zZ!Szl+fe*sl9_q8KGx)>TY=I&;Z)07yhLk~=*62j=U$hkQY*5MyE(4HD#dtQRnJ7O zM0K>cm0?Xop(;(T>Lp}@Z0Xw~sbnwT?>iYpqS_K5d3kW-8v#=l6B;FwyitWSfAj-? zAbnG#Ig~M&Mj5t%XigP|y61^wuq{{cU&NFxoN%;@p#XZBbxg^DoYzIJwZ>6oxatU; zs&9h#%2VTtbxYzj0qQUYbu|g6FM;uN`tcb{vmQsFNx7xHxs@32p7F8Dri)XbgB%B+ zJL)zO+Y}yVuZsszC2gNI(lQFPf9rDBO0DDthky>A3^^HWtHS2Q*PVK*F`rH9$>uy) znP*$GV0f}&$9IShf7D0WVK0}K({$;yZ^104 z62}@3B_vW|_T8%iih}yYz#7UXxqk3N%$T$h50R(tv*sApR~g#^7%edpfAlFVbR1QO zpi@7CDA&F9)IV3Lo|dX?HB%(n@~F^27xI)UQYzSKtUcXL4~-b_{j(wdlyaU^B2GX; z_L~N{61;f{Q`m z7CyaStoWwOXRmij7Y;vIW;*s_BXl*7s?f%?M*4o6&^4?PHeE(I@`~%d?XFd2rL&lC*XDPV-6#j93kboZ%{_2AYwG;fgMW(uU1r z!e!xH$&$Lg)KuGryT-n?D$0^WSb8#~bzJ$8JhGLsz@qOfO%JOb2DI8cAL))bREeyJ z8ojim!}};IIP~rVe;l!aCv;?eZs z5;ae!O^fK{dhWZb*H-0M8SjD)T^1@}^-&Qj{Ag1da8&*tknQ_2f$Y%))j{aUx9mXb zSy+UfrWI9oeI|h}x=f$-y*tMaZEx2u+``ng$R@w}H7&s~$slccru;Zj!#h&!uIMeOu03Aoc`MAPn)U>4#p>%wh$MH^Cq zsc+=-oH%utpd%NUimvJ=WK3$cwD7A!Ap*D`#2I6;J8b$yaGP3@(L_Kv{*r zXvJG{nxd^p=xaEY0W!z1QIMBOT>8@E%xnDuy|MVbC=!oF;Cx7=sWlo$cKcf;N^X{o z#$kf-QEs-ngW}L2Nc%7k7=D-9&OvRdxhhp0`z7_Eu7#?x%))>KX?y!mqHQRteS*#P zx%PJVf8{mnkMLlH-KLwPa;PIgl*fE4nM6yG87$G)ecb)1z!#3RCUI$@HpQN+n_j`} za=U*L#cY%xaK?vW)!NDI>8^g8oYq?_VaTg>)TQr+C$;&!zzUb5>Nl@-^rt{L8eiSV zWMT*~Y^z7cn_b(r3HS<>T@n0ooXp#imnPj7f1baiWutl;s>URx*veXrUsHLEH>kwk z%Q5p>)*M(VG~LT43*CYbtjA3%k-M8#^z-a_mFaqq81mGOk4ew=(WlrEV;+9@)RuJw z(FlMTvrp;=2lA(88xE+?|zlZ2+CNV?NLu6`_R~)-LZP4hd_R=iC>7=9H4`4FWv- zb-A)ZfYtgWbul(Ya#EuZ-M76%&l6{)e=+N_f@^+n21Eu@Es++3aVD8Je_F1hT2tpz)I~kroREgb1}{fGZcv$5F;P3vOW)s* zP>NsxOCNF#%A*jNg^KkpXUBdkeX$)y`W)V8PrnD;k9V6otit%k4HBDS&rH!QTG*56 z#C;t%OV?>leeK*@BP>W1J@YM5{pgl>~P$1|nU3?|>Gm07-gu8m8v?h@e zp@*ZYv(oYj#TDpiok|nUe>Qoc*m$1E5y(!FXBa;GyC@gq<6oYWr={~stMwnu{aie@ zEC583Cmirvy2S^(`9?1kQ+V5Mv5)VJjxTW(J1F+SZn@^LR|eHhs4U&gvTieK&`amf z*wH&)5Q!lWm`0Q0wt*q2S|FX3-}PaWsvnhDcA+|@HuTLbw913%f5?i8HV9Wbi#9EIwCzuBE?R3872I#J)X-}b>h4MgVI%Mmm8ynIrJNpf;qy@S3Xzr%+u{1+( zwgQ_{H&>mYZlm*9S!H&*0lJr^0cfsZSI4H2M9+nh$TjOf7VKTQpnM)C9??q4pG^vr zkF|hit|*I5zYn+uf3>HJEyE-q$+vb^6>5R;i#cC@Hr3WD3Z7>Oi$-7W2+*&%kpnnY zTuzOJgMYFNt5Cx30!cbP`X@t*7~m3C&uLx#r{#eD_5)lw0j)&6e-l(j@)@FTE9RC6kAZtN zz|GVt2!$_sfe!4Z9b3wsUH2E8P1gr~hZi77*?&~or8lZho)-8+a=3kZ?;hnKz&Y(PW3$V*}j;Iymf3pDSvKVm|-brWL)q0v4|cz z;X1scVv{;GJ4Fy(vG z6qiN4e;STiIi&F-@S&&g!=m=+ zM1WbIV*K?(b)%07S4*~TbAd=S$K&_Kxpn*CZ~|c1k;FXfPA=GLQ-3xOy)LNF3j+;z z_$Z3fdi=g+_?aH)RKjKn#BcmOtLg<($g033f8b~k0etU9;2ofmIzr+$z2Tr>U*amz zB)qn_1j(<6j15+3@(AE=vN3tP`{%-0yFVX5cy7s%pO~*7(su^vF^MvWZPfW$_*m|f zT+yZNT;}Dl8`yFtdp_25XlKhDa|&X@R&0HtWL^+D;Y~`sIM7mc5UDt{5Qs{`k*Ry~ zf1wy`fu^01D*%ecxKB3?1F$BUKUoC1El?Jsy3MoG(R;oL7+`Vn9b)?i++Xbf**K_e4?$l}{_v97oa7eeFGgx9DrB?oo*_(p*jQN-bEQ$M(5PYOK`X8B}f zxA5_EG5<17kXtZZW**e`&? z-l3NQde4_6(67^sJAEiBnGw_jaqn@n1BV9=%N_;2xWeaB=$R7?cGU zAF=S^cin4l*HiKHMGY!3&Y~#4r*z?`(|x{TRrB=;Q|T`~8bjIH3LOI{3S{>NZf9Em-i2g0pb@1dXW0KR8} zYWx$L3BD7+qNTjTOu`(Ee*nWXG3>O##TUs*2*@3ZNS>VPT-D3ew#zm`2h5CiX*I8+ zPTdYIR6-zwy#vgdR07zml-t17*83z00$?A{yQFW6ULZ}@b8zYYx7c>AgOLN4!XXnO zNB-b430p>izj}Ex8O?A4HHFp=-#%z$`>FPyCnyoaz#$xOk#c`Of1BBm)+Bwp1x{4* z2pgTE!Y3BgOfy~`g;8?>JV#+Zxthu)VG)TBpoMny-{eBfUK)yGI!kfiUJ^DF+iqB$ z1lmOJT%^$KPw$pZ7x)YEdElp4C<0pd=;>U*mzK44FTjNzSfd6|fhT<|UX3m{-1MmS zgkB|fD=Sv|GAl7ye{h}N#Sj-KUJ`e@(2d?x;Ax(bhfk6(a8X+!lwWWWd}2|G7xgnI zaZ8~X~4wjYud% zgo#@V{ZVS+*X$$4&7$odsgW-U>ZEjo_^~?aYvxHnN5+lN>!eZcuF$E1Nk(wj$1|4 zR}b!7)o2i*=u;dNP43m;fj<;z$IAFDK7@l~>P9G}qSxIE(FulK9Mv%JK26UdV@lu$ zggb=uZn4QHe=9JLW|;Q(4tCQV0cQZauw$%S1iF;a15~U(Jrk*7cTux5H02&&5)H0? zGJF>j1dN2M9KfPFcUVp-=nEk45+?!=GOPpVu66?&zUWr0Mi=uzzPq7oKUZnF<1?2# zt2eb=v(pXiy(MqJf$4D$JrxLm?agQQ8U^ z-E_`E=yM{|0Ae&z05vSx_CPaG?5ciGInT=kq6S_a%X|?<-A75i#*ASSA=EY`H(od3 zIUZ2+f0&EMMgeeqM<~;je&#fsubTiCcCbvfRR-a(4Z*;y4I5+uQR{)_+ekNYqN{!c zr+niZi@s&_tx(j0O`DP*udV|!6MRFGEzo%g%U@X7li=aQwl@2eyMT?zy8?5ZIJmNQ za3QCQ=#UkQTi#6EhzWvJl0(SsW<#@`T7sQ@f8+Q0#LETx3}!4JWzcu(gE|kX^+QFZ z1DFqTiXi+;OB^Ao(*%72)~&P%J*cCcP5k|$R3gZ&e`e(9MTUYx=<4h1v?Tp00ofh) zozj#;@&!n6d-tlS^7aW492ClDJiU!cM0CBF5)9n-`gEGq7nLmXH>{ChmP5A=VV#4y ze@_VzuzZ`D0vJXN5MR>_Cz$m@`LnK^FDo}7&WL1`rEJfM!p+v&iONlJkM8Ya2a7e17lSkcH)eBQ*>VeV!OQ$q+Jb zSqnwrNTGbsZZ~*}zG+tasp1oep3F6?{x6?vEm4~u?`rT^O z$~1?vFkOi>tH-y%d_F4D#QQEExk9{FeM{B-$eLd% z@COy;YEP?&v1xkzIo-_*P2%Ii*!v?E~L9gHpu?f5P!ss}&a2 zx>q`SmL0?JN!9LkoRIzI6eSQ1At9P>ye1wt5&{=I^x{+jy^TRjej0jsr;<|K$j9T&(QQiqSMg|`+uX)Xp z=1hE?t`K2{h7`-y+L@t!e{K0{P>xgRF>S*_XxJB4`={^?UD0)tqiGyzQv#eW_j3*$z zo{5qCUQqR;t`f4&g&ScH=Z?|wxjekw+9Pm#o04iW!7IZIdMY-Of9FByAcCk^{M2PA zunUALq+~-qsfXo*47io&XG{-o${Y&HEEl?vZFxf4EQgH}qWB$Mn5oFVJEpJkip{%S z6NK>7=L&lJaJq#I(-s|E-;$As=-8e-v^jyHs++P4A~HyZ@WhE#wD5YBkybD5?H=^n zlmXvbkM)FM!|@OWf9#>gTy8Z~R#?%&dden}oGUv-$(?I_UuE)ga%xmgp305J&}Bbv6I&0a$5=Le@h%i;k4Y59?Wut995k+ z&{!eEZ_j0SAZVE^dp^82U-95dWzK z_>+0_lb*Dcw5|`CkpBc1c|S^-^`E#UEow-zK=L|6|LI@$x0c^OWnirsx$(IAcP|+6 zryO$JFJw!Y@DvC9PrYO}{U?j#=So-_iFuxFf_jPrf9|Ig;6F;lpY-0ZB|(>QCISo< zKiwt%8P897Lb~MJm`3m|Mf9KRiCy)-RHqGuxPS(@E`BaoVl?fqKAL_kB1i{*k1d)O z*?$T&w?C5O;N{g9zEV{GDL$CP|LFVa3Nsnm@|q~~#Q)6S@lPx5KWT<}XDvup=yPAt z{s(~?fAkD;VW_z`Na9GF;#SDi%+ileN%Qi^PVF?eTYI%vqqI_Nf1dOf-I@mj2mpN$ z!#+pdD9bDZJ`>{`y8m6_iDHb&w&mH8C}=rsu}zC|G+MeqL15YhfKhXgLP4Rl(~m$o z3o%f*K(l)R4f5s)SB} zy=4%5@180%%2Ga9UT=Zzl3f#MpTi>d`=uvT=E3m{F*mmt0_|ovulwx0pG)oAwn@yV z&W$SH)=<<(L_uo^*9!_Sq3GQ4F=dXiSm@T>a!`TJ$+TgDz_JBf9VTB6-+uvrXE_Xx*D~N{YBCO4Tz|GSYfQcZ$g*Qkj%qZ2Eqoe z_r^*V5@QP)q=^nugL}>fzb*!OCE8{nFu>!*^~K2bg%JS-Z!EC{!x$194^TzsDXD|} z?nq-CfL?VUk_KARxp};1gudqee@Pkzwswz%A5tJDXbJNqjKrstxKXp5$)VlN&M;cm zwVjcZH~D0oyXHVI4Ls2pqWKWa18uZ65ttt;O_t6$_+4I-@E~bgMFWgMKrgtDXp$$0 z*zhBcVPU3ed*iwTx!kSb7i2CJdfPZeG9AfBy3AdLi%~btG+=(b7+%1ne{3yggD;T3nm*B~~-<1p0sj%s)z~SeWL=)x!+*gLuD+n={jP!unu87-KU*|5O7ETCGVoEPx$-fUJzo1$6l z*GYL~&ns8EB-wP~Xp@vuE}61BBxef9VR;8dfoWQM?oRWx?=?#GfA`g34+w@ZTRZ}* z9CU{Bx}*RUNVdwr;Yek^a4x_dI53LY#{ zKTrGt#^a`#l9^?3Qb*m@!|7gJ$1xmTsVB2%Y@26IX4uMt?K~nf5!16tB!Zr5Q&RFs zCMd zva-ieG9_mUHq4rjuo>u0zjmO+BosX9`l8#l3op}`fBV`a*LvfoQEj{O+wJ+X;G{=F zHH`hhyEtAkl7tol+yUQF2i|%H5ed2AgJYT=z9`+G;v+@Y*&s$k@uNL{8;3J8v#QYI z7kNkE$d?Z}3Islx$2lZ3 zn+p>se*i&=BcV)tGc15L^Ki-uc21MB6QRYJISfLJnP=3w5<=7G!5NP_S-OKa@wOWy z9-A6k#1Qp!YTWRnBjxtIWQPh2Ww8Cu5o*?4liJd)GtJ^fG@B%30g51>S5B9_@3sE&V zy0K*sBrlrrkX;{St)^0&hS3=R-9B47Wftnk zIAwvRwYhPiL5@AifEh#gT6FjQ@;Ha(T&8`?6l+zdM+d`>(hkzx>A8+Hcs9?Y&Mdlp z!^1LWU{}%xu|)$gjC_&(21|o6e?FK=>GV~r$2xvcl7bHw$-4Wl9yRfES74l$hAGM! z1tUiJa~O2ZVjK14$#5^$cRNe!BhnYyY2KK(oJw6QLfcbMs&pxf5t%wxy$f7eCEQH<8bA-*jHmQ3m7brVN|h}#vV$B!es8e zcY_i@VEoPYz}8*r`07m3(>@Rt+hJfu082z6JjCY%0CigwX2!j1pdG+xx*Cu{eJz{9 z4{DHlt0|vCK)Br{!LN7@WyfD1#29qx*7(C0TMBC=C?<*d7~A#6e{u#}C>%9dGEDuK z^kKumOOqlKxgY?9xt|pP>+<8W`eLi z3IjKLJO8*+hyvf0e<-T>Nk+QBqUhqrt-3*hIb7jeBin*FQGAd$7@qcmw8OfK+MBNC zn1^KFq3Z#NSz>ykIihSI4i(yBp;Ow<*gZnJJ2XTRDGDFutB=Wj_7J5_V_11i0|x^> zBgE?72$4x|Vw`~=Q+%gyhHQsX4v|LYag!*L1gP25nY9;be?K9Lf{j3bigSIfw1#}l zcGyZBp=6S-`lF72ymr#U9SVvvRt4N)HbG{6@Np?l*eWIxu*NZ&bL@ z_(ctOLH#XYwHDA?skuE?vfvz*lBWnYWG2TC?Jf+zf88Zvy9x(I%w%jJc(y#L8-1#T z^IFm)VJb)0#t3$d(rVUnz*ZexXmut)X zwK9lgNHoLkV-W|CZFj8w>0&T_M>Y@7DW{ka+lan@wj|0AvqO;1gmNISpp9WW`3qSw zlQIZ%5{_RxFV1Wjrveq7;=PN>fBC2c`V}RKrtgGEL|8rs>1cSm6$82g zpDOf91^n3eC3U$e1%S}ypLbU$OJc5XBG^rmVHHiAAnl?$AzWnN$D&HW?TQ&Q(>s~* z32f~m!w@`8VEpQWVsO6wGGFEDYKzrxATq;S1y}NJ$qH?0TmDjr_*gGkd|!uv?v2#c ze>Uz$$?7Wx?z++a6;`lj^eb?*jShx?#e zm|z$5g;B`fLeB&pcTD~wZ<*mWn7ZvFCR=0-Cqf0)GslLrE!3|SNTzK~g=2l#dS$rky|uIuVc z2U*#^@#d`Iz0W$#B>aGBAvMT*j1wi2QXAZ8+yyUCfwIExSj6KXC4d4-Uymv{`dr*W zk96(NM=D4ihc@uD?c=?L$YXZygs~|kYi~J?xY43*9Ik?HH|6F6I~1EyLo(xCe~S3g z=brqIWQLJNV_~1Mi=Dx>dW{nuF$bVR2^@6kb=Uz{mb1BQq6*PTMCX<)+7==Q@`Pmq z5I&M=LOp6Hab%`;#&(3CFjnUX365qGpR)%Nj;>d_L;_bDT#O0M4n{v~nP4V?Rc<4k z1(*?Z6<+~X(G$CVR}i^d3=UB_e}RpRORFVp^a-pwzt<6RM9_sgxzAg5HV963F@WG% zf-T*+{K(iZrhSK9&{5M>BBpTt0atNc!?uSBrzr949B9=vkBMl7FZ11x?wt`Z*Ix%C z7jU`ql8MC~yFNvhul|^f98XhtEH0y#C2AMAovrBSnA;bVE=4AgA{?1We_jM^m15_YE}5o`y*v72U@h&Ckv|IFAU;DX@2v3xW5m%HUZFM)cFL$bRN#3+YjEcz zXHO*7f`nbebJw^BdEi(E4}|0}KVkx_XYoxslw$Mr63Du!l(5PX2Y&Uf7BhhZ3*1zc z0|b(fggN}HEl8Nle9DV}e{*i3myl>29FAFj_eB>`EUE6qRH*>i83wu~0-vDJ#jny? zh54&2j6jj^LB6K3JH_YiXy;XP>`@_%tMCEpY;F7aT3i$*u&QX*?Wxymy{|@@pRHR} zpPzmRvvqeyJ6SZ^wouL70Fv!r{L&73Ay&lX%{se%?b$9l8Vxpoe>R&mPqs;jO$q7; zbyP-rs1I#upGet`MCcj{U=kTn`QYxgIhW_=jP}MXL@`(f4o$K5qa0X1kQ=_)yRJwD zY7)9yjt$u*Z2ErWFX?EBY3JsBe?aVQz>iLbXdAgd=@Q>@sizXElg*q(6Vzek(f=Rb z-u+iypKBESU*@;Tf6hcyyuBxr%xWo6Zd$IwFB3&Ah>D6@@MQk?NpbJb<^4YIbLQkE zvy&Zxu9kM)*J^3^eic5(B_&x^&V(Ce>{8KoGFw*TiLu?SMRC^(^3?Tr^;#L^rH6yt zP|R%QbPd?-Ft8k@rQtF!66bDDD}B*S*M>i_-)=*0sJK5?e<_J8V|sFmc@ddFMAhk4 zkph0_v$+G*DxAm5PfT-c_-o@!@xA%}Yw#92X=^)4H!xm}#$Vd3YDe1$z&vi!5vo!H zv#cw4YEXb9T*VF?%U-K{B6nEa`H+0(UVe^iBpf?(+oqhE2~>03fk)xXB_R(MC?@)o zXC7+i=8Ekbf2`GX#7-RNuDudBAGgAJf#@{1al7y1j^upK*GANBJYTI=5on@6bM9Xp z*)%u}hC&aRN5AJ=@b9u(Zrj#ReuV7%L_1h!fO9;Pd!C&m(7~GnVt6g!1f423jJ$zH zwUK}bYG*KEt^-=6WpW$1mVwRU5^$>B4ucxh8dk!+e*t3JO&}E(D(LdeAynZ4;0(=c zu;5Mss3H}E4BiBm;EVw?t>5NP_U9MAwb2(EXv9r6i`C%zQlHJG0zbyE0&Ci#x`w@X zJ*Eatvc+8!?j3~3^ed{;eF=&xT!Sg;8I>AODF(zh#zIhcyHGnubU2?In~S9ejf=B- zJrjkX=)A(z|f1S#*W?$JOcB$J5?E(t13viRKnTTLJ zo>FD6j1%v^xT%9xXo;n$=WI53rZ*)Yd@0Fb*9+>cOzA(}hQQg`ue+{v{ndv+>=4rBb7SDA1a`*TB=&LlpmSyX! zgYro^_O3ps=?>|N+&9oVj+utk#=cPtuL8?GtnW_yo?X-XLb)+FBFAeQJgc*H!@C6< zse){b&1id?zYmjrpfvAOQ4JB&LgyTe5_@NO_mSw7_lTe7M$D1w-Jaol6YiJie^@=^ zgW&pOr?BT{><+e@=niSD(7D(hNO1JoS#0N91T4;H!4cUt*Hq*QhrLs4*V0=%j4r9} z*Wqy79_!5o>%;vO8_m#llZm~r>4CSQjd#~wlddg|kTzd-wwsjC<1u*a9bOk-7A_Hq zce)nbq6!~;HPgp8x4GT}Z20%mf6dlaUQ9=|P|UVXCh)}Rp;1hQ(rK|=&PP*oxA$%@ z93}S?Vq81jA|u!-*d%mG3}qRX*YWc`2{y)pgl|$st4Ws|-TuCMycQSlWM}%<#lGFA z1qqke#cFcSuP8H|YL-H@vWL52b#w;Cls&yO_i^)#E+HBeyS^`agk9vw0a#A`;@kYF}jUzq`uJoy7>yLdc8tj?9>yQHaC5~ zdU zRqdnQDf83uDYne{V`e{3f0uncKS9?%to3W2c2#vPBSl$pbKIupke!aR{HV_6@A{&l zrRiS=ReM6%@7~TlwrXc`j?m>2tVN=-6oIlNiE>4#NZZFuHJF5lx(u*Z| zPm@EJ%^f4B=KN{!TdbsLehn_o8@bA9W2o04d5gEJ-VR21q_f-Vf7r%Ay4m$wzt5I7 zNbdtCjKe5g*UwPN9FwT~RW`B9=&+uT0;GS1!EvJ84w<;^uajsHPo29u8L!GT^B!^6 z=(Fg>U-MD?^z(Vs)$Su*w~#B&v^o^&>};Cd)i~}G)9PmLBJ#$8Gu|gR|K=qHL`qbh z2hQzXJiFV*722Gye;$}^&xfUJ*Kv9&P!yk3z7Y0qeYZ@G&)aL$p9*Ae%&lyUlTziacJ{1zVEeDy9PeFl}5Co;ZAdw=lAd(k-9w0?!%C! z@uQ5Q-NP+*O6Jk&1Rr*zte|i?t-5G@Yc|Y1t3l{n^{JrGe;{zxvh(erHRz&l7Mb=(qiJ?ebH7K2NgFac}Ek=pC!;Zfu+~<%KfR9Y-s7 z+hF#n-fUs{!|4#}W|&;Y)2IJBO*>0hhO@`;8TjMb(8tMiIGwV?7tV`ad!J{wKrtZk z28OY?AKXihf6eO#`)^doqq#YZ7U?ekdPc*oA{Mur-q#v>mG`1(EPXlMZuV|Uhj#KD zdSJXyHfbkX_?5lwGhK+psk0rLFdV&DWv+Jf_bzr%W&Lc!E>SkquW0AF$KbrE!^(PBimG)9H!c? z;D?RAdu*yeX}5QGG`hEruUVsV{l5EVwzyr~c3iug$trx};<$FauJkVDz)tySdW?(F z&Nt82dAO?ISv4fb!)1B#wC2Ga|2Q$xtIkggf9-7#hf7m$HElHIXm;3m<<@zQR@GNH zRtBI?2e;zgHY8juA=|%jxs8|IOWzrn`vSceU&W-IPqVh2Gye5e!{HEZ%;0ry-&UOH zhw2*-E*a5mh#rnl zf9psI{awuD!i*(412IZAc4@QnRP~bL7bN<7_&O{EHp|@+!D3nMk42RYNdDefq8Rei z{F$z2#=J3#<$67Tr{!K@R%5&^vmV~> z(khRqXWFDA+j_1<8cxs@#mxz|PTirYYZT$w^w*2+ZoE_U_5S`Gys{bhZd3Abe_Xvw z->0eHL>r$Zi(Cm}C&H`3Ihlhb$TuE}QsySa*rMg7JPG@Cn^f1Q65UP1nE8Td*N_$S zIt*9)X!51(I(9zV#j_KQAHB85z;e`NJXpc$SepBjPpse&wVAsfWS{kmC?^nLey@zVnxf5yt$(2}*UMMpJy?~;a1@KsrOmZrb7i~i~s`}Dw_ z-4L5uR;RC*O>Ud>q1mSCY~Nl_>#xDy{<<6=)zW^y>3&b=At}!1>)qv~$>Obt`Rq8a z%|p!f6%Lzxu**+=9C_VA*BAALo`SmEH084%o{gICo?*OK)Zsj8^Jv`+f7NroyRXdQ zak-ml6MuD?mMV((gJhqEvsrUX7I3DU>`~0m>g{X64N|fIgy$W-`ET*{o@q4m~X$d&j)f+ca_CR9=s~cmDQ5HujF`*gd9ix5yLS zXMx_PYyIfT%|gd+o5nii1`?@B%lvGW`Oy-oiFj^6n9krBf1Ry2J6rF_M73J4+RE2h zwL*+t42ZBjN`jq;b_nV(lTinRu%L)mgb5QfmO7|%2v&A}K#L_)Edw@w#VThViZl2; zFqFpz3xb}scZJ-eOW%HJU3`Z zKkXdz5rkrm1tmy$_k$0ijnt{zMJg86$k%9O8fOEef6Q_V1%n?o9%}|mYTZJ?Z>UXA zCa49+)N>Sd>lq|BH8<3|{A7Z78YN$Ya1-0F*DiDx_9=d~^xz9Z5cjs4ph#=gg%{G3 zDN;fdWfExWMnAD7_>G5zt7&c2K<5jeqz2*hv&M7BZY$g9NTml!mm6(rwvad3n<13v7tW?YIUXgg*<<;L`k0eytch<@bs{XMqPO3;1`l&7C@|M1T{?fK^;tLYn| ze(XIDQ9%?dpl90UvAoyAkNkS)W(A^FDKINx3bIaafH)(X;vRHCmvQK4d!ZAfIGMo7 zf7=On;`XTQMDASo5;|d&8V0e@_Bskt~visFg22w*Xt_DAHQTq(mt05e}4baUdmzo1M`C~-DmQ-YzNb}*?AUP;*LJd zwM&dkN(|xZ%cFRd5i>B5bk#;z5Y%Fr8@S~TLG@+nSE=rMOS?!34#`GqC!T`|g*ctq zVYGKZ_ZZv&1g(||d^rj+ff0+~%EQ=Ik)TWD&!s6hb)R2MeKXZ zp*+!l+kj)wU*(V9lCtF{zjd3GyPgJj;JIwG(2KmM4_vDNZ7bXD*kSW#Izj2u3#bju zUr`Ym2qKIG*-{ZeLXYI0)!&RR|{t$kv34N7_{nS#$9f z8Y58$q-q9x2atqH5asRAv|IU|uAq_@ylnZJ6p9Iqc3MFVD2dpX zE)NanTd}%xIQ3Cd=UbXKf36S+3s*zDT9MYxfinw4R3uE@&~>~wTC2Qj(FAdl=@@)H z5Kp~O04@{P6mZjd*|Si;x{@_uKdJ!v0s*RcJ?hn_(B}f708u4HWs+WmHEk zH@AtZ`F`yMs7Z16urx8hVm>#JuWLq;YReOvP&U5<%cY&5TH%K&2yO^P*AP6dHqr@R?z+Q=sqATGn-z={z>gpO8 zHT8~0^zBX&@U_FKe+$FK#bB-ujGWdLaNH7?`e4|x?gu~-W>x621_L9m7U?d62HUN; z8l7~r&A~+a#m^ewIWnc*bwQDOH~hl~k$R%@TmbsPyUqgkAL{X z%kJC#=#tJYzje8405!EPrlBd29TOdIxp8&boE=u(6P;O3p$2epeFQwwHL|D7MVNC_^I3_6^mDSNiNOX|U7@8K z#+m-{B`MJm#D-|h@czYCOxSwBEhJ~kwDsV~mshx{5HK*Bw>V3E;I-_>W_P}dNl93S2JG9Y7}r6!5!Mi)`;596z= z<(R8M;qri;?uAFX}4Np-P2ia6Fh(GFZ<+tW$tTYEZTH`Q#3zuNy-b(s<(sce-Dm6G zFaP%QK7J2fyKpwJ@5Y!%CKPGT09Qb$zgpG7O3Dq9nGR}nqipc4Xf!fGxxv$=Ud+$7 zQaivYHA%TjG=G5!k4~f&s>z!G_&n6G|F+A>BuC^LF!xF4Zlu5K@(P;;%}t(ZASw+3 z57Ki&A3BZnPQe8D@;04Z)-eL?N|NMwq-83~`EG zTxFVHjeiKZAwo6RK-s37ta6w@ItT=QJtg4MWh`7^N(h{9d;jt^SfQwr{$(lgvi6C@ z%h=Ptd8z#qH@#2Y^U=3o_@f7V-h@Bmh@WTQa{RabXX#)1@SC46zx?zrmCw!&N_C9# zN@MkQMIc5%s2|WJX6!2nTnr<*Wk5Ow?oyqlD}NJmqR9=b{c1~EraFF_5K)DwbWL2O zXj-KX1NEx0iXxWE7G4fShD9=iwQgu?5k-idBbFx!h@o4X!xhy;?^KJd@Yz456%yE>aV?TDsw5c4Lf;->T(-;Fu8%E&mFV(USNZk1kG=Cy9Uo=ZvlAVF5i`AyoP*FFtI@SE>2t3s_ z7CgH+e>ep}(sxyN?aWo!f#B%q^GZWO~~{9ET%CMn@oL#VZl6tNT##ebwW1-Rvp~T^GLz>gghd< zOlUmL#%5uqd!HBtJ=0A<@QYC&@qA~t>MT4nqVWK;uF-XSGDLAEFt68Use~G?H6kL- zEL%5YiiQxHJ$2vVrI-VsykcMF-hVMW+(PEdgWUAh*8&t3-{paWBnRw7Gq;dO8PmJ| z8FxV5eV*oD#@pVTe(Q!m^q~0m|2a9+&W&~W)cu2>`#AJgVK9XYY3u% zv^3IKy0)Z$b40!oC=`6cErq1Uuv-~?p1EVi)youb10cY-i`I6D3W_mS4xUJTP2@MJ zFZZj(6VzT`7E(@^MgA}2@jr4fZ}*sX`hT|Zf8-DQcscIDYK9Wg8KVtr=%70w7n3PPav~-0qPh%rLIq`6NU>?mXxgC|7Kv0cL#`lBqX@!}ttdGjbi-Zj)ouK!6PYOs(#7zX$OQUAE96#2f z%w@1N;`+i5PA4x2r93L~MAwbWD8)R|!ehR{oFI(z9J3k`mQvE?6n{8zc0>J!#PG03 z^>(8Prd9r`{}5irOJ8$s$bQ932k7Iay!WF>;SOcILdC!EH9z)fA{}|UU(YmoM1gt zj)W};@rhwD;(#C7z<7v#=Ak0L5boNpPBMpagbNy&KH2H4nYxjCmGS-xt41IiY<6h? zB4zp;R1nI^P&{U!)Ix&#NTiV^r0<)L&5pfDK?$@WbSIJPWq&T|lnn-e6=n}W7l7^_ zvv?oZc6$4b^N*hWga0Gt4;&Bwj(`8qDSzaytSx6n~=}(E^qlav6f$(|DR1IQ4O>raDqHyNlMQuUs&4Ev-$jArcQ9xE!%M zC2jhIf)K)^wZ^0k6qI1P(F>Pp1>9Vm-d85n9utff%G)T0mLa&r^La39v_?4ch-4`b z5As+3ppU&qRLj5PtY5hSgO;w@%SuT)WS*!T9ZPQQf|F&XBwpV@&bDBiaOBW;eRQI>dR=Gwbhb!0P`Ww*kaX!qoD{9 zuEaW03kZ3nALB9-3D%kGnnUqig}cWZ&tph6=7qhyVyjC#BNQb@vF|Er+~>cqacDDDgEuP;!~+c)x{R3R z9)A{>u%IU4jOa$1LZyVdCiP@Ah#T^?4(T#C>bJ#G)974{w#uCF2U{VvY8P4J@MdHW zPk|%jg@W{3kr;=s7CEWR=j^53W`g$loIm0m+3$G8=`S1m@rge{9?yzHVE=DDr|@iV`vFbKSE`PkHE zJFiqX4p%h{s(H+=Qf342^C5^F0h=fIaXA}24T@2o>x|n525e#7!pz0NyJmz8i+_ug zSxBS=l`p$ZAsf6dl`C>$G1UmB6q$+}VBXtn6@X}EpWmGKemDBgyZ^)*?$Q0x+tMD7 z{=oZBtoa+KZItLml>A|%|C6uu{ongVeO~m(ZhXg|f9CGf%eM{bZ8s7r!hQyJm(DOW zu7TJN3$hI2%11{RVWOPOpovLfk$*E%5#ec0d1ca)q)_m8Aexf}`GtsvVyO!mF~Un! zqGBHFm^7$$Eu)KK(jiY_ssT*u)GEMjZQ&rrE+XlYs~#{G7)M1bfFb)>F(eq%m|{T7 zTk6-D9N1FNeZ=fL1oai6G}L|GVnMKm)5sZ;(1zM^3ELyVqCU6ZvH}ZwG=EUOH~cH? zSgr&2QQ$`vVU3~p^Sa?+4en0LeI(`n4f=1n@AcMI!Z{PiAPUKgv`p+EJhc@4_ zDAR*gZ_Zw3|C$`Q@!b?*tW z@*?gaoJSgu9Me|{4DQB*NPouQ2uI`{v-%8W6A-ksk-t;c*&?KzpS96sXo_lQ6?|#D zYa2rKh5h{OOr8gR*0ILjFQ;H8J*rZt2EfuiLUe^#6cQl7&ykDTy4Nd?`B^~zo9 zm8httUO}H+(NCTu|HCG`-9P-yHt*bzZv89A_p!~=KTX%}ul&oOaeqc1pO$~+3Oo0& z`M=McL5{aIvh}v|5!7SqlfVGiUnQtD{4p2-iwU@iG4xM5cL^3zftiBQ+ub8UO~>bY zoX&U4;t5PK{SY>p9Q zn6QRv#Z_!udDypN-~zS?Xu8sP_-nNbn#%*uf4W?KeV-QOE^(O+Mbl!9zw zP0)~AD`>bD!0L#IN?t7X(Y7{1wq2pOV%~vIu~%6sTr_-~+kX{nay%2R%ioUXHs3pg z5rEaXvEBHfz28HCxi>iGq}K0fDP73 zehG2_h~P8_i@8rJ(Koq1X3@wmL-61$(y)LbjB>D<8v(BLl*1XOm%$X4HfVFrmh)uW|X{%^~3)1zRT9@M_J!NqtklkxPCz+b>;W@_)Dg0!WvIed0*2%KD}Q21q#O zgTee$pAe{5gTdfyH4uZv*I%baIsgvF0`xfb9Ut$#tDMh(Q-onW1KxCtVS%>c8ZuTn>zP63!HUXeLKBD3s(Q_MyPH&{QJ*bq>8 zTEo>CBAQbNFzYj=*=jn{=U|3m__(ie> zWPcA&8Jz({dU2Bd&KyW2E|Xv+P$0M;V1aO4WEcyicr03#t_OQ{BzKs-*1S-V` zr{LbH5wvJzVd?;tu&9btV@8k`d3xIaE(Ih=PiLUpCE*Pe8l^Qa?||$X5hzFm!kM2K zh#4b2NMe3W0zIP+f@oL@2%0V3B1nQ?w10avkT(sSG(n=6Bwi=?%Qn$;z_a*ka*m8KI6n?*I}jtQ(|nF zMki(Jo4*{#`KK&sSa3=r-T=8vJhK@PD*^7O^)U@lMIGQDn!&cwki;)HW(bnFl7Dn% zi4kWFxF$29VZ9J{lHWiS8O!;Mf}08f|wFgmwL zg1T_FuyP!Np-Mz%=)~CqbNcrKZ(yMXcgZuHH!wf4(^(MW**za(>E(U+hXjH*2$k6# zP}*S?0XmkYS2Rf1QjG0X!Ui6u-G4}svUOs;Y?0xUBnX6-M8`cI3DUJr6c+iCn~n@Ya9z+Dpfvb>FZox}wE47wt>p7O*}{q`dpUkHbpZd9cYox&CQ|MQ z7?k%kl`{_dsprGgkx3EvC~3=P0IR#qG3X!uMNtE?yo#g5fq*KM`&(E_nbLw@hA=ho z*gzZsB0z*(E@1a~)g@g)FB+dwdxjmmZAwWKy_EQh?#>q0jzf?hG>-=HreQR;yOA)0 zu7$QQ-#88-pYo<@Sk#xBxPNO=(UE)i4h)di&e4YXWAAg92%QJ3u;P-|$ zccZgpTZQ8gptyN7m^KX~T#k=+T?-}1@l}pPv`=}|G>rA-R@t>E=*usTLyS+k#Bbi0 z+d7G9Mh0*d-`bf4;~~-RK>Uf zk~A8*H7r7U6sN9UZRttEx(zt6vSiaf zXA6^FhobaEvC91!fRtS*W2>yMCRK8&uT$wuNjpjvyITT5y3ZtCr_!G2cUbo}6ib^J zf9hTOyB*1y0#Ia29s;PmL9iE4C9fh*6DG9df%>4z@)*O30=zoLQEClEBp1~RGJ zkv#)l>TBs&{HuOTe=q&`IXuHl>F4oG;>B-{G_L;YkAKufD=fdm-w3_)FzxMObf$Rn zO;2|&`DPXT0poxzbvMPxpvu1QTN?65qp6ML)duB9b_dPc0=A+Ltr|d4f>pl4uEZJs zvy3{RTU%+feqhs61JNni_I7E4WlUhm6<^=HmHgLo?vyU4T4yh@+w0j?_jkWp2hVTX z^V9*J8h@LAiNoJKh(PLnr`kw9HyX6l5*@bD>6B8f&73UF~{IAKd zKl?YbBITpQc6pY5xUZ!KJ2nt&Y0~?UgpJ6yzt^Sz_^;(B?Rhin?aKOF>#w8!+IJj^zMlvE_2+r`fzRc2^nvfM;}3j)oqXW?>$T&^JOI+< zyA#~K`xv8Z9*tf#jUWKTm2ex!agp?2dw;l2!U6igiKdZxf~bev^>Co~aO<*r10&yX z^QPedzVzV?kk{yQO`3+2@8?ure?ND7_(0OaU_JcjI_lv+*J%&`xz1%j>9zu}}`riuF>dP7M6ZGVNI zg2}ojk+h%NGlV33UiLED*J%iq@Nel4|AuGM=lcb}3%=pQU+|;8{h;}+JItQ97xoN+ zth4X=TDdobNchPI{wY`~VQbm`Z&=y?@E7d7@5?V(MYj9(r_3CrAFrwRhDcu$4f?jr zr(lv!6h81Xge5GOK7IHDi*eufFMn89_Tv|9(f4`&2OhEBx7~f?g-ZPdgCD#U-3R`I z6HNQIf5BF=?O(8^>~r`F)^QwZk!0Li3RFBh1XS~5exx$Si%-~|0nN6cSvpex@Eso@ zb5(8;Bo0B4PO=7m**wuOFxj2TeBSMI?Yao$iYo$0vEPJKgv(nCP~a!``hVOf?&SRm zKRNojpU7|ZfCQ}t294rUPHLx51{xSXX#$Q)T~D9Dq14N7SU)pnb%Y?hk9Y z4+b+Kzc1|nEHG)GCqy7WZGXh&cmD8m&!7bRk8%zaT=x53AQxQm>mC6#;Pc=24B`HS zhM#->ce%!_4$}1ZJqHB_9GE~aFah=qbzoGG2~>m0Wb(2G!v&iGm-%B?*3>Hby=5n{ z<}wg6ujvCif$74Ap$ix)P(Y!qw_)fc?^A(JfQ5W+eeU5YoBp48Z+{CK`}4i3d~YD( z4FX&K$)~1rJ(a;AW9R{9CCdvrpPw<5fNaA zCHuLP>wJy5+}ZRZ=}P6NqF9yv;h@!WJdS*SEXS{NSPP`}{Q-0ZBbW6GIlfJQ?}8)g zW6WiF1*DPhk7TIe}5#$@%kR;^A~>2RO0a|nZV)j23-9w@gAT>9><~G-Q82e75CCk^KrK-3$dX+5i7JKI!EO za|z$m``yduFMj^eV=K^E;%O)QFZ(OsV^oj}`71v1@S)ec*JrYx+22=v|0TXmCi(g# z+m8gYfb{fVNq>4|5z2u+m_I#kH62AM^AFq%8fVP)2_8Cw7J-+{{e-z|6Qi+$V zEKeCKU~XfDuE;%6iEgvsRlpWpp#{^UnT zpcRl5Liq6CPJT(Q)!+ztN%&NJ_%rX@cO{>^<^2}m`F~5l;5mc>84^!_PFD?7_h%=# z|Kfn4`R9ln=!St$^Z)brt^|+!bN}eB`~U5K|9}23xg){GQm z*qd;_{(rym=2Y_g=312bf9p$|M|ax)$3-@C`aBW)k{6HvP!;9BHTV2K{&|Md=+8rx z++T+%-RFNDqJ(ajzoyXs6{gP8RPhF9!X=E`jVS9iCnG53)Tc~JF^X@kZ%{| zNiVjj1!D5GnczV;H!m({%W9b|I$EC2M^x&DgMX$8-)*MD?qrhkt|o1Jnl+yLHQt_G zZ+KYO-SvJOF7EMWH&ZPAHF!-omL;avpBR-iJjv9LKxo zy;XwIlSbp^9o-1pyKH}Tlm!$6e0S{Oc|DG11>W9nqs3{YRQh>LCQjT1*tSc2jEJ)s zpMUe(t8I4AGG;Z8#WT39GS5&Z!>MJR*6%wkIM`yy&|?4EC3$lc2VpL+?mFMk;_W6! zB!;vMO1im`Uo@mk~?w z)}bMJ&2H)}ot~I|y4(Zw{CbVf>FVpf&VRS#+pA5xG95KrG}O1YIcsV1G=t>Ojg&4h zh;mvV^WAE`FeZMvsAF=$hjX-C(?QEEIUw_RZ z8tkpJoAYu3xihQc9GL4J7SmIP{cvW@wR||v&B4sRRkPU5_W2D{FH> zClWQr1-^W-wAm!ut4rb4F(aobXPBNBohz;~7M+2s?gT7%=eJVU6Z^RdJ8HGkS2b3^3)pmXyska(_J?d5l;!UI)+R1252i^YE$^PRAwRhdFVhmvXzsmhS9d z>J--6dp9<>L11jJljz327Uze1dZu9MkN4q}n7cJ;d|ocGvM<%(GFcdIHU+GacFgCH zHtzvWaIUpa!1a=&v{cCJ5!9N#Vw>%1yCU{Es9Bn4S+)5}?yBXBU4O0Y($UzI)AY}G zSsq+#1Jz!Mf|YjTxAGcUuOUH&nH5HzhrH8;yR7v=HYyIu3Jj$aMYChWrYzfWx4Z>w zZK$NTRn|F2OArH(QFfb~wc$MKz+WP9C^xTaT-{Z2Z^x!}g`rBemuwjrdpfl3GZgBg zjc#F9Xlj93;+=+gdVgK#X!Al}Vk^EDhg>viq1fqUfGa(40Uq70O2Ff7O4rslb@4H4 zk4>>diDn~>=he$rEemmVqGk87=M>L_9@HCuFOQ zzntA&b@I<#zp;B|$P2=Or^1K`ID*`G^v%QL{Le+1Z!x9-C1nFgQb1c%X{n=GL(wjBcTaVS#Hn!q$bUH=5 z>`P@$Xoj?T8ExZ_#yDG+#oarj$6?WwQ++oWmXELe#h1mlt(4-DzQ+No$z#wyZsiu) zn^QFk^M8U|JmFP40=45&{7>*5-Y z@3#8HJ#J&-=T2;^^J-e{s@*-d>5Z4F>rdX(HNB0N#T89W@w{RS{ChOsXI}+Am(N{N zIumlfyq>d`hf%nDlf!iXZg)o~G4iFmjJa`{lz;ASbXylYv)G4LUAdv+c*pgud&Yyq z({oJkd>(92>YLrT8WTj0k9c`ClcjQ)(KK3aghQ+x+5TyI&dD3*{N*ahUB{DaZT-K5 zy|>P`K)YsnKm>Rfbb$BXli|IG2XpkJ_xtzs-lJ)$MM8m!suBt2y4QN=+4G;yZZ$*! z(SK4d!0`ul>*y^gi4co2pWc*9LC&>_hu?xCw2v7c6m|bJX)upNRDYuiCPjIe%q{{S zGMI|)xwz#x8YI13Kx`v}h(GT(pSydfS)G<`QQ|d`Gv+k$E>K1bF^DBp_{h;rZs+z( z6AJFfB$Ak%vdrvUYYghES1BLK!0>sAQ&z5h1V7YJHoD5Xtd!IqOfOhbCXlr zix6^)kg}84{c`>u$U?D*Z`f;v9PTYSnotrh2)fKQS4NymkwevEq!NKiUsby`*Ex~5 zO$^w(gVcR?^Ime3T7T#(GVRli2yKV*?QlI@=B9vOrh#83hhL_KU#5g#riEW7i+`U+ z+!03H;YQq%M%`gX-4RCp*M`7)x$8o<$)KD>r`T}%+4|mfa4?gjdE(FVVs!w$46gAf zO~J^$Qe~bxJNIS*#avY3%Xm(6%W4`}q1>kyJ=R9|3Wy{{2s_Pu5czyjfohvzg(K*b zm+fC`Pt&Sp6Yoqz4+wY(V*W+=41XZ{N84ReS6h7rU7hamP)q zGsteR*-e=L6~ErFqx(IEFwcrH`|#n6+hazo1mnI#kH_#WJwrXMT z1yic|zX(g!G5JpH2t=PJC%bS7R!;gA)0Oh!2inS7D!$PkQ$>w9+4emKitoS_W=g0N zKTfJlFMz#6ST?uW*bxnweD`8%GRZgq`l!Gb|+!iw{|`ixoM07Zp7UsetS15SRCC z+@Bh-o!K{8iQ#RtB7luC5bnjICnF~@9^aTtvJoAc?<8rS%Ejs#g@2xX{ekQoBJr!- zIEk)QaOBp{!F_eM9I892MI`hY^BNi>&$e1$HfAF|8^g>W^)Ijzrs!@4cqmxF9`uwjAsIw12JMmBF2>N0zMH7wsi^)l8CVx6GP@(*->nWt&YA zL7BU4YlJ6@eA$~ZOne_~ zb|SmKN7^yCfZ42wqeJ*jtrPL?t`B( z^`;i8O-be9sO%9;zucrsj)vAPk7}YJ#!Vr;Aj!}BU%dM#*8M2zCUWKmOU4BfJ&Fz@ z9)ybsYJ5mcUu0)})N;m+f=32s2*@k#AtDp*ouHBl4hZ$Sf(QSsq5LJnxr`WQ9Fh*Z zrGFgTY8ex7Bw{}¬Dl%H%BqALigDAmb|<;5M89xfuq*oc(K|=Ce8KB9w%lWKd79 zo6^0?l;ZlgITsB8Q&eS4yji9e+qpMgQ$)ifOO0j;fn?Z90^#QN*wpM^8p^Vd*RD%C zD5bC$4I*@xageg$0OAf;^#SJD0H09CGJg%^(R#0&zt~rfvPjN>3nQ3teb|xK!MhyQ zjl~jhrPs|u7(PB0FagBvcJKDaHy7G{!mUnH(Cm*;T5K6dW4g*yJ(DiWt6EJiWb@m? zY#$;&vdYp#2J{|`TFzHkyxDJrE9;4%XMQ1W?*QA?-afczpCXk-8Y}TAiaf3_lYiU8 zk#>}i1NB^Tsdj?+y?|CeK$Om42r}yHppU@`y6Z%cH(4UHtlf5-CV#y`{AX4w%#6%o zduTyy#D085h@J4}(z4RAavsad#N<-hf^_-ucxh$m69!C|*S)OSEnLy*H zZdSDZ__BLp=K%|7bIJ8Fl)PDg=6`RNdB?MTno}yZg|AZk41Z36sWmVKr}8bt-RLpk z`zl3)9jkyDf_@uP`u@eefdZ`o**HQaKuPjqfcVq<&IBgoeu2PhIG@tdp)fR83*Ci1 z>mWj#^3E><04|IR5goI)>b6agVcm?Kz($U1MLPe!7*jgn_xIW6Z0aRA*ndLP2{g|- zpsFpuMv|KPx<*(kKrT*g?^W&>k5T9)X@QSwC`tdL9)DuraSK4-8`vt(F& zQ_y%0KB5AsY;-Kt9e$YeN&@0};V+SK2%OK-d3l&IrK(ICVQy;+#icKu8);zCJVO8S@yk( z3fqvUs*c-?SC|@)&eRcYXXIO;noBR;D^4JMS{F^RlDY!wE$#Kk>VM!jJ7|7&1uI}i z8!DJl=8MPdh9a-yoeh~!Wg1mdb!8vH*}?P}BLdeOBiEfMnE3-n0R&IMn3HX|!Q`eN zQaZlL!11)+`_o1k<0&|S+d=aIQrB$(*tqa1jDlqFls_9V;Ao{-vFZW~;fZR_fQ%R* z(D0+A-EagSzLq<$bAP4dujng#4`Jm+oLrMOij*u@G20t6(|)LSra^1@bp_#X=d$J$ z;hgMv*=bcgz_NgGJooQ4B?EJ&D?J$P<|Ips;p{C{NWPIY$w?LtFXqKPIQA#9l53zC znjZ~-D0)c;&vHTdE=A641|aip@FO6oyx5%(`wKHjPaS%6mQTV_-_LjJFP6$|r1~!eED|7$G6yb8h6}wZHTx2%Jp${u zADvKrGF=J9Dj>%}lHAsyipD%Lk?-H<&_-Jr{U7x%j;VcS%pk z&0egrnxK~oA2#C#x7yJre|kI6KaBtis$I0J-49r}2q0nh zca&=oHdH3RFifS$=08+JTpcV|q2Ao5*cQr;TlE!opfU)oG6@UtC8ace2q@^-|cu(Z3x$7o2wsccu9_ z%!IX!O@H}Ewz+#;Tn8dc+SMK;jh_p{a?Qihm5FsWI)UM1y`WLZvWm39qd(uxVmU*8 z5MBL>@j*$oN!C!6H~{iTGC3^yL(q!HRCHi1ndX)GeVT}_5YaX&)dpZ@@!0Hm75J+GvM#0{T6b-8JIAA2%f0|#5 z^>Y#ChGI$97wxH8G%Z!T?Itqdcx24{-WH_}IPBkoevw`&nNa|}xU zQq};!WY+gBB}L$YD;Q~&urH{Nt6%cAEM2Zg#+5sG$@r$@g!w9|+Zwsz3CjV0^5A&P zUz>}DYbbf4x4}o39dN&4rdA#(lMu-QvB%~6ufLv9cg;I@g1Yk-UWzk8;`?Jq{(rCl zZa3==hnrSs57QD`xFkA@n9~ucO7%~30JbF49Tztvs0WbNNostIAmKu>DvB$Gd0}Os zp)+y%-K>x;b)kh08z}FD(9j~pj2G=Zn|yCA7T*|kqodC^SZT9I@-Q3zowRm1$%nH? z9{F@Q$6@9wu-he=1+#FocIuDpIbRKwIH$P29 z@)Dm&_MYi_0cP$F4LoE>#S3%}_p|``D)wm(Mw^%JWerD=uPTo?%723}>Mhn5*@C)E zB4_#6{YG|n4HC#UIg1G+*w zkWU-dkQLsrTPlm8G@vTUd zRB7f*EC<$bGrl4>Hlo=$v(he^!$rx{5wwJ1{D3fQOHc>cQdISYv(Q zzp7rP1Ml)9s)(-zOVewP4W@BdGz#+!aSit1`(55=x&0tbfSNoN^iBX9%EuDYu5xuLYv2{$%c_Pwv2B>f$V}3Y zPKoj1d#9dB)!TgOqkq`lh(pBiz)dMi`%U=jLv0?=;c^clI~8s(D34Mic&H$FgU{?t znz$Nc*xuA3RKOp6V%gVc9Fwbf`kVa&+Dq`Q+hgi4tO9ju$_j-fJZDy(C?-IE&t42b z4|j%E0e_eI=^;tWXTR5vueId2_TMBrVL#p-Ih8R?Ng@nu41XY6c3*T%K)3_K?y&7h z0N;TFeR{7CeMv(x8_VlzK=C_j5{Z$|o*BNx3D=rHlou8+`K;h9h#1->)|2!Am!b=O6y?EO*XZoHM!`yFTKFHi9x*H1S zg0p#0VSi7T|9vwZ!i*1k->9?JVbcnl?66y*LN;++J;#l&ZW+O~xId$xPf=@$o*8jw z$q-Ki_Fi|@>qog4GrnC2_adXc|1rNF|FZJN!b`kCFc>-=aR6`!oaba};%~J=hP6I5 z?r0x0B;9*k%VP(`V?n1YZNSNNT*!~{2}%3SSAX)v8}W9u8P4v{4jMks#}4*!09y8`?a3U3F-;P4ZXB zoU6McmX>Z2T%y7=H9+yqZ&^hF`6 ziht&*B~3MM3eoB5$3cDx+6sEE``X=ivb`bBop51#2G;x>kN^y zy5%G@t}+CrYXPU!p=b=B!P{Ke@4~Fc;wh^6Q4@;vcEdX$fMwKtd?2(mijJjjzPPzP zv2tV^J{w7Rqs=vEBp8qHWtm^)4)(v0v46~Bb?gLVC45gbDOOP{w35|g^!Sy5|8m3rzw!BhQ$Z3m7VyS`dgZC zNYUIf>YbzJA77%J{>!k+FzjL#+h?PH3YNd-`H#oY|8>>Me@d3hT+4_F6N&m|-_cvw zbWlj%@*+1nQP&06`4T8j`tn)>4*`(Z|JXcmoP|fj>L}T>kNedYO&zb0WPeM_Edk9~ zn9Jo{A3)cq7P8|l6cPt=nCLkv*mIE^U`U~p^VdA z;b;N?P*x_dJVYD8ra~G?f`1MDF%LG^gZbi&4UKX*mZTDX>gTH+ubnfxydUH3?ipyt z5-yI+3QAy&B;kgpUrDt*WU}j0{7Q;8I_`5tdB3#9#U7R3CYE6j1T9X`LP;0M4;Mr@ zQ$iUdYXznmLzEKdTL2|}Ib9Q@X2&BAo#aX$c*KS@Q{!teG_*^7Oe_|wnGFuP_V67h$^ z)*}1{i<85j&Mb|gY1`oWbHrk5AR28AZa{2zMfdVQ_*ZMiM6Un*9{7XM7Uuf?u*{z*BVNJlcK9% zmzI^6Y!L)S)Ia0WAfe1Lm!o)>XqC`U(wPkfxq z6*1+a(RTVGLw|4e?Ib9hM|O3!%-t-5-)Q`}wuQLAoCZ?AN6BHsRk@tR=%AFY{5C^i zy!@mNh9L#XM@Ad)>GZ3fsFJ2ov?fi+#j9}mzG z8b(A4Eikus<_U_Cmib3QF8T$C4M88A^r3PHWnAUiE`KU#+P6#)MKa#Omail$x z9&KhHeZiGMTxhdhx`#m^^`-LZ{C< z`xMjBhJW8n31PGyV8fe%8dw2)`^CtO4Kn(wXx#|%xNUgvm+6gk>b5Lsq0u43O>Mjc zoaBzwNc(;Fxtk#d#9V9dN(fDHL$J;fxbHDLp4@aY5C>EY{-%S)&@{*}T%}=JC83KJ zy%)~3^oj-IYT`pA7%>n$=WNK*$TSGRhrgv8Q-4=(NLD$kKgLuH4cvoN zq7n~x6H1)=D;7;d?ZSyuYLF|*N7sX(aYMMMIS1r<%&~&P^UIAZ%=9}01#ePCYxtUj zG__|z2XD=rEjvO3T4x|r30X`;st^4QXriQaUh(HvS>dPrb*_u#rnZRUXw4$o#XORm zZGSseIo;}FFynQzlUrww<=D-QKwEeire0v(=}=-cTuR1w{fgZrf#9nKEbx2!!wL8i zk5E)wF7n(?Y~YA1#7lxp0zS+9+#yod>+qfT({QVYj)Wj{m z+hV)J4Wg!N3@K1k@`;R2!^h0tQwA=NPJacK_p8(wI$EnVNK&+#jsj<_?D`=bdxk_w zt>YDXlK2W{M--(Dmr1;!W#N^34#dJX&98t*9LMoW;SQZUTL@tiZ=y=wAlQWOS1*)3 zBCc<3R1Q=`ftq9_hXKf~ZoZxtI)S~2SU7+dSsP)L0wC`3cX(p^<_e8+f$E2%Mt@Zj zQJ^Nz<2AfkV%tfGkr3g4aOjMM64QgM?H1P?*kM9rh)%~W6k~3^gYdBXd9=K{k%(WX zA{~6h*ieHjX1~gFR(%M&nKI`^DV8lrY-<^<%+byD2f-zJxRE>~u##svhHBw^ltgju zbWhQ{BiS=F1RAA=i}D0ST=OM3bAMC{mA&fE`S~lt?mj@S&S5qi%G2-(##^KQ$C-;PjvdM zsIu$^gbwy`^H_?8669;NYmdi(SvC7FGFz0mP1<$$ujD)z++$4D;RQjdlz-nXE%&k_ z^<}9JHMF!QG@W~5B)ZHIv5ZJ#Nx9fH_U;1Z?qwyF<6xeF{ClPN-VP9V%%R+~Ro=<8 z?bri%zK$~wd^VM|*L95~K}fmv{jx&1M+ER7O;pH49Hi<)SjSI^Mkd8ns-5AwYCt{yi2tNu!r_1-$HRUK$?~&z8_w^pki1q} z3|44w9r6`^K;A4lgk4|m2bh&FO@T6tWeGPP;X)u3vO~Z8rn}jL{X~YV8GZ%n=D{7E zS=Ot$L0lnih4)i6t$$P;#K_T0HaTge7v?@v5UrdX5J@j%+YPDr(uy%_F7n2q8QZHf z*&|k7l-MNtqlbFV06sv$zZ92%L=n-dRlfq|ec%FYBY=|~y7YUy$N+u$E#DPD?b9ZN zc(QIOx zqmMYQ5Mc2@idmFAse7A-(;Fzoe#{TWA>f&Gu88h+S;Nutf2mgGUZ?yZPIzwJuo54oFw$adfG@;B8y485uLs7EJhY9np2@Cb<4f3sb` zt>6j#gDUcoDWh8=eU5)LCMX%}hZFlz@t3hZu;ZLrqa^fe{%X)ec5H86+<;* zj9GA$1VGoDEAD+`u#VQ}8)^EZA+?|$>GwGzvvDq#atDaYKW@~i<*z!lkvdy?&v_@w zFK4?R-_`n3B2~ammkD;2HIC2pJ;pk2bAO+~K>ve*8T|Rs~{d zF1E>>p?2PwZc=~Ge((&gT+1KWy9x%IHv**)K+=0&(pwb|^PBYQSrl`GUdIU-68Qmi za#hl~=8XTNfk%|-2L&OUXl_Re6x8Yf^3{sUzybZS0P@(`n`TmVC!Qo}Z%s7ofpZMY zIyn0VK<{T#AuSj$Vs<@Coe_jgh&!P~W*|L%rwPGgy8(YM4s*SdUAp%zur-M+b;im3p>XTmt9&jU|vAfP= zhJHnU(B0ih&T%uTBoO0+>ps?j0C2fZ5_OIn#6 zb;g$p9Y=p`^+QX#4NY`p>#>EWCk!z2 zDgvpjP8YF8+zAfnWlCh13?*HoQR|c%w;t~4?cTX3#FsWen($@6kqjEQ14uGHi@z+B zzOO|g&77(dRNR&SG%SsZ8y>ugiCQ^+9j zL|T8)A)b&P*chMwPP5=xbC#v-UX>Mf*Jl4ph0MEgv-o7)sOzi87BO7As?FTo)HQoF ztw1l#yY;%O&{8flPe~kuxi#1*qP%<^Rs^d9tS0R~H${0Br7%(5Y4|u3QEVwy?k#ub z-fg_q9nis{BLB^S|@H70{zR1dIf(1 zEe7!)l@p^}s13i*p;2E68X0!deul!hIh{*qtQ>0Jd{ok1TWUV^t>Xxx;>K%f+F8Jh z>JLxCwK(Dp#giiSIg^?5Sv6~Doy z1i;HrGBebl(9JbtqvzJ)*a#4`65Pe`eu>Cu;-Ugx*T=9jq1Bd0eNmqC3+JjL;t56* z%E!a;{h8>N-?m_3dtcrNO|U^-w^%FHrbj9 zl2p2^*Spu_i*MEr8|E%Sdr3CR_3NWCmoV^3{T}jnQBL2c<74`+8&`j6=3Hcs!k`Q_ zi=jV0`XRo1cd}-ST>k4@lzI;x+}*4k^%URXxmA9OCDE&{k2r8}&Gv!1_1hoz1=1@9 zpIh9gP4HYP z*&O`8l;zGsm|>gMZ<2p~?~ro75h19V(^H!ilA#)g3*^gV4SOs3yD;+@qB@(O4PIfB zR8NP%>a2O{HfYja{TLaM{5=OQ2}Y3|!DBx6?&iwZ6Yw*$9Iq^HEE1d$WYbC%aRNGC zKA63#si_~CszyXTS0K92!hs~~pgvigH(22pE@T+aiPEanDW8Asm3*Tvx-pdTTc$%n z*9WFcS$wrKZ3E)JSFc%r)ZpT;G~*J_U(r0$#41z zqd=Q~28@#=D^h)jD;LOv*8M$jY7-Xyu!6 zEE_>hX#fW`+)hhNY+eY{S5|0;Ml)gadG3lX>s)d>v*U1{L>3ooX6@I}a;Wl7*G;5r zO4xe%({dOAKQKAR3OU(2&BL@Wb8-#KON?J!6x}K^w?w_}>z3D=qFf}9%8Hr~vSi`B zsfi(z*`$Ae;CQK66687HHIpISW2??2VNQ z0lLGMEe)x3a+HNB8Ft#7vlF#wW`!FHZDx*&BLhQWUX<61|3+Hth>hXq{}pMC{pg34 z1^ypO<$ofrbNE(Bj z{N@ejo;soTNX5H1S_=(b;=yzd<&bf09qfo_eT^Ta`}40|T?9l+6H1LTh|rcW`?sJE zij8G}hX)6K%m*p0Pz(n=xDrus??NP-fAdQac1R>90z*&{jOe9mps#N_;TECW1g2uN zv}9c`h^frAPb59N`HpSUh;6b-&m4+pAj58@l)5Eec@2+Y*(H%oOgHog=Kw=q^=UHr z=GJ)TNW>h=DZzQXPlGg&xlN{i?$=Ah#Z*}*kr)}Y82TSu;6LNB%^z-8mt`vvFn>Bv zgKv@T3ae1mGG;3J-ApPCq9Oeu9R#go^3{+*txAA?cl(y1G_Xb#KpK5eRT4Hw3aaAHR2gqaw$?A^;}o(b%e^NsIO>=Zt-@E}6z z94t~KL}dy+F4fq7^*!G-Vo$kG;STYp7bA4gSc ztKBkr;W*f3Me@o>2w&?Po6%J(B?+sDYHmq)zR3v{N>mb6I(;Sika0XiA5y2r)9 z?ot}gT-`=VN>+pfCU?{xb$Av#v`tYt+f$7?-u34r*JP)m%`pO@UlZ>dCUxUVnnkc% z$fOL(N9wEb%7cz?YzLTSd4FG@JS5Ue$=s3qg+8PiDJg7N%-k;j%lyikLTuvdyAjV4 za;^+Dz&s0fCJ7;hy31Pdde@omoXT0o2@BHeS#4=}j5TlF`4Y8!l(z{?yZYYy#FvX(K!4>rRtQUx_qJf* zY8OjIbTf{1_c+PjJ0p#Uq9u^!hUDjoaOUEY`ha9eQHa*ED1?y9%G`cj5bY!DH2k;) zl2<~UEGU*BL6+aG==I@VCeecb2}5AxZy~+aZl6DTHb*TLfmz#jq7{5=rEPpFRyLLN zflQdd*5DEzdL5Jq=YKM%#^sJHn0*#(%gRVB3&naaSac^2SBI?P4PG6fUVITz zT0sUYb-FVA`bZm-rVTo<_4fO?Al4BDo`)5vAmIzYZULE!YoRkY* z6xQtz%<#kRG%QQ9FXK<#=?eK(f3iufsYZ3rZzFsL8eMg$f`4*M-2@#&Ri;i-yT-W+ zsgZe~Q;mOA@(#n=1rTI6GWsMk?2J8G^cYEf%gnQu?9Sluj4bEApH7A9qT^G*&a)Ta z5$0GMam#J;4ysIDa;wneTyxLNsg!Qh0wA`Q@DWk{rj#xficDQnvxQK?QmZ!~A8SBk zCSj`=efMWzTYoM2y+Uqw|AdV2lIyyBlgfk14m!6~BYwPGK9uURfK_&iYh6&s6ymWW z;Cnp|ZFPEY)rtpTNa#C;qjs01<{7?+NCv;=MaIP+qv{jgSwBG>Tj?6J8%=2A!8C&J@ z{tDt>SAX*!Nuig6PStO7JEHxBcqYgmS9{bD`HqAyhWEs+B4VQ%mubS&S2wm8_RW4+ zOH>+D=*OnQ+atD-ADnXzpF^uIOu)=?z|o8nm-`!uR}~?cJmJf8Q4)5!h=2*gsR6EN z%V($3&4VN+GC@hA_u7~XPF^$4{n#Z$C>ajuq<<`cD}Zz8rng1eW1jtWd)pqWXc2i< z8HoaG(FL1~^o^SE;Jgo{pEfD<9qCJ2vbe(qKWC8blKRTf7`x?q{5OI!0tiaeh%AjW z5pNahu#2asnTXO)BZbX>@;lPYiM07~T>8wP=IveGqOK)?k&Oi77sgx!o>4z22U-k! zG(;%`Jzzt*%GH095ZQ#1n~{*v=ZWp|)KfJ@!-JcUBI7!MnfN6$8j~wVjee2lM3~o? zU%$0O_)wd*$wyH1p2^16maLafF%dO?t2H{}{Txyn1f4gmL(-gs+5EihdM6vQf+6^_ zZR$)c&us9ZC*KVO*&p5IRiul*-IY^%n~euuj10H^i);H>H{Z^jJ5Wm+gy_Q-DP4I| zC)i%Tbby7vo~_yBkJHiJ9+%nQQj9eJX)tWb;=}U@(XeElnO&}5h&=#43W1D&NPt`W zJ%`}a7L!&#uFFH~{fVVC?=<_-K#M;Vkv*FV z#WD3IT?IMnA$_X6!22$YWMWRU?7J@`7desz0qsVEbY3-x(a>*H12~Z@lc=v?;ptTE zIKZ*e?%lor_xbN;Qm-+&@J{0uv+3B^6ocoPj1fhOJ%|51zfNp}NzRggOzf0Me#`1g z%=nV<6??gv6vANgD7;-Q0t| zh|%jQSkq0?0?(F)E>SwA1AfuYZO{~uKklTa$h)p#;Prt>jA>w( zZ!aC`WJ(NlHw3ir-G;Y+DK;0PfxYAX(2<=|B^HR(wtO|P@voWRw1sHPei)jzq|ID6 zG{x^^#HVYEXHU$ek>VK!9lDP_dvpt`zd_IX>B1C{oL_#Qi3rT&>K1-t47mlhZJ1BH z+}8@`ag~he-Rw%?WhwjZuQ$wt$kPPaaV-3Z`s336AgF0iS*v}2BEE)<61&ifNJZ-R z)vQQQpGkaoM`7P*quT9!WWI1EnQwKUUU|R~C)qS`t-$}o+qX5Tt~2fZm)xbO z6b1~KFE?q)vMrxLK4Rabz<_Mb*???P`R^&g%v!Vd-tU{FQkA(G+iJ^FcdNVA5_&Fu z@_G)h&jXC1OCTzLdXMvPW%^sbz0EJP@&2ri4`-aTo6(F;hmWjJ(PC?8Pt`@MwGg@1 z_8g}>jB35yB>d7JJcDT8F1Gi@5FM5q(=L>*UA^FZ~*eo zPi^F_cfNLeW>R0)9x)&b^TyNBk@aFdsBc-sLLjx**o@YhlVx>4GuUTJLwCB2= z?Yw0brZ~a*(lL_jK3ScXT{}{g+sosMXMqCC4X4`sh%>9-7qIwqFW}#{0^aYxb^?BF zX`9Ql(Lf*`4ZNpU$Fo%~cgj89JKHkzv$Id`yfP~yuIM>auoqV6Ut@ywMm8Z3JI1cyc(0aE(0HDKTWhP~A`a?F9?_b|1 zxB18S&b{|eJ^B6Oy&W((ua|)uD<(LAX%lmz1es=1~oq2Ni z;qv`HdSvO~kgv8x&X}E6PMbqfn6uYLKRL}y1Hk=%_fEW(C3ohb_f9+sZzSahP9bO7 zawjBoAP+C~9>7N)ji>6@-W31fZpTQSd<0+ z0}TE*82RW80RIit$BV>35+LpakoXI1^sivXuXt~ZZ_x4VU{}RJ(&(Hf-m4gSt-90M zS%<2BPB3GK$i6=a;}&pAy|xHJD2C2dgYc%(TX_(wxR`Am8ko)z)N{ls;;QFUF)RGe z8$xv^h~II-id1LB!wtDvx&+M=n3&I%8d8h|qY_Gdzj8LwUQw|qOadyw2$nj}OcSzd zUTG0<50MuOchPbf7TL_mRup$M2FAz-JhR__pKvFGhQNkN5WQw6LG&Q}0nYXYIQ)R! z{sAoh3v4(W{QxN(_%HSYJnjv$_XTE}S;mL%BbPtGjK4tScp(9hPxc$+^bJ1!0X>br z!FIh0ankC6Hg_?nQP6Br8*4_(US3NA4`*|u%jSBhE--L}D>d^_BZ1jNQ)rDQmC2IB?xL8-B99+dazY-=2>}VcnWoS`%P-X8q|Jj@JRf)joN9KO77(A0sCV_SlML%9;G zR@oXyQZ#O77aLm7NLtS_p%Eki3mg85f^u_oEz?`IvCEd(F!mP-8R9*S34+(D z9vE2}Hw8DY4q=_EZ{|pli_X-w&D0!=!qJ6#4-hXrq^v*)M=w72g#;coJPGEi>g)~X`V(Y*0{g7l>u~3oLZ#zf|mF59uN21Y#ENW!|J21eLYK_^dBb&48VQl}Qms z-c|>Dx|JZJ;_P~X=OLHrL3dsT*fWh-;|Q9a%^L#+w9-XQ5Q-0fai?L-a}$md&VhMW z$0d*}!sxtmm?q)Gqz>}vpd5iw6G0u2pmEU+`c|b_5CcV`b}91+?wEsERweFng1hq? zZ$_D5gRQT6e`~qQ> zV7fnm(0`Z33Vnlr<7QS(O3SY3gmUJhh-Z3vzPfeNn$^Z#AV7Mc)fk#-Ao!AmtRy@( zLW72+E}A)0Hdm{-d-!ZQI5QwB5{lVEXX*yH!NV6BMvS4w0UeAkXE^i%n>EgIeDI1= zv7l94#LjbzdxllgQ-XyFf+;$qjXyl!z0vAvae+`o;Kf8_@YT0C@EUIKMB33qgkviif{HFOlG!irNLvQrFJc9#bvMv|=;jmJxdt zcOoL9R-LYYaHZL{8Q&w`tnDCnjgcY4@-wbtousn|YMa`C%y&!549t$GQ)`_uY#|m| zB_}r6n0CcY=k*puZIM9h$042Tfq{yIA(LEZI%q3kjc(fW&E90T!Gvx@9O1SxfUCBF z&kB@q!zUbqV8}G?+!qBinHVik+7dWH5`Yz#je{9~L4-E)kYO4BV$Q$qgAaxTqwnIk zu72t4=m&VMl+Ew)zRzA?Jb%ZQ5}Y1Y-tYM62f(ejmp0dvvi$g>pD~m$oRwe*7&UOI z9tii`IPzk9vX%hbzS%((wq8bBUKu6Z+S~^{EK+!h6mxFiR;h_#S^m0jVec>Oyl~u> zT8o!|wlLaaV=-;(@Y0ZS0>(YuD!L9arF#i)Nf0n*VWH$m(9G2^G=zzh{HO@l_rw4Xc+4#F`+e~sWz`{KWf{m=Jk_f^Ji%P_}(1I#G(gDlebac6}; zfZZeyK}9ltf#?UY@~59~zWA?i0A>j|omuoZNV4&cyCZh|wvUKspZUzk2Mzz`tG@up zeVl#!J{bqTbrM+PKnbRfnwy7Igmu}LGsb)GW`p7*#MT7S9Xe7AGjoH*84(aBblu)V zv2S#U+pO`NePg2vBEng;8-Ncl_H+z?Fd&56Gj1BIRZ5FHec#P z%^^As^;WCxAbz-*+>&8|bzn2;BlUnh31}FJxWFXyo#V_ZVs>>!V}k@T?maW;p3U_(HlS9cZ6CH0QdfuAN1b9KIGY${E4_?;ql!0wh@DruP;}sg zQbr9_X7)u<%aj)caZXWJuLwT-ddS9p$Pbix)Z7;)lqy9cHb_@^82bhbY7>_fKLyi4 zG`q)?w)Xzu8<~f!kRgPMP*>pIe;$pXr9ML7=>c_d>zTkb-$CjcwT*Fd6&vwvg&RAv zjeH^+G+`sk?Z86*dIQ0z=VQGCpS+LcxwSYszuyNk4Xwc@RZ-i3B%5H}48k zzuThv_U-?`yN}GIu1a~nzU}O{4u9!SueZ#K-+7b0!sm#37z%>Q z_^9(Dp$W;Qb90cl01>xAlqo1+J3CaL#w$pn1YKvmzyTfFs39nSpl2ytV8pW9%)uZY zZp9xq=NN@#;#FGiRWu`&;vtbGh!aCUx~EYGR4=&&ZTh(9L5}0uq+Er>)R_(JJa<~v zhL+ATt4G{-16}$)P7!deK&9GK3(*LCP&)gwDIVMPjV8KuLUE)e)C5qzHlKWZuQ4hlz@N z(^1g^Gi8j)GlMb`0n;qI6jFzO&)FFuu-yKRd6CqoRL1`bt-j*_H2;#r z`Ah!n%V+(z$3JxM&phV$oL9#Azj8s`+2_W+h~hYZ!&o^kIz<>lje&`R?^#1lW9d&& zV3jl7PXX!6an54qgHm!<76}qDrByM97B%w{mUh)V^Mhz9S3eR0H)YFQcnaArkf?AnNSU?m;r@L)4W7uh)g%J^qHznxiD~K_?UYX zS20I_oOzh5F_yUzCR_s@GXKE7=KW_qV9r&7te~6s#NzMzDdW=5oc)Jxe)+af-ZA?b z|Nog2mbuV<+=G45y~q4#KK0H|8tm^e94WoQKj|=89zyS6$H|Z%T;}YQ&`ZNFXe!c)12mi{$eCLgepL2|N>@bODE5u^g z3bPP6o9lH}rDbi{K{ECB17br9C%pxP}vMRN&Gj-~tItUZGwsoa(gy&)w zr&xF{K}Bv7J2V3=L<%Uhqz9K}5#jtyyD>ui4Qf~Qr$T4U^(Ckdx_Qxeoc1do`^-ZV z_|2~`9;o=uhwjZ=^QXP|k(a^aJ*-FoY~G4xKD1l2iX)#jzAkue0G-U`oSNBx`!!P* z26&bmFdu`qwnlV>dkwyJ)YeNDQJxvV+#X8?u3b`sMLdsdPA3mafh!n6d?q5c^guIF z0JUAU7zw#tWc8>0ehoJw=as3{7a!txKJ!aQ{+KuYo;PMc=8p7}yZg|@Mp=ApB>NiE z?|PW|hfY=5kM)mVIq{st-ejSF$7{XwX1X}-7z;Y!I9u2Gj(Jj1t|O8JrOe z=dIp?u?$#fl=Txt-ma8*Oh64VW+&WweV1!C7Fuc`1qUD`*ttTL>>ShcgogCyqfAUJ zWA!yZ6(Kf_t8_!^xiAwN%pt&&BOcxn{0>Q_GdpfdjdxOh0 zL+%r%Z08kUz^M+Y<+I`#XuyfoCA-e=PYj+Z(HSk4Sz*uW2xnEavUhp_8#|(R%&2J^ zEF@0U>tDPD=Bg3S*UOM+X8(TbX>EfHp@n8L8!z|eHqk#jsoy8TbcGy`7dam1g1_ze zk2Sv!pWe1QON{S-JWj;EbV#nv{2H^?*LuTuzNa&O&KG5#I8^bE*yuAi;eW0_yz39Y z!U@jxB2(@dxMKlzf!jxaTdwDUckxOH9Mtx(jNi(3 zV+!U|kV*EsG+Q+@N-<49Ga$K-$(~b7Pz#4~Y_o!eb>kpj_Z-8w4aR&MG9B@AV0L6} zSM|!QW~nJm-G8Yv_bSKpu0JBUljE7Z$MX$+#p54&yvKB4K$dv>;zztKeZjB%`)_~o zhYifSw;aEJ-}~)PPe|kuDcLJ9SeI~$9ojtci!hz3kq((fQQctF1uLavj!bTrwDN)` z12MC)$Zl}}CPMbw#I=J1fkUpihMpLk-Sn)-+;;OJqb0P2Ww{Y%CX{R+lqz4(?@e@+ z@kr*ekT0D!CsIy0rY^#H&AQhuD5bfn-|>bmw=z6`wO{(`Fp&EC`ZZQN?<lOZp>E@++Qvueo_bU0t#!^RVXHP>!h? zXL?y*n4%PR=sBoWwr1-EICNmaNrbwA5uiA6L5+;$UYlHf>OgD4&KOtcz_ex^3Oy|! zu$K{k5(ib<3UM)qlJdo%ap-VBd*My8;pczIyhNCu)#B($n|v=n}~VH7+op}u#3TKoWLuQpf+%!Iz;Y8IJ=mO zFYrKl!jn81ihljP;{87Z`u-uvO8?No*P%{-;pdkj*oBCpN>~#(sCXd!1bSzZizq@V zF%*#x(6g|XVwYf}p)-EyaKW8GTnpT)?wtv)yM=cbUng}jfM?d<*8^>_Sj3=<#rYq9 z6td`7{ShGr^OCQVv+(b-^EX%`irD8Kz}o^+am`={Tmv0E9T3F?U<-`~gxe8dp+^jV z0;sIM%AF%%4fi-~;YNW%*a8x|zvv&J6)@+H3W(~S1g#uMe%M=CAC=&}C$~Qp_5o0> zj{tILYVhS|8l>TY%rk$wB>SryyIBjynKZk8)qemf*cTuE@@Gki-J3*=1>=nv%$@hX zpFKDZ+@xGA3I2-UuON|9gmaC-{B{3-al#qP>!g<3qNqfO?DYN|&54{wiSy#K%VRmHN<`#QPoTMks#=jd{$*82%#nD_hrzS{5W{gL{|32Y5O zBVneKq%_hHcsS*t9}KsFJW3Ch@X3*7=p$k+K1QlsABM zs)XGejD3i(y9VUCz%w(uH$WTOkv$EE0y@ES*V7FY1FPqPC?3yDgd0XMz->Jvj-Y8=&^2JD%sah|5i13kv{Z3AB1k0LK&Jq)Q6ADY zO@Ybx=)D2fG~nGDl;|n;O4^x!!l-@y7j4k+9AhC2ED+9DO~sKO`$ndu0@X=ICE#{) z2-5*CX-C%8fy)(WmSR&dqh3=N)Y#x!y(U2BX2++Bw6npKrA(rJPlFZpj@Ylox5u= zT#O9r0>G%$G{gdb6OUIFc4T{+2n>4-#6B2VX23d2>^#MOPLrB91VyO}$Z4Ch zAt?VPyKpE_*9jbZw??Wc@j0u80^?5L(8sjH0HQ?;H5|GXXq*$Qtd~cv`=%=4^ zHk_*nP$kEO1Yc(+%JUnbY>3ELKONVlw$70r#PmwnGUh;E`qeal|1cIj7XuGMEFf$o zpY;aVA;ZY1D!-&A;|>1k0-%TR43sq(*0OJDDCLT>JUI&J;(Zne-)->!eVw7OWj*kz z=l56u-46XSAfbR1?hUB#@lO@!r85DbBrL^bO*95n z3tjP@x6a7x&d2G08W>ZqZ#|?k68q>+;9@L@X&)Of_SEGB%(+8{rXaD;DZOy&LO>O| zch?ZQS)$mtcf9d1PdQ8zAXhAG(C4E&j?>QH)r-e+rIPDS<&80C+9slUL6 zSbXZBhtG4V|1&+9%KmSp9s0ZObXe%M9Cr-l&J-!rh9I7%QV)IJFu6kEOWEoTK(CRx zr1+!1Mkt_RfP*B+8btT3LPi6s98*4hk0Hu4x5!MHOJWcKu`xTp`5IHj5s;d#0sPj` z=4N(pnWU0`Uq;y68&ot6BZ<=ug^9Zs%J}B4A$MJ(f6C8I!`P?1>sl1N%L~^f=BHd~ z8dki^t*UD=dYA89m)P%e+%%HU-|}xD=iwlbbNWXE#zy)NIc{An_EKlQ$E9&ZZes8~ z#S;Nb^0qPV8tyLkc!N;&=IIoUnba5TarXwOD4UXhCiT6+AQ6`r&XfSuNu0V6Qibwh z4Rgs4U5=>~VQON@n}LZqlDO%tcAU&5F~$#3b0b7;xD?T6L{lw{_RBcU^)h zmi=Q*!@PI79d<3k@A9MT65~@Y`J27V9o)63dzW8am(WkSdRj=ct}SLsVV+zGdP2v^!Kok;s-y~pLh(K9+ED($2%=RD|uH!#+- zsT`BiP9v7O2r}dFmQ{xRa*mp}e7x$14l>mYVn;x*5?*+15hF>EaFGU0^``9vG@#x7 zPw9OuxWE4N+kfl&7k!-ki+;=ngpYsI&rdmjpq26!bCE^msZGR?5uQtT4MP`$MoIDS z`|kp7m!Ca$G# z_AROu`z-X$%mq{0)qm76>6^RnI400DlkFQw^ehcn>H=XyvVPwi(x;^Yj{m^>(Btoz zqw>ux;%hnuhbb!oRVr}()s9r^Z?6A;qrDx%{?m@RK((~(V2iT&|Q zHI;LMZ0PO3b?aY`bMHYr;0tg0OctLEe$8F z2PBsEm35t%SIu8(nK`ikMjPONPRzT9Bm46nFK5Y{|6P9`0@dE5(WQ+?#upDia;&_^2c=gh3$Z}N=Q*AW z6#HfVjv)g*3m)>GzvIV$96z!z^bj1Se*ulY_?m{mgUYwwbxXgK9DnzJuj^XS&o}o* z1O!a-i^`Z}-ZkZ07hN!AOgMKx<+APW4TO9*4CQ!|M zl*?}eiOCd3@;;KX4$t?7C~vzNgr2n1ri`P%zwfTm+;zc`@k1oz8lp7~2MDU>-WVXS z(dU{p4JTjEsl5Jr?tSQgJ9*7M^zZBVL;t?cKJ@SFqG{y(op|!wnd^G*Ifm?i8g?%^ zhi7lPbJGYJzrN|#Z#ppEbW7a5WZyq@ilhUG-*j8om21RbbY0g0AVXefSY8{S>#*y9 zeLb(__1E(cJ(bt_hyHW@q5oWe=s(w;YzIIf$Fa#}zYvf&PcQF(b^s!hZdJ*CkuSRD z)BpHQ*L?bKyy@1T{-Y0_`sx3N?kRO>ft%lU!vYzxi+&?x;^-Is;ENvprq5*kGpGA)J2r3KR&EV>+X406rtl${)Ele*&`%MT zv|AZdM!#sMu1nB=Z(1SS@{4x&ZoB$(%=F%MR}aD7eU)=H=s8n_Bt6F;`i9V(yT<5U z_ctx2vhLrsvdz&iTJo;9`@u&V%ewpDd?{tSb?u8U*!s|a@q}_&_b=K?wmbSo>%80X zi?(SR-hAerE|}(&65zfeb2HLsr7^uVP#zla>e#A&$Bv0f5IAw18v#t52`KLzoBO_xoyj}3_P(z{=HddE)R2{d z^qG}Wjpc9gauT)n1EvWf<9m%k9*JFk0@DulIlixy;+7Yj2K# z%$*xYeKi$-hz?ueh%_Rp5l19|e~Xe(gUKL}zh`Nz{)Efl2Lcwm{uRpji>8Kxzw?9y zzfb!A&L6(-5nu}E-{n{$G_=3VbucI>zsm{e6xqMa4ImUHzst?Pl>2a^==l474mu<9 zlq4`UC;*MYqOfAVdk}N^UI8118cA1hWXkrfIZ}XRuokG3@R6#J7U?-6wnTcC+{0DYsla45!uhPyw3#Or5tnFUn`dFDP{jdfl4r#e4fdD z*G%4<;=>n<%KC)t?@IO)EC1Bq^eqRDUh+F@@!^Zd;34@p7r?<{umN7`<{;nmz+wMy`tgzbyI-#_e!Sa%6$>e^T0VbB{^txD`8@o(cloFO1)vdF z1#AR7q+GLiyE&}h^5fug^px$&1@r*d{~z?~pW4+(K9-V?+283O@AhzX0$l}1_7l#q zk>kT+5*CGj>BH+wena^`fAcN-$kAf5UGum8GIV(J>HkV!c{9ypWfeD$#!A*8aLiYUe^16EuU}R@_M(Ud*8qHJeKuGa=b_He!S&zko@`U zccZ`6`_Y3$e&3>Bc18Qx6`e@EXus@7{MEmI(LUgBJ(qeS$HQAbRQ`YW>ks~$GpWOY+upbWEu52OKGwn`M7-CzcKtqofc67{ z=ze=_@7-nlpZ@dz{BQEaf&a2w72E&ub#<%`-G4Z%XuJIH#%)#KAGiPdE@|%FVf!B^ zS;gV=L~JWw-v48&DF3y&760*fJfq=HKqK!P&Ge#G1 zEObODer!M)HOS|*2G!QEDpwYN8An;!Ln`GuB`0%uKtbqV3#8JBS(}Fv#OqVO=-;$M zjE>#ri4@(B*HBlEUGZ~tntPYWjVKUayYWaxTXRt?YYUC!a{;+%_Noc4=ld9<%VcEL|qYprv@ zhFMkjuKjJ2UGlNg=Jb9USj{F?Mx5C$-IlG+)P8?Au-)Eb?2ICR<92xh4LO-p-_D-L zi#fZu<8d)vDbZ%ytry4D-B-0?vbj%ZDUMI2F_@xtc-5Nepj6=qfMYbKA%pU662v!b zgS%vO^wel_ya}Cm_sKG?eRXr3@tCuSF7oH;wT{VndY|lhqTOPR-1zQNcAcjw`~Bs* zMa!Y3&i0Spt0!K6tI>Q8+u_dLxSo5c`@Q`!DYUx-8@-=I2f7Id1z?R-TF|29s;|79 zY)A5SvbrUqQ;*uoMA2@eK|glV>l)O@w#Wy?u}J%q?eo&_t9d?~w)b66yDpzkoVQag zA13=aw)I=PC}{K2CkuC~jZR{+Z5G3%Xv0VUHNU$#*>rJ#TDk-+HI1FD-r?~S$M#jtGS&iSDdo^Twmi8X!%a2t z=UyzI26gRCj>mkwYW(dZs$+V30-7p+u{f{F_?|)QM6cob5ln~S!CS?> z-R!O&ujj@tGEwy>bW(xkjVDRVH`B{<{%Tk2O}`zSkHv03jL*;Lu_>+^**cSO+@!7B zp>wC{SlRgEGJY6QaxPvWFfll;Huso0uVuK6=BGuNUefVm&lS-5syM}md^p)WUyr?0 zB6&Q2wl_BCqBWykYUi4!`q?p_+0nuA*knr1i_|HG?NX`8Gv+skOd@$EQ*Np`ohw3s^iPX$yU|^}*II>< zY2`&Zz-YpQ#yuKZ%k833$0U2O=VcwuhV-h&p1vNcWE`s^Kl*eOUsHPNrAF1duAdlx z3AyU*WCTtYBGt{aof*fGfx>-O*Ad^_PHr{!&sW3N-H^R{&zt<0j18X)ii;KGlZ`p0^@pA76bhsEo9w0!o+nD+pX)N8l%eZ?8p(MYpnJNEpTn^^to<3k4V`8)tLq&YKJoGF zk|wv&BW~eZjdY9}Gi0$|y4BvWo z(?d$~0luEDaqG_K`gL8!lipqw;ca-0o~QBTVqfyXy{r80F+fwtHk|f#zGZ4q>v6lV zhtS+F`|0(t3_91ePswysxV$!hhsQL(nDJ?&^ZaghR(Z;DFYu~|%_~#|r|t4~HBRi@ zbW8F`u>Cr*K{4~wBH|m%Td_l&q{qjy{|x`E6P`7r zEqm!GXxt#jc|Tj(r^$ZoD)%Esx41d>`SzsltC`EgFuC>VHTUw z0V$+8i1PekvUy`%P_Q@egTk55hj80- z_eo%vwes+!-MyXe!(Fm9R=zPy)QDAT%k+`C;QIFLFZ^8@@}V~9k(&0r44y4N_Nz#H z9D1X0K<@g~xosCJAD!x#f3>k<13hn5JDEmNOvWoGI2WCLySpQ}T?;g^-SeXI>)y#& z9LA@;s5aBVkZb*mw%3F_YNk04)@kRQeBW6Q2H~bA!^<^npHXmk*5K+pLF(13A6s*E zyms}~jPIir9$qf3QXN(SOwejqn_3ZHtZ20l&JiMWYaglUcs(ytf8AMSYPz}%{jJ`S zl{>Z0vv^zqX_U)MFihdu*7J6OU4X~D9J5g76v{33cIcQujgxAg9ev)_EatHfP2p_y zU|1{GsbMrN3$ zGY{{N_Of%5E4VOUffBJ|aK7m$*gsm&NwRJ}S{A9P<)lqcTU$|jlZ)5R){W@T^XquL znk<9fX|!IYxA{o#B~yE>>;~)pBs)xve75a7uj;ZO<;ljFXIi>DPUqE(>f!2i@-3ua<7@A3(Gwks zncrN8v;8Cy&CSe z&aeqrb%_nU%qlPAqtM=^+T=+e>s#>B%;78=#CiWY3ys29M`Z7A!t-^YB{w^96{l+% z@6XTT0wV?1ziRVsTQMDosf5(ddJ| zeF?S9;ND<_j@LBxj<>}zGur-T7WK-5UXRDSkyztN6>^yLk`+eVIz7d)E5cznSSOF& zoxTRc_S#hWYv4@7rN_)nSUGoY$vAc`^*EfwPh_pT9{1L#$lQ?Nbw359?kx|uQZ={f ze{8(W0%f^oQBy0M2=^AD>KSTqoy`W`BU*S(td@^zbUx;XcC=1cr^M~YMqej$`;m3= zf~l66b?0W7^n)^d-Uran4RIci`p-bg*IXHiWl^oxfmnw$v)pYm4DBYen|agw&^QV`#w3I^2yo0vhKjjQf9e*STibGNa4e&?+mdSGubcI~_Z*?^>ZOfzCD9qN zHUCxL)BEN0tRCd~%5U5`mHnnWHOlItcJtR^?I7msyG?Jn8SK2?9iHdroHX@x{@PfY zRgctWcE9hcu!uWT^u1wvIaPyMq4}wC>}?i&yLFTOL}tDA<|rKMm*8N|e|Z)aO{=&T zD%0KIFgYy=Q~2!Jzn`?)RLWr%9eX|T3ZVDwo{O>lv~B3L(YA`sAZgP3?Lx4`TS)BrybeGyZDuD+Vx~LP~3%BYGZqeYvXj;Oxs~v1c!ke1pUd?9ai?*N;bpe z;FLCE{j$Z_i%x!f-TCuOe?^Bn&6}NdSk0bZA+i=fkFVY-5ALD88wXv95ND%t9-R++ zD@wg`-M@|lo{eUQ?cN{Op)wQ4hga@+J2QjKbhIIgCe66S9@YAf5~FZ#j8?nhYSSpC zp1Gpkp39P4(Iwlu0eSNAc)ZT`BitWb{=JH|(ZI8Z;q-EMh`1d`e|f)~IXxtt%6f&R zK5=HX81_}QSU=iubDfTE>d{rhag?f~ezv?$4r1tlaLl;M*PU^dJJ}yYziO z{nd27^#-$1nmkc9;p2XY^-5j%`|HVFoJY-CEmoQCX8Go>?ilL(cQvYCZhc=g__E+d z@4U+Ux8-s*K9MEae_U_%^zpdm7n4x`<&krMqun(bXOC`nDZP!_od(r@{n8Brp>iJd z9mlxz9d9rRSGeC(7s=A%+pD-J#Br87rj1HPrKEi z3ix%sIvsR&(bsSYw)O6IeXSPJWjI%tMT%d$YT1i740YC^e>ujJ;hMX(?+ukG^yZ~{ znfv2qm~Z1=U!Q16QjKb~Cgrx*g{^`P895&Zx73|XR64jhqq}0L!SuMER|z5efvRCP z)NYrH8en5y9Z2BtMKya=Zgv-b9^_46B4dX8!(P{A-852G<7jwS_Vf67PS#!#Jw-4& zYkLcO)^y|^f9CyNvz@W}c3rkl_kK_L=y`SS$z6TgU7x0pu`#mUOV~W;#X$GYb4yQg zewrS($MrgV9EPuEV+?qDXWk?22&H~*FOPOLfyLO0mwllIy=X=+%4W`c?!hj#6JWCM zWvVrtVP}=17!CyH$A!1LwxRo+#eOtDWT@7xAT2Gue;Yc}t%vtP@3A3jwp<*Y^=sqI z_lxuWmR{S{1o5L6(}=406czPySvQB(l+vA%Mf+htJ($gi8M-w;;4=-{E52>Dh)+o}zc#a|$1b8D zbSwT?e<;ax(zjO6O;ydJ-aMtVG>=SdxNR z$=+%Y=WYMIzZvzhUV8CW9hAhRVG?I#b(vS*uuzu4ab<{ATrCV`5lxjO!twapb$~C<+8W=Y!D|wGTXJWwN$mUQe*u{XX0my#;GixSH|J`FwZpOXm_&`; zZSDuYpALt)Va>{@7wFaCTHbv$^P?+!?T1y^o0W-4R_Sw?ZZ@mfS}|d-Mip7Kmo3Q} zh}+%vv1M*h<|$85ymM}~IBrhO%OHsr&IifFzi3f%>lv{j@=Oo3$Lg4vI=k)J&<{t>50jg7 zz9=jGa*Pk|=(*kgFSgEGSzDl4uNOpscYy-DH!mJCy!W1~SMBdVI;B%mI#e|c$x!q> zYrQ{^BXzglxethXiNOhIgQK3tR6VgCE{2pB%(TOcU*Fb-9o<5Wc0j{7v zab}e$5-fc-PW&XdI86tc^3rR#!$@OML20~~6F&!^pi_*9gzF!zjyfN&$Mnk^(Wnr@ zxn?mint|*+etd?!{#Ot%;mZ6ehn|jkA{w3{lyx7O&NtB?$D(y0|K7pVroBq7!pTpC2qHOpba3^t-+5UjD z5lL5ZK?pD!h6}FReHji=)6qQHxb_(%^MXd2*lu1!)Kc+$TU7RlK7_sB;s;XD9b3`t zS?%dfcs9`@>}fj=q8vnkEmxc?f5Sc1n0CVO{>;}K4X6kh0xXz7XtNz?Hz0Y9P@RAM zy{)%W{SIhuhp7!S$}pG@hYGJ%rT(K@FJ78V9`R(^ZdlHgl?8q#l0~%OTead6q~g=Q z6a+=S9sXl?xz3@W?bRVsb#&WT(1Fi#{30cS!YvV*+()_j%C)rSo}2Rhe|{$?No=7a z;skjvU8@N#vi80ViabIEkE|d&_z4$@B5>KQuUF^Rsa_HMWK7ucC z=Se#RmYd!v2v3hY_yQ9tDi;l95`nBec9wHH919^jhoq8*9XQ?PZ7ZpX^$F%?{J}n7 zCl0IBBjP>`i#5}owu?{`e|E1#yFQnklSunl&x`Q5nR%l5nWRsJEk4B2>v8rNnEzc@ zD}!x*Uw(%^_GH@_5B*vSui%mFY;V29oS{S7JZ(w++1ZMhcubQmngi%b)6`FGys@eK zY=Vtwh8GE+jWmO;`DczU78$XhO3=)kGNnU)9iqzpL?DC7Wx-swfBCbq4ITAziA_gs zP|LpK4s0O{T3VvHgS^-opeRe z6Ws9qc?MpZg4b(ERZvI~CrtCmA%ff28If!FijnQI)*60L6|M;mCbfK_0@E!h;=`g5 zeNXd8vw#D*5Qe#re?FT%nL1`vaaWWSsBS{4CI|@GNzazCwTZb!y7>*VGm#Cpe%imE zh%$U`^7Ry$(g4hMc<7T#o_=5TLPw42xvdpQ3}R$|l64UdS%!rs?SK#MRTg-w4~EDW z)<3WyGXMB}jB-01#o4cmg7ZXo^+b{W_3g{@%i++3iY}2fe-%}mh@Ca^1dOJR+i#IO zaGTcIrgePrp3(?_E7OrN&bTAPI$^S9S7tYD*HcwL?QKMs(-?aw%Ob;EXNLqbLqE?H zzMzDbCC7*9kZV3Lj;!F*Bd@#O4Nb3T@s^!~?WWXKkki*Zha05n6>#QfBSqU15TjhT zb4lbHR`>Oke-!0!kr<gIoXL||UBDub>xeH# zqI9WEfBNm&yk>Tkp;Ix*RS6b2_Kx^gA_t(1&RsZHlf&vUZi*Jm@jV{k;>D~>ZM-VS7%~WL zX=QI9troV$B#pZHOF>ZPdaE1~`y>z2AMEn|e;u@Cq{=5ho_Lms$pwjwykrFB$A-vh z9i-E>bLNT(jb?X=i43e>rm z=_?qn?0O&v99-I4!LB4;9Gg58;fR#tg;6|Cds8}fWpQ0<2+Ka?MY#@=(=L0cMUW8r ze+zm<5`0?=gxLBec@S4UD{}^sH_LZiHkg_^h<)ir%19cdk~k8LEoqQmBgtVabIaN~ z`(Z6<%Uno2YIIG({)NtPMnl{rVh2D`J%b>3$5 zOCo^tCxZ6D*Mh(0c}?S!D&1Sg1c8i7f9UOvRz_n}@Mj=YE0qxzEwfw=1aQJ`97xLq z70#?{vBEDlb6L#uxsT76S+O2pQzb!cKC0yePFC_mQUty}H%~C`pN!>NQ3mt0(eY>< z#R*W5s@pn~s|E3zbj7Dffu7yq8D@4ev&n@H5zK!Pz7;2|0Ha)-f>_|bbkTX_ccfM!CwK4d&reyEvs(dn zy1*sQIAl`0I|aBJnX=&AesJb-ew&JEv!#l5Au_oB=&w;f{V9S2xBig-%ZimY0&}8 z0cZhd;XDge?u^@ZYJB1l*GUy)Fe*6#XNWIW4SGHj#;Msd96~mBn^RO9e~3u$+6chP z=W^UA96`A-V%+Z2>Z}%P2P!2k=jVvl7g(Wu=3ZeHh=>;vGwRZ_dXy=1OC|}~?ImGz z?l?rhX4E);7BFoO4hIjoObECWa!+I2d(hktrRi#-Ds!UB;HoJJBOY_{HOM1s`JY$V zRc^d_R9CH^I-t0xQvt7&f4gOX)gi^pzcY;r;PirH7N>aiMtp1OpayW&!skaCTDoElD0#QET(Vp~sC!Tli7ABD!9 z7F%HGY+FG)ae&5<4+9Bx1zPuvyd?yS2|K*z*C;ygIAZwuef;wBe<-r&dyIL!0T5tC z_2OGm@!slXGUkF~0EKTLfIkZA+sl!P^tLR2t<4gvmeskxEh$w1-h_Uo28>vuhMoWx=@SS}Z0c~!?SD>jqSc^dvj_!-eepG1Jv#2OT|Lvx7ULy{9 z*n3!RLIDxi?V%c-X_%+J zp+?2}?f0o2Q3c6VrO`T)UR*)S0f6L0m!hFcXO>(Bj=513fAx$&CYs(FekSkKq?zQa zV0;KH775lo_E*(v^~2QjQB^5;c;`pyGKltVG9-oE(?5`b332QeKQrc4^@L_0UQ$!Q z6-JWH(urdEB#_M)uy&G)j*zYeo=`uLvX8C8y-g@ghtC*=Hq8L`2F z6mUGJS)zj?g;*U#?(lgU0)d1+bkD$sFc7lNarq6Cp57kol~vg2t+W**jqpMwXkPEpp)P{cb=Ss`8-9Mr`Nw z(j2A|)B?KZ2BgkWa%|xdL2->&LHp!91W3>dG@f{Yp?uQ`W$~8?)3;ojCDq2`+CurY zf#Qk`t(GQOu+cE|T_N%P>!FHyXInG+spxk@e==_xX;?zP^AUMN(!y8jDVL*!l2ttv%XNc(Cg}BTqKYVwDfyqPXq^i#^37 ze@~s_!Ti~vxo(ObFiQXGm+AsG8BoOTo)@b#bWX&v+@WMT&4W7ix*PW^Sm9K@c(Uij zmxlnp1eclxKgqfXr<05ad5bwDa`kI6f24_jDEc8Pg}7P0Iiz#FzjE31<+ZhP$<*Mb z=M2y(`7AoVFHiQy$$CmiWa#OkbEs!0QOjGP@tbM`mf*Ip1(`JBZoLMs@6Lty=54dz ztcPX)Pu{lHe+7VY zpoCSynm+J$@ms+>H0b*>V;(Cx@wxEBL<&`vAw7%}o_StL8KDGZ8!r&`x{6b*6_k@% z9g>0^Rdc{36MrM={g^=136XKw{MD#k=%c1C%!b7bYL`og zlQDEp*1hN6L=>@U_?WU8B)UWDY63hxAq0P{$30DCCZV8h+TsH2#A6tANH-sMcX7sj zlg&&s#)}t2c#XBTVgS^@MQVHpWAVZBpqVVeb~xVgorTZw(-?ce8Nuute_7Y~-a6SR zxXF=XE$YgLg2Up<0J7BB3OZ5J{kHe4AQ?Gy4fuk!pTXXqJ;V&*^7uA>Wk&Bc``F`|^XO5!a>crO7r+z+Y<@9>R z*OCjNe1mpp&A%i&pGQmqkPGM8;2PK)KmbLupa;x>|0V5p1Fum26-2kzO1sKdWA`>}uj@C*vYjH0| zd2peCkHo5{^dupbW|f9n2-$*HYsl}qC1GmA1^saLPlX0DHS<1Tw8uxNMRT=d6Fo#kAr z2Y>YTPBaHm0ucskg=>X%t9?nt*B2?mv`?gBFtuJf>zh?hKgG{5yWCNE{)%#GCK9E_ zI8U4()WO7qbV3MS$7^Plp1*QoAso^jE*lfht!%>#oA8=Uf3yTp$J0C8qOvuhdDccy z>z-qFAXFvlHLn969tgJnYP;lC`kNsMx6kQ8}=Aw@3I$6QPm2&n@ zk0dY_Bpi7mqVH7Xt$J0*5|07FbysWEsLa8sj4>A2 z6CC!rf80S%`RRBc{o%yFRhd6@I6@d{JUok+({L-s;N^-D$qy_nac9Pm-fN8@({9Y+ zhYfU09*3h67rB2=l`-u){$dwkH z^kW&cm9!Lx<%l6wu&VpLVJ1AjVBTH@7Lt^*eye&9kRo%V3G~>B5i^1*!@`{t!3TZ-s=z+W6>XE#(n zB=68m!ccsa{(Amc7khxVLRn<=@8}(ue`e{=cA4S_m|En5S~;!f8%I_-g!#Qi!>D_9 zWv}JZg7LDQpLTZ3lTT#Q`w>MgXgfql=JygD2ElB~7XzO}{&{ABghkD=xwggA^0vhMmG$z@!yn66Zy}`&*#IH>MBa7O4y}hn{Pe>M#agt*{PDd%5Tyn2gAnEFR~4CDm=D^p)4e-*Th&Rxvx29Nv~H*EyIm2Dyy$^y6Ah!XO9;eHDT zD_jx9H!_yYmZR02F0dzw_XR-_75ipvzj+sUX^n>qgG# zuP%*L18Y7ZArDj)BF*R1`z9HzmWei{%g^>LaPTtRA8|v_cs(?6W-?e(f4>5wbe4N) zIN%dQMPK>|>@j#a6vG3NxnX1^L+z#Gb;vi-k27_Z{L4ni9FK~-x8#RnFvr1cIgj!K z-+^xl6`LQPOn~)<&=hnv+(Xo4Q80PmEZZ{KU(p;d%VsfBoN9Ke35em|B~tAVD$_*i zRHA^0K(gDQ`;u$uRW*12f95B4CUm}m&%oZK)y#QW7j~^}uH`!?*xV6OkTCKnY@Zev z#`AlYI@yWHMY^HyRcp5bYM+gXBgcV_COe4BMA#BF`_Tcu=;Z4*yYgi?cgmg*i_2FI z+QhngK|1OB_`YlxRtu=NahU`{XoM{K4IvuJ7N&cx2k9&fZe>&%e|(}`-Yr?U;NuhX zqnqJCkeOmJzKMMhr$ZD{C-TLl4DJr6ianNEC<*c*OI>uZEEv}EcBqNgd={CSivV2W zZMC%5d3KCXsI=6hffxGdgJqc)H?02mYdL8D^@%>tX`|2`?bJm^gl)y)JW-TQ_67<^ zpksA)Fhe}5L-6ExnR+z6r66zF41mq=^;L*ZsyUwX$S58Ue?4&?rBuv%LS~wko0ifW ztQW9!00>WQNR70jWNVm5e>~PgIDiIcvFi2T~ST8qSR+p|9;V#lFEsvbIXyHN&#S zy9X5_u9WkuKQ=j%e#r=s+e*GVXs+oLQZS@bU|ggViaD%YtdIm{d{0G@e0os}nUKtnhlx%K&-fz?=`W9=30w$HaXTB!*mxH_ zq@mq>T++PHIEt|;v0;m8d4K1{gpzo8`dPWSbD760ILAzg1^N=dQ+I7G#}o?MPst~ z>stzKsH^7YU!60JqDgfe@(soczdm7q5YIikEZ#u%9EIB|wCVVQI|kAC34s)pr7_iMEP;loO6h<(Ma?Ii<}b~)xjYaoGpASWt3GYC z9qb2ZzXoSxgH|iZI?6hkzC1j(#RW>LXt_lKnkLNOCF$ugQA)Sqs>!JbLFw;L1IsexmgT#QzA<{p+IKyx~^#AhIhmN=*Af?qy8HV%rPv_5O+3;$KUfC z@!w6P?yqPyN=M}ccZQmO@8|nhe@d_afutkDcM0@QKmJ$#KTUUkm$YJb_cbrgH7M(# zM&K4XP9o5qVGz_Q>6hVnV)9TGl81N(ay{|z&*Zcoq%3ob{876>-Y9SUDP=(WjHxtf zD{4dhQLdV0&is?0DHgqIGm;GFm6U{*G9~U7As)7>*F(J&_LnX?B&vbuf2;7FCm|V~ zB$5`u1f_Iz=9|r{46avR-gx{|^k=+l8P4p<=BKOPOPJT;oi^ z!1aQ)#W4V{vLH|r)7nJTB-u~KmEGEpUvK@eYxwP)AuxJe3%94w)%MX0ICE2-4b&RR zlofM(p)0u{$V+BKBt93mexVp+d@q4>7~GtQc^L^3M`dHtkt6G$e_OWXxQG3CZFO(> zw;}A_k_7sBimO|DMX{>ilYI%0kTehwId!Vrc)yAi6rdJ%{&Z7eff7g-*gF{s02|%s z@umuFC?+o$v1r#|(Pg5xJ0xs2KV6D5*-Kn?&kM2P(w4?Rq}ch*tT6DLLBlyVU%HtI z(2cTIWuLWIabN@ce|Na7J`TUM7v>PP*O^fy7R zrQV?fppC>4a~XbpsAoNwhOo4}TWm9Cx#yMaHY~L)nhv}Ma?n>pgA1vUqK2>{vOjp% zveuhwzzHul#)uz7OZ9a*+rd_Ek3wn+s(Pi&LRCNZxJA`)G{jx&eok2-a`h*FHLv1Z zom#n&Q{DtAe-gISlfTe*NB;MCS!m@r%Z{ANQ3xMf>7n>&7(0ekft-7&<9I~Qr4%E5 zXZ#LASh$t~_!};UKcyKB>rH}UGDfFqupLSV!9lf;Jp#8eC5<1Q!O;BF<;j z14Muxz@?#x^a9?uZ}w?fvbE}s*=>gv6-a>3#Qy~Ve-n!r5)kyfh3x?7@zV%2NRf#l zf{KYAnyKvU=I5!;ITr(i*KEI5|C?DL*=Q)Uj2UWZYi2I*`==q%dk$(5SC1@;_7V?gtA944)=A#XlX)?WkTOX-89mjEHt5R-e$V}#f9&fi#=ICI2SIp_BCJ|B>KzTI6$Ma0 zcauWggFwbXBa2#{xJLnaAz6i09t7CvhrumQmKOM1d1^HDaPJj57SU5%qN(Q*Yh&iC zmOsIJS`~z(*!{6?u!tQb-V%4-z7r&js@(JF+I-?wQP@yY87A%xR=D=FP!Q=P$Ah;WQ4VQQO?A8H$HuILR$?sYeO%)vIBh4+RA-by& zb|}W_Yw@bQfWn3e5>%zy5T%q6ZKnnTzDv#Txz`WQLR~|w=(!4OdY%vl&P)vBdy>eN z+07vlXmq_Zo>Pom>^+3mFC(0re;YB!OO+sUgBvcQJY*YD=awM-C9Fxp$S69etjbLo zMd={M04twglYYIBy{zmAPnjP)Y);7sA!!6unkb}k_ztTLox<+^#AJ17{)FavM5zR1 zwWs?9ksucs^%vr1FOUMa&J&jEZX4M_rH%2wnoBafY_>{$h=EpSOMuTg2{y4+>s1|S!)gAypC%I;y zev~CsGel!wDMzvC;-<>jf6lxbYLyYQg{mB~JOWoo__jt@g+BIvpBanphr@8CpC!wAme~>r{f7$ln1?x#Q347u6r-y5qN8=>B}*|4kefAubtaIOte68n9*_w z_JCln;hd?Y{GMP{e*k_=uAsq+-rq|)sq(FBBTd>3(5y#2X~$WwA}jV76_-*aSn=bAMCTu-)^0mGq+{cr|jL z55pMQl*dyK6Cbrpg}H2713Et1bpoyk@iM=;CYIa=b7aVUf9im`fJ2TgQptmTi*1e* zH^v5D!NvpFDh{Lm{k>;mBCbD#@7@W%y6}b5i|!ym?F!K>E~2KEP7=_Y4~KW!Khxm&0dzkv;%XFYl_lI)?%weq8-U*~Q}M|Q|Vnzk^g z%;bl}UgDl)e|?pt_Jh`ibO3gb#O*iWSaRY!n+CIZRE=B+b)U8CgpSZMUE zu-QnnrQXOn(qdBaDWKXV6HH}aWEaMPH3Zjo-{nDXMFXxezV#`qIz&562tb>`tDjCC zXS+HmUYP)Lw@K#t;~S|z=Ek+`-io`$%Ev%eosAs=Yq7x}NKZ4aPZ+*jg_Ifcb&spd zT#@YPe~d|P6ub8l89T@(a7n)6RAqQ($RJVq;^1}aIV6Br1=?&Bz(M-KPd_0NhsIQ&Ff?BJkJv#}ADy$YO&Z zFw<2VPfZaY0$I)nPI=FiEDB5^eT4Jhz*~9*6>&w$GFG)O*SRJ@<_qCvOu<9*@cO5# ze~8sBJHeY3+Z8a5+d63_K4cC@i;!)&;xirM$u`jF#bXZsH}UBdgRxRg3yhM8eiPt9 zH@Dl=;vg)#;j=~Hy939}OX$Hjrb(K>X6EhJ zRZjs}Oi!^4W#9f2S8vkpuMF;-qEljZr+&_>U5#2aOw3V|=tB@pvh#)Zn0&q}eZ;Fn} zdVl6Upi|m+83ewSu)NX?fyIrI)bgj?41G+g4Wq;Z%%$jeQ}u*UZl7|&S^Vh(eSpwkDKBEIA}#92 z$iMGk(Z2WcR=ZSGS*&WU*4L(bD%ju=#RqN0NXx$i$?_8G&zM{T>l~4YcY)PawcrN4 z2}54MpTEr;xx$#vHQSx$f5 z>riT^iO@4F=X^r zc%F)O)u}t5DN>pQuOLd};^MR9z;%$mw*>3Gonxt;)7vj&R5X*re|CGSxCWI~%828paDJ$5V`JCeJ@@u(^XW)Snt z*#Rv#hYLpjO%R!8v^h3<hD@Xub>Y7SU>oKSt*C+52+#x%<*1rw6ADxUOPiL%Lh zB*upZIU-^N_ypw0f8KL5GT97WzUyt}14YSV|J54}=dJ7xx+p%|C^R>9@12i035}oC z^P+48)a=0nGCU_)Zb0uz-xOj5XdPRWK|<2Hv0szSmS@0ptNI`**0F+n!uhf?-dRv~ zoLnLp3>5!rE1jQ&8hTp7+|utNZzbC>H}~Ij$j1+#DHvNte|H?aL@GcQ(iSpKP)sjQUnM?jgoW7f}dgqqU6m{)}86Z4SGe^;FvyR1-D+cy%By9y0}qH=wR z3j5zTp5|yuNvJ)<_>|K*!Ccp8TZ;Dw7k|)D+S&?zlv2f(sYkQ_hTNoe!;%MnWt5RJ z0aLW%U3=lz!8Q6X5Xpc~Zzm;0|7TR{ix6}my*(!Ld38x>Vs$tj6Dv*GaCXE?bySPI zY_kR%fBOOrvjjM{y}UmWER8i9pU=uQbM$WhoY7&srF4X~pVr(AqjVMtPnUOw@dFQ4 z>op4T>@bEe8Fh7V={7&(DxNY!Iu|Wr2>w9lfU`Rl1H=lipYDM`HaL7YO0r-W1;R$x z#|_#OrOmv!<`gswj3*jZ99Gv|4`#@jX$og|e@_V(f8xDW-*srR>kh&AOw5kv_~b^_ zt+mSo`!j{z10sAQHo(QU&R4AAc)bvEcO{3#5}2gRtEQn*HfcX_;Kjq=WN(Vm0jH%I zw#n1)ehD4p2R}E0b>R5#hG|zk+W?2fxU*Z@^vM?VW2kMP$ah^;54o9AfHH%cQlYYDiBM6ME5>KWnU}a9F-tcG^lFu0O|aFKtODe ztaY41`o<7~I8f>v=Ib`UF~so8reLm84y7+}rPPXRLKGU0(J=Av1$ml?Z1C+ZAy7Ha zsO}PA;Y)x=zu#ErpwQBryQunb=k1 zbZG_kmuUh#2+1|&trUq&JSaC_Mk^}C&!Z8ypk~b*$>Y9H;w@fq)!rU|3;3B<=o$^B zOJJSxv%FY+`VtT0o8BhuFe~$PUhic)EhTgF+0PSbKbs`QEu_X=6Mc~xTx`xBe-Ny{ zbD8v*K1)_gv>(`7#lsDJv<;MeBsy@oWh2&_4yOC9VM`pnW4v{A^{2>PG{=gCmh@sG zg?>6U@P-})C=7i!Z6up;pWawn-T4E(vm)TAH74B- z&C#FLWc<*%I6x{*JBOeybn=88e;F^tI#CKN4TO_t8oU1@D$w7qK`K*N?hd4MwB;u= z7JdoGd{c3L3z&@&9{s4&E3+NgRIt|O^Y|_a1(XutiM{FEnaUKY9bS!FV6VrOxBG#6 zp|nNDQS0hW>;yu#s#m~$wn_b^9(vt|PT6ch`kyqJG} z0DWTsU?FukXO0{bz+@DUXsA41va$=BYn{Bi_eL?E7Qf;3 z4Jq-OmWAr@X@PcnBtJj%(^e=ToBWA}R7Dj;5jS4Gti_s~jvHmgf2?SWky|7Bs^GGD z#AfYbZz3bRcp=rEXksPQ2}M+gkexc}@h+X_&zB%-$|+2_?5U%kr{W5l$S@IJqcN%* z#6W>b?F1g9VPFsP8!P-&-@9|b?hK9!1sI=3hvNfE$0CtRWPajMSqD_2Rg=gW6jis} zA0QJxu;(pDx}9f#f5m}0Mj|cisNM?^a6O2)MqI&zT z!J)&GM6HxUe|_x{Ylv?;QIP8tP!Prxl|8e|BG6_+CC2XBhf7jcnmUPW$To zkC$4Wf<;daDpQ$kMu2DE==hdHUj(_-u85Je6qd}GfeE=R>EEm=1U}lNE_mYFi^j`j zcSgfAAYwMwLgoaaJ?aPhl|VK-5)ArMY7n(@i6ejwwcGlHb=$z3zhaitPwTnVB_{^(LUbT@if8N`oIQh|SM6qJEpPCT*>7}m6$?B~l z8DeVs`U$emNnL6Q2%7Xnk3vWo^YNjJpfto%I4Belf~%BGvW>%PZc?Lpk?~Fz`m^)1 zpdst(q1W3pr0ecv)318wT*Q;Y4iViXTVZZK0+QB0zhsA8td~|sugL= ze>lMuk$hewFIk$`9} z#Uz7vPnsOVPtjm_`S2gKKNHS*4vN0o_YTo3d~DCV!2~5)B#H3(<6-C4+`l^W!R@)z zyb^LluV4ad`1ECYWg|#_d?1WtBM8MPf4qcrmMzClxv)#xW9l469p@uq0)5!q36#l{ zqkdP(eIAT-(9J31m{nbTaWOfkkNQV9T08uzAV93*IUS6;5$IgKx6G`uPm()V56j%g z5Cmf&mtdS?lI;Bre5#k_!V4l)g#KQpSFlh5w7!ckeR2Rv&5NSgep6o-Le?5bU zvCiKBM?kp0QkODc3qp{zx;!-4gdzPcAL20*ss1UN_K~`;TG9pGs*VprnaTzgB9mxp zVL}ke@Wi6X$$D75PYa8mu{u3ISmC6N8P3#~fYcBpKfox;uk{dQ)!bd7ow85W7qz?N zlsZ^N z4y65+u4Ic&i~M*xx{GGRzGuPv$gYN{nyi$4r7uYR@A>qJ>w}UDT}85Y zi7QW0(hv^h@Z4-}J{ohLb-iFMDD?o;cP?SD1F1QTca4C2@61fbflRcM=Q(N>J&S@% zpAg=oP46Y&F+$+BRfk@)4v>%Co=4h<%9Cy`F=%6Y9e?{Z6L!et zK<(JQZG6uNlcVHfH}V8N(iTD(jr}4M_!=fM+6OgHRp(rZnkvph(R?+R%QT7GL|GC} zh<&Vd9cMn;zKm8tFYRLl_DN)Z`bl4DHZ@6&d`u!egjK7C8{rmulvr ze!VD5-k?9A)9ecogQU-E8<$l6LSeEEMI?Q)At#c-sYNsH80niqu<~WTAAwTba~bG} zAP=`!?6ZE^`+PJ3pFX)=Z-(ga>jAWE{<;FRi{B%nB=aMX@S~Mf?Hulnp|(pJ(AW4D6I`9-= zo{PhGB*q-`{3*R}ivDQ2W13?cRsPG zBK_hv{$TG+`$?LaAwK&Sm$LV|Khx`TsBpM{f}KG!iGN-JJ)512gc(JsZ55s-pVMXd zAQUsak{XRg8!pTZp40e7vv3#!3lryUw|^_7*LdJApU83#^#pr~_@u$T<@e~v zLhb}TC8*C4K1?wPwiH$%8rly`ov}d+$PF!HnLR4MvEeBX=YeM3vPQr3Q;~Qgg7=cx zJ~RQA!ll)78l7-MoW4-&OhZOwgMdZ}Un)IaW#;7yOSDnl)-fDk;$=ZpgbpxZ(+>1X z(SL&MK^xD)B)4x_lhg39v?LWSjU*4$B-ju^H_OjMe9KzrX?@cD5gQ3*NQm?Cfj4B@ig2I!(2y}5eVO`R3S4D` zQ8lv>wuGV_8J8ka=LJjFFW;&oB>NU{8Grbya4c#$l@*sc@sYl!xje;-P*T-WDQoII zeM4jkAP|(U)Nxm4GrgMX+*C;V%rXNMXs= zIb0^zFQ{al+RF3;^SO=_N^_H64W5WC`Wj4G9q2bJT(9dem`;ZrwqB}lpimG1dVkW& zomyOE_FCEv^C*R=HbI#q*sScLLq5~Pb9{l4tugP6R31}#EwUP5uNJnZKG}EGD4rtj z7~TO{2PKc|Sld}7#7C38pIed{LOL&%6wo!Iah+8I%^|H*Zz7o8Aucy!Zde;JT!@BA z(30_puWHi1xLb7cw1z&nh_Mj{sDB3U=e4vPC4kH>$%F*uf-QBANKSPE(N%Lkx%81> zbAyq>`8l^qZ+T^=e9P|btqicE?lBpu_p++RT*j4N4m9(^wBf?k~#AkL{ZOJP4WZJuSKPoSVs_T?1t8amm4P zpg#12)5$^%q<%!wGy*MF3ZD2dus299JyTXRfcZJf{JrsGte3*!4P>KrUgi1 ze2j`bziab&&$VD}NyKF|-%t!J}d%+xrf~UP^z;CN-8CsJn2^Az{Vg{)yLf zwFB5(-mgdb`f|g#&58LK6!?XOC6jGv@Df!9T|>#o(B@UMDDF8mMDPT;<9APS z$N2#E(5<}u2qc9slp4ES>qNhElmWiHv@=N$@<1(gth-z)pZ5y|yMKve97#Rf51d7I zxY+slAwAj3#5DbW2=$7NLjDwp@-z;!PY4|Du&Lax9LDQ-UXU=GSf+bmNRdP9wvoQ7 zF|=kKN20A?SHWc{IUXR=Tl4IW(v6>}hdi?)arZ}Z4hbV%?DK4S<~4~aO@HIXkXqFF zHV-0tznn3zBRG9kTz^}9@TahCJotRr`#?(Z9;zX=Wi2CVpJ{Wa6i~~DoMcp~Rt*|G+J;29BUtl6Ly49ZbbtH!`>!zH{#7EX@P zQ-;ygf%$J^8e4Nd~C%Lew_<# zBKEY$G-Y1IDB|0~th-abp2tv3_5!tAi6LK|my`E)S>}on_!C?5Gvl<$DKO#8-AtWp z{b;8#d=8jhc_@#+WGqAN4Xkd75-*tRHt}U!(?yy~wp3z=VjOMf#wUS%^}!qu`#}_t zxk(QpdP+&u6My95eCE1B#UoyEXSE^7Z7+Un&!*(#I`E#y9u__ED!b0oI%C_=s4g`Q z*}7~Z1cy5wpze!t@!P=eHrqq6j@x4V}|Oj29L-l3Pt!&8ZXz;ukM zb+4{s!W1WW#TGq+pW+94$7?8HMiNqI|J5OkA-#)7>VGvvixQ3Y5ev^~vESd0?P8mi zcKG}bF;<;$Y`(N0T;9FleWl63IDm1_CDewP@wr;8w>tDS|bA1_??RZxc;BxJ~H4d;ND-&`&voE z22Y#!fHyHY5Dk#=%w9<$C~;N-iVVh`{Zv{5j+l#Vg9cBWd)Rit@Hn6B?#kGc(Am*+ zUz|JcdBotA^xSK}DxJdZ{l?y}G}x=k5`VeQ?PtZ=yb;+b!F90-*>uf4c-v)|ikB+8 zbvT{3cgHvcg3om4$;c!>@$Ob@Cx5|S>R6`0$i8#8AlLJNtAk94*Wh$wO&H^FcXC@m z{3vh=R3P98tE*Jx*SKfeKn(I^+5eu8vH|dW`mG0)Wm!Jp_CM#5{?|VXnxGx$QGcNI ze~tR@^^K+RZXSA2-S?fh8UgUSwA;a-faU;h9E*zQbLfIEh<@pxJ6Rmi*W@GU{Pbq< zhFkP{$s~PwwH{&mnUs{GKXDd8r(yw}!S?zlso%KrQMWoH(e{K~S~`ql0t^ID)r$7U zju`ydD>&vqM*G%`7bMl2aq>EQ_}YamKHHp;i8j?& zg@d^`v7O6Zaef2BpBd6Q=@{e3z;?p1XCT3-(oVLJa*EV06wuZAKIX_m+<%Hy0R_!W z>R9qBKMDa`Lo4W)CkJp4)u4BJ+29R3#t@gjnm1^QD4`g3CE@t-g$Bj6D+Wggqe?`k zD!=oqJw=sxEsy}wuhrN=GqY?7I7YN91W0WfqfGPCUAV@@po+ye3s;wJIUv^ z0|(f39snEGIg=7ZgQSv}Wq&Kqiv$o$%oItZzN^p)$J4th_3?0Hk(*jn7s>BaEC4z= zvdHi@W3IJ=0>I#CeJYOAu{OgBuj4Q`Z4-BOU|<|vsarFmun(tbLQ>SJjLh4eq{kF< z4XonGo9meM!F$Pk#rdY?YX9Z=TC?Mg%@}7jm!?m4Q*}M&LDPleaep)dKm2Qf@JOpG z%{VF^6trD#_pvppO&1}H%FjpVYINR$G7!`h40*d4ajC*ItQ}7q#a_|#RMAc;iwAu! z)-J(`Z}!*&t+Y;2o{!`c=(2w9+z4P^t$`jsp1Th+jFjOL#zJGEXB@ITWjFVUvY*fv z&u|Vs;1Ei+sH?(0QGbE#hXVoEtN@eTds1a@pKzT}T;y#gbp%^vz96nJKgCtHuR;-C zPS$HyN?q@-{{TB7hvx8ZH~h zLaUw}aQSWCf&rv*L>AdVff$>> zR#<{%nXFBdLP@D8jrq&^;nFj9#r3lz8Qt{b!@5YB=tCFxImJqvdpLvTv zscRZJ#*~~9>W-dao%7PDK$$dR@R@ORqdmlJb0&AGj?Ye!s7N3K^b6WzQU&%g~Z^DNHQd@8Wvzjoa#GARn~{Kv*Bgx>J-Fd&Gqj+)HySyHeH{e5urJ^^+ z{yf-+wPY zdR@SYA(I6AD$t~s!Pr};kIidp>jUn3jxzPNyt%H9H>abzhbYt2N+-%SI}KT1d^D2J zmQY$u7r5q$Y=Coxxg;aK>Zf5Kl2g;?2=R=zdl#74C-9d^`195j*~;3V?y_mcK}ZBl zD-w(o*!g$9qfrXY=1~1Bx%1MY+e_~LM5{ciy$;0QKXlbKIW|SPFd$- zrU0cR#HZsTe{4RI!8<)q@)B{(w^{0XRRojF(o&CQL=88@0OHlF{6TQuBe~Q;ruR^r zl7b;@*8kd_WEIp^1bW|0+u@xdoRSZ}F!az=Sp#+7{3~dp;iv|(@RCqwvVX61_Vn~Y z>ug;&ZR9@2?rzPbyo(3^tVD#^aIld?N`2w%QRDcbiUH(4W{fQz?K|+HMar2G@-hAS z;EOc0)Qh5#&WWD_aO{T>dj;tA!4bkr+n&Zi8rzj^-*Xiz`C+%GL$QB(%mqT+n1#O~haioNv zNBzJiJtZt}W9v53jI%Mu22$e+CEK7z3r8B10kNhl=@7nsjpk@@>wm}AoR*N7@09m6 zcc}=`Eu1cH}h59~Q7 ztyRWy?|lUPIV80&N?oNXiJ0thkus~%I7c+c>(TDhfJ01eb>bw1rz_%BGf7hjt^T^N zmwp*n(R7ziQ-2SQI`c;pF7I$aEDM!{c)nkELcB!w4O=FgLg}R;vvt^i$y2KD5^~=O z*T<6I=qNhI^3zMvT+?4D|Lv79gB+(Uw6cBgJ5z}6!hu70S>_Y3ts^P&)qe`={0XDX zTkH2p6_Le^pMLXaJR|SgkBCrsy@})ML6&r0qOeTfN8($=qC)44zX8F22uOuP2*Yk|}W4mbb+H;E3t_<6A9Ek$~up*(C7mfjfhz^!Y`&~yVnp{=if{6z*81Q{G3>nIN>p3~XD7Sd*T z@r^12AhQ=%L)o3r8s7~+>CnaT;5&4^>`r5MDu4bM;R5T#bD#?05Xpgh-(5Zn;TTQ& zpcOMr((K!P=H>Ubz~aSBrv~mf!u8=|-sfT`>m$>{C)1(H8xgPMgJwyK5CqNacwt6a zgDCwaoJ6rjS&O`}XK0+#g;Y&HoqfFZvGPt0CmFH3UGP+fHAWXwjc1(vkQ_>=af{o7 z@qdepYq)pNuO1*l&=0DvXGb|Ow&jCRFkHfz}cIPW_p15`3HV;0KW16u*$UUwzJ%$|r88F9 z)`fO=Vni@0BB12`Hrozn2m+i7R0420ux&;IV8)zIhRntfz$Guh%R~K=G3Jb0q4s^| zh%ddxS>wn>`IJYd(VKf5Mw{RB#*FqiKA)>(NvPw_I+dQE+T#*?Gc@ni;@O$n+mpXW~|@a8>j|)rXTaLQU82yWHTFMenMY!|A(^BtYW< zt}-HG8{#18AP?~@27h{5r^MYeX zLM3E9y9dG%g#Ot*r9nalwgdkj zCky!BI1MLu1dxkNl_U3ajMhgCHs8Q779!*Lkq1bg^&|G^NU*NH0ugbC_CZ<$n|Iea z{^V5?%HLm%0v;$pVt_NT-wG7>bGZGFJ>)CKm?|G{pHQwm6fS$z_gFI8JYuRm6!v^j zj+D|t$5;#=o9^^B>VLh__HJh@8x|Qtkj@;E86N|c#F(wT$Zi!&4xGvkRoiSen~k3; z4jf2z)d@{}7eIk&Jw5>T*`Na2Z{E9qkB5=%yyorPk-fa;y_`XT@Zl}8)xV_6+d`pN ze5HS%F24(mUhxxmcd!yUNZC0W$kZM0;(F)`@Z?3f;@f89AAg_MPvM9|!LdbVFHi>@ zBFj>auv3oG0s>G$ra6`6?~^ASxF-Zus{{rIZp7zeKzT>F>{D!F^hzFF-GB^G(%Uv4ePxbnXJ53BX*FLN5K3DEUsm(;)_E2Bc6Err9 z*g*XGYbF2eA%E=rsO&rjYFL6=Ffu?gxJt7)=y2ha z!`&{`$_sG*%v(O?AKnRje8=*Q6%TO7e8m$nHc**Re7XkTw2%>1PY;*! zADO3`io(|5$~xavhUQ4s?TE1x@38;kK9d zYma3!{|>q+mffWZu>L*& z7jw`5^dq5;0|CUX+~nBbdBe&Xw}vO9|24)x{Nvan@FqTxhOHEZis#?5kLHt3>~P4( zPwhNpesDSqWg-NuT;SJ3%~gpK(30Emvr^l%?tjLY$|#X_`DkF@I}aFSRx$!g3UE6J z0B8{Y01~j}8^0uzUUBN2bb|eW(OzshFtORkmAK9`zop84JE5 zmMWN^k}dJvjtbCc=dFJ5fQxf3z25e4A(&^GN~!@v-S)74{=f5*?8<@Q{N@||U6cQ5 zn}0v{a6o03&{7#}+lz%a(XKG?MKC!IEL15UD8^-HpcD%JEepcM!-0s?p692kWH?rz zzuAh1ew!~G2hl+8!20X-_POV;o_@;z=Fdy_^SDZvge$)Ef7c3U?-=>Rjd27VoYyb; z9}Z+ir<-)n2~4(f!TxSN#CWg6iU%!-@_#~jf?7{mi&TJ}2eOxUR!d~g)v>uPM=D1L z0lRYM;D}+oUVc!0Qi@c;t~1c!mlhwxGCaXnPyONwx+d1!rPtk=k6Co>XU8LNm7TxG z)5?`U8Ydrsx!)LD@Rq{%c*Qy`U$karWGS0)H47 zC_7L;r(KFR#RH(W$To zGfyCuKb>jc`fr#2>OWVnW1nf{!C>@1xPOn8OKOXqW{Z8xQnM~n4KM00m47!EsO|N* zZaz3yO09lP4pHdL0sLG49Ei8RbXMp8=kMRXF#ge-w;$)>FSe>&in%{LBXqXzRMnSv za*KE4-L!QEJO4Zjij)5v@3DZ1!5he55WqL@Nw@u(%4Pxh#ZPUVyma$4Xd;@jSC*@m zzhC)9U%K<=B?W>ekml@m`+pm|-@H`tcZ$-=i*e)LAK!wFT9^Lt^gpqfOHA$i!r}k< z{L24Zheor5FY198>bV+Tuj}zZK5*%YSVu7MjCv8t*suRoEtK zm$I`*tV_i@+m*qk^h#H?veWrDzeq2r2G`2206cDfQU2(%o;h(=K7VCN`TvYZs^ojj ze4KUd7~ho)L7k z>bMho=NXFq%O{3^?Qq3^e6WdI`4?|Ed+TfEin0H;$B*YOoYm{Rym{F;d`bADO?u{t zqaP3av8lwgIcl8os8w<+-#XN>xn|x66P6YW334v}NIDz)%zu9PX2Tx}yLKyMlU%jK z2A`BU*Jq-8fC$Dz&hcN8v)7kc$54wp3I6Vk4uQjyIvBf5;?oy^y;m~7KN!^c3Vd~n zJUptb5ZK+`S)%=xD(bCpz_)_VP#oRn4uIGU-dF)oYbO{LN7e#fS4A-wI3W;C#F;=t zHDMn#?6H&bWq$`EFl=+sTf2sDohU&Mz@oO=1FpOAeFyuGCnKZ%lgA16FE$2c|6$ne zUnclz^&hs~HZ{R7Mt+(G6ZSuO-k*4hQQ&n+GgYBkQDNLnzQ)L|+ElgR=c{(ugVo#@ zL3eHvro4I>Fr&s_WvO5JNqH4e95Kkl)Q%IC%?qjOzJL5E=a#d(at$o5njSK1vf_@* z3`eUf@eYHkLq#-K{Xs25VYS>p)rb;$=v)QV58!_KaFFaj^Zdp4pI8t7%;6tD(i>)v zCFS~?qyEKq=APUAy(<3s)-Sj9?3)9a(TN|1m)7RGZn8U(La1~XsrZiVC#fh+xRhkx zKx!JAZhyA3qLPz48oGDdHI=C?_w1#|0*A07j6Yc~H=Mru41(c1*L!4eBKFb1sImGR z`EX;_QzkM76YjPy9aWUov~brj06#+bt`%UWfOY2yC*It70&wC;u=7DV^FO}*?;8H$ z(|3fBE~F|M*$3l|j0cM1QsQjs>~Ox#li3ImOkTXz?1T(V!fz z9iIl_RDI%7(I%HXU`v(BfXbG5Cw?ky%v@eob6g&Sx)WJ{g}I74B~D*l2ln~uAS`U; z&H6L4h)|EBVVT_STFtRB$2tM$tqVu@s+`-Ofdip1i&g3hgj)w2JpX5%F`>?-zj0iG zAAdLfqbGiTe1AUviSrJo)<|M|HeKm7HVUyk}GK7fe!jodk4H$?~J z%4siJfZA?Xt6aS%0=x4WU!{uEE$X!&N@EvNr59{H>3mg5F|1YwLX~1295F*Xs>duM z)9ak1cY0UX!2Mt%tW&(hTe2G;?y){HSbvbLI&dV$uY@C~WhO4`dhd;ihF$bWa@ zT4QK7Z*ZEEfUTWVq5@-&MAi!6CbUk$Z-eR7g}1}P`pmg2Ee7T*Mv^Y+e8wiBWF@>&6ba z4oyMaEI|;}6@7(gbW?O}n+Ki2*nj)4o&l@}1HL@F+^?m9^wJLhFF*hB6RVPMEp*Ry zyK`LQKmPKk7IypDjUjdXCl@*8-{;0%18Mcss%+$^u}@iKe)uCa?{*p#H@D@pW&}>( zC3ia1El~CIEpWY!iFLq-Zoo%9yk#{`>br`FT)dbEkQK)cq(j#kTeLV-b;#xl+5hhoT8j>|3sQa-;VhA$@kbLD3(0>`d-*YB-n?KoQYlny@h z{ihdS&?9HFU0rV7rr&ze0~5D*fBfgh)1Pf(Nxt*q>z`aaW^OE%D*e)LuKvRvx3B%> zoF5*sN!5R&fnen;;AK0vM1R-E?WTIK%;aguNaG-XCIEJ1Hjd(&=RH(Tyh8)UMKj=y zTfvEUe7hrDD^s>ZbNnIE2_sWp-DRp{rret^<>M1rV8TroRb;ezY4^Oo5DsB7o9ycQ zuEUuXkUuUyaQpgaUUn?q^|;?U{k!hB-oKl4+2Y^&!#{k9-n#3DpMQVqA*0gd8)$AF z6SE0Q97+?2+JPTJ&AZ|yL^|_`)kb^tUCj8Gy;~qK5~ug{H}ZBJs%Q6D6rs16bH`49skltis69(0ILPM<9}A_ua0Bi+WxOL_{$T2*7T>3ZeD4WE5H8HQGaWc|KR0MQ~yV! z{q$IQ`+>3FIa9uYQ*W3fX7SZ0RC}`A1MeA>X8f8rM8=1;*5;TYNSuZaxq6>9#4nBI zJsTZW^R^xefD}Fdvk8Rpkf9BX?2({}ZtQ!&H6V@B1SfR=_ z919HZiy_<_eIQxx%QW45^U*F84&CBT`@wXK;lMiV#(SZ@v;S=Mhm!?6|64QvTc7x87T*UU zb=Z6Ad8o+Bhkve%=fP=G7#0m5DvqD>f?#k}&FP+;>Ll^@TsdMI`&yVPW#Q^HV8;SB z|I}P6uI%-2;Ji27E)ream@C=Qk3b7fyHW+ll^h@AIG~BO@9GZjb6qZJ93ZuGC^ZNC z@KF3hFUA1uQnAX7hv~ohO8$%IKYspO-@krvYpgq7x_=ude|5k=936}Q<7dwN!QT5A zf3>_;>Hql8pIU15H|P1QIsa(c@iEtY|I@z{CR@pst)4S&RDK(8Oo?G|pgE9Tf$(WWB7 z-e;ZBZGWelGS#;ig^Ovk?I?{=l#IzgEiABU-H$$Bp_RB7Ts7qi(}g0B4~ot6?7^1; ztEZlbe4m~Jl-)L1ep)OS#=qFyYYs5EJulcJR(KFAVAtRBC^+`X0Jp(SuI5oydAx?p zzsJ?G&MpfeR>8Fz9L!QmS8K+prxH`q=Ft_FYk!BlE01DG6#_%i&P8D!oXi~PC=FDu zdcb*5$ED)OAKj^xf7TgL6+7EZc(1d)d(G|!GGQUonL=!;YU)X|H(OiHv6kRFP!_;t(zSF z?T`Gh;BOrKgE2qdigWTGJuv{c(uH&NM0*uD8&CLC*E5BYMMTiEpytevJzHEAgicp0 zhyApGP?$E3OSK@_?!hlGA;@lE%ba4m90=VSlyaq1{E@qIUD;gaOAOFiVDk*qGJjs= zvrG=#h}!X}FyJCv+l9aoe#r7wR8yvM#=6ZUFX?nl$y@KPl(ZeduvB%tX#aGC2c0~( z-uW88-Zyy2pHQ*Xoxk~3xG~dk|Iy-q@$=SL|6=2f9XQLJ|E0Zt+WsFr(0@9NyY0*T z&Fk*G=CB>!$^X@-e`>J##&Caew||nTrd~6nVFb6LAc?aH;jkSUwm*8a;wH0GGArAb zgSXP0%yyWk-55`NHJPkKCEPJdvTKu)uzTu&KOJHC>dqdCkv_>Mu-ST2*d}h&_F?!H zVSkP!=8-+Wk2ffYVC&6P@a+l-b+u~D?Zxq%JMBVqa=L|M@&zbaZ>WJebbpI)_=;x( zN;;>^3aHToG6YQ(#5=nuz4@2m`Gsi9EL%~9;I3bF|K-UVMB-d;YvJGgiDmZRKEgk> zfLrJMcirUyu3xN7UJ8$h@~)I4PH~{>*_&l*u|I%+ALceM0Vt1uytC&J|M0&4`L9^jCvNkjBL`XM@z6X7IK96<{3o9OM?A~!hPfKn!8{ie zG}cP>b5EP0OI>u4U!vA44UxQcX&Zno7fDQmc4HST@Fv$fUr+bD^L1p$@>RX$CtZMC zagUbgRu}^7=zmQ$cCF_zDXg*K3uC=swU)pfDa-D%U)hi`H3(1|ZtZ=vbNpQ)^a^d~ zS`auJ`IBc?U*iHg5w{H1+`eqw^kbhtw)ckyL(FT?b)Pk9@t3&PXWh!}%qQ+KJd>)w z^4Q$YFza~qn%JeVnNYk$S^YN5Fk@S%AJt9Ta+84lie zvhTO;rm~iBwWktYZFST%G!PDYT|H%2^Aw`19@VzoPf@1fc)f|Z3U!^Ofo zY+9c4Mu4R8l-d`#6WCBb@78I^k5ev3LtO6@2Oidyzt&$rzTp=2pSZG`AA zhFJ3U<9~BM_F8_NnwX9tOYIo~1!Pttq^6Ef;>n_-` zXL)8hDg931`X=G}rpt^M-yRq9^>#S1)lYr~{eR&*Mc=i@ckX}IYM=GnN0-O5O=fYw z0e|xe(NA5@CyzDA%LsBy{m|@JUBl$-=lzQ3fVVc2Y?YbBS}UEFk&t;hs<2tLGNGv$ zRU&iZ!^C>8H`+ICY0BU0op_Nwv5YtDWMy@iwv^KN4~Z;qK8TqTJ)Dv{NsW1Ia@XvX6l8kvG|6g>wg0VdHTlEKlz%U*nk7S`EqK1>m|%g^U+5a zP3`y4?-RIecf-1f`+{HpdRSA=KD_<)wKgqXox^TSa9r8lYa=i`;Eei-KR)4kT zRb=J0x$%Ir)eBP>w1`NSBfT5Cku(R+)miQSjjiZ|l^@Ur?N^Q^L9O?5ETJmXNLDHH z^y0gdrQ}vQrs+i%+NE5gz?>jz_Mz?;uTc`Tvf+vsg7tV)rZ+1{1=7)+^v`Vj&URh} z^`od7vlNh=wI=aXaZ`)jAIipSL4TsmxP902D>)>5P#=$$!mdzxYwYABr4iq`eIv^2#g5}OxY4Op?!u0IT-{VDy**& z7JN<4UFWR@GNI3l?;<0r$Bsv@q;ju;O0{?^+5}hJ-^8?nEye3N0lI+x;eQ7bIV$J@ zI2S}4hlPczN#F;D*61m`Gd#hhmPz7aSw{z49aK*nK8Occ8XKAY;`%kpcwWKUbN%`c zeEyAVq|vY3$%h`>Pi%A0HRMNUAoioDzwz{6{QgDllZG%>fNUowSNnZRTy*@J~DSo=w#l*LZ2HAcS zCb(5x!SWN_5Xe*8+&w045q$C$vlt(#=>Xw*DO;9@`U-3Itp!o7Tz{0mal((E@=pJy zX6IKO)%#Q1^I0dx-};O}$Me|Q2W~v=EYiJ#Zq;tQ_NAhY+CX)|MfLL8Y3 zZEAUITfj`*gW!?$;(z6FP3Xjx!fb7HFr`Izv4YS#&c!8KS;dJ zN0!DS=q4O8UOe}y6UmtlpE?oscfBUBZ~U&?yOi5x-gbeXd_Wg(txWEor6z#udg&WE zG|OCgQ(ey@SM4hdLxPMz5&+1dp()vR^OJ~i#u>^IH!H>+%e&>CH?V$HM?hA4M&`--^okHgn z`%rsIR0xzU%!nl1Zoc-=yc7jxP(zG%avyZoEH5H2otqb_s?LJuS7|JH54us>t3sKg zwq)zFgK@&E!H*5c<1_3sG^H@anPmEnGxgRtA3e}A9Di`_KXy*el*Z4R=>`#HgX>#H zLZBJ7nb4qn1WfAc+8}JI;wB7w`gEn-h~D*G;&F%5csw3iXv}mG@K86Ief9nJ6FdAF zH~ooeUB3G5r{6 zHRhD`7=J9_;dqT;yVTdb2Al6-W|e3C*vpMAiLqUy)EVk?5FL*wx>I$_>mGUV{^73n zm|5KatK~oO;dpW#i$q>)_NldHF=}W%_eo+Py12_>Dq2fX!jQVytt^$L*QRcmE%Dd}0+p~pE#6|Iy?;^uu@~F1lqnM2D$s^{*mS6eC2uQ4 zy*1G4tKTQYvnKZ2;`}z)3_iaCv@_bRmHOfgaPfj$G=8Rd%zhiW)dh)QN~{ znqHR5DpG79v8T;lkVA}P8Vl~hsQX9>7-G5s}CqogdohCFcT^xv$I`J z&AkR^2nE>-D;JeP9f+ZcP_{Lc{Jjm>C7$TAUQ~+CT0<^)C7 z+nNx{*SqD|rs|UC`;Eaar{-5*a{uBtUw{78!EjuEWaq;l13%l{CohE_4BIkW2h`en zvmlOK%abIbv$H^!J;GeX8Ul^4C@mu;t>N_Ko0Cs!mB^iP-G)?LyF`C@SB@+~w#054 zrm2I4>vVot?6caUZnaVQ$8R&h<57%UW;}VlusYP2iF+xd1OE>Ad}4SPoqc8PCx0jS z*%vV6_4SNICIqvcP#MUJFQi^wl`@l|6K$M^d)=$4S;sypWE#}=!4cY!P&U?FT5SX7 z`>tghO!GZ2J9+((?|Y2j1eO`kWB$eq(;@!o|1Dkr%+I_Boj#Lu1M3ozmB^kDX_c8w zLo!++J-xaP%3YMnqr`1b+n~=+iht~qBF)S>iy}8L4OaD5t}Z=~?L3Hw`fBYozQ7*v zjX?!i?9FGb%h&urYd8hv;RA)laiI;B zcej(?LE*pKxmxQP*~My<9f(3NN=6Aj77$xSbyd_2(qFELNsCOcqH3tk9e*?@nwuR% zoIExHO(oZU$B-VD0wj7N>^8$^oe3nHaRYvfZy6sLue<-r6UCFy8hxla`O&$M*jMi}t_OboHo&7k{^Tn9R2}Hf;_%C(Xgs$KHxv1m z9u3i^kb>l-6_o>T;b}99oo%+nnRfC@LAGbwRiPQtv1sQ#y&>6_wmp+zo1Zk-b)qs? z^sJIe9yhK8S7W+XH>fqWl8&{?)x#Q)eNOu@6J4#61XZIk+c4*jv)>0?hoIpqdw|_=(|7Uyt)tL!kq2E0#?;pEI0{F3W`GEJ#2KAY`mh)0H zNm=RG5n@X{KYzSU&XCcQle1{A_&mMul{DARJG{#q!(ho}e3t`Ah@$z9hKtH{+{V&& zL;aS;?LkM?gxh6wx6*I@*5#M~g+wy=e82cKzVLXR+y%35UN`*K*C5jq4GfeoEgT80 zoaiPz0uYGmYAqHY;f`4z?g+)SP1Gt=P)?9vqzcCFc7KBe1=G~Td1|U%cr&Hu6~Ei1 zN?S{++mSkHiNZvmZ`V_?NcfJ-#b$hfqlQF)Zm0)B+);4qzTUsVKyADt? z4-FZ|GUP(Af0MKKB$|G8YbkFx(%n+^VTbEwJfnKb{WO1_>+7!BeD(GAcYWSq%aniN zg)fceJe}Df_eUzaYOZNW(v*vU+@MTy8(D!|SWccDRD`WEs4R5DOi%aBaH$qZ+x)5p z?Nuk&){<$2a?9Cfsj!pm)?{g^+r$1#k^A2xrWJrvzK=nBco5@Hp0fS}FTOU}qMU9K zkAHmX4<&ys8|+B%jZY7H01n%=Ri!#AuC>h1alaZSFwN___69|~mb)whVM6zkb=M=I z+ZJM|O)4T|Hd4cCZTXt8kr|TRw<~1<*74EXcw7cm+yvZYTo6L;O% zpVqnrByd^0H^5S@sW4MIOVMe|kK?0~K==Dfb_2)+LSqaUQ9(3QHMcfkt5<5bh`fpQ zuy{-BNcRS6c5Dtg;aVKZ(Xj2GVy|g3} zIxl~T*N*bM{ANucC1}FnHRR=K^;5s1pugllI1L6H`PK7Wx5r!__?l{{F_C#pFi(4% zDVKKArpC3x+qgDO8armTiiyICtqM1zT7?d<05c0og_aR=@b*GQtgc-t(9X1Sb&C4* zzU}J*^ISU-sZ+Rn2jwQhsUsC}DW6G(z_NekF39*eHk(xenT1Wp_bu<?keC4M5%lr;&o=v9xVY^}sU|FQI{B4vS{Y*%85GC~1o({{}?OP~FDeob( zWXLs_ik$=r8*fxaCw&&n=S)z%*pP)s?fR)k!}cVh%T|GEwk6r#If{l&#Y31`5(0ls zQQV+5aedIUZ~a|gp(9mezMsqB^#ER+$?#*Zw875uILfzX_xIZTJm}Q z*wN{7V^m*7jE(=GVTaczVrNbAn?- z%OQ2V<_vA%gdT}ypV(?XKk4Woee?8lh#O#FOn9B;ul;SEb=hP;c^UdEK8UK-269lP zEY_ieGcGi$O4UuLiXmlIg1gk=d-4<%bdoC`w>^z%U-A6`!2Lu3eFk{QEpC5NVg2ki z*>19L{Z1eM84LN=$^Y`l5Xx3p$ORcn^3YMb1?DdIu!^OQ8QU@$-I(>dN{R16Y}ib7 zv1jd$@ARTbBqhCmaiym86e~xx^-b2&%xmr!u{*0$CDO{a+~Om@`f24rlJUIfh1;-C z9qy+l!Ri9Ow~*W2pEaAS@wk7=`;VG2etJQYxBOr|-EZ;&p*dmHNnP3?TIbS4QzV%r zO5bi}kKl4j?@C9*$fK%0>RxhQQlyZ=2`CC_cHVm|G%L-Jfcd}hi!HvN=L7)$A2?lp z@nl zoD}8M_5v!&v8!+SdK*dH?<0)+{Su$36sPy^r7{I~B z`XuaaSwX==B8TN^BHVv)7ezDHTx8UWfB^aX}EuD&w3VFfX*Vei8T)|^>w2^_NE(zw#(R22BLbc;rMan6AK3g)%1GN!R$Pez7cuO3udJqS7wTcB%7 z4m%sKZUzV!Ouq-b{{JWC{(pW_E^HbY&QbtCKKNdbin z)){DEs(=XM=MMlSAZ5D4-xo1Np%Nj?R0?v6{nL2N^I`!g1aN?d{>~{xAe#Uh@;BoL zKM_QuvI`28&J-wLcLLLMP+Gj*j7LyY2${+^hhoh4^TrjJOt%3lyyctI=P}q!H32g0 zQ&@jNIvbBbTJQ-RXE1?{2oBXvAb^StYKSe+Iem$R3JM2U2cS@MJ07cH6Q!5+4Gl(*?hK-g1;LSGUqOMM1II@2TvhTT0pKiH{;YfVtU^t{g9a`ey7X}(& zjUU0Z25AKmLq|{>XfPxq+m@Km(-3KcGe`n#7(yTm5L29Ks4U@VY2)#DERM(7cs(A^ zUgN`f{T@$WqlqYqhp?N3Uwr=Ik8i+cSqmtPNSJ6CnNI*G2%Zep_AW9(P?pd{5juZs z+BJ+!z{6R*d(BfA$Y%z`DqU0KeyS7`?a=~hTa2=#m@+6>sd-jz?m;p! zs#<0RGDVm!!KXdSVq$cVyAr*V8U%+yWd}!$nS_UHDULjbWF)qkhM}@EdWSr5S>A#! z(WU0dRK_aC3@WRncg+Xa<~WosAR2!uVbbO?8lEer4Dpi=`X`eEJJxzqYI}&NVEuws zdQ@qq1c3=euW!8p#HK=6x_eOEFw0x8>L}Fr5hkVPK8*-t7RV^p_6Q=k%~~m^5muNZ z*NJ8lBn7q{3t^s^&;)XpxAfbehnY1z2W)uGLNvmh&Nn0WFvB&9W7K9XMnivp_zZi^ zgz3o}h$t$e zu&(0p%)omjk>@awQJjQzqA4DeIW6nz(L&H_B*N?gYRUUbmQQ3}LAa|u#7?rDEb7$^BHBHcI{~XR5pi_j+J!t$l zP0&h>YdAsYd{ZQRSKJa%Y=UYOQ-(dh-L*b z%_;|oEz_dOTb|P(AO6C9|2MM0S`8yvGXWXB!+_z8=~Sn29d=JTObyjGTwtrr;rtA$XY~7T3C-3X{|SLqOkK8!A;{b5s}~bF6Un)#pbt z$~&3~&LDSy3eJaoZm5Eife)v+<7-e&@E8OR18LFo{TpBY4Nc(fx*W;{BZZAM5;3XZ zG9{E;hG>DZN0)!ayoL#o1+>!fhpx1wy#GB3Fn+S4Z}-|TCo!j2X!KJi^Z9jyySazh zGzpfy_R*rMZK*Z1D@ETSG_>zclgOWT*0wYr+EuUbP&c$6O_OM!_Ep=`YG^llR{oxU zhoLhvLpduoykHOtd(D%%+_omRvKFBqxr}1+2qJ7{<{f`f>T(@ZJz6ZZT8|k~^ns== zPBP9fS`W__o2=(XyD!z&?l-AV9c1 zYOY!9Qq5Rm)9Bh_=*#Onz`(~>m?l9QZXYeS+Lme=AD@o<4*H?JHcdi4?VP^Br@ikx zR1EEnX%c_+)6VIe4edtJci0=+pC*PCpHFiSFB7AzMR}$X>odp8!S*tUFf)sW+frtl z@G_S0;`+-P^jDhwGoL{j`&l{Ia*R}s+t|u32bnDJAj>U5SrqZPvNLT;M5Q0*r~iWo zJ1I+K&R6msI^#}ca6>tgP@?oYhkcuXnbbf3pTd6&CWgs(+!OwuA9y$VH{)0WGM4`^ z&QCkY_pdWY=tz4Poed;33i4>BxYZ<84@Mb7~3 zm^^>jTgq)>{dq>|BtW=@hh$76!z`>!lX_!#U;nSqXn3w_LRxa)0&(3z8PGeJv~ROk z;XVept8(1({gWQ-TSm{wy(Y98?4~i;W+5%z-p(~4$y!JxZb$!y$1-l4fBYCk>@2|! zLxG(qoW9^nvHRrr*^(dcJQ7Id12lTeue&bJVW#|`S>{sExDbH6_-Ys?6qD_{(_HOJz9yiEkM+w zw4}vg4`sc8=XXBKjl}sz_yYzr%yl=0%q?tnz->xIa}OiaXb(D=qw-s*MHRe z7dTIupnruc{;5BX-;R%4Y0*K7WNm*R$aSnqU1Ht|1DD>-J(5fl7DoS{VPQY`mVbpM zrU^?!Z1{Kvc^PQ)UyX|WVZpjbFwL?=91WIoGwm$Mt3+bALi3 zJMKq|zwr~>7MhaYkF^!t0kufM;4;8lJde&9dXANbGp z2mW*2_wONm*0+yV&f1p85s1adpO~h_GguObi;Diu<2@g^mcs!A16*pFJnk(Ia9fYV z0s6p+ZOaJy!ktXMRw3MnL)ma04A;47YUulWFa{lb05k3MkhGyY(Jdwj+(4{-a>_`h)6?}U}n z|JEBR_s=ADZ^^(r9iRIRelp7aR8{-PD1T;&@Bq*KVgDDrza5B>frGpfk@ z9vMpn13brF67MY`cXN;71OH^SXdkUcc;CNZGmf|a3sxHN_ys$xr^bKMF(2?=ed&Ug zq3`}n7cKq3e`!(-c>jXchrWNoHXM)NuzlZS_E~qf5bs-lo}hma^4JWwSqUwk3@Z_X z-la$Lx9@0D9;=eHpe{O~iWBVt{n)(JB!C2?S11AFK3u;g$Pg}W9l!y%U913YZY>3k z-Wy(SjsRMIuhV-2Kle|{WBiD;9z_M$s|0J2$nm1~j);@pkQOam>Ce_$i`+38Ht zg0NU%9%iI26pDWxe4Ybv)EAc1~BA^72GIRwcg?$DY$QH!IeE|Cm zmHa;sMgyrn_q<*=`=5C)0UQ2&FORn)lf%apJvkiZ`5lEo0;uLkjUV}W>*d6J*KD_#6d~PmIH*eBM8Jwj94b0Ep!NRJ>XJ4u9PR&b-rKIW5@y` z@bhUmJlPgdY*`_y1w+{Nwijl<&Lh@x1@Hp{0HF_3=M0yo>tt3p*BB{`e13 z5dK?xPygeuAD-b~qnZAEdgdUkmEnknPxepXQwc-;69d4n|h3;$u=|)rFYWv7%0JQx35OC<93$5<`ADwq4#<-L$HO{tgpi9Ypx3+y4n#IAPlJO0a#5prT7fcP(e9AqlpR6qRZK~ z=*wnquP5hSDLAzqI$o7`4InZqy74ZyAx7uJJHcR}*_!g`)l+{wt+H7!U^IFdChSLa z<=C6;^>%A}Y8qKS+LZ{&!aaNkTw4hj3owk0*#=x_%$zDVesrD`DEG&-&_-to#Z_QT zqNjVG%%!TkOqSauY~?!Z54#CDZ~WW1TusdJQW-DLV@6Q=6^(Q(ul(S$SUU zr^f@VCY$=OgZBukwv|1nwlJ8xd9c#ZW7pCdqKCHHx|#0N3ia$%GylCUQvL+g(%+!C^G7_)x)aZn}0W z#bs|;t#PKWZ9nO??M>)z_(0dUX`0rfX6B8n+jub-PwR75_c@#D-FSrk>NfFq8Xddl zBa-oCr#0eK9oN$6G1<{8nP2vloX@lDwDgXL-a;dW*m<`?Dqg;K!9v&>PkesO&fe^P zzUr5q=JbEtLoxNP88L}e_#16%^py*Sfr;k79o#iHImvblWfx5{(rA-ob!)bGZI`i- z=lR1ZHd;GMqI>5B)jnPwLi0GjOuZEeXP~Q_yYXxu_!hbJm9N^^3q~%n57!XGv`GyC zR`0#{a)Y;Tp;6Qe3tg)T`RNaE``hyPX)pvT~RG zwh4b#cYJ=}N2ENB#fFLFs922pw`Jd+#-h~J$60!>Qc<{V165K8nvWLACSUbBwND63 zs*@6GPCnZ^j;;Mlo zHZx1RpkrlAb0==;QF*@`F?{8@87F5*ZYY26AEVlUWtSO}qzIaX8HVgW$;6_I-BUK^jDI@_H}exNllTgI;h}5~yR*TyM=)9H6LJ&J$^RIFIe; zfL7bGFz^5DKI7%X;NUQ)tdQ!VXR&G_CBt}ZCbalJfG5& z{7@b;Dav=(mjrf3=U9G@`-<)aM|OY5&~~&9bb1l$s(Wt6>t(Es_EkCAUiaxKX|7Te z<)WR^E<>eG^3zDDPTTc#x5!&%G83#Jg_j`KBb5?kLk1xyS9JHXCp9 z=nk_q$U}esJm|q=;UDLCB`%=fCDr}tO;6XWpgo&Dx2#F@aFdPf%iCppHKKpw;+OdJ z_V2g6xcccRVzq84)5Wxuo`#twEAdi4wX5-dMD*+)T6yC%qfkHrlHT{kr$sqQiu2sN z7^^tSR-OQc;y++s?DDnS#FvYU)!Z`$0hg^}zvxig+eZ$3VprnlSeJ&r0h zOfSbJ(&pv%xiTE>yn4CHc^RS6NHcFML4Cc?Qhnri4!Mz2jaue*uDx@bf=A3 zay(?abM%PZaXgFXzR{J)drw!#RpYz&S zi{%PA1vO5S>;=|&;o>^eGjT+hPwaw{5}TY11x`iY5{L;CJ4DhrJ6l#vD|T0jak%zK zJsz?|46==t)=w2=v+jST;(cmj6yBJ-zr`rnWmpi9{AiH5r;5_&_>e{ZQPpqmcz(yN z)5iWqvoCmkYK+>H+pfo2m@8zGldM0-PP4n_%3{ax#D341W~h-C=c4E~vPBkBuUbaP zuCA-Vq$O zN~_gE&2AIvy49<#I|q3>4lVQ)_olKOd{T(-V_K5p~+ zwhtz=X|>#)Pp*Gv#P(UE$Iup?b+Q=S^Ldo4+AY(T8X1cvRjMps~*1|C+B%HU(2}ccT06OUvK(u`I6P^tJ$s{ zeHBjEt5?u=ot-|-`?!CH_z-222yfms;JI}4?%vumRdj#BjhA>i|AfRyJ$dhYvEEOw z*UJf651?v3d z9C!C;0Iz??baI+L;dLwz&U0p7a;6^lua^@BlkwVrK4!1kH991jh#xlVYkDC2@oM{g z&EBDOSZIh{$Lx}i^{ppJucaWcl(3nfom6ZUp*$6}=A@l7hQ!PHri<%*A-D4;K~mR` z;gBb-_+%b)QjB)8XGrJ$c&mxcp)N}QzKxzWEp>lVB&Yi$s;7@pgV7f$Lv^~%>`TK)8~jaQ?q>0KzKaT$K+u9tM_w~jyK}@uFdhOOcu%6k++)-dSBMZ z`96OiKQ$ps#O+HwwtigOn{9k97pM8t9CfuBaeT?uEN$vZ2f|$4JkxkG+b@LILSPe^JErVx*0(}c=9(Jif%&K}+v_pU10_FO z>y12j>ajgzcjG2$=e%@JDmlvG;{w4x3UGh&HebE>+-}%vKYqRny-Cl9*DW}p$)($B z{W+hH)D;|rS-3bDX0ydwo<+$~3zy1dZPx9oNaDhpzy+n&(qG(;I9Qn8eyhz6>aFpM z&_14e)6ac(b6nH;)$5LBvATIyduLR+MYG2{)X1FWrgms=$v${1JK@pk`poIN$98|b ziiFuCm^#E*?Q1=pdCzovJICr~c|Qd1o*iqe#B!wWCO0JNs`6AQDBJFY=Fv5Mww5@b zPtVB>gZ^ovn-~@IG@{^IA!%{OwRaS-1FPvTo%RLU6lEopQD%E=?&+>XfZeC zULR#!=N$>kAK@tQt*_FUTKVnoN`>R$9ar#xBbcyt(b z?11YB#JMa^y{YSgfB_v~9a+xhbEwZrLf->V?&@uP^mZ-8^6MDLQ}7(OF%( zzEP%uuHGhn(cb-#*-qrVf$@)0zE}{n=|MOpms`6ticK&nfsW6TA8SX|v4v@UJ6$At zK$C{ir1urA-|XkRE?V4I)2zMBi`itgxcDzGz_zsZCTHy+oEbVa{^~GSo$GWeNM%%@ z>A`=Uws`MBsx3D2d~M5zU0i>^=92RY``5#s_)ZkwVXx`~OAL8TnM7L$VSS$hfX-R*FD-mIzQjVJmZmFG9s-&MDr6vzE& zf7{jO@r~QGKkrO!H@_o0d9LPOQO2D`%-!oiPH0`0xpt$=oe*pwQOxIcJApdOZ?+t1 zYvxvnJWq?mVl@tXE%n%LzKomA_7P557qKnzqJWiN>V|#6LBWdthy&Ter2XcO(#Va_K53^stcC)dyjvtBb=F(+# zQeJ!UgpB$yU0y-LfeInVlB#d9s=_J-I(FO%qPS_i+JMT_o1aeG{j)HTiM8KQ@=(PzkfsdP&(U zN%&Pv89= z?M$nhhweCQdbB>s?gi~^{nWEjP#r`s|oB8~%u)S)!$| zj~{<;F|A0$Qb~4u!|#?B&DZLC=H5%;v|hDxY)-|=W|3IYv3+%~^=fQPr|5KX#LV0w z8%^Ry)@MRZ&O$OW(D?M!URFQ8U$umm*ZCw8^aa}8k8Y8v71w>M zbVJj=YOFRrCgvt~x0AztrOyP(#f@}C_5FW2IV#kYoW=P7ZDzf`6^4X)}Pn?^6=>!?dwdnbvM?WF7fVCy_qtp~d8`Y!Gn8I)5~Ip?gN za*kX1>CO4D{a?852_%-PN>GSJjWLT4NiMeSP+4>V{tFLQ#8aNg(x0?-t-thKq6r`M!hWS#vKT)Y5(?hdy zi&r0B#4HlY(si!-n-tCQx{x4O$N_#v+r`GOx#`GG#WWDMIqJyW<9CDQX5IB+_>kfajixyjIt-O<;0(*}bS9tjF^|{Wl!_ZzexgTM5 zO73NqSG+8$J<`3{WOr=@K&Xzuo);`BuHd-4Z_jMIPEb%Gm^fxQUIk`oV|SDITXx#w+@>Xwgo0=%#~WnpIrnH*jR9ItC=&bo_BE)RGW!1I_gl@TWX#R; z3OYr8?BI|z!bLT5Tnw?thX{YLE;_65wnz0KBFger?od6D%(e05f^%g={BT>%i)hMU z(u*{tKz%YiZz=BmsU8sJjLA%0@d9Hh)7RX&n};>~F#@Wm5t|-@wuZ#DOn`T0YRR}# zPUafs^d`-}4*0@Fv&z3TNjR!?!Zya*(NJoE(eLC}n0hd~- zhE1v8B8I%^w=$;!ipU@J)Q*BPT)axX<};zcd*{6X+^O>j>6$V9LzPA`l$mZ#bs>qsOB@UL!(g;(ph%l@xu~#t+8Q8 zDk+NLIw8-QW4E*?F;#zr3!%7)H)dreKK=CGTf99w_kbgXkKRhv-O&TeTv z2p%28oN?O;AUQo&b=Jw3-Cs6imQJ_A0y(Q-z5bw_nSVeiUR_mvC|QY!u?M7w+{Fk@ zP*U@_{AFnFsB~}W(euz4rXE+7cJ5Qxp>NBJ&Ku#{=_rZ|A|Nn;3Zy~JEi!b|bh;$0 zkdwH2|FC}=G?CIm$+ql30Y7aynfm($)*V=VmlS?+FA*)86QAVQhq=Yaf}`RahrATi zI@E!s2=1YIa{KSic=^Oa)X*onN^)hZy+k)C5^*Lp?W1w%;NRy-o`D=&8;k9wADwFX z$W>W1SfD0Ef8o9sJ_1xw5%3zMFX89IKouk3s=R+9hZ6YR#mDTmhi2#N-A-WLee>mP zsf$>3wdmYv`g-Lkd3rcGc_Ie_^Pt^D8$%EVXX{{?ty4WtBfdOO&PDU1r+Be=J?`6r z&{UBX+PJ#OV48F*VJh??m&XV3rXjYllD!>Pem;c&ln@KwVxWCrP{CZ&M{Xdj1WX#{ zsN8>rTQ`E>_0wTt7V+NenBFzfk*m3?XZ9xmmQCCO=WM~jVO-Wp`@iI`nVeQi`Ydub3N#&Hl zv2&(xtG0_p*kw@^9$7XjxY=7+U>h$~WY2#|qc07TbJyvjTa%$1A5^}-KxW1}HUcPF zcQ%zCynTqFr{m|XL9aKnt38#N+9jwQ;MR5)KoltNo%oW99B@0n8n%5hWqf|m7Roez z&)Kg-fS5KA78_aNGXLZs0&;5u2l>#;nj&plJGi!mu!;2m>~5lKGZ84xXNfpvpW=T~ zBE1Ag+=V+EQ$XL;Z30`xrsdrHg|Z}tx<;S|9%>RNS^US%?`DH^XidKZjnZ8!0Yqoh&a`us+`tKmZGxCcN?#xErlP2)CWDU zyZ>mZf;u+nfG-$vS1LU>oSkzx+si9=7YaXUu4(wmvz5~*hbZMV1j35O9vz1-#?HZ< z)RUSfL-O0vFuVqztKmnEZ8}cau?^#4S1ME10ru%ZE4bnECi!0^=x#RM5O;s{Bn%H8 zjhVw;`NqoM@~kc7MVxgy)8i5&x4B0<8CFjkex{ZeKVz(I}u5y39XOH)6U2!mMcfQS)C&NhspvrK$nn3fXGB)1vUF{B=IkY%O8Fc zY!fie65|O)&~`X)yOgVkddz>NdfCXqcybI>H|0G8O2ZNAts@T8KJ;$eoLBq=&cFFM zA1sHkIHu{2G1k{1YaM|D;z7j0gk5uUcZo#;T#;`r#Os8+V!L`Y$?cREk= zLhh`3>laG8UMwXmCBZ&{nI!gOpFOD)IH66`hx0_`&!O!ik~6)Wb8 z6My@n0#p`I=WnLEmGyrV=vT*U--ZCUThT7eWP-HRnet<+c~5cjw1FEGrLSdoU~UlY z^Cd#zIz(s{{Ro{DrTrd*YMAzysPY{WDDmz4=AzmUiV8622mi|Psv+1UJHJG76N;sZUUO-tUrG`aCpo$qhJxbfkDQ8 zWq}}aT!ex=Bkbh6DhSpyIEJu4q{r}fEbO3l>-_n4svf5+K?lU@0@9|co&FjvXh%Q% zLY~UlO|nMO2&SrG|BFlhX`T6AQsM^6zDs*e_b673 z(1j!DcHI|S(m*+upyC`I)nh^EMTaR%mQF+4^T9^V%(YR)$>aM+SymBfZ>kzwu^?0~ z^g3wah~tFkn#^WiBCzHO1T2=Ii?5u(7MV)>DL8!F96usO)al1<;#05k`$#j@wJ6T-C$479H3Wj%AWLxW~Ca0 ziylTUvG;#sEC$aHRNz{0Mu6%%Tj8xKDv-&sxP3h1sI{T=;!7K=9t*Cdzrh+h`;X|qgcua-EXrYL~7 zi)L*wh?Z3jna=Se&u&rex`th?&)aMcG=ko-5u!W?4LCOAN@22-oPwO0g8*!EA%~rY zjoExRfQuL8SP6b6C?JpJGJOqXkyhe+qdLjW<>G$~Untj=&G*5686dGZa#k4({=5Q! z=%uO>ZK~XT8OlR-^KvcIUmYY^GZ2qLBpFMZeBBx%&Cc!Ey7Z_h$cV4=?eH5kq%`K@ zu?VObx`S~&rR#dgR?Jg4)eVuRDd+5Md1mVB9iN=7(Ut271Vr5!E$<2~1(pb+*8t9h z-4=fxv9DIvY?Qmp>DD`>iJ5>5qP3#V<_gzfNSBT;M~+Ss5qeGmgkf_&HLG;1vqjV9x(2uxGicGu9*dtaz9ly< zHgwQR{@{k`=Mun**AUqY3p;IJOr&GZ6`z=7bT-uEHIcKuIq+-VSO_U%@sXv>)lX^Y3$nZz18E{GR9{{4mu7?9c3ShOy(<)Pd?Jc^W1thr*F27;e?iEKc6G1DnmT+CB%2<@rm*MEQ5ycKHR ztdKwxxc>udL_*pLfis_qs+ppDr>Ki8MZ)+OmaXOhY#*KeFBszvJ8A@wIucEL>r1y;tRRQclvKdeLb@5&*anJ22l$SsS)~0oD{kL8 zoTd(ji8<#EwU(*GZD-lgC(}<`r>M5QlL`ou@-ra<>br%q20^KzK-ho2KxE@kZ~PYY zs#7?(hR2+%2D`~bYws?F+m(8Qwxr3oiq#?qj5A1--xAN#8B5ZO>87Ire*q}p@a5A_ zY>KKIezR>ZnwJPjjy>7+`QF6Z4{2m1{{xr|VTSo3DMRGNq)@@_%y4d{(W0iAqs$6< z3StD1D(v)o7v8Q5!4!YpgikyQbNor+jk}E4P0~}B$*m4dF+E<p>Mk zW1}ZE5Jkluzt)3CM?);q0s^EtMS6n@jBc7dRt~k`G1ent4pe{m$K;?&n%G|Y1gcq* zdQ9tP*6JlKT0oR_XvH-Z4tg7iazY3ss};`KzS(x_U*9YIKK@ct)j3vM=>_MIi;(lg zjRkmjB_B8nITIO3$}^SC>NG15da<&#?es1mi08#8>$E`%Yodk zj-WE&AZN?;$q|40qx=jqVo;f9+T9me+b7Fw!he?8pLMcT+R{SpV4^6$o9VA{R|2-j zSUSf|-7@Wlru}s*{{CSqUDt7LctU(o-<=1P`|O)Rk`bVm3>nC9tCUDs6$~qNBS~(X z1Qt|(bm9X5i2=lQL>0~v!OAhawzi$KRD5-WcT?V%G0|fkK54XX+Ub zD)o4mHe*L>7`y;2`^o#=479wZTWMBZO18o2x8l5W`h#-D#yOvy8dx3OVswQr>)tVt_(wGeeaDgz_RGin3WEj}Ze zsM#(np(b7VDlS5c-kM_5Pb#wPdXCNpA+=EI*Y9uO)Y5@K)5tzB^K-=5rd>&=6&3$w z_e(J|;Q}R2u|4_~&2*Gq`ZyRO^xtJJhawMS)zXmY9`zuwlaU%^4V8E~KTq2kiaYk|+r3cQ@j=>U0M~zW ziv z4X(?9mTXy+Z9~@X;!H$j3n}Pz!x4XjEa{u(7joM!@GZ0lU{#skv^Rdl1-{^tNdO-_ zr3bXpBDq76Pfh;x=H5B%O8cSiW`p&j!cjdM{8F;J&sF6<7C@Wv{gr-U`I+5FnP;o* zU^^Oe6hE=Nq%Cb8S`e54O4FT@#hBGehw8My5X4I^xs*v`$Rn~fRue}^rQ&~B2%8rb z(`xped94bjq3E<11c*U#E7!0Gz;nISCR|RjgeTvGepT(--D3E4@rfVJSZYkt{{|>B zT`7d^DV7qB{16K_lSl^?f)n7p_MR2?Y3fz0vZ~=?#B++K95p06F&zoGd8|wGQBIR~ z!Bk&$QuNoynGRHq`%LglL$H6c%Em*==S`dU@+d?3C?RnOR{J7C_B(HA5o#WgB!aWF zvfO3{HC?X-R6+=C+Z@bmIuDRa?ZoT%7s3{U%7`meKB_l2)aPN0``RJig>;Ev$ck!& z-69Y0>NE6*rM`3pLlwTihly7u;3J)gYa=e^No7kXH`R!K;3ypUuAEuQP$4kM5c9{LLRCaYaI3zu#lp=A#Gpk1I+DAJBg7P%q^NWp||<u*#!VbY?<(w-YKxrOX(^voz^Xr0>iVML#ii*JJvXYaPmIO5^#9T$)=V<|xy*Z%|Eh3c{w_`oqJXoG>wj{|*vE zUkl6|jbWNt*|I$&@FYW0V!DQk@Q@RQJUb74bKpL76A?Fk`=|+j4mt-KH%XHS4`~`B z_#R%#NeTHK03}HHG2v$^LhFe=!!wyP!~G}atl8akb=5Ew93_cOlCzIXz(3KOVI^i3 zhLxH*v9LhEZAAda_TgrG$#Q0X99PHxVvBCo#sMy9huxMetlt>JyJi$65%qAr7od1P z+MT4QX?xX|f=s-BBYV^E&u;OWOgukqX?Xnxu+~Yb#N!0Gu0EzW4I+DbTPgXQz-c@io2Uhx#*)F4g(Tk-O#SoCVIrG4 zAv24LbDe8{mE3U+jb!~F^7E#64=g$!{m#tDw;o0AS~Igf#NNA81hLM+iFP!j>86mb z?$$VXIEaWfnl9Jo(jLuWbX&r57Yd3_h;WNy;X}C|dR;5ji0|V_-PgDp3*ASh*CqsB zW^4atLVYzXP!kj%?iq#6d^Buv<^HHl5q1)3m?ay3N>+2xeCh95$n?7T9G_z(x9OzH zDHs7kH=QdktL3fD*Te`s2Fk?6oaJDP-#-mga0cP^mydE)r72FsZ3fzPM=7J0ohE@> zqc{_Z9vK`fV~B1odG_X$d_`ji1onuN<=5k~o^YpB<6oZ$88gsE7%{c4U20Ee!BF=2v#CevZmAf2}4NE-#-vv9|3uGC^Sf&tQ$ z72{~R;BN+)V=^zXBw^Y^+-UJYQ)J-eGc{oHF}X!G(q<246^la8YjqXq@`s8>wD&qBUpM@J zA1c-kUg~e>T>m-2#ece+#w~GUip(R4|5)4cA8;N2k=XLPTq1xaXbSxaI1J|>p8ZdI zZ8ONkBgva`Z`3Y-X(arKsek{>B7ZOB9NBfuA(LRJVi-oH{)!6y zze(w!%zN-JN6f!yqUoQb$oQKD&xIDQI1}7ff0|=1`~4q%@!ud>-XWHJ5|7kj7~`MA zTl%Xf43dI2)_Uit`Ij5xLqLqvH+ltD6+{~(p z7;NmLGQ>HdV;rUz3!&pM2jjM;TQ#jZ-n;@%`kdDx%^tFaQ5qgm^g-#^%uBik3p7M# zK%}^^G(L(R70Ht?bhd4~ssQm}0?lI-s*eKq;{;dbx+`kqZq9JZ_*|!v#~1{3W^UAY zw_U)ihY0BPju~9RGWT8`1t`#e-e6k-?pEbj)ik#_M$)e;g%r%!nu@ruTRA`*w{msDpt|%zEc3T9H-Em~?iS5e4^^&V)N9H^tI6Jz zRrb3#-8zW6+&^d7ELoq$^%Z}=LtM=!JYtA#XnVG7LIV355D60|7 z35Ql{Bas!Q)fnAGX0yIJH*<^06H=Sj7rcEMTq#mRJ>1`{`&}*@aw@@Jh!FzZN(~7I zHa)$kYR1{zD6zQjGV!y~>z7F)jOL^lrfZs)%2$gG^br9Q>^_bl2`KvbR9*6c)io%5 zrYI@r46#d-OIO_vQ3j%aQoV%FZ+EaWOfOK_LiMLw!!ng5-Je^Psy#bAU8E4!xdv5T zjbJ_0IgD}Br~|;-vSBR;0%J64f_47w7DcgR;d3Rv(0ElijI5KdNhHN9x%ErQ-V6ySWFHB#!L!NZc zg!-8JTmvF;!%Ji}#@YOSzlzrA2eiz;qq+Il6jWa?8r9izgHAZ1#-gAGiv$Ou%j(=H zHru-_V^T)U-lD#L(7%k2Wg@9onhGS?4Y@XQhQeZ=KJmYuIeupsU@z#kwyAed@z!0YbI{zAN0l$zfMTT`qxZ5yXsU7iL^{nLp`p>~ z5}{y93P+8hu_2*y*yc!ISPM!CTh8xozg|DKr(X?S#mn1&Cd{aq-~mFyxgMhy6aE^{ z7v_P^-U!SK$aGxEn7A)1O@iL5+7}TU6{1ZASu1~hL#GUvqB2d1$I*DG2DFAjwpwKJ z<965}8h|4SyKs`TAQVOw8E?nf9jCpA;Y*lbOkxIWrE;H+XgTY>E32q)vu0vSDyj^g z^}LUgfOnjKH5v8zeLMtI@`ShNnQrTt{C;yu)Tc)X0Czq5G4UvfyoWwf6lv%coTK|S zt1Zg&R}%re;6Xee7>%6q6$_#X;l8>aSLO+L|n?j~0P+AOHy&$lz6JbsTf+zw~B6C!3( z;nhrb3W~A6j9WNvpvPeo>@`hB1Fzstj;uorO6d=$s;{M-X&Ou56B4Aj<++FnS{oy; zxw+SW5UCV!G~AHmM@BiTp4eAy7fC?+FLlJ%40HxWqYk76!OC{8GZjow_T zBXYk@i;XVUnnG)=UnHtctk)eoJ&v&v-S0_X00dY_0s$tPaQV6?dX>j7fPxo(T~N!P z42o}}f_w1$$WBe;qc_;P#VH09_tzRiWiY&d5Bt%}NQ5&^1akZ(XN?eO>HfYHPfju{ zck$4OayqaeJ1~Z?C*7ht`)7X!QF!}Wpk@js&@|r6XDAMw(xDBF2n#PgBO?_(1rk(x ze->j|owab*7IoL|5n%mZFny3Y1Ypp0gZw$v3^xQaK3}03&Fd#r`ORh zX4G+3+#+uB0Sm8hzSeuIf>Yv=siYy+nVKNcCP?aOhJ#Hta#T45AE!mzf=@-7UStTW zWLWhYL*1k_W8zM`chZF>4~GvH*v%;%Er8OhUl0wqy*$Nb3B-@jo}aiG@zZ~QD2npX z5nmZtV>AcMzeooANs_oz#?SndHsQ>R^23Q67OxYe)EXs6xi`VZRH!{3LHq$+9+NI- zq-QK!krzHFxkTx$WWF++(YRzyxqR1}wgdGjS0w|OoifO(l1jiL1k7w{n>_dFTr3H- z=m?jPT@2nJ_xUM}3yjO@egnaOHV@N8ak@MfeOTNTi`cFpkWXi(nEfy&F>jC0fTxLAK$OB?(HMc~(esueP=+fnWM|!@K z!y5n_Z^nduyR^pDE|@a-zMIutM6TIm^s05I`>{6|2tjE&Uqvfd5Qpq0M&+h|%flc+ zQ+nh}Ch-9$cVt{EuHXcYVq%a?lKY9OQo!DKN7M{_ztmlfvsPNp<|?%9shJ^-YKy_l zz|)m^(;$ipvOP7uvKt_OlZSq~Ss|y}8~5tvx>1wF7eZapA}X{82Jpc%qF=_F9k>hI zyuHQjk9liyS)L+ltF)mJO=Of%I_pA}w8<%kEd$-wm^(sY?Y?yUd3e+z4yEw6bW3BD1J%KW%6Q={A zKf_QT>Gb;fW8X#YgNG~?_bp-?qkkBe-cgA{5OIXxl=9qQkxbJY#g#$hyXwhoXb!an zlhiSS+cgE9#1(YVPoVulk2&YiGv;1dKqwN>8RV&pPj7O2eFgr_i!$xT)J*jjC;~58y9_w`@dXKr~^_#%$=ux7aYDPb0P{VFWAQ zdClbBRqw+rj>AS_`O>CT{g4iVA&!g*-J52l>yDVGOL62K?P=D#a3j5MoI8bzY+R}8 z;$4iChW0bz`wup<3g5DQjD2US6{iN6P`L-d=eQC`68lcpn<)PYs|!ceh}`mb#Fz>{(<=W|)`y z5uKeOvv+^Vfr(?=2LpfU5Y;eDMm~Q-92({t?VJQ%9nI>QQz>Eli&gG&p+;@7c?xC7|V*I!9q(GyGBj-A=gmD-GB=Q%~{F^~)f_ zb-UWXgsN$!-OJ#O89DbUHx&ZU$3Bozqok;eH3jqmsfobX9t~_4oow%IPgX1}9;Nr_ z_;&tuE3XO6;M#Xcy((T;Ef;XfeaKrl9CIr_?F^-V{N?)Q_d~5iR{-I2SqUSxUKJV* z5_Ir7EVM@pVj_xYex4fH7f3q1G5dm)Q70ntLaER%C!K!I`$>AbMFoGCzKGqp+OX{y z)RS>4`IZt2GNeLFSA+#_JM!b81$ASHF932n`FN$89S>`wcfoa2CE*#ihHMQgj3Y7= z@5S(cjrNqOBqhB>Z!!=?zZViB*Kp~wdu~cZU{3T7q|KqI4`!}JKt1B^_m@8lo=a3r*dISeJ@V8#e__gZ}`i0e~< zHpX16X_tqsGb~JntyX7O%2MRS^TU_IpdKxM);5GO^zYv?=0)^-3w=koXom_ZUc%&2qgV^@U`f2%(n?!;AQ)4i-(DX5Vk#vPsjJdhr^SQ9$$NZ z#9-&NIR+0@(W^ekd?`soSy?yh#ScnRrQju5Lf$_DAxy=ZAzgQMUfY%t+G;NUEw>4> zXL|SQup$DuGr0c6q`1N1U;&*Gr_>gEww}Xyjk)*qLP11)bh$EE*a0w)f93J|7pt0c zOBk;?rN5s}117sz(H8onHcBe|3y)8K1>{<5A8>g-UVGT_NTD5f&o?829LLOuuk4Kwa>Qz$_+< zIbBLO?E?5mQsHVQf|!ZaZ;(ZgpWW`^)q$XY#7J}viG|%0d=Nt^ z=r?cUeSBoqxs+X2`4E{rCcjcRFNS*EsYjM4J6YJZy>n9M=#at1G^&==kPMfX&jmlk zncWPl()7rutHB>$T(Kr~{7X^20*@29(`iw4_pPT-FMmCT-J$eOW zqQP^FPSpv>E#`aY@e}KYl5Bc0Z1=ADGS+Trbfn@Q7gsP;>>~lEa6$i*JAyeObu|fK1djfRm-jkSz zHzfl))kBjY2K2O-s>rH;9Y}02{t^u+j$2XlKKDbiXJlp{>D9Wv0d-Dz4n_gWhUk)i#|{!@?-3{AdEW@T z$$lFvOc!J5uFi8=DVmu0tY5AbVtK5HjMD%zYo>P0!Q=4Vf8q}kZh=(DB>7^7mCfkH z<7rM!vAVdZH#)V~gHn)I>!75@NVv=W-TwTx+Tiiwt~gTc67fx9z|_HbTG2OLWmfc? z95l?J^~pk%)9fXG50kyF=SJ)x9qV2)AtGZoR)9HxnLeH#A=@B_K*c|kE^hn|Bd67SO0b*(xG)6;ywumyRM zOsxky2m|VaYy1XocJdV_f(Y|$lJc3rj4BA7-x=-a^NX~9m0PLM$&{wzQ`_#$XCu^5X45F2@uare@lQXizmGjVT3_ODoZ&HpCqq4F zTyy3?`AXTp!iYO$Zn+lnF8HY&RPS*Pj>;-ij%|DiTGd2r&}ni&6}G4Nev-47NYsO% zFskUv77d|)244IC<{~?oj8ztAkOboF+P4N6gETdi{Tkmb*a0^6y$nQ2mky&Lz$ft$ z;fv(DgI3)TgudRNv?<0=sbquo=c=U;f)XjGN7U8#(77j9cR+jR0ltZjT`<= z1%A?hLmeqh{?aFFNr^igCmTIBwH@U$pW}8)s@-Fauv~<9zqEQY&G`s_=}!TWWr}6F zyXENk)F-!rJQLZ1A?ORe6BAKB5ifaOSiz4|^R(i<^tmonyiVQyB{+G7Wnd+>>Zz2l<@2TU}p}E{3WgYWs+&56bfyZ=n^8M z?PU^5b5Cp^i)H~L1vU}}m$7VISLz&T(@KLFh=^W&tZ9T&FMrK1#}mEMZ~*fcWu?1n zeDMZlaDJ2#+XD!30_6_=5b^s{CSvbIj{iMj3*fx)K|~#2tbhN`o~nJG=^U!QJ^|=NTm&t z-;kiV>)~~P8d-Gnb|U$rvG{JHQ#G-FY2QZ-g{(42lHcEAz?X~RrfsDunOYN-{9F)z zt2d0wB4UX!D2xoNPATok8fGh973He{zbwT(^f#L#C+S=7)8%)P zmOTSscc?Zf(dL)q1}kn5j#MRWhFUF=B=Q|1xng^0AZNYKDm62d=FfKd`s>AifpnLj z6Y+Y5jf%T*OuKjDZgMy=03q96F}7Ov;BP6~fT~GkKnf%^RFjI5y$NWyCG~`^!0G@i zI-=rVG6%?{G($`J%IxzmPe|Z4RMp*k5<$u^XJUKU`T)utu}$o53=C1^0$GVa{`}Yu zL;C@CbgarCyCUys_|}9lIL*pV9>D}AXVJW zQ#U|>h3SuLqbAJ)v8b!@E9cFL4iKMRH(X(*e6J`GViK(cs-x_8kv`dfIz!3dD+8e} z>C18q0mfFov4?}(3hDsYaHA}PfJ^Jyksslrvsti#4^mLLk3c1 zCn3IBeyLvnq-0L)(aw;4C&0*K>4~{gAN_sZ=9XKjfRS%-jX(6nearcJgmK)&6~Suh z|EBCqwj5=WE+0q>K!5~)i(a@h&-1tg#5_qL#PIZeP()^Z^;h-j?p0<+D!|?B+O=zj z+`up|K@gTA?iqeJrBL~d625hvRX;~<0MFRE#!?`dW5{;4p(E@r z!MnAEy@}ADB+QMfy&|8sz!S!0i;JhTfRRe69ol%P7(F(8({+T^zuVM>(0Tqz=B>v< zwY~clY%%N+BVK(i!}Muy(mmL2G*Md5F!a1}dVkoPi!gOKI~9F*miwe5jgo$ldx`hq zil{epdkFsfqeh5-zkjmCmEYZ@Bq-|P%IzM9#s#i(n<0^_ro#;paYfdiohAuAj^=U9 zENpLfx3pr{{o>}f0WLwf+7xbMe%jKr<_>RxOEKZ%Zc~oD9z-fQGyNIj3fp)QEiHWZ z?rX$gdgbPgV(qG<6+{2zhH-BZeK^>xH9M{>G|W}h9CNXM3z5@B{=82)Txf7NigCX@ zr3<-#62Zx*WQf=~5||PXb-THc>QlDksMs#c5ezx!y`u%Sm5HAk$DL7HeMnj^gT zA^iw|P@$54*}JmQ(t7^-d6@EAqD!iLhRV9Wbi=JvvYZ<^Ia-q1&6Oo``^clj(EX322dg6Zb%N39{tS`_w&k@( zY}V$V$5Gw4$;Q3bbg!3KJ&oLZTcdBIf^~Td+uk96TF)_}Cd4A22=Z(;NGfU~6MVX7 zb6+!aqEU)-r|ScWKNw1MQIy|~0-w^gz3wI?{ZA%T!?A4`zCTubeae^j4WR`(X9jT; z?oJ#iddF}J7F_Klw3XpNjPckr6C152sr>2s?y&t(@RYByt1lt8jgzZex*lTX?{|E6 zE+5%{%8gju^BZw(Mq=FvF-YPK=SvT%tU|o8td7%f(%F|6x|3;)jO>G_p62$bF)Cb{ zPF{p|#mrS+3C742IQz|9XoD9hs5jA678}ae>dW=Ud<2v8c{Z8F~XCoc8W6Uhu*VY}DUQeO@F z>S#~pck20kXd&ufI3GcV9A^gKpCRxmrXb7la!aci^ydx3=>wU9f9%G%BY&Gd7pQrML;4mhkUHs($EWBd^QTc8K~)Q0+9( z6!=)q{oXQmX4qm2lBe&<_HFriLUHBL(0kN*ih0H;$oY=MrGxZvv2R&Yb6J?>!JxWN z9m{b;$Etqk)`w(_uVwP52byYI@>ZaKyGG^F-9OKjmtXtaybuCK$8r-RQ}H>_Eb$AP z8%&<9WJW%!3m`5)tU+mOtGddSEggl0NvgAeIYa!s5mg&8*r%=Cd(L$4GVAo8kDK+| zE4^d4foD@d^>A^gr0kFZ4Iq8@uSNByU~_$$bnZ_R`LSIJ;^Fl&H=5D5<0rF!Bnla4 zk=yER5WHiO1N34&A`TN)Q+4@h>!v8cr7)FEQ7g|%)n{sDKbCr=$nbuo z8GF}slt6-hFvZc18dR7M0u*aK-K9sIqmMfui~uG~#z}9BSM8EkJ2AybyG`m#Mnm$5B<>F$8H(j7wQ~*`+>fm4J;c~CDm?wCUH4)`Q*B>U*b(DtcS@; zxHNmY+2=P;EJ|F)_sx7sAJJ<)#M?ASKE!)A&zjl^XreopqhKAx>>MNijKQ64l$u3W3cDJ1h&9L>n)Z2y5$vNE%(w0<-g znNGh(t`laby*Wd%OoMOSHF9j`Pm^cJWzaAq9n6o}N}+vgoawE`2zH<@KVA%ER(tAv z1IYs+m_P@nHAJEWOiC@&X=ooW(O|IM_lgDaMxIlBqr6>6l?!}-z3ZM_AiB!f@Yw;D z?e<`4 z_8Br2Od-ZU>;=WMr#ovPNR;+5!pocs^r878mBj~0;L37E;k~%$*o3!74dlDG-lOKe zq^8)__nYe6w6}eKTPp;)RQ7fY?o=0&dezK#LnKVs<2g?TMd(nENX;ISe!K2l^W^Eo zpo$qN+#?-uM|x89#667?Ep>)P{Pa1zO|DlucrDSsU`*j9aM|bH6(QS!&yjojEu}>C zX~eRz%twllrBK|DkTh?R))^8T-4xPLxP)Kk=lfnhKikWHVdI0oZ=WrawvV5>H}Nu2 z@&g)E*F^>T`DFWeFDE%IB5jpORJkdsVk>4&pl|=Z=NT?V3Fc5dW%HCmD@A(zIGoLZOF2?b^ zRPwW8xg{-es)qy@zAG-^nf&0G&40=Glh3XjEKFg4v)=j$n3Rb%*^Vf<+iqCbDN%u~ zOL=$qR;Npw8+V2NcDDy*Ie1bK@@w2?Ocqe%;=U*)H(vKiY4>MQr`zo%+SzSii_${k z!4wG@FKL>cq*BbRCXkg3nsu$s7fu-BT$#frE_Yn}6 z%l&?TM{TKF_?{p%<2^hb{cf+vHa^}85eh{03fz*rm(^#T6LraCC4bKHZ8V44*HDMV zOg~M#48En_BdvGMuSZSbhNtAOUTAD6pr`<_x)tpnUF2>dE)xBARB0GIclE@NjH#`< z-qeSNQ!1JoDZ|TPd*#~NX9LUKpHaWQX$-f25gpwIi@lgl;THbz=nyT-u|2nc z%|`njVN$j7kc`0V(cQ~tQy4qGY@dj@Z~xGy`#XP~t(Th2=fhEh^>d)m%%RAj?slxx+)S=S6Pq5n}q)G;!Nh zT@tSkL(-D$l_RpU299;*S&56q0p!ttV&sRIOFWUjNoF}r4zD!w=o1_fO#jB`HHzag z${k9~gDf527DzIqUp~u)pSnOwSM9EI)qxU!G)5P3frvsqf#zIl+wf)46c2uYR>%1m&#j3`={;_t$X4@Ss;HoCeE zsLtJsTp;gFuWofW9DWcfGwt=wqXJyGu%i1Z5d27D-`~XV<}#zT8#XLO+M_%zeqB z=}8-#du^Ov%_iZTCTcxJc0xP@0PJ zIjhy9GF08ZR3Q=~HO|7}ZYtTD1*)9)gf!X4Wr`{6Rva3QTt3DoyV)TstClmPzdhL} zQmceO>}A6pjN#Y8(?p7w+M+3~26tGZ6)`z0t6idlM5E<;mX@q0JJ3g2!$ zLj+~Lh;*g$m)49Mj0l;h+4&nAkUcEbfaJ)1^`9C$2^^L=U{4sQR&7Khr)?!(sj-`7 z{(+MN`M@-gkCH5xEyw78;q{2Tupm`aZs`N22YM97rE}LjUx-3w>86x4w_d2~NH&=; zx-v~JnK+RW6K+82OJYmL8@7r3max>5-b%N$g+ek9JJnZK&Ri**0>9QCt4AHq!*v2H zygjv@5kx0xG0m>3rPAPZ;{qAX4bSy&=B9_`eRA7oY08eonMYiIKG$YajJm#PRXdm} z+d4yA9HM$&B#*A_1%hz7l;zGGHq2Y3Vg_gUZ48msU(>Kr9iPkkWt&zfJG9%d#E+VpD>IG| z)nf%xxbyBuUb^`zVH_~GB_7yobYHOEZiA%8+d!VNU3;$sMH*x@p)Zx_jYb@bctt5Q$6CS zEV;1vft%iV&)%8*@W!%t*M2giO82uy&H|}~%-zmCr1o3_XYSJ-|!#oplm)=2EY<@74*mk^svE>-k zy&DLBiF360AWNXh&75|s$6cr;r!LeGr#3cz;jv1n241DAcrBkDURQ%~IggUptGMCP zn@Y6wbHB&LHO*10_CwMVSPEqQcoUBfxuCo%&uV6{Xu`a!`l7>+ozvAl`nWbtS5)O0 zENi_uz=eZs-cgOJxz+(VB2Z3G-+07(&qu_6+^UZ#ea?w73TQ@7q$^p^$++DNjhPAZ zLj+`MXhe9t2Uxm@O59~4GeW1j1jUt-hvyCNZwb!#iW;c%a6{=U0>qEamue~l`Ln2t za-Z_-(=<_z(3esn*%Z6BH`6erD`UxJ61X)nB0E=u)z7z}<8va=swC$ZDm1E`Mp2Wgur$e{4yT?+Ut9MiIyHg7_JmWTfhXD|NgE=D6~h*2je z`gu9RS-C2bpd#ZZMJ@fBA5npTd>RRqq9$o2Upab5-I21SYYqvqTekXrnXdNq@H67k zG1zNc=wr2pn8^EDzE3*7yU8_w4K+&-U5HihD_U=Za!W_# zMNhJag1@UP_8tsPnJD)Cf%;0#G~&5Ji_D{#{-*bRk$$g?Nh|gAu92sz29x@p2np=lPlth63F#u6A5-1iPP23@jkVSb7Y_%Bv_{$_zzJ0u z;pM!3#8xHNyj_y$45HB#;=%t^pVel0T6SzN65!}w3?XUm#zWUfDDkk300$}SvO?lo z9a3opjAa1<&&StIj{PG(ToB z-ZGrUU(1X-zW%w`*CHhRhu%Kp;`15W!O!zcfK1@{#Eil`1Hgeu*0SWp=W7cvd`ewF zJYG{5cDB=(@*v%gK&;>9gQ0R%f8*GNs^3p09Ewl&Iw(jBKKpzi?~x%HCrA3!cNtcj z0lDgs87-FoEEEra1o~3z;#1W6Y2K=+KD6_YcNd#}7j6M6HWYmJc-{|uax;rZwC zkgV!ql(&SbIZ82D!|~pU+ebdlUIvBss$|(FhSPAz4$**gj_XIerI=Z2hQl|{S9BF8 zZ_mS>=C%al`Rq5Tkf6!Ex|Yw-y2pvG>0?*#2c;QJz?wFHIxE_d2D*14DI>u;ROILv zK_AP`U{qEWEsD)Xutt3hj5#BMv4U2cJbJd~E@p>zwl=_%Jq zITY4o(V$TE@186){1UcrRU@NVw=4_Y$@bmVgWV+bti3gQlndBJAZL@QrLHaBJiA;j z2TFoz_~6w3eVEs4r>p}1V9fLU2sfghJPU}hS{xsL8tAw>F9}?Ln=z<%Wx~u+=7%p| zp_g#^bW%%YblX>=%4R9V_$2~>jBX)Ybc$%M-WO@@lw`;CM!Q!JFkqI$ zJ-GqX(eA!NPBX0!KI6VXg7IX-3pBnQj0CC&E_3{ z;hluJ#2tu9w4?ZiV_(Y*i9mZYHlkL9W5zMQ&hc6o%z@Zu6}gpm=?1DVm~CJnqE@5Q zJp@oF-t(@l?pbIpir3AQb{NXAwZv}R8iMjIs4cf7(a73Dt_p%`aG+c4rna=;Y?~oA z*W4L95kvI0w2KxUFArt7X3Fr&eG|)n>)XY697Hd;sh>?S$5cW^^e$~fQ`KIq8t{}Q z>dfUS5`l4!sC?NY-c@_HP=wDu(3-}(SDQ5cXz@BP%b)!=6Y2cw8x0^5d$(GF9FB&0 z@qtWOJ|E|jnK6TZdtUGIEW*vv7cuiRM%8?jT;!*Ity=QR%S(fOR+%Z3;FBVMOUCe; zNQH9w!b>vqSIJ1msuOAsxAx;X(-{U8X}H{I4;A<{>Uc4ArSKuRj=GJLvyEYTm44mxNM$4FOJb77&ixbfYw1|aHOMq6%uyltc zC-QR%vieXu+B+gr2_M@tc&KZCU~41Y9Ks#6YOJBA<2~}XrKr09q%TvOlKEWQ4?Vv> z(zbnEXwT}FcfPbDQVvp>C5Vu;Z&>n8u~nZ0KV%$Kx1emZHQ^s0Nc(0WhUV1i0j-(I zi&3z3O^Fu~rHt(^{dH&rMt8dD?OJdqol46s6zQ}5S(_PO--OW0JOV!><=^e!>yxj$R_O;hrPVE30Wa-1?mU0Fqp^>P5yFeTsU z{8h>9akVV1PcrEFV6EOt-}`~A*j~6LuJ+P0++pHZqfxlq+v(^^%RHmT_JquOuoJ9X z6)5c$k498@3Ii}oJc`6?6{f+Hduze%(BCi0L9P8UcGL~EuSN2IJhaD+6&4r0mt%zO zM>aMw@!5MxUq&o?44zHa@!P#=*I2hDoIIX~byqzGjV|_@-bA^usQvwLt1YuudKp2E zUz@1k@9%~*Tbedkn160#Mx~HL+)@aqb?h;Bp0?0_-ViH8iZRa$qtCK}lg!Txh&U!x z=tR!h?)z0pBgl|{jxMQPU}wDbwBH{F`JqF$r?39wvg%yB@*EJGM9{VNL=j;l|e7rbI_pUERdzd>Ixe;+rfo7v}>vpaBZUZF9FY??-s#)g^5a9}HY?Q(|O)aa%06VA4- z&@WN!QIv_~2(ntKg{E7*tnsG3W`Nk^Cb_x!R_eT<4{s;#SdOJvm_JAoC%cG`-|C!a zbcUOM4i^%~tP56@jk?lZ-feg>67rzvCz2JT<^z>~H@bJPT3veGo^}vD(4e(@OVoTV zqF0)yiHghD@LGLLngKqqP|Kg zCkow-#IOb8V#6CPJ|!F5iVuayR|3A?{nb5x>y!R~9x8oDVXS;lrTY8YU-5M7t~ZZH zm!ioUdMPG1kC2IIIPTS!A;B|t^zs!x=tId@{ABW9d@zKgF1YavOLU`h?pgjRCR8}! zxU0Y(sYKh+J<^3RY549SYb<)7pKgNm-5=>3e9TX2h)uQLvwYGPYYC@rKd)kZtlS)b zUTU-u)L1{H(Ple4eOh4=UEDTWXq`daRby?yId`G@fP~Y=)N;Be5l|+!uGK^CJ$=m1 zIVJKT{nY;jf&f7MuYaF=@*iVQ-tWItPky@w#<%!iVd-MpM{C2}m0JMV^gA#l63!_m z%y()D006hYqe8akF<2#=MDUDJd-x82B%$H~4rw4Q7kmbQExy-gh2NU2gBW{EL!0=5jsO7p@6;0|JG)Wm)EBS=$p=6Ka0NiyxPx=DgLAaK>#bcY zB_GM>9i$upB}@>+0|!uc09wG8KziFifvd!5^?cRY=+LUjgSA%ctJdqThw6>C^R^`d zY^(*CRg#@W44hfPZae5$p;}9SwOXqz2&omUStl~YpqV%Q?B+0*uma6u-YZZ5i~ys0 zCbL=qG}6TFS+Bd+c|V`GvP0e@OlP&7uRIT~S-B zkk!#70gSQ)kaGwP3s@1MEj0(z2n+xr;84I!57BCduO5Vysxj_fpMR^A$SN$OQzMo&He)NDgQLsMWLk8!bpC}@>#l6Au?S2aLIzr$|6STd5Uar4(#q}ohVa0i3 zsX^6yxEz!`CA>&r1F4>95ghF>CKe{vh(OrYOX|{D(do?h`h1F5E9Zd)0N(o7^|`|f zINLHH$S}P*edVF@PB8vNaLhK2R{b`FT5_36L&5B&GD3zrFMoWF`oOC6+&(p>v8y$g zmcRFbD_jBl0h0xP#P5A5@V(EjUG+5*Y%YL*+vg;1S{c`o)PT*I6()r34(J z;GJlbcOsQ5LQ+7+$rTZ+%!*6)m?}VmD+>^ru^MidG6AkFe^8Zlthod!U))Pew28u8 z0R+q)d6y~=+7U48xt^In*8?0*lx$Aq91fRk4hPW+4J*8V987cB%M7Y}33;ZupjHO; zx`Y~t0I(p7>;%9{MmhVlaMBb2xLw(@L2pjG{Tu#;^FTQ3MWVb}jqb7z+hwazL6@;P zR}86M{!WuYEZjp)A$~Vs`SBFs~7F z?niH|>ksFD$hv%#gM5^c!DNHMWG7n>2nyh_o(vnE>lQ%C?aIo9z{v*SW^gaVOTGgZ zM3M^TSw$ouv87IYJM|@aB8UceAie>ZPyC#Gr$GyxdzrOeZb|#!Q z%)PU`a4&X_@-)c<&DzqXII_nFr8q9#`GyJYG ztY&6F`@l7O11sW=YxLkPiU;=m^^0-hC->mB1lj2qwv@hp;n%E^kL(X$otXaNUyuo~ z{I2tV&`U98s^OAZa@#drEq0O;qw~v5QJw)B$C;Q6TgeJX%$8hk<+_$u{1Cu0;~?N9 z<|oVc2s*;46cfOdr{5ZQ`t8U_ZmN@)0;!1}OG_+$7wSlWPk~rDO^_RVZ%OF(^T5fp+3T zIdMglCm1ClfD;#y%Yt8AtdY;d?5eL>!^7-c3nsZ91c!#oyC#5e&(x<`eIdpC&&H8$>}$OIk`eV ze^zqmx-gyj`VStU6-AqNT_0s2)|FO&=h#6b>&oVH))3!COF-_Uy{-(jd86 z0GvTaJ~rl9I&uRXk4#qSm!iU%H;!7b8wdgKmnS7@KrxXk$WUf2fpP9%13-Cy@8}?H z9uI)FE1M?#gA*3WW&ywblQRfL|H+m3R~P(h0r|v_zAhSy;yIv9rQ#^3mm--UTQ`UR zKS03061@T4gMX^;xGW^pn;pPM0zachueiKZ#?p%RdK<;c@s1zeQ8a48Fz1daYqVPv!o$>bN zpg(Iq+oLwEPYFkQdX-9Qv!DPuEa;1=UFw^!3hvm2)ug4L? zO*UmI*Zf3aD!o$gv-d~WI{(zQ-2SWU_8)uv zjk5;)y$4*W59>%Wvl5QLP_tllDZ7WRbzc&1+^_Xz2Hc1ECiRVMKZNn|Mlf;sub*D`E z{l~s|4HDJPN`V8DU1v{cFH0G=qu{JJx6Meo(GruHQQiUGkA6xK(o%Uaf3?6p^WQcq z-~4A~JHMT{<52m?!k;|jdHjjPA3c8Rr=PsIXSi8LKY8)|cjb9vDV_25~Q`ELE+a?;Z z^;V7pD&tTgw^@!5YM2vbf6%;HlLmMXcwQa_Ko!e1==sf#VI~K5Yh|omcYQfBn8!OXe*(YQZ##u=w)og)9?o6{yg936grzY&aff9g|4RU?rCHtv$4(Z==3oNC?$fVZt2(6H2>3qAI>e_L>Qjx-plZM5W;&VaVTN7{1 zhG4Hgzm^@ht!m)&1>Zl4d|eT4rxFEv8B=DR#Agjt`e{=IMj{s{f?P$y(ur$WbQpZ_Sf8DksXrZ3owjY*hD-IH@ z=5dk7C#qUK15E&}`m_^we}|xJEzPxU#>zy-&Kg)jAJ)KUY^t*$y4yPtIWSfn>$y2x zO?Kru-(`G{&vbROMzgG9cY=lAYG=Mok&DONN@WM^dgjZg^ph_`_~nVO<~wuQZk8IZ zSKz14{nk0we{XDM!Jj-K)!83ShHK3?4!@WKA+UtwnRKjgKkBI9>{;ZPCm+0$%sB|2 z_r108s+Z&2i%wDqkM+9gmIuegywOJ`%Z!Y=&W^dYZfp+MzVZaPM{wD!2V*~a$=O(AEkzzmL2rXv zt+*1hJR|ThtPiZxyK$09-}9Mg>eNuEt5slDDiXMP7E1YmdlU#Z#@j5hh*sMxrDw)4 zvtUKJe|ilsV#S%jvwcnWE&o_Ol^rFYH`cIw@Dc=@&T~3bmVvZJ=FhipK1giI@6!(c zX$Mr(zjko5fM0+1{_h&*^e4ec`YQIby8h&tZ!CN{{p_WD zu?6@wuUlQ&+Kg}bkWU_sakL^VR|&3CB3Ztre{F5TR%j(2m?UlE18kPWS*-?3KQ_fD z3~FF8fgLHrqmtlwV?3$kZLU-tf&`9_DRp#L&Z8~O^HK8|#c`4&KuQCf#MV-JAU!9a)aLdR}T5hxo7$ zpI6=L(Kn#6LTd1tEp&FcGL4sL2Q|y?D%g;_id7aMEoI4vmxBXB;XL2rM^`)8fB?r% zwdA}nwN7n)G`N#$^s~K zRdRC!XCZ!vEgh23_sVpT0oCe^eL-|{ed*52_t3_PvU%91>#DBxf5#hgf zIOFUB*)06V*`Iprkp5~@spMA;N+Z8ZHe)G@({UM7bs_7PmFReyz)264L49xye|eL$ z1g0!aW*ZeYpbD*zHl6F{w9Qflr+Vf32C}1lT-O_T-LGud>sd#BGKV#?^^EA%I4S4b zxyO+LtEMuxC9&5P)m&msz4f98&>F4~`k}Z~M>K_hFg!e_tP$+VDqr z{?QY^y}y6toS$5B_UWWu=GkqFb(<7yd+jVSdy2u>C~h@FKicK3Ds?u)K~p&#qt(h- zId(-Qu(Tw;(ppde#lE7L?f~B`QNd%UiwH1L-S^CAOA>1|m1{|cskOW6AdSEWsszWo z?0DJlOmKv>lpVa<^Q)Wwf9XfhYZun~`cdRvDHKcXhu^-rfZ+bb>e>I*5AA>Y@`pQr z`qzK;>u>EG{Kh>0%{l+jU|@KSqRpw?idKR%mH^8((;`>~5WZwble+p=#73QAjSmGz z8EaX0Sm$jFX{cA6m2j09RrbZJqM9vNSGHyad0~w1wi91*qP*Fne`U=u$l%@`dJ!&IpD&u+3^o9y#790`1YN6E31EMmw#!0 zev5u;vtPahgJ*Fre@AG6K+14!G;4({@N@>(`bez&I`X;AqOy7+#*td zBL_ig@DAr2e~hO%%6Le%S2726l{^p&TkCLs$E$~xN!!L6fJ)>NJ$u-uwsEt!XREh) zENI$M!?d?ux4UAsrsPma)z2l*L_T-LE)(mknlnPNjz-a!CHZ*`yLz^`l^o@&>V3}& zM4KWdXO1|p9F-X(I@fz~!g%B17`rH)D1I&~a7?wGf17w`Mno_hsvRkuHNFHUa@9k# zrCK+&_2+4(SRf5Jh!fYviEE+!;u?Nq_{(+w(1h!c&A*&@)@y8b=C@z2`{T1;U0G+i z_KSfZy#M(Ax32pSpMCFD3ca{_f$xl{vaSlZX)({VAKCH@(HUPY2Oumpg$HX2Hpzgt zRSTFbf3o{*fYJI03}qT%m1Q_SUH6-J){5uR*G}3UN4mQE=t(ZEfzD$2 ze=T!Fhf~;c&6tvdz={^9voN&1suGYaxMmo~5;Mz24J-+T1DOM%1=)Rc__amDT~f6j zEeA)+jNOYFkmVepcK@o~-0Mp4>$mSEAiKxPKtTa)L2^FdaJ`H#v#(y|TrG#}uZI8a zoBUu+I&JdBuz2#pf9B|chuIGfos&-pfAQpRaGYH9!_WU0tR}$G6>-;mmXSVD4RWFJ zSWvwMtF{_NdNA_&E-ic11>h}S`z9PPdyjIK{mn}aSjwEu=^PkMxN!_qfkbh&g4?jb z5#szmF{?+Qz%PY0CrmlAilbGHBl#Mf1C}G6ay`GHyJgz39t;GLy{(T&{ji%ge|p8f zCz|Vf7e`P#mXktis<)D?UkScB0N0HtXEC1HGwTh3vLp} z1I|<`DP&x`kjnW`9p)jm51p~17k7+@HN#l!hh9*F1&fClj^?dxhk(XZ+v1R3FNg` z@RKc9%WztlR@{Ed_!8~4=8THQ(P-l#mNafypxIhKQY>GjDGUNGhccA#?$n_V%uN;mRTn}p1GFUTpgL) zUVj|h8YZbxj>o9j)=_OP(=yJf>T)>ieCmF|Usl-GcF&3|Wb-0G4~r(c z-+|%-(Md#iM#k9YV(#5@l{@SSw%S~iVuh?IOVa{;t)dnA`WxEe;_3Q$Y4l8fhpL%sX*=X=3=x0-4C~xIXpLCsc1Gc9Vx4PI5r6;u zS+5J5ccxRXH@`R^p6jUlt#f|G4FC8WT&(4X_~9=e>Af%$&;GKpzhd`SN5o8i@=&n3 z9rTYm*+gb;BkBTcov>KlO)3|dHFq6Yk*o3DD|xLl{`66EHa2l!uxyp;?K_AvXD#a? z)QPdxx_{Ck-gwVKYHrPQlNuk|7=Oi03b2<1fob88O)pDV+2NR>0#0eC1{(d zQ34bp!#>qnLGof3{8#!y8S<*yNvaT9igDclZsEwle#-LO>{=hV$ZL?P`F9RN_-8_A z|BAH(`Mu`*$xRwRcYRBAYc!EmN1{ zyOFr(5?iYDj&C*zbU)YpGk>Z+1|@e?bP?lZrA9Tfw6=Ee3}X~m;YX>fK5mV2HC~tI z5yHIoJA^zH#hd7m6>)s@Kl$#z@NdFF4+{UhjfMUn*7p`~2AtS8mz)bbDRjO1V`s%$ zR_Heo+rAEKqVMlwJ}#D!9sU>oLtradK6O9zCnj?6Yr7&yR`7^lkD9 z#>-4AAo3ZXYGr6exY40}rkkjFXm#W40$b)j<&C*6S?iaYYIm({F;@D>B#2ltxim++Z!8e87^p%u z03Y>%vBvBbLwim>l$KQY(yTU(Npt)!cxCTj+Pke<^f4+{@qe{Cn{s~fI?m`}7Zw_s z6L3QZu}I%()dphOMSyjdZ*kz7nA4kiZq`=Jamc@UjDv`?J}aOCSJ-FX7x(uwPCveb z7yYaai@*Jl{3j3kC$9QWxT3*t8L%wF@6WJZWxaCHr=}UiF~w+^>u~eQ*bH4{g*{54tzMo7A_XHiP>&Sqr1=D}@Uv?pBNnDCDYn3|1?@s65$5M}ez z^{~f%b~A%!O_BVu0{Ih<*()JD`QjyxI`UET~0QK>EYqw zg90t{q_rv5QEJ9SD&CMJX{3Us;28z$i|eFcT<0h5y?>5e%uhT$Tn~!%2_6S-FVui( zlBH-m$UnQd4Fbm{o^lc5-hBjQP&}{k{EGV1Vueps335D@s{QyWyF)yc7}6As4i34% z6`w^s)eIUFL_D>Rjo(<%F5CS02%AA3cz*T5ycCBu{>nK)e0kfYj5WJ@xRq zjAJh&FMp{b!d=y4a-2?=Pai3(L;buPv@Ovkxum`F6!E})Ra>i~jUtt{R$pUb?~}Jw zz@7`cbP#q4JMrB_F&g69pL}DouV1DW@y0v6x_^2a+D=%{$lQm-04MV`mIDhx_(z*mp{-|I%CXO>#MYGMAwqf_Q zn$^TbWI7rhw7`lK#i-bgAccr|3D?pY(j3XcuEOqv;b)!M|H?xRIljR+{fRS*exU7~ z_iKpVTew_#Q0-{fCaCj}C0qG1eCpxnx%Ks&*kq|{-SV7A)!jI^t~JQvRGKpV1s4i7 z2!A;=2mGxi^!%Uv=uiABPyfno{p6Gfef0}Bg{u9N+050Ga2Yv;>mj~8t^?OXXVX3F zaRR;Kxph%LxNGmT6>>|t@upT_aN0XPS(sxQX{BsOiiYXUMWle-rt`H|)#8(>7|32y zsMf;46?}LUd}yFN82GSrh5i-3+eRz0jejd}B;9v@#rSDojIV6=R@{^`^Bgk`s`vnb ztA;Gu#u}6;o`v37enZiq{Ux5eNmSovs?sUgF3A(8xDLzXEVcK#uZBJ7%Z=h2`qa){$M>5CIaW-a_QvR=L+&(T=m zL~;Cse`_x$k~2&N{!Qd6eq*@P-#N~2n||4G!=jn-J6HV4W$zb0zRRdP_DlpTb<;nM z5V8%>_^=^)rfSct$;l$qJsM!MIe%}!I-Hvj1)*xn#WuiRo%M=h1kj*zZfaVbl5RW7 z<=9Nnx})QSdSSfx$2Q3j?46{h17;WqMC;`o_AYKITnB%p%Y~e5yq`Gwj~z7z-SZ>Y z-q{^uVCWxg^r@nVpAc<2t0yZIZ9|W@x{8W1LPRkcx$eW8jkg%B` zU%XV2e`6ZI@M}Dqe)W=|X+8IxpLNQ7tuxQ)b08jbi56Veeoz&t3$BHJ>TXI%wOt;-MWV@&2-n51r!b z)Q!|+8CYw3=Yq9b1MmEB!0l$zC__7LzX0cgU(IJIH-r!4^<9^XHGf1n$kEm76s$!< zj`fSz|K?3TsCN^s$VX7-U!E1%N2(pw(AIZ+tK\&EQEnue%(g=jKgqeHsk^OG47 z(?Rpfmf73x>X+Kh_M1*VJj3-O4dl8f#y*(42qW-xXn%JQ#yKfbs)m)1g4=pc zc^!X{pX>e@!WtJ4?Dz_zth(tuvoXy~;Nlx7jcvzAh zE8@A(b%LLNpOgxA$L%wu@uoOe9|%`3on?%o z{_enWL4Uj4_vvGb5w%sw4FqNtLzQaSec^(YA*Oj4z3_U{zVOCH_Y;}88M(y;NiYWt z>RU2FFNKEFhVz}uvEIdW@E3SI=o1kG{*Bit@^7E8(0^3;3twLr^fczJ8KiX#-J$bT zdJ7nfoyitz~B#ldA0kAn+T>9M~lEU#Cq3 z3p%#^mAtE`)m#)rI= zyC|O`vuoQ>BRB6X?=c_ztyUUQ{1kBoHHJ|553xD1x%|y5{j^bKUmf+=-XyVR807rt z_um-U-)q@u&U0YjA>y(Ej^?J!he-xqSgjZ zniv%7)aK2T>$TW3sTMH?=Hj@NN2&@a*YnCK52e;uJDavW(&^pQW?^t9AGHwaWV!Ox zo8uKVq`k5-xcnfYM-i$L?L(O#va=bux=Jewal@>>slx-mIK|*6*pf_9&WtUngFn(4 z?6;pF)|J=4@cSR{Yb$@r?|(+b{(kcHlRtj8*xM$?<^=nIcC;fm2nj_ro@xQbYwjne zN>^c|K_0p%19H{i6*0N#S(RlVswrs*XP>h>1azZga-^{`M?gw)sVeJSy==@h9J2ko zW0KV^yF}fZW=}#K=Cd^)@X$}GkI|EwF5so;*G7DK)cy=>*niEZJb(Mk$NloB$?_Qe z+GkPbla!hW!h)&=yv|i~!<{15Fqd@+g~?*aDb18}6FG6qbpDn?mV0*LmcMycZ(4Oc z%d1Ppj$s=Z-I5*an~P(sHfZ+I&TQo3^JzjAjX-d>i`w#4vs?K%Ocs!F%WBTAoEm}cuZ zZZ2wFFd9WH+)`Cb3>rJMhs?n8SS8iRQ%RzZ^PToNtV6TOt*D*of%4hM zWHrRMu#9Skuh7OohWM5|>}w_6(5@jy{KG@TTJz8YwScD{j(>B5DIFBF(^=&isab1s zR$FWqo5gqC=YZ~j?TY=(1CB)fiCg`Q$$#GG>O<8~VD1f=d{#^9YN{J+Awwl^RTT5H z+Oe6&8^^eqg?1*)6MBzazwwYe2Yv7(0sW=deyx?i`&7QV;Rr0h{`Tkp#7EGbyEa*T z;~`4ve8~0Xcgg_t*Sww6i&d#50VL>@9-E zYMO^xXLcJ{tubkzMwk-}F1`dPGP~u@ORKYVmCF}DrX?dYvu$ zGbBiMul^?^2jFNUE$k3yx44%H6oKb6q8Oh?V9$zu)aAOPrjAw}P7uM=iAa6OG^S#uCPv@}Avvi%)969I6onhaa#t~^ z4sN|%t1I{bIM`@2dW>gxlwd z2Ec)$1(-2Z3?HdIhF7d=+}pr3S2C+V@`sIOaAej6s-Jn3Pb&44`$t$_RHdb)KyP9)+8`{GH3|GctH~}8rk%MV7Z%=; zQ-9PYm9LwBdDQJ%s-*Xl7~H9cqhE%9zsef1jvnfa z==UCgGy;z>7W-viCb(1;#u612b&$o7Fp>h|cm?o+AOp%B8L)yhhCDcBV1IFA0=Pme z1}|~%5BHh4x5d3P+{@y=7WXz76F{xl%+Q0!zy(JJHA6cFPiPg;m?wiu@Rq_ttoU?p zZ$Qg%Ce97%2vFuwNn+R*apdwWEts7POMhX&5Nbf@9srV6U(kOzAm%OrT6GPuhmQKnfa(8DbSgKsEph(ofP5Dzp@Nuzy$vgK3s%&x*rm zsE=wsAQ`Qw5HgW(b86|mgp|>pG>W7-qo<#9k^a*kVgM$(FrC7%?bf#8F@$SfXd1DP ze~8q&l;HpTq+qc{2e9zvKR;^d5~NhHF;jT1n(L(kT7zelAR+zdu_6w*Ml4L6 zUD;Pp#PGGB0B8Wi4}bixG4YElT3d?Ku-6nM&|26qGzS})esm2k|afvn6#3l>03HTtB*8!kH(6s9MWIy_^*{xYqf!O4wa|2Oi3U%bJGD#dk?pTi-;`R;B(MWwoNX*@cYGc&K`*4fNC!wp<{M7 zL$_$rR;B$f+J9XfxEqAk!%`Y;=yV>L)Ut%a?Yig)(OHm;wUcve?#ndu-JYpiGq z+Uh;#zJ@Y$4@#O;tfj=Ga&it*trEMzQml8s>N6eS!*lBQIWC?PWkpbcFyq zi{G;a(?GYjX&Myi9*{k*x8?#M){0uRd+R_d#xe~j9EF?(A?64t#TH|ZWRIo}7^ZA< zj6JOPWT+#<6OKy$9aC^0rXF@^Vqwz(Q_!L+$@B$5YV2v*@W0O?MOwB1%`ORRp-@e; zVdVpmy?-Dag(*jP8>Bj-`a}!Ugq_n++nNr7cwBM_+r7C*kcPkK?%6`#wot3~La-=O zGh#jkps0oFOiP!w0B3s9Ib5tu3);B_jkS>9ojq6%hM}Ko<(*$ZAuI7vZ%YtM(;bJ@ zV}bMSCocOQQw6A;V7t_Np@V+-%W<54mIW1aW`B~HwLl(Xoaq9Hm4es@ea=EunGUe; zoWZ_Tkr-d5FA$_jCFYf;da`Z7t<8x>ezidj|9zb;4n#dL)bo3cE*$#!oW%hVeRuZY z5Au&a+;V|W@HEj5y?2W=Yzk)&YsVoNs+4bajW~Ot&hUO=@l)~)C^-g6SSd+&3?j3**S?;c1RCM8irg= zOg49CJXl93z<^GL58rYm9vGye@w#8XcM%BDdZOO*6cAS`j1?qEepp%X7=!ELF)z`6Hd4j z^0Vmx%`Re0eQsTllW6R0;yH(nxqml9ZR_lTIe@v^zd8U*OSF=c(!(Z6xmt?xkO#S^ zC1PT1o&n2%LIA|Vi_#^sjOpqJGQc zwxx6^x5~al;ZS~c9HM{A#rWn!xgGW$#zXntaftOTS6Wl;Ii&Dx@v=2_%zt$q*kGGd zq<(ELpv-}nU6N<3Lfhiyqs2?1zq&zx_3-cc410x--NBZdBxj}rKAreNCQH*1vRrbi zqwO$P6F6rP+Yj@jnt$;i@lloF7D(EvpRdTD3Q8LF=~Qz}jB?Kc%Owsixn4 zs{^n9&}-U%)eoWb{#`%6<$sVqf83lvRnERcWC-HB8BVxYa<<0E`1k$C9EcdN3HN&b z(mwpzs;o6t&Vv5m+ldyCD&~q#T*cYLY|x=7t4*x1U;!Xx7s|vco10$7mWk9(A_3Rx+x!rwq+4}wyM$! zUTsl+rVmi9Jzy*T&Q%L2PO-w)*cD@j|1L8f(5$_%S-)trx_^bZ4)=pynhiic-8-B8 z4{xRawVXSp%S^Smee4c;cHRGDyJ-)vpSfq511!^b|8flf@E`)A_nm4d_*`$%K}|K- z$90{VYPWSBghN0d&K{wErZe}k_D^*9-_?&wucrX=&T4?V~J#t-taj#^%56kKi@u0bJ<#;Aif?@|XS0`=MxopK1BS zCwp*c-=2j|aY|SD^Zu-DX(RlRzv}p(li^_TZ)8QtN006DBK+_`&2)NhAyKn*@F5AE zko|D43IFk5%TG4&=60|zYwRj|Pnb!vbFdI~3q7iB|9|u^Z(9m;yPtP1kXVdYD111z zeVF0EiyR4?{VBiYqV4^hyTh}Uke9?H$FJWi{Oll(Nk8Xa**sg){5R#oj{&41pL@Ny z=WVN2cV~~hL2p*^dC=2MbIzjjcl)BU=UDg;L>BzGb{r9s$iy`ruH)f4a2x@38Lq?O z`ultIrGL-Gb^N6tu9GkQaGieXhwF{wh`0e##k*775BUk&x6f9q+LjBLt1IYsj^iTX zzYcW0pab-!<83Q&sqsK}80bJ7=r(2l0ZM-8mTfD}RTe&+4&oYpuSwf7^7FeXu77@a z2l`OVg-!?h?{z%Tf3LHF{(GH^b^wq;j+>!h6n_!Xj$J6)0T2vyjp!Hpp?iG$uMKqj zZ~xJkPX6{k9_U`*{{PSkzf9$izw`zd{@WVAI2F$tM9lrtULYjs^Ky{U;hC10g8n1? z;Xm|L_P5dyo zK;izqc3lj_wjTtLY=1PCAza=&fC9gmH{?WdFYasnV(8cXY>3&Bu(Lon+pt{>=!d|Tq`8b6ZlqE_l^lWB0Kji)^n?dNABc3sJCz!K5%G4%up@fd!VQVorrQB^nWGn8IlG4 zo}*rrZz(WPP0?(sB-~Zp$?@$zq@-0VWk%Z60{QrM_PRqo2_F%ZMqv-wZH*|m% zF+K~?u7emqJrnd}aX9#%i=;;N?ay~yCt~~@2Fmaphui<;{15XQ0IGl5b^V!-NX*BM zit$y%(ciT=9NYwCeedD*t$!bu;+*XRM^n+By@;zDRJ{K#=Hn>xM*32;>k6s@HvRvB zZ~jHQcEdOlF^nRxKSD0pziFR1Nrb>*n1@P~7oeNs|Dj(8QSZP%U%H`Pgov-3 ze4hiD12kWLF{l8E&VL2Y%HcwPxnF)VhW>s;d%o@D;D~uLOxSXGi+%^T) zPzCUWK36%=AqL6B6i~q$slX5D0NY^-xtKDM=M(uXQ*64cMSot);?M}Wu>}qQbAi`- ziT=1kzo@3r6SBY|dJLvwf4xs(KIF>)Rzfc6@Er>Z0Ttt&i=?|HQo|ufuK2!1#6-$n zxQr+S%@ur|2z{1tlz~=3%FrG(4jYjVbTW8yq5`D|QKAFQk-+}~ZHz-YySd^dA2^Kn zms)c_z|k~$?SJC<-g+% z$N%Mj{lETq@l}}r?X=5}|L4!urL6n^uy@gM^MAOz^7=o9pWA0&AOGVjp0B^Z@MFn} z=YK3@`G2&J{6GHjD>LHLGm~H6nK{TS-TlEifbO(nUVn#gr|csC5Np&@&x~_XxhyjGn;l#`ZM>Kn=3;5@m)qHF z%A0YwI+xThY`3{=gQULnlC+epW0i3}T>+V#=jUbB%V}`FX}r|>Y&plk`$|2ZV_fj; zu)I${D}OMo+PSSSq5es1+nQ{etvk|_CoXs@-;71(_WiNymQABQ)9tC7C!$lO z%r_-%)#uzW)Z^juuETNhK8)X^9=5wu8cUCR>qtww#F^E;S2UcTyUR;T!+52a7jLbR zgT5jASG|qQ)upzd+lgA-jrBdtrg6Slr>#3%zkd|-yqSdlhfB}ZLYhePQW`JA@UttQ zg|SLQI_vH%Uy-1Jc|uyTot`Ld^cuPDXd%;Tv3{w}eeK8-XCdFwbGlr!E25RjsEzgU z<+{2giNA64x7YCbXLB5hbF72Wbp1hd<xA;@tmL#nFaqQ2n(-ZSx zqO7+QegE_>p2SusZ7n(Nv(PQ`lOVS|PCqjPlx-1eYAVC*$irm!OiVjRtMH|{kAD@O zWc}wRRmW|vC`mDur=!()**ovs{`?|Aa}Fh!S;TrOt-wLN#`9#`>a!&&Cnvf&U5{5# zrCmOemy^uygC5((ls?me%vWTU6rJ#rIX zhjZyRx^JU*wX6N>JAE7;$wb>2!GCn|{A{D+DV!Yz++Z#K(@!Cv_$j-JwB|Q)KL#WTJljeqYCeC=DEkD&}F+cm4Y{lHi(SI!02R3dQ zO<#8R(Bd)6qvjlM_D55o6JO);M3Sbmv^sqB^K`TC^M;R0lZJU~ICJlQnp}^`g+h+t>6nk`Z`OxY6coNj;B^gVb58Zd;_5OxBb0 z%o^L)KB~4c(kIIpZ2X>r41fA@7Eu-vosX+SPVLciGhGos4_KOJ%4nsYAMHh5hm!O< zem>KQwF>loyRUEiJ~c-9+7?IIL`mX)l1(1gU1;vjooXK={Q1<&y58Mt|4^U&`ns+! z8n0Hfwb45#d$oLCXQ^a=maSBIs}vwXG_j~=Q0?|cL5OHyBz6%u^nYFz$-Ic`B0jL> zLhsk6x&`-1kvdxJ+DUfv%5q}AvzuBzj`6rlCA(0X9evchCb~Zl$$tJgCZ|cWy0Z;B zD~_bT`E;hnmF=B+e}9F@qO;6Z?AM}H7v)PTW=phJjCGG)X+B;RrrW1vmt>J0e8}=L zQceL^PStD_y%lt0m4EuAD3OZOXaUNkjiaoo%gWIDN7HrjOJPWIPgmWJ-+Ei8cN3i+ z(dT`gNc^=M?N-v^vY#)kOY~UG<=tUw@6Yd}b)8mxw3-k(Y_x1iRFT)}- ztI_KKO2BKKcFHR-97DT$5g%2b)2400daoY#SJI5Yr*ffrFNra*SJ_F*q{C9vgPl{< z*E~HnOK(4(Z$~e2WJ6F$tI5=#uSXVfRcp@@i|v+czMsrC&-?hZRK3+cJ7t|S)7o`l z&3sigsxG!B*MD_RR=h~vLV;S_#ASipeqomEdhWC!Wz+Hc;>@ODT_{1*W%6`#yLrih zNKS+Af!;;YwCmCvoayfD7VqQIbY47Bl=r?zt^AaimU)$zw^6Qc+DQq~q&TBZ-0+N` zT{Mpp|M@D=bR{31_c9-`OV`#Vy~xv%+iaiG<;~nj{eK!?r}K7mSU>g9SKc3A8R^hY z<97af>g}|)K1E=jSC=fiE)O5sYEz%wgR^BmR@$?svfVw=%nov*@ijuH%ae^}vQ#O0 zFuG=Pp2}~O)yLzwm2OqxlXm6OBlL;(*qL@j)b<{%)N3*Y*`oTWT`cliITnZgMk7}e z9GADWMSn>*krx|Wmhw5h9o|z_l^4~@f2!r#-0lP8x(HxVkFvCLdStC~Bqv(A&rf6f zb!Yv7Kx~$lduU}Xz^*gH=COL?_uV-z_S7={$>bHC_2v7sy{rSvJ*nl(VF^!FzfbSw z?lZfUPAr!}cjsna#MQZV_a!Zr`{>rE4a&UZ%6~cZa{aQ)%ki*0ZQg}FeYEddQ%k-u zqfE&c?aC|H_P#hS$z=7oUK<5p9_{jyc$5Ufdbmm=I>9ES{ zHqWM)pp@?$a=P67OMD+a+s$#k9BV4iCgVxp?D+k7EhXbHlhPxZPrK>o`kWb%Ij5J$ z`hP;(+=@pPnX40CO)Xy?=_NDo>vVm8h1bt!luC!Kx4s>(b#zDkYA#E@(A8b9){lE+ z6Uek(cw)XoQQoSgz}ak+#gD~%LE*;l^5~P$i*Dh`EOJY;DUE95+!XTA-Mq+J!(N$dcHZ;wp#&V+< zs`%t6mg9AKR%D;syRE)COK#aTC^`9)kBXxM7=H3NWTtyoX4CUhax@p*D&y57e}5fD z?Y*9>{OV5KtNq?*`$u-@^vm?F*9Cda_r-F)+>f)@ej-COM(h2{jQ5WBaC&eJ_hp-< zD$#AVeb3Z!dn0)LS-w*ltNLcEwZ17e=3{@im$J4w&g1x?HH%=X@zgqap=XuF%RYGT ziE%l@vyHZ-f>t3tfY7r}wRzAb*zJz{Ny_Nkg zv65Y);mSDASL;!=deLr?x})p9d4EQ`$L*c=261YdoNe!ollQ$izvp>p+zgp#{VKj} zn#o$tXPv>`-dH_EiesnA<+3}B$Cs-*-}!NnUdsas+_)thqC;_7rPMe)Rey4%?|xst z_J?pauY>U`mo9*cC*I@}cGK$h_4AQKJy_xgLZz0H{DWo=|(-n=oMU>eW@?!EC_)I;Zb_gX5)SE zq->qrx;lTH>1O|_rp9LPNJ;uwcin$uiB}V?n5f5@5j{56W0ijv)9BH(yB(3o{xMcB zx0`1r%0>286P-&Jj2>xw;j?u!gZbE>J+Ape()O;4t`b|b&$w15tv_2nvX#Qyv-`>L z(rorVa+Pj&iJEtPh0{#Bs~73RL0=q7`V4pWb73A2?@cU_x9`#`v)Al$ohE==C9xq2LJ51gv-bhYHn@=1lTenktb%_Rdr3qo&2KZ@2Z3G?a$NgGqG zu#aVetJQLm8Ak@E3}0_ob@5z2q%5loAW`$|4Fg%Qb!%kCAG_SQxjWsDKQ{%4%dN5m z>j0A@Amim9OM;e5WhFj|cv^p)P$0{`d!!%DCuOG1aD#nS11lPrsJzy3zISp~`f~)w zd+5QHs<-2I9y?`sNcZ<{h3@#s{N^C5bJ(r}OFo#tY@K*>B&EEZ-O9i=T9Tx>0R*EU7{_$#S!M%6fAw)Z^rK zIQC>bKh6Wi-}l|byU+Gx$=;wxjK`yE>Wm&ylRi)Bw#4r3vGspcUXPl^c80if8{L(t zoy}&2yOp(>*+@loTaMZtU-EPD9Ou%0ciAOMq*{x&Gw*NSNjZ(e_v%9XwecQh=IEGh zR-fv4+LFam+0^}NZ!I@!>N(!~ICGYt!ztO#7j<2f?(8C49_lA%`Cy9$T1-)UIGhhE zdd(DW7t?wcLtTH4oXh5V(D)2ix3zLv`M2bFL*AiOSdeC6RIYVXzTczKdjm7?sfP_M zu4bt`fjd|gpx?AMVtW{@Z=146i)Ge>m_%Ru4%-V0F!)Vg)9XA~J7k}^T%hTur=k@BQ zg%DSY|X|SD8nVPJal>&uTNk^2GU!3o}-Tip7ua6 zy&Et-^bUW$L3EdT(`u}%KGlOLxOb`K6X^Hud(Cjp?CN|t%+&E@RXoeA=gF}^g@+`? z=yIaX@}OySYGZ$M(DglDEuNdpVm_JFTYJ8Jzt3ye&!5(Pt=Nyx9XxW#lDqY^I%9s0 z=N@7ckE8i4Us#jb284sFJvwPyzR&la0scI_ULK}7i+%OZ zF8XZ0n`~y;I90plb+ulsdWLSSc}v|#FBBP}*A%u&aapCLOeGt}DOYE=;GKCp4IB4> zjjyMb8cz&VMGgAYUak7YM!M%`rnyodgkFDkB$Hw>k)Qs3lOiuVM8x_SMN@#ME?u(Q zz5wDdzgNye-5SK&Q^(M+hd^7Iw;5C2@jiUgt1?T}_Cb^M;5f<3b%Vp(NTs)}B+HpT z+k|01(yr>l+}}?d+1w@3b*D@U*auFqy5x= z-1v2}Z@SzvC+j%r+r-^Xd|xYJ9~P3r9F7hdTo$ewsn1Cq9(}1R+DBmR$YiIfu5|@m zE-Y>4p?y{qVO_DHL{(4b>lcrr+ogY$CQ{@+BK!URSn>FR^g=ssGCPevULLV<#nf6Q zv&-AU@7ZJ#sN2_k&dk?*ZBO{EjGykEJ;%qzy?#V3TbU?l5_i^;Yjv5njFkLnpGIRg z+Ay-4QPL{);y$@O+Pu+cHhJ!DmYx{y!p=KTa9BQGlU=J}=Da82TG^NVbhUqJ(20De zBR}mY;p7x)=jmb|FCXr6_TV^P*RSb{dh7ZB&Cyw7TMoofa6t~t6#AK&F*7r>S3l>^ z78FQ*u3A-A6BlMfuc_EYlV@tghTC)tkE?RV6sbippCemlM$W}^zm6i=9{;&s-r2lq z$l;h9zn-m-u1IhX{sEkcJhp$wuoSKdjItJd()DU;>RdEgSHiZ3XY9}eS_b(EZL`jU zG_?$KWW~d+BlM$yEoM34D6=2f8$X@fwrBMNO!r6m^^R1wM0 zhz-mNAy@&q{_4<`gjVT(xz0D3Iw6MbVOu7{7HuC5UzbkQbf@mPzQiYy->(OA_GjAn z5wSunqSYUZ zOJ@&JlyPiUpQ2Dyvdj+XwIHFLD11X`^`)Xhm1*Wrl_|atEDV4BDV{G-@;!-!NMR93 zC2lBGL2xIEZV|)hbP+tckcl~7(NteonpW5{qp9M0IB0uf|F>1C4c^F#xT(VFcSHR` z&GJMtf%)^QhJ1ZJ+vLLJ*{qFpInR3a%!eUH>2#VhR9&9+KH0f|y{V~v69_+Q?S{Ta zdhc2gskbmKQQ&`B+Ko_|-{c30&d z02SW}r_j+(q+i%+YRZJlrb-?|*ljS;xJs@=KzMSbx=T(aZkw(M6dmHO)d|j0wwfv` z&H1k|`iA9ZdEVt+or+?CEo}TIA5i?tB1V^6_$l*P*C?Z>ef_4zD&`s z_i+Wl<_<3|eBQcOTYT#DjNFx0hydPR5BG7eWBA;W?OG~EQ($9Yt3QLvy6KL0wsPomow#f3t&BuyKQ~?XZoGX3s6P3kksf6m358_cA;82!2}ae6!ts8 zpq%t!lqo8+G<1Atq2^-0GB=Zsey-39Bz4(qyH9D8)*78ZbkN7Yi%j)Hm-!w>_Ym`&6J~xkuc{gznXgUyL?25f zj6E$DPxuWPkO$`l1o6{=?{+ma6=Wz^{YV^tFdw03Tewi+Twqn|cCQLu*WnFT_}gJK z`sshv+Z_-)J}YAxMn-#&$Yb^~Nz|`m|LPNZShQjB8~1R*0fiw{h*Zdc!kKv=3ZVe# zLgTWa$&!H^NuM*LXZ~5_p&sp%b>r~PAd~r;voGpdKAzeZv`WpMX7ewE^USNVnU{mZ z_ibfahDkyDg&x;_VtW#BoTXK4`LL|Wn8Sa)rTuM~`?Zd$uU{ix)NlTAC7KhRyqhU_ z!$Ngaf{_$qnZlIvE1zTnzni6+{))5DqLkPage~{O^&5fJI6IG)gSXgY5N^!b`O&H? z-m|b{_LnJg|29xoN9%;3^982#|xH|Tx0-?;gm~$wOx8nuQ1It;mfGVprd_3u*#Ke zrIEWn@7_PM%=GT&aCW!Pp>|x(x;)qV#bI8Z05p)pO~;hDj5Zh+dSNB@P_V7qc~$wFkcy`VNrG+198k6@~X_ z2)q$83N;TE4JbC#{A%do%Fmh2UuU^VFQeYWN{rbfg?9PTn4f9DP1E=(CB-JNl%or7FPSMOW zWgRJr({>Xdpu-@%U9qq&b#wiDwHX2VLLX+LFOpu1U@uD`_wp^K(L8towM0NVC3I}~ zsDBF_tiQ?`U6=q*2^k|FrT>Y* zwk-y8Ohfb}?I+hpmAFVNJl~7yG!>%_#e{BRFWs3K9>`I0@!QVMvK*OmVQ&&u0*finFkJ^%nSq5KgNMh%!I_KQMR;eUr zz)tK~f~i|}N)w_NwXeW)#P!A4%2><91@#3p^5I{CdTpuf`R?!w_sJW{ zAN0DDT>q;TEjn15cU0jv{L}VZj1SO025Q@%TXl|(Yw(aCx*>cn_HFnMS1J+UZC1N; zEZUjXCt^h8gWrEB>C!T9NsO_g$UdAK03s22l^D&59bPlI!wZkF*_hC?DX?92!{Is2 zL41awlGwEy-Gl~r+au*IKuj()x$|t55j0}d8uvSa>b&!lUf?Q%nElB>uUnH#fg3&R zwcHd$40+*>GevR;z9(`=T<0@a-h}x@2XkdgCL5Zp3GRIGbc+;pKv1w1 z_4ilGkmI}Ci`&o%pR|=X!sg6T@#702f)_b-4fEoanbCKPBvwbpWH9OEgp%CVm#I<; zt;?5uT$>28MaHoCji`))lH38B|8hiFRaTN~52MOE9htOz$2$MZF)<9mCH@#LJV&%S** zV8@B{mD-G>Q_E)kLVcg(lVPH`6k?%`<$o6o^GNw~m7%ao=9h#L;sxYzcPn_du2=5x zR3ASzsY^D!7|)P1fz!fZK;^(mb`1sU+!Q*Hkn?{zko~@3KM(j-AU${@w=2n`v~RaL zwJ4}7yShZBDj%!Fmd*D~zAVjJTw3q{PN=9F;}nBU#NWCbRT~J0!w>aMW)29~a8toI5-K1rjv<`SvkS&GXenuVXjD4_}h2X~!Vm@65O z9}OWO0W{g!mNNFUrMxU3{tQb*NrMRKn>gCJIEZ4Io=e3GM0~gth?YdTM5fl} z)N3gldMYvV1fa*nC)7za8pr5x=CEFsq(gW{9jfl51?S_lX0$ zYa!b*YLgi6!nT>+45d?oBA82YA*C`>ve?uk`(o5ADA~KXRwLmwhhC03uA9j z12dXGE1hHifNSw>aWng2lOt#IP%+-CX0M>6DqJWSNHV~FUiySB#@>HM&nl*r&}wGk zbH?!FiNSA-2xv%$H(d36I$6_0KdRo5gCcYG{rZ3IE8WKnPObT>c)6)k>ko8SZY zYbFJJQ<|VF^#(&<2%L0$pfbAuq!UA)J1%Iw*1a%4R6qw)b`p=Bfj_0wxRJOXP9*td z-kcI?Wr<&ZSUZ2`8Z2$nLBBy5!Kik_4EWP^uk)@(*i5%KzXNcEHmuadY?gtFRs+qZ z!6o>~1GmnZCMpgpw~i$;Me)IM0cG#MD?tYibZv4)=iG7SdZ>o?2pj#rla^ku^AdBd zp5%&N!YUcHd25+O^Mo(B3Djkw3t(6QfOicC53#I-1b}~$NCvd*VO|3$pgh?<0>ShZ z#y6nU3EfzkAw31|XMrgh7F+A2wKE0`Bvr_APXNDK)Ppv(TMA>QXSjWS?y?hAipeY` zN8_4$Jdne^#J<4>@72nVzh5yP8B=m21r|kcaCo2H=cZwPUZ4+e|9T~HUzSbDBA0__ zFP3jxq)>kyx#x+~4Iuz|Tc|rV#Pd*|6=1q~J&T|yOe0*rp@z#`xdyc1`0YS>X#u)B zFgK|NnY1xU zZ7|TA2RBk$WLn8!ljQyVzrU|Lv-!=T_j2^W=gNOWCH>&0BS+A*^eJup)nq}~kSs!R z0gd(!pk+0MPe&o7p>#a>Idu6Hb5Qsd!d7@B?`%P;*B!X#DX4Lwd6kEAQiQ-@wVQ^VQI>W&iS$Z_#WS-f^kJbdKnuYH$gfN|(Z4x5 z^F{f!BE!!9vG{C5zw0f&c0aUi*rZSOFr#<5AU-kouZsIMV~lY$W=80Xq`t*;s9`qGQ2LXVNNNxF9(T=d^#2rTB~m-P*X?%cKf&?sAr#a0Xs0#>QkMq>rk6 zI?|Q8kCAFmDzr7+QiN1tdXpzsnKL#AoUUs@qQ+x8foI+6JAT*(IbeD&rgPT$jRG#G z-3%;$Pyytk`&|CiK|fwNg~}@-s|PDunVe@~cpF;gJ_E74Qr>48;%88OpJRVWm5-O= z@zX9DU4AVuK~f4T3^>XB5ECQ{VY!8;k@qMvP2gO^<+i+SX-~qw>~1vsv@jcfqi4uF z9P1%uqWc?vX@YMCChvmq+)XcH33B#8!c&#De!NJkK(#8Uhg4e@hi)T^2S_AUlU4eK zECHB9vfTC`HDy2xx=43bf>wVK*8Y0!E(xAQXWTtzQw%I2s7Dx<3Wwugch*mr^m%bRNrTFeIej2feW5QoQ1! zwT7&bXAxM0d%%p;eu()6BYzp7w(;)4;bK&fd$mt7IP=I*2~7KiDL;QY!}XW$HJ^qb znk$Q&6rFUj=Om_cr60yAt>Xv%+0Y}>({YsNO8=C|))Wz0Jvd11Y|O^3+?QT(sVb_3 zI+?x&#i=W)OR)4jMZw_kHn`%Hh2&tmf==b2&WWmD7%zlb8$g*sM zE#lAC;a>b;hMsfPc!+=WV)e#1*++A4Yx7wNzW`GPvU_s9T{AT-?yR18k&{81)1>mj zkSkN)@&Tn)VUV1<9)bL%tM`U@&V#Sip!OmjT|#9?8N23WF|4&TczY=>Jh+)X`A)%w z;y-J}RTmfajwo9iZgo;(DZ8WbIjXRKdQC;E+CPnTvO<(m(tN6zINIw zZvR^6%471X#VBIYX^NXYpVB4z<@Te}-rm2M_U4K?1=d|xM`f3-1#q;n1tCR@g_vPl z5}StM12X2)k@bIt*El?l3_5pv)E113Satqp2vXn+EIwjLR}!r-nR@F=_&f>9uSb&r zMfRf@wQxgZA;MCy3J$`PJy$1Mf|>_4 zGl6JnbhrDeinD&WSPtn9ETktXz!GV~_LQTcsmz#k#Iik5!VD4xC$cVf?5!p1wYVxn z1+H9I)E?PP;rtv&g(^M)O)|EKMxzGZxK_FQ@_^2Rp&Le82)4wNFVo-F$H8#&fWs4z z(-P_RZ>)dYv*i(QDBN<|noO!v%hMx%@hK?|OmioSZQs`y8Lyy&RIH1+K75>>dCW6| zD0FgG*G=DP6+2;5k+xc2tPBGl62Sr2B}(YxX3Y`{Iew7Ob+QRt^}DKgKT=MGov5XZ zEGkacLf|H$vd=NI89i$T>7P%q@4I%1A4U^3F|&WVK!W-Km@Jsj_c&lnGNHR~sBKbX zeG`C8R+cyrv9s?sOUSc9mHs$mF6AxmXie@lioK}Oc@-VIsrG)K{Hr*(gXn31a#BKS5eK={rXXc1A!)+l`gh}?Ubpk2 z=BDNoY8j&@?F#tdPi+9tI;*)~-w3Xjc+=X&H(Tm*dxrE9j_Tw)0`W4HXZAO{tZjdt zDZET35dBI!jqH4HZZRBYMUv=<@HweAuf^ABu>2{D=iT*@h@>LIvwPwmgKGjhO6YG0 z{@^t{SjUPRy^x!9&ZoVT&q_m*bHM7!5tgE$TwwFhe0fuYF$R8faSQgdu6MB3+ZtbF z0e7-Cj5?8kYH}9g&sTN5PZO92h53IN?JI$o&X|*Y_W7KgZO=orNlAvwXdBy+oz;?S zNF&A|UhLEps1ZAyik~luVp0YM!8tyY%~&ndg|Knot0Z2CT(8&M(B`8IOVba@0vSTx zL}xW55pTe=3n*NyJnhqC$?QoPGXT3KIwxDK`{oXTU+m{ivqn9zQK@Gjl#+i<@f46M zbhi<5sTAhvOixV{fdk3e^bU)qNK(kLjGbTzAJEVe0sSL22(qUy*iT}#9s}tG`bUN! zt#nsH%qokbwA5rhD(m>mwt^JCsxL+#H6p|z{HBuHNaTzQlZI+1r3;v<8wS^|?}_h9eRL zq_%t_N0(WjCK*5OapJx?+Hv4EFPS?(u4VUp74%hm&8CwP^c_%mTvOqpiNfU_-@nsp zOfe_Nm#4xQDZ#(KL`5V8UeKTbeqIcdifw0ZhX*8x8pzEt%&OA!vG;$9v>Px>6^8pM z!eJMLlx0IZx_uzm2Q8HqdxY%i<}^o7$>SziSyozA&?h2v{WkWn z^1Z|)JUM=sMbOP`yP%_@n0K*1(0BRDZeYv5it=zJ8WmEMNd+atH)86rXc9RFhT`Z{ zRWqHgRCojNYJP_9^W_v|!{FJ>sGwjS;A+k^(Ow&i^eyh8KU9AzRuz_HWbo$aJJV$S z(|Wm6@Lb38GY8@HaeWfZ9Wa2ewqP?fEWtA#W@PAth6XGadBO~A?vp`GAyzQo)9KHY zOWxZV!U*6HDl8B*kylXmH6LP%lpHyFg4ZJp)F87hC3Rr1^kOC!3nE6mFrd48{o=Tj z0m-6)iUjb7KWBfwdQm=6l}F3GVHc1}0nM85c-K7#yJ%E@(fdzfVj|uRnqwepi*L<* zf8ob6vE>nperbW;{Q#f<+wxQWMsLIiUOP5Bp!J?FwY`28c}bE-ADU2eD#lgsRw?nA ze}3O^G-cdPw6Ll&G>9=&=OoT?Tu&Zk^lq?|DQoM5CHNzmT?v{GZpA)?hB2+%#Qw`Xog)|-5zn(h(_ z9t(dKk>LYUOR(Sg^FCugkjHL0(@_R=Bx7b90hrMrOH|jx7Pcp!VMP7mKYg-N-NW)Y zBY@_du$Tfffcw}n;>uR>1=tKvyw8=kR8C-E$M7KhT*ZQn4#J$S^P!_v#B80V)%iFW zPNdKFtD10bP6m)ZW8&$!3Y2mUYIZAnrcr;49V>s*PAuWG-V%7lzQ*(Bx^|4IYD98k z|8SW&cGPS6u?%2cwh|FPY9=>8{zMoo4SIxkS_DbG#OS<0JpxKOfeLt)D%b0g)#x(TvVYc&8>b@NW%5W=s{b|dRP4`!Ae~q<@UMKslc07iU6yMfz=MC- zCo_pz){?^P&n|SN7G+0Tq9QT4abp;-2axn_s{6Axmg92ymNcioj+p7*??v4PTIvN0 zUcU>;F@JDzJ@9~?LIW8w$^zL(-Fz<0j`jfw=E18woRi~`w{AB@C8`-s|HbdGD(kyE{oH+r)sY{Q} zu>(*cQw3h@+dO*cCp_}$Sf7t0V_Q$u`dPhi67hHHoe>j>@5LmTC{Z>a*wt4-HG`m9 zPRul5ypCy;vLPU%2iY9#Nw^tY8S_(^d`!*-}1v*b<4+|!%|s&nXVdK7D0U@;@x*(P(2Ic-;G2Zj=W}eX=~D- z@}$nWg%^zX5a54;%of<~yKdt|9reD^a7-lije{w2hv`!27^iQ{Qwf7)0cRjx_dqb{ zH{TRisA|c{mcB&NFxfETsJc-9YmKJKFRqr4EwJ8r($KsO2aG^_J%ygQll^5kf(77o zkX*Nr0L6Uz2r%(6KHA>JR@%*5Ff_T;+n{}7dU@+%ML=GmJA8u)0rciG?{*r zFnk~0ZO;X`fb_F-JRF=&_)(Eq2Gn$9+@dBpHN3?92rh|QT(M2N6JYpPauy5D%_8;q z27^v)BU_!Is?a+E3;#+HE36y~G}iA-*fgd(+Hf% zC^oOozc+s+nwL+o(}&O9zwiDyN?^Y=!J6U9P%uM+rf8>+sv6a!-vP7LXodjG>#!N| zb=DwYEPp+`jm#SNcAi!VH3LB#Eq-g8cN@*Q)g3)MWeUiX)|L7&mnOjuMeRwF#U%hr zSktxM91Q~Gf87kZbgd7h%a=pv74!1OaxvNIFIRu-sEFeuT$^}hnhfWi17X}iDcCdD zS$9+2!a6eK71X?|36uM@hmw9X3=lq%c*|v#X?$A;%Af%@w@0~gpZY8(pbo?-Ey2n1 z=ZPec?IS}G9X5**JgU37;~BQ*r#W~;>$d<4AhVm1<^gafqkD=#cfqBq^C?dVOik8uYcA)rft0`u zZM28^mK;tu+Op$y4^hZHlPj#Rc92rOKUc#ZSIYa#BX&e_j|Cdmw`wfet!6kN_}2Ob z>K2LH>k_T0{$LCu=bk6Xy-d5Nvm0Iq)uAO^iSj~pmy(%Dc|CTK&Xp;j$7Y2$m5YCM z*EuS@ESBF5m=e#M`9;srnnec8=iNXMomb zg0@C`N89E&ctxRh+jdug-A^k5oXCG1042XjZD#~%O)a^Nj8OBk>G^8C2IVpUB@8Ll z*XHSZDKm{ChkZ`%OssSZ&N+04C0SH&J2td9W4b0XM0uNz?x^;3Ud>q8=n6$h9-jmffp$Oj*sM%I;rkLIKup;bO9ROmnp`9yj6 zQtZ^HVa|n7Kw7pFC^3{2#oge!p8$Pa0xi4`@ww#|kxoV0`RXrd={6a&m!=aqU;$QL z6dY3$_{Zf6Lq!U$rcda+A@hI}8eer48yzbKua?TM_oIa8kZ^yZ4KiCr=8e6^`btWx ze#wjN3#N#i1;$4{Ufd&pt1{ajJVgUk6>!oVMVPr++7qWlS?+CDDzw7piGOz6+_S6a z;(jpiRo=1RyxGlBi2&QdM(}W5@XHl5F(!SGDzsdo!}=UO?H(=5@oV7H~H&+q{1r5NW$J+3Yeu4|ifp$?3hl zh4Q#fY4A%dU~~iR$*kruGIbx16iWqpdAQ6*4!;Av7gfV-Lo?!ajF>miH~)#kM%tp$ zCqmV}@B|zeQyTC6w+@&nKOWnVzips7Mb!9->j|=d>6b~i??b$aW$&XZVQnr7LZAx_1iv`vJFj>vXy35tD!Q2E% z5n*9z2gPAsu5Ai0sI>slDAJQ|GP2Iq`OnA|o!Rs|>T7QHDWnl?}O-Au}fpYUQqJjdZcWt>gNv;bf4*kUD{4qOR;D3M;_1d37}V@Zk< zJB~Pz82MG{VzeEO0vgBIS5wMKrxnI6%rAe_dG@YkxuUg02XNF5RPecUQ}n$IvxQNvK=C3l&}RF4 z+iAfWKk5s0>?=Xa4cujrb%JA-kwcW2ekkqQcT(c*LsCln=DyBL+^-AN%bN_V1c5vU_`%CiEp`(X*Sg|L&yp#<$UB;-^ zPQUHzeU9Jl6_I#aK~HpqnHwK4Fq7_2mqK7EU8&@|>04Rhx^Zr4@V0)vJZ43SG6vLe z3AZ#iwLHWZcEy3{KeT_&DeRsbeZJ1jgglu>3TP_>e_ob?Ry`GvV1FfzY+FT`erl-8 zz6@~|;oE0nC16Jg@YK>UQyEO%5&?swP;b(rCIpUK3M4kg*etbb%RnoPE~Z>v>RVqxq?;SVUi# zE_mQSnuV*i4QYS$8=fkK{s^$5cz*SEJmI;vD5B=788T1p>U&(Q?^!kOJiZNU9%%)F z0Yf=7bcCIAGon)hFuRj*c7$+|i6-zr98%rwhzcN`0~)U}Lb_W50D`uo5Pb>T)|wge z$q(DoLb{02x}w%t(UggcCztnM5|#sc`f5qW^dLPN5K(`VJ^<9Wf_oXcY8td@ZAjsH z>5*-q7R3a@6JcQ1xu78;?P>HE;{liO2Bm$%BgsS_9v~xq}Vs|4VDWk>E_Y$O!vp)Q{Lle(-zaQ;O z%}!rqk(PfR8{4f_GynMdZk^p>D7jZLw(>#?C?^|RDCeBjDd(KOr#JrF7;emhpoqia zP*6choH;#ZhM6W%g+o!i4Pc@E!2%B?(73qrH3^OC=xywt^DeO+JGVYgLx`Z@$vWW- zL8>yVX-lYZoN*&~q)+;$A#ia?+NYnGWZu~k350*#_Ab}X;!BRJeCGfYPqgpsnrb*u z8c$@K@qSUmwi^K~;7!k3aOuMf^|d-(Y1_!VsYvFX5OF-?bP>}M-P77>_>^SPQp2}Z zIM)tzjH!$j5h{2a`U4t=r|u0uJ2Pm_4{7Z(-H0TtWaugqyf^ ztJZ(+jx-+-m?W zn=-ufbrDTmQO1=HeCaFXFN%_oh^o-2k4w#@wxJ~|3ftBL9SA5!AUEp9*8AF=>C&>$%e)e9ARWEc3a~H$O6x?LZ9hFK!w*<#@3DzaUbK!ZWAYBV8j9P%2ZGeIF#?@Q|B+W!3;bNjzwU5CK8S0kbVtHXu`wzLoH+%4X*W08hkLoF1LrCh@YP!{L9= z73clDJ&msF;uvC{1~d}TNZ$r5XT-2P=0bf4=!OQ>i`N1y+!K##YM;Ca43w6e0I-+p z1qS*lq9%^CE?GU@E~K7@<{=QjZ~PS(Guh;z3&LsRmt6CowesWx*Q2-3pf68uGHsS9 zEEtQ(tVibT5LfpTtlSAir@3 zCs!oSRSZw$o=I&LKGGQsB3x{iad^SYNUUz;c|ik>6ZrwO6(c zEv3fP+|&bCl^SVi2-LGU+&xw1^L{mF5UUyJBdY>JD4#M>nSi6?cI|1YrY&j}}@O z^)VedCxX`!b9gpe9PYLuosN8qBIU{1mI`5o^;#_-^!V(9g=8makH|1YLN?eh@*Nmm zJxI^+vw0N%`>sjq!EKvLr1gL1pzl};zd2fX9=m75PDOYBf{z;Sx}Ci7FZWAY?M+t# zKCA35XO)O_+13$VrpFZ#<4&f1!fq{B0tY;SCSaRemMTLD@9p~?XdXPm5d4DcktjN9 z0eka6%!HkAs`j~}Z2~P|Fj!CywmHL;<>Lkfu>Q#kELXZ@pZ=@g?+<^7x;W{UMijjY zqFE2T1NUKC9O@2f%K zDv#qZ6kWZf_(eq;d2>^CsjXYKzY?Z9$$4X0v>cqdMFV!iyBBo(=#gLuQ2 z`M~q*wH!=6m#=|S>7e%iIF3!uVsTjVxa0|h!(6qs${~D}8}mRXKa1wdVWK^`&auhu z$8?I=H_-hsrNR2N=`p|8v&~UJm@O4o3($(oiC74xe25KvMpr+Ew(_ea#TwDfo^+$bBe_>N2POfX@N}>DpZ%m2?17j8z2d1I;LCxg71&{?)xlUzae3 zF}`Q~m?-MO!@6F20y`2f*I0bxQBrG9#0qRe#m70j{6&>DS!>h3yzrwFAhjgtL>Z`R~u zH$^15^O*iX`+21ftUh98$wO4e`Gcz-V(%wO-mk{U#uhv5o6bT5-$e+ zFjsLg`0RgNd1QM;ge#lc=YZuRf!2sd}7lVx^I@e%Ds_AKVhdBL5!C;>MA?_<$RnmNbj6tU_z%T z2{0^;EBql25n|}u;HQ1*XU65PnE8aoZc~^!h7o@Or5d8MZttt|EoVDzpfV=jfyv#G zFU&@)i>qVzb5T^1WBPMx=AhLNgHMPV(@T z1cq4-YD#qh2>^Zb0~80px5t~T#nQSI`K6>~&_Y)O_$DLvRt*CE0?L1GJ;nxZ^@4fsVIV!Sh;nA4EIQq#oYEb zqtm^&060X!$!?mUzGoS0=&0oRmwy7{L}qg*p`FYTEO^}rj|b}XpYW-OTYrGbPUZYm z+3f&-P*@hK{e_H|OIB`NE+4aq>slK9f%9c~4Rbj_KyqV@(y{`ZqutBF4?_I&fKo z_4Ce?1InG+Y`ZcJn3qs!S`_6a;RVv%l6%^J%({g3^RdGwkS6t#)S+rU$`nY%CPoCm z)^Fi*s=VsPg56n2tQ$rb6*GcsS@%E9{x}HiWGga2#B`YC@z?goPxn#uBXK7gL`RD6 zeh!M-@l{aqPeqSh2+*DefpYnx+^&DKe5j&jIPIQyR6mde>`pT`FnEy$lh<=pu0?u( znn!f*nrEJ?pv@U6W3MEPRzJ%+N(sPFGf}2OcH%%5c%0w(0b+&XxBEp3s>#>84ND3C_t$Bkh~lZJ6>I^0SD=&et?8O& z>@!Jw&#BzrjN9AWUjXSYKpRlwQn{Ib1*(n`QpFHDL2*7y?tLW`O?r(}MYs?9^#;oq zErqOOR4Pa@uasWrI91x}=F#G;2W0vL$G%EYlVoM4w%Su2yjEn+jv;|jqr1-sad=SI z=d{9?qg*@2UYn_&SMx?i!EGN|K{++Atp8n#Sj~0>?6w&x=9pF11du)I+T{j+3NLt` z9yqlP;B2&2F4t2%_+z!xLxD~#l_MTf}b;468?KNxY-XQ() zxE%^)tQu905k{P7ocr6D^Mt@GeiX${APj9h0(;z_ei_DQ&=OEe2l_=^PCQo*`3u(1(0*VJ2Z+2`S` z_)4r@mHhN~G8mwZz>{>xfE0^;aP{rKO|8t~S~8Y2caiqmYD_+;lG`$W27skE6WJbm z>oeL!(q%(<7bsWZB7mgjF!YVS8ThJAdK|5-(fty6S{xEBB>!GR6*%YWU`kH<8uP8x zv_`+!c zOFgi>kX~nk?-m!nb&mtz58@+i68%QaG-{p|jYK|?Nh%i6oY@?-JoJz;~|T8*yTJsX}l;V!}9%Y>vp|DVLvTR-g zp`W>zrw6>X$*SIeR388?8MCqzd9o+W&x9|;*bpKV9y3@K4_N|$LBgYWN6wjaLyIu7 zz(~0aB({l6&D^hDu@RC|G~vho)-r%4?~UVl@}Z*n#@5TScKiU&l>Jsn=U`g9CUO~t zefPxPjfSb_n-gTBt?5{$YwV3Swa!6Omx^Ln9c9vRfPBV((w|!zE$0!!Ny|PqL6d(^ zm0%P@R8gp7O`Chz?Z+3ewLb0ni3e1zHtr-{Y^%TUdUiQ4gu6*=nS zGLM;m#2^fJ26wp(M0rrHz$=4W*1oVJhP+*T@H&=bk@V81sdotg0Xy8jH$DMa4$IPU zhpP31KQ;^swss^4(cp?b8PaMRQG6BC?oMbO8@#liNM^}E*|`LH544vb;B_oi$A`#5 z%9Yho2sCf3tJ*>ERd26W{+U!Fbnq4!UB+jB0c7+~?T?l_gfsBK&#vM@UIZc7_??PH z3Grdk`yia%73z8DRCmkwr_1uyZBn5>B$wF;Qel_h5X$HEE=!KQ4p@N8Z&Tqd8?Hm5 zx8}w;dQuP1G&3!!xYPy;r!6u<_mP4UDHoZoT3B?20859%lTpv#Z;i2fO+>qn3ZJWF ze1lRL3TEJJl|XxP-Sr-1_z50aHtKk`Sm_yni*S63(v_N4GT+L=s&7-Qp1(4;a2(}S zeyqn#as_p*Sp}yD|JKlMuk;h2_fLqLU-c8}q%f`^bkIW{40fRL<`NVp^|r2mSS7~n zl*^WCqhyVZZC%Oyn%A^zs ziD0z((iWgwW5z5ge~PGTSTxj z-fp)20Vs?$A)|-(Ll9QZyA8l14LcQom#*PTZLemG z!Yri)022DM1UoGDJE*1leR=swU9-o(t!5L#8SEvZ=2#;Ub{Fo}iMR;9GIv{Q%0%4*+KjkUb-RhpqFCrSwCF z^bAK_-h{SAs$ajQZLJ?BA{2Mggykb;C<5`kWfFfXI;OTxQ|R=ErypNVkFH3g4rCM> zVrpbW1>szdPNBN^6PNeMn*pS$E`A~@(mh;Q8g zsXKljt|4??;}a2Jl!JhO5#H%=jrXU1ZWIAq>86WC{S!K+iF|yoRFGC-QsLj-2hknV zXZ?C-YU7cfFeH9fAm~hX35j`(VtcZUfzkpx#zVa_#)YOq&vn4h#e=4@9_i2Hm9+E4 zXa;3B*UzFKxj1L{GYvCcNT-4|=FHlv+9Lv6PTLGX)hfch6zgPvIL_uqE>}*1ctQ|Y zFwM0C+J3@YAnD-VtxOgOECOKM*Dj*)MF@A=_%_;}W^nddceKf1_$!i4#mqrG!gp7{ z<}OO#t`Iu9<4(VRO1ngSt@ zdq)LsroXI(?_>$ud&aON#T?K2j`x@^LA$MlEL6Zn&p9ANSNu6RHBvuK}=9o!IGgJS}9L-W>ZJ%G#$ z+7s?kUA6h<50vPsfy;Kn-YHom&1bL)nf>o9t0NuhT5I==Nkm&&wX6N8p^hez2oTqz ziw|Lceah2nRK>rqEcL&$N-elfHiK>aAU%>bY0_{B1`=&oJ)MNm_E#tW;7_ly>~7|h zECZz)0x-G!gn)%?M)>szae0MFr{TPp43_h(LU@hyHdEfm7dMOthu_qZk2#^$cckihD^HS$oNc~B_X1@-`ryCa^lGh)? z)9!sUK5Uz9)Tmg{9kBA|EkMe;O6Zl+)hz*g1UnmbsX*bm z#P2u{(zud~U=SK!UAY{Gb6#*gvC*1toY3RgY`BLhoJ(|< zq`expwf${M1*x5?skftSw}i#Ax3jeIwu1GeL_1u)^^yDl=E`-%_~Oi*Z8Aik-&k1z zzZrfE0%8&_YN=rzN#at6*=p^7KXIJoJf(v_WHpVM&`BDA)K8#bn(;1_C@)laq&=cp z-dB{?J~C!N+zqEfsaffH>@CnSO@zTKVx4&@Pabpu@i+~-fpy1c4{+#CbBWBPx2E-9 zP=vmJM80()om?``WK-lwR5j1_k58XkA|&5oW+=B&zgHfl`^oFS)bT}sm&0LR@b8() z7GR53%Ls9BDdo<)JP2ax0t$J2BLLxAvqEa$tODO%GP5C0Ya!ZfO(^7#uzuJ+RhSX) z1TA~>bXWdBxu_iPy>b?^rqumd!EnU)>Aqe8G6jS=lfewBpTjsoHR@`-w}_hyv?ol4cE!9Vg?eeWKEVvC(AU88*M8Ek-~I?k zZHk!&un?!ur<;b0Cl#6}kt{~`3!jcGwh7;x!L)RH4R2@z8(*V;c_NB1N>fF%){}@$ zT=lBsTt$9pXevBYWsPt&t@Jsz8aAtVG+M@^N-fVCZqS4Kr%W|z|M=O455ok8^=V<% zNle}gN9GH?BrxRI<~(Nfr-cxgjB~GVWPlskV+0I_s&(w=XIK0$SI)uL5IJ~#tBzoG zPvOLOaCw`5ZIZ@tr$VAa(fzI5IYIE~iTlL6ZpVHnkFaQ^g|L`+EY^|qu+)Wl zap1RkBG>Rc5*`K8BP{~ey2TQNc7923NSs*%ChK_ycN)i4xTgeIDrXMh6 zohS$9-GtVu_xvu%l!e^0uV;pSGP$?<3zip{tFkyd1ekG4#5iJHyawgj_RXR$mri{?WshS+mu3oQ&*)g~FNs!F4pyF2(7Q%I1{p z3&k!4YwccW?py?F7b!L<#zcyNStRW|yZ2w-@d_y8`$O7kMz5x7*hQ&*MnCX)T^GNw z7gcm@k7u+e_o89^0fA?v&zcOA*0g)#l@uo+7fv>RmUCCWr6ewC2uCLO`<&o-L-vFl zrJ)6D_00;-Y9PiE2+U*sq+p9(1XxbYRq$jJZ5k0GedM-vW<>SXn#jBMVcwsg*8R+J zx`OLW$1O16*n*qEnFH&YkTY;VWM9LjanDUL?jSex)yc-Q-N|A(3ZdDGzg%9R^3Xsy zG+Us5Q`4owKGrGv1WP(#Mqu5ij`NOJ!Q29f^mQ~>B|O8$BWYov1f8RLPNXVazu0Y! zhwq4Y&@LdR`6i)i(lC;syLn^S@e|{pa5J`W!l^=`p0Fzp8_B_lyEC`v>(*K|OKNfy zBsD2=Ya$bUjrBiZSw7n?27ySVB(P-gvLtIL%)&t5TVwo0;94-^ne?cnWqEuIB3> zsG69U)k}TTHA?dJH93;)>^9bqbbDBYX3m;p38wYq(&l=bB@F8I5SI3HwJE_h{urBo zpN{8>N2v)phvgWg6^>W#*%pTT$>S_F(oD2xn-IC2h$`^5Mt?fC>`5A6V4)i-qBSke zE9Fth`t~TXZL~+2NwwUm-Vbzjma|rVG$468Z;FiFuAul6IFJdMZt+I0(!}xo>!mgwlqj+jy7@SfdLTyt>DEVmfUKx z%-j_fP00nk37B(x#$kVJuR4ABNtZ)#y{1Y(SbPrwAf4FElru6dWwj&Vaom=F@+G|< zhXXKo%=!T^R-^YFWqCPq1%Pn0WKL7=$~D zn>NPbYPGWM>jn3gw=H>50$s}=WtZuR&36iB znQ*?&PQrU^c@2pPG7)8z$}ZjZS>+ua4>oP^_(?o?i(2mr!f87$w$t>j%PEPTb#`u+ zvd`qh_K@|LB6!wa?txi^;Tnv60N@lFf`-9^gZ$@jX8#C{%BsBTROKx|6kG&?Q@*> zvCh^PcypXhp)b@eD7zqRqd{K$f?L0im~v*}gq;`9b#$eD&*jK}y3X@3<%RqG9Wa^i zMd5B?KdR1M&p10bgk%wsC=1jB|F*;833}=z()wJz5MG=E3vIA zL}K&R=s_{sfbG7BtXznR^|_iK)ldP`+NMJ!Bcf(=TrTZA9WCmoi4^jf5kLlrdRT3O z<|~SU;jW9Qh{-E|$jbnmJURZ}Ol?V=S0W=qdQzUQvO+en6e{|O@au~)BtzOoe{O>s zTf;bNAjN=v8&MeAtJZ@Xq(IaCaa~wVy262sx2*1soXZ8dCN{uKx@&g*0$ssMrz%{RP@ftdRjv3}>rmRgOtQVtu@#C@b z8Ik3i@P>z)6WST>2JE?naucpFQ!>f{ZtDttQ#o1ofXgo=q~RQau4(?#njPly0+k$% zAYW4)LgN^{%)9RrAhFR1x)u&5uRzE9Qs$S3<_j4eRar>F|WP zieM4H29oZ7cRms9ZCORsi>a8~N}@);omwh)5u|PA_Cj(6nVobO0Ze7BXV%e4`}J_x zHboOSctVEQg5(6*)^_=LGA+Hl6`+tVf&Dat|^~3|63|C^KB4A9}TV%@5I?jfm z%-}5NGGW4%O-5Q{EZS~t0ArY+R!n{Ri6d@Xe2e!?)=zvibj4C65WpTgR~M|}*# zj{Dz#rI`HnHSV=~dt{zT%rK0U{88fAKb+r-ACQ+PXbLF|!{o?65c02z8aS!!eLzx6a6~izI{5O^I$5MznsIgD{ z$Pu&sTO;)s-~I7ZKJ}3S9f8~GZxV@j{x2TGehw+;lX#{M!{qp1yp;dpr6V?0d->$3 z2{8=&T7U8Q-y({1*xLe**)RQf|Hyw((_f{lO951&znW_#HQpEUw;_+x(GyXzSPh&-}HLR89X>~RCn_p3dXhVHz zoHk(JC+Br-X9EuV89Mt3A`+3pDIQgOqlVeFj1&1QiDy;ONJoqC4fdqEQ|t;?FAMbx zvQ%=9TNeE-!xDU&VrMZ%ZH%=$+Ro6J9K5iDGT%=&-G#k{lkLciE$G9};klZBymPa1 zD_ie?OA_dT#t5zB@?UVn!Plgb^qaTP=@&Sn>R1?h)JO}@<0#577{^l~sPWk3w|C4a zt#BwE)e3Yk_YL=?1s)sgl@>&UB00hU(+{!9(M6ajd;1Il#W%RBTW)?df=a-fsN?dU zSHV8dQF^&Es*FK*a0Qm_l3Tie7`w!ApA@`vwk)M5bc<3k_whtem3%R0v0OAjzx<^z z#USP1NdGQ3u)-z?nu*+|r&vc$5j)m?HB!U$BiqSlr=@O2$6xkDu+g;Q@ik1l9nrEl z;(+k7Nx$sl7@^}-S5hi&i2ZK2<`YE&cZlJ^^i4nHo$22G3i3qr5J4?}MNson9&sgt z20cAn^wr*A8E3*wR<7B@V2!$vm}jTG(Bgu%X|~hv_c0h1?7}k+)P?ndTp7cNuCRi9 z`$UD_;{8k@`i!FZrw<6T$$4BFhFJm#8l<((;zVU3l}+^lCIw-&B$hQMM(Ly#Excx- zwk#3gx7W9jzA9*tZQl8RiQw2+&HsweGhNJkLtBnKGc1DlE$UrJaqNHx07G5m@xLAWS*F&Ka>szXIncxR7 zzqr+Esw0ITvGVs0R_si@yJL@l=TKZ<+5{U<$Wz@cQ~C^o1!dllyF{H&+P!wCEJY)gw*W+YrsK*WOza z9J-*~s{A}23Q49#-wsnkqHb6Y?Ewm7bq)0f5wj4*?Za<>kdax&P4lsVii3gFem4r- z3i1$#{fdkbM-!;m>c-`x$T})%zidu}UAL=r(*}-+Rj@^-@>#DE7ciR6;gH(h`jP8x zG6b3JkIqXrfMJUD?{SmuJCBW3rzjqqY~;rcy7%qOe7L(yn<9hA1{g^ke&l&OfZZq5 z=3z9~-3cdu>xRJ{EgLgJf`tj#fWgZgxON3R|Er?x!hW(feqUCKF;NXD`kf4nF)3J> zS*qD9g&pdQ7VbNC1yjsz+@yo{2C)t*O!mo_JNdw{V(NGdT=z=?}ywm=;`c z_Nk;|9G>2NF-+dzLz^XO##*UNbGSK|e-{H<6GH1g`hI8$ugvShTG(~8cgr4g(xksL z-9}`8Ct5J(%9ReW+xO?)Ojprzfl3--1nrKOl0(rF(vrA{_;cEM*ZTq{N+gmd!#qs) z!#bfs${UJI=*-G#rv#timdOtbUbBAHxR;(^i`|rx!Y()7wkBO{i7J%40?VE0gOWt% zxo}tY>-v6t^Tx{}{kcJ+IR@Hyy(~Y%9hJ3zi<|OCi&=Cb{Ll?4sd4D6}On4xH7x)$w)O8L#w}@3qr$x zczCWW5PZ%bF{ZMC5fr%fvyIOS9l)um_BV0X(BX_;WvKR8#i-l~gWgo={B(s$lpK;v zGr)taeYdxnWX!Z52zF&6MFF*xojdu-v@4W~_0vNkVL1Z_(cnP;c2cGeGZv|<4Kr8c zYyd22T&he)*w5(t-R?J5Vksh-v$>rKqelht1=DG5h9_2aCng=FpoY4RZ%Rf zd$-sxmXNy<$KnZQ6Y!j_Kqwkz#4A>-sG0d(jTLB734}v*h;WZXqqcn-e(m%#xpq#` zTq5FKZ(KC*F(O5bcdhp8LU%Z&%;mh^ya#L$=wYlB+X$r=TjH%FjyZMMTqI3@Rj@vS ztGi6`;JQo$ZhsQ-1_EtM0jqe|mx9C@5*6xCYaSJ)sh$ zi%b~>u?KJGD1q$beI*Oc55OXM?ZY~cBuH>YZ-z+)=>{(nXwpap6Sg8V;dwuT5ht;b zo24ZP^!Fh+p{;$_s5q#91TB|T8tkSN%3%F(8uaXp=E_P7NW-ex+E&cNL7~C-ZczrU z&$c?Co6G%KjB`AN^!=(kV&|5P=D(-M-WW!y_|mb_npd_Uh$T_KZ!jCSetWvSk05#u zx61Fgaq=T31Tp<*N<35dwQ!t`z5)I&DWAbeniiA&_zDK;FLX=N3=*mzhYbBLj-Kb2K z$;V{Q-`kp9N?Hx8y_W?vn#W1nVzz`(=HHKl^m}L)j~y6x_3wV9Nqo@- zpD93F;vPwyBvpVr!Rch94&S_&0*h}BzBCGdTbtErK#j5$8N1!ezc>vPZ0T?9ZK0Ej%&%PaJ#j0|paSq#Z$dFICX z`zseVepk;yae&skRxtU1sIDUv;6b@#lLF%k#o5$RrJr6cFMi<+Ir~F!n?#eZcxVB1)xeG>%sQ-kEZR+lcjGz7) zHOUil={xEsMp6)7;$pNg+Jw)~Z#86aX#%1ijV=OuH>De4#?+cmc$e zT8oGYc;nTLKcu76IDC5~Z)Omd7f+gW7KD+1glB)uY~uHZbZ4pKs~Gk6(I<#H{Wm4F zMVcc-1+?zwb|r}VK_!qP^;BFl7zQDJzEf)vac>&1ZHkEOhYyC@Ng+R8)gs}lJ{UMl zD_Jkmz=}_8)v|~p<(LMh3uQ54jD8wF2>>+OZ#EhEsq1uEjnoS0?P!~f7)*H0h1daq z{Zv=OY^j!lQaR%;1-0W~$!mNJorgF*-t^MDbZ%YI;TXevozxx$nU(Csfqp5iMaal> zslUg-Ztw*U;;TXUPioQw4X?Gf$_6Ql%HxEiAy$;#6`VM*{bUJf!`%MuO6q-A_?Cel zQn@SDS*Lm#OI1ogE5OhqiVNAguy^!-d&mRyY(;eUCFO~1@}`?wG^o4vNZFI*>RVMYBfW#g>=v`#P#0{B~9gzOpFTpMvuMBCyk0`zK+AK1+|J(>S zFfD&{CK7qLd6XXM60wgm2^;5D*R@VD| z-_+DxtXhOPL5EiN(@#IGIB5hbBmPb9FuF9auzZ|0d1Wt67_VAh$|kIw&5oAE7gd*% zSi;W-1RppO-d9y)|4Lz<&yClU7k$i}XBNe%y{dVL*VkIZ|k+tEt$o>i?So~0ytX;}T;uUWRJ)66 z$9JpPmgC7{{qFC_pRr}t0o#7Qt$#LRG@Z7iX;fdCr6!+eG<_V?uk5jXt0vp&aXMOG zzfSf1^)P#k=yf`|y`M~}-q7U3{_?l?n`h6gc#@w#VFbYSKzp#VQCxyz=*C&RllCWjA>& zC(84@kfl{!U()8`eH2x3x_o?itHpbMsW0q!TAhkxxlUhRyl}odU+YWyHb8k^6}DyD z_KIiWHr=qxWpl}o<+)m|Y=3cij7oje;iI^(ZfQ8VuK3LM^DFpYoAwwQnHh0skkNd9 zwu9rSEr@uVZZP^9thPd>J0;V%nVF9cgsgdn7u@HUIyKd(Jc?xz!HxuDbe zq|~WKTa=%t)7U<(R@+_rGM!G3cUi~rYBlyQo6TmIZj&TA-plKHy?+{yaT!IsUHWmq z-_OgClwbGT{r3KKzhBQoWAVuYbS=*`O|yNr7fmxAsp{%47Vl$zduvpCuGR})H`Zhp zA$YSXZ~zXU;&pl-zwUza-q6Tua9mxs!Ev)#S5^=vM=Oa(dol9oD`UTo-iBcCGHv{b zPxw51+^_cN_WG{&j(Aw`PUa!YZ zw74&ie#OT8YdIcI7x!C!m<4Q}s^$7UYu&Rk^z-e7U@~}DqnEZGZ`S6vbzZV?zMP`B zbiexA6=X7O@@z1e4VKgVGAPZ{yk1Yt>1?w>}BoAc^~-h z<#RRLM%NMlocPoEep~o=b%-Z#v%vL=@g~0Azqb4(t(Ufu9@*ke-(zcgUOO+|kITk1 zuDcESdVed$3u&Z(+BsjZZ)J#i+{o?CC{BFNPx-sLqja&i?dHDahsI9EFXMaB+>OzB zu`~Dn{&=c}?tf;x8!uNYb8)^qacMWTc>na5t1W-EtKB_Z7GKfBv(EE&7F+oJx|m;* zOZ0W6Z%cvOVUlBy-{WarC9m1ucw{l1=AS`(;_@w+y|}0Jb};M5Je@h_=E{R+s#J6x zH)||b)9KZyKDNbl*;dW)t1#o&*IXK+e&!eR6ef$U3>Sm1$9sa`i?2&?2l8H(;x?O|k5MwJ#s1Cq)NY>NQQd-fS><2h z)Cf-Xu%PcTz1(+PwkF>fOSd|FtaSCK2tVGv+O-yga)?G2rOWY$ysk$3V*lov^wQes z^DS!Y$bZ~TM?tt6$@%Ex@^Yzmv9U>%5 z?BbDpJ+`+sn@yWx;raC%fC3zbLx9=;{lET?Lw!AM@Bj0EsQPgDAOGure_oIObHBZB z|Ib5xz7OyB!~eD0-rir#|K0r*z@jT^DI}v2|9|4u?j!y%0K7~0&D8eq=^9OMMz{mO z9~b>*+m!pWkZ?~onF9Fo^qXkAIivG##Xg5?1>jAj{@ujh=h@`DZa|pqUgqbWb3Bp! zl-Rm}w$RVJd}}B9#*QN~1psfw^^^W7uQw~|AvhTN^Sk;R4(i=HFy|o*^ru`h0RTa^ zvwz>pG|bSjm;rcxs-NViQU2}#dWcsC0GEsI=h0`n8Xea~@^aVXU+O^wr_6rSe{RiO z`!N8wD$Uv2YxLNx(NRgy2Eebc`se2iK*B@6d;m+dJ3tra&Gt~U+MQz z!A`4)=BzLoL@5RlQ`%G%$_()^E$5FnkApj6Smn0jj0|6V1RR6RD6HpiuQ*mv%qlm6 zudX12Qxpp42GpO4seDAOJh;|uhT-NTmAdmJbuFM2M;7DN%yb^XVF_|hL}ue~w8DWv z2&%4HE0(hXObmRNC*?Tuu&sg z-ga~Q`@JSxS%GlSgF6&j_U}sDCj+h^-2< zre0Aal*gPNU6~5;;Yn)4YNKDTEZMz>tYC+!R2~R{LDVr7h7a+@0m%?}ww72_Kr zh%V?*_e5+=sf16J5eO*|DK{A4o2=&(mlZYb5F;E-im=!<;NQH^8=7D7T ze=ks-|AFC5qM@9B0ySX;sDA+Xn5>g#6&d0HExXc|={2>{gE=fuS5}j*t&%_dC_cMI zruS0sP0QQ3wIbeszdJYl>=xQC(}~}vC-;$_Srf0U{JBRmOIRD1ZoyC8zUSPw0&$ZTcj451=>x5L;5Dd#ANId=R9LV?T6 zhHip*;S+A7hn3TS=bmJ?8&s(a8HRvamNkqq{7+-$?HT5w4lJIIflT&4`?3$c8)IWfk`gdv{ z9cdS(Kn;g634e54pa!fIx@f!KjHa8ID7UlwZ=YIbk1iH8>9^fF-CB1(DJ%Vmck9WU zxJ1gtNM+A;)BY+R@uu4=y$@$sb^CGp<*z3`PD|Fu9=$gcU-G!y$4920JL#Wkaf-=N z4j^Z?ozvXQ<04VQn%ngpmzO!CsY~t4iooW;aFz3r3V)|fTc@NhDou<=^&maL8Xxy% zY#Rddnu{`u7`pj5Oy`i-Vs)Qg=TwaFNWLM+FbYf*9gnH4j6p0b1fmiFHjj)X4Aec? zshhhvv#+J(WLeDK=8;)a5Q&GXkU&Kc2ybqHB9M-Q(&Qwp3mV0j>4Q%Y%)T(X_)C}u z!)y8)Vt-)4sj{)+j@zLY^61LvAnAXIkDbka$H$2u^>!RSW1nSIOC)2D z$P2P0CI60Z>XksXAX0uMkS>sj`xqwBd8jfZ)%+9CO@Dp_N`F4r_RyV~?(=14TLG*s zsrQwidVd()==$rEDl) zQ~AgYt^fgf9U^6|;nM^rz`iwU=rbQl*9R)=Y*r@}`M{v{Y z;(tO!(VO;HlgCuaCNCzW1+*$G?3Q>tlC+PKvDe zagyd%NRyos45Rejn%vLB`Zfo(w?anXD9|>DLp!e`YkyUt1Q=mW{EiECBgjaY0gSO-mgow0)Sk^Y8lXRP~oz)EkD<-9jh}$!dXxSa`X>NOA;9gjSe-C1I*q=;KA} z;<>6^B25O;JTFwXrob7OTBT&bMbZHTsrIk-O{?-EcJh(aK&1vi>pYtxWF@QsGzuEVoPQZQ4!e&O zVQHT5oF*vb)8mzW(p^UaN$+89cOv)f)vQ?P^|!Qxbv~hSF@g5=G0W*-0dkI(uZQ7%Ht5a`>2d-K zY7>_F+&tvG^JC5PtoQlN_^-9=;$hZf4{aA^+*+brJAHzC>1i66dw=P7plW#YsFYO#!Q zoTGeUZ_XBSA5%Ur^nU>`YY}ip8#CZVW+Pc2Py=%)#F63Kc~1C^h3pE4Jxz$3;@nkS ziiG2vvhqEPDth)GLU9}`OT`q!ZECfZ={J<=8+mENy8MOEdNjvbYLU3|!dsUYc059E z;tJpZSOA{KIfFhgWMMZCji*K|iYI61Wbp+ z7$e~YKy|$LiKp>Kn|))AC$=`R9|@h;oWIZW(}qWWnN9rsxnKYAy)2ItZR3C9T^A$! zy{viem7Jh3n}7Pq{&OTXx?JcLSX~b@n4_A4u6NBJpPlbcQ&5>W!rwso@c$w z5j)&~9(wP;@zhTqGjfX*RgJmwj5r`NC7^h0H&uBcwRAA8;^=Lj#Ln2b^L2I{w ztI@;@nt!LcQ;xVr;;VJuyH9)y+Fs_;t90Imt~}{+P+a`sYqAOJFk-6n+%YXRKhgRA zHP5>^*FMkv)8Fa;DKMNNJU0AD`eh|jhejr1`Rss2-C)ct*sLf>FUO(jf}#j=n0p~# z;>{qW#~Oo3ZILqyO4|%~k;o`WmfFCryBx7YFn_gtkv5t?oK>+Y57l1^DCFhI(tTu_DO%`kZanq@$=ktxsDZxRT}Ky1-{MW z!aRVA{UucMd1+c?GCd>{8Dm+kLn*xw=zNi6j77684QduBa}X9WswK0nV(LS2yZe%Q z^M67Zc0k9jh#mW=#*K&6h` z7-uU78@IQ)4YQ@|R1s=wpP<`IzRuoUpy|17#P}GW0x~;zMJBzRqO6#``M?l4@z68O z&<+6;RWB^f4@u-{T=gdGnZ|dH(){Ty#H!;@!(TYFUjwP}O!nt|e`C+8Zdl7w5`WSR zi&;JY510<(gz`rf+kv`c;k`KzlR^0XHH&QM$1`kA7(hRsdB^A0;=hk4HJe2{7d{c=f_p*4 zhkIPlu`nGFT=T^Y9_igKI6)dG+ka&Eq4*vurTJ+4)2q%pL|wFe-45@!5JG`0h}{LK;51bP3*}=GW+%ZD ztdxt`gX8*>AW`Xc1+WdW6t>%echKv~d?Y$t8$Eqm&VpxL3;Q$ALb|qc;eRAe79(Mu zVUlaS&@fOE6C7+P>1|;0ld_Lj{?}?Norq?d$J6 z(r?qFTVz$gcWG-$_*TF-o%3+&xu6i-g^@H!)?VGArg2SMe+@a1KAZ4xRBL4ZZ; z5B3sFYYm`%DxASX3YQ4w!hiI!>*5@x0K$2Ps{*cIsN>v{q$yGzKVSTfSD!pY?*o4P zfLngYK#kjeutM_0dZe4f4-VK?OPpb{y0}Hj4x*F?TWP+9VkAUlfza{aqNSOFG<@=) zg%vUc`7H1253{3N%+TzSNh&%nM9yQVS(xRn;PRrqQA?sW4z6x8I_Ku{Hr+b1sVW!597&d4 zy_eHNpQjv8syrK!^y8B#`nrEmVi^_+Azcf~(;GidM?})wa&2fe)7=3xK!PDu%geH4 zJ1aD{PFsE4yl{CrUEVo)_iLF(PS6*Syasbr%m=g7*U01iy#1W`b5lW zJ>F4i+~-hp>ua{MB)!d(;N%mAH-hVY3BgHGBpAjbdOHHe9p@+M)+y!0R}?Ql4Rn~i zzJUM&SkonQrLUKE1YCcBaKP8>YaYghEhy>gZqd^xfAd#8zW9^-={Yyl(lr3L;H4FD zeV*1n&d#i%tzHLr;GA%h7(_B;r70riOfBW@dQ{PHuu`bTRfR^=EIx8KMfde~bu3bK z+`$uBg~Z1{C85HAQ2T+ngeZOw3QSuZ4*5Oj2z5iD7?9XE0L~cX=JvICwsQ zYXwW4Key>Kultk3>UtH=yWB46=G4cX-+6=h;nv%y?f>QtdL9wex79~vFy97^T}6n1 zS|Cz{ut|}k_7cnUf;F~ZZtOBpY#qTnQv3tdp04W8%Sh*(=_0Bhf>DTvIxm9)@}i;` zG^Hl$af)Ob$IgFXQdt26h|OC%N5C+G={m4Jj#Vz!c2~F5`^p==uNG*Mv>4ezDnMwz zsQ2q1d#E|dHg2hiPwc@1{>{<$e8+h@P;^Ez9)h_K&vgFsxxYaKEHpODRX&<_fMW~?CbG_ zeAMqlR#IJGdhP*lbnPcgSkynmg05Djv3t+M>l~$tQA~^*l;!~PIHO@EgRy^L`7FSt zi99ublc(keCj&b`H84$nAJkeav zM-Vn=6!V>}@LJG4L?*)}|6;ZUPIUW#)*Qg2es$7}Kx^VTF)0i=Mo`@r?QD*h8FpJ4 z5AQNw;0(XD<})u!NhG`}SWJ)82cgFqfAhLejNE_uvDNduto=59SURt0;*R&|>fU{f z=<^7f#{Ics{doyHd($p}PNvnc4jhq8xt=h}n(T)-h~T>+#0jc*Xa4;m0QXeaiH> zy0MhOa-fj#vorH84<9TTJ+dyIutyhHz9x%h$0wJXZ&+%7<8?TB=65`CBb*=KS;l|n z-~IdB7W$k_^JQ#hZOwPws5n5JLTW7Smx^;R7Tw++DMI*jAX`Xy#U5D^zk^JGRE~f3 zMoa>NqUG0`F5zvm4C)ic^y)q6_qikqc%MM|e?6DK*P!DCe#Xpney!ivKC^HnJ3z@3 z_W0q#9)6hOjz`ekm$1e6KJ7q7XSa@tt?0#bE!=oE4k5IqiP89CY*md=aBDdepL11* z@6&ioLQz7-kcHBZ9sFMN*8=K+|6_lbmhv#(SmMgiGuC4eSwJ5V?(*ZOKa$G(1l0K> z*45_}CE6zcX{R*$$(JWwdBhn`NN01KhqAkLn#OT|bx%rb_(SpZ4)MX7>%G z-=9kzz22nrFYeEJ_v;m~wyVRetfs>ZnwQ9*wpaexrQqMS@iP>>paE?yL0^B!r@6qZ z<_4m+FuHFt!rxZGjUh={cXkf1D=PH&oxwoG%nDLA5I6j5Ev?1vbugTSep}WSw6AY% zX=?w8EJYJIjE_i22FLkeHhfi`opHZCr>FpGV6L zP;q8FwiAnF(;(T5_lZmCn0e}qWqrP4{g(j0am_sF@G9scXE9>I%mdA9{_`C7y#4RI zPsi7UcG{L6mv-?<<5t<7)t&aWJAXS4nwPl?^?|Qu@0;Z9Hdpx~G%kttFc(DU$3H%h z*^aHG-NyG22@Bro08W1L^^kZYUW3=pz^8TX=lV0o~ik}SfTe$zhiBXpU-Tq zV{jw3vP4(&`;Mt+4*%Md@iTw>Ra5SGcb!)lzU0q1+~<2%lRf%4-tG0;`h)Ap567o} zR`Lj|47*WX2Vqxh)4p`c3iI^6r>;ACSS988iwGbv)lS@2mYjbbL-AE}dTYEEwk)h# zq^N^iQ7lU~2Vr^(C3vO`>EMbZna(|Ige0eJ5;m4yWt0d(G*+hWgIMSoh|&FqC0*8O z>baKdZd-0SaAUTV9#r+w4h%!oet8JFg`^>DWDbd>KCX%t4<9hh`Oa=KPEf4#wzJlc ztfclsGl(<~6(@gaE^GWB;`MJn(6RC-PX4z0-ex+kyyJrVx?#63ea^tO|I{{ig*@(U zEb429MZVaay7=9aF}87lIkKD?LeeEfX;LT+IT9PqA0XW2a3fcV2! zsX&JrETx;;%Nasqy0%IPZ1~QbSeG{?8kZX))P7Fu5dYLv{%{68Z>6*zW&!w9i1*F}OS z76=E#rHX$`HglVqbp{_2dBfy!AL7t$1C?Q4>bxw*AQ_{1Opyvq-lj&F*O)09ZjCB; z$=ybRexX2adR+MNur)U*#UdP~%{-sov0?=T?mOz~ zR>J0ZXd}Ft6B+PaBs#bGI0zekafIaN#0u&o;k=EbF1BG;QHzAM*vPcUvBTjA3`66> zj}8~A4i}>Dxb?f<`P4}K#z;L+_8k*KK7z7dyh_vE0dnE(;H}}bvs)L_#&3ovqSF%Zm%$UqY=*;%gc`^6>&TyamLKc_9j}*_ zy%vlAjlVnlOeg+>kKhQ&Zp=Bh7uF7$3`N|=XnM<6d``{xjLK2E9s?(YCF_yI*B*Dx z2Qjm>d_{a!$nVzkbwAE;*`7GN4j-U$I1qm!*YPQau8!Q{0|@@ke?Xbo66p_r(BXkD z4>vJ;`V=4kdk!p*Bfr(p|Fs8&74QcwSh*y@X7|8UV zIdDf(&!cv=9srB@utfKE4YyEU_=beW{oI-ZOO7(!c(`gRe;$qxu$dm=;~QVUieP^_ zU4pDFkiv(?nnJi7Ox9HZO6oIdVd*Z*RV^?4Tq?H_d72z9LeL#zD*6P<1;^2;n|sK<>KKw`Bu=ItMj#@D=NAh6bf zuxFhF2shY|7B3MgDdr%pH$?q~eO5jQ z`*V9qF$yvP!g*^XMYze9Ho1RSAS51sRu}`bY(y!PHv}TxR5E4I`4J35l_FQ`;vm!Z zhB|v|vf!EjB+^Uj?Xo5>WHPHt<8Qr^cf5K0dTg4<0No@C=014K(0vs_r@ z5$Db$#_8yRmMCUO(1nYHi`8rId3mPVu-}bX9tjRLxg>>4D*!> zX?rofEk$O-he^@$we;4bTAWzM+FND=MRScy0y-FeR1;4fTGYm$9=fx*?>0IQhc%i= z$9nRV?^e$X6kUI=*Kv4VT>o7^`>g|O`LpLnpKbO?k!!Fow`7q6)vF_7W}%oMC89LW zx62g_#AN$$V>l5J3rMF?LYs)FJtF#0stV&$yn0bH+_ZOVgK6c4u%VK5_U3*KU*JUX zUBVTJq3vxt4}fTUgQ%-%1+=R>@x;P>p}7J-ihpb6$KQW7nI6OP(ht6KC0(;J@sq2X zKr%zC(arAHo62=GV<0XNy27+nCQ?JW_9&ip-;ZWZ@VLEO%Agdp9m;)sE;D?Ug6h1y zRdL`nGmXhtmduX(mJY1Kw-b3oZt`~M_U8(Xg90q@t>HoKK$NF#6Q{}EAl7HQT>4}e}iSXIgzP_4Ph#=iiOTuoQHc^H4yD% zcVGu%>qGB9yB4@i;IA%k-?mi!N4|9Aw}0vQb=`lp_aTq(8fN^%Px>%=-3ikTVqfdU z#I7>24e5)G(t1(I1w1MP3G1=5(k=!IV!kd^{jyXkf(L9^K0CR=Y)2z@Mo~3he!Xhl zBL(z2^cDMrEQ3(?eIA+ACBMr@njC{Dz9R}kYtDL({P9~)lKrij(!6GyI-PE$F`b{f zQCEK_?e^upv4#wGsKWeFxd!bNIDo0_7ouFovHs}BkXRE(jh&CFn*s7WJq+~?Q|GPjBbMkP)lvdU7n2Wqj# zb?rjH5|73nNPb`_`(lV~S!B)Ek2L&HH0 z)F#AS+mAH1f^1H7y$@^8=6>mTxPad}POW3(N#6(Z-A9-uJX;D7QnnVTfawtdZvIED(Hcw?4y?p+HoN-D$SKag0$!efTh0y`?VC zDdt4Zi&S>FF=x_Y$Q;bo@pISbX^2R=(7vJf1D$621lLTz%(WlTx{|&R@$dde)MgX@ zkJ`eIo~upRPah<@Y*#WJU}(-eVYq*Rpu`R;4kz|UUtltdmeW~>*)NL(fwGqp@bL9K zTmTK9H5-SkF9iuh)MxRNfd! z*DsR&^fpF>%W)^+d7f7uIq z+RI&^1FOS@*|Q&|kCT7deM5gmS1)bjD3*bCW<+erB6`J_wp@#|`<9 zHR)^&eM))t%2^C0nAkkAZtQi)&;#C=#mwEEjBd_HS6DRaaIYK=Z`hFz)qNFShkNN7 ztAF)X{6lM*jjhMCf9`)9U-orD!=q^;jgJt@e`YFhK{Y zYJpXa^k1jX!}^%4W(f|{*Tbk92I7X2kR>#_d!jW1RQ;Ct6S1F zjtbM{>94wTZ$;mI%-{VW|JB~CUKf&SuAz^|xIgP4bCTG>SWtie7LdyQmrbo_J7%PJ z4C&=ihy->uMYytD=hYJw3iOe7YyEbJEgDTd`*Cw-A~OQqkW}e7u+#I%^*kI2_pTp3 zn60Ru$0N4?VBK``Q?Dgdt-faVQ-{*IldePcT=%aYswQ1cPvVnA8{*K6Ly*?*E!TZTE|$FG96|S~K}LK$ zI&zg9(JJ1;HRo<{TVYvuoHG(4wuGQanzMOno+zX)0i#^Xco9zUwTa)AuIp}FTmJA9NzW6f>t9&( zrZ1Cdneb~pq$1o2(j~3 zTrtd!=n;>w@)LSlxV*{MtIof!hXdLB*JY<8s-J(jPh+8V*rl%9pj)DRyVuddb)`UyM$OBi!CaWN}~jf1$_B z@6P(%9-)SE1nHFl;{v#4=cvokLA+1yeXeLb&1}vem4VF=Fqj<=XMYXfpc3diep}uEB_3EOkt-q4yZAU@7W{nY|~xydoE<%#g6l)h4&bD+-}Ef4QKF((HEdlkK2;J z#{CCh>2WVm$ZS@yuHR-Z*_p^LH!q%Z`6qw>JMOsff4t9h-KgY!zqfD|Nn+a;W`Y&9 z1j=vfonsoT=b+YXGe|XpCiWR%bDxMx%g|Lb(tiRv+$LW0+%4Q|f4E6X8qeXu` z66fm44|jOMgPqc3qkU7ejvwZ~u&35%{0EM><1hZ^ho73wzp<R37+{)s_1{pAzHJd=X)3~z@>V=zUs~|*YmX+tyfO}<6K$Z z;iiAj^-r()=;zLu(2wmVowy)l8XXO7*Bg80<3O&rf!zB99FkZLhK;?o6>^CxT&lLH z;CnlaNnodmExAFkag@uaw(2uBmJ`vvo_;DK8Y2+GOq_yCTSy&8Gzdqnv}b=>SF8Co z)bY$ZKZ-|xY0}$o{mYX#7pdkseqpDsZuQ->6|)CdgcCq;Lr^b>*UFU}3I_$V0|buK zDBXjFfj-AxUk86cy!@+=qvz7TYrA>8{??ahZmz?HzwX1BD>egtCqVS0 z+dOr5kd%GDj!(pR32fxIDf*D?9itmXDhtv^YMuRDgp7lroE5A>+Rs*EOhx7(RN!Ti za*$e=!gvsx0g`Djuj2l_Ak%&H7$F`aMJ+CF#l{4C8L|Kb4QNr9=xculC@_y8sY7#4 zp^&L3J{;1_wsyc42n52IeSoBlNr|AS3*C@_zzpr6mc-nj^}5MFyuPQ`4fVYKTI(ih z06pe>H9b#O7$cG3Pr9*%vxEJOT0wWefvxKm1DJnmF)zCEYp2uv;s3I=1AJwnxUa?j zslWF;Mj&F8D{P7ESAT!`Hz%Ktf9y-2p)l+93_1L%@mD6XQ#$lS-Ss&T8h=ue+BBS6 zx`(}c5643p7!*_%rg^tFwu@OofFgojEoabsGoms>@TsteZeO{YNjtAXA_WB@3s2vp zgPHD2{m|=)KrCopZ5=gma8acmq?F|P`>DbY_beeTy}aKqRvLdc2(Xj6B#+*~ZT~Mx#n%VmAuj`J_-h2&BXefRNRDyle_by#Ff%$#! z(#Vf`9YlAZQY3$VaJ`O~&;EP;{T^HP8lh*;W;Xt##m)M=mVW1B>bG7f{niDo{`ACj zb9YexJ8X;a@qI+?II1KCLA+8@r1gqeNC-1xO`6ba`HCrCTS2yQ9`P1QYC#T@Wj^zr zKp?MB2#X3M!tWM}F;(g7{87EnrLM(fdjx`zF$|G3vPFM>yEv*IQk8Bn5_2Nt0~J{V ziqB8d)Vc}>9w|mrSVxgN-JMPkF^HOaVK>=!uDRVHrMy&|EYWe8Vl3hcLE^_kejqtp zy3K8lAhrn!d<=As!I+~W0g#opBMdfROXp3BHLG1u zSb7>hW|Dsu<_v==ci|hJ7qiNF7`TYV=RD*u7BCzY>~mhQY7Xq)-&X;!6y5;VTpw|* zEBxjMa;9;04u4`=NRRrn6VSaP^h`T#O182@hKHYDgO?^lnP5Ir>%p+Qiy~1OP|xZx z1%nz`3L6uD$!?~)j`h1H0kyO~J13d+YtU#w3=;utD|{&zZ_bT|eMc7dFW& zXRhiR#63P0b-R!=3i)7*8#Y=Jt>YC0nmt7NeL-6YXG`30qW8ehW*axWBi7kMtdE zU2jsLHAvlS-~Ny1)eG1^;0BwAtE`WdnYjVf^!lu4ClbuI|4(_d;Q*Fi(w2X1W80-HBW=|1pI7gPM=D8B22ZHJ|G|7Vl^OM~a?c+uSr(BVw` zr}z1ho@gshu7Y28Tx_pgQq0KhzzBSPocA(N`c6XXlcxUUsxd7Ycod`a`RIQ#Dj2cz z8w&9qQ%5LE7@-4Rc9v|f??xEjQe;2w^c1lL#_ke=tP#~_wM_z%Gzvk$*IsVY2y$_1 zDk&2)v5nap3HISh&!D17+@uMU6CB?T0@3AbzFO7gFt0HvWAz^Ttk0_^!+~N3TyI(j zYTq8P==v1?(hdK|J5HQl2l{{O9+W>dj6XW!KXKvNPulg51vJz=LRz?&<5BZdK^)uE zK60);=eA<0RhS+|ZjIJ%<|AXxf+}T*I`f7x2ge5{Ya0m>CJ*_UK&CkdmqML%c}NU{ zWhDbnB%&k3rG-(lCbN*Z*yX4>GNg2oO4>mdBO=y*HC;PFJSTalWpjVF;p`sB!@yj= zRyv2(TCBeA%-=y@p907Q>jwC`Uq>`R44!>~q~p9MzwY=dyE_BRe`yA^eVzZpo~`C5 z3*qidxWyr3;ASWU_sv6}SYKK9FlxY0#X<-<=Z-5^~v zsF<5qnj`R*Dt(^gfrM7!M8F)7)V#EJf6yHjx{k)R4&#rF>9v26@w3;Qe&1bX{kf~4 zi_d69)OAj(q5DuT!>`%ef*5U-b<(kKShDFntlk!n+j*&+FRxVHbIZ@#>tdnok7vr?d`<|iv2dTh4Yr|nppSUzb+bypv)#>GTJOhE=IC?V@GSr&?+pyUfxKB zK3C1mg(QCa(JO>0Qc~l;ZHwen_tyT4Pyepblzuq<&ozHl&Hep{Ix1Mp;hP8)TqOj6 zI-D69nZ9CAuWR4q_mbIy3*i#_&Q#ucU7+5DQ|E1pBBDe{Zx*?;lKb0N3B zNlLYyZOyF*nduu32{CG$!>QL_pIm+c)&+U5c zKf=I3T-qDzS;=O-8jt?@WJ9xFZ@MoHAL5s;-MJalj<0sr#A(+ux$-LRwKgwuB(N{p z_J)Y-%xBj2@#DqF1?Az}e>k6>!XzPV_?+7E7&*r0j_ZUx&rjk~oHBCQeNO3fZ>n== zIvjuOxXdU^o`bN_ee+#To_+lH!QAznDpxjk2>9ruAb`lX|{n@=V{Sk zh>mz(Swy1LQkamv_FiD8&qQ$~M`R>EQrN8*(CV&H2X)tW<6ruC9OLU0$U%u*8h-Qn zPvDFo*Hq&cMqDVg_3a%~ihh2=adp zlHN4TtQN0I&alZzq9znze5bGJci72%?xU7Nqd~end&Mc`m&ydm6K>QlEspfH&?nAA z3&JY%XyWijM~m!XBgwGAFIEj7lC*gGbIU=#!~5YHi5q;`9P?pqVBdGuSkgiW|DqtY}Fu+R+3KN`(B6YrY8t` z$GH~TjgtkHZYd(|K*DWSCVGN;c#2=3P}Il#nKb^x;xjbnRB?LK6TE6$U2|yC-b2k@ z)b3|o^JH55f05IoZVXTl+WJAK2YSutJIw)saX8lM7C)T3rl;K2-7tRqJo;Fy@Rb+l zrO-MiaAdGG9Z|x{E&r%T)du}U9t;tg&^iQNF{m=fsqXmlUM zXPC^0>hxpEKXid~zWC>Bua6ex6%-s}rUxEJ^7r4DEPDwn0bQ3^dkHpwGcKp>ZXI1Z zUgw;gxpL3hC{Qp8ibhKo<6ErN5v}CPMoxsb&Y%;&zn}Fo2_Z=EwQ$LXf-|DOM@*Lh&t7iRZ;uu3*PF=iwaQ7ho@eDU?|v{ql#2= zF%Yh^7_Y=xtTwgb_77&Sj?G>zMHr_?!j7L8FQ#-}Col2XwbT7B&9$8Gj)YH^LJ^&- zsvPk3dh3gFg7tgsmteq2`x5n4RI5ku2M+{m96#$2G53G+JeqlbaIn_}p%LZj*e8P% zF~PT(;!8_qE|D_@>1o1`(Yn!6scIjpH-AzZ*q36k8{14J7^C=fHrztT-?;cEdLAFK zU&Rm^4=|a+yN~#)64()SQxJQ(AX>Ef(M77JC*E*WTHJtuyzT20&H0LTmb9taNPq8M z+8%UM&n;ne<59#nzD+Kf3rBDOG2k-st(iHrh}by8V-FJ1n|(j_-X!+XIiuRosj|0iTy}eF+)?eV2=U2_yk)m#}>a9|7-|(0vIY0rQvf zeF-2n{l6GIBA$TK?38Mg-1s-R?n>J|=rvOmm$POIJ=~bFS~^l~xd7%Lm>^GluS)5|9u5ix1Vzm$rTh zEdkk=b=)ZEqn?1EAgfO@I&!MigsMCQnVMdy-ifrG zZYl>H_tJTxEOg`$HTlJxisf)H%46z{$9ubivPaXKSGVJ;UBf|hI4y=k6&zS zQSft4U+Y@~MFiHjrSOOnHjDYemMLRAeDi2cQ)@X02F3)WKM6{8vZK6aSZl}Ui%lw& zb$Y?)k;%@1T=d_BAhq5R$tN(ZPoSMCARa4(3)Km0Fi2D(+`V+U< zmdul8u9we%2`U)xaSPAiHt(=p)Rp~It!`n*-;4h6=V|Ac5rGLOe{=i=SdM+SH%A$? z;V<2^^lxjlJI@+C#iG?jPyRMu9mX!Rt#(}#FBV%X6l<}QDbv-A5x$;4Q+r%lW%dp7 zbQX0IIgy)Ka_lm#LnqRWKCNA3`-!$+-56X}`S0tUMBKcjT2HM+9jVrY#s2nx_<76M z!smU}INsOiJvi#re>^YW$}2CRm&Jo9@WM~l8iR> z@glE==l|fV1Y#U9cQz&CYH`tmYQrI_`b<@Yn&GaW{0Cnp+35ebN<|G{Ueqqzk0x2=XOhqU-A0~l;X#`hnYOMf3zC3ikPKMgv09NyQ309 zGxmpw3(pTFbFVn&ur^+V78dd-8Q#k(JNakWi1O%g3YQ6FsC(gC5v~P4 zxxMF?euD`cfA1mV2S;nn4+smBVp@v(_q;l1|DDwQbsCO4?3Ix={iMc6BDOjoi7lEY z(T}|L$as$0FP#s@=r6UJ-T&_OrRUZrr?I?h11qZ@lSld&4dYjJaFlcdq5=_Wmz`nZ z43f=6DRR>ZF8s{xUU-gdhxhN-*78k^C-t?~Of7=;GyLxN0K6w*QylD3#85v7T zmEuz^Spjn9DM|f{D5%m={|Gh@6$34Nc;SvvCj#I+n@W)W^Oz2rx(TpbPx2a<{HpJ^ z{zvWleN8PQd`ljsTT7o)b7wXBy12H6rw{Z}J611TX~flpv=&nadFY0I#K1(o$thwc zxN3Qwe-mu69|iiyA-*5+;G>3bS_-VMT|LLHSaJB)cgkPSXnMB1miCoAyj3?G>J1UCKJTKOJi;kb4Fw? z3DWs$5r&0gT#y+MvcOzwoO=oUiPrzq=TbB3s$D^dTJioo<(j zx?SIjFmMG{l^fl>jxV~f3iMpKkMCRjrTEcmIo#>~wm7$2pW~OWRs4qz`t|3B`(E8I ze=V)6?op@j6U5D4Tv&C(p~gG!+8?>q>Vi0a1Zh|V#w<$AlcYk>!tYx}!Vs1_QPN?M ztrD43^za5&ebTfx3m>6>N_a~yXGCR;S?38;qL>U!t}n~?iGHh^I7%H8OIH-6eu;>% zL;;asnlmr<*M7M!(>kdamt^hK#tikTf550Gj2IW>6j-G{fsj`dB?At|X-fPRd>90z z`m!&uBvKNhlL_MdK*_|GNlt*jbD}jRYu8tL#-tvGS`t$8m_SfQQoT$suFJvcomocc zMWmL>8f_E?W8c;yoEAzl^bIPGpR+R0n5rzsdBxC*tHCj%Cb@9zqRJgL#__#ae*5g3A;xPq{_QEmbjR~4uKjtFb@|bq=38zH-Ra(<*G*Fsx%ndu ztxK0v>lPyebI$43kf8D~pf%#7vA<+lVfL6P?w}SAXw>T#Vpb8Doc$B8fDoBe#K1v) z+^{;Eal%Jw=~@<1u@#=SbcpwbVO>{ zx#Xl`hW8s^5y&kl@)aw5F-d}nze+9J7*}5TsY?A>FN<*#heWgS7kp;qB8fzPH_)Wz z%m1#8{`kH9c@~wqao5`(-t-Wz@pvc~cXgRN-x!Qba4&b7VO!q*ez1KvK1BSe+C?;x z93I4aZ|q(2x#2pW&S*8^p)Y1i6Gvs)l4m~Dzz8I+%Q8wW@|~tTm?KuOD1#LpKfdEV zXW&?u&xZ+Ae_QFj-v>5@i3376#ogf2#ayIWpUt>*Sfp0xgSHx=EiRLYUYI1e})$RnibqiYoh{{aSy%Tl+P*3 zJxjfwRHw&ut1{QvJ0vu1OGzu(DTf?Ub*4GV#SGIdJ5dYp@k{~1&@w_%K&Cr2KOFeV~d0JnX6DN{Q(>n9VaucKR_Zj(wBg} z!$z-oOgG9Kfm7jqIK1txI^+ID-t?+}^bT)&e{tU3v$a}N8Oq_xR=C;RIGM} zS($Xn#Te1rJUaw7MEg>PW9uOtA1Af)d7-xsWGg~ZCq*z`qYr|?XqX%weTpq*E`kd0 z3Sqb!Vz-=t5TU=9ve|c7!bJY*L*KcVt*Vh_m!reMa`};{R+n?q{ z9p_w0n}p$L4(4?3pw~slc%6Bq@E=`*y?ef7L@bE5R(a@8oA0{|O-suulXCHDVO}N( zD}p#Ia}NP<08jmy7B$uz(Q|>fSo`*l*ycUL@?oc8bmKmE>*N46Z?$V{-mV{ce-eo0 zOB^G`^$uDxqAHt&yT>HtsL?TD<+_}yBVk0$DR58P`~aEuS)DXEz%?IJ3O+}#DFhEE zj))P@ZppFFqQGUd$Fl#9dNbB5q^2K5*h|Clc5u9(alG+48vII>-1kh*zn%k3>lS0< zIoZYng&Qr2rR+5<%~-_woWO~?e}G;E!5|G=P6((3v90Mk^es{+bE>jFZ8JsO7{5f$f9|npw5;p>RGj)Lr_ySBiYm zacXc+hy0TN@~yn%ICT7$e9u%f-i3rCr6h}q_6glSl&&@>6cWx;>(Q3J8l)=c)?y&J zl2&{jXTb5*HrGOu*<;av7tL6gHjD{161(=!9Din$AHCQgzL?@;sQq@AeT)es8U13{ zhv)hH-}aTj_1DactC1UIR8a~s!@HNcj0r%0?l)5NySVF`Ti$6cUUe+5wYWMD+nk2C z^SvWaHYkru#qGsg$R%pCo~G_l^w6X7pKIg#-p?7XjrP43Q1u2`l#_wfGfQR`A;CEz zxyMp+=2}{$NM^{*j^tVQ->GK2+Oq%gy_#yMU7?NkKD?NG29p$ZOz>2?@ek_!7236b zb6>kU07ZOTa?1<4v(>o6*bcAQHdq<10}uJSD8B6DixnE@(km z0_0Me{n#chSBTrpim7b-&*)Q+`(_A#@X+iVmX{#!G-WOsY48|7ab4RJzQyyM9#)I@ zxyI?uzJYOO!^Y_EHTr)Tl0r@<15E-lc(Ria9Hu!Gna$+$bZMlNs4SvezaUyrOG%r` z%op0JCdKCYU}gkr9#DOBK!!7r)pAvwd>0hzi7i28!L;Oz%WM`69{ia$Jpzb-1@k%7 zJNJ4{3y{x^juGBBKj$0x=iCUQ`#wPzo3zh`vF$Z)zw2fA4WGI>^Uh1cK|4gxDmdi- zK8N#AGROtf=5*m?*-PjMY^RL<4 zs}WHXyHe;OWrP$qJtMvJF8A}Q%o&AzlI%lEs?6rGrpI7KgQ^XK_td%io6%uyplX{xr_gHLgasWrb|vL#DiY%IbTqs z8UZ+Rtv!*_-;(G*+*??Kud}OXwR!{KnchK**1HFua2+37Qs{=UP5V(j(fEs8bIYmMPxj6AUO|7(1;)X7&|gHO~T z3eLRr;*6|H{{-s)G^e#oDrME;W2*ez);{%ghsy~xpMkKEUkkv0JHNSB!)~UMYtQ`N zgWVxg9;|TDvo>Z7${B+QtWLz1I+J}) zKxKlG<0y)y0C|^BgcXQi`9zn)wchWY=CWybe2O-Q*e!agd;{++@bfK8CQ}N9x7EH4 zO?=8Hpq)<$E1wsCxdT!SBbKiyacNptugTO*>gA;i~ z9bU{4g3JNx+L27p%xZX zN<^;D+S-eMLHkAAv%-u$6#t8Lko*6|xR*(rYjP1(YGf-VFm~8+`GZlVXc}U;y~tAJI3@dP~Tbo9L)E=OLZ9WBdPRuWf&7_}8`8f^LIvoy~Ia*egxQ_WpJ)*)4u%|+ola%@w` z1(h-@Q;cunwSO7YBjVE=52K`XYkp{TOR%Pgb{C=^=9)ZU-CVE_f_LV?AY%nPn%9*Z2_K{54sm_^}+y&H>5Q&Ih z8ZI6sei!%lBf?B7TD;u-{^b9j(6s@7S1w2yy!l(~k5wLzbxkF5)p-E&xV0O9 zjVMl7aopfK!trpv$31ul>^IKvjGd{L55+^c+~&NS>+a5(5-Psnod5KqZuOe{{8KMz z`dcCXzdQTL3H`Md_V+NJNJLX_iS*f1I%sO1buUhe!~=y7xQYa;X}O z8|si?ZBbYBV{n|q!GD^-_w$EN-lf(%Dcx{dYAWz-)5clq@ysFP!j^^uDwB?fx=slv z5_2|Q7U^+DlyF!G--4VxTAb8>q=tZKVY9*B77D#_P`1|Rr17rt$?Q>l_CJl%RTXzj z#Gz(qyY&ry?SCiq2$MeW>N@Thzvz0UKl#l5cl{zi-{F9MeD<#KW$OnJc5D0&o6@ZR zz1ziG9Ph_YLy1c(Bj9s<4oLez0DBPG#F~p0YcL&VH6uRyv)Tk=n#nwWB14D%dY>|1 zU8ZBI8x0Pwt+Aa(Nl8HFLZ2oMJzD&+;% zF%!g=GN{TCRqvw+e!mw=aIH{mKq-rd8AlD3lNw%JI_9KQ1?>+WX?+?IRh=w-q6P9Q z!Re2Y!TFIP+pKEO&70SM)k4I#x}ZbUONsau=E8BpwT?kf zdLx+##5NIk8bRI*E6NWMujn3(5VTyT_IeW+Rce8D`wOtJF$t)T@GVHR+n2}2w-?6! z-k4T$Vj~_VYSx9oxoU8xS8t|YTGD^|KmFiWf8kUCHD7M`ANE9lMCV(M&ue`Jv0i{g zO_sosZ2ZO`5e&v>r_L22ZWYV%8@1O4kBTrc!M5!2V<1n~xMOa? z(S#?3+&p>!IY7q0iW{^v-)9xPG0;=%%)gy5a~V%D2AXa5#yRwYmjxi7CU8Wwev{iv z@5qyH>#UyLP_E}Br+A+`e}f?I@X-XeCgKRU?L3-1IP-z{(cDH}PDoT<+-Zg&7AS1a z1dKP9hjf7a_t)(KYjk{Lep|J$VD%APq>Lavp(!`kXGmZ3jt^bQ{z6AYwfZ2ameS(Z z;j90YQx20J@;)Nj;9h?CeuzOQoz7L)FTxxT;}fRj6^}Fe!E@g@rRiI6q?pfC>$MHOOGny@bhV7 zh2vG0z+s|f<1Ia@^AC?97EoGH=3l9R+>PXbsv~c?Ae^{ollIj(AjQ~WI^m$?_ukoy zx3j-M{amUIZb3F7e}wy_@hXt9FLqzqC^IH;5-@25EF?XPWryJSw#nG~QfdzUm*c`a z!%JFBj-2WR=CvT*^A^=<*rxvoQ}j1L+!%Vs#_l#REX*StZNPln)bGlC28tv%Y>+M~+EWXTyxU714 zK~0*2J0!K5#P&F$vv`1<{pXDod{={0B+KKlRP4u*r3GC zB{5Tb5uPpWVxBuotkmE{&e0uF#x;hlsGAhi6@Y9~z>6po-33MIaP`1&7{7 zPo}Kk;pE%Q`}@RYA=XBxhqg{qfdkF}))>O#e>0%}!(9)jIS;RzHJH~l{d?@^+Z`^X zy!CjR@fNK%=(W$&#!84&YsLankwRoVFrM4Y{-<(k&{gr1KKJ#2+WeZsd-t3aMPcVA0!qSD`5#F$w6H1U)_afsZ4 zf7GS7H|CsFiw6IP=0qEl3^htaG#b1^>$jIor>P+fw-mNTvNOmdvxea?8|2cOi)ZQh z`-j0QORaeZ>x#nRty&yl2HVrqYRQMZ|Fqg$%lY2Q+Ik+*q_3P8XnSyf|2)M#sMpnc zA1pXY%UMD$u5y;3+{dN$pqUm=)8Rf0e~LDLjlZ9Lc-AbCu)n(}6ZCY&`Y-DK|Hi3a z>)$yvXN{UZvx1ZUYtAU)A(7VmO7fkn5(hB?`w%=Lc^ul31ji~xo^%lU0g*oGbImCF zu;(AH9UkXISDXwp^7Kf*ulNicj=bEN zB5`y6-FrU%##?px)BGMU)2*F`hxap<^^FgYW3u~&SG{@)Hoxm-?VjHcD*{eBG`3Dv zKooKUst5DoGMbb}Ne1grQ9pRIstO@8-(f_iK>?muu0ttv63BrQskTFOD^1hSNpaF#7dspBuV4s{HGt=-54{Cc zl+_P01<_-bF@;NVwB99^<=C<2{L4~)xYCNw)PuzR5MZtEJ)vjW}g^Qi#LckzCrQ^+Wh6}Cn=Br8{LM#%9AXdwfl^8o!` zh)e`k-UKxExUnaLdQz4=R3E*O<}THu?N14(PZaIJm)TfISUSU+n{0EX*B&j8E+~~) z@}ofgf2z$`z^c(X^?{fGIg~sGSuFC9yE>gM>jLy?*)K^VBi6x_$)aQySbHd1gTJYrX{C z=g%R$VV4P0(jmL?luNz_hiG8k|~-6Wxjh#94IPFDh?+t;>8bB@b;?5qg;ip^_dg za;%*%`rlYL(t=9Sc+RkbyZz0>9zxF{O`x2A_BVo3Q0@<&%Im@I)Zk*QQXm#?&|+Ed z`V9@HO|Aj8wa?92^aX9txQSuONF1A5X!te?c^+*hkb#thf0oKcZP$2M!)v8XyyTG!M_gIz%5j|$iawMf#` zbenq8rOk61Pxv2Q%Ee8er_oBz>6R)Fznk?wLhb@-;=Wp>6a=BmuzKak=6k|UFVCT= zl{Q&C*kuWDf69j1VaAh7NYZ}KDs_CCjL0b_r0XY5q-(+{Z~E{ZzbL(GoZa}MKmHw8 zEiR+g)6H%??5iH{t>3rx&lj4liL9mpF$IMx2|!a;gisHOLP*%eq;|*_qMMeP3SAv0eWio z({F4~e}21LyF2I_tK8K;8n27JB&RT}G$Ao;s5&Z3Wx|r0Pps1$D>5~a39)3gq;kB1 zEqB@6opMMn^Q~4+F@B3Q;_@{oWSP5XUCB$GO?2W1Q65@Np3v|%p5bDUX9Yd5_|JH@ zns}(y)Aai&yZZfgIEV1wGud>S8bB?lgvpVAe>t*Xd!+uk?Xik9bD7Mi%Opo#i-!o3 z4?;}O*2WU+eT=1&VT}ruKa0jY&sav}eB?qOH>}X9t&Kr~m{64nhagWr-*}NZnsTmS zp3kjsL-&ePWRnLn2X0SL3$RXXuCf5}`_uFB2!+a+Jod9XPXdWQR}RDK?em#~?1d$`U5iG2SnuIqQ7rilmc#^vUR-r>TD%KxE-_oMlMwg2^; z%0U?x;7%?{IlCCIlkb^%fLYf4_h7eSYhF`#*S+ACJi)oU_GhAXj$0eu<-G zFXA9X?@Ew)KtKj7WBh7wm+M*LZ;2k*lQW;tI`Me=%p6h~3w}DfrMM&#b9?SEex_?te?| zgr|Yj*(8XLQx^UC{^O{9f41Rpifq~Tkxd-eA^}&nxXOl8tAFuZ?r?u5|Iq+%u#)9n z&+rly`CZTOHojLb2y>JfQh7S&e~0XI5}BXtgE&%Mg-l9|K*Fh9D%4gKB~?XEtZqxAC`Dh0 zv`849`1feSYvt1V;m}|%RkcV9b3#KE^F21IW%5EqtFwXUyPSlLm>IP7e~VxHrNM37 z*F>ZP@tr@--Px4>$G!5>ljpy1Efd4zP5-Q|Wtf09oT}>si|Jqalz3kB@q0TDsC(uB zeEP zb5lk@mUTAo$3VWZwxTlJe;bTwWpVYbu(pI+KTZAZIe(|XB`bvA>~ov-r&w?LI3f;Z z*v9#oCnnXkPjid&L+LcwUY+2YB;DP6nJRxcRpKIAoC9-09?;q2Q~%@0XefL*_=tJ( zxvZ!h?#7#Wu7u;q@8_y;9ievl=$f~?Q36fgyY(B5toCe5EOPUs z`hACOnmr*&q&VrnD2Ik(N+rg%l?(mH$Nq3rPDhrv7Fb;T|^}fRYPBy8Gvx{dMk$GAJQA z9ThyFa<$mco&usO7q#lJ*TzB-vf0zQ|8*ob^3$Ys;L>o(Te;|;44cZzg}vaQrO#{b z_rE8-N5xkV9<8WQ5e$Ol9eH$MxI|fa4n$4f@wQqW+>=Gg4gdU}zdrv@e)I{YyuX*z zr3pEI#-jiA(HkorVs1rMLQ{Qmsr-I^mx;3`JYb-#{lj^+Q<=-PjtIu+nQk73 zp#6|ZEy(gK-@j!x4Sk<0$@(9eOC_$6L+=xHQ?QOd>%Bp3&`34s-<>`qVoCmiBj3AG zcF*hSL3=mQzBFP|EEn-Y$80-VxsXMSoGcoD4YvI+K3s)}VX1^=etW0#^h$I>BgBms z^)Za~25VK(qmff}ibbM^Ytr~-i1zy`0>x(_sqrXeSD(lIRvpp*7x+!H;EM0=V&EVD zhbwNYiwD+Zakr!|(Yetx{#wqRfyYJ3p2~@yH2n>% zlcT^V$SAACeiDxJWRwR8w`s9RS(q7rJF8m{?@(*(>zulJ*8C6cJggO5>-Rr=zpi%o ztIzQ=J+tB0tF2Fj?Ur#~m*#tsN94seC`qDlf0{FyMP*vKpA)`-3VF&RC~f{WDNK8v zrc;jjHHy0GhVy3ey-VHFe08i~)M_kr^?gND|57mr^>nBKQnvW;;ButFR$Q)sV|vqh z`o;gwFWvJMSC25>)GF`p%jV~6^(_)&MgD0@zP@n0PajfPp=+<|EFgvF&gI@XVejJm zX`dF)Q9*?YLd1q;v3-_DpYaDrb`%3qL)e=HHj0s0kMW zvIdvBs0kuMHCdR~HQDs1`l9Ok%^GtDe9oqR^sU?+bzF`$sy)%azY>7tkMAw8PW=Qtf4RmDmtPzob*D9=_*H+s!Do>u zzhY#D!8zGC_}@BT+&J0FKak?uM_T`<`Jd1AW_&*`R|TI6!*#a6HIZCT z=exI+%rkNJC-3t^Xjjk2R|~w}`_@Ij&-W!v+%lPL9}NlRE;fpc2!^WRLQnVGXg9AG zwy|8jf9`hn);~G;IV-!WzCfAeFzDG@2Sq)ymzG83|tdPj=m6=|6(1J``#JG zf0l;i$K$KzsQ<@#-lnsay^7o@ppUOAl^e6h=Ugt6Ej~9>i+`VM#ZQyL(hGNQkMq~< z8@pV3zLat@SbO1zE1G&jqXYG8pKgJU|L1uxe>)!)yx(vxRmXEIJHIo7J|9VPp%3;O zdNT3VFydsePgiOQ%Ikl;P1EXevxOkbdDg8 zdq%3I^)QWTsV3EfKeuJ;F9uj2Lp-DgaNE0|-MqD$dV?5Fm4CVO)gmKlL|04-)9TMU zF~;Y+x3F4Tn%59>JNRi>Zrx`rj2%( z^iM@VZ*{k3JZ-$M%MAM$!wxq)X!NB>mR@@p&D^0rON$e~_Jdl#-Wvu-4ukWxUNAYj zDw~rs`ECeV2~K5TdFFIo4?^nT2!B`PxR|m#|cp< zMr$ECOV6iwtS3@wGiz(S6UAMEnEREo7b?N`wf;4m{u0JHbpRZoIrtL5jpddA`XPx$ zcg|F9>s8&{l{~)c(`1XFlE;`@}~I-i~-jw!E7M{x9?J zZ?*J)dtF6;siD98W!$yh|I7R^{twNYa1*g{c}33Z2czBO7yhJA!TCho3d4EWiUp6m+SA$JyS3xH z>auLk_L^JpWiI>nxT^^zAv~czC4_zSqQ~cMpuW)MDE<}W8gk7iu7A=N-;5VrEi#ak zFMCr@xXL;|Nymfy*u@FtM3^Q!3FUpAJotsN{atfxWAhAgDX7Odnt3ma8Ka-7Fb}p& z2$#l;jpE8RT;`5!9q8v1hq2GOdf$LrzYz85XvL`YVu($kIsk*rl@Zk8zj_OKwZ9Tl^f2nO}u|QmRDH|`XS)B<+P-j(=TB1h9q9>uEw(QRJC|^K< z-_PmwbkfwS$>yntxRj^yXnK*{U!D!D5AgokWM9%Tx;P*B#UJ`-e*f9+InmYKNt~6U z`MJP;Rul|Q^p|k52`qo-3w&Cj{*XR}Q_Z=zdCGGK7=&fejnnF9oN#R??Rr&kxW`xd zMa}~1b5cw0qgQu#Crae`$&7sKijgoAGkLo=r#3XTPTtnOGvwhDd-G&oa!nvNI$Y-X zlQ2zd4Xfr}K;H7zOZ?CLw#8i{&TxOrhw#1EzIX8I!OOcp)Z}#x=d3c|c zi`v@q^szMGp2AFbIH%`6#=jR0Pk7#!bF&GWf6ZU^<8#p9Asfxgg`*n3j>T;pS}h2~ zfpna>?BE`ZK}b$X#(c*v;W=WJkx1dsLSK0)yiY>k#r`ut8*d3cDtrRE=k6JvRK&d`(V3t-B4Ty2!sKI0 ze_ilO$YP6oVy@5gDw~n>TNua5eiG>^D~;3ay#8EncOlbj>W9YKyO=I?_=TnN$JkNz zWW$3ezj(`q{NeTfU`4(NlRaoPu7*~Ni8rn~3oULdZ?S(({lqq~cE%zYu0*y#p1O5h z9)l9Fzv~OOd@HmFTHX+915W5hYT<(Ke>L6%(t3Q8M^4_?CXpxg@|IZSozsi-7t>@v zT@}JZjVQi)vyv~=O^dv3w*OeDq&s4?L^Z{cW>Y&a&GPyM&S zn8>+<`$pit8Q{Jl4L0>2L)EPztOqpv!{7ea@c$q7SW2$ia(6g4NZbh2r%qB=e^YIR zHgbRK>HxKGg|>h2Dt{pkmH5kEvt)eQTHA+tbjhW~)H^Ja#|*}PI47G>CG(6OiKhj# zLD84gg&@vDBEt!VXF98;!xifIsfg6FFV$Fhk)$Lf$FNEdOvyusEBl5;6E`ifFM#LZ z-B_T;E3ON9=e-(ZpGuqsYK?c8f5~NUM)f$BnohSpmxla>pcbn>N*fZoCJCb#^b11> z?qJS9kh*w-Kvj!*N;2fGP$j9UqyT;5+u76O6p)*~3b%#zvBbDZ!9)MdNg*dYppEY` z_G!BB1eci-4L`f(-(TlOZrhqqzHf4O#SNje85GGmxa55(1!bs>F0J84e>aEm zhi|jDE3y}VxtjYiZ|YkOTNE0GlR((Ta?>FdxXbi;X6LWCMm!CUGU?9EQoY0y%S3Sy(|S^$S|^TeV_YJZ~jfE6ADo zQUL^$V+tz>M5V$9e_KB7N!*hSr)7VGBEMs9nEZ}Oe(=%2XgR;*#To8rZy)#vP0%^O zsbSlrS&{$6{_G7RfH;M^B%g!58lM=)uoZ}6?Y8>emNIK&U`D*hW{;Y z?<`oqVX;FQ0+i~Cy-~?|Vg$x06k)`H(&69j7lkywyio`lXya0Dp{}RU4zk&Y+(QOsJxA|oZfAAayDGfQi z4b#yg_r$tiZ7g{pUcub{d)Qzd4TC%XjqB)LVZw8m<^?wM_~ov+1t;oGuo3?a-F@TU zNE{%z6wP1b-uxD)C|h3k86&@9V8#Ul=qGNG(Ep4FfAlqGew|$fF_w8d6d-7Bs?`we z6BT3t`uKA$lX9hhfQQ3d!2gLE9I02l>IVFhtww-ZC~y{5T)Ewv<7UB45rUpJEcr$p z5LlsYOot^#<@*~s8V?ta*Z}KLLbCdy{WJJD5G;CI@H|5Dz3gZ2_%ASCvY)ugFZ`)5 zY@9#fe`x-8H`wnuP^6EM@4Orc5=2|Vi_7UP8MgtJBf}|8>JhK1U-6Wv1^+IVOxk_L zOhc6GF}be_SiC7_L}y>I)mHm9Mp+{F6v7OqxlO)zkV}rk;^N8_4?+cH{6SxSlaTqF za?r9yfhQnt{?fnxEj*on#?KtSBG1d3XThKOf8_f+|NS#2(?f@j)jXu3C1-XCo|u6^ zx$eZ)|KKg{^ouK($ifD+LlIg{rVhobqlgaw8GqzW(0TAOY&z2-SRXPHUGKrYcfn%I zzw-U;K!S`R%y?E?VtD=qJ6Pp-f;_>!wsO8qIkH9=umWMOoNv42FR*_8+%JA`_dfH+ ze<$wggP*96$)taL``^Ga=5vBc_=$%}Neq^?-2Z9u39t|Ul{w{z$-Qfqr}1MbgG&<# z4>H#N*KifT@8Nu5D176Ze7)LrM$&~l&m zx(1qqkFbn5J9Ht|jx&}gfGz)Xe-RzDQ zz!zT1Pu$lp+=8#QT@QY(CBEY~;orx-gEwvP!&e;SusQBv10}l^;JA6U?Nbv5f2^qT zcP!`mSbpN)rO)~&M7{mT*iJ9r@Cfn0#ZY2}gPUJ*owo`6A$ENDV_da@m;HX{7e4WV z6ZF~Jz`tS8|KKoveK#5PzdIB2cd?%+aRsATZMnl04g{AUlXJcqiDTuthkx?PZ?ZqO ztr)Q*yG>{DbPO)WxCza2&jwAt-;w>_ z^Y?e~KgLi43BG8ZC-8A3#%Ppv-qIxs>l{jDm&5^s!60j5o3l!RyG<<^mNVMqUkFql zEZ2)IaaUxdz@S)m1~w)Ke*PRlB6Bcs$ocL&=2sxupkUBgFeO|7Obe_wf4pMZ7buo| zJ~#tBm4ZA30A2}h`^AKBT{>CUJNcerfm}g4FCRczp)gT7{7BJ@*Erl6XiIA>7+p^B z^d8I%=j$Z2{KRFlc^-B(H|wWI1`1x-Soe&JdGBC9GwI0}rQ_Pl$Tyzzfa=VTmV1U~ z%2TCh+m0wWt$%~TB;3ltf6bz`&|7ESPHAC(2w|29qDqH5SRTbtj*i*GBFKk(yURtk zjSp~mNvKW6Y#t;Vr*6g=Qruj1dk3Uabd*oax6 zDqfA6$1vR)^`5-!so!fn=%k!AG(7bFw9B;~zx|2gbx03s!JTv`8E>=X=!G?1W;N5h zaOPZN(vO%n3x~v?uA4dRje201BC(Ba$xa1MP0896Zef__VaszBYhL8-P&Cl2+ue?9 z3v({_m!eP2Lwi8;f2f?4!-1LUe@@9nEL^MSPY2H=uDgWH$B`4FB41c`h$kxv9*Mf`#BK=er?inOOVfQIOs z2ap;de?{hLmDpcRh&=$PJRn&*wIDg+55Ei+dHK?UF0l`3AGIg;g)2hGgG>Q|*q3Wb zI$7!qKmNpcP#-?^p9Um+>|?F~7kF`MWc}2bp z3CmJqX`f2c=)Sl*=?4Y5E6QH(96k+T_whF+e+D9%H$YAFWdVuUU*L#*J|0h3;mN`=)ckgTbKs|S&ILn zFXjDR8c^VM0Mb%EK<@!u30)gh3cwBV4+!oSIDy4~N&P+Pzs8_ZK;(&im;Mxf^hf9v zf06e@^v{={{=W1_(*HNyPyLqW0PdInl$N%Xzxd0W@PE>#^=%J+=|77;zWeWAr@!R+ zX)pMZPyC(!Tl#(Y9uG(wk+h9T{oy~TkJyL*^w%Fc&&~ML9yBcD4bewPoJO{m^kRSL zv;dR-tILogaUr(@7zkbxAxp3 zo=OKuc9QiG@@XJ3r$&-1oe}Fl6 zzZGXirC%=RpiO|m#8D=I7qCwtM|QpB<8_JdPh|AM%Z}pLJhl|OEaimd|0ST2d_3Y7 z3gg7TORx1GpP7L$Ct;2z(tGgC>EtMGV$bu(h}vm@(LH zxCWR7*oEBF0ip~VEFcickjAnDe`=YN@p|G!&m6Mw!FLE`K zpI-h%nkct8-%r{o4{ae&!K}n(^>)sjay``H6up&Id0*wXgIXM(tqCwie;q!PxX6PB zpaQ5EyvXLchQWxrGcXQdi#e9WYChL=ShA-3Xn@Zf43b)mPDo#3b>dbOV-tkm>CRyv+_gCF47Rynv*VU!@8@fhZVR?zufcKav(@S99JdOa@2^_A zky(684=Ecn^c>sAe~xEnDi(}q$vobXd@Mu8z6I+vs^wGi)sO!FKXTweLudekAKHG2 zWxn44u(h2%5WV#yY&k1#Q5J<_i1W%1TOe0qT#^qEAY}9+%-T$j0SICMtjXY8GH&t) zwsK|w&NRp(thE>*4e6OY@m&k_Ru-KZjFu00g=?X~5j1@3f5jt#H&E+%oV5qYTH700 z1G-F}X(Nj?zmR<2=_oxa~`)K<7BzP`3pdG&kxWE1_D|Xh|m%l!3lg^1db5N8U)@tAAr-w zatuM>fBe|4YeCYspyJg1@{NoOdAVx=Zd*_mR<4sM=)$VS0n|#}0-^i1D0}rbW{&u@`}*2dD3<*2IkWLk%8KQ)&VdCB8NEXS>QC87K4+f@d%W`5Qze> z#lX^*`T=gik#koX`dWSmZfD<`c+G zf30drcpU@D6EpDnsQE%ZLeb^}#~HZ5E3Swc_-WKUKhwPR7tL-nLg$$eT|mCD~l_vg6Y}$UxKuYj=AH?8ff6-Uceks%d`m^tly0)a>@j#UEVpEOmQdWx} zhT3umNkMFD?EyHwNZf+(@n7h*fY3|d3pC)!w2bW-n50i}&eDgFEFxBfWYNo$Q(DL> z)kUuYu`3X=*0yAIk2G~`b7B)AC;IYUS`5yb#zVnryB6ZMwPatNm_c-=5sVA{f1+tT z|Dm^SYk3P@LemJdg`R2}FMa5nwzcqwK20OC7y3&>`Y-$Mff(|g7a$#LF96SWo5`f# zqM?;1H1OhAzl0HX)^;dQpl^XNE@O6u}qrfzJ&j^TEL7e?%X2LB7;jIXM*}WSzZSTx0`YL3xcv0pM3Fw=Z#0 zNLiVt@jymx*CM%XEgqBh$vn#OYM4}H_h+%$jybS*lxvWNyz1bCU70?2*lUI0T4^lHv`Ek{=De}`9#4rKfq zm+@-?|Jcuvk(o$e=5m=&u&e>oVocH}b6F#OIUg($MvJW&P=qdy7yEGXZ$8MIu=G(2 z_@L$ZXG?IxJU`ThGL|dCs}sPzle+(B`K?ywf=|1-#n-?11>rwy2O0DJQ9D0$5PbjG zvbjxRYCH{W?Yh_Z{lR?%NqD-qJe+M9HLC2{R8bY$h zAoHr9&kzAYhW!jtD2P>sKJ(^-%*`afBr|+r*Jywx8GHW9#}JvDzt=Im#lTYD1Hod1 zlIR!mMkX$4xXfuSQ2s*qd?Ayy1_}$i<~QPG4yZIl(g%nvA2|f#zam*L{*?{LcmkZZ zXhB5AcBt-wK*npbf3El|xygDj{H$|ugiNx|!x7Po-4?Q6{y`ox7M;*E0C*XM$w2Zl zA9}TnwqN$t55#Xy&Tl*#P|4UWBPh!Bb&)M}ao2L2miTH5lfrrdPKg0hypOF~0KEWh z@CoTeXXtmDtN}rZ$(;4g8&)mwDkom%r7a>K3Phujr!U_MfBy4+uH`zGxk)vVzKpY{ z>5p<-6X?Hek3$XMP`v!54uAO|1Z3RT$_w#xv4trY2q5;VDwMe~(q8-kZ0n`P5dN7@ zjy>T|eCQvQN9GBe%*Wnk5NIRxvd#*8j88(@w=RGJ8DDw5ZMo+4Ctlb&f9AE?mTO;m z4)E#IUx;r0f0ykczNWpvK*o6c%l4oC7k=42%~~)NR{rHDG0?23Ey$Rn<@)AV{bAc$ z4Ox%;R>ps=58L~H&{s(Rn2?&+WF2mEp=cE?@VPKp){wl#@bQus$olcG`%k#|X0*&N zC!}x0iN`V@7WST$O~z3n`qjRqZ7r}Qf5;k+YhSXn_~HP`oIU3sI+1L|B>lyg`RixhO4ix7=ke2SF!H8r`R0E?C+iph1!xY&MAAdsa^+Ktp|OmcrB^KD zbkh`}oBzn?=F@=4`hlm3Kb|yACflv#*;<~x<=NIW8GgNYmgmoV_rsr%XYa$mJo_L1 zmXquPb7*pG|LfMkj^nlk1-$m2N@r^IOiSPmV;TzhPeAo6C zzI5RO!oqjXrciLZLhqU_BT|3;={YlEbzA$p;`G_*E46^;)v94S{T`e=iKF zO)x;28OfQrcJNDxA#eYapYucjLa=OYs3dq4Ef`s^f;y7B8^}JzfKKF4r~%ZT&^s&= z&P_#2cP?YK4}YhK>dx5fef_pS9$;| zplyEA5P&N<4ZqUloR8i7O5;D$e>qDjf&W{Y+@pH{`A>gCi2Kcj*678W6+cOWB_(w_5CO1fe!F8DW@g%|Fe8PCVG=W zGQh_o4`~O-FIy@EPf;eI^O6N33^Q_0Bn9vdL`?MB|L92yi9UK_OpU~Oq4K|I&&5tW z5WO$Cl8;=1PW1aA?_&%LfAYUZJd`g#X|(y2&kZ0QAPb}fXc%_HJ}EimDK;Rq;D+$U z1Cc8MCrohK|M&aBU&!?$dJKsk#Xs|FKYjpKM4uV}D#>@TAWibg06Jg#S?re*JBmbJ zD1Ph+X(E?P>?YD6BJ%Sssm~+%0swF1zQoWX`rHt^k=;e$FM29Le?sCme)G#ue>xC9 zpotveq9>2Me~^AB6EHBb{PZ(%^O5T#&zhto@R4)zClP<-5p;6Tn7+t4z~N)pKh>-M zsoal$1WWrdoEN_){0MycA^&NQKrYw%2l636+QAV0Uo1lBL5Ru!OS@RYzxd6<_mRu8 zp#7DPCikQv{PB~Ne;l^v8iLRDCiR8G$B&jv8zi0B^CKq!J<(%=;ho?FxdRl*A@a*t zKTDQ=egN7)_;tefAbyzwI1|79^oQYh|9AtQ?Ef$4Gk(Q4omSjWeA8e4k>L7eADU>5 zjN`&GMhVz~C-{4i z-17>o7IJ{x^}2x^IaguxkjK0W$UPqx9{@XnB0v++bkJeMlEI|25CXako&z5NyMf|B z;yo6r(+f)^J7p#nij z?h0!_e^9~;7G5A2U~CCJ2o5j-!U)bXOa_4lXAh=WV8I!~t0nBfxrH|%9Km@5Zv$B& z@epH?Bb_m`L=l7u%z>zYuz~p?>L8?HfyEPufRqN?ht5H$gLEC&<)9Zj3t7LvxL~s8 z@{6u;mi0w14FtV3i-UFc#W$O-CHP`ir6Mo7f1=QZwVi<~dw>c)pl>`PW}U&cKO{`m z#>|HE#OwjqaK3z(N%PxJS@NKB@~6%M4EnIpC!GWIFS_?bH}DVrSAL!s2cbd6Z=+yc#dk|iSiiVPb`zez+aixYK2h-Nuev7P z4fT+eeO4X`({5#_vG%AHvYV~zim`_17iF!4JsxK#wxCj>ZhEN|K?|`Z=Wer_(P{n&)O=!^ZV%A-iD|D&}}ai5*B9Bk3WXYw;VDH!cJWe@`#nD(G5JkUCJ<@j1-f3wAp7$;Aw zR97Li);JQU;;Hgwby2AIDzx~s$YjjrW{c1o9jK9LgPos}E6NZwn< zDPo;qdp-?if3@?sKRL?FqpK^W`k=E)?aJHgQI|e#Xf9GP)Tzjgmy^vd}TSm#^ z(8RJOrt%hwsqC+6k2=vD1*n3H@7n>W^KdgBy7qkX!mUs_wx@yn3QK-aR^#O0PT9`% zxof&zf9+lzb;=B9Rh|31U(r#wxy#c#E!4R?x8_i2vj1xOZ3vUBx7bze9h>x`A!p@O zw8*6IdOm`_f811KL0$1VYv5s8Q}NwY{UC_YF?ZfOdz-iS_wd+U>j!F~m!wXZo}28n zI?siDR<9*`L_RZ3RocCx+iGU2Ii^hZIIm_UiPmor;1~+vxyRn?IxxY7x{A3y9}dKeqGc`JazeI4$*N41)dvf@vxP;w5v|f+dGLMGi=FiCDijtU`m)^d%tf^a&w7Y*$s`mdt2SJ zGI=>$u?}AAYV6l*S`hB<>vUn}{VeWz#V^&Pe^*l;wd(4biu>CO-*$WUXq(jA+O(~k zin|+i5U_X&l+x_%Y|!84bl&ZQod8Qfw7*}ZyVZQ^cNQ8t zj9CGm=6@sgOog3JWv;Q}qT5=ZUt3<+S2##AhWD=L1EMH=T)m@vC{5gWH!lwRUP0-- zil@-!=2L2<7j)TjO+!8&r&}na?F?zN3z{o1PF;RTCTG7jvxjME8WkO%ap=~<;f(8c zSnVr#&gorz7oC%SuFEIe?oU&-O?|DrUz+`_I)4{$?IzG=!Yom`A)l6o z0cDvT^^N%2M#sd`#=DvI#W6I>t~eSu;69A&&P&cNJ4TlvG`h?EzSBKFIT&0(`-K)V zntXu=NrbcNA7A-BJPAI}BD=M=JwG{|z(_6FYk7hr9i;s&VKb)a(k;Db$`wj=Jux5r zs z?f~ujB74Q_?JA+$h&oMYHPNo>%(a>8P1o_s*>){$jRp*FAxrzH-~f#Fl3SReu#ffU z0Zql8E9X1iSxFd<>4n$S^i(Hzy<^#$zkhDdsy5LKJScXbb|EZxCUwD230ir_jSbQmc27NUgh3*3aBX zcXVNyQeZZ#L!X;`LOUWzSLwhH&S5JK7$s8$I_Fdi)+=xBdcn|J@2NWJBUUatcYjnL zkN4?t$=}0uG`mgVY@Q`oANk{!3ZjdiMt7ZOBD*xVRB}{Cw6>1VFgrS3c0_&{#tjO5 z8?EwDT^WXQTE&lb2jmSw`7| z%ccwVwkPZ1+?#db(iafZmprB~X><_}FUpqeEj0&?Z*coDklPK#tdq6GY}r!U)~|@A zk8y#{!O&8!$E}mHf6d&c4a1@Ir(G6WiT6~y$ZEr#%K3KN)UwvOy!^D1mVcAH8kOf= z&^%VIS+;_#KZ0=->EX_9?Q(DDU=j+UnC)&C*@sk?XI57pw^2X8U(PKt_*k^Psu#_z zq#c>@AP?px9Jzaj-fjSuccdJviT8XtI*%J{Hd>;e`%Dz0Yx3Hz0>{H$NHx$oq4&gz z!mgUTn%mLbT?*bTFJ`_$TYpm%WSj1( zd|U6dEBt)h1xZ@OmmxJC;_g0LeW=pCkPcBNWrN!7uU?V7&!JB}ul1OhU9xZXl-<>9 zHQ0x8s@Q|O-M!_GZ4Y==-5lF(rlOAS=Q7)N$pgMqOX;3|E!;{e@PCZWpvB?l8faTX zVGi`k7D{}Th-r^_C-t#!1&{%I7`yoWi zsGJ3*yTm8M5V!W-wDMpbR7QxNQZ-qI%G9ep``Yccs=hIdvTH-jSl#$a&U?M0vy%x< z`&dslfgDdc?-h}FzJH5OREy2YYn>NY_Vl{GY=-;xsZhI~kzSFU0Hto|z`Zb_XYN_y z?0LECOsOu_IlN;O3v}Y08#OYcO7Hh7eZDfUAn=3yl@8pM!hym*-Wr}yYB zw`h87bM;BRaa5l6$`KiYla_1Y)$Tk~SD|(gkE%J0v<=(9wbpnSB1Y)Rjf}?ZgyJ1g zkBf;-B6y6jb2 zdUrMqv1{Uo*?)p9zb_l_mDRxL)>vxm?QrnIO!T6i$OkX^eCcJ=%=&C!3~wpthB@cv z+2U6G#?7B7i;BmqqBA#c{C1zi63+=Yo6dEJz&Uv7C=gQI!9W*l?ldUZq})GezTEno zc2sYVD|Z6Ad5d<%i0{;Wyj>)lInr%@4)FR>gp_rh!GC2m*Z1V*TsJx9>^o3)q1Tv_ zR>)Ti&7kEb;r3c3n7ivwj2kyN_uTdIP74|u_}X8TU9C5)P{&-iIiKJ~AEfp?iDKA1 zPFO!FHgg&V<$@4A9eayA8mbXJ1uIWsShuoTjxk?uGaV~u)4r^=DH}_7=SosyJ;Chf zrMsmoHGkq*NqGQ2c)srz!=cNjV(gRP)QzDG!el=EV<>=a#pwt35eEF_>FM?}2#@Gt zmMr2=PInS^DgBssy?>~OgZmJ#rEf;(?il(u-3z^;PSex&HjSy!Wu>8qw%w?!<})eH^+n@7t@EBwk$(=FwKzg!oMObr_FpPk-*h(d1TRX3uQ8N4*i1>69^TZ&>Z~+)s%in&X>wQ#wHIz-h|6 zcz^clXnFiTO7kH~D*KRgqPK-qImH#TW2u$S(gn3*T8*yr_wyaodl%;%?HaZdW`@%{ zTsJ(uQC2AimZ?!{hrz<^W*nP2@0o{lgl4=NSd%K;b?#B=&^5J^D1pAcukwvnq>WX% z@#=EP+;jME+IWKdUig)JCuz%sEb0f>tIihol zq+}am@u^x{RyaQ%t9#Jea(lRRR21eytv+a&^d$^>>#V3*k()c1Fwa@t8Ik4!iuZE8 zd?a=e+DHz;NlafRRICoK_u)0HSDx-(bWLt!p9N932Su!n2UKkyL7`RkOCV=I3G{Pp4tu3Db#zniEc)Li#D-n<}H}+ zOWQFN<2>Q;u#4G}nKpRs(mvR2w@=+koch?^P^7G2G@T_6<~KpSn%Sn}1AmrEf|v0= z&pg}IiQHWVJuhz8$30w^tgjVJOJ`2+oJ<21dRDk;$Z-8$^CK3WjdkqR7}{66AFbrh zijO_J^IP7Y@2Zs>C;d2yEPQL3_eYP;QY zG;ckAJ-6fGy=9}4Mne*W$A2L=iifhklrXrQJas&t_}<3$x^SgeJ1f=Y_~zH0e1?M> zUnQ`5n8j4jYe(s(Q0uA(A6?IqAJg%ru3j1?uWB6ng_Xyl*X#Fj-KDGIpajO2;i!lE zl=uL9?LN)#)B84i+wtO1N@f_Eq)>{!Z7RuJRra~&f;PIek>!u;M}KxqHtXl}>3H4t zl$@LQ(DU)FeQhZ&HyjS6*T*V+yJII>4<+wzw5N}oZNik@A<-yyI~R|~W!E0FE|>;& zPAJP+zgPOKiP9a6!T~>-RTS*!?Wpa7s$MtS^>p?6qZGZ`r-+Ox-Nl07N~u6=+D&HK zLCn~zWiUxHj4RtPw|`>53stecWKUI!)?0fcO8TAM@hcsl4`;oNyW9Q`qD2 zUa>u=B36?apD~}VP%_}@joateTjCLd4U;1_B`Q15X4L$T9 znwT}aYC4=!%Ea~ajbv@8^DC0~vMbKYj%TTTl-=GCU*4_T7EK(KYEiFu7wG#A+w!8_ zN;!BVm7A3J`hOnH!K@fe$#wLzd}h-6p`}Br%fm*B-bZy`)b7;Xp0~`1>C)OUZ*X^x zqdr`G-btEtyrt*OwRE@XFxV(}wsu$8H0S0r2Gkbw*LAr^9TOe#{%{JZ>&9kH*kK^V|%?n%H8H>V_CeFM{RA$R*#idX&)PS6YfppaGr$qQ}muv z+}$3iYQ}oR!)}<_`CvCUZGTE+^zeh5rMA|*uW@p+$ANW@!G&w6lrG+6=sCpYJLSSR z?bx%KPk-R$tN#8Fuat{qc*?+BEor#C;;L;1{bU`k8U*o$(IR#|_0H-QLcSGBHR-tC zDMc+TrkiH(B0Qij)zPd>J4!awRpPt!?W!nwTVK}LU3c%Z=}E;zja%zcKBmV-+gI|; zx9p@nj5tfp!Ek-!rP4bcnl59-RNbCqPRsiD+kbEm{)Ue97b=I_^A@6%B^H1SGH-}9kqx341` zAG4!%?X(s|Z$^5>eK+iX^Ayu-as-B!*XRiXV1T_K8km2Wg%vlnRx@YmnIa7AG6}LWPxU zR_ifM5xj&i&NZjAE$iGR?55~$#?J#kJ>IwO!8A{nL#g0$YA=Qdy~I)f40mcx&6E3> zxtfbF0*`N8(0GUSwXM`AJfO>1#?QLGlz$}?ZI52Z?=o+DlAb$Zt-s=dHzqE-uCJ5p z_fP9}v#C?F4j$`VqmUiskUV^xp`+pr% z55s|f=ZZZiTZV^@JtxJZVWvq)&#rK?ZZG7%6k~2mZV;aA(Z$Ep^olUQYFfvg=44&8 zr)xlE-9?=9gH|5|Gg6eDoecBSfAW*5vF||>(RLo+iM`IG-Sw!uTfJ~w|1?{bg6U#p z)qdrbRWq~VK}a*dHZ1Q!!|VIuwSQ-S<31(I-p21BovC3SbepL!{WXjA%F>dIvxM*| zxXn)T{ma<0hHd%6<#ELA$(N4BORk))=lKAW;3@0=wYp@+RtD|z*?i8w<;Pplw z)P|NM{1z!s6>0C9|OJ49j&^bI$>p%;=cov-`;RgR9>O#)6P5&;uG95k9)r}*6$z`j0dT(ib z!t@a6o1Mk<(e9!q*PP`tMt@xw_b7luFuUp)uC3%euSNX<{8M!lUF%eYa#~}(V(LZ& zN=@&b)C~{AD(R|IeliaiL1iB`wu1dOc(bqWc;*!2EDe2MJ^d^7UgwD&oi+z*m#rB= zlIGnmKEKUiWyOtb#MK$a)#_-13wJ@W4i3+}U%v~Vu}|G@ld#C0;(wj(?yN4A*}S>) z$@s_y?&+WQJBr>1VD-w_`2I{4q(@e%2#|k?$}2jr9a_t#cX!NQj)AN0bS~g4tqP?- zr=>7aNA5Pg4tH8hU-#B+vMA17{pg(CS?1_*=FF`TGMh?ssi_tW#Ds-%mz_&0V6A-* zk5|jo!}?y{PEbiz*MGmU@~f3zM|%hFw@P_d`?an2D(fEAp;GSn;L-D5v)*9C0rk+_ zb^V^N2I$pADagT?vs{H7aLsZPx9V1Bt8AEFl7z$PF_iw>mDkVpp`#B;+SRj~B(bmD zH)9zobC2oMdv|Jc<+k1@>6zK04P%ZET@7#@j6{vF!`ZoQ-hc9TzKg}BTUQKw=AZhR zpOR*+GlN&#Qey?S*hjWd)2VVD)_q1lA4kJ-tX$Q*Q@!5G33H@q_gNdO7{cvMhw=Tj zeG8Agf9|%HrEJ*g67|e#c+M>I6j#t{H0JeIx^~CGI*MRrX3FO7?hc#lJ;XV4yI$#ND1D5jMW1EEleEL;^?cMZ zE28!}!y7k9v~@ado?*N#@8$jJ(l4&K^lO-3>e9CLX`Ei*y-V_g^jryJwVHHl^@y6g zlJoBkKSlGst>(0NZ%_NOzn7`iMrF_jTeL?NW6-YVa%J9Xkzx5A{=i zR??THo+rAL!c&^;%9HD!t=r}iX`Or1BWGH#pXWWE)U`K12lqMcU%Mt)iF+ZvrB{ET zoGh8`R47*vH^w8F)r1{yN%od@8zw>w8dAH&FWHnRiL69!Y#S9Dr zX)Eq!OYhbDu{PQ`mZe*u+)_0%SnXL}?MICpk#WIQ!9~_tM)}yAkk&7^Qgkaxjl_#Q ztUT|iZm+Q+c2Y40=E*G-+Mcb)>%y_;fKMJpzkhAZTqZw(!g-GFcT;rs9b2y}_SpF2 z+IAl1xyMN=C`#5zxZ!Vxh^Z=Wtx)Fid~>@FHtDqYS2Z$@+ngJ_^`1S_IA)8t<`%uL zT&Mngub=bde)JuXR^$B++?H7f+e0;k)5y%Tea^xy)T{aS20fM6;|9Z+wc2DgxaZx~ zpf}QN)89>T3ifK>#!t{NFNuCf?qz$qL&n0Z?aE&pmM7&m4qii6xIy8a&L;zm`%P!l zU04nE{+H432`zuoTI4x!PsF@=cvsZL1_UM8gOt_{Tt=PAF!Sj7v(f<*&L}(~X{v)w zUU|R#?6OoKXLG3_8S{S+YmO;Ktwo=oFo|Yszu6ykA0J@0f*m%!Fq3uxfTA9ad+a1Q z)m)TMz?H=QmLz4P%HP76PLgq5(oP~r5^BCodq*w+&^mv&$DRz&DuxeEP>Ww_5p?u{ z90O?p==GoQlVIE;sVKMY9;%gO9Sb5;A5{n%j8w8Z_Eo)1^u%2^yIH38YHnk=Ow0pziSr znkj5Zt)PGSf{$$Bm}Z>EW_?_1NUn#?MK)(goSx!_Xe-;R{Nu-IPN629#H|ST$yn~a z-#5tE{_HE}`J)e$}2H-&$=RI^OTd^(u;cN)5nQg`%QClCUz z;faYNk~5$#NNflYB`2rg6Uc7D={gT^o}QA5>pKKah)|z(Splpn$G;q>0vgp>!-d*+JsVe|($QEz z%<=0+5A)3wh5O}XJk$F`ovNG(BPiO%5R3A97z}Ws5CD?FZx!kU*8^X`2kzca<6kf` z;j8)&bKh_k9&lhsYiTYhv;&oL^N)JJU31cfxdzKFNm6LOm&zwDo-vVF$~xK6Ka76? zT^jKCOqySp^7G<6v@Tk4!`1O zSR`hyQD3`S29VmbA&RCN(Mc0&G?qKeQ<*NlIx!=-$p@Oq&{{&WW#}(mfsFizQYzQMT~Rk>ZA0xo|z&ak|G z*(bx&d`O*$#=CluT|o6_S|VS#*e(dJ)c&5csIh%!{{&hlFA9HM_Uf(6?WRj!7a z`Qhkuuh`gPNqPl90&1QhRET6!aU;J1SmuZ)TeEI`_wAIpTZamJAJBM~a=OeSk}}#z ze&df#C&-Qs>mkY<)k8tk@IE8lu~Ow;_!$tL_2^t`91rlMa3OG8#M|~!Jr>qza$>bk zyqfz2pDDfAu;9asruu)-OxG4imtlI!1V;|zLG6L3?|Au+MRP@7^S`t5(;}?gGsE?7 z5;=h9C|iB7=!^UQ;!h0o6Dto?p^_PT9s0ycH=*7&06kSmh&3bjPG}XBB+y`WmfUmE zy~etZYF{DTQG={h?PUNM;>oU8CrEBjS&w_|C2x}|r>?4blP-Vn-ZS2Oz{jm4ajM-$ zJwdku-cy?&a17Mh_W{@=yqjgY*&x8dGU)ZQB>?nl&FRJU7G&3^qbu0AH;0~&@no5i zl++yIo7k`G(nDve;UWrbiM$qNRm_foLOzmnV5d*c`@|-J#W_iv+IMNd%4<+ZB5F!! zZR?3mi>tS671@8Z49_VD3&k4fwfn4&y1M;((Z+~p*aa) z=VIHzO=X6FQ;TDfXi@`rLm64Dr)aDyv7f~b(#3qi)pCCl@@Tem)cx6>Zj)#NcTpHw z>R@46Q^DH@DtL0aI*Ldtuqa@-HWTI$&Kl7a%IgPa3OHw5S{+FQ>j_kNf?i8}+J%1T zX^DUf_UvUQ8|OmKC@BVWmv3a-e@ue?+!9q!?W%XCXwA5kgx-)BECWJx0&^-cw8gb@dW9CUJ=WhV0CzmSJ!} z>Vj#Qjw*ROkK$w5*Ov{twHSGpj0e)+4PX}5j*0r>I$934Nxu>1f~)7|#%oVn+@ZKS zlC+YQVG)U}9-!dsZTWzS+4HSFWy5#sJT(}AyYqjO*F)(|`vS7@rU@mHH#Gfywh;sP zHTgujV119@1yJ~cxzhMp64ZdA%O|W)l$=5LXaF(es{Ay*Ywk00UG3CX_fI>f8TVJ? zH>aK1$rdk-qDW|6CVtD1^-ijPXJqI0EPUZ(V$8s1DV4$55bzez=jkGF+S>ghxE34` z2{3;S>zyiZ5$|E1jsr1_5*d!7|a?|Q_0yRSgJm%ZunoNFt8WsCz zJ6$FHc;V@d`wgQZjspf?688RlDn>HhyCiTr_3k_i6{YFO_eg{=*JC$Tf@z0txr)am zF6c?hGM6QNoQRFI5u9)&dRI^>u6`=Ws}_FIf$C z#+ji&7CKXW)U_1bqm-@MB{XQEY)IDna0Wah(;-XmA;ye=E+|x8?dGD3)uvLlu#A8E zXwBTbqGQMdY<#X9gKKSAZm}R;m1Mln(SlienL=#fHV4lPl4EzqQ)Jze#RdeHUQ-{f zYFv)HhVdnnMKb74Y6KxaD<2IdlZXowWr@XbI6 z<~RTJ^W<3?FTgZ0;EP{cTeH^D$u57;H%TjHAziUz1vq6-N2j?ez0M`Q$bk-QRLV&g z#9e;o0$O>lug*kc=H`LB6Ka8nJQRN3X*2xbx>`zN zs`kTyhO>UZ`lKdKnm5EDZL0+_Vg>_G*qE(uDBcniVbR#jDO`&D0nZMVPT+qaQF66) zVH#bjs}AULm{UCScEgju*#J#fv(8Zx%QCEquOYQ^a5CKnLtviZ0MR(St zq`s9BIw~*`C(buIYi3EVegh{^3R;m>oIBZSfp96%@y98gd@O zLL|v~4O0*T>%WC=X!C%c5+2BC<#qA)c^SDJ*rOJQ%K)WnB6!(akw=Nfy)bC%kkvjpY25?pn|#iTO`Ao7@C>mSXKvn zOli&QFwJHkx%j&G6(xDNQ7D+364-~-fxK4R7OPeG-Ccjh8bfnWci_u^`w;D7^!%v# z=F-m{Y)NWlM%A;?c!T{UT;ccCIhe6$C=B}*Pq^iszQP2^O(8*mHMV>&Qo#-Wh94vS z+avdY5+Yzx;u2PBN;x<(3Ed)mP$s4q~Y`-R>(q5JXgS z{?ebr&ryHpVh3dUWOpnX6{^Hk>hAkRY>q5$tEtF6^Fn`uAJ|Zv^i!0*pt>V6m&=)P zK!td7cskZZ*|VN*8su-w(&yg+cOiES$~qw-T+{P_>0TdTxU!^063nwtrp$0LeLrv@ z23DBY9-@(p$rwlNrwGaH!g_Apjp6+g6mR*4NVI=jCuSMe`<3mr2_cGQzg`uqE z3MYa;hB>hCZaM4y*^#+<>&q>r^)8yq9K`#EzxBNhB3jt4oZmLSb&9lm%i(TBjVl7l z1A7Z|AB3a(va8;}zDQoj{4*$kGU77<@A!YYgR}b~-cAelD50Z9>zE}+IEcYsj1j8e$>^aD{*3J`c?1h^-u+2BN||J0cO4z^ylrhIT70Ooe~e zTQld>ruyGJNZoX=^l9GxY88sJQ_Kz2GJZt7>aW63BbS7qC)Uc-*n2$opk@H{XJ4TM z^jD<{j@bYl=)&t@=#?v(V5R~j21+7dKT0(G5cEa*o3wG}f z1b&sj*DnGD{}LKQQM?nf#{`yK$ujUDS+@6quTDZA<3^{JdtImHr&9qzWHNvK6!-m6 zgdi$wl&#p$qGZTtCJ~jRC(IY7s=`+?tjM)91BCHVqX<@E0!q}Ef9Hxp3iZU;rHy0| znRQTs=Otohm2W%d8}n70n4gu2=v!oQvD;SE<`Rddmaqfz3KG_i5!~A7mO0w+cx|tF zE1)Q@TQX1a5bUKeUi$?=D(-(v>%)x#hait}K0hzc8#$DV_$r4nAaAj2DM}gPf64Zf zV)4t}7t%pZZp!pRN_67|!0Ff2AYSp7;Hu;zDml0z8gDQZh|+oXdCJd(T=*#=)`1d_ zwNpBXCDpG^hb7ri9is}})~+o_s$yLA0dH6VBxn`L`U(sBA!v6}1>}Dir09TV51IA$ zqgI;6OK-v`Ud!rulm5Z%N>dT#nsv%4U%vtg+n{^iu)q(1OEloRL+p!;o-YSYtcj)i zYRv3cU&%F|U`M}2kJ44fc1j_Dhj(9^ZnZQ93ERZZVt(K=O&yO4y?yVRn$#OjFV9fd zffJN(=6N43a~mQD0Z@NtiF+4QVJ!4nbDWZ9z>noz81Zk}#u@TYD$AS{(@a>QS~l7S zqP&u7zWMPyQe?BMAdt4!vLa1cq7Ut?%yq$Y_=n3aj{d_unsM~lgs9-cGr!4_AD)*b zS|+@YjIn`9nJ-h^X1|6rFk&<{(uK@szK?Jz>4zN~FVvkhT}*!p4Zk58H*Z411iO@Y zJ!x)P%rH=2j!let{aLP8gVh9S*mf_}_RH3WsWcnQmPmb2r_^dN%ykCmZ+bB~%+Mm; zxWa%fnDphS6>RzsXGHwbls=#2CnMmw5}n`}+1%}kpGymEb~b_kb^@?SREf!>{pZ~5 z(U^~Q%$l0p40(Tb`gHQO{~oVA5wDE4_#-naa3U%bg%CbdA)Yq24XAY(o-Uqf7H(7W z$zt#>lMh;jIK z7YLYifeg-RLKQPC5kX@jEkXUPIzmGu7Yo>rH$kb-M^b;@&53EIajk)wnYo$&fc@p~ zYoo88M^Qu?ybqXZe(Ji$vb*%w;L;5V>d;`=P>Uw{dHH%8Qso$?d?9%Id|* zoyU`Ox#NGW1M6|e_x9l4w+X<+ z097(-Hq=z)RkqIzf~eU&`qD73o#Q7k_yDJTFKyS5!zy!Af6u-MP9#{&S2KXqjFxj; z{VIQC4kNq8Kc4}(1NWvl2S$?c=918R2@lrD@n_DNz##80Q-}`SafK4p(LQHQf$2$=%$4%#1<&i%00aM1ao!Dl6KR+*(AjNhDe?uQ#^H2$wVzw zNxKYVFtT9}{O#(s|>yd#RsYCyb2IDbvtWjq)al=}dP4?BM*-3)vq zDs!ZsEZw;R@5YnKLsZ$*1=ARvIR>IXJ|2{Ta5O~wWB)%v}HHlsu=Zwus5i!bFV5k$)>e9&lkD2ded;uX}w zuSRaTdmQ4{c88RHQ?{~?KYD+70Q`4Dz=m2iNBmZbiTuRcp1afE{d5UhY-yA3mQ%o6 zk=X-o9a`DgLJqF5oV+j8?k=2mRud+*CG*%p64NRRN4XCS~~ zpN9*$rj0O*q2*;S?793@fz~}QPeHa}5J|P|SH6HCY>cVviy^+DbTz>6+SGt9Ne~JF zR^LIspeiVG8;+Nps!A)0#Joik%xQ}KXm1ybm;T7@>q%7ZB?Wzr3z5|v5<8XxrNh4u zka{kA6kHJSRNzi~*d~9Ntd!9)c2jdk(Cc~N`6})^ zu~FGlMYDc%_^MF9Yt%Vq$k0jXwcHRP^VoPR~?`b=pfFYbHjoSJF zj;II__i~0VHR~*Ag~*J<8&N*z3M{0^#Wz85}18C znlUaqbNszl#pZwcLsK6auT}2PLo2SirCpl%O1WP(Z6OgO!*6O^big4fYQ#(~={ghYIBp|3Y@@DSaQe=Fom{ zXC)5M>4EEf*#MOSE)>({J2>>hN+NM%0%W}jKlftKB$5`&ko;C!$FISkiW=>C!r>)b3|g72-AIE&s}h8mOV}; z1TT|rS)Pc-fM@U4QomyG7M%zt4<$Rp^1_4wcTD6rhp7t!*dO0CH+SiG?S3{`f++}G z5$W=NgXE={uJnXw4q1dl#|mdd1$?b0e@CyK>Cf;c#kVTb0WluCJQs3phkJ+6NXg*YD-*F2uF=1^qx)}?n^>`gym>f#hPX{wPV~` zc#qcH0%6m#MibcC?g?&&1ua!0qYIrD^q7BsBfTTl$C%>q>(|b=#{R+v1V>J47Fx5O z#8UFis31hCWlXHU+*43F51(0fMF;0LqP%Xu3^(Uje&wa!k+Pwv(9GqL@OxWcjJ5mF z+t75iA5euRB>uMvv88+inV9xPu}Bvsws|+qX6zfj6F85wUoiff^!O03>dwmf<$ZrP zRKguB4e8LBp6+%2vPd+OuB_DY6ak~ef4))o*wjpkKOrfTHE|5=Te%rgQ0)*ZRqq*a z8e5WrNC%dV38KfF4LTK-XSd?M?fDW@@GxIgdA!u*o%P8908GiHp5ooiUC@aVcTorM z%!e#&qMI^F`EHN|ha4|~v7!iNd>MakgM_Y+`D2sBBwVtlME2K|rR35^`BLj zJ3bsd>Q0n7_W>Kl@#ORER%DXMlceD%!z}U@Ziw|ZU38` zM0O>G?ht9?|Ih_FFhT2ADEw3Ea0XpAYV9A4<@6%{Oo-QxT*}g|wC4 z-V=KXMJep!HA&lS&m8j!z#0n=3nY zO4uTm#_hjK{#V|8V9$KEpG-s+^f~0_LXL9!$*fAg70fO{aJnpVZ(t`fut@h*U`X_9 z^K5WX+to$4zdIl1@DJfno#<&7p-_CI<_g{J!8xp$j;b>qangU%+oUsdWe52~lE}Hf zDbf2XYicl=XNpIm0vz4hH_p4e_BqG^7`#~DqcJ=BZojp@R1vz|Cxb13#VikP%J6<} z!P>&zuv1f9*}Yq@&dVRr?;<*r%-7Mv9aS#mMf5rvD}amjtkJEp*D806SXiA~-^}UE^ul&c zdXqr0x6o1PuYKJtb?zs7O5t?QIas!l}c#? zC_cSztVhc%ZO75A-6xK$k~|E{c-nw28HzI@-3Z$R!p$`@i%Z6X!v<*5Z@w{V`*z&S z^7KLlsI^&};zQDT?M9IEs`JZ`oBGhq`oNx9X=;bjDkCcSX+Qec`U;FEp=|Mb`k3f; z{aDkacgcSi)GHu4NjA)1SNA#QY;bMryOub$^h@Q`;6>WbGL~0E@AsK?IYI2@Qlr7; z(C5u_!PmN|k>naN1rPjy3N(UQp$|g=^lVH!B+k|+@qRBfyKX-e<#e&l`5E=1rLAJL%vtjE5E1<9R+sU$Z9`8`W*sk zee;ezqtVkd`?Hl$0A=Wv%a5j{p2^R`1n*LtgCv;2qAg#4^QUH*Vw6}KNd_Q0C ziUvBux=vmQhEW_nD=4AtY}&tkU7lO~g)XF{aVLSMpCY-8cBL>TUChrhJ3>$Wa!UH; zx>yeQ9)esJ4!%j(sE;wwl6}pvE?1S=+jW1(_&Z&Vytp~6ALt_1`GOv-JB1b}HpDZJ zbK^QohVmk8){lhr0}a-n2iCghYe4HBdCp&^2#4tn8Dt4Tbl>zxhpEJp%?_ijS-SzZ zy#}O6T{xiaBchWHu_QogL%WcHU%nJ>G{K=N@Rwv7w>AI(G8;)g_i;IdXe9Z`@F;&9 z7!|u3Iru#C$RXlv+)+PNMFe&j6eI-tPX1^oQ&pmN68aqxm0$KXh2x+SNH~4_{A?Y4*YVhQA_p$IngmMZ6ta`uAz_?XvgTsYC$x8+`E=SMSEAkzB(vkR= zST>@$cs&vzPfqdDD?br}i^PX}!3N%60vq%_liKxoG2_e8bpO;!@I7!1J@~`s9XM+c z!rd?;n;v+8Qb13EmT-N#_DgYyKj70_4e-wq8`-VaB9m1QMz7&0EQ~!*)?vSaPg<+#EMOp?=SP`A35fvBG)YmTuQ0t=}w{T zBg)~$X2Y4`<!Z4Wx7wsd#_5dLM(oHUo$_=`y7ANCffK1I_B2X;Q9^?|s=C_ATn7 z_8|X8wNHj$#NoJ(sDD!nSO|KhJbwr=HqcxiiF!akf^q8@<4fiE(Y1<6G|$Mt>io1b zWNM?iqnK!6u2oGU41)uAaj^_%N>SPX-`h^j&7%au@|`rm;D_cBSxtYN-8Vl_AD9J3 zMLxCZefHDcgdOv2^?tbon?GgB6Nrmc1p97lUfeX%E*SP*zDwZ#bW%QES&G`C0FTh&8wprGnDGc}?d7}l-jyY2d~s`?bw+ezJ$wnB_RiB2#B_QP=_V+578T`{ zSBhtlX~bxD@-m)b@A-dMu!0Tmy9QH%5|g3Nd?VeG!Rq{G77~YOAd*;(R=~;Egl(HE zG)#}Td<1r~m1f-fp4UO0zR>pw(v}qu%9D(Q`+#{lrURj9c&TJ^(i!6Bp*aCAQ7%n-Dz(%rR-0~vO#5>NC8Rq)NmqlD!O4ej^UU1o)vviZ8qOqBxu$Zo#a#jzR_e@pzT-;^gzVuRJ;+QyRn zK!kknpxdfaF6n>xiHJ6W(?EJy%e>Tj0j4w&PM^5o@P(5w0amixzRki%-3JFBh$p)p zs?6~TOy9*;2(ui!CLO6bByd+Yg(YpsQsO`#z1 z9Vq0M3nZz6_uO3obn%q4oD8;Xv5)e24(i9lq;RQt)|!9K^}8Mv^j8J3if%6aiajG( zwC~ei1njPy0Fm$3D1M4Dka+SnB_*;t83|{?Or`t6pWK7z?V7>0M6XRts1^6i-&^N^ z>cvMxfsdyPOAm@MzsNp3Oy_{wG&!Zo_N>+?>hOthBeVmZv3jD_AdGT};(|5QWBdgU zybH9`FZF+skYa`(#meXlsWFbxqzGkJ+T8AUaYZ{~pEeo(a2=QW9En?X8rM!3??Z3p z>!B4XSexkP5BNpSC+n{hFnRMNPVLJS?8A|_*l1*niN331(u3k|%RL>S?TyQ%t-Zer_M?=4Uo9l>@I}-Of=gEAr8iaI)Qmugoj&QR08>ZYs zETL{jm@4%G=qkvXf|%qnL&jGxN%BCjWE?p zgaSh3wDTJL5YY~P+M!WHzk5S24OHFeKL=+W*==Xvu%Scpp!BA?pQgNjCTH1CHDQ+r zV+VnQpTs;e<-t|#K8zPXEmXXWIy}i&SMz_%ldZH~g}K?)elFvSgT1f*3UiCQ`RU=z zJXRo3h$Q=w<7Ee4wx3tr9n$Fi27-8jd2Sf+LFrTfzVGXbk!8ut%yTgYVHg>|p4URx zSs051X7LUm680!3#YIM`t@LGnU;D(cFcIAD+S3L)DS=RUmc;6tCTjoELs^vyrQx(s2U=Xx%t~ zFkna1XHPZ8!l0@`O;rV+<;IAtf$)EH;q;+Ni<6>DKr90nbQyF1=YReW-}fJW_y7Fg z|M>r*y#Ftg*ZFTI@8$nDlUJ|*7k4>p`7ibUVTJz*e}q4m16wLds@NLg7y!1Pp(f{6C!pk}wK~i+`~{uNedV z9RM*v&cBrYy5elvu+1_RVW|AmQ~VQ&3DaMM?UT?-i#%|FaH(i<`;d8T}r60PbQ{XO6ETpehRM5Z@1oxRIfvl`2(3KQ_iA&@+ zS$$wHY^XRACYsJK#}Yv{X~Kl*>&ed}BgYl)B@52r?PdTqhatbExCw%A0ZKT#t+F-t zU3E6>_<+eDZsYHhuR}QkV|;}K3d0LWx+KD5lY{J!zR;z>Vhh|D1lybM_$zm2b4xa6 zzWIrs!+(8RUj){IVO7V%>vCd$ui@H>;I2Ye_Cv+n1Ko`g9(@r$6* zuUEY_MtzX)l?S|jYKQ}XL<*3aOxoJlL&NF{_+G~Yp|7+dM_|Nl2C=t9BK^pL%wrpk z7xPQCq#2$s#Xj40JXEDlfN&2u>nUh4Cem;kTqx8YmHQ@O}vNzT9Vk;0TcCwrrp zpy1dRZz|LrkNE6AWHivCZlk|72RjnnFx~lG6LusZYhgTK9jckhK;CtC40-uR<2SyG z#-g-{;~Ot+)j4(0=vw&JxHGkOI-~M29=9zO73WEU+N4RK!NiJxJ5GHd#11UP9FBBpjIp7}4tAQ3=+xJ$d3f8}t#Pci)O>L6 z!g=s&UQG0b$wREoiky$RqQ7y0G${5>7*+o2p8IW>w3am5qk3Ueozg6HK#3Z&KhWAzIVoUA;rVxEVuJ_1ND}fWbu7`M^>|DHt+;PpN*5zO@ee168I#f4D9` zL3?2dqS!<7QG9cyvR?soYLN*LOiKFuMR{rcJbiVccijW8jXn{JX2|!))Xn-p{X*N~ zf$~H2-}}>hqss0Mfvyzh+T*9JY7hDSHH51dPcp?jQJB6{$cent_6wb#a(ug-6HB2F z3u@srkeI)J(K;5cqCCD*p=lmIE(BhgiGJLiqi?#p(DHFmUi6KcZ25Gbap=;?&k_Se zQeRJpm*DB2p@~?Vg`mBTzryot04e${$K*%%luzz^AM(dtbszhIvwqd^M&FlK?FIWh zj)-C!e2M)zy35;ce{n}i&CC7iRr|TUy{Gr19TkCpFxk5QqhDwyb1>q$L(eaNx(Ese7m|&o0+2~I0X-$N_PV7GtcbC0xn%4}%rNt2!Sb7j`-R_clpubY+}})-!QwS9ZWB=5)*%kSZL-n(jkObv zUl$k`0BaLs+NL>~>2Cci5Pg>L9SfTiHSK>WA5&UEwu{^9jKWVvKODz2zFSy+$8a829en{ihd}A?S~Iki=yZOtf!_W@FDkW%Ik{NbCv-0`RTtkbhK)A z1eXxA)CTpeKvC_&e~kP##eW&R48PCSlR>L(RINgRJwop0)p$J+t?*iZ2t6$3wa_w%z2I#Wk zO8}zywFENBq6u@ZyAG8))-Wc{0xbWgV&_ndF{)z9&RRPcP1J#LT7T=}AR$GtZ8C~4 zr!OZjpMQO7bUk>I(zS+?c#$>fgDmF$;$`=lm-CWNU|EF{#NG5C&5c>UEu(vY=#8%- z65+h35=k!ZBOu`u05dA(cU3DD0JZZdd?$rrhdgTK2XvSMCQ%6y&J7QmZc|pBjDyNw zy9LYgTIjdEpM8?b%;)y`%1z-_h~9TJ9ha7bQ3Eg#>=-!i{x$d zZL69I!{a6)c;Rg#`~*Y$tdFW9QBZV5JR9xjp*Ar+^Y)cw+8cfgY(c)^-AP`e656q| z<*hU_Y9&Dj^!~KLX8pc`?!IqZad)yQa?#>;Z~Q z{dnaj7?oQi2{%4E;1#pO$mU^uFkJ8k?a!*Dkye_GI9utpPY)izewz4jirff6 zB$8sg@u)72cRtC-&mr@&`Z~%2E-X!AEJv1K19lHQ?6``_8jpj6DC7lXW{&~H7jjYe zW)ginL}IWA+CuMVwIRcKsGGmBl}(1ie(mlv8Q9B}UVqj1TmRq(l`w^iy50AVP{Btc zn9ng85eX=Vf2@wzZ z04TAvD}Tn@e1gsVqsUF)Wcr@p496Pcy&a0bCLbJ|#}Ly)QOh|bs1OsUl72Ir9+Ij* zg{hyhP6aOQoT9_9U2bC4)4Dc7?HzHbq-Ob10P;T(^Ra%BoZZd8Yd(PdMi)~%27FcV%qmfsO6`l9P8nQxYXhW z7CJ*hx@ZtAq9b2t2@sZr3H>)7b?Z6tmjueiur_a4-NsIU4(};V1)i_M%JMb{<(3wmur>Im9{>!1m zUgpmmfb0b#w`bYryVG%dDHNpj>BrW7VOljRU^HhO$>{zwg1BV_Xu~%KBkr)TeT06; z>=hY5xqVOKX&rK%^MGXN7JCdC@SVpKihmVA^z>&t&+_eOj5$Mo0(AWopN1&Ayv3Eo zeU95*+liCEkB))RSvP@$Lva79rxse%QtjDvfL{u~3ztJ4IPJ$aAU{faX1y*X)kq1E zB$#?#IM<#OZkzd2-pYl%WM7D9k9lbm@lVE}R*&bV5CC@T)SHDv3MgFl@>;O_!hb(! z-p?c2nR#pThiD#lIeK#SBk)puQkO-S_tTtcL)BM0x+S0e%Ui#ik}cGaN$pi54}E|< zVWd-(bS@`NDmv<3YAq92)lv4~9p)K1t}3$c+HtT=X@YVg%)RZk`tYO1fsEGNm?g~2 z)kqXmt$q2(HPG8{x}j=V*8&%dn}3;P60_~?8bnoW;yA&KpHdyj5NkCIMg!nH^61zO zZ>aF3V7J1n^7aiT`U(w6KnNgA*9BJ(R(+n;9V%}1Mxm)Yp2^g}KleU)DjiU!JP;ww z5(>ce+qb%^4Hd^7I;PsS7EnnH)Vjj3@N^AYuO&jb(sFw_!DdR1 zD5t34D+ju;b+>rR2Q8;dzD(`#0FZ{o8b+Thruaai*&!z-pdm8jJGNgPc(sIvhc1F? zw~cxknGS%B|1~k)gT|ho1SMpUF_btaNrJ`{D9U-(lhzy;rHYGfEq`e>KHe=Pjy4xN z6%DXsJGL;gZP1$OY0~s_Hxq|+K5T22RUW&OC6-ep2a1Xw(932fm`vP10ZM|_THi|K z@^Cw}uk>-4?-{xTI*}Hv$Hry8zf75zAwNrTh?!A;Vvgxlvp zfVmCU&AFNqm4lr;sekh;8|8M4u2eojvQ2eY$hW|=9aK_$*~qHNQ;H&VgCZvuZ-g|e z>IP0?0CFxxGID;;p$U^SHYZVilP9-OF?^%c-hTx5{><3()-u<~_(Q5O^k=;?QQ%5` z-}V;K_fi*_CbN<>OowFMmS~zkeh5WVwa6u3I*i3r!PY z*D7p$!P5}jzW}gnQy8uf-2lA(^|jF5t)4MGg-~RmJ2dWiR4md%>cTClpCz}nj7T(A zf$p>dr7HwV?C-^ZS?%Hf>ToS~W_RJ035~cr2XcmzzXcy8gA=8FeH@0W zi;x@p@-}&e6MvbZS<8Qw9#jY-a;#FHo*r7Bm#YXl8B_KS6y|1XjpgDc6LjHc#V@RG z8|HE9Z~a0=XIMTj(BVXYh>?&x6gi^v?ykDL;PvH}zq~y>N1YIt^iA=#fO-*vUw_b?browN>WEhXPa7kj3&UCNy_^$lmtbs%5N#Ud7H`UeWvFbT6Z%DjgA-_p7RqM*r+d8n6t6H$+`pcyhQl#g{qXj z$us5XR`T?^{&m8Y>Q`!YR9ihl0d7CCHc-iRy?=%5DZ5*e6N?vnzGFKKNZ0RB&XtRD zbidmK`&3VuIrKD}7F49p1l_^08=mJcnv>tR>o=ap8NgD%7`J*2^D97%7Wu58=?7A> z@?r-C<~1cfwdR2CRPEV9KVahk5B%jtSsB8wRuIacEN^FM>;D{nD#M7q`TX~MZxx9m z@PA$71F=tovX;>DV&SDC4#zX89nhxU2;XzCPOj&$fzpH%@b2T9yr#j`ob@VUUutyT zz#&_!KM&x2m)PM)bif+zZJh@XqDj+Mx4K>h(ToevS!+u;p{HghY5dMj##^@uB5qtU z4vO87aP@rtoY--K2KEks@Vy@MoraZI;eS*Wl_2@o!igOaRf#_Tdxs5!)z^&yXHCQ0 zWQ>T{e1!`>STht&Rae7mk4TYj6Xb~+QnI>NV76eVZP?R6a}R*@^m~?n>S5pNVK;$^ zv&)C{HlPam{Ci6;P@acF&64_lmqV$1$67FWufrWpzq6I7Ck8==7>Ic3-+^yk$$wM$ zVCj(f+n5TU#PZAAB(Qi1sFylb5wqM<@3n;a_vs+Y7Y9VnlT*H*yga^j3Z^3SIWBd) zJSAl@NyblN8UPqay9=R_qD}uuX~)b{zf%kwOD4!|U)6Y8YUDgQF`W;;(CC`IO6feH zQ-GibSlzIB!C4}t&ECM9%bHt|PJbtAgO1~xS5ze0C?O&<@~M>qsiI$bW0+f88I6Vj z^*B-7TGXvK#yJxyL_M0jl<0A9=b2itXJ^h)jJS2aV}2Ed&KpjbM4y~;dFBuW3LL_K zVg6>)R3fJpt8!>I@W7~9_{!F(_r5eaH6)0MZaA)RbKZ<#K9+CYTC2qE9DkB@moe^l zuO(^7O>*oPC!z3@W6}$UG-b8!{d2&JhbYFiNM*V8K^-Q*K1x6gEX1t)&B%D9GCkWJ zHza??2)&o_hD^bIw{JsXq>eQ1rcdM(0LZTF)Z74MfWyhk0uD~fiW$PKqeps(E_N~n zb_#Fh1z3QEbm>tF86dL7m4E7a|Bvc0ZHnxu401R!4+12}EF4WN9Eja*__q3r*-dK* zyW)7U@xM|=9TGP=6|IPnVbx8?JR~}4YYvNViwmFDd;B+$dmw=bm~#6!H&rR?f}k}Y zqi|cBTQkA|`KwgG;>IMl#w27%6Wo-kXh`ltTwOCc!5!?J zgU-kG45eM50*)F2h;W8s8N*S+oZ z#yAo`mdio^!G37NUw>e~rn`{O)ddK{5O=v%06DbwZ#%f#oA~82O%rxW?NB?mG$aZ| zOz9%%L8R}8R4^qesP&q%EU+{}^WZo^+Ye(|C2lJ*=LVY*+YLyY0xHb2{k)T{RnDWJ z+HdGOl_W72CL#T&=e_3v!)A%a-e%wZt2u*kd$Drm+eCP2v3~_9TYX1T1vF~Kt7OdZ zlwhTDFy%elt($^@qlyy{%(}$Z$mx zgX^Br))fn0e}C&hb_5M+Fu!LfBWMlc7i}XlF#1U-ANmX2H-5x5U1_4^i!}vc|V@}`3$>fA)BsH&9EvXZWdT(Ju26dC`rC^3V$JtkZ#j=v2`#=Ssz`!nrByh z1!M404zApcEO3OPl8qTyDP2oZ&k>7=Bl<|MrXoiO6^aiOZ}J0c-uR_$41KzNX}WLW zs73NZ!9}u!LK65PNu~E&2$pCTknBb8lXb&R!^2zLmcOKKxDmA+w|zbEBCh>;=C<9l z&_8ww0e>DDrm<1SoR7fbx)>eCDGJ{RFp^t$SqhCM&0y%aB0lfLjAem_d4TeVBCer= zIW?zcX`%AkBJ__C&uP3y|6$;6PZE8?4?|h3M|`P+oTg0tJUc?$S_iHA!pGRQT-q?w ztfbIo43^=)&ESp02^p)iU6~p^cV%=RiJp%n3xCEk{DI&0(kGwkSXASBym#<;>x2b1 zQ^oUxT{d9dZbC=F05z7roE2NGDwieYxs@oR@2ec~12Jz)fY{;Ly9D1YxnzAqgM>Gh zF%UZIV5@oLd7f$+ZKq!4^{Z-dWJ+e=+v5q$A^Dr&jqjcTkZYZM9}9}Wr-um_yWX_g zv44p^G@blawyU%Rdce)wRZ|YHPWU`f9g(GCF{o2A$vNQf-SnIR4^g>xW!;sLvDAgf z;UUHE2X}I)2DuRlsxu9Tv8i~3EMBBrtZX8LZ7dQ_#3Wqo{5aetL{xFtIBm;z!CQ?oz4{!&=RW*jBg2$_ve zMxJ)c(BVH(HWGN=#4GY%mHHeTjvfgQq3!etE#INNzzb6%OsuP&mREc#WD9BaNlV(^ z&ot4@LDk&<(rF~V40{tMUAaCvS?bk6{N;`XII-n$z84PnFdaSe`KJThlTz6wjyHHdNq^3T43fxcQhM$1J9WQs#PIn*87L#xqG<*z>oXmH zaM+<`3arDzc}eq@d%F&GIQ+-?Lgdf?@Hw~J`JK7poj&2Oyt0#8z{B;s@T zBJ)s*@AFd@hc)T<=oTXT=I7^yu8qE_`$(;GWi__=lX0@tuQT8(%aHBfbALQv%=l*| zzlw7mh+!USi}tG==L|%_Dd518mdPKM3*x)b#6#KI>zyrquR225wXQVjfc+0R7@`Ax zBkG2}ueBGG5L$ZHGJ0xp-c0C{@-c+zv><2{4Bf~D#qXr0K@fPUcNa!cZ{lH+owL=V zI9t&lCF7}MUH!ctOfN=2Wq$|ss&nw*#1r3k_*0U1aSXQiK+^;}a=K`czW@g_1!u^1 zU|eWp-bNZP)KLdL{YS)UcM&Kah1OMx@B;ZFyYWFj4>W_ewZ#wd?+#WXddy-ZpPaDn z6f7;tkhCy6Zt_Xa_&%4`N54h+FsZ@&Wk<_nprN?nGPxC}*?KqZs((%b66pY>;vIFM z6r_W46g$YIk3%mFK{oL6%+X*bZ4scSfh9spbtk&;wRu&4kOee6F-Q7BXj*Lvt*6@r z85G&Vw|e`0aUpnF02~g&z}wT&_1nj9Xy5IA`P?(?c+2``2c_Mn4@uiUrJKCTFvjNA zO%56ztM*2%>-P&5M}HJ?*a3ZCwpT&q4VP3n->kqctsIu@`ziKq_u5ZyIiy%-nw~Ao zpI=PYDl)9_E0RyK;zJ!c$*{_%hK@5MPV97DRd70Qv5PzyQ{8NbHwgfW_=u&4qEt#j zY{y1AX0|c6$w8p_x=4kK9}4oKvQr~D*|gpCiIn(*>yR5N^nbRnmpu7HHT*;eXzA-2 z54YPMNo#LIA9L)r+)gyb$dC*CZiHP*1L=cx#MX6Pw~98>E8==y2|OOr0vZe~?jqxLGu2+>T-T_TXShe$ zL*J5y`>w1=rnmBB2IWN*Ta*H$spB3KZuuCeQ2aMIDe0|%bnZSklB(*gfMl;_hH zmB5;8_J5xk0@CN6su~V7w1omE0}CN)QVkGIH42 z1G}Tt#Wo8ubFy46hfhIhzFRFaU;xPUz*1`@X@B$m02KL=Ge-BCVIey9i-%Jam7FuJ zO50&V*uE{vrr@*Dp>ugng)FOXIT48YwXq46Y8-m?uR8shaMo;eCn55H6Q#|Ixp`RE zBBJ$f80TWqHgF)>fDv`}XIEXrMY=bj;YFL9EG#`imt}4YxRnMw6dE`Y_Q%hu`_^pv z{eP@=>D+nLLCDj=-G-S0xi8pyld@MBGKfwhZm{JdpBGa?FHl`N-`|A;pYgAEL1PSdr4SXWV$N4TB4-O)cEdS6F2UyXs&jT;-N|z zm(w}8Sz4K{h!j+GLztil(XpG-Qv(4EoqrnDo!iW4pLGljcaK-4hplW?UsC-N!&T2l zo+vz3ITCFLuM$)I&-tW?N(@H-;i=He^D800*?SyG?sZO=B3&_)IF@U)8a4~V&Zs)0 zmLtlaWL5>Cz^e3~B0~`yY+(vyP7w&L!JfM^FB^j+{VzY<0L2o^hCnGCq~p41wSUvI z{Ar_HqhASe&RvA)S0Is5UARleg;M2$_QtfT`5`=+djf^=RuiIssh zf=5%yFiOkgMV8PcJ@0yuk5c0@WEn+m32T_9&>*R6xFWyfC;L$PHdUdyu74zGALDIt zJh-^bm+ygRVLLkqYH+_EM z1KE5Wr(hD5=w4~|<@>?VmXK1%>853MV)m=om<$XR(OOjc!n&z|{L7CyTQm2~T1%VJ zc-v89sFg3`Z2an)JCJ$e4u6mBhgD>L8L6?EgoWT8ecYBpi;0;Q$vpacyC*XKU5bh3 z-UWmpAGqnWab`C0JK8gmI0_yP=d?|clo_G9Mhw<1;NqKzi*3QlS9rk0|LT+*qXHr5 z$y`48u3Zuh7VbaDd(SvLW)B%5Cm5il990&%X2S%8KXZ-fz*Um>N`Ieh#6iPCi&{+_ zHMuqlG+w}jFV7DT4;f_nI-7A`r_U0xpg55r{SaReA^o`>Gg~jLSa~p2tAVvo%(Y%v zzHZ}O-;06n`;2hoW~u6%4*r))558I7EHGZ{-@(^=%zo7yKUoJfMktoZ%JLi zdm3>po{wcy#CdaDD}Q%oUYzaYOP_d`q>!?qt$WjF2FihVVCqgZ6fUh6^zXFr{`S!j zStjB1cYQ_3F8l49Ljy!%Y73P|GtyGgNaE1a;&2~+cC#5TA2(eYm~hWG>5%-=xRbB8 z!8-#fRYRTl&zYt~96%cDtp-{=m1qY**V zwALiaJ&C0U-p@j1p+gP?5aXnBv!AA3X|n}bNuo-&CL0>4u&I^ul*l|SU2&>i#V*eTmLsG~+E)l* zUa#-q3Hsy_w98nz0X@Lsb3Dt>Nnkq!04DA2){_rxQZJBD(fs1p>ys?`>QYc^;tP?I0O34+q&QuYLQ+e{T++9h7bResG3l;QBe z{ByawZTkn5Cr4h~x&0|TWqqH(zPJv}ygK9^VJuZ-UMz4sbUM1xtkS{I@GtI7c|>_+ z6!{YGn}62`U#bnOoD04zc&rc-M^L~?$@DHn<8?v4>CaE#ev966ik4FW(O9aA%#=zs- zkoKK;C@t(z{1;9=?Lh}8(CuPLv8slzIk6(T$j2T6$ zR0f)wV2YAPM|wZqPNpv2#e3-oL7^h9q&FXtf=3X=PaEjhl83p4 z?0=0HC0oDlm=^4O-}Xz4k)iC5$WM$qtIVy-oDqU$^a1?uYXfJJGSH@t|Vd2L*A44_0mV6K? zWFUx$2_wC|VM&b82sj{BfVr!c)n?-vj(_SIYz;d{Pl4yWHdhg`@+D**U`+0}uI}H) zx4|r#gyp~zmNtu^Luyq~14_V3E)>iBg%r=2(ayyyjvGn%L?miX^13#YvUErlQfKN&wq|0^7%@wR&+B)?R&>Z&G%@3K>hpuXw_9> zhSFkcm+!U!Ie1&JbMp`pzgs1VYblx1kG zQdR-+v8q_owm4%-cu)t>L?T{xM3?fz=@{{}`tVBnA>0!6fTfGon>{IX&4{lCz&D?jxeKhrQZTpX^+s7}qO?NM}RFTg0c}dgCNpXSF{Q0)O!Wcoa{^ zGb``tj!*NCR1X3(yzN1}e!~#2)cJ-(m`(JKOr2niCmw6}kkZ3KXu8|8B#c*cZU59O zdMkoFBfL?WFFNYvCOGG`+Ih6(X*cc$i zh9}@>E4)!xwDs(sn#Wbl)_)gy7((?D!CF~;8_`y?e0rgRxhDs#!9zG+Ez(Et7L%Xk z{qOzZz~z+~%h2+L=$qi5<}B96+_!} zXNnm>wM;3+%yL6UDuHJ`MryHPQohn8ze+~whbvCY%dku$`-1&-Y8;sTp~Ts>v0xB< zhslgPjE{E```J>E-+%HWzzh+o(b@Jz(|d2A{U})?#@E&aO%cO2o5epmwLCmAPFi2EmELdQ+nfEpFJg3zlIv3kG81d9zI#0>;IY>7}C!t4V5C=}rbo zr3#Su0f7=ln$$K3ImBz%8Dw;t>jJE&FxL7XE+k6d>fOkyP=CO9ziYvsLFiLwadX8< zCq;A5)gVTix^Dhi5_Z( zpi`X_QTm+Bl&dNcM8ZZk4K=-rSig#k)HHFkVWpB}j)O))gOKu257`eLEa-77Z)C7|5F>s+t5|Hc22pGDX^j zg7`yHcYny%)aT|FeC?Z>%wOW^4`Qr=?9-$-M^s3;wz4Vm{MpQ@K@VaJgF$V~?+Uf90IKEOE{koVPUiSOw zZolPG6O%nBjs6fi`y>x*))6ViA9Z9XL%Onp`9@1ZF zj}SBg#v%fKdqJJ0=bg2Q-m8#pqq71!RDTim{z%T|5lh!Amc}HWjG~hs(UIogVvo-k zt^wbG2hfulW;9H_>9+bN04dFu)}gf-v^>0BdtyjBOBvK@PwH`Sr)>n6!Ph{dHF$sd z(TUreZb`mDyN8<8%VCf_dT)xd`OV1wsm{oWS>lePB($&;*n_X!arKT;=yJb7NPqmh zf8A*}BcU-Wft%+^_stjAztf3dLTQ6|_17O2Hea^dB<3UCERUt0v+iZuN;Weo;G&Se zyuW}ckuBKLSkH>CI8+-E`Ro-iEA}=m zq$2K_V)%9YxXuN)9`oj8!K?px(SKnEnw^}Syjr)@jFD7gdOC37~(r-;u7aKJQ+Qw-^ z%?$X<>>dy_p*3>Z=+ESXkj!Mayky%8;MamcK@S=21NHMeYGe%}oBh$0hc?yzI}_O# z{Gd;l-^>{&Gt*`m#Pk;EyZZ_mZ2kyOTUAE*LACFdN-vGiHu(e9<1A2Il`m;O@@w(4 zgr$mmHbOO3Czf|6NoDDkF@HXRXn9OJ)wq9<2bwhoxmZ~7*4)`u=88kbpe7_u&rgr6 z%^`PAUwjxx=ru*tE^XX#_3Dl!;v5+E`A(Yp?Tbo?g7fxo+EEglfVe(Dy}>zEw=`YE ziHFEBpx05CprOnKHSEdy;89E6uoxaiYod9C$Dg!}4Z>$ua+qYi<$p#ho)P>2fzqf> z3fwwdkz>jz4D;{y9Y(#s=X%3`@6&j#L+ivD8b*J;%`_^DUOQ?C2d}GyZ&cvbrzp3) zbVIO{GUG_d+sEAEML(h{>$fa_YOAyiR(X9<71L$OS9RoIC(>;6JJFxJu~m9hE44p8 z&m2BKfUqzmSA7K=tbd+bzh|%#Oiyt$&2`{NG93lQZ$J}$MPEvajb_^y0|1a^!D*6$ z;EK-_a=upoB>NCUNXx9s1e0{iCbS7hXSFae3l9a(k>Fk)Wuuf&TPT`_Q53ea@vUiN zDSnF~1yfEP88~5Nc6{9;zP|Elo9Kum%R$uGu+_P>bG94faer7NmuZ9XMC2A4OS6Zkl z^MV6f9vI7&WAN5zP+}nv8#se7H8&m261qoV_!bONQ4NG_bLC{jgFBLq6e(A^3-EJ0a3E&?ho7lGTmpw@{8g#keC%a7-m{4;K}VE$|$> zcs&@@*1BI3#kdko`Zel@8TP^WVDp&E#U6si2v`3GUw^Urt*}mpa9{aYAtYMfT1LCS zI%RRtccA1jL^D`l4S40R0wsbqeNmU#lDrFTx9L*4=yzRIFgUF#n7nTtGv$b!)TDkR zZ5ClvJ8m&$B9fqL{Atycqb^)0vbNu%qPKY!WG-;hzb5J2$TsAMDkEW^ooRvTN?%a` zfs{sZxPR&$9@u!;tzOX4U851plt~bI@-dc zNwORH{jMspD_;i=VArcP2)(RU@!%geCunMbW7^#uST5kG`4QX5T7?1!h@;?U2S91i zV1EGnAT>@-Q2EVV^_HVzzX|km8%HD(gSR76F%x(~0I0F>r7?cX4VK>TW+YXSqI$VE z^@BX<)c|hBhEZmw&{XwHyqz%YZ6epFA}~qp&=`}jK^4PG_G8)&8+;X;6Tm zYn+zRzlIY3P)O1&kX zj)YKxUab1WEh+%0_KxDndWJm_U2j~dmQje{zUGioyAO<9Bkb^kg8T|H+PxoP34doT z3u~lTYnouwkDk}k#o#9@EKC08ou9Vpbn(-P+cB{;MY51}mevSNdO~FsR#r{YZw89) zTY}KsfDDR0q)dW|XrN{D360?AB#3=@HRg3pw>lzR&a8H!v_q)I@e#(cvnM5BTo0!2 z(o**2{R9YnFZiRGKLQrM$Y~DUISCtw$&IkY#+CrT(mz=GxmobA%A^!89{6* z!&GK}@OmxqqtnxD)a0da9w;Z!-T#6Jk|yZso6^z+lX#{Bo>rffD}7HnMBI{vd^41p zh(vAd<>Q$YLj$p=JFn&VAJEFIgzj|LUNMV#9g^5y#W-G*xO}x9pwmW>tj+etggTpp z>xC@$YDo;AM2-yB>~$-u?9krL^!g?+F;8Rt-KsMu00f}+u*uT>Ttr(2?Q1dX&}&2 z*#|La=K>q5VO*M?X@8t92Gqbp&}pg$;=86yW+S-BnL=7xXAPNnq^{z27c2_pDFHtK zH`L6&yr;pPYc{h`v}3FdsB~Nl4P~)~CR%@Qg>|{v$XEBzGnS2M4FC&l(aW#jH3RRV zWW%mHG<3&LbfSz1o{I2S%Y3x^a8dF)C8Rez{`%>myfUryj75AuneN+H(BYe@-4pD{ z{A5I3;JA0JYoJB(G!?B9M?CfBvB^fS_DnPDl73BPhGmdjEJA}egWIX~({cNuL&#Wz zqbRzUE>Q|Kf9KUw)n|9l8t$LVA>F@*$eGzG5K-{F-0XDln)%gpanc^q@hJz}ecQhC z2A1({D0D6KcKTj@hVH;}v-;{rl4K2e`On?!nalNHWsJrrf8DNk%Y*ThBY*r}ZVurD z?L7ZAvF2gsj<(B-S!JjB#vBeRyqHh)XjaVmx|p2me-T@aDviPD#ZFoMSZ?Yt*i5F2 z%Ql>L5jKe2P!JgK*HtUcJRi~Xe%v-2Zp+59v=XbW|}G*Pz9+d{nag*Kke3e znUcOj?Xr0`x9jOz4u`25H!J!EiK>KHkG17f7dBU)lR0@vInP#jP*1VtR;1o6iuSEo zyV+V3f1EyfF-WKL2wNDpAbJ#sapa7`Vu-ORta@(lqQ{e6iA~Eo-?2)cH!Jzx@_2eX zIo{j1Puv0JYMb7zwdUYQZ1_2G3vKS%H>IS$gf?pEt0KTJwRZu7_o80EhR>Neg(qX? zU~8~dlHtMM>*In=ly&6~l}n>9=E|}z_P*)9e^6E)k54~~lK0!Tb;a1(ud}vXw#S-g zrMiDiUkmhnoyG*s#L~t0?Ily6wTPDQ)tOwY2B+^{AH{ooQf-j7)4M-dM(^q_fL_qLOou1Bx%$*7NV6L86Y}loNYz$tUryqP zv{<}e3Gt5iG_kFsSWrV>Ny}8n#+C+y0G}glHhPPjyM{Y*yS$N8<<2FOv7;W>f6J{KF0NI*$H$vZ7UZ(KO^eA)O~`sZzh7te z#9cZ0V?Rc--DBpiRekL>!a8WtE5F{i`@6QYj&S*yz2n2d*gYp_TBN3Tuaruw#>d^f zT~x!uOE!x0IBXr<6PsPOE54f3Hh3 z=2>om0SmH~y310FJ|6yK{@#>{^?J#aA{+$f_*HClQZB8D1%YDmN(?>ymBwH+b5`Ea zI*ZwCiPB8<_r}E9P2NtpKC8%W*H9a7o)(NRrE`D3BSXCnpP4P6LvvGX?+-(_EIWND z3%5KcxXRJN+gSj3J zlKV^$aEP~?v#o<^_R1{lemS4V?&aWkr3dm=C6mczuq*7rV~EGbM2SK}3-o7Lh-WgT z=_@Rgr?0DAe;z|R2w&sPP}x^h->tb;$oXh{pVP&1Rm`xthxm2Wz1L=>e|Q@$HCM0c z;V5RqxiMR;?Z*ME7B9SiFAt?8nTsFJ3R?1q>c+fAVVVT%E0^*&7(IQ#^U1VmufJ<8v8|-RwlrebrG^2ch!O zK6M`_w^go4vb?h)Z1`(GaSqOQwQ$zeUWS*~`-%74X9u2-7i!>y&Ff%3b7ESr z>-jo*S+p6?*yy)QtVEpgje?z zJvPVvCFN9Q%49yj%-{Rh%hxb(@X{ZUQm0$%!lG=v7^N-q_#Dset0nm>lo$84f4#Cj zo;|mQSS)V?4;WJre;#MG@tVHCWV)np7e0qRUT-#d;~5&+t>e+!f0S(9%pvsfxOEld z3JR+!E173jJt~6eonP8oOSR$Pl_axOIB~bjldJ-gA3{1>9su4yjH{{b>-Bbd&`Z6V z9}dUEVmf=z=Oe4J2t>D9EO;K!^Xn08@(_r1_ZW3`gciqXe}S7e)hF&`KCQJe)=dUb z{2=9y&hCZMuuxyV?_OxFC-d#=wNje%{a($7j34&oWWeE5o5RC?qnS5Yj}Ij-uW7b^ z)BW?fjbBYX4D79$zi*FwJR2VNp0ZX$r0Hwr{C2O!#M-knUG69Le48EhbQ8*#z75av znb%ETwnOJIf0OMXPH!u(u+TK#wd>cG$XwmM_Q%C>v74&8S?c)aF7xB_Ok2f&s)KdX zych4fd6?yUF#uyvZqNMg?#`?0pjMZ|HSoNZRNQwwJfw3s(Vx*>9ggSe;Z+Qaqqd(e zHM4z}tF!z(tg(M$qCnd}4|VLHQkVeRKLYW9h5UW8eX)}IWVa~x;th|x~17LurZZYkq4Yx zL@C3-pIA;>F3=iSB{|RXePs=3wakmTy{@n0pt9`}+>-oe+xPS~&uhWf(%{rOhEUR&2)`wWUjTe@@=d-t^?V9(eG6Db8AI+Jb{67>NJ+ zl+EE_J%l2hfmiumH2U-WO?8})IZ)pR?)&;&YqV&k+;8W}?r|Q^@v7RThp}`kb_?&1 z^K+RP+v#vKw7hi5TDR2h(YlpwpxJA0tF-&;e(=QU4Ccc$iIvuv#+CYt$oNrC2j%eT zf8HXg_(PvY#*a(xDE&IK3K~A_@+2} zKQU&HW?lt{=ap@@1F9&-oHNgE7T1>nPt!TM?y7S2wA)7=8|U=3ou2O3#r}C|?8@-n zG{*keq~Yc@y5h+!J|~lvepz{2#e6KBe{`(iaH|j5~xXQVlzUs_V%;E9kbjtX8z1Z!*ay_vdz1HG*+uY{M zxiPuP*XGd9N7vVNAOn42>C;#EFxK9dERO0rkqZ!dH*(+mgU3-HRkQ9n{YzI{f9?bZQ*&+5LIi}`D%zl^Of_Os;!z5%!a zTrWqv>7i6CctxJ;38~UOu@Ku0f(*i4)f7fqC(X!wGLA1-Us~~!@D#HD8$YHTMgzi0=vnKDhFkyR>Q5cF2(A!u>p-%=c$q*U76u)Wz$erDw|rt=zr>MvbD0D zrT0;w-{wY5dAqmtqrW>j%dOU4U(o=_DP4KzYmpotp*1}^N~l-ne%7qqdUN;hPjib- zd;B<6iX7)F$5~1_a}%~O16j51BN>JBi=5t9Yo3PV3J=GcgGRbGSF|l%oT|gAXpizD z7vtr57i!b<9JtTjM!yxirhh3;`<8QPO-m&XH{R_jy>xS|>%-o@E;F!8IaiPPe5s5ZS>oSbyazr`5Db3E<&KRcW=@*<0i~Y;9Q{o|QJyeO<2HTNd)^RL{y| zTIKKaBcWU4K;-GUd7JZ9f#q7)?*gdT)361*K;S1)=aWt4PUqvT;`nwR={aSqO`2F{ zL{m_ze8;EWt=P|xC#%{?zDy53KVEEkcc=AWycn}td@XmgbbprDH?nbO6Gb~)ZL}?u zI^5TYr(cU!=uA;(HoiER>g`Uqxn>HRj`j*Q{wvC-dxVI0x zM{;taH?hep4`yyM^K+wIZ|}E~=jmg1D5lE*@`ZU#>Y^-nUUtOucJnmO5T4Xs^<*QE zof~pGt<~Ld{D0hGHGdjsqaKU-A)*(k<-4Xm+N)q^GO4o{ljd?ORt7zp;#} zhxh%unVgOmXZ0?K=R<>CEju~6E>__}Tfs$SC4&hactbX`qzp4)JXpPv`4P24e}9=JVa5ihi*`Pu=7QkR$twG_ zH^9-3u5%{&W4nD$uR+krkj-yHGC?Y@^Tkz;{(>cby!&H{kT!D2Wz?-p5E5IT1=LCx^*A%yN>74OB93gDECvXY%A@W ztQ&AtoPT$D%=qklS<36`o~}$w&Wr2XH3Mhj9j^Y?T4gd(q9`9Y+hP7h=N%KyP%JaR z=eNC=dR1H%8I0nS%13W|5neC5>LfKsrz_>y2*s*l7@m{oWwJ&7B}q@S<5mNaJGkDO z$6%^C5eVCKY}^j`A=kWAZ^&3}lQ6}OQMRM`nt#yzFn1OKPxg;IiuRB6lq!~P&8FzM zH}(Y;mvR|IlUsTS*jWZ)W?fGicm_G0=5OB#pzSOYG1yEem5*k5TdvF5?lrgY#PbtR zjy#3}{h7`#G&`yJdAgZd%s!68{iVFRt$y0554d4Xv2vC>4FwO<1`EqqW;(godT^O+ zpMT+NG=5trRhjQc&%mEgN4^)&^x2%hW-x6w@4FV?gVM&UeM^kR{m3pUGN)VQ7uG%Y zio^Xnc+aod?mBCq*-l^kt6WFJO^m`cavoJPN|V;Vf#-g^GTaMV(c*1f*!&W+jh3-x z&~9-n$TbnwHl0mp!`sGz{4qY2uJ%@HWq-J>s!gPX)hXwTTRCTW6Dzav`{tuIS;o5o zzd!Wx)Y`9~VY=nl1L5moS@w2G9`-M2zJzuDpR;peOKDtv(S zvN|8yyAL*_YkBd?``2ygxcIMlwd)nmVm9@C8-m9}2 zUg2{RoR{G}TwW(vP8ZE_O%WcCqkruvIe76iYMMqd=Ih%Gg?6xbr~Ewh?v|l?`@_bV z+NX7a-(KkO={zTw)9{);9bsIq60~D=v`d=RUfok|_^J-+Qdzp@C1l>^2F!MWus$B| z%I9jqUXF&>_W>>@Lj_+76}mGGM2i7c?bC5{I=`P+`_<01`5qsqcZF@T+kZVh$Jezx zys2sXdJpg8Y*ByO2rp`CAp4FSOvEQ*S3u>rt<+^1X_I^KK0{_`_3p5ARj;IGqg{}P1oid zp(g<5M}2wsPulA~tecTqG=JvAJ^JVdusd^KBVjiosUG)cHJ+FLX~?e2yQ8f4#-$a{ z{n8go&f@D5%CILcgyAH1BLp->H8C1-d}j4*=^U4zgdZ7KwK`hRdtBuhlQqur2Y z8cSGT&&eLl#Cc8zhIJfz8 zUay@){k(g_qdNo0;mtMk0aI>{QCpo{~F4N^eZ`E^FdzrJ$@8mY; z%o^;s4q6?odUoHC$!Z^8%5sj6_kBIOzih5(+hVjH6!W&c1=@+`3-D%aTPxNmj03fZ zl~a96WOL!J@z`$F_raWO#c)_JYNl#VvVa@^d~#mnv@N#SLw{^FJ-Wl!`hGpLw7wq> z=~SC9wDZhg>}qr!Jk ze9Bq+noQ=};Kg1RlDVBHPa2dbCt!DLwn#0RKT!q@01%x#$@6@?!0vf7(%qN$ax%VT z6K{v|Y1)mz^VAPV({aAK)Z_IY_~jJx=bD!*b_uV_bU!eI)9Q6NE=8z$(PXC|wx{t) zsD=R7YJbeyz)BnzB^Cz6*|=0$l5BIu`SE|tSs`~$+GFmCDzFW*7+ef z#g+3_2GiBXJ7&|CljNK#u(8SjSMLaf@d$k+7y`g?I0QiZ*FGx$v5m^(@w$1zvJe`4d+|2qb=#G!yTO-ak21mS;oQy*+8M)_v z*OPvi69C}#-S1N3SG%>Z`ln=Zl>lt|{YI#|fi z?|-L);4@i`PKoaiJ00m;s#rN*M&8~QVBH~^;BT`DZr-L59E-^By~IPG_3acH-Z zOMfYEKw|(HfD!=Kkd_2#O9J*jo%x#y8V?Wk8|(-G0^#m_wRB7XYhZIg9|1{`0Ep!^ zWLXmHAT@wU4VbgsdM6_n(urFn6IQ5*wSNiz9Fr6@PeyLYC$7-%G)NJsR0Kr1G9)!z zKy<4qAp{HqZUgh3EG$@G9bcm6O?OaK=J29m2q=O))j`6Do<}C&1cKvaXEpW33)l{Z zWKTM?2&93SA_43%fE952>n;TrQGeb2wcw=Hzt<0CcYW!(zdoFfU7581k7Js*#nvmg*1Serl!9;n>p9)oRu6UDwdIfL4xo}?sY*Unkjb=P-!WTjT@8}0TqFq zB2>aH3h3T$1!8FLtitGud|*QmAt0_`CCXlyJRY?Hm;>|>@CrH&-ihT_0X8V;-SYeE zHK|NJ(5*=kDQ=sL-GG@`(9b$_*MDC`f4)z_TDMIV`lgrf@hxk+V?O^_OINzjyZ6U> z%O?H#x??)q@YDk$HT9PD$G|r;VZ&Ri?5>toiihs3zp1xSZr3M0f3?uA- zw899()9m4SK*UT43`835VoFH9K#0>b@vFpaWDKRUG!WmAth87LavM;~%jMCO3;|Bp zK=7?S9zs873^5=wI&vyK6R3@mFCI)0RBz=7aUOt1q!5a|fj~o>S|>nt zyg*~P(Gr*e2MEe&0N%qFg*uM{AcaG?v|Pzy-+gjx7=H<)1oxzaZAL9s9fb4_f|`*@`eP>XU6K|`al~P*LtbalSD^r8<7EOPsD?yHZ089AGC?q4A@NhlCY1paak7L8 z9e+`8kYp5RNd{$Vf}pwpU*=1Y`g+5bP`CrgqqqYZAdk*ikSA(N2T0-RIWeJ3p`EAD zM05dtx+YRHz+~$8OUTh?IIZwXSX+Ri&YuF%+3z=i~?z_N?t8HKZ{@50QeGA0wkTEAxS$wn{+siee>$~+LVuYOFgdkfS1#f5Bhp`-9)XPwq*_0H(S$uv-M62`l+o32)qg4$}5~#esa=wHaYoCt`=6r zmU>VfAdij)P<>R0!a&-C>{yD->$uI}YTZ6=Rk_%TfZte7n(L)3jV9VzJbytbP{ni3 z`}}n5M>Z665}@hVQU2H`ih!V|j_LZR-uKtm)%kC(= zH0k>nXY_e4>HYXm+@#&wTHTk&A_TVMU!0o`1R+aLe$MuHHjb zh%+mBsw>I?Wv)edBDl{_2vDx0)WuxeLe4zW(3LOrt^FpmK%qMf$fm&QnfjU%_-StP z3PF%E6abC{F(Fq6n+`9EQdpc&9MmoZ1VPTJ9V;j29dX=713i|pFNx`NtXU+k4rp%p ziUXU{Kt9DTP+}{WN`L2fyX8=V3qvOW&z;{HLcssv0_`8)gUuH|esSgxw)@(%vv=Ir z(|=&aq~i!(%~!s)tHXtTeeLmJzdp2!yFUK^!k~~MS~{cr-wg#^Z-B%4d+uVpq9$v#Cm7?hOwKg+BAZG-K1Yu4f!SsNEf)R+Q zrO}FZhgG#dUgrLc%k35p{GdFd{dD-9v- zY-K{uF@KsEy2Tm7LNnmIeuXF1B$MgJ%oqa9#v~wq}<{a z?$0Ci7{~}hzurQ@bs|&)jRp2_Z@)N-s?-?8Z^ENaha#QbQej5`a}WqX9}yTksMpVc zi<61l@{5&!cv;6KfBF9}zfAtkE5{#v5q)y9_ybjflg`0=~h<$Q#A*tyg>w7&^O9m)YFPvZWExVl%&gQAXOs)(014Ly4yBEfOeMXP4DqdO=rdR3gdmDpC=fxs0c#KiK^>TgDx>VL5>O;k!PMZ((TzZs`O>6a&G{oR=8A2Kk829nNjSOOL$y1^bVWZez}e z8n9|`<$!TWI^AX&@|(%&+iufeeSby&`i)i99K~ z7#Ko%w*6!Uc!f=rXf-Y`Tz^#SrBt7grxpT9z%pe^L}?mt@*meb&c!V9t1>DDw>d#j z)D{NXodv21a`w^RIedUg0&@;*0HcoQERp0r&mo@}%;-ml*Zb?s$%h|na;g1p;>(Yw(XR`{O)8MT*L=x1EVQfb z(=}U*?)E$icog-y6)=c+=@@{Uu<3Sy9=V3lLZB*8Z-9v#c6CD%6Od+3Qyr~BDUEme zFoB_&FkKpvRU%G~6o0FC3zJg-BMiB^>^TvvI0PvMk{}3y=fVUAxpLE`L@PkJ>4TLa z)X)an8u>)fU^tlO>CS4)!jMs8`h9<^91aB*z+M2IfuyU?@hG-o-P_eg$zK|JYQT@a zKKa-$`LSQmiT>EIxBvPF57BpW`8$7nYf}7+>$~;&)T)b>e}8zB?(-jg7JH^!&>*8r zpKW|Z)?hz?83cs|m8xTTHJRQ^N-6$i*pC{LcE<^mbM9g->({d%MeS(|i z2GK|t)Lu#oAulh+`|#95UYKvvnoyZBQJq1m=UX5=2Phlgpu^>YODHHIfO|%el`Fr@ z-8KtD-7gRzv43cRvfgT$E?EI{7zS@z?pt|@C4)dz#z1N=0>lW;)6(^|?5Kb??f>nWfSzOa>k#OE z^hb;N!{0;e!>3wTnwh~X+eQ@xlIr6|JHjy8ba5&{7!&ia@X>qc{FesvsO)I^8_6y#{&L~(iKgB(yG3G8@0kQE`2Q79M! zXANVZkzt}i2&j%^$MG5Ik6WDvUK9@iCtbh0Q!4OM+RqM8atVC^Zj+Ftb2qPD(9*lx zCIKmU{(qNGw!eLcFUHnCT&45BpFGiPuRpy5Cdrw0w$6C#{_se^6ziQ9oeg#T=>a~Z zZuqg#p=1P#l#JcdDFtUi%a53I3`Oe14bzeTuvu9y)hBTv=+=QZ;3$kZW_wdP*(m0F z)TTAO%Zze8S@F8%g=%ySGa~5MRtW}zKz#z3UVrE|kqYecfZl3LEM;oMd%0D&ELeI6IwH4t|Cd@E1AKL4rnKl$?; zkLdu0bj&)8{V$tZ|HS7XZrG$hdU31+`oXTBd@*qaBg^m^#w|bUr^Pys%7#dS0cV{L z7=MuSK5rs_R;(MkNvC{;fLyU~RV0llyiybpi3$Sd=utPMP$jtQKTC=LH$^bo+&Q%( zrYwBFWg3trhN25$N)Slb@mgbGY$fJf88h9t!!h?Dlt<{RM7fh-S*66zMnbcvdTAf9YYe|lU3J|?(&U>VuE z5KAOn1SF+;RvJ)5rir>-Mfd1kJAYgebQLJV<~Hlp)eHsc74d301{Bh^_MwPuxSWr{=Nyp<`3iG#zoRau33K!K z2tnQXFajd7(XrC|L@GJ&*9;KAI}X`I==RmqVxfHW|86D){nht=lb?D}e1B}*ug4Px zHR`!y%YL-XpEyU~V?Xcjb3@w2^Ah~i{+}A2v-~eE@mU89bSO$43Yy@o!mt~q7TkhD zIEK$}rY{XwlB1cWXVRzvQ5VNz_kALm;qOx?qo4XJgaE=eAfQV0ZA`;Za=WNd&M z9{dL+K*K;p4)hh=YtNi=1WJxL?d@I-hZ3j~y(Bz-bj-fLclyp6T>fazAA9sRZGW&( zr?L0Fw7zF#{oMl)fAu80+TQzo`j6Q9YMqlWuPbQ7fA|{8$Vrx^7Js8$VBIK{^QbaW z2Ch)it~L?`$U5eo4Kv|k(B%Y*SO^TgDe19H-V|=KI7k;*a2I1S1+u0%Jm4w-RtBe24a-9 zVX5VY6y5yQ-}N5?o_{z53LR(?!GQMtivH%tL5=PL&N_Ps^6Tdr4P5lqdOta!=eX>P zGkd%I%;a?QKHpru6tv@5?T2gqeJn#d{_p<%C&#l1`ONy@|MS>+3jH!pK6olIA!P4e zGs+zyP=|Y1NcLuepy`jk$8d~z9dA*3P8zmGpwsfAhDEXIAb;ftcioJ0b>2YcL;k@Q zjCWy1An)r()vuYIU2oQzheZ73sQ*?I=%4<|B_4xyJO-R%Bsj-HbZZxMG0^G$!)Gy= zfxq(`fk0^A^K|_gN`|j)#_b<*v;1O{ejVLz9lLwu1?bn&7V1CO@vk1!$NxL~{_wVM z-R$)p{xjS4AAh~68gv*<{nP*+5z_jKxe{x_quvfK_I3ZQ$xof_?Y-I|aoJ?Nvp0mC zD@&0;1CRjx@lDj(3V(C&k1cvT0+n`};Kz>LalOty?R3z(!>xpLy55XR@<)&PU;~qV zHJNX}DCo80zheFC&)qD|PyW6G@zvCS=bE;unI|ccw14lHA(+Np?K2DFf=i_>Idi_> zEB4I&+Vtz*Hc*xa3(F;iWRvOB9fNW%-*j(a;B+%zfOmW=dfOm{(+*Sqt}E>@#_#WN z$Y;lrL7&K*iK6GH&+x(Rt+1cKR6l(N(dnAt{g?jz$#>l>9i%J0{ddnXHGgLLIve!( z9wfiLyMI5i_npPeX^$WN^pXAxKI?O5^8VpQT@LPgNL@aq@2@8I`8@0vSAb+fA&+#F zk;VkoWj)D5ke33;$_Bj3b^V$eIbQ-9*(7J4TSO2x5w}2v96{<%uj_(<`0*t)c=+Rk zEF{n`qwaMR$UcTccGmVuu2LX~4l!+LK+)|cb$}tEay!?w9!`}`eUCg!?#&Kv@ z`VT*D=m>}}pLl=pg$wA#1q|!Ozhi*q@PEX+#H*u5Q%D)>UETQMtH*Vh7nz{l<4L!y z03xB_a+(97{Q8|ArM=&FwFftTX$Ag)G(_*~7bZ_?pj$FGYtUA;9O|j4bNs(R$vpyn~wxVh)VDNkGS2*pH11Hg+4< z<=%g22whK#bU7KZj*EQz{vV$E51;OGh)6$}?xU;2-#YaV4Q3Pd0EgE{*0}oLI>LUW z?V+o{wWO5!f`Lwh7YRmC56x!k;(sinK%}-_QvLk#|BcDR!F+us+J+m z1UZR--|?2ee4mE{E-5^4j)7o0Z1}p1b&%1nruS<#+%gP$yVCnGtJuHldC!ZhZti7F z&`W>_Af@BQr(U=H%)R{f75>eKd)y@cq0@G+|H%cys1Av?ve`2tZ)tBP+SPSPB*a}Svh}N zh@MdZCv3DRyYJESr#9{KmLI`mz;v?&+$|OZd57f)XYxQ%aS$Dx}aj9T57|uA1FXiK+^T=Mx(?p(tiUI>}|iJzvqVk z%n0&d9r?#TpLqGTYZ~h?`SlIZ{?Vqsc8_(KuD`XqUyonDCHlGFuf~v0zza|$xsBhA ziG14FfHDVJ$um%c9{KD5jsTa95uMgu@}g zQHQg173{k{$tufSuA=|I)!%y!e-Y`+xfTch2gs zIsNu8y(<i7vr@Jp*=ATmk0dUQr|iHbC|L8Ti&B~b8{A-G{=T8*;j95*1x1dKqa z{^$$@p=!Iy<6MWox&Tx}43`L)gmc15bx+?VL%hTo@&Obi0e**NxecWH;r5(C;QQEw zTI&#Ugd!=I;(tsajt~xk3jyu&b!_FO2QGkd0+{170|I9P`2Q9^`oJs!C!A7*($Ylr z68ovwjg|oYGMr0$2x_?=PeG=hK6M-}X|Mn9Y&r@$%jDk3|gJS=rqqWu%Z?4Ja7k zosO)9I)9NnEc^M2SI0xPz%GE^&sUT``0cA*_c@}=8>Gw4QYHVqXQa>58HNAAZT)LM zx=n{?zq;7>o+19_r8~>$^sjPn0$y6Qfx3msT8CPsi;{Ko8HS$_6jdFD>Fz11AgDSG zk%F%G+~p!E-b=L6@tae`4Fe0-_iPYPcNwP5(SIjlKPMTmGFK=hf>4-=*l9#YL}K(h z*h2RB3Gy0`Wz_5C5R{m-V&^ok!Mq*&H1EDcZ6d*32s#by%tH0(3((wl?r@8f`^?tC{KeU)`UgO(8fbHsS0!dr;w>JKlj?!^_R*;d$ z8GqWr(#qeeo^ax!M$Mu48%K>8ZaN&5WL*3YbMM;Qs`B&;ewERAb&fJ4lZ31|&!Zzv z^#y2dc7w5v-;8XWG#Do&!AXNh^W8_)*om|D+WY^U+h`&!En^zom+I=e{%ZaLl$LPT)`-m0}eBQm28#2au zyEgNN`Vq!stdCrN$Jd_h+#bq7VEAh>*bG&~#6=v-VE-6$olbhe+#zC)v5bCYp22p# zX{RP52Wn2i?j0#PJl&0mj3T$Q7*!DosA_y|`JfvaK1Zu%Z~Igh#8K^QT$eX9h<|uU zYKNeH+LMXfy;69y)4`Tna2VZeQ4hxRib_r{7RkmsQ8%;`57!uqn|QXcR>pS4S9x_;knc!t!2A^wBw;HHngAl z7k24$jHuJ)Uu!_jTIL=(>29#gEPps(E2oV6s3vfd%R2bTN%`oM1t&7+LE88flph?E zN6ga{y&Uu`h&-D38`1kuyfbmnzJF?)t}}Pq;rlY-C#xt?)*N)piLB*RHtc_CnyJ=aU~fnw$H1Zt-lEPd!WDQctSCUa-d^eH?q1}y??B3TjJ_G zL3=EP$gpBixujhrsG45gOz%TxWFStd%*eB@ozb5ALpx*FL4}w#s1J)Vt|Kx0^z64D z&L_0BKKt6H#gAZ`cx%zmfgCVBlBF}b*G4x-1{EEyjZ9;_`=#x%$ipdrguFd)E zjrvn;?a4-B*0eJIdw*jCH1Uro{lcF)u6kK@s=_&qXFTy)?1r?#_(L+tZjA_UiuI~2 zGAfnus;H%9;S%Z1e2@XrJTi8V(n&mCPaKW5(H{{HoV)@d+UXC(3Q&IJ`*A#ywa-+w1nc#fO%=6Ur z;V2~qnNM;@&42Nnq5$+K}_DM{n#qbQGa$1^vqv;s;DCY4znmisb z=jal7H9zI@ed&ySgn9Wt>@F*1k;j*s5V;1XB8SoD-CsquU*()Cw2^Fn5!pEKsEDn( z13C|hZstU~>1eY~jqQV2lwg_?c%u>aC$E3+4?!Blynh+=_kZhJDC_+6-NCYrVflIo zGIJH!j7a?!^eN^IJcDl0{^G)i?NGJ6a;NqdWH!iUCGrh7h5GqtpQO?4>0AXi=KYiR zXaahOA12E1#glB74?cuJ&L3o0?NMlQhib_7EP?H5K1@p@BXHeg* zNblUmd+XLqrR$x1WR6nk(_eZlig_Y4>Z}4@%KRn*@@@A;AEKJVOZUr}uR^c8T28u_ zEO+gSkd{-`G<=@FyNQxa+AnTDmots@w|^X6fmlKjL4dKOfX2Z_pmUwdcvAXp+;{XeunqQYh47)r_W1;3z& z3~kO+GP=1TiO3d7jTwYx#OSVJK} zXoR$>u7f^bupur!yU4{k)5gn%zkl)4ZsL($VVz=Cz33>e;KNUHtB8oqBS&(=_+v7g z`4gv~Ojhi5LJ~im-sneApD%*cmA5OlPL=ITSkPF;Y!-`*^t%nsw2G z+ht)e77Gu?>s%FyFM*7%Mx$7Z9NsOMD{$Lus0aRf>SLMp&M8(8j6R+H$bX$V=OO=v z*1zLTjJ@?o-rZSm|DqQ|Un4geuZWi1m5t{q?sLf3FjF>>wU6@ER_~*?&7Km9gbHnExav*jjQ#{ zqU>1gNoPx^w0=z#UzE=JF_Kp5QkWAMKL#^B0d*LAWnE0A<`XhbsDEl|LF}!QWb>>X z_ikU@hdXgaB;S&RuN=L5zXN_dt*vWE6@}B;IrMEss-C-@y?5khPfCt4y~?{yvcW(0 zG`a%cM!bd$>8sD8-{L_Bz4~kuo8>dZM=OD{{#mv;$9ZE;eD*U}u(-4>pW~FjeU`(X zBW(u@x#KIS!J(;~34h%>69<81RZNu@AT{p)`P5P|S5YZs#3CzUGbMvC$pkM2;%0ab z6gk?7xRcqT#ivtnl$#U}LMlFzb>ehwK;`in=J{WE4Hf807S!LVU$j&9BU4kBh;`Hn z7Z7g(YI}YZ4eKj~^pt##?jX15K|zdp3K$>CbYc8pXS|E=ynnv@gm*I7WR34o=W|l# z$M3WG5_`$)xtsVnVE<)e2f!+@-%Uz?Sq~_GM!NZiqlL2e2(^`V?i1f+yJzg*Ch@!u|ps2 zfQ#`pM>uY8$U{IKC95tmLjIV;{cWo??=ZBwJH<1W;s%oTQ;_;8?D6phPU;Jr<01!gu)p?Sm^;_se5m-&)i5$%{^xzE^W8UQ z&yXMEpMPY-$0XbSVP8_X%O_k#I+7xBXl8kR_L9>26*~14R6t128@|&!re(D0B1$Fq zD9MqhIh07($PIloVh}~-mC1y|9Wt2%U)T08Ljo*m>xQh1al@EmnAhilPIr7T>^ zUx~_u2iBd?nmKGNxYRb$txqbL1tu;*3P#Gp(SH$*`>`vas94EKW~mj$7y-oBn2v0i zoBeI#)na1u)o;=FHNTM)3C8y%8C8X$N9Y|7^Smc|9J^hX zNGS;gD`21q0~0&Q_R3D-PLPDuKDU5V6GIUVdv00JptF_^QOSD0VnaOFZ}Wz773yys z<$t8<3gx6nq&rK_knZ^LlrOK&F~dK6h>viI4O13B^Ot|K`Nu!=2flNIE^-cj$QrXV z&olR@pls^M{30K#u;SJ>GAAWGz`Oz8@pqxJH0^AT7$eZzV@6hvuk3_XvYXWYgg|q^ zI44@x3$9v>fAH{G=$!e4J^(vv>C~cgrGFq({65JVhuEc1QR>uVo0PW4W#jotJ@=~o z8Kh&AGF~|*o>NW3wD_cwvq%dEGbm&JC%n$TsX9Vs1v)TK*{6oWP?NPprK zM_-Sdh7~L4Xq6N~i-AQ&(qu#hex>-7lX|hx`>~ga8p?d@mePgSw+*N<+KF7$AAefy zad3xr_i>R%xCIT!g$Qf z)pP>+BkC2T&sI)iKl_T4N62R|6hd zTvH#!!W-s#{j9}w&>v8*^jD}?Itt!Ue-Ndbf-?D}7k@hkSg5;{@1kd~6f2%&gr)grg0cHVk@`FL!q`?%xd8q+ zolhqHcmpy^&-NAy<3mhT27QBmh_~$J`e~{yx}N3gLYFKZl^eYwAC0e|&PXSr2VS6@ zcNJ3+e(MbvdBaDsypVtY5KlhL^*MeJV}JROqldh4et}hoxSO}Ih<`qV`XPrd;46%S zOpB$P79mj|k=yBWOt^ASV;g;>?h-zl$v^-9Xs>Y#3Azr z{R2>16C&Gy$lZe_e~a(_%G;Bk>vFQ*i5IteeDssZ6G_coZSpC;_)jQ3ezH>^|0Vx1 zy45*v>;f~Dec?MyzJG95zj@1)B~5<8E99h#0=lcwG4VO2*PJQ!;S==~5m~uJ&K(_< zzRbLUWZ4DJelqzVcvj91NS8Yb`QA^)sgq`wM^xt$IbmtU%e!dmkY?>bPM=>f=im6j z=A)4!cM8XA;+rwA9qkIIwUY*O&ip&$i=OP#XC8Wz$;^5?7k`)qCmh6YdG@_{4CZ=; zIZCebSA>dpZ$}QZDocY37H(oKJ=SIv8qcB8tuwR3$R8`w+kIHcm8aw#n`z%sSwc0R zfod(kYCn@+&Ab#^w~Go-Hm@9-h$`iw9p7i;mFG1ct;Ur<6OV3H5Ax8{$cA-8l_s9a zdl~%N%hS4JN8kC6w3>XkHM7a!i}U(|rn3`)*>bg5<1E?|r`%vCwtO zl=U^A0jkFX9|?cr?tSZgC*Ar2zx9WldV7g2dSQRK{=x@+%$4ZCqvdN5bO1L%$iMaD zYbSSH7wG4mUhtQB+~;jr!$c(6JqPPopd}Ms(?gM20mXkJRd>OvjKY%6-6azV?Jn&F zdl8hD4w9@|@TN*+zIh`&@)DJJ#I^?e3(DWo9EuEk-`3QiF0+en&J=E+g5JXwb~P{dI26^`{66tPpGAy7}9{(9xA zR(aa~qN@e&Q`g{OWyxZjU0@6S*sp`H;KF~Fb3UuieB3?ru|Mk5C;Re+UFYn1FlhY;STBC{K#yLHMNJ!^l?iaOSIDvo^X^suBb_XM?=@jXhThY(=Kzx9Oj{KD90 zTQcb9Usy-W=y+jQxQwr#edU(jqP};c#UbA8C#`>-*(V<04RHd+IcNB6>%EQJq78o% z5jv@Sb|4pA)ff|l_)rcl+w1i%TgROXl#V~B6mFy%O3A8bL5OFk+>55 z|Fv*tWdtgBoD$=4qrjI6^L(e#tbTuuD{C2HE}O}H&t0^C@>DF%Jj)iYbwJU{;u}A9 zYwU0SS3bk(%2FCaeVb5p(MV59$VB~m%@QXLangoukD_56v#_(^p3 z)a|YK8uPij=8Z|kK}K`8s(q^Mkfh@c)nemR1>%hori6l*oE|iUK#U%eF_eG*cR$Yy zBV!(aUW2r6eNw+nXucL>+xAIQ$5p=e!7s#7m2NL@Mb}gl8m+}n?eO<<5DIFWb_(%L z75?7N)h!u*FaHs<|Gk$nuUwolqXwRcJXSfsd@Cv`I01 zM>2NKpEBncKCifq4Zf%N@{bSu$Ek^oyots#AluhK+B4{`KV50LG*v4mQKpoiA;yiX zC<7TyBNEO0jz>i5Edjn|+Df3~9Al-2oIAvQK^aXmwW#=e1$KWB0@gA zLyRPt$%=FX)^BtyNYbJFUF@7pZ=}W~#4K=kRse5rjnf${SVmMtJm4E9c>yvC9MZU; zv$d+12RxbclbJm{;aQCIyokgK!-pmE<$nc!@=#j4+Y3bGNg6Q2FW`kHMB0 z<3l^^9TIE1<6D<1M>&e$%@Xc!E4ac8X^c(6e11q|ir5O6lkNbYVeA$s7)vJ{s%B51 z6(d>-mVxW1BBHtR4NdsYN{6}}yQ_Px1nHuRCAmj#=hq-b>?D6t8Sr9(c$Gvp{*}rw zzab>5Sr8rH7Rs@v?zO>MP7sV-SR+4;hcJ8gHZt0fz&9-}(s~Uxy{IAntfNz3Fp$9b166> zJjVTpIAWmDEZ~1LkewT^9+aP4iy7xG)#EArik{{5o=7Scm$_d7Pkp_AUOlf~{95PY zMclbWZJ(2tIf_+qgm-@%ws?Nt*oE<{bkaM?YKUKb zeW^Q~&(ru={DSg#e9FRg2HWW=rs8**?qVKi^5w^RobkpCw&QS4!}+{6c8-B(Kc!L3 zq~)_CX*2}xR!@3NSchN@(Sdd8{dM7 zCC!*K6OXg3!MNi05VY@TD_J?S_~q)MzncZD-O&C)M5TXa^31Z&kqs6{zr77ShfE_bN4izZ=uj_k=p+nQ%w! zlja8fSR%hUJ;LFPQH4%2A~0Z(|C8!0N$5UQ5>!{Bu?fHtP?dMe43D=pR zwX8^~Qxsu zHoxOVbnzD(H^zy5`2q}%TVY+q04(EO{*!;tBw26j3uy7vzKQYnwZ`0_n59-rD^1wS zZ#faKzSd-+C8wn%bIJVxlMn3ch7)lRT9bLh*B%wcq_dWoD+w>_>3t^>m$XkIZD(AH z4}$hAo!hZ$+pLVd%?fKgSR9r>nQk0-)`V7N;C1_Er!2~(B%8HJSb3c#ycV8P1YUpR zYIHy>T;;KHo6a)W^RwSfx-rV-Q0@0G(#f|3+F3LkqrLVc8t!Y)V1yeRMh0u~ zKgW-+=r!hXysF>|+!jgzaabNcVyb_PAK%Bkr*E8~^L|7Ov7fo`ml&fzFn*0KbnvUd zMXZC-8*LV7;To)>^B(^LR?#U})D@RG#+UXDHEE%=Yxf8x=`fzvAt`y~L1H(cjDJc{ zr4|JCd&If6LimWXZ2R;o9X_+}TUXcKG5-vHEeVkY^?);4O2kH{1jh3y%^H7lY!-g1 zJExRSOlM&Tm$*ior;v_a4t_SZRK;hOidlZm)%Y`nj6*WcnRJO)M+rA^)K>5BElzdh z4zgq;9uOC5z{>HJ*Q8F@y)B*ntEpQcQFT)MR%5M*0_)FgJ(_1!`j0z}eT~Lk7C*6% zdLYv@TY~+fR&jw+76^J=w<)amtf2z0N7S z^_Ol^>^YZDGq*f<*C!p zm$N>~NZMAeH3^SI6(qqJXg;AdH}T|m7>~9blMXPgW_=8k0S_PJUk2UVcEmZpWq09Y zCvrmc?^wNG^2c$=vnzjYKsGKT5C@$_2-Xt;70?82;e09>93p1eDq@uhDlIRPa<)2- zJ(jk=!b;?V6f7h)TcOUiEKYWC8*@P%lLrTl=gpMzc6kdrLpeWnC{%Iz>C)NUqre+K z86Q287oL1@h;}7Y`u`ts>xo6~wO>F}fM?=uBn)=fFMNkHb^(9pN(?@`6Fv;C17rANC2fbuNzIC>AAoB>s8REteYA}NcOfxE! ze-FB>PqDqnA9@+)k%UAox^}?oKme$U=sFab(}Otzl%MhzIN2)i2+tgo zVY5HgLm?+j(Tjh5OG8pTdU`QoO8G=0K1jew7tv2*Eqx;SOcrQg$(YO;QSm)dXlLau zf(8H5!2{YSOx>IzU&R{qN}CSYKZ9>?z}?PC$oR&AvGB_=V#*T^h!e+sqUJ{enr=z0 znSLb`#6Q&06tpt2F0fa8sQ2a5JxD&YEYr zgPdyVv$A&hGar*~z7^U+Y;&xMN1$n+{JvRu%5pgraeOTz$kbXS!sogcR7HugCrxYW zDEATG3&eYNTV23q9jwc^K%)4A*JmF-M82R!>nTN@ia+`@j`=~4Vu!W8FiuMjNX65JxxY?u)}l}- zfjtd&8P@Pw2UzTasoUi*6#aj=iZ9`*rV7jX5P|&^hs4f)%;z`F zyj<@8zwRB3n|Dg55(`mAVx`e0T?;&?<$bmgWnOVzz7QxpS0ep74Bia>Va|0~;MvY3 z=BmZFq#HjvotjZcW6oGiRp!HOamFK}R}eJ7BjaahgWnlzOnld~-2Dmjaj?Jp=bAR! zVAg+R7joG*p4#7IYLcrTn3@=Ct)>Bihgi8jEaec7(20DTwr?x+ub!}&BCI%E8JxH; zygwov_f}@U$%E_UtDC~q1B@+{WVF>6Z43I1HD~fAaF4MZuuiMt-?(mTrlO%v>lgnn z7JOL0Qc$yYfcn0VwGni&jFkUQWHuok=E#5N^bHz1YGzlZjC`Ua@}RPCQtqQu=tWJU`L#Y8++tdduEy*MaUDpGm1mz&UP^YnG4|8r z*?yWYf2(J3&P{QC#*hD{CgN?Z2Ag4ov2eTW<}$|Md(DemTxW8wo$;NEc7*Mm&z*nQ zkyo}^@bOr$^JYm^ws@6pB5QCNv&XZ3oCukdDOG)HMJqRJp2+9DE7W9$ux13ZHCMSG zfN&jIr$kvUtPlfRa*{MDQFQ|<&)_Xv%cFgrvIkn<-{rGD(!4B+GV1emo_UaF-KKxN z3U$3*QMHar@w5!>JuM@|yCFo22xEV=iELP-Gpym35cP-ejXxC$BY*wuBZq#)%;jW4h;-|0;u;JetQRSK z0#1XtBpzu4D2tF*(Ib^;%S1Uv34X0>@hmb&VqRkX21!ebnX>6ltO&*)VxE7c%87$E zNqj~|bc!AA^Qkh})u?OvM0?M)Y8;bOuSH6#EH*Y`Xq`92uQM3u$s$b<7h!)>fVT1Zm?LI5 zSnL&XJY|FU!i1UFh)j?mV?^_stI03K2sS+OUU)ceQf3v0#$adDQmSDdus>gTXK#qp zyg<35Fa8OqdHEfSb2<+$@aQgZeJ*en&u2|!FgpGuObB;>|%`8*G2mKet03G_*To0AJvhePJaN@E&e|-n+ymzD#2b>H7k~Nkyh=SrwWIZS;VBGbB#Wc`}OEhbE-uES;T)9dmaM%LuF3iQE!RI zjAC7dj{8E&ainuzH8o3kQ8Iv*8-Ddv^FEP4-t!hFm zj}W`VEo+Y@t3`jPq)SW7&Gnek8NZS|vJ%-kv@BLww}UAAb9#iGAcM zAlQbm{)}xng9o(FzT*}?)`%ZdU_HYpw9nqIbiTiyc+!91s@=qCK43$ApG|*<$MYM; z@d+d43kOV|YjN!coA+P#zeg@`5m9DGd*uZV8)92y4(b`}>qkA?Ro4hw-|d*pCf;kg zMT|bin!9gIQLXQQZQsczVlNju#=iVyUpziS zEb)Idt>i(_eCrU@SL_)b>Kxc^mEC$-vogABUjH55le4&KOzuXwtfzcl2S|qu&kU)u z_`nO8$!@5Fd5$$VI34JyVhiMZt~nPv24{PgiGlcQe$cM?5f?mQVHW+_XZT%5_(x3C zxxVP_53IgnO)l&6T;z)z8llPDZv!1o2o!(L($YDRSpc%;4$3x|e{|%}4D#HA#-G1T z?uy)!gvsq}#;2l(14UaaL3|UGtL(-hXz!js%?Qf0a+X&0(!-j8%|X>SVzwg5dXgYM z0Q!kr_Z2e4$#(Ixuk+O8)2X87$KooWuwU<2?^$L^r+$sYJ-2vOBaWU!L4P=yHzI#6 zwxrg9m|7;%AzGw~=;FxA#;kkA`izh(b_Xvx2?3$T(aseO^@&+=s!34{cqq>zCepL8 z)ZNu*!>H!rZ&(zV&J0&FeTG#3z_s8%a4op<`!!*RnnFXM5vGXZw+3ak2kyAk~jP)lYwvf{14A+XS>nWImzK{LxXKB}%)B(LYrB%+yDR z#zmjtcYSoWfw>^Bh#NsnobrU6>kJxNj21s)3(oz}IhmApJ*7?Mv$mNSxmvr!?}|*6 z*82p+r*9qRldASPG@QbgE%Z=7S#6zf)NA+GR>imc&}X zldQZ-Sh!BSPQt}3f1xU$QOpyo$(p9(QS1zkS(@;}Dz1uvlo9fUKspW9Z-Y;8QfL6j zxdvwP%3lT8&Sf6(pX=V6@wk6c|F7FX(js2_o9Gi9!xXUx$Y~6B!y4^;4$l)9)(F&H z;UzhBsMrQaMI*Wm7qje@CJ+!4Bk&3fRaaix+OUI{^t!$dWGzh|;n&y<{56`KDEN)> zQkXxKg^7Y4(iX8N48HzSw4-sXyX_S4rknttfc84PgBZO#9d~q&#t+0n|ec%!@<3Z zimW?=H!rsD$=%XBcy_qRWNN|AN!Y8pqe*HU#p;L-VXiyN1*NZS@jVGW4n#g?hsDfL*Cn)@YVjL z)fJC8*LO$fE?VW?fqA<~jr-LuxHGdWPwO$g4@UCv#E!>nfH8mj1%LWu=wEWlw4qn> zv}o$<-Y;Nc_?TJ4HLj?l44y6cgoy^vVlB`XZgbd4r-ryCHN`vBm2am`xw>`cNQ@)Chuzq1hROd}8o-z-sn{^~2s9kRUtYFxr~~8`N(n3Zb^Q!0HoDg$1D4@@c=&vxoO*PX;eCG((i-MC-#n+4A}+qqJWn{@ z-@#K3iSk~+$AhesuTrV3TQc+84k539NVM?ciO5!Io+o0|pBMv2BB~;arFV zBE@eHnsB^N4y?VucyH#eoE7$;g7RdfMn?q;L`Z*_A~F-5E|tLuV`z67HXA>`45Mto zbV3g-)WMsnVWKWJBNAMo5zg8JI5p; z`|W@Id-$ak?;#iDI2w&cyU|rRvY$q3^e`ITevUSy`}a{hx@t$imZRtCl{d1z(c|dh zZFCGr{L}s%t-Xh4dXtS-)A8-+aWjfv#@D0cVieED*65hp?u%M|pkl<~f8}2b(25XX zJ};d`3M#!J}3D7@{uhvz!N7EV4pufJ$r9voa&H)C$`QE9klAGwg%72gsaqzu13> z1xS2O99ocV3(X?1COUl0U!2J0> zU0kbD^n6q86=u@Q1$8lbej`G#Ge?yzD_4t@b|9}=vWT60k?O!9x!Ca2`!a=5Ecjex zo@0UfXufSa5g)MCD0x%WW*4Mg$=d5)jAKtl5@ zbv5JBCcTG8^E|l=^No_=PBi23?gKK&_xsh#byh9#_r<*4wLs*;ov#)tsavvQ!tz() z2Aq%*M})xrg;R?VJb6v>7K)vG2O%u65h=dIQQ-#M@HD5;w`vt~FRz7aL1BMe#5>=# zlE>1T@67-AKyyCdwUE7ulxoT&RaG>={q_#~{8>Yv?eZ)+_aXl0cc$9M z_rUO;@4W!bPvbJ-sfGm{TCmOj7glg&QUh3XxuAR_dS?|>{LsRR;#G+d z5@Cs5xPdg}hinmYzT(Ns4Kn4}xwcwN& z*GRu0Z>S#LA@;-fNdFHkxySKukq`Zqc6zE=jr&eQUQx5AEhlQRKst<7E76i3)Tp9lSoc4^;1nZ1V_ScM^H!~Ldhr5x_}eMb^=zX(I_ z4)?9Lm4&(QReeW^x&IV~+&A}W+u}T**T038O0Sb#^zhMwv&HiR_iae^4r%7RFP&!+ z&KvBNe^_Day7K^ZL+IXd#4YaL)nidA0~4 z6_!xR1|~Qk4ENMCp~9s&H`gvv1mj6$9RzvEGfF>n)wdh5$9M6zMQ>SuqeazskOGcx z7lxb)isO@QDGpGyC!%=uF&jTl9FAE0y%1 zTDW8RJ?=^5(?l*p0=cs{Yn9WuoRHS6PgGsdptRj23p6Ba9i?v_k$j%am0l!mW!m-{ zS{T3oN85kLRI7M1j5{{8erli1|7IK$4e7@}jB~gTWNi&n4&l=(vfG&?PTG{=25v$M zW5J7m9)Ac}Efx98ZAk6ql-}qG)oP0K{Ez1Z)xcw1E0!PCp=CAlP;jbDg%$By0NlIO z&z^&-E_O*Dm%=G4T?nDdsKGcg;UX{84F4x#K1`>L{c zg2K4k;qSG7eI|vEq>nvugR(`wlkYqN9%yPs?;<29W8k@(EW(Hj$dr2@abIQYMdq7x zlh$}P$DXi?U7p5n9)fGm??2HPp3VJPj!9VV%3TX{mM;>y2!YERzA@Jho@omW&wrTv zG*W*?=BQ#fpQ(+whd^3u4b?)^`qYmF((j7>k>B3HZs^BElNxzDe{Uf}evNVk{S(a) zx+(`bht%R882*p4liNq-;zxf< ze3#r~-{NvYZ}hb4kyf?S$FzlLkLSv;R<3^rC&|MK#wcsz_uuz$ZnHg}S?9K-T4<8a z{o!1yu|N*wH_`4)TYtaLqYu%8nu^B8b3fd`=x4uQnV@&4wI?X6sMY(Q<}muEK@KhQ z_{7lpT02&pXXKDB=1NVS`lFozaghIU{?~TpLH@O!_^0>n1if?3+`Pk5UYDtRO;&#k z3gboktBZhV$X{jO>q*uA-d4KT{$JWAwAYjVBf#;P{nFsv{HyVNXi`TXEkhnpoYNc* z`|rH&H?C$hre`{7;lP4EQd0^a;#htEKAxpY?&bT%JN}3EF!G(h-798)2=Nx=Db<9> zYDFiSWO2DNGGxN?+VxK(Y`2u5cR2z|kN~+j1@5Xay{`@_@zVCoF z@8LVwkVu>_filUYZ%adqA-E_LPQu|n_C0(DjnAej?w3p@_eO3-Ih#0%oQ3aj6I3@k zt^UP*{B5isaSyk6J;Sw5n?uVj!#9&OLO$)H9DV-%a6=|x2m#6uGn8xixNUzGKvK1b z)&cxXhR?ih6+eA1@bl^WXlUQ#XF0StpR=L8`J4~!&F7+R(Qd&b{OvppBco&B_!nt^ zM4OK^?V7g5`UgY1jcEstX}6^P5yw8Xv#=dNX4*Z5A?AUe+Vy=8Kpgy>Q~c}&!_K)_5wezhW5kfq5XgGd1ya;?y(;L3GQQ8;kXEJGpDbnA3!*^TUR(Pernea z0OhxFa8CfAEmEXA{oZnp?fYSm;xivw z;yQT|%EMpnw`(+LQ10IlliP=)ZZ#*3h=~-g3hSY}2RoLrZ_88?PSP-xG>$ zA5o^9eYL$0Lr%WiYV6Bb+b7d^`*P1*nRizoTAHz0I#e2A+#Jv05 zRtS9ecU$c9?5iy|@9i(^B>q=7oAg`_r5Vh421qTVcOvUu^@^kFU0E z+p2cZJ40~Yd-R#c4%&Za@XT6d;=P4(-$GPpUiB?IrWT;Bsz;lL?1^P7S&c(08eGYNSO zd70q(Svl1m#;dG&+6CZ>Y+l(Es91!jwobii(}GlvlYb~oB;tQuigGtUJqJ8lp$%uT z$+nbm4$6M}0B=-ofbTY(0#A=*Azk3ff7A$=qhHF!|BW41%+d1sIU0RJY2c3U!E4bu zY{A?8=nDOz#cXsn9bXOigPYOSm>!hJ(WDtG^ZeDpzat7Aa~#iSWEJP z&gCtx7*&vKctyK!q(I}76A1_O`$TB>X1LOXgPTF2$t$=8Y!oPPBFN2k1{+0o_wGp2YXG;MUl07yGqQ#KQQzjV3*gYb8P9?}ykQ3NXQO@)n=VaC{4l&*m0h zzk+K+h;Xa;9w0T2dkdt*IIK1H^S}2wCx?|JJ$Mmbe`0%q^FeKhB-~@awm>Y5AIT-T z#&K;lX>fmBW;_%i0CI!#87i>Nb2!gD!gd-wq|ad;*YMSThjk(ZK#%Jk0o-A|*7LeV zX#S=qpKpKa*UQi^G6UCK@9{dk(ZG zNGFQCAWmK&A^30VuLSF|zY`eNZEfb&^pjveb4h<{L$8wD{i$DTGmjkSvHWX$`-@Jd z3h7oOec$4|Z!tNI0%U5KpJDyXI@V_1)Ub|*?zxTcJ>Yu>`u-dGHTwqwjE`KSztD@Y z{Wsm>?`eE*j=z74&lULl6!7^Y{=dQJT+^SAQ@>1qws`Hzp`QaiHK+c}kRE5IpJw0g zPxF8G<9BC&dGA0Ub2EN%;uGV6PYiU}o%lh9`}yzt2MWirHS=HN{Sxa&ZD1JJ7T;^= zW@u;n#W`f>=K%4Sn}JR|2?bu$F@%~N@plXzd1kyv?9b3o4I!?NCt#)F6UU9|l>?u` z_ZU9#iSH>i*`4^9Ht$FJ9`-|fNgtsmQ{;auv$J0h07iC%^!fLH!kiNXx$Tf`pWzWD z@I0XkS*wXJnjpz?vlRt!jg2xxI_5kxq|%;-X!u5j{z4yMk9aq}Mps@9fr59W-%#TI z9H02DKt5q%Tnf7QkpE63dhuGA0{R1O6eY2=2mZK&<4ZJlqJ~tF5IZxIWPVh~QRH2|kBPBE@sK*4R134q-V=IR2;uGKU+@ z3WzPZk?aX%K`3C&xrR{9Bajwhichg~T+$xGXmCk}5)N{Ey?L%R#!Qgg`Nw(&eRM3m z?c(k=Vm_Ok@7FOo-4dA4@vRxVdjzPPSdE~g8(rkPti+0Fh1 z+s>EdxL$Oh8x_m7TeHlIUr+Bm%2^F3|1=qm#=!J{{jdM!e|u`)pVx=~{GSf*ACHIr z_3=Obm;dMgd}>|}|Hsq1c>KRlpPs9I|8K#1_PG4NqILE1@8;|F(C;7rtvY|K_f;`` zqaQ0(9{$a?tbcFci+}rg-~Z?B`mp{tygAa%)6>nrZ`SSOFZ+MrS`TidPN6#7FAy@r zIHb{;%?0^a{#5b|plT@=>bYQq3)*;cC0w-yegl8*@6E2&FY+*H5@#ovGrF@{j9+GkU z>n0p4EOzpK71_&uw%uCyt?srrZAhKcrr!19dV9S8a5v=c@wIi^#dY#FJ7n^wg(-_~4I4hWy8^fI#tJboG2xG`^Vr!I zXcFtacxN#ZIW5UKJY?9BbIx47^?sK8kL1~~X?1nMh7Q%|02Mf)VoA4C_X@r+fB%QE z_gH!zR+$YkOLSGIGvzwla)jyr30P?a z+9VJq0LR5o&&)@)GG-JkAvjAPe*sAnxc3~=FYpcA+w@!`#wQP0MA>E-q0X|f8wh7G zP#HZexD_V|h6JeM@K=bA(I?Sce5h*fs0{+bFR3c6`!J{I*t*Ekj7)6fE+;FO22ZEz zpL{4$o4MU)YN42N&14RW-R5S(VO)#j*oz*fr; zc9)d-LzDdQ*J=#EpY8t$e~YhtBnj+7N9V%h_FdvMDp7Jvt5NvpVm(p4M#o|62107l zZ6cZ#wvHiDx55Te@yiWs4iOqLh=v&6?63#!8Mw6VG3dmM%S{-oSNQ`tlwxh%N(LG~ z@!lNGbmJQ=cJEc^0rj8+ti*HXr7YZrNfv>9R?qIHUuY_UpCD|@e}W$0XMhv;(BorB zU!;y@?Hhw$FsEf7Q66|o5m!m*Y={C&^)Xgy-{rx|M=C34AkvCJp?&&*Fp+sod>>KC zGMn8Gpej!x4Gn6gnFke%8; zE;fvFBno_6m;l7ke{JW%j^VK{4?oa^kocvw1vn{1;)s#2i)*oz_tU!`agADg|i53(2)`Ug_N-;N0 z&r4MmrU?$;zAOhE2|)z`nc%z9hpJ#o4nv6sq4g*Mt@}CjZU&M&L!Z%YF{CK-S-w>X z4uZQp1z9XYx@SU;{dV41x_z0oG%BH1To{)rCNze0e;0m>Xdl`q%X!Cl>Q$NsJm>uU z)4i9nldb*W>$e80)A-rfymJ`_oCAr66>4p(ZNw z3!(m%q~rML6r^3|!C$DpAUxP^SUan!-Y(JykH2io7cjK$r@sA`GVs6;NO&|A?7vQ| zw(mK!f2tuA#MVzSOztf=vT_6RJFH zp+&{dR%G9tDPYJpi*yV;rl9zP7O2EMnd15EU&{dlT+ya;N7F7Teq!8&8Lh75_9@O^ z!cn3@nV2K7rs7o_O+?Ui806FZYI@nPdUj5V(zI`D_?V09mm8qC9}5Iw^5pL`cnp#@ ze|8;q2#u%-5D^0B#?2mDzXg&+rFSjF>7w_t2yk{8caL8kDv;I^UV=Aj3`TqjY(R5> zpEAy<6gQQzY2Ysxl5fXOsN%D3h<*C-fbF9uM!F#G{fx!jqMCUaKmb+8zWVzsoR_z5 zk5G$UH{q{2?X_a8&A-lojJNTKBzoRk~iymTMkCNF4ZX|F-UGl*#6dx zb+;ZF$&BY5t`*e5{9x}MIKA)Q$iV|7?ty2^G2GNHrCVDI;9A<} zdtHX*Sk;I#@9<=aI`#?``c{aqdAivD9#4+oOq--k7hUvn0tZS9zlG3593 z@DBs#uPU_@A0Y(7pe0p8=v@Wmcv1Y?xEheI9!ql>g%fbhuQ`P#dD}|*DvAUoTS91; zOJ%OqygexAfMLgaCMCe1e=LK!OQS@Dk`zA3V7Um$v*RTxz&$Fmx&W$CmljM8AkFZe zi3qmkHgeEiNp9Y9Ouhp0{WKC<-bvnxD&X_{2MeN3ramZlKge_yz6Ql!E7vrdno{51 z5pTftjmy;YIvkIM?=O|Yok)C^({Gr2+#<~+wbhh3dAOr|$pRn&e@e+QA?Gq@I$V~s zAzvC670+a0^q{mS&Cxh%&gTsbJx7GN3FLctF%ShPvaZnCugW`f(`Rs@`5e1~bg*Sr zn@Ot)VV(J{e}BP3HN!?ActTFVYx3}*z#{JPS1oM@R@6bR`H1|Yr`}!^KRkurx1m6#MmF(vn$Y#Iez_=K6$j!AK zI)-Nnvo6S6oU6~ba(~>1Pp&0Z00^uAvpq*C4YBb63Y4e&e?z@ba{o$Eu?KSuvvf=- zR7WS0R_q2J)Bkx(rLpa-dYR-Yui+PVgpz)KMO||YM?K~?$eOzQotaFVDHipC0082@ zuyYIa6Vjf!R~yy1~Og5)Q6cn3cPV52esM#iV+xpOSU>rNg z<$!U^UV!e-ch&W~3cp(_{?q~d7GisgzAkHv)lh7ve|cPj>{y2x*&7k|YKQ|Twhd>| zY9Om&^tDT+L=AQ+9?K0n)!!{~Hc7~5#)lV!QDL8KEznA#ve_PyOAzX!fehk|Bj6_U zbbCZKihFvrNvT^+Ob$HxAbAj4>o~Tot@H=&@~nYRs(~tWnNNo2ba`-}$WJqYl(2>i zkU&bIf6UvFF1p!`u(9fskyiF(mHYq>YrA$$&w@T78G%?AUnzyN+(8@X`c{Ooa%{|w*=mU- z#mk-)@!c05LVU3VdX6Dgv2u;FmPp2CJ7niie5l~NDHmC`j zm9n3o1kE8$qZp8`sESsBxxAU4!MZ8Hnfoqhiqc=aAVMB){8;&PphDjz!02REg_SCm{+|%Ao7mmz~{gb$VrS2e>zkh~%lz2|ujVxO;zYdb-s=5V!J__#D zg^iZDp5gKkV%-Hypb#Hy)>g@XR|4%te7>L2h7!WSU#HR;y0{q$WMhkme_482lBr%Z zY#tS&8GUPo430J&$!3tp=rjMA&USwJf4xJeYD*DBPw??e-!2-g2gio)_ zycM%37>*BHS7|7?C$4|2Bkom%q1Gj0B$8GlIQpaK|OB3 zthJxBox`uo24k<7B7&ky7lN&_Y*ppOH?IYLvYA zw?%dfX(}~v>g=bzXh;}clY1WEMm)Yb;)#pnHDbqj9bx?}C5tVyH7#?Mon_8A(yx9y zsnkqIZ1gB`LCY6&22cts=O z6izHrMOV-MVJ-tg$r|&mtZ99+gi_0v3capEJm+IiA%YZ-3mGK7LA|Ebc(&~#jv8TqX;$6cFXBd7%$oJT|tmB>*S6I&jon|hwrQ^18P}OlFrewFsNRcmt_OqkP z2rUgS2=6EG9w59WdBHea7yZol6G%u^6sh!8`wq_N?l6*bRWH3p? zVzWletViu`p%Hk>OXy7P(0Cz!vclv#n0_9ec(TIZHPY)h!T#)GP>bA5m083n0c2%` z%AQI54({g|fAlU`jElD7*z>c^@d=oSSzS1mudDr(TGAaH(%+w_EEL;G>*MPAim`i#Exl}r$Tp0KfQp^m!{Yyt~? zyjW_nqm?UBErc%@Tkq;l}6Dn+M3zuVEHpU zJ+xo^^)g%grQC;;mX)g$fwskuB`E(X{Hn10XVPVz+7{(<4gJee!8`Xh^&{eexzUUb zOmfg&Xs{7a)pSIFGQ0!Bkr;0r0GLXUIat!;e;tMdhd*gi(If9Bh-#t3!kjc(x|B={ z>p~srUecEyT=?=w;Ow=yR7YNlC^tFr$HsEGB8RRT;L~rD@d;ur8K$Mp63ACztY(`p zLadBhx+%I42BK4GTq*1L_JP?SY5*2l*ngkQ)?pWYv~9Y{3AQx3vw?*E4h|d$WGh>z zf9zQ3;2zMbTbww}9oS`e3TD!XC9(1f{BxJl?6X9Q9*r49NNGljW(@Uv_ct)fIz-7bDHHSlqt z^M6|LWm256t2O|a1p|NNt>p>!O^K%%W8koL!r6%33*=cyB>-5aA>2;K`#~g$XJ2{3 zgD?fsxIb=Nk?rvC?L!ucGU>UT(Lq@NyXnm0i@#a3OO(I+)!-gTW!pvSjQfD(5nh_h zH3KUpnbMO5{wug`xW15PgH`=@O1hc&A%F0EEg3Dd*V+z(=w%;tKH>!vIAW$V@uNS= z(+|5P86yQ}%;)*&R1%8DrxIUDYcZ240+2Mfn)tUYr#+o3gW8YjD6Zah4?_X=C0GgN zr_FcS*0YRqxxX88PFK33kNI1r-$`|CFbO`WynRTiVB_Y6C%a5G!M!Vp=LJf@ihpPY z;vVPRjbE)eQ+IYo-bm_j67>7F;Ywx5nW@tRHhVUd{rG!!IM!sjI^c3JW;j<=BP#*X zIX6$l?7?m5bWHkir~D$uqIB1$AB3iIr2QC_z)>uM%O!H>1SvYTsAT%U;<#_m2%|xlaE?r_EDA=ZubUIHKaLRK{erEv5aC zVh+L`_nS%K(AxYMcnZ^6r|wf|3FFXkG?(6;i{Y`b z?O+;uZ}FfN4tZ6#j>HK8NdXv$IF_({Qg{b>*ge=jX&3dh(wub;uRF?tx|66W#3DjD z8o}PS`_5T^>d^&9>ve|syZDjb*B+L$pCVmnNY`RUvg(}|`J+hVaDOaO^|VcrBLuTv zY)fhh=A@?=;i^Jpywyt=21gXA6x&ZR#mCTc?_Pc8qvvV4RY|61-uSz_%SjyawUFcr zGVfPu8r^wmh7m;=1b-ky1CU_80-l`KeEC|U*pP4ylh8mk__EZUpX>ozmTXF>A!QH4 zio}YFE#}NxDmJ%`yiauvE>6IY;u#BdvJ0aD`fTwS=eJ^rt-wrOKjWyUI@g`;ReI{N z4E-%JQ`Luo&{#$S+=-U~#@TI0{xtwKOg8>p)XG2MLRnb|R)0393M34hUE^1h;? zFLR<$h(;o$&yEb@XdDYA%(>;bOI3rfG4I+oE?YYs!+*`@RYp)fP*`G}_i$Ucr{Rv` zmDA>c)W-}pO`cFC)Xp=UW8YC1pEE<^r3yz~D!PX8w&~i3_=$)^8~}G($8j2W?3*}A zOQkujJe#gG^n|JLYjR#g69FqW0oO^|Zr&Zz%@%p7MZ}A2e=B<1& zEcZ47&Ge4o5Prt-Dl=;MqSQKfDzBzdJYBI4ZsVM?FtC1!Z_<(`ZvI&VT_?mk@yW7M zc{4+07@_YE%L^+W)TQ%iEWg4&LGTivQnQkb&wq`xfoSm7_;+=64Bq$1cqPSW2c1s= z%AQA!78`sLhI&hX%XXXRXR_-APbHGUMO@8Rj0a{d2vJZA2%#gvWH`!hUVg90Hcxiv zav{55a_DRk!iMVruo^kLSptE%2Gn4xk5VuMAA_a_l-)6G*TD*_Y)s7xndmCJ9ka{{ zE`Qv`70j<__>FvsAAwj^aG?2wrwK z%RmZwohr9XzGO;kCPW5c@)UuTXLs&g`D=|=d3gi#>URa6QcE_(qG#7)xW?rt-&-E! zjh)a{cGRmvFR=ou55+WDItK%YchGT;se5A*Mvi*f;B;0KaZb$>+w zRf$+r9R!-&ILdCAtc9AeImm&K2iBi&SZ%PxuenEN#yu{JsH0gpfHH4Z^p*^@a#Mvo z&gfl<4KLnEG3LA|PxnvGl<0f=d>)AOBIZsBbPl2@rgmB!8wl_G9uG*gQ_z%~?|s_` zK1gpi+Y-xga1{6PjSNW!_brfd=i|=$p97iwTk< zLEn&K)q3YulU`v89noxChfH65cq`H%7}uCxi%jt`T-xLJM+!9j{g@eSx7yY`FY)F! z=ayFG$4*%h&kIBIl)FS_99F#(=-|69`G8yp}tQ6+nwjFKN8zoHm|C1|%~J zfTv%){D?bG;LJ(xsUJ^5D1Yg?^#J0xd-kBL8%|nKOn4?F^P2F2?fE| zUj`CAqSds*irsS)5fG+!s8%c;pLvnS(QP{pcl~tdQ!xEy8`|bL|9>;WysL@onoNQv zR@k5k_avtP_DX~8zd)|JwtE7%nvLUS#wXJ-KcXZ})tkhw`+Z+S=wp7mqi3>M3)w7$ z%qfx>??*I%*V_uG)ERpuS$Rhe$k9b40KxD|%ylsiMPVbyibg5g$K@m{Bt!70j+4C=5O3JmaJyA3G?mf3SX6P#9if8jnJ;@ z3P2jAb0+#@W)_TJLDQ?pL+WL_*xIUr3&x}OeTE&sNY_YM1l1c2FOUrHW^M|A_L zr?>4lcU*HKdOFrc$Mb!&mit}D3hn3Fodi(2hFAU@DYR!h$0GEifTi#N_VH(vDYrYiQ(rF6_r}6NG+#Pa z=uk0_CN|7+o?uaK>nfQr_8#e)pU4@(@&tvLLJ`Ymw}?6k!#1U)+{yWLkK{UO=T5WK z?b*%lxr7^qz`m?(&1K-Cquf$I=%H;HXKLx?EX4{M&y!xIP1?FOWCg=|hI2tx$NH=F z7uDT|uzyV;b%Umut6oj&=i3_@tqIn=U3+cER*KtG?Sr=LJw=TyHoCnjUA=dxg)_Yr z&hDjZ9h+~#=S?FKUiMQqqley~ukFUId~s)=3?nfA!GlF5Qj+7FqG*S)8KC~ZvY3JZ z53;q}F(>w4(3Jn-H^q<^aTwzHIW8xD)Ezk`Qh&Samv!ToA2Tu^Gg2Q-)a3(5cF_zX zU`e0V=vVAmqKTlv;_Gc1An3^1AU}eTmt`}4-{WbQa^*&c)NrVm3SS(TTIS&*&06Va zSoR5*!@Vh!;I*hB%;zh4N(f>JM5c%_}V2MJPEto3Oi{z=96Sm65%}Qqr&9DP!<1 zCRQ3umTfEL@!5WZPP(G;CVMztV|-HZ{C~KkJg?(a3Rh*SD%MU>Lz46CudqzlEW~o_ zTdppXbAE2touMFU8H;*GTK^-Vh(6nNdzjovI`G5!_d>dvT3)q;7kj5zyFf9AQ zCaV3gGe2|b63ML-c|#`*YT9e6z`FuDc=T#dn`gFX_NtJ(Km13tPKHmj6p0-YiGN*_ zW8rUucWyvPr;kaDPI(kYkeWI{>;1muUdQfVe`d*T@yx8*%BO)ZnkY^C+pF8dBzE=z zOUfEjq|J3^6_M=|hF=PYs#y$~d_xT#@lV(klIRIzZ;!lI2s;UvXOIc1cdJbZ@+1%V zvg8wDFGRQoLKMv_>OSv04~A8|X3XjI(}j<4aMz#CSOxDmtH3-td9ec`nvA>_Hgvds-Az)(~sLM5?xFS~5d7$))x2cM3Q?0~glY8M(CmqoWbkJI-Uo-)9uU~$VfhxrlV8u| ziAkNaxWR;wHnmU(!;a3=-2ckPtf# zdoZJ2GKSpV>sQAo}^#+}nGJ>$;=Qf|Xy zQA$oj8Z|`=i#H-}-=PoSltRhEgo}B1CN?kljS^2MVNbRm-_rP_+nXW(5z`hs!>qDr zqmO)tw9v}(`-qCS!-%m3tv7cn48XVrcbf?3S6)_^VT!OpW`6+Xo?uc6wWYlJ4axfa zusRdLiP)aJHR^@=Xx3s4F8#K;mcey1 zr!Fg6#<(0>muvOuwMJ34-3jwas{jqbI4tM<-rk8YL!CX?x|Zu^+Xt+H?F7b!;2 z5729!I!jJ&_bLRH80y!O-|avG^%_B-<|kkcE%Y$siA!+7CbZxu+y;lQII{<+`V4oA zq-1Wno~z5LFS-zRPb(r1{f73nJa;EV%D&7I%Mw9~n}4&#zJQZ*!HdGGxxftH?N-CG zB>OV{z@4^`U-gnrYDqP!eXf=88EA0T{uh)(>L%zAsxozu+Br_zmeD7XVQ1*bqQgk)n>*+NyzR$N%X;vsFIwkQ+qHi!Y`OjZ^Pd}-`cZR__)9S(z)Q=R+Jm0j0h@*# zh?Vi|&;n4O8G&Q%i4PCpY%-5j@q2x%dxxeHSbxyDG#3#0H7h`F$c`r|CjPXB0xFfO zz==VehfHQW*Y$Ml*Z>XF(*0#FfB^)SKNkqUi6&|-L3E?I;bUckt+71Yn7{X+$z4$5 zo$(e;D{YCYoK>l8-gu$6n{`lP+$5Bo?!#ei^GmV?vrx9={B~B-bZN}n*fZabkHTqH za(};Im2D72bzKC0!_Vo1Y3o%QN)e;weh#hrzvaP|H)mB#7{^H9a82n(T@?;Rm@l~H z@oD-sg1+jO`}3C*+{gWs|JI_F;#SLtY3}a@OikqH^{5erQwxdxCz;7k zU$?43zNtfl?9YLB-l0~ND{g^tu8eW#AAe|stMWC40v^xtc5vV&fAcdN= zwvrX!Sq9Ja3OB8rvu!cn&~nZ4ZPNxU{VsB77CGprd!J9QZ8P|;)kZ3htp5J$Vt?#! zxwZ(pS<~T#b0-w%{Q&=P^I5n7bJ;T9k*N?+kbM~#bo7{}+X^$~s0`k#_27{Tzs1C! z_aR)q+@DTg_bU8R0pEe>dA6jV%pS+w+wV6ENBz$4%^Y8O#NvkP;X_jql&1_4deR31 z30eH@r0XM5;OFqZXI6nlmT8)hV}IL@2sH!37**<6l_9b=soTc)qeB>sRht)kge5gg zQww6>GD}J85ZEaiy;4r*p-I>Qu7^lu)F{j6F#DX^ckC!D7>T&AW9^Y})YD`_hFr~( z(&{C$)Dqe{3}E^VOoxheQ7d+yx?9*_c!c}1EIaw}+h)~RbX0w~oFeu!Vm#uOU4~xkmIWXkXs}nc#Ri&nT_On?^%{0$cN0pblJ_ zjF<)!lo&GVH@b^AGjN^B-hY^&|UJ|r0jvMRCK zuC_|9_mn^>q(<9d{@Fx{pDwI}_4l0sECd%;xI&NN^bBfoV-2!7{*I9eYerJ3063)h z1uo!|wxmE#=3VR!~NW`PAM@LmK1bC}s%dF`-QkOq>YlTY9RkL!VxsyQ5a_d(#G zl7}f6a_jx+@3BtqvQ1URtvOsli0u?4y3eqR(Xfq=Z*^pUk#k0KAh{H#)2(^Z`itjy zVi)nn(aEqjw#T?1gMXyBT##!i&mJ*uqcX(g=``N_*6kfRM*Cv80jUr+B&koQo(Lxv zmL__`{zV$-&&C^#P681On%*_5$X9IkhiDY8r*G&khFa?+9$m!(w7jl|F;42zxM5!pP0tb|Wkini`GL(;s zbY865F*4SI(1X*)=AC96EIhR;$5&jf?i)FZkP$h2nS@u+>PD1KkbV2rtz8vx{}k|{ z1PVC_)@Su$aewGr@SXZ5%%Z37?7qP+6bconr`b(Q{V`%`XV}R}jDM6&^D@#>3wjZ4 zO?DtA4r&GJ$^`&eot*ATDAfErH_}%H#Ui#paxiI<(h!9B?M3=yPF$b*5GO$Jttx#C zt(+88WTGlMm$`ZJZ%51t`XzG*+;T!uje;=rLzoZu{ph0zK|a*MNHW)RdAg30W}4=KADEZrDds&b z`;|qR;5dvrr6~*p3_q|R$7sy^dR~=9LDT&Ay$EaCc4Zgb3((Xt=Q)XfXkAoQrKciH z(-aNaHh+k|2uLf0DZPsTfMsn+zPrD&(0^fTghbIM2g19X6)SE_zCqHogQgTi)=2tq zkvet0v*N)_UEeec)lHtgO@{Z~Js$;gOND+Wow1+7s}fN25ex%NpxJ>{&Vp4{Hq}o+ zH7{~CPxCd8^G(10tb>~FL8AH!q|4$A^V3-O7 z8HW0Q>D2sxWNMD%f2q{`H>zX(3(nkZ?Zd`p^wHeTHN!~HKWUfbpXv|(N7;9$)#S^H;1 zF>;CSk_3&`KY<*n`@1YGC;d~A`KKDgFzo#P>(8D^1P?J@Qar;jmHXGrQJvrf|Hh@X32gGo_iuczBeV6t zF-~e1K!0O2yWp?<{r_yoa%_?7EEUU+5;|_5aP=;B1vxY(>WvK+(eK1x)hwk zt@{3*SAiIx zftugH^OHrUu$a1#jjure7dDS0vrqAfjUy{}A&LIq@gfqLLSkE^V{c4k?|(v4US%|W z2Q~QuV7dZT?!sc7LjH&CziZ{7=4(u98+7cQgn#2G9_~`xbVzH%;2HBN1l(ooUf~&$ z6F4PioN@FALE`?#6W0s&%qdJDJlffA0!&EB|e)I7(BnTo(Rc z>i+CNHd7*l+Gc;Bl&crhXMY5abs&SSwQ8;tn{zBd5t5T>{*`r(F*pX#LdKZ`u7?fA z5d^kTB$kmXp$&ern>ef!NZeiXO#Q$6{$J{%p%&!x&qe?n|6H6ZgD+#+e^F)2VI4*i zn37>khgt3fZd{LtQpRy!?G_N@QzG$6KBFs;|J;B7#p(XT|Np?D{(m#@uRN*Gzxw}R z@h1PN?~>T$it*p^x{yRS2*f8rjV}3*-CFpM-TJ>a{xIwcMEsmi`I`g(nFpcB7SVs& zi<$WJiNrEU+|+UcDJIJ#9D747a}f`?1sv-n3ePA!&cz>lC)YB(25> zvX5aUcSEJbI)PAQDu3_HHpdK{P%A9o&a2}qwc^hO=R1=D7b{yEfm%Zb-!Ku+GNa1X z9L8X_7$hv;_cr4wI_DT1d*Ku~8{A)Bo2e6#2@2M!1wn9kMd)cOeiDwU7r|s1$-_e5 zWhjxk=SdMoQLgKruX~oQ3)Mz%2fPN^>{mm523h7rru2%h3V&;s_3lRg21fBPgRK+W z6|g`52!g=OE46O?Nr$z6ekc)Xvo-$tA(2Ts_0Pxr2T#QO3h2KW{$U;en`d_s{m1tt zozngvzURMl_^oc@CUhqSlrPF%^ohd)p*FbV0VFQt01%cczF7r@*l zPEs~bLK3ID41b^gzhna;ea`&mne|-snmGvpTvd{`wzgC+VXgdOkBa~=&&zRNRloOU zr4Q<+{|=dX4eMgqV^}U7MoS41f0OS|cRFX!3P6bguCC;FNKG zxHm%DahWyh`muHk0eK3xbTe0yy;dT*zq(26y~N3*w1No`t5OrNRnTR~FIm52q&Z@~aG3`Aw zbe?uE{(ruhGNbPEgRrS6v)Kl^S?U_rd5zkum7i#%lW87Qg#@Dn)C$1xC9Ewznjs$H zhfdH4vgyG;4*Rkk_M2?AJlbh_9UIy|(aNrEgRXplO5G})M9dUANj#09HEg|&tR}7M zcA>PNi)3#zv|b1Y3f5!u%8y50PYUL0qVM4tGf7l?n5z-S@9ZD>^P`{?k>L5UMqBZ^^iXpVN|KW zNPjd&ayB`LZR987^~c5MGoMNNe1%qH{~edF-rG3u`s$z7XdqPNe{&0dd)TFrsSk1J zZE&T#VlDf1>DOi-Z}m!chV1hAvy%_*ZI%2et(D*V+q$55H{hR>Z4U8ru=#Cy6V~NR zxWE77cYp{&Ms@3>+evgb24k7DYQw=j27lWrM_WV!i&~|k)(0)nh%>#~t zF*kAW>iO917}$I@&x<0+j6f6ac-JDN&AUpQU(%KeK`;r+IAnP)S%c8QOxo56gauU7 zM}$1)T`96=_fy(ZbF;%|NK)wB455kr8Wti2fesMVVbr;qgL(oHtz%a*T1Uu=Yk%1l zO9@3Fx!%jJ$outhWe)ZtX6^sNB{sN*MEIOfd8Kl}Tr28bg9!+VrCO!FtPEj?c~sY` zhMLZRmK4z1|F#}b$a%N(v8zFGgirhnh>xEAh8nYcEn@`qW{u7h-UFDvh(w|~<=Qy*R$ zveW*1F(>Hn@e+5Sqctk80YZt;2HXl9Js84AF<8N7FneLpV=z7vgy*IqxcP|i>iwV5 z2W%C4F_6;|uvy+n*8t&F$kc^D)OGI%j<^T9iBQ&zY3u#S+hD&7AT{f#yuFH(62Ks(kT{0u4!9t{T5|I5ovttZ8TktGDLGU<-dgM4 zE`~Fu_R3dBoOOJ*EQZp*LZcwjm7nL=41_@Vk)dllMKRs9>8e)MvVT?q7rtK3(6{W9R5;)Fk5dYobV z-{*3xkNaEpulxGUB7eBX#SF!>Ej zsC{eKYN>CPnPh8kSNMnB&FnDU@LKGd6`EV2bS*TRobzSwqJIRIfz3GsujZh9hNzy> z)-8Ay2c6w%r^DrT`r+H@46dzUW;$oaaK0~6?(9wG44pGOy4o>qqdSm&{Jxq&k0ZQz zo6~w9*Yb$h2;Xttsu zl*rM0Md=mLG=KMj&=s$A>RU^>tYl_7R^85m?YPN{apES5*;|an45OS(qgW*WfN53n=E&l z?Fw0yRpjkfSU;z$n`sDH`3I|H)qfXfwo$#KvHEw~On>NoiOu;vI!y3})|c*){)l}! z;G^sKU`}WEH5~jWtwVNR(<&eA;4s+0U)Y@XBJb_G8n6_-mU8OBHfFz+&5QZo-pWJ= z&_U|Eb-J>XhF1!)`&_+OZPQAy z&6;<>p?@kkLrCzx!F$$b+z1KjM*oU-g>Ookd1k|CS{+pPD)nltMpx_kk-x$&)? z_wnS<=pD8tdo`#j9z?F7c#zRq5|kfy#s0`A48Hr6Ki2w`gV68R$GBFX&f!=7qL&wE ztUt%cHTTuj1WO}?;Hfa+1lS@9mR>8t-TST-;D2<4NpUmZJ}_pZcy09(vuEFQdZoIO zBlDioPC8XhR-$U}lLPrrhZ=GTPUhemwd-bt=i`GfV+BpxRU0Q11V3u$QANFSLoLYF0n(Uuilx%Hv09GMdE%=}8YR6P8Y7>cehu9Gg)KH8@ zA5b^!#lj2l9b^`wPp|o<_hky8XB{xvM40x%yiL0{4Bi@i?d)-&8 zg(3I<^5=E=dd&TC$H)2(cN)@Cg@0BsK6WFOnn`9R0cV@v+;|RbZtnP0@+@AW%YU<^ z@X=^r??oOhem=!@^JC>4GAPn;3&L-9cg#e0hF6aXY5}PTBcbomlh4w&`>4x{5XTR5 zC}&YdUEZB#R=lO@G#1u-Wu$^LLIhj$9;9VEviV2cxC2(O7N$)VYf{~`_kaDJ>i!?H z!#Bb)^P?RmeCHydJIl)x4-ycEb%7=6eEI+6%gP?_pU%~njpkl#h4&X`+4pHkRba+Y|@AiaH;Rr?Xt6JK(tlrmi#;9xEJ=6tEp6on3t8pfiqXB;0#7r zH%~tw5u64mwvaADkruG?F<(`A4Z=ZZmJ>#o5vgqHc9&be@CeRYOy39f4&}Z~7k`x(`c9d7;c|?=n zAv4{#TF>2dDZrS;%032Q7&Q6BM>~q(88wva0Lv9mJRhx&L^a(-$#gmckp^bse3-n3 z3FsIh4I8Dw7NuV=t}682{I0m0Rc*hHFN@I_x|}(K7J`!2@w3=AZ?uk~DgHgjbVbjZ ze(~_-fN5-bnSbXDx%%PE8SQ2F7be=8ZG6JnXAW@6^r6pt1K1a5u3aW(efBHEcVTx% z&_Rw>vfH4M%M>Be*`|3B`OhFNAA<4tfmO|lS%J(fKOR~6dwsKu!HBd-^|E+r8F$NT zbI2K{%`!T;gM=K>nA>^wYxXj3Ugy?H2{u!;=w#xB(0@&S$vWS$O00uaeaNoNew#_i zaJkM8Wgy1g4F|oR#n&uf!;)Du3mx5X(@5BAgRjpOOQB-BU)Nz3<{acQ8BaHIJ7=oV zugn@m(0^85^7~`&)y#?zi`0kAF2R()X5uZ%x-t-GZr4jE1uOe~q*5zg{3R7l(YDcsgEQVW#)JJ?wR#x6*#l z-_cnR`w8pczcUBz1m`)d7vLG@3WQ3G0$U&~1#xj)*@B5m3o!P#Fa<*}K2Lt7QcO|- zitqzRD5U`{$2HW*cA9t-aQ7cztO64!OF4c3D}SiwCwwP-WNidSu@%J@t9Zdvwh2OKgdZ1(BcyMt>G4%QB$ z&+)@N$#(Z(HSrqP@jEL!?~px>Eg7vfg8P5hXIc$!W!eS)>QkDggg4R zu7CA_Wjb>pj7NKegAL>Z#w9zVpXXlq*$Vu)yXJnAgDn;XZ)fa~_*jK+rYm>5>&+7w z#nI#vVG^jWl(1_Nw`Gb};p?{b(@H%{Cn*1)Les@;D??_(Unt+!km~I+*QMagFmzGD zsm5BxD~6^NT+Rj1rRR1mlYF}Fzz4#6Mt?7a(Fmpp5mZauD%G0sI*Eq(VZ|?_7#J02 zTOB$O2)iE|j3GWS*=(n8bK+mZ`_T`lTWB>2_5JoxJPbUlIp++g+S?Jk4f%t;%(pIU z)Z~YH|3f~6^3SCNHTZn;Q``I+TJdvn{qUeu@y@LH)XkfD>KcxAx&$i%e-dVXs(tcU(wc&ZRU3 zvg{H>5;Q}9oD3vJn2{q)JM$FKmXS`8?eGxp4AoW6W6Cs?~|Dq zvgMxB{EC-1UonsJ&eB(z7oanbA(m3@V;`Y;QJ@&OT)>~v2s;m<$`=f@R0l|lLm|1< zY?1t0#R+h5Iu&FqsR)^!NPkGcfGZ>e&(8Oq%;fGVXT@Xab{t_6BWx@h!$*I10V2$i z*0p#KA+Z{araX*lM-1JzpgIIXzEDiL$dtqd1u&vo^_X!e1UiKkQ*CT>1ZQ%ED&cN> z^~-t$QMBGC);In67jQL2PCOOuivfhs_Pj4@s4qOLzQ%gYA^MoE_J4ZFJ+8%;a92sb zk=C-3l{warj+|3AV3GbkYDG=Yeqn6y22qyoCt@pxOIv_tWuRR*RM0}!`W|AeaELfn&Mgl8eOTb$yXVw_wHnb zb{sl-tq8CWpG6SXEoYWuTP|TV1%xudR8+nz{HGf?2d5d@c8R9v8&*`0pTUfSY>02j z`}(KL>;J`H#{clegawO>8oJ(|Y)=n-)`;LJkU@l3uY*D|T22iB~X)*;_oKcm-~ z?bVenWzUmXxAMKu`&Bl6$V9XCW&Av2CZ45U(AOQDG2G^TOcUVw{8LEYsb%VV72GxX z=QTth)WwP)dVk%o>*3^A{>AqW>%8jy?-%Z>k9!9aU<*zujL8q ztWk_`oio1J78-3+UB_`8lzQ&0Sz}r z0NrinH?ZJtwiEyM8cVp}%F%<7;ChbTCk~qGcVNFh1FQ(a`axwZL&B@3-~xM0)B8J3 zHulux#(!TuKWukHy^!fIU$@mmUNzJl`ZYh=5j*8MviqtTk)56PK0X}Jkri!JmFwf$ zF}{DkPjPxr=a@7kk##=WXvUoPCau9?&GLMmygdcb_Im-=JcLz4ahsLK(eozI`>p<& znQxR&xR{y6uQ8YatPT*oeE5BD1DyQS5UZ(f_J4Eq!-ap~y@dtgw!Myyxb4Wz4KcUR z zb8AEF^R@gT#6p7SaChUz2#V^42+lJpmVXBcQ_`K}xSd?1HCV_gSBJ?xPupy$*B3g- zZJMDTJ{R)Y>_zkWVS+Q0YnJ3PfWg=BJ=tdQA8=y1;HEg9O|sWq;o|3_erf)ov&gL; z>b-pqqJtq;*mtyHlEZWLo?Dbw1kefc3lx`Q0p%SUk;L2xxSP%>mVz1lKiIS1XMYdm z_xa?1^!w_1;5vE^{d51McsHed#4r0g ztWWj+!%)*YKl{I(^}d!^@Kn2P}) zu4bb*i&&|QYJw&gKLrpz)&WnZ3`|o{4yRnn1mWh(b#}8~*G{9D{Eom-?0@Nhj^K8E z4C?WwDY-B%04scOQ}WG-wqenqYYV3I;rV_9i=Q{p>yLE8p=7;&fzt-XCHdjP^*SUV zKYZeWR9^=tEKpvHeers+FU0AkvNKo6d!MsaM@-oJr)~f5`hp`momnPLzrU|D(oO6Y zyX1!?Oqj9-iVl?&Zplm|5r4{sK_M!q|0pxq_Da|l*l16hL3A$Re2Id1ecQgpcGoc2 zl|*G9eU1syDl7fgCv(X0Z0^!a*q<3xbBaIbhU6Q?hVj!3U6>nFSN?#t|6v8MCx?eY$ z=Xv?BEt=b>eZ-?4E~i^Ob- zgST0ai^qWM*6l)W{)so?5qx8&l>`x?5nEA(n3x9$6E7iNf`7UH3u6k#J!M0dziV%@ zKr3$^oMqtt!iKMHUq4|G#nc`Frg#qW^3zA<&xmVuY&kkixFnex2gn@W0leQV$6;*t zyBQqR2byzT{2w*(zK%y{>Ps6xXjU%kzjb-!*vs{$-YDFk_0G$4Es*uNVTeU8$Y*Ky zG@P+t%l_=GEPt}iuNUTokeG8;x3#ncmK(6K#Q zubj$AgnuGicGbOa)qTXQKL>X&EZGFri1pmB=)g}OmG#sy7p8b5pmu`J_?d4fF2mj~ zbS-S;&K3DuA}d|^w+NM6(YY>A8RCEwZ!*^Cx>Z@`{#gJ^@iDPD7xK~NS@yp2+@)?bKiJO zZW|6AD9Zc5B%gD+yOTwE!(Hx+rTwAOA#*7of5iVAGZb_?GaNOwW!m23G^h_SS zC4Y2c#WBQ;d83sr*}_a8xK%!fr{|vk0TW=-JlGfY%inuJlz3Mn5pz{N@Vmsu3%6s; zPK4}z)MAIW;D}V)1%!xdh57~FbJAn@?E(Bj4_UnSHNL&r%PwC_?DYhBuZKPTj$P;- z|1+*T)}@B{A-bNQ_~Ms-DhUnwZ~xDu6Mq;I2Cac|J;4%Ph;9B!8bjDQj$W}l<`il4 zT0Gt*HbpCtGq~|xCljmv*-C9?4KyKkOkCf-g8=D#S>azpUqj+xOepzo>Vx(B9)>?R zDS{&X=vRK=%>AIpD>Nh}3RiNd-QBv{i9uIC8~v~Y4qNs(O!qhp)1zOYILY(ZZ+~%e ztN-T8$d~kI?Xpir4CnJ$V;%T}i`+3d=hLVEzkW+L&fw<%;$|v!&7p`R=iPHFWW1IB zV!OMC6`pX|4?R{98*(uz7!i(zTsyTS)D~x^kS}B7qX}SRzWW5iz6r!=ORl44f1f$7 z664Je=CA~Vk9P9NY_7lOZ!Qw>aer002Z(E}BkO`;U2kYz8(P<7;2DyhXGn@vCgv+& zKIYx}7!&1%t`kXLMfrs7wRy+s1fXAng-p4v3gMW2@w)($u zY6G^|kM8Ju81`tzoH%=MeGbcs`|tBm5O7$G-qG*l+I7!6VXsR>We45D!s>HvXVAfx zhhxkdeAdxM;fOI{n+N^5>ov-Xv{bB*P5*_%l8H%4&lev`s=rab``72`0q0rm$F2>T z5&NVm4q(K_H}1H$o=pxWrGLH-aJwIYF=F&F@ns@>8ujhV_Q~v3Ad$3X4I(S=j=TZm zX*gR{le;4iww~D*#cTc)S7hI+=8k$uayrYebAOUhE&|SQc=eJWS1rQwo!Rk({KtaB z<$er&!HjB7cf_p;ymn?ii<;ZVL$8lk2;ntQpPW1L`F)LA1%t+1%zweyV)Sgyn7=iO zsRo?K5e?-UoF%ZJ$DxW2<<;|^NACIg5?r5GlP@@~<%y#&`+V!7R&wUn$d~Hwq!ZG< zq<(>I@z)tByoK%g;_rOYNnbR?z4aG1FTdyE+haYNaEtrWX`D1lYJay5>s^3O`)G{M zJ$HbY=PiQ%Ju^C-RevX2Xy50yeP`Bfg;NnTRQGHn+-qha^1+okH!TOs$!w2RIQN=O zoVbipj-2gY1)l?QdxOyY4$P77>|B!J=bSr_@^-RW&W(P|oM@x5WysXL=ahI~_Yo2% z0*pSu>!-)zQCZL-@gTx@KIPtEnv|38vrPaWzn`ergE`uJ9e;yssP|;R{jTiue&4pY z&u&%kjEN|h=Sj~iQ{9LQ>aQfe(G+-V#6iC_99>y}-AO)hO9-06Z~Xq1$2Wca_>NP} z&_gv~o729YZS82=M{KXo`hk*nFr&x%Lt1p#-CQPf|MP@e00K5s|%cQ@3>M1Q}Q5;VW4@e<}c)_6%@11HTv z3Y>bti!aUDnG-8LeJBp_(MlX`CuRkesnuiPO37Sdz-|(R#Z^X}1q2NkK$cfAdUh99 zmNfK9L zQ5|c?;eV5yRR7%J$QAg|=RWm73jk6;t-sQ-oN7gT9nf=2KED#+ zZs$gesmu91Bi)1+_-ybN1WjU0IMCkg_ zOs}gS_@|3CzKAFBf#xZg?X&{jf966ysiA6=Eyg5kH+ zL!PM*+4p}mE5g~UvnHI@;>EC=*Ue{-Q>{{>eSIKq*C&3<2rirwR4?YP*PT9(MmbZ) z{{g&!jC8KKnasN8G_N9uv)=BkWBAU`?2ECw(CHt~{E2f|$wP6{E%_I{^(S6p&=Nur zb6f`CsCUI>!WC50lfo#8XlYi2^UZ`f6w1$1Q8#~kR>>@`Z5bvu6reZr4qk(9x(X7J zN!KQ#GY`J=0$DZqR>5lYyIe3V9>AJdingy?A%?4+L=|r~P{f0-`Y`%sf($+i2myYO{#DmS4%FY!kuM!=20L>( zpWuH>=h*3i4;Xlb?8x`ueAAq|VgdH<5%LG0GquHf@-g>{F3g&4woN3w?0nXF%bp8O z+=QF*EkLstR?R}!ETXjZMbff&m;228R+Uq}8h`SoR99hgyG(Fr;2wzNM6@OmqZ^i6 zwa|5ZpQhys_l~BLh34Az^{vflbPW$V{x*LwCi@J#w#SzgbG~LMTxc}6J(nC@=&>$( z_0PJPljas*E4z^ia#h&{WI7PFd5ZT<#LzHslj{{WPd+pgWAa7enc9c9dtp`OS7NQK z90{j=`K*7etj*FyFrJu>0y3KENVuG_@rLEui z>Z_-HHWrhn)XyOP2b)Qpag{Gv`4n`;gGl)XvzR9Io$7l)Q2;#*vG=&-6 zgD<)%r+RPN8Ole5crKpgUL=fqr?!9E=g)8()Su}SOjypoY$h<%N3_DSmboSNpJ_tPgoQK{)B+Fu@?3=0>6xZQt7k+4Q` zuUtS{?w9-A8o$_ZqcFrNt>sg)ZHrlAAF+Cv`)-Flf`1yljSdE#ogwMd@goj>ckEL< z^R2zUW8gwuxy3KMu>G$8FOU3U&sEXT}vl>c~6wiOc=5y}Kh+Cb5(WRfd7W>|x@^fYd=^fUx^KO_ex0;E2 zkrb@J$r`&dG(|hqu&lxQ=WbGL#UBppMPd6Q6ZIsmlCp!>)c(3>@Oxf?BK}-cj~P}) z_{6L?yxx13WR;fs5H*=Bq@;j#`p4W-ZRyRFb3z1xfQ$v9m-52!|CgtmmmB5 zM`NfJ%|+__-~;EKbSd?@pTwSfUb8_bIh?WHE{C4&@9}K#`RBNP^y|TQ?z_B(!rL)e za&6kk@7jpLB;$$3#uc+5$teev3}lelE?`_j%snm8OTel`RQi8TI8E3%Aznx|trDzg z1*17O@?j1lTqefey-X~+Ny1IIq`bY)+Y&9YnqA#Gib(d?R~egzFv3c*}m-ZSWh*@xi9Km zzxo?bo$yn3|E_d(X_sx zx@H>A7{bd8G_Fj+mR;URijmd5&!It4xv2n+X!7|o&F8S6ya<=`Uq+MIBMAicJSid8 zs(^a3pX}2TOwVhw&SrwaE;+5!fzXZy0 zr&Lc5uhgu50*A+R^<#qZkN~w zyb>`y=U@1h=C8=7(Jvz=gyUv?9=GGe9Gj#6x}4Ybg+ssjtA$lJm%i`Sl3&{AU?(w0 zouPjs7P%~E1Jb}p+8sf+Dxvbh^mYRqqr=NdMN!uQ?MD%-0`f%$1;plm2ZkYfxn}<+> zY1%Y_=*~7;;3NSnOq_TejT`)1Vmp5?o@b=?n)10;X( z?_;b;!^-?tZl|$oW;UbQQB;5YGx}x1CEV{|N#|}~>a*_gh%<&gP_Hr4+@r7A6vLXH zwbJJvgJVzp_Zd`YjPh@L()$W@%qqtH)2{b(F} z_OLL{1c_i!iBxpE3|v0Lie_fzb?Set32R4J<}g@#6EM#!vB!^LO$%IDGv`b^`Fz^1 zDY*7icjYEFg+%aZBz`&DekQX@Xhdbc#~Ak_)3>YGHhrG3q4RI){DT2T{fPO4Ak0I3 zo#(j<-{R)_qW6D%?xi}d`RB9B@+7m*6C<041e4-6OwD6(u=nKP0;?ma^_73^_$)AAT{_%@o?F+Z#eJ( zU-6H08HextdBH37eO>1~;G};!;fUe6(xHDr*Izh8ZHJukAG|?8=Pe)hxKP#f6P>+@ zx1YYwmW523l;LCrRlZcV6uy7y^=dw=P>IH5$J(ARnUe5Ya%tL`NOJOBF+%v0^Sq2J zp(QHD_GQTT%`4ibhNL*}JJe#scye#XNxDiQ0K2!L49B2v&Q|KF1GK{Lsw)M`m5yyr!jC>S=P++Lhz;i8KJvT&_tqEob6nV&TNL- zvk~ZSrC&XBZg#w{JyhZkbHz$R(>~xAZ-vf#;>OEkv*X%+mZoZ!>Fe}C93W)YC1PC?mZY6Km5X#&`CCcN3p+lVjP)UIu7zHX7x43zSFoeD*6k zaXg>rIY#S$#VChj{KZB6U;Zmj&?-$c)>xir9NS7%<=cF(?gW+=-SmkW<04~gWzEqi z8HGu?Ir${$=O(z9mGN+H-2=1bN0_v)qJtgvKyfF+9Ij?HG?#xqNAT<3kJEE%dd?l? zi^9-zPUR`z{+e4e=nn>+)R!OGf5W1~Go3Yer#V08e!`Ov+A1q&%6TTp)S8)zQgyx& zse1AGMy->H3OYwPlj>E7rYUqS3fWEPe7}2UnSPor)#sv`tOFx`g-SB(eF)NVfDIVz zjHzpy(oPptzmy)h*2Z)8!>fX%k4FxM zc+~6Zk8^m=8jOHjv;5+*j`8J`pM1~dE_jDxm{GUwXQkfrM4{*m&eDnBYc8)vgEVu_ z<;aaQKAR+@`Rrr3Yv;kfc?xQK2vd!-)EoK{jEckkG!uW&C~m{OFivHHIC}CGE@+a% zg@g%AktraJX!#>@t)Lkke$sS)BD@H$vu!`?(JV;A083=PquD*DIIX5|-wZkyuh@Lk zW6*Pb^Ree=?f3kzI6RI?zr3XxrE1V=e$5cQxs1=ld_%Gq5zn{NSYW>9U7q;~%`y`y zX@(K?Td;rE?=veBYB15MSBZSb`PQ-22`*@!vhxzG1m(ejpMjBex#mVA+qcnTYnxe? z|5&6h3%H~4KoZTZ47=$^loX#u?L9~@Z>1|Zq-EfDo*I1?0q@iUm~tNOogv;3@uBgZ z{6aWNDs57YJ`e8zD? z`jkfeRQWPr?01*z;(Vk2gh}z4r_DXfdU%H9LEFUelbN@fB<0hS^LMng84n-tDuQ3T;v1%W%xn|l-Rvf)(9 z-EoZ#{Gd7%P#sE6i650>m09Z$a&aVC_7Xn&v!2oT(4hc@+EJ zIrUsQJ)havOu%%ZjarzL`t1u&{Vg*y?PLXe%8}`BWTI+e%)+?Wfb{AM2ivP{$CDV7wif4q$6kc}sPr2F6 z!Q!xPJ1)D)hpNjZb4Dn)yP7le8E3m+Gt}h*f!0IvCVvg~?VGH}^(^t$NviEjXl8!r zvC(ri-|Wi@P4gIGo<^j>+~;p5p_w3>%i%VIYr+v8;RspKdN~;8SRQ$-9zTB^*QfXi zRq@qtKg@`^oJsJVv+22-FJ8IN$A`Ra=#Q~ypVIr0ebqiOuq(f63^R05K|%cROh6F` zH~F~Vwheuk7cJuE8EfHN@_w|Na3OyTqV zZB*JOW}P;~gD}X$nBwlD1P|0t$Q6V=F9SUu2`TRKnC$x-zu65l#^qkU9%rc>=V%ELRo*+M&4a{49ogLa zuoDv7Pk3%N(cV2o)$VSV>>9LcBa`V1fWkTTqgcfbbn?(qzFP?Dx7Z%>+GV~t*z4*T zUj`hqEwN?hH%rCR0it=MW zDG7_k4?k`H2t$2rc_3gipQU;g^|~pI%sS!%L!PQfgdNlUJr;jdUkCn4eJ5%PuLjiT z<~{dq-*-|SdzJn0tV!mOLWd_%9la+d9KAB&UK`|FR#~5v7^S~ z<%?U;#~5=_FFNTS665#&^6W$Vm_LGk_G5W5GiSnqo8K^;X(ls$6Vu$BY`R_;FQMMk zUT@~|`Jxu-Hne{a0az*#;V-L2Ky&-;Ej^} z*@lt!^)>URYS668Uejn=fy{gSWyFj$Ny2K5_Wlm~eu002OXjM`P7E_5!rfiSO!*QS z9f6}hxSWq@Fbtg!oGxib_tSd9(5hLSK0NsL`;3WW@AmvDcz(OHxuTih2zkmt2VlZa z>Nh3r@#yyip(v@(MvSI{M`rd3YzoFwz5o3t_4G)mWkP*6@u?T+!jq}d+ilZOrU3CCpd&jvYj*v9P;MFn6dryQ{@F?qH)eR3!fAzs1OT!VR*#;@v? z+1Z#So6%VRoZBh6<{7SP1Wm0)N@8kE3Vniie^;#~$=p<#rM{8t|hy{nu$D%IBeplQhTFT#-%uEM>n3ih6aoThWriM}*Rl`n#dybxN zewdZ`2%urAn?8nJ-p{*QQXT7)^cl!O8rT?!KCf|=HN{@i+|7Odm%eA1Ir@cd%FDXc zmp1HRZvLT7_{N16L45eyn*nD|SnR|J$5UP3JMwYpth<+;ORy>S=t?I0zKvaF@^*hZ z11(ssV#HjL!R@XW%p0j%-Y;kFDL_cmRB4_zKNhJLBkQiX z-R&c5Y>+IIheY`|2lqHnXzsx;!79>F=-fXP2Q-s+?i?3X^GTlrQ56f8iO*6B#L5~D zN@Hz)N2yl_o#r;N512e;KKbDU8h?NE^PoW|soP8+Krps{zRLScDm2WWt`<;xOd0s> z=b!@3K{Z!%Hr;WfNe}*Y&q=Rs(5L=}^Ia9koT%T2p3b(ecw zUO%&{TQYGU?jg)A-8;}NRo~m_uA-%xL^xHD+V zx^$C7)_NN!yA?CyUW>QK-CoUvi~nGMKi0+}cq4A@%7^MPODEUD84T!46MbYFKem4D zgZEK4;>Cv_WzGuB^Y;=>GxO_*x#6BV^E_0)zi(jP^Iy~Kc%Q|Ky42X5W@-%geJt^7 z*5H59G!*0b(+gOq)9?9=^{9XUA!IEBBR4=CeE0AA1#ohMmYUH2|J39a?EZ{C5L@Of zwxA60;Xm&W?Prf4lVaee>hqkczSpI{qvM-~S*Wax?reW6YURm0wcR}h zwbJZ_QJfN=Kk_=N-IL!{rr0HqsW+7jOzPK-6t?C3?#hAFk6=tHT#p$NxmDW`yax$F zBZwyf)9Zy1yTE!JN4q-gM4kwImqxv)Y;qe^%?6*TX*&3W#ss53QyPEA`Cc2}lwm!g z`UPn%pVtA+c(jK9bYg!2Jr-&rO#G;LJ=__t`uT*+3UTZYkk?j8v^twr?Lu}FbDU?! z9}nJ?dPc$n%<+0s2PRLFn-`DQ&o(}o$3QkIIQZGn;FAmiQ8#Im`-l1 zKLNI{VKTj!nMx%u9(>~%B-RG7li_xf@oKDBwq;V@@?bw!59ngbb93q$t-RM>vzHgv zcX~G4>(9BZD_oM%b6Rf6%-se)J_K!9# z-La8mTL{H|vKFaXi=-^kOsURi;M#C7P3%|RqclhTOJ9Gdn|NWmqxm^5s%C!lGWB39 zC39*m5__jQZGtXGnKXZx@o6qaA}Vno9P*Ry1rbyX6|mnxSGC<8eCX%h;GpTHoV^^* z!Ejf=>sj|Y)LxjtRuZQh*7_H(A!JbaV5bN!x(8EAcrNIEj(5syAcN1DYd2_1^0iBv z*$F$K{D=!Z_5&9L3Kv{M<^Gq;l?y|Eh0cUQ6MV<9fj@Bgi{COK+tV|_rkVi>k3eEE z9|*;Kvq7?2gVrp55WCC6;KK9**2`ixMM%p5Qx5|SmO!JRB*-UUabf5b7smhK&VBpc zf4&1>09|X>VZJGntH{?0_tZlEWN*uG_b{`abfgJKOeTO?pHgE9VDQnmfBa8>zV*%$ z)qbtXFP)`^rJU#x_nzTWW)~qNAWGn-NfI%ZG7yTjSllai8$W_H zKu^M4DgU77{*5O|zx*D?7;5gu!=#f*K7EhkgvGOKI!}dIuK@EkV0aq_fFz#y1T$aS zl1Gm(zYa)0^YUw7`oh2YfgksOjd87jnD9H=NNCU`69v6bHW<>z2RXb0YOD6he0v&M zrrE`;NQe4>=}|Yyq{owZP!N~_1Xc$Dg6uz{kNyks1i;h8QDp z5up&=Ne%%K9f}oc0Id$O&hSk@TVa({D#AL3gsQL)jU8loGx+a*|BWBoVTMx){=dKv zW!NQuz;pZZv7P4O{SP>QpwD;r3kMYTmaKTd^Cdn8yyxzGH+kQp3g zruUur{IeHe7XPk~bL|t02r%E3$rp$+&8{|wE)F3vRM4BVZnqkLK3EQF0eW4xN(*wg z9*)%1K_6o-#3uam>-?Apn_)m;U>*L!!&95l$dhOT!Ka1h?{<7a?X$O)G!?(VGyth=cj1O*8UN73q zob#Ngg(rVDXs*0}O?(enwuF5-)U7?D%*GuZqH6LbmM6bsOZxZ(TPzj<0+;$W`JWEi zj`^&8Fk>`H>O~!N07>z@@$l_>_W=3EMX_`DAl!Er7Lcd7UC}T6vI#5dx4-v)W2QHlm+7(FZ&4JDO61h8R?kp?n`8U{lQ5fC$( z$=2*&Z(-2S6=k@YU%0KzZ+wAzssgGDDeWYUXa5e9LNv7O7;^U zZqi>*nBhJC@y+&ihO@c{@|Hib9Dl?X`m!VZaX-3$=`)yL0R(Q(e3?}Qfdna0A(V_K z%`^-Gg@I!L8sETPgOK1W$gP0+9fH`C8I~HzEtknOcwqj{7KPJTMsf}U8FHcnhYd54 zl9-{{WE;b<0vVQLZy8UEg^^*n$FYJa^M{Qej23Vrx=#BU(~B1hV6lmahUL++KKk;2 z$CTNBm1VFvF`P8z$gn1~$#2Cf01221-S(MZKEvr?x?-q78Coh=H7W$o{-bkecM;Sx zXk|{m+CA1K)Px8yJO(A6u;_o|{z*}9G(ZXd4)>4A@%$ff6AGOF@?H9pkN;_m@ z1)oGUab?P4#Zt-eL(14uehC<$~%ox5Kzj_ z7A&z?Mg4@o`I9FkBi9l@sO^lgC+Ar47R(l^Z_jLhac#?ys?MOTXL^mnbs~g$s7&!G zri*DyWc$u}vLozwlRbk^ybs3fpKx;ig!2mC|FYk_wdi}kC!W}#|LJ-6KDLHGFy|KE zaUmE_ed>c$n6L;HXZ)vsxpwap?&uYFiWgY2N6x5cG3;rA2jJk*)+i7ezI0^fjr75M z`^fY1g&q&Xk9AIYgMDwHUX!qUyraHKFbE`AWIb}JDOH-<0}bBkg#Gk4^L(8}Y#;RQ z{eEEAoXkIjkF6@6r#V5U_ZRE)8*?f~AH5Efy{qPgvfhBC0zD3Y5R)^;{%A?0+q(^% zzPA;&d72B*VX&X@&S><7C%1>g*POiBe)oOH$ZUar`X&GL+4v!EePR;)tDoQ>^n4TE z=a0La@4INngOj_ywbh3B49N?gK&*j&IG$Yg-B;@vZDtCjo>p`kasz`!keB+S0;nAX z?9K@uP0J1Lb*sdbi};mTRX_&zV)L8$P_O) z0XvEeijM$JaS!d^c2<*}O(@e?W%d->mmOdN%+vA0Pci|2sDk`_@MPqFu_}>9gKya|nanv|I1hd zjYR+yLEbRE=}Lok^`J5pfO=T zmsQe#W6+QD-}4qi#ia)7Gy8Am?chOXKg64WO!IbR_)0LZBDLsCg;o)o-!#?5jK^3! z5z!mi8LTNE+-Ql}$G6OX@<0IigI)ZWFPqeBziqvtHfP_mUor*<%9!nmAAh#um(F0) zeJQf92|s+tnLo*dfAEJ`xop48f9iq1Y;351a?;Wt|xw+yFq-br&x5{P@85>JLkg|{Sx@W?)3Qxm=`^-k#y zX}!jg>WU&>ztSc_9dq$}xsp^_H^6Uyr6X}dve`?EcsGa0!_LMnTU-#%d8cNml-US$cd2mmQ zdu4RueS^Bhc=PRHe9Fc;Yqv$eG4;OVi}}Gk;%7N-s3|r*_9fej@d)2qYgPPs@{P{w zwfDZxegho9gu>ucUgZGPAlJhu zIB!5oRSw{Wr726BEUipwk);d3(iTg%pYl_Ei0dKG^84Q8fKJh}2W!v)-~iJCdW<$b zSb%#BY--p*Vz_)L0?!QCR6#&uzFW{?p{Q-5zkw7>l*I*L`GD0Ipc}(~4S#JwP0R#H z8L}ueK-YtNZ(f0=p=B@GV3vV5Ft0!YEUkb`00P3=S23rq0h~7wvi8drYkv*09;86E zfRcTuS%0`$1>XQnKV5VR5~K9)t{g z6L)Zmr+A}`o&c$VOIALAsesMKcb9$U^H=+PHkmL33kx{Z42p@HviaT^R)#Q2ac5GFDCl_Gu{@3@@ISv9!Tt7 z4;r8OF)e_3#4&#AdT?d)YxOKXBpWH$q<#rRET_0Jhb`Hty-F;EZk9$26K2(4cm z1;~xWy*SD(Fpm4rKJ~AEv4?4&H-%EuG;vt>@}PMTKR^#8PuKhA+) zn3;&MEkGBek(e=f^itJ3F<5rel{yrxEa^s^MuQexb(uzgIQCa^Gm{gU-NBaQ7B4R4 z)-Bg@vyNNsxyx>*a-zzexb2;Ln;d?LFYcgjo1GAF$!@cFy;IJA3^`C{!9Q&gnA%{u^U!Y!))_hy51Tn!Tc6?5=lFhC$SRKq7LH@Xmy-c z8XMMFu*x`p?{X~#AW8usqm8>sA81oj2tDvdb{b#I6d)5>xi;cQ2dqw0yf+v~2(T`( z8*UcSoU|Cp*hZZfNMsEVnY*H90!w4hyqzZ|z=2wf?Aw+I5OqkdRiyNwvZaoSAqccg zGXWx_jk?@|m}hNpdLY9%2`{5e@l?)C0X9t_8EX)Kk~u9k&&oTPNM!s)Tcw&pXlFsv z0dk32tpQ{%xsfe*Adw^X&M{1a`Vq6Tig=4PlSTKWd52-5jHq{H7;7mIqp+eK!8YN; z(v9S*Lph0%GW@DfW61ohob)NzvvO>ln_I4#zzF1=oyQEBiPdMsu%n%nusVAC0@7Hn zlgNF4_MnJ*3EO1{pck0{cxzUc<{cO$g@5*HJ(WLsf+Au7OZI!brZQ0UB#PWTN&z(H zb)u~S2;{tG?M_S(v9XLip@&{iT@UpTg?WNd4|$tJCQvor=n=A!GUBO@7zqrVhKz27%pqmIx{gf)Zd;$XE0!{UH7}6*@^dPdSYkvBPig83l1cVQN zkx?ik+vC3Y%QVeDuLTM9RKo%70B+ehlQqD41gIM~W$eL9GuhWWn=_Dh5@h2`l{ExW zSh0B}krE~yuxd?^@vk;W-apsbA(zzyrh2}|=+c8RDC5AztnX$DoN4_d8+AQz0bt9l z9m;6sk=NvA3U;Q+!bwDZqR7}x0Wqh4_XFC2t>m?gXDIE!rX=;H$JjO|K4ezz=)*tQ z>uCpJk+cBDBN8EiWl`992W2Tn$cP!OXCrMCdRz=H}%TpW2`FGWFlLPC$n!3@IO z4CY^XZkMxsEPV21G!6jjQWzEk&UL8+a!y*{8uLpJu+xJgL><{6Jun6gw3@Vk0Ll|T zSal#5%#&Y=O@Q^lFp2^hnxOO;HMzD1F!p|@8UM05?LKuNJ^K6}rC>++6YIYkn*jZp zci6j@Y~5j?XmpxWlSQGeCq87}WB^(vn_IF5VArxcX!*mxFz5iwufiZQfv52rbyMK5 zH6?k|8bX?a77C;gKn!pX5vh!SXYFbsL2SG;im!<2t8O^iSJUDuId%2&}iz z$^a)_fOS+{&>#cac3Rgeqt5EeO_N3U?H(P{x{hGV`lpTK01CkRSD7Z3KKVh{5i;fL z#W+B3%DRzCS`e6uNk-|)r(p(+1vnppqji=8HoJ&0JO)=j4=s8^U57dx9GD1FXe z0Amw8jRHS9iAh46wnH1Xe+|?AHG+TUGi2uuvZk|K!x_`;Iz({Tnk+Pvt>vO83QVTC z55v4&bPh@Vz4el!Gv~eG-}Qs-dH>YUPd6%`Q1M{ zi3Q8-sXPAjInegw5-UohNdC3-;3)irASLe#)fS0@&ygZ?#F>Jhc(foz}4Tt^Zsmpk|eo?r1aF zLeY3*kN`v-cu2;G`k2WW>|70tH$*_z;PVYpEGu=5KIi6*oz1MgstLGsFQ|tV+k5_s z#}GT4f5|bJ2(Z%DKqi7*vGIl2&Xp+*r!%bs`d^GaX5QX^Jw$QoUa+W)PLC>EVQYZ3 z<01P4|(jBtcn4zC}nh9h*WoOne+PLmOP0?Om6PB7%esz$l`*ac3*dl_m&ar{scSWF z;>8}aGy7e9@>$!*Rc}pYE4E%j4WB-8E0eR`Ai(3Qw~OY{;pTsk&*T^YW$5(w$kKh+ zN!8sd(-ph zcmG45vFG4JKRt&Z`sq3P&`-~M(`4EbD6w}(XnarK7-RS7W^QjWx`Sy>@QWY1 zk4}1g?l{o_d7|4FBa8L&MYriX3V_RGI0dk0_<6=%NAcJ18hifw-J0ls zJvJ8#ndm>y!9@RgjwkxhbH>^M0G`%yGxdu=%-V7ASUUi4C%T693x3hHpZ?1eUHa)i z{Lt~A{s$A?^V9z?IwqG%efwkI=$ZVsL(iVDvMOeCzcJPjGWx8T*638G!y2Q1GkN$o zJz+B6H~rE3qW8b)g{gk8`@MIl6K`*SjWqzW_P=RO(_|-q68r8f zdYC^)cF*gEwA9!)URWbz4YVW$k}gB>hwS(OOjl*)USzV+yUU~l-)9~}SyV}DH7seg z7(m{bD?MegeT-jeox~xapEL@LHS8xvB?6$kcLbPhHz$*tRRd_eFPWVYz}}z(MGL~B zCt4mq3G@GQapzH3$a4Gy6cloQ@kQ$<${JrmoV$_PXcPblG1B6>`zZ$?j>0;L!pmGG z)P+EEjFv)9jC!aE1E3IheON&XvHKrX=PB%-3R0keDVimkFuXgJTTzW+2ZN9~v!DRz z#ArD#Mh~j2IBvpEoUnx-FYXT77(QRzoZ;2Tzx9tgP&EL0!0dA;)$;Uz>#sBc0Wf~# zQ(yrz{+;e20@m+*2tdKiSHIH)BH;a|hxCv6645IV(cftbc(CaK>ER2gfQ1+tfl`5_ zhYbrJq5;qV#R3Ka6i6DIjUGr9!=oM9$g}8OY^GfRYj%ZkAn*zl5oiGVtlWXXnx$iQ zufd|V0;qq|;K{1~pLp+oz~TV^c(2Ib>oIx-gOmaaFlO(`nY~Lw;Dd#m@mPBD|AsNc zW%3EjuRuzn&fXIf*s*lnLj@@8-U;#m#~|%ldot!XKE2NQ#WLN-1!1^XV_EPWvG2@ub4HuWeclnk7f|S@zF*^}#%z3^Y;sKs-pt?NKR3VAykXYgPrI^DyJU;? z@5X*lDJG(t{ZrPZI!xl{-xHN7?vzU*j{)-;vFu!rJ?TuMl8YYB^B& z4`2NgA5nI@(lEYhSpT!0(NBE&HP38)Z5SW$tR3+*P67M<#@3+*zWHxje^a|Y?U}Q5 z0zU1W_(`Fk_Q)3FM{8O@HT^X`}^~LBB_~JwS>5ojn;h)-v z0PBbFwLZFO9+)5ar(X+3Kk>~(_h}bl@vT1Pc;=^y=!Z{^%(+c!fZ2l=dtcmt_$Zs! zkHzNm(@u83s%%)huM9uMXZrx|Sv#k7ma%v^Xamfi(OZn}#`y9G;G6O7XFc>k`Qruf zhM9U>pNqtQuU&sPlNi7L{U4g{`4~ztJM-XluNSj~&iDXLSC~9o(mR%k3A2~57*WVV zyvRK}uPQc`DeJmQz=;)RUY!6D=4W?d_`?SCV?zxvdUiF)vw3-at(y@@7O(-Bvp7n2 z4@9;PI224ci95os4gni)#9ro*fEh-x_<#*SIScjP0W<(&v3tv$y_DJ61G@tYARU2S0T#EM zf!#379Y8uL0c-;zLaAqs!=R(+4W_F{*2w^3dYq@1s<$xi;Uat*qrPml3-`Y-?O z|M@@9&GqqroDbRQ|NeTq6!rLT#=$@B|1ax*P+b3g`nh|I_37WPtbG0Xr9TxkfBc)s z^Z(wp*}py8`oA6zkHf#QOs;RvXW`$|Lw8z<|ND{ODi-~^2UiXhfXVIy@HGWMNz|`< z=fC}f7Xb2_Xc4_+62uy6%)PD|n%QFk6htaiV&;+v*tc|{QnjoCi~3sH9x^qNMf2K! zc1K~S&a7@b)3F&l8jO|oVdvTf_Pu+$+Hu^=AVs`N6W_LM*@c8lYY!bB#P(W;-cEWE z=(TDZZeCi~b+N9RP1$Vd@Rr>TKli4V2fk;^r(&EASJm(8*?tqm zxfbd$2(h8}qpu0?PJ31PYB-nMjiMEQsb6-LuxSLIYu-vSyKe_=eP3GKT0V2jGH;8x z(PH^fzMJ+UzJj=OHtDO_U4imw78@^EpTwwkxb6IywO7$|dkqs45=y)qmI@5Cs&XW` z)spZoYxlso1%rJI=hrCYjtO^6BF;L@pNH}INM`fdP2AIGvR%9u=KQ7Pg|^*);9LmC zyo-(Ni@S1bkk3RQ$STjecPJ(22CuQUET6G3$ z|EyGBcyE{WD!UJXC?0z~meNCfH1D7`Gi|{eDCN8?j}Xm8Ejk3D@^ly77B6P2*KF~6 zb^|BNuk%Y4nuB@6`pXrZ?WSpe+v=`j=4YFhlu7er0Jzuv61H5o=d!2WQ>%#gmq%nJ zZqRciQh)@g~~NxVjZ23F|^) zTYNt+Hn07h*Im90sm<>%ToW34A&+WQ*+|07=c`kFo>fXASb62>^dhLk^k6{vPcm0rw zm-#y09aJ5S&Su+;1 z>__by+L%3IE6GF@4AKHgyIEvL*wdPn{>zPg$ttl-c!knxECdLWt`p}nM%?a^v^=FrGJ=>n2M>%l?P*u%t zb6Cm2_Cz8v#nm8x^^#+jTL@>;PNGU**qhZu$+Wk%%V#1wJ5Oo346o&R5I8JXw`~(Q z+`S5|C3yFJ-qDQR;2E%-tU!dp}a0*Gmv?z418J*w1EH`ZQ#?`DA>$7Sd2ER|%>tSeQ2nA@d-5804Ybb7@@xO!%Kbe`e8 z9abFkOELA0ql%F!8h(n=>AK#^yOlLdUr%RvY~9)HOdjMe*~+5b31X<6$JcCMEZ1Wl zuG@s0-@|r)Uy~H)`H_L z`&E0KY3iGE5YMyYvphK7c2?Ko^|iWQ*U7mRR`2bjv{u=9$2C>Lw{<@_hfI5IwhQ4c z`PQ4`X32tE+iz~Ui8qI}eHjktKzN>(hyDI83O2WY8|VDF$kc*!;{AJAZZ((F{6gV|Va+ zSD{@u-?-Z7J3-#c&Ah&&Q7>uffCel5>z;92|JsbJ$?|q_dAHCI@Y@O zlB*Xh?oJ1*0Ybgkg}eycAWYtxuUv*Xo@t`)f2g_VJWiS9&b1Z184-{a>a8!1pqSBB z>u%AdGqFVR?Ueh^2n~u5xF%sofj4ElK zf9~s`9g2%dFP?zSm}Pc?gj3 zV7Sp_(CetaSDvojmMeZZgF^1Z&0L|LH^^#c z0dw$h4WdFDs~ZYmb^-X^rp`~kdCXyu15U@c>-wF}?P0lHKW3 zI2XKsE6&MrW3Rkx#Cz&Qt`d1?G+v-PS^&*^^*A306#A7j+FPjcSw6q-^(M=Mf8FB( z1M3*7^CuTBs`6Pw0T}80?$ysdzP1~XZ4&dG8Y8;QJJ)^(g3|@NYP9;ndI$L{))zB3 zr@2_9!wOZ;uIzDc_j)te zbKb3#bi=-{0?!@!joV&*XK4t*ELxa;k<+^tFFnWf(#7WCtUCW;cGq6wQ7u~Ua-a7u zDuNTH`btLTadkWNNd}I6VajDBZtbf)Gx60{pIbNeE5JF6ez$z(U9_9QN=?n(aAvUt zB7L;DED7_)Wk^Ugk5>KZf3~-{f)q|@^zN;m_sfu~#;~MbWr_bN-vZ`CJH?&>Xv)+rUu6Cy8Ry(080Ol>x!s@szLONUe?1MI+*@6BTdN20 z=#|nUF`Oruzagh=ta!b+?*Fd5BaWDr1*x%CmQ*5}J&<%#{<@$Kf4fP4^LhHwQb~E1gypZdC8lR%& z_R&jeb#>)a0()_1e=M!ttsKx<$?wO#Ut%&p4OGTiF9}Ek^=qfmt6A5V=YkB+Xt7t) zg@E^9*MV&$w@$u3Zcd()EKZvxc<+Hp<-5a?tvZU!t+n0H*2{NZ&44SlWiPMBp6A4H zw@jpmkFM6St;vF3%jMSJGjYSHl6HY8+Y>z{J{4@FL*hdB;0pVh#XQ9Z0bkd zkx%KoinTRY5H+%QP(DACU|C0paKxS|-5WhCiGn?Acn5|b4bFuux1xK1*IS@(&DTq~ zoYx?KI`w`%fA`TvR?oXpSu0ItFOH3H(YMyy?`F4daO9a;;(FN%qH{l-v&_3(yr&K^!@ZFZ^WEQ`Zd6wh*bcXhj7(&N4(U8{}4#ngl4 zyL>Dh_~@!l|xZ+NFBICBq+iCzF+gFb+?x(cfWC{ zKZoMk^beAkT!)+DWwZI-+nyB5OJ{B4B2#_1_e8%1%iUPYkK%0{l#tHfXHQR`@q4=m zL6|gEfADZ;_VVb5xT&flKB3t3dH%lTXG=|8kvr-dWm7|k{ zEx~jz~MPBR7cyGhMn!VM;DM| zl6m$expc3~EJSI)%bFy(>;pblxBMYja?;y97I1$!9>F8`gm5do@u66hnp{ejksR|X zf8w2#pO>W~72}C0Z{-l5n<6&M?zps_;Z4RBH7^**z5X_fWjK}x4~HjF%3bHmRo?0y ztLjb6H^VV2N0RgJsGdIrCz!{zxCE_lmcwPsA8?euEhyduhd+g89QI(@j9NDa>HTyw zsPMd;LuDIwL9W`~Wzd4>NnXltygHPHe_^SYupN|HcrOmE>A#!$CKpZv54Wd?-_X%G zjoiSSsvAq-wd5sAAbB~`es{`N?Y(?#oYr0- zc|D7C=XK+60_JqfQLW=+=NUyqiAppw99jzr{J6QiFX3U?$?;o8>T!ws{x(!le@M={ z7+BJAd@RPAXqQw*$N9yTua|?iSZSBeufS zch0}x2D&VBe97)!E55yX79?^ffB9xB>xF*Bukit&f_!)L2MX?QhpTs)yjeToP#B@G zap+Aq=J*L|Sr~z0c?0&ZoNMZb6ElcUx_-Qiswn!lv&*J?Rh5 z?}+Qm8~nCaZ6(FA9^cz07|1Q}?EIj-?G=S&Uk=8rJRYF&NR`W88!R6Sf^r;`q;GiH zi*)CPozSRL5}LQ@-ju?~Z?Ye$eDzXM8M{)SFE8}Dx zM&n__+jTtpYqYVpg|e>60w7z;v3att=T6GW4FVyIm9mb8TEu36!8t0cO>|e!)?uD0 zcwcq7fuFOJIJjF=e5viAaWAzyHi&FwI$K{aLTF;48&-3ZUQ6%de|hsNc0@&yma4kc zUh&Dj>~DK>*$WSKbKH-DzD<+OrAyw&>n&gGv!cnvvC=PBv3}&iZsyeYlRbaD*AMxU z-Mnp|pT#S8eXjPtrTBYsZ^fK3uWeuPq|4zvmJ;Bi`98iYi<+pG-JSEW_V{vrPZxZ( zDaP_5MP!+zbMlPif2@@wSUIG$ckdQtHH`G4mx0VRQF3{#IN6V*=P=&3`N|io3P0x* zvk&CBEjWnkV*Bn_+G2+1{qeAR&Gp+wZv92nz1B0T2nXwRS01#a6Vc+c{Y1))1=DSN$a;{w&!l12d+0X8|z5}E2*qvYt?EOe>%G4#M5j1)Hrl!5?@{#h(Ipw^nEURs zFj^mnu!*@?d7Hvsi?8#8nO0*r#MsHL&QPv-T=3j;k&>BF*QefnYFtxt4b-OI zd~edn`Xo2$f4p&wd-U#-LUZr&D3PR3?*fPCNT4%zS`_e?jI%Zb34CsE!@MOM6I*nXD-) zA6I(d-;PDH8qEG#XP00^LklVio^P>zy02eHe$0&~d%n)j7yw5TlS{$xNcL=mV(10y zVZimXtNr-w7GwEXoQY$3-SYL&@)NbQBM>a|;?G%k5;u#%IHBsIAwIRuRDAAcZkrtQ z^KPlDe|R9~J)VUx-bkx^?BB>^QCoq}WvAX(&y}qgk6C$yjS_XId2Y7B$~^8~kMP|m z`MNz6k7c@htrR|sj=B60g4LnC7FSNUq%hoRRj1!fXdFG`q8^rEu&tzXx>#P?KtHTs zZFUMWbVj=R>Xx~XUxYLtkELwQ6o0FNW!A3jfBqSpk#N17G=V=xz411R=H;V(6JK|@ zT?Y3>ZD#b|E3~d%bNPf>t*Q^PP*2z3a$gd`qx$<$R`;rZ1ll`N?q-kpir)lO6@}`M zB3TjUhB55MBoEW7O8vvkKDa{|&pk;#Uyn%FHU^T!{<+H!0@Cj`?ko>qE?M@L_!1np zfBvh#Q>P5lxm-KV3_R^q?5L5{9#Vhv@cOHbb1&}RDb(BA3T4nzYYueYe;(MFDY z_qrS4?Lr=_K+=RRe_Y&j;m%=bNnN?(b^X+>XB=8oJ~dXFsUBuYOYT6Zg_!RPbC}JR z^?~q}QfoHiE-Ob0b{UzYg*676${lZ-f4^!|(qBDqv6gSrq2-+}2oYT!u8X!@%ZsM> zq#e1)W7rhKyRq&wF@7%#Y})T)JU^ahwZ2)8<2$v^o$ki?f3;{c z>s7^P3&(gd_vRrScx|m~%cCK^mvc!Jc!2~wo#U~tc6np$O#p4~JO`KjW?AVVHvRP! zFHNv3(-|6#qB_%gbG)A#GhOh9*+GTjz2A}7SrdahR>UOcNPr9OHJoP%Dz$vH!V$D8 z2%J;Z)uWQX;m%2Xr-y-Ag*t*|e{#@uBjk`ie@kv5(756bP~8R>Yo@F#V}!aXzq;m* zlJt0q))}eze0}lLWa+3IX*ij<8{E9p4_0v7m^*t1s`j$$&24F4>!9hK>eSfcO@`D7 zHCmkRe3h&7QQlC?kRGk#NcVY1hwwTC%OtdSuz1#LOk#4HyK*7y9~a&_e}rvBL~l{d zh4ujlyw!quejeOeUMKd$?9S&+bi0b+XryuAywGeztS~5mt>t2Uxy4Umy?eUm%zUX? zaN}3+pn39Vi};=vYIlYAd10>?DqU#9&WF8bK90_VQa5~*hvE?QPNf>F*zb3aUy8yw z3L}MN^mIX;qxHHupEB*Te{dXHZBF*Ae))C`*F&{SE7eR_DR>sRK9(dyL)I;JU}upr zh#~dwx5sN<4teiM(mTtgx^o~%?t@(3V|gr#br6FrJjT$9tqp!Zov}PNGrZMg6BhCM zwWihPY}Z@cbFE77`(h_Pam#_xG~I2?EOT&Uj<1h;oJ zZ7)kNcx#X3C1XP{Y`UsDw0Fz-reA~-e^m^Besj9ho(h*Xn_v5dYmUxxd9B?}ESw!! z20H53MR4XeX0;uxe{y};;>Y<3+~iQSuz}I4-t5m~u`C>s+_htncIS&|bIDy`;fw}a z9bq1yAG`jl@WW{@?rCJLPyJ<9MAd1Gonc(5E!S2k6WyZoxa#Dldi>ng3SXL`t0)zR z!p;Vq=b!_*r*^Jv9|gFEZ|PRG0w{+h3Kv$}*T&kvydVC0e-Li!Zhz?N>!1PM$CfMU zu)leGvM}X+H;VCi^Xt0N9<_ScQ5U;Gc-%(VpQ$|9I#M8?7K7q>57|ENlr9HB5bo8SceVp8t7+Ra=c&b(?rUuV@gYT~sk6g;r?mp7(Ha37S>$ZaC zSxG%Nl9KRpLaPyEu0?Gbhv)e|ykE`Jx;jQi{xaM)uXDQJX~=ocg1p%XC<*ToQu$-h z15GLg_k4V#w`Gj+N;od{APk()aLo(eRojhFzXdFNf8Y|XhGtj{%G&jJk0-sKEvKYQ zy7ZiFExyyi*XSd;*4m99Zdh6XQ&$#;;4%-aSN`(#R!j#IA2^~a>!MDRTk|Z5NtSO9 zZr+EQM(y*ByKu&>ag)zUd$s)hf{x;;3qu-JdrW97=6HEOfEUdEUy9D!%3=Tpf(1D+ zQ)p&pe`m&TX0HC;wV*)3YBamq{$e=2U7aFoRk%i#x&zn5nLy$8G@3B}ddbdu?|TG; z^1NaT=F2e@AY%J|`0nbLUdr@^tTQd%8^Fp6KeW9O$BU(f4Gtl!3WWm8hFsVh4gJ9PmoO+Lrx(B zx+|!axH;a4LTjG+@%XZbxk;gyR%5>r_CRXR}^6y~aRXf3>Th ziyfUxatqDjfV*?L^f7Z&(-$Ake${h}zPZTFcI|rGsg!&%k4Oj-*0l1KcTg5LmRA_V z(J!T-18ukogWHAWka+E6aFVM1Rvk7?`)JQ#sWbTOmI@Z|_ZKI#of>HRZ?R;vPkVMv zVG0qWS}K^K*~FVyniVwttqX2!f1GJeu=^Xl;?pRrx)K<>v{po_1jJre{7uWxKsB=g zU&YTTF(rR)n<+gCbUBWTt3?ZEiqW2oYI_7t0$lZJeB6PswR^_p!)wGogbTwsb3aBj zX`?d%NAM9YW(+E-#9^V{j$Tx@TkIt9Y+@q0Ga40)a}JOfxk$d@#J%N(2}-BdLKH2I+(+p|5ArsE{efA_&_buVk8R^2qr53n5+6U$FOI(rS>B^DKh&0w8o=V)V*L#^n2;69E7>r&CN@czf%>)M9)d|XUBd{0wn zbyluoq<5WWoa<4ddbYc%YcLn=6iwMSJ6eQ((d$#2h8*Md5HzQzzB$bOeP9bQzn~v# zcVS4xb}lF~L)eKg zt_i5Sc{;JK&8RDebAu+2?4IJwO+I*;oT{hok}E&%zW~V0#ovCZIT9J~z6&pdPQR^l z-hvv!aqD;JR@pD86CpRg*6#z^4+>N1IDL`yv1(_O)6KRNe=JEA_>`BPK=dLT+RR0e ztXLdq!1~`Zj@vPHBaH zJ1nqhalBNd?Bq9tG+UI=K`q2S8tFWzh!UgImxZ&CG^K1EcKR=@Oml`*C7jA;)8^1& zuuPJ?UnoUge;sm!#rIA@-L|w1Ng>NnI!pHseeMa6Q7q(w6z2QJBpo!*i_eWvG1{M#NG=)tBcw0`u{qZld9x2L(0(ii zxRQEd=v6w(ay3k47BU`eqr2g;>)_gxq&wA$POP;)f47HvE0>f8N^Wko^vRUmvi>H6 zEK`pS4o47pEP&&kikA6@ENXhp^8xR_S14Of%b04#$?_(f`Dzn4=u%a5^Ku_0T;{`} zH;0Jp3z9OGCN_-GeQ)4rsmz+*fnFWCgXR}&7NX!C#IZ#r0zt|!4N(#7;}W71@GTB= z&$xIJe_`~+0loviaN%n0>jA**hAI4nNW@4CWSV!nJB@e~e_+n{@q`Cvd>*c-3l`mlV_@e^ph=lXWk(l7F{CBgjZz?PdTC#pwed}W9}gtM6l|&P`}PcSuE2nR(M ze|PQJk9{BM=|!(Ook~&ppj><1Y-9V3zhxK#}rc4lkE#Vop?*KnN9SwXj}^h`Y@+DI>!T z3khGDF@h@Rm>%KjRwiN7bh~1u6tUO!e>fh3E$p`KjUUTHiDkrSUMm_)|0`uj&Wp}$ z*TuE5D$>Hfc8jtyC(uOiSohu}THHKo7=`Y>Zv~dO*77zO$!q!PHxF49EhdZh;d+ux zQL4QeAc4=m3_Tz7c;I4sEG^XF;%p6pKr7UoHy#6Ga7xAbd*6(hgo#MJ4v#6GmMb) zalb|ImQil4a+wtA5#8%bv}PH|e=SC5a>6lsiUhi~2cVe(`H~J4i{i_fW>5Dd$0<`s z6Z5i>m7yoJGy{IENRYOX8@uq5yp*NCTxtwG0pP;u0o9m2zRby#$bP`(RxR5?Xh<%q z2C`ccdU6tM(5OKGgNRDIFAk8iFmYH*GmZ4)el0SVj;+uxbEH2Ihs&41mbeaEhCWVay%_N5ira$G^my0T@?SjjU5!4|6ZE~zzD8R;c}Cz1IKR1GRs@PvZ}fANzQ*Lj70fh3gi z>k5rYTJWP1##ElkiLL!MbJD`!+fmM>XqsLBxO}_B9FMv}gR=Rq2A$`|Vki@0XQ9>9 zCpVrqD!OYpBXuSZyu_;ogkYoyJ~V^tzb+`~0y0C*=7%utx}~Oh)5!A6#MIvhklR-M zhUX;dw(<6$oYnU5f8Y|^#OdbHE@Stj3aFnCKJ?}{E2%-*@w_5wMKY;7ND`bz%Yi{a z2YKC|>MK}1KLWJL`>o|As|eSv%aX`}Yj}j#JprGF4kilQO;NdLK-F7$dk;(`gYv$B zg}LkctJJ4WjY87IQ~e}>vY&V!+B6eNz8!J=-iA#L4J%gQe;n_d^g7uDZTgG#z2`zQ zHVqb&a;jB!m1|_hC%w_#s{eEn^6ES-3xk=Z5?N$j$YDav9}k)emMk~qM~pI?xPw%d zTyIc2I>fvmEwymxJk@xnYF%z*b*1OI3oqoUaYp^UjT7v@GnKc(fH8m&wC0Y#vlX@d z&=UOu~_-6RC_*Ibo9EO?rs(Lb3Q9~b`OvXf$GEY!+ zBN_b7WaYymX*bs;ezB%VZHZPb2fVHgZwtx85lA>pe}W-1>5D?cq6Z%WY{2k)mh#M)XgnsihxqHu(h>dLq+_*Nsn z#*d2)D#K-F@PK|WEcL+NcBfRr-C=s_qX?7f-!(f1>&JI)^Z5V!{z=I-8gfr?XB=Iz(O2>25Kd7$*IoZkw0lBv`{I08us zJ8-6Z_CDrS=|M2^*(<%q{kFBq<;GGY^b!9>yi}cF|I5=QwwL;AwAeNd$-IFr$j*I z2I?!<@1ADKF#Y}P;m-47?MeYCDj{+Ce;OxYKR_9^J%sIH3A&N6hVoZz0tWnk1inI1 z={A{sTR9}^y?yT3GlsA{K+9zqd`dg+Avl%Lx!m71PQ%7ji}%#1s;^5Sh^Iuf3ZoL zaevA7IJ#Szflh`Ug^_0WQYRU2!=H*4W?*<%$X4!5^sI zINbao38dnH9e9i{k9146-zYBr&n$sR$|c7_`es~tw8gj7BGZ?PVmw#XS=*ZA8y-ny z{xuyx4_x+UK#b!60j|DD60soNe|a7fSc}jnFq`v>F6i^PR{*dQeF6u18$Rz()nwgo zz#Put`28^zsXCm5+3p;f4nua17lkDY%D_6qB6rQR5RVE!e^ukBdJv}4?qRaI3A?ia z?Sqwf!*$7E$fv>z;hak@eIi^5oC%?K<`Qa0%=H2qrbA))p(B>uRFg2%f4EBQQJd=Q z(h+AC*hLx>-ZXjWDJRF=p3y7m!k3i!9V06SoNH%5r{S zMQL?1?tO>R;izGW!#luT>4p#<$S4XSfopg5n%UJJ%~hiwW4Mr$E}=R(WL;Qm1ON$~ z=3IECOX|T6Q1oG4f5R=;ZlY#lr@Bwem@wmE_bZ66ckuj;k*XTS_|DLum@ig0vh=Lg z+ml^BSpD+-_A4;W@M;XQL`NTOEu1ykz2a8{f|jJ+H#%y1QMow9v)07AVC!5@a>eR9 zaCI!JL(Q0=oZdRRUc}}h%_k3qcI%hc@i=F4%_e)^ATEZ^a;IjfU7df@xzA*bG0r&TS}b zNRxT0x@yyGk$}+Tr19|Ls@iyM9T&Qr&2l9!AT_7gwsedsK3~JO#FYaSY z*g=l=%J+i|f4Ys-K9))1n}>_sya0}*4}fPLV5q*%A)uzl(-L4@L52QLwP zGBf;IqjYOKs6I~a83)pW?3+%0R1_WZWcACzP?yi5&bhnQ{^%u~DHsG?Tcr(nqg2j{ z+{%92qci(gQ42SN|1@R*iD=4UU68XKh7e8(+_B(-EyJ$?~S-;GrMfiIC zk&uW)e-Uu>OMb5lQUmhn|3W^_VqGA|Z)v=6QZ`Qk-43;`}O?YuXMNsQEca(0L=S)eGfqh?${;PX!f= z&-eN^cIwhwXrxYinN3t?}d%Edm_7ld(+Kjr+p|TR71AxMlV?NLv4;LNct)yJNVJ33VSLC=~sjJXw znYJ}N(1C$oP=xmYI-?e~8Vd?n7py7Yf550=-)K~jNjlJ~`^2&b@_y)5ycG^0N%i~) z=W9#bu`lLO%{tz_Sbh{GnYjA5b?-B(F=FlP)dzLpe_A4Rl4_ za2eXOcr*oK5j+GrF@HY>=>V6}dMwym1vsbAG5|YBgTum8QKTH~v%}r#e`T*2jP(RV zqKP4Kxyem#i)2hN3=#IkmsDX3f3r4r^6xQp4XG_Dg=y0!J1U);oR+G6;9>x(%F+?u zIurBdUWP%=L^oC2=bhHsE>NzH`u^7fCPMFMmRBheIIhJ*A_oEh&Ou+*r{?!0>y z`~m`XLF&@n?plK4{h|0uLn-=behDv~CoSTaFIn0OXvO^VV{&x3<)*&Ie^@Ro)%-rb z1F-uFs(uaZtUjKA_=k5;f#6#)qXmYw=`j>98VyT!FZ>MiX;$siW_qEElU|lmPTABL z+92As9)<*XUMHf!|BO8Nz-fo-W9y?~)Uq6F?IXgACrYGj`DMJyU7MSnxGr`|6Z>|) z+JtQq-vq?*d=Us|8VndYe>k`xup6Ku?Ys}q&uEB&Za%q5-&rA2NuP=bjM0#`d^f)itV-r4Wr@W*Jxc2^ocej+e_=H6Txq zPJr-nZ2)Eqe+(hysQP=vm}*9tYg%)7W=Dop#9|^guJ|-lA*B0X~ImDPav$Gmxw~s8NFe-97b)1=BMB&ZKrV0O-MA0=K9C<3Qf`}tKYULX%x!`zR)Pllg7A4W4YHdP zw5k>Qau$UwyDl2-XL<-KH5!P>5ahvPx`FfBe}L|L07)1s>^x z^sDy}NUK9I%G0j;_GxXWKwYeZHc97$xRsyH*>|&Xx2ZpRlf6V^$Lg@oi<4Dmc%Ps( zn@4L)*|v*?aBR*UbBiO9J&n2XpOsRz^^@^mu3`?QabNGNHr_dzMr%FjP`|$L4ClF6 zf8dnG8`|ffeF&K##EhRKR0+y-7@pK9U`BG2wRLp1N-&}8s1WpvMniV-$`38dFS6q+Utr$wU}6O zc1o8B4}&-JX*6m>omlJCGa#tsNE zi`6!-!UPx;=f7A_fsn(Cl-{I_uFFo4s_6svd0?4KohQ zP(?W;S4)r%bCstZG5Zo_07T1ylvB>90v}K--e~j)gn%Eg! zmPR6Zo2ilN->DG#Zyf>nrI}Jdfv9sv9gOPJQn{p6Z>+siyf)2Ff`^D{`f@xSTPM)p zvl2SfZjba3odJ1viS+th!4V74a8J83Rm+sC9R0#gkF@8>Z5_ku!=hRbOX}-`ijZ#+ z_Ox3Cg_Xlyl=sJn?NccHe>z#xB(;A9w#^pXDO_HXU?wIhFbgHY8TT0S)?$t1mJAm~ zQ-7L7!LMA_=P{XZu^H~v&W>|z2exs2nqL#R5A*V{f%VQteVVQ|X689!L){@EiGp!$ zo5pMgw@$yQ{Q~E!+Dy(E0+z(S)oTa{J&FL4^35YqK|R3F54nfDf6=<1-RP~TodT5G zD1pY#>vUkY3#E6qtm51T%h%>2C!yD$miWyhatt9!;Lq$c5O+hPrcK{X1sI-IrJp9o zN&V=<_`~l(JS93_(H&;R%wZ_6D+Hm$vn=<`Xtg~sU<0n_9q=0c)hQ0&@$8FdCcp=! z!ibC_({<=98ao6Df8@5{GQmf~1sKiwt=n}d8!qg!7f|h2k8(WLhQ?&tiev$i&(WNQ zbUm?)FX#Ka=)7cxzPwD=}OeZbO>yG&z0I7Y>#M_fhNU>HG+m@PLSbyJW==!dF$nDpS1OM?$Dwy`_F~DtkN!Q42FY z;TLImH1mU@JNc729r98%BNg!wM1|+?Df3d~0Oj{N>_z)y;`!kLAR@RUg3rnI;5uME z66IbB;3U3se^ot|15XUeNFTc=d{6?bJ-SCq`v)Ca-V7_Xmo-WoRG&9=hC}LC>eSLp zGOe$&GLREpze*?{$J2v&Awu-aP9}yCXQzPP)3E{vQ_E6EhpEO)V~!pJ${;~#-nd)? zKlSzC`YePTpopz) z!Ckj^9}or`r=G6Q;CqCnHkShwPbkBrWj*~%m9v?SlJLN@p{T1L@)%YIc`Hej+nCd# zRJyO*K>(Iq@$<6oMu4z_cKBQbcaO(Ni`ZFz?2o9uN{cH`Kw6cs4bV(fyUiivTXMqW z-$!aGe||p?lhW7S<(3~@ytMP_ix%PJIlFn&cbi0JB&%QhnMY4?wkFuR5X@1GTn>KN zk(w~k(IRnqOp2MBKbJSHsIFrOk)jx+7EE%Ep>4>1&Vq|&q&)Y zxAfG0WdS&Ti4bV6K-kV{J^=ND zi);J^wK!`9zYv=!awBEF7Cl>S2cVSzn%$&7Be@SM6he58O-YPPqvvyVJw-`GJWh$W zf8K=6H=PVa1x;1A!iOOwq`n3)cTBmb^3@QETm=*mx6Si_Ky-a4isKU*nz1X%;AqZL zh>`#XK^j`f;Xmtv)jMN(BZVGp>wkSHh>D-q_Ym9-gTNx&L1f4F6(8Jbk}hqc8?(Zm zwFsa!APkh<-B%_Yslh8W4cL6rAV?iMe}fmc`jTD#*`s?m+RV{7YbA*FgQ>G&x1LDQ zzeIG`xi?~sB(CZmqjCa_;+jzd)j9J}O#+qJhYU2+%+LYf0Eth zlu}sE)3oIrlF?}*8%|*mlW|1t7L=xRA+4Q~@@+=6V&jjTl+8VUKlwn}BOQ6rchxe3 zySFP~e|Q7KtIL-y%#&=p3OW~0%R>{~H2hwsXLTK|^tYS2 zmqFzcWe{-)5`@FXGU*{tBOE|znyDhuJ$5KZLibaW7 z=K0gg%O{k1q&I39!6Vi?L`sgAnA84UIkjyAYB2JdP`NS zWc?l!(jOt*4wRtVS^Ob~CRa6I(QYs-N%;tm$~ZlbdzLVe;nI!Sa#h~PtcN9 z%X_V%_X_ohN(Pn2F{Sb7jf1zqj}XmOvu*gB51I?c%rR94)4^sx(zDlfi$n(c5Ua@{ z6G6Mx+G1Eho1OqJad2lU+XO@=f%o_-#Uz(V;!5Wh5Ky`0emU;`^` zP`-^uA~XHM1aJHaf7$D`wT;eMGS(e=3Ze~v!a3L-0VQb)UN>aw_F-K8Y9PNu4e?R1 z&${r!P!%nC=MLxPsj6ev`Exr`eUe>VP4(D*pL1Zfb-xnZl3@Q*hqmNXT>%S~48}DR zj~zN@@-3rq$e7q~duP!-HbDK{n-lPUe90oD3k5;_T1p^yl_om{8U=mu1&Z6_E6b_WD&iz z6ZPh5;JOBi#b%?QpFQLwh16T;Gx=^A)~?wWe@;v1@QN?z6dCDiE!&%^5mxcbf+2Zy z+^<6wA?hYviRR@eqs&W9JGJUvP+2(S5_|TRpD<=HVmdG*8etuy<=5SJowF{=mgK!P z%+qI|41|{IAvY085o$bE{PCQu+M6E$=L4b9t5{R-sR@jknLA*by(>^7_NTid+!d+J ze}n^gN3*({4?bKa{tMso=VW5-T<*c#d6Lryj`(9dxew3a^A!R9(pOS{`H=*?CA-H+ zS)~IG@>p_K(V^@Ip1D%>2@NH_IkN5JIzso^vNez+!8Yt)y2*)m7?;nkOrjM2__Wp( z<(`5fz_8>Iyd}}%$ZS4xcVOBDYUz$tf2xj2E2%=T|3;db08J5E#>q&R!AbTuZhw&} z^)4=u)R9#(mxVLB_%*W!A`^)5;U{MUJjqE0W{G*R`1Hq8G80P8+%?=~~X3qzv z?+16=T45lpkVLM6=Jis&T$>a`133suTmu_sRl&~F>!q!rfcGnpD1m_3rU)AXNkhH#Y1V+W@+D>RZ zsn@XgouLa$w!NWZWbxk6WkGP)e~lwmw9PJc?^v-;<#3{umkOZa8He9jYQ{mpC}=G$ zeUR}2H!9rD*bHpy#_1CgX2 z$|g2uY4cm%_uV`FHmXjbgmVM1kWf$GENM`zs!g}Op8YLyG;2~OH zB7B&1_MrMr#vz+`Z-#I#E;d;W1>vsCoJn<`?HITaUY{6rjX`jOT8N*TNWYC9*7@*# zy+PLm=OJZ+=|&z^CvS#|-LRobrx%YZD+^55MYQujk$YClIwhZ*e^NFL#nd&C$87yW zD9${Nv$uRURBmaa&M+zz#?ty^V6Xd!0H0sO*!c)Ii~DlVHHnP@;@>obCXea)`H;*T zf-5O+Q@z;Qz)p*Xn+=?mcTahMjQ+mZo38w4J3&ge-68(;8IU--2>XqKsxv9aZ`JP& zGP#9_`E#`=B2b+&1RFw-O~d(kd!e#hrKKhz3E?8lO!uIMUBS9vGJi@S9|)p6&Qnzu zX#fCUP}Fh_)-Ph726*qDtEzbuW3z*3fsh_^H8T6y_8g7^X2_M{==VYcQZ`FsA5XOW ztcq3u);D96y)~)E1YsMimOsSFX~NK}8W_zu;rchz;j3ezf5B5*2oJ)tM-Bluw>^TLryr(Z z%SVP*9ceJg-a&{blEWG(fbhZ~XX=Kn&jMk5x~8^1V)#`ttACUue%5N=ot?>1%Wb&* zPL2baE1kzIF;$+o*3T`It8umU&*PHn=62(KZlNr#FS+ZKEmcEOy=`ThjY`c*}el{xq#{MEh!W#o8I6d-hVYc)G>9cW7WFyD?&0?91AN*U1>u~3KZFGWx zs;w;Gy52ZXLVp#yRu^@5!hJPTv)VrV-cPFTUYV~g1@wLIn^5SDIWen4s*bY&IFmy8 z9Pk7PtN_5=i_<$u0M~0n*y$lX`C$AD;5O(-)%V({qo?(_QJM17m2xHs>?O0Sdb9pl zuJI68->{F&JfMkyAaL-Z&tqgxYP(;6YdLe-cP44&F@He^uU2SRa>-$5Q@%~M#X;|P z_n#kjxV0TPsRNMH`fN@0jv3>Gx*%iY3{}qp+)BPMGtZ$BpYO%5Z#jiIVyPp9tr))e zzxh@}s@jxjnqZvA@B{1vU`$>vhSd+qFY4DXY62r5ekg9vf7^&@qD9v(WCNDW_+aBJ zobA=D9e+a>l)@llf>Z1A`S&o58Gyo;b#;S^_y(#Fj%jhIH0Rd$ z)z}k~0w7a=Fi~YTo1|=Q1hyiC91yoxt|Y;9`1MBNRn7CGn<7u|OBFaD7trsB2i60C z`(~Y*eN&(``(;t~s5WCLRIU`YaGztW8cbzRSby6W^WEwOV&5gy!jp|=Z*m$}P}LH) z_t4s|3;QFp*Zv5j{akKn?AAUe37ELS7lcX(9453(2i)wRNFziZ#q?3i4mFn~nF8on z%xpAj1^r@+Z`Q(zphpTa$Z~_(t+~zY?eDTt)hpg_8NY`j!nk5JKc3VdK1|c-}mHp*u6#hD5$^31y3gFgD5x7)v?t_>e{GOviY|6%6w>;X|IhIVJUt0P* zuR8wd(d`+2!*<3I$Z!n|GFkFKMZJCh=6{zzQ}w2(1Oh%c74r-FJst0OOD1WlJND`!U8LVh`C$V6mmF_|GrTGXh$V}BE5f#Mhb;jIw zlZF*%k$GSvl-Zi+qp@Y$81Pyk5GMCI6&hzwq^DAj#w(ZS*YD!X8O~0Fv5o=pL&94c zEjAY=f;(x-hVxfbKz}_8GRc5&Z5X>CJAkfqfblp$HduvuB7+JkeKjO_Zu3?)@C=}d~8Cgh(Ck|*=hsFo;EYO2>>@W{3^QG zJ~7ym@Z@pEx7rB}$WcJz5ho7ESG1l)!%%&K-R&VIAx?Y|pwY-}mti-;XCPB@l)G%Q z&PlX(jzKWa8-FD`{HFbBk%3`iw&6mBBk_z?R%7*VAwj>tlIB^ zoSwIbyvn@I`HmE?ERSa)xT@0kS6XRTDC9T!!IAuC%VCQ7Z4BQbrnXH26C_QQ6FCsX zQ(F9;yts;^(qt2#!5$`aaGr>58#(r(mV9Tnayck+cz?+XaX6p;UGs?ekOaQPA;62g z5rQE%_FrXV`m5;KH8$|d<9-iLIJNs*OQng~B*;Jxt7j$^E8j{2$3Lr0;JbRmz9IDDmQg2@yPd2;fN|Aq!`R(l5ac!)CQ z3$L8T{iURHbwpxdjW3dcp_2XW%j}RGe@hG|#(y0iKkkW7s#h+tL`Ogf2PW4VI&4T; z2-oGMCulwtdTZ|XT+^VYWelpH`n@T81pLf>kk|tY^oB_jjhq``r-)%sBBMIg@|4Gk z!a+3i8LyhV9~{3`6?as05-V8QzAV8AtT3$I2X7W7H=oSIZ$E6GM2-&S8;>M z`?mHtW?$;p?YwT}U?Guk>ee*+?$0PHMSq*D#zqX(#Ej#bkRxkR^-HE<)cj;DvNuX! zAIv>3-Je{&xBE_LXI(msi1%H%G{`-3R@0bPus6-xIc&s-1`rA4V-zPtO>?oNe3U?J z0^aJny)?`fdcGW|qPdD}>QJ}oxH*xDHB4{PqgQ?JGdhATe^(2(uNNKAL!JzKj(@R6 znwt4~r0L)9SaS%WRpD{9aYL;3P0bn4biU4Az3IB&{FYBO({m|;M@pRXN_EQP1x@dd9wF#wB(>=K^ZmHhZzR2NXwcda@uh4 zq*fC|oXQvAe^T5O-=$-V@*4oMj&kIjeg~+$H?!hXp~S?Uby(RpC*M@6g>;Axybwuu z+?JkT(x8KbP|ckl;Ov7HJI|mxf+cK3Qr#1g$$!JX5=di<&61Y;`bNbPet+1<+v@%q z_kp|j!Ph+{S0+oqF;c!$Sy_aJ4!nUip82-FTs|=maG#SJ*D^P9NsrCHdTUZU4~&vE z5CpraVhzDb2n2NnbJNV7fPj)n4vd@U`iJF(kNCE1urCP<(<;k?YwZj{B(7{phJcAj zw1gn|R*j?f^ zBh<$GBY$|`Za+dN^ahoHFR2H$DZS-FDzEm9Ke__~OP&DxKB%15z-HRkfWto9y zih@!zERr9LWn)qh;d@ksb7TONCR_>7=jCu!BKv5reN%o zVlNtydfU|qtza2J)Ii1x8%pPwP}KoN(}(I4*XMpd+#>7Ah%omd6_D|HUt{N~5uO?7 zU527=XFmksqJKd;Yx2I#YuaQ6wPZr|kwanj_O@?kkKhys_A0q+Z2x$R@^jV<+X6MQ zaLhSG-Q6uE(fswv6g1->2u2{J1wSy!6Nun{fQfsD9*p7BR1ashX#sq?8Gmq-_eg_Y zE^hRbiFsYFy#{h2-(@T_aY`2|i|8I(ko3k?WMG%W?tfTN=R2@`cs!d_jUPJ6niAej z#R%qEb0Rdl>nFYi@C(4((NNhV-$tCV7nv$B6qAoGJho%Z63 zAM74HS%0mwH;P!&bOtbzhB392tpFm^^2gmSj_Ug_*a<8vh%5Ixys z@TjbY4=AeGw>viWmLvhxZMoOJBJk2e?kv!B^nW;gDTr1PXX#v_u96CXR}ccF)t*&$ z$B2xM)nqkpvCWhm~1P-z2H`_R_B}3<7NtJKgA}1`5oTHQG*=3eMV=9c2YZdaYx%+M8UXLRg5l#YrCHn^UNm3Ov403T?Qh&QzfObu-S163{^$lK=Hh*_! z2~IJ8TSkV?fDT7UI(!Nf1p8nM3A{cPey5M3e0mYM9kN0d4~eD@BZoYog08-Y355Ow zO0JJtf)~;U)M_s!hD%tiAE$`NPh^Y{d}2Hdyv4H{>(-EhBfvc} zOFuRXC~!$NNj0dFL1Z^7)2mi9gOQ zM#Ks559WJ(O88ZBLEUK%j=7W!7eOJ9w^P0X#T~Tm%i>FVXwwY4k1n?2-yJ;rNk* zm~fz`kDV!Z4LO&eSbOxAl7C4M#I}O+s{9RAaLI~Tsy|5=xs_jk#`X7{Z^00bV4Jf@ z44k5ml!o1ud+A%6k-BAm(&fbj6xx~n?>hSxa=OzEmI%Pm*$TlH}W{xC_EcVNEGaci-vqozP4&x9OeSa57xW5&oT!H8m z6%6BPaXvrer}yfl0KL;gCYa^CP54=$hEC;#bUZhYOdiF{=fi8nW z#w1sCGjLR~^72Y@r0ALWUeL{npV3hRNMH+zCYdzS`)q}5x|wfNkTQVE5LAXDj-x#+WIRBM)!9xDd zlt4Y3Bax~30NdO;NcUZ}0wmP9gfKDUHMGf0_|!8M zUK_p?w)>_YNhJRU^f-^@_*D(=u_x_XkPU{N<7cn6d<``p!R<*O!&a4I3N^*=$c7W& zM(~)9{el;j-h+X+U5WIY--txE&(GJEuE+-IO~>^fNOFFLgUedkP;kms_mtb_%jrT; z4m7s1E`I=VN*G_sZbxfk$0`0T=s}j%I%$5pCZam~R?1BbfT47?oiWoWA(=@=)lIj@ z$X=paM~PyVCC`HHvE0QC7{7JAZS9yAE5pfPcOof4{wn2HP)@x~-y~|DHyKUSEVVVV zDncujea;ZiPLAci&%G1{My+gJvjUaNtYx^=`uo>4`Ckshn#_t^`yJuSi`#1XuEmeQx}tpMD~Rf zP=B_aP3Q&ay^`0rXadi=n(-aa)m7?oVY>4((xBkMsHTJBXA6n%xgqb6oTsURMbC%G z%+wdLy{7;jz)!`Klp#M~sqL2O!ihL{ke_s$C+INB84fptW}bd5da0NPw|c$_QB#E= z@MRX6Qylu+bxEECJ~QYulO1Sc`T~yGE`Jb4066l+H;3meF~&FZ-)q3UH+Hg`?WRu= zQm%-X`k6se=LFdAo$M_{L0x60y-nQ{$87hbjFktrs|2TvqvNTLK7}?%5OPI%sTOvh z3y%8lEDVO~_YNgNd%Uzx!PkIe9M9*p<)#5tI>Y_$c%t)6n<#UeoV}xrytAA`CVyOr zsRQm>l^EUkjDqvt8MbZbu)#X{7A^Vw*iSSPo4E!xypuzd*Fk@nI`v37;37Ibu(65a z;4yFb4q%n=7TLq^jdPGXW}$1y^*7aig6je)|NR+^7E0}ODz7+_+wmNHeA3+f#Hv$MX<~9)gb8ZP>wSP)XH8kNmUfh`V3u_KGAG*Aw0}lRv`&N; zZAK$C>}_@QY~jiy=*Z6nI6({hTD;x8fHP;Oh&mw|2c`f4(($~iuSyzb; z7<}#mVn}gtQxzvWp8Qog69&|Br!2R4Q#po_!fT%#EP6thP=I+*P?=HU)3 z=XYhHJ61Ybw&2yF#R%GMEr0byZ!}ORp#eKMOzU%R7}ur!hC=(!PmCim_A#f>Zp-^R zdbOy0{G{-2?>vzP5!6hFmE|-P7DytA@>a%Ate`NV}yg6CWQk$wQmxjYo=mdMTmUpMN20vw1v9v-p=X znNl@v5jsyrlc7O^)O`}=07-gbGVU`&&*At-70_+U>X;y!Vt#R|CxSa?QrEi?iW6s3|z6&JFo zU(fhrI86F4xxboxX<5WQHo_pf)Cq91BTfxC{$g>lSPHpWQ~B$*3r#>C9^3))%9;UjNP=Ae?Zy;32l_L?_`60)` zTchg;fN^FBkun?NCDIZLYC}Z$UqxeP|i^fx?t{zXMOw2f?GwgY?cBkDz)ujbEnHv81{SYmC zFxxWJF*|myW`C8<^x<&u&}Tz?zCOE=bepIPQi+x1*64a4(gkTxEL-T==`~RV$Z^Camau zs5(a8rBiMS59b@dg%uCR>>V-eia@qfU0V(Rky1sKaDUO!G!vPq7&t6O-Ez%`6E0UY zPa8jpnLnI)czE;{Cl4#2A1&!+Li>m4o-!5$qh}f|JHud>d$^sms9n1f-0DI+my^y$ z`%Ipeu{kUUK;A~mSEK%@-&O&;0LEGJ!b5V_g!wn#8)lF{0~m4w>jJ{0P(;gHRvu8v zmORE@WPdGF3{31*3i~Ar{lUtUFDkfF!1K&{!Gg}?J$Mx^+30hE3@#{#RW0cg>3IV` zPeH;=0ApP5uvsWmD=sfine^8e+dhl(gFyRL`348>5U+wAIdc@o{z=xujmh8clcvRg z0dsh3l)YxesrA>(4q_LNL__4z*e_`-$JK^SUw;QiYQq^(62BqZT?x4!_iHXGPTx+e z9Ut2mdS%vZ!5A`O-n&4I-}xE`U#D}F4ur_tzwT#Wo3^)ts`??3CXXA{oWU$#>PvD< zWFXcksE$E$ML{^$zo+*lYVN{=)n(;LmM=nU;oZNVD9$hkVBCo)fb|fw_A5fu(aDZd z*?)$ZwH*)kGk@y=^#l$ z)SML&jrLhN*zYFZdHzU84y(Do{#8h$4`W9)E^7wNJso29R|wilTO0(cVC2N4vhYgF z*)=P=R49p(_{wb|C)=Q^XlzBzN}U3bV5oX-5W746%)UNsTKS@;N=BlMiYq-g+<(I4 zF>A7BjWrFj&D_YaB%c%Fz2cKjR}}Z$xS*x59%AWwnMo)wv+QHe_qbRO&lO4h!0WO{ zNGw>`+K(%zDhT1l>mN3=E)UVAdyij(frDwMKlY&5A~;@NIh4936T*nWQMPn7i1R~YC><3yLwUN`bZSDU#JeR=q;g?!C4nBaFY(x^$U@4v_!|D z6MU>BJ?BG~%<;jnEY&zBTTRE@(Bj$%Y5S~No;N=Z+PCaEa_iB&pqT@0(0}$;v#hG5 zRPq~7^bRasOoz4|&Q?~sFl&RQoM$sg@|jK(40~;PRexBkdBLH)`=r0zyl9ongrOh9 zpgfosqp9yuIsL=w_hTuG{0Jg9ANDiyV{ia!dhvKIh|VG=$`Y$$@JDQ64O?by3qm)U zvIG;;x?Fivns{wQ z`+`mN>E&9>Mm9w9gTB%@n4gE4G(BDUWY@c%^1gdvc2Ih9M@skIaG;9^uNVfowYZPu-+GT>IPcbtfMR#2~6@L|rDd{he_JQuX(Km;@P((!F`U z?r%L|*$Ihfihs9kKQXg5o(&DFLjyp1X;!4sR1c5T?kZAb7{U#PDOg-(=8RXJ@@R-i zcIHg3P=(pedv<3{$LsN{09oQKKG`Uyg-T9t^{7;h&}LW*7!Dk{?OeQrQ;hKmP51=svCHSB_b$>~cStSEj+soA-92Fj{|24xsQL_mW~)mnpy zTXk!)I&oAZ42a;9_Y{9q=o+$}miDyjbbksBr|3A#->(jj2T4=R0z5Z^67J`7V|_TL zrvM>M{E*}BA(0C1k}7^l@t&SvXO@}lA1E%>oNK6`z`uS!;jtU|ls}1s;xO&=*6>!R z)ZsF)9N(0jd)ZU+sqZ3&a4Ebzh7#0#{)K*hb{7JeT%{C^a)dV~a=I|uNi#6_~Uw1aC+Z)45(gc2tWn71v5 z_Qo!m!%d1kS|a_zBJLGUfd0!(ceSD#1q*dw5}DdL6;M8p%9BP7mM& z>B}drxSPCw)~92jL# zK_LlAapN05H?TFaY7(c61D-XbWeRTju!s%K=fkrBJSAhAZMTDWN)Mu=vlT_ls3Gym z=ogTSxQ9>zYr{2X&i~rQ}>L9JzD3 z^j@P_4pF9R38E>jT5w!uqA$w8!B_?YUhOnAAQKb)@xiyoy-Ifu#;T9|&W)7{BQjPp zVx!(*S>UE&k2o9y;(r^QrxsWSW`i&Ix2~?3E-_antAjk)-tw(`p=_wsL})NAu&=hk zK--6h8qntF-&Y1Xpy>N`R;)MQchUq81YecTaVd=aGZJX^^M$qu@<2z%NAFtbJLU4P zhE61|$Cu2?ibc{f7bFjLrx_#Th3{xkS%|_JMHAwe&hCc)fq%aslnRUg(h;xihCA9r zzcKu&Q4^PxFhU?bxv$|HgHvn3Z9E^Vu?_C{gxaXhwetA7$tx|Ts}slC9T_vFMqCq# zmCB~EFBoN7hh_gUljR^ww*)q3g02+t3d)R%;Vb7)tc&ALcMebC)-OPjJL=my63Ufv zBCCh%u~e|RoPYC@iWH-K`^}Oe23uIOK!rD%{XSV+hOb8zVp$&ZAzH(+$G_C`DCLVv zrDFSED@7RkZ6uiEmC9B1Fmtq6GBFJTSPxHEdEmBpG9-%5#8u;t%=8n+r~t>6N%C0% zZ_uuIUoTr1*8|I%?-jO0ehJm!3%1?ETzLYPA!L+2&VOM(n{Lxf`%*xQ?NPQFaU-mf zvaF4?nhz=M(V@{SwZ0~$e}d@KP6Q~oY9a9xi+I;W%#s?MIh!3d68N<5fu6B~*=w9= ziV^JbOkD0oC`ZhJHRa^aQ;fN_bD4&hhmL0kEtDN&wvu}s2pOvFhUzt z-h_hydVf%(M1wQ|a-d|{TJ0JHt7j-|h8jOq#k3lLn|=+j55}a|_!GT{w}h`~Qp7k- z6&HlxSUJ}EzLvtW^3P_pXvP9+uySUX`2L-lH2*pme3MF7G~cLM)x@VlhOUamV$!ON zF>>s#0ZHo7kfMl3bx+!ae7~`Q$tdKvx1j_B7k^~l>5%A+hssTc9?G%g-RwlN1Lq=$ z*!p3w?X2P)<*4VWjb#_u)-+&p6}|VXA6ly4mFyx#;ghaQY~zZy2KTcN zHGjGV;m5#9A1Y7Mgh|z^3`OoMR_f=3;za4Sd%W=ktqS?Gm}-U0@tA8Cj+F8icZIZ| zrc74jt6>XEP9<#>xHbS9adbnXny=9vW(E-^q_x5~$^6d`S88S8PI#~l%GjKzky%o=M}$mu^2VeIvt&4k5UB zxRl&`%u8>^6{I#?h7lUl7``0F!+(WN&%@`_-||gC2!W<#R9?$q1duWv02Pz&N1UrB zq9cu)2F0`iEeAEB&<}bgE#$#Z3QTrHv4$xDI}A}SszZ4El?6U}u^;HtmbV$OgfMaZ z=DzSE=~q|Rqqd4dSn-`$l9-c>0ba91FeIxudZ+;&U9r8N9kB+lvO*0yWq)c2GzCkq zx#jizDkvaR2@=Tu9MsXsP(u`=8%`-TaGW+K zSZ*Y9S$bYDip@Ku97{w^pS`9+HReLT2JHtEc(p%+{YJ1j7)$DSUKnK*K+Ai&_ZjoR}c6hMt}HiJv{C;_^W<2 zSPu`qwU+J%Ca6d8m8(weQK4SR(kwBv$@?ABN7tPKO*A?uF&j!QqXj3x;oKoMAy)-=wTNiQa>Nrc!3Z%^ano^kztj~y|7NpT?r z1(7m@9#R=V&c<_ld0h6f80EQq1so>!4NVC`^|3;vSGAGI;Dwm`l zZS6J=$>%T+OL9v5cPsp~fZA8{>AMn^`cbVs zFuES%D5hntVC$DMz;!Oi9~RHZU`0|(aJp%3;5h=frNN67iGQx2j9gCNP9l|WxiJ%b zR7Lj4rLZPaBdxjn8gejn?1NE|JJ(ltYan+2;PVfRsRC@eFZ>MXlg|eg zg0dho0`A_M{WaGr;5xrNNG=E-$YBPXJs-6L1)wF%x_|JMC@)02?k(r{y2Z(8)R!i~ zO_wYqb=r$xx^hC6UGq0{ZE#j%QLbR!mhEQ*NC1!CM(_MRE2bI+_j==5NPi&qeL#bb z4GFG0kPInf(a%Zm^Rz#8OO@)jrK;_)b_JlywvAcOM1OEyJVH?L=7^upKRO8dc5VgJ z=-7qd1b=*cEwTSty=mB0lT+f*8Ggb~gx2U^fe<=Z44>3xL6K%;dbfm2 zsVqpdQsn|@or6>h`GWq&01cWXB9n>|8j*`yynj>I#c%TL;wtM%1;9ZzcT`GvbnZbH z9eQoVGJy?~ke2n^r>~v2NZgTLz>aNlSJHX9oetCpVLc1|X-|~L z(T~zb)t}2RLJI;)$sHz@839zdXUFPk7{#3TWz>0&oZh-ss2v+gXKqA{@k<>DKISfE zw=bn(rWX2#lAY<#Gksx6GTq!)7iaf=rGJLm_Qd5g2jMQ!(P`OVSgdqCK*V((1d7*V zM#rE%0Wls@DM0n;!%(i&v+Zib{i-YIyoIqr+JD-}qW7=^sM=cb5<*$?*LPlMb+$KB zAN@J~Mt^KcISOD!04ORVgQ?^A^laLwtF4L2ZZ(?$~k+28UjOtj|$6bbnC} zy$HhyVkd}0#Hz=IR10aNAM&Dxp(Pn+H7YPJc%g{Q09C>Ab0cv$xZmqj{ICmn0r2*y zWe@3MYiCitE3Nv8gS5n(bvih&N8Ic@fuP9zN}-l8WUQeB$NbRWZCGOBdKW*}kbh$o z?f;C-dT!NDxGEvlAYTiPOn*CFIDhzdE8!~tA`QSY26%dmk4IE$xXlNblE+P`o#c~( zSBMnH`H_0~K2zUCf@greU2O@%i-z7M-a0#c=)7_6#}Bxi4;4-8Vq$oxcwbKPlM=RB zBnzw*M5;t?Yx^4iz9ti1Q#{lhnSMVajJSpu8d&2pJ-i%Y-;~=y-AtmPpMSy+Y-rOv zsN4+m(fG@~EaRDIRgdV+v(s5oeESJ*Nr@@25z@5kl?`D}V{vzqtL9D*&bmQI8*>1P zdfeDPYdvRH&W*$b@wxjw@`-K8lXEx)qz2L0d4LyvL!H8zA8jz!W;Y8 z<7a`Ex&ryAgXzO6JjT8Fd0^BnW3_h6#dJk784_coUlXzQB z-Pmk{({rWu%#$-ww6|hWYF!aBZ5=y1h#p0Rk>|1BE)v9;(V~1qi1%b{7&IKc^%ZzS z#lygU*`Aj@f{SDW_D=fF1dT`c)R_(?5s~obHoOIDq2A@{gVYDaqkl6*o_bYp6OF^I z@G7lYpIi}v%EJPbGt?I-q``Dw^EVee*ic zdl8nuZ9~#@W5M!k1b>bzVT?gpZjU@D{!-0NBz{8I82cuM#=}o}CBcYDu!SyqN$^o6 zhB1J>4_|gjL(O1AWAWLzDQpLS$?qO1dTxaMWhU19u_`oxa|{s}`*t`u=&-?}H)W0A zh3%C_JB#0Sar|+|gNgiHPc8Z0;pHN2#ge|IO%?SKK;kdH`$VBjdo%n^^&LP&kiMbmmjb5A6lF)-q zF2IaX{#(H}L$?TQlWFU2gw|khiFfSqDXOY`2^Uo7OLEW~gnQwtJL8>{MGwQ(INA3> z^V%r@d?n43?|<^&gB-YUkk9Uy1@ybGGA{RJ5cM4XGDEZW!w*+kHRPWJ;W&lDJ;4WH zYbV7?h{IU55&}{WhWG&t))te4CqR~#W)h?5rD*8c^J1G9hs*n6_nFkN$ET3i&ne|n zr`%?p$BpSAn~WRj4wVkGlmd&juzeZDJ3{=;n&y8wI)95?%b^&EE+~UA4a3ZwS@4CK zSKs=*1z*Xwe5+0chB-7zA%F!1W*ZsSMe&sSd$yn#Maj}>(7qtyb`%fYswm5cEA2|b zMhApHh;S`={&5FXw$Q0^@3L}?Nt?6}%yphEi)PvPWb#B-&z|&zaB=HF+;(lJ)^NJS4$tkds0% zaXJ7_K*I1?(#xu-m5+CWaG&{w$k_%Zx6t;6ESqU7fh*e(%3{`2J!dm3m{bC6D+I3p zC#%i|!3xCpE`_-I2K!<=WgUpu^&A@!`^P=Z$bW0PDhj{_%R-~cB$QHN)AkMMroemD zS!@^mv4G!?un`xEK!qRh5naGhp61Zm#0kM9*MBRwm2~k~A;-1|Bf9AB^ z(NBB^oQ6zdB9`gi39<$!i_me)ogJAORo(osN{TvNY-SFJh!H@5Zn>Ej%5=n<>~Y^H z_U)0feG)GHO+aZeykbjEmtp0m#InGv0wh`)Z&qH zBwUxm57#zaDf|rxv!5{ox(oy9$GrmgrmFN1FerJC+F4cG-DMpMTwDQ^KqFIBUfZ5* zUi_hB)UH2?%?HWj+@&I&GX9lNO_I8G8h`sFy;gk%YnZSmGNZVu{4qjOvde#eFb;s- znkn=Rk#}Q>{yFUuOUeMsgg(+0X8YsH)n$gY*<}E?_N@=$(pFW?n1R@Pk3k?)4W17E zWK-P6IobL zJt&QUhy6n{{@X6AOg#r?dP*}Zx;ja5QOo3U2r>G;+Nb4)o}Cm$L3#t16QxtmLz|Lvg@+M&Mq*Jc%{>kC0Vh^f)cE)S12RBlPx$db{sQCtqI1lI_A``@bnEw0mV!CEQ(&oI_(0oZZ=^AUEM%(5 znI9znr66Rvy*M3)pMRAyvf#zlWu@OYB))Ylkl0^vb)4o9&oE=c^y57c^KX{IRFOyt ze7bSNAX2#-Rtf2s(PDU+x%%zy+`NUP<-C^LVJB7<;k*1+F~*>Wkz$UKoQ%NvNj31S z6eXSJPO$AlvkKt?hk&c^2O3U%`7~7va^(oIQwjSNC2*lqvwyzv4$8?13G-!u#fr^v z_aeh-vZE2>&rYuqTB*Jagj-iiA}tB0?^(yMDZUMiM*o4EEVSu97vdn3N1fJh0|TmJ zuxg_`Wl}4ylj^oY_UZLB+EgEUJ&t=QCTo!8EHEZS{hOlXBJSo$)b<#GHiK@X)jm`2 z^>?c&Spj#AFMlt&d~sRP6W+o>W3HPv{p_%-%t!?BxFFSv(Wx3EOUIpJRY=Z0rjFT! zDBIZxp$xE}I5jmpnkP3(I<94Q8rzf$;yrp^tyR<+o)E@heT!x$kB$QDH4sK`9g z)F{Lx4{$o;IAX{DI3ltF3gaR>Fi$40O&0p*<0dwk5q|{9vh8gNLAeg%A+o-}H(S5q zs)cgl(COX!X&DU;a}VYpx6tVULY>D`%SS(j9jxg=9vD%-?XE8hcd`21{^`NniVdFt zqCOhU8aJk15g`0RGp=yZ(e-rZa*NE?F(*NfI>r~3xK1*d;@34t!kls*BL~Q$0`(ip zAJ$OdV}Gj*BcisqbCKZ9`lxdkSccOzZ(|!44=pFGJu*M@+@5T?2;i#bXD^UbN8)3< zch`+~$30fTBoDAG@+N~v>i5ECMTe;IIG7mkynl2vx5N|pS4N=+aLi70q>)w~+?V|r zOH(8GS->A#oE}ynaWXVjq6FGGn&&{cs(y^#G=Iy2;%soaKfuRlgh4OV!^K&O{UtHo zO$y}8)uBkjO^TKSh}>xnZXC@wpPRe)3btB&ARQf=FOldVe5|QjTti5 zpzoJdRgmZ-VG&M39~q;Q-#sMQ4dhYVB$yM-?$c(E9yQxL8$ZB5c*?6t&75?+3-dR< zjEB~@Oj}&{Y!LX(t@i+uZBb8F;vc+G2#;2IeFSB+0{}+;e6j}PB{orlL#G0VPJiK~ zSy?JHJ-sqo&AuKM!2KQ_WD{@ipo)mwcfvf}Q+>nhKq&1-Ecyrn;{8&PeVn0G|> zVs4V_x`>YVjflVvIzziS)^b^#9aR7;K-9lf%}kOk1Z&KG3I&(i?}}d)8|#>9Elc64HUgVdKDv4__SoYr9`0?{}w z*DqQ`8wKDG;3qEe(J4Mh4o}wGv3Gw0tWU1wkwa?M+tN<+Bfcn`bVLeq2+ciXrX4Q4 zPO0G)OWO6c$V$q9D~qHi{xn6?W>s|X(CW$e<{W)!B}Sk4c^A?HxxofwO?m(bq@&g` zF*pz9?g}f*D1SM)%V1OT}bC~_?3G|91SRB0G$0B%)#~JcbJGm3yD|_QxbXR9FWl6t% z_Dp+Z!82i%SZ}`(ou-YF@X79wn%@yA7DJvK=(+8(uNlV}Tl0KOjCF*p#I*AucTn%3 z?k6)J{~DlJT=#psu<`)&(YJp|rB97E(|(CXx|r77iOHf=Kt|Q%>|yxyL%C^*Ns%dy zp&Q`D&@c1SLD|GM$4@_;qwl07$cz6#xe44T{4|q$B@zV~4%&&JR}*$kiO?Ef#kw_k zP+Ag5{f5F5D~FlCVZI(>?7Y;aAqO2_ zTTiP=n9w(0^E*A9cesCFNca>Gs4+ix8L$Am#=J5NFL4O@La5enHYq6V?(YDO+Xdp( zsggV1$*{k>njDi-2A3_A98Hp0G4w}7;cG>i&c|#RQ;MyCWgI5QSe|$%6k@};_(5Ku(oPno|LN(S{soLEVE~$5!S>Mnit(cBr>F< zm#}D0sU_AJWU+t!1V~jW)~YDU7XcgJm+whbnHTeX=WRp9YsYF(FL&4*xfgy)(HHS1 zCkd=X6OTs?4m_LFTV0z73-0s1+}Yv8_{+&j z2`Z6gFO|<65f-MV7?!DWu&Ix)Tm071`mXgbC&9KPcQqk35JpW|$nOti>G$_D=s%L2 z6;4wHoaO~yJdT+5(G|}6VloO#8;ym6UD}_l0SsUwx`bWwFX)7Mk)ux0)U&*B)xYF? z(Vc(B`qF1~DbHY?SwR^K{M@)+@W?J_kgkhEsmgp>BvqM@(Kdu7{8inH-vNC1*kUG) z4{(5NE%ETp6gaRsb)Vy-?oj0({GIs}88mt+7+o4`6cIu*^2kfpNNeyl z7^V%s>YY@IOold?rZW<{4-OgGvfYajL{fiooAQ`;e$wo=M+YZZ+-h16wEJN z&E2TdhmSz1R1FszKQ{sn^jF_Pg~al}Q#LwjOJ3%E3QIHW9dMiVJv}Z0B#nv0(@TF> z(TpLyHDXDvmE6sB8>2)x7quf-aui0Vw&nK9Kg8( z{_bqnNv*z4pv6PUHl>q z50}e`9l2&75##2&`TYR$LtNi_6|8@%%dFR>)mMjzq_pa8%I2+C;HPB^-1tM7gqB4U zA!+s2WV-p~5+Y$c?a&@#2TBJQ%d`roc-%o#w-Ga-! z_;KreH)+!(N)8`SEhgFh_m1pdMW^qT1_w{>OQfJHu zfl|C4UNp<{>{g^F4n24!6L$1*3A|3L*smkMZ~a%NMhlrm%MB34qrc;e>#{5v-J5Q~4~zZa0!aW|5fF`n&^2inK#x`g@!Z)}7l4^xGMPp?WS z@mxs)Ok+F6&bxLo6(JAxCyBLW#ixfbC3RO}F3)>EHq9aRO97_3X8lE__A<97MH$)M zU^Zzfo9#9pUem2)lgtHG_O;exG8>ShMl0IaVW2YQ1#xRNggVXbHqC#)X2ND$+bTtn zcWY{?8U`a46|!k;9OT#Dyv>#yauuI)To&$L3<_7m145M&r;N?gdMu#W{g%xvRu@+NU?8H>f7e!I*)Ug_ z!_$f%=MvlN?fCBs@e+SiE5X0P>&D5d+d3AlplvKjM(yopM1GJviddmmLyK84C9`CI zOga6VRt-eyM5s-pv(zY=zYK)fj`?XcJv z1Rb5(FQLZ{N?us>;=|xkw+~DmPrKPkbmN%0qThi#S=yzj$IE}o!zK|s8_GK1WH}x2 z!Y@8-VeYR+&d%fSccE`(wka;hx3yxJsxwo;i5*K1Ly&T1tYBTf5-g=2lw7B6Q@nwa zX&9y13*7XC0D+Ixsk|i!^CVsMf7`2-W4Tg%fN)TcObFuHfw#E%oZx@}Wmqv%p(m+v zd^shoyeFIeWrcs&0gy|BaUVTAQCcDKDa!`bboRJmawc-O=9HX_io91+k~<>h!AVG$ z_)NO4BFsbG4*gYSEDMHGS-hQ1Gv=idu%}9vJg7W8E59}kdPxIX^cVApi}9qPb&#|y zin--MY*!bFl6LE)4xp9WJ#1K===xIIK{vIFsAs%+kJEoL;+G|)zHsk#tTNq3qk6O( zwP6d|50nym!j(LyxndIM%~*SNEeH?#qj(mi6i#Ah%WlO~Ky(Trf0dE9hOo1=wlF9R zvb(ROdL{h|!M_<6hVDq1Zx<5#ZqcpmW1&@Af(e*|IA#;4G_bn)M5_9|u> zQqWyCTJC?n%C02NR-+KHBC?r91+nXJZakmB3i8Ps584{;h933(; z%PO12eXG8^x6w4s6Ad&)ImvFjXW>@jE3f3%XQ;lZJDI7*Gu(%&mW~w(Fk@8b8zaZ8tbU5xa_#&`Z^K zU^B?Vw@39~(7|F|Y#$}4Y5mZpiFv>g4yoJ1AMpegld};`U0OKQ={bODTOEf()Y8k& zI-{v_i%SH~EJ+3>Mtgz3eI7HUZV$-s$Lv_^f<9{sR$b_mPh@@3*`z-jBlg@)EYB)h;XUk2Afu5B$ed}R4}R|MQ}0`C zS|D*NdQ<5v>*+SVDl%);niFi%^SX)nOQ(|$_C5fNVpoTr4pyi--93FQM-RE{ldp}JhUD) zXX~e~Lw#uXHUV^1$)#BLT@kZ&9K(O;9zXL#ZRpl}?5`?%VE*cOn#Td6rQ(;_LUbhL z$!C#t&MQ9W(Q`~p$lpZu@w;B(1l)q}lM@5=tM~wvV=UR+Jcf|o3c3oMnMDc#uFkm? zbK}QM-d_tWp2EzdTsNKrROH84mF%j!2ba2@!J<#5^3ULdM;pT;=igYtM{j@qJZ`8K zk=2%97bVYj=~EvHvyo$OF@AxwF7l_qTgP;oF7s)btDUe}@AXLo7vBDSeiamT92gHwG`UN+46&{Yw+lNsP3J+6=oM7OpFS@X%_r}k8#BB}uDVCve!!;CLusVOwGsOrPp9~t* z-hTjCYw@j2NA$w_RmTuxJ`gaLS=!+IlqwPRJ@dN1;u|(&mLc}Cce?8a8>e{X~M?gjUYCr&?xqm5o*pa+1NR2 ze~$~WvX8f;5Z|t1{U~k!n6=W&hvHh4foyI?g8SRc-seg-KFoiRk2~jU#S))&aL*Us zz2S0I4@|J(>d!T=7>o8x^*A%7ooius>01UpmA(|cS((Ge&bA4zIA23M`&UlbQ$J~Dji(_X#6nMxTe@%NzPvnZ~ zICRX9FZG0(cLe!Ccvq>10z=u${A!yG`+O=@T1M%pmi#uJ^>X1KMH{D3g4Ze>T%b1b z;6s^S>wPoNJWucU9v&_P0I?p-%@LE@3gI7I2%m}`e^x%7dl&0HaUbFc>%E*>!GILH zHy|VQpv`}IaMdE{k1gkb7fhos#mnn4cV6rPzX&|t4}OwcAG5+?Eb zs<^gCCo$(6WivWMav-W50mcYY31HO>>t;)EBwRviJe@{{XW{PHGBfeY z$bm!4fvw(INXhN*O};0WWXiM4gQ;KJp67 zNG4IoqOyqoud?VIYn@@%Nz|A@WZ3f%Q1su?0XXJ@Wq1;*GQ|fm8Pe)HldI6N*ln^z zSa*L;R)45wbAx(6)tKjDHT8XcF7OULTxu;~sOBZVh#0L9ca)0AJqolfcv5VlBlz+d z*L^s2{g-@L$>Zk?%y64TObF=`AvKN15hUkx0V=jqc=v5!+9ZF&mk7u_dEp4o9-vZU zR1&YkoAYmo1$hR@{ugtS%lY)^}goL7dF@3w`>9o?eb>}p>O2jarH_`h8aI2UJxH%jdJ>QNNna4x#WH5aYVE(D{CTr^HA0u`aKM5^8v)X7MlJL${sAw z$YMbV1S4lF(s@C3VJV&qzN$kzqow@4v;D0q{WANB1OjquYqb20@?BD+qaQ=HbwOmAnN9BHeY)2op_{T%dzSBE<5@N+#!rKnh4yH zaXs|oh`RvKl<8-6jo|hPZ-h8AA4qRHuIx*ba(bpxV05^P@XzeuL1v+4q7kr0zRodA zhsMkG=|;UStb!?RA(iHZ*dKop0nP5>D}quJgRFDdiKw*IsJF!HIdr>a%KfF`4WFNM z<$%#BcF9xI@ap;ihiC`@NY8I&HLgNc9STgF=^je6DtPZpLwD~#k$%xN3{crGiPg9d z7PrA&Y;%RgnVp8Lnbef=!s1Nw_wd1piR+VuP;;C&0fXWQj)w#)jr8Mw#A$%ON<}G@j#dGh*i;Sv{xhHl2LsRQfE_< z7ho|F39Dyi9j^*&ru%;}cS}*^7;9~f`4Q>NX&_UtEBsS8k*gnEI1QwbJ#6D+H!CCz zKvTeC0@6__1YnKylVa#ZRXdWc8{Nm;g?egK^l}{QC#eHnDHqCm6KaB|-Y9H6d_{iO zk{?PFbbHHs_vyc)0dsujksnrwPq&*OepTFbgOf>U5y7(CT~2>AL)8xg_6o@GD1YHm z!ui#;gov58*V&kYe{}ebKU|$DYs$vYkG;%8J01)T{O*FUWoRzz3zUe(e#G8G-5&ED z*IK1|RL6NnM4ot3!GD3$^p^d3T(;P<|LjyWGp;K@sy$!7A*d{2m4*Icy)z13bA>1& zOC-wbvmOh5KXHGmhDcp2QEC7=s4JSLl_q|j-6!~hn%h%}0J8xgc;M5*#qqC&QV7Y! zXMqwD54U8w?f6qb&}O4vA8T3Q`$YgL9nU}yzxABfv2n3_e#LNMONegIoK=Lc4@q3%P2ZMxaXu^Gx~|_*{g{8k$|CZ=y?UY@B{-->UJkmY}f}xJ+sW>t~ z`iAj6F*#b~M~8&V`f*G~>`Gh*SQ8)l2_X304BZnUpHTAn0DbK!DsZ!eM^n|+KO^s@ zNznC6Fm-<}Ij(5pDC8KUqwO9FsAcXqr3Z9rWN13|s(F*p$6{#dT&OLaHSV%;z`;ou6$WiIo>8*3?jLDyEB_)_9jGDzJ)*RDH8lq%(3nwY?QAdrN$63VkU9(nG z<^%uoWz1TvF?_;^?`NefBsC2C?LTAIJMRfjItqVMTFFofvtH*Dx;I0z?N9N9n>5M=#&2$NIJRh}lSGD-SkcLEt7&3rZy_oV`s{_< z5yyY7ztos*VP_eC8hc;8ly zSo%OWvGn?u!TZiTmJay!R%J5$4$Hk)>iz}|u5?4y<~N<%55Pbnb2O0*uhI8-a^*Kt zqTkfe&@kSG&Sp!fkpw0!-I-miEgs9esr`Q{ytHz<{CK5i&y@l2zCd{M2-yaH2h+|u z!n`WzaJ88|FQDg}-fMz^OW3>dc>YmClLwgiNy2a0Rn|xRJ>|m8)d; z@{X?^L|%;ZN6M5m?9|96C$$E5-41`(fg!oX__{y&{x8>W9>Qg;)V10Gz(tq6f|7sbg@xud8Q+BJQ}A1QZ;aCnEej#&)uPk;sS?oNM-dSQ~yTFp$$ zDFFp!`~@<3Mij5jgVJQ$5U6U7(`A3&M=4u(6zRJWhnq{m>jlOz=|}NIpMYS2il=o= z8NS%3ooTZBGxv?}>w_1IljXMSmDNt9?~I~K=hN3BXqFfxbF{h-tE+hsVN0BZQ*WkTf$p_bG`~dxn zd;)}u3DV214R^-~PqkDuvkr`_3J!JB-sPo-XsMxnD&sKH3lsvAtbbMwJM4X`_lu*s z*`nkjr_1(+ucz724t6F#b;2BFo6>+k(p9TFfI{Ph*%Idw6l8zkOG`WCAmyBk-BT+} zWp2|RR9!~f{Z+hF&b9yJdiHu7^wB`OY;RPfy z*z0n_L(&g18f!7US!MzmrAN+RKabjpPp4XK6E*4#QgQ3jNVwcATbF ze)vj2sUlAPw$Y~|F z@W97%pxBag6^-32$i6C^T(EUfZdpa7^VAEo8FcPinR$PVrLz}K$-`&4$$^z8UF-!T zlkIu>u{5m-7SCJvul^MCrOAXX}X<2Zi}zOc(~YdpxOqb;yx;(-uCmQ2r}HOTLf`UQjI+*(pFJKD72<4W2iVSf(C zqe)Nu?yUQ@quu(HI`zWs35E~Kd9hTiN!KV& zPOhv)5n3)ob%do38mT;kwh);%O|Ltf^lJI+a$%FYLNT}myGa=%&M{U_QUez}Ibla9 zi}HV($`EJKAa9R0(~~20^V|SLp#sm$(D(}Na;Cb+Pq`bWiGXvVjJLU0H!~V#Ft(+Y zs$J`I2^e3vZXghJPK^J6CLE^N>MP$KtxYY$$_B(%X}Z@P4oRTW9KNFcxs_tlzYzyM zCT@)a@grJ=#NT+%(T=WKuPwXO;N0!*@?L-0U`jcTR;{Wl#g48=gCw961)^@CJ~fhI zbq7z49#_)PwNerWIxoF%WdV6UJlCD@?r_N!oww;Nvfw8ih?wnENB7=n9C3{@a1(QR zRqnJGN}HHfBo(hLcJ!^XwHYC8&$6F{=Hcu;woE0KsxS+B`jK|a2T$=ry5;PYYBzru zMM8Gyk=M<}YOQN%)Q_S^g?&*@=I>Ld``mmo4m#NEoX4Nfsnh10W{uQU@cL}4JzV7C z1kYc%-|1Sp%%p!%Hzt8rA00VXGY`$XD-G^AzLnOcvUz%O)X^{M z^zViKCNLTO?2_xS6y8S#D3SE=+yK3tKZmx3UPIx}zw{*Xh#oc;kr%E4hsFZ>U#=lu7!(J*S1Id30&sIBbB@XreDZp z{BTzuZEB`gYIrQaBQ(FKb1c4L%JK>I251Te%FMF^AbU%U$MV?H--`dz*FBTaxkG<} z(`!{5Ye-N89amWh2fy1gxJ-X7#{q(tqFGG(O)K$rcrt4uUEx;TzH%$zjQ}ZcXn>m# zBvwpeF-gpY%w1vi@UF2=tfs&uE7mvDv89jC^=s);-0vQ^c$bVPZh$cD2PwU> ztQWs3dl&VbRnFR60>Dv`4Qe+j5_W4VXl_IT9Ye!>EB_Mrm+ z$7|@n@Y1I-Hh*=MQFEYo(Wge%rz)t(#HC4!?lYnk#j1sGj9p`0mnXPF z#`IT_uPs>x(Nm^*&$9eZC?sY>R*^8?{5EroUsjzqPQ!n9sU*u(*!E{f6)ADB(>%_u zrGMOT>oBb3mp*z)|LPZa$LeGFelE?S0ov1M>#*Y|4L<^({w-b``Y8~XX;k=@-5zcX ztH`xcD&NaKMU9Q{sPSo_lnir6aM|iK!cc5$)3(sVnz#oAR+ffv=P70#%yPct8L^H4 zOE?5A;bwp1jhkNkQ{&QDAsvH2Ks9b0Eo3bgfyPEmj@Ke3QG)X>gG|5nDZUUd`$+*2P5euoL`C7jV8l%?`2 z$q&1!#SAP)&59iuUh`W&t9GE5f45C2&5k_Fh~j^h8um}V%FHI{%kkVN1V~^^N*Fp3 z!p{^R`9y1<{y73Eegpzjz%{;UOgw z>{x$x+8#)FhNj}~r53gY_WmS5dc= z%Q85H9r^=^!=sAf6xGBHzik$Jsr^?UzI#`aNMRfWmOMXnBi6>)y>q+hiD{3*1 zB4R?7yM+%`D-l^U1OZ80GlQ6QJ1t%sU7M}(1gWYu?W+!&o7Et9n&XWH=c4^H3MWQh z=8sbdyllBdLv6RCf^1EfIpl*x<4J!72Af(M&XxQf`@3GY#z(IQ$p+?luWDEFgyTT# zS*Md8Cba9fs07OJe!rlYr?!9Dsi`HHTrke_im|lkeq1Eq*|;sjg9*7rxG#@B8{Pcel(37m7c!0su2Y;m(E# z1AiNuXNfKjX>|ZjCw*UEP%0)Qj<(EpUvjL9Ck8XbgDIvkDJy+vh+Z-gU zZpP1-*Z6+2v|ZnQ)GKYlk#)veZ8A_72ityKhe5Awqw7*^;((P@vs#8je9n-#K?8i0 zAlHrdnRt;2>d||jSngie2nXuAXNfRf$hZj|3QLh`1~KFHIh+jYwsU{Y;&s>*O87oY zvWC=J3o8_IP)JkLUmDB4!%(;p_^wAG>TMy;SFd*f=J%J$tr3$ov?xwr(k?C66>S2F4$=hWL$k1x3xtLxbJFM@SG_8wJq0&k~`w{OlEJ1i5xweft z#(wZtQgg!T&K-K{q%}QtJbcu2*N9XPs=y7RQn6-2&wT*Ydm2QjgZXt(J$proL~GNT z?xGC!-g(5tIAcYlFW z1+ACxJ-*@18*>@Njgrul@KJn!V%`GjO!<|VAr|WR@H$mE@&jNcb|f{-iRD&-s@Dp{ zO)xW5YOoYRHuS3g23nj@B$Gal=<4)EXkrO6L%dw*A5U&9;;pYP<}J~sIjlPxoF>0} z+pG9{R?$ijLP&r1vH}~Vso|b}opDeBYJNtDjJ5}D22JYG1p(#4(#U5EE4g)1nVGDE zD~t7g`hA3V)IIr@Gn1S^#TJnj-gVqPPix_&zHp1NS5o8Ss0_B}*aZHrk*`)Of0I@@ z;wc`%Hmvj-t!91Wj37+WU67GPQ7mYwzVlOOpa-SaCINrGu0~ywsnJ||epr{_>DKadQFhV2Yn+EQ)-1zmI2hs9O%Z{A1bpdiQ zI-7b~xlDfqaxs0#M^bpwARxh)EmR_#0?B$5g!$1Le8W_Chy%nPIQO>~_>1n}pKYsd z@TjU8mt%iDfCLb)zoM7(bcWCYXRq72WkmQ#zXR0*;!7>BXOQuw#@x^MR&+#JlM0Z( zW1vn6x{-WZUI5#k;QMSyqEcFhBhPQzM8+BIpLu_PlT7XEaDwdbzECk_cVkOWI8Q63 z7~WDgPL#~F2QJ}X*S>IEMsLW?QS@L9<)R0YFZ-re+Z!T;mEEV@(-Qw3I|MVkI0ut3 z$l4Ti6VY%4O}8#-(-%tff&aD{8As5b_@8!$%TT3e>s(qfmZ3%_{+H4{4hVCV5j93Uhbad)rC5MuPe&;C> z9Baxw>;Oh;M4pVrmym8jqkU&kDXC-(6qYBu7ZrJ}By)H!)MhL~iw&jBn$&NHf{7}F z3F~Aod~3D~<4OJES8#f`2 zb^JQ!jiTu(;yJ5=*vd)sQu7D2aQ;Ff3ZI1CK)Y{8?mR<-l1zyZ@Kg=Al5E_dlGA?^ zbAB;*NZXrS0jnS|5Ef+G`C%Z8Wt7EKSOwl%;n{6xZfW2OEST{sgED-S1l=WO-EY&m z{oq*&6jv@gRgh=CF95|^AcoZbw(Qfqf{WdUF7ukMMtHoRz+rnpuxBLDwWb4SY zaS7U#$r@cRP|j(vXXN}4AkK=ic_n{S)iDm%*uFbYMn2WnvJv6e=dKhEU1*kq1m-@+ zo%pirE5G)9aj4rF=Vr!a&XSBt46>?>xWR8;;&rrJ$qARB?~<@&A>bz1X!`}j+_EbD z_upuN2!Q?Zf|O=6dzNxS##&!{LR%;y8sZ8A>#u_z4p=KCyF|nCw2zNj!99O6w`qcb z$S{VlT{i+UB01-Pd zZV`a+kdy5h(a{5%Bbf;CezjlQgkdWlid<#Dl0W|1TuoQJ9vp-Em8ZaAQWRJ)1LS*x zK`cD&Y@pJ_H}#`>+x%#>^K#@87Vcen){ZJsF_}`# zH9u70o<1<0zMnGjmX<#+@j_Khg3^IkvR8tTV(>RU%u-fkkI{5kK39O*17=UwV4k;1 z(SDx-B2k$}N9o@0^Y-aSkk#+P^U#5+m3u0*gD6--DU>Vhdv zmZY-->1^u6{~$%M7nY^i1G|+W3dt#=Ywrs2;cP3vdanCmgjS++2BC?#c$|w?1>yYf zntHuMh>Z1XtlCJ-J*r=F`kcDX43muB1Pjlx9(x&0jU%hQU7FWaUxrXoA&t8(f1-y8EzTS`o@%@p>E!?IIa;FKJYBm1CUR6d(fB}X5+hCFTvka zLY6Jv;%a|}-u9~vptPgcJnfPcHE_(GdD3L^&4%Db?Eno2el66W5qReceS6Qxn;|vh ze{Q=KU&m^)(Jxh8*YQN@`t%4yUHv9*^zFq1dtb*|osP*9a*B965tP>w2OwbYO~F=T zU{iq`l=roCAPywyd(t7LvBG}jZ~_q-v>;=vCTD*pKx8c*ho3|_U4hYPLVF_T9Q1Gz zL>}w#o+)P1_I`zQUvBpMY9PcG6-^A~E+!W>pVha3n0dX4m%+NJcsW~XAMZ(O+`87+=* zA`E)K<~-AL9v92VjC*o{({ft+_JIAe4ROSR{UlaHSYaAaNfGR70(1hr_Xe7thcJKB za^626{TtK$D}+7C?s2tlf(nwFmQHq0Wl+is$HcK?(>$-*3+4O{$3QMwpcdJR^sMv2 zV0JG|${g`BQuYEu01^`#rl<^z+et?!H2j?7gPbopF$y{j{0k!o9_Z*L9g7D6dR2;X z^gv=+?J_KbtQg_Ryf-%RK&$JfhYx?Ib%c+nB&qe%(4Ij%3=SQ{9ae8jL=32H@Q|o# zqAmZ)WyF>GG%}sc08s^O0C4(cq5LOQ^nzvoW!3m}%j=Op8s`0u|4ri%Q>qH4x6QY7 zT47j~*dMM?V-UO0ep-qb2m3V?E8%9itFdIG`L8m zj#S5Dy>_c-Cx_*mid)d+MffoQ!p;T-6GXB-&mzwYMMzXq3aRwm_Oz>h`7@md;v-isO0&Cw6k+JIrAr{;2a0!Q?njy{-M;ruMD>4P%30@hT=2Vts zThoxg0^rOqKVUn@Qj>opwl)4TVlpf61KreTzoiWXa!OCN1BVY=(eV}qUq8$X#C|BO zYC6@4EVozkb^>7Q%pXfFm0nwvHCItpx|mda7+E|s)@d$45d;Z-RoRvy_BB@pFB1kC z?<7@sp61t`;)ePUzwssWz-nBfNDqPfLZs~$u!`R?#UOU3= zMt2?G3+Ff3-=haMu2=4_Ybu12&$dZmHwA^`zN9E3xQgDqO02HR0NLRX78&ywWnYx) zROY&*-Ym26hL{Y`v$+5j63b5t@#3dsj~}PljK57m%Y8PdDTZ!WemBa0#VG2Vw5mt0zN{cRix!UoOjj~W?Q7=%&)u~~Q`=Qb@rKmG}d z4s||h+lvJq?T`Qvk;OgvKqlkK@}Z5xEBtm9Dc#MwH|ztQ2%N2}EvpM2?2)k$hGMOV z$51e4tR;Uzcp#$RQ_(#0J}N*ooqA3y(Q`$6-rH;qL#&1M{IY~%Bbil3vBd=f16#iZ zvf>4N^!K2=$umy)K#0kpb@$D}%tf`-K)S3rTEt#mu-#(e5NtPBC<`JmOaG>)(Z+St`!zEFhXv6DmB{YL-hFHFL9Y zI1gpFsBG_;gUn7@)G~Vnz%BzCrVocfu;a&b5!tyFgro>p-F% z&)9!$uNdMpxt+22e)uI3+;UD_J^hg}rzjQrLnEBd5%=A!wjEE`Ks@rUI)2-{?=k|V zs=!TzDPXY%W&UBN@B`y7iikr*g90-3ryx{ml09hB3uR!jv z_E_OGSlZ_F1OAmT78#zlQM@3{kM3Z0<>CYt{MhhoBhNvwFHhhc1Jmv6R2@Jmg1b!T zbdSDe=;EZDn6B?_`WV(>p+5%AV)@#aRnVk4FdO5H?mq#oE-4%E(wCR3KGz$x@UVX$ z4{*{wcyMGuyc+>hxM%NE`MEf8mr}VpL?J!X71LfJJyV6*dZskd;1^^>e@ux;K)3`z z5! zB~5YMT*8Qb*a-Wa;r1zs`r9$)zd`Kd%Zn+%?54(+s1LXh@^zCM4V?p74a z6*72H=T@VrMvy2~%w1e;MF9UIb4gPTZvzvh*GfXBNlu=mj%5U19hrQvRc z%P%0MqC_j!YY~h&A_s9dXMo3-q5@>9Yn)!8a5cVZIiRYOEu}ugVpS8xM?AhK9yC@S zKyzkeQxgx@Eqzta^&ic0!=^#75McbL~oQ7?Qf_SIu{2vla6bz1HbBvanOm~`T$zg@W4LcR1oZX z=fhvhU>bmx25eoL*1BG`({<3HL12+AnO`v9NZK7t95T{xHPm-HGHZY0BU7^(!|!1Z zRs+NdIr*#ipMy1nOI)TS{DqB1eWOobb#`W2kCt0!XZnz8T}{DAYbDk_dzVoou<>XDvrx7pHdu@ z2?>rWY)WGeTN8h{d)gSWZ^JfSp{9!{>27!fjKyJ5(EtH8%OZbE#dG{*Ge{AnY=~Pa zBcnkaEk+|d^r+ACLAtVs2A~z=erJ<0Nspt4!kg5%K2#<$b1?gHFvobL9l}+&E8+>+T?(XT;8JD{&5 zlvRV3E)EfOFTbv87G-r*U;a&)ho4^T3z+DffI43i-(u=c{s=g4KqrHuC zMF8%TQ?ir^d1(ahzt!ZC z{kXqIb=sXD02#d17EnpU93Bee$ zkLy0Bc1@#YCqm{h&vpTn5GDx__ok2q74}(Bd`0bAm_WeMz!@d!RuT1NR~5kL!fDo^ zQvDJVUvz&PKHHUv(I_?W{?=vHuE`9Z_ivHtFILwsg$=(sUt)oXEb`1tTRD~> zlxg@pu)WBv~>EvaT{l>E9|;3;STO%42wOhr0vU7Wx?m+5Mch%3H~EqOCOkoB}n3x0&@!kI(5c-7}Voc0h3w73wSRC)bG9J0Dr`?&!_ zeL{3m+4YYc4m)V+DyX4!(wE1^FJE;RAOUrMd6UzJ->(E7&2lw*#L`H^-8EihibGiy zu7uN~ym|NnX><}qt=gQ4Ai({U&X29eL}_Cj-V050)e|+aA?=XadNKXg7)gS`%vp zf|oeQsl_b9qGiWgM)*opU7V!_*wYU8A(iCEK6Nc5RVc(qS~eGe8svajy?wW^HwtSG?Gq%8{f zP2>f13PPs}112d+LCPc+5D!AfwV0#81zWi64HS=K03Pl0{HsP2$Az*0PJ%H6hn|An zwREEd)#Nc#71;&GbdD!BY}$!k&82sKF8hNMCpCp{Nfz-WjSQojbBc<+mva?=LkFmX zx;J-6f3mNY*h3AQ&_5e}SnMzgAqA!dkxEhMj z#rlF3EP^}=gguj$s!`9*Dfw{Q55BADH5vN#gNUur9jy*ucFRR(G0YN@q4P1X(#zv` z#&3Gf6X2VrO%D~u<@A(T?vx*Yh1c}@i)W3*1Kd(^a|_O(#Xz_n+tLRKA2*0Ce!0F7 z6vS;oIp+q|nV(nrSw>U5&a}0=J>G0eF>>t{3QDmwHt{xmn8gqpQSO!TjLbi-nK#b0 z+P)Mc)zM`)kP?wa_T73#w|13b4CCvRh=}5Vi-Y=ne?o~nH!2ae%e&!!lO+f|h7Wf7 ztqd%$90jDwa?V%e8B5{ifsh#WQ?9~`9i!d~D;rM|gF9meDJHuA5S2JzBsQg(Aq*M` zZ?Law=ip#gR61+1TEZOUG#tzqbc$DVkbDg~Ugi2?*_Fz19d+iNJq?vk?M}e9?aM*0 z%o=dBe(3T}kS?6YlD9U*GgBEV`#I_S zVA^~;L;&2N0c7$M({r4AY8rv@X%Lh*l>E(4DrfL?Yw4eQ`B$kYswv6{`V$5zB@L`S zmJfEEtsvNb(iw}X{E5L8YKOW2w?~Y#X%j(SGfP2N6S%Adb($g-l6Y!$<=qLCJvi_$O=^GCYXzkC?t#mFMNwFP_hUV;+@q z0_#HSZ@Jz^ArM$N)`4ujO0IF&VPA=T!!Dj@Q_;u{9cLPUMrtUxwYSo8Nt}ukFs#aw zh7z3R7`dClPj69A206_a7k@jHd}{-!%}A~9jwYFfdD`TAn<;ekZq`EeGF&0t_zt?l zKG8-@@KUJ3gDc<;iMo$~hiu}i2=J1+I`D(@Ow2sWyfAC_8=(%@p{Ry!;ovf)@7>`s zsdRM3QNv(=JZsN25eJFQe3|TEl>n(9A zcM;t18yqE|oIys5s(x*Gtm9aKY6a#O%}Tf3NNR;rS3Ng=C(4nF0z_0If68 z#=uD&(WO-+F(^u*?zI&iryd+abhvg2qP>AI31w`(Vb2!A@3}y;^Jj-i>2BgF)Bxp$ zkzr!Stm3=@xr^Ne>U;yd@lJ5H?<(q-wsjQwo&CO`!d$PxJ^_ZE>Ftz1eGS8SVqns=rFdN0@IjiMTbO+2V#EOh7qF(*6pefHl-vL09`SUvF+W=$OO{ zDqWKennJ!$f_CwRknQm{r975KVnhWC;{qYc!eN-esQjN4`k}XH`s7C`P2$b!Uaj8HwgwChYis}Tz4KA~<#+Udgz2rg zGW9*}fuaq-ZVM|lY&^8iLr~>*pQ?1+?Y*RDemag{*=k*JBI#!Kl2)q+G_ ztAV{++|#z?($;{wV)N;qojcO9b=3>PcUY-3mn)#m7I2p@F?@AjXv{E-@HYZjhT%)! zvf1dSDB5N$2B?|;6<|976q9Rz43_^jm4D@5uoyBR9uqsh@3<0(Ru!x%+WB2sB4-uM zC#sPmyImEIUhHdQFP9Mf;-Od8K~t0yn$R#j+TVZO(-Z_?a;{M0_=m(bAGf?1U zfdEC|H2&f<4X!Tyd!zOjLWe5iYDo~u?={5D%d1cV*Yvb?yTZ%}d4KzVoTCcHF%dM| zMWV&{%_=F(^S=73=r4dGIS#H5OU!o{BFaInW-GR4Ew*LQzQ|kgCAi!D0mW-}L4Pml z{92%l=&Dz2++{Cl5|VDnP|&lP$9vs7yg;AnP@<9*r0Duh{R{_%AzZ|`N7pW4lU8cL zMbVAlnda!D*>xR|d=UVD&WdD5i-s5N)<*U8atObNAb&J;^(Yx|ST;_rOFRX>!EnX8 zX0_>8)t-QPEjERG$63-+IG3tv?B%bh%-tj~@C?A>mSAw_U_f|L7}61H>_5ySr-q_s zIV>na{wCx4F14X~4rt4?*QOpMu4tYpHSKT9DtXc!?5YptnV&>|2<^-O_e#PA8^|1= zj1xh4>EZy@9;M0^aC9cFhYmUm<-8o*=GYkC2MsvTuMC38VB_n?O^H?J3t$C^C>KE{ ziEfz`8C7de(kgQVtxoVOjJ?&0vXe==BH?b;`d{uZ+p?7*_N*%&J+;;LREE!Jb2d#= z2N{`_R-Y~AC?ySlOoH9X8CQRxy}3c-0=(`u2!=tLJy~k<#u3wL(mxFcZ(3UljKN8O zKP^4qq+!U!(aeD#MgH8qkILY5^Ps$xO3g_>xEZE2mM)f@I+jR~#)PIlP<#S1uqwaY zW(EJ|YM%K)0NedAWHjn{_;^#1*ddnKH9i*p)_MB|gk<`Em{{+WM_~l1sbjR>?@R7A z>@NE=jc=1@W=mx2;A*bh;YC!(=2_<%hy`WdaOoE5VPEMv=G7UVn(eY?F=YHKA7cyM zVI<5L`jkEn{VdyKL^6#G{(o&dkn(j#q(nX&WDzR7Wz_#l+(jJ8ls@%t1vR2R~ZEc(gO65SuA2z>#Al__osnx zon!HTQcAE%I-;OwSP4W42{xfkRnfr?teSIsiz3sKOv?oA0H7a&es~3s1wR&?8gM4k zRqH=;^*d+@g|$jJOGf}k!6ZbpWv!m&XO>BI-l=1qH|q%E$~u5Jv^KK!{H^t~9#6X1 z>2|DJ!>uc|mJ$8?I5qYVs3B7?J7yL^+Aec{OZX7dut&C(O^k);p~lS|lU?{Kt)n8< zLs+R(pGQ_34rgHur;K&I_^7X}xPVLNGbStP488JV!;U<>#I5R4IHXn-cS$w64tn$R zCIgv*FLj3Dp7(y%f9uQogfO~g9)lI@HbjHtj`eR+h%c1e!zwtcgp31)0jfO~E}E); zD_=i~rKNyh3h&{5Zk zpb&RI=Mx+ zmgroPIJIClO5`W6=yA;r*v-3NTG-YqOqcpGC&{)`u#5z8F%2R#e4N?azAOYJB{ah3BF&pu?CNz1eW+Q zS-TXk-fWJ5(JyI~>k5>d6m@KWHWBlcf`eSkqu|22mk8mEO)J&hKh12GM=ZkxIqCjU z7iu13#am<0_-D|+m4$;T7M=OSk*BMZ$^Aj)V#>Ryo8iJ$vl6vWr}D)wIAfG$1c5G1 zPPM{!tULD;U)QUFg=<*f$;B?6eaI}GIFV6R*_SQkq-JXLUpD3q0M9pnB|c7yF{YYS z)Ga|XLpG*KBPm~&^muTPI}dE!^5PgtoC9+*^D_krGQD+1OB8ogi4u5J?tK}5cl9eX z>$r-}9MVJrCUf4@M)Jyowtj66ON*olXvXyN4>q2u+fbRXSADD#b_eXqbI`6ZnZz_d zymRes+j3O*S{ZUZpH;$t!3BNPoklYJxv<`FjfVAS7n*(CEF!WJi=a$eIn96v)xPBr zp{C$J>6@b)ppdLtYC9<{c};?b5_~$ zSxW-`rh8BBx_YscTOy~H-$5-uUC_dDJ5#^kbKx_I>+WO3B6ti4YSuVaH`ABZ10@cG zcB7{Sn$Ub)av$7(l6j(Z<3)liW2zr-fUvhzQKVF(whS&jZ&Si5K5UY@EXX&pgf&b7 zk0b6Q>!%Zz8j6%H(;T(Pm~Sr2N2$Szip`sw7!iC9)Y9Ly`e7T)xnziO2=U7;?s3IKrecGC%NQTRM{S{zx=#@KS5C;q zh~j}90inlpGBuj_~Df4d_ljLn}92 z^(0;FZBiO0T4a4+0)_7lWmBGz2Go%r<|`-J$Q`esOQ5i}@*gP?*QQ_V$kOa0vD)=@h3M1!Rv( zy&5%w2$x@4+}|=Q7r{V2Nk7+z4#>wgEwCgA z?J(MZMp1a~z=_&Lc(v0Q4mA)s53l@)Wc10Lqvw}E$t)1>jz@+W(}{iCY$Bekh0gZ{ zP7-}ZQmw_GKd)x`R2O=FS${33iiVLwuK&(!+XI;&56jS#w)}&aX(u3CD$?W!6>S@e zwG@S15c|xKEN2sJ2FDrsGbn(&3L3Dsn9NOhFPZthfAFW-`OKQ>FS zwfsv{E3%1zUU2C^VU%NYwfc42Lu@88;% z!4Kqm`};9cBh7jWA#08BnMB*S4gx4c9T#xj+RK9<2pm-WFgeo%n046y_Frv(Eb(>1 zHEk>RpClgS~H0NqL&x$S%Z`i*gfZr*5$8cNZ6^-&MdkD}qjw9{oCXni_;eE1JFt>fOk2w^8DMx6VJREB5Ya zpDds-nk>EksVG3T&XO99_u}+-|DGsDvW?ET{%4_GtemyoxLQDeIl3}kbOy+I0X4}G z0h5J*22O`m$4#}H8@l6L-Ke>2^`MJ-SNeO%hClA;z4rgm_HE6{I`5WWWooYWuIwsN zNn&PdE}lL?1C2I_igyMjV{=e3!KBeoKh>Z~-uL&P!>*~hNF~NYp!+$lwVuuwn~@ha zezz-<41hX+|DVL2TX(8T^X@-TE<{5Tb6I@=8f>&VsCZ{EHj0rXBx&^1?@{0>bFFu+ z_rJ$JT+A_N5~mU9Lv__tzXz=EFbP*41R<#WRhrbHI&)oBeM1z*c{8knV%Jm==d!P- z5L;h#dWdK$x_%9=UkI(rWa_?8XXdN7wpp(nk})@bqn>VaHlCaCU7P>Jcx;tl`kwu$ z)otJZuI08ltbUtIIzRi<{(fczk#kJ7y>NJN{#-Q(?f!k?8(q0@I(MPrTkHiDnD!2H@Xi$(KR9Q@JSxZrI$n@*QtI@pXE}iJ~MXe7CLcFO+n>^#AfYoc6Z~= zi7~D@i&&93I{g%8eRn^3T6KFk^26r6XBFHdnIi@Aib_cOx%yg6-{U~p$PbIfEkCTu z<|pmm)WJ3NqVc&fC}$@POs7rL7hcYpGjrB|HFay`Ij_;>YInDF6^S}OXyYu)Kc?Po zjw7t@oL$ z;-a2Lf)Y6RfimAd_N*A`@Hl^DP$b{m^=ceXtC*-UmHm#n4PTj*8uTHq=sLf$@){V2ipQuhA)ol6JbD8xMdkj^!Qy z33XZdjfcQVE1%eJ-d06$$h3?E>-|)JW7wMs>karF&0a&onZJRCYe}qPSV^VFf@X$> z`H~o&QK{AeH*vSg z+ni{7j_T^fw__03N9)2R!ZC5o?*VI+w|Q;sL7O%J1uk1-H&LlGnRb^O1coKJvG@P*~nDv-Uqc3K2vGxx#fbOzH_I!?STu`f0u+>XoIu5go ze@%v%CvW3E`djz&SAG47>3-^e!MDV}=~b4)f6%80h>#Qc++wL;z{7ZWTFoQfo_C?^ zUEcm;LLRc4Uc*bzXva~Ea2-}}r?b+vAE=FevbJ~a#Nfc!Z2W_~YTdgv zNhVwpzp-0>hZk$2j)s7#M6`28-XTS8C=DjI<0Khp%Ae)~Cvc0hahg)Ji~?n|-cQY1 zEr`xX${CcGy^YyE)p?hX!Wan@)_8eV3^N9tQ0+gxD&YhPZ}}%OE*+>|MHesGxv+sz z%&{5g-T^-P^1iGeQVEBDgn@XOS)wev`G)nC?^cg_1fC*CIQZ&2!Q_}47w6ldWxcic z_f%)Otk);~toMhW@A^Eq@|@q;o%g_B&mmDx_lz$(?4vZJFVcI$1u~AK64%2en9Si* zcygPS8o`1rnB7mYx}qQUiD|~;D3`6)k7G0E{%%gwpMaT=HgY9@;@bEWeKPVpg1Lbb zBJx)3pSAW_z?ZN?)}~}6;zy;jM44SGj;jw!^eZj<@0408hn$o0*#uM`{Mg~Dtd`zJ znU|Suy|lg&=`3loal3@Adw%aIVW8c|Qyne1)irU|?sHeqm9FM~xwrhP-yM)eMOqH1 zdXC4m&%KSkc8QIDXR{Bi#Sl_$9B_YV@9FNI4kx3n^%F$-tJmZU2U5nYQo_Qx4j|{L z%a8xR*8k^P5Xd+!pyPNmb^3iRH^!zC<0_MT{$_P>kU_Y~A2@K%Ovs|heuFnzOq#Ty zDxHxnZ=9k#=SE6sSlTqvzxoOiJTpbgd4~;rkJ|oxHiC73Nj55Y!paNzl;TuZJ6w2v zhx|?Ao`I~=sfFt^iLHFXG(Uz~#qe1#c?4rf>_?_;qO9C?Fi-7PW{aqg2w z#Gc9}H9Qk7pNu@h+9vN13<2!3ztn`7PM3ON`)}k8FY(Xsy-avxIO;&eX2|K2m3W}& z%<#>B$fyx~ngz#(Y;PEW3Ir|mpI={i7Lg7rz}+1CdjFmuW_9)R?8@!w@}Ym~F)yql zDEOFdG>-2uxhD|?jAgjs#Ekr?mTuMbxHXvyxyvV}64~z4QBl6Tr*8ASyJ!*}4 z-5b^l=2x(1QO3weWE!TJU*T%#p81IA6>vU(^R_)#hui|wo+XY%Imp_~9FsDbN@v&0 z?8y5W0LsTdQ9F_IUaBck{tFmykY_(bILFm@6}9xJ)j5W^$iN52Rv#^I`Pgza|K#i4 z-}I5=J?pe#8+m1A6u;PV_w~yjJ{~nf8%G|Szu-a42=eWr$xfProH{^ zT5N)~*mOPw?OcCP=~2N$rysodHXP3NCSKTF7TMtsX4)Z)Gm3M(?TK#Jwm+C^$1H{v z9ONo6oilg~OVUE;)rb_pU85(~@9dwa;y2Hy&HoQ@jd;U;Th{W1 zXDzSb(DE;*B8I)w_Oi4!WUO&wzHA__IoSSLFVsCJjDc@>h^#%6&^L{|rKh6zh3qK7 zpg~G(|7pFik$+4C@U=(vc+yYzda$cphW%uUJ+H0lB(XCe`7QDG9_}oEMgY-unZp}j zi0ttBsqKVdexf}OL^@R^-yR&v8{rY;I4K#sw8+6|-%C69-fKURE30G zaU)3>OUpY7^b%4Y)H~OIn56fWB`2QMm>Lx?bQX(aO0kiWM8cb-w&Q)K6HpI_@GuN{ zYJXFs98f9oHrXchhH>@C1!=imXU^3o{oK$2N3SHzUojjI!J^|?krew6=;vxRO`pR6 zfc=G6x6RKUc7L#r@?#j=>~A@CI-fef`Ln2vy>RWiUUq)s9N*S|24w`=j(cBKe7z5X zFiYP}3#^+vvq`BZ=vzUQnK0Adw;1B=h6DBxzBc8AHM^%o*{A*W=dR=t=Hmm0>Hc#% zWfNwFfNS5O<&BCNwh~ccVx1*iF9J!?Q_WcrB{}@MpTz?7yDKB3MNv1AuINmi_T%Zo zjdevN>WIF|QOs9=EJ7XX`YCLAE2gVYeHhaE)Fh34_Jz5rQ=hp*`5ZOUBP}nSJ;5#v z2KVIicz@M=w7laf*_^o!r#|>!amnxJ+THVoLy;?LtZiWOErW`{F?;b)*P&C%%&?75 zaQtv>UVK&O98jyA&Sq__Y_{ZJY64P_A|Elu8H)!gyNPXo_|Mbw{C}DL$%x@0HQqBR zkaa02jjL?D9}Zkyvi@_9Tu*Y1Z@5Fl^_(6C+i{^kDDtg~(uGDZY|by3}YZ|ebJBIl?4Nt=TSW1Idy&!73{H$D(uzvAHP(S!Yp z7)zy>dgqOQf6pJj5T~5oAq-gqGyslXfgFrA7(uyUMISg+!l@qCM3J33u4P0zY-6Dw!2u;- zWRZU@j@eES`z0P&0#fUsiuzU1bg(#DYd=CZKlTxS4!@XA9nk)Cin;`)0xuEOe8BI6 z;yVmUG?l5n@!6`FA9Uks%jJuo4u54&>{h&#i|#E6gjB|Khrfe}4_AY>W4X8UDw; z;feb_{LeqrUL-ea@5RF9{T++T~78Es}^8?Hi7*+Xy zv{7ZuVMUY_jO3+bET0@?KH0uL9XFLrHlO3~V}9pR=}~3PdBHOVB_&ZGVyaTFT`DLk#Ewo&(!Q(qv`y7^Q-TF(to8Stpw zQIcXVQ_?1TbFQ!1xrFB~k?~0BDb^hre8miMV?hmZgCi{#s|G5rVO2fnln{;?nXa(QMPcS?E14tL@ro9X{u`6SZ!V${c8wM;nj5h zH9>pMlewd85Ad9K{HuSAi2veiQ~Zms4SB!ylmU1DPriu%_BHPFo`!d$ueSq4k^9*DV06_Q#P>EOCF|Z6 z>IzzhaHnHC-1g02T)?z!eHBjQZQIS~)z$@TjLdRcVE=6WDbRBd8Hq+w3g+kIu@0H? zpZp>f3{uXp_F!u4lbHpIw|z2Hmq`E|Ya?wuDY34M>=XKnIb1oIXpI+ykqiGQf|0S9jD%1}>4^LXuD9z1=w44kn2l5wQKh$|D*N zcbqg25e(;JBgUyl-LZM$U`C)8&+Asl-K}HKI_Tp%HU{c+_w~X9ot^pz@IG}K;v0{f z8`h_Lj`wc$d;TGF@SdLYhG%a9T>o$CgUH=F>nQH!NdqVwn|7Li?OUxy8*83@m(iEp z701=Xls}BZZLOHD*Al%!HTN;kLzcxq4?$GLy0)LPt_(Y{a;-NQ_Uow@` z(E&n2At8sXpe7?Mo~~$O4QY`bMN;}>1_Ab;ktPq=CL>;=UqoDgcyazfO~xtBYHA8k zZ1Q7_eG1xpO83ft^3yt@g-9v#z;q^A0;$8(Awv)MRMw=KiqWC+s4?CwO`~_EMsAGW zXs~chS-4H5oTz?@8o=7qiyK>Bfc*tcS3B?;{Tf78K;xn2k(f}DZbYLarVB?z+zPNy z1@on_?mdAU;&X0&$N|cBUq0zD#94=d>lgOSUt_4l?>M7>J&WD=lp)JeUieC#M&^_! z(awSrh^9$DbCe_#rBL$|1TiPhVG&9?q{N6H27-D^Be=nuxNw{(eH-a=<3_nlh+caK z6SGEn%cgdgLZN=(on(b=ShZ}K%1 zW&>(RqmO5QmjG9;dSjaieX$7Z9W>2b4$2TR4S;-_i_dn)1-;=LpY$lr)#tO_zi)Xp zU2VfX>4%Tke9Mj3><0TX1iAd4ZjXQSP3M;fvufrw*@u6~=A4Sc**N4iw&$29q?_8I zR@o3J&@+8C-x@(MpQp>YTcodKK>d)(m9yK;Mu}X1c(F#W(X96~j2u%rPRt!h@fM1N zD*ww-X6nV|ewGZiAKM-qOveETK^h}NQ!U)>driUFByT$MlVbU445eU60`lFPrF;LQ zymyq|=I;A4Xt+O$^QWGH+%;k}p%60g=b)uGVf870-}0tMn7`jDitG7(r>#~ezx$gU z(=}gzI{kxh(fQ;Z)vtU_yLKJ%`xlRj89)1|o%Zi`{2tHiNOsFM?)G&&#~#zIb7I|` z*0a{Lx_#ECSgK_i6v#6`PoeD83*9}xwMhjrR*_)+LTAi5m{*^|iwCHKG`WYm7$Qcj z?Q0jF9}6G0GD~P$I7W_{iu^UmgrH`yvM9-a>J+_r1!ygdbX}Cj?=atH4+P0ppi}on zai1rA6VSTHkRuAU;8B}54*~5G3r^E1S2Je4dfN2F;r+9Xmm!^x`h4F!fev&dDXn|wuyl+B}4(6~@x%9Np$LX13&(TYgiQ z#2C5N7@3=&UHQJ(@pjSvU3F~de&P0iYt$W3?18U1G($ooIpL&qu>U)N)J2bb63GB_ z!_1_}S0l95>T&9Upq!Nx7I8te=01d`KtH$$J==$CzA*(!@(cFJT={1f%Giw1FV7w25!HFj4r(?r2K@lS3Fv?-84?14*QnUh)*P@6itw0!m{NX3 zJjLAYq%aGQ4(C)^%?QG$vM~-}@82BM+^4eQ#M-tjJrE(qelcwlUI--)X6q39Zc6Za z^xiY+*i*yZiObxFC*;Gl#x;N%^>g0ncmHmF%2C^ zJ!WJa4bm8U*1#x%++FXjWbk8u2r0Ax>-ovia)dE3Eq4bGGKVRZ=Vi$-i48`Kpjq6f zMt1#%>tPRGCH*$&3kwO>dpggC?6x0m?hssKvMzoqGSr-<$gk{TraVF~1g_e>@`Y+n zi4q=^eZ#Pi8qYL^Q^!VlZ9P&|E}ch&pzhw}-^o^xeIYqf3hP)DqUa%iRW&1KNQlm9 zvX=?^gMn^elfr=%^vBvTZPR=G?)pXE_DA%3DXYQjp>qRzeV_~#rS zcRayUT+n*j|MWc{^-YnI#0W;o;w^&GkKAlVjH1MtDJh=oo|=5QI`79v_`&{tYEs`J zQ?`t9^roLLQ#s}6k!R$8Y&~u3H&O3c5*v9zECrw`?x(1&z_W4}gER_jZ0>bzAj?3# zA)})G{5XCZt}y@L6nWIeBi`bAzrSkUjtk2L)i;Uy^L`7LDdUmD!$dk+LLlXc=|s=Z z+-iG62{SoiQ|G}!9lFLEV#*zmI#U&SG0qS2`q_`Qp=I`V>ifQbzub4;)=&qRSV8BW zh`U_sX;1poo6Nn$8QXuGADAbRE+eP${{HYOAe%Dub~SpoD3wps=*`iX+Rmk;vh~ZP zzkxxnNs;2Sq)_;~$P5`TS3C4{8>))q^I7#JOFe4_rgmtgrfN{mQLXL_cFc)nKvd0~ zeP8{<+;Tf!vv)Ut1aAvQLDsrxh>j0~zO37m$M893`j`21@uK(-W8}n0 zvd-H7zs|+4AEqS-qp%c3`K%uR_MRv~sLuQq=iRsCO5x$lx#R$mBOIc%V zi)_e%=$)lh?u|~cyWyY(%WhhH)3LChQjw>n|cj_Z#beJBqE5SCIYe(tl*JgFam!aEuRc`X+HbP-hlTdeJf^ z>b_gs;Bv;txf5=)?M3IPeb<3(Jm%>{B-S0(88S#-B$VibFba+O9=%3kZ=Z6a@obKv z%TI3$=t}`_~5)+-;SZ z`=onz4oJ?Dg7cV=F~(5}6DPIV>MZ5c-QFR8Y66OvQ+^jo?FL{(NU@ebd9Zqw?SRHH zoa6J~Qq;F<)M=!YZrd>cf5bmln}50X?fuX8cC0tp3Al(I-QrPJjdOC zr{4ZHZn`7M%|4i9XNCwqx1-eRd&>6D1j7AgAJZaUr1=CI-&>$xAm-sfyN=3UmrBwe zGfQ@?Kir~vfUI9C6HZ>6j8T&Y#Ju}?VO*jt?s>is1{Y6B&-)vRA$B&R0wZDvF8ekg zX|D%N0+7zP`kGHvGv|LAKgHsHnv@HFciVl=|2~$I4-oH4)u5m%Z~H+e8Z{^%VuH0>C>;)hku9mf$dH-d8_`-2y^2c1 z8O|Z1RiyjCut)TEd@~L6ZhW8nOimo8-<+O%P#Me2aMGJd$1^MGL&l&0Np#wOEAR}Z z0M9}hh4h734+U4sCGeQ0nDcJ?VlUS>!!_mRd09&*An-x zf4sM}ZWP|f;i^ELrSy8fHD+3lWnaw6dcgnn-oO8+`*DBwVzk@#1xK})yzvOd;OsrZ zy4~52w1Ov~tUI^w{{EbY-aDs$I?T^s@{U(;`gR}aIBkWeC99$LyjRtKQJ6HPCg*WA z{1|Dh-@d=?V;kprOk#PY+TSwsboV`ZwCUF21hV@ODq14L#Eci%10I$s7L^!X_;{9N zL{$znA8ZbI9xoZ==>Y2y)M!F!{t8B90qEB6zT3Zgy@~c2)_IX#?iicnUJDJsXo@fW zg7}Bo^rXY;>PYIGThAGP@BSU(eL3k{*aL8kOOYQo0%ZmhJiq9i)Bc9JT?~l(vg6tz z8oi^xYN(i!pi0Sf>idOFp{RCLxkr4~xL-jch*b5#GumUyxMcHPWJa`mKSa*9ijnbQ zLSnR*>gVp*dm6Kd(=C`rY2}O)kiAdc&4)Y9Y7i~A7Kp_8u)nN-)Uz_G2z@FoL!kPi9)f}X1P7;+vt_-v7(leW+HXRu-g(5K zh6ieR0}0BLltWB{0;q>lOZ6;<<>z_m0@mxvwwZ9sVS8u(HZKIgm(ZJyZ>7re(%XZ zDH#Fcg43Q;?%;UxYry-|<{fj)|3|Gxv&}`OH~O}0bIT!Y=S{Z`M%{U>(@J!_XMCUb z|9Ko^udbGJ&lTTb9Tv3@q4x4g~Y<3R$To%=8oEQl> zZ_4yYOn!_aJgW&gVY}7pkZLeuEtD313=E24L1A=ky>}ZYE;if)ZN;Fu=Wds?-d8*e z338D{%#`rl*q(~5^~?+LJc+F3sL_|#dmGDvopwf%eSJNBo~by0B%lVu?4M`4+>f>5 z;YRv@@X_v@fI9tAt6!>GojSki^%qIoGkDrtp!Fv8HIR_Pls7_Bs2n($8WQ-VMII7m z$GsoJNTeL-Jw3zX{%$CWC-~sV4PmQ!baaok231n7ljkXVR2xO#drxvfc$TY-v<$*In_6wm4~4oc84)Kgf|!j3=UccX%s* z6(sN`b}J#KL&8$4m?Y(uYqD0@w}$as-1Q~rQF}Hxv@upB;?v(4S1mc7yGB;X$4t9! zuhL_t9lN)$=q{=}3hU=3q&_Ijq4IUdEg7QzZr^(n|SAxroapg$N92S3tdP->AjvZnXLWpL0Nk7NuY z$%t`M_|&0ih9MXpa4PnURQu`U>~*@SU!@vyP?qwMQBlCgH~MAdEMH3}^h8{L#v_mk z9ATs9FrDaEPa6KJ{BqmNVd*vfm(1jMPC`Yf&|7bGXLtPM@80u}Aw&GshYrZY649@D z=it0^3UTBc+kvmj>3vv?#I^&y$1kXU?kJ2NU|rF7Pz?1Wy|nAmfzM)$df6W4%-VAr zNqf(lMn>dqe-LZWHOP`b`@ndA*IG|{ij%KA*wbsa=e>LyI7f)Va$PxLv*O1szdpFm z$PWxBy;8aS?aOxu&lOY8#YvUUyDIb~7dWDVBexsUU9is!D+49|HO3lAX&0v2QczJxyUHJNCfj=?CY0 zd7tfw%0|SWO)u=C_H!UN&q8kAq&+WUO&5I(;i8)ZSmX{vZOQr(-y@)4$Q(u`)m#!Y zA4aMQDEfszoz8^@#|ip%+3oY}&eU+_DKMSje1@q(zL^WX#}dAO>V)d`P?_APy5jA7 zA=eLJ3onvbEzcN|Zmkgmd9GzAxSm{de8l9lTXuv0i))s3o>}I^Qp7s>;g_$_>_J%> zvbI(f}%ON4v`BdMF5l7K`MVHGjM&o;qGCzFjC|*6O_ZYf(T#6(<}_ zv^E(J8ks_)msE>?XDG6oDtqE6L8l^9ObsV2a(YCoPlrh6k}80RZ&)#$F(}8Z!5k^- zmU0M_(Jb7V>bbI|R|KZw)PJX;-JZ0q|pE~I{S$z7(^vf@O8Zh_?_~;>> z^WoSbQO=AlqiQMnvy!A*N{38NeGT{UV#p6b__x1TyTcWKf57){(y4j~e|Axx7&~)$cJ_fQ8up6n7^PF_OCMm*gg;* z)vf0(j)!x9U!HrT>F+qHr?cwFsH=9OQJ4Pqb4~ojsI?c4znj7^BZsegZ1k3ZR$RX) zlcIVWZOp>OXWLY9WQpyl^|w#G>E7Omg!lYZJvpkFO-S`LVVJY1`Dpu-&9Ys|51EWp z?fB>${U$4_rkNsH2wUlfyyDunz2y%6tnUkVcEXWl039Wn2YHZNA z!Z$ELKO`XXrf2FV{<`(}ZT;1LX-AxspXJH7qU-q-`j?O0nQQ!(l3g~LE++m)jfpjl zgF{|;+jp6=;fvQ_gnxb?>IEb_Lc=3}3Y(#({OK-C9XZ}uGiR%;@it!zC|qR3L`g&@ z(eQ_VoA<4VVC38{WU^1EG_lsBMh4k|E846E3EQt1sS_+W@HUI}JADHZ&msUxK(@a+ zapjm&-hFSU0?_9^v)~0j)3#5r_3?@?JlTuRl4;?AN()B^OV%37ibU0p z^nS3OhTfVgBqk3*X8}|-162#6Q$ZyJ1Nn|n@)-4s$_1@l zY7{%@e^XxhuA-UoXB{u@?AavZZfh+Hl8#lxSPhh=`aD zCwILv9^X%^XRftFBH{MF)_3>l*KIfvhQI&$IfB#k4inYy)|KqWAF?|mw|O-< z7-uanR}mTw-(J%qE+~H?=lD!lYj03JuC07SG@r71xRx2RJ^0?I)nUTqd!SyzgBpgB zf8uEr4-2tb1%7NhhoS!C`RP-BD7|_-pBb&)`QV&0n%{8jF@76BIULcJMO?1^dEZT{ zY~yP5kSPO-{-OHOo6et+g+++*XKZ-;IqPPUh_{hm%Y(AQs`_4fDD+=_Z~$-g=s3XWjdKf9ubC>IFOZgo#ad?@^6@4X9y4E_T2g9`Z;} zJzhFAHggu4$|G5Xbx#&SOihVRK(Q$5(myoF_{be1mRQ%Ongx7YNAc)=loQfA=O7e2xvBUfc z#!H;bA6Ar>UzFNA@DZ`zF&D*1VK3doaO_5962(e#by^o5Ex6ad2Q)p$kGK7J=Y6=V zr~Re>E`JM~hQ>+-!J+gCtx~6`e~XEgbGoVDsZt-Um(o^GGONsZ{2V56xnxuRfm#*@ z5BsRl(WrbF1w|F6ah0MR3hNO-6}7adfYuPI^L=jb=dC8ZV@r4UxMKvJbA?;kahq61 zx_G~34!z;MR$KHds5ahV;V3HBo>qgz;C+5bBd9h(WX(bj8?Ui>SlXOUfBKIc{6`Kg zuasD5un^(g9`>cOWt?QMA}zKQL6lTCOgAFbb=rH=6Qc5vfyVw0jOS+9r;`MsE_0lMOJ>pC&5yt7Ohl#b#gA5D(;UD^pBY#sP@%l+PMYfK?@+p`g!(N8M z-%w<#niMq&B!-T+0s_w`K><*qKmG31_j9QQ`UU!`@>gh1f9uF!`?74N{b#KEnV%=h z3D9+0?~{{WCdZK4zk0N0n3pluhyr(N^!7zJe8zxb4ePal?z6qLH8;sr?ikYLO+NL| z!CDVRAg3ltfA2n{U)mp{FNELLhfn>Mm@|TW=9p8D;X&^FRv&~qNcg?>zLzeTazu<1 z_rC7RTwno6F|J?k?8(zBJ`d#BjwbiOjL&er{Wdmz)gt7O5#Qd8!_*Z7`LH?=8}w91 zUjQ_Bh{T9q)>@zlXI&vDdT;P~7MFe_t@tU|<&ao*e>v|>dhf?sa{F0RQ+gEm4k8;h zv9$N~bZ29%YZSr3`Up9>{ftYMg`2D-ZSKB$6M!+Wo{81Z(Cfw3&1VxZp4>fSeR_QA zKL!L%bn9B#-n;IMbMI{BX)osBnE&QAevg56sM_B>VodJ#$*kJKMGNv=uTMrS9RJA@Ni`Ox7!;sA%p!Qp(V+tATeXs8x6nFoN z*L3=VCmZXv_n4gh=<>5CX41L$Sb5iVyzOO0KZ9e~{*kS>fjs^D^!2~|?ELQ`MZKDz z{J;HuVIy3sZOU7n=IHLp-gw05glLEAt=+L!e_J05e*O2KfBxxb-5=Wg-1m^RE2BSu z->dk)`1`16ltX$?Yn+Qjr+jDaurFMY&D0bB{yvvKFPhK6WWs^e>wOLJyQO-!qw@Z3`)O8m%_zHkrbiA$i(c7t?;(QiqYjkbQ+gvm+^sB%G ze*&Gq-ibZIUP!uP$(l@kqbXG;ru2FA)3;pbu&@Q>kxMqtgduksIX;4rrr3{;)`Wqa zCdVT88%;^Whc737M%wEke~-Eb+Oz1npH`Sl-KRT;{^G=3N+4LVC-N<0=_J)(+`Xa67lJI7@ zm&N9VnZf#la6I)CbI?{CuA?_)Mt>@OAFz?Q#uQeaNW~{{D7MT>3#TI>P)dHP_EWqVaM8oBT}; z!~G*N%yTW+YJ~hjTGscXF{f(~SidE7|GaW}HlJtcCxF_X5*g2gf}LkuTk%4w$=l^R zFds$nzMuE8k#2cYosY;?4CJ5nf2N!M*o$uK^lUDyxZ>nZ0eR8Zog%sRk0N_B=WAF+ zFB-7N!txf2G9i%~!Lq2* zg$F~o?J-iQjp2cIta-1@x>`xgZ#=~nn8R{&OtC}fV6vx z>0>HxF|;$yz` zh$=Vl!w?yqQVl1)d`OpoWAeQ>kN&3KRWzGP0WojjE{n{Bv^g2>_oCtXUeRB9Dj=d; zFVKs+&Hqd7Tf(dE;Sf2%mVYVOm1sQGD}x96pEo*DfqaDLwUl=uHf`J&##loPrE z-R}JymO}ABP@=3cvWTX-H}Q_H%d{Z=L4C@^I`|vgJIXR-d2dY1FpM|pH_3kl>pgi? z)jsVzG3Gc)G$pAjXLRaB1nLoSoCxEn>Ew9;bw$^3iH8U0vE6!*e{}Kh9?Wf=ym&=^ zVbVUAnuY`>y%+RV-~k7JhVk6HvFQsKGzJwtZBVNwK-Yx#>;v}A;n&?cmB2MFhL)Gj zThEhh&Q*5KSL0`4x3Q%HWX$FQ^#ZlF6>KNZ2V+lQmn+sA?3wmoI&YZ!8PyW=3=XW5 zXY}B?#N@!CWw7roe>&BOVuSm(*Ra!qoplWOb+J(S2>DYO#}cTp-$BwJ+c0Aj4txcX z@v)(@m+LV)&v*9gRJwAE417L$ z620$X_qFCvy&vQ?J#0(xcG;W1FGx0C`ss=t)UN24-dL}!0+2&1UW+Lg)cu$+HM$#4 zo&-c7memmZe^3RTJQ>h~N$0;x_$xTA1~YT~f3AOh-s&@3AIaQnz1ww)tlU;( z=KRw&zBZ#5V&>DtBbmz?{dpQ^RJ_I1^ykK5F*R1^5-m0_cZ0VFg%*-Z5D#9bo9yis z5yjOmvL-&Ac{7zilKUtbt2C0iQ)D3gjSHJ)GW|*8x}UkvQE;5<48QmK%)ur^WyI?^ zfZN_Df98kF9Hn>rPRB2AdrPjo`coY0$2S_Dru7BFd%{kf;!~adpT@FR#{-i)gTR-O zA;}te^~kDH7A!oE zxE`}#mOKO+$w3raZ!YR2D1Q$Q;^3&9Jk0#f!d5f(wwZcrSo+y&L5`KT5)~!UG)V@! zH|AuCEH-BvPL@=a@pk9;zRl|h`S`Rd(EppA>S(7v@ByH8Fw0x-7M^sypCZCN)LFOd zf2*$0FN*x&K2u8Mek*v-6sV`W1E4+?yqJJty}f;J zr`sP5zvsZmy;-|%lkm#tOyJ!B!+F?gdCq#5?(?QO=5e1}AFgh1peraxRMoXct2 z`<8c)`DM%Ny|?-008rncybmC zMk~|uRPW{Y^Ut)eGw|NpuQfJCC$A_wfG^`x(9Zq%?fGX$R`a&*LdGz%p)vdc^=iQK zzi^*6w+yF|UAiRGL}4&3Qg_q!iGzU&tU3BoeV@+-A(JhO4!My=Z$R-S)W}iEWdmF@ zNY`yIIEb%WXC7?$J}R@lP?!gfznw^bq3;0e@~z8WuVzc-Jpg3Lk;HRN$*JHDsJV0I z^=z(MNMxn*1lNiS;~IUN+kcamEh&^dW+hmh_C7S38u3oSP@;qK5Lx6Wf8+xDW&!gR ztxqhcG<09RrsuqC;iAu0Z!4SH)v7%UO!EUQm+>Cc1`YejgQry z6B*}k$js>(&&{6c`oTb+b-b^i-UejOiYxC6oWo!8#_TUVFD?IhJP>`GOU`cjUfy4F z$L{hkOs@s}&3HmS8NDMse;34JVXtSQUHECby#)Qo@cmS)e{`OwHU3>^ASHwF`$bjkv0dZG2K#oX$!L3~Q1l!1gzswf-R(!H={xO3Nt<_SIZi*? z@OGUa5Z!&w<0ApX4g?7MX^>@>wbDxkGkibRz8|jl?S6sr%;xN)B#$x*e`?G{jb5hu zg=)hx7XtXU3Hvbvf1(h_UI~VM5l^5xT^xD9zB`2d;<@LBr*frW%lDnau$ya5$RP|{%kS6N7U{iRF{%&?_*G((L|XM*PM@|)!TL(f=YtPf1=1AaBW;@Em@x|XFE{X zgMbXeNfgaIRR!KOODC$PCmpGTSxO%PK5!gijE!gb_V*wDsMBMfj-_vx+U8Nt&-MK$ z!->Cn2r0g=;cQl%b0E`HL7svsGHwJ}zm+-Y=4ZtVn$V0Q-x3tR%k`u3kJc-wcT1aT zxT08RC_};;f9o7C^^744HF`(iEnue}g2+%uXETSK7@Z2adKYbI-`}P~*kl7_Y@GTM z`X`RUvGe!D_nH5kaxiMHG;kSQT+0Z^Ca2mENCj!cXFtBkAUUF zp}&w!xQlpo09_1SS}h&)V}t+{NMy4R(A8UPt@x_Vf5Tw!u^d8e;SM}I_ZOb-IeJ0h zSV%bfK*RzZC5PSvqNc2Fay^qhk4`UH{ zOP=@8)=zN1Bm>+piNbHOM4=L`S8_wjwCzV6my-x@$M{bP@n&<0& z2${IufA`3v>h`u3<_nBw)QfCnz0kX1zSvrRZx5tt{Y7BdJCtk!o`v(5^HZmRp~MHa`*A`_|ev|v_oN< zh7@elTNt{)=RiJtcg9=5y2Shh_EYRm9Et1sf8(UJg90{$DDD_<(ZvMb=%AI4eC`s` zcJB{hboT+y9CS5-{lWs1&WM`qkqgx;H7R9I8_l7s-woffes?giI&mNd*PP{F&q5dW ztmf8@PSv~Njh$=k$g-=|wx3SC!y8v3^FLN*bkv`o^SY3hm_U?o5%XmEr60t@_ti3+ ze-dRf?RT_LFlz&aXB8e2w%HQ{3_==f8U>)K#ytIkR-+mK;W}<7v78e=U>b zFyxHsU2J&pQWV0cxaXobdOdPzR>q|GrkXdSXc#XJ)h>^#$SqJuF`V=GT?OT$7jEi` zDm9^|WHja9q$qtnmmHbhfJv({WW-~eF)1X?=k+q$sE5|0q^Z8gapb!DuLcU~CU0iW z<0^UQjiX$9?^02sm!MbZeUdCzf3qj+Jz!-3w?A<{=X#XDSC0GSPuCZZ_t0jy zJ)9Q}*VQ+c_?wgNuetUS!d!$zr`YEvU_Sn0`mz72)slG29YbA5WNnHW4$+u_nfBATa)aPf- zN&c8}tb37TZ>>k)@Rh+7b;#Zbgw_fXV&6v-+{r)*O){ccRKKTu{ zbLYYq?}xD@6BBu(WA-f@pc87x&JlYn6>WP_He3iTWOB4zPOEiVAZyXOC#o8?``<=b zUZLM*lueUF8fDUhW6}}ff5;V2=W;vTk2cx+lt2`{WfOA1GaYi_%uFbAR8i$mcDWA` zuNJyC)T(|+l+9ZY!HpO@E&*&58&mU6OfgCb=F*Xykq0DlWQ<;ty4A0e1p8D--g169 z=8=>9#t*m6bK#s~sFCK_*W2=_vZ;ewlG2PQl+~Dp4(7a^+EPp*e^ZO|I>{Cib<%15 z?&{Gu`FRRR@kOQ1GFQJXx2ASsUXfQLyxuNaI|Ua(HQvVkdWiF1t%n7q3Ol(veKcv~ z=o8nSpPm=*w7;=%JKkezwRKtxeO(2m4T#pSS(U2g`qy?|V}4?4H@*ul95BXGE?qzE zf=%i6;xr_~ujD1of64n}w&%v~?$d#jM2!+~taelL6ESzx$#Vj_-dn8Apclxw7uP?3 zmeyaUn>V=Dba{+3b0fb!^GwmFqGgt0eelE&Z`bDY@A_Zty;-xfI5$ z10KF}(~_9PXn_I4O$T5sMq?Xnz#aYHJA~I-@4IX7v(G+peej=i7v3cl}GL@hLb>83~*eGYriWid(NJ+0x8 zz$0x(!pjy_Jle^OuAp$vcJ6C=l*4Uyjy2a!PN zw#=uYv7z+ZY@eJW(ngu*SexfqLD^Ti+G;KUsRB>>@pt(1p8Cr_GI>vU0-v+`GswQy zDk4>2umjzu!e6}u02`jsK@@ba>dJuV>Il$Xa4Ika*3SopSjCz&M!`SFVH4{qMHm;d+-GP`yq`{(YfGWoLqdWZR|_klG8BE#@ik2zPf5BU&@ z&m;S01u*t)QTXjW04;kFp*+-W&gRr3O~3$q+Pia;>uZLyBmlBc24AK9U+w>YaCxLX zVK{~puGx3|;(v%WlvdDXXXf|$-(cUd7|aaRf6Idze*e0yr0fWjuQ^yS3IcyIu_w?Se+YJnz_kUCKO75z3&)UszVWx ze@YCqKw*BO6K3J^y@CBd#d;F0f85r8jQ7dLM={@kzy_3?@YYo4=bQKUd)?3d{OR78 z@a_8$ynn?S{lWrr#9aLyx26{zcn~4%4MCW&_aLGKfBHoH&*Imx&tl(rC*SxyFkgS;^Q``a#WLao zIv+m;?EL2U^b3#Z`yK0-?Y<`r7XnH9BQJU_-fSM2w})Au0Sj8Nc}N{AAj&GlE#wP5cOXx~Mj?Xv+f>|>yP`L8p48U;~ovU5@If77q4 zpKtD?c*VIPVe;c|_Md9!KgTi>9cXiZ*2m*Ddx9VQnk6*cf1;P|e{4Ti_d8Y*_|aj$ z`6s$#JfHDVJ9g$8fMKW+Bch95pUxcmK{Dyjd@PG>J{HLLIpnmj`A^R&EWSkkS#K<2 z5O(hsy52v+NSi%ch%&4JBHvKof4B#?{fGEyP(Lu-m^{+%pY??uvLTLWk1xEI&spen zr&uJ@_kmyMtQkhXzVEW{2BHlG=JX_{khYQl0zAc5Bd+@62lNP-A5b&l8leAaY)t-4lfHhvi?g%K&-?rj zRumMbZ&z&YWxk&}0+Y{Ve}8n1f8hUl**9Q~Yq?Cso6L|MG=e%jQ+xNkMYz6qA?CW1hdxZV1Mk^s>1e_$=wxiAC*w2i&K zOjs;BIH&7x|1}8kd#(|6_VwMA-5>b+W)6Pcy`cd46g#lL&I~qD4TdNXc?(+hU6jST zIPKCN3nHP4OwDw{;+RSkQ=l~yG~V<#MV6tUU}?Cv1n~3cOjgAFrNjhO>~Y~ za^TK!w(6C#PwQ+uQ-xqjX~^Knb`m+{i_+1-%E{w+u+qY%gC+lY*1@NSC3+msS+ zEx4Q+l#8_xY|W-zE`_8rx!o!~rkS~O*J6=1mXYw%v3gn;yh)4uXpQ35IlfG5p6m|7 zC?8b18{hByfh7C*y*wY{V81!?=Hb!LHplJk@Gj?xe=sW!GYKdhTo6MAGfvV;+#$Qv zD)LkX(KTJN=7o)WbZuLb+|m#1^IkT}D7X7cvC8w!UJuHHqn0DxaPO9mbF$U4tOqCA zt6QBM>*x7aZ#_jPDg@#=Y<53F@zT_51Uoz`*g^!oeI+mOAI7)|@ zo8Mf-f51S6#w80|ucWIROZIkwAMVX`wv{QkyR$2Kj^UX%PpdrrgIn`8UQC|sfpihx z?*#xmVFUi^PncaM4zRN*IZ>c zog^@&1;W}h(b@S-&+@yBj){x`BrvBSWBnmg+yYoKkDMdZhe$DD@=fWfK80hDtP+JQ zCLdwtNd}lPd+0^rGxeugmIF$xk|A z`jF{i3B-WqKZBz_A6b10(+3<`{WeP*tiR#Z|EI}*c4*7a#lP)EX;8x2&xt^%Q~3<% zi!%8%_B|JttbdkF9<$dSAW&A{ipisY^e^!JAMBgCKmnk!tr|Rj^$++cfz|)ZfBq@U zk1)e+VfZb;>>Zccx7dSp8dqe(Ut;yIKqLa#t2-bXEZuzcb!a~01K6y5>vO)wgqZX% zl0yR^_esy_hyN=L~NcBVC{@z5XrD?C8bxEK#|1k;X&mcfDE?@e?3`Rq91uD z{e(>4a>CLn0b8a|nVLKhdRBh)Y2RPy1NJau@6&+`2>Hyf8G%ohHhVb%8V9e;KFSi9 zeOLpDD5U?SEf6xCD1}h4hBebS8z22gf0Dt#Oqe%_kTG0IsZq#$eVcFB3Jd)E7Yhgb zxPw5{v&cD9Ot7443_Af7e?Zf~gAF2u0)gG9_@8pJCB@niV7Mdf3+ys7{ZNOHw*%b za^PZEQpEuoMgcW);e^SK-2*scn`R0U7dR}lTQ1*m%w@f!82Iew0AT@BH%AETI5ToN z$0^Q1jz?VKyv+|t1-LYF+fgwtx!gr+hHE#EMs>vYEP@80g3WgifO?<@ z;0Xj7myTe5TL)%WOX1zS%46a=z8)?PXTZ`QyBJ z&G39x%*kO#f9J(&|CleI^GM$}mv=k2t~-AIuIAQpw|>2|xh2rtLh@SK(sTy-wQ`}! z3^uFX@lu>;#%le#Wc!)8TD~u0x06@>G2QLA!|HiSd2_Q_J+A4~+^klQQ~KC%o~!#i zz3(@-)xAh>`^{-}JEmo~u~*GJ&G#F1b-kwNJ^x27tGnEs)*CEocQ2wn6UPYYm6TOnj+^kxm>K&nFDe)b^udqFgdmO+x~061Hg zf0t%07zCMn_DVJ7Po1T_76B~S^I%IQAgiJ8+nFB&s4c1xZvk+ntYUJ9I`Ek;eP`K2 zr>C}qdWgI%M5u?HNqikBnA3WMOeBxuR7QZ7J`T}6KaT|gfAtiL zE^UGMV0tE7AgVr~cF+`o152y}`sp(ew*qAPrScX6-z!Ge_j+#ZL&jzdJ%o>M|67HLYRjS0D}>&A%I1mn^_Ab zAwWoDUYVj}B0Mg8F&}hK$_FUhqVG7p;JZc2PC>^(cfCw^7hg2H%@|!=e&`Y!2LM&U z^YTH=RG|e@MjpV{7IhCqs|UGsKIUtxd4DMY=Y$e>v+DDQEg}A9@T;RA8*#c>p<``$$>rP)_9gv`AI<Ir1IPhuU#aV&@S(eHOUOi5^KpRAM8E2~D1YcozsW?`oN+*C zqTh90R6q1}i-mDQf7)u%uEjFyWrl{x)+QecpS2gjKnG7P7rRyfZHsn~7VX&lHB9r@ z2>#yAkeOM?nAUPECKT(yB1?;n$sDtfEK=SR8HQ6|EkO~m7|yJJ1pJ!~iWV$A)B@3O z1@Ybz7fP-d7*ZMIit!pP(7{35{!@O^V!{8kTbOM9i(iudf4z3FIq&b<`Jn^<`(?}) z)`g{u9I)Lxc9y-F_w<0E^7rjWJrJ3`mc2RlPwCxvtw38W^wb{zew}Cm6j)!8rLopy zz)y213@VJ))ZPNX#xBS`HE(VT<`Onv2W&6J)=|u9qXk4L*@X4$hOH;k1DvKc6tOkY z`{ap51BhCWe>me22F(+XfF5we>f8P~k4yD3DeTc?dkYzJ+8_XkTF?pFh!~uz~&R+v_%U- zHn&4<3nVsQvwg*1$<6kA!DpXCLu9di9u1jZ?6$yO`3HH}Ty$Vw)G+F$(L{J!58WCg z{;NNY9kZK*^Ba#2G&Xl*1ZB*nlWf6}cCD~(nXR@kDaZo|Y7CI-e)_5f(Dl)p9+1Iw zhJL4Ee;r6_%+{=L-k@rMTRHKxF7b$cNRS!I$(L>U|Nc1FbAw`G)GTC7^Xy^#qugW! z&#%6xfevUOM}MiqUp5EoHVsAmGKjSa>>i2Wjg1#X6mz~5wCu3W%IYrNn^{@Jawj~YQkNj4~ ze{Brg+rJqrY&pQdYk3qY ze-?wn-m;e_^qoyit^$kyz4{-HD3$4zfcc}q54 zJS?W4eJk5%+pa5)o8HWut`(dAfX?f@A(8 z-!UJ1#P$y&$L#T->kOhlV86-q+ns*ff4Yw0*K=q3{ds=;@F(oI`{AE{dmsMkxBuav zexG!m-ED*dtJ_E8dwRzh+pd+$w(Ug#M1%1~y3XzneesQq55Ny!-?r>Kr#tbTCq5ue ze5ZVz{AoUXySAkO*lZ6c1NIw!ev55O#jod>{r-AxO#BY(3z?X)Gh)=CP(BjIRLOHzJ|36e(^m%?UyFL_|ty);p0B-cPGB*r~O}iY+oky zkKg8vp6%Zb=-CsNR}op?@3bw1j6cbzF*@aGMa=l$Y(M;qpRhgO5C7Tu;y?cIFQ@XI z_Sf8@Ot!t(wh*v9>15a3qZLKQf6ssTpFPPLZ^O#}#mnSB{_yT5S$=pqR_^TgF>_)2 zcwzl$MYflO^Qr9o*%Q`J7<~A*5HViL_VmYZUWBHye|QZh#}98d$-Mhbk4T!z9>4U0 z+580N-+IZ$5C2Ce2v24I@RqFXAKskFeEi|nb)Bt9?7cJZVfGv$z=G}3e*&|2Jg-99 z7T_=jf-1rFZ+piMuy0jH=J+}bbH#@(_*wJ%mH>r3jAn`{^bi5$w1wQ$0|Z$6POBvF z814~EBUjtP4>AM-puM*Q=xjD8(=h=CXuR|6o-q)eK?O2)y}Tzl3!V%YU$({8p)i-E z;0Y+mrTmN64&|+Q1wm%}e|p2NzCkD=94zclJ^)4Et3uza7Yd;^1lZCZ6wcRv5AkvU z6pC%{m5@Tw{)eCQWOgYXQXn&@5&#gp5(HXhzM*)EY#C6Pvv}s}6mJHgj><0pg!$n^ zM;(j5^ax;om@npU>Ui0=WcX4;u!`j+sA|Y_NOa zl%4k*ET>Si=*}b4yg@+EdfEoC0*V150?-L${dZ>l zC^LDX{OL!?e=)g^tlwk~giL<1W%ao%pU?a!M2@9zY@FyHoD$$odMZJ}jx2w()6Y2D zF+1Rx9FrZ(E_;4s;}J6u2f6%=KY9I;>m$#Cr9<$MbFwFye&mr1;1Fm`a`v$Q^y{DM zHU3oY$3Fb2{S>|@`)2$IeA%J+X^%ue^mp zKcmcCe_{|@H`r-x&rF@UT7Uza{6)h284Y&6?J+-diN#rlCF{zVHC-Vf1#rhC$N)P_ z?Lok%$hk4JZN81NP*98~`1IwrA)62~)}fU<`VS9>6Z4 ze@w*~*b}Z@3hW*8B+Y<>SuBwfI4)GFcm)n+F;4ElX`v3p4mfM5_hQfd2Mj8{+2n5$ zaS4tIw1CJ1l|qY(TV`|zBF=!dgNL3eO@xF>0&oeq5lLbv=r)xYzBzP=qye{uM}QLG za_~r{1MrxCl4Ju9L$_z0>@sD0zzd+)e>0_*(5F(&lz$>p3cMZbgber?20iNx7lu^2 zvQCMJbZ1{uVjy+EU&FX(og~79N^jO#XCf`ZDPf()f9_REJ8hI ztFJtRAqLp-!!htX?tH>u`NhHLf4f!aS546R7lXIUvyXGzv;vkzv+m4=XX`{hyDiNbiw%6iE9difBY?{+Qzb1 zpTFC&wDVZ;dcpriPwMI#Da81azYOZ3j92I9?_~t_o{PTw$t4SP@}AKtWID}a&{e5i zz0b1uJj+!UgRF{AkV{Wybq3irsiiXVz2_O6t628B(C0a%n2TVUagi)>Uu_wUW8QJDStG{ zt&NxGTd+tkCf@M$BhT9Y zDZd=P>$i?S_47vmL+4O!-jdpE!xL=JrypJNCgqFhq!z4i&;GzgC)lRH%3c2|_uJN0 zXngCh6yM);awKLVe|uvhd(Tj=iu3P!E`Rkkc@rg+_uL8E|ACx6rJ z;(JUPz4e{{4Zg?R+Yo=z$#?y4JpRys?8rpMcYY3k>u*7R=pOo>yUbr>c0t}hbjhaP zJCUXG+YZbk_%3H@7FZ0H!y=&Hv{3w^mF`y=!P5SwbMCi}e^&O7uFpR@Fu5NctnA`D z|5-MF@&~J5boxX8X#@9Tqbs}oYC|c$zw0Ni^i{qJN%5JN>?0VBue}%m%6#5~!~gg{ zF3qEky8r&)y2mr^##{Q|{>T6Ke_xtw_kUfYB>jKCe${z3{!5FF>FNJ5qAvTl>2ceQ zRr;^Ii>f^Ne>~~aoMzp>wjB3w?IZcu^HKfxbJRuuV#ImhTrSIhi=#H(tpCq5x0hMe z#;+?bTp3xOr0t-w_xe~#e@a$~rNhhcua^Q9nA>C$cu zS}tWVGR&}4VC-`JG~8e3 z6WYHbsiZfbtcp}$FRz0rL@~Ky^{ZL5r$7Smm_wwv7S1V-|?~l*bYrmT7zFWQL z%I!T{xOaE&ns1H>MJ6jnDS4Zkt9ig>wbLa=oEP(R96a5F)jaKUSi};?Z{*EWPT!~f z1D$YfJ03URJRd9h+M(fzJd+>NvtNOi*3yOGe{Q0q@7y0-?r~O>md4x;^PHljQTfQxT(cfA$iY#f|%x3ukC z>z}L21o_%H@&@UzyX$T19^cu$-4@z#yvtZw z(h;8|OwX#Jh~0i}gm`{RzADH$K$ps z(+w@J$8b~ci+HYCmx5b!ZR8iZGw&`Ad?)5xfM2p5+yhN->`;lO-#V>;#^B_Mp zFP;lI!*3;L@xCKYk9<*uYaNB!e@YW)Pk)g<*WUcPe=T-%K7EGSrA=nmRV$SJ>L@+z zw^Degy?%b~w|!Y2j^f%(%g9sR{d+$@%~yr2nR2fa$=jQ%yqoE>*&&_P{Qdbb9)yU8 zZE}gN0$`4%po#d9@eXXhP|JKgW07rn5dm|7=qPiu*Xe_Lla2+}w| zn%Qxm27!vxF_ed@0`DdgIG<+NP;Ls4ACD!JWnY&qWTFJ$o-6e_o>V9o`W)7a;c|qa zdT;wjf{;)` zD#0HGY1;}Jq>J^CsQ%15e?J`e)fV%c=xovL(52^9FDZ35mPcuvC*I7{t)kIanJo$8 z!;EdvbQazao;xCV%4*qoq1t3&=#YP_O4G`o>ua2Ennz6NHnn@(p&_cGd9u6(@Z?!{ z%k552+?R`$E>-f@Q}YVmg&R5pQ+4BhV>IGyJP0mD<7RV{s_QM|e}mcL{wVJGrAUb_ zWdT|5Ql1K0T;Vp*4QK>@f*opetSau?fvJ zae!C&o0~JDppgcs-3=QAts)fKwFXloC{7P%nvmXRiK`rCO7N2hOgabx%S>3Nl+ zbx^h&*>14?42`ZYtCDbbup&>%NFO(M07XE$zpg9EQ^m)Z9tWq>>VM#`+E-TcT}CDG zdRpnBtciksz@4)cmGF|mU3AFiuxk)LEsC8l-sRc0S2y(~1LNy4I@@@J;#z-Jr^iCa zYd>~p3gJoAR+VP>NN=(O!H!lhpmUqO>Yf~-_g;eXRCCb<91fNIR34(Uzdmme9L+J> z?Kw(zBsg|RO9g-XRDXoQw);jZuOzv7jbdxDU~B8b*2H%%saNslTjJwn;jB61q1>da zW~qX;nV)0t<)8)$8*S*QxY~xWNbVPZ)xP=J{LrZd&9>cvUgO!Jb(DHmkfuzA24?mV zWpyR?F5((L&)j{oM$yx_@9nm^KMQOX=M^DEm>-`Hi|^r^ZhvdRYy#QP`gGw6&Q&_D zm-)$VM3ih1*U)F}q{adZee;elC&6Ex{Nz&K!mdh`m!MMr%GVHF-Rp4+fqaXuJK*{k z;zh^RF%7j-cN=F8KY+I{$Jbn>ul|<29Qv$`t<;R6oe1{wHA4eDRrcWQLX zQVI;MtuKyxk4!R;vzkhVB_~n(KAF#BL-@TY*^T_r7L}~%TR8&{&l_&W!iq$FUY;(- zqA8W?o`3P{lCx;(XBKQwKAMlt-7H$3>O<4hv1u$(R$}a~It7TY@|5(-PFP(rj;iE+ z(}EdIP<$@lOZk3TK4cWUpCH_wyG6t|gITW!?}^ApaUS=^IOxapVEB4jUIlgEBwp)W zwz`zxxXmuu@fmL%OYfzUM$GDJ!)bSI*Q&3F7=OOUctiUf#BzNfVQX9Msu7cQT?}M^ zr5hunyJv!k8J2hdZarmJ;S>>Y@`8(FjpT@y_7Mo~3uw6N<NOY6_Djh`qHH*XB2wynZ z3V-iQtF3rjm|5w2zqe;_vCcZV&F&Y}e@#tl%ZC z&SO{EVKf?L8@?B~Ht&)LV`tOro4ONpKg_O=ws|l7B$AJQRsf&7C-P=_O-Y&@c5~|7 z4n_^im3$=H>14hGr4<*&3nfnA>-K%WOn*~_;QWPmBzmMF+_|&Ni+cp@+jCU#9-O%_ z6%)GIU9Bq@fNW>YjmBQoLgdNMV1ci~JdK2VLHk1tBvR$H)wktP(dHE0as#rgX6?(p7Pxsr!X( z-LKWzfl9*#3LN>g`&K885EPP08cP-7gnY zFl_ziHb$1Z*0m72ONp9e@_zz~?Mn+&b-Tk+K!dV>+!5OBoM;1YN8i*l&Kbh@ZaqAs zXy$-prrENBaB!kwBG4=xon{Mgi5pvKuI2J}kHb}aFXy%{nGbn)HhI3jXF9@(H!kc| z=(*3iy)rIt##6azq}L-iE?eJ<)F(Y&A6aXaQhtr{jpmtkBm1Uh4S$#_ugzhZug-gW zKGsej9V%aIHXy#z!pT+Ll;;j8j**(bE#r03;DB#{E@YHF>Eg1ieQi&dw{9K@(N&xp6dsUo^_8bHbF@;mIQd_OeS>2N>ny-df+<9>4jIKLOxTAX3RIYy@r zw`3Myp)aTL`f3UB_RP=o^m&O++4kwYn(Og?l|*;-R_EF>FMlpkW+k!oSQw)7;EOc5 zQksS429}~duZ&oMPf52{hLV;EdDyQf?-^ay8>wk&mDRX*)Rtk`saSgt?_Tw;i35)N zKvq>*ZFlZ63&T#}_S-wCp9F5!CnFe|_PXrboggS(7_YAf>A9R6Am6)&CzM2uZiY<8 zd1LP!wT)j_4S(+5@%)lgFBiL$Wu>Lf*Wu1e8}(Fh+a>V(+YH|J{WX~%j(4;IY1`gf zt|8An3*4JFENkMVRkKf&b-#^$*lTZRytbO<7s;WJyzbTI=QwjNX(o-rEpChrzGt#+s0(rP3fw`1De>sad~hgTl^5~VMF4l?Kk%iizwUFIJ3;e-zT138z& zPQMD56@QfHvv>yaoHt|;ZLX&_EMgr>+8vEutk-bH;Y#1nZyR3icsUoTen>QFmOWB& zw%_)>+L|{(vXq4rU1Q_E+2OuzmcpTWa(BS!|!HQFFE&mipzA?CLXlkz&8K%3i7NWB107$3m``*W2sNKt_aW+ppMM*blJ=8E`-J*x%R7U z)#pvp7OR)DGnHK1pW@pqAyv}P(ss7E9e-CF_vX3@HD=@MM(iYgY(!U?*TZG~ir(fc zm|0^*7Bhan-p|hJe3trI52AVrxb?t`9by5?=PQp`t4*ztvFi9IZ!Gw%I7n`Mv&=EG z6$N<s$bSz4 zw-fKX#*-fU*kWMsWQiV@-TYNo%W~kkt37+3v>njri)_(2=E^u#;`{7iel_Z;y2}Ra zF4sw6^8;=wbv(STywJ24$;v%l&$x9dZFRRZC_iqkvynF>FOSr*2fi4di%esivawww z%q3mx3B`ghHp#KPZn*Mx^L8ya*ME7gc;xT0EAt&LF4lr*+Henjm;n?D&s&TIjK_zt zGxqnQD+2F&w%^5l2dX@C&|#16x-D7g66IWK$?74Gd()=I?6h3)Q8IVUWxm=B20E!m z4|>UsvSJ6V_izpNns(&xyf;Wm; zCHpX^7o!=3!rxp-pxT1w#xI{8kzcdR%Rt=vR;SQ+soR+R>vb}cW2H` zZs+WwU8Qw)GaT(|R$D~mnSa4abGsWL$}gzd!?1(>F}z;OA&I(8;O&%}t|Z~L>c{s3 zYZBPG=Hr&X`8ip|K2Dwa%8*yW#;9R;(YSLo^f%`P7K(H)vb{oHsC&QlI$3DjIt{Mn zePKIp4~pE$HkQ>%(s0`h!V!koLm8ZE<8}-xqFgVlQ19}H&q4fd<$wNgGSB&KwoijN zQ_$MPD-?%UhsGw=q>f28LzJaKJw&B z9If$$=ld0uS=gZ4l3QHYvPyOJ(eBrnKcU2XU)w=43~tUh`hUWz7w{o*uR5qv2j_Ed zy*%`K5f1dR8T4R09$RClE^dj*FE0pO5m%=AdW*`^c=HT;Vq2GU`@$66N%0%8M77f^k~Hi`lX={;jz%D^O%!1aE_)w7nSqk z{P0AOa&z%?yesILo@H9-_seDxXkKb|OFmh78(eIA-G8#fiyHS?xy}}w_%6K8M&hjY zJa6PP_cU{cB<`BWG3pMxyo>y`;Fval=GqMj)v6La^S6%>0>+rBM5tvGUO;4}4ApXb)mW{tucKV4X- z(hwM#=I$5a!UOBp)3FnATQxRfIbM|gqknGF+2#}~?mM!GW_5n^En|8;Q;o3pn%ssqqDeSdnX5xr$?Dx?`Tk>z&#qy60?Db0LCtC$4w5Kz|}v zy_&bz2Tk&J+3(wpUq96JPU?0sYFa;9(0IzPBawwYo+U4Ld@H*_0QXaZxb>T7QQE?d zm$l?9<=b^w@rMO@o;BYyqXT;Gi_A-dVOApf#gk6(4JBz-+XPhE>i!nETh~2IULaL* zc2@DNb*mj`L|C-9X?!a1lFWkYaDV6ds(25DLpd&UQXmbcl+W)$;ac*}RphgM$4a|s zZqLO{iA(KqEUrzEialu}=*&A@fjWMPbyj-bjdq9U|4-O?tZWN43-(@m*4SW9gC3}l zIp;7l=bXLz=KUJ}K5zz&uu&*BWjRu-);cLcQSz{P0{YcAe?*kiMIu5PLw~C{;xco3 zgpJE%cQMyj)5q={BOJ>$NtU}Tf}4bdEESV}+Kd_4NIAxR9>8$bj#NRtv0D4oL_VV_ zN~N}Hi})lYfp03=X`9aP>3w*#+v*^rQu^}PSp~XgFCRC_*Ow=bQnedUN9kD1AdT#y zqi&zh**w!^anCE_)oxI#seepJ2h6D+(5d<6?U!Npz%Y9LMF&jc@&{7O9*1lsy=q{q z+PkknsTD9ogk6*#RTVRn#lEg#$>F^iN`&Wj{4;Q^bool#4R zTQQ?l;L!5zS=}Hmj#AM_|U{ z48B6KY}cyR$L`8Ndw3`W?q!ou5B2c2Y!bwdCHjUm`m^H|K%nMO5tvCASzt3T}!1Rx>3e$Cf2>5>(<*Nt?eg(DK8VqTCxr#u~=iLU@0o!#>` zXvs~zYY?y|Dp$39`F~U42ILxNKEqaZMuXpmz;K~2$~TJJJ|irjiA*4sX%r`AcJ~uBxq)BY)u3Tw8=S@VgNw^&G9i z3tnU3mH3!NY(-)|g3rN@)m>>92T1mmUF7Sck zfWi`WTih>`M}IAgsB0#!rl(|4k$aDETVn>G`MLH57KGFL_(Zmoh>sfxTHkJk9Dx4A zyKaijsXfdRD$*bJ4l;L4m9M(Wq*^dB?oORPA}`Xu%U@gwxAdXjQNFC@J}Vke z@!Y4ZBBQb0>e!=n$tt>1I;lAQ#!b)Y4FyFxDU?Sf2T^VHP3gUMKtbNxCoEUxGE%&s zTMfKvL_%x)T{SB|4}_*ZUG580lNAENr{A}zrA^d$lh_O4QJncDMByy;xI!2m6l6zj zP)cd-0Dqbw!OC*valRyy4>ZWVP6QsP3UF(?&r750puNKyUmdPiuK)yHv~^D?0Y12{&4s;=ROm|_=YXa)Y=*7eoqjr9NBXYxvk}k zz2R5m)Eh}}wF8g;^JK;YXNx$@KkU~yC8kO65`Pu#9fjux%zP2&RO(YX*mzK{sd@Ws z5=y*R@e}c<4RQX9O)G+d`PODV9=pu#pc2;TbBm3b&92er-slucO;dV-3S-w5AADQ% zT-q?u#}w`z-rV4PVHsa2$;1|`Fctero>%Y0ko1zEt|=dr|FCrd=A3=Is&lBhBdL>!k*vK0!ZOAzzP!!?a=AI#NYvsw< zUvGe`KwE=GW)jiP(;3Vr0gNO{VzAm@1m$QAs{9lbC{s|1I=Hw&{ynKs{HT&9i2JZ* zh8qaaDu7bjeIFby?#!yahoKK{cPpN-%YQ-OwH^i}gy)AY(XwxlScR*)q>}?q&9Qe~ zcd+8lJe;T138LYK&vjuKyAxXRy0#}E@)IYuC*{U6u=q^tOxP^T0ZYB|_H4y@D^rPE z6?&Grx;+{xiE2I9e%WdB>q0wGE^o1dSg=$I1B51v3KA{i^m~QGAP#glG)K&QiGNUa zx-70TH;0b*KAty+p&>ZC8tBIeg5*Ex8IT0)wZ1C0$gU;zkYN$V>P0b<-V1C5MT{~* z3vXYE>5CmX8i^)a++3(ucQYzK`^<3c?NPtOlpMYyV$pCerHQ@CCAeng4UX@kip&0~ z>v6n=javJK@8uD3v~#LEw86Y_uzyq9$7`hX_xRT(XGzbQ&>2HE=Rdhw>rdgb*>GPG z3-*_~g`BU6x}oO9p`S9B4&tjMN4)RY`L1zUHKjCht~8nqRmkdyNOMx3NS>OYrz!5& z6oAbXIh(>vkOx79M%U?g;{-+u78lw(BDiLmq7_J=1e{~BNl4^wPcY5+^M6&A(UgO* z@XyJ_Qg2t|sMy5-V4I9%$YAvWQL<=ypu9pkxo(#f(TisC4X;&_ZY7tPmNEK%ByJFi{^)>Qo#MXf=orlS$dZ_I~%IoZwAoN zhwo;hVb3!@P{+^)J7(vCK~E^a5an79Si%mOL`<9gLSqEuvx}UYet!*$d~-0RMlr*@ z@p%jclgty42m6&fqGSni)szfv$T0KeJq!cSw;1N}#%qtdFWXcX?^00hDS*Zc$b3U= z3M&OK?8EUJk^`jtlF5VRp-LFWym9~)wJ=tHWMZGVghs;=zJ7DgJbZzuMW z^0Ne1&RYIFG}AZAJ-Kuk`Bz|!0P#x zRLc6Uy)g}5w14$o+&o!47Ca@zZ&0Km{Qak>atrepMiz1)RhaDAQ@Y|BNB9wd2!+wK zIp7)5M)gFo;1t3OvntrUI)__ee9lK^^2!mJ1}ki9zU}d7u~XuvQA2*?k zKCK$k__z?2eV$~Go1cEj{?IZYh^ybHIbPdIIfGXVHN6ByG(PL|3ZZK$lfOXYN6Ts z6hpRbntyW?FF@Q#;VoflITEeb1kdL%>`_p<$b%!rJOaZ3--=#9zZ$hRQ@i9}Lt?@5 z+)yNoxuYNp(2=HiHz)sC8y&^pOq+AF=wE&l4c=hn?1*zo(%$Za4Hk7LX>*I`yQ4*dB&of97)?ODjBM33;pA|Y<_sw%x z+SQc4mEbk_@u`kF6;Psx(UDuJQ;{pNX78()%Y8nN~faDS>7KfOogC9^BtFpkkg&M2yrJj}tgJq@ws z(+AJsCN@ja;;G;YN>gjzmD6lMD@eY7w$AXI@N+;(eJXYqBsQGOoYjEW}C^_1`G zGb-wSj!y?p_>5-o_(1^paB{)djd++^ryWI`WIo~hx9^ejvGAy}$5`b&Mr7)|~obio`Pmtl4vVV)!v-ARL(eRzb>kHqz3cX#nKW<2B^p>*?U!}WA6#kEPG0;>IDAD zJ=13Zi@3h#dzxU|IPWjwOmiz-ErvY-e*+#BUC!M@J22L)FMe}U+S(5Mdw<%ZFV>n6 zLhtK3$(cyjL+G(a{rg?$c%(s?-8k~8#|F+=g2ubr&f?c%c$e%$)0~Dsu$J?H^ThEz zqLUF3?t;=jutdzzth()_&2nT^g>fTzX;$irlM-k-6Gj6&XVf^jxx`W zL1|daUoUJYG(Hy6&zG$1=N2u=+{7odMkxby!*8-CJ-^4ZTPJ>?wSNrjw1i?8O*J-d zP4rqD&no`%cGtqYEHe?2x;%qFI&_HNPe-_G;g5GOrJef2DUx6_v0-8e?kqd~kQ73j z*#>+tOyuXM%!27$>hBD!~j#*Id< z0qC%Z!>g&j15)jL9e*=)e$Qa@AhcLYHcb9_*u(;j4V`K4n^ZOkhh4encwVmb zhmlQ82nvJ|6yKd)X6pQ050T4cV=yLj%sF<7iWFp+n4!-EF@K>%Y*^-K(s$8@FK2R3uSlJ%MwjqvO#}632r6 zcwoSJ2xnacEq@}Scx*%Qiw1oT=03~_kBbYDOOt+~%PPFz&nBN`IK&Ci*UBV;>zF&~ zW^zz@FJ)j07e}&-?0vW;1uN8|gD!km?kbR>s_*w>u_o6umMhUd34;OjSUhf+K{n?B zX~X)6ECe_S@ibiFBYy(1Zu^%8Y-CjD(csZXWQig)(|--AU?rk7)vrG>gS%f0{Zcng z0o}`tHy0Vy&NCI<;`g5M?8m(>v0AaI?bfDI@7%?;5!-5KS z5mnvaF-*}9spCm~#Ae)C)H0NPr-1H-twX)#VcjdKuhJ%+;-N=6O~WY5($%3MZ`iAI z%d9TC#eaa5FE0}yK!rvsd&CokX?=VDIjmIBwMGQ`+Vdx$JcvM2nVIi>?&#wY==Ce;?yX5h?b~jQUH<_7r*vkSPYf)4G~r=BN53 zyHA1`<0b%62p~C>MP1SRMv$^ghS&#Z@J-i;27lC+ZU30LYO4Tb2Q`5SG}yEc*aAO% zg%;SF|=DrvRLG2B1T~L!Sna#Z2^XDKW^WRVaPU>G?jw4RQ?jsAsVBNCPE9zhX z0u@4VgRv`|2JLb1 zfJJb8QBQ#CeO7Y${RKC_2Wrzymv5&9 z?ESWd&zV@gb$2tatlcvG(YJwjWFcfBgit;S)V|4u88C zG?KA1(DK#u-%J&)ZG%EJT8^n4-&y(6l{5o@~3Dq#B9fr|_vSPQcUWuB{Au~mD zkTHRSagBPZ>0Y)Akp`_qPq1m21F5X2qR?@dlJ;hANzORDUC^v#a!9>*K#@&C%#Jitb*&x*9jM>Vl$6)!w|`2hRl@Gf zb^6|V0>=ip5&y|0V#1d%VJ*E6KbW5YhvE*3V9K}cpLQZS%x7sndRx1#0kT3Dn)3Y+ zXlxI`pRQH&byhb<`6{BQdtlob8~1H}lW>5}8$avqjP>mxO#MrxO{y&&yh7q{S7AV# zi7rIIbjYthwMPg*1m|aNl7BMV={SI29z)HOU0cG1xPZJ#LLo0m)p`lzTdY0k*i%84 zE#vgl4t|G!G&+~+#N z08Cth=~3 zp)Ci40W;~t#G7SXTclchVG0Z0PXqRc=YdRo&ShBx7kqUM)DJ>rExFeTL?ibNPQ-O~E+iI7FvqEh# zH&BBw2dHG6X@55%TVKzd;(#4B*n)svZQQ;wnM$A%HZuJWSyodqXRX*Q2x66Y8)L=>wk_pH&T^89N1mxFA7E*AeA>*sS%+$n?^TR`<^))of82>enFu}wX2Spi9 z2gxfHYGr4#y!$pTLta-83B&zEn8DY9)KygQsD+V~X1+um=2;nXd5D#oEzf|?v)(rt z1}qW1=702#rf-`)e>0qRlB-CT?p>PBM2+0?bR~JT&G6bCW&&KoCBZP)w*CAJauHKS1xowGcoNx^6t78u5zY-Wb@E|VRX!dr|$ zx#p?oDH9dB=rSEmHtHnie=Gq*;aHeQA}@KN*x8McnD#~MLpNKRzzg4^w!Z0=;y}L# zC4a{@?ftA+$NR_hmQ_I-^3`Y$zUk502Y)0ci1)FH_t&$UC8IYGPDQ*P16m__3`#8& zT^(a!+L3Ao4G=1rM}K*0Ab3zba28l@@wZ&Ndg$>=Ow^ehq*84MaQTeZ-iE{|9E?(= zUiGCwA%qB~ufHbEit-bp$Xm*Kk~+!2)`F=Un1CV2p-V0s#L`;wvtmX&o%@V^bbr~K zZ-G5DYEy-+?~XTLsi}P-nt#7VJ%?;#bGeVaU-%vT9ZxmOp8}s~XEWrikE*x@v*nJ% z1O@y6uAN7y1AvbhMM(Bl?eV2#R&MWe2(w+Y9v62JL}wGet?P?npdQyivapqDOc0Gt z?i~!-%IEbI{bskI3vN;}CTQAT6n`9dM#%akk#aQ((tQcz`}#FhqWA$W2N#Es4mq~f z0g0=ljSMUeZq<>%mDp^b03HG5Z3p>K0nrCw0v^A+(uu&Q^CZ@&*xD`tjHky*3#@o) z=FomiouGv0Dkb}aFmZphT_%t5Q-U8@C`}?AWf58Mj9zXM^OKC?{Riw83xDa>)?i}t zIDiq{NxLvSEss3Hjz?AmXT*XYB2%KMv4?86duM;3EDYO%Dq4(1?2wuk85m6m&DysZ z3rNFr^J`ST@BNHOZ=sA5Voiu28g@)*{5*oz-kk`GG8gwAR`L}i`vW;fKm1pIV4OLnGwI74 z2I&%cMD;y2?j1~Wnt&q_y9;jk(oUk$(xm2$pW&}qdDOH~qO_JK_gZ^aXU3@pm!(7SsY*!2v>4i{mgGXgzR9Dfe()NG5ci};0LrNSQ&%V_(^!cEV( z53bf>@)%?6=n@Q6BY#V+$yt9w=o7W&T|z}Z)3z`@gvDq`L915zHdq3f?Vkqi5FE)^SWFiX|@pS*!o>VF(aR4^nLemkVzig=$M<;oBBs9*F4x6wG}!aE*o_)T)*XG)2Zs9#SC zks_F>c>;A@<|%x01dQa9=HkD!yzu zQqAV4gqwgN6F1QU6)gG*z(3=4xuZLuRkhk_5N(6NU4K4e7t*q~(Kk%`K%7IaQ>R@9 z!2o^hw$NT;FyJlMb6}!t(U(((%RIK6W2zRXWk8w-$_l%`JNiv|4QDa~-Z@q`K2Qj9 z2*KOuDqmT%+F3&2KrVKkp1 z`D7jW3keuf-(AMD7!;K8R0rt=)RTkCk~-riWYt*ce)~_Rw&ADhvE$3e&)KlR3NLoy z+GzL66Jltt_2RTw5(^%L-7AM`7pW8Myqy`XZGk%Au=I%@vFkzT2-7e>e|kl`Z970n zI)5E@VHlIjs(J%4C5RTFfMET1=OJ_l_>}`~+3wDaAg41~@&$W_=WdP5C~w1GJnsqUo-6+9owo#ifj?-{cRztd_$Pu-$0H-~;#OG3a;|4ca#c=F5ITZNBGuTB_ z*c~L9`}8iB;zd}9I#*B0OQJ;CH_|d!jlg>*BBkIex((rk;PF#(M@Z`}mw!ze-C}Qa zIo33im=;%D29s>L>{&PAp{c0BE_|1nN-2pmWNqX<<4Z?bR3QSFN~4s{0$i|jceU0O z56sDTfkW)HE2PnczxQH2M)r|my7H?Eq*Qa;eu6ge6sPm0GcN0~Nyeb&ccqa^b~uu> zV7xO4ji%adHQgfPelY4pV}GP>fSa*Pfa2#?y<2dsi|W36QBS|eM`#?8$9=!o9!;4z z=;HxO89*V{dSv+`q>X5U`fN|*H*ibAxLR@m0pH{6;X)%?nma`lH~tG=~+kU@u5 zA0(x=)=rml@Z%`X&4PR_kA{F0Gr8qpb5pWuw}CHJ=kY4a79hgH@LbmF{Pt_qyM+IF zCr+v8wEqf|aGlni(SKpE2-5K)ne5r>SH=j}jJcQ&w?jdt&|7ITT%E@TD_l03I*#Lb zv=Lh_u)Yk8sJMfM-AD3mz6^TKvd8PO5jc;eG*8j_HE8k3nP|4P_2 z+}2==Z3j1pW`7RFd347zhh(*$NV%+*)pAtgW|2&?BK0RL`ggv}+PJbxEVf;P{&qop)8lUR1Mp$I$&RzkEgUohOLnN9GovlrJV~dST!5EOaY=nh zO(5u~H!qAXs0yZL*6@!;)drQ);Jhx3%JNVyC;rlfbljr1(?j_ipoZWe4)J!rg>V6) zo9Dlzzkh9i?-kaJ*cai&CdNmJMZ*kRcl|u-I$T0w60y)Zl|2wVKYsjG88S*^d^l!X zdV#2qiH$M#Z95qRew3}%jq-U@LRF|$CF|3VDFc?F;z4iabxzer@)cfj2XV2P-aTz? zooB|Ce;+Pri^(xiq?28b!he=*YAUzGYI>GIJq`+MT90e>||6YKS4m5THvd6 zObf$dSWJ(E-iY``%~+c|+4?A))AUu$Es}&6`GLzzVQ1i984O^f=%wE zq)R{;=_O6Nfv|QVd7?}bX#inQ#o`@MmL|6g=1JFFYm(Ag(K(g_hER}Yo{#8ixd8z1 z*U_(Ce41a^xmjmq5WMq!Q5V$@H!IHGJ8+_v^^ zT|+R$8eq6ACRg!+9lHd)Q3~$-Eflz@9})Qz6XOJ`Bk0{@f&G&&l!}>CY135Na!jO1 z=S-TdXfuArk_UsCnV*p8YZ_E3A>JLW5EH#bV~$4T06z=S?sx1|SflCyPK6`YK!5vr zH_3vs*ALYM+S?M07|tJ7!Z}9M(&>JuuJ0pa25AKnmSe~f-NCk`K?D*VRUVQAsaEbt zd!bDpFlh5kKH0hY(9~H`h+uJsiRI#T1Xd9XQr{xhJS&bl=Apd z{j!mg@ElEU!p~-%h#Y_b-?LF-mmEA5Q;*R>vDnHPqoD3Cl%<#myMga1n@+!*I%k9{ zNi-6>gE~$eHvrC&c}!&8%bn42VdeNdZ5#D&vO zTnN4=%v;?si;(0CNx}naW`8W);{>uJB(2hr@d%E`H5O!QDgf&axqjI0^!3xu#4(T+ z^;Rv*Zc&rJCUp^!PnCcDIhCuZG0O+4)8nk0NAknHp5vrj_Dc{mk}K#0t{yteO2kUm zo&&W^0(rJ4o*#N{3B=@MlZ+vv=gDO!RF!~>Ernm-d0dG&SP=}JWq-M~A1+?~PI+d_ugg^ic`i@)+C8hO5Y1^+D?{Kx=Usa5#s>YF`Gbwmg zQ@=_Ztm@K2l5JB`A%Dpsm4On$qk?k-<0@cu9CqSJR~NJrLtn&UhP2cWs4lL$P2_Wq zQRSwSi&=rxmt{9427@xk8GD!*bDj7WeGYqJ{?7}**3QQ29%3AE^rr|w{>j(-`k$Ykm%$p2(FFR7 zbkcuv0Pt@~Yk$9zPq|HqLXaE4(^N*Ye5P39yN{^H5|pROSOTalQeIc#j^B;*<92mg~9NPhvGEupne zry>lM48xe<-=>23m&(b`5ziNUh8PXQFvPzK8Gq}q6o1%?3_`dmrN7JVUuN?EO_CLF z0?9_5|KWSbzvrMS zx{2b6eDA-Ft|gRpfhGV&vGHpkh#%eS59Y#FR5XHgs=v1uupEky{P00EHa ze~zhIz<&o}lg%msC}_Urs6Nolak|MkzYp9*2I^3Dhpp~|7+L+aPKz^Gly?pU?YiJ6 zz~-<1a*IS&=B;{Pai`Y>kNm3nb%T270nT6IO2iPVa7;;^@t`|?KU=57YaASa8k`+t z!l3p8yqm&d3MGx#OMj*`u(}lCcB;KO$JKvkOMjsB8lLmzXCuGntCE(jPBNDgR#7wu z>g_vqsZozwavMw6)%{jY?Xeg9kwYnAT0&8|ygsS<1w4c}TVvRF&LyBq3ZHhTMSH+Q zl7V`z{;`DTyV&y>!>75;3Up3je1}lY)nO!TZPw}=!4wS;QRT2UgO&ji{0G+ zq<{SNx?jU{xeu88R%RCz1lMe=6sIbl{9=)9`OvmTa%iCPA%QYmoyOHYz*MIj8Avjz9A<-ITv13R9%R%=KGG@I~FJtnt z*tyr)Q4j~%yVL)Ak{4pamU>7v%;-}b+t*Y|H3THRN%?pCdT&Vujoqq-XY?C#n7on< zPNES%hLNy7zSYHs6teJrwch@*r^HRVFfVk1PWWIMzru7dnJfoI(9UZtrjKJ#f`2-O zmW3I0Vr=yW4F|DTq_L~7ku7(!CrF%BV!K_`*$>2nK#^nMOLSMbus@h~W2N=so;}MI zDTrf-KJB;IZ{cj;D6?5Z?kZXO56A@PAW|^I_>g}b&R@;F|RiRie&FK{SyY*Pgfb8^DEG9+r(W<2(tUuWT@Gw zcE%Aq&s9X>hfSH|KjTPwdaSCmhAryv8kgS6Y3kOz_E@PPxi94oSoEc29Dft6)$azO z1rdsI9tK2Crf%R?u(ieXh+<8fp`jkwIeHGp_eRYagBWIdTDBJXC)wE^A z<06!$ZZb(9RHxa=pz*V*U6YY=ga=~zNu`mL#jCGR#L1=hQ_5p3;9v6|>*`)7T93_d zCYQZ1GY@b&u|%tregKFP?gQGUV#wEv8&pB%p1}N&l%#y)g5M8n)#VWWAUc00t;Vgs z$oJmo5HZUd@u)hI=NwzO&ko558kfLWt4EmNr62^~z+yxP&8_iUiM3G~L)?eFacArB z`e%6Y(wsMe<-9a^6IKZCPf)Qy&*j^ltyq&m_{z$$vJ&1$%x`N^X?tgE*UX&l{af)I zF?zXI>6$j$DG>Hm_X-w1P^y3G(kX#ygxuq3BuVns>+vs+Hfv3(^~U=g0}wnOyi-$O zjf?1N;N((|u@*Fck}o3HA{DXRW+svf!FYPrcLf}rD0}FR02-^vsDRKUtbO%B3^JkXx z*v938`FG)uJh5lT$;<=ePgfRNB(C-+= zZ4)n9u5hz8w!4QO!4rS9+4>@3t)c~kA02%ruL_f~88=1^ci4l$>98_j=@kyy%-QW0K_}}WkyggUo>y3Y9e?Nj-8?=>6oFW>p zFrVf7tg1arE^zWs#qmC`HzV&2^C*fB zeO7GE8RETi?BukJ!fXUP`#4-Xg8kVI>Vk`!Np1@{UXWO_O_I0Tz$n31H%&}si|e#} z&P91}fCWc=dC`A~ATMUx_iUy9ACAr<*K#0+q6^9(Ov5lUXBOtxLxrrZ%!~ZGn+U&3zi;Lq*NR#r>r!~P zvY#SjQohvd@Ut{&daAX2MrL7mfL3;Gxt~LXEQv!dp7no-MFMu?pQ+xMl(k1t3qf_N;jnLVL*Ha+j3{-k?cuc}U?^YZHeJgKYNZj+#JfX=&f~a~*KIrk>7PJf z$cA1RgV-WGZ_m}L%AVp`_$-(B?VP5WR2(!h)+~P*aB$OHb)72zFxhkRrXNfam}6UZ z6oJPE2>eO&2bTEdR^S&^@r=H^+L574oj&e#z?z-CxKd{*j}?~GG=?Hw2Q^n*oc>@g zwxmBDJPtiPSdL%J|t;rc{e5GIIro^eA6>(DZ+963(6IKZ!tS1E18k@2JzatNcVdpOQE zMK_a3dXRt#^x9$ALaTg!*4BaYi`~GZ?Qef|O{IBonsH&&q$>G5Mf=mp@u&}BJ_g#O zK|AZzH|tbhNbSp`P^jkD5_4oKJmnNwo!wiPJ8cNl1(M`g;-yEfYQmTyP+ZD?$cWTA zp*QaZ)+kFQ^0Dy+#)wib(;9pxnMwRE1(pLlk+2tX(N>ct@H9I%a~wDCkAF57m6w06 zca-<`vD{EA?iKk>F{3ENPr6birl!Wdl(zzJyV5|NS}bmS;2Xv02|Kk$E7ANvOhx0^$=WXZN7Et?~hm*thVrh%Z}GMt!d zK#hh?D{cckKTTfN8WKJfNsufg^`p~VcY%#f;LxW4Jv(10d`3cyqPMNwj|HwY3#Q|= zxAx2)7ME=B3N71nQ)-KgoXA&8ygatGm+4{gpfWqTg1|xF>M6w8K{fW+_2gl*|-s3N>22srD+GUGzL=3i-6$Q%6Y*Ade8C1=Vqj(_+P)TFf{q`;hKqxC z&?YUEet7^EavkYLHd0*;dc+vr%OFX{CNR7PZ|Ztm4nc;4w(py23W}E2jV?PrS7}^} zFC|vziwODeOkgH*5T+&gRK-mr2dNHNbgN-i|7R&597<8>lTi>-6oG#*OI0f;r*Twt zB{~(NJziDfk&pyz$<=sMiQ$rcS>vDlz(ahh788^X8B=!6ybi8_o94p(I4Q7_E!aJ4m_qksO!!Nt_KkM;0)L?oWZOv_|iNkZWZ;_pNJ1@HR z(MSz|sDJi!(3YOsmpPZk zD0~X%t=Wt})yLto>>HCNrOB_^O~o|!Ic#1v-(1<{kqJV#XDuaMabNc*!0^dE+ zx-jE0YIJtTVR2cz6cdOcz)q?aOBSGAm+PJRVD>cbo4Yp>q@aR{%k|R?#D<-p{qQIm zp**E>K)`IF0`0E`^`q9?2`)X_m?TS@&pXfT65js~l+v`Tad0DiHxSZ-;ZYt~Jr{yKnWc|?oIW+eYndOhJ{HH+meyosZ z;qKxNoXtwl0 z&S}>ovi5@qhvyC4eTnRqpKqXQ??IzjERhI_E@|@=rlo_H&a2D1kDowasC3zA2+_ou ztBI*i+jR^s_g-|qIgf-RTalp=aqO(s=jPlrEHknFT_hn+3S(M?b=8ib35`HK=W15C za_5j3L}SOb%0{L$v}^@nql16&8A!P#bB?pzu%5CFaGRT9rLc6p(EA(cdMn~k1UR?i zbBaT^jr{(ZeAs;<u9@-qL&Ni}})mrbhGG*zNnC!Qr+ynnpG|zuAq;1@bJ$;e9tZ;X) z4G6$SXn4rDLJs7MXxH=tZOM%)ub#f6b#e|QCP^HneoPRQhrnO}p=c*5IjyrFZoBFo zLo)@!%agK-+=tnMJtR>)jTDRr_caV*jVDZkQ25^T9u={4STFb`DAsda6NvVWSq5tcs|OEbnJ zqr^(P$5jk@xghdXlE!qPnF|%T+QBI{^MxK>^FQ301m)bE$F&tavgY65>@EEyz)ZK_ zno2|0sT(r2RRVT8LZ*0}yuvcn?^&5s({ji9?3M$#(3XEBcVa?$A>Sf<$TjS-IB6nW z!sd*1Dh#KOy*56CmVF3)>hMZlWH4pPqESqB&U!?dB+IF;c0RQ*eki9Amk>U+y7gya zJKI*)7jPkQjCib&J(X`H))r-h))I)yEjz*P{dC&}0{-Tkl)*YmUM>r|&>6>&FYVml zt3L1bXo`Q8*CGl5_qBd_^pNmQ&v5QAG++7)9COIXoEAvo#3bl(de(U7tNPxTl5){Q zj0wC{?6fN?R1Hx8^kndig7&cy5e;Onyj{0i&VF*%Ca7Dd28scd{>Z8BTPw1|Ffho1 zq`EmO-~EI_SCuD9vh`$#rBCt)8go0wra!uOyda>=az=GKC z3NVJvPs=@qk?R)#b693o#9eq5rQkP$0 zzSe&zZi5D)I`UEYY1bHyT~rf9bW}Fq^|-s3gXoX!*IE3_CZvWpQl4_%`xV=ppxGBk zb&!mC7|$SOfKR8A-;Y{)8c|eR;r>uxTvj-XFR6g%9366T$9z{%f0eG0383yFFFR`9 zN2`>U5>mI#6d{+%Uxuu4k-P7O4rd4 z2Dq*H4e0Dfms)Jx9d0Eo#PnP`XbaMy5#1b8#O`)KWD+^a(LLkgk?P<@37PJN*+8Z_ zb?Nx~c`5KIAB05HNX|F-P@nOc#kJL{Ngmk-1cAL|Z-^fd-_zeW+ksqI%y364^C*Av zp2M?4i>|gKcHa`TJk*HH9#ytH+Qw#lA_+r=OkFbTG7V1z-Io9}@ZhoEO)JI_AsNi{ z7M5kDutwF)TThxKP8eEw#+W_fSNjMbPfrO1_MKkM16e(6*NMPFf?mkg1moQzMzT9 zkq%~HR>xDCmYDG9p*p70EA`kYcnHu5%o|ORIm8hxNilOe^9}bi>QfY2oe4OI4p%gh zMAsV6thRLRAVD-9j4tiNec;T)Jaa8;KEgrg}Uz*D%(4kx>N=7@eBk)cRTWpo#H*LO$Ri zMH(z`yNrS!=l2TH&RJ>9U2VFm)#d7awjz#p^F#PKihP0L^RwuSab3e`OKC9wkI+`lQsa(t=M?wsKatR1h}1`E*zS}Qp-^j3lz*M)mCr;a;>U)PldYwPB^ z?E}T>F-Nvh6vnOKS^cbFc%x~zZPg8lYh1W?rhNTqihzk4-zD@PP{?QgXDTSmPAbbx zmWptfean(Yv{8cl78-vkx$slf_0L}uYRsGj9jA3lq}vd0NH>FmU&`0@_>3eDZ~a-N zVD3NmS3NmW9L;B1niV+LtY#q6TeQ(G!@Nflm3tXK{%W-u8y(u!P>N5cYfJ!69;5}0 zXX@NhPqK)K*Po7S13+-_O@}xBCQQPqm=>a~XeQ1VfP$Dk;PR-t6v; zm(!@~U@l>G`fq(_ibUm+7b5xDd%Hp0N-w&+^040!K6p1UFx)HJ3|ies#BTXiVa$@* z!e^)E!Qy`fos2l6=U%z$(!F|lR(`2$2?C(ij3|1hs;-qLuKN|H2KD}Eu*)4a zif=28_xGeLBGKbD@;gY?Srd$nG8IKqOTu-|`&CYGa-qw8RjO`^)YVv{`( z{n^AGKma1lF{7jqaAWLfj((sX0UReU6XCYX`Q|-L*XG0cd`?S;*sGl-NwP%&WH*&Co7_Qmb{`2!adEs%cw5K-AdC!}=%_2N&(lWnPT;cu?`eUi)%RxXsRZNUMTs67zwG0 z@Du)drJJh2%qX}mqj8!q=75N+V3GHj(q#4GAm>P+A=zmHcv`&hs}{h(8L)^MxCjo` z#}Tl0%@3v<9dsT#jIn|GukFa!sX{*O#XR>Qow9$XtyMJ8jrJmNm2{=OtEeAI3vGQ! z8jY%0)~+sB6Kt?QH30aYbDN9z_s}hvHfq!@Wwpv@M&pmO(w}n`-vP_AKF%Df>vTj4 z!^X|kjtLaq2wU*Eki<>cq*W0^GpXN3t9NhHq)O79Q{Q`YfceW!yE>S#Bhp%bM?<~q zUVne^jfcxlTg1_&&rsM^ORv*$7A;{H{)0mAOqXeI==Mv;pCXp`>?bpx-gmwr(V))02_6yQx$`Pfo~Nq z=z%Ni1;eCdLC^!E*jQ_^82h+BNY0KYWkOzP$ z-vyG`M*P}UYn;y(JLG@cn+FtLfUXtS6*aET0OTLQt2G!Mg901op%(eM zL$b<+`xr`-s9t28(B2T|3ePz(Flg68(p=VW&r2Wl8ECJu3ni6z8+PqsWdjui8DZ|0 zF$tYv+nes?e)SyWjBghQPUCAfHs6>EOe9sHakY~o`M&3)Wux!ReCh-_(QrOzn`l*i>4~#$y^35zroE$8?i(e_aQtLBgUbi)TuS7%EGcy4|~L^0OVi z@CCE(8|4_O%5PluvE%#&vJ@}2SIpHCE4Xq@giYng70z+=3mlx-i0b#Yr_H^LJ%Jdd zdjsYNJvJ!E3mFm0O#bEy=C^+)GYuG8{vehj(V{r zO17c_AwnVDuKfCDM{9(l7rQmf9BcN@sb2>GCAI3tR-NYGtqVNaV+r*4(-ZARV;Bx_2K@0l;~qYjuI;Om{z94n*sSFK<-3VI+6dE68T+;H{1 z=VXGf3QX1Z-|#wfh9f8jPej-mo_WOcii_Qb_N~| z(8{o8>zm%i5lg^`y<=u=*PIFqd_G*^0Wf}G@RwoSAMQphUs$otB-4nZkh22 zxw6X7tNfs%333d`c~)!>VC_@&S;1d4x6NwiSuXbOFg#dBy? z!!3frH@VkHrQQH3U+{kFRXX-|x4*dl;VBQ@?*L~7}0vNWiI<~$|QJ94!-xV>}G>?LGa~N93K7Tqv%{FwqSfY0d zy5`yaWNyWnG>U(UxRfk4oq(_VjwF^0@zZa(OP|v2wAxg1mQ+IwQvVs1ji$QxJwVBwCY6hQybg+cW(6rhupZkk<5G1Gys1lEN2#uiO!&7@AHb1q0k4? zO&|6);Q?_rD9&whS~zXzB0iRXDXAW)uoTlhaK!gn%T}Xd1;XvA2O7AH0Yejj7YUQ) zp~MuZyIE)o`PpR1!u-;hmVD-%z+L2oCWAjG{%+YW<=uQdSkUQof067LSnJ+INsqSH)vzvw}GpyA8mKt^6p$>u7*CSt!`jN?FrH(n_p(s7a&MD*Bd#|&T7rW!bCIl)O+K))QbPFje zIF)|^gC^3V6t?G#=abZNe5jhGL-klPSV4tFH-)Qg~woApJAMSX|s&3(XR?Kf#Gc9-J)Nf zVTk|C3I#iUUsKaICjbe-{O1S@tzZHMJ&J!}i~`f z5jeqM(7NnOy^^{bD*wv6S4Bmg` z#So%uJzsoRkYT0yP%P~muW|CI&D$DsLiH7z)r{K;sKvm$WYM6Wo48xmuX`Nv$s;fd z87SWaB*s0Op^qrZX2A~TTpw@02j7{(Q!#4y4ZtXM%{8#f-Ui@u735k-fH5?B^H@^K zo4l@N`wpJqomEY>ZF=Yqkho9h5H^1XG|iQw!qW%fTLrDNPhndn(s&NPpX~S5#={7P zFNY)gL6q2H+%lcVxI86OeQYGB)EIP+t=&W;xIJzbiDlqLLpHr1bqq>G@i9rWnjwp! z7(u&B#^i!Xm|I1t>P~OsZEDmfDZ;F9V77KGv+GIaTc{;!#Ow=*%Iim_e5rq5-3#+; z6Vu)IR$cPSV(N;EJG)5cs%9x6gpf$78*NqL8fQl9 z(U$0WFyu%1onMc)H2yZw|B=TR25cu6k9&Ue!q%)Mqgz^HA%Qm6J%ZmsWRurJ2!aGX zewk*>RJ(w1_C4J~Ocj6ZZ@Y26t%IIlAl3`#Vx?gKOhtas*&4t{@;dTlRX{0Hi7dkO z#?%}Mq~wuAEVnVm4-aY4gy}&E>}LNoh1nC>wOL-6N(mzejOhT-Wjv{_S@%|!{)YCf z349L3M9s`S`I_R!ci<_#&Tta-2C?Jqx`&+s~>m)KNdbp6vJ6NI0zv3m-~ zKcjhI4V;PQXE&1fjko3&44<6H0pCNkOUM)H6m#9xT}1~TsY#3gMTE+(rLs(uBW9Z7 zH;KUUd#5#(XIOtZHH>R>$xzj_j73dPR#=$3BH!>sS300GtJ?6TA?R^1S)R>l6J>Fp z6++zj1cnejV{ncvQk7?POmVoN7AisD)t}K1AE*czi26D>hSDwjfvF$99veb0DNXpa zWL5DZyUULRX;Tgi#fjb3I2ke8pm_&tAyz@x?yHN6xA}kJ7G3-sXL6x9?s5?FR3@#1 z>;eQ_`c#H*6t`IK`Me$A4T6wvz%*>$VkpebQMY4Ckw!e(@<)Vp+Mb5gFAm60lKqJ zd1F!@Z8q~uaEcRQU}4x?ovo2TpP}mouYGbd{;oBkiYQrs`K03S9Q(uji<6ial&qbi zr+I(Ho0cRe&oev>#6;pdhyuRgOh}&+1>U4w%Sb7Xk#cT}kd#JcIfRoMc+NVF()aZ> zkF+0EbDW@Xapv#YtDhXkFvj}@=0oRm-hKY=|JIZyFYrnU(Ul(>o3&-NC8flAdwhrH z7VYwii2`X2(v^a$S>&<_yCO`YUjS7MNLwr%7Jg$hxA_2ij1PA3%1I?a``GEski zl&w2ZCg`SQm9~$e;uYPx)137W-<8yVkTJ*Wo%8@8mG!J1Q&K(lifQmQs7Bi~Q3AX+ zo;to?#J{(V?O1IYN2mH%u0V{{izYCOc3h69Q`mpk*t_F>!Zo1Zl~cy=7*AU z#o65kSSN_WxHaCe-bE}iQ5pn`;0k|HYM@lviVb)dz#rlGW&iZz54g_LpV~5oq+*0qJALj*HGkV*A;zKJit)8u4n6s@p~039^WVQtk3GRO2~t@;B=;bH|Iq_{6pg|aVt_C*M-mFJRnV3 z{h~cCo|QV;+NmPl1{cys^sj%wI>j?g+uK=M7^bj=Rm^=~Qt`M*pXD)_4p~7BvAZsb zpk?2)Q-)<@T6tYe!0HL5mY5a=qHu?eU~hxD@KC!0HyZX6@=Tg+JT1bljUZtDIG+$% z`)xjfJrL@o?;b~9#_SEaE6C_p*5EOqAm7K?iG(Zv>OGk^Oa_0-?|>ACt{2#*G1&JM{QQo-g0p&9vXqz6V?dU@G^rZQM+}=V znc2YeckSosK5jG4?^}H6{x)z}vaLS*EAsh&Bcn1guqMnQhHoUs6|^{&$~5`bq`Axe z3us)h3*%s7sU42K-8t<}!FYhxKoFFeDM{DzenV4i@PUzn`2&C3{W_NQ@yav!BT2b2g?%_>GBqZ1~dM_m(M4h*i77?2KU0rCTZbg!zG;df7G4mUBD@ zWGNOLCn9TF3!l*LjWvQf!wqUF-KtN39dr}R;IdjouVod3%>L0m)x%>=m6owj9XvGr zQpa>j8+cN1Y+HW=;X#ZuzlL>PT<6Pl$th{P%=`Ru;K(}R%%pwvU|9Q``^uWBX(?<@)T(G^z-q(aesO<-&?%Ge^v}a1~moK?2aFDUPkkl zFiIAPjppeSG2V!-R#a6MievaH2t*js*7&YB#Nhx2nqq&Z;}BJTip0dnkCPk9y#ADs zTpA6GqCyA{Dk|HbHgo4-5Fhm?rTF(kL^f)>hJHxPGO+WO>pqh6w3;_XBQE>x<*wB0 zFQPTe7!e;-hzVDdIrZg%R=66_3!|RdO6K!g&+Ym!cNhhJ{_ygfP~W_>2PE~*T3Ul^ zy&<8LhsS@`cCdjhojbN$str`IV53C(J~#2+Hp~xaXA8K}ghRao6qOdv6}?bwh!WXq z=;A?B6$L2KN1jgV%E5Ne*$lV51%bDS^81}?i(3By+sJ0n_eBBSZ8pXYI?r^Eqa3KO zySS%qC^2?u&AZ}jwkaJ6I9i&Zsd9R>CU@yxW+i_gUz)@=?Q3>x36K%0pLjAlj#j73 zS{h7J+`_w8(~6Sj>&r<$Bq-gi?%c{f7Ze6;P{w%ztl@?0hBiPcC?Jg^_ITo!Vv0<6 z>eyXDE0=aK_BuA)TBi|}pVRKF6 zkv_mF&OOVXuy+Mc(iS1NgBJvqD_P=+Q2lL zI+4Q}UkfuiNR`+}=)!+r^(ZT+AIx`zQG<;tmg88vI*3_r`{>x? z;+g|&p9E5BH=T3dT*?{#$RB%J*Y*SnqkisYjV_6Wq~oE&eAzqbZMG~lU|p8Eq+HVC zY<=h=SS5a_eOMl`a0i)|eO3Tt%o-as$4h*{R7yemZD<$8-~VQwJ;pKl+ z*uXjOj)C|rNlP&V-#w2wNw7^GwrXn*Q>LoKyo*2VFMq))2=x>&38*w3QN=2I_Db6Onnis&RI*ttXM>mvHvLCIp^*dcq z4{%Z6MkQ`ub{!_^NBP7XnJM=B+! z?L7LDE~1V+0^{eIR(%}}%9r;!yRB;i9s((VOCh~2_r(smN<*%i^sYicjK7hpCHcWb zk7XFv6!F7U(Wn{?_BL|SgFSz1*$jWJKxjyQL4ACU^c(-dCCWOz9j2 zxbAwA?gKuD5Ky!@*LuGb}2FU(B%E*5%+L@$Q#TOwLH7Or3h@}3wf}*3f(kDdTi^-iUHtQX^OBv}I z-@J;#F=X1y)srb>^_0iteFIt!0dS*O*w1aF9DV;0v!O}7+#-KYv;f4?6dnVkP7xqy z0Sc*8ticb(-e@q=?_%7fK3q(k4Jwi%nt7xfjp5ns!xazR8$j59d6C5Wi}Mx#0gH}9 zbHa{Z%8qXBp%{1Fz~{<-XsBmN7ZY$zc6p&*f0hr#afZH=w{Pb-4q{ayzjm`5OrxP3 zFP9|hq?hbk&Gdh+KZIQ>&l02l(^LR(ev!KqAHL`s#lY7GwPAyEbEmFeRI+jh8}OHv z%HpXwP1o=Td~DG`c4|{`^41#=x67=+V19b(kvW)@-UNKT4148X+P63ba{}9x9-yR= zfv_a5o&Nj-X3uAH`22KpYCkgw-dJ5%scjmw5s** zVj&SJC1A67Mb`i#L*ZigtAagFlcIfOMm%!=!YXn282^`_3;S>@*Ic6S6P4cktv>V` zKe+K|n6$wG)2v~^Pyj8abttHD!Ft_0X75=w0Z(?W(aH%-hPl?k9FW*>3_Nd(ENF}> z8#@i`o=$(^yQB|;;!Wy1DW-pwr^!%tpLtS(*;EOLJFo-d@=IvdRyq=a1sP3!4Zh2i z*5tm2UZO!T65)AxsTl^(!92$?;FOcljuilZyI$f*iDSIFz2Et4KEk%R4UbWCvplk( z!1~@HVEO3?i3!z`_fG!%R8baY)b>R{f?YMqnaiKr3Dqy`)zn1@Y#5{uljeP)& zrSw%-rkLa0jtW`oB-HK*pYUarNXoN3WqD1W%is?nLJ7oX(5>8S$g(J>3Ubu|q?Nxe z@T3tv+YM$AM5=cra?q^dKn1to{ed%KN(l`r zyC9H?;@nM(^=Y+MzBl@^+Xowl%0q%~iR_tcR$WP_;Xgcb;@@M)uyM1k`T_5}6Vrc* zR-=1QlFvc3&TBp!n?Muki@6{@O1O_H40O629U+-yO5`3Rp*>GsGpU3{a#NQUck_Zf zHUQ{f!C<{BnX5|lFX$dKB>dLq17w}GK6#_)HwE(8$vY^v>W3pcqmY&BmUI3BX;Jbz z6kWI<$toOY70J2FXtGMnI{iSWm%e}FA7}5@rmC*Af&L&b#8Zoxl@K7DG#pgCQ`B%! zQBec@`uzm$9zEuoYkhm~i#dBdEw)mrdh4yXD&%;J)=`5G=*8$PxkXcRIyB7lpt>cq z(G3oh?Q@mI@PZ!S?&r6x-`sEZRu21W6+Rk3~E2qsoH`bMH-rIbr z?U=W$UJot4HJYF0kZ_gs=$3y>+M984wj?xTx8ip4Alcw`xjzrZSlN%@JXt%IGU})5 zAp4rnQelbxQV)_#K_9#NQ88XWwpT6RzS>1m?#+8y>gwQYdh6eJyGP$JFRq$d*NwBl zC`B$lY-@p^5#B%2XJNbJ7Kmx;`?>F>)Y^S{N)$OkHo1kbr?Dtb zXq=maZ1yl$PfP8eg5hY;yEx00ZHK2iwU+tJL-TFbFCW`OsTIw2HE7MTyN*zT#Q!2IW5xRBoj(p ziA8U5IG~0-++B`cwDFhMy%&aOb*}9+aOu_IDxAE%2OQt)wt!L?c`q0r8zwgWdOi&v z{oZsiV37AG)7ij=3%lRgp%=Ucm&;-*yz9e&@xyq|(k4E-+ERZzPZlJ;(>(eE~tV z1Qr{l7i7qn4%&Yz2IIRq8n|Ykr!KZ#Ys)S=J}34Rsv3WB*M;_=trTyb+&z3 zb#jRtzPEL?teawEugUe9UHbM}y-k8gYWI|C*(kab^C!+Gedobmb~bxIl&^appM1vO z=@JFYqt>Oi3+jLK$h03h8Eu@$*^AlaSZ@Y&~)8$+ozuu8S*0&DCp9wd&pa@fw~; zIy4t;xqVF;YE2f-f_*Otl7V*Ed>OlbWb7xlahcw5vK^_1hhbm5Sq;&-Dz)QOJNjAD z4n#3&R6XclXFD`2V`pcw(W@v2ujg}@SkxU}jXz1vt7Qz0&E!5GmT1<01jnA4PY$!m z%kCX#>JgS^|CIY*VsNbWtBUmIr61ZCGV06ePAaG13)fLM_nFs=4pVj<-|4ho3p#0Q z?YMtrmyw!Jhy4vgN5ssdvF;5I%YC@_%=QOwkOx>g# z>NvMsx^^1q7hz1DgLieaTeJZVQ3}1eZ^Yp8 z!X~G)R;M?Y@19d-!eOtzD4f&jXnfqMqvHq$fGL27-DP{50V(NXc)VR+il94_=3##r z&a_-}=DOonm0MkxZ7Zr=uflzDPmJTBa_#wCAM|&0Gx46&O0D*X{p_gk*Xd(D@q*L% z7F{u#G||;iQtIEQdGV;q{^sbclJf@ins}g#`yz8Y``-24`tW$TS~|QvmxrZpZ{eKe zcP~EJ%1m+UDI1QC>uF;T3+u4&&6G5n5s{N|jCf&-kV!{hqSwZ=H3)SEr2)G+Lb}N{V&mOS_gf z4MjP2Csr`H&Nt~=S$rk;L7?BJMnze>xAddGJ2{K3+FoB#55$D7=I3i3A0B_9H90y; zsF&vct6q84=I-C0<`$p!+T&Cz^0Yd~SqSmv#qP`uMA>?eco0r6Vsc-tSrQIQtv^&9 zJkZstqHgKzR31)wdlVNj8!pbfP@SBo;5~O6{g&_QIzR1O#*j5Blx4V?-=1QgY>riR z*xT1d3cG}{@pHa1uV4(dqjP^l+u0#C;uulg+RY8IyP2`NG}L%?aAwbV_i_*S!?2f` zy?Ivq<9_y}9D7%v(|C_`g}{zk`G{#~T-i5j;YDD%$MxN*+vzpF&y*W;Ba&a^;8{q~ zbw5MHu^?MxJKUYr_IPHuo?iQBCr8&T!>6@-dR9bP?2h;J-_-!Ge=DRN9^wBjsG0 z%hjGQjeK$08FO^r(~JLAj1M9blWk%3czSRQxAc_LYLX`eSU4DuX*t{3TkJXR+Op=_ zS78&~*Tu@ar6HS4^t69CCS}%~A2Hn;2O>_-O=C`1xhB@Se&;a0p875907o=vpx;@&f;6}(aM`F1HbPan)`J#Ivp+MR=Z$1l~2>K?D*t)+8E8$6`Dm>+#Au} zyzhQlLWHTZ=}+7ZV?|OmeLdLX-sS|GiE>i)C(&-t3XNCp75jZB9S4ieW+bYO$d0(} z`-?OVQ@3}zsOKZDFBtWmw6srq1CDldow|ZOw%h0A8U%l}2;J$ekNrIEmy3L|0x}$5 zt*SX+7i#SE12GJha8y3RV&j;Fnk=ppD=T(Ob$HP9`8v$4@%^|NUk}z~)ju`Xy_}5} zS+eyW%cfdRqZiM6!$IaJYSEVJHD1?nRGg+brtIr{S%~ZEo~%quO!Mp7GXrNdKV1E- zwMs>-L{Wd%bGH5LiO)NiJAJ-LflY7wc``4TWuBsFc^b1pW6#3tWmler>d5Ck9BZyv z6$-<1{Je~|*uTWd>Fc;vA@X|HTm9%wR40P4O@_wppgqKz72_K=R@*pCG{-2~!E{Y% zc9=S|fW`Yq7Df9F)Q8J%Q}?*zzp zW-;$=CZp2FUs+qMi?7{lYH6dnAJ4^L?&3gyCSMntp2pdEviY)H`#22um*VQR`e`$M zpp9GSD`&A&aqu8*FtdE+ODFeA4=$taGkgt(jddC;)BWHX_|wV2pD(}kuPJ+dp`_k4 zcXfYx4+>jb?OS5Z?nn2MU~{s?es0~D^ZanX_L}K6-Ce)hXS&nZ{wmXPf3w745;>2u z9wc$=-{865t_<&jS2S;oi#xq6-Hn>Mi=f?VEhpEQm)qoP^3}g>9K;^OQ{kzNQYrmy zS#BaFEKeDm-HNH3)l21T*xY>F#*5{y08K!$zsK%>4}CbX_UmVuY}xfd*m^njt|R32 zMZ&$Kf(QL$eCcWa;Iee@B(1kaFgm#u+h$qo3Fb$Sif`__h-|4`Ba8B;oP-H zmm6|o_vTjMB%|H3-p-mqay;q}W60*GD6RVa^L#{(5lr;dhc7*GZ3sM{F&Zx!kIUXE z$h|p#V80Xro1e=wrDdcH?!Fb;+070o4?kO-SK$Ma%j$e+?>=k>*W%(2Z7W{EW*=M2 zl;P$1{H3odg%U*@oG*Z$l{4L5z0)QeuB_R0)vV5Dcty`qa9)J>aB&@78J*R~HO1O+ z7;OjfVZMAubzLjQbbb56p&iVcgq{2IyJd`j=ljFPnAoRvt~K+}VUuY_E~oxAc{<#< zTm`hl@n9F%tNnOS)&8qIBnxHXnU~O=Uv6NwGpy;u;jVbDX70;Twe`KH<)gl$UGg#V zzEp^2Jvz2e$Ia>7Jg@evovPElcAVT5caz@k$$5EQd;QxuX#b$@@LIl*4iTHt-<``77|O4Kc<^qOWzQ;tw{&mz>kD;4vX8pFW?t5;q2@qdh`u+YM`YGY-`bhn+bcY7PNDJyG z8f-_}md_h{yn=urZSb|@-)F?f1}oz;I>$zsB8RRaG)S9p4o@EqiEv}y?x-Pu_*b$N zsJfoxJ<4=CZ&v;}G0>3jw75u?UuWHo&$jhkh&Y@WTePfB>C>jy^ycj***&?tt=lG} z!`{^v{bQnzTP^FA^Mil&tLgKE0(-NpwcuG6rqy`(`I_AG^&`4h+vLKG$IHJSTieKw zXdsTRYZc@%=sBDD*mcxu5Sb@`?>t(c)@!;*yykr1JIC>Rc5*0>!s_#Ne{{v0M%#}k zLk%{)qw_-X>h^4cZB(^S?K($Socnt2r_Cz-MvC^Cs*|qh`bDN&P5fzS5 zAGMXPpt$8qX&BG-YimqMa1e8sx{7C{S-Nww)4uPIgXwj?yyfHZz*Cohn@4yxZcROM zzmEE0NuFmzpPs(Fk)dDio3Rmo?c>YWlW)>6-%L+law=cr2cBG3y|TwZXJoZAzDgxM zPbLFr`5H9Y3*})IhX=2xxDRuh9!|OUtomM-F|Rl6*T;Bvx1Xld5?DAJE|Uewd2`ge`GqME8<6?USs*}IRCVnuM6D| zmM47~4fbr{IV)wq4rhH(+&r$m&>gu>&^+$Opqd`U-n53vdfD!ON1CF(b`zyy3&e%i zJ+uFD9u6LlqgP+flWcolKH}BsvKxB$&AjDb^WAg&yq~5A^E&DE^RHf{8hi=GsYp*r zxeTuRO}Rhbm#ev}eCeFAnNjqu`jW0*yJUNHr?Yvyp7!-=(e9e@;tS6g_17jnWU7Z! z<2F3+-Nm-KBeU{<(YDlk;3#m6kHlHmXY0_0bo5noeP(A=3pIZ|ZK2G}xIACV)2;ea zi8nI#n(+nwoz~Rg2dCbIocMXD)i;0A6N$NcMG&cr5^k61kwvHFWW`i z-bT-_ui`5q1?TGquMG{e=RPW}{vz;!eXO4HK98t0RZLl6a;YVe1Bma`Reh= z0FsBi6x}u#blI)wv%eMq29rkz%(d@R5;~m-O$OaR)6330oe!&tVM+C8E&%kY*c!`p zV=NZzNdXLhE$I$4c%O>Uoq_;>vbH&EkQl#Un3wTQ^NsdKyL0#5v`Ofp`+3G>Y^(Uh6p>d3iGQLU#6o z_zau&YfolN0~L|WPQ~hr?&dAT8Wac^#fW_!G#+8#1YutnMj_YS4zBy)aU`YuZSmA~q?H4cfS8>jVe^F(FO+0TU(xh8LP3m?v}nqOTH^ z%#oJk<&5!mY!cQs&yU@ z32an{M1T)qR846>Nbe+*-;YV#H=T{sC_>WiA~q3^P1MFCx0P%2xyPElUz7LO#Gm&j z!_T^0tjx`H|NVDPAxpLFKo?M#E>4z)&xcgF#7&cR31bICqT$r_5fsQ26eAuwEx0Cs zf`}w5_R<=iQCwEQs09Jfj8uekMqDRjBeQXRg4i4wjj^Z@uA(`A;bh?ZF4cvyAy_>lNDvMn({SNqOl}U- zC!~ek#ylfT9k7->3}313tk@!8%LS${UvmpPdSJw>!pW4(P9p@uMF^-tTCw(jlP<`BB&MXu{ot66eyp9VC=smHDX=t%BfxJjDojh}qbHl3@_Brq?lrSYQwYgNR6-FpNUbmWQqFfkn8f3GrhM z7+t%~mCxHN@q&pFXlc!&%pzfR^f6N{;7iNrD=iQPY9VKQYywMg#7bgkP;EiQW~%h( z=Z&{P<;o(jLw-un8j`C%E-b&gD5;AvQ>slt3$ux9W9`|t?5+`it?w}UguQ?{2cv4^ zxWm_)t=+Wq_vQyT#2a7##*jBI(8e`>`;wI<${Sy$O}3dUc!rpnw?h;%A-Lx_lp(?~A^1^lUXrhD5y7(Ke?!IcTW3&tXg; zrN;U^VHmiwplM|X12go=4#KN8*F}JQl`TlZP_}6oQE|n81!p!ZZB&<42)d;Y;4#-A z(0$beu1EfW3dE^qF2{p-rPyS7LN=z#P$LRtiic}ui5rl~IPah7!x72ZVElspL`P{^ zEkZHt&YaS;zQd`EHE;kK0NG&Pj57ly`hUlNJn{niYt4V`_U$*m^p!W4U_&>R_us|R zJ4bi+8e_MAbw9S2c(Q%zP5fo0JOV*0oNtF?bM(V`EEJ3=c5-e2I7jUAD$}kib zVo9ohlXAFEHn(vf&NtMS1!^ZVmcHUQW6p_ zIqdR=7@c^6X~3DSBdT|6^I|8KjW34pIvC_@2IhhQ2$08}X-j55CEsHV1pv2Epu|(J zDLiQD-D~53OfrSFU~rA#k2Oejp0+JeW(01^PF8s?1Y+{H4-8RxPvPsveY9I&x} zFoW_$6>EVwAy|B_JMb}K`uG48L8@=)aa#C3LlCcYU=#r#ICCWH02uXqs7eVXkrY zbeP0m?lGVm_M;^Rihn<4F}nv$RcJ;KaBiT*v=1VZe@{nH1absY1N~?h^qd5xNeY(fOgWKspLjymflC9O z{9{_UeyC2C`hK^kYByD4%aV9P@twmNv+rVW zg$ftzTF0wDc1wmO*A0KkbtGJZctVEzv9-O7Jz#`28+LT2{XG_}d(CC`B2Xf_)RE!0 z{#h@|k(0Af#f0fF8P4hvcPM9*{2?09bQxaFAns^?@Z=9*O1um{@m|}Q%WC`*XCVa z`N@rLS1Z4LO4@Bq+;80DiOel&$0gpmxGl#8a}8Yhk2CJM3B(3kH|HDYCA6e#6TV)bC^GAD?xau=CUXEGI3lu^VclAZB+@i` zHbt_KdcZs{H7p7pwdAnJ*HTAB|KMrOIKdVWS<5-dS2Pez$6~#IN35VIjpj7fZH-3= zfj2$?F|>mFNJzfE>-9ur0_T*VDlkN%!@Afp9~K~}j#D>4tV@xLADhd1Y0(!jh_L{) zll78~*x&V1=H@?spC#wF%_VMjxqtNb-(>yGA3I&>T_c^p>+WyO;iFkBb@a^{lJ6Lk zwtm;++|6}S$eJ;Kr!7N>C6*6`hKf1PZk5E0xIc|o4nHwhM2~;P+kv~aSIy4G+(&Da zE`!9;gy=1JWMLMWZiZgAULhS(pKzI*7cTH9$BgK6Gp(^EtouxOz(j*ENr6#{c-ZTA zu|qIoF+qMp4ngd$PZrxX(Q0|3FRkUxUmSqLw_g(w_EXA#*XZs!8teReNK&fE`~UH^ zu8xeGk8gLjmpt5UWv>3g;~q#me{=O;*MgwYFFY|&lYH~8KRDhdp@-$)b^p(6|B2hA z^=flPpBN|s9@Va|>`xZko8Q-fYvFo%vzkpo>V$S2DkQZZC6&XN-Ng?sx7%i4{PuZ> zjOYZq#5#h1d53j3<6K=fi1~=Uap_;NpVL6bh^(#l>ln%ZWc`E?%rQh5QmUIwayzH@ zwV0N_F%1~O+9yw-F5bdkpG!TmlJAj!69HEVL9zB(N7)&}i;lNZsWYBCo$=Q?esV?o zYi&ODf~!Hw{_qEBYo3_jc;-$@X@dd9_E-;q6W808fzjQ^@-XfOmYU`PBR! zN220~6Lqor(MWz`UFNfojyCdAI*`8gb6<6gb$@msBBqE%Ii8+Y`6!El_8VYy!eFCJJ3X=@}gABi|vVgG1k(LzoH|GY1Y4Rc60W*DsvApb{f z{;myJ=#I!f&c;8nO&fnz6ntz5VVGPs5X1yAelfn+O}Gw-7)dP*gE57jEr7Wc)vl(0 zF=ha#2w3bOA#fW)VL+tp_UluQAI>3_iu5cwco|@=v_mi=5NhR&%dJa1HY~ zB*3N4{_PLAmUygH(jS`t1CLwxkKUgDgWrDZhsJx|$)ERrQ5n-QVu{w-61nZZo-{_v zDR81AP3`+pQuwLuE;zSz1Qaof)`DVxd>ru~J{*n>@Hry9)5<@1L^0u=P4G{gr-!F5 zKTAY9T8w4>{KJERbbMb_)(Z`m=NRN{ggjf3J=-YaV%<=c2Kqc8 zac_ymPJ_fxO9)YUeJu@t|FmkNJyQ<%W#OV;@rdk<^^?&Fh0EeTFLf`(rTpHqXw||G zeueUTN2KS#SblGrgM-c|-gW0kJL>Y;ca38pnP=qpui3xWJ|A6A`V8u7DxS2T`}@`{ zzcnMfiyOY!O%TCISuP!7>vXl@hKOaYQ@IcsQn$0y>)jf%NN19NC&rq~%rP41o?X}8 zGUbh4N51Am_>0pJ%-uPWS{s+Si`XHt(0q`@i0LmQtmYc;bP%6#*;gQ#0Jq-PQvu8< z6Gn6miI}y?+5k0_eQ@un_MlPmOUP%5l&}J@9|PT)2kI7s+t^Li=znUTHx{xVU3c`O z^Z&8i`lm<4UFFSx6TbD{RZV&TWfZ~w^M?tMWAqOq9kM*)36*K3V?B#pKj zW4_4~+1m_%bS`;P_0+RCe+hMAaS_Z5iWdAI91Cx9QvYx)yvVB73EZA35+8wY1R`l^ zKm*+VTka>iroqWYu^Z%MrFGgV%NAmbQ9a%Rj|-ib1b9qwNHYQz+16yb0Hr1@LBJTE ziNExP-xUx2>er|4dxn&kgfTob5^*k;2%Qc-tDhyJWhQ^rW z2^d{agG8)rHMd!K)jSr>ZB&|=jTu6zZJ_bSAW7@965)pF5HhUsFn)^&1a$dBFkA{0 zNHTTPhk;-SWMHq?7)_IJr+Kmv45feH=T|G)X%9$ng6Fn{0esk5ehEdE6 zL*s5&qhOZo(wx@EuAdTV5V%FAP((N}aIGGeELuo9AbTk2!bV=lsUim+%);`sp}<;y z?j|%N7G6wWiw()-ePEH;AQU-{gFp3qzuyOKC+k2-e(!DwvVIrQN249%e{$b{(^$T- z{cZ2D;e{-xP2AOM^nY=!u9y758m`m*)x{9Hjnq8;(pL<&(j~uP5pi>(c*50=Dj6fj z=A@>9CVNhtN}M^!870JHseiG^pUg>rT6J16%5;oY9GesUgng8*g&N>$Wc-_UBI_n) zKaL1O{3e#1^&LRT+Nd|(=jkMG-(77qZ(8m02pPD^3Cw8Vxbx~HHt(rxF(ySpE z$B0Kd;>?8GxVz^!?z$8Sru8>MlXE95^+_8LjeMM&*J z)(O&1qyAI`6KSWtkanW@W2fPNM{AI}sqt-}Z~g)8hqwLFU^Hl>U)t%vYEGH%vp|nc);a+RANFH zAy48il%LrsD*L)b`n$H6oDiv#TOgiYt@cM7_eo79Ub+3Vi(uy8w<7Wpj2Q;^T{kc| zOaA@r^dwi}NUeSHpzPs)=qPF2`X8KHBtP?e9X>R24gYB03m*MhzjvHusBxtp9(j2( zvSc0i_u9Q{`4)oyxaa`Ugw{jzKzeYPw*zBjUyt8gpd#Ay%_o?AbzpXAYQF% z*HBrZLjkH2NKLkzIU|Ud)HRarrsjUPCT)okj!&?J(guFf1n=~Jbt_GmQBAf|v&2w2 za~wN}FZ_^*ZyrRn+T&KX%ylG-x9UhiKZ0*vJWbAh5lu@;AlFis7oWKD;FZ`K3%nF$ zZ4kR4#u#-vJN?v*dKd}aw}V3K>W7ql)>U59>7cah^*`hZeyhu`nh_ID0wN*=%nZ8v zTh2-i$Flz3c5{b+jLLiq-*a(YKdIxN|Iq)w<2L&b{ZFpr|A^T*rwxrr?!MRQR}8u> z#Y1fa8YCYbwZ=qNn)2!8fl|rd8)C7prT#fI+>r2(c9av}A~VvhAruBm;jv90Hv>f) zV&x8F^aLOLIYwhGRbqMbnh*F6RO)j}B#5h9+w*qS<(#g69@B-?IllR#W#;$n1cF&Q zqDH4n0k?FX;Qp)hNvPzUGxpXsy4dCJh9p4)*__r$hf*3JTRx!IY5ytN-uAX_AIHe>)&uLGj% z2s8{lq0M;ZJc@NL^Fj*cPACRq%z}qX(3`V6@(~q?5z>DydSgKdKI=Ovlp?9s%DE)G z>-WGqLc+im@42K!8(Hw4OUi!N_|hjO$NS8<8Xq2iF7?arn)P?B|DInEarjfO%Ikk> zEH)aQmN`!K+z-Zr1x(fxOT$n0X2S^hbZJ0|fJKzBXbB>*EE2h>@sXxjLeEQ=gf!M_ z7Exb$QBkfo68e#MQl%F#~GZzho4lL8URe(?dX;mg*!V zS*SCAmHNWe`Q}T`FF42kSoY!`9PVuIHp`hmk-C`Egh$QB0OgSQdR7DxVo5 z!pPI9A6==o_ulghdA|qdOv;IlAWg=x<$4OqzGEI6l)%a+UA+ z|8Q;Le*5}IuXBIRalZT2DE*$r>wfp$vl@EZ@vvOEwmwO3bEL|8d-LDcw_wC4NcQOd zzx#bmNsJ6H?{d*wFVT-Yb1L~p7Ae1Q;62N7 zd+d*e`(EFzzfX9Hyc0em#MMv`*J+1Gi@Ub$BbqI_+9)Ovq#G-r^`dsXfI4=6G#lg?uEXD`L~ypHs#if0%qUlUP) zGLkGb@t(tB5m~fjM8t$GpxUEH74!Bv&=pitZypOb0{erm}%rn6e(hb+xcFmRUfw z4==*79@frC9Z?~$gL&qro&E;>HpV8HI%p+?Qh$r$8HECOy1`F;CBtri_VSyXe)_uI z9trV#=KGD4Qt$j=qFe{*GnY@^=iN?Hr4sR5K>NY`P_!cELA;PfDS zXTa2*&X+|-YO3>_0gPs^PS0-1~){sW`LuBQs3VO$Y?Sp-D84((d@mn%f- z@ryt#am%M!@`X5E7sOX+aRSoFuRQe~d@fD{s^ZW>RzJPq$W^=mFOHh-`wjsBg# z{Jp2=W9vxy$!BkW9{oL^Bp-3RxBS9JX>s%n*7$;=m`5)5g<*>Ek8N%irrLQnYc6|Rbq-~PZtY{xub&5Y^?^v)PNyQ2|;D+H_;I@ zRlYt0pSoK5kJqKYz4}VOjZq`>AXLCaw;x01oo+wIcieq{@J##c!~5YZ-~5Dp?^pW6 zlfU;fedgG@`egaXe|6+EurV7o)f??+_V&!DnlXlCvJ@W+3||px7FJaHIjp2yIZn#sLh8W@;X$S#hC_ z??cUx5c*gXQvX-y3&an=5?xEhigKmJzM_#xVTe&#=N*@bn_i|G6p|ZZX|NOuH)`gB zuOO5E2j?{UB}h*HS|E4j2Kxfb&7ur;{XR!_$Is+ErW?QYQu2#$CI6LgS$Q-d((YND z(JcOdnF;FV(0w<62|*-v133qY5z%D~Ld{MzC>kG1DO3K7ZR}ykm-slLC^~=hB^(uJ zipvb8WC2|EGl-^j^AVBkZ*;t(!=c_7)qq;bD~xwPef_hqwd*^)x&J$EBB?bFfA)yA zzvC$R9aGUS9{r#8_-oJ9&u9F>`pz!j^TfY@UMtll#ZgE&^(|R22Bn4s!Fp32EPT>5 zStkYJrDxugvIyaztZ86tx}c^ONI$F^P4d(?c3HX}weyB;w1zEkP`&IRP9PUV59yL( z$biw`xn_w`xy07h3K;jzr5Z@glKm^hJ>T+qY6xziAa+r%*oYNZ0*@uG%H7D^g3t)6JXY*&6p98d&hU?P4p8>-JlEPmGOrponN{{LIA z`aWY;pUOOTKj-Bd;TFNqu@8@dYN zHd zXWv&>`(FOcIKAiGzcIIe_LLRT3&VFQW*dHB#WK~a^q^nwcAB`rxNGE=W7kE0NLadE z4hKtyTpgC_(#-T|kEoHJta=eC20=xP>!fEb^-k~T_b&Yfw|x11eg58W6cxop<`iMt}7aI=$MOHcM}wsdJ6H3p1ccBmL1x?i!aG30g%=Ic%?hR<-h} z5wFv(Zp@GuiMNil;Pm{_k@QYS3Zn!5Pd{k%-*epQUp;h*>m9fM>5+ARbyuRG-#XGS z9Q494SsIj(Vi_fQEVO`ENW@N#58ap-PmgprXJ>%jaWWY{<0bQp8t)Ia*})EB3QFifGsZ!Xy2@vK7)e%;LDMr zT0ZPR>bvk4zWfub^S?2FqMQHtGXwi)pT}>0(#1=jMqKhX@Z}!qGDIv1kRNHNCdGn9 zjzyw54(ZHkMP;S@XmDkvapCDH+n7OghBTo@8rhi@GsFc4F|j{+Z(*HE7yN38n&QjB z$;d=JHzZesSjVC|pEj}P2-$bKNS!E=IuVn6s!gJ9pUY?8a@YTV&5|d`nf?6p9E3l- z>bs8qb8p{!KmYU~{;m%F;v;HCVg}eL^r- z8&)erGe0a2P3$-s+nAhunhHoj5TIjUc3JJu|I*>#k}+XZL70i>mAB_`-2hPynX(orGC$c{LOK? z-otw?#rp>jdg}>0m={$>oAC=l>7qPtNlJA+8?Vij63i%GL$@CRBh76gqcR6mEsIYE zqCb2a1@3e1C#IXJi>xfWJ_hl*9z=z0*VPI~$-;s{_N+dC{SJar+?XJW1>s|j)@81V zj&EhMaO{LW1E_=d1suIpVe98#Tl%pxBa2-hF zLKqLWqBHubW=Ki%z+*Av5r_Wvh2+26J%8zsvHR5PXiJOlJ+=I0?*BU0ZGGvtz5Ja6 zVi{TTXz)*@&jHJx%lUFzs1HE?o3$$#Q3?%|s8pr<{*^ zP*f=At$%uw4ajeJ;m00A{cd|EzpW_`&ZMWUi8+p+_vp`jo1ePkD`s&|x~*&nnHLMs z@5O|J(@ix3CM}i|JC3G;N%o{;cko9vv#j>pD zH2;+xSaGG(8LX`n!`}KCJ4z$HhEuR z>_jq~w0KRAcT$hs-x--7CISTVvaC99%J<+rKqz7b1_OFdU%e(;QVIGm`eggXweyqr zi#&1=zT*i$bc5yVEq}+ur=ISAVi=9-gY)5+T(#WGAzu2WV(yHQ-kGF#-D5X>*EQx= z+J~TCL>jzJsOs{-NTXfA$6B;tuZgaK$7sL>DD6y$l*G(R#DcI4bUiGP=WEP~X>96& zJsXaOOQDEyWEam|zK$a;(ZM>;jKPOByRv9wY^Upic;vo{YQZrk0v_2&ZqquMP*qZh*8H*86%*D~MUs^}T`AM4rEu$V zFD~Lh|k#gwt{6`M*AGkvbOOH5vgK3B>P%VZh z0);ez9`b(y^gLR&@|IwD(7E*2$)wRfnDSb?0XckllOTRLf8d94e)C^>5L&#=x9r$W zVzl{j7;}F3+;DIF(5pE9t&Z`mAKVFtXN=7H(%C4#0e^CI@Rm>OSG%Jf?~KzgI~aMc z4AwbI9divtiI5r0v9}3g5w{WMXA@7dhCrDE;(9P2e>2!yYAj34KT+NueLaToOh-lx zL?}zaYyya#!kkh`3FKEoSgW_FVC=P1CInKK$;cvvBOhKL<0GBw@X`q3GBq;cgKcmfxHc!ffw7eL`QkLO=5M zNG>o({NI%`kQ=4=u_ow0I3|8yFMiAM7Ej&_K{{*oPxDUO_!PrAH8v5)*77vo%l$Jx z_2fEuteLIOesGD+zwPO7{NN|v?V)ct>iWJb`S)5}f9WON6U@^K+hJy%M_n{AK6WR| z+<&=Radm^zq5;aJqA;r{sO^iIymnAmLfGBdZ%kjBD1VU&rRPIX5E95KRTd#xWw=3= zo5~}3f+R(eweDAhsH$=_4`Y?4j_3g#2N}5(k}Sx2ge*@xnxh`_3@WD^P?go%b1ms8 zr#JP$O?v7#doGJ+f7w^uU>o&*aPYKvzJG6D?{7KtpZ)~I?BA2_)6V}S$}+}n<)1B1wdQJsCfXX=Fd1#{I9?_tx3 zP{B+iDzC0g7?mP6M9SFSqc_QzVU`C+X};L{8$slTqX~(;dzjNtxMB1LcSkHmj(uw_X}w zWJH8K<}<%P%F>`ro_rH8lNxNcoh5-x ziJnEhSt-Z5(2!5%lQ-gK9nvbzlJLnJ(J2+9I&QGDG_&dfVvFC8Vq{vX8WX`x5)q~Y zFDH9IADTVRD`C*lB_=rb#;(Pz{E>Gp2owqW>r6{8_C!=mD_7gGK)X^r@_$%hb=Z4T7PWkKvJSi(atf090s))TnrnNnuJPhAxC2s*+hz^PLio* zca<*G_-jtm(Z^e+jz8uk)n$qKD_eRLKBUQ(sx(D>Kb@9qbDON)spr=t+4@XD4RG&- z*s5pFXiDoUb*y5r@Ee$WP@*5er^q)ct61!YcdD4?3FM&pXY;k{I)5$J%iHFe-Ri$N zhMW5d>c67Ze;GXdfQak<_J94rHYPv!wC2-a_Ve9*a#4Q9)!X?tk^DPm+|IY3*z>2o z*4qBhcxWKzWfp0+tQ<2>IBAORt)%lv^Okxo(?Bm1lZ=MCk|hx$FjS&D=*tIgEDTr> z5iz4kODivQm=U@eFn?P||8yJ(v*o2maI`+`^)=Ms8q{f?LZ%IBVbVvG2%+<(Cj{-|$%!>2z1-$V8* zr(^5C$cPiqUSOwChL1a_)z<|(N8&Jg!9g>s?KGlgLR4-3VB^^r&&)?e{ zBNrVvOJ{m!EayFe*f)n7SuB{rXHoKQ)e)mpZ^mnQ@VYfd9LiA?HlD64GIk2&> z63!b=WSv|s-G3X1`Z5iaWIHWw_W{;q^a2Ns-UJ*lD4ZzFe;&q@En5J14}8aSER0|F zM(N&u#5}g2d0Xs>=kOH94O`rO_iJB@d!%3Taf_UU_+&vE3=s|C>COn>OJ=YXUGQlh=RRk)b2Nf-r1uM*Lbbs(LucVH)mQ-e7CJWFwg@X6P zMokT+O+XZsQg! z*3Wx(i{XkG>#s4l+$AJ(!hE#BJp>`^fFRCQ4LmxF-l=4US;IsR#F)WgZemPXXvf8_ z*i)72tZjc{Qm9_$LQ=*?jmQ9FOxi)Auc|PZ9e>BB9P(uZGgha8q3;$!AjZ*Mg-jQc z)*NvNV0Ruk{;q%q?P}^qFMhy&;`rP3>Nk5P{^A~P`DP#ee2merdB=ABC4b##{PLk| zZ@Z5*x(|ud6<#iD^gGTTrA;lqJhN zFE1v^GFRILP>Yc3>+M6IO*?PO%8-{fXnF4QZWtvoK|I7aPr-MLsQG1#=Iy@j$pQZr zx4rY`x15JI_VUcx{>0E9__uHRU`sQjFn{nT{B>&rQ&dDY$TFLjzTpQ2;h8sPB;8m8 zwEvKMD7sC;;zGsXLK?5@g%P8Lcz5YY(;;8Dw1^1uGOA#ti^?&KwIZUX9+VEtihzd6 zHVlPeb&^vxay(fEh)ZNOUo+6=RIt!$q)X~_3^fmQ3n>xMU&zTPpRX^u-TH+DiGN!h ziB`QE4;)GJ$RSOB&oA%(kS&Z+-k;Y`+X?e!zTzYEAAGDo@byW*<-SFG>7nPg?Ru^Z zlSvJ6Y{|Hg)<&^;cIP8dg=5H4fM6W+)TmO+IgI83g%2jmdKQRqDwY$<1fd$-xRG)~ zJcra`aZ;lX9XQ&g#3ZTMDJ68NIDZZJjOb=ysr@0l^PRaZVgr;za=vNvj+()Y1ya`u z2v_97D66A{qQC1CL!pX7C}lfHdw&noaOo)K zO*BoEXO5>x{=w$fJR*>Zg`Zbj1H2c=qAxs!4#n?zM~JEU#{I<)E92r1JECNIQ)eEToaH_H9>!FGRqM~4|BMfL+du8@CkE*g!#wlu3HgQd zV%qvz%s<{a5fV6Ok447IYs_SmNMD7!O99@W92F#J(5A7 z@wD45H|E8TWJT=l`ppP}{q;VK7;rky(`MWxw?j{aO3S!+u`_{&I zO~Mg3b3pr+{({T<@oxXbMgF?ym#JA!ea>=EQXvz&PB$b^kGd>+F@Fpg5iM7YrIVTK z<|+56=6l9_#|CX{W{G1ic1NpD$K$5-(9eayACsm-T~uvcsm%{7v7HY;1Qmba2ft%+ zSrmP8)&9VuKi1PH1_<*ZTHfk!tnp`D>eo2-=UjO3ksewu61B*U5H3cRa62es4udhR znPr5vZa3;+X4=gr5`TSDYa=f`vu3Q#@gF5Ew#*_PqZILl)z3%=k-tUUa|Fa3~@e}SKYrPc|l+UdXoLne_Ir0Yzi5BVV4 z8bn~aw44!XTM(o-BP8zM^j{zpw;A`{ZDNC#&GoQa8B$Rr6Ol#An9gTCQj2NOL{p3n zn$o+OSv2Wfd18fwf=JBsZE?Em6=>8V5j^Ml3+w^sxqrg>;}Y-TKSpd^^A#V&eQ5Jz zowR=SE&KpZK(W8r+dl0OCm#@K#EgiRb4{lK6CAo+uSMm`_fu7gN z4Ly(+6(jw;k_`D2P`|Z#Qi3sE#o9$AyP%uhnNVAcB(`p@qg}2M0tvQ$-npmbRk{|K z=ue?K@~v+U>-&GICw9oUPNw`P=ZyB5qyJpDP+xQ>Qhwt89=tJL=E2qAPX!5)>wcudAY%pz7DAVD zXVBr`!Jm#f`pCr~IH=a9V2|8*Rn94!8!h(><7Ox<^PYcv*uk*xhXaf+5t>1&D^KW| zqys?iEP1z~h{I`VGg(=z$tn{nOV8i;cy@Z^P}{$jE{eQ_Q1Jtim>uV8!Rza@mw~+`#FiY{I5mu$u9B zx4z6Gm=a+mvtW?TED@4&5vJ3wPcm4TP|YX#ED(Q>e6u;Wx0L4SsekG(UYf7AKH@AW zFKu3){JGa_@g{kj+nE1>QT_7Qa`h>H=7jd?n9cn?fkrw+)pH0mTHz{g=w zOhQrJO;yDP`Y?9>6xOGUFJx@i2@6?5Vai+|q?&}uV8Rl_h7e1aiL;YOl0_dDiCiku zMW%nT0@o6=Jp&5!BZrEj-wd>+kqgXg#N%O4Van8#8J$dvGsp655>+Dg1~N(Qi{4)k}R`Jy)HN7S`Fwz zqeKptqVcCe-ty6AoncBE-1qCq2kl+i<9&bH4EyJqUA-r-)6esX4!?42ed&_VJ;fMb74^`t^v9HXP$8U)ZBsnXgxoj9HVomSJomh+ED*-$jdPri?R2pAJgB&*KqJ?0^=qh^P5YRj-HNFb=3^vmWkvM2ugfjMhG8{&(5f&9iD`oK({ zWsitJpDTqNetpCw8i8a)RVTA9Y%A7$ooLHh-=jI7sV+o*v24m(%I3DYbu|xt*W#g=_V%_`WAM`8X@bFUZ|4dBSsV#$@FiDnegb zofl^-C?g0)J?nsW^T5%iV3MN^5>EdPcx!x*FkmX0D#411}Gd4}d~2(5wT-gqWaGTGW5Kykz%xmnpH=!J|4< zYX^1bc3m;;JvQr+^MxzDD0-13$N4&=^3&da>_PvE2md9vZQjP1GXBW}+@3*#_UJcn z>lgm~kG*)yJuZJ@dy^+lWYG`Q-*VbUxuFIZ0yZ^~Uu!Wyyw}8pNIytsJK@^*DD!BL zCPXDiBS@z#PnLg%QI0}=#En`a1|b5K&J2b5uGA>rWQ2M$UczRf_b)}NAMvU=3&FpS>gMN1clFW7mL*GAq=Rw>d zjh=i&Vv6!>Bi5Frzw%K&{Cmw8#^rOYeu=sLq9eX_z$3}L(4Wd)Ae5;Ga>x*m8nVeG@VK9=vm=n#ygWGB+pAS5oytMq z4pc~#mzp*E;M1t@glo;xj0FQZpu}_eU8_wRO{`|Pnr7S}6au{qLz9S;TT5o?${; zQS{Sa+cWMhXXP*Zss{(Jxw5kLJW>j6O^ZSKJSU1_teBu57P3IN`Ofj8%L3;l$|7Xm zC#Mnegd+A_H)tfCtzl5f=VlwnA7yqhTG_Y{hl_vV+!IgN_rf5|HfSjg#q5xlF35og zXU|Dvb{!+`tM#EGUKe>A5wnGSu9=N$eam|k$fY?y`jwQKau8uE+3>ZbfusAMBGZZG3 z0_x&pTnu!`j+v7W7W%zFc(K-UmceK;t}lNzX(}~SBO_sp2{km|4!nrAd?bRy?zqqg zscphwVwk~zNJC?JwJ@Ro(%-ypK#J1Zl0>hIlNk(5%qtYyI&rdMVqktH!MO8;pq=Zt zc>#G>Ow)=Aj5n~yS;Ifi(D;qd@e#X09Y3N!=ffA)GWU${GjrN;?Q-$kSvp_mPvU=^ z@b`$xTSl<9BC%(pGC>$-9HdJ(gerh&=7yu`myOx=2*f2>LXtVT^-@uFadjq*tk64~ zf_k;5J0aWJ^xe^==NbVsDDLMY&9Ml~jjl-yQFhHKLq0wl*O6(4Yj@t`Tn{`K&w8T} zjZ-}HS<2iB7k6JDWkw!d{5thO*`|MxYRoAQj9GHmA<{{Zgs6wY&gkX24}1dydA?@W zBp-xNo}=%%U;l{vediW{#5XtAj~H70GgkeX2cC{$C^>`CUT4AH3NS}l za3c^8o;Dfbd7Dcs7?q;yjY%fwx<-c#_m(?jKAXQUJ$acr)O)ov??AyIL0Er;OmLsb zv`eOmV@eEMaY7+UOwc52*D9qy^oG#QP>lKZcv*$b)j*tfl=WuasG;}K*4}zaHWii; zMQ^wP_5#)BJGzt35a|MMfH?T&-TK6#w0_CfCTMx{zU-+VbJ@RtV&kK)b5MW71m%WU zUf0O~l(QaF($GQZPR;jTxBGv}MAX=aBc2C6@s#9*B+66Ai3PiKF8S1ybD&9`fk-ww zM9hK6FhnvA3N*roq1dt!VzVd=654V%cDVsaPnd=#o;{WoxQ|YdpJMg2+kt%Na~mT0 zoB#Eef6_bDVzF?sagWvB1 zfYo_NKCGoc`ESpx!~MfkcYNm-ZGW7{D?ac2mZNq5I->{kvyK$BgB{3_;F)9@%5t>z zcg$;fUABy|{?VV-h2`cvvFeoRqC6YQ&e6}f4#0!PTk^*lgN_*vb^XEPV%M@5OzFU9 z!RT%U>B}f~ z(g)jyQ>ugT6}%TL7Skn(D&s5`lvD#NQkxgb6ff28NYlt!sB=Z{%sqipxUhp395j%TRLdL*4QIuxWjS(fY9!YjamCk=f@S&R^l4ZR`zL!Ow zmA1Jrmha>WW@nJfy3wsMtj(-9vt;yftKX^hs-)ZKd!DG=Y#h#OwcX^Cqiqa|elz3t z5_bJ&@nN4v2cubU9io-+H|ZQZjbuu>GiZb5`0w*x30_+zkg0fF1Mm*e z0i7r!U^@a&@eW2jCkiB9kHzOTf%ZPM*ZM-h1$ubSq*sHU>L=uBY)WvHQv zC*UZn17twg3QjPpz)`LSv|?*4|AzN{1N4Y$n77c2z!!gh3ACaY0UbDKU`S7bKB7kn zFN8%f5#u}11oIq9%zY5b9^y5{_DJM)wR607B%EL?K_Es0fF1xYu!z6}Yu@PyI>+mo z0#gYNMAAk;0!0C~vKO6~f?9ZPX2!q-J_WrX2L&@Y6?okTBK3)a1dbj!${yqMj)MF# zo=$-MzhQrVQqKcldV8Qs4+XqD#>4NTx;WtY-bP>v*8!5~D@E3D!{2uzAc{^U;=oF< z6%T+4(!m+Ellc1$P)QCLhdwHBg|h)#&>A2|>H}L^BOoaa5&v~6!BBPql#>%ci!0`e z+jxJC?Y~yw3wMm3ymtu+R2biOw01lI?l2lXo@aj*8Q}X#N-)9fcRHe#N4U&84v;0W zR-M<)ND_Xf7Q*u#^k{Y7qRl2T^qO8!iBEKkALl zg|WbCLG|2Mp6uKu#L}ICFC`w~HkTOD)U#ca0Vl(<)baTHw@+4;-Z z1=iI?8@TBACVIm7r2*9@%4Pj7=QU!=yz$z{fUhD`DNV zH}FJnJRpMP@oZNlv%P}wvWUa60)r%r*>1tjP#HLuyW&zoRB;x`3dAmng{*&Itw%<# zNaNEAwe}#|RUr1!ERzwrWYKPy@NSXCAyvZglM2|`Y)mDL;_KbHE3jM_Y|&gB z6j9M5)Lj9`@3_Xd93v8%jdFjJFU~-cHz0`%mXr4s4V8TKbEqe85_9ckTd#{OE4zIK z#x`N$v!7*E$zlx5H$+}$mBd`3hkVhd`O92T6{p1DE)b^~D_~gKEoX#fgVKr8iBDRNs{uWC86dhVv%|$z7NT?bON)e&(9Js0h)+ zGs>Zq6@iER5LR&)M6zEhh&_2lgMLtMlNAJ_2QuG-tO%Z%=v^wptFRj+l?aN!D2fWY zJKmR~I&x?Ysk$qa5P95hhu*a@fOV8>Ux5B0-b$kEAbs2x$A61no z+WStUw%_l6tjK?!xHCg;Iek?y(^fn*aNbrf^<+iJ=e&#)c8Ox@N#r<51Jp6arJ`EJ z8>*OTw_{q-g+zJb4O%cpV>T>{!=B<4jg-;35B@E)$#s zt0*xMjezS>d6ZQkxbsF`r>^q0uCS?r0`@O=T}C*qE){?Es!DK){j+IPL(|^hxh`Xm z`#8SQ<9^fBByaDRuFI^)eH>re-gok*reS;k>Qc0f4e74n=29Xnl1a{RZ8D^IYu8={ zF;#9};##U)7jDiQ+)#gct^P{ozm_xDCR4d>*K$fMj%#e;Z`(PU({?UbjL0~~cjZgo zV*|5k-%o$@Lk4SIE)0|_;|#TPW(EW^Bi6~nY-}uhchS&ouK(HcnoFVk>~|M`=d*ke z|C4^a0dd~`ML!?+!8kvSMJ8VOJ+FOWGbd-4N>3`TD;YHZy8oy{4duD7eMq(1(ClU)1Dfm3`vOX5U!kkC}@AzL%s`>lM^()h59-+`KnzBWwldy?yo)Baxb=QErREv}_ov@Hng z4xvDt!4*vk)gdEwVAGwiO)p;0)AWfg_&n zNGMgL82?q@Ca6!a9dBTwZin!yNI1)?)~>MsCNs?5vxl8yiY=~_i5ZBPvK4QP#V_qK zic6)dsse~Bl5R!bu7`%5!}a+-w-X)ZM*DwJhAN7>n}Q5--)h^|V*_^uFRqKS3UjVG z3Yr+#mB+hU0n7|}<+?&!Q$haY9*%7oREmCYr6|iy2bO|BU_e9_J1A2=352@Ra;JnCdzqmhPmiw!tC0Mn8JT3 z4qs~zScLjdgnyI^%`W+~T>h`>!}cw)-)2SDdTj7jWb6{I*9-T|6a?bzu88is@aixh}jpgA)7G&X;y<{#}1i)|yx4p^j(KV+5Cu`BBNdJfa=WM2{5w4GhsUS{sruKQr5F z(%O%IpZ8`vrtSKotL>On8K=bL+6UH+>rNK0zWQgmE`xDiU8^Y&qa81aHp!cV`-gTN zOwcAAJnml=DZFc?<2p&~cD{tfdf2UKXS10>@P2Y~SH$@@_g(yL2*k9~Q@nrnTo;C@ zYo{UV&4q#X1ZPXMqX*yZ8&nmPTX7~n%HySa&D%GB=gZ*E)u15ffj^*OKxnrK$qg;jM0HU^B z>bmHMG1_wT5X%91lp9nP#@BxzCy#RK(f_C|cX{;R zXv?i0{ePB2yAwuE^Q|{hw9j;Pu2?HOHNN`^y()ors>L636rXt`v@L&+_VDzpyoNU4 zclqa>7VNA1I>-98^AFw8Z+|1aDuHdXTDg9A-Uz|+AK{~X#W7)E}O!BtfUuEsK?ys^Xw&S~O(=?bngZs`3l2wH~ zZQ2ZdGq`3gM1ED&j1@G2_;9~%M^~V)O6SoouR)mOcm?Eh%}ZSdM04&QQ55K@23XTe z=%L9+Y5sYwAs+B~y`xuEJRE@%T<6SyYR((5hr1o1I$e>^2?&3yLP2uWv5fNAEdUfL z0+*FYml3;?1J9DrvT<;&ZDsOkmcLSTX^H@H(~lm{0R+Zw5zlsuONi&pNUG83 z3PQgU90J!f7LY)|#DC76$32i3Khg!gVg62)lG~&+!1F;2tm541yeZd$blQ2Dj4qI2 z4gl5hIUhiS(;I){WO~p0FOU0956^?WNQ-qkTgiAMzMaU_1k`46ZEf^ z-}fo<9bCSzbv&2({#~8{Es61c-vCUP`@V1fcHf7+AoBZq47$*VNF#5KAO$i6i(^Rb z0H%aBzF8_U5Bi926ti0J$uY(YbPgW+optmY5e8-isUm+zFjH7bkYIR_wAT@=B-!Hs zF4o72$2A?GEB?Rmy9~$s$L~t~T^GwIf?Tnj5coR`7C?4jYVaZOK7$!v8xrP<_;|er zQes~5N|6dof%Vv639P3Av%q$n*shf%8sg4ZiZs}-g#d}a?_zt<&yD?L6+RbX|6(8; zU>aU)*e-t;Y5K7qAO-e&4I~3oVL!M3%g-4JQe`w?d3b%p_yEW8pabv(ykWc6NQPo2 zSYH?WwNj+SezpCf=%gp@xdkr9S@1a=XXar!g?Oe^KHlRJzRGWb7Wg|2zHbj;2vqxe zFCOB5DWD@|_iWeU(JnFtv%TKob)v`)-RJFix7UAgebf&lq+nhv(#Q6!1j(@7OCTqF zUmpP1R!%Tv@S9_@U{Q~7Mq6yV3$^tnEL4>WR@qPdwzXe(Hk=<^P|z4`kc_cKi#xpCP~N3LgDh zW4*21Jj%80Vhp0s=Kx*-%7>g-0#dxDBXEBex#RD&a%8pry<>YG?Ns36`>{Blw{pXN z2a5~xdQ7$TqkKQ+L-kWWTtz0RSEfJW%bkJu8DIYUAI1UO5L+81 zo8mfQAVoDvF|T&fz9I!FPysfc!%9<>-y=|rE93ZUIL^1{d}uIl#0ThNKCX}PX1#yF ze!PLMU?K=ZzJWjzs|6Vd;^2Il0`XBkrx@?X%t%S5*oYbCksB#CW&uVCu_7rL+5=bE za>OtbL`o=FqwL<*cyTXu@p9j z5iZ#l8)XZ&hcLy)9U&}1Vv7Xr>iMy9X z%!)-^Yl&FzQp-QC%=as?j=QzV^u;{L$UG?9|IOWPZ8BroGgN9ab7dNrWFLQIU-cIr zT^%C&VHVb^*F7eE8{5$=&5@Zt+E%m1}mrXTBs#4^_G|H zKiaRWOSAj8Wy{>1N%S|%#w~Im<<^IfHkHyXjs3T5HZ7bW{hqGimUQVl=xg#w(B%9HC zw)yCsPK$0Y&5LL-y$yZQym@=I(9f@X!NS34I+OOajVtj2%Iar}DiO_#@0$C=&B za&GP3R)v;Tesvbt-8t4+W;7r5rW?=JM>}(*cFa1pHrYD6>lFN>6Bdnj)~70};%Zza zY0|Sk3Qr4EjI_lTWM*XvQrZMaJa{p&CBTYwr+~u)$jG}X##U6G2Z9#o0Xrd zV{>V4Y--Oo2kU>me;J6o0dE28#by|*Vlx?bJLl#8Wiq^lr+mKc*!IdUUKg*lKODJJ z-MjG7WIE`4Ox`xVdd$-q+3#M)mrkyydANmvck7*wt7|V*zElfoe{2JDaoF`=m-mK~ zXcXjX-Wg67otNJG=*_We`8tL7^my7%{jv9^?aDiU>D_c7CtLF;vA^W8OmZHkIg%bHBmMbkvr?KWp4Gmpin;^q2w z&>-)0JHvmM-pN?Yms{?cE@nM9NvG3$pIYf=XwGlJ$9*vX+T6Nt;2Fb-byH$O?v`4U z@$2i$c(XOEU>myk%6d7zskiyZ$9@#(SJG()o1svz{o|`~z*DQ&?@BTmyu4Et)RUbW z3oYAgOBhdl+&g<-UdkYWe?V;;kir%=hUj|OUHr_iO%ky@dGNe9j z?uoJ6*27Kb<;C8byM3Kr>FJhjPA}3Lboq{`hl-U>Qa(wSG*={A}@dUbT@qQ5XP;-1 z?}kg=ng$0drq&@doR7%THp}JpT1CyKJM0)-bN7`|4F|>kw#;5;+_^PiX^rEggL8i> z^&p33B{su@b#D5B(O!9)_!z$I)`d`^raO-2$wf%_*YVrn90#`56#UJM|)3Yjd zUPiH>%OKZJ)O#DG$N6b>-Em<@!?HAw5ajD=%J@*cMz?)%b+;{pjE6<#ZJ~n@*SA-%R=xdtZIi!Qsj`xebcm9~aynl4dOf8b?>g1&& zk;8BaTsy1bNG)0K z_2uQQK3Q_UU#<9l`?letuZn-k}6;WS-sc7Y>R z5*WJ~h>wk2?3Vnc-xYt=>pPPJw?5zdb`g(7Kbl|kX41s*yqRALZFCHbmtOy_yS_Y*evE+Q1(DLSrV4%o3)&Ez@44yFavva ze4WpXczH<{`l<08@i9z_-sm)(DPMR&(Tw`%*=Rd3%We{N%X2ZA1S`qn%WD1ht|sRF zI_LYt+BqKX`Tc)A`Uu26J&5xr9W2br@pkOOnz&(f@LwBw7@ig%+hDmEuCF`Wp1-@D z`p)i~4?(h(`*9n~EMB;sHD5tvk6xGItrAgpKhbq7UK_=Fdu&c!c57zM#3+Uu^sb|7 zY3OczBqL%J1GaoSbdGOHr(RCue6fEwtzMnKkGYp6KyzZoWU+t&Y zG%OD=F5lnZlT?4#7yaqm*xHGDbJ%40WXf464Yyn~*zMk7(4Udf(UkQzU+jB9#$Fd) zAxiN5@?%rDIT<`I2MAEGvOl(Q) zGff(g7rTEW*OQ?m7Wut(08zIXE9n$&89xymVJMfl6XW$w308eCpYhJAM2XrFUouM zcdw(^^k?&EKjMAf%;&@X^fg0n*=jKnfm7dm)_H33K=13NQ@KOZqf_N|uKOg(vcdb+ z+y#VWS~5JXQu&s>_I9IpyGy6AjI^->*-~7P8@kbU-T8YA+cP_i=N&h=m8-&j7h$$i zNfm!bqggM{cY8VD&aQkNcIm|=`+;Y?^!eG`%}%9rN>1Iu)V8j#y>0s9zF+4eX69R%ne+P9 z{-GsHwyXNN*3fvC6ssr2nnbxj!`Ak`_TGPBd^wCX=x05=dcRkpZfNMY;h3*|?YO?XCUv!t8-&J$S$D@{DjH35)WJ_sq*BtPF=tCLc`sa1sM~EdES8A)tSV zxEB}w43d+LT-QaKP8m~DctI-!Fn6Z6Hm#pD?M>z zoV`;-6xP3Naw&Fq^$F@3_^sb&KP!J+3#WAhgfCu)%HOCt$k!Ed0I$*cv$uM6v(Y5Y zYtlh~m=DcVK<7I?KvEZL!q&SVbM=O)XRTiw8P8p9WAhW{7%>tnDez>10#^hA z&EYOpEI)#gQOHqP*6-2Foay#L4LWW-?WZp9TOA3$ud>J;0t7Z_UG)Re$Phs*zuH)I zQnzOm(>$qXLL_nihu<-fKFTJ9b$TAL{~G zA+s-~u}i&E(X`$p&!0@e7~|RGGHvd2S(BAz;BZ6Z$?O7&Nb-lFLGgMmmueQZ5+|m- z@{~y!RcxOEi&P1ME}?%fFPuGTlbq?rEiI=a=YcX#%~?h-l}ex9kp(X|%lL>Yp}Qh0 z8QP7Y-E?!At50oH-((llgkw)*MFcB3qry=6YQ0=sexcvJ=eggSUKRwtX~vga2-^>H zKw_}suleT;r`&FBUCz#f6fn$FWxZls8P6H`u;yXHs5L)=kDg_&}64o9~MH5~`klt9!(O_c%M@imh zSdO>C5$YhP<^XA!+rY$?Zy2{Nn>82cEIl5>FfXqP9Q%A#g%+ExYQT1Z${EBJC~EhQ z*xJQA!uOY49GK$_B3+2&i6{gURJ?1%6p zM1*|{;Q`sg;^9JvjY9j^(T#S|71}w(v1H!M^B#hVRIc5sLYNVJYfgS@UqRp3$c4eD zRrCs`1qtUoQ723z``;{Be0Q^?HUh_?@XF_9|dUP56y~ zVX)%J>j2@r7eA!)t>hAHWCz^+3jwyOLdpv zON>%wRQR<9%y83H39ANq3X%F{id+?nFE||lzYz9w;QNPKoM)M3QOU;T&ZRf^r1%Hk zJSX7YH>wK`=h<(3GCF(uy=X~EtlX^F`0z21Jiu}C8L3$MNl2(U~$ z4i`JR-L7`GZdsjw_RpeMO2$z5W0bBGSCA~QXF)YHf)Jhe#5etw?88_iq(C(LgHB-v zdmdwcUKA>tk~LNl@k<;H*T6b~rwy}ncnA2E9@{+Pz07+jIG0oN*x zZFSOh&z#71yR2dGFdWq1-OA9ed)4$oI|Q3DDwteW0x;v>h#-!HO`nv(;$#NXM6}}B zhN*gany~mxXNfN)3AceDl7m)Z%eOMQTo6UCY7<#h8m-wi7=?J$05cPHv$T=%9jJfj zTx^%(o(ghJjM|S=$Xw!F?QQd3X+zoBQzBcFA!pPRus^z6Pk4iaoGArO*#yE&5c`ni zo~N*hJ~c44fjuYokmSb_9(DAS@213|6o%vJ;=Izfno8_E?8azWmR=|$*9oxL?IjW^ zK~!o9?QTARsupPkt-Kwx)z9S@4Kjb}&yK>9l!`xYMXblMyICO(*-qkjHeE_oT44_l zjpji%x&h}L7*+V4Lqi-X;8v^vu{^tnpGT>=rwTMQT|($ zQY~ye<7OL{uLi%q6thpq*slJ^TFOFvc+KHL#si$+iD7v7d#miKi<*on*X4gAVY>!f z;n3)H2F?L*8LN0Mm?aiP!C>yvy01~196R4hPs)AcF&C|lmB~1Y_NI%fc)ATqkDSO& zz&(x@ytA)sjlsmA_Ym-?U$6i`yNCGo=SSKc5{4TvyrtK36yxiXs~U=3^gadvSE+OE zBBGr~BHLi$8vFnq5(U6t*6k;bi^5)j5k7yS2sxFj5@y^zKVkIQ?7xFSt1H1n0|L}DR1FL`B|PBOksY6&gF#YPE8?C+j? zREfE{=CJG#^zCKpE^~jK#s6%9R5u=bc8%P`fod=)m&1NYJ!RVW7t>(ULZlXH2_8}m zThi%@=SwF0_=X)0L}`WerdAE998XWUt)P1oUOp`}Lf+M?WZk!FFM5UijM~iLxAa@r zch(T)Q(B9@=d?%`T-^Y2x{U=T==|zVo_+7swGq^UETCMssHcAc9+#q#IcV(B4ss3P zfObnH#UHt_!{>(ABg5%O@^Wvh|NOPM`OFp;39)T70PoZgM6{laSq%r?K~C)ZQVw(&>gdH}N*u7nWj% zSgtM9BjlY9u-Sii{=4L?63b870}7->ka=5l zLY>G(*p0xRov^!MZ1y_M_(NM*i;R&)kswoKM^nK`xo-REbO7rt5Epc4_7z5NYxjP* zT7CAhh7pqzn&@KM5Sg*=WxHMQdf7O!(@dQmp%LFSbKWvh|R zfX-{jKo5T*i~dCMYtgL3OTS(%B%3t4)RLl;BPJS<@k5}%82UUjQ=Ig?*Br19adM0+ z`5zhXEU>+nA2eNM(GQTII2SZAX5h36Jy})%*gQeV3Z^L7O{Jx&-Unf3a&Ht38|gkL zU6@`Wy!bpAU;mSU#OIo+HYTQA6_Y?o7>-@BQX_vI=s|S2W7xGA8^Srh0rdqfdT+wH8-QN7o$5<9)8&Cm3NV86Mo44QdJXdNOWscha0F%1~m81*?H+S5f~ z1Wc@>@DGZZll0MBcx;f&56mMpO*@%h*7P_9>l4jtWk9#mi2yf}v)wvq@Ne<^SWfFa5W?6IVZlFXm%I{pIf%YW5o0b#NOOxs3%T&Sy zyab`6u+2IG0HE97+Ws_iCCr*HP+2JIBR9OOr{@hk-e<)G&QbgviZ49Z3!DTaTr zVx#HmgQ?we*>sF&n~cdHo9-v8SIHomQ@AS4ntZl;GYtv~?Zuu9$`Gz7oT!hKuPtLa z8`kD2F`gQ<&YoEZBJ_Y=?_sco`D5t?x!>zeu|59W;bjN-fbGT>?*-)F$Gk@$?3^7W z3Z>q6ojtTuJ#h``na~!Q3|@{nd1-%)NI+!WSfO>S-7YUeA?l^2!6K9HqIYH$dF3I(KfF+ z){TsaO}QFzwSrHKc^e8Mju(CaH!Ex>>x!UGbgBU!yCP1D2v~w+5C9M+pV?ni%=dA= zosL;EdxKss`z!_W0?F94p=As+&n%vTeVig(lFDQ{&Jjy!*oXQzeI@)m7J zhB3WRy|?vLQL`NF;xxuFnH+OqekN0oP7HiOO!o^yat|0oyw*ZDPUo?LGM%0(z>d-FO)j&l3vZG@UTXR<{Ziz(y zO$6t|v%}`{Yo@yyChdFx_ZEF0#Xoyxth23~)2p5Tx17V8ZM&h!rO zr>Gx6S3p8{VK?QECuDIf1XN~m*W6W~fTdlpenD#EVOLS>7U?7UW?{p9VuU^EwFCZ` ze4r!XqoZ{GcklFQh-zH4I9D)@oK9~xSn|yTv(1o&< zx*Moyr8-!l>LS6ZQ8V)b%Om*RWY&vMry9BQ=I{G(4LdpTfMBPM$g+;MI zw`?h)N_HXS0{&f;N1!-SOZx^P@K&6u5#66*qZ@gr*sy;W4pvQ^S(hOTrlt}1tE3CK z%;O^^U?5O;;he`GKz{x{Go=v?stCT1B)*C~5rd-lzN@w;BvLHRw*0K-fwkFiklbJ4 zcZL6>Tg8Wh`3WD-=zsX>&x0I77}+ zxBS~$^T%ly`18+dNeR&Q4CM5O2LjX6HJyaTqw&^E7>AHtK%oc%?y(*57TOC)4OR?0 zU)$p^RQ7}zQP!Ifqm1%}RC2dfxJDwHZg2d`UF?58z4shIp?QZWI`A?I1)|R-+-RYv zwT!4C2Dd_Fu*Br_H5-NMgGXZVicGvtq74_S7Up(;u|9g_-k4hO7&Y7d5*P?lke7*_ zNrAR%nZ?9m%j{yqZ50O`vqIZ-pP4Wk58yWW7$5r#(L=#iT8CX4rGvm9BC1kP7baw4 z8ytVSKEYW$x|Y(3KzIW`{Sm%w^;;!XbmB%}?uKzW3UpXO`fb4$VxjK>%#E7|=~-H! zt$aA|tNdWlG>SP%PxE1=DY|2w;L?#aGhp6I=`1_XTG45Zvorf^dItKY{ds=Hn1Zfk895vh8X4X5zTCXK_?C<6Ss) z^~&GDu&(BaI}OYEA8wR7U8Q7vT@+v{=b^?Ikv3WFUDjl|h?z^dWU@RWr+^a1WqRPI zXs@6|Tg<4ME^A*xpu^Q$dd^o#mVw{9_kk;+@YrMXBO1H0r|R7t*7S%~a7TaDs1yTI zu)FGnZ>cW@`=Xq$9RmnWmLrvtN|tp8>K_UU2ev(G_q_-RKon&B6_)lN@+D7}QR#qC zGG+`~5&RAx)yi10LTbWBVqY5-*8}-ErE`3=$1b0q z)*OlvqyT;6`}BR9mFQ5_pPGMyd(x2GTVz+)75&Y$J8q>p{}2}R4Khzw@kGC`15968 zuSA}LjO5idNmcKshxhm+2O^8OfJRx=J=qE%ht9x0?og==d!xMnMXfQZS;wZZ>iFp# zg+tr_Kf%SIXhNplpJnAsV@f7Sq~M`wSQHk~oU_B3Z?_LKo-8{5ZUlc-Y^z&`5>)7_vaW~CwHWMj2tW1s zs-t*`g)vv|*a7G7d5t37LkI{)YpZ`vNCU&ivrwak`Bk%4fs@yWI+?~qm3?vL-oR{bSpG_HUJGt7)yfKDxozPRRbwF9;Ni`= z*rjaBtDbqZ>JWRX!*xY$QzkAp-%Pb_f+YCK@ti1Dpky~0q~m+o?pD$xAAuMRhJ3kZ z%FK;D+adAH0}t3&GRDAv;iY>TB4@TX@A#E5doD0&AzQDD8=56CCTat$o$*nxJgAWO zn*6Zu!FSB?$6;L5K-L511j*kq5O!@_KkW3|pE0!6MVSJHr>xWxLf?7`$Fp?tqd+U9 z53qA6wWmU;N|7*iOml#6`);3@xY)FDsIZ!4LvVc&a1AWk4KIyGS^Qf4$0_&>hRD zbx}fvG*X2(tmn;tD+-XNeTGqsDbFmYzUs%$WFYeP9bqU7X)PPV>|1pMw=dv04rVFJ zUIw*>7M-&sl}&O;ZfNHlW)O-L%^C;#dB3_%xHfNJ2A5@BtG6F&tW*PPYp0q@C>K(I z=x`i?U)VM-7+kIzSdgX?)f?M^+P|*qistP!(2rk-pPAQxH=lF~`nJ=Ory3anM_Ef% zo8q!pql!bf%K~-T)Q}2pVnGNm&)R5hgc@peT0>XtPRlv#R`o$hYJjpKICObKcm zFp(zEN<>P3GG`w3i`UF54kE^xg}=_>DK=yr^9Epgv6(ei-Y!MgB`QD-xzQ`IvRmOA2J zLcmL0P!W?zs(!^HV$MJQdns8^HO;FlUSPKB;U~At8s%4*`({LUhJf`(J50fVdDF3j#Y34QT|DCW zqg=(r&Lm0G$nu)@*Ku|v9_5cRy1r z>3<7(g@QGwl7o;=V{ir*W{ zs&`K%wBIw#$X@uVsA8dggsM+h!I9|cqb&U=>-vS~~Q!SJkG1zX2|OH7Pz z=n}xp@>i0I8fpXcZ51_=kYsl9MQDzaSfOVt$VrOg{5n|eTHA-EAUB)d?^qOyE3tpQ z2xxo=i|B(Wqx!Epm1TLAlDG6VIinN!RyhrRWUAmwr71Q3)p2zGOde3H)rCIRWddQ0 zy=cJ#9bJGk+MKa4QJJfyhkr+Z-pC2rWma)02pamKF&K+EMYwjYd?Xd(_vt2&&cf15V^PIPCWnTb{Aqxe+DX znS+Ko&0^L;pu7O#wV9LEzYV7kbs}b!_TDZdFiYAL=h911e(yPdLoUS27(>Kkv>ks5 z-<4msJE|K@@`~){&q#g+lV@*D^;{mlbXRDJ=Im>2#st2+n)+#AnM z%xOGP767)iNhn2s7BH)n^aBw2Kqy$T!|riU)bFk$v;^)}qJXc|gJY^3s;-ktDHZOO z*r)Nr(KU6;+1OGi)PxJ3be|C7@!Dokfz?QfGOGgB^oLVyvpu*?Hw?3F<9M+(mC}S6 zy_O6Gr?TSrhpmj~m*W`Jem|Y|iy;{YT#LtCyozTefk{$-3sk(ja(Y9VzXJ*sFi#}* zOr_fI<#Tj`3cMH8`QJ(r)S>V$j zBW1i%glJxc13U}Y0lsk;x7>H`-Ry2_b8XF>2zEpTGCYu0xpu{iM{#Ue zY8Fh5+$cJKxY?*~T}#8^a*;1!-Tr0G&C~q0+toGrGAq!GD45ZRA^xans2xxMZI$>K+AQ8^AM_o8!2_|Z0{TBn>KLpf(- z0pgu(`(Ry=s`%ZYG-ox~;>PxdeAd>1_5iW-Lr{XO^j}aUF?aPTu@HL7P(hQd5Gk9+ z;K(lqn7{`Ir$FWp=fNT}z3-I8%S9?eRaKsUr95@T;=tZBoE1%S1|osr&wQ+!f4j?PtW;X_%8aiV?_858n!ge$%N*KFpIg*bHP+Op-o!pKPn*&Hu<+zh3;eM)+> zY1&kZr+X5CEfrqcawyy(QCD8Z_aZ56rI8~N$xQGN(}C$9BM27>Xy#9TEloIcRit3x zox}%dU(g5r?^-U`#5TFuh=d^6Uk7J@L@OUJPsLX7W5-m}yTuT!N(vqfz+kIFehv@J zDWYjm>xBAjlbNJ{9wotQ3AhC7#X~Z&7$^`?1{w6Uhp!I_-f|+N?qtuYa`jl+LJq*c>8vX+7Ep;@QVcoi{y84jt`_0EPK?Dzq5FKEn?$0{O!v;B$& zi?%An`$^c?kCd$~q^#5NDFgt@%C+CfS4IvxZE2+pCvXaj>cX&lo|_+t<#Cl$NYbwV zGn+|rJ&ZW-^r3@@3KJqsS1k05NXaj9JqJ!ZkIN~&M&8EkW7qS_ar}$7@wCwq)gj@~ zOptWy&5t@0h#W(shYPulp`gisu#K;0#;V~+WW@<9K3sx zf$b|Fumu?d$ECF)ZgAbF2Q;}0rf-EN70D3Q@`kW4%fU6=Z2vXW2h~UL&^3|_FMerQ z*2HBJwl_l&^Zm$sW=e|}t<#Fo7UZ}QX-CHKkX^C#Y}>sD1-QXAUJOs1Pph|?3MABAAW%7GljvXv$E9oM8aDv-%fO$BOTRzQNg?Z7{Z`% zJyAqYHrkvx*|u3uyXPg2GzslqXMxwlg}C-M5wd2wRqkK{+f>l>DM7SuR;`)@St53+-^I-@y^^R;z3M z&etv099a9n69nF!dojW{+V26SE>~PI_v*{1>tl47dkwhCAa?(~f~zB=kx^GMv9BID zj;3(Q_|(Mu89SGM5F)&hDOp&awvnh_b4}KEG~ui6nzI#wc0~ERm9)tSEfDrg6g@@b zJ(BR4_UD+n(x;MDZEnUaEvv?wtt6mgA2j*Bf+RU!JG=w+@AwI4sR8J4e`7*DursuOJNCM%p~xfOvQqb(-MyQg z*rB3As2)`B8^znXX#Sk`AbFFr9nX<`MrAF&W*WO)l!N#9)FUdj0ViGyB@%izuO~IB zmk^eD^IGn1m42sf5%_l~99w!oP-M>iF~Y+aarHu@&-~Ke0k;eLkDCBzwPyne{Q;j; zEVe?9Pi(V)_cT#w3uMD+ESjB{-0KtjzJF;S-j2N9a>4;zYNOETA#y{lf+8*O#Pd)! z)6V+&0YwK_?W#%n)3gE56&?fdb5*vzDiH-KcY{|loUX#He?GZ0Y2gE6afa;e_Y3D@ zp13fWkD-4n>#*HwUj3BWGeHt=@@!w#@RFlWmIx*D@a zX)1RgNPkrb?cDjy2_3Lp{rF0Eo)551kVa5SIvxbkG&M%gYfPF)n^)0yvt_;W&;y`TogFMmHiIlg>j6;j>r@`f;eCU@{Ddb z+lwN9i*WXo7^Uuvhvh7Ut5F^sasv}My!;Dp6THnb=8FuH;Gi=m> zmbcwwLD3Vo7z_8jkCzUJOPdnMR(YdWE4d|_()BZ2rvXwXJYOxWY^w66XL^mgnbxFQcC(phoY`m57_CJh&gP zT<;xB;o(XfAPJ){$~ly@!bdxlxcd}^C76L|toRO}L_ixS=R^^>8?F+X$lE_WvckT9 z)|*k@6P!X@pbn#T(^8ws>O>S>%;ZFWBZTob@sJ3%NP$WN=H4u#yRc*DUN{kQ$F9;l z$n)$(Z_zEtGPA$0tU*o)h$tRg3-@m|3~pGji1Ju3UfkUZ4kn~X(7*Zw7N*Hyw=5a9%yMN7zk&)XiY$45(h(QY zMWGCUXe(5vMsE5&Tr&P>=Con4}zz(4Zoildbhz{ai^IrDBP?m;F zVVyP31w|jZ_CUtkg01HVyxUJ(oSdPMOU;-R=<3@2z8>oFqD z5?ft<B^h4VyysP-Wrj{+sjF;bEqF5n7(hdekX=iH4p zUcSPB<+={O+~@ba-*XwQbh-w*D42_pRJDu4?^~JBhR2tr6;b6nEV~>U=-p;9;~)5K4T(Zs+PW_m0lidy4=8< zheT3Vfqi4JYGV{zBB{^eRrQ^f`6nd$yDKt)4WP}pDyj$8Op~--qkExQoZdq}4{K|l z4hZ}StOof~fJxMUwQHX~I@h|m`g~{Xs};97Y{lu*(9+9-)#(Qy`uWtk58UkYlLulT zhknZWD`d?Q;1c2D(&xHnS~E)HgiGF4@;snNgj$WY=Qfw=SFMv};>G1F>KJS(<*oyCvHF&!p21P% zGga!%QQ2fviKR&@Q5azPhm1sA^rBbki%PTFoM=h6O7hJ`EF^x1msrpgEC@pdQp<$2&OFGA? z4@7rC+z7gQcl(^3@4AK~m6u->XB6NMSrYPk)>4 z7yU%ya4!)SFAxU}C1lO%< zFM*mYDdQJ)@T+}HqrfY znGw1g+BI+Qf|ny6!YdIvB45Mh zJ!o%1bv6Co${%;)1E_6^(N;+Y^csHMkzCQ9tGT>LFWUt9+7WJjH;X-|iC`c+^2`Bo znM5Pl5#U7vy8QiGJ|vqMp+>JsxA=H&DZ(nR_<_SC`{~=s?Rn{q~rIuIwN3DEOi~n$z;}x-?k-a&b1|nZ4X`^yLvV9Aoqu)-!@{jy zqH=$Q%hG9HfxHs+q(Zqt!@V7;SNPmXJ^fU&R`4K%2 z7wgq!6zWrp7}Xs&aq*%`qjNxJm-TgjtkfFL<0yjgOrd{IM)hzt#h!pqOtStFa0w72 zcsiR^>z19)C!zR*gk{*b=c+Up5tpxO4~Hpf-t9+WS5znRoti>Ni6>ZEn5UP=Fcr6irhc8Kw@SgI9z_6_%qS!w!v3_FfGpP zniN;?IXGkdAzk6701PdO@QhCv<9BgCU9|mg(#BdH_DUW>{igenIm1}$ItOUxz>Piw z%oHEF-QXz1fL(}92+uIvFa!ucyt@+JPaSw(-%7jLRJ7YwvnEW{SrxP2=?=@Ap1jDc zy30ha&7TY~JKWXUiGY^4mchJ##9XfEtZq0t){)q<>JSa#d&E&ht3S5RV^>>P3#{*w zSwtjy8`694%;>$tM|t|_{EF_4lnjgv7RDF`*le%mVZBtSM}Y?Fm66)=R$^&QJYvE- zxY|YZ%5FL!;IwXouYm+z87F~&3Q@w@&DlpbWG8$Sd+)2O*a;O_`AA%U*^OG_ZcY+M zdH+6_GP^9a#>lsMr{9a@E&>@#Xcw$LNn@2XW>#ap?p6%Pi-5LqSr z%m>sUrR$tEGOc53RrA+>m!=YyB~$IYB%dmb)m^SuU?3kGHubnla8r35Gy& z3)->4QEVyfN|!R{c8Z43MyWrKwzi4@Bjkf-Nn4PJo20=03MoUo1R<+`;v1}n7>>EwJn3tU zN8FEre%cr(KdrD7@1sQ0gahIVTd{E*!b(?47yBz*$AC<@+Jm`ntoJ*R+x*m(o8W}| zwF1tu*ag$EXxGI@qn=h6gl!eRfso5?RzimohmnbAVD5E}t@C8`rlw*`c&Qan^;&Z4 zrqCgDOIy({(Kxh!#B+lGHX=v2!-I2J#)~$_lqR##)-0-DeJ&1K@{rE--0g9w7?NWw z*z&CEjHM28neSZueOn2C<3G~@*0pV-jHN6_y;Hs%@`0p(r|1@KL5UGg&W7U+`o@l` zxL4%GjT60Z1Z@Hn?mz$`KBFA4_g;UHAwe8W(M5J1rVWDSW(s|H9l_A2w(}v6WfA1g z9_X`y`u1s5DTM}~{uw&N7@)Ddb%*qTlcvDF_Ac~2-KME7GU6f;(a;x$rN zE0_lp?VH$tFNN65uRvEk+w2PMfA^evxT0`_$HkcwZMK!K5w$zsk`xZ?>10P#{b<4# zx$LjG8F`ZS+xlGk+VCVZH3nG#i{i{E)8MkA?p}YI9(nBT?3#)Ad2D-d9IKOqC3cN0 z_m9s#>`O~5|IkLU1hk}8j&#cjo#|=?|L&WPwyC#&IB{J+?RgX|F(!(82;9nH|0y&N zj}NuHNXrv=u|E~D#C+M}8B(*~-%IS%0oY#Ag4=kxVKWg}M6l0~K3$SVMEnY8NHl;7 zB}!#cDevv~l+=0ag*!FsuT9C06_Sy9#-L(mTq@4q6r04D?6!68~|Dru_ zNbEG#gyaWNbpx39pg+k|arWbTS^uQI-{C1`YA@yo>vcS?pJf-RDgjp~?)47sQd#Yn+g-aV zqCuX1qC!(K!d*82h|@D(2c%d+k5lD;!Eh%U`_XuQ%a!9UB+@GqrTagnIzqeQe~u0f zvn7_Q;KLLswisx9Tkx1?#_P^GED)r!mJmufwt~kZuRjKs3m`n{kL~55b5*UU@{sh*H{C6qNk>y64G6{yNKdrFvckrKifW2&2=DYLA5%bT( zOPYTtnScGnS0-oVK?Jvz{I`1q|Hq&D*GN{nNhP1eGj$lo?f>eN{;C#=5ol{Ry>rxr z{=6#xz`o%>mo|;$zJH;A?3@1epAi1L$?2~Zx=Owcu~p_~R7{9T)byU&mZd}155AW{ zPII1;7s!>Ak%fO4yfMGz4>Wp{rvEYvkWT6FGeDF6jK?TK($3Ty1Aut?rP6bO(MsTt zzY+`CC?cV-JjR&|E3_QzaCX*-Ne+cNO;BJ=AZ-n72LY;X5s)Zg4*wqD z=qq9(12=?VFoEi+=0<(wX=9?f5k7EWeqO5e6Qx?+;E1k2lcqo7DnOn&r4nqv@Q^Bf z>)9KaEc{-%9(!sKg)Q5*+jy}Z3r$`+Km!H^+83a9JBSW0|CnO-yhGhe?Yh&3F|L$I!soAop} zGgq`VTp&Us0QcKK#XYGQ`udyW7GqtzXOWoQVffuyeDX=*$9_L)-MsNa`ZqIEr13|` zD%s^hSi99!T*L&1RS;pI?{B+btDjvQ)z{Q`-A2-XHS4~f4IqJG!OP1YHSaIYl>nHv z;o-K1?t8+2&#weLMOIa^pU<3b^k=cF4j&Y(sAq}n>B6(SUH=iN${g!JL~xUe$I^x> z9JgQ_8-z>t5rm$@PsJ4+sy|nR7uT@K>9k!`)g;a4SxeFvSkQ%PV&1sr##V$IN74s+FLAL}R?`wYIWh2ymWmrJ*uiGp=J zN-m$WV$8SLqu`*>B87@aOygRu1;&3cQLZa(zFM|H(F$J)1I@_Nkmu-G`-`GBEWVj{ zh88TORD9y&k_k`aj-2JI9XtwkG_QQHc1BDn!l2lUlbi8sc!K-XHhY-V@`K;}0;a^I z1XTimIOge(s!xvxm|nBCgZJ*{@{+WX<)~Y8!8bC#3k2P_`MHs;zF}?m znni9I-P?(`()Pi2Yu&HTgufUZZ;scRRe2Pm6qD1anD*Wfu%fgHP)|Hr+t;+xu5llK zLl&*AxCb|6rK1i}v_MZko)6(ute5X>{Ph=BVq~yH z)g_sWs5FtNODTQ9A4_@Sn%Lvt)Z{!8X-UqjCy@ZlV`V0+SJADC`Mv}pCj{>e9anc! zsc%+h4^SB*P3(OSd3f13G_EF@hDf4+VUE7RumvG>GD>y^EMN5iX*vpRp|ratf>wl} z8wg;>I)dqSZN#_uc){eEKgD_mn}>chLYUCx|Jn2`)?w>f$U!YY6PDsMlz%hxT;ohG>J03a`YL#0XB}uiYGfParpU z1l2#8R-B&*I*m{4%=kd503Z`vRN8q3(EY zV=xw5!YiD1jiKAl+ij%^`6o=;9AbcoT&JS9Q{K>r$^LqUH2VzSfYJ}Q0UT(KbxqyO z=3U?=QbiKX#a0RLvP!fTkN_;NeJk;JNQOP7(xUtD0y2`N24_bc3zP?cV#L>M0|s|f zc=M&g`_U?6dn*oTRZ7Vj+S{D<#*gyo7o=qib6| zeDzPXx3Dz)D0>H=uVji?{8l~cWx%6Zf?_TubSNYmr(q)ndD2O0R znsKni>GW}+?tojpX};!vD>j4c3<)N8$`(X24gW#%9Y^GoSbRE9`k^>AbVKL)(Ihpi zLr4`*lxWR~+E=;rY6F(-8S%xduRHa~lOxlV4ZC@@uoQ$T<`YUek~szEdmmwGp=w>Y5O^GR_szQ#XKPTlioU7FH3wF%t26dmyH;^;%>1@%^neuRtFoKWC8& zT%-3(m}KB1gED=8XVGe3242YG1DnSPxxc9J~~+T?Vyy>_mO9lBA4D za0Ox(F5_Dx%1olTCXo}G)ukVLHR=L&6}kEs(hYPT``jx`fvZiV=V2R9^j@E;|e^NX7a>r_YpO+o=J@ zZiSI`(i!)E8v;UEk7NQy4a0C^{9Ba8;oGa$*BmW&cfdD*1RwS`QSTrvJ-!Mch>Hb7 z^TM06b3bc%>rufd%25(Pv{IPtil? zS)yqSD;VH>dN{FW#msMd(P#p$afVJSxZ+o!mGganzfaw&H@cEtk_+{hMn3J6#7lD8 z@QANKNzukzWg!C$ridD|c4SkR+hB~Y&pM2YTa>$8VKZ;-1zKE3fw%54-4l?~v7B8T zIM}D>IvJ^VCsv2iL}c@joXss-A$f^A%23LdAVW^nQ0}Eda>VqD_S~1oEqkpA8d=3@cH!jDPhE~N zhhDs`z>tI?7YSAdx57p6jKB&Lo%t#mL!0A&p5%x@vzF=R9%3}rXQE_EnibDi@V980 zS?`B)=RGdhzKOd-0~7|Iw;zSn4WyX@QQvvW%j>!OFGQTGA!m58q(43NBwb-QKwM>R z{U~M^X9HzGQ+Z)3o>rul4UoQk9+=uKV;cItCNo*+b_}z-Yn>zwBh||CzC)7JvbeO*)xaM7xuPGbTZ>z|nmvYDe(Zvg} zJPf^LgK@#8za=`eC*|mKdtG*t0Eim-!{zp!9E2x1TUU9R2mm8R`W{wDI<&cNO zd+rf5)F`mzOTSc0bYa-))BA$%9Nrd`R&6ibv{%qakw_Jn~rWtAGz!!W053Rs|K1z3}%gInmm10umnXDu!@|jbJ8>nB2ZcKQxNvl- zx+)vce7cg-1Xha%HXcDgeu}6=-))ht#Fx{~B_|z5R&>dk7kujeoL;*g)JwsXj5M?h zSIqbo;yP1o8ff}tMVuv^7_2?NSpZ?m?*8;w^yTvdYNX{Wl}n45r$NNYBYEIrKsbS| z$o#kPMZO8<;WX}lXBABwS68tqRXv9MWiC$?JAA@SGIFBaOpbO|rS{Tg4uKtOiXtSa7DM=9(uwwKexd ze$m6BZc&Lh=h?ML7^|dKRy0o9L^jpDBWZHSh8-3Ij>q3YtnzG;Wd^qmRe?)t!zopg z$O;iVC6&%okKnh)=cbe!XKw)-BXM zoI%G;RnmEXeguM`Is(ZQa!>VF82WIhfq%ScEWc+tJ?^ok)7&is6{C$UOR-1M$6h=( z2@;hsk}!(mpDq3J9@t;BlfIHr<8LEsp4dV;2WV|%=+6`Cw@wjv!PH-c=H!Tw(X5E% z16F`}B1V`~?5U$XWmR`EFjxdb4*at^9>0Ht8Q-*jNty~vS2;IV`mv=%MbLdWy0-sh zB5=6M`Opv_5Q9>I$&OI#8i3}R2sXDF5Ok?w@U;7<9xP8kzu3$X)f)yD@OtWW>(WH7 zkd`!X)Yc(4*M4W`ME0ADyc3H&2jF}|DV3>REiYo+Im>^Y_}LV&DQ6Y8dPwJVIs9J3R*2f*L&tu#hjrVt;aD`Z`oqo#oj-{bo@ zr$4PWk^rQAPZ@L#I8#kL=6(G+^ z?n~fe3n=JT04qV`mqJAtpKz5xlm`;q4?yYFRY#2|&KY`MFLElRWN#OwgUU`kj)vtH z?2`8E-9u|er|+*dUWm{>B2?KJP2;day3IUtboe`+Cb&;6{B=lD5`mkh;rQ#XREwM&IG8>&PKQxs$tabc5me zv=K6U=nbScvLm)&l~DY}&yy5^gn&PPMJ)zhkdw)vZJPkaPxcTW^AxDxGDB_$`qYwC zL5|G)&0nfDFnpHD+Nsz}u|lo9vd(eKr%}~hV5Sy!-B1X80-&;SHYz@=ff9q38s;i& z-UmAuh$z+3-%Jg>?l3}{ypwh%p6{4I;P2ZkZ#4ci(nqm6kv`+x+>MNr9Nu?-^df0- zf6{Bx0%`-yp8Ev3C|`;ezO`MW2_Z2vRL;1YjeHqgwgb76@M?{ReQrMH$9M-h446sG zG`6k2D?*Wke@pRu6lcF@`AWFo@6R?lEbZPh+~9{d9U8wi9VlT}CT^1;=>Pq zs}#qKB2mehlcext((Y?(&xb>QNeoN^qjC1VJM5pZv)kv`C6s32u$W&d+RIbQX_}tD zv#T3ILn1zd>4mjmUjk{ztAOwY)mBBy$#a_nkD#GS_Tw5cgReeoPsPjGYIzNq0?>K` zRZoUtl3ZlAPaI4OxNrpa%%2H|Ld8W`Pzt}(Ju{H=$`$Um`U@&bmC+f0rpBNs0}vhB z$7A7dKCr9?Z(c9R-F*O$QeV46U$FQ0j${b%DQEW@q(7bQ%Nct=#)r71kA_d66kdyk zy)XH7&q!>iE<~|!#J(ZXKOvVY|FX4~Bfut|*I`(qKmcUrA^JeP6DwsS&>%wpx>94n zcI~Cw=*HZpmt{~=)O%}x9bH?PtcvT!aSBmpTV!&&oUaG=Fs3BmmAG%392lk9O3Du` z`&64!)wkcS?Lpc$LX4M-$lr<4g+o91c-~wc-r^vqISvj@jlnXKDjj>Z#?|AbrgTV$ z9W@SpHa_FSZd>1p6{}v1@VN(|b67f6i~Qt|oNWl2f-d>WA$Ki*MeCC6zQM!fH0geA zatyly7thV0J=88jCsN1c&9{Ste=(Ct0^Kbucl$rK1@IQWB`Fbxcj$wCnm*v~jZ-e| zBl@SZ;e;djYvGLLT3;Cq=w&a$+#@=WJ5T(FS zm0(7VG5*Pc?!(W2pWQ_2cAx-oX@gK2rt7vy?wX%J4FfH7N77NTU7o<$DL&s5yI zQ9t2nd?cj%#~fBJu^Hj8$pYzGM{e&fxg(cH8Au<=YI;=inpCZvu-ZaB6)!(L%EzOi z<0U9w?Z$@BcMT*L1dQ;H%m>Ie+4GM<$DY!kG-MCC;IM6fBqaI_k8BraigNC{o8{C7 zT75bh{=2u2(m>`_I&@Oj4}OOjoVn9)msjfNZA^SR_gT^9mZ;rk5Ol-H!1&VsguO6d zx_LlqUp=Na(CyZzO zDswyuyH|35rLud-W@Lh#v+J50(RWl7X{i_&>!ZM7xUBnuIIbFt#ip|=HIu6kH;etO zu!6K@1hC! z^{{=C2aicwgCkl)p68*Asg|ouORfyXOwa@D#pW(%w3qE_N zTejVbW@e}qO&zV${v{}@;+xW}jN1uI+(PLSngW&MpPM|SvOJ$vq`$5Ja>2W5*Dmq2 z`GHq|j!Vo{$GN8pXx>(K3^lCNa4lR}UM36)=7V4XX+N;Bn-{DUmm=qhb6(EZw8OqY zBhZsuco>5JzzFa@mmFNC)87G~MrP!yG2}O*FIt{sPuUmr%%3hEq)f0csrYMWtKFma z{(A<*kEf9#w5;Dn)&TYFsx*_ob?JA(e20{Oo#GdkspZG>52|^>wMLGnH;GOKBVT(g z&vMO&!~E=Mx1PuvetvpxHCO(tL+=}2V1pVf32SnL(<==dse;fFfRL9X9vEVRkhjM~ zP5Zr@GBvvpb}JDNzygnG+QoAv@y?B;HS~soi&*K?jWkPt z%wQ-gB9|2d51YhKyjWIi#Y0ye%^;-6!8vHk(uh3BWokx1<`Z z%;_nQ$078XWm7d>IgfID9A|7;H+`aq4;`MFzeRFg6$PK={)$1_ssf3@p^ScE2)G}G z8sN&artG3UNWOneWtkHv8_&WiNPzoEZvZgv?R5Cpl_FxwqgG4c%Ms z5w+dbh0F;nq_(4PuIq^Slr;RHns|1LTdUjRsV51;3uL)Csn@tCtn4-$Qu;(9Do+1h3aD&;|#u^=I$``;7?RvsA zu8MMkL=h2hsJ*rxI*Vsnr2UI&qXbsIo@QOln()-3kSS>0_tfN4Ywst=?Ryu zIF{6i&Qb1T4lOwsG86=0vGd$KC-6%n=2Yq-oLcfvOEaCN9)SpF!fY>pvaHwcFZjv( z@pd9`v}Z3e3yPq`2LeK54#Fm0q$fpCrCI;mfZ zE+%!Vw$~c)&qv|1#cx`UE6tHoDC$LyQIR7oka4Z$WrJ^$C`-4E#*o~U>}nSNLQFWn z$hzTM#`MiNjbBS|_sJciAK&^M##|u+*2D=BD^D>QG2eLzx+&^^TxiPKWn8985wCD| zagHrg-Hi!Um7 zYC7dC7oys0ZB?QX)29yg2^(M?jIDGiKjffy9NosDZxd{e51mIZx_b;QQa^q|zkMe_ zJ^OL`5?s}-#sY1BWwOni%)H{>+7MDT4oBzoLab%t@0jJCu(l=G#z;X|Lf_(xXmd0OYF!k(`BuXiV$%XnM&I^Xyrh}iT{b@<*GVcGD!-NO%5X58JSSd9k> zeOg~9EC6%!Q!+u|x+0Ui&v#(%L*bDF<}EHQF=ZP)1$Xm*y)%E@4K4Q1n8424D0(eV zd9B_;kJ%I{j8svUlC{IlmF>i98Iy3l=gy0)G&zDtKbc9whn?ZWV@|cNYq#>mxW?6{ zj&$x#bXk=uib+LfeEBmq2gPZsR={a32h6REkye^+)fmBqjT`Z*28h^5Dl%Zb2Qiq( zp`VJ<8p(ctL8e7T2Da9hNa6J3z8`@+h(Z3Tz{v`4KAx42H%0XBo~hxvA#cbU43oWx z3)N8g30~I?L*_rm_tUNk2_ChZuN0yuJK6zQJEKj^G=E>cgAGn!?@g(u8h^M|FWe<$ zh!KD5RP(e}B$Y;KgO4Q*)X*KCk>eFGAdc}hSl+dNUjC&#^7;wFOCATPoW};w6EJU2 zw~Cpa9*@6`5^N)Hh4i+w3eFtKUPzjp0lNZh64Yvtkmwfo6SfyF+xRkM7oqkp(^`SM z7=g3cPNqQOymzTa?9ON?3? zN>s>yL;y?B8xhp5)dDqglCUL=qoR=Gj@bb&oMGZnAiVn5FfGUS#YY>L{!P&j>^eX; zY6ix_-}FoXp2+rg!-S2j=I7Ay15(Be?1d#lv-KO}B=fWWD&nezz>RFCV!Xm2yB;8v zOMj~$DQd)#tX*1PdKiiHwcS~+6l*0n=hf|hv$2GOYH&#Nn6)%hqM+lhZmk}CWt()8 zA}N||Yc3yOEAvf*qz#8!1@T`U&xrkz#983n7AFX4G?s9#_Z^;l#yE(TM3H!wgAp&0 zk#xw6dRvJ{5Rp*PYyoU1ZEa{1%3m8I+*BOxAMnWHds)aF&c%iGm|cy#uI^M<*AWMQ zPzaQcM&8}njuxO(%r#Whj1-iB$(#+gC(+@`CFLWw_7T1T?i293W~H>MzHzMLU#lXF zK?lkL=jw=nMR_u+<=pcC$U8Kcu8s}4Ma2a40GPrUh&^iuO7f@jg>_3Bc|+>+{j1pG z_iN!YDqo^TgU{^IAK9WQczvcFv_0FhI^x z9>g8FhiZ!0R@18sWynF*^lnMknV$ubLpaB>E1&R!dFFNR@8CI3f`bMHr;nk{pfR8q zzo|G1{i+|Jj%^b#xxuBvs_V=KjYe8qN?kPW^Z7D&NIwT>qo?q$-v{Jr>HfwxJrPp! zf7ynmx+9*vE`zl^2wT(7&EM5CH<*;_&X~8R&0%xH&cF+;qx}fS&*Js?N@$)pW=-=o z-te#33XgU|x7{h*c1ws!QTW>58*wOCF_zK}`>k)o3(8diS9*e)xUZYVUjX}V_BFFX za=)OtiH;=ICb8p8l^j8N=k{zSmcf3IVRhvM8)jTftwl;I?FMZ(-;j+FxTFJeDm zZaMl$GVA9MfR(yGz4nOD@b|cPyEads07^Bg6~`k1%CBpDyPBS7J=gh_1){Mxc&fI_ zt7zQ1s^BG+V#hp-&Ir_f_9T1whg83gXAe)I@8`kU<8l&PqS0LYi zCnFgxp8|?sL5pqfuj1e{&bG%f?+=7K(fO<0=?S8)LWV!uL-$lw7==@w>B}^H3|L0t z)IkL<)6Ef1CH85{)4*uzz1#>~e;Et}h^{pzZs3w4vBjK zLbB@vmEfZrxYHLWY}AD?~-G0Yxqp2n%0(hf7U5V0le1i z->MFU_byy!Tly=hqhO-JSJmy?N-td%qW-Hor?9EaKl^;mDv9I?XPv_%kynQ~veZ^t z6;q?rYx4cfc8E?PKp(gb)?S=F6J?7Q@{*OZNVr8hTDXWt;CO>+6nL?!gQ7l+f5jMk*lTtR+bK~%&?LCv!&C@Dwk8Sg!DlR;*I&kKLAXF8 zr3}4C&m0>=ZZUq~c(y-Y*h@afb@dW(0K-W$f!b{{-4>dkNg2mYAJ6lf#GY-Cd@T%_L^4A@{K2=P?_ha|6fB04(@Q-{9V9-dD zfTpYuit{yGwkoc!20!cWpnMDl#cueBMm+a|pnwM!e-gW#@LN-dMO@12Jq=}_j^bsJ z12~Pjm??lAaJ=T%3GA}}@&t2++VF$LXf=7@y&$cH-^OM4&Rr-Dk@g>wu9-i{cC7vS zq5GZydI*3$PMJn5f8JDE_X7%ja>sPDOB!M#FrwZ=S8rWU41P`^tpP62M!Uh5+Lv60X$kvo zAw!?tzQSi_wbBnsZjLx(u@!W}u3;91e?-8JD>_WRjij#e(`LL} zy2vHtK#n}a_1!*U@WLy1!PI)rF!?m9Hu%E~?$0D?bAcs<;ii0?0!wZ?rWrv}EBiSb zc;2u|6=;mjhG}p1PAICfU;2I@jIWFDkP)Z zd=67Ntx4*u#$z#P;S-A@;Sgpq2UFidqw zC*1T704zCV`@(uYB5Hh~tC1|;Kpd(Zln#QIvP}GSUBfFgO@298Yt_g`y6uD~X~d)%rJ!7Nl(g}@km0EaMOl9?>!frD23k<0%3wl0vx zX>f8?XVO63+H}XgW~FuxTfL$>!;(4}-UqOBx|eU}_u-ea=ejH@f}T)H=p3y$$(q%0 zf0%fYhA#7c(S*n*AO6dpG09%s{IJ$z`7(O6ANcDj?9&*)>~R3b=8?jHjZ=pB+cCtn z5%?8F_JkRUmyr5C+o}NV2|l^Uq|7dzmyVWgH;g>niB;XrOu>>o|3>=b01yVGqgdtv zm~zlkG|7FJ0jH!ic=P~iHBj$vE`tI(e@x#PCwr$Xd|4F8$^wG4S6mQY5n9%)Cc#=r zi8X{IR>S3{tAf0xRe=5n| zZqOs0v4S|3o+tUohe4LGM-;K|GN4%8ZXY(%nW6`Q5cWOg#@YZL^=@sdeW{+c7rNRc zX8}^efee_)U8d1`fN|1{-&%xoY0I?P41SZr;e_};@?!tB_ zYGyav*k8vHRY?BSK6Hg)>xuI!-JMYU9HMPw6ZOzI+hbiaePc0sEG~eOv6;2$qvYWU z9PFQJQ6kdaSFU_9iP9Gck=cEi-V`6P!v%T`uH;I+On25I7alx6D~TuQ%_eD-YQFAw zr?22JRbARfpfRRieWiUUe}ZwEC?b4pjTs!p3e2*MH!pM>pGuRi*RJBCNPxbP z3;d7QL&E+3!m^+w2}$JPVN_pc?Ab3l(7YU|F26`QD`I-tL!{?b2Z+mztz|uV!2G24 zw7TG$d_=z+v%=tEe}~r}@xpqadL;{=b47-@hTZ+`C5|U1U2F>l3;C$n*7??MIMo1BGAW{o^P$Xc#Pa z2@ZTCA&h39n7p0Ci>b9;yhMCPw~Ag*se~ubAnY)zeLN8^e{DWWIe=98T&MQOPRu2S zN;qz*E|elH)U_K?JBY~2|thqv=V4gg9!Sbpof`2B4`B? zl4zd(Uuf@he=={~uWHpl$hCH=4-YqUa|;emz5TH{n*2acYGOP+EQL#+$MPlmK>vuZ z2ku&=OIayb=}sujStjB8+0emNnn&%?E@C*<*PMvtD^Sb1O+Tz1+-$^?kCB_nMZ+C{rKBf1|1do2B%SB$wi5new^Th48bQ z2j$$hq`vWug6cGOh%LK({0di4g!dKzTNGFRv4EOsJxP~iWqhg8vA8)wa82gX;Vk~x zJYMk}DOy{lv)BkPY7-8n-V1{gOl3W7CGo^l=rWE4rTZxIsy`3&M~YOW;^uu;#yNXD z?p%M9f2T($8fTx4<=N@lZ9dZ5t9Yw1(^&}kx?+29K+^3qgqomTp5*N)g|W?C5aXd zf4oU2m}Pr2+1SFG^X+AQq}&i2`<$cUG8geU4i-h6MD}>&+A0^c+#AS?>J8diP2AY5 zLk$%mdSBn=u?p!U6~r#@)s1i(Mk0?k>7rFzZZogM&nAK>YE?8n`$_2s*}m`r`OGqz zLkXkg@A6IPvG(RlVjdX3dsIcXvdLkPf0yUr>wL2Lcdi5)V0+Ia4o1RAq8FOde|)3e zVLB*p?jsIS>X_4^C$lObk2Cqsj@;)`RvTZu)(2jKy41IzXIJm4&abmned;C-dGBfv zlhB(`f`_4No_3e4@ymbPZYAEES98}trws(lf9oI{GzZhG zPwFW=4dbU?#;0lX89d9dayXu2jKsJ;Z{cFd>pMFQAQa1^JSIXyygx&A(zE5Fp(y!0 z$yi>Gz`e2exrhZT8(xy=ZAbl-Oyl|Oa=y;1osXu4G;HMsg-8!44Pb=u=E zPHp-u;>a&kF1&2mg*&Ugf`%^3T!@t&G3ubvm?!t~%G@6COhU^_nVUXl@`RDUjI z9$@C6W*7OHuiVyhZ?|uJ>tdfxC$n63c)i^w@z=EVKMOVLuY)qKf6;UC^0cJ5n{nz- zYF5+soDtVa`bgn9L60NrrWDm*?&HI{Z2hr$PULDkQODeWH{9#sIDjMIR95t@;ZhXe zwj+Jk2yT&8^!7fb@|s!u1PUvnpNZ_70d?)kMw9&T;*8{@%Er_CV9LeGs`c?^=kcWs zpK9Hl#>OZO77OU2=XZWOxmSKIj<>_J9ow4tJaZja^ue*Ae@;ywXw==U=&eM5Td&?; zm|!`)KmGei=ZaT&d^yuhPUWoj4vVgRZ)(1NqQ|GZC%$j7E*;`#Rq3RfyGXQc%Q;I0 zb$Z<|n`K58dCmk)n;N{<+f^%#qW*Yqw3JJ{g)2i&43Jx|J}Z@qK9BgWO6K!T3@49@ zBs&Vux23uZFz_!`K!`1Mi)=5uE_s`EL0)ZD!1&xTpa`osE=WKek2m5>2RhF9Ztp)2l2 zyixqpqw~jyU=|MKXtB(sQ1Xj-dW>}m*=^2Wrrx>Me~E9qfssUge3!DI&1U0r`TX?j z+eIJf=U}RQQJk1q61yiIXi0b(^jOf#a{=Cu;kgdbW-9u!4)O6}U3=_^(k_T>{eDqn z?em$dlQ(_r+p*53K_IlcJ4HveD*3+JNaukU@1MQ&9Qm*FTe1%`YBi}{r2J0RPHVe)gBQZ+uw z??V({!^z4gMK~0T$S2-MR2`j11Fmn2Lx7$=*U@4ePlz1-@VYEsaaQd@OBu&`Ce*41 z(Xxzi-I9aThc!OdBu1u7zXtti3Iv<=b|$xbfAde`XJ{PavC%K~>nOzc;GmWB)w~yG z#Nm<*f-acz(&LLsgc_GD>Rf}FGMjU&Jw1gfCX8)YLct^zndqFI;yqvGhHtTU8lGKE zYj3rkvfHa((lB5Fd7={2xjw-&h1#h~J$f+fH>}1iQP|i3x5;!me86 ze?iRI?om}eW}9&%m9mg8kH#W;QmjX@xj(`%WXuZphZKtr;woOCjvFNmU|nPm-yn;e zON^I&z(BOf#`9!cC)9-j=JYdmnYf$BcO4(6^qFXJo75KJLxvAbq_?;}{su>EI2ib%-cI zJou08V{*#F!KK(QWp%U>?d9Z4&8IM;XIUl}48p`?4L+2_dz%gK*Oqpp@~-PCL@TWi zBJmP#d}W_QZR+Q!GgcG>e_xMG*sizOs2T2cHb?q6>EJQD@3Kf=s zoNL+lNRo@iZAEf>8T{~~dAmE7&O6W)%I6GU;mCyP4y*4W_eE!)=k0cx2--Q8oGLOK z-SCa{JbK4%cxGa4k)+E*#%6WQFp8-d@|g+U=zRg(zG(O7hoMFL_H@sR^R`mvGj{7aC339Am-U>=r!<+~$q10qFo!dnylGNrS$W`>r}vaX{m{G1ixB)%7|z9|X=RryyH8r3h{juq&L0PFd-f;$ zCa}kwc5l!0`{qcMiHudWy_eTv=2UO?N8!BQJa}YsHOHRks0an_f4y6nO}jeW6NhEW zbQ<|dJ{1dQlVTR0K9@N^tkR1)mMdu$W5&8T@!B(~P(-F=^C1!MF*=l|^|;yn^F(8T zgVR}}((x)y=d2~Gp?#Sk_bRwa+i@9Q)~p}sE|9m6wA`=7loZF0VhoRXlU?x>vg4~V zSYFTKT#g7g6qZwLfB7vt)=Kfbr1#@RVpglp8bdM2!{XJ{tM=xnd;_;QhY3w)3xDIR z4=uM=xZU^z=l3Lk4eWLWt-L5FGrcqYDR6mDScjl?h1>OTE1h6{O2U%F%9`+p%82Fi zs)mnaCgfBu>n=^C1DmR4t~T54KrwUwY@H*WSnu$2;hD=Pe{^WrU*9b8rk*F^_9LWPChF}Ih`P0rNzhVO^{8~*D8>jUX_l<%T0aJQRZX6 zsaZRoy2o?z$@A-dKX(WIM4zy%s~_|h&QW}$Dw?Iw5U3ivT(4|?xCKuMrO$J5IH;aM zxO#uoPfOe7f7+u>!gi3737`Xc)SX+LKW`#A$Jip>q%`lfdBbbU>Ae~mn2F4B5H5nL z+_K2>SGZKbFL?UFSO?=cfuXqF!uQsU8&7N`i|x+TNA7usBzN*`&^meZ(Z!@cvuY}5 zRR2DGs{XOiZ>o}dy|&ZOHDuk;90yN7ne&i7urnXIf9Ga<;;e62~PDu~0QbJ^{{zInd4c=-{{r_e5p$LW^9BJyEJ+FAiM zxtozE=v@7I8)KAnc-9QEJ`ZU4T&3)xnr<#mVd_27GZHQ?CPs=U;rsSp*Op}67iNhK zTsbp1f6dcl9B;@)>Oa^z~~D?5#>$&5Ek9g0Zs*&fy!etBY!7&pC5A|swtl_l%Zc1mCE zU!ujl>XLQM(?&mUv2#%TU_0F=)dE6yvgX& zcXxf&jG23$ zGiOj(8T~u-U`JoM<9ey+PN28=GBC=EWnRbY>9etn;V~bgOU0&rtartFx$EmWe>`67 z<1rYWR7fp*uop}LOt$2v83qmz$TGlyXPumH^&b!4Byt3j6wm-?40(9fiN z140&@j`5KU$WOmrU+eUq`}M6&+U7J-?td<6U1QprUn7|9%SnH5`5Krbf1{7iCn11I z);!UIKR1%gu}}kG{obGMdl$A8Xy04L6(XMh-nmY~zz0wR5{3xq2%NF}0R<^o$jHHa z7Bl2}P)a2lM-t4D65@P-Osa*bJ_@MSw|TUL&urf2k15K?Vi28+mFCdPT@){Tbd(;ri0g z)}YokWw_}W3;ke}?893hmhnN)%>SgOJWpgz_2x}_rvm4g%k;Ude+TXxsPEnTBmjVa znWb|nZW{#vFP{F#`vHmsUGhDIldz%R`}5)7r_|a*y+i=CT*fRu79# zqo`5y*oPBzYDp4YNg^(^e;Vp$k2!Z8jyahr-sMpqA2b{ES45W`MB9oPWNluwUCkTRf|EGj zPq7u8uJcXtZ7q%6F``ccLsoa6XIZc3n`$10qi3b5@EUXBYFbaa=er{* zY#-_}N4A-cPS?5X=5fREaeoB!N|`}GTi1sh-dDY7#{A=qf6P@#%Tjk)eKYXgm6Rk{ zFGH;Gw|kiCc}kb9UYzJQd^|;PW?y&3qR+}Rd8r*EQP=TJTBYCW<>A@K-~jT+^LhA; z&$c6}aw}@K^nB;@c(FA7c@6aIMJacj^WJiAokKTzXYVt*CwDybpCP!VXuD*b5#`k| ztNIGKo?(eOe+2SidwSAbQX00Y0!|0&U~B2~NV$icI8)iEN%cbjE-)+!PA$0EW0NXo zF>f`*Sq$-|H!p}60cp9FlJT=5YOCb~2!~X$S<^C?qj0d)(d>Im#IqsJYewa4&P!-X z{Fu#B0gyJKiZ(ivr=AKyC*air!gi8!uz#ObKf(-Pe-KvS;CM<3EZiDW*NaQ_uyFq7 z$Rh?<{-c4u;uyBfd8T9l1q2NMTd<<-aR$<$xpeusf5=(ORjmnv;4C(x>By!vP1ZC< z+cr;)JO$U{StunnS%%}t6p?G$(SCCiKV5Ivb17ew>XjUAzhfWaD8S!LZ@R6G&WmZ=MLE`vNUU*#XBG+qXJfZ;N}qI%t<3}Boo`u^MfVa zK%bp?;1Nf>(+61)2QnruX42xg%M74}aepK$e?cZs;y4&uFb}MC)O~7nvMM3nQ}LvH z`N1iZhPJCNmtNkP?uFmRPj~?HQHz~^Y|`5+(2slJkq7m9T4H<3p5F7VUWD!{zPf>& z={hz|cb-pnYMuAtd2mT#nODh%lTF* zX$m!2&mCpXA8N`+3ZKR|^ZZWpk9B+A=tHd@oOwutyJT=@rVP1ahP|3=$zb z84pbtTwk}!S@s`9M}EHDl_(f7@Pv2-h>D*;%?M7dr3wt?bd`ATM6O-@&A= zg)XtW_au>SUOubNYu?bDttXJT{9JUsR3FY^wN2qxV=m>iEN0&0i6ZrjfbzjVV)i_| z42(GawGMf2G<*xk+>P z&Q_I+8*xr#`H2BLcEkp9AB%5^<(O=+j~sm=P0y%9xz?;WD^gtzwF0y(%F)qzG*_m;D!9x}?!`4r_S z4C@89%61g4wtBNk7Pc(s_O>p(->vU_5t`E^!9n-04+uW&2k=wGj(nHv>`S7+VQ1y${bqSjjfAl*S+4`;Y z%0y%C7HW6M*ZH!QM(lsIDL=j2+15ipQ#&<_!`MDXzDJn1^*LrtSL(=`azEk-#Ts=Z z?QC0Q;hB%&In>hwqa7T=I_dKqRAfdRtqh^Im{vmeFTx?~Kz1ZMUVcmB<2^w(wno7t z%!>8hEMw*=o9j~82+yt3f1t7s-JS1CVYv&CFss4nt_moPaOA8Ry)KvcarFIV4lM3L z2wsyCgWDWqAkploPisHAd}I@KL1-Vw$MQNHc{2pTR?y1hCcfqbwd_Ms;!7G7qN%hS z#U+wf-#GK^qxAZ!f5Y(lD!PKyW&Ly%%Bym1G<pZzWgpucdhE!x<6(o-7-o#xZc~*-sCQ0 zUof+lCN|xi7yOsB`DfiDPqOr!WdF`w-ob~WSG{9N?n=FVe~&v-o)ewq6Nf%0SS;E4 zoOp393q%$PK@pm-VWEn2 z^B&-Ry~K)r7h&VJv~(_`N)7J&fIf&le8gWY#Ge`0nOBUSQ9V{CXbfO zd8LZ7lwS&soGz>;g6JJaDnoFTD-WFH!ZLtwdkAT1e}r&496G?5ZidQBTb zl6rc2VAKyIIKI3(?&IJEihJDOwx`=Cb*{HeIT~Hkh^rHr=*ZR5y(7lrZDQp<&S0ZiDo>aA%3d{|_zbtgC0$#+ z2bObx0ChVHqD<3y`;O1n$K=K2c@P_hAjNybLi@yn@O;o%qbswnU*Dtg**KoNiskW1 zT3PjY7JW1(Ht}GK?Xwb3x;i%jkky+#*1XNZe*)-e*iaGS6|nw+&R_9(eZHV033jpR zz&4R2Xvu2tcjRZ;! zDl=-zDU_bYO3S5unj%JG*bzpUn! zf5Uzf6S|Urw2~~_68L&AR=31e-}UGAy>GfbHZve=EDJJ9lW}$rp8ymYMG(ZHJBWVf zLr_!%yak`?qj{$;6{(nN7|@2V*?H1LPRN-+2s~!sM}nJOl#$w8(jH<~K||WZK2?GW znpy$^McN9C!P|qhG+Qp%a!r}>sqkjOe;WHVf_0`TYwY!fV#hca)3%6Cy=KEnD*H`%yk0A+e;XHc z$C`b*-ei&WTt|t5w@}`UhOXY@vYA4E_lq;*k*$0n%QJl^F5;mh7*%KQcUeML?v(e1 zEVVvG%pWR}!u71i%xHYj$8CHZb;L`y`CA;9AMU`D#3f6e+IaN#!KIR`cG40zyVvV{ zb&hIQi@8JQ41DVef7kk4=H};gf5<;!FsCP;^`8Q|(5qBkZ;JN_ed7U`+wn%e8IwYx zG8$FNk=adszWd6!QeNpe8f=2mIT&~F@kG6$jf1YsG;!)biK5&sI(GkdA&k8wPyik%+@nz>jkf3O_-eMH~< zt5Zb-m>_IXN-bIfH!KLO-TwF=-7~$*TH#%pf@W;yNX(>9xD9or&LEVk)RR3ck#yhh zMkQ*GHx$Ww97a(PQDR8Ga*w*Y3@@V9ZY0Xuiuq+4^pwZji-Tlw&z5UqGld(h=g-|= z0I!HYhD=Q7c}!D0qulade}7~#RbAriC#7F4=ZM4c&DE!O@?vlIJ9luO$6&s|Q1D}4 ziPkDII;yXz!z)VJEf``5Me-8xcjYEu&adUsZXfbkl^;HN2lmIr#GtW+I`;C(oJmEx zJTDnJpP$Ag-4;oH;DiMq1kV+ZUZs+DoNYDbQvnJ2(6DXUa`~}@e_FfN_2pK_kL@`l zrpQPuAH?(Ini7*b7KFdwk+xcap2N6GyvAYOiwLp@+OZy z_k0B7?E3;NHN8kJT{u$u(PU1|JeB5p+x)Z%=-ZG4OxJBc{tF}s9*Fb*`d|M!)8W5H zI=o(gCp!E%Ho*BBe_%7j0cCxxHrD-92!Qe(TL4n!JE7q_kpTeM(EEed(p>KdLmA(= z8GwTR4nKHgcX-cZ?M);z0E)lcZI1Y7BE#1sf+z3$XLZljuCE4a{Z1@!W&q?feNmz+ zQ>+BNFaXMTA_K7d?*xYL#D)F)qwmB7Z^#pmJxhgS%mAd>e|KU6n=9ug4N#*{5&(+u zT^!SYCoX&^EbLP!vhVxhJB_I>Imx_?0VvM{NF@7&1^At) z003O4{iUY1J68Kd3$Ow$+MPr=?_rTP$uBm^x7MyltA(|QxJ@g_rTz7H;xNDss1g7z z&}X2%Jl==1enb6y}@eLdTrGZ7YdxyNcK=HFX?7AtY`<}JyWjB z9dSXkf{%_Y;(r&-c;e_j<&f+|FwIgh#}>7s;+ zP}zo;G0$HI?>k50GQb6a34qHcqZC^86xdC%*-b%Pf18%b9!lh@h^M?>d@-wzrhBQL1X|HHJV-vj!Uwg!@HZiWGuW0LYYR*+`*fLV!~QB@N6{;#Y$T zGD`yUzV;l>KvMw@70ClCz?v#SNr4C{V5AEHYwFk0753k*LQsSN%|MyfQ~-hYlr}_v zf8&xye=;Z>2nq-6@vP8hZ`S@Bf1aOHRR+cW`|p+*_I7|6`_CGLw9)W>5Ayvp(pqn> zPrvgP5laGuDA1Sm0xQb`xO!YvONTR-ASNw{tQ_DARj)h*Sc4A4HP zFldIe8<2`r?dv0bd7dmVw+cY$0I+7N6;J@ae+d9f`(qGPzy;V|jCp>U8(>td3Btk# ziZ|@&dJrpy>z-dS0`O_uHj-9jDOR}(Lx>AihG`-<*7~!qqSx27(bj#>iq&q4{c+k3 z(7xXI`<`0+jYZ1XjkW&9QGs)j$W{8y0og+;#Hk8vtJc4tl?{q7T_halLOpy`4^t8x zf8MZedx%pBAxjY|wu^&dRrXsj2|yIWaFftEutnBW$->z88QjwtV9id*5XoxD%eACx z3yw4tLajh4!?F=BRugaqf51WD06Y0!XbRCI3DlYZC0PWxg%DVh1_}Y-`~dc7x zB7=P`U#$PZ!~M%k^N#m@{b>Jhvw!%2vA;PH?woMfsxbP?8GAm>>GyHHF34{_jr>(c zg=|D%bx|RU3%Y0&xWk$EcFaH=f2y9;SuSE_K+=M_0H}Cct9LD92_VnJ`Dl`%qIyHB z)^noFn(m3yTdi5nUBV2_tpbhW8iI)iSSu8q%3z-Dc&?(T3NW zcbTuNC(BMd<)N|R28AKp_ectQF_L5ZfGw(o-dlW zU%t;85J;kcCso>b`a;SPf21Qds!?eyL*%yIw-y4iBn+T-px}WGfHEz-zVocTYQ4T8 z+TQjWvFH1DUE2>9zV%CmKl}fiGwBb%25GJR#t+V!DS}Ycup~hg$|5K_{aBqKuvH&f zn}97rsEdbyB`JdB*_@>z9I-5bNrJ$n0WPEkX%aw%Bw{2&pjAn)f5{o{@u-D}ykVqgrud>sCK>?ch-S?mR{3nl9`9E{u zyVpPbx7Us0{^f>MQ+{h@VsHDJLsDPU9_&~O(DyZZUEH-lH6CX2MH=N4vrvI8F9LSZ z7R`emdqg-dC|q%(gYADjtArjtFsqJ zii1+$SkqQVSXXMT*4M(4pbR2e2LjB1Sr-LFgC_=oY^@VkaQo6hQJJ2UOBnZ7NjKgG z?i!vyf|ttB(?%Djt`qX#6IA2NAMLIiNFcR9Fe7N!Yz-Rie<}aygl~=b(J=eohkt6u zU2j72FNZeiDmT_|Y?1PlGhbWvG-Gy5h-gp*Z+f70zQg)uNq{BHMvKKQ z&GkBjA|4#jE~zkQIuhHXEAh9jLXF|gsR$%df)lAP(?`P7qAp<~Vp5SLiNm1KJ)l`m zvm~xSUhSEUe*lsU9I=B%*Ojp%&i2|H$XQpZM(C}DvoRM-F% zfHdv#F+2}}JwDd1pT4yr?Z0{QqfNk%KCxoiv;1$|sv`cAYyb87wP^m}Oy0DgI$c?U zKL6Bf>oa+NWvep43P6?wFmk*aC?Mhmh+1DlX`up5e|BLg0%VC0O@6p^Hxl~+Jlhyt>7P@c}jlN5!ykVM4q8cu;8 zu)%Bz;xRqBAk^T}N&*gb5n~8}G(vyMLmk6iSAWmuKS+3damj)d0pmqcfw%E@30tc% zOQeade_E^4`WJ&=Zn0MIpZa7;fB*jF>ZTxnFlqhKt3C|S{{H({%l`1v&vQ*dgAM)Y zC0zW~Qs3vWgaJm908#_av!sAQ0N9ctP#{2-uwQ3ylbzTnbYfu%wu$q-ux ztforDp;%*i)D}{WJT!@jbRf`(Fp?}IY(qIne?L0N>?@WU$*-;X>YAoWk`|<=FYH5O!&Bk03#nD3jrjFU~VN)d$QI5?aPHI6$zlJ){1)K z-A)b;AwY)XwW+jg3rMHCzFdjJARK@Rfm8taw2m5(zVZLX!7*MRG%3X&MC*;(g1C|6Alxw@5Vh1=#M+H>4QEMv&e~6U9 zz$Pe?jD4tzS>u1@S0Rd(C46(iGlVqI<|0}K8EFuroi~9DW=n@aYg=iI;k@S*mWTq2 z5J*rYr8d>sug?5VQ2flo1Hjj=GgA`pIuqLBUv2rL7jPQMUu=Hye1L(qs@O`t&(*(i z{MJyb`M&mx$MiR^(r>Tw#k#+1f0O?$u1!d4JI;UkQ}e#P-JYL~j*Szn$WBYD2!cWc z0huVMgN_Fj;YVl7a86~25d^bTX~+^4Y$*^Gq0|8qHhkjA>{$(DsN+GAXG^)DP$b?- z>H|ptGt#8xi+Q1*k!E`QmtP^)9&i~Ts5A@CsvtD8W(utSjpba3JkX%ze1 zQXxo^y*K{qJh<0%^93FlTy~vDr}?kWQ~u`gZw}g(M%jYZ>#Nm&A77<^>h@O;?LFmh zzxj7>3)R}vKRq^Wi>zS%f9gi?<>%ke`m?`t`bSUh^-tB-%TblvPz2WhEuQ&5@suDG z*vOGH1OXO+FJXI|6%Fyx-p7CW8iy9#c;WkctVwWbebf|K1F7XL?ayac(g?65g9RXn zEKxwAU;w0oSyLcleJ~diycPnSQwiYEfF61E;f{-a4^xPTFW0(Kf0K#=u;gzpA(sFE zsSp*83egSD8Bu~+fH;*UKq_FLxGI{&37agKB#?b|cMA6XfWG$shfDu@8NWQe*T-Lc z{b2789;`2a?|Ng`wydnJKe`LfJ3iC94Ziq<=zh8UzjCf0BFgPOM#5TXGP7cLg9jFz zn|1gnwnGL-O`KWbf6F;D(jac{a2Mlqs0bb}Lx8FrUZOIo7B3zy5|#`DXh?*^D9(jO zdig!T-jETd07AmLX12S?G_Xk#V?-rddI@;TerhTz>)|S zzW>WtOe;KQq5AR_)PDF%U`4IgdhJh5`_U5LKKa}4?s4AZfA$Zbsj_Ho_TDG${@(9< zfA-H><>tTM_qR9NIe+h^{>lG+AJazLb@TtAW4<2F{LdcC)%SCnZ=L@6PWq3>Wo>_P z3{~{6S_1Lz!-x=sg1C02B8P#v*UmYVAqri%h1%{-IJ4rlNaDbpsv>Az=pzNptRJ;7 zkO2FGt-YVyf3dNW+WwheoPNES9jAPkdR-*LozMPL6QNP@G+z_p2Fkt8K#;n6EM<|0 zJTU+7`bj4Nvfx+G;7}BopV|en0<|Uidv%n}rU zCc=^lcFqO35GAx@P|^Szq<`rrdDl!Jy(ti-5X_nie-_DxSlh+D31&%@zL@{1iF+O? zyUkJ#!efTgo=2#*_B>+#?&A+1?j95KS34~lr8i;!mk)Q{@UzdG{L?cwzrCXX=i=)( z{fi?vC7QyYwOfja)3dnqAex^d$P|EwvkX#U_k;FJvHa!=1T^Sy*ZK=8wSCYhcgc|Kg%Qz0a46gg<@Ip37g{?3`4gAAj?!ofIPR zT4mSfJ8xgSmm`7cM+42gZh};}AuB2+{v7yie{G_$QZQDPM(TYJNY`0kL=dXt{N4Xw z&Szz!niMy}XUn%X6Osc&ng|z4w7?9RpHq(VJMI}6PoPb^51U9x`akQy@9+JikqhM) z^P+{pZ*Qyp<0<{r+&{gv^25nrKL5v${V$%_asS&BZ5m|*%LQM(RK)VPw?x0X{D&KU zfBL7t9=;DeQrJZg#DgjVRPVVB5&Fvs(pPVy!p2wvlmtP{OJda`I7^~fK~oXaP@J&} z7HGOiQvlY46-&C{r^ugo?ZS5Q?&;1*5mwAx7tMe>|ISDh8lCbal*qHCT98-Z9GX&8 zRGMtkK2S<%K#2{Z(W27Rv`-CUymIt zH{hjlT-OWS{dFw;V%rkG{VhPEf(j^$A(ce4{Qa%C6(sQ0SpU)nNYH=s;kutVRkQKj zC!fQoEl61UKliPF^1b=({eSAg?_PiRJCv)hUr+z)r(G9?-~MXPA7QUW|Bu{`e~>7O znJzZS64xmK^G~l2{cQK&92bICK&@}J6QS35Q9vO+1@ zl^lp11Khvbc;{CEBFmM&8uqJwj5A7#IC{4vAR&ZGOGo>fnke^&o+;g1LR#mhgnu-^6QpBVq~@w7-5S;gMIab*5_ z)K#ebyYBwt>D#CM`Y!*I2KM)}E-f zX;2U4Ac?3Dc_{e7hqmJ&+V$*?DOfTgn2Wsz>YxG+{Pu)@`wR(qBvgQ6K0cf(ph5h%itlGA#bpZ|wU?x!792Jwx^Hy)6Em@A>jl{Nc;r zy+H@iX!~COOUtHv4=sLqZntWxF#Z?D`fl-L3Gm{#PGPKj5XlwRC>71{$J5cvz$*lB z1|kSj3c;N}cAnYm9TLU1&6U=&MO+B7RQMgz0uj3efN+KF&TBgce^XnXcb%XaU`FPy zRdmn_>fRfEb&K|+gFb8kaS)L+dx3lZjjVms_U{t}Z|5tkBl7)hRKGX`OACS|!_xl@RWOr@q(JU!TSN?0JGP%y*w9B=P%y@LB$1 z{U4tm{hl+V>Q@{8Ki0p0d4gX(E@Exh@${cq0gwN8ef*2%e-Fpn4#`^pX_hTTFG%a9 zTCxAeZKC+kJwR7sZwBFYVO=o?fd0{Mp`?yueI+r3jcVBMf}g1eA@i^dyn@Y zuj@awz7U{-ZU7r!hP6 z%6AZ6c@v~6nnPKvumgn@L6q($%*_Bexjav+|4eVt9v z@JIoZEdhKHnig3VR`(@V`}HaRA8GH_)hM%l4}Typf-$!7a@7e6D3XGZgxnd_f)GVT zEy$<;j|E9gKi$vX`}e+%9z$&s40TwCx#pUs6+P!Q+7OD^U<5o9-aO@=dJN@W=*8pSYb#c0J{Jd@&ugv3i@KqO&56mQnUM*@Sly(2=`;Z= z??Y;&Lu$=Jmfi*|k_7Vbf!23oSGg?4@s={?Z+{Q-V%ktVYm30ga#4++T1z9_JbPA4 zxH2xzha0~U4KiHploSrEAyJg2jiX+g#}FE#Ik*g-_rz2#!6q)~sV)Ew&^Jv=j0Z#e zTReEmeNJolFpNt+#D%AN{4uY*@PP%^Yz67<*RgIrp?)7%r|4(UA9%#bV;2Fd_R~f!+Maf%htmErrfsyDfpV3*QhmUC;IwaJlLM>h)r=9^UuQ- zPnqfS#KY*SW}t%);1=s^TNvtV$GpGa-!nkFd&Coo$$?qf`(EFmZExsly zwH~J{N*p#b!cR7mlEhhnRW7_Y6nZ>c$-Z{cI+$0xWIp}?tA?CH`x64qF@`zNM;+2JQs8IxuU7&ihbW61~TE7Z2>1%w*(S-N9 z$I1jaZrB~J4&M`kYaE_(kC&@o(L|3x*B)5^7Icq?4)u!P^8C~#`uQp_IT5nn2S)L} zmKgWgD@D$Zi+PxF7=gpp%vuI^y7uYh%AY9(EyUEQ0FYCOIV<9*h<~YEBOicVp}lu) z&r{nU?fuE-46#p{Q@{ILT_Zc&AktLGaE)_Pkd@LODXbhp7g>Is-c@V~_LR8NZK-#Jz#d zRYjm86q+T$`mt7;_4Srq!q@j4j0w2S2sL@Ybc%NH34qfpgMWD06z$+EMLXD~sk-11 z`Q;fK7_gpxANHKq%YFIJc`QY&6LH9|;E?~*4j!?Be0jQ4 zE8up0J0hkjBdzewnqdi3)!u1hv?B@sIRNu``IS5_1%Y^9p*MA*y^w35anSg16`E@N9BFlPS zM$aYSOoE9iuGOpOSE9G~Q{Zg1oXTXG=EPQ#*owHO;iK;w;}^!T#sHEohkFERRrh-f z{TO|*4>K|$IA267>dwcXeVm~8Jc*GbT%Yo|TaDW)^q>lR{{d1GqL?UQEf>=b<~(4e z4Q;m2W%C63Q7HBXsx;a&PCEKXpS{8<7_B($R4B@-p2zUWfRobPT#aM-4%(y(qI+Hj z6WWLaj|LkoKdIq2A=jEzlN(kn&P4~4S$8jjDlSSj&2s$DdRm3vm4!O_XR#O z2n^P<%IkSzQ!vPrKKnr&vjL+!VF5p5;~?&e&wo)1Sw!@rUfSv-Co=!ZXZ97@S*~J} zm4UGjjOnY_rG0d(iK}*saK7f6cY@``p*P5~~@Q`PIuF2tD+*5VF(tv%7$dQ&?*UspA~v_vjiP zrsT`O1ou%yt9~B|IP(|#8qOJ4JJsM%nA%T_>t1lHFKhecy~`eXo249(SJN!yF8_2( z(DX2sTu=2Hh2^dG5kqr#x6h&wA3>oOUVjz^#5g-|XGqpgUq!59l)56e`E;v-g>5Uj zm6TO!K(!UL%_q$GnV&dcihsUgP1Eh?-5cE-8REQaLzX!23rBDJdAIyL@0hb*@H-Bt znE&)#BKg~M9&w>viu=a7@Z0QipPaFcLpFxhq+bt}l{wW7a(Flx!?{%}N%CUeVt-ts zDD2>y6cG*DTRyMsQn37yXnu`(xEJ^Q13`-HI$(}Qtk-KN*E3^$AA%2RjW^ozfR;6( zBvu0J6`<9U!dhu(;N6!_0E@hd#r1YUlp7$<9w~uAKIn||cz*Tb-1Cs%FQEnQ(^k?h ziP7%(qw~6X2Zp@R)Cr;Laaz&+#((eljekr`UXoT1*v+tpPBl`G6za@1N4s^ZCm+)G zfsz$l1D``z-%-t}Seka3U{2q>2d~7ZT5+l^Bjhca>Fi2&Y3o9h%Gln`3gyvpJon4x zY-?)bC%{=Dy|~RH#3LEw&*9;5Nlm4QZq^BiCL;QAUQ7_L6fUvz6OUY3`G10ZmIU+1 z!?foYop^&B*G_VZr;c@&AN=BMOZu@nbJZq_cG& z?+`if^Kb1Nx}WZ)mbHwXI)5nBnCR@qds+6J7}}-+*6*Xh&&QIQ!YiwHKjyt3qTka4 zJOQqU4tRFtQ8)Q-JnDgGa>95nIMT=cR-C?PKOCT8pN%1zPe|be7bHLd#0f)CmqdR;6vt)s99f4Qwz*SuXOS_M*>^(+k`Nd42L2vkmpSh^R|ja;^noZSQA{j{N(fR&8 zEQ~Y84Y&2TA4C3>>bW>Hl;P>(6RBJ~&7i_$c;F{-+xT>T_lVpQ#88Bt9-eo6#Y}y# zY{QwUKH`@9eO-jiNB5drcuRp=!ur0Y1i8JvKOTB*PX5>A&VP|Z;R%>fiJ0P6O-}6gj~x^l-Qq!ah7(OF~Z%+t=AccX#N@MG-jr0 zT&;u$$N`T$MY9%+*b?n$qRTQ~WVNfROmzId-$%yGm!({WWgc>RD3y$(YVc z8QT|8sES+u%ZEicwvN~Lb>WLMLlnC}5;gM_(yD@rwWsj0kDm0qhhvdBFKwhRmP z8WtI5I)9q#X3NcVBS0ps?;~j5-Pn9ZHgj2`3x@o|wqhi%=Zrl1ev)EL$dKzLG~z6p zf$e>|KYE#RPGmWU={BIe3gaws6UD&Tl6Cnf=48C%iO|lJQZskcDfX_$s}7Q=oA|yl z5hK-%c9YKyn26h{qJ}rshR=vqf+LU7_IB%KG=EA7HNhBIhIVVy<9cmAtfy9u{hj_I zxD$V3;qi&T*^f8LSC_d*zlQsHdGNzUMhs%Kpj-l#phjhwzvHv+tWuScgc+>~u`6OU zrJynU^gL<4%*3_iJ02qzkP!)YO@acoO0-X5z9!XD)F@u5JiDTgwnM2I=InB`62TEp~;n= zFrp_ZA#a=))r;GjSqQfG3-OM7GU0;7lYfkYDk4FL^~mCYoc^MXHD7I=Q4P3hre^I`pC^Y@Rwsjzu;;jhzs*5-F$?o%co7%iHV z>Mt&N`T=ITo>@6xLgyM=Z_o2oX`H8dkDKYsdCGt0$)XmB+Wn00vG13uo3rgZ_J0{2 zdqV%GdFjUsG-BC&k+z{fBM&;y{NnP*wU#lhCj0s zVWT3plTmxKKx&Q*uG2fSZ&;Tww!4c8WQ^<7P{2w(>g|^Fl88 z&Y@HzFGMnkGZ(CRofUPap?`X(nD2@R!owrYTImKlB_Dl%212Xc!$xUx+0%w_$fKMQ zUA>Vu%KP}FY(E}smeH>I^r=5fsINgc_h2;k5f?Zw2XKD@GmxO(Lz@0VOj?zVxTvrH z@W^(#^Y0i^|Gke}#X(;5LLB=d|CS%?ERAEBgVi3V(~uNd3Oe5f@P9jwG)vTmbW+jU zmc$O@KJb*BRg{=}P8IoHO` zW**aazSK$+s+$Cws_OY)Ys8bjU|Yi+^TD?6`y2ZFm&o%zx`_u3%=Ny5b!{)`G4L1_v_Mmb>pnpnD7_i3Yvb&)>*g z=XGE*Mrr=bEzE-B8Ijt&QN!PT3Lm17EwfJ0Gj?7RMdJZ?jSz2qP-u@Mw8!z04}Ofn z27d4GY{!1tKCb#A$q6qx=`sA|e(E4!&zynwhYx(@E)I6;HGf-(7WuP?DRRbI=X*h> zbBXIZPCce`pOn>Gxc3VI^2v=M^^8FrI}4M1%xwJ@n#niEex!x#J+#*IA7I4`ZYW^x z%LRpL>&B_!kjcKb3`!8L&&!=614J0MA7WA+s{!Rz1zY1T#q8a0j2{ z08ubDcYnf8LXi+S<%Dr!OxUGqIG$Wnn7#y5rwo$xiB^)?>1@x|Y83Bgn^9I^$m)xkAq+apPAI=54=%BXp0lz(6gxfP4a8>M;VgL@ZJ*rnXLH7OtQ z0&I4Ggp-Q#7v9H@l#sdKrj3vFscv~6#Z4Tv@U|y538_-7&0fyy=0lP%JMd(DVvcp5 zqrEw4Hk@OZ{DhmW2LAI;p6CDcUi{Z}_FFvPp0LXNy6;=s^;}ZI3VjDgKJenv3yNGq zqJOm?BlkqU`-SRKlSN^wXQFf}ea$H;^A*vR5N(P*wm)O3Yu{hgE>9}UOUwOIe@KmS z0w&K6`LNR&d&*+knvUD{*PKZ~ck+qFYUKkD=Uta5&uWOLxfkEUylwozt^QblUg6_< z{q~9ee%$dWpw2yw5mOf9XPx))w#4x^HGl5YeP0i-IXT8-Kl`cpTi)Gc--Vse&q0wTVZjM88RbmMgLUStHF?}MSo0@zH~I1x(@$pQ8;|p+ zpse@&=(n(r$zNeTBNOIYn|VC_q*`IVy%-%7nPS}K_Wc$i)khvpq3Ll z$WgE7+<&#u|96|Jdo8KUc^mgVe}Bjo&L=bBN{{pWJ1Xy$YwLxG^9E~En6pDGkx&%V zWW%{>IHBpMa%2EndVZS;s?)XTYsWtrn@H-?PwEZ~pC4`PI*3#4Cx$WYezW%D>akF5 z+L{LilLuL-1>(_tT}1aBwlHO;==)gO#)dk#EcvWwZ9b!IYBX@{-;~VBUVkN>ym2dC ze>HV_yGzVkfV+G#*>=$3>g{Waaon=!g-yty!_}|x)rEaI>csqTvI*)`wSTPhpYbn` zyx`sjpY^$JqGA0X*Y>HFzuhxP=dN*PUYq$O=38PFqt~t&WAhXoNFEnfQ`;;Uu6am(|6q=NWSlxW zsE>Ge2?Gz}cfJWb*Vz7hZ+m&fD7 zw1@`nEp|WZ`!3W!`Y!aKSK^pa&^jT2`X_$yCwKm+`Ox!QPI=3b1N>9-$;Sk_s0V$j zbM<@VCnv&tV?9h=^vImz8&yKy#J+heANTofy4t@#I6oYENPlkqSj&ilLldb#6ii6G zm`c$afBwpmS3r%C%uE!+{$Hl8fR@K#RV3ig|0i0av7#1w&b;@ul!utd!827sxmLy# zhuv~j2cWrpST2i4U7PepD=;!x@#w9LgE!=ffSw?yjS~AvVvvj)8B)V%%p(6X_cTY- z+2gIK8%bYHsDFT?4xUQ-eRDWs88$oH{a#~dF|2)FVC`8AJt4<_Ald!KG3ui3q}K$$ z>+F2PEGGFq8#qFNra)m?c_QX6#5%=2M5)>w_e|tntK||4kA_pesOAj!i;-kxfkHV& z&ZHPLs*nSeS+M>Nbne3QudC5Vjrv#@oUDN##J}a;Q-3ivqaE_I%mZ>IDeCQ@z7^+@W-H0-O+({`4!(HLPuY{@H z>LXjNsdjvmxzwoT(s1f_-hT9zqVCkM*MDQq1eWIzBiExL1g45&S*UE}qR0a@PxSk4 z4;DFfG7_|ztahkFndiZ>Nr-L8?ZD^79;hDrOKi#+L3^xRa$iw0@Hum|$KR+MC9oe} z-4KiQFSN&J+-l&pwCK~ai`r1u`^eLJKG`Ao(HH!Ejh;9VCp`5qVHZ7R$37275`Uu} zX4lTaJy`VPzRO7}6SbfveoOar2U0l|eA(7q4Dl6UYl+V#(e>w6@(vq7gs z?Jjx(2JMKW9tb(|(E1*|3rr0AGJ4J19@qH8AHJOVvd6W1T-z@Lx4p-n8RE7V93;dd z7sTY}912$P#_jbdw9A~^^*qgkK8Cbf_*JTiO?^NeGSliOGu;#d^}cfCzJD(NF(<9( zrvI3eI^xut2du%G{qr-2Sgbx3#4=kT9!MsiP3%~@w~bmjGzaI@J4=s!2gvb?q^*~j zo6_YHykc^_jC*c8Ap#Wkc%qISYJMP2hJ9YdNa-CaN{oG5SnE?`ofVT3_ODk9!->+| z6D+&cJGDVw<;Z)#17pWCXMbwL9-l(n9$e#mKB69mQ%8MY&EZE2>A%dS_hV1adBH^w z#4z9eUie@4(C(y%+}p69-p)bya*1*2WS3pDF5jK!^O~gd$^_J2&ZqMcX|BR9zjbXc zy@eD(38<6Ih|V$*=X2NFZ=|gm6q(C-eu|gkV9oPQoQJ46MB^pr#(#L?mkCKRiHTNj zKx-c8I)sCH`2O7D^DwX{?T%h=a7HwjB)Mv?LC^f%=d0Vs5w+ZBP`hr?+^)UJih{oa z)D}5wFCjNHMeP9>OtFHteDDu^&C4l=7`gde&ULf#cTg{5J43&WkruR)Vh2x;vkm=d z2fxgJ>N)-R^<&rd^?&Wd*+0AtJ!Zezz*8-}udfX~RM~ls`+Mp9q__2QG50@uPe&f@ zxd*z8)_uJn`;yv}BQKw6KKFFp63CuCja-w|pOqU(cFkntWN+cYtQRTId8<-J72&oK zw5?F5dkr?*`=Hor*=tvaZXv$|QXf(th4Fs6=zRpF0*Vdx|9^yp+h45jJ5wK`z_=av zmCgL6P=TqQB&k1FWa5ZceYn@sCL*m?yso{z=S^~|r(@ddc4;@|sIQ$$)|rD}T{HDU zX1a0?@taZsy%lMxmwssiY#!`+-pJG4Hv>;sXyn(;X5EAieYWzXC$JWaf8cHHP>YHF zflu^(gWV^Xsa2i@C0>&+4yYN^$n>VIm-lk9z-@RB*zUSsVUB#6=U z3U&^)8cM_|Xqf&AwR7aJ9(g}UoAlFrlX;>&;}ST2_?`59sXw}JC(SYX8wR@2uV?OO z;kd6>ZxyLm;lfa6^=NT0mMvzpOxN30l5q_(fkpn}Lw3!qBy2ZXvO}D?W{H@@o!Y;> zkVL%)&VQ7#2KTw5Pv$nAQ#tpIA<4nE>jWZoOT2w9e)vsgAG6?VP3WRAZllAS+`Cz20PV zQgPan=S`Ut(8!}$jP5ZlYaECS@p@g%L)E&2UPF4;|2<)QM?LJHT1PR+L1$y3bKi}prCTaX;^`cpKb%h;c+AmnF8g|k(MMg-2|e`> z^fa!J z)8RysG`@1%EP=rwV?u)yCt;DO37msuv2qt z<)fe~K|e#2+o{?;K7Lax_>oT$fOms9d|&76oQMAtp8|WGF6wr0s{0-E;(u?vu9Vc_ zB@c(0EaGjQd5h_WS{}|T2YGb#(WpqmB!PAHJ7-QebJy@IUgnp)IO!kihchp3;Bl!@ z*|;w1aar%>AN;o-f0d&V)>+UWJpipBG7x+o#sg+LC zBOg8^3#@bX(WZJDc%{E{^v?CHL$9&gUF?b9dT($~eVb`YlX_3F$A5a)w64Z}^h{82 zT$jR)-=*%X%>r7=4{D@vwfl>ki7S6aJFm?&J*eC>h7&ENplRo}>QfY4d!l@kHVwG$ zagw&@@tc9KMn07L!-v($Q5*TlGrp`LU+{WPy&Z!eT=#XP3ti-6eD+sf@}L{AS+F90 zB4A4vt3vAfNhQr=n1AR`pEL_s%fMTtUSu5sMS=%+3u2l%C$Qlbx%`TXB;ERS;t*r= z4f@nL-+~>uL65e|A%+NLO`zbOC{4vlRl7;P<_0xmrk|v2d@#417Io?sdd2Nzlk|m! zcJ9Z`I5gEOC44?;^oZw;8S2X6Xym{S=%<06&n?Wr=Z;w3>wm%Uf%Aj;9Ioeq%YW7x zMNFi}lXkLvc;RvM;DR@U7?XiF?&|&1oZ0DI+r$SWcIlFDFfg!r0Pj4apuXYzhQSUZ z3e}81PidSkMQp5L#(HJyV@$Q8o#PKW!gyYU&(RLWy>2bYqaAv^$VJhvT+aJN@4=M* z)K!)HUwSYdVt=>eh*lwsQ%MOVtk9&g12<=GB-JfX!cqz!MQY>8OmIv7z2~wX(>8CS zrQA}O#7x&Tfh-3-jo4hzXUN&y1by&Qo&1R&YO{WsyMM$mdNT9_*fR&^sAYLNF9dzlai5=agG;|X zoVOv%?0@B~{nurL5hw1sXGN2sq&}D`nUTqg)9jWSEn$&wc${w-@JHEBzFc!HJY7+i zRLp{UlL>FU>mGmH`K|UvEa{gP6!C1D)Q^F!7-;Q5xp|05&1lOL0*3iclW{YVve0Cq`F~Q;W+!8dJg5{ob7&i=_9evAD@O8& zEV5g_ow$g5L7838SJ;1UfAZQ#i~Y+Bpx8H-8e!_c=)Gl^8vN`WJO9Sn?(@?bo(-6g z>AuX`6>1{0TM~v{mhh^ogqe_ZH?=ZOQ`lC7UcY&>!>3#=2_XeMaPdALHPwf*S+?A5 zVt-QXi&oRKUoG$9VG6omKYWer$3LDWc2N(#MlK7k|DRs;^m>SC&u62#mwKw| zr1duFhWD``Jm5J)$bVIYv_@oj#*B?n^YB^N3N?mGN}=}J4D&gc5n!M{g*ML*HRx1A zHqxM!XY)OeCz8>Y`Ou0C`6Ph2$>D0$=6^ZYX-M1dDQ}I-VfsMvoQK$dLrpl+rq9&M zIHC8Og!Jdft?|6B5&A(gMSCDWlcuB==)lX*2VVX;Z$9g(4S7+SANy;-`8g-waiE5O zz#9U1`9az|7RfXsHj|>t*BtZRbbmr*m7-Q{l#I=c5FRqq%t7bt|Hh0>C50iMQ zH|{=jTrwf$q)t7X-OV9QmW#p{xA*H=SG@{tKKw)W8N^vguitpIs3xct0)Lh(UQ7cp z^-Fj|mNPt;ArU(0_QsFRyGo`Xt7JOxYUiT^;_Dw4dYsZV$9|Auf1yo1+kzqPy3m{b z;fj=Kzi|(KmAs>E4%8Zj|EAkXC|CbO};v7 z{It0XZl+JRDqO)IJVO)!2}`UkW&7uD(kL+TmRDO|Hdx5?Xn&mT`+gM`j(p?4 zV_Y$VFN1Rf-)T)-rbdVP|Ztss5W+F4Qt8>+eBsj4{Q@@=yY582oA0cy3(a%LtzGn>05 zohmTyuf=Y%;O=sk5P#Go%41<<%*^<`&za-cBt++ib4l>vlGv_XJ-zV=C%oOPqH)vs z@A^Xt;>oJ{guVWr=^XQq&FG-%3iGBxXA6bp`g-IAul{tt)CHgB2Lo0=XN;U;vwpqx zIPgC>#J}NNCU=n#<8h*-X(Bg9-Iw_@F&X1V(x=zTmq}gkmyXKv>)A73a7=x8D_9GUVHqZQ?xkLtd z76@(i3kCBX)>T2ivCd*W(x2^6!8wNi$A4bB%s~fV=Kb(Z^uRZ{=pnL*=*5;>%GJ`;5! zuHgxE_JwEHZ%wm|U=k3U2P_sYwfsFmav#-rCPLua-1n>sgzh!%LU+MmCDt4}j@*h0 zwx4s_`hP0*>qp`^lpJzJHdzynqnR)%XSxc+Sj>o-9_GMh2xj4Tb;Go(sIg6mnsG)6 zPP6BbPuC)uK5@VvU$lL4=xf}i)M^rgn_@lZc}bmKH#KT6_!V9La-j`=84n+hIPzK; z#1Zq?Q@(ZKKmJn>EBX%?P!rXh^Bc@zu7~#~j(={QT(r3Pb2X1AcPT@hoQI0qsMlY# zG9Uysw~e4h8VYwh3Fv!7K(5{RU$C{#-oQ=~*Qym_&YqQOV>2JDTxXSe6rFRu6cu;l z%EoE*PEn74d>hSnmFo2mbxH|AtSx%{r0Mn%53gA+_kYFUSo(=lT!*CEi+01M&Q)C2 zNPklb==J1EK(814F*+!!QJ;ZHm|N^KeZNUkeH#^qpk6R6`mr9cou7X4Q>^|o=5*-c z`R7?-y-q1&z{qPn$0kQUE83CvjAfA*-Sc>+^SF%!tdwdnk24x z*$A_F&X7ZDlbfFJ=j0BL>sBq|M8urK)#(2AumQ5|?x7%wx7<9c-9*C2*A%sx*)E7& zOyvwDqfy2p>wY7DblIOjI=K<%m=~Ne{Znu0k;~q5p}7%1TJ_6+`2GH@)?+`a<9~Pj z4}F+pth!`IF3%WDbDOgw?MRWE(dWx1_|1H_#0iBnw?4T}u5-Cbh)cDcG22 zrXsic`DzB~J-7MDFVn9i1_pLABmKE^*E78{zv!#^iqe8p-Ne!&r!Gw%u)kdAOBeZ7 z3FKaJB@k?xxQD& z_$a{mC|)#Q>@r^e*IcXbsk_vpUj2TaZO@ZD>QYPRw==s>asBk#jr$_%qr2)kKbeYB z>SmBF(_wrcFxQRP~1qV2_qgN$1pGvu={j99{)X4!N6#8g&Yu4X(Q$ zmwo2Q(>naSPSwwux2IfA4Oq|VJisF_R8Hz3Z1aLy$N0#P0xiI1uR+V0~IB zdG2>w^g23tEDLIj{XLGj^q=-p+u|JGU2tO3@kh+5OEQDC*aRk?s zFeK=s@Eosn5g*`1X-QT#CC|jH<-!?yx8Qyyi}cI4P7JU6TVd)ADSwuD04F7c-!AF& zh3kw?tCw9kJtB}9pIKTJu=iO01sjumrdL3ghft8}%wRiF$CDi%WkF!J;c-BweNn=Q9 zCN8HYbRA4tGRA4+xPKbYRgg|d*wquHw~N+pdiq)){{Uo-$!Io1)(~ ze6rsm0{b_5yk&))y_)G0yXAU;`9oI6cd`G(n{?r}vj;xP(tnIB0j5U zo`@L9Otm}ZF6?IC)~@?vA13W5ai>kbiVv^eNTx zveUb|)Y^BQ>Ly9={cZi?(v9^$($|nae){*S_R8`)m}hS*`cenallQoLSHZXb*d*Us zC~V`D*Dv7{|9{Hp-7IXI?}zT{4hFrXK3@h%9xT6qW{1Nyz?A)hzkRXM_<8z*=c*j~ z_nH3R@!Bu62mM#-48H@Adw^GXM*rkFD7JGC@Q%NG;5oqL`S-};^=A0}YXZ`SH^>a= zEOcnOH*JN*AYzidnF@PT%KDyVDvjW2lu zP^e+~k1t%hcP+r(%=!6GL0;k&J_MM_E@yH+oq(w6L9+*3-Mi~aU)%UX{q4a5k@L+4 zyML7S-+N1pAU4GVO(8JQJ5P$=Sp_RYen9|i53>wz8}@2u27W2_W@c|Nt(c?dD;RNh z|HRwL?s+uo9|`YF2)G<-m@VMeuopXff?vRDJKMuA!x}x0;XY+kJ-?nTyfPY%Mx(3w z=z2biM$_j}{oUD7a5H@$b#J3!H+}o=_J5;bF@1gRRwI8teR^)k-l2Jl%<;MYKYTR+ zjSK;n%R;OYFv*<(wijUoo5gBh+K7DFK`y@YY(O@$<2#v*Ep*mwLd670L02=&9fho0+Gv?-Cun^gv3~}9 zw)Bi?!0zy8Jr-31v3?4XyWz8p2Uf1qTyBACXV$SICAzQ%Un`xUx@katG-SzER28ywt*9b)er?>C4W-r!Y6;yuG^HUdjYQ_GEUA{6+ye8L z`($-(3fb{(xi_W}M6al8wWzi-gnvmYOwq7nvr1?S`kKY7NMx(T27+XA%a8XB5;7zYw!26m0e(+}8j@oFC_keLmKFw;m-?(=`{Vgaxo>>rn)&L3n z_fpp~uC~b?JoWDJlcIh3fX?Y*|=iU0RLDmt6c*`uiWKkm4A?`AsZ$g ze-my&gp>#w0{2&+JN*GtI&v!$yoWI17ToYSrqDHJ6LK%Bq-nt5Yu$Kh zwvFbIc9y&T`yJ4XFLw>3?=qp5a%IYr2KafJ1qgo&+~12V6R^#e{$@p^ja*jPMY~vq zK!Ua@8csInxl6{Dbe*NC`hRF}iSo$ka%?H>EKMRRJFLV+G}{8rULGv0W)@b|lCJ)2 zplamsoHIS1O$d+@_eCm7Jf&;1`2doQ2|qq(5QfrchG1FxU2Y|SCdc*FG=JF(Dx_fI zgjz@%9B0;m-e;7EvWyDpYSYkREdcpKwVbS>u*GLvB{;7n+bmx!bbskCOXJgLi2n0E z6YJxBV7SlEUI6ANQ4#aR!U{ecnDqN!I>CWq-O*X<73EvmiA_-Q&jzMVqa}^Eq{>qL z*$`;(?@LN8bE2SMGx&2C(bX2TcHaASC_{`4`}Grtlv?ikFAa#D3=Ic&7Ra${c^Vlh z-%x!vmX6$oTS!9wnSZWA&Nn>X_#s4_B$|X6+UKyuLVIPs5M{ucGBD zQ%@It`!_VUB3rX*1K#AIw<+B z0a0B1j_VinnA+hUazFe(uK#-$-{JE=q8<8k?R3<#3iq9c41X(vI5(V_)e6^PWDwcs zU4(PX8i1j&iYe`N+KGV;GT+PzlWsB!aQ>^N5Ytd`&8HcxTewx zEUesCfca=-^Wk>=oUCxXMACdVBySqcmq9nSF6~+<(vNTpn=s^TxL!An*2DF#Yf0Q+ z&%= zXlL^{&MTbSW^Fx*owgi4p<*j&5B5Fi|M6$DwxQ-4aoY}qwixyO6)rqN*V`DEF( zkN`g4E(|%96rWEwjaL12Kj~UR`|B@Z$i;9S$2aS*2efNR*I$1NLoSEwIKEYXJuI!6 zhd?qqwqa*Y7DAR$g1(7leAfG3+}b5|mZGi@T_hqA1a(HNbcJLry^RrJy*kb<+T>8q)h)-uuLKcu%7(=nn{W z_qi|2$xc$3mOK1=&7bc{;4AJTPmVFR$aeaJE8u~pj{1-xHmw@;tMMvSTtaHxJH>sI zK7XuIzxSK8Lf;&D(#dyO61iCjZtuVUw#Lvmcc(ZeW|^yZ4J=u6@+AUTni%3!V;z!YvTX^T*I+V_vo`u zuO;Qml4R)*eyQpSDcM-j(Ef4$=j)13{paiCf4bjJ(p$G5n-5s) z>mqTl$!0}ix=Mbv3V4e4Rd=18mCZk2tKDn=kJrYuv$O6iz~|}rOO0dm-=62ol1hCw z3~f9S$1xoC-)Y{*u4Oc$r*+c6Cw~h%W#$yVIQe}o_0LacGTB*rAOt||Jp7=nv2;Vc}k<7W>)z~Xz81oum7;(xn7ZpAp8 zh*&Se4;TmK*e2zFxsHDuc*^hK27h1RTqpHs!=2u4#)(2Z?P45#`upLQ%)$@?j2{*l z*YIi67=XBJJ{tk}GZ}v7|Btu-+JT;D*2TbE8TiPCJWx(+pIEt-Te+nZLpkRxWA1&R z_x;xYFJtc+*cf2?q)F)^S$|b5s?}9w)w-6#KiB2A{Lgj$FaCbZ)?fUeviBGNryTyp z|0$>I`q~i${T(~hH2u$ijF-y$i`HM^b;27}_?#JEF%lls(^x_!C z`-96h?LDXP5AMEihGE6G9CB}2{9AVJo3Z}6UccpkuK&f$Z#nvl|9@Nli~n2xi~n1` zUpg4Ze%G;7zI?Ha@JFW8A37LTcyV$4@+JI(TmSO!{lTq&`4|7-lE3`3e{tAf{{P~R zSBJjb`tRq)hk4)MD3xTHw{7aYdUr0XAiVhd?;3sE?5Fk4(?@^bhyR7o-gjd6UwHT* zJo|5a`1Vh$_rISV*?%A1hL%E#cTdeg7v`mfybr zjeTi$|AiI)^n3poIm+JKH~61&sb0Ey>L0mqpTGEjWFnVu-~YnaFWrCczp&~bI{t-S z*R@>#{_oTnNg`!|DGl+p&Ah)^fhR93Q*|q&x8A4z_q(G5F|V!aer+C>z&2ffGXMFT zhjJIgWZe2NDN{1@YB3*c@tXIQk15ChtZ!`9O8?XqYgv`QhCrF&?)hSv?R}d6HDi{H zrY`O44`*ymXqm9G-`{`FTCDNc$`=XFr81K2`8CEdoE&KXV8u3Bos_LbG2F}gUCe?d zSB?KqJ4Udh*M<`$UdOS*>%X3N<^xsR z)bHluf7LCPW1N5E+4BE>P4GA{Ox>%4eLV3ODFEC}y7)oUt`EOui)?R&p^&o@WbJI@s3 zT5R?9AHQ^b-ndlk`i>i(Z+do$NlF%a(pPq zc;OxYdh_DQ3=6&eN9D_h_x5wVNUw+rRji##c_*i~CP`e@kZi z|A$`u59nH6IfkzsrT-qUl`lOq6P*n3%2Q()^=%jbGw(9PI{(P$&-`WYJW4OU|3$Aj zhH>9{lc2wP_5V$e%GaSazIrph{HN!O|D%`x%=3S{zNl9ZpqGyDkDR>s`t+_tnfaIg z{aJs1==w_!{MIApFP(q%Nh<%+BkHd{>VN1w+4Jwb{&(Z*|6RX-^&|Mh|C|3nYX9On z<{v#2{_-RKS$F@R>EoD}59F19{Ac|LOjy3(fA~UQ{NHu+;{MWQz5Ct%wu2acWVrIL zo@9Rvd%kt;_3wTD=;TpquU=*h`@DMlcRif{RX+|!pcuPk^?&=#7>4-*{J;Fi|Ks1G{U3kzA1|12zNzND zKaY&}Aujz9-TDsQGTa-#j2T{Mn`$vL!@Yl|J7#<7(av}M3XJ})d)|8YM<5KJGA6^| zuRP4xAIwAV^ICl6XP@sTf|hYgR(l6^E!KSL_ZZfG&n0^2ON946mwqoXfyKse{GuPj zW`EA>o`Ue_d{HTU{=|y5}KPhAjB*2}6SfGZaD_t<3~ z)3rojs=7=PBv%r*u(vCwt0tynC)PKn2!SgW_RkdQ)&#+LPpT#cW}n#lr%y=0*+5)V zsj65>dHWF~MBI`<5F^Q7GfDVFKlXo#gX>62n}~jP#UMTuT0Rs@@3%cAs9mwaR=rsO z*#4@LKxxB&^{?Hq1Rb_KW5ECifLo3(#+Y|TRUC9vct zO}NJU;aafdF~9$){a5gr^Pa@L)K9~R;{fANCB(pOqq(v4$GMv$@a7Z411qK<@QfYs*C36gFbJq(ty=;$jfU(Oo{TZKjxQQL3*pK_AZcZ=R{o>ia z@%DS$+8#%7l}!|6=yo$L?}V_r{BeNd#K(%O{VId`Mykq-Y6^D2J^DMiL|;^;CFiSF zR9~2m2?|NCJQaUbN*m=31%RT71QkMTK3C$Cp}uWslL5H-0kf{)$1jIq-L`v+Sca%q zy}RbRD4qi(KV3>2;>HLAbf`*#O6UfwE@)59?TGaio&egzIb&w^FO?aLNdb0bH!5Qo z(XJ?dm9pIaouFR>Ku~;aiQqperwvA0Mm*3g3;Dp$eNumKaH{XT=qFQ#;h4x?`(uA2 zEgb&Gw_}H%BgYtN;)$X#%oaLYPh2DxhXX6_>lX+7-L@Y_+&5|9-x}-r!D2_N(?`KC^#$VJLX$)dtHW{Z z`)&q3LPCE?%oua?N#~#F|4hhUIk$-p(4SAtbUqf_&DLp=iw=_A28d6+C65aN2?3US z6X~r~wo_svJkA_kb5rhKbUnf2!->tg+@*l<^`E*Nzld8B9$Y#w`ynQ_WZ2_LBRS$$ z2s=+(KFpfoX+2Tk7_n8SYz{Y6QD-uA!~;STN=twEtRJ37ZGzBQvY7~rM(Z*rln>Y> zAx-haN37YlcP3_4tklZ)Rthj(u%s~+IDc+Quj~z#*sYSzQHPC-^4-^+6Eilzeq~%6 z_)GjXUINwBf8X87kVMZFz@c*0s@rDlEuJ&rvF5jqtyEG9QTDB)n@zEh zKF)t|x~><_y)mLhS=q}X2U72`q{88f0FhHgcmZU&N@D0Lew%;jSAg3M|G5 zD}oS&UIz!+5WZ<+sKmD$pKy-*T36IZ8(2NAXLb8jyYs1nA!{GrS-;gs0rBoq^un?= zdkq$#I;Z%-7K=ZAX%GdTYlV-gHwr7(DPMo;X^17ls-G%{`wmNeE|pM@Al3|B_jo7g z36g`|n3_nR5~Cmcfnk)@69(B_Q1kJ{lXz2YOK)lQdFCu%%mg$x&1y_?h`C3**NNLR zh{pD6tPNmdp!Zy}h{mGMTP*8uUAifCCF1RbgbyuHPQXuHG3B!79|y4;LDe~n5r2QG zhLB1a*?@rlR80PbzCfFd4%O@KN&UBKL;D-3n!Z9c;RiZyBMGE{>eulP*-P1We3DH+ z5}aI|5z_CSwUk=lGA}q2A(*9c;O9W&5=Cp6bC`v!w1q;ET*7tgqms~2ujN#Yy9?Z> zxe0~*yihS;$57ZSigTStv=EHW4rYJnBwx1WmfslFhj-ml8 z6j{@{%^Asf>v}degd@9g+Y^cT3+`NSJ6Mufxl-(=bvfO$C9ajr$r5a5;Vfz9WDBmw zq2}Zpu9`&T5{cFpjywWEGzrdd2!&nZySB|A`0cx^&JK(YZ(I5>P*j?rnH-rrMqqoSumjvLO<9gopBSdaJy~5NczC!FYpU7Ve#dq~|gLI2R^FN3Tnq zD9X5MIpZ0uu2leT_X^*LCYgT}ej^|w9aH)VBFIwlZG{NJKZWWleI~C~*k#H0(iL%Q zAKVyAU2s$~&{C)y7p909ImhdsZEs<*X$t*G8q3QnOGEI7PCFWNee`M5j~tR#=;^ru zziS&;3)j+5eY7M56FQp2Alg1WVva9=srSsvy4k1%@zZDQOyG#4-G}-GMsyeF^GZcgp%AmALtri2z z5Zdqjy7oJyRhaOqE76hXD3{y8`NRqGMww;L@K(`5G^Ndz4c9lyvuZyxMqVsbKD7yd(9!G z`JC1w7FP%4ZKrc#JndGUx7;#9cj#Y46=gFjh_dA+2R>PG9JkC_Si6MVu1MDyk0#E# zDolCJ(tKInh)h0?Wr_8=T5j1~v)cxkm}2SacDPmk_}y_TfotVjfVrTPaV0GHNj95K zEUJTLT0KKPTVa-Dza0Sv7WvKZ4c z7r&F{n^h%o8F860>W$HZr|KbT*v8bjH*vU&h4R&jK|*ooNhxr@!{5(nghPLG2KRKI z=##<3x2u0=PqR;I81$y9=ddm`Ae>-xbV^8G+j{Y@{aov*o7${~tVPVus<=e=2JC#c z`XlD|(wbPFLXoJgp&%bigqXfEez+=!oq@BgPSZLg`crksvvgR~tp+0XGi|ndYTMe7 zBIQ)Oe8%7bo7PMBQP~`V@wz3;X*z4xolS22vBQ7ty7guxudt)Nvl+`ps2@^zu&G02 zmX6c9bTpaM3|60Am@jZa2_SzgWJvNgxS0_zDNM$7r@a*0H_Ro@dwLY_wD;SX@E}rs zoGP*<>3gPLwL+kU{?;EvPbee$Y_P06z4=ISdG!h$W9OK&1&%jsvRFmOiY@&<1d5y0 zH{X98qrPo;Yh$YvjY&(BZ={y0(52r%YuD>K9UI&gVLmi+vc{r8#5|@kGdAjwxnM@E z#)SJK-C5IV3`t`zba{F{TIVriKJ&J&d4&}n>XAC8)}+>XxS_vT%33z5jk$d#>kw>= zz6q;IbBz1H45s7ODY_b~VHn1qkXrIyQ_2VYvWmSKd zsj~<_@fX%ZAf)o6IPSHQmx|f$NKVb*6M`~1 z`tYLJ)A+G}B{{bp+i5MzE0#TXVyoX76P*dk@tbTEFx_=uH#J&0Si;v+9}I=O7SsX* zOd&WwsBbCAH*2#Cly~5LI)fuWqiqLj9?R z#o>4g_#@1xMQsB4B&`mz`s^s|Au#Ga%(!)>7QLnbb(W$IgB3aYgU^BZBCvn7t~4i5 z>MWfYwG#K%?XrqYK@umLg-Ty~g0gX(cWMM0!`L!KWAu3*6$LZ8dm6g73C_9{0iTWQ z4bIC1tkdCMeosWMMNyuqaUU;dt~xmpJT#%NurY;2O4Mh+ zYPC1hk%wWA9g7KI9Gpt~?KXgl=x8RX(7UwAi(8f@)vIY2`4pz(fpheah0W3R3B-g= zjhH)04SKrG@>yxSLLv={tz^#Mkbk(W>|$(EF@9bOWFGs*!;M5J5~_a)Jw7vkV2A2R zZ-;h0iY`f*JER&W@son5`gEG61b(#DLG|76tL20VPZb8g^itPCf6=WSVe&jqqCg55 zXO?8D29T8>9SR{wKi|DaPm^Ll)w5KS=rgnr#*u4179a7G3xW#OB(OU)o9}GGhvM|f zOVf3{?Po%Z!lVcl#H4?%Oi;`9p9N@nk+i;ryj^>Xw~c=$NT+{j^9SNzJUDun0^4{Y zz>;I1$hBH*7d_etBlu@J`Ii?+bX?uErGM2rf64F7@1v0)Va+`ow4Re#+Z>_AIIjT6 z?}S^y^J*;jN;d@6&RhzZrjY_vk57oAko3cTYtu?#*XjZ&26{HL#X5ifLp}$Gs2?8GYTGXQINdkYZbepm5IaG;x2cF@EV+^v7)c~ zjm=sOYy-Sonh2n;_$7VNb$;wP;zESmMcGay_RZ9uLRrq8ElToeI7g#K!e<)kKh_&I<0l(bE&)tP$-A==Ta@T8utTKCsA4=bXUDBlI3WBm@J}H^LKWstDijLYQc@~W7<8og zHLSS|{fx?^H&$L#Y>QwZt;zdRcCk8Z2|zOr0}oOKZ_#3ri_pkNyV4xQ6j|HwTGt64 z<_3W0wyH^QKDbFL!e=Zn`c|Kgcx09x_E1eCSYt~EcSDgTFlb+^cepxK&vmw)Ij zmh_|1Hq0Tu*-6$Ah$0&9T^1#pRdbc8X3f{qA+jRONUxkqRb}{F+_W&m3yT{xnH^7b zRG|O_=ZV}oUo3(~mW zZJX`|4ru(4^I>6Iae4DGD60%WxZU6AU+0uLHfLRRriVj;29-r?lmrUDCkd)~`dBkt zVyJ%=YBxT@0j8KIoaK3`6gdxigk)l_LxyhCkB5~-FzASp2mI_HmbP;!*pdVn_*Cg{ zpeWUqSIL(%bd+<_)IQjcB*s5Yl1AxJ09=m^(bHFvFT9(-zy+im+8FO~CF62*b?smdRkwf8N4MUkR2ZywZe`67@3Lc6q(1Bu!*iW% zJ)w1PXUvk#tEvro0ZQwUXX~V^V%7C9{8bpLgRAAO16kS|u{oU|AyO+4X$V8!jiYYT z5(ZE46gbGL3Sj*q?)z#7J}Vyk(>9&gr7KaH=$=xO<3f}baYC*4&xYnShLSM!wQhfO z3@x=}RoQh6)((F;)!UCgsU9)vR9_iDfhX3;PE;k**Iwmv;7EXFIhxGPh39`(wHjuJ zT?yMCz~2EmOYASrQv_DTWL0_?MQmIRDZ1A9ZEAT>j3+Gc`ZP-=!M{q3AxMXEULa2O zLyQQlL>s{Mz{Xe4;J94#`N>SAdCY&^PGr}fIjC6mE{xciH#)i(R^nZ*9X*FKb%ycd zhAvvPd7E&3nG=KWdImTsTC!dD;c*N?*4sE}6{yo5?Z+0kremg?qArl}+Dg?XIQyQ3 z*rhN6_r$xJZI|Cb`BTr|qCd-P4fq36P#|koNNig3I0Y3><^#Y z`lRtX=!0e58jB)*>P~G2vIBoPT+}LCI_~(*HBYr^DK#4TIFS*EfBG{-A&D3IGkC+P zqxk%M2I1{>E=`Rr9Y;-{Y7|;#zp{22&GiD;~ORUWk8qCmQLn5J+DO zpSF`h`CHg=^&Jmymt|RLM@Mj%UV50-Q#goloigarZmo4bD`T)6AELJUHQ$)^|3U9+zORp~r}e1f>yBT3=kDwJd<-Oe(7 z@hj?B-_7V!?CFElly8yP4LK+~$qx5`j?FWEjaegLEK>qo7M!V$gIkUDM-W zAr@KqF}UxI|Djjv+$=Jx_3$L-)NIac1%97xGy4HgBo7R>rk{UlSN3H3V}+M}Cp3(a zNE>cCUNnbJeF%JwS6n~$DMtXL?GOy8%1NugN0H^2YaeNkprj}n<%%}!iFuTtEmWlg zvzfU%ctF7jPF3;sd0^MPdFx^|kvR~wcrR0KP`U0*N5sdd`8>7fa8=FMBPh?E!<}5z zBwp4@Qr~}tb%cspnZ&?U5wLj2=0R6HFV`UAVfgE26{`dn7&z$$$At>Cz2F545s5YGLfj9QLhXRYUt zvHPRA0CDc8hHzSg{H*#zA$eYqsb_sC2KfT;pgVtUJ@$c##&{iVCbo{FSCI!=IhWeo zmqt68FRVz-1RnwQ54Oe@kc@!w`;)oPyk8Eq#1Bpqeb$lJkPJdjkv?`L4tf=7=(??t zIh03hmajG>RV<44BrN{ho-hc|Jc?r3nyZ4Y@SM zdJTV7`amQcDEyC&60Ay=^&SrB*np2t!!}#^ofa9@BNd1p#9p|?iPCc>NvV=Nz)nzf z_Xa*+-AB{f)IhlFC13uoh-rR^Pk5lk*g>L%5i9(I$W6**8boZ^UdxcTAsM?rJr&Cr zp2HDGku=Zu06Yzb^sMrSAW1AVCNnh4)9!zNB7Do-*4(+YaI8t@eqVR%4|b~F3M>0= zwr?8$d~5L`iMJ1C{zwtsM`j9i41qT}x0C~Ql+ktBNlYYsr;Msyz z*pThm19*#=P6W=jCvqp65-tebBnOe|Xv? zj$dm441^efXb)j+B>6*)M3ga0A^@l4HtaOXQ(iWa!=u*h40G7wzUqI=!|-%D5N74J zor4@|K9rH^;L^}G#KYCU`I>^ZF_UlVp$FMQ^t$BtWq*o>8DeU6`6r*3InOsF-?Bc> zXU!JfOVL8=n>{r@uQ&e*BqZHElqcB_J)u(}#?oXa>1S?k^ICZjy}M;ye$ZXOs;Vy! zkl@#<%fZY+PcbN}r~H2ay&47UXh!0kdj@TLzaZ{oFN!YuK2m1fO(Q796)YfNxaSgD#_IERbq``YL;Myg2xSOU37mpx0+-jOOA&a46pkl!F5!7P68atyE zV=;BTgSxN*bfZd5NBreISWrAi$8h}%T_`^K z;PR5(JkhFy(cynb5ug@hi2PN2+l;T4)S$LVkb@OZBY^d3GCxfzP10}0_o;T_Pc>!f zH{MczKsjC!bl!{DJP92V;WHfod}f39j_xRJwYkXnD02pgx?3{E`7tW~H53PB_;7TOO^_2i3L@2$Zt*PwNwkmyav9BEe?bdu>;+ zFz8rpq2Y7RP0K~OR6|mpv}>Y`p~y{$p40J0;%&V&=i2pPQjz2r-9sv<)4V^%D~s3A z)jT(f29tlBmNj}vlOm0$giwV96sv9_AZpm#<3S3`p}s&ihoqN)UXGoiksW$+rz2EE ztLJ?MOVL~eTF^dY*RSBgk?dIf-m#Y8#0g1zZdS`_!x zSibNPA9-Dz5m0(|T_MAs-(9@0EXl1JjJlwN;%d?-LuTy={m**v$O%AT#E@b71zru(z^q}#J*xEn74e$jAG%w076WPi?fwb!spM5V7+++p$#X{JAtbrGlXjnH`f$(+U2^QDzZKKOIKs3SQ5< z{tFZd+>tRYzG6DMD@!#vqli0;>O_C{nZ}`pX`3p4UcfUpM+b*^@ivF@z^<4I?)~1Q z33&L!CJjkv)Y9!+%k&ZCXBn-a{4P10p>#N=KWaP-9Bjhc7go<1?&dySI%+_E z1NxY-oIRM2HFmR3Qha|vF@M%H>YtN18*e8Ju)%AFjex2VGm#bd5{FI@N`AMmvNVBy zXwZS}1Sgymb&`FE;UxTkn&N*ms*+~hs4c?nHv<{Y5gw~HzatvBKcVVs1)WRFQ_9hV zH580*WM~y1tp;mGU?m<(Nw*8~{0I0w6$6PbI52`lpG0I6mIC}y>aEMjIA88=Jjs58 zuY`T5rk06Q`g!^pFb3HNqPpWCq|Y&Zd0|qS4>;p#JNRKEI2EW{6}f+>kZxiiMI(RI;y{|_98q%~210I@X-qUM2a#*fTU6o-T)54W1U?ZjgLJT2 z1vHy*ZCV(#b_NqE@1dV|CHH+gYbMd$r)m!p`RC+_&S@eEy5<8UD5!sm!^Fw~^e>#g zqKhBa+!#<0TFYt=?Jy+EJ6!c3bMhbbg~4gDW)k?}TA>nFF;Rb0XlFPX6s&6f^hJp5 zJeGE{+E=ttu=yK5i}d@9K>N){Wtr~egQu`F*P(G-y-Q|}r}p#Z7dWBcVuu}@6?v?p z0A~C=1b*9DQ~`wlb3eyKW~&um*KbAE&3)SD=u9=>){h{9>hldWpL&RBZzSN3+XDVT zhKc3fLdaBbMOl9^0%TD`ERP7$?4|s~n5obHK1F%OZfl=)uF%B!fJyM3&>q(ZS#6O# zkIcM~Yri!6jVG@ddhVjnkUFEhx|dFPor_`lbqDE#Cf>1f<%EcBuH@-6)k-yh)7d5s zE~;(|Fuv&?!Hk=egTXgWFO!Yd!^!u!7h`-3y%f_IEp31OFKuSt*9dIA-@XrMd*x3h zyx$tp$Y|B#7kK5}<{2Qdt2+r!B$yg$DG@+a0$J;YW)rEUT}l)9kjo3x z!eqH|ohN@yNr_^togY6KJTVXI1b>J;3`xk$HrYt~I*&E#t$$8`Vismf2&aHPYu zM-YF;Jm>4WFh!)T581DTx1dTh$S5A$?L;rrZ$VK|9;M-#e!k+5e&F`Tz7~LcdQge5 z?l0fa&d$2Z_yE63-zj>tkvZNK#Nhg!TC(rD7_dL&Plq;XQ>QkX1Bu2<2Ozt;KWl)7 zh?ML2C+3P?YfM%Tj_4Y96{}SVBsDB<@0EXdx1P4zvo@WxC2+A}DtN&DK3l8F-}$!$ zE2*j_s6x{Pa>OU9>^{u$|hitVO?DMX2@qhi5>viv&4JRl(xWDDr=J zHxmc1~8G}J@#;|IcTXjd=fk;MJ1ycg03xb|#5^x9nK{Ntg4 ztB;9CrNlUP3Y4h(2eAkiE@A?YWkgoJma z`_n=_NfyPQL+9h;q}U4Ylg@CksiZj2IM0`d4(`#0*_{=?E4>s0Y-%n-%3VHo4}O7~08bNZ1aKTN%Dq6T=do@9Ty#p~YZPw5lg zXuqfpRR-eZd3JBJHS|MBE|jR*taiOQWl9S(1}A~=KCR1gcU$6W_bYHIEDU6?wCu;3 zI&kvM>nG5N9D_!qLv|w`kO4_xNlgjo@U0A}oBasqz{*w)&CN3e9}iC5g@;`Cwtel~ zakUe*7EYQlSqv*+;4yzO+VztXDte%iAuQ0$v{Q%cAs5=sl!&LSXfo!F$`qUoxLo@$ z_+BIG9ikvU>&44Iak~h&*=r9)_!foe5`$vA{~Dx2`||FQg^?_tUwCC z$O!-~n%|8@v=40f7&Jbhp~Gkwx7)N7X;)>Yh;Fryj8Ep=c{EaOhOIb^#GA$9LkJQr*EliSH&$6(H!M6@qZ=!OiFmI!=DlC@ z#HyRE&E~I~88Lr43tbx9tNi05kE>E&yIPgyzE+W=;z#8u~DIQoh%?} z8bqn4II!t_$3Qml?Pl$hoEScMhM0NqGui$3~lBus`E#gDf|ou&5wA9L44`o3wU%pae>KifTEVg+y8#tH!C&X=yMET zP{{J473hD~a;!>YO#uH8+N2)@2K5Pk0o+542X!^f$Z8_n9Mr;+5xnqG@RP(ez3b_; zZIbZJC20UUV$|?D$`*4CNy%$VRkWF!vg{#h)%pHnf{ZfAw!hH`78C@tc~fpYr`EFX z+zrl;MVUxxtF6f}Lf^faN!T6Lb=CADAUc$k0P%m)61=YatX$T3)Vh41_kKdLRBEcr z3$rbmenwN_>9HyrEj%zaPy#6CIlt3|{s`s6n8fiYjrYqPC=2HD<4|~T-AYvv)OAc* zLg=(+1V_tMS8ba+1W!e$V458`7&qGqu7Bmym0A0=&WzJ(iRbGzDojPK3C4$Aw&%3n zV@7|Z3lC>~b_fw{n+tAhD%s_k<4!T>PS;c~nw?XY==|;r)(shR;*l=cjeHaYqj@Hye#U%Y(DamwFl2+!(pC5BFo=?npWuh=#UTnY7;U|6E zyU^}Nin-EBPa~Fa5}}M3QCE7+F3XJ+H^+Zb>0E0jjKXnmJ#DvlEeeuE-oRZ^mPM~^ zgxnpB)1Qb5`1;gXMCN9d`D zuj`FvXyX(_OrT2XX5ua!xLJXeB+G1aFjenKMVSy`iNRO+Rm| z7YL{-*}ecZ>8G0ctQ!{Ot} zMQc*k#H&vdfe#yKKMoSOB!f@8gz`S8gQ)voqJ(;qRv_d*i}qeQe8yZnA47kU?Th5G zNoUU`x9b{dRHwpMY`b*07U}pbM57I`prkSalDQE{!3^8@=mT=W zHN+vFWp}kknBWbnBJ~*jQ~*vVSLVK3e1d|S##q~Hzk4E)(FxDuTC^9ab`~d*3^>@x z-VL*u+H!H}&1lbqw(oNpZV7*<>RiwKpgD+ST-WP>+MV`Dh38dV@(x9!8HUM(8j9za z)PoO=H#v}mw3Hb3IR?N{;MduvAZ-$cfX5J1VMgrYau%HjJ)Fg*r#sX-nbVrIHU_+p z;E0kIE~+`CK2em-Y0i0q9EvnPmqd*#mvNujr=H#cPnIJiPclk1?^b;LqRJJdbss|n|PlCS;|9AGy111`){Qw}#f zZil01t#dBtp(38P z3$r5x5A*C{Bu}-?0W4%QcE=(rNn?j7e+Tv#WCUCbOZJyBdtAgwq8x1v z=eqD94EZSQwr>nE&0GXWoon84E@MprNaiyxODa*tVhD?E%Rq?V)Xz4C;j6O6Lo z3AJZ@%xmPPGH{(b2GDyxjwNzMe8BDsO|dd^Va_jf5%*r*Xq;0^!e2mW4S#AdCEZf= zLa2?=8ve{^ju(GZ+5(B~&;2MN8EtM3os~_K`?#V>^k_i7;W>q?1cyUZURD~N@)$s6 zxcT$9e&grQ0Sw-DFqw=xtH?KR0-H~&rvt2!WBAH9&6iXxkuo9Rq)0n(%ZzY4Csz3o zBQgMvWRI9%P=sb6OHt)h?%bF9)w35>+|qzIsSY$72;_e^RSo*037S&er=#g#F_f6H zDg*>hlrmYx_j|dthwiZV&uVEY4yNA5mD8f#{IvyN3nFrNdbV>_2d7jG0c&rgE_e8Y zy-}BBgN^U_7cChY5)X%5C&jiUZ1CX9fGY?(F9EtzC9G5`yNyCJ2J_!?fPdKj|M;K( z>;HuL|Nnn1zx!V-Ki?|<#q!5w^Pgdc|0l(72QT&hxz_1_()#4@y~aHN_ma#biT~UH z@IS%EKWm2bkoPM|37SHGiVZXTXWjVMc!(N9m&$f3r1JRN{cjldpFkfol8&TnscZ}M z-+cd}9Qbc*hm#@?o;Lr4Y7D~-#-DTz<@=AY%O!t}#6Nq}mi=4X-+T;`yd{T3?UF_U z!*Ji9J&KY4iR~TPzpW;dV5s_&<^P*Z_!F(NM60#3`0P1i{tfUe{>eiAzLdu_86%nq zZmYli9Jf0EUlN34SiK>#d=ih;VHopo!${@dLHo(n>!rMN)ciX>YW`cv->Bk+IyM2y z?3;i7{eHavZ>7`UMpwtX(WJ`U%!`Q%iLPM#BWG-7F?ZZpZ(LW^LwHCeA*4Mo(vS1b zg)P}ASqo_HOH0rA*9D_K*ze;{$<@;D{@{VI`uNxb=y%?Ne$gP>Xb$tV zChsW?3lY`OYvw!-+(~)+ifl2L4ky@C&EPCS%-&hC;%Gf?5Ye|WnPtxI{!li(jevt)aF0B0rFS0?dt+<6 zsUgap34!%W$v2#0LsAJ}d9Kw=X<31n~;{P<>^k|ByrZddLl!YNxqt#Ns(@e3~3@JCwqTjG3Unk z-Pzb6v*c~i>gK6NG5sM=Ty`@7<_D6C5SmQd^CA(RDxjzoyEJS?^ zx5CE7uqf2e*oQ#fd#1!wf*b(TJEx6&VD3KCqC}SO~qK(li zK+wd2-ua~V7-Bz98ZW+R9yJQlsucj%LB226&_!@jE!p?))6Y2(1Ppw5Ohy431r=z~ zX;|csrCy7$o-qX>Tfl#A{hpQ2PTSH>EfrdJgo3m=ZaN;=ku7cwkXNN@>X-KH`AI%jmc zem>7FZXW0ogY{RU2X=p>QgE4JU_QUz2oQZ+0bz#^?WnRm=NfU0!6(n4lL*c{Go;GAJ5ko0=p#QLYYY~1jH<^zHxbwUvl9i z?z^R0?AH)vh>D^IQw(M<`la-x-4jV35;7eo*wvOtp7J)FrpV$Rp*jche#%2U%E^L405dB9zfl&yYq__vhB_!XG zu53KD9Xrfai4<6CDp@yx{}CAWAZ+$cw7OZ%8YbeMvD_K81)fmj9`9jBBN114UikLL z6E5oNoWkG9k55*eTm`|H%2}ig-U#1XcvnHOCXlN!#PmwPR*?8DDlrD*f4rQmP}))Y zC@6c(jJ$t^GkO^?+h9zABFdX+7`qtoSp%j)ews@viADiRq^!kW6}0^3q1tne84Y{k zPzGp0UJBRdK+`TaI>4yYmu>+MyrbtIHOED_eAc#n*G1avr(G`rljc5dC{SiNNcRA% z`SJd$XMp1YDSpvTm%y0hMXkXZmvoDLd#x}keZYTM)!JX)Nh7(HP*8BzU!W_rP+8SF z(sock0me+)1a%4fu)#WI3N|dUXy8pZzmh;5zAV&wKCuiHV06QZ8f4uA(DD+KaLmVx zcQLudXGIg#nuF#kv*RE= zjpcuYudVb zVi~WKF`<(p)1K^21=>#^a}qHi`kC*-ppSnG996Sy*HF3jQxf!Mk`!65<6|38S{@mONeWpm;bd+hT@GAKzh=fPfjI%(x*>$vi+%zPqYdnKk-xi%GZ(0`xi#^AVF8LPm=QJduHcb&ZU29 z>8`QoVs2S4umTz11XZ4aqQf#+DeQf+EpIrW8IM>rWZD}kU=dp6vsNcLP}B)jB|=yk zvq0Hv=(1kB>Tzk>=W`&yQwii@4KHY{h6Mo%a z2~v@mOOn4lbdM(jA#u+eCG#_Ux4wT65|l6dmEIRDq@-{JVdXNkK(z-QF&~#W`eiPDa7dT%(E z7(t+gv+$j`7c}AQ9d$%D0RTxrw!b#_jw0rCe{d3}0E0Z!@aylswG}Bv*M~EQ;Af|Q z)*|z;q7r&xtgX4XlTAo#_%Ofm1fKogeE|c3vD?hSEBG_G$|xX-;B8~-u^3G=ee%_D zze98@-~?R-bi$lW*cDJxpH_Qa>C#H;2szMI(5a&@(^nLgcF3 zWXWBE7HnwPl(ZrH%TWFP=5bFkN40Vg1buJVv^$!X4t!>4v&*ET zbYyK^(WWB@0lF-!&6LKteO07Lv$6X==j%{wvX6+TR5wh9^cs1_P24{EA3u?nbB&aP zJpyY8O6*?EKt6GTI4H!4gJ`frHEa0FWVISheHE=UG!7+gdA|Gi*|GSkoleZ;Jv~3C z-Fo6cUZ+Lf7UWCWyHPcQ`P&8I~v)ERAT|)4`@nMYkhxh35tInWFn*>2S(b{HuSHyLY-=OWbGa z8~QA3ah%+gL91K}p6FfQ&71*gp%e2&w;l?9m9}3N+&IUt6w}>;5Js%@obDhhckoN zMSB{BjN@3GXhim{zwfjs@8vV2oRI<}lu4KX2t9~fcXDCaj!oeBEx#eVF5YiRIjSrB z6gsXiy)oL;pKmkzVCy^Yy54z&`fpg^L*b!7=#5G5dRxliU>T0eX%zyL_84XtC|B9% z1PO_uxhB_tl`5gzu(Nfy!jwMHE$X4qhoxUu&|@8g(`^KV0Wg-$az;sFM4^IAX?&#L z1)ABUh_*Q&b_WANxT$Z~7;K48`?z`pm27b_Malah2yTSzv||_9$id>z=TN92IJ`ru?}fMSZWs z>dW(MR3`yz>8%ImYr)Ji6+_sZ6cc5G0MU`0lPJsOvy`oozZjEsZxe{mRtb!*ADPr8 zx>*d!P)*#zm*QohwnjC7$J$vmFN606)&+}S2V7gQ$Triuwm$_j%BwzX6g8VWcqAypsy=qA^Q-iezO?HxAK)Z#uGJ)$2mHD49CF<^pTe+SXojw z-o@J#m%?YGb2}hw#cY5sI4u<~xWBdPdU5ivNA%%oOq1vBM)`pj}#yW|^xVKhf@xxCf(EAGcloU(P#kcEzkN)JcM z!9M#*t=E0LJd!ANt;?@$8QO)(jGfvgM@grhDou{h@SEVGJe!2?NR9en+bu&mOnj}3 z+;#j5&uTn1T9lKYN9CU#F~}=JX|FW2KT`y%i&Kcuv`bu009E70-*qM$kiep2Dgt}ee(zbG&9{T~TDJ;CGh`*^6b_pScZx2d;UXg7t@vAG)qR}-de#}R zDloYa9z_GlJWAVaVQi*=;KQj7rD23XQ1nb%l?c2n-QMaI!_Z9Yao9t8Du8;Drv*ry z4~D7nbT0YvZDQ4N3u+EZ#R4Ir-@a_doz}Fp6vb8(UMYh$AdvGDLG88U7pu>I4NzTo z85dmq!DZY|YJ>#)?5klzTabqo+}sOxuDzEI{9$&tPI)jy#2F>u_%63SzFTcO{?ORi zsPI$_ImL}XwA4bl^P8lRX7AUuKI`;>r6z#aiQBS>%`0OBPw^8-5WWEYCf`XcR-QO% zbmSHz@W`>*vcj^kqk%Gy#UqV>9V2jHg=_Kal}df3Suh@Z=fKk-_$~bq$pck*6t1`y z{{`~v9UZt3t&Ofw^VO?# z7<(x65hLj_Kfk~Ok}W$~>9tJK;0Phe(Gjx*Kqz%^IHs@o@CEWxr%xQVIclJ@FqxS5-Y4f4^#ZYhcgXY*jVBwz-zgGa=K^9yu}_^=_=ZZSO`> z!^=~PQq64{Ir&PupxR}ObNzXU{9S|^BdEH7t>6YzhD1zsi(Re}Fs2|N%s^25Sw9B_ zu+j#!{(}&khQW6wiI_`ELVWQV&y90Y3RuM2KsYflcL$y`Y^$VyR2`J9z`|wjb(!Rf zrt-A2j#`Z=JO-+<{<7lhHn=81)6D&z|yk&)#)QDN`d!QIF1LsW#_{fc*Ez9}M zG08S|dN{3Z3JxYjpGwUJ!BM~Ggaf74TY#s-=B%ZKaIeyTU$wnhI_Ev0ek)IV3{N+L zgWQM@v@my!{Na84DPd{fY!+*2Vf-3!;%{^;e_YlxYUjfwc#h*qe~FCFyv#5u#unT! zUm(@?fChhH%WzxxQuPV0jRCa@IN?=XRMD0=0X1ImhX`dOJM%Ay| zz_p?R1Q=0~a-%25sl^Q*VMuazLz2k`4s*%WrA!_I*Qm^%Z@{(CE;_Df@zaz&<)E`_ zltS}0(C6e2;)A66@twtcaARLbU(QBzsPo=lM?Kwtxi;twB$V{M3c}zaPz{8=ge5n# zK&h@CaDtl*{lT0G{sdc#;wSzd7(zR?(Da7PkCkYvORZa$_McyH_N@9bp=hkB!#i+| z(``Jlmi7U-SPr@63lB$wo06UonO@3V?ubF+yd$@f>6~il)t?<@ODKD_}A0)kCH#fg=+Rn zp^nZ#n>{ow`x%LMVrnidg6Zy(lStsFnRi<{Q>!{|8osk0cCYmkR-kLD_8YsZwQ=nR z9hs_`Zz@rlPgZl|MPr2h!D`B&|8)NW*zV(hZSN9)Sh!1wHw17pS5%Ab9Ew(js7U2QqI`15ASgGu>VR!*dfFlPPc8P@Y%sxF;hmxMjS&b~1=B z95p3*pwd53j->d`^R?02blZxdv>9M8$^MZS*)NB!Gvkt3&{^1>`XJ{d7F2FvLm16} zIOu$Z&cgOnHKKPnidgSoCQy1OG5pB6W1h1)&?4~oxzqVaWIe0WAG}hoYi1{({c;DJ zmY=5$nUA)7dS@n9zdHk0)l*KI;jo+y3g|L8aKohRuiojr>gl~um0!Y!Je25!)30kY zS?Waetl3FsU)r^s?&SK;=G^DcFwHU({BCxffDfsJ8Rt^k3)(n|1boFN>v zf`uUHOX?1X_1e+8G~vwXg=;c@#sHYDxBO?wqRdX<_e7fxl-%4~ms`1`i>=hX1DM}` z;Y7UpYI1W&zpeQK<*n(#bQ8AX;giKsBj!pV1Kk7E1e2|GyXWEE7e%2osdqDd4$*pb z2XkLjvHNqmoryehNMMy&4VFG1PxJ}sA+34^H45^RDO~fxjNFyOkA}yd+dY1NdTgvI2kP~haWL8v zb>MuZ-kyc&n$W@oPkXQUfx+B;0`HQVJMkEpV4CVE@d66N1=@`q0D;%&;w45Vm65`JZB+Hg5_s4pL0C!>1JOiFG2T=+GE(11nsoGJ-cZK6zXg1_o z=K@x52)NIEc>XldXGjT}CWo zKb)r?D{N75LIHT};coNE=hxF=HANFl_D?FUu`Nz9?!H-Zev#D*NpUoDTruyU4cpt4 zy|TgIhO-PLAwQw$s`@%+)v5!0se-hi4C>PT!e-S@lyd5*Oa`@AuevcXuAmI9|gq zv><$cj)5gIM_=+iJB+HwV{x7S(&*1v^?~7wc+epQzbp-Qf%Hqp$zPxa@Q(;paCH-N zOoD0m>jyR{vmbk|nhxSC7mWC(b{Y7l7`({4;wV;LoM6VGJ>^l);FZMbtGM`Rzb>>i z+21mM0>hS%ZFMN!8zF)1jZ&avT@?>YI&y`+57Ip1(ik4kYe2y|(U-MVQtv@As)N%a*%gy`JP{ z-4EDdzH+R=7tTzYiCQS?@{uDq2o9$Xw!^xAhA$oy%mF>^ntid*g+da8m30!QqGRG0 zF@d2kv6m&FT9btM0(2@~KSVXbL2llUWytUR5ky>PUp!ls^LE_QV++MxX%PPi>!(vD zpvUU*EjfR>ndp=Z5Eywjq#qVeBR}*6gc2`dV)7AGyfLSqnUXn;2+5a7IsD-B=9V1KwWCwWP_< z)d;g>5>OqIB@%8T3I>QzKj@WS6Ze~c_T=_udEwBuLDNjY&WL-cr;?Ii!s){Z5P7^% zcn#Kddjd9#vQQcGzJJKzny@#CRE#oO3k4WuMpDt2?FtuMzXLa1&gEZ!qyuYxIctpPkPfe|df6jK6|;yaPs9TOc4?vtD}c4xr!Xb1%dwDuNQ+&{ znV`g6KLX)+5Mv39lij?7e1s}G`6C=PsJv#^rY^DLIr_0Go4|SLw0403T zEWl$;psSYZcJ%soxoD$+uvflQIT}!oVvSQcHV3EeVF658e7_4>L!oPDrTI*cC+oq- z#+Z+cX+#XtuYm^1c@UObHa6*hMT}YiwR-$df|12oB7My#&oMKTa9Q+NvBOY`pM$J- zNgOD;<49!sPrxq@LAn(nFJk-EUQ*eYPG%rxkv=N(m?L50N&(=kZ73aMoy6b-e1}+@ zkhQa*4|ckQ3dZa>R`_o!F(}KwScwDb_$`JyWyIPSXmD`{R93;ayp=?M_zeBYjD}t- z*z8eW+O3Ax%<;Sy#1(%&$e?l@$%A&PdwpAjdTkC50PT8UONSR7Mj6IkvcyoE)-EN9 zFE-Y8a019qg8x(+C_3l5Xy>2dnYBSLlEJw`F#Q0%M#OB=&;5h?bcx$c#pLEYGP5Xu zTQHBt09h=Zth zA~0`5xDaQ+8E%=xH#KF;Y)0yD8j=j{`z{Y$|72bZpA`5!tBg3J*i}FBpy75v3S40| zD@a@?3&Otq>e$nVjX40IRVWPbrKFm#R1o8A^@K8;_IZO%1(TtF;-s`3n$v~P?P(-7 z;hiT)!>aj@uHQDsZL1-#RRome)mg*^zn1-h%`c zj(Isv1=K$NB1OQYOaQO?>rF`d$(xxO%B2Yck@l+dd-!A|Kb@dP*M>~|whZ1$Ieb=) zJ}`wLVRDNc6_2cc9CpB>^2`r}z;<+=`cw?1QmMe)umfCtEpol8SBpbz-I znf4Ceb5>R$;Z9uSro*QyNt*FQ9EX+_``aVp^V}hgf=Xfx91-}Kt*$fKD*MyZyMDD} zgT1i}g@R9Yyl~=0)(YXP)(+kfFA7NP`Mw8ST`W1T1;WRF?2^ofOWt2X)cM7Ipbt$G z3_2PG^l2BM{MbH}286XLn^73QzlN+b+85G$MX~aci1-4SP<&Sa3@UCex~|m?rs7yG zK0tqe;Fa-VM;37 z176jpp~0)57yzD}pMQ5l+Yg@~taN=I-@*O4v~vf?xq&)<55diK5kCnL0BC1cc4nB% zdT;*G>VuRHf8tw#7^Nvh;O?2OONGshVHK8KPs=xdxW@XZziJe*E>C3L-QN zL1JEi*B)A(f9?dwl+&>8hzL!;?9IrA3Fmf#3}wx1i4Jgq!XUc`TCgfHzhm=KHE&;` z?*c>D@;f9Vc5*dLQf%HDTW?LFh#}e*RN&mXgG}m%o0cWNrgMi#h!Usq$Dkx!N}Cql zV+F-sl|5mDEOP}{$vUG-F$nL9EPcC$&72i~lLKb~h?(d8PPCCLAB)h|-16(=3me-O zP|3f~4=9?8hE9i{&EO~=^SK`ee%uGbZT3tkoB;36!XODN)H1#mL#A3Wd5PnM#xBzr zbMRcBpv5J_&2HuvGXC|5yut~enFozm+>KK*I}@-3%}o-XkwqOjq3@#PZgbE8rieX% zER}1sYII$2&^3qn$!fuqBGFeYiAZN@ws<8&L%>CRw<9WsZ-6frrrbkoCCc0;$ zj)v=&HKhxN`M@(7Oi_u$n@m3=D6>=pmN!hKx!)K1JQQiry0PD{0GH|SsIUUyAgULM z3um)RPdoiWLm&x*dqUQaH$w5giH>Z4?ktT8KI^57Yg~iV#)FpV&mn@4=cd81VagP!n_K{)xynBc#m#P z1{3J){j60vHG=4*oTRt{Y#WfwHBigFV_2v~5U^tw0I|tU*RDdm)uh4i&Sjc^0)^9o zKtOyS2IR6^z_FZORJ?Fn?4k&iMc1568mDEM7X^k9C&)Q^o)emV=jIZ_glQ6m&86oE^w_-Uy6 z6VYIe@L8*J50pmDB3oUN`M$!sfDi7$C@pZ)IG}9MO~vT0T^MNt_Y2jZ@$Hm)?GvD!@!aJfZXi@(ue7Z891aTODmz9IkUh;xC z5mLk-a!b=6-MXzXI9>JmUhP)^A?=mv`yF7pK;N2oxN{xD8Tz7>`TYZQdG;9wqpc1S zgsAcgR)wl-yt~5az%|$0(EEiZ$f5P90P6K-)2K&zSCsEAuW`byNYyk~gI#G!`d52P z(VUYo!&kr$e>F9@TRf?M1P6ZNQ>$ zanit^KwDba=pm~jTr$k+6B;^6`lF_HJkt5VLSvv&@D#XMiAdV$Zuwy-ZK8WTnc_>J z9SZfbh-o)P&6hm6ax#bR)I z$2t3#Kx#DYqtP5c0Uj7?i|oxSc44ep(vOk*D@bWBc>$?Dji3>+y*di~=rYvCx+q|I zF|J+zeOS$7uF1Qsauf$sT{0DYr+R^DC|E%m5>w@k}qG(_{8qbH&iTjb4SXWavQnl`Z&iv*wZSA0UO>@y7D5jC9s zlH1j5YP|!adhTQ)kqh?E9;KW9)!7HO+07<`bb*(RaQO$HF+h>%M4??L;>J%VHH#^# z4t~pt^2ZZq_sW~hoXPrO&yqZ0J`#X!9Fz!S)0?j8|-BOU4EEVDh)BT1r zk{2sCX=lm{0Amtdb5O(E^1T?AV>`Vmo<%!UL#n~OcPS3O+c4CRQ8)+yt+j+H${gYKQ4}gOs@=22hT;vo>Km{GrS#@qDs-4%dGL?mlE387 zAemndAV9fFWuzl*)UdmGYVNwDTe11b8lJ>X_(Zz+oc!JFV8BY(xHX&7jfx!L+(H>% z)x5&;(j#Q{1DrJeEvxSGnqA=dYtlYZHw&c@F@>jpd#I%B_K#+)+;Vv5{+uhp&av2; zHvvSlH0Q7h*mnxAIt?nLejYP7XRjeqTJUq(=b4+>=M~#WffDC(PaWVrSyKH+ghjhu<|pt zJ4VufhoaiGkS~2d*%@(NO_0dT|5yvirk1L&vDnw!2C zr8;#7Jib7MzamOX0lZ@(?ZLQMLM{kM->(3F^9{ZF)AP$~? zJK{toewHiE;(PEly=K!Z;L;F@;8nnTDmMj}f6q4wBJs{(OVrh3$skT*TO=^`*HxHf zy6V<$SUMdi@f{0*cw$wtOcKNKq{(k(&iOE-*o&u78@ST+sGI?M*J7*@rAt_k*^wHk zMg4_P)@en%ih~fMTk|%6Kn#W$3G#1$rh|ai zTfb?PCwCig*+(!NS93*D$RHPO6+`GCuX}V$coIKlt)Bp5LDTw<=q_Q=#)?>BQS@Yo zcq*~(mY_)9iO{x_q$dg3j+ImhnaHlbv|w4^Zjp2sx!xYWgbw-PUcJmFg~dLztUATx zLW~>;W!WNt+)!2ckeU1hb?$q+ljmq1#Qbohf+43!l~zfwwPIl;g+s!+W4j@ z+F@)4X#THMCrHj9+qgP&V*iOU{#Ov4#gO53)X)&TNl`DR2*tP)PJWY$_C zy1)T(2!uK=Uj|KXP(iE#gDm9~WDROz9C=}E(Lnb|(gtZPf4rD#c6_^k3UWB46f7Za zK|&p%lP?FEfjmvMK))QRP1WRv#`U}`1?E)7p6E#OPu&uQGA7sxBD??D$1on;Tmv)L z+2RCt$DyYkf@3Zc>}O#kp{2#aTiZd{bFqi^~ zgn0E5j6kamqy2tnyjPik{0YBpYzJwYCy&`4S&HfXe3vDjE&bd2TqhpXM4h-;ZO&1Y>eORyP zm@LZ`vg~1N7v9nKHuLVusqYU5n+nY$=l6I5s%!ge6VQg#S&v&EX7~a-LZ+o48KQ6~;b;*~T%MpEDj*U_;S^I82yIE(z%*!H>*fcMX zEfeW>Du-*F(UChP(;&L9!&5vTlh)ZCN9p=e-7w{Uy}(}@i$U<}KKXFdnMpnk%){jx zckNqL@0RtLR%_JTmGi^KS{v)Hv@0)5@wHle+$)Y*{TlJ{gx` zb;Ij_tt4k_`8Bc6!8Pm)t4HZ{ayX_#f$p+b59Q*yEc}xN0SXqcw6un4cV_xe%RFc8 z&B^+D+1K9vS)}blQAWF2|CTX0tfLrPw(z$IECa`51JrJUuNf8xBMa@$upT9?;lgci z-JF<+4Hxuwak8~M@5T1Pk(HkAr@@~02D$rxOp|Dp_0>8q9&{7DrL{-oF!i-?-$eVw6^B=uj}r<#S58RL;8rPv-Kn>=(OG>O9U`azJeRL zeM$PrR1@gMjyHC$s53vureC}U-RdA$=fE*`m#JNve7d;ceZiJlb#TennLpwl0!qh! zyD4^Oi~1xF!ON_JP_`~OMUkJFhjkGd)14o$?4HZY%WZ+J^qg{Mb4t=yr%)ra^4EQs zx~u(2Sw;b0>2x~ouE}mUKHaeq6h{u5Qh02yV zhkoy{ox*B5eGLwQ(0~7~$HUAvf-pjCaSYl~FF2pR7EaLNDNZN-{Bc-J^Ekbj-Rtl| z*?F=%7p>5c(aaw%CT!qXuVSfMqwXfXNU_>k!^y!DY*qTsYTmNiWS65T{sEm^yClx?)+#TcJnCT*S+#OTx)GM?nq~>7{jIfcg;qozd%~Ypbdt~0N?5zCvqzg0hPNB+%!4cP zecj*DLwB(qO{|2SRoOPe{(|;@!`WrkcFw9?DY>uvYkXPy(yov0?#1b3ak@R$QC=qW zH61=x9(QL$(r?Wr@*Zxm;g@UJws6}hy?ji^!$~x-f@(0c$#56M7pt@P&aw)RM!(*< zz3$FkX1n=wkYwxOZ2Rh;t;1%X+{|5P(r4}aJEj7JxLyB|$2m)FRh zUwdX+C)56L{_>4%{5oHwMK&)I%QH>(%(~D1OHEAv&?c6R6P6HKzduFK8-vBw>tqn?*u&+;oV z4QJ}g^msba9@N!^k9VDa3bXU0Zb!LgZ{nkAZ-e$)xZstYPHx+#3JNzCwq0LYi-TUCche_X5a{Hqev zS$j7ePVQcoGDh%yFqA=;xQ7Ygp4aUpi)V6Q+P!My-*a+*nX}ETJviJey8}}{j>I%w z5-jN}Vb}D$wGl*han179CW?=jWBw&lnb-LpH1;)*T5_LE`zi*I*q2ApbJ-=j9D+&+Byxkv zMekvKTBU{sk(@XyYo+4J8kJt=SIcWWs_!8qcb)4xTMvxK}Rx+tuo%&iGm;IeN`I=Z%$b4%an* zeLUROO}w&jHfL-16kV!?&Ca(=xqjAu&-Pk=XgnWQlA`C@4YvL~+_k=(b9UeDTHeaF zZnNW*beAOR8OvqZx=FO`}^& zJg0+EmwJoQtZ;j$>lM#thc&x>t=FrW<9o;Xcv`xrw7)#(%jFpDto38njcyASUrfBV z+$`$%7nl1beNxlu&aacN!}AgN>0;7F`*X|ehs2QWR`!UO=VkA5>WWQwi|F`&9_Qyr zx6YGUm1W!NxpWq+wKmE#S_H=9o=LTspKe~_v`gbO7!2Ia*S<4*ff%pv;!|&*UMMzL zu5T_fW;<`Tcy6jkb?Sa?ZuYF3^m>cs-kI@yEX-X#q7U4^Jrd(8i>x?JGJ0_W1hj9RUhd)s_{ef8Jd{r>qp+y+j& zeZ7lobnRHT<+Woy#nn7b<_}}Io!)|5w0q8;hDA_OZ+KAf9Y2LmQE!rUIoo1p%==Ry_WK>Ww&+GKD=_t z27FKu85X7OQ;ECX*Y)`3H7LNo-3A!_KmXpF^*^r7y5E0p&HAt^(}o;TNQNE$5@& z?^~}L-VE)_=EL;7v5%+eY*vi*tyNR~^I@{K=IP4fXK~K~=%%4|yi;?_RdcJAe*fl* zFYFharQfGln&=Cr{7V0*ecf2&LmZ^7X+Z(#vU&cN)!b^;++qcPfXupiJ$l58&M`Lb ztyBRlHu{UfG_l67t!I$dQvsNR#(XzDeTx-pZmj}9aq~+jOkdp9{Hhw*9DoB-0KB3v z8YC|o@cMWXHJ5DO-jy|MD1bzdH-qJN0a0)Su)A-BNCM_r8JcN=k%}#r#b+l=PF^T_ z-hmVEz?Np3EKEp$ETrOtQ^#lLC3@a7H$f^B)L=>^CrHI67*ruMrc^~3WE`MC-AoZ9 zkP*Ji#A+Y7jRZwVgqPZ95va+yD;Y*Uv*r+o2pGbM-&_@k4XcSG^pI8#w-_vd3Lxt^ zzT%)*?N+?YU-64q@zNCUXJg;b{SpVKPW1D4`HL^iDzY_ymj}NOnOVbo;9GCUQc2b8Gv3-lD(fT?tY%{vliZlL!|e;rUf& zb0#veO~PM)-YeGYt<5xw^f@AxMEu2<@6VG1Qn77@|9*}#{vYqjn|*>XIxx*prQ+qi z_9;zeh7mIn9nY9wT8oZ~%q3F#<5m_OK^1G7<{7FuMbXh=xG+g20t+*ns)&XiFs< zOC%!LQT_3FAJ8ijqv-Ry_3Ic53X>>W2Q)_zY@Gmqh1g1|y-o8LQ6mhZ+}WZa5Um&} zILKH zqWAiLZFgCcIT=5je*E?)U)ZQ>&g!6j&I|ZS`<6#O=cr$Q;4d1A<}6HK=+A5ViaQ?O zpM%cmWoP-0C~mTPWcow}AaaI*TLkwN$QVNgQ@cvdQwr;5uNAWw1{dHn1WKCtu%ZMg z29bEw=Tijc(#c}PVEhVCykjfF#aVpDKpjValqL$H#!wuwpy(tDi=q`-O^hw5e20Ms zCKO@Z^50MQ>8+%laai>3to=_)CJTgBJ&cfR2C|z=NE( zy&&G2YQ#W9wX0~vYNqYK)_bG(%}nDLKO|i*6lW*n8Itjw*6i&w68`!EAQd0KUu#x> zAHUK2p7PW0-m;!|hzwY-U-yfFDJ(jDDdwigZI&?``_UZXQPA82l#VQdm~V-Kws*|L z%1q9JYN<}?XW&^u1@{NzLMA2>TTCFr4nMmO1gK8AWT`=^Io?qaJC>1gp5{JQIfXme z1SJ7chSd(aiTSBdfo>Fe@EHk}iMg*d8xG`|;9t$zQDJrOEs!US`Qz$BHH|eC&KC zS??W>V*RrjN~5UR*FzJVIv$R7%se?No|~w#OYC<5MOkX0VG_{eBmdj{dQW1AZEo=%oMd|nu_`vKy21rtSQpKSu zMwVs3(M}_QAwO`X@XPXCD?kt`!<+>DJ2tZgm9d9qWMOPgS4ttQq6mrO4M{FQ7H5$1 z3Pv)f)y3v=v_PV;Kz2kppVwG_Xc8modKDm0amxIWv~iz7yFV2#wU}1@ClYK53J{op z9+G6A{AHC&5eV6hzT%gT6{gPL4?;hr7r$Y$>6A8hoQ^y2SwFkulszx&;-RJ+F zVQ8ChxqM@sqZ96E#2geJ(wZ#GR&N zBfoPr>w53}+~rlQ{psxJ_1u%pOy${1MeO)=V&f==B|}>zk2aLP6R5Q#rRP-Cg0zS% z9Qn9XM+?jvmijenLW8IT!E8_lex`hDBVoc)AX>#45%grEk>>z^JraMEIFfoS%h497 zWh(>Q(s2GSKWk%TcpAxqfN{HD5=%N4zzoohuaKg(`$`P2rrH)i@gbTs|C6t3-q_0r zM|CttnFSO%z?fb2Cm?B#{8Hu@hH+)=uylhwvxioSMK0onctWc*sMPfPDLYE(J+{ffEqa1j9r~Quy(&29H3f ze=ioz?_IL2i632fpWElT!A7SzFcrlxPcTl4z!KDKUrS16#hQwkAUO>Y()cMNbXHXe zv8G||w%b?O7r0ii0ulk9L>EgbfJ2i{cA&9K8vUGy^Ae3eXe?>g@f$zBwZ!@xTu~Yj6^?4P4qptsKJrFpk@-~pFmYFhas%R0v z*bEC{Pawu~RNib=+9ZtwDh~BXmVud3?JtyzJ_0|t+V^Fv<2Qb(OH}kFCmcwj{oqL@ z)fmWVaL#YOc-Lu7Es!ieSf|17X2|Ow4Eg=_j5oR53*T#iIq$lvu!NtRSVjvBLV^@* z*6*u-Np4O#IwDv4&lFV{r6cR7rrWf}R7r%KirPq_UYJr4_XYKpqX!g`8jL8qbOJ++ z?U&4vB`64HQW@`T^{(+y8f_V5d4QXnuMyy{_E6eAC>2b#A6CUo`yqtl#A^1g-5P)S zQ{z;PZ+>GGbG5^KD`*`>DrigRk^| z*|PexR(Lg6VCGv^0^6evc@WQ5pYfscZHG`(mt1o zc^Bv?guz5*JoFBUMWsIPC_vg)^#56OLc%&>m0%DN!6H$Yaum|xPM#fDgV5)=)aUrn zcxS5dPBpR`yi><#s?RQi-`c0amwax265jo>VPE{l_P70NJo=0ARcvqkm|{dJgoXLG z$tXk%{{S*xp!B%+!rM5``I({E!Xg$T>kpYsCM=Bh*au`u8NotCNKwr@m{z4sy;eUG z6cp`D&r3rD(G*^BM(XG>iw2RIjs{dDOeAj|$ zQkzv|q2~N_eUi-or?1B3v##&@?t`C`6Zl)Ye6a{+Bu6M3TPmSQC89 zD^$W_pCneYt9-3yvx1>9%^UMTBe(~V1Eq>~M^|uXvI1#6g zPfjRKP7{NgT=uRxh4#V3(BE(3QyT3M>73)*m|P$22P>Im}w~px+LEP!yV3J`Yg2VPLrb*ek%j*@sEgpQs#Zd!>y%^2Hyldv*@V zk3TlG;34U;pIY#P*$;8YlhOCub?vwOt;JWbnYCo5@}Iuhp&Uj5lFi{u4=E zAARQ$zA#n88_pnOr=~V*YPMMhD$pDsLc(k6GxT4u8-yc_V1+y%2uMNN<%mLG5K!N* zi&4V|nxuWy^QyF!C@&6f)QZbML%4_&$|-Px@yUbWBa=f$!Y^H{N?vi>2o?3P(vW^v{0B z<1QIExQ;S^EGb5rtgWLL(Du*+sd3UOLMf*pDWi3HLn(i+d*ANBWrP*L6DZf~9&usV zdfkp6&wp#9xI=ODj%~mA?>naHuP-&8U^VkU@vH~Gcs=^?jtv*))Gaw#Swqd%S8N zDm9+M0=Txv5UyBJZI9@?Z$ZD;|ISTkr>1Z9blt7fB2)L4ifm5)J|$_kOZB_ zdQO)3P0VV}TJssP{g>m^>ta>(&RhCSC4cclpLj3Pf9waV<~{whCQO&-%n>nFBXY^F zlLg4z8SH?Hmyhzh7kWJu6xXS*7I29B$QVpOh$WmyfsWru+~bgRCPZ+VQl&iEr;zs% zYz%^4;^t&w%a+Wcwb*!v+y!CB?|Qb~fg;g%{b=gsH@+}`?fA{36UDxd!@9zAQ;61h@$*ayahC?xM4gPyLao4T7Y=AjJd(b--u+ z5FVXUm?4#=k40k7BA}q;6|3!VWS4U$n8t`2b5iKW6p3O_m5$^3oF<7rClerR;<%r% zrmyo#PEJMt!R-jWZIJ2SLCt;ssVluuoVJy)e^DwdR3(YyyJHjR?!fxud>*mDSlLlp z^{G~t|L#4u+QadAEoUH{DG0k|0=3M(C?;p`U0>11^l3M z-LK6!{~fPqC}TRloLI)+Hua7%zkGTXAyX_3D&+)eK3wdp&wpyCzjes}eeD=#eRI3~^PJ0hr-}EWcu^%S z6F53U^v1F&Lr5q*gXy!WpZq-5&kco(e?RfvGKDjuQs-LzePgF_ZBs{)UVALu5K0PM zpI1G?4KJ0#ca5v_*bKfgx2_>l(cm@S)QltC@D*>Y;D2L<+2&*<3z&k?{-6kgh*CVa zr;AP&cMWAJc66G&#aHvL{yxWOaqpJH9IYnefgwT^Ghcupq!w~N9gHv$FA(u#f1

n zu3iV#>rnmwvJUvg51q&|ff-!>f7pWQ5guZ2H$Uh7?6O}wXn3DTyH%j8()oQfcVktU-DiKIAj0HVS>Or#j<(lxmVyxkyAhWV~lm$V8FmOa)elx*8eU#+a9(3#_O<%O@{Wn{DV;-N1#DWhu_=AJ8mj^WaK>x=+2vb_7>^prCvF7=dKW(_Q=HvNfmP&^x zX>WUgFuZ--#mOHZf9Da|_a*V08s7R{=VqLJ#~w&H-uGURhOPGgYxqw*kkD$k3fxr_ z4v2%R@ofNeB1KbgQ5?W3F| zns;UjP4shee;oO!(c^H@;G-J+x6glig!P*z@o=o`M4QQPTl$9&dGC82cRqTfn(MPV zupL6l(tKmR76b}gpl{|-kqz4kR1Os?vaymJ>AzJ~C+u@JrL!EQ6Vx{erDLsynF<}3 z@+}Es8;e{|b8@I+%S6%VyzExy-Zz%d&6N}u1`8y5*?uf8q}l>pD6o^@t&9PPV~FVkv9_u?d0_qyj%@ne*#=mW;8oLD((` zYB%|x*zKWYM{Os*=Cxlm_~7S;t9`HQAKl!$cUTEa_qtKW4`k5cji<$Hd6m8dgJ1SUb>rwUn zGZDjwYQnm!1r_2#Wi-WV6ikmJlbm8>T8Wj;zdx8R6T9HW=in=Bo-b_E;}!{&#`1!t zCLrL>)aQak<=e-(-}KdfIpdD^T4{c)fFSwg ze;S!VUCu+)PyS$RWTOE$gOZq{)NvVUJ;XbIMq67(D-e^`==oLso+lOHgSLGOt9WOB zk|IpXR+2K^`)aTZ zSJFhHsaMkWGd?5vg9Qbio-0HOwea_9e+W__ttnKoO{=-LYe}~=QQM8SLsaTIvx%!r zHFz3*>q&n3o@D8z{4XAuo6z*o4%$9-e_{fn<4Lh3;7#YEpl~v0Ak-2C{X0~X3}m@F z`o#nu?1K|yv)g8Of)ZqTB{oHa1tQe$x{z{)5yx_5OMlb?J9sR?RZ1%*k9O6BFXGP$kD8 zYF{OYL`)&6=I|0a9CK0@N235onK^~}d;)2kx`x91PW$#e+yT_Y-~}*cq{ZKQG|TK9<#3nlSMdFR34S_eytd> z>iK1GSiN9Q+r-zeZT3$qs`0(qzew-joj{|52;VuU;d^yX*EvQ1d!PB?2Lpa%YiuI+ zowJ(wG?C)$s7A-~=D+GjOIU8KJ_b3WNc4SE6f-8v2@yN+gIw1QHp-z*f2AO+?raE> z`hFfyG40RHigAs7u*eHa3mb`tMQT8iux7d97RZqef>QM5FSE4+Mrj|g$gLP?`;sd&uhcSkPs*= z1d`#qXs~Wb+03nQSg)C?*dm+YYkxK(Bz12manSNUdfp5QDiQCzqB(7+ z(ChMaUa1qe>3a)tc=y+Sdoh2sC^o8v#uBgp=GG_FNNNn&Ddq1mSX=+G<(8y(tK$5#`6bYMT{!W!<2{mB)L?+fp>x*+qo!#?#f zj`wyQGwM@kd6eIEe~8wDnQzU_a#YEPTWQJBTW3J7G7g&kMTfT52I^ZFZ1~mp&?c7u zXFc$fyS08^?`3Ssp#Jbx8uPD~BKN6~C%YP@2o+dm-2Pzt(fr+y{5(@crjj*!w6$O> z*eIQ6;zkb#qT!)*?rZQ*!^{5BVm5ZF`@bB`B}ESAE)re=f0A`Q!O%*?7S4DMbB1Fh zM<5Ct6|CtK?v-?O4d#EQ9ohqioqhYF_5SF_y+%(}efI%=Z0F6ZrUe~y-fy%`uRR$< zv=%?wrgHtGZIXJ88J)M=-pG=T%YW{9;=FU#yV4e{+^EjvpOVgBx|d01<6_G6_*I zy1zL@c6U|O^apU*39qv+WY@{q^uePqB_~kaEX`FleCX)IX#v~z3d^1Ls;xh|*d51b zI9Y4^b2Pu6w3hB$9vYc0btX-COu*f{!o7mdMCou8N3` ziAYAme}3_s74Q%%?%OF1aehZcH>xCM;5(DW<5lz#Lnh)TlJV7%TH9%Cs2VNZK<;Ka zn_{f>-pqxLq)13T7x8}neX0E?144D(ab^{7`qz$#|3fd5d~+fH!O1mx#WO!7ogWRA z*65=ctw_gO1S2EL%CR5jZi6*VNq_hbY&;OUe*_a8?_4~JiHa;%%a`}MkIn~~II!(+ zMT6P?gG)>{F5Yy^N(!@DQ7fk{y&n77?!pdGS3(N7)a$uAsE_q}G{*cV9yU7npFVu| zr^lgVeHNX**J6fz*Qw|YyU7~GSJ(1c;&vPx$w=?N4L%Z!bC*EGO(*MbEmXMWW1ka6 zf0xfJ*7Ga55dFv#f`BYYq_~Od`QRS1JFVgoddHKkj;VW%KS`s{RJe&JT*t51`wX;A zvBk&kKQTT&|9Q^8v48&WOi^@TsevgH6O?v22x$=|u1qNtWW;n2*v$;BncFc{_*4|B z{l?b!3E~VyUcj2psbu9eJwEh|g4QSVe~=L*8MFP7H9Hue_`JAYinWe@9$nB3E2MSw^d)pI*0%PIIX?QHzU9lR93wwd%R1+?g`wQ`jye8O0FP7Y6YfwALI`3n@mh z%umA1Z>&l>)}LNY z`-i(Pa-9eDJRKO|_5c+y(erdcz9XM`I$LVr74J3G{98lv=1CO)`}}{nf5%-1WPy>= zS`to8%ZQqpI5lDzm0qC!KKmcrza^Thw?Il0E5&S)b1Vyv0-l+IqxMt&^gh~W@rOs9 zv>*G?3#L-*yFws|jzO-hKl+QQG^Znx75~Q6pO5EV zH!aUtMc=u+!VsCD>tugwe+FTHN+l(5`}AhQzWLLq(H}owBhwdbTJC%A^V51}X8FEI zg`AYt*rPU%j%%Q9ox>h4+X4qj1x1 z%6#4XZF&t2ZqmGpf8)L&(f0oT!`DqSE6O)lW8o+-*_>O! zeg0HVdBmPx8tt1n=W{yIm(J>bnV^%o}ga`>WA??>p4YHvQx`|4NhiA^VdzqQwLorYT})YA8yL z{On$2G7F*$N>$GNV(G}Tj8E*>SB{auidnXdPg)x^C09~aeG_6Hk43b0@!a#Hx310E zk6N%VQK!+h3B8WesU-cug@=!C{lkm>^sRn+<=^&0f4^~vUQf`+tX{l8mQ=i`ONFdl z75((fCqcil7fe=kk{wUOj)+V=dN|UU+5WQZR1{5A&aF2V3()f`Y~RIq&s<>3KciZO z9B7-ct?ktL%|J|$H=m_P^wA3Jxz=f+8XgMbdeulKDCie z4LE^Ne+0?G!G{$`;*;Y86=JYD6iTf7b+)fuyEaS^+Y^$MOoQCqEXsf58Tx9ErNv=e zymjk!;l1a8KK#D+!^7XYIr+?3H@v^*vA=t5|7dCd%ycF*(C`pKsxtfh$)NAe_(qi@v#}*7n^WZt{qw^Z7PM zwS4zkm=xdhjK6U}|NEn@;J2V-xm za*0U4jZiWnLgkL1qf#XvH1Wc>eIsS8zbXidbUbj8WKP{Rvrk`O?&9NXdwkEqf9XLx zC`bDFznXb9X!LzZ`_O;(0zT*TzxM*tBz(^sH0#!T@XdMSN5#=va@Q2`xWQd2j%}Uy zs=fvaOO)kTltwOeiwJ5c8Kf2_eTrYKBEp|InVyf6)DgCeQ?S}+t%=lmE}9X`yiP^o zP0q(1N+a{F>x@wbQHX(ZtNqnwe~;3!s*<%I3LWBtnj%6!>)%mSOI=0ldD!puH?eL0 z=@sjk@zHYWd26kUdFxQW`Hh)m=Rf?v)~MHJ)yL-NJ_y)IA33yeP0hy}E(zi%hJ5A( z3WLmDB9aq^#MSjMQ4~c{9|d`dLG}qpqYV*k;`$c{MB;kNcCk_>`pOXxf1MMkzK0|y zCOJ}2+@mO(%w%b6Jv&qycl*T(C|Vr*6jVGzl1@O3y37&;HSHh@wyrbYcHH10EdT6- z{KWeoJ^PPO_PqLs-&y>b(U|kYpLslGTh!=8;QLdKU>!i@5|!+nESY0bI?K7?LtR{v zA5ADOHP25yX^`f;r9N*)f4}y9!i{8tV7x^lH}y;h!93}B2_r)Dtjwl3eXn73iy)chYY3bX@e;nYa9j$EIcJ~nB>u(GZ_Ib*CO^;x0y?3DUa0QF=$>Y&CPWD}QL(S*f)PlY z2Dzf3@Le#EC@PSjgw33oh#gI4l3Fdy=J_Lm5nmuF?MBDt3jm1+kQV5s^?Qhc-Q2HJ z&i_3B_dL|!8EV5k{O~^u{&Gk3CG7R_3Z%me|D-;rkv4K(kZP~izxlS zF<~#5k}8Rb?vr^jwzO_R>lAferup(_jA|fo~1f?Sz z%aP>~PT@;fe}^Vd3LA{vH)~Ec5A~;x)$!v0TOGR|{V&`HAbm?WSEOkCwweDP{YO9j z9c%vf`QN(Qrf&Gr)O)Yhn+--qyL zE=>2NS$8TMJ^4%DG4ttPIY_LIAXvwf46lLPL@?qXRuTC)lH)&XJI*dFP69ZMPO zxqbAWf7514&uJgwpE-K;nWI1aouh}X_G3Ywr_NIMB=zjmQ@IfWPiqf#+#zmOSnobN z5|ByM`PF%vADu$|2g7$f^6}Z<_}9z^UF*B=7NFgno==bwMKW=a&m0P|O_c5#L^CAcbNwU5*0B5VL#U~jH>Oa3<^lis z@An>{_&s~BYxo1N$Eg#8GAK{WA`&w5(5QGIL)uk8VFAZ`QXrD@awbRm|2wX^B675} ze_*8deGQpFwa_}}2B*_M`jEdf9`;Y1Z~W}%PNW7GK*B5tM>UcCz2kp!O~$v)oFT+t z6JIXON+l#sG+xJdDJ?JW6O2^kN^Jv72lj`M(lJwU#X>gwvyp5CiPZZO0&e)w+)BZE z?~lH_;IGCa*7Je-`HYYD;O9=2=8lJYfAt{xu8GJeC`&FN*}(yB-c?`cMJiN$b_K(% z76b{F6I)>f9AOZa9vNzS^(y9jXy#5B(wAlk9h*OKxY1d%g`VrM8{AR+>eBzg8~=+A ztD~Bkh5ffqUEj%8_v5H8PJc94_=LVS%8kZqs%Na)PtqE}KfI6zL1Y1M{raTQf2$@L zh3|Q+AD#?tJLrZHT6$>Yn;k{bRQ;VT>3(YuU@p*ZHgwvT`6i90ZZ1nSNmV&L z)ov!u{YO82b=B`ogIgCmhv=wz(R!k7Gbg8-y7)6!(3?vB=~J=y^Zx1)FR@cK9R9zU zAryg*eMrv|r2Wj4bcN+~fJN-PuZUPs(8-*-=-^veu_+Rza`Xjlmp+gWLVwCyo+~v1 z#ifeQ?c6S-1reW45W?dKMIu+NVK|FrV4=8E=;Ri+s1sa+Pl^^q|4?cRfHW{m7@&6X zntW8y>U;mphxd;!;jdf?pKpUofq!ZXq;F+F{x$YTx&PSCD6{;o(Y|XnW&ayIFZd%a zlrl?o7YhjXH8ur+@+6@?;D7MIU7)o5IsRuqNRAy5&M#Y=NTmG+mq0}OHKLnI%En*B zvNiu4FNICifvN)OyPs8ZXpG>TR)KYV|79DgoesrEfmrz|8|`-wL=eFetGX9>*!+nX zmxMBIRp(cJ#ZE0h@dx0~e)Qu4%D-P{{m9S#n|Pc`a25Bj{_>#yQ-6%W&;AZJ3xAG% ziGRnwFd{a_zhiDnApov@U~bI#Cm;A9|EOeDZ>*9@>YEH{);opM>r^SJE7^ zcmA>8_V;mINKe#nJ|hV(A_syX;Z`7lk1YqS8!w0b#@Nzga0TU9w{GD+Z3PT2F8|cF zm0(~`4h0`cH}Dgy#(xPv>&ranoLFq)mJ^ij4wkl+|8@*TJoEd~+T1$%pEyX=SJ#%? zmam;acRulIzWS5b=eMo&kMG`J^YkD4i@$QE{QbTQ+GAFm;PK^!fnOmZ31cGN0{AeA2wQs&Iu_H*FW751MJVXbblw*A5Fn#=`PAJ$a zD1kQNgkkez1svmV;-N(M6CWD=Sr3$tz4NcW1;|eO+dmumDdZn}C#b+ruFG#81F_0c zOf8D-L2|}H*ne||$9{Et5d^sCo(zp!M;(1f4#7{~jqkt_k`aRNf{TDh>*hNC^{t{JTJa$k{&IsLsNH+zf<8KM(+a>t*-J2lt7WUw?ZPNAy`g+toJ?@^_A?T>52m z`>WhOF+=LoB=`NF=4<+87y8Mo z2ph2rIHw=f61Su>JY7HX`}iFDkSpB)I=Jf4M}M#ZsH@O3`Ti#QB_)Kw)qvx}>W}YZ z^sJzOD%R!QkwIs0ML+~9XS^<3=2nuc4)xk%h~a>)^@7$ zlaocRe#Poquu^b)gHqPSni7WkH{TinEKoWSJ$!!XFHxMdGI>GH1B3g|$j$tZa;a?b zKhMEAjc|>KsRPF$>sNDT3 zb2KW?-}2}AZ+%1wkbgDcagIXtOgPX=y#V6Ey%%r>YY#3!L-Ctjyucj6ZO2G~xIk}# zAsQF;QfCp|MRC$+6c_k`^HK>8V1Ho0WuE5V==r+=3`Z{jS8^Em)+a5F4Pc4h-+js` zvX#eSxIi0(=(}6FBfF>i$?<{%8RZh-V|YUN4a!?DV*t|VBd|dH2@ZmP_F-n=GT7`W zKHxttt0W)|V1D7Kz7joemdc9q$`=Tp#y~Em06>Q`9DswH1Ef;#0W9GefPWa3Wk=4x z@`Wj|QTBc14#bg59qx$UPn7Z#DC75i-}nE}4<88bfABx@R}jA}YCo|@@ENoLxBx2z z{K7{798e^h_K_FLvjlV09=3z@f&R+((|$<4!bd((K!yv{zQVVCYZ}xg=>9nE0L%;q zOPE4o$L1U6Lt*^h|NEcZ1AkCT3>X&hiGlBX?)}m5g{w=A{T!o#^6xv!?}6#p75^3h zKQe!+H;@uB`E)B#%8pTDdkLaapMl+%0C4uDL_^tkpW+kg{OyO|Y7t9_h~>Lp5Py9f zffK@j0Hnx2*LGkAL>632ctF93FJ#=%9hB^C1x?Pai>`vH=P1Vtz<;si?~8Q>C&2fM zbqA-m6giM6kixKUBrsYhV?e|~?EI7ZN>pFTQNZ&5uD%YOO4Q3@1@0JzVGVZBG5Us; z*crnp`@mp~j}5oO3p?E~9iv}ZvY+o5X=3rt64)MKrq1H_Zes?;?)I+Jl3rd8NJRzk z|KtzFYAQMaVOWWg2Y*VBDFNS!`Uny~>gvUW^P-7enbe#XhV=sKu$r9`Z~!*(X4cXs zR3m`N2q3TE-Y3q-At9${0O~sOG>NAXD5J6ojpC6D*p@QU4PJx+a9V2-aHEVR$c>H? zIJmZ|j>(Te+{6%Ldt9}C)|L_w}(fPZsvYvZc`I@MAlTm{!Q zVG(>pgTeq1h*cM5N(3DAoHC8T8Ao6YBMkY7Gb&}17O^$?U%2jACLG%7<`&2$=~$PL z4@@z}qXo8MtYm;!!_JF{vQjo7tGlusMd(DFv^Zj$DLSA7v8+bEN+OS2fkW_aWQuX* zEt&{pB@=$?3xCRy|2}7Yp5s9vmLECSccC1?(}^U2>MK9>foCag++ud94nW3mDxdH4D1)HaofUqCZ}xN z0yvY4Gk?Iv_jCb~wSb6Q;MPwgXh-GCr{$0q4FRlCzX-&7aU_@|1AsA!M#~SMnyxD< zUqapul$Vkz=-c2&zsB(&^aVrbK1GDM#+B3joe2(jp!? zkRje8549_dlkmQPEokDv{euScU)Sj^Dyj#3>wo#(7XYt%X5|Xr0wOStfd2G<1=n^S z<=nj8E_o@_0dB` z2hNZFxH`uzf~4L@AU29#3P7?ng_IT{Fi}!@6g}lCxYFnu@N{7cB3%#?!A(>hn2Bf7 zdVdCdU6|ZRnEXFr(k3Ff{{s(nT>wIBhLJ9KLW|l}`4#_RwI+X5vAL;*@cNxTg zfySY~>dwY3e2e;>1LB(dwCO0oGK!EMnt$c!pjq2QZOe}U97eR)8GQ9u^cFBWfOA7P z9--1WH55lcqdsLSpFX6Nv7W~g_~b12fuM3u|B-8;;MxKrX%#^^jLJAj+ow$;Brnk$ zrxE7k=m?>dIe++p z-*jD2KX7gwk(~c%{}CjmN4T-Ffq4XCf#wg?Z*^{Ikk0(kFU335L$PcW?zDi5@EK$D z3`{y_lp}=oN2Uz8$4Fl|qtpig^J=V=u$@K@j6esr#s~nkVG!xSQ*fnpGhiWk@TgDS zA*V#wf$fwv@3B;+TVI zZi2C3edb=&_ePF>p}ExHwng9OMMvWoFQD;D{`AX#p=Syy(8`A9^0IU{P=Ai#$j1!o zlW3fVPNYK+i`}QKV$Q5AIDXp4sQ;vc4lTKXwg5L>4DYo7P4oba-@`$p~4QNcd zYjvA9?q*hsSk-7u8-dU`gF*vG2t^LHYd?+vL?isnx58&m48G5d&H|hP=*~8Ebxk)4 z2iUqn?;HR6j0a|SGi8NDYm2yX2EhP=ws5#HC^|%I3^cF0k!}bI2!Be6F$A>?(kM_r z^Cs7#x!IHzJa1yplwuu7pZWVA=@^=uw?Eb~E+0!yGlGzhBiD&^AULFueaa%5)1n33 z|AFj1qx`HnYEz2+GE*3!Ie5B&I$F5)hmHv8Z^Hk>@s@SX~DI?cSoX1@$}V#ijm7d+rO;oAaM> zM$k7-4N66b8<#<}nU%$ni(PL>R~KFzhOwWL5Jq7 zKX73kEzk?q@4VE04up$Q2A&_?`se-Jn3WBbw$jsR-q$mL{eLapPhR1UKWz`uTBS^e zeaWf9!W< zz{MddPaEDGr@!<_HvbVn^8#CA z9(gpzL-9w3-+%u1N4t~K!gD6~hn|dJ;1Sld{AbWsNqT7MH-`WKF9*9WkmXFU(~m!Nc+ z&$<<@v!UoZzCHj_tI5Ty{{uLBHxL4W0Te3xsI9$z8U_3_Zn~a<#%T$yqyKono^PPHxb--VB0$&hdo7G(Dg4}5(Dl!KOG|GCb` z@ngy{(0{vQ9l4-5kVpJS-nJu~jq}0l#}WBlfAO*p9uPlx7jD}T?_WF|;Q!15yd<9)6U;OX&7yo>?OKZfJ@Pjvg`42vLRB{o*r_L%3J8ru2W|GP*9n-?$5+X)szxY2gQA$+zFWef@?f=5nK6L!T9mla8zu%oYkR%5AE1Dv*&7e6e3Y1|4RW|~o z4dnXw+R?)SvQ=%!=7C(yyZZwEnDZ!Z1%II6#;yG11Rs&fb_g;KumULiKgu1-|Myc? zone%}hEN4?+iL}2LZkWD+*Sfz8`_T@($$3)#FdNY&swaH$Uh+wyj25|5`;C)0ZxhZ zA6&7`MlXl8D8@5NT>%7?G_47Lo&$oLARA7U1;yw5l9GPSK*p*J2u&uS;}drnIe!um z-2bf6FGl-b68axLIpUE|GqW}jGK?gG>tM96Oav9oF<@^c0@d-%e2i1JG=j+|mx zh8fG}`JZ14a&(Rc5Cjg1N&DD+|0qLb4eo!#IV7~@H=IYCQ`_&d0Ju*Q(?7~05EN+r zE=zw0H^H5T*WdM9fOq&4fIN^`kbeVs1hRsTEP0^dr~kd&7+ov0+0fbFYpPZ9As!eKr}%7DF+}cVT1m4RG-j(%N&hJ`|o(Kfbux~>AfX- zua59z2hIr3anO4Nkf3uKB^+EG1Q$Sv%3=vF3-nPrE8&jlOB}cXsRc<1Ie$b)24IWM zdCV3#qaXvfgl|MwvcT?#j`_>)*Iw=|umYlk>WPuPJV0fMMsn~!@+cAiX#e-m^^b_& z0Kp>wZyz}!`eg$>-~Gtpj^b?{^nM8)5(cB6`gTO`hdx9PGNJPVI%2s*^cDc#5r2gP zarC|vp!d_i$et!;c0L}TIYxi$^e)#{;mHf~ZET|n?bbkgCk~0`T^7<+3 z<+pwtKnln?D3Bkb^w}T4(fiX9R>e+u5_yv;32k%Rl zgZxdt&*7hQO-k7R^naHHm36dVEPuwU&v=j{er2SmKXigWwKK;ez9kf#lYa0VkblMl z7U&DYH@@v7qfJz}qyF|C7neWl^BbMV>weDB{O;0ye)fS!`ufj*konnvb z(eY{U%5E+Nk=@(HXpJKbiYkkU>IZMPd`AqWsw*Tx@+EPK?`{RU>IpjdgrWINNTPy7 zxa4FvW+crAQuPD`+*9YLPFy&C{UKzuFKYj{|K-*X_kVQyum5El=IgXQum9!0{a^q4 zt$$4a+bzwn|L4!uz3#Su(^CI>`9Hk$F?!dbzWw_LHcnf2{kM8byE^~gS*~?iP5(yY z_`i=s{%`Zp{nsL$(tkrhZ0T<|^53&`yq@U)wZPAkQGPdp0mrZae4H*oc>cKIwg1uP zD7*tumVb^CmG_*7g2KJKfGPnrOQi+0Idiml0%-b*+Z56U!|(%uFI<2Q{}wG}kiQ@*t)=U8Kpy&+#+bEKz@ zzO9v;!sKY({a3;{H;Eq)JMkgPceNSwt$kgnwmH3h2e_V z9Dkn;DW6Xab?rz)?p(*=$;UY6^qKD1@^HRJ_kO&`PADDH$M{@GCk~90(W{5R)~4Z9 z6@>MA;cnj_a+zT%Ha|bkdo&(HZ%$fGF4dE8v?vuHbMJ7$PMF6Clvf;PT;@ts^;(*fl%KYT6dbSD^GK~v za$T#Dr@Y8GZtKm=&#>+_T@l0)ESI4a6b;w~E3(-6H8ITerlv5ur}!+Kg{V~uI(8bv ztO`Z-ykLDe?2zJ&N`XYo)H5Te(~2D@XUnEIe^O@pJRXxA>#GWC&pF0YCj=}w(SLk` zzVF}8&EbHbA0B+ef~0T2p1VGfcN5nnFg z3J)h(at^pG&!yxmd0A#_mP>fTjm_f7ZArr(zVOoRN+m2cz6U-HG#cb&FY=`NXfN+x z>PC@@A5-_9={7TRwhR&JBb--&}%~~4^f|9 zv@cz;i)`3$t>T&e-94rvFXFX&AEJa|7(K$sZn~aN_uzfa<^6GgkLBUOa~of5HbFG@ z(klGuV749RG&dW4RmLa9Qz=^At=p3Z2d@p^hwifTZ&Mh`SLOsHKS>Qw5r1u&W8j51 zVTFtBP4Ux0P$$CC1k4STlQ}ug6AT3&Eu7j@SD`nSo8@VWc%2oH;|cMgmt*s8jY!8~ z7BKHu<@<)uJU3Vfl<+Wfm-30RRfEF5-gM*E6VpARR?;^Yr&GA(zmId##s*;)(q&h^oGKO6Sx%DLkB)9*5eJ1n72{zy z%R$*)PpSx=>#rtVKS-`t%WT1+5-Sm_SORE*RM@2U$tdR`rPiZIbtyiT_E5Jp`R$A~@`(V6Ogz5lH@S`U%?tiS@4#5LPZ;70u zg~6L%4lm^{D3^%EW!0_(l=k z8mq8(t7Z=mq)*$-xkNgHDA8W;m@g%6bSuy1IPG>1>Le?qqyHS_u>kX8xz81sbB$s(u+}fk8yd0GtsecPTgkR(Vc=XpEO4Y~?(+&Z@l; zn8)>;cBs`RpHJ1;RTr9>_bQ>~f>K8#wokWNy^G7}9UO=3YnP6>%GE@!4^r2WG^cV! zjd%KqtH}{az3t|4yjlD0;tA^+l=VePd+{wn*cVL}+kZxB#Dqlz&RJtsyEz4f+CT}e z483zZM?-zRnbQ*r&-HeF4cZn481Y=FtQ1(y=P)Rlck`JY38yanN5Ptky>KRe*D2CF zkb)~8eOu%wCw^&yH(P>u6l?9&$x~#;QgC&bAtfsrgf4O7xl#a$nZ?63w=}t}IE&Bv zd3~f48Gmwx#n@Jt0>>`zQYq)QcDWy%xk$#no)dG#1+LiklmFsYs9xfkyOK$H2aoCL z5{we;7Xcp}e0^VE4LTJ%NlFVg!;JXWRGl&eyhzCVGzPLpjffB6Uz6iQ7Kho0znbV+g((_`R zNt@^k-dl%a%1rFwl{;v}3xAcfF`84e3dm=8xx?F%_PG&Pz?G+Gm__Llb^LtiVqtW! z-iTazm>$p5MzWDm%##BopV2j9h%=te5`Xzo%1{k7>DI-v3p^UmA$l66% zlX=-Dus2O^>&KWI)7Vf|#XB6Q6_#-Aw~GLY{>3fQ1pF*LfhZI^kLmedkaqjDzTVa{ zDVyXpy>MS~d{rHv7UOJ*xkKq?*iElc5Y#sBeq3+ zp_DEqnwRPAyYx!gY7V=&9_vXBT%x;n2}cg^BqlE7ln!KA+>7SKNd#ZPF*_&MO`A6C zvhhMkal$p6gM1L!?xH{S4xV(9dw-GkHr@huEdhVPcGjRICo0d*jaKCy$ovJ={0r{+ zjy(RK4hPk)f6I>1mO!KhpX9T<0a`aBF~9WiNGV^o8i;aIUI_)7ixDKjLm%6Vaph7K z^B|r6?fxYKbB{f1PH@6|$ z_9*U8HB=T}=!K>P4qUB;Z)HKaigcs|Wi?#%s^t!#u6^3QOUILNo_H&@?BzheVis_t z*R|;-hYRm1-0_fgFBPmlaeuq5KwgOLsv42EOz&fHbR1(*UmN!nZC!bWoTn$p#OU*q zt&h8at($?)4v#%QG$^VTd4mYs+v}QpozaF-w_IM2R6kZq+V;-}r5z`(dp3E@MmQgg zEzz~cdkx$pC~0v`x}c1bw87n2ZE*&(Ac zz~!|xKBSh$)t#JHml~N+NM``bL4?xeuF~*`3Bi<$msCPOw3W~fQe5+HXXh+%e!kS+ zYB7O-|#tnZ`|R=C(a4>052|D!`mfOuCeVX_VAUzj>pej%+rB_K+KQ1%UhOL=5@z;O@7U z&8a4KqQxmJ^IhWCcuMOOi);Da7-AoYBQWpc-2gd2#=lmzfbDSSe(b%gQF^g_;ep`&{^aeVD;16HY{6=IXW-dKSw3h1cZ`d#vIrisJkptlNJprJ$G5V#$Yj zf7YGwbaY;cWNjD9=H?8S`bOhonx%43BHwN%&?4^xE+or!8O<5CPZBlkJ)(b%usgeip-C#l85`Vy%uTLYxYbmiEAC$ zMf7Lks#9zVbB`kC5?p^v=6w&iL+04lZ40r@sO9S{Glg*6+`2m%+rXU8iBnK+oIj7D z^;`~>XQV7EF#WrTa3%{*UbSDe|zt*3JEQb@v9 zqtm0}sJm8yiqLmHBuxAnv^=)d>|VC^U8v)m9j3{{=k4>lzN3HXB=Cis_wb>Ll8$ZzTTxvbJ(j}=+66gc`19(t-Ks-v_BS} z#=|WX828wpwf6XWrf;j_hC}fT6EnP|>9Yg3c30sW&u=F4Hki{39FBzxl!mMCm*Y4O zJx!Rw_mTuQv3^jR>hl(@R*g0NkNP86k!-=U_9 z^`;JS(H?!?{&b=l+_^*egrxhrkeOt?e|pIq3KycaLr zKec^%==x_iZD~eZBHcN4wK0_c!a4C!=Y1bVul*X{LSBDs58=Y**Aw?RMCPWEdBiAD z7cEb|T&O&JXy@Dhf{|zMd0+^}2@rrSAMEw0T$k-zyP8sYlNo5FQH?dzM@ z<2dPbialD=2z+~?gp1-`i_FX{XVNdSW@?Wtk`CQ+bAgtmoDRl!T|G`)E57wv3=$TM zl%ea~#>!OFI+0pE>2U53e6<@Q?BeAquzZ^nFLQqr)@hlT^Y#+A_S#pfV@ap;L1=}r z&sFz1<=gq#$dujduT*kePRl}X$GS(5@4(?}9M$pX^Q+#R&^XeT$DLD`zI5>dw5T*G z!?5WwYj@eM%WEcGTXj$TLcH?B%eA}knv_f6Yu1r*r%zp` zJqk|KITnZAE=DXXf+l#E>gLnQ)J==e7 zrnn_sN=JcBnZ}CMWfo>+G#pt=f>xGl;`qwC2~RSbijSw

XgQgg9xQr0Yqociu8DemHBw&2iwVVWe!S2hc6}D>^ZVgVjGjZywgX({3?@H3^ z$6cNC%}+>!u8tXgj^}o<1fwy?M6@$~Tp-om9r^F1F|sJ$CvCHM>OrUes9 z^h5JZ+X;_)Quog}yqMVD!H0U1(N}H19vnQrSJH5<%dw3bBd}OPcipNd5(x~U@gRM( zxZ=ty2q;RA(!SZ0a|HeEI!%9qD-g>VZc{n(DyJT6zmCj!NW&^W7P0EE?qRALt5NPO znMqpd`CNW^7x$oaHdRY5AHEBh8gEmHrLh8-Etpysbd6r z-96pOKp(I5X@5*tLXe3ftc-!ePsPhljgxJWEF;4D>N%GiLEks-*=c`z9+U^2XWh#L z?nK`3WQ?hebbfEoRH&znxel2}Q#YgHkLj%8Lo|DvR&Vg4PcF+R0R>5({MFenP zQfntWLsG@FGj??>PPJJ{x>8XOf~TCjp|1G!B*uH6U+ki!p5!@xlEb6OYK6T+R~Rz? zI@S2ax)kEsX!B#hSiXNEdQ@=lZc>jovJ5VV@seSn>_PX~l}a3ijjOvu$qRUf@grXr zzA9%Wp|S^{F(>WlTtxilh?jdXsdJ$5@L-EICwq@(7~ao^b5bwQ>lTNeaT9%iucPXg z(;Nv_czr!9eak9toZKOKK2^qB{aK2^rAUmzeI5mM#4nPpJJ)~vX(@&FpfK;va!&8T zZMwsAzYmU=Rq|Y%{N8Z0k{aJ!f->c|GxZw7M!SdnUc-HjT6tmZ%e_)0nV3Y2AWSJt zH2)e3^W(Bv2j(2(n|;+JMG09jq@8+r8B;ZQHT{Y)tmESG1cQ?`KU<{%H(?KqGx)|fufoe&PI z%bPoCnQ3qrI!ra_>9+M3zCfGr{!_-|%?Q|FyDw`xNDqH0X0n3kH$uoTQhv8Nn=c+4 zTGGJ8N2NWTU;bb&l6asKmB6^`#btxm7%^GaPFWIB;v*4a@GygzRC18kal8h0HtLyJ z46un}&T^>sH8K55r%1YMPxmwbe)Rf5h;7pr@zPoHIDJseF*>?&SfXgBcf)Ps@!ko> zr6nD=qS=3ka^*yESQGM^OymtbwY^%dTK=fVm&&H`6SJf$9o)zJUCsQKS``y=Z)JBa zT92bPvIR>~F$r+&BtEF3P)E~E&2*6pM%?!5O#pIyiwFCD*yT);wtV25$Yvdnyza>$ z^RC}UlfJZSsGl9eV%UmTIDb$GoCy^6d0pk4@8y4O$}K5bi5{z$aTkRPzd2^}RK>eR z)pL?`GY<=TOK@!w_6%1tNlUHZF=bqNO7v*xXS!)0ho^CgqJ1XBTJO6uI0@&n_Y$gT zZX-N!@aRi{r_J4aVX8E}_V*SCZLLmMPx0piIrQ3{J&WdvsuyZ!#_pcIJDKk4B)ih7 zip_uOjVBKI7xQf@Eu{(e@*M2@n*Q9)>_DyH>!pJ1cmftVR==w2YO~s`5^W9*V zq$TkYJ6#t63=$?juhI+5t8|txi5q6;6N_y(C9d_6ff&|0MqyWCWOH^3|i(x=Iu@^6*q)!EG(ovjB_ zO}o22Tk>`}>L*Gk-X~o>A5(^A-`9TvimY6_tZOY7A4)yXlN;Xe7P{i>oo=*sKKP$uf#pP$@P=-YqZnw{<^i@+=g79w2sfN8b%gbl-m6CV)D@uJ*agL!6- zsl`*4-^jw1X|ch5*5F#!j~I*8O+cq@VFDpB_CPj`c!b@e6z2!#Vx67F~??w z_uzea@Ow>bI6Kt0A`ikBG3tkN!^%ucUp?&64Nb0(gY3Tb_fuL)Zk@3X`g#$JJtM5* ze8B7{HnJB#NqYt?++{Ki%u(37aMjQj`tfisB0a$&cymXrA~H>F&SrlfrfPc|cE(`M zZ}t*@7dnr0*?|`-ejv0d}F5=+*T&h0#e<_592gN3LVw-JtTxfQLK&JzunZ2 z4;jz-FN5_qacyt@pvE^Rvzel_y0cU-g>ZjA*@IJ5@%xohYjGm&Q2nu(zN2kv0^k-0BQJ@HL$7BIo%2aDj0_ z2V)k<0bx-5NrV8?#AAJkJNhh5!-#j$XzXX3^s(wFS#FeB{O-vw;gokQS*+ zJRRQ;9D03q>d!cg9xj~Bi&VuzC9CLr}uhPPo2kaJAsd4Uv z<9!b@mBVLUfOnUstezrMEXwK6y#sRD4R$n;ukM2~)1RyXOZXUyY0mC4oLCWA_?#`CzYc zi(%91df`KC>H~lO4Av%y+Dm~^=3@(w#V~cmHm6n+D|22XXq;i%#6tTiJe_m$7Ev!u ziH+N~QRXAPH{_c+gEZ5*J=glue=o0k3A^cWX_LHo(|ZqJZAj%TAw>7akmcvG(sU*p z3KO?u&`}cE$_Fqr{PxIRrcS1Qf9R$lJqf8QQI@E8C+zLMGJ+QaDto}-;uSk* zGfvoFc!GaF4%lRtBx#-P1LoT&YRzsh-LzOeXK7C-PkwBkxr_Hmg!g8K=l8h@-vhy9 z&rx5M@+52Wix_b~e&n?#_o99?Pie0+%}B_ijuj7jQ3?0TngVfvx|-52JYbJAC9$MK zVtpE6u8XIrl+UD>@Z`#pQa&Dcl&?JbL=$wVs`KjPSz!l;xG=*7#~~p zt(^3kdfb~ys$Ry$T->QTQ1j#-j>lf#HlY!3F*SPZ*~k1bKVkIYjGH`OWQ1cx+x3xG zB<-Iy{PiH*T#c(yj~}}@_x_fg$WA$?k@%ug-DwQn;*PI*zPEvYrHc&`~#vu$IyM^u|+c|GR6%}_EZh*N!Btn;=97t(q-^_{r0 zFr-uQa23P8*F!iSoY215DXfJ`x7mG-FA9HgeeH&HI#GDilh#XD&dtG%$^^U&zfL9+ zJRx1&j+Y(J;Vr`7BmUgzT?)+8!!aMQj2IoT50<#da2uxM#U)=uK}h#ZU{!TprGxh> z*SG$@t{agD*W-IBdfGahhyFFJhD_4hlO$=rU#XGKDSpFkevKstJ_og@Y>U64QS5)E zs;S;xl!PB&-N4`17liqRRg4DKhrK)6L)0z7(|6%MWj>Ip&A4htbPO(gDG5-qIjlA0 z_<}?CYRC&G_J?w-(}U8+MiR%4cxs|b&nEBjnA8*9G{DttPH|j{nUtGQ-8yQM81CGS zu9)#bK=XrE3#+cF;m&!Cb9Nq;b0mMo;QXRs^3-$!D(bdBn9%O*F_VD@K*ae#sj@uh$!4uJAx~BG-1jxo0_PCjo@M2JwQq?DIu#pNVz8MZ{TL zISd}%M-O9yGWJzGDZ%ADsZNL!L7_b@YsQcDB5YFcP5FjPCqT|5At@S_>=&$SsSIH3 zB^J`gA=YrqIjT!oDS1}d2~>Z{qbsDH5U|8LPdfe#14e*8wPzC>DrY`?61B~m#S4G$ zGjW>;Yie2cM)IzBvtw)ZE_ac;Fw|6wM=3QQXW_}G;S{>8T3z$yL@-yTzzH>$ZCyWj z%8cbkg|nB!^ypX~`F< z?yzW#2t;vn-~HG>fiP)RpH(wD^7)slj)QZq-T`)~!5P zUVJk}dA$^9x98Fa=RRntC>wVn$a6#82`1P6I$9`>)e8dRLQo?P1-NkzqOd!nm=y z7vZ?1eKJAqUC+m9&O=b4+_^VP^I9IxoHG+=p6eW}_o^_+h?Cmbs3X!cR4)*K60 zJ9_9QhR3JoA&&ba(Ukhi#B)^a$3tSWEOEX2XFG1dErT)h3aJ>sVzaLtN!(DBwhdhS zZB9|^y@-X5t9qjchdg-6U0l=1A8L{`-O@Ce^?0W3dq6oSS!~-`(1xLOUdpCpgG(Q! zZBANG@=s=d!Ww@!lGyhaRYY;wB~nTV^%1?NWs_f_$O*S?xc=X4owv^RFp!<^#hoDo z%Bk2wIp?fSIp_3BSHJjvhc#fgl1L6&jEKRU=LB8MD>Cc{VMeF^*ptSuc}YZU>r_Af z`WsSn_-POu!a?{8fZA`;v}_-|GeG+6}6>3QBg4$Q92W|YqjC)Ce zw$0f#QByl^)aS$_y^-vA-Dx8Z{b$PFHzmd5HfK*=F3VUG) zi)i5!9UUICrh$~-^GF9cL7B!k+O3mwM#gCjl0gj1&85p-S!t?zdXp{nwPX1>zFk7! zZbcHGI}25!7Biksqpg+32zjYL71zyO$=->dmpw`k6DZRru}klQ>8mC&~gm zhyu$O?D303&_CQZPj-}Rvd3Y^g5oemcEEp%4q=7EBV+ocl~;Jh2)yfu>%`ou8lBTA zF}6oY-c%qkiUpiC-KH9ceUEk7-9DZuF4zt?FHgJnEqVPu$lu9|J)6RkU>z8kLbI|L z8P>?H)+oDwwdEn7;U)ZqfM?9`y6n9qscymW(i@t=kSNS%Fqz_}VldJBdPu?#WKMqs z^xobiu7gWm?`79-W*?>K%Nad^KphaTmw$y*x0GE?y5g%xEi6zANJ@yMl5q~~ttE;J zz)R#$^<_|MRYDG*@iJ@U8?5Z!2y&U2m4^R&Uu5IJJgLtDw_o~DqfW^|j06|z!%6{f zQT;XOHh6aC6TRZG!hlb|@a*%w9gXv$Z+S#8>h*%TlVzARuI3Z~ytE*YkH%x<1TZtjO-13{6scircJ z0PQbZjqp`q)C*zjJ|M7o(bRvxafm{*2k05B>s-QIQC zSG__i)JR9bKjaKnLK>P*(8oI1z^+^Xe&2tnZSo}N@4#Ak4JD_m^pqW@Z~(LC!ydHf zmR5OjBkOs2qdB%gQ-3|kc_UI~iCi$DgzNbo z6r>-GXmV6Gxml-~L7%;f=W4tDPWFT9bmMzsoq!(B-Bo{vo4`AnZQ-?K*-meOhT;*9 zt(LBZEiq`tmf!{zE?;{0ib*-$Tbx=Zq6|Il4n_u9g>2?{t36Q=@fLU8N@+n}-kK>T z!1UcJd^6h_Jz0RC!N`B@;@LIbQ%@sa6ioff1u)Jnr{v@Uf7-{on5SHLvJKKM8bQh< z;-$xe-NyjZb=K^N)4W8}adb{nDhpq0KzeP<6M4;RP1mOESyKLLy--vqQtP~{TZ}@` zk{H-p!(~-u(~L&``P~z&4gDY{t)bw3q|Fr<!6v_uH!U0AkQR z?)l^$5XQc6@Va#2GT^Ld_?I?upDVg-WtbMoh0Z8G1@NIpvqywNY0SYnqV@U!;QFt%T*~(BpI*%3 zUhf#aVC*jm2pJXYO-D(5Cn^T^-RACTTj}Gc(Lp!bx$A#^Y7|7SWr}B}LRF&Ro%wAD zM%pu89<*1tb!JIdezEu0)jU5F85oVpS8KRs=o<&6Sby==BOy&YQH#nDI)Yx&5iZzh>&RCyr z*Db>w##Vorm<4{2d!nSdhq^VY-SQV&_2{rf20hX`eT%#(FEXcJDPDR1p_^#MH(d8N znX0+<$3Y{aT;G6NMy+wRGQ2IuhdG@b`igvGo`cLRUF=_vap-A}l8DgxQp;;U_S&1Y zWIy*FjSUOSmVtS<<}Bt|PZ}T-w8P^WPkjf_rk#ITv=_m39HBlti-6uZ=uV;6tn8~gl zzmpxxv>P$($>9M5d{LP`I0IQx%?oEx=O$62Wq<4<4)`Upd%b!C%ds&#>^16(8P@_p z4B~$v3hFNcLzKar)DCbQf8CqFuEFpdkQiaN97|S-pYp!RnMWuXY&FtUTolP2@z~8@ zmODmhn_*u8ZrLW`g+ji-s{PV#4P@1+r@va}hsCe^IoKoVt(98#2C>YH-K;@ccB<>h zWckT4$L}u0rqNC>{=Qq-w&~V37UzXi05g9km#2}f+U#NpuLA6JH83p$Fzv&+HpS@xPk8ip@u6^J+Yk6VanccfdzDkEq^QoUj4~2g%GB2YCE&d=LS@M7jY+6 zHbOk03>g2j45m>wsaZvNn8VJwK@Sh!b6LpO8e^$V2@BGEo#>2ct$`A+l~wFdxgUST zV0LYw8J%zp$-Z!uZNYf5!BPRa!Ph`XZ9%LzZtK!jyeT{Y+A1K^N)dTw0Titdrn3j| zn&$`r0JF7BNRTNP<-ISBf#~!dZ|I42Nv|9Yu3tR^{K#&PUs- zFqg+`Hng?})%q+H1sv~6E;A&Rs&#+-lrCf7b1vH^Khu#qJnH<`5uC`pLmcY>20Fo}$A ztW0@URNl(`%Tkrq2)!AI&Wb%HENEnTKpi*k%Qe@^X$~U+JZNE6`hxgh<<}qj(k-RwRr|*PwbMc$gwr+R{ zUgAj7I|O`>=Lo)CO_KYKLu4$15#-IlD~NdPa^FFV(yCDzg+ha3SfB{=uVN$B$y%H+w;Up_dx7B%Ko5YxZDwq0kKErH}Kcn1IGO_G5+(!&-w2rj5yS$51ZV5$fL5 zOfKjlJH+TZ$cOyFnAe&L4uD7&Bc%Ef0qlV^=p@pNe5$f@+)Q%&TBJ3qDhjwKWYJ=j zjIr8NC#4Mhy}#h(3_gDmAt%)lCT-#}KOf&u=tHx7jOz$S%z@=@n(_Mc9ddhgkqF0| z0tP}0@CPCs+A;~5*p44`jlW_t(d&lRlS>ews{`iEMYEig(>26WT%q!b_r$Csbzm2wGI;svT1w45Bu^!gCvS*4HK4bVwVl$dtRM8cd7p)`ucJP2g?H` zqgVBC?n2`rSNIlYh78?ijZvRdcSH!&d8>|j-$#Xo={3%^Ynz~#wCXY~Fi_oIllVlR z4Eh$;NegJ1pqt}uK&%Z0UYta$GI7fCpkHStWICk>o2h^46#I%pq{Vld=Ec;HBTrB< zrFE~IrhfE3tj~x6+{He4hue(97qQ1Xq&j*0?f~90InBvH1+ZF?(&<u|q0uQ!< z8*8dVM>KzO32}!~9W6?#Wa*+~m5Y@}Aa-aiI8y>+UGpv7!JKv&*n)9rM1Veum_uA1 zQ>~&|qNEDI-X7v`jDnJ6f|aAr2jUVPf85<{@2T0~jyS)xmhGARfla+zZK7#vbCk@_ z(50t{c^qH{p+jVS7O0_BUawiUtO+&I6&&c4cf5aNhh*@K=h&y8imb&JY%sTadyw%h z#|f^_bE(!zY2LG;@ zy##@i%eJ$rJ@?I<`2yi1blB+;I_oExNfp%rzV)S3AWftdAy8*BB;-q;cBB=!y%`rd zu9JVOd|=S+m1n=Dlz(q!C%VWs$qzf9d_T>Uzt*eK3Qu>77F9KZ(dprAFPbx&>(h;H z>KO8{J<$?r{YCW3#nIE3y#BjLW#lq1WHbkM3hdYZ_L~tBC{o=nsp#Nas%Tam4vN|A z*~UuC+O_p3kN$aNGfiNALNY}%9yr{pJxhP1_GQ`E)^WYVakWPVtWXIl=nV58|M6cU z`G4~D|MuVi$Nwv8|Gzcu?*G=b^F{N2H0|*7{|B@Gmzv!UUh2&+rfO{Rh%=96%Q_IYCqC57l6De-HoXVo{^bQoK%uRGxpIBY!jPfA4=W zBk4%Gmddt3|HYU57nS`V{lQ6*AD%Y;cd-1RRA`5$oIkrhU}G6{yNKfciPZ&TJkdc^&c^|?KAxl~4tU?psXVlWK?MOFr#V6@%SGh}7 zFuh2H}2VBKi@gV)}L&0M8!{D!2 zp!Qq5M9zx+R+_P{23D{N9u;*KTy5dt33>ODsF-vNG8iLf?svP$f@9B5rthx_P9s&j zfkVI2Shgl5khr+T6V7aLB>R75;BK(wd5#AMTREhm5JV~{+%l1UFZjCKi=dt!bWg`= zZT8}OP2*E+XJ-QXBolc}GOM`G#$J5qB)$1C(Z;QE3j?r6+pE8tsqMRr!e5sk(naUfn0`h_6bZ z`aaFec+R>I7KzG43|J_FQlupeQn<%mdG335E!p~&Cz|Q0y+$rC7zEgeZAc}{jw8AI zMVE3u(yTlzP}Oo6rp#$giY{5hAjk>pO!oY=}$oJ zR4s~;FSc*FEz6R%iHm;~k{no^ny5yGJ_SdAguoAoOOlu!=+4G8z;oKpqj$fC`XLrD zeD#!rymk}X1Y}4Nl|P~>Q4-!#GOqk!Drr``S6^MXAM1fX!-2peKIyolKjni~Y%)|m z43oh|AOoScFgsOKNr5J(x9zlQB*W|wx6dj&SKjaz*xNy;1dunj~Is@>M&(hFqTr90V{VJoq`*LjZ?1o?y$7c0=Vj;tuukfD@0HU9zOIYef26B zULTf;eqT}wup@?^E|Lnhs9IoT&n&=6>aXx#T9g@G*;Qz449N}0@guWmWLV~L+_aK= zKIjaJLn`Eys}Fw-C~yvB&I(6UUV6WVxOGN60z7ef;bkL5egJqmxBKA*fx$i9=x<#Z zA)E8llf24C{)msQB1b_D$nd=q)Uu6TtPQeEh~q9t-~^}lLS7)$-~k$45EEcI0rBqg z*H(${6i-!<2L_)$4uaAvHK zA7sieFGbYY;i*CEK5V}CDJKzl6OQ>rh$!sOPX+fm0ZwVZ93-?vwH(izT~ALLK_;L= z_+1+IgUTx;lY2P9VBAf&P0^JeYRHDsxJD-pbUj`ROD|pzl9E8Iu&L54_zMqcFYd2j ziTu%+M%RB1pj>r+DC_-@hPb0(dH`yjcWs$G4W(VSy(YgPI# zIP1Lf&Aqxh_Cw6p1kkxAWRju_&ms}Ja`3JgmK+bXOcGd{TjD6wOPm?IC=)+4^u{+xkV_VJ2*bMbJ zd@WL*rOtH&LZ{(M^~#aoFegCc+PIBv-dzhS6oFF4So<;?I3{d$JbF+;Q49cgr;$M-28X12W?` zn;3sm3IIRD$peg=BaK|@1g4{9z-?u3%B^BWpzEk17+FYX6uc`yUYq#K9u`15~8bbj?1 z5IJ+pcbVXKZ|Akoh%}9#+v4@2gfLt@wIU5*Ox2%p}IsHMnOAM18 z_&(qf9Yu)ZDf^t?>0&t=-J?U%v9%cP5yZZ_zaYw3;TVwxvMDJy6R<3T%`5ZLoDJ|C z2<3`*O%~JjtOB{sLn$}#?Xu*qLs}TgvXu;j0b|q9R)1_!vptArv+7WW|C8VkDZGE6 zf_N{MB|S#FlWpuHV_X)-(|JX+ZMCq*Ib(DO%EH66G3=R$@oV2n`N9dsVdE=8A51{ zIFnbnik=ncnDA@K==f_Tza|Zd1T?SXIFSV37mQa%>1ebH^x`&zTjpuOMn8YK{pAQ# zIF5x=D&K#t9rV|g4(?_U0hh>X)NWgZQIGJ=a_Bx!El@w*ot&Q38Pa|8Px#K)kP(sGU)da6odSf_T<1 zVsJ$(;P$%b8z-;GZncJrqCqms8t&y<8f~UWt#&)4uYiWQ@8jYFJ-wf1DeD_hca3&t z1y3Jjpbv}kSVGU;18#p`RcPx4%tVlejLXoF{(K4(C^9*`(y24@8>wt~Il+?X3%f$s zjU+tKS$+65IN@=UG7HzJXpZjkQRp5fo5sRSfdREf-}14jjx7>yUN)S}Vn zG%zDT`%zSW_7{Ili)%Ruk1}R=Sj0UuJ_{+9=xgwv&i#JK)3N-k`lYnXy`5HALd`f>#f*x=*O|ma+AtjO5Z;6g8XQ3TI zzO_Gwfv2+y$i$usa6@!>+Fmc}AIX=)&|Gt$k2*bQ; zeYl=S&M?_P9~qf79tl+a2qj zp6~Q=uIGDT(6xTHGqY{_Sycc7X>5s6_icZ4$eceyHBOn@FEacZ`S$Pol;{598s`g6hp&yr3#oZ%CtDx4db8w776)kPu|P(JWd$jPu{Eb zhZs6HLt^69oL9~`5;-nYGI6xbV{}ay1qIV+ev%9QpcTvk6A#b=gDzq3tRdLOyyAb! z#0P-W^^t9p8|k#=F}3HJX#L2HXL35hHU)V%#=g{>kDNq(EoW1I!{b=pakzn(RlDdI zUe8zNPX}TGMvl0^0>$XyT_+J-E4gM z>k!!@bW&d|{F}rWliC`G+O9bkf9W@1niHOFtDZV}YbrwJs?v81AxMljlVy!XU8cZU zgYccT5Zy{1G(noKB~A5SRsBhHpHgG&i?RuS#ZLS+Og2sByFSUOsxL_!@bt_9zn~sg{kp#mvd>L1Cxde)e&NDwj_)YYU zS!np}nWcMFRUZ^^l-rL}KZl}-m9KY(-;J9C7ude#B{jsWZc=E~`H4lx3q6XrhC!Fc z0dbKC6HNv-BroKN?lcTHsVdrCOS&k4yLX9QqOimOvUZ9Ks<8@gKKy^Op^QT+=6=KR z%$P8@uJFN!U3&ga$|1JA8fMGm{wNYsQ4~m3LK}Z(Wz`Q3=bQe1LXkx6bgMfNXLqw6 z1RG92F2_$I_ih48XA*t;+y3I5qF4(kbTrUVuAD_Y2{_xcH6$++aC+wiQY-!r-4N6J z9+#te9YAHh^@ihJz>0ryQfVaSH&4EcBWzdUVtaZ>fRa^9(NC0ZEzi)VFPbfv>|%kX z<2q+_M_WT$sHzTGJ^~6CU-88DHj2Nc_91Ih6y}elPhmtmc$=(~i-0ZC78d|uEF5!ZJoo64 zg-qF$`r$+T>u`U&v}m#t^!14pdFrIQl?@0 z$#W-lLU!dSuMJbcu;#%tNCLX*26ycl{3In=39G@%hALc)ZCjzJawjmFBC#8*zOU70 z2sh$NBPdIA9Is86s=j%k$kX?Iu9VLTye;ZBX`WmIK_!0>1Z?B46Q!|%oUKL5A*^Ea5n706%i}8akSGVXJsr})!&`(7peX7S8lHBR=9IjNj zTAuWLX=OckvszwZX6}%l2J6Qy*4jBtPz?_rH(-Cpwh9+g#Yt1l)+<=2m@2Ytz&YGHtA^jpRhhjJ|2!_{qk#d#=~v)QsQW*XB~^ATZR58=K6wRD+(8Q^6# z0r~LNO?f=!UUih6A1kyuo35z)>W#;wCugI=SZ?H_Y58y zWF>z>+TK;1?nW=+qG(HRMt2|e0KzhfkHKpfar`CBd`!mEJTAk?R;^RhKGdC=4;y`s z3nTuv2{yb@ay%-Mz>2*@e-noA{$9Z+Pr)}TG6zOkGu1Aofz1+H)dFbLRfgY}`V!?I zw-yI0xbA6C6wF^Kzng!>-x(@NHM3rK<~Dx>!2;kAS_rJ0Eo(UkjS+WFr|E^+u1no! zrN<8_ugbWp(yH#8ui^u2;hd(Id>*%zl?^2MCc)4cMg9IPO~$9so< zw0|b!NMvPI`7_qo0CkI6>XlI5;eUT=ketu=$mwx$Nz*QIAC=EE9A|?6QaPJxl;@bs zGJQ&k`<{H@IZqlS>XfS*iJV9<1e{`~ivcc$Bv3~rKi4-{s%?RV(V*&;X-7~u@NJV1 zG3aI0343hdpiI&DaGKs9rKO>TqRNw~GB@%VaitNk-QpHFI*yEWU8)fEJ8yr}1vIDnO$_9-((EPWG)8Mw6#{dEMgMY$*$3LpoMGKZo?$8G zIAGLe>VFn~F3K=Oh%$IoD`|gVOy9vQNIh+Fh;=c!D7c@Qxd+p=)Ka_9H_Ca%Kw;%r zC9RcS9gu4qt)5iTM^^|x~Usz!rk44db%w_5`&$$(%gSn9YUP-q1vwv zJ6^x7w$SE}U{hjb#m1*kS#Bb(LPHNecIPS zPo||g&iFMz(<++UYvekwNty0K3I|pA^j7xbeYg$?OC=V8Z+kRRT$+tK{8LfBP{zIl zRkOYW&L|SU?=4cS&@fwL9gajRjYO=E0!A8~=}Mq!2tTf%3haOV4fh+GvgIgZ|72h^ z%g>3?EfPA|>w;XF@f@rT%~b>;kaJ4KG7`qDwlv9ks7dr&@U{aicD;hna zM5%lqb#n#?!dQQ0=W+n4(qVSc*AuSe^#u4EsMLOP#XW*=#VJCe$i$-I1}bjf#ywgI>z(#rhV|E% zgUc_t)_!7UbgV1sAT8e@ zJA9($AjU`Ak{o505FA6_n8=AtYNH~+fpDU?wA<^Eb%3Cndk$Yf>jhPX4Z}TrX$x& z0O2(Erp$j#!>9gO)c9@(M>4ft_eM$yahCx!X49?tVe8O(!BhAU65;GCbeCKiT*ItD z2>@%&JKvIt8{zK@U&yndRZ^v*)ECS`%J8Ko)?PhVPr?jZy#q91Fcs5Wi3(5tU`DtY zm|B^LV^!Hls8K_*N%B*f#O8CsfIRLKT7^Y5FVCkyZ!o$PdMHib1A&%CIV_G)K*WCM z2wwPR`L^W_SWMy6_5)NHvxnA7PI@p2fHPUiI;In$mhig0$t9r**~QUHy#SSTbpA3U zGC_X|vn9SznA2YTlEYe|?YaGMk%1MreqVBvV%$@=T&~vuDk;M;4pX;Kz7bG{+br%G zojDvrD{`q6eU2O;zJHIFYnLJ2t3BRZrJh(*n3`!Gelsz1-N_Ytf_KOh8U;$x$8vMQ z8(cl}q|A10#2Dx2?dg3@9b)_Ad|cn)E46>OLU)D^@5noe%m&@^R$<{!Bq0u}s)4RV znf5ou^2Put7*n3uXvB@9IV>oI=@;29vQvMcvbNt$FU5uQ=1}!t=DX2jV!5AL*MA$?VOo@Y8rn% zKf-ODqvl*6;$a?ENHWuUdOO2-9N$S!(aD~Ol3jvSwqTlXQzDctgDF#kfcmnJL)e}4 znmOG5XgynqS(k81v7(#O>5l~~TPV=^4fZw{h!s(slm6kxPkrzpbFb1JdWL(t&t{o@ zb+BO?^^&cM!T%*I1;t74E znfOI`EF<7Szu-E#VS}RR*A%WuoUZxBNwGHk6vj+}$yu}`?qv99Dr#o{TR^10iH{e5 z;wRN!VOku102UKKuoU& z@j-^MX>VVs*V2G)t+m4I?{}gLQOT`;8ak(|yUJ@|6~DGom+?4VRcXNlX&br(LpgHb z9vT{hbmt)ZPVDqjTOISg6m)k(;~oOV`5|9Nm@@;)F$*${64PoM=ONl={65(^Kqyp;}se$w@F;lOMBaM^_|w4aq8=loTzz2`KrXN^S< zyR*ssl6VOf-70J4>Uo#xWT?A;nt9bPY!EVUk+R^!^j+_I-M^2#CPGp-^31~!!vVaV z5OtTJpO+6qlm+jm)vs4x46^lMmK+$6)@PdE^o|T;UH!JQt>J2BnU<++y4y}F9~)Vi zZ`?U8D9sBxE#Y}_63$NWHg>CA|FPGHWF{jB{Bm;hZU^aE(V?F29imfzc3r6OK};!V z#C=nPmXn|#GZbdF!;vPj=oUQAHhAN zyk?%@Li1OwEs;~x(+oFTUOiW&U`N0FPUNNfokeoHc59&n;f9CWl&2%J8w7-lekf~YtLzB6_7%8AboQ~G%z^*Q$Hb6_|PHk}K-;*A%*b`xDa zzl#EWIf*jNGo`fB4^w_1K4^)+#NJyjx0+s41g2%a@L=XaPfZruKQqB&f2Sk-RFbM*sl$kd~zS?w_3jEoa5M(liNqw zt}{$4=gLQ($sjoa7mf6_iyv&s<8NB)cXsn~f3COJh~r0pvR>cNPTzuz6Wg2!r-`h_ zn|zpywTkP8M#^H~4j?bWWB9lq7U&LEA!=kzqa%)2oSxa;db9R+25u5x`o1|m4mG)b ztAY*d(e^ptJh1?3=a?N8GlC0MSsQ}*A9nvJ`pS=`LjLgSyOn@H_rDB(tP3r1`0WoI zfpSAcJ1e?>zv06fDs<4oD+@d9Ys=^2?TEbScgTDc!9lzm5Y?J2;ge}rSqzQ0QsR1! zk}+|q8o931MGVr!Lh?SW$-10CgZTOngorJ1%BpAQNXZCb4+R$!dl^_C7`Pn3 zFby8PpR(xaq}DQ+Z5JGOyUT=wm*lgOuHyShOe`XQUTZ){qJ5{I?ZN(@nkzjh3Ejy_ z#i*f{8ZZ6@wga_OA)Ig-SFy`c=NS2(JwIEE&FOE3_y$c}`P1p!a#zs}kitY&*tz#H zS*pBaQIgop+?aW+y8u4WUjpKbU^E4=zKK3PXbJHXSe{O_v4d*04Mqn{uUqbk^6%|t zqaT`o>M=+MK(H`2 zw6*8ey|IGk%Px7g(zWZG%9nGGiWH@XYprPZ=E7xs?t`*g-i0xFaqXm)VL5BIz-AVC zD`w;NAJ3f7x+wW9ezogkrhXdb>OJ&-b)=KH=pY;6~nROs<+J`rHqud5h;-7OPP1mJPr>1T17OG6Q(P`f2fiBi<({#1h*Qz)1 z?23g({)%p9LCR&Pz1niMiyhx__=3|DYo(=q=akxM;`VN6xXjG!H-AG@`--J*?@CH*wA(QL!~Sx_$u*WdNRU7)X!C#^nF zMK33N-`!V6XvTpYS#!N#9KYXx^~#|4Ti7IOwz^IhH=UNqncr1>Q=xsk=e zQ`<$+Sya^hZZ`S1@1C1Y#ulN$@U2b{3p{(olis;(FDvsLZG>`Aywfs1+H-$thEir> z`MV@5VULRnA=BuzRj<$aZk9#2^IY*h(?yz+M?RN-t5ia`FWk9= zh1VC&M4qhYXywZFJ548A(;*xAr&2ye_Cb%V=V~z{`eJ7u=}~`evO{-8_M*-o3w|@4 ziJLhNopA3KU7To=)#rAXefKNuiRKcN4vTta^0{B=c`}oFWh<=?(x&*nvdw;_)Z46` zPl_2m9#_-r3vcG@-NBoG8oTW-+ODWk&Dl15>)jgl->p2s_S9#_rtoi z=Y5{OSEY8Y+-0_%O_%Lco#MSX=fxTLOh%GI^{0BSmgcFCv~Hud^ILjM@FZ{-`eg5T zx7T(1m7jI@WlHMI(%y%2{xKpMklGRJBm-0^JQ% zS;jN5FAEwwPQFXelPZi_b6IcKThNZgVoQ0x(9Ods(o6HZKEDr#Dm8a~d%7rUip$r& zJr}d``&riQ(On%|4{uLy5l+!e%HP|(SX@cdp~EZ@;iVz0B|e3RYvd){pq z3-fuYj-Pq)cEJ9B*z$fm;ko~{k>{UX(mY|)W_!|6j@r`sOnu~ox#tzT- zQ3r7Wd9f)jH!%sxANV;@J`{BkJM zGAYkzV%o1Y9M|#uh#UTx)cSXF{{+5VCZDT*ndHWDt)ukWUX0bJIp~!^uZq9-j#mY5 zAD?-Dgp}QU&h#`6?POhO+dyAT)y1h^qFrIS(_<~=j$Rmca6l{lkLIU*c!(ZkhP7xv7rB33pQ)4;C(V53r`vk11cYi@Nplo` zW4f{Y?@rkigib2^VPEQdcaC<_3ye_eNWrDIE4`HLJ9j?YV<}E?9^5IK1@hs0yH9%9 zUXQ_)Bp0-4^ITa~2W#zYdq2t!<-tkJT{=q-*HB(fr;FLsTZt!{J)$1pZ|8Xbz1^f; z>B@N?T(t93*B9;K!VOZANxGR&w3T;%pEuLvPIezwsF!V+-ky5(nE>3|%_I+%gF5Ts znO-tIe{K86gu)4z_yY)CC*N+?CE-Mw+~?^PE#j{1R?OK3=JBQp>6&dn;brqgULcG0 z{Gn`TkWAhO@2xMMPjSrmi`Tk8pOrcJo=bVVJFWCa?dOUYwO_96r_Etr&dG`H&O4dPU zh3^}OlN;6bZq}Fc=3Uwnx=Y98dXT12&4P8`2@WTwv!2IxYff1z)N*iye<%ny6PmS&Jdxsyct+Lkx%eiyr z9~Do)^`M<q3#V2K+auRFkvK2_}uUrkhCsIng+~A?-v$a`NZ;~A4zj&|Vd<$jM7j{$v^`OMrEI^&`EH;l^Mb-P z%vW|RO3Cn=Of721h}bR)E)(OxcJrbEqg3`4dV(RcgNxp8V^!x~+uhB89-U(9np<|f z74wT({jBDn^*?6EIZD)j&1MsWCdLoN{qMicgi9hL9ys6MO~QZvbn=>Y9c}XIjKsl8 znoCdX{hmdK9awR_Fh`|$GVu1|ZrU3@RSZ%O^cxv=;9Zq>~!Hr)4euUs$t z>z5@nuhr*$F8AB)@wuYIF;;W$Fh~1TE}iXZZhSV@3Zo=(umQ`cm#OsBx&zrHB-#_qPsjj(NFRzV(@F z(pRMAcXlDMJM$cj{Ve6(#tK<3=fB5$+=C^qgZ65D?n?J16 zzDV!dH+uBPV)+&6u`Q?QBa+D!OSyM)PT{-AaHJw77sLL4-K)$9cxFDOn<5wG_oho? zD{Yg)<=Ac-)IjrX3|Sy$cJ+96?|q*d=#m-ek-hJla5W|-WOwT!b1OQ0q{}FHHs>$# zHETx^r~0$cgs;JHz&cy%<{2%@(@PSeVxT6P_{P3iX=``ZAgZG}13Z(NG>ms?9VXib z1ZXYE;?@~|%hcF?+|wlVNjzbP&4gBMI*q+_NiIEH%F|V zWbZwnPSo;!$J^Wan?HILt&K#xAEkGL7dSjg*FzFv&3(Z_cnCL4)4v~82V*@cJ@f5N ztG%>;|7?1>%e9xiA-=S_+HZ3sz4h;`oGz$-PeYD%{r)*pRvdl4+~bm{wy*hN{N6{` zELm%lo4t;U{il6m;?BchE2oRFpMBlsVY=b2AL`Zpkoa8yN}Qg*IBS{W@?@-wz*6oJncL^o<)!6OS>7{m1Ew*N2D-gPKfVR zmtx7#o6ox?PrutHxEc_mFzt(%u`FgAJ@^{>t&|Q|W7V6F`TN@xQq{cT(?)rg6smoH zoMFRc!4ZKW<7_R1I63%HVphpzWKNfznopLgEgkJz(-B>!eFX z3oZF{_sy5?`sp2?8@esq%GQon^2z212w9(eY;V%;%B!%~yD;LDJ1$Th4B3w!&HOZ9gnL6J+T5&TbJ4x*{&d*9zsJhI?03fG<~L8fFrw3Yy|K!C zYTONn%nwkT*3@fn)pKqx*~7i^-O%ay76tR_=)=zU`$ZM9=kuA*-KaF>Fp3dPFQF9A z!`pS0TH(|!aN#KRr%L@gEAIM#?6JHPdUH;@N1HcZri;pPzkPzjD(1%de=j*=DIJlY}GhV}}R#{)i+4OY+OB*JQnP;)M~m zi;bx*Phoa9rf2WQ+vp;HE8WSS&rbHXJSE>nUhbEhW3Swd>X775{WNtHD!t!Xz1>*u za$~2u%FH!Z@1_#__j|9}$z!e;UG6K(qz~kB-QON@S|&|yc)Cu%e7jJnp>&kkOS_${Nj5&2v`q`@vWgeDNN;lWjaY@*xc`fvXe_c&Fw(jdx zdwEd>b#b`)_m1v=PW1jsPi2H}>w3Oh*iqi@(v!o|$jzo@Ka-$-2Cn=KG=E6?UcO^Kz2zO=htVMrOVT6)ue4ACr!b=vxL=i9k2H8acj8po;$aH^iotiaVe=<+;+ZwsPNPyPNVqO)pJZM%l|c|NlwE+2Q($evhEdT6(PJ+Dm1yc2KM z8}8@Pu9H=N%lwUWS}vWpR~D3N_`O*2aHiRn19&- znp5zDEM|VO9UjWsEoZcG>}7zc0o+HLJ_v;^z4geWuF~ueUX(r&Y3JwFt7I6S38L<;MjI$Q{$|7llioGY}@p@^N_hlIaB&JWZKoPE|tE* za(2Eg?rXlZeOSnLKnqrgJ$ZcNjq!>z|0I=voA~S4=T)(t)FfOd6zKaII(2ss9>v+8 zWhe#6Zf&%bcV}lQt=!{@s=>P7%Iv|RKUIfyhF@`OJ9{;Ge->nS?(JyVt@U|uR~6?L z$(ctheIE(>VYm?GhYfsMfLv1x-Cf~ob;sFFx=FZZeG7KsDDGG8_k5-uj13d6)XhPE zeSark;dH&!OZY=Huo6TxOPO6Qp82S@4U6xts8(3m>4LO{JEs)RH9HorrU@5y=tcdT zmczl`T}_tlLU>*3-3=AT#t<{h)KB-SH@vbuRH{KH?8B3Tl|75n&Q8VdyOikj)w{jh zACH03t7~a~vFg>a?Vakej9(>MpNoSi5QJ${LbWTD%_ zFUg~7KGo*-eIKxKy{+-Kt9^G<-Z!)NCm`}JxiqDnmCx(e`tl7`2?>|D> zUSFN=CN`&gcHyoSU0hyiT5uI$;t!xXVGWG`@#_av9ba zL%GSWmd0XF=g~Tc&w8^iHUHIzt>Gt@+hEp}UZexF~~(sg}y+F5_>`pQ3@7Wq;- zMbY%>a^K%2L3^u0+6ndaE}wgA(!VFx>9E4*+Ru~4Y(Dwq*s@FOK^FVF+^(uavx=AN z88#A*zl=6Dn&$1|x z)A~#g+5EZNt<-x&V6lAcUn}^OhmW@omRINS)mEJ}S-e)$>CJtsLyxv0KI@S@BQ+of z!n{0{S$}vBJt|J}M17v6Ww`xVMs^9AzpEMcuf^SWn-8n4FudH2q@EssS)yBMoFvgv zz2A3>cG}7=kv`On4+rdR}l4mKNt1Od#?Ifd-;C9n==X) zYjyofwsUDNr*|?Jj}AtGpeATXJHmDs_30&lUmD8 zL3s=6gZF9oYj3i{?sgJ?7Q~J3K~>-5rrk|(yML!nQv2JFDnDBngl9sJ-aAA1%}#zd z2UcCoU3rWpw~u9_D&id_h2~x?hrBN;x%p9C*gIO;JE*K3?>7-=cZ`*$&n2h7g4?aTucN|J!hfA(tnpH@xp_uo!U0AzTlKeiWt5{O4r_v_YaUuG`= zg2yIInK5==8apikAT-KiEU1n*#^I6gP8=ZihIUD4zq_H^A*CN_D*y-`+9UbepH54o zwu83G_-%1Jn3rp=%lEM%lrI3ZmhX+E*cr~8y&C{+F)BaaPiLjElhQEBvHcScn!`3w zB$9uI&Ptod*mLQBb1>FbcI=?f6##Ae8h$(Z>7+DvP6B{+G`=5wvUGacd7jh`_a6;+ z5$R}h};}N-?Fj&)OP5cgvL%u0Ek&*=Oj8biyAsrK?gJeZ~+MbwlzYF6N-Z)(gG(s@Wyq8yfa+tng}); ze_3usFt8YZV=iQEkWY?$zbRb=-2FI$n5d{7r%hs=k^hh156!sHR_YZk7WN$U7wj3( zcZzbK*TXz6dJ)H{b4|o^gLd@lMOoc@q=d{bO!(7}xpAA2I*e$It zn|fM4bNhTgtg~0-DH0h*BJaK7b$!;mMK)a9)O)&r9mf)xF^<>eJ`}9~*Yz|b7}^BA z9T2gNL150N$8G%gXB5aqnh1F#a7=X;>$yoZ4I1OLZQW4N7=etT5eSm$F_2#v3E9$s zXhB@031l{6ER1Jxv+gyr%H44r{#<8kHx?GPbKA)p5;Da=JcAj>8*@#Kas9H35oV2t zWF@eFMPmb}hQ?k7a3jAlsv~f9O`IolMl{&@M<@nGxeP8qFyg{Oj3G!6Yryg&Zuum9 z*?6!qaKYb56sj1nII=*GAp@}A)B7%#Uke@F&hGOGRPoj7vSIyy*0iGInz=nq5YJ=1 z&9?fmt|A{^S6L!A-6Bu*fDQJx0Y3gMrX8JsdQ~>{uzR)oZ05Cjs<-ZZ*b_ZqsorK& zugGPq%uTn-hOuS~pa)&J#I4aW41@`cAP|&TA)~;=$`LPwMeNg#XIR9U-qMnB>K_QV z5fBX+xmZ9fl&Eu5#5(RR;|$cMJsgOL@wO1DBZ#emTUd4Gon|ruM!cxM>*w}mBi1;7 zx*!b#g7GkBA^R3~T@3pBP{i*odO49<2Tq6Y2QFTJ<^Jvkn5cA1t7kv2uX2F#jQrWFf39Z@Phak{ zt**JfTMcIe1j$?1yfH5p1;*E~s!N%3Mb)c%eh>QY>S+ZJ0GhMU;Sl14VO- zkX+?jYnbgza$0^eLEde1(4e3?;+r+%uBnLux6XA|1LEfecCl!Dq&$*{R~k@LiyL8} z#ktXP-_S+tv)7OaUJvIXkfaBEUbqK@LFaH_V83Tc9?R?o%yIEqy@L_MadgYGS81@nlwT>gUB%x1Wg&8tp{osR}86E95kZTBH z8uWo17D2H!c#SyKwN8osP+;JhANLva!kXZszGHX|XCd%|CLxA0F+_fUv_hZ|#U`oX zZV8G&I1a@5+ztb4%Vf`z;2Ve1nSwC1ctYt-?%D?>`0zUXVxzmFGm%kgvLyaUd zv=q(4#bmQXE%``+w>nIoJUbLyqv z)%7Vatmo{dYi8K)sToy$J+Ll3j92hsJU)yUff&b=tuF8NHcS71g$bi=(mtP&K0^*s zXaqIJ-14l*strs_oxfe=wnXRV<}qt*%v}#=-TkP1W;{@88W0QF3W~1-eIsxK(m}=X zN+1hh-&+7^z#_L8$25O^?eF;ckFPD=!GGVZ;Y*Bu;K!e9xetH+{kSgkA74IV(FrQI z^Qw5}aU;`$jFD7-HZdr%CU%~kC?-WNK`k6{5bKCB9W%C}poSD@GGR=^1q1|+rjy7P zZK;kqyHY_B5$p&bHPjeaH!;<8P$Nxb7hP*iEpSWR)+S+$n3$+IkOQr?UU~^Oa1v{+ zyFndEeU=4!pb{wIcmnDUWD<;CdNS99ItMDic(siK5q)}pKm4TdA_f1mJ~W|07Nd^f zzx)Q#sfRt4AV3Z$IRzIJ$h{3C7fnbLxoQl=6pH&_av z(DI&`EJwgpAIH-R63WC5sAmA7VZb?v`xXiu?u;oQL2?jmn^r`QM1wYrf1MU|#_2QQ zHHJbt(NJ74BZwl^ibRb}P&Q`AF$ODt+!S#LJiez~c9TpGaLf^ixN>wNA-?d*aMo&$ z5hpb0gg?~VYzAq8*SH4Q1EzqT_-M35gK9kg;b?n6O+9i4TpHp(la3sTjIpd0Wzo;M zN_^i_=3ceSAbByTlmn6|6bI%+;sha}h`^CT5!r_SHXyhd@LiN3C=Qvvi7|VBMIzR9 zs4rCL0n$y$6lAnZY0_X@*lYkx^mLE#!yh6~bNgYBMxXk_8_eJR`xE2*!GWI`smkYn z_}}l@zgdGX9BjDr^?3hzfM6I&YJ{NCfHU3h11%0|agYfkNv7a-;IY{rI#lSuI|eV) z6oR2Pi|o(}AaK!njBj2s#=${Sxb2k5tIZ~bFrjGOL-#Ft3gZm#IJjT~Q z>ruGBvB)nL{#lRyFpKKuXqtiLYuP~;YJq_}R;j%0nmsh#JD>oNQBt3OFcx_2G_=v< z7$ZX5xG@lxeNw3(15``DE)JPto z8++OY7$Ly((GLgyL83TzG9oi%0{PS$b_jQ5ZpgkMLC&GUEcdxLJn1 zB5dy35FM!#z70EsbW7@&)ys_r3L5-^Rr9Yy#8D&AYeK!6a#kZCu2akEuA@&V6W{Nt zLCX9doypx1gl^;P)Ez zt1%6H^sm4Ejk*4R`0Gv|xM!cCqgdvo^X}zrsH}@$Wc?VH}Vivp={ZEn@LFZqyDO zF1}e2@w8UnXu@z^AgR#<(!a~#e|aT^6flNXpqzt0#BbP%BTaBL;1KDvf}%VfY?_8} z8+aZHMjJ1GtcuMm?xGDRYXrhYo0TTLsHTC%3DVXDC_Zcf10S3t#fZy}fRRI+2G0kb zb$GG>HozG={ev^;lc8t~q2Pb~_Fy;p)zh+HP4QoxtmCkLr;Rj&mx~3184w28Nsu25t*7bJis;<^)eio{{{Pd5{n3&B%yHE6 z=pWuSoDcrP(f;Ul?ikMo$tEENoyiIg?%>A9G%P1H3i&I!xO-i>*xQb~$68K}j-0iK z7<%h}(%l7_MC|SmysosZ_f|wMLFo||G&=1Wg&;l^$#iw7FBcu&mvnkLrnk1B;iYU4 zH?)dufRiBA_1f&v=_qv9?&Zk1W{TY_6&vh&5c9#Bz%v7#4g9y98f~tPc+tBv(z-nv z0o67!BG&;q;9}M*G9fF>aXd*i=G_6Ih;VF5OA>Hay+*& z8Sr+%we+uU{S()hUNr7qFX0@2 zuT$YVVFVpx*ZqYzmR0C>nW_091j`85FN+mkIO?0O#x8Zu!sdjE+oTn=hh(?MTCpXp zKGRC|B3VUyaq+z(cAY6?25$xp>FwC-XjmH*96deAuT~wOB}UMp;Ht(Ij01R344&4T z6Gb?)K}Q9~rxtP(s}&p%S`!dBk9Mwq4tD+?{51XX)4!T%o(}UI{Nev{k8JK0v7rfS z{0CR-GqjuD&k1>LAY>RZrY42DiyIcBE32IT;&{BlAmW`zvV{fh<&^GR6KvmYWdjlw z17n&avh+H0&lXh$3384&iS)8@je=lp;fejBR*muDBxoRBcsc4+F4zO!1ahK(v7h`A z>D`g<|C?8i`_0LfcCI%5u4VnN@BiJ)eHq@<6zl7E*e&-d6y0;!N3N&nV);H-PDm6< z!rEn6sv9Wy9TI_3P7ui!FP=`VJ>v*M`Sld~1p)HWq8b_ z>cdjo1w36aVbo>Fuih}^n7Gcyvr<_?!Fs@YcgzhDbBsTx8Lok#4^Rq!B5WQ&GJWC4 zAYNE{b50{B5BKo|f`QXoM+_lTe*q_+E4u7D=2r-u&(aj$&0w$nV6Wy+{C)Y+B?R+(63$;dlY$$kVGcxj%F#YccLO0CSn%{ zt$M|srF$s0cXxLKR}`3kI-4bfZd$?x#9;xf$w=C!RyR!=)3OjFToYc23pIzrm>vqg z+1}i7KFNSr0tUP)hxrgz|IDYLQ?H~yaoaGjF~8&Xqn;3w`R_bq7-Duob%)23*$9^t~1&{Ai%4EXKn|-VUBuJ^P@Kf zKlAy;;$MIH<44CF*vK8(%;Q6m9Ye?Av4u79^*CNyjMFhL76egZef0p&u;)nQt|1*2 zvbj(P3ceYP*yLgUj0j4B-HO^c%!05YU3i(ceN@jI9nc*dZ$E3$^P!@% zAZRw^RfaX1x&wA#>5<`RW_%&QbTjIEL$wGc!@7++T}!NgpLHAWVK`s^?8D%{gdXyG z7hk!e^)F@Ek089}_$AEsjT02G`d{M>*B>_7FNW*8Y}glBAFn^`3IEM6{;|6t+ma$r zJWl?xyFQ+gg3diYaMuhC7&Z7osAA-I2{t(SI=}v}5X57`%^`5lrA5(}5ot5{AWj46 zt0)eB5nwHUf~)b6UYGUYul})7J(jK)htv$oeXPNG8V&jbF!F#<(GYVH!u=$;sECL) zh8;40@M<_*&d67+aejjQo!@`%>sCj3sJHoiZ z?Hmgi5$yU*??TwQ8gzPxgjK+KR{hug-=td1G!~o=wzHJkxc1`hmpPc9zL zMvdo(d;P?mP5tm|DD<@F$E^nGPM1%n#^hm7ASaT?k8T)rpvU12tV3N>sj1$d&gT#fi8Xl)chrA_}(@n+B8Xw}~_`f-S z_gBmL!!cR~>up{QxkRgHz1!wn{U6=>pE%%;raf>=Zekm+LsrW2uG;(Iz;y#r#WO~< zTjC?ZY@IciQ!$_;Y44dI&0a(P?N`Si_j5VoH-qn}}>iE&f z^~V|z_rJLG@bw$l_4&w=|LA>s^w&1qlg8w*Od}eS-u=Uu71?4_;Q}Wt4COwZaW9_I z3^9d({Lb_ltFX|K^i)xQ-S4mBA5IK^?2SF(434V)1N?cw2eZaSPq{RijAJP{f7Z&GN2 zRv@UUkM&Ji=tZ%44+wIH>IufFL=eEh&oVTDD@+I@nvJ80d|nN?91OzR!cY?f1o&^h z|IOhx@-y~V_ZxY`f5Z{fc7L6%xb z*{8-aJ&4zkPe$EI*5IpDP(Y9c+;mFP_^^F0_-4Y;0u3G2gRAZhbq%^uyq|R$>ipdE zuU#9jHXnHj5$>RUrF#2Y<23AR@mIIe2ERDOJ9F(d_jqQyLU_QkGI~RQ>U4`^o+~qt z=hWnSKG>s$`^nU>Qj@<3sU`z1U0~Nq(j!9-?%HA9nxP6%W5c@1V_xZ(2kMY_KeZ_C z;CKIMk-tAP4Te1Yn8Wy4vtNBdL&NWZ-mKo;5>74%2y^z?Vz48^T|!;DN|v+tt^fnW z=u0u`+zGFO!sP_*INg1J2tNSNn}g8(T*UM(En=UmcFGLn$j!<9ySyMUH3!1yV zXLs+L@8B6fwnd)7-HzDB2oXOmlEEIVfp59@(Ql1>i~ZQc4~B>Oh*tyu$zyEe6}Ajqnka>eq2S13F9hes6N!|ToM6Zv zR?o+3(4i_6fB6u9&yIPM9TsCe2wpg1(U24V!#_!)`eo147t!O>2`LX-PzR>F6h1PR z%G*Rkjm^OJp-Meqa;L#>JquT{&W3dzo_gx)f9p$zy1CFD>nesCma#r4iu>pQN^glIbqmD7w zVgBUF27R%ze(@Gs_9yo_{Jk3e7$-9nWb06n#z-1}>WX;gx0z{{Ovlly3ezS=EWV-E z+cFj-V9oHDYH7L z`T?Q)z-5z2%a0MrjmhI?%r~+Z`^isyr8T36;XG0S?Z7*nGW&-|_QSRF_@CVGf7b8~ zIpDv4@lhH7YOT3}+L3e3E^+g6B7rc!yKx(vRwxjjS|pTnG1MD5-55h5uczM)u1&2~ zbiLBu><8De^oa+pO1A{Hld9feYHfq^WkR4BKm1a)U5(#SUHUj|SA(5|LtgCcq8~`#WE5Cg$ZD!ccR>JfFfb#17 z)gX$OuY-Gnz`GIjiqX~o*gP}Vp*javH`ut`fn^(A5Hqf`!;jsar8P4%$OTaT6TjGH z9NP)i z9PGqJWR|Z`0Y@DPOn_Kss9T1JU9F0zAUJV*hN4S@f$8txx+R49F+p0JB0&gZ6orBz z5CXaUwcZF4(t>Ef%}G6ABDkP!)S*rewE_j8t8g6Zsb}wEtFy^(J@tR*)TdUN--h!v z?8#3pNdAk1`PC}_o4=27rZLoi75?S+zk2q+>X2@OZF&*A{K;R9dpy`KcgLLXPfjOK z$KUSqd{ijmb&`DtH|)_6p98Mw@02$?Voxx_D??Se|0t= z#E9!7rvKHOxWQv+Y^MW%&n(2(;k7-)iw>4G%;ZeFSoVR(Xe0C^)*8g325XUWtTOr^ zTAS91MK1pz_TH>#d1h%B`!6|*P=o<9eOr-I-JaSLv`5TIgblPOFkrw#l>Z*J;eMXy zUGG}!`=T9fDN*i|+osFy9;&OWtF9``tH@z{bYAYbPC`HqKI?yf@DnfL&ba?J82rRc zFc1v;4nOZttk%y^W(A~)1S|cKb%!-L6dgeJA!I>V3d6hTg<`l+oHByPg`fZE9 zIoN<#P4$j!;VCwLzx`pq`{GG_m+vL6@EmMK zgI&l%`!x5%@DL_RVVVZ=fOo!X?8gtoeHqCHS{i(?4_}$dG3D(qoXB7KH1~hS z4saXRKmD!KbqH3q!Q0K}prL!O^XEz{5RP9l^CBUtsDQUZJIWTn`My13=|(ZD-UNh` zhF35C{F{=0BZ5lhvH}`wN9c+Yj%ngJF1Oy|P>xi(8%-J?Lk38p)dH9>(xNH%xiD~i zW5bBOO{X^d`di!~=14~Eun%&)BYgIgln7RDX%e&i!5*If#w-8JG5F{GU2W;ldi^i- z$!!D`70URTDF3w03@PKWk8z{ffMg&af1r;kaQ%6INB-)o|LTk!%k#@O&uymzwrVZR z3*zb`<+0~28@q=mwec!Z0}|mpdb<+5@if?O^R*N*h7rtDce<#-CGCb*EXqh~^-n~H zF@$W~i&*9Sh2a~2R%A-!4|+XH{oh!KKXEtQmzC0-^gbVRj)nMziT&D9CN_=`-)~W1 z#}ci7N)!8UA6um-5oF=j3pY<<1h==8%5OXMBWQhrUaAYk^#v%AOh$aCCyk+D7Ny5kXV-L(M>5CfxOF*>0Qh~RZH$_yX05yBs>n_w@?0a zF4$@8_xXp%0etL|$&X#~$42zk!JSIF>^ED z#6`@{dIRn^_5HQmJQ0YZmC|6t*BHoGoiU9E>S6FzBdctwuqD23s9-4uvt^Ar7=gb- ze_)47D;RTXVaWV!(6^icUxnn1IM5J%f?#6}_8N2;k0Y#7bp?39U+ci_7j5!A8;IDJ zc>Bng$GI($_veexb$NfdK62AUq4Dm6en%L^>zhG;;kM1d?xUX=+bv|9 z)(HevZ-`f={?nu?Fpmm|soXc8jRb3tC599?LU5cV@QK^u@x*w<)D=iI@BibAU${W- zU;nUK`sfEk!)?AFyqRx#$q4$N*vAzn{SX_;W4dO);%?>?=}d`%5z6C}fE9vPe}Ee} zkp!%Ms6wJDxT&pv-C4aw5Tk=%7&%QFu>Fbq2OrdufsuQy5JrILIJ4i|tC*wFWUIAQ zLh1Tsi-M^)N^v?02zkoL>={o*h*e){D%K49;n3%_KyUcq|7AMZF=r+tSMKK-^qwZk zP)fNUnfpU?8=~Jf%>JColK6(uf3HoKRPM#n;b>UUTbU#yK(!2bBA`@LLX zSEw_?ul1gq41ux>L5_^{7)eYVsk_fU4d+!4X~Ww!tRwXYZ~L*Wetj3dk6Qot902^R zuj^n(;J$We@Ed;cFm7_uFT&ew2K-2d8X_Y|3Zt}=@Vzu8SqB77ATysLe-;@WU_>0) zsXRWi6VZziZuiq~3JYad-TB_G?`$Hw!6AD%LL>g&c=#x)wBn-zL3YZa+|c+wIH_KdHO zP#G@)sLs!zSapt%3CHdXfBY+)qWL zXjr~X%Ats`21W%?aLt`OooK^t_8-5B-*Y?=X=yfKC3?R{E&bqwe@0divC4qZl;lRq zxE%o)$FS;T#|Y+oEaG?#N+h7d9%lj?_IF3dQloY7@HuBC!@0zf)$jiPWApfnKU(jI zI$(bCsOLT~*g?5T*=5_-ismL;xd27Wq81^74Io?Hl}{?J5X!7XN&UU139*`r7x5CH% zCr&k*waV=YY@(So3Je$S3at%bmZv+#4O4Z=&)s7tE0!D?e?S|Gm{rP-EJ0fAM7N(kOR&cZ27q0W*A>@sC&@OtPhr}OD@==jjk>flG{S#Fm2`M;zp*mB1uJny}HCeN4ETle=8Gs1`U6O?_Yk$jDG#5QEz)z z!2Lcv=LK%6ANvBPIq87u1q`u3D(3tWHO6Gfc{1eD@Eyd4a}WAC_x#09p7DLk|Hgks z6C1Qt_o_(GS21#gtJLa%9PwgN>O#h_)cL8FvOCZy*H9FwPPIAGV3CAm03Dz3Y z@OT1ke~6Ea27Lou?#J<`L)`p;>-rTN_rX~G_%FH$ZQ7OY!9=#wA||bGVY)K(jQf>} zyV}+f1{!D>aOnM~vJ->tV~JyG9}$OEe(gAo_Qfsywy*rsCH-H#g>msRQ+Be8MuDSs zGcz&L^1a1zWCx)}x;ytUwW-_4rCIXW>L0w8f1z@Gq&&o9uKoo+lIwl;4)`bH_RWCR zyxi^p7lwfu_25&_sIP#wuDEX&2pD{h+rfXTaz7b1eVg;hU~hM^B%F87tJ%)Uh}s(AYLg5NU?| ze>b!7sFHLrO}-9H1`8fPW9Lw+JnKwP9?1P$T+{9koRaq|hVv)ZKK)B9m+}ueUJ%nR z)aZh-C)2S`4`hphs-m;^L@_ryno%!`Ww~~hlFe*yZy%_L$7NwWO-n)aF4T=5R=e`J zlSe=t5y`V+>`TQW+OKMz8M4fp&~}<#f4p0XrsvPqE<5d8vM+3*0Vj%iY{g(-`rzHw zU-*E=0zRJ>kBQ}YhLi*YX2Sg=X6S40fF{v2uu$CCSMY_Y-T1D|*l^xMAdWXW98B=> z1AG~6CjpNKm6!MHDT5d;spf80t> zQ{oIfoDOjl7zXUHfoh(6Te-Aijpbf%2elTPl5jT4IIy(HuETeHix!npuSxe}> zw8wSx<3E>7F1PXEPfjEze{uiGFCE1iVzuS+CvOD#=B@d(hsPx<2gO1>&YK>C=LDC(fS|X*k>J){>R=noZtSmz2QYej%r(l)b&kn7dpVx zl)8E&(XUKBf5FJvr_xTcLfx825{!HW4%? z2=p$dav;3aG}Y|Zw>e*)f4Q#mNF<2RVa)zLq`X*v>Yf;(cx(A|# z?DRH9Y#r17sUeU}9$Td_9=oI)`$kkG)Fn|W7_MrWK$~cTA{|^Sf1Bm24w$8Y@1wIs z+(Swy8pmohq_;SBm5Pr|U`0xPyAVe(ff2!KWN8l;0y=RwtsLtvvXd52-pEK+N(nh~ zu42)}hLN6g2XG^!@uNn{R4%z5aOWat!sR>~a0DLqzxeiOVt=vhAA1fzuSFai@hf-D zud(^ylRsnE5B8sLe{91yhg0iwJi-r;4gbIAC=_rX2=%oqt%=lHM52~pLlVodHlg7` zmEtsV`BVWpN+fjWDQXfT(pgkM#mZ!R_W(KN{8c@7# zDh#X^ba`Uxm*&i6NmBJ|XcW6SF-zO&$qus(N+T699`A0Qf3!(jA}6c1Mgb(IIrxrxxK zuuVi;Z<7_Re>I&{8zv?yBiJA8-JD0sJ@BK_NY|r#QzO6d6RiwdSs+iN6J;)kcXY^8?G`1u|+(E(?x zTL3b_Sl(6$An^DkO#q2W zl;RTJ5jc$;^7WolR6*neFiilxkuXCwI1y|cf5isLLJC6%I8#8pTw!R<4>e~B78~kA zYk<+Odu4dvmfw3+6F?R|&kyxk27i7E_zAZfPGH1<8hmNc0j?`JgXx08aKQE7ynuii zF=zsA0fqeepwS}0+@FpjNJR)5`vFSB%OiLJ9GO1m&x?SjP(=^|sX>bHo%i3Agb_W$%J1xdh&B9q&*Tplwfb2~QDz$38Z{e^~? z>#$jY%VcxB0JNaU`rKOwkc&_GnSw)!0+`#$5P={9mnmbw7CKOr0^{Zh%?7o>_XgFF z2mn;Gxm~AjAmPuqjkMtNy9O!eIf-QMe?L)WST@K2`W{!H34A`+AniH*i?`2<`2B_7 zPx%?mma)j~9z%Vaz|Z>{Lp#nNL+%7^7>|UHkHJhCln4t9X3$jdpr#8N=MVgVCjpbc zPa?cg9iN{!`|#5mWEY74!{2y0Ua6o|(t`7^KxO-f@BiWN#O)9J1QjsIA)GN|e@I80 zgJHmfD|s3NiR(rG@E|u|ZB`UE{qy&ai0+G^;9{C!I6^0lH-HZCTEU1%tdR9-z|zJ8 z9`Cb~=uc%Zz+aW&|6p14dEw9)X)vzI9Ot`fV9oQ1?i#cJO@s`*?6Rl#qIb!UiahIcqEJ$v+?b{ zdX2N|`ZAs##?f+}pZjQR?^eRNcgD7~>fgORwt5zg5EB*gJ0Abf{M7(8e=z}qAXP#a zn7d>km5N!z-V4jDvC*qBl#=Z;Yy?5pKo+F0PXK)$b#|t9a#{o0*MKAxAk`&|vEjEoTSRDO{Yf5}77_ClisRmsNG{$OZa(+#x zN>PJDPbT*<>rx~e4GH~(f5d>Pu<4}KM>K61rzL{X$&KvsHjpKNucT!dlMpC>jy!9i zbq&^|Mu8M!DbRYbo7_E^dKNBqy|$D0CY`K@o}NZ+$+if2GfJ*6^GJzQTra z&Mq>m;YLFhcztZB4{l8DaATLe4wy#h$-KYk8c_7824oJ;VzrTED|VM~ z8lFdUGm04nb<;?)oxi|`s4Vw;RoT}dKf<8MYd|zqgMH|tqDFg(2>RZPK`}8>hzYVD zDg`k?qZ1U>kmU3Me}o7mCN8f-VP*`f`5jZ}YPL6JEh-4BfpJ>Z30PcH*``8}5C4~d zBRR-x@bARM4VEpIW!T{DsXb8i7-YRvqDX-_3iN%5?L7$!R?El>O+ZFlr?rgszvj-J zHmK_w>|2kTv#4w{noS#wy9T>9CLL}7+c=5=N0&nb;iiEQe>bSBo;4IT`Sf!`ryCIg zM`iGOvxcf@zkdLVrbzzw(=bhs{f{NV*auf|ePRAY#F-$IwqD~k5PrGd`RBj5RHgZ=O8bQ_)51D|?+uLY2F z*G?VTZ6Jhae+}ZW{)u4p5^Vs4fQDF(NvG_MOn%m|s%uKH6Qd&X>9a;q^Y;^s8%eN@ zfoJ%2XX7viN`?Ethr|T>bB~51B$1;2Pv`;D;`tp(!KID}4NDbvFp9@+bysWO81ElKyee`4{LEH=jC4 z`T3pm|5|dEy#GfohyKnx9d4w8uR97+hKD(=WyHde^P$anlz`mXd~T&0U{rHEga6@Q z#ciObe}J99iE2c0njxhIk*_J63~Pw6$V?&?WS-u|16jwk8phQpg3&;zwhFLEP3ni* z4RaFm@lxFSS)-(`sT>&Hv{8BdR_Z^@81|+qOP}(pt||SLcU_B|p*%57S^bm?bxo6@ zT+6x^qoMq4nzBBWzFf373fT3$~c~0F}rYVz8IUnC*C^xf)ZMkOxPaCsqSY$#J zVZ`?)6v>-m@8xT+#^Rmtr55fQvwJlrmtT_1<(D?B%YVaXWGaiK<9j({P6w=pxfCt= zn&fgC(a=N?DdjL%5w_C}s)zZpvOjo`e~PN(8x1&z#&Yj1qH^E_tO=bu8)&-*YFIt~ ztMoQ*m>qxmZ4bQurT2q>(T`07?(5(5^HUD!AbfRBd7D{c?NOw-RcgjCH8HG+oc#O# zn*ekwbNMnevAEmfX|ur%+wyV#xBCQcK(_f@Nm0+9wN!8!3QFcPGecSpz}GI;fBlP* z6-n-Nc1n~q<1(!VIj0XI0+j;=^Lef68X&CEu;0qVp6Gv`sc8fB0VeI9WtNFK!LZ75 z{=WL}&$#e%I$I-Cu3JPoeaIHbMuP?GK}^JT48E^Ap=rqke5NifS!ezs^z~ui#0B4* zZH)*?&cB=ynV=5)_dnw?zBhM2e|n4~MJml}2&5=e?yhi3>cB4 z+*$Ir+35yPn{C!{UTu`l#x+ze%c-(`Gk{gUrlKeQ+d8{`K1 zXJ0CY4_HJ~EC1oGf4$DF7x%tzNhR`q-;81W*LL3|g6H44M_gB7`^P^UL+dvW%20C| z?+ko?XcWPYeDrj~fU&Jof4piBkizAkEdQM@ebT?vk$+cTPVihE#^wP#I!Rr1g7zVX zRp>5S3uK?mSK4*WCaeFEmZ}r|Z?q2XnoajIcz=fVQt`3*&-(LfP+`4liOYDTe8=#! z{(k0tsx}g~@eiNW@RW?sVp|L^(r3Tl&-YSBOZxbuj{mMT%;nC%e_SiW`XJUy*X8TQXU=laTe+T%6o-1|1Exkc zZQSfXDes0jbzfe>e~jOIe6HQkAaGlik0MoSiyCc)IhBF55 zf9Q5Sw*k-41s>@B4>N;B)}~KzA}tf9}`&(sf-2fE0cmVScTA zuHCK!`u#oQ*Wcei^n_pUKlGpL5B=x*L;tz%csl?bzK&JK`^7`}c{^@HI{?X??vU|* z$zQtq(|>=Ut3Ul$2D@gWlrCH z)+m4G)lfc9f11Vh;a~Jet`mE|=qF!#^qW57^{By^6@c5SI&SlzGRn(K z13!D-47UJBTz8D$-fIdPefTkkl?<%8|e`Cr8M5)8~&rHeY95=9+-PQn=_(3LOz$7%kv`P`zZh|sJiC&BwH)WZUG!MSd zf$Z$L4JVBKBSt(wuy}ugHcn%Z^H|0pyFCi23bG^r7tMJQw|}bq|AWf`GWha(o_Cz* z4+eDPc1({VoBQ>E`^Vx2V7n4O<;IXl=E*c`e>~oeH=pOfe+49Lu)E5b50mj@j~(t2 zAQ7tH<(LmZ^SfN)=~w#S_cGuHslV?PkO^{r->d&nZo@@H>F@eCAW>-zNCI^Q37|P> z1P&BkIk@%6XACol8o2S6SPUBe@28gv6SftbewDi%b;y$>-% ze>wke^7`cBbB{&9?ElVtW$v{8ie9PA7mILELb<1ojjdf4|Awg8??YpK{Hg z^LhV5{yibyZ{8lw;-7FRzx}!Z7Qm%Qe;D4+)4r(|xD85qsOO%dh{rMe{9YCLIC{f) zGENtPN&nWq@LS&V`?&e^-=gs16e`Y)s zE9I^clc6_n*S>)f;NpJ(|NOId)qEU>d5Zo{{~FrE&=n++$2Hdg6&(3G&?p>uyA<*r zpJD!{{QrI!x9BsE8s+WU417XSIgHbPl}`p-bH(|+;^Un(f2-Hy?b~y{))Lq{na?1e%lFu%rjo`eydzA9|pQ1NM*r)HyciGedb34UVqQmf7?e+Hh=Q- z8{`!@41I72?3X`W;{5ur|Iq5|Umx-!puwdV@m>(V&tS%vWW(d2HvlWX1v>-2f#*3# z{vZ;EeQp@y`rAa2Kb&!#qz+I5j(8sUimx|=$NO@Z9iJb(`mQ4aRs#b6&WO$Zs1oCH zVZr0Rw!_EOM6O~YkO_aef5OM#X?PEXl?|XZl0deBib!f? z%D-MzGu*@Aa>N3dzIF`fH%mgug5xB<6pfWzGc*fc2cEaNFTUlve?;lGys$Cyk7*oE z(l1@V@u(80X?av-)l-#cba7HB(;JlNh64MxpH5OV37TYD*6~!7d^Nqn1oelXIt6+D znOh#eH68=3{@34MDSbX(r)PIN{ZIe%|M|c1FO>c-=Rbb3I9z%>(=tz}2>)KELKgtWC<)mmk&sOAOJ`jx@?eevMEiwBWdVGaqJkmOLK2ZX zby1D>(aAb`FPb55A!p>i(e2B4yneo1xf7%{ogHq`F{5s5e@4MYb$nZ#UZ0EeEg}=7 z?!vdPHLm_pqN=jh_Pn#!6Qe&F=EjrUb9{YeH&+mk`|#PTuIc7$cZE;uM_91wBWL^>W$l9B8Utw~vl@p%rp6?~TZK z%cq%$pZvp9=l60pg5;Q9M(1}a+gmHOyIrho+RVDF&9yE0Ydx60SICj_vAH;pXAjkD zo}IX{%_!RSKs0=sj<%zsHJ`I`f1mE`O|UczYdhMZeli_j%OaEl8C8l?PXE*9kWo(oo!wiiWC>)#-%uE=tD9`=m{qc1fte z!S8lsifpWf=ff%pwQdWIwD1cH#Ld&&z+7yNe-_`+_|!}o+CC=z)(VWkC?>Ou6E8K5 ztbBHVua^FC7S12<(AYhd%T$>;bL09 zwl8x#dw{dj^6}+8-M+m3WWP4*-Crys`w&mwW9dA4p4JQEzi`$s7Q~*-t~W0+T#N5A zfA>`!PhV99c;v0CI;yOQlFlTtvQQZvmF9MP6{SdAy=UyT?}b2Y1AQ;MnxS|i&jX5A zV_)gVxNM{SIcd$CeAZTMbq()X7oU_Rx;?P7z~QrUh<(}4QVku-VqsM*SS%PcOR+o7 zuD0EDMAar~A`0(Euy9w$+9>5%)!DSjf7Uy4x>nIPd>=3KeWs0Ob8B7C>f>9^1@%yR zwF_xHJ(AY*rlB0mRTCBXqAyi9jCTmMrEuG}kEOA1kJW1v<;Q5B%qQdg#bzhEnZwt>vMt#?1@W`p;8s?`g+=B<=5~937uIkqf4!to z8&r{z=tWyqc$@uOZ#P$^Y}|I8bzeDhgMkM*1>-Pac;te-w>}@pu?7 zqcuZJ-zO~3mFFwi(x`|>R=D4i-F31Wp~yG}_qDUAA9s<+{w&7nVm(VA3c_`?^1|#`V^m(RrC$=biRyc7ATMtU95QvDi1odUm-4hd8<2!f|RX4w0i>JWZ-V1>3u8T* zFXeJ?KjcbOv~D|z-*?rpVAD3p(?E2bFFn&$*#*J@Pzk+}eKci(aXs$Mvp;eU4#m3s*3rX0O~_=k4C( z)7gmn%N{R8uRSg!=^Sldh7j(u!*WHWNq~>`YEsvdWj4yD>?bpMdL`*=Ox#w%xffK2 z&34>`PdqBgv@168f8;H1LWZh-;!pAAmT%4bW)Ws1^SL*+ma|<~Z4mnVT+Uh5(^5CP zcxl$N*)oJOos3=a>cRMNr^0!9JtkC;`|CNA&ib2>_5P-}=H=Gc^H@g5V1eXG`ldKA zZe5D07P;|wqPP2*SSbBX;kJ7iHgT@2 zwdcTIdKV*TZRL(#x_?C(1oAt%<#< z&*$xVmaO4@HrD&s3T7v?K4tYi(nRf1EYg{DXm`Dvi!pMam&27r$e&k-w47KI?ZoPt zdRt=46Z}$L7@pH*YJRQzc08NoUN$3}YNI83Odfg~e;Z5)%h1vEmF7e;*{-h9`=}X8 z?%xjiB00ReC>MMBs;%8EoSXgL?#g_k!6_{gRC>xty3=evL-Y1Xqu>^o4si}k*3;;H zT-Ot)B=|@TiR4NN|Z13uuCq4Ab(QX#?Hy{zB?Tx z3eQtfe}a5hOSPwm#5RlK0W+tX7;}Tj_4;K~Ep#4mbCpD`)zS*OL!%yTKg^}mtE%^V zX?D#E5?!>Wfrq>@i2=$ye)tNVOq(}WNq=!NRB!#J-dfG{Dk>x|fNhh6o^e94ckL3_ zVAV@Y4=*j=Z1^dfzdfx1GHWz5>H%hW5#h^t zy9E(RIemK|T{g-~5UyBw-lCJLpPx}zp0f%sZst@`*n}!4-#^BkD6ic0bDOXt^Gc(C ze*{M`&$tm+fpDMSg({nfW86QY=WS&PvHl1RV~^TwJulXg)OUh-evW6cXe(=oKW zpLXiremg?!RG0fLRrd1rP)^L^xD%ecf5{_!nu~McmW#IUtz-;_w0kNMfkRUkHZiEq zSg+oN!#t0b)kyK|;)sIBMI-lT3=yN6(iUD>=(;?80q)xp=h@bqyVj+x6@a?oY6&-7XhTia)FAOWUM$ zRQA{L-M<`8TK?#SLZLDE!#eJ)9{%eZn3di9&3{>5uCI5 z=ESh`N?f2l0^Q*k$mW-qkA-FnMRL-BlyZ`EtHSJOhz;s_~h zdsvdn+qiubj&1$Xh2v=Y60AdRi3dYUl=oA67{WLbaN*D1>Q25H#(tM6Q7@I<-t;cF zqo%i#7q}YTykG7mQ0!jXJgU>X7sHM4nC$&&-m0e?GW2~?)R~kY<_Dsbf5jr!A-*l; z)v8zbImWwqyxy2o<#0W;Yp?~g4!YQmFZIY4kmHjLrOY6_t7> zUTQxv^Ym;hd9f&$&$ag&S;F#|G_5=7)%_{iMapqKxvrM;6{mm+=_H)tM3a~Z`f1Zt=yO~Lk_jf#<%$a*0$zFfO@Z6~COBTIRJDap-qBob*Bws9c zaVE4EmI}sbxkPRIxQ9yRu3Fa)uCDE5xAnVtghV|#yGugmuWH?`#JN6MKx?dz6U~3S ziEmz{1D&?#AZv}B>X*@>%gv^j{M1)%eAX`5Z!~dpp!!X8kWx!l4yGk zNs@Y^A>Vbwmvk&8Q$&kL?{B4RolRi7#%(xfXPDkO`~>tnHAp+dy6&)>ajd4u4Lf&t z(c)k=zdzh@f0EO)>i+wwNv`6qJuFH?s`VV$=bM? zqvzOCFOS_$Ip0QyQV${zX|+`AheAsZ;-szI``ueze^cQQEq8LE9?Gbi(;3U`i++=9 z^OzTEw_1zOx|x_25pBi4Xa2~an|C9%C(=vR_WJGO?_S59Wm#@@9jzuoqeteU;mlK-WcJ*Ub&7@GA6JI-g=~95^{aMkYGm@|mUkW%#Cr;n=3*vxPVvheW&j zkGF8@f6=Y?%ji9RYU%BDxi-OFSG3hKsP58)qU-tOxa#Bee%kDUkv2QBMp}2#M4~U@ z9@xhhIn2)!)4r9~CCp%R-sXxHj8AX7o1P?M%GRl)`9pmQsXx9guggulKFrAeQlw%N z&}lI8@76AOBVFjdOOqGD+DDG3x|{j?Egtv6e<6KM9^w1Z^!|K5+r-DwVy6qq)!uG# z{G^$^OR;_RZzy4f+jqhAy48H6kl)v}_$E(y=NoBbPB7jHT@1JRyuA386ugex-Mg|+ z1ZBQrMD@d~Ffl2VOJN$Y+1y&0#iU5rw^=^jZ`OJ-QFig9*u@X?d{}gNBX9Gv*}G=x ze_ZE%T~*o`gnSzy?QySg?;oq{vfmrGbQWv+ZO;pDimko+OB&C$Nj*)ybzF$}n2`l5e*+_(U35#L1~}^7mx{hVmsPS(R=1Zps*=Ut z3$7=l+hD)E?6jgU-A8xnSZU8L7CY5yKCN(28LXL7IXG6q(S3SYf5yeJ zJnS7?RiA!RcxO7%ow~w#xX_B}?S9>8?YWveBDqco^LSZS|T4 zM5z7Ca$_jlqFNrG4O)zx;xIX{T0q;Q>8Quiy(E58*08^~SC`}M8b!(0 zIo#f%-Q)Fp<6lRyqt6!eYQ58Se>V{BOD5i|sEzmH`eg1d^W$Sfy=LMRt&&fslX@K= z*G4}L4$rJhtkbdTXOG<+dFL?nX_{?Cb!rj>i^EJ8-@ECn9j%krQ%?8MStc#DiuKZX zB==p=I-+3vUAwQtV>irf7ITPPh%@;BR0)W`m}IP_UUOGZcyx2(&4gmMY`_i{=As3 zaJ9Jju5A@ zy>l5aS(LAvLl~QvPA#g{#FeYQd398cvd+{>^EggVSMPdvix@4>b)UUQu6x;zgROo$ z7vy|09wVk+Y>W;;y1Hzn$GdFhk&*=+EO!^S293IVr`X%2ypjUFY_W4WhHLlI9(~vd zWG+(2d|+L){dsl~e<#%fCAlVznk8P|Vz+YR($9nLt@g*qbQZpecz33=-RwTT*{{_^ zvZ9L2{IOXiW3%k3T5j?gmhH)1igUlXQNysx_wG(rXr{L!SzP6jcA9iZ8;y`j9*;Mk z5_|hH_ewb~VSm}3=k0xI6#|i$&w_d_>9UpJl;fqy*>G-@%S}2FX4x zp6zj3Rq{oc=8t!~JdPfbUZL<7EXpfVyz|MjZ)~+Yb-V3yv~-<^Zr;+@?mfNbx%q65 z(FT@#^KELvy$!GPb4%AFviH^Fbp^+h`_#=ypyuiGCfLdFj9M5Pu9=+fy>+IY#lA( z7H4mL7v?8vY&L?JhC6dp&9-+-RD){0bLN36Zcn$B$kuxJ3@2-=h{B1sk1x4h%wKqM zMX!sheS{N z#dxu{=A&@iuB}aU-q%v=CQZyT*P+R^71Ll9I@=(svoJ5;-g^etX0i63*Yo`Kv>K^6 zymn}2mi=~{Jpvb(`JtJ3sxvPm(iq+rFshH;f2a`;M3Q&bBwOm+Teg0VCf0ju#pc`Y z{DqdB?@@2K#!XtiCfe1S98l>z$&)OFwenhp{b)8zm%;V6K&QXFcq$zC`Fq4?_F z&C4_AXd&|iB;^Mcw zmnQfqEMLdj>Cm5Tj5Q&hn1{YA@#GaoeT?#@KOS$dc)C7;c@MWB#xq@(FOK#YC1Lk} z5q91>*8|aYeHV8I4KSyYIcHv`jydPdrx(6okYvDxn`s6(B&M9%Yi$K_;(65wf1>(I zTU(qgJ6+3DnY}KBKuTqa?-?We>D3p{`r#er?5q2`4bGTahEt{Y~7MaLw<&Yo^Rsm z)Rm@}Qx;6GPAzK09I>5B_vE|>{Y1-M(mQ)Fl!Br9Knv+yuj;O+H2e0#eb+x5)e@%_{odRS! z#zi%Nxg(}u?2Kdrq?>;q$&Sh@)9{FY6c_xOOLuj268n z=K1sb$m{t9dd$XuOYWyBj88%^;Xmu}Gu?1zY+j)?1lFW@P2aa2EIAv)o@FPm4Nt5H zf_E7j;L0@G%_F zG{Ljehb`yh`$&g^`@419_y^fyR}cJnyQa-+#_bgsj2SA#j?CbMk#M?(Q%}^$c*@06DY*QR-R4;YCgFNgvxq-GgX6pRto)QR=38bdJi57q6nNf3E}AZRdLGeB zs@JYZMe^~-rHX7}Q!yfz@ANdyc|CU$OXO-*o#977h$7=LiQao47_P9y3$RPxI3gJFwtbo=4JYNaie+nX zS)e}t;lH%yW)LRl4kO3Uhh!5U4hq4-D#{^qHevpS6}n(^1$`NKGeBej;EJbr68T=LU?4f6qG1i5@iS^?LEq^3T~H55o2%q&dqt?t zPwqbe)@m>D=9Y!WZq`#O9z?~5_2P=YsT10%`FSsO`S^Y1Q8h+ReI1~;chYxnc^F@~ zuEce@#F42N6nsjgmE8a-Nj6!m@-ai6exI?OLTPU6e?-0@!~qkcc4_D%hd3mToNb5$ zAHAcCbLzKizA&4Qq`dcHP*nC9v@3hDc4|W&aQHgDM`4w7&UqjT5=~DbJY=TR^qJey zxnj~zhn^hf!QNr*l{CJN*?1Xb?Q-y?;cZhJu}3sOd@+NSmEGHsmfEHm!dYZTucmX- zA>_R`f8U4ak?*gL=GHX@BS&Wh#O=tvFY|oFO0hw?Z6Ff_V*x1smdRe%X35WUzzCh( z(tai14zRYgg^>o498VN9!%~IKObQ7%hG^8M7?Ue%p-x@q$#5q6o%+409FHlcNy)v;=Z!Iz#j8{Si1rWA#V~ zf7{x4r5Xh7x;!C8M4?Et7=|9-COgf9+UF_@WDDWQphT1GP`Cj+8~6y(SCzg)-Zj8=&0S2qbc*fJzl=NafFRUKHywWA7S?zXX^?}(L|$2z0W%DMZY&taX8$cQccY-so`qx=n(Xo z`jJ@X6B8hi3|R3svPiMkoXvaJyOtw|r|Ab*6l*M`*?Lf30*0 zS6$}2==2kztr*h zBWPBZ8}P?JLhy1J7O}*FEcSx1e>RK^xD@q@p;^}X*;)ne9#OLH{n3t@p2}%A;C<8Y zpHsDl4+p;f+gvf>Bu92@?X(S4^s!2=vbL-n0}xaK&9qFMpQU9v?oUz&y5^N&CGDn1 z&`7Bv=A zOuTN{+BP2jw)?$7u&BxH#7G1J%9-m;TbbL8p1~mS zuwZvX3a^dKu}PyZ$KLN3q!J?id00W6SUs_kSuksVd&nF zibp5YSGaUG95Q{O*4P4lb1L5oyykMflX%vkx*Zt+>HCz$Zlb5`_wwViLtMEnWx1}p z0B$WuXA-Ovr@2n-e{Xb0G{(RA&`1yv@O<3r5{Ep0)YhH8(eEl5~tf*z^kPm~UI^d@!(W%g1dgnrxIt%6+ z13q}+T?OB*o2)nKjo)+?#aUQ?%U#mDp7}2a{osI;aC=r%fBBX!lw2_aYx~0N{e~e^ zF%o4gA!5u_XWEa-XyRDZ1%%lEQ9!Q0Q~I`4;hP#%jGnhuo9vIoKpTFtrA3Q0YW!LC z)DgtVvAZtK{s1W_NG+qUZ8~(_gw~5Gp*NQa^{cnkdF-Vxu73mUB1(5_WKSgeV2u0% zlOpJN=`KKscoFDVK-W~&^ea0kHC}b&O2F1xKE#^YBaCu=p<4V!10!3D%Tq9TU)Dy; zk+BzXcR^)LR^1HvGdAUTob*@!1O^%z%Q6o*5nMmWc&+#Q8}sLTDt0(QcU@@fXh)yu zyv@Qzi&}#-4u8^>0}eA=LaXS5(DG&>@z#qZyg&WY!{9s6F6OdR+A~>dffNkgiAbDt z=~BUa;n;(Qn@vB$gf)Z!V)dX!+r6`bY@@RhoXXLbqqguaGsDFP`Qw<`4AHk%2uFrXRq}llf zsD*=&jwX?)03-7GE3 z5M_8!)Abi^@=lsUs_GouZi?E{1QB&KfV~^FwM?n_OvlxSA=#OZu6Bh7b zT4kcof`7MZdtiQ(e6PMtn$x_mC(SP0{Id$g`2#gJi=@5NtE?{ghDKDM$s<;Zu(Z+2 zZi{F9u>`fOeDgO#H=K8)O4;;)P(51vZb_3DRL0_bQ!U0F*snDJrsjFVmwsapF1!c3 z$qZ0G7w%s6*PJ{pBy>?=(#gg1eY`6kR4TsVRZ#a;qnAIO9);iw|mAxCX*LnN*}t?s5&H zwbMHPBJx^d&S3^F#q1AEB05Du7uin6PS%x30Vm)gkP02_XqIT^sv`JFgYa>K=6~4b z>?&L|IaPS!s@p5niTfIMiO_&Zt@RpTYIB1-^Xl(vxS_zq83-ws{{TsRCH+r`QD zc$A;PhQU@=-VMmAu5SWGgaFmX@-~{1BgrQl`)|Nl2yi)etM;hkK#4LT=Huuh=~e@b z&LS7;2%bXAtTzDeVR9}e4R&51B9?i0lHTL1tEXpj{`X#@W^o*o3jFRuvVY{hPz2>0 zg$jW$R(8ik)!a!0NzQaTu<|wdJKUusQ%YO&kd$OfFY%(wEom0fWpwUr;nmf3k|3{? z1MxQLxT_vraX0j|N!~^-Z|Do%fKK{R1c!1K-qzAu!`ZVvZUXmjCNRHth<=zB3rWOC z88aM~wM8h@ZnVG2gU1v*Qh$rPt1qYBvN&FScw;nBeenwBeKtf>jRg`v*Mb${Rv3(( zVjl!}HDrvL?M?QXTgJQfs4f|7{4(b?$jx{xE(W_@*1i3XnIV9vSA&eqUu!m|0)I&^ zA`91i$L25CV9kjle6E$O{0WV*4eD^WIXXc^7cqcQNLe8!t5mW`et&9%f25c7_R7!d z6P9I)3Jo%v`577IEbeAvB*54^bigyBo^*3WSdk)pK`JP5*g~{NT485lM46HowzT{O zDv@K`{djk6;G^%Df$9ct|1FIWoVq9hfO%=M4SZd=dQ(`uT+`L^U2o2i-(9h5uE52M z(wRCq?3eeAbn6Fe+<#FBZ^^Lg)qMdkD+CM~Q`va22hKV79+)h_&t8|6#}n$4AAmD~ z1p}=F&?9?ljR$1azF5DBd?zDwcc_(#Xm&aAE%x~LGh8PgDH3CxPDqDb4L`_N?|Z%V zpmm|DHyc6!b&BrnORVf zZR)@`AV_LS~@4yl|y%FwHKD z5YTI;*gnd}4{xLHjl->lqAdAd_wAsYouIDn{hpQHYcMx^y6Ad~>-|LC)kN!b+TS7j z5U)JO*C}DpC4UItV`tXDWsr6xa7&ykr35LkFGfh&Eiix9!61qvTkTw?5jmAjl$!f_ zLh33NrwD1>bo;ZhiQHPDWG&xY|$vb62kfX{A+&52%4Mh(kwrjaxM z1DE~7zJLDL|NcM!FAVm7vtaIjv0(fk=lGv2mbh`^Eo6nt!qi#{qN^lVdc2{_scL`#0nt7#G$WC?)EcOU0iIb@(%4u>a(# zm=U)mT}efg|Eqgoe*bNH{!gOPNstFm>VGl}hJWFB_fHc~>JTf2#hirVUn6n%&j`)` z`xPYKX$}dSISB=Z;k>{7KaVuSaE?UZCYwqyRQ~A!kbi~|>|Z(AwI>o@>=|M<48yb* z^Ix)Jp3wHVDn_^|<-Zw9{NFlOj$!qiaI#T65{F^*;BTHZ{0*HYJE(`}&QSAD!gc*S ze1GTfQxY4Ytnr}ioBTV!_ur`TZ_(A^t~IGNSGc^xT!PB5AnBFeqTjqEqsj5Bd~nRJ z_Z}17K8!VrHG~ZqkgyX@G1LbKR0m ztpwRE@nUx#Cs`^Gwgd@V!bc;_Lk{~4p!%{Fst!`5+q}c{GmhvekpPpuIOG=X%zv78 zhSEU(kR4$kGs#=aH*3$xzXn(bG#UqRDw;|8Kt^x?%J%1y6~XmIMJ=-O5rU1)G4lJY zy;CBz&z>d;5lW7JrdV$SX=(j;{YC^&7j_f_gt7PrPZxev^UuYu@Y?)&%51}b{4<9) zKu2L-w<2d{2d+UjeP{jC;2sYp`jpqlYnzr*bI!WA&x%fh2T=fV}FM0CtYZr zT^TIbx&nT{O~s4hd!~Fn3>CvdJD-+s;bGjAYKL)vT?`EQYaCX+)<((FEtIX!($&k< zcA3HNGo~r;5iJipD-{c;0<_7tLihsFOoG9xN&tL!RQE&-_uH3tj|i~6kC+*L+WZWX zCWcyBGDe4nQ#e{gOIYzH%zv(N2*sPZ;SWRSM#|90(bk*#(r0<%x~E)RWa*~Urrl0x z5G=w7jEsVEec;*n(QhwDKLR>*rYBt!H4U&TG?aE_xZT*_yVsbhlb~;k;Bxb%K8Bz7 zl;)4lPxN^{wHb&7*L&{b)9 zHXOuycNhyTQ`ILW%}A3ttTiDcWi_ zz55ppxLi>&FesHJUFv2(O?~1Diejs;_85$NVo$P@Xn*exKRq~L23VYOJCGn& zP)irePOa^g3Rxw=uuPM^l!upF8fZ~Q(i{AC*!t=SURmPt)6FV#6a>u%m`NMjHbbLx(cMxG$HaslhdQ)itAtrQutmfX{HHks7> z<(|8u&!OcEF|S8hT=SOw4l+mexD^0YMF8?wMdU2%^nc6@D(C7nwu_X+wr2J(=!1!< zfbhGL)z{LJbwNi|;E7R!DKy91zq192?$V}_TGQ1M_l6`x%q^v^af&TF2H(P!PC-0< zuj;@{cXFLONYSbtk(+^cQhZSE8L)@=21yE8{q>=0Z=I;=23mlONQ;ZN8pfoWtF37T zj&&Fg{eQ;waXVU5F1)Oe=4b2`a>XgSyU7y)ky7P=S+V&)jR0%*f;b3sj4 zLeZ}ws3X3j?8PZUj2V~E^*T#&S+72D`hcb^*?*rSC|(a^(+}v}x92z5+62y3OoQ`% z*Z!g=9cd|a@pyljnlKA8g8$--y@cS=f~vCMS(n-LxM{!4P5t~C8}|bZysFthM7L{h z&-3DKo4Po2V&P{*+pEnjnk7p5B>9<#X10kPvkS=gm)r2&Y|i};Waq7yXv;`Ca~Yo( zSAYHPl|+xPR9#y#NgwnsBVH>EGB6lRTs5;2t^;$+l`=!{!=s92ogP4xi@{xZ6#`+& zY?15e@*9vsys|!FqOLM|Be+0ZLFF045e~n6=4{k@TCZG~Gtxn=4as)Fu5G4R?5vOFWga>JfcZT=&b6d zW-5vHyUde6c+4yXm=)fZC3fI02|RzX%k6&89>2yeLY$^%)=|SaFneWrN}a*OxPO}1 z?{_8>i`=GC@ITEom|_Qa^b}Z{K$hQ!Q^smv6fd~B-lgHfH9zP`yMQ;~PKwS(n)SI^_Y=7TRK>2aB%TswsS{xp}WtzlMJfkt?gDSIlMn{Lf zP!KmBQO2|i-`CLSq`+|cL3`RtoZl^_L_qu`k+v;o!E*bntz`S+5(>VYY)61?hzf4N zw(d;+Gv2d?h+$2N=tU@%F7!+*Gw|wu_|5mpx=E$|hT$jv(lsFVI6=nc>VNFP*~@g5 zxV55>Gr~*G^pZN3B4ABTN*GL~>X^IC&7cKQ$-d<_EPu7xo^ANVLDuta?6hw|Oq*T| zg{&SZ?2FKvd5AW(SJPDYfZ=d(o?6xNQdwzNkC5Fs)<^}b8_FE%mdyWNgx2M6%?;8r!Q#8RdB@1FCuAHHA zS)__r*=qWY-7GC}m{F`AYy&fOspzNiA&3pNpJ_geu$>d+^{m8M$xels!f=2<0FZwV z=EPx>n@>29b9^h7dp;u7g*>`!mf_pKg#o4~w_3;(pPN4>1B_fDe1EeiuI+@HO13}v zNoygBVD;Vw?~PKIz26%ojiKRu9@kE(Jqh<6jx37`m!W+5l_6LCa^X{5JyJ%j@0Dt7 z!Fpd|yBmw`h|WBKU-m-l=3p&SK=>Nt#iFwguyM-HOPmKf1L~L2?(b`oSQ^WpV9(gT9FT$!mRo3KM;vpr%HL zx<|%|e+n;2pv+dcI)I)h9<`frvpOpObASnz-@93XlPqR-xY3%%qlQ zK-klvQNT#8x8_&!2j<(XaWkJuWk_cOuvm6m^JJJ7_LP&N%BQ;tm*Qt1_HB!jf}b}Y zwk@&A{gExL(0`^+OE~n_z_Mu@hBgKmfk0`sXtjR_Wz)Z7ahz^8pRQ=6S!PPnK?V{zQ_aJDBxWJ@$Jn=0s-JkR z2c7`(VOH!AYp&%2Myiarwo3FRsrC%v4LmI#w@AvOZI6RL#rqw1qKWz5!e&b8fZYb< zLf&T*$A84q9p;a11rpLxcuqlII}-~UaG=S%ojAxC(oF9?Xtl*Pa7k&sF>P z!@nU6RHn?Xjt9Kq1jMSHx(`!tEVQR@qf2tqzSMX{S($7P{GdJ@00McOa7{|M%5LV!PglYNZf3nm!?J+E{CpxDtlpNcV=4c_Vq zko8d%(;Qk*uJ89mwm8l(6ai1mSo6u8WPhE=q~!aCaesxSZ`b9T6Sv(vh(pD6uWI!? z`kT7nac3!W0#KlVE@iFmp@IQPOLc}#U4?WRgBn8w-}Dlmj7=(VyswT4E*T`-G{5ot#qB6ZHP4aCcnLX6f4ks28PX+Z{j{bJXJi0r#>-5uw1h`` z#i4Fd!BXlLdc;X+WUQLXQm+pv_@HnoFXYiXN2N#apxeVR>Nu&R5UBRlFOg!!3(+G> zR;Pq25I;`nT63P~XP*up7cdn%m4A*&sTcHS5E|MEUN!Cd+VzfLc8uGSPDf_TdN2HX zRH`+iW<&J5p&lF`Qxfh0*`4+BPA@~>Gx%taLSDg*l51!b6`BKis|GE2-o0JsPe`>J z`o8WEm^pPZN7ngDsiq4$6l}$EadT5arHG&E2X^9q_m&3Ic-e8K3;Gh79hn|eiO;FvS(_3N)JK5 zacp{qQ<#O{MF}v}#k=z&WPg>OD{!3s_-R`gv)m(isrcuc5_>+wQpn#jTuD2naNGJ? zMVLDS)?QbX!glY(u9xw6!X5SJxN^bHb2ON8cG8VyYX}2>B1~ef5HW4tmGM~Q&!SB0 zxhTpAwl(Z4Gid!u-($Z+adzj30=~o*ks=FhXHMDD5@dZQ3xfj$V1JAhoA}4yUbyQu zD2cs$!I4VRguvk7?{ptZszv+KqMMCkvgRokZ@!TqUp^cu&c&E0e}PSe$I}HRF<~|H zD-k<`=XnlpHJ5&}dlE_47F$#!V1N2eR)e~hK_(c8YQeX`Z4)JMZmbkooC5Jaor+?rzOG~ zorg3*=AXCd;$`G_yV9%=D=of28~L=YIwhz*#}-%IvTAAkO^dAxd0PbL_2U z4o5!rIWXa*E`Q>`{`as&&p@liAvKXM8uQ2oBom=K_H$R=Fe(X*l3k|_i}%Ge>V=D* z6dgOzHWyOd9R|*s)2DWJDzm4O72V|JtIKRl+{kT37M06QI5bsl1OT~lEOQz&!MpZN zcoP+uvg2k9OQ2W3y(MXNJAquN6O5jxXnn-N%1L^`VSoD;PX<^avv+!FBeFgCIS#!@ zyXGj%y;{)UTcDwHh9AAPs5uCdLeVpJT zkr(d5X@8LwBj>YRAIMJ$#cWz5uzq#+iYv_dkzLf!6Zhz#6dN(BXnc3HEo{#dd)Dl) z{fylE?FB(*kO=OdoEycl-$P+6&*6SbBm>OG!MGYdwcm z2e}Vx1hI>X8@9rH4;j2^vc}|#&O?TfBnxTgQTt~KjBuyX=jcwurEvLB)CHb}*P+b~ z>Kp|Zb{u0)QoaCd0h|sffXkLvLJuKbeMXZN5{?I2(5&)s>0`s?RY^XgOg-aFIVD{K zmVZlj`M}g!egJQTG)C=NB>BzuZb0y#3bK2c`fQ<~ErUKdN&6A>dv-bndeC-_%B*57 zk0ye`m&~AH5DAgFR>%bA)RUS!fQvth$O>1=V+Fdg4i%MUjLYN6<3n!>V5NXP3|-8F zFD&RN`A2-rT`Q}1K`N9b5!K&+IAqm!H-E(~d+;NB!slw1SSFN26v%$=%4MJElnFhk zD;oZK{FRt8X9`f)Ck8?7KjedH0#>W6iRn*ycO49+YpF1;NCm#~`*x6dF8SUb@wuPK z^iW9Z;n!PC_tTCbRJU`V?xs?CtyURMnIJ@TV3U*vhHGw`U_3=D7|x!GSC^e0hku_o z&T0xBX^yRb>|PB!`~*kFG6J)7*Fv(KJRLUsQ$_h$rAqB?^o}mAB52TPoKrq7C@XXlSA?x!cQyM^4O1)Voz$@JDSsq0uZ+68 zyR%Toogeq|;kMwHi zBF_@2Y%DcgSmfN36wlJYStzV<A_v4$H$A2!AH4L?ff}G5V;g+;QCWmuMM)HXRD%DCH3Y*nvwgJ`_ zJw>Z)zg^P>wf*NCzL-W5e0>ealt#8}4D)fbn(W|WnhGo_?KK41-ARQHR%qh5kuGE) z-ZSfoD~HXN2c6B7EvN>f0e2m)M%fS@INS?)Ex2Xo)eh0d`yCawl7ANo_XHLOwE%HN zCvTf&c6qHIq<$zBKyf@ziUqnYC)fojHvJrOc|2IDTnq$Ey9p5rkURIwoZFT=3kSFz z`?=hS0yjdWz~aznz1qRieePuCe8wN5I<``Z6)0UZ82iz>gC2rgwnVYS-eyJ@>){*- zA{g$GN^0N@CxilP8GpUlFwS+O?Gx}x0r1Y7L^?Bes$8a{s$S_8%^&;Ispq3L4$8@> zkSd}1g`-FD_)Q&~2dSNI8`R8J#A+37cf1FRjP4rJj1iG_KbiePI`h2)?TQ>+Mg<(1 z?l6FY!!<{B3+s1vydGwwB#bbn7OZpt09r9eY>mkvsEN@#u74!3NBP5>rNr)8ptn~9 z9&=pnzf1vgL6uRXRHK%EewOQx5&Ua>T%Cz$Jm8F*`eG4GrK~E%M_R1=wwgf|u+G1~@KH^!yqv&l? zHD4g%&3UZ-`OZ-a5AG`4NLWh@C<^vrBsfW+fIGXm+J8A`WYZZrjN@b%bjVRoB^t*K zpNrzPMJ`~=dG}{~4F-F2?GL<<#i`_B>z5cCxLlTu?LksSNZxoxBL=9H--;fQk*9BA z5?!i&0=YyBami*8_)DniQ2Mj*O56Z-dD=Xrx*If;=gs-R+Xn=-4SoGYza$e^dcdzH;TC~dNF3V52Xftjj_0QjB?Xc6r)1Yz4jfyV1PGRkLX6?7gsD@=X}WjEXn zP8EGoThWFI6vy?!G$3w?Qf*`zzx$a4DTE? z+JFuTo`JctvQ>4WwZ$|)YT7mB%{*nAh~^MZ#s^r~43q;I2>WBcvcQ?p&2jUnBPb%N zDm+dHAQ^=PX2hQPgMFo+=K93_vZfL^u|^P_ymMR6;_;9|)xK3X;)U?$_FD|opWk?J z@qg5@oN1R)L^Q1`45@kqX_B(?aCuY`Fmg4w)g#x=@Q*qNXZoDd4(kp7o~@~G8FT@W zKlhTFWfw`FNCwV;$MTv-e$R3prj!vjkn

oG}mU)p-xvFi-xpL*mj`ggtFP{pc0C zc)kifTpyz0D#%~2p#2L_x6P54xfkGnSbs)E9ym$@Up}dxW%M~qLHbROO5Y^Z>kOwS z;dEnU2aOnAuzySA)|s$MVewK#F}3iFPJ&1+!X(>s^a{gu?e!0yQ?$sxS-gFzS-kXj z#p%*jtug1Gt7>xKvOodlQ*?tR#Wr%eSid1>3rsI)I4A`n-j0|_|IF#Xv}qFN^?$*e z-$6J081&KC;kEWS@-g}7NlD+xFJ9?~$^p@Y=?CY7o9?6UBZwt8R0t^CZ5M#WzF*_c z`XUr5{cE3#h!>q}#%$U`m7;&Yh=_3^@V+*#>e~JV*;hmlbp0X&umi_QQc)i$C)6#& zcO)8oQKRjiHZ(Jr%p8QxfCj28<$o=qo<-EjM|isq-9Gkwj+*@`yG^`@4)1`U9V#Tx zT1^vmRe~{SGATQPI65fdYRhiZl=}qhapf2>_~ByB^ERCWJU6aaiQg}ZmDWsj3zlOn zV~u91P}&#v1=mHK*-i8snTl2i+2|%6x=ngXWNb;~4}`I_uoxd4T6zz%5P$SOcyq1R za1vM;Fz=L!=}y`2yKz{dWAFyiWioFLY#5*h-oXwxqr89P_{t1PR10`CdAeV}N9W)i zt<^kHxU(y7?hSJyW;A}HbhY0C`Ns%AO}zUoaC_GL4(A5$y{vG1v}oBdDGL;DW;ej7 z^ymqq7>A)xv@KpKr#*#(lz-Ux%d-%=$zW29a9lYRHHjeB*doyRJDh!{Ai#xd2iwyb zLiz66OU`;_-OvCS5E`gatHwc`$=O`}GNod_c6jhBg;G#(h4#=+ez1g}2b<7S+o#{q zj_>G21xiO=taEJ|m47VQ6fn z6U=T$AbBe!-?3Sp?td|tgD(#PMhWwj!XjLo{|9OB(ypowY=QnD8=#_M<2I5IAVARY zQtT;ecquAskY9hl;9PUBz0Wz{ZQREG_MCGmkECkUs8N-qd#I%XMt&B~ta*{Qb8Jin zdu-zD{5mY{tr}$A?lq3#YpKN5rSIyeep-0pvKUX!VY^5lg@14++Bw@e(c)U(jmq|x zx_*uq24#!wNR2l2t=`X;1-^>a=Eclsujr9;4DIwl#M?Qv?p6!RcD;W+J`FpR`FQLr zulKgO-LJ8RZBH48YFN5HU;1;y=@eGE?}>6>ZTpvapKa%H?~1D$l$IDTM+g03iKdO3 z*bbX^`FJk1;(xMzuILb5RO(XpOg!T5efjtDUcQ>~f5 zSvB3$I>M`_G78U}sxc~;PuDsD?q847v_q5UW3O$_4_DhLSMynh_cPg;PyPB*H<3_S z+0544+kMwH$H}x_-w*!y{)i0iH3@FG5z*l_z31m;9e?__Zsd#tbJ=^*W1jaTa=E3> z_VMW&yPv)~PgO_7=`=ZC+4?XaU7qjv+us|zop*>)JuX5r)!VO$8N*ohJ1^PiHeWALKb6n}6->;3%huUbY3PawkqdI%Yn-WE)^k~|zwW(mf6_GG#7%4y9P4&vZ(e550&EO)7qTwmU@Ti)gZ z)qj*P?KruXXRf!eU_Lt-<>g)92vBD5MHhQ(U)GoFc{c~EtrpLx7cyR3iDhQ=df3my zQ(7HXtK!C@5@wk*f>zMedmJk7;KEPq_h<@`fr0=#VU>3K^P0YCZ*8Y2eI#@By$vnE zi{+7?4)rO~z3iTcX`Nr9SYo=GmM~pJar4p-x*dtNy^CpeNEX+_=x`~TVG&+^27jPU_`3JCirEtUL^6Jlo9j=%$DLV$q!+O4A=Je2(!Mu10A>i^ZZfw||D~ zdv|zQ9L6(t7u#^o6ru*FPqqoNQLigXeg=x3y!y}G{2l}|wEUg{yn}B`^PjHa6p-Ou zUT$#l+i8tC2!}&n8kRyV< zlr3_6!<#h713SSy4QcKq_*R8|P7~b6QQG+>p7Q9HuT8vUkpfv^Vjgc`Fn=|jIDX}k zW3Dr~kYi`2z{dDKOIjT|!`N_f}J@TJJScdV3_`GPn zt9= zS6%zc-utcTX5DR{=7-B_+<#x^?`RZM_G~H-hQn*k5m^CtT>+PS>h&aQ9yJYmyqbib z(=v8#guQP@=}zmGrC!ZS_c~WpcGorrGDq&UJT05l-#@nPikjQpT^~NgQ^VDmnp>~% zy<6cMr`(^9Zfla->Sf1b0fZIEQ7M`behu8q z!tJ`-oZbtsyiSvowI%)5Jo~eG&}>B4!)_d(@1Z-s9C~B=+~4N=!=b4;;iK$I4KHpi8 zpWdf)8Cvcr+bWGRdedIFi#tMlj+BgZ9oXeQoxtKsHI1iM!c*&3OsX{yzF}y= z*JD1lmKty{8F}MTMZkFX!8G_hk)Ot0ISCRQ9RlYy4UFZuU-!q}x*mJ~9QA~Z$nBgv z6|4};d0#avbbrpY)nuMNUa!@Bn`y}IVqbTT2Hn1<=Jhpo?ALTPzVs*O(zE8N7*}I| z9q((^eFtu@Byp)E#Fz%>3?X#y~=SGqun#1gnoo^1uAxP zKsTd>h{0hxC+bO^Zv4mct|{xt5i8N?Y@xi;yP7E@b))RPZN)eB&IKgiTxM#HOm%MD z8ka<((df(x$G6ok->U0*_85oeePOkP^+(4%2Z!2Q@Adxnjz>Vu$Yyo9mFejjIkU5; zMrLCl=YQSCZ+DO2@v`@*J{r%uQN^U#c-~rwxu4Rd9g3#+pXoSST*d6M*|IE}G=?(K zJv27;g{tq#vTjaw*`LK#EGO&BA<}1;1@K>oop~=0U02q}o-ydmDm96AtNTl=vfa6D zPe=E*&cPvLOna?1_6=AQle55u~hGj#<;wt;j6kh zcXV9Bi6-#g+D{I(-k&e*J~F%D;+61L6sX?P`qg4}eK zIDhZkxG(r3U69>o`nt`>sI%=VQt;`zz5XydE(W>#hxuxPAlq*)%^J@P%eb)1zk zuUu>frr7xRJYut%nOEnmDL$8HO7_+Xi+}pH``C+3X^5?9J~+^BwbFw_$c-1)7Smnh z&lZ!t>IH5Qn+2hpU6wj_Ofpbu?7(KLdwE=(Yo|E~w$4rgJ6~P#@MrC4vYgO)a;pyW zY+kf?yz}Q%RlhiWysy$WI<|;q*LZ53R`2Y5L(OLoE32QOqi>&lRfCnbSYLbxy?NG95_D1`9ATM(a}8}o@wpHAME0H5zhT|9u!u!-9PRXE3)VORL<5RWJ~*&wq;cv zR{7aj^t+dJfw0yN&5Mpfe(7*sZ?%Imc^wR`cv%;#o$$peCReBzhps=noA6EA!~9hV z--*MWUE8}&yzysA7%1N9^SJG%^?%u6v^|8$Lb2%0ansuO4Gk{!4P3@fI+~KvN}+Q{ zh$vTg%8c$9tFrc^nBg9cb~V_|)LJ)R*tox1sBP#C1&Wtz;d-~5igqW8GwPLKou^Sw zNA*>|oKbg0Q1SA{t&bKsyYX#71$*xIuh}gOI}y>vT|q&aD$Tl_Z2+EVH-D%7TyASU z^^{OdA~l*emYcBNgj^jo^^z*+S^)j4PL%XKYon@6^>9kD2HTi*`7>FRU~_Rc03sT#+{ z$lEK$3tbMBdkSCYfGzIFRe!c>lBUdIoYWc{f81qsyB?Za=$=V7>bc{p)50jaq_69A zkAiEO)${XS2eCi8-Mi;#rh74n`fOs|PsUShS*6{Pwb`dpW_VWBj~81^iqpbdhAcfk zi#R?$vpQ29)0xlE`Dh(W!mrgjjHmbP6w-?bqujaGIe3KynH8VF3xA>OEmJ<)&8AI& z=0#s^tNG!*aE$3HNLOOKq9`<9+5AfKS}QKI-Q1z>c@iD3)y?nCdZ#_%j&|k7TOV{3 zK5-u|oj{$N_|cl-b-I5=@A2g0)S9|Dj$dK0n2m!~GB@W7_MXG6+kGB-@(3%}*c^Lo zEgxrk&5%9Yqo8yi$$zRmJ#M4V;+7w7^Zu0|%x$nKOr-1*6lJmZY`Srl_Q4&zj{A+} zU(tq?AL~jN*M#o$oUX%uZ}c4BQr_&d`E0J-cOGQVNnQE+M{QMQ-!!{ejheb(%X_t; zMVF}a$>$!RK3yk=5qmt%$;>%!Us1Maw-aXDWZ~bY&{ssp{eQEH#>zRpj*MV@P3S|b zp1Zf1&7%{|+Ec}*2bzB74|}|Qn?O^Ri%0WPCoblky3^|&dMUlr7I@k8kaYXyXPlkS z=F^(6RUPN8qFh!}e2&4)%mXktGwOoS4;Uk9!g$(@>ag@z00oT*+3M0<2x($<{0N-L zE$(hKdj`ekvVV!5AiHiZr~VOu-S}2rgNf^;8?Za3PLeZ}TrP8S)2alk#`tmt@YQ;Y z{mrj;#bo0wZ=28NVn;XlnueEk^oZ8C>5Y+Pcis|YOeXPuoSs(6EAF~ZwHDj^97S%p z{ABE+tR9Y~t&XRiHFN82X?#|Z$7YKHUu)%-y*zGRZ+`;x6K#A*yUkHM5?y&Wr);gR zeft{G)%6bSeu)foGC5SQ&62)7-Pk@xMmbef<63IapX(rAj)>;g=UsjIyl(EhU+9aY zah^R?y36m6?2_EJzH-;H{{2xNlYCi5z_d^Ptpq-w&nj##-6gQ~w;GJ23pY{~mL8tH_1u%#VO(vC(`+ zhW7+hGS&B5uG#s!8>h<4P!d(xR;K8RayVO**#LvmxX^F@ba8mPf|YAIe0PDB#=?&& zaF^35A>drGV@?fQQ22o@SzYZ6EZp72X6Y=1ZC&lIuvj*hm^ik1x|O|U7sa8}Ej$WN zJbyVj>7hgV*{w&e$ zW7R>%^xzg9+>LMTn&P!m@AJs*-rL!c(LU7Et5YWKtS!Y0IcIlIr;~AVa_kH_lly(M zKt*^r8vU%{GkA{1j{ho6d~qsudfPnNrhkqT6K(oP9iPp#t@Y~i@Hg|LH>IO-cU#=g zJ7f8nE_r|6@Z~D*%IeX8<^lId*Kk6p9nQx)tRGdhDw{xArEg}dDzxwO$H8jP$sIkK zna2I{a9VDjXSP-s&s~=2mu~x54JTz^-4 zjoiC~zIf~Gg{>NR$zh{4&!93%JKvRcl9)Rog3amZoTlR{o;#*!%%!&d%vozXZv!U1 zJ!&4S?Qu6L?pN!xtf%ISUdN|kH7@GcoYs9n>jy1@4RXeLJzgU)?Y3eCPS3})jb1Nf zIo~L!HCTzxtIsm)KKZv+9R1vp!hZ>cv;D|#CyV3NTG-F^+np+-s$LYq#fdB9vcsUf zHT$S^y~;sxb=wE^&fmsn5GJ*m#M868-<+~zHk+B_PW^0>?bhD<#^IbD+v9P$?56^+ z*6Z~F$GWaBb;mRIBYoNK4vR%U74hkOULVY4sVsHhf1j0p+)wrA=8@dSbAL0cC))D# z-roF)!72-vL8I>c3BKT0sxFet`UYh8-c8+E{h{5wqw|H8rcyP5VY+cTa}$?tj;!50 zl486w>c!i4^$1WS5L6sCL2KNtqqnzIG7Bc@HCgv`?8Ehndhs}Ay%UC=w>v|puf`<2 zgyeH!uWHz3{N>W=`hTO(xIs&I2ehUqL6dpRJnGT>{c{L0R1_T;;{W-7{=KKb z|JYLC`TX5c;74Iu4MK^4JQ~wCtu}p;HUJpOaN*P%b_f`D1ONbxo8fLjdAu?X9{R50 z00bMk{Q%tWuJC%;ke9L*02q6@Vt-})*%4saqX2r}^L6v?1>QD;+=to|}ejTi3nT8L`o~8g8 z_8Rmm;DW0Qkm-Z4$O}g+Clt z^h+hG9ai`gxv7ENQGWq^07-`9pXGCOxW{4EzySlm;Y)sA1?~iZ z78nAMCsQfn4AG*U`hbjmgCr&(iHV5qM>IKsTY)OgO&{$A?6nJE_@dP00uHw#j>Dt|2hoPxX8H@dGmTOZqYMECxJ_ zX-~$u%qFxa8#7=Q6!LF$V}i^gA8JFM*=A3+W**1zLkp7;8y2*1%2@TwZ0e)`^>5|y zb3eSxlzg%ajDNC;UuGo~sSUeKGp@IJEGRZWM&+68+j_x>n%4=2FoCb zQjAz*!}THp4FCLI*~qz(kYQqh*sy|3^942_Zz%%y=zqhRgIGo`Rqg=x*Y9!d0G9cN zh<8>WPXXIp?0m^^S8Z2D2BqD3YlVP=Kno6rJxkUcxX+%Cy+S=eK)>@+W^(R(nr9oc zOB2|!t5am98KmDi>EhB4Wt5DkW|wZwKE66ddUf(_3`LrlFx@&Il9h9i(deP5JYQ8k=xFX59Y2U++YSi_=P}NJ=scnMCd*L7 zu=Ak3uYVnW?dg+_eIuKgpR^=Av&?{PXfN$I zoA{9&qj<1eDCDv3X-SIj=K+-V9mkMM8zz_k&+D{Qiz8lpm_ICo)oaELlWQv*rUrtI z2U+ZaG{Xj{C}7@*iP(gkQ@}BN1#}i0%?m@^1OfJa6fY5=8-r_^$tkgTYeLqd#WhKG zrGI)!dIzSFXd_NCTXpb?6L%HRixbdUBr1wnsVkgfgK-2oL8JF-Y|x@Lf5!Fdh}8Oy zS$1)X?BYjk@YO$KMgPo$^#45D&@%mwSAU;&G*5fjNvx2*oc6TLJ|@7bQ_y0t$2{Bm z{rB_WlU@lQ?lst{S<;U$T%Gp%1{>9k-3L=X!Faq+Jdn^v3eNxO@@Sfl7K-@8}49wWd4M>atUd0iBW>+D*pcx97!hb}z z2E3`{{D4ki6!FF`e1{6{V0R`AEp_Qxfq{AMr(F9)F)NL6qCt?FCL;*))%C#J6RrvJ zK3V5=X|;|j*Fbbz9COA&$dpVer-r(Kk+7xf9z!|?MdPq`H-vAWt-vOq+J%-^nY(Z_WSHI z*lMZoF-zSoNDZ?CYYPx-3o-`>jR%cbH9K_-LHqL=j6lZ&3^{_euu*{bNU*UFv!lPczhqMq#ozbtaBk+zCIPlckp(2;D-gC2@ydq+ z-wmOR#ys0YWsxj`#;$LvZzzk6bbjTTt_+Uva=bU^kpVmb(w}*Q49oHEB@cl-D+hl= z{yxnyr(9nHw)~qLNOtD|uYo4R#=wUj6as)hZl8@w5PN{#=V4q)d)-SO^ag*h`1j|e zy(atg;uP|ER%_h?hru4#UppF}2l^%5m^_OC_+qagF#vX%X)sQ~d0s=7x(Eok1zv?X zrciA4fHf`>+yx6&^0uooYG4oC3xF6n9FS+AMu=fF*w-+oY4YVKGT(S;1EM4 zCVgY!XE$EO)xz71{S}^FRtA4FjALWr7-Uw?A~y$#32|=+%@$nLnXR6+ae}o?aw-^d z*DfccRWotcWfLR7_xX13uSzV(DH=A{;_^9%ak7CJ{94*=k0Y9pDDM4;l`;-Nk&U-k zbq>4ZLb>0TfMMS=8XsJk<@3X`QEO+prVs)Rf?*!OU0z=>x0cOv@Iil1hMWiSgR?xd zosaab?8_UzSpCIho=s^N=YBNc_0;F)Ip~}BI>6Y-qqndWK7C-z$f>u+(A~M8wVq4 z534Yh_AoVbITz{JN9ljn`OvX%(C;&S|2D*9_{lDQm=1rZW!ew8@iWK&126h)N-Q#? zz4*8v{SBBNxRB&-5AL*xPgWOiK!JTfj*-Ecv7XQX7I8pPg+tK7^EiLUpt8#%>q4U$ z2cUsYkwU-;F-rC3qIQozDz^ne4ruNOn9zrGvnpvya%Rw2*hYW7(_+xnvVuM2JI4%F z0Qdk_0D1A+;28uZd4TB04hH^)$9|tin5Ex&l5y3b&=6+389z_Tdp0}43<7rXp)B`} z87jFq+NhuNRqdI2o}Iy#nXo?eQ0OKcazhNrS$TjA%livC?hYLMa5}_mlwHV&4EHkw zRuYLtAx%t!{$PJoy8VkMNnes2U2@FLw9f&ksBPptBsn$Ru!6~1gQ#GgAI_i&j^P@` z?JiC?AYd+HY!mgn72E15;1zQjHvqFhG=A=3&be1-rK#4P1!lzL43Zo$7_x5I-)-P6 zwgc`la0N7E-m(fc4rJaU@W<@pn0%;w7YlaGhGq8g`@w&wADD_?FSF=zNU@F73% z{~uiFrR`-G5ULZOVd(XAln8a;7Lemf$kG)C+Jp3DO5(U%_vA${$CLya1-!+EXl{O> z#}4N{65?S6gjv9?@h7fK%TEAjK43AVTsDvJNjS-?AZ%*4IGuO)Xh!&SqzR0W$+S(0 zD+3UON>YD?RaR2DgCKe+OJ+OZ1^^boYDERXz>VYFWSIpd&V_XR!^bAR$nIdLyUhpN zsbIu)bQ1~p0?OMa5>ukQ+quWuZ5S%=R-``QSX#?N;_OG_ETX{UU;)5=>i6L0z!-sE z-j7ZG<%=Tg&HTq^{^Wc8;I~&f?q*3OpC0DP4A_5mz^OcrzvCXO{uD;*36Bt9QNl7v zE9`)47sXeBP>er-#|AC$!Omzj-=OnXpnqt8j8Ndez&UXCaW=E^jG9$bLFhXkZ<5V5 zP&aw(dQ}wxMxYt1BfrGzDymevC+9G_-Av({>_2wq0f>I=%=^j5<-7jDn&13`N`H`? z_4|LG6TgrD{M%1nkxu z2mx`|V-^?&v=Jbwo$p6hpmr;nud=wZ0+3oj%UzIppdqE5ll}I9W_wH<$Vg7Y*g#`4 zf{^d93;$d;*=+Ch%niG<1Jv z`bufNk;nl9R{^1*hM`Oh90*o+~n!6?GASPjR9k)_=&?mx$%$v4tWI>sqrsv_4S1>ex{)(0+L@X z%3>olRu$l6>>MOtm0)bY+JQ!_QbIx|E2N(?@ z(-T|*sBp4!Ehg*_yBbAaQMd%Z@%pNvCzqBAq`jX_L56Xo0|w9t=b4N*SHdzG8*7i# zt0Qx@A!p8Fn)f=0n0mgd$S;3)NO^I%1~Fuyy_=nI0IL(o8Uwr-dNO--Q|4E|-6`O! zi?P=8Q@q;~k(vIPQX6FUVE1C+-6-YmL|UQCzIaTLINJ;ZiN{jEH2;owf8&*$Tb|lj z=G$G|`+vnb{7EmaAPN2QuIUdi-WNNv%?z?bpWqF{V{3yrFq3L#7A}9c~>X z*}&xJe8`8Qr=8U8mSRRgK|=6IA%uzReA3jyS&2T!#FWgZXh%rI5HRNnn9Tvp0@}qf zEx=l7S%A(0Le{8+H?|v+aL6YVWelno2_~bST$6xp zY+gn9QxP*CdFl3kI+(x%LFIx0FQD9^7F~wg=M^X2oA zqIL}6SnXSoq`%iOhU+x{*L8w~TiV^_&`}mU@bDQp1ypH=osSoa5~D;&T;=lnF_QYn zC;s?-T>23WeszDSpIq*XOM~yp82g=n$s_V?15z&7^$?-)vhu>^wdnOOsvL7u(GCNQ zVSFnLaZHc(*s&|d3bh}NMBEy|NI!3ba~M{+6=ZnJu>lPYqFX1@8A;`%0aH$?^Ze24 ztyW+ldDpjYfCJj$QvN@=a2QPTMtM}|`CjZmcwUCwG?agbD#vR(CAt7P|_9WXPxGmkdMt6N>*cZrKUsS!j0I1os*m^6QWWpkpf_GMjudx`b9dd288P#i#2bQcC_|`Pg6*`*t?mAS89|1cm7ZC@{W7&0bvYeFEf+gP{m z4haIe)eL#wciik@++@=}D8^bNYV|w;FO%8hqm!{#QKPkonq3c5ldLuTb3f z$vn2>YPpX2u{8lsiL@a`{A1OIrs^CK^F|9k#G=sxV2JC+G7Fu1?92NUjiNj7JZX1< zS z!ZWn3N)L$Kxx$Qe)yK&Fa1}e)3Idp+Y!1N^;!bySv%V7qAmr&98nau%Q=caB-rocP z%Hs`W1cQ*mpwe9$GmL8f&%7XV1^l*V=5j`Y`W**+k6?v3$F0X=3$uisC!@vyct4srh+8j)*kWWu@i1#9b--qmRQQLW5QcPjpx|{X>6Ktc@-Ff-F>ID zEjj=K7aUtArm_dd9sCsaGT&2WJ!CEQ**@D1`fUHhBZeB#*QT8B-2U;MLBoGGL23_w z|LSc+PW!{Ze(dzuCR$uR&vu|v9Doar(v@vc#q50p<-2h-KKdra5C~QWWvntG@keyn z7y~Oh8r));mE;We4qtWDgj_6LLocO9YzUEVdciH#TsfkFsnl;jAoGgM!dx>c%!tte zYE9DRW~*bvz^24|XTW=gS|ERa-1m^xc%4cK%20Ph3JZgZUgW~1eLJ3%q1ytvYExM0gOt5!EH00NKFJPzYhKe=MiMdg2?0jM0}=9f?W z)KgU2Q;EGB+Ow&jQ(YPT9BNM4)d#5!#SF68 zfu~#>5n#@vI)mu`>;UsWzM;b+t!KMSHt{Y_wi$O>6CNX%JFgQX{*n`;iEk-5oF>(c zz(@_l?|4gqcDHi8?V-Ms{fzg|vl!VAUzEA3hM^ejdZ>*YN6TcuS3@i{@_c_UjjKkTRA=k z`^TTakXw<7U(((T@fdexJYYuQo8YvcJU_x+nboO0YxeVz*%m(@UGChXXg!)Ki>t)h zIEtag@a~Jb7;I&|j}3J?`!Bw>{p!iMH(w=#bLyA1XKJb=+i-ZS>N_P_9+xb)eNZv9&?lzNBEZPP!xuPti}P8mlg{DW8V zm<;y2JFqw_V7a4+nS0dF5|i^A<+65=Pq4tI+l@U2s13NRJJUBUNagd^M7^#$P{4aZ zAW~Hg?e3`~HCX;6v;)`;ngRw@21DGsjL3R#hzoy?m8W_jA=n#%D%&R<{LLdhMTwy{ z3C&OVF0anSOx2dxaKwYSBQQ>t`S#YL-sR)$DS!Q*jpTXkp@G=NsYeUVuhll z@XCK}3Xm@Cruwm)U(FbH?DE%K{Kb7T8ArG9f9pZG#G|;ZrY>(+K+wWR+8%{k?-prd z;~E)qE*L?8S4L2Y1FUX^0WdUx)vNBQqz+NQmyo8be`9%pi_UK%~~112%3W zk?t*#hu`{xtt`1UlQ?67FTWuEXtSuImXVX(1XZ=lL70j(!kk|%k-`hWCnauL_|Xwo_4&n4QzNW>gKNGM zXqB&#EnviRK=swJ*3wUO*fWVrn)H*41HSoz!~WaXk^WQf3Gv^xkg4Qs;5$dw;1_>+ z3@`W=TH@&Ln2G@Js*q)V+=C}vFGBQz6oM{hEVTlr36D_+l>CT3;11aM83iE59b9Fg zg6rtkV8Qlwf_)IGM!;hWF#Mzfd#jEOJjIeekPN!dS>gf~11>BG%lzs^)}CZ7;hz0?T-SL7|SrMBf5I92_!7{>>=V+<}=*dwt}Cp()Oo z*W%+R=~#KRrJXZ|MTP)pxygTpX~9;;8%a&!OLBUcN!-{#f&Z?*B&3i3(S5!edqC$@ zF1U*!Fpq%YRrsN?EmF8m8d&sQZO%{Cb*GuDoSj$Vgt?4ofB+`#%Wcv9|E_OHUB=j3 z<{nhA*{`Pn9QsiJ0Fo<~B|?Hbm-Vu}#03My-@NDBzhhvjUwyUL!RLPlZ1{t7iD4ZPQ#N-ZKQ*4-G zLv!6Qm=Ff}*7HNj2VH+A?`!<`zG2;<=X>l9pOX~r31BB=zrwypzy6zU|EbUZ^e=vG zTKdHQKF3`=0TJj{Iki{g{B24xjbUh1ZD~T?lDPw5Gmmua47qtsdL4EjS|beq=lt&` zCn)y_>03J|7>cq{exLWy%I7_NpO<}koUAFA^#6X|4)=|5#DITm7)ct{9*ihF3zq#ZWvnR4wS7OGW9*F@5 zzHuIor?ayqaw8R*osBDMi(P0`bVS3rSwqp79ZW%V(Q`rkZA=dK_~IdsS7Y@UlTXfe zsEJ_sU4$Ph`UroVs2q<(*1d*$NQeK!xBlu8DCjTm_$PlJp3%Nld1 zrGFa4r~()+K~Sc>!z)-DRF)LhNki5kY^*tOM&^34@&Wb|LyWd)XnGss@><5_Z0N75 zhPVtlMO^9*nflxN(#ihWZ;XatK3MD=YF{gySOC~pHF$8gQ{ zo6}XP6LyYoR5g5d#~f4;eHdAt$J4n;A3&sx-_+U#B8tK#+?0g;`bnJ0@%#2ciJ!b+ z$OY)2W6Azq$=mO$6G72v31|>VZV5;e6!QT4tk#agMuTm#ubu3HO=Z6@e9-NngLcv$ zm<>OF*+YN+S6um12kEk(dHEAZvVZE-?^n_Zh?kTBTNT?B*W5Wzm(KB;c^;_HP_JRok>mOjmM(cVe9-8xKFsa6 zPid&xkN?Z}LaABgdJ%Q436k+1s=*f;UUq&n68*;PP6MMKpFN}6|Dsazj^gR zll6b~V8GEYKK7uX2+u!e#k8p*t&;9Y-; zd+eeQ?>4J%d^O&>`ZGTdIX@N{>{9Z)t8dUB4JRJ-9Gb`WpeOxeVR{|(-XQy@pS;bp zEyHgL#x3NS>;!*$5$X4g9UPcmEY?_<$-GCLiBK&==$0+);}l>>&N_%e<0JyxCa@5H zFI0r-0n0Lv)H-7gLLdwT1&w8$WoUm~utGWi#*ptf|LPNh;!hu7I)$GG?k=;73A)8e znY%|~s$lE5G#y!14qz0m_Ba01*GMsJfN29b^eampSkBMnH-{MTbm;B)^%2R_&CriP z&s(ct1s2SJwkr zphbW;ODE)W#NBFbz!qGygM(;>fbe0bmJSrQ!G!OIRU_@EcrO766lD&6@c*l`Zio0S z=YZ>ol(H9i|8`J%fs6(41MYt=k4-$eg@Cu7+xdaa?PReBFetTaaQo@2`A_}O-Ed1+ zDh9w=F_oUmBfj;Y`r#T4{h@;`N-ZS#AKWy`O8b+h4vW!d=yg*SFXV#o?re8)0J-G) z=h0||Ke*s3W-4|h&R`uG!m5A9*J{OobU2Rd?(0&(@YrYRC6s#F7|MGN{vI&WfW3x};$OUW(D4Fh-awJBqv3fB zTk2xX(6>L}6d75hVDV_nNa*mT=@c{Kow)>Q$74KNw6vmQ0aeDM^vV7N{-n)U6h(A z%)WI1;Q-+2Kl#&N;}y{)nQ+9cypcR|D##ifhWc_O`&W<`?@ZaF^W~BM?j!uE5BL$= z4(rDP_R(0gaKcPSLiR$cwQDpkxHFwv0v)@yBAp-xP;EP;l?^iA0P4+^U``6DBuVU#JeCQV#>gvVV5H0?}s}UIZX)HNlzvkqp z9+BERMw!X=x5s@9>c+q@U3(wR2mnEbXkP3iAe?a<6*I#F*uKcaq`!RyP#3ZfBmH0c zFw8C3efAgb_A#)R*MUDv?V4_e=VP)3pcrGWT0i}c5$%6yXm*VIQ3?mmT%-oF!##+FuHL95l?KeUj|69 z|ChZt+jU)67RCN0Z-Eh5k|j$%_azk7w^rZ01Xg49maN8Vg8X|xkGbYrd+&AjJ;6-^ z1ju+9q**K#i$zw6tSUv(FrGd|0l(ky{?4zmBFp7RUrva>3IrYxEWrJ~pFljacogUH z0rh`zVu8qHYZ&H&1|0a(f-G!Ki_{ppH_53Cr(?VqWupz(O>-Dk&n?gp7Q%H z|KqRu+iyJl@hkY+2fn;2IH-zT*mMRKcF&edD6=2|I3|R=ZgGv(2-C6l`sQYJ{D$BA2qyri@1O=|D&BXFrB>5OJ(eSj(DttiOY*=pXBejf9=$EtxQaNrX;Ws(Am-H)=HM zaP3#D@{3}ELe#KU#9D>Up2uE;&^Lb|8v}JOe#--fcP&^?z z+x?!O7Lf#y4}qiOzkIB}F<>%!pFe)b$o}#-NLgQ$d)`Scxbjb)wK#E|FtG9>C@~il zwFyPp^t)aMMG85afo#EZ2Ip}Sn2?PiK(|?zRie*@XLCD$R=txhTO^7xl<$A_cw+X5 zO~e)Q(pGSxr|DLK+9Vamja6c#yE0!ri2}mF+k?l~ul#WctNiO6Ao@#uPV`GAzjsmJ zF(cdj&pC_!dYABPUcWwa``4cY{t*x7VYW?V55-7r!1HpoPOu;$T;*BtC5DBT1$Rc{ zBsTU%3>|`M+B#VZa2e0`W>9}%Cag9Nt{xOUybG~7=B#uMJnAkwC_YgLxJHN-wtQ)@ zA3f2C?IJSOy`SQ(XabKnePlm6dhA|GR@|)ak4&62^80eX_WN(YDC4{TNC(Yt`k!$3 z*Wbh4U%%mmp2b&q``Sxi`)1;P>`ru_{*6sXvFg5z_lj&4+fVG|tpa~_;rpz2yIVYr zzJ7fT!o&RO*XRAd7yU>7blH|U`OdBU9klj8`K7}JWGRb%{rXlES}V_XVZl}Zbx#-l z_)Blr&%FLKjaO2ULGzo&wJ0jc@!w~0gbYpA_XPJ>OuzkS`R5$$|J6JK|9?i+|IcSs z1*psZQxc3{bpTujB*=ds3Ifiu012KbDAZs9SHN2V-oTDPV2J|Yd$vUT_tOIRRaXI5 zutLGcz{3Ds`2elU8~Ew3tP%!OK$^zgX?8q6{XfZ_uof+n&{h6$Lf#seWj{X>uyKt+G+Q7E8<>cj*>fZ7X4 z0&oE@1_UUwiA!OL{I#nUXh0nRM1E0hi12X*N^U};=bNa`0;L9A%E}g-0|%j!Fu*h4 z=mC(UcvM*c{DjVnDo}Vpw$S_c%IZj;KP?~u)ZPFZ#j#nR^JYl{dTFLtRs~xR9P8VT z@TI7~v&sU1JnMfKtO`N~NI)0>i|lPy0VxK!LLCA6E)6<2`_{$dhrg@~?#TX=Fd(Mv zb7gIG4pbG~2hATmp9MsZi{^(X5&XBl{hlvYLG$geY61-A2j9~Iu_fm~NRfQJf96N2 z0lOmpyuoAvAH_R!DF;!!%q#=rD<3JuP=1RiBJxWVnxB8ik32~jP|07CiR4>b&T9mM z;@%_q#n=1mm)!r;A4>xuv+XKcKl_lXf)VXIPzQYvziD-HfWCJQ|G6g;b$SDUHnBf{ z6xa^s!Lb1H=D;}l&X7pX=z_Z<+G?ES&_jRX^uH^K{f!KHdwq$}f6g!2ukpG5B-mI2D*I$$P%BDo%zHMmCH+X0@; zHih&&pMO371~>+U7Fz;|1U)kLH2@aCZ=g=$R%i;C3wQyh3#fwYRx<}z!1b%yf@@Lm zJBVcVIG@kw`h3#PSOkK%^jIi)hp`At$mUu0ZHk!0WZGed8Uth?nOH@Y4j@BCd!PRBMesudXG4(;(YS ztL}dZ77HyJxYo`V5wC8bv^i_!TY#A>XbdlhaOL2)%te8-vI0v^g$zMs6xImJ*j~#C zD4a#hreFzBf^Y?mUR$H=0^2||)DvK>K=k9OjM#6#idb26xVJB>;2Dv;^R=9Sz{2$3 zu;R`(vc{pz0=&|`cNOr4h55Ce0MYh~asq!k`d)AnK)#mkMGYWCSx6!|VL@O|#l)3X z0L9f5+E~_jmh%iD!u&GUK<^1fIa$B(IdIj#{lNO|r?XZ75ue{`9v1tB;!pPIB>NDBbop|io3X4Li2m>v;;_+1{7;w z_OH(Y!9pzoGWt!%=l#4H34P_l^ zy9o$=#n*-ul^n1D9~tLn??dp7$cUvS_5W-Y9f3|T?e0ITli3V3s6pCY0l?Wl8_D)m%YO5g9b)D7eFKFeOrl-dkXU)NegaU)8n|27`ztf3 zEMrPMu`K6|NDhl)V!@C6u>5}{Yru-S#|i?ySbzm0vah2A*cJhbssqKu^0igz^Tf)N zgWJ($*^MnU%>?jPz7+Gs>XU;Dk%Jq5v?a3NG!tw5YPY7z0*;0I(PYbxh33^$SKgW9 z39uYmBi6{@LFL2A22k4zLxHLo_#*$i`neBp`~vcV*03foHCO zyYA>b@0a}g8bIxe;0x8tR|b^%VC5>brmz~VA**7Gc`G7d0UMp!Ta5yL_-fS_!k_RS zgv(m(QzC!oBtrc1)t!G>1vJ05H);upG>wOXGtCp*Op~R5+r?_%SoYhVXc{m4w6D`- zMc;PG**A9kw&$A0%RlYRG+E8JUESCE&-pi3wqrG4C~6sASKuQ*OaZOiNrQSZvFsn~ zGKl!u3!q+TBVbb-?J>>_D0?CWKgJlB3F7AJP892l$!xc<`87B)s z+qa{Mm20bFg$a;|ABIc=81Zrup*4_rk%;cakv{5Cb{$JrX{f2&RDE$d`vxFDv?}*%Dp15?<%9PO(K%zN9eEzq3LIc1e#1(i(Z9-x-kqi|e zP#mzxe7=IOK9pN0daL$sNNB$fL^=8DQOAH&U9iCRJT$LnwXcAEtio5{T1Y3hKYhfF z11y}t%&C7wplX#~$lR`epSS;O9~n}TjHNZQEm*X7SSzqbpjrD=Od_%|kX{v{X|)R0 z0m-yl9cvG<)qM4)`qj-?THCu(%2-@XAV&K8?_{h%y7><~hUJYShUUt8Mj0di0-KBM z1=WkMPD=p%8{U`lj__EWjK!tNPW1!^&K?0&5}Vp0xqw6L}Znf1pQb=85}WCI%Cm)>ie^hk{ap z<=1@HcMi!7_7@nqLbP{7P}r3reQ6_jQcJ)|(~zu!P{!01>;cj%KXa7;;0whfb1Skn zS5SZbr48}c36ReE16EWLAShn<)k}+oF67xd+Vx+u)&A$}d?#xe0;Z~q`n%6^%s

  • 6gBDzC8B!vWb9)4#g_o9yD{%k;6FApR2fT;&Kg;?g1C3jp*o+GVIq`%xlDRF`Mt*+{euFBVT&d9=)}MRsTEf8hSRcEqCHlRM#~PP-w4p9kg%80(cSkQ09JoQWe=jV@Kbj79az79uTc5d??3PoD*He1Kjk0z zpYjjG&}EqmW`scAC%>-Xwg{`tNBfiFiw8F1sn|^4I|u$N6YuZ1tY2_H<7a>1)}QfvU%2^a{K6M5 z`5FH&IAnJMM&1A1H&SGuaqBqoU)kBv+~47{TF6e7^?a zyZamV_Qm_xH6xJ-_erH$@P$XTbL2Sj{HP@m{J<}(RZa6G^9c8E*aFe*{)Xkgbo_?> zzE7S1%E$6cH}i`xQ2gTFfAK|5f8c-dr25kR8&>(^{teqBI{t*+_Z@$(KkuCrh?1$W zAi#WdHiL9lY*ot&#b5=yZROgZwxedDvno#Z+G+!qXa)&j|Im4<#sd~p5A}Oz^WFf+ z;A|t4&w{Y}Rd+lQ4*lk<0@Hgzd0kOnYsX?Mw8!#L z4m6bQjcsg=??F`BfmVOWKrW*{U;}&ElvR{Hzq&&Cn-3gRqT&7Y1HcyAIPlwewlSHv zEXY-F5!i?P_n&iD>nDGt`H%D1Z=DBH0jodj zGT;jCFYP*T3(owrZUL>}(?9DTunIx_vmX6ZI|dWvkNCSz0t4KyfY(8?vMZ1pkfJ=> zVhSF}nH2({0=7o?`Z7vA{>`)rC;%~_7s&sGNA&{bKaMTr0^5LiWf#=XTiNfe;Tnk`-eeU<1nx7DS2}E=r&$0+_WU&npJ&;%|h1w$vQgMfX;=1(FON7TW<8h^04pMw#I#^v2+x-uNdAnQ z>(O&HAo3{XcZ>20ge&W!dQz|ldR~m?8}Xq4^WQzkS=oOQ1p~wu8v%hqbbZH@e(`v} z_!0rlVjWbER~ARkM-jp+L_hc?9gXAj!w0Z^=|lOJAVBnEsDEU!I0`yWR<=jS*q3iMG<9D9Re}yLzJv*Y~UV!0yJy5V0 z#P{s$Oo&wd4}X5mppmv6u24$(y-yz`IT{g>rzK_Q>8KyqIoejopWSNQU)LwwEF zA32OIP@w+livF*tU;P~q!jI$*^^?B!=0`43{X2gz?XUH&pnE|Xuy^FS^GM<0M zt9|MDw{rX)j}Lz2_CKRf{;sF5oDaX&M}_`>$0el*9z*>2!u`<20)hY4&jKdekHq3< zy+{DnKPV13Mf=##KCvRW6v^oiozq|ESzFmV;G-XBQN&l|%+B>F#qkHhN9)i0q+k4K z|NDR0FO$E@Gh~3fvLfOW_~VNFe&PTp|2t*H{QBAj{Q6e|AU{qG{J1Kb zNQC+C(v;C7JmhDMMCq&VzEg?rs}bp6{#W0;fdG8-^(re8ss>#Fj!2$0G%t`r>wQD? z=_|kh=K%xVqjCi>Ep&S~At&-1TBjCTSG<3W9*$Dl0DDD5G{7lX4H4rae`y}Y3$QlY zCq0z!UGRX9?!kou1SpTQfbPTD42URSIy8Z-gG<4MC?=l(TnQ|TWi4I=#)OE-fYE@< zH{P>gp~b7fnph6-2Czt$TX_rEJeUf204)OZ3NfI0QWj8y&atcxGCb*;Rj>66d+~pl zhYNP=XnwUToMruLZw$zKS(yZQ`3vqaT}$xAvdu(ZbVXr~I=cjIIe?ZOSl<|80xwzI zUsIx;5`yGBu^a%-9*;k8Vmt-@xL^L>pE7^>%E~-0`MeML|MWlqAO9=*!pi^he5CpR z{;Skg?fxI?!_CA0%}K{uy>QaRaWlatsykp_!1p=kiPc36_MFS?D}aXTn|L;E(sWhb{^of#Ebx2Nx zrBgJPCEE&xa&z03tW|_N1%>>IgYuayXQF@k<3f*@jgIS!{~B_MI-PyyL>uB8CUULd zf@L^bTMi!f3V}RCb~--Kv=e_{G1dmoYi-#xL4%}^czu^5<%GV0zC6kdH{b8!5|~S+ zU53#N$63MYyCf#yIc)}Ycly-$7_P|QapyF~r}FV(1L8Cro#$iB*73NC#Zq(5mA8ud zNi*Mbkawj@D(AC@F<5xq0eEx%UBg4-iq}_5i6ilK;&GeJESW{jX8g##%<_C_xI1)kjiZ|Uo$ZH>o=9Gu+p8X)J^ zExRlI?cxjR91cS8pp@x6FMHga!sk;;2OpO;O)CQ5JTukhG^-j0UZxm(tPED7=xF|I zvBwISlq(3Kd}Pf&Y!H8%m(O*1r4vTEy|)DK+hr)MgL|QG_e6QsM87?Uks_P!9N|ZwEFzRT#gqRO(nzB^aA1 zb|FZFiGPTRy!97&ON$!3>Wvf6Sjdw}d|WO=#kuY2GHg-Os z!tJQ?ikqSSakGC1^m#B|BLWl5F`9bGm-U`JF+6Vtchg~u3waUZ-oIQiF&X7GEWt)U>nIyaMyUqe9gSR@iL`5*_*g*r~(*%Xv!^L`dEV}(Wy9@dTH zk3#3QcX=6Qdu!5W?J3atuio;L+=!ieYz>8U^)5*^uIYb{$6)xf;?33_Tdxq1qt&p* z1`JW=ys>VbP>|mBTqi+3w5gLFh0(lVQhLK3|jN@8Zm~C9VUN?Weoe2j4%B>W;l~I=d{G;%WFSr z@$-^kUaesJ$E288t`maJsVA$j6jpU&nQ)}5y{gGpSz&_mD5FM|Qj!v7c)qW8MommJ&>y;{od&R-dZi*y?sdCj3!SlcmX%{DPkK@1qIJN^ zTHlqmcf}?4OgO=Mo5%6R+98`W^!nt_?TgoiZ8`brO8KI6`*ds}l*DT`dEt`2C4+;r zU399~zj>X@{l1eQBCKuQW^~=qH-6oBtSNuqJ-H>`+hc7DV{GE7Ps2m;%~ul-j>ul%oK6cdOEJsnn_O*I!F#GgE>pIoa2BD#07XRYXIh21- zdbLC4S~*JPYTZ-WyK`F1aW1HHmhSyO?T| zzLmQY@|RMF@6sVFn!E`GPm5Ecy$Jp0EZrh(OY(lz-F3{MrkuDnX2;>KyfM8so?%zU zK(zaEn`_5uSCub0(@e};9G`&-OGo4xI@VA z#Cu|aJfCWeE0y_TS-)LsakY`3XLFbvT2IXzNFk3I5G!;?Y1*%jww z(*vYieXe4fCQaKo-=%+BJGXX? zzsJ!K4eh1)*NcBSjO0eFuHKy(fnY=!a>)R|bX0Z*iPF%AG9?8~t>R)&ZLb2}`w2pF zZkCr0w!GVm^6ejdt1J{Z@FyYvs8fBqq}y66vE8}EA&gzkhejp%^BQ8VZ$QC*70svL zA~R+UecRcvN|#fH1^eqxcdvhU6ouvdG<_!q6(a6lt)TCl5HqJK?dlDT^?r@WrZZw-8g(Ny zm|K?JOI#v6*^N`mQFx&TG2o{IsM~`(VIrF*TT=6?9xMG;g+Vz-Z4s<@@ZsTW?%{*P zoEP-HNF8fiNh1O;5<7nZKKt;7!GDSQC)vQBpJWYEc|69fQqS2VxO>LjVN$cv4W3hc z3IXsdQ77fuFE>9f!6@D3yR3)>Rh!F{v&szQoYMfevEYp>w#g`;g34tGG5@764&x@9 zV~CX>ouc*PVojW_ZYz`^obPusCW=b>xa+&_wt`FDnUb`PU9Nx3{0>sP#YI=v z=X8q^x_1k^dtwa7p(=@K>!m7JiaPjy7p1(dTw0#C%FRg(TabC+Wm#qO5}j!luZY!p z7wOQshEk_a@I>lUz3Ix8U$?iWjf6zj_V`% zG$qMsA7#R{*uQ@|Ej03`E%>r8E~0E}p$I{>23XFYY*k-@%`0B~4%9s3n-tEf;F+0x zBlY5$?(XfT)aLc1%7L0YPXQ!O8;qv2+&Y6*Pwg;lcrgg~Q|L1lS@Hus!0*sGg%k!IKM$FixcAbXCT$5oC*x++d~Y1Tl5pe??X3TJ#ADcfs<>wIWYq<8qL|5$cfpS>dfCp zPWCU(DDO6&lHndMxqa5X_wEmGN9HWwawXA~^3c7Se5~oL7GcL*3xnp(G~yDNe$Bk~ z!pW^vu0DUxB3k>%f$qxNIe91R$7cs0S*2|dRHhMlar@dn*dyo)pvGbOq&y*5Z&ndz zdB&vzUaw(}U1rm!2P?l6lHYl>{N(q6cVn?2CW8_^dVIj63ypi{;&10Z*3nF?dk-p- zS|-WskLpBtOM!=i)~4BKE8D>^lvsHr^rJ6d&%%EfmW0Gj;ub{FLoF&Xdo10{2`9^~ zge0Lb(X^qJH+g>3U2l;d!9yg4LBy6rAqLF}#jOnw*)!(PO^oD|b17I%%(utTiqh%U zD57N_>yXi!rMY8K7E+<;oKS6wGnUT-pXZhOP=VDwZH$ON8Gm{A1(yn2@8h+``C?r% zWvzeSOWqhLMtN1&ig(V$#(NKYGjQdwosN4I6W7yek<^#eFwOBZ$LgUzf#$B<&9b2!qA5;>5h)>I!r)p!mxq7&fv6#h?&Vwm8Gi^t% zawz)!RaJ`IijrB`;<}0fdVaf| zEOv(S7%-~B_=e}hMQ~nF%dLzL8otSk{LB~tML@d0&7CKFCve`8K!uFgR&(RHheMyz zv#Uz$d`RzQ*K4h$DVu%alJJ_Dx%RVrK304Gyc&{ww&We&Aj@IdER!**U%T*zNf!Zc zI;^XI{W#G0d!10DL!iscrm{4ao{5vBH@>!X-qoYCbdK@|9Q%b?H6AMpj|k5#I~EBu zT&B%lk_#-k*}Apom-CvBSCEc^_AtnxzaQI>;r()i#vu&T!HZnWEb!x*1#Mj2P$8{W z$nSh#VuHoHXBbqan47Z5r-o4ybWgwsWeIP8(-LvlGHhy*6ebO=70NETRuhlO^P@js ztx$|COy}#DsMXJ;?lXSLbaY*Ob_?b&gI5AUZ_cX{R%pKpMI8mH!k3)LXG(*VTJCA3`S{F# z##i52ioEiB?p8B)mK~#kYpGPjW9zJ^ytzB=x2lA#&Gg*+wa1^8{iy>=Hrm0US*`U4 zkyFmtNbQd5%me8KR@dw8M8B%{CoPDEWI56Hh+eCnrMAFZ!-$*RJVNzy_lHGdXG;^V zxV_Hcei_-jm|kF2g*@e~w$xsXAaovo)p)kgrbi^5=w*eS34LcruCTk-dLIbS?utZM z!aVWEaU4d)EwC`)x`Tu7=TyI+r1mh+(xZHZs&7Ii{$(;Hp9NaC(pYTXt2*% z>mTo!@eO;Emc$+b$hm!SUZ4Ep+>TD2KPBa>r4esV-X(PDoijyxD@4lrpB>|5De0!# z3QauXM?YA@V<0Yd351<2XztxN-qPuT9oZ>U=Lc>xazk)1`1FDk zv}UR5WVTD_T1kaz&>@=>zvo96bDS&DQBA2-3t1hnV2nJqSzv_Yjon(`X#d*7n@!iet3R^%##LUZha? zWbjCyEL2l#GkNb+XxVTO%&Xy(F;cX<&UuVYWV4XQvJy<;P2G zujm&R%(+(zj{|4Y{vks<)0){~DOP?@qS7sA;V}Zx3T2yzjsDi1IO@ZDT_yVBT|aG; z2)ORmx2uvq_I)MGC}rK{CbSl&xOc{)0(Ym|G@aga!wnKf(C&JF6+^bIwrg{|o2sza zao`1)@2U3QydE<>-0q{VRmsAnrvg?zr@b@>o88Z|-r0LC&@!!Xqg8Quvyt$6s~cM3 zJl4lB*E+0#RkGJ>m7)Kn=!m-ji)&t&b%eTpyEwz?is4`*>gUu?Ew+tE2F!)}79Y+>#eH)?qRlfAx#mA)p!(Vwhpda*^1Qvw&urejZ< zAAH_SCoCWPS>hL+k@KvI+T}Vh?XhCrC?r{sM~Q!vPJ18Yqrzi8hKIEn$8-o)Ce^O#-))}W{&zwJpr;A6r*C;b{{+(KX_MJ3uZD;t75p@lfVXeG* z=IC53mYzI#Jr(bfjrmM_>2X>+Z==YVg|%bL!^rR@s0Y89$sq-w+emFDkDKDv#C0ji zhFPW0y2KC4(K31;Y1g;;2v_oX1NyZhYYMXC<_y(a>x_x-uG}GOrW{qXqMoWm3v1t)8BR&zYWzVckPk zoP+$%T?UqpmoAI9W)gAD5?@E9I(U6k4gMJrwOH7XCBZ$hEuREszt5-1s<1fEb%JcwA=LofbuqEm^@-$_p@l>a__sx(kwe;Ia z!%u{ezXng0F2p#xPLX_Wp#w>wz3Eg4p2~I8YN&V^`a=u zInPqGliKBaC(gK&4Qq3{?apxVEkAeU=_Y;7b@q84{xPQJ?N(f3qVwpz=AFqp=sHXS z>$-1$Z9ji*ckdav=c_6;8i6^lTB`*OXY5F-DD!i)5%WwkS=oBWXqK~Z^ZXt5suW0J z5biM3Cn{VK4S%bq-wiIt}VDFjCo#@B?d?NI_C-Cbl%d&Xw3lSqgz)n=( z4^Wf2Nd6Q$mYD_$1Xjp*iP0D0HmqVNkKT*1n|Hx8VTqY1n@;ajDoO7;sz&OQ}{qdi!UEBk?+GidkaxY=I#H1W$L z86LUbbg{>+(qrZFMtFmL;J*SG~;CovrairO&;3IveCW z!REexYgdBnp30}Q;6YERJ=4B=o6qSuXs2a-E%_Rr_ArJdxtdSWRKi@nr0NH*FT1oK zVlI>6nJNqNGTH*ij{Jk+RKp@c8PAdT(k+ zD!NNa?@UkZn{pO9*6{gF7p~N^n~E^~Y#KELnmfXHIp*l9Sufq3y5=G~&8SY!>2_MX z$A?a=`I%oKSP#3j6??rcI#reuW1a+TEDPmyzZWC4)%9$1TCa$o)1e6XLuEF3!-j;2 zpY=6Q_E)cb!UJA!#Vc~P>s|?ex6KQ>WY}Gh3^o)3j=B)Dpg(AKuuDP1(WVqUjP*#Y)^h*U-=DZ9&X}TU za@tIHOQzHFSeV1}%?Ha2$=PBa9^ZWQg4cxSTd&$WmM>v_tKx+8uTDVpYgz|)fy{So z(VB`ZY0-jG?VC?ufcB)irTnm!k>EW=fLKtHQt+U*73*lR%x8vZW{uomnA;Kkb&1xd)#yw`(tH zr%xsi+!{S8Ch4Sq=;EsQLwaGPXzu=t5gUTY@{DLcgxe)lQCM_SgRJ3M{8g9h3@=qo zJcXRWVmfhR8A@iOa{g+ik$Ac=0uDy`b&oaU-p=||wC2Fw%M1p-Jb@uUD@=xY)8i#$ za!%i6EDeV0)xWuU!vgVE#(R1NMDDndrDdwb zvJq}Ge1u_{F=<+S{Ip~!*7!K)+l*!6uLh}>3G3ANtX&ytGEmKu3k;L+$~$*RFuX! zxN_tg3DQSxA+=6g5f~Sl^0VmGdNU1gk55ay|Vv`z{cKW@X?xb7q{qqsY&omyd z;u)WLc@s(E;WOa*nn88DM@D6zxa#q`oj*s`sp;()qM4)~5*?DT*Hu595Kcd5Tt(T9r-yMJg0mL(X#?SITs&8 z+i-d}vDxXD~{`n%e6c+C3qTd~jd%kDbXdPA4+)O|LI&DP)NJ0rE$t6%S}r zNAmbcp)<>$))E}G9kTF-5z%T7Z?h~!=9El{t?f^M@;9B)bqNd+wHD@ z%)AH7-5>7jKGNRXAJX+|N&XSqEUjtWmUw)ad6_%qb{=A7aHgIM`)ue%H#H?&?<+}A zag<*sPOuGW)^CaPkS>6~^_TOxSW0fV7q)$^KHFtljz?e!(HSk2f)rw-J2sEBP^KPfXf>{d`IYnyKj{tnbFV-yiano(%ulfp--K`Gt9i zG$jciYU3v3xZH?9WS6U`z8GjXI@3M*g*{u=%WNiBpE@VZZ#g%%*Vlx**Fm*mkv{5) zrA|7BS$Y$N^yT!q-6%qJnwHrk|5Ln6@Y2WrnwiH&ld|-o){v&OLaYaSO+7GwySFac zjLSzRKK<~#p7%k<&_1|M8Rz`Q>^g2!U4|*iL&u}rB{XG!#br+fe>!6y zjh~UOlLIN-P4?M{wx1iFn-w`Hx8!j&oMHQy=o4hIMKjuEc2<5T(nSll=F@y+mzibN z2*-S;!ZTzX>TsgdXzLWIt8dUzF`md;Uy{m+dMT-W_ZI!~dTXj=H%yRj%4dG6?p0^7 zF*#0IQ&*j6EZx~9U$dlt4Bn80hITxpbXC0~QLy5Aqg-FOzV8bCywS<^@{DJlWwhm< zJ(fC@PD?{t_ArE0n!)@;tJ;3nf*sh`*I>N#C~2Bg+hN6XWgo?F#=#!CKf-RE>P)bygs zu0Csml)UM;W5=t1!V+MRyTGvXkO`%mGRwYNSY?#S^s$7JNp^uBIMe5;$2Hru&U(+D zF9lj-kl*)>R?j89S zf%brdp~o+UU##cJI-PDovOkMM52t+wORM^NP6(+N&nb9+>S-?J#;X8PG<05g%NUPg zz597D8v3=R=VdW3cO`JQ#PuK9Y*=?Z>}X2LS5q}UU4D1jo6ZH7irYiol;o`rLTx0D zB-7sZg)tfWM)VncKkamVrnGM)#Vk&+8v-$0gXj@nbK!N>J5lJ}@U0wde!E|(GGB`> zRog{SE@3Tywxe#JPSo-8FX)}R`kZ=t&cz6OsPpylx&kKQ*E?-PGpptBX}BKqR$%u^ z-j#hBuNO*PK7oIMedm45yW%pSyfqB=zCL>torMnyL7znDcjk;-k*2ClQIV32J4_vM+51zdz4% z?OdyAh+*Q=_WMHG>MdA8GZOA)aB1+;QTO!j*mICiq}|}N^=iEWfb{-2wR@DSkC9#pn58Ww`hI2b{4r{$3l(6Dtm+kGnUqvaNiyqigOr;j z96xcqmcAxz8Y|>gpcU4JCCiI@6%WCq>Jz@_>Sp-%jhXIm-UMn*-?Heo7E+)+KBrVA zlbe}Lq)lrl^LZy=7Z;0p#1^)2SO{GL9;M%ZvCuLP_b^fkzf7V~yXoc=P4B!QG(Ycp zd-M;V_Ft^TKii@q^P3rkuTQe-zK~njgjSP_+l5ybHd%p#cZ1#fDaL)`YJFJx@;DoK zbmpOWl2?l~28Mj@2RSJD3!ZVYu(Rd!QHF?_TL3T2mHmWj%8OI?JXGKAVQ@TY@%i_*K5PBD3HtosSO_f-kZ==exx z{(b`Q*^#AH=lb5HaykasN8Qrw$<4igUTWux{XgmYkEQ2fty>qn7x!Fr6?1Bw26N7t zJ?5Onc=fLLTWkM&+s9Tzlt2Q6gsBoHiDwM$RL4>ixdDo&LbjKCj7{nu9E}v_cAPEg zm_k)mD+yy&PV2+ylAcWq1v9g~5?_PteWqY5wWqxwn}2$Ja#MjZ?7_e|WVXD21u;mr z?tdR=QvGL(GllsDu*nfXTS$d}ky-L&mv?O*)H4mxHiEF2i*v=@2T=KCKMPF0>*`H` zPK;hsB_St+a)(Z|H-GX-w=@f>o;D8UeOX($`>e3R2rSM|~hQIX$rX-^Y zk5R(DDt_v5jT>OejS5@sMFk6Ps^kj~wZh~IvDC3;FbwMFX+RKHesaXJmF?F_^_6&4 zQuM2)3+o%NJ5@t@^|RwEmN5WXKjWpjMKn4ict_zGnnS!S2N;&|V_`&p{0H7!ZQ2EK zM47w*eqjMFAwa(`3~#B#f%g2yS+P)HPJfIh^wn>&0>bkqOt5#cCC(T&0HU ztLn1UxZ|Caso$Dh;AZc5mSy&JZ92$J<+d~m_cs0zO9jk;gKBrXr<#CzubWpPa5D`F zL-6~_;90+@hpH^tQvwZtyu|p*51+Iw%_f4AatF|&xERzCFly5Os8aHO!N=>X?N;xes*FuXbK;CRS8) zbo|l9gGBAG%CH{|(#Y?6oEj_#Rvs`wkC~sCy@{YtErGY(42&i;Q8vq8BqgZA8fAnp z&BvfJcA zK`WBFH7MTpb2^z2s044w++~i~8%O*H(PrzBK&Zimsf=D=gu+5ccIHLk_r0MC3oFY9 z*_x`3k+yijls{u+9jO%7Z#%IFYVM25HGn0XRbGzK6po7$|p zk#AZ#a)uRu(+p1UHIFe_`UDd_`&(M*V8D&G%H4}`$)cLm_Gh2v*&I&@+&tmXp3a-j zLCrrBk?0V(v4F&#pyyieJH~db%!b>gGipofm4Et}oa+PtiRTM!?1hC%ldi%jBi)?p zm=t7NeO*ZG(aG~d^od3pZSAA(>+f$#9uBOxAz4v>rd?!N^V~i+&ML1ZSU}ycGb%GG zD|rK?jHVx#Ul8?Wh*RsTZDB6wE<8$_L$;=$mvxrgA$&gJz%-9vlz9p|Z5cEMP!?Z& z%K=(oLLlnw1C8L-=}kpjTQW_WAUbnjI+h!wAJALm+TaoK=7oR22*qFQXkiVRv9po} zlk>fQ*_agQ!qq2O0lI6%F_Yn!v%bSa^@~OM1~mFBD=h2qhj(lrDnTmKPX!*yrfVQ( z8hnPjdk=3zh{*TOWp%N&u9PPdo$oMEm1i?$-RSRYg(Z9&bpZ~A_J81HMawhKoa3*8%Kb(nw)BG=&mH&T)i5O2{vo_> zldiX9{*r$>^WZ;(%KVRt4b$Dik)SE`SExe&P^%|PKctuY-;#<>iadDw&ppZc^MfJ(^pjIYP;M?=(n$QXIjsJqBpE(` z`FlnZaOJwGUD8Nk7?%Fmp8QV`(ouw4{$&yjRexpw5aa()UbWa=Y#<&vV*d5FO8yU# z)p6V!FM`|ZPr9AO|F_O9$1vaDec4I;r4GZW$Um2r`mbgu$$E{scaEBW(vmCxUkO}> zVHc~|J{x7<^zZxQvVVio|8t+KCs#v%s>)p9stGX(nvh?$Mr9a$TN(V^<@k9|mLUoD zd_1ofs>XiSeN*s}#8Nt?Ka!R|<1;=B{C&QPBHKQo9Z-ifz6ifCo52~C?Q0JW%h8yW zV_bkA6jqiv)-isIR7BAbodBMslK?3fh!}qD;|#NpI!eiP6H=FHF`Dp6 z{Jh6g>C;Wf;x!LUr}iHmv#i8_@*Vrl;P|u&8MB(|+y`*H0rMzYX9p}Ra18eAg?;nm zYwJ5Kp)z)Am$h|@(B}+*#{8_2lDDb=vKy%AQvwLp?!|g1DOIFZ*We#vK4z+S72ovS z^H6SK2_CkEUcTOqERjdVcL-9|avFlx*0s!C+w!Gfc;PX9#OlmYj~i-#2ox(AWiUD` zt4tU%O;!@6fh`rn()f3FUAQEuKbbH<>XWHB4WwKB)>mUKv}A-nZEAa$_R9)2eIJNG zX9ZFenez^hb0kK+#^Rd&sIZc*M!>|2HN2}nM_)!inQ;end;6TFcYdeCy&6AQ$_=;> zzX5<-0tChn8};reQ&3)i_*x}*Umm%LplDn}(JoIbV%aZ)ySsXovu7}V1P3Blq17Zo zL#9&EC3!Tc9G4wHUF1t{zx_1Nm(+@rxI&QN7-p;7LlkK&Gd305k9`cqH`an1_WYGo zYJ1~@+aH0X`^7K)`%W}saoX$CJR%MKK)v02*47xlaRtBz`g{X_l<_6!veH`HQ^od9 zTL$EW&(tFQ6$lIh;qmH=JOafy2GB9Ww5}pz9kg?}Id27iZDA{P;!-i8aVyp5QwvrL zCB~>WCpCq^v#0ENl(@1vujQ+t_Pe6h+bsX-9H#6Rny zB$G9c(Za8<(giBncaw=gGe1tGjG2_&$*1vmi5KmXBNOt1&WIK*wN`K)V(#KjeHvTz@~w? zb->o(n1UyNkkK)Cd63f^d!ekEeXd_KXy&!&X^YJT50xD^btt5>ACt} zXXW#M2BBB3qcr;&Av7Td)#7p`rEo^VPPBk(nIvKty3su=h-T6-fPfoo2*uC}2&)sr zkPcuCp1c<_OZ8kUfrx9p)XT86cKWz6X9SF5kSu!PZeSnO5k?D8ZYg;%0%uSKP?ndd z-+bC%o9iK@*{qat@%!!e^%kQ$BIH<2rzH=6C-t@s-)vWu@SBaX_T{aTADk_=nhyN? zrZXZV2q(8ow2Cd!$-?hgLT(=cblS{PWK*U_A3IFrx?emPOBwfrgZ@pO%@A-0S9vz2 zG-R4YqelQClLg2Q21dlZ^C|28b8lWt>(rDS@0|Wj1{PWXvT$7qd6y~z_%w4{nxxcBYlWp!^Pv9F%XvM{WdNr&cd znaSRe8J@ZlNm#Wr0WNcn_!;@^BWBEU)i%K!$5(QVz zohFN2oZZ&8CCP=_mf_e91H07+x%yUrL1}Hwn(J>ZENU6N?g$|_zApFCs0~Hv(q6LmDBlAnvV)jhf_@>_Ms6!a@VM z99+Gn*5Uj53MWSX0RKiKYKc1i6~LSlG5RP%;yjCID|-mwCgyu=mjQO#rk%@w!9}!| z_9BySG}d}qWF8d|*c~0zVe#G?bRU_*0Zd+OzxEdsB!n4g4Z(|gI7ZvF7tN+FDWM&T z)S8NyZ$jqoyH-G$5XAl@z+$xb@v9V_tSqhKc$yFgu{MI312!sv zWH6VbObFEXXS;KeoBaVeE+N|M*rGcFKssa%=2i48rAIJ`3&ZOXbC{xkI0MG{Bc*Je zTP5;SM;hPg!0^4r!(IHMqHC3W10m*5A&cWZl|MCi95_FYw-T0t>HV_=*#NM@p*$oE zA`7v^h2E#}O^mI;{oga_p5QUZ-Y19Mt~x)KKg@97xQ(~r>GLT%GAaHobkTA5a#He& zBpb=xzUFy*-uM?(H1z&|tHeP07(v-=aJ{kAF@krgJsKtt>F#@$7urG*Y&H?PyXv0Y zNRp3xMpjazNoxAuf&-5+C~uH{o@z_$C>3#w=ZY{7+b;MAh4@+| z^5gldcpYRMQ>{KCBF@@;v;b6g%D#YSqsHU(U=a8ushz)qmao@;!xMcr<@3k*0xw2k z>CXzk#wlsg{I0?y85r0Zl8^N_{VjLT3g5m6Co;}%N*cvST}|B1UmoudI{iklS!dUV z>X}t3tl1RS?iD^% zKS?Cr@5erW@%z@NmB>8+7!%qG^`Tqf@FwCbsG7F59GmxzPJaAFZB411CZ$(etq^U2(FRn<`1~_ zm6R2KhRPUCIs^B11xAnpVZmMu#KgWTiC-?518f>wzn2sw9`>bg3~YJ@1Kz*7H{uNa z`YTz~YszHx_@VJm3~tt3er4l8I!?X%)U!Qd3=Jgh{cRdvAcD&H^V5b3_1QzfYOWQga$dVSmcC!m4jVM?+bJ|dsNa>1%HM}I1Yj!s3gBHj!0<{SuUpO6 zkEf<6o+(PWe!E@36o%K4jKl@*08}NQK!H@*#6T?JUyh&C>!5ey#_T@SAC3#@yscN|=fr+tQ zCUWVg=_b#i_VyEL#LkrbWvPM4GY8K3b6&i2*Cn+KP;ZN5E3!e%y*w~_8%ZbTj^F`> zrJg|YxPojZs_N;AB^TZ8g$YK9s*sz1hU*rja6(6?{ioj5=GMt1mT(^+Ct`g7qWt=%Oo=#xba0l|+D{k~kAIN`L%Vv< zG&y?`0mf98&;i`hwQ1J1WpiiLL03-hd={UGIOTbd*$RT-1p@XTUdkTI8~zF11IdR{ z*hWy@c_E@T&yC1y230!ZH(=R+H{I9ad7HQpYyLieFXMz4awiCV*$iA^ykB779*viP zisM-!7i+Wc3Is1zzModhlj@roxCpo`__DsXvm9igJ1h?K=u}W6WTtBzTlaRH zs~eFy&{q~Fq+93TDCAL+woW>!)ib1{1yu_{GiK3 zw~F_P@AN7<>b@+C4$vZhyY8qj8G8J{QaYKD+nt82T~G>hKBFhKNIew6)k=84TC+UP zW|heAu~&4@?38OZfWx6n`bX)NKCliBKKy~P@op6P&R882o#$MKARdv_0kA_*q5W38_9 zs?sH{ylH7-gW=kHp@h`^y8GJw`RNDT+?M*hx|(0|7Q<=~Zy8zBq{HOwnB-K>1L?XVRqhV!~H_PJVO9Q7Bq#_M_$ZKwDRecY(;-sm0z=J(q9sR zmHk`ekZ(?#Cl&+D0Py}YxI3?Bm2?4`1dkE2mb(5+3$DcJHA6hmY9)#J5D4z zGzw|L9=+FXA6wV42LAfHYpJq65UR)}q~yERr+#Jm$bw{+LGUA&;bTbNfGcDqQf}RU zS?EK6T#GG`p{DMOwbxRMU?%(9f5hK6RV~d@r2uG&=Nq}dq3s~f%ALE zN(AStNW*aeTQM}G$}dH7CT6Sd=tHZmCSplGE7^9Gq9SvvSk`P^*k_#iRtz zejNV%R=1Jx#TSwK{MXfe$fQ?oy#tx$Bfq|ZM8KZ z)Lz$NGUxAhZW;DY;KJDwT_^}*K>~b;Ms$3~ilEN_#&_q>x&+^k$=B3^rnwBTDhvbzqYfe>Z&`137t)UKaRjL34(8# ztm1pj0?6_vW`+%`8K>~0)P!0cE7AR~8Ge;3EBD-wC}bpIqaaS0f=JfJty@O+Vv&=U z07J8vAT+^FU-e)L926jS?uSgIabMc^SDzMn({CpVTxXb{Py3m)PPyg%Pc^tD%(!_2uwqOG4A94DW>pt-qZDpo#%N`a} zXh|FacfxEMLep!{r@&rD%n&eo09hIv`?JW;oDwH8;@KQDmo?|$FIe$j9*Tn}S#8p6L zh}c3}6}fT0GA3?D_9V0ZBW*12JA22_3R%x9Su`QqJBtr#YX<4{w;OG{#2DhqY1x=aA$sc73O!~G zwfqDjVuk3L-*}dP&?*E{;KuAO6s1~bj4Z@7sOGm5WLeHIzIFrq46RNj-IfB`n-1PI z>%to}o>-#t@2uzJp<#+%JDngPlc$S4d<(LWW{k<3p0q!nwixEHjBGb!s2o{vKuY#35&){#AZhY6R4npAK!~E`9o13IM#g9vyhu+De>cOzM_-KqU5(~$tRgR4c3Ne z-s#jRR9sZAR;Bwaw}W}gbnptIO28!kCa}hf#GQd4Xycgq4m(Huj^zTs42Z&v_0yyU zaoLa!tZ5y8o~NLP!bR!WI~DYF7sIqAdk;`6zVXM=F}J@y{3kbrVy{BuokpWjOzg@9 z-Ki#*I<;+_Ls9}vxAA%BkcB}A(Cry!lfPJkqhO!Ukn;Dthwa3_7kIFoLNKgE;c$JK zlC>doqf?l?_li87kf}q^cgBcL(mhsgz2mfIoe zU0zSeHdr7~U)3kC5%?=ru0_3})xtzAL6I^_v&k=iM=^J_4Yl2b;`P`QUPjHq{(2jw4>yxpH@DT1$ zA*HSqDn%clGMkLFCWD<+&fw6!vB$A?bt0}b@8T*R=GWmTK_NWYSl>IqBew$RCF{ro z$vde3%=~O58#Le45;USMd&4gx%v2RH>*q*+%uQlFpnM8e!}$TOTwvJcCBC!x9(o2c zN$w+UKJfJ1jkkU9iICL~d~ZkP?s8{+dxx`#8&XkdqJw!K1wVhj4!y;>-PmHluO%`E) zFvrRCksxk3N0XJlL@>WRDp~S_2nxX|AHjs#&HW45z7N2ReCv;E>Ifb@PvJAtya00% zJ-v~Bgc3Z<{e20RA|PwzA_uK)o-V8J<0V7VWuSiruPE}vF(;I3Q%!?^Srw#D9u16e z+uWyhptr0ZdWlXya*U>H_m^jAXps=QttkB4S-V1!$OGq+3eQx0Xcp*qk`3h-&G+p% zb#QLy-Lct_f&i1S%YaXVc=a)V4sM$4rynLYb1jD6o*c2s?X-o=#>;|z{A?KCa@&}S)28HKK&syj%T07+0^ytuF=|m8KZ%VdMkjJKNjjV zI)a9Xu@Njx;0)oG=_?uq7211&1Rc^swF3V{%z_HF7V+*N- zdq+VBsDnO=yf3mmTgBsyOCBdVxsogl7*m(Xz9aKr*JJtSga5sV@c+a8z54&{{?6}A zt#G?z{#YIo%_dRVhmm)urMY;~y!+5Nj54eniuuy8h#Rnf1^#d-R#Jt5@}1UEL|+O2 zM>pGH7!({DB&2NH4jBIa=PLbwf7l=Z9*KZBdi~#}d;i-F+Ft%#3;r$k^Z&<~+UOb1 z;QK#=`*mTY#*i36;EX>P0F+|XTkRO;=HEX-`gdTg+bJ0F=7&F_dt_nw#=#gC`?8DdIyJPg@ujXN-X27>B9MwNE4^O4Es6v3DY}O=NI?7xhCpCE0g7(_-c}nfke3FZCBy zWt~J~WG`yy_1{4MJ0IIHlR7DE!Vb&ROiRo%Li;hCsQ8P}c9YD4Al;0erO-9_fRqgq z7`rVF!w{gMA90`UVjfq&pTW1tc7|0bY8f*X{ca|e2GNlIkPd>@G5Ko9pjIV7zq@_Q zP#Rc&BMKmmKBy`Qq`%NmjQ8ee1`@QEF(QDx113ia1*#x4GIx5zK-*Vu>s}~_1kYrH z3J`!IFw^6qiN;i8lpvaLsO3b6;MYY{{CmL+T=n49O4}H+iDHL6aU-QealTR&Yd0Tt zJEOk!3IOKDmB^abKB<1biq|^ii;G+mPh=i{>F(~QoN;5x|HcKnkuK78`@RV?F=*Ml zm(Tnquusl6zDKcB_{hS82&HqdNRbegDfGBhWB=9neA9?MD?>0EK>PZRc$jX?+}3sjYU)R$BmPRrIWyD% z^DNkzB!m>|E^EQ-U1z#;DrXrdEJ&|swWZ-P*1UD+OVsjFwg(R8M06J)MxTDkW=@2w zm6HzGFd|&57Nu?;U*pQPJd#-D1>q^IS2LpS?0fGMUoLI|mFrj`EJfbif`O}DEEUns zIM&_cNhEjgj5HpKmOz#plAkBSnUhQE1Ck*{AzI6#5JE01bGx`8+DF!D_;Cv)uY@>R zP%J@$EWcaP^TWMNq6PmGhQP+(LVBy+{#}Jo-BHdq`qb5Sxa_jaCki|+_?tc|$k zHhBksRi;k4Rp@c9xo757O1EhN5L-+5h^T&3N|y>nrcSBZLMUOW)tirxHJ~w*u+@vc z`!leumi%5JH@kmAMtI3}UA{@>!DI)WTdEO1UQQoMby>hFJH@pwsACH8SP}5O9*4F% zy|-$`1281?9m7$(OH%U}zK2K#S94KFc)A6Db-au5OX7zqAe|WMHX$cjepjilB3XUO zuMJGp)6pv0+|7(yCV+xr0`=`{&v(m=o2MH4$2s{_TZXoT$=tF*IARTbYh}F6`7zyv zEGh_AKF+NiPwY?FG@zN}&q**5b_6nmO05}v0r;&1cl{I-w^#i5#7u*4ofKq>U-G&V zHGS|G2uoL!JM242`0Ik8{27-x1P2^j>3Ig$y{_gxl0q*BovPpFc0~IL@l23CuJ))Q z@*N4E4DX3sMZ`ukF4KgkuWoEH?3?|tmZ&tQ(8Z?0+atD-ADr_WK8IFan1Gq(fTI~D zPWKv#R}~?c{K2P}uLK7ye`f&a&`ocPvd27ob$i<$s%Q~;RvC!`YtadtjP#9~@Zh`; zq@Ok^^d0F-TC%vq1(!3(c1e9@X#5FNJ^mX(836>PX+)OBnTWRvb=bwz(@aF^r;)J(?7Pe;%-*T;=N5Bt$l$ zQEc2%8ExG~ENzj(1VKS_uTZ^mlF3(pg>a-c( zS6Zofwnj(1pF>K6f1vY*bx4|XFq@y3UGHQ=Rxku#woRRh}*3Gvw=ML181|j;eMM_tm)CsniFCAc^uV-sE`Qzv4Zja0C zwE#arz`qnD&3_sUTeA4@JVG=qS!ZUK>nCClfR92TBNE`&e$OHJw135<)x~vrXuX$M zO7l*$9}TqlLlNooP)-8?3scuYCVSWPd}$>}kN;3aoQ znX3pI?7}qxp;?n@sl_wYZRy4WRkNuV~Q=n69N}5}9`qh>U5#8r=Nr|IOr3 zV{)03tZW-dB2n%j54|1FJPgny3fr*O#$@n+=Zjq;@lDFLU4J5_O==0k(H5+c_~y$o zEH222&pZ^bA5G5j>m4)PJEbioF+roNsg&M{p0S77xlg<$#?vj!gn?{p5AGsH@2e^~ zr|3%z9Kp{w&Bzgj}R}zh;8OmK^JzH#}l!}BEpLaT-nB=j$ z0Kt0G;^0zL4y!alKrQ1@J8z#SOYA#n)r}v_>|sV4#Ppf~mWuiS4Z}X_-AkZB>Vszc zGu>d~I;fyPyUYTIp4J{3tFsUZ)ea z!@H@?gIFy75D7Vt0YywjPT^GG=X^k5hc`a9m@rT-Ff|8iXRKiMwR&bd5J zjMFpfU&doF_rer+PT-m3 zHNZugGd8dVR@r#R@Oic|SVoY<=6@&oE`uoUC~7X|bIw5ckHWKV6S7P|$)BUJjlc^Z zD?4Mzn@pp1SfTI@#xjl~Ga$KuM-y_c@#7pN;O>SNScm7Bgku^(WN*x3o55$9)evtg z7F?y{nF_rI}~#Wo23jbUeW|KjXFaTJP7A^p$T^Z#NIHo5c4nNKI4DI%74FY0jY1&E6sn~Z3;?ubuB-`(DB_BE@pSvG-eHvCg8gsHyM{)Ui$?GIwHg_c3#9 z>+>GPiSU~Hp7B%Nl<`~U|IORGHmR#L`-0zP-c58&h)TD*&zqBV0e_0C1p?xm6N+|$ zXrrQC;KY3Q6NUEWzt(!5y(f0W>4*-2N{YH<<>i-2VPlAc6jy;?u|bS#O75aFT8|o9 zb>J)Zf?7D38dN5R$mp8H0v{Yv1%XywxzSq7q4e0okDbW&E8^v@5SZVq*65~p#Mx4P zj1YHOY9C^hw)B&r_C9PAizB&cnW zxUrE)SVqBR zo$Z)xipDX)j_2!g@)es#w&_3B=eGo=sqynbq`&!)#!Pykg62Mv;v} zK`ny_m+Jh@I$WoIL6H{aKDoE6dECq~=YgKk63(2Mz<&_5GKs<|#@t^4pT*H@HV4jx z6=rJ(hh-pj+n~*Bv|yDjo7B^?3j(n)gy-3JzNH{7B$-EGy0l96G3O0&r5>VAd-XKQ zpgItSu1-<_3JU$~r*vpW)SyZJwkl!-cjL{IK!92Uo%@-RSDE@QE@OZ%+v#p;d@xxn zd9~=roqv4Eie5(DAF_b=@&y&~m%Mt;(~mB*yRUoxkTu7j`O2&(AC3L}Fa7n zUSva`Kj}ptt2R7k6EgPR-B#`Byb=u>ZQe}_Z@4h>IaV7F7l*=_lbkpwJ%6T0JF&PvA;LgddS%o$V=C3ROC<$a z8AxW4pz%9w86tSL2?=r*MHL&JcPR|e1{>zEw&8||EetV%5Qg5Ob^Pc=Owry=s7#Q0 zfVt0<5H4%Jc?1#5q#iy3Er_dtDM4=Okj0cjML1~jva05w( zfPZ@y6sw>@y5I0TCZg%yM=(kVV9fIk8RY~lbgMc`>~srtjvx~=1ZfUfD-HVUOc(5`QBwcS(5E%$uh0MA?mQ;fqJgD?YHG%=>bH-D42 zAB4wEj2N?uI_CvT5$F!)%Jym!RxrnqQ`bdaMA~d_N@~0K8av1{p!FCI`O*+^W*p;B ztEe2ZpYEuZZ;@2VZ5k%jI}=VZ79zFFS};*zyYnb+!Hdkib3?wi!rfwC00ORfo%Y`) zwg-b8W;z~A7^shSUjUrzTX{GfvwyJ*p5mA8OAb73+T-{9f7vOIxjjB*{PMSb^JVuw z=U2_2nq%|_Z>afKcExX-=*tHA;+=c;!yAh3?v3V>J#|WKe(NTY2k`aWQq}q3Q`WIB zUh1CqH+tIX783K2ggin(ZL_Q1=d?NDlFxi|2#ic;aVQrcDK^2Q7}bcBvw!9G&>$S~ zx6dTAXs9KQ=1mzU(+|mOdy$=-eTeODYN0YO!!S3g)J)Qzv0p%B?5huRk=-z41X@}< zIB!5SYePuWDW?HJQ-q6nOq6ZRJ?vnPPzxc*_=7HpyFyyYjrw4Q&BZ~2!&!y|)cUVN zs2dA~a;^G;Q7w>Qk$r?u7JtcMfW-!ij6k7eN)|bwoAd6(7F;)-JMuPCeSBMNO(E<- zE&|j}IcaMVbu)@Z<`}49NO&e`tE-YrW``0#Y~CypQ!FZqLM{WcEfHrFL^UxXdnbJQ zZa|D6(v+MGFvLq!eG59pXH(=ScL4{3vx5;<2tl(zP-Q9-Oy+|P+|ETW zAY#Xf;s8g}2}mzxi7r`~Rgtm9gbN2FSO*oHQ7dpzOm|)@mzEXV2(9zHh|ydyjKk@{ zTgs#Q3ey~PX123Yf~yp`r+l>>N7);EOczja27=Em%nBOCQ-9uhEf1#~Qn1(qEi+2C zL4uh`au*dm1CcF_d+XYQtM7roBgw9|6x731+Ffe|lraQNK*H<${06M{G=ZMLMMF)`#R-e)%| zG-gE^$I{u`q7EZV*FaRZDR+zc-9skxDmRT|-nGvxqlztQ}C;m7Mo> z9Wo80&wL36gHM=v_|8!TrU2|FA|aludUr!%DEq2w8cX{dH0{S@r_CS*l$l6t9Z%!NI7#;c90^fJdn!AXPZ2hM+apArb3#6 zQIOsWWK2NDfoZ`T;o`81!)$Rk91L`5KlsDPkR!9uh6Dj$4EdI}5(OcUc-c%Wl9^Bn z7k^|$?HFF|a#L7`+^6L95{1)>@>n^%agy@;lJFH21!7U1;#Lq4HKT&?&yJ z_EuNFGvKyE8a0;`1hWjG&r!Ha&9*LJRR?BSPpu;rsD)1hI3gl@TUGi;kCjT4s|=ux zk}qZqJHQ^mc`U}K`$`iC?(71KFaP#S7k?>CP{-c4kJMu-y`A)sK@Z!TNESd5$Nd=J zf5{=)>#;9=^OPlhJqnK;_O{r!98mLg z`g0KCYLVBwH>#IptP_OeD?!b`G28d|g@++QA3@|J0@*ti-eEyrrMxQJ!Du}ULarEMG2I&_k)DrfJVla4 z4pQU7P1>>##$#fz*ImFh9ERLtRezSa5zGx4t8@?yTzhj0y^I(2!7G}*C)kRW+BPfe zgm!sMxc9mfqT{rgj>3@|^AIN(8 zINFmRzHN`c;3~cD`C|?DeDqyM%)0F3i#MM5@-LsQzuwD^UuDwo%X+_U{C~wiN+xigM4fBX482;8Ely7__A$G)zN|jP?~7!edsKM;OUhU*tUp=E%?qPYjC_Fcg-!8yTgwIom>hlkllJ0S;G3)0cg-mYo$oyGIHSH6l(lG_%F zDFW$I&^+9V91F?lqKbff6ouNs7csKJDl?UD5HV_k!g!}6M^HqiAb*kXDNGMpf}3c6 zp1V}(=NuU#N3eiBDg~RH;vI>m^oagxg_RD)HX|aVxXm{)G-Q6y?moH#5?(_jb(y67 zev0IC@QI*N`lbU$)eeHrRX@|+IBVd`@B5iwi-~W)>x(zXzp0UcJY9QxsC9c zfIK>jg_y(pT|+1{-7xn+tdLz)8Z_G3F143zQ7tKOKM%8s2X%rA@rHS3V0^VHmk5Qr zjY8=vjG2kbbU`+4Fk`SjFrqpP*%dK@}1RIJ1i)17M{H% z*}z0$WVB#Hdw+SE%9z-(KAnr@OcON5B36EZ*JtXY$>1&)7_JnA0J9ltb6Bv8-~Kh; zpniJ{=P5UyIPs4h8hyz+^oPHe$IsZ?crH|o&Ff=if58yOPg?vJ+k5=0o&dgh@!Qsk zV^sQ_lzEEot-&h8vbFVoQ+WavF+xz|96YSI`Os`DjDI3FI#l*1Mwx@1hF~PfoM?e_ zXERPD0a@*Nq2(YsLOf#>vXepC$S6YMFZip?wXG{=>^0IY9L2Avg-(If51NQilg@~P z^y567y2f5h(VZ1E%uhx+4MDtGY-nT-P2S>ronyfa!UGCo{&MLI&nu_IS=`!%F|?`F z+#|0o(0>lTLBn!;c+Ff{8*@}2mHq^gVYHx&d6C<(cA|?>0t7WJHI^^aHYZTF>*Tgd z5NF8ZL(?k`yG18*t78@vQ->6~Mq3)2vX373e$|b;OE4-x7w=9uRq`Qz$cLZ#SMzQ_ zmLU0MLq6xFWDnbP(PM|2|5LXAi!AtO9VnsqbANiDO}JAE%4bMd>uTi@@Jx_Q*(lr$ zw~O@k^@@dunP3!0N>(&3v8VUKoURRV-5cC92-cqdvgBw{pA9(KX?gG=(hMkoBhj=V zXf9B5vV?Bod2tO(&j6ufN;L(^89|`G&n8Pl#ZE;QMnk%^E3r-Y`kdZiTI1NB^U5Oy zD}UTJD@~a%twLgY&@LNpUO_SwAtUVZT9XW3z&j|s){;n5_JSMH7#{Y*FPl=ueZTQq zwKk5weNf~6?f)s-zsCc6{I$pL9<9}%psi(fMFi+wP^EzNyvq3ZgNIf#z zm??jg+%+FZ#?sm6IsuL$Qu;izjKwV8U{7E^z0G+|5X#Of_9HueV?*ZIah4#S313S+ z&!_|7ezZVft*w_^(xLMLc^)D5Y4@s6-Fr@O2bW}AcqXqfaOzMYl*n@l&m0WG0Dmc3 zkXO%-SGNU<5eVG`QSVm6;Uo%k8_MPbOEv$E(hI#Gmwpa52y%P}?1Nn^+-k8OoowaD zfx;bSrebXY3LiMY-(n|!kE^NQul139`v!lpxq81r>CA7P`n1=+Fw>si`@#?Yj34)S zsCsUup8LdOvu}UW@A~z;^u90!aDUz?XBf3!q%Z>IA7Cks-Ja`)Y{rLSvD#3vab#|= zLU##{YqzU)s_YiQHXtg6Cw*3Xonx%sQHMr*ho-)qea?CAy3(@{cnB352F`0QGIS0i zqf$Hg$w$02MFKM3&O^F%!fFAMY4+mGBa&ZfDJ%rs3Onw;fQPeGAiTeZEPt#JY^KDN zj2M#H2yz1h53oF!!M$1Fdwy^YZxPp|>%s6Y(izMq39I zwXo$oQgMnF3r(|Iq42EH;D2RGw!OcpKjXH^XN=>&;90#){gS_bVDG7T0gDLM z8R2(qM%i)-M~eTD%YWkW4>{E4{}k`FkmzHEUzq+2Jldt=Jx7R7EY)a@B@YyCtn$PN z*FB7PiQ5e$lUGTQ6=BRp7>z_ic(c=3Z6MB(M{A#y+qngK2*UGFG=EJ)q9s%e?KET> ztybUE(TYtU}Gj5Q0h}vNr?LBAEXbCBleUoD>Dg>gAK{5=m7z{oKpD@l9 z?wx`o;3m}G-$THcZqmG+%VT}85K*?*FAS&nL(Y%w@^A9wS9v!6$@}= zpK-n1`Xlzz=dk2YJpBjfe)s@SxzWptXS}bMJ70d{7hgVkQh!j*?((N@%eyZ-&==nL z+c$W~y0NF)E%LD^@)GG#n@R@&LqSw;*bwO=g>lHI*d*aPq&p*SwF?pkHi|5Bq1xMp zipkNvl0T@*c2o;3#OtdMY#tw0)8#N6hC;g{5o01X#)RjxC^2!%6A+#SAutbT78Sw( zt}To*R+tF(V}DqxybCCVN90t0AI2vaO&7#~k`}RNVj3FKxl{0XKpRO4k5TC%eDE|D zYd#GrLA=@~5sR?GUfLoMz*U?M?}~V>elg0{6z>k#5qtBwMsb)g-4w(k)J(Dv0VSI6 zM0HJt(q+?71p%_M3Aw0|8#OUD>?qm8!RGmfLX8lw7=Mo9#t`!Abn>9i5yXKJ);mi1 z7#Dg*q)@T&9R+QU;^D@I$^uTQX}l4q&0N&6$SC-gAd$ej*(=}0JG8zbD1o94y!Y1` z<;=w0)*yY(R_kYiedVPnyHe%lQ7|;vDEXEiM0C$^pNsdW?K1YUcl?!?r>@seuB0)*W zt13o&6q-ZUf6`#7;&Lw>k0sIp)^|VTNa-V=d}E)FvHfe#&%>Yp6-&-O>^6^o&BtGG z>VJO`kNe3>dh$m6;ER8icks|tM<|;dq!ytCFXLmlhH!Cs=UHeKtPSt=)<(K-AVz^t zwEGR?66ZOss zUu}UYH&mWim{6x}FmOA7!V|dl&Wl2f7k>!Mbi(JrJ;sn=nCFItY6_egN&py%(AfRD z@sjO?fcI`HJDGrQBIQ$L7>S%xQSAv1EH@qQorT_E$p}a~w^Te=yb6w4WO8Yu&@kOH zS1~{3*Nv5ZA(ee$O#xl}#)ICk)BecO_-mYgpUYoy&Y!%1Kj!;eSM+P>X{-Fg34at8 z)boP*O9!c3f-fDV;%r~_tN9=PQ$7KdTsk1l_^lqBT*-FrWVh4OSyT=J4_lBij1s~R z)H7mOfMDV0NT0aWa6vRJay8SHD%Pr3X~eK$r4BFTCq`;a>HcG>@=d2ku-76YGJ~{~ zVwip`w94?nb4@ujJ?aS<`H_d%JAWLZvVpQZ@eT}fPGp=Zm^dMM1e~gPhn6qu36boM z5ro)qT08V1kTM9Vum+zoi*OZy$fgcw`RRgU#;MiEbj!POI>Z|vG=z!d1_+FJ<`@wh z5DFY4ZaYJjHe^8Hml^mqNeucOv}ZU@=Zz%z=8K5?q})9LC%l+qm$mCGPg z`K1{VQw)NA+V^}-fYpRg6Mt8#c%6qZaKo;+4W;BpsQHLfig~j}qPcF6Mv?R!9@v6) z^?^d|h4i|3U(_w*Sxg`^5Dvi|C8i%$2Y$#^P{KokQF1?Sa_TjR3)2$b*=S|Z<7s#@ zm`IM*yp$ZwMyqW#FSJ#5+7$QpxRRIOxh~&x@q2AO%l?YLJu$C8eSi6{Jj!2jyBGZvxhvi=Kk=O;eAsjaT*uxud$#g~+^vMRlWv5gmpRVZ^zVQP{EZQZE3B|OK# zU-MPhlzv&~KFxL*n29(gCg=CClv|^ip52d?^u$!nd1=8lL?OFVMDcrG2n#5ph$D1= zJ=(2w-7?7xl*fO@j(>FxBU(5`k+@t@3)KDk^xmJl)sB8Dy0-`dAkVoqV(n z!=Mp_9y*U1c@p8Ah04kP@52t><<0(>3?cv1Y&D1Uw!vYsCxV#hz$W-k-- zuk#)B*yhu2|H_|IwmEsqSN_zgkL$`t|I(d(e!8-ae%b#IEJ^<2nes2T`ZG7wZvU3M z_27$MXM0~*lhViklnbi)`}oV|`nJs=nI0XkXDf})+zbk&UNO^x(idW7v?BA63=P2z zBh;+wto!p<`7a0rEIWh2|9?FXbH!ho62{Df*W=z(n3naa(bFYfgX z6@|?T*wL1!wqT*lYld5))6k{~mSat)Q3tpA;B^=H0c_NNT`!mY@!cOH5^^M3iUxyq~hZC8Bp)Hg=?H;hc# z4u4=A>Y3|}yNEFzv%#L|^A)Z_B7}ow@u~6(NTEl)Z!3=n8j72vscBVy7b7Urj$T#0 zIE!+>Sg;#E@IvL=fygT=g$2^ET`rB~M;?IWn16`EF~IW?SsA!^%_%}*P2!7P(#Q<8 z))NxY~QMnPkc{RdW0HL@#t;d#WJmfMz(Quj@ks6oV4vDE{p=^t!XEQ$5jCOjm za1|iV4Upc(C1-DYPuzD@hxQN(l}CpK_YLqYvk-_dJ?v8+K(0()_!;u`1A75oU32hwfKn<7z=L)U;dWLyAnO#pmL;C?ECIN;z7Ujsy=M_hu!)0o|ekb z+bcXw#gPKZzVVR1>9W7_?46GvKFkwC`+u=vkdJ%+L&>opnJ4JO-}s5Mk5z65EcnS4 z)V6UoT$>6<;MXAJuGv;>9@TmPP3(%Pg@vyB_v%i85!z}hCX6=69)#I6Bi-uzG<%(k`(Fav}~5r3)$ zDVvNew3UO{mbF^8)O6uUjsmIK2GUYTddT9862VO@GM}_C3c0`s%=ZD(7p9=0Rce*j zQezOyU9g`95PO}=DZxBPkW4>pq4G#!XjhsIQCym#7UGr$h+iERO+6?fpvYTTI9n*J z9im*fz@s23LO=4``AU}HtJ%4vDf(U{J zO@(uXkw)h{9wAX#6n0l^Y4%Q>Tr6vMFXe=Uxv$B|h3Hcrv_#UZiWaSp4G@V}#NGF& z1%bT3L}ay;)9H-E9A3mfZeBDPOkP^CSiyDr+-8Y=m%TjSrvxzk=Q%*<0h56TXo`;l z)*v(BoOOV8@-E>6jtmIQZGS)-aX3s6lCXpzQs-*{0U`n{5twR+B3&Q>gBVib-+^Q# z{v5o(h=ax5i`wys0gU0OL!jbcSSJmj4`?z7xHkmDdv;0n*L@!}8Ki>isy}zRSUtz# zuYd0Vj2PVH0P3w%$AL@#`9=@w8h`e$kOO#Xt^MUsJ!;abmrb>wi+=_>hACH&l>i3_ zn@@ExYCM+oO^xDys6VILwV#8Fem*{_!sn_H-eZ$1)bopc?jZFw4+4xD>iJ;sxE~>K z$Y3xS?A0zu@8XBW0I&osMFS{FX|iG%xi^3yj&Bkj{B`PrW=^0^Q*>nPl%y_TJ2hYX zXPN$~+%n3p4bY~+NUX|{QvQ1ud4;gymYVUEx$ zO{z@;?9f0KR?Li~p~mu1T^ooSMq|+cKV&Hv4QxqlloEA5Zb0mL(G*P8k;1ista*+p zaXM5rd|UDoONw?X@Tc=o*j}Ue@z|=LSu|0DEm}V=! zDOVRj*#uDLg5{JWi6bc<{d^_aL{Ko9=<9V6WX9pxfU(bL^z3KdG@=TC`UWrSqETy# zKKKiF-G4u<1=%o+4CVrIxq z0I?*2B){rfsys8l`B*Yh(*ihtS2prH#W!y`#bwf+2&?hK zWq)0Zvh<4v!I`Nwk@Q1LG!nft7xOdi*hqa>Pdss4u}EaCWf-eE2T{kj8GDn3+m z!%ej6`EZ0=09Y_Bk`Mu=&=}*JEW2jI`Z>wecz-2MHFgud2gn$c^VVo$giKzXQSOm%Ir*02ssYTEsnHh3izzpxH&4H%u-(SIV1 zMv`F(4NNZ$2OK70rrxIKB{CY&)p+^EUb`-}9_4C$MPKfe zU5lf>{N`eqs`+#`@abYC8dL~Iy?@6XPkP-eA;r=yYo$x2>%ylqflnpByk36g;@|Tb z?6ZZ~_qrS-izyvj`TKrNmbhQb4MieN)m-_4ji_#R{rq&lc(5g6WhlNfF3>s`YCxD6 zDM2f#1uRK-`>)b#E{5T=-$V7BFZwY37yXz3awh+#pO11dF3+7nDSv-q_phr7LhSDYJd%OpApxl! z!Hv;8TZ!+)<9)AGHtEs&BOTmrv<%ezK2rm``VUNy2ta)Bb)G?E`t zcv|tQ*PBkSo3(r&c7DOMm1tl^lpg&D9urD8f7vmJ`eKCxiDJK4squx7WZ_Mni(aQ~ z0RO|-7pznD@FUY!MSz-p11Az*0NWdI__G}u#u8P*Kg!!o$rII%34fSMwnKE2C`yW^ zw=3*_;u&fm>BG)3`CjSC%nW&o>4tU2>W}sq)wLnAX#mm&<$GTC`p~dTxIO1{KPQTB zv|lt>VkNsVh_LjnzHK9_yBm0MUB#=gWSS$POO;-E%+&@UF~p4#sccON<*za|wo$5d z))%d5Z@{qQW3QKPTz{w+2liR#J$dW!uh)5}?-_<^xflCh&XVb$?XEoGR?oRtUbBgg z|2T%-4-axI_YA+9iqFFfEcvkD=FncH7^Z3T)_Oo=CI5KzH@e6iqQBFX|E@mOzP0MN z*`O^+4YrAmL&iwEaxYXu$S>{&5!Zz`r~4}{YdY}`*YOWkN>>x&3;UK{X*nEKuy6gGMTnhbt}D7 zq?_;lS*}YVD1Vz~als&=dTG`S1Py6(yfqKs`$XEQh z=(-L7nd&&k>ezT3M_mW}dA(G}&+89*OC2W<`p5Bu{(o`&pnn{9-M!!Us%x%|A(}=i zAH!4qU%9S_01#1hY1!RD*{?laqv!zQo-TJ?g%cP(-7->ifF5*u)2RI^9(3=n@>Qs^ z;gIN$bQE)_s53^eSds>(D%n5eLDbA)s8h*{h|=6F*PgI z4yB(J-G9077k|<{`i~xT^3nger@KD-Z}fDVNB^I6%I<`z)BVVeT-j$tUK-l-POIjA zj&G7HJJn*3HLC8JAl%a{dwB6puPK}FhyJ;yIsK;Jma2a3@=JD%`)5Qq$yJ-Ip08)l z1R+Jgiyrh1Ws3H#um792X&O$yY2Q`b7vHqAzJK1|YsS)5-7n4rK~JyRPV`Gd>BZe( zc+lS@FPm$_sH*#$R;YF_zG+F{j&IuDpBnrg$D(id^f@lr?Cb8H{c}vpzTMxn zc3<~5ZLQkzL)mJh$Pfo#*L*u9o0Y)0v<6)$fRo2i5@IjXJmarS3PlQ|C7h zDx6B2h@(=;Rf<{aR_fi&Ds*WLSrMkLV8Dv(i#82!tu0L!C*>Pfa2yYzH2wH-A3$gv zR%vorH7!u!`uPUdpx6N2Zzu*D-BHR-pnu^%0+Bzkoer*m37{HmfC{dg!L;AM4)9Rr zdlvDNu!3BucB%XW1zq;|;MX})_;USIjsbMI{hSj8GIu}p6tF<_=Uj#6QSfta{-xZ9 zqd>c#`T`)F=mHTir;r1B1&hI&qYw3d@>b!NSU@J>q*~$$P&*Gnz~KsnDa5l?=YI?k zRqxB@5}m-zVIvgY>8X8*PU`wbpuPHaReiL2oMT-z?f=JfDfsaFxk5eHRrDE0H$}&} zdJa|l{~f^8V8>NCg+!ej0+tdj!1N#&=mZwP7Swy#F{~sSf+zB{OX<^6Ae-+xe@_x-b&I`8LW#UJZA;>c6&TSz1y>-ulc_5M~bmg@Sy z<=?(P27mDDziJ-_)sK-Hr}cAPWJ3M+{TeI!ts0M>?$IvJQTV)115zvgsh{gas&iLu zA!7Yy{aK1XpY4QS^NXjd-zHG0+Nr5}@?H*1!Ig^s54mzz>+B&9<$rHJ{scNza>e_J z&yj(UU-7wr{_FQXQE@OOx3N-)a0HwyCO84E)mA8tk{hmoPqi*R4xL)xzIv&EWZ+BC zfmCXD%Lxz(bM@Z+IJvDCAo=27?Qna&+;-RQwEMsQ@BipZ9gX`+uKD$MVqq zhqGSnmj5d%@4Mmu=zp&_*Y2?Uj}KMB;c-RoN>N<@L)Wzb(OmNX_;b5lwTJHLYs1{F zHZ#9J*@BW>DZW1BHc+ZRrvD-k`2B~D077s^#4#s`GJJle_1#0hU=`@}1&^3|Q`{&y zSmHzXL?}17toC%XCRb)#G5=iH#bP?-#ym_mF9cPQem-4J=6^|N^R^hjw}4OJ-4}w?zqMCXmbN+IId^Q!Emr2&h^z~v$HPOa72bi_VHTAr=#CG z@moIKf9hV1>VIuH-$#AbE~MrrBa8$Q`!?*+YB=g(?c6Z&d)+bFc&y76Sx%(ipGOg87k zHZ#|a;bM7vDXL<%KI{iCljUwgMxRT$UD>7HQ9fDSwtv^p)sH4O<8C;wY zsvS*E&&xMoTL_E-T}%XoXx~x|My@^Z#yWeg0oXD>NSH-0^K9}$KlYU6p9+PZ3VWQTR4{N%6d%o#Wtg zeVxDG?Ac~gb+_fP8;ut7Q@;${`HMXt>_2Ab%YVihO(&cC^)?;N{PFFQPA1!z+bw-P zyoWC@v(J$~NIu`L6A#3q2oC->3$Yl!??@NaNyE0un|po#xJYAk%Jsu}GcpNFxyVV|F;f{U#i0q1PovaM{+F9-Q~8@i+AC)%9^=9`r*od|7JeBEY+Dha+rd zi+{y3p2Y_@f#PH9lmU5@-Du^sKSkG* z2^MjN6Zx9jLwrVi7Ee`|&+&&gU&TrP}*dyRUKbMo8vf@AXSiuXC=|nSb-b zP;ruJImTSKtMwM2QHys>qRRl0`HLo)K|e#+)xtP=XLj74)?~A{M#iaG)W_J=gKjdM ztdSF(Q?g5K&Ajj-e=A(yHN!(t-7odL%#u@axQcElVI7;FuP>LEG8lSxr};NeOPcgl zHi_SoX*pa^KhraeJ~tVq$91XWJb&6lc}x!=qRT;aW^KM@xIP_A)5k|HZo9UL*Kf6M zPBj!B_da?>+mn5Zvq>^vy_`OVuX_1e*0x+3?(9A~Pd=BLKHOe|@+EskXnY-wUu`f; zZGLW(~g>6qd*%JXyOZVt)#;PKbPrES8QO-94ferCTGRYtQ-;OrQ}*?*R`3-ozP z%kk-TvAJ^?Um=egH_N8)>)cu3knWnG)a9R>j8_y>9cD z%ON0En9uKHK3u!2Ww)J@=6_Ipbdzqp_o(G)Lya9i{Y#s^h4x_~YPJH<=o$qTUs23) zMpIErwzaWC6E<{jamB{Q>UKSiZg0t9My%218q@sq{-Uj9dc?FlIOo*Q$0uS9CL+@d z5#3+UWBX{nOoFr>9JOiZmY3Io`RWeyPhWd^Gn2VjexP7;$aHtx&VMH3T=#a>&{^_j zWy<=gsg z%%q=$!y)~cRU075HI74Lv3)mQImF1hkmmdnyldCh(2&$y26^ho)w({th_;LlhBGb~ z{H<;saolMuQyULXQGb-?udrO5d~8m{P&B7~o=J_G`Q)_xxP*bV{G6=TkrrofV|fhf zVSN%_{^F9$>XO>_GEPn`&`xbwpcY1C{xMmY`wMChSFveIM5ocL%&OV04x_%vfp2n_zt#R`?x)t=cf6HIa+i>Wg%XsCwyS%*54%f+<$~z!yoB^CrqU6x*QVj9I@7o z%v1+458SO7o$5}*!9K0bL<35^;mJ5$=>l!|XcufIrGZA1qDVe<-P%l}VdMzo5{*g5 zj`*E_gwyw>Ut@c1-map^&W<@L@zJc=CLV_wnuqmh)?Und6Yc*4oBO#+) zHu?ojtACiGwA$ccbjE>C_~vxB_bHtirF}o|!#6D2jASd{&FrnKuhF5Inojw$jLKpe zm8WG?9+pvgT}I`88I_m#+D%Y!IGQup^UO3``zThQ1P=VSWWUetT^$ z+?VHE9&Xj$)+bcryDUGewNqwYjL&O)Oi%Q%a6PAp-B zVK)oU0h(%ZcezRgqG>h`%rU;IMY=5WevAZkVSutL-XgO8iO-`?aO*L71?=OXP zKKAX~<+R&2<=Ec69`%voHiPi}=I7IVJ2+3z%XrWXU#9JDlfS>uZ}ZfOMywgVe~d_)tJ>tOlViItW-Hu*^cvd8IQ%iSAQ^RuTl~D_}pDkKD>-kNfj()kZ%}Do>|G_86>!a z5(i0?*t>+OM-x?azIz+KLoWWN;E;GM8OZ}U55LEyzg{7N@o-9});0Er>b)yHw5#{I zD^-!h=e!9pd%m=OAcny|-zE~hKW%C7$Ssnz`};&EJr)Jkr?w`llX_ljLVsn{NdPWC zF|}y8eDSS+_JXaG0`kjn(2KeWDgazSqrcC4EkhESC|sc9G{H|K)V9qaAM*x2XBSd3 zD&r+b$%HoT)%!5o*~yg^M?|!TEoc9HBm42lp}1Y09RB3#T0Ey}WfH71;S3+*36935 z(qF}>!a~!wo&%eNKG&sEOsAN-LsEZ89_wS+W`%2B8UgN#fNbfyI@2z6uCAA2ub?>v zkHYzh;v9bDS6jY511!>OrDR`uVJgD5ATM`%A#0HEW?%W2+ax$z`BvE&4;jcDc5vN z!Ji-)kf7yNKh;8vl1g3q%HJ!6{Nl29G)0bulk4Pi3DjoGO)CofeF(y-Kl5pQVJlLR zjjiXp@sS9;7GGB9j_7=)Uz<0EWjeV|uqb8@C`}p3Ao^sL<|^b@g)onN^@I#wMggnL zNj+^;cy2SarQB@S-C6VKFdu*5`wevsvFA?{Pp`{|n`J3d4_%^{)#HKIs?R0LkcA3;mf~T~%bdn994=?rzj)vS$UF;jO5Aotd_O*g~naq#j3YoOmKg5(I)(7eze$b zDWwVnNnM9nf6Wqmq@GANsMI?fiEm`9;U z9V%xUY+^eEuAuAZq@>B3l4Imzr+$f7fi1EHF#D-PxVO?p}ADUF7MY`k>hO zL^r70;mwVVa9jNJO9g)}RJ6t_eCr^KVRB^%l5M)DRftj&r?ln6qS zDeCfxLI#0@RD@Owml70CnGC^;uAr%aH2mz{%0A;hzw=O1YQe+7;CDsgTNJRD)ck_? zxlbEAcwA!RfsgO$w!0=H2lTAII1il%Jy==zp?}Ng6RP*`nKXZ^Z=>@uU%@=>8~Pq1 zuOITuq*7OP@Zm)XV!8A~E{0g%zPc$)76=J_``R_3Vr;1TT6{nnreMEuH;b&nk`>$5 zxU_5AbwL*`_3$rr!UHz?+Of}vIhM`0Nghm!$4pAHN1mQZf!AwuK8Pg2zj;2bK8#iu zg%{Ss2sS1cbZ zq(O~qKA1vgy7wM0Sn>McFiq|FrR^NVm6Q0S%_sC&-YXj89i#5tw!I5nvMQd#c@)}| za9F}(rOuqbwtSO`gy?$z(dF>dVCtBWjv~XK)~aUM)+vA2#eK-kzI9*Gy=H_$hfs?wy<5^|NUmQklIJT7+wusZo`Dqy5$- z0-`=_H&1_4l~OXp6F&*@Ehx{KBu#L?FJrZXu!rz*3vq7SkNlFpLGkXzqs;QhlpEel zS9sSag7~r@vg@DF!=p7wlSweQUkJ-j;^D^9BcdG{R-me7WGj}L3f<(xo?CBtE1TDA zhP4R-sfS4V#MJ}cPJi-(_xpSG-s_RR__kPD4BvmR=ceHijga_89W@&15wnlXR(_Eb zrZ?#5qVM)9VitL+F`q5rUdhIl!jNF{LJ&MKqxhD@dwH{OWS$+iVLv(9%x{*W3A1|O z6#Iao3mWE9NXPM8AQ#!yPg8v>%tPp&=OdKp@y%lOS%*b~rMfiYQ1uE6@!i1Wr0hd7 z*ouE$DHmOQg}sT48mcfzq_B&0R5FNi}F?L0D0AR^gP(@? z=lww*cKKSe5!&1YZ@wS=Be$~wiQclW`?No?SpQz+>B*`0Mb1rpkO*s?a&qc@^q6S_ zEoLsp7V6@lz4Ub?Y+v(5x}Pm$zMX%Xzdo``&D+{ODBJvQ@9$oD<%|Bz7N$}y(e9&M zQna31!md8wI|5b@I5??#pq$Em#+Zk5@mK68F~2Oh_dy|=4zc#&6*jA|;E85i6kBgJ zj<+XYL)iNXkYO)miH1=K=SX00*SndvtX<`zaQH#CEydVo<=VDB-m#@>6O=lp!lc zmDh3{pVigAFh6I-$a&Roy{z+IGb48EzMtc2oWTel9-l53BiFcwe%=Vz=%+77ylcFX zomrl|s5Ic3CgXpK&azfGoN$r850ny_8yWcO&MDz87-TCd-C;35UjBv*%T0g4z| zJ{(iW)Yvl5A4PYs#}Naoj@ATH33Y^@I$Eoq5x${tMqWBU6cLLuklrhd9~FhF*|5Nw zdJ_Qv{tbWYiTw{;&-;J>b)VAxd!O=E|GiI1JHH12|K~fDc5u@7Mz8hx+0P|@PLn#A zb(~3@t9b|g{Gq{c2ZrpAx8c%<;PtY-q#Qx@j{_HpNKfj74b;@DU z4=5A>KnZ_uNOO+V*LsvmD00&~`RcwX_j5pUd$SG~Epv{UHv`oCc{J&M3buVBpQ9Om z!Jz;kKc`h?=hw)6e`NU}E~#@r?|l=gy**Wnuv>lEJ;%-4KS}>{?()8$2o1LEPxuWI zy)S>C{`{!Cf21DJ(ydr&exDHtX$<+E`0{#wBhS^t-YKk0@rV8G*R~YcR$UmuH_m&x zDJrA0`lG;MIj?BrpfBU+k0@^8L%+_sA|!N0Rt7fA%hEZ`n4GnJVnQICP8WXFPdL2> zky;EZ)V3t98cPhRpNhARR^i^j6?baW`89u|7asQM6w-x92Rwsfev*B3{A|unk4;N- z4PAGGwJkRVfK*lSW+?$ zEeklNBQ0@`7rYsGzSHomR$9NM(aOHoGMfi2^PK8pu9M=IUEJkQ+|=bcScvHI!x>p<8+zb)s~D8CX*N6* z>O0Q#-0dGHwd11hhR#%U5!5_*EpdDEx!dD3_aiKCM7G3^0K4nBelnsfUACp^wtJN5 z!q?a8!Vd!7Y{8Sx2b_jlVv}945`KS5cgr)`sZDokDGt6kt0O^k1Q~-RQ;#LzZ3!nd z%J8tVL|pC6gE;layWwNnM~|AOkNvm`lVk(HwLOVzlRphBx#pe-xFEG&nsjjkuP%lZ z4sWku&m#`0;$4-mp7+oZ8Ht#_z7ZVoHHp7GGNkW=lJf`gKV`P3ZCO>G`{jQvH!WC0 zg4Rx|O!C6~SFB6vKuQ%%$p0a`<+6PlU94Sf_a`quhvu`4!>%L$=B_8h)P)hFtTJWw78(t)A zPec8=;Rgb}msVg`H(|kd4|&M2$4kdi_FTgp%|w&UmS^B8Rg}tFRP}#?TE5<|;iYl# z+||Mm5T!$^SG!5g+h$y`Jgw&9)v2J7|TGgew5ZeWee~E4E{~xQT$^*M%15NTihf}?nRe=DtXVR* z9bSByiVO2f8@7C`R!aP;0B1PTLFB4A`x3vVJfW5iQfEZviV?=g;_OqxLK#hxKKHlJ zEs}y*Mgx5y6;Ua(agNdZ;AZ_0g%P*k#h8+yd6prmy?i8iNuGbvM$Fif`REX?a#cty zpYTiFiS@ha{rRcAYLMtni+9vU32zv9u|jBGBVwtS`Z@@`a?9j!=5SScW?VBI4yHnF zQ0X>G#IoKAL6@jwL+!7G@VjUvDRib7l6cF=_w`0%y{0z`Yo*WQL6gDQ;fzQ#a^2Nb zl28eK6rBy(%?E#YwoVVtjAXNB?7~J?f_xonJTLg&URx2?P3-m zGL=7C6=n9zn2<4*AVPUwNtwMBv7&?(-`v6<<+UKLOqaNT^dY83o~{qxA`oaOr{a{P z+E&^s$4-{14v;esp&Hv>R?4pIC{;&HvQHMQ&hvrx7Oa2n^5QMUU<#q$wNnTgK_gXp z0yXQRbJdQ+X6aDVmK6EPkPm0Qlmsqok0lFsgj8488wf6HjA7LC`$4_#tsX|xZJeAq zZfQlOoiUJD@*B}E4-y^|Ut(~iQzW5%@SK~jU}7DTQk9K?R{l=Auqa?b%D~jyJbq+D`_+O~>4rg55sD1L`JWPW8DB>KjL&(O)o!4;0O)DD@FV`5zd-_%^bC-$A} zx^+;6thidDJr;CbO%)Zsnj<*asQ^d-=j3IaH!Ob?_B>B2OkSQ=zbgbN9Ci@y=rPPz z)(cG5Xv%CG+F@9U6Z^#ogF8SnM0Yh-r_PH)?F82R?&8dm_TQ~j>R3)5Y0D&KGq*n`*2w{#i9)Skhelj&R$C8+&F1plsH%X!p|qI^YOJY6&%0$<8$TIDa?_| z$F8@R@TY2wip3cxE>Mo}{RqE!0r_}YPH*I%8#qG&7PvynQCK4q7ekZmMsr2EBOlLQ zp>Or^(gAj1%{hE|blwhQy;}*F-lBk(^sIlvyj~G-wjU&c*WG?Frf$r`=0P>(f8^)& zBAk;ap7|pyG?Q4-Bc>sD|B$LM*2zpC(JLj^VBuOY&HM+DVkA}&(Y~6leCf+TbR6B8 zuhe=IiUeLzuDoF9Q@>he4x_V>HRP2aIOi%lO-&??=)r4OJx-gjXOBF#lN8?ZSo?pQ zL^%(}L6@VlCcnAjRJ;nu@T(YQp7M|(#RWwWHR2p2N+tSa!Y1x#1bwHW3f^?ZczS?9_9`e~PKvS8^CbS}D9?BVVa`)RnrlsOB@;)cpa%TNs4K$5k$)I4C6aVzyAs)+~} z*|SHe>liTZ4a(F@)b?d2U^w;GMn^k>5S4`v?>_<*6lLyCCo(-7L}ZQ^>D7OButpDY zC4DCztoVe@^@c~+`?#-HuH~OVe9ZNi58-UG zpnFTRiTr9&d10_~#XTs9dOUVLPwcK-A&Pn(uo1~R@u@gH)`>nMM}sZusaFE-KsTF? z9{scH+;Th~*f)80mX&-ML-^9qPn>+}S@JVQ;nmeo$6{+q+E_HmiudAD<>t7Mg6sf{Og7CLX~@Q%uJRYM;fSI>b` zWtf+aMU*RDR3ZFHrz{h}1+=D}Es~HC!;+ew$rNwzCVx2WS48Yey5Kih(+-C(tT7#h(lz*DYQ!sACau{dx8!OL;neh>+Or;_<9#B`P^c=t!xgC+eExO`Z^ec@R_V z;Q$rVprWfsRn>pg`obP*Rg#jJQGuoED2N?yMFpg0vi2R*z@mEQOEgSI^~x1yhpIn$ zE(@p@`3f)5e>Yi-dEqx0XvUJlOesWa-_so~WxmAOaNy~OEhFb_0L~Dr#u?wNwQc6w zbTnh>tA;+_mK2vOhDGJ&(2{3nvo`TT>Z~y*H66JlG{JvYbE7gJMF5YOB5a<9?Jqx# z`YrJL$-bN=cgK`06?2WODBHz?W)FkqobV(CBG4sRE->sHN?QPDtj9~a^@zxDO+u_J z2SWG6%45KQT(5}HxA$OUl)_!`+~*EuA~A}$)XjtOcVx&@dD5~FW2_IBSr|c+n^hl{ z2O|0+58i)UYT59yLaMw#y+rCU^_?)h+xJl4gO3+5bq9yq*{=%Cbj=)@5gV=btKu~D z9htA1Xx@Z89vj-+Yk#@R5{&H=3G4~q@w+hs&>!{OJO$d8fCoUG@6!BNX68q@>sP?* zx`mp`JCC9DV6BA@NE7x|OAx3CPOa59c#qr&^c#OV!~46X0@@Me+VxpQ5YRdx*=m#} zP(>U_6c4l>XZ8vOI-GM7Rdo863r4lr-!1nNGy*IcM~r`jzJ3bvfj3XNEYh}Z2Y}!0 z@js@LY+Q#YMU;T){mJkAEp%0ypYUu2W$!`I!1A_*OO~R;9>4VcnmX}mzGhQ;Z&6tL zul0Wt@x@Bncs)f-sG)({XPfUz_#hc>2P%*lGd9qIK|E5(uMoe?c1voGcG?7wWa#LABfx&V zw4WRrOJIXprxIK9+l05AqTv>p0`P05_G}lC(NMW5wpBC$F zu$w$jxI<;I^=WiM4V+A3yOO&&x^%XVr9AS}0m9&#arT8TtUv2kQmkY={6yl%4zr{T z6@OG)sLHYK6j^D0x?A274CqV-k6;$898N$EB?Wr* z&Z4V46Meq13kJs;X(EUK;=f_Kf*>jyEmWfKob3&7B7Mk+YcMn4%NiJ;;Q78usBADB3N_v@dUK7a ze&Sy-!H!{(R^mBoA!1=s=kuAG%rngMBU)d`Q|CGZL%YykD}GqgCwtggZNvFK>p^^N z{xNn~)KHOa!d%7U36w7OHROLYz{sVKCUcV|E5uC}2=BpnL$jx*2Oy6ZbRHx!^^m|W z2g|nvnUp=q6O&#?@rzPI+SEcFyk;V2*YVjML4GtTv0-Gh(g(FXa!RQPN*Co0(ufu) z*h>OUg9x$XunQRMCotr8pT@dZ7^Od$a^7TneYG3zeZb*()~nGdFByME6-VT}1MZl9 z(>1)9!4M$Bp!f4;Q9bVVmKCbn|P zRP1`Y__SWTCZi{m0n)D|COoh=yT_{V1HSWv$;jPZJZW8KfT1#;H%*Vu7meBm3-*${%8E1R)=t>dTPG0sOH&3KR<@ZE&t} z^1U1~;k+_fBn*GLPK@*2ohKvgN>wLQ{H3bzHC$e z2etII7A>j5z`7C(i{ePq2Gh8HYZWyMLt72H zE$FrqSk(4dTIb>>fd_}yrCGSBC?GtWOld`?m(M;|=XHO7cCIF2Ue2sK)`K&}EP|ww z3l}pl1C>vSbSOgms~^G>mydm_x%7zyNP6`RJcsy{bT+Ae=~&H-ch?rSY#?uUlD|-{ za&4u4-GYj3V~=fwo+^!zQM!1%UBeop_7Szr81FHiYq+wOQ5s0)tP{1By%^7S`sc-e z$)Ix=ou7ZnCrM0F{*fQH9xAByb!slr&b)es?vAISDG@B~$XKH+m-&{=Kr`~Qp16~C zB1*KrsJ~!tvhr>JC@ey8q$ue=7o9t~sq5Fw`OkOvtWN_9mI#N7UD6VnSba9!Sc>=p zr$7&0GkbMzUYnm%$H*b6&4lN7oH^{6Gy@x)nKFNd{z-rAH5c~2&d)c=cnRt&tQFY-b{tHF{SRq?#)QmA}^iFz9yKgjw0&oKAx3^drU60RK{Q@t|||P8XNtY zp_PAP@I;0+I<+u*M}A1thrj+pi+c}go+YmH-#dJ##9Mnd>)>X;07 z2a;MjG#6Vk1nQ;6*--Q8`Q{S3`&0t?x`cwzO~kNjXl%G}JoWbqrSl|tR}ed2Qupj9 ztXX$LQ*0zsMJ9VX^doWd%KG$QpPKhT`?X27(RUz`)ijsCM3?aT>%= zdMRVdgq)yh~|v%1#aaDn=} zo9I(|R8s6Nc#K>*JH33-*B|+V@R?nH1)`&jp>cd!unSGl*PgVcXO z5^kKDhR;INn2aoH$E`)AG)mI77p|w2lIzRhaBpy*c{+xT$BE7Gj#yENxbX8@54W~G zQ{tZNaPHf}t=P*`vt&buJPng&YJ`WYr_9tD5bh7o1Ch+)t-`$HeqRHbN|zhI4JUuP z6G2+z_YyAejga%6wdr!b^S+77QB;3Ox&t_7c~D2bJmSqyRKs4CQ=`iwk@gGsz}MKC*k&q}n0O^B+q5LXV1Ke0R#`9uRB)(3$rcI=@PAz{yn45w%65l*IAd!Nc_~4*OvDln!Z6E~J+ZagI~4HFc3R zEAdOy6Fi&_Bn(r{&d`tg2|XX7xK>)V0&U&Si!AB$YOIx*zS**K8%BTDpKt#uj08Pr z%q3z;++JRZ3`bn1(nGkKCSWj53=`wrt+T=fh^@kNtfv z`X`p8i8vUn!yu!GW->&X03U%jYB|ag0jAJTHTiN=o)?rQ<;!Zoi!Jy4v0x)1dq!U?1q%n?z#@o zaX8nm-H#qlS2}mkox}trd1^uCn}d~Z-C zl#!=mfO4AQC#+Jxs|;0E1z*pvCmP?jn|H^#bctwBk*kOP!&{2Nx?Sm8Uc*!R3$Sdj zJq#UUG0q6+{eN=s{jT=-A4))$;ZcWc>CorRUY*>@f(D-@3au=9O!i(WUxOK7yy9Jk5TG=7oq$}9h^VXIL0l+0E|Pp1h2RR&p2O$a|_mma5-iU%0tkSVd0|hLZBMi_e|K_K=9~(dTo?qN+5sPf_`;1NUuZWHN`TxRi z{~N~nA9VU({{4f7UO{`~y4;m@0+E0B!d3;eaw8vsb7=alnkkIN-~N%5UIqyZj)0L5 zN4jZsLaXNdeiSPE|!E^^d^HS8akr@W74`@{ezXMresAwt%oK4Ko@@$K{<=T zyngYvV(rAYPt-H%fm=xM*0Y?`bHbsjN`mmfO(f1q1mMIFylc;AqjHH}p*@VRkt zhzK0>7ro%9A!tVqslgqz{*5Rxn&kQuYd0bU7mwsd<^GMC{DZ=6*}eEdqyi6oJx<|y zN9hfJzVVw6XCDo)CoJ%U(CL5n8y7WNrz3LRa`!Qwtp%=VkXCU7NjydY_>Rigjtpw0 zH4<%!PfN*!*?Fs-Du7kD!9MH!bJph2vvYc__S%3S`kqF`)Be#9_m}Qk4mBtq}VBu%Lg%y7cfkpW(L-Yym z#y1+jG5U+@Iez+J{5LZ2*Dp8HM^?{dP+>e&xaG#KnxJ(K>kLHB|NGfl#SJ-|6(Y&2 zA1Vkm3o*1G{JW!`vZpk_Nn)vXSQGTtjo=0iVHMnK^?uw41m3(9tJ z=J=#xU4hm1K$m~S`7gWK+wKVb*!{ZO>mCFDUG?t1k|`~KZM)f&h&&( z`5D6vj~8|eo)yb&01poc`>&tD!Ag8>vV2>ASaQBtt{z!#RsbeLK1@2s?u;66z8o;V z>^Tb>S96m4>rWG=C4kr5qXw_LT=sus$8GC>woRx1#TEhjv1J$7em5MqCRn>BM$IYn z|7^>4+i8CTOp&#ppDU@f|Dr3|+J4`DyW>^3l16ch5;R5kGgkbxtI&#W0>mSc!4Jw@ z!HjBP-FSz&oO<|?}A|ETkvYeoCjUd7K zi-do_rvaRTg{$2Vo$*FHXFn@GhjU?A02@#*z}@Q>;}_rMI)T{i65BYT=={9bP#Y+N z4IIqm9kj0xIEE)Uw=}HU^E<6^p71rV&%FyqndjVa&S>-ds4Cu!TfG4}*NAJU zp#E%DsxnMFQU=}zP;ZGw0Q}#%XmC?oieG;WaPaf~F#VlVyIn9l2f4?j|64Xb2|;jZ z5jX-bxC(z``cgpKT-`U?RZz}F@>W-T6{pw!Po9OV|Km0O^t*=z28CB|iA4aSf59Ni zH!|&iL>&L1@-X9`Q@D=wH*)O{a_n#H@xNmmMPriL`o^2`-0nEyVVDxwU4ADVsdj%- zkL^oP)1rP6h1T4RrZE?*&dR@HR(BGK&Oqy|Z9fSk|cxlSN<6_vF94S8Fu_iS8WT5_dm-cp$%iQ110g za{e=aErP%24c&i5oEqId*SdX{wEur#hYRG-T;jjr2LD3Dsl4(JLhf*)e?uDo_4kd< z{6!nK%D;{q@Be~C{xA4&*Z<=?0qnCNu!YaNwIMjPC9tC)1$8Rdip_d7By(dS0YL$~ zV+(FFi{M_n5snDPXXuWl#}WpJi$85<2)}KnSpSu44sduTMCnF< z2?l6(UjKi!KXA_fA7B5E#@Eh1&)<8pz|dJ1#6nFZJS*6I12UFb8?CN^dRO^2*{=u1ok!&82as+Qs(vKJ-*H_F zA_sOvjv{GOtl8AS#QQ`j{p)xBzZd)bP42sDwFPJJE5)if|C{mecis38%Cdk!SKN5a z|7L%<`_i9W$PYgH#%&h&I_Y0x&GF6q^Ns&*il1DUc@)UK9uS?sP@RAG{nGy&lYr(~ z$1y=*#2l!zU=M8J#_&3$0^I*)Ecjeq$IekG9Q*NEhjv*YaNM34{60R7Ke5X=^=$j# zbz^Y(N3bZMWK7vTTGq$=@mJN>i^3qTI6VZ1UInZ z)rY7$v7u&k+1gM5>F}bf2ykb=$$At7xA(#MrE#b5|J+~xbN_#z_1~ZIR%UDEAG~U9 ztl&2$er%F!CpH`9jyVi0v@1EP6KD#@H-W|Ng}|%9D>!==G=(ef1Z#z=W7i6kUG*dY z#Re;p2(_b;YXup*<8mO-WFXC1#iHOZ8z=cgeg*kk3+EV*GdPU@>aSaJ8GwKC2aVwT zj?KwsHpO50ut7dtnTnl!yGb{B6gXX##x{FPVU@BAYO{>dvBU;KjqO@%MW0qx9?P5fATky42`>_-RrC0{LRn0Dge9T3(fC(Lai0fWEBrY&d*w^ zA~;a(g2y`vSHBs!u1$$M1Q}jsh_D|uyb7}5%rbxQ3|6jdR<4Eaj#s~;IuIv(Q2*?s z`9JUcBk_dBxz~QbYZun=!LFL1F$~txxC)0jPzTn1oV8Y17$Cp{&MdU* z?z7A8LiMK}wfaE={H&9m@{X$?>G_|XfS-MY=nsB7{s)Qu_grZB#kv=mSqu00{?DAL zapHe}$z%SYkyYQh`&cCY^L}BU;>!XD5OhyPcRm<+Ahha7Z6b+n!$|VgR%z9b!7WUg zbKV?jN(F>kV1@AW>^$%>&jXu|K!O`*kR2yv*}l;b=L_b!15Oci*vF3R`JIYx)3-^d zoSBz|ZlAo;rr*eLvK$XML414!%>|R1V10kW)^j<>d#$o@K@9kx-%UCTHWBs-LNY>FhpWYvttbPy^1TQ2?|xbVKi`t`>Hndh#6R;{nf=Yr zz13R4%rkla{NHbkvFhdg&heY#cMfyov%P=zZT_I(|GTlqx=#NU+ver(6yGQw&%eHq zJbHXhII8=9v3DQqt}1DRpf894UWk{er4c57=Y2C96~Is^@^P&S!sJ=sKN)BfUR>Vd zD~@Z2L*Rg)qP_WF1B$Mu&oM`C>)m%pQ358N}e>YrxfExi|lq??HW} zQ?6{Q4nSB?pgjBeT^iuab^Dwt&E`1VM)S1Xsu`=S;=307;9R8TSvL7kpZF)wT;Ab-zx}NjS|2th)ezwLFPOWC|o3ou0-yBZcC8g2u2&ImeR*j~tgc#QxR zX~fWx%oL@}Dzc2(bf5h^MSHFKO>3WT7%=KSOzD55#M1@p z3qLyV@BUYmc-n&wNGf1;)-C`J?mXBFg#}#1^fm}VlY+agNGGZAiLVZrUghbtUly(b zz|p9!tLN`>-APLf6G(xs0RwKk^5A1)xyN)BE_8lV3Esys}k)p?;o|AWoXZ zmj8Ld^RmD2fSag4*6WM05TB=!1tU}F1$-1Ix$c61C;(usTcQ7xbk6@Z_{aLC$-(-4 z(?mEEtxpPZMx3v!o}a1>79b!XUl|&xfZYl$1*={NmFUQ^ULvmK$t7Qjm0Zh}Uul)b z$?H@j>|;<1d1TmLJfEn40Y4;*wlWEjCh)M?vs|+{&{>&@%ElEvIpa%Ymo2&MD|we& zxtgk#)n%Cm&&$bL6!yq0FbqQ$j9c&xz8r83Lne$Jv;!kMY~v4zzA$*tjz?~xvjgmpDB~<5V_)mhw|EoU@0PQ6DgdlQ)rjYr6Hvs_YxD*h5bciO- zPA0aDBQYwfHi%B5FiINv%$5k#>*yr969B^GC#@L1qZBX=;XeB=uVC| zPKMhJH2_@dT=wMTH%>&}tMnz?a~Bf{9x(NnbT0%TDsdP(<;k0==pP(-(453~c;@BF z6f^ExoTNO%2Ia|#Gl*e_8jwy>^yJq+G;;DKznPQLJo6OjFIjXq22({HUVrTVAV?eJZte&zO<8w{O7vBvA+oT4;u@A?308ILE%Jw9l1;+YoPd^GmuQ& zNoayXq!ltY$XdG=d=cz6Aa-`?4i zRj$8(?76^2EUTVGDDN7qykje0ZQ98?Gor(XuDf!fU*{(czi*l!l$wwJq&ZW0rt{c0 zT{u}lM&W{>+W1o8CLomd3pgs;efath&u(8&T7Ba`WIJGlPkGQ=U#Nc3>%8KtYE`l7 z$*1ycUw$}m_pk3~`906+mwo@T+2{N)Im!Be{5*c?B;5bC?CAf2bw2N`>h*bUZ3+PE zB{L_l`6<7QKY=s73H(q4u8hcqlK}IqQ=I#$L#!;axX2XDDcJGr`M8k!$yedSK?~(S z)8oQPG=Ayz+es*XY4wwnl)p6lNiu%v_4`SWf74Gw(*M%z?7lJOCp|mq@o)M`Mi$?H z^nKshFa7@dCw^W0lV896=GWJMc`K8wzP2;(5x<5^}{iWGY z+Wt$gKQ)aPzv*ZGreB)f-8c42zmJ-K#w*`+Vme7+5Q2GPuhYpPiy|mK*QSu+zSmyB zL>hR1sc|q(?DaLV*R%cFKJC|slKwa4j5SJ=)j!wr|B`f1zHGDnM;YWzraAdwKAbpy zoNS*F{2~lo5)A5A3n=9KGA0qq1wSV?QILohi_80%29{=n|eRA@VcJ?WNaJJWH zzvA!q`|R&U-+s=Al?QuJoUFX;OebCb5BqbrMHYLK&i7T=zx4V9{O&J*Mq9t0<;HUV zh|z$0w%z|kSI-ki{6(U_zHGhXf6vtf7(Zl*!eaF^&)7fHJ}i84UbBA1M(s(^yUFLe zM6%~UyrNOd`%l>l|L1zHRlj(3{nO4``#vK{O{Fahx#{l<*Xk&t5)6F4|ljowCf23Toiu#ki3oY$>+O&aP}YnzWzj?vKfD_ zFBYp4;?5T`AJq@WJKN)EE!)Z@1+5MEo4?`~>)i?dz_x-Z!pJkh~ zz62*b|LM1${cOkg`TepQRXq0PT!`wwmYjk``$0f2M% z;Y4swGv8@`nhO4BzI;yq%-28h?uSLUKk?sb{}cb6jy~~!-|6&A$A2mI{!7RIQ0!ki z{)cSuzjXXJ?)5$XABz2Z{67>s@;&}9+5Y|>|4-chCE5RIH`d=Q`-e5>ryr92!=R_t zr;h$G?jH*M_|39^;!m!e{~e#XKk?r0cjV?0?jmNA-(k|B&+VZ++QsFhfZgI zv+SQ({TJc>i7nYL$^KV9vchkc{X9lF`o*$;@}=B=5$nzv3ikn9%^0QU`qw`_Tf1uX&;!kS*AJPrbMN9_%ky^+8e?_e~AU`Q{j{iH- zTs-$DIiHLA64Lvb{-LHh`J{~3b35jTnEqEX{cbat^SgfL&zIPK0?7x-Np(x|-x1ma zkot4|+y5oj|1Yur@5{$oU;c$8|Es)y^M3(ibe0GFf2aI>)_Xtqg9T?f9KPq}-2UgY zIp=4e^Yhv6{`vkP-2E@v&NsmO?Dr@CKlJ%OZ`b>m^?uKLP<+YuZ@X~)whLcc{pWbj z`cwRy{R73PT%YCQL!Y0Y^7;8Wu7AjHk^k-&&!2i=@C zCq2vif44lYpY@;RM$*3&JWKp;KeUyx&hzg5F(MjF=->b!W=xnwH_QY_f%WfkKSMU~a0K?n7 z3mlyh03QG+VuY2CfYUHy!6(338Oi2z;9R9Fe%!SChTcPd$Zklek>|eb_4paw5Z!Y>0k0)-gOUrw^z%% zzW%JYmeJqojs82`0l(4};a9qS{90b&bo(d0(f7a7e=V=C=C9=sK>nm3I?w-3w|^ZM ze`2pc>A2$+Kk1oZ{;c2jz)!lu=-kip4ZrHe?{ws^^yue&N-(?v>wk^3$No=~_D=Mh zig3;cfo7pR?#6qf04iDc*k}`bf6UaoshZB+OHG47X>_0qWhob~RKM>P0UON> zJkCryX0XS`L~dg2KFF|7$*{KwNt9E1LAM3rUV5O2_Z?}fEWw9G*b4O8VjjYXLF2ol z`6}tjM?$)1?B*R(nD`uljDSe31M-T*F2jb`L1+`ej@S+|!0)|s(PiMbFx?8&e_e%M z<(SES!$$4x0#kH&&tN)8d_AjMUTTWxI^84d0$s@2gBNn*f>bw(19^sYNJY7ub0rvq z=X~{QJLh&s>GaajxG58MAsuvbH?5bWVjJ|hz-Zjg=;jYP&qn(62vmsLB;XWKkwgqw zxuh@<4Jhx%MWV+!hJF#$wcdlre`axW1~b=tp9zng5!#_4%-nLs@hO-{2VOBkR5~gf zTlV&EN{>i#1v1gL=VXz~S$3XLCbX|fv?7A8F6Mo|-5C`%Afhs3xK>N0k$K$clmfWh zoiMCT@~Spo(E8Fx+Z!@;9;YFqqE<9XJc+nnTliS6JcU^c{J?V*g3v|He+V5mP&{Am z=n(?jIylq;o}I1?+8DMRe;HC{Hlxepzn&MRGI@pZO}Dvt2(4VkCK!k-6IBpRy5{uC zC-68_pQ!*>~2cDV&lmMi@DWdZc5O3 ztd0d`yrg#Yau5I}Lega%Jc?-Sri(K5em8j9BEyrtZ8R)eav17ae{?AXQBMWosz`;$5 z3Q3R4nwB+EtmFO6^aRGzJKrQ(y@6L>GwZxU>RotstCwSynQu4nu%p+3aKYrDQDUc( z{cI?^pkO{w)o#;re@pB`|C#L>x+70cC=Uw42qGii8)>=8X_i+43!l<-#Capl_!8im=eIYA4`l3Z>vH2Isbl)_e?1^xXo0fwyL><7ciq4J zecwAW!cq8H3&@N`$}TANC|QGxF{Pig=gR=Lc0=r=(o&^sb*3iK((B~evC6K1gIv(^ zxf`ayjjXD!)yt(T`3(paVt7d%A9Ld$d9;eo-O2b!&4h7&-@Hh4K#NMuDh@f+l$i{J zBHJ`*obRG?e^VM~s`2NPyVj+!{m|id8ynvK6S)g~ku`W-7ZtBIytDA^usV#dDuu5o zh5u3tKN^Y#9aeKXYPoJ|1Eb>ivKrR1u$8_QRVjok9|YK2 zy!pPu+1p)|?DZlFa_RC0Q|P(?Hc{mFbt;hEz$is(f1o~nH~sw>ZdlGZ7ZIa-y(xnW8(b1_OtgnwoLT%q&mB zcQEBzAJ^1!dzYAP=V*5p^Qo%v_aKXR6SC*sHl3E|ZFZ7&Mq|b8d2MJ6%)6YfR|m_&_YE$0Ak&LizLV;@ zqS7nHFgqODc9K0S?KEG*>o834*Sy+U)rekYdoC$FsP~*!opDr${TN@8lYj_P?zf!t zGB63gx*pxTj<r3aACw3(rf9GxP9CK4>DZIE{5ZhRrD}pHS%ylb_b=%G`4>+4u7?@6bR4_w;m5;@QO3P@faWScc35^PbO4e?{Z`^qrUDBL$x% zROt+{mKv0QA;}VJ?aEiehXp@Zt)*JHe?Jx$4+xyRA%UtgY29!B3%GHmzTWIRzNo_u z5mB586XH%BgCscS%eH6tCV*FK4x;Rfv~OU*k=s62IiRw(%=QRQ*fP)7>Rr2sA>!Be zeQe+MIyG1ys@y16{-u6j`?W$Mq(&mWMxv>fe6#USbWmXajGu1zkGVGiw%$o0-w3f{xr!%( ziBH~`-Q1~rIRlb zWH5@#YP+12tlcj9*m&o!e!xh=UR0cL6C7CWhe*nDJm`{iL)(YcEPZY#w-!o{*?EeB z!8HXckT#H+bW|8=*;tlEe_7i{dRr*O+kn0~mbx7%OQe~(=}Q9(KIvC;9=?q(R9*Il#e)}>ZK9Hof6GcIVNF|GH3{1Y z`1ndGCi%E0fib^f>R!oLyQnb+NpJ5rlD|}jrtZ{xPn%YmMlxH`*QW3{-EpbQ{gx~E zXo?c^AP#Zo2M@hgAqh18; zLI?ucFXv4n&O(Cle;{B=v|iA$dN8&2}B071taC zsYvip=BGxEWbmLwIc(!BwV85+Ug*59kJrNPe4g_>RAkU{Fh>|D(9m?+7Uw&>5@CfX@a686mYY9)xe`)*ERNrzya$~g9kGR1a ztX;rFN6MJBc|sey=PHjD!I$zqzHEk_UynKEW57E>d3|>;VOZu1T+Q%q;ctwltS7IJ zyJdhGeWusq_Krxpc|jI}W|Rowj_a4G*Qj@3pya}**h7hV8e;Z=PcL(ZN?DSSaEQv{ zsAwQ9{MIhof157VWHST>r)6oyOoU;(K1u3blJz{=)My&NaW!4aw!(M#GkBclbLd;_ZHvy*^NbGJiNQ2pB7v}%Zt)i9Lyi9uUssgmq-hy)*A#KgGP~<7PZJIAze_ zmic{7Kl&sbjYi%tG!%Pff!{#%#LIUf>BNzxe^#0mt zP-ks0r+HMtGbq0W6kPNrz$_{sDeR~mj&~dxUOvYWO9y_R~ z7adz^v$b908FYKEwlF=o`VwC;aTNw}S0S~&NM34@Y;4I1cYlfyD{boe4vTpli$vQ> zf8qHtU%DF8Um*Ghy~DOlXov~DFkr5I=(f)awqNiD$26xUp#agL>CBoJ!VmIQuOkm! zNg}A@of;3HdUISyG}{P2RI;b7*NG&ix9;WH(z7=o9XF5n88a^kVZoAogE~2=2OLa6 zuF7$JCGYFaBE-_c9&})H7`L{;uAs+be_P0K-{*Ig42TEI`Adp??d9{HJ5PZZL&@VR z<<3(qb7FgPnTqE$$Z*KD_JI?hwK!6gb7XjWTZQcL#u_8IcS7h+>M9X!?TjVOcDv&_x#KOyaJv!G6fRhT=-8tyrC67Cw|h|= z)H$m#!^*+n3l<#`uv9PuzF`LBibZy}_ns4sZW8qO5;~R04zVFi+ki zIIP@P5cOglqiCC0m`*15HlOMSe|Zjp;_E+bZV<@(m53#fibM zXa}-X&ZA&djQnX8@p~Z*h3GYid92*}#(kET-__vl7aS^=z?<`0T5l^{P}4RU zE`h!@?PAZ4qP_8Q^YPcSxkUI?@MGy7RoHl^xQn$NdpNYQKt(Ag<_t|{fBr)&5J{tb ztQwvd5rW7OQz6pJ6*wFnaK4 zi&$)WHH`_%AAuhR(zOvKF3TdBBm^pbj0j!c6>Al&)YNml9_b96HDC$ii|clWKHi|E zP`PDOwPwCMw>HG@zB6ox7GK3&evCEl3S_ukm6A|++I=Ti>=ocpe?RAwZ&33NQ|TLG zGrQ&&cug&NIT(!08Qy*iYLjzRZrssP6`xnq0EE$dOZuvo>FPc6P4gT_e7%Z85@POE zT*b{L^Y?{n7QOJet2&z)obtwnr~(2OMdw<-$ZG>9>q$-zrobaQzej{ahx>vC z57tYDLAPi79&@57mk<4>-S%U~w|Kz38{EfpISsecy{HB>e>=BnvqeZmIzp#RDp{*> zu9BUrYmzEo(Ch93H<1TX+5}+n+xs?Od|yM~W|Z#20#{2P(SAaYTS{)xG9BC1RE<$> z21_H7^VB6G3-xuoUiE1~qA_5V1x2YtwQN<|C?T0tZhz74J)fJ$B~c$pc4?+mSaXHD zny%v9@F6{1e>ouh$`0Tm@E0W+lC4nGwFvW7v7^b3&PPmQ%UbN@jws4*U+A7?s{v*>d5> z`hwt;zo4Y3+3X5DGw9zNYKM#8Z_f?hm5@SvU9?uie=93xeAkOy-hl|`SmyFa4wPl> zQ=Te>?aFdG7B!S{#T3TM$#VGsh&VmNe6}GKWx$2dvkY+BAB&#^)p$CAK9_DtJ zA_CVHe-|t(d1yewxpzZ1mw@05XE2w6;anS|xxx`^vYc*NpE3~tKVtJl!yMny_waUJ(q%x5!?AUd1$i~TGMP!7-QLqv>gQ&R%}hJ4(x zbLvG>!fU}Kq1|aV)(=IayQ1|`&ReL)3{qum@AZYJ173a4Li9E?m8RDAKz*=Ui)Nn)Si@ma8wwd#%-=Pj2F9Zh`jw&fSQ38gdPr9EF}Rb<`<0+>8Qy~$*y1rA z`j{fK^U!G;Ux5y}=1n2#a4-pu>0l79eGj+$2D$fKNx6(S<&)Y1V%3q$lNoSL{LuZ zqZqPXTjZg{@V&V^+jUGYqcLm{PdpzEl_HkFA6txX8+YetO|X#Zof-}&)Q~6xe+8X- znnV$>3bT5SQmkr6*y90u1k&>EqL#M1k@N}d5V{5BFS;1toj3i|03(fuuNsHDy-ho2 z5fbg=C6o_OoP2!IE~3Fkjt~CV+4b{%q@R0f)_={e8_OT;x)X{4{hM7UK5tIe=fifo zn^(FZ|7O=S`+s8B&FBBft{2U}e`nXvk)c7r^?r3s!7p}Qzy8UtFA9rdzu0vw{exXs z9zcM8v+L*J7=Zb_l>2-pGf2^2?7IH^yy^L2*WJ($d>;G7u5bUquD@4G`^~QBAFI9i zVb{IKOZmmF-#$m8p8LZ$sd`Tn=@+{mM1RAsx4{}*v1@gp?^R8V&eejYf7oPVYaR5h z>C^j4hH>XWeoky{{`7PhGdfRNzT7(49jJ{%q1}FR$a-5U8DFkqw7s>vCk7J<9c%J3AFbV_wG}>} zplX6y@pFMiJjM@hdLvgC51r7eidD8#Yc&UM=_PWJzJUbALWA@0LX3QdrB^tm0-8RO zJJ^*5vCkJ5F)tGxTqX-x)?`5*-o@Y#0<+`7bT@*TgT z`G-V=J&es@*RO|1F%Xci@M{-e0KMm^`k0#rXEUi`6Y6ql<4b8tF}cPRKXjWyin;7r zB5!u7d|dPrKsxG}6mHgeA24|-Cg*rEA_?Q(n|?g}=pOR>i>**%9tIgX7^zO5 z5cH_zY-OlY28k`vRVm~ET`m`EsS3iIxgp-{fAgeJl7_zvE67NsjC}% zWBUCbr5*9S(jD%1wvY}pYGRKrzAXoNNlfV7@xH8+)TjotXm~g2Fk0i8EE4@5EqqBx zus31gHqHiJ^Wf!ta#J@-eY+#K*2UUnXD67VU!0VNm%AYh; zV6V(l%UH_at@aAc8KI8{lZb?dg2@~0%U$2y?fK|eK^wbcgmKG%Sr3pSwzZ~F8Y0XW4py-nA*>!f0^ap%L8BxvWY|X&RVlR4aZfoGY7L( z2GA|55qsA|4PA;hAgStpcbDb~_fj!lr^cg_QQXvxv%ZyJY?wMOCWME%-nbM*{@5i$L_-} zHT|9PV=#0I-)}S0G$xeW4wp_xZ!l(E1*_f~46xod?JrzLbwDwyrTDe^>{d zN6Yo5UAFbe@W5Ff0gUe3LMv3Sg$v<6A9%;Vt+r8Lp&rgDX%B~fDU*oQ)4q)NoWjkd z;wn55<5-)_(yIvGuU3*rx`&TCL++}FyGt)kQr-pSDwKRNL~8W7NHTTaz@;1XAsxXE>oZlzxqwybOFFBW6y&z#n2x!&Fd&Gb;BYb7d!XxfY-4DL1RWh+rrS9db-jQi}ud8zjs7F-_?K$oYOr#R%7`5`%-MlEJPzJ#0 zXgy~Q!gpKpI?e18iN=1Hf6R2FubyfKq9PLB++%)(Hhr8p-i(`kk$Strw1nJ5^NY2O zmlN8=L<+nH_r{GIu1~#SK?F5Dm$GnXj_aB(FuEGRq4muJQN>(v0owsF-n{Fq_6psO zl)0nB*mzuU1M>ssxHjo#74N9!z~r`d4cZ{o8E)UWWIEMlTd>FFf9*YQWyfP7V|Um* zRbKISJgV_#;YLzg1aWu(5JeACOV=e0ddnM`W@%ozRz7(R0l)MhEq6;~Y&*BCw0`3R zA-($WC28?An8N2l8j7c;=jy#{$er8T=i-i+ zM~ZAcGDldVx=qcve?r;n^hoj6BNVTMf`s*A9Y;P|$GuCDIEn7`Vue_~aim&5UjjK$ zQN!yk$@s2+NbDdn_pBJMwoz^&dVnC$AL8bRG+}q_t>XNwe_7kL7R!A1AEj~+D|E7X z=h+a3GGjX-9CLr{lCjE(4W3|5%z$uLhOFw1n~6tc^K79u$B1gU4poUijze}Ny>4zt zaw}~Io#Qv;HbLi*!_GbZ?7qCK&}G(M07#&z>r-QpOx&;2JzG{teYORDgEkbyMLbXs zeWu@2?dI0Xe`BnAV5O_k8X)0PF<55I$ZD^!KvU7{KTn~faLjR zjMNAwu1~I>=a~#{l=d#0j<4vADvdfanD&*#X*cLlo=+AH4MZVdZPXZcI-l|>mSrT3 zI$RUo>T%@Z5xAS6JJ|GM#o+2NBT~?@I7IsZQnDOle{f)K8yGT@#dx~$hwC#=HJbfY zBJ!vt&D59+RZE10=F@H7qlnclMvg~8*IIyyFk>!r0AV zbEsaWm#?_(;ytMeTk1!{1+02Vz~!ymaw96PeZa*-o$EAo$4b+$$&DwH3)i=JJi5xI zc^gBXf55B(QAeBkp8F2i-j<|nHL;+r1)hICC4Ov0i&YqNeKx39wQG2>6P3FLJZ4_W z2(^rs^>JKcxa0Pspj(J-`Q3FFBwA{PeO=Q{L$>Up>z7p|pIf5Tw<4ngS0rds*bTUo zACV=}M5?ipQPYB5K}d8lX7lpk@Zwek^>(Kue`u&XM>8&5Jtb289R1 zPj<9!d$a16C+Vq)svKBB!NprabiY{h=zKk_iwdC--CAVtWyo`l5f?sScT;8?rPm>Q z-#Ek9=XF)P2-)yX=>kRxo7QrDu9}aLg8gV#DkRh>n1mQ`FI{jccCl&Ne`b?mUkx#& z5L#~WZ(*_8?c?KwA258<;4lTlrxX5s!SkcnpesGR-Z1sLTtvLR(Q?undGK=Q)GtD! zJjWP?o94}}41tYXh~lrr14K5y+FR?mU70$h2G2c9vThD{N#L>Pe-IbsYA|eXbHehz3P7Zx%8+IQt?%&tZ5_y-a+7NDCQNne zELrd@ELeq-vM_{h!3x6g&SbmH%;IuqEB>mQoay@W-Rq)&=Afra`{n(;y;xqyUFU{` zqYk0dmFh`gG2v_&MI9sx`*>^eISleWalito}8UGb}OwX&M*5ZH|0E0=VbbMLWh z8j-gY;engShTvn~_?oz5d*@i#X4lTHTfxo0$uIzZBte>zz&nF5`wrIf<7N zvjk@mg=YmPok$(he~8aZU&3`i`ME{BDjC#wKVn76b}|Nt2piD|SaXdXE7q=&%9GH` zeodiEsKS}rZ~`M=LjE-<)xCH=?Ch;GV%g0d`OxudW?s=v66MhpMNw1RbPX?8&z?*A z-DQGF=nG&A?v_2Vy-OYKwb>H1+2&oF>a^_=2y0$9yXgmJe=eyvsKp8y8Efv&k@Ht0 zH2HC7O#mQ}J=j9P**fv=cIL+-bGxTlKJ`~V_QzDZdj+*6puO1bg%yZN#!7s5qRv=7 zX4Cw79V=9yU$WnCDc=*8>JK)`2TZ^B0xh*oXxjANDXL0VrRr&6iTkfgEq>Vd) z`u)5;0GLN!eS~zjL*?nr41n?Z?%{9==cn((|D;LJzRWtVX9&cASLF zCetO(&!1Jr+s?aWpap+=JY!axaNJ@$8e`6^218j0oSv_%aJGZzNn_lPj5 z#0Qj#e=*{1V^b6Ob(i;wiQs!{f~SwJ>qs2Ca_q|0mV8>4)=9~Q8j87fy04x!#9k@G z?Q<<2JN9HZShLRWoGH{;z^*NzCK(Bagt!+^21ak{yIQqG z?EB*$vq!(*9ei&fa`IT0S=2|PEJ5V=v9UHybvPR9_)Qa&Lc{uPH93?+URLj!mIA7E zvL&JT^WsTwTb5dUNP*aCK8n0aMssFke?9m-@t3Mgo{CGo4fPm8un(;?K{gsSL#%Ye z#pP;rWa4C;r5U}xG7X(vZFu2{mNxZ*j}IAacUD$UNE`!x48m^D*Qat={Oj*?0%WQ%#k6e>dB! z*WGmIC??!qFWjZ#m9UoIIYoARA8Uq_vF9&tG=A+b_el3cK53qA554$BD4F?s>D z@z82D%1LGxk4gV#KJUr@!`_{2yQ-s!qJAJZ0E2DZxD5$JU5V1rGuV`_u}$&xX9zs! zTx(YuCo*%~(});j$AXCrNC+VWfBat|g%Te3tF21`%NX+x9@v3d_==1aSn+|1nlIj@7xIOvJQJkMn$scIze4>1bLSc}flMv}o+rDKhjQ$twL`eq-h zr&~ou$@3Ie%RBO)=y_S*&2e+E6Y-5>X2>qQ1?5+io|@E>jv0BsPkz+Rx_X-C+J~1XCF>%nFM}!OfAPv#JLk}-GW&5yinSHQXi2BdVs|iegI6w(1rQmE5id0d zPHP@?+oL5f_3(1k2ghO8;$^cHM{z}uV!bXQL*ZhvxCZhme?HFMz11m)dMD8Xj*NYKJStq(4uI$8HBIT~=##Y2_=B$``jYe-$ul&~1~YMla18 zUU5{sB;|Smohe$q_Rz>DoSb9010b}jq^+;xP2IzT z0y?v@ zr3a&1i(MCuo5Ghf6Xkcv`19%ZeHTJ4=>C5 z{aRj!+xs3}gyP-Dsh#(^QzeJ^bXrLFyf{*$x05AsQMrS|pC8_H6FsO{Td{_kI$I~+ zEu6D|3+u3AFs*myXgRg<$mq-Uz1}ywP&B-yuD_~vx?1n0(6Q{*xQ)Gp9m~b~o{q&$ zW+M9f#tgL%fA>tCK2GiKqA-fIz1)_L6-LpLS@hkzk+PE;ZBENv+g77I= zZ?Y?OsEX?%XDL`LH_LRrP|C9`m%Ghnqwk+-_H4LrfAxMv?=0C~=%=x{8$0Q_4c9As zJS^Y7^c=nBnOuf+w|P)1w zr|Mg(^&ZUg!U{VQoht2Y(aW${#K(3&vNe==T~*bqnLS29E}uufOGfsr%TluExslEn zW^r-ef41~s$J!n#Zt%7Xo;y;M?NC%&`r&Xt`={GUX>umKpR$b+8$HMM(T!#VAKh*( zcRuN*q;BticsWUzzVNvr+v@u*cWI9SdW2V3&#!D@%Y}G(eeSLS$SDc{{=ZFohvg|J zG{vF)p}mFaEdlaWoWuh51%Mi7;?u zf5F6YmfVV)CAtCKce3bm1yi6v%$ERk$!@XQZ#ZkQ1-{L|!bD<)uGC9hfSd)?GqcI@ zRVjuAdw}v{V%dF5j7M|k4hWGWz+q;&tq!;dm=i4{0NVQAZb#d&Lnd(jNQImU6;*{& zzBxVKCHr6(V;Ye7G&b-u0T;>^cn1Ad!G06hp#WIYQUTpwlfX=3}A2^0EjFvAc>3e{UjS zFty9Twj=Ayz6lu1H(g}smnU!yld4x{w=YR~A4qvuCdvyq5#&?(PPAwI2Gtk+jX!)E zFIZwgrkM|^_F=zgno8`{U}CDuVCE1Anb1$vL?F4}6nG?IwT1<4iH4cal(j0sLAwbB zi80?)unUBVymrG(A6r%EI2m-se_5e3XQG2UwA+uSEEGek6lfwNMLNs1q7bPaEkkij zBk$qxMH`DuFoG_N1z?UL<`aQJjh(bM=p@T_n+Ozyx(-CK&jz=9^m@Ip#R9lpZe~li`oV~~i zA419d{L5Oy7gzl$f5L*ZnIv|!VTz@$*< z09;Agnnu;$jIPzPiuQ$x14B?a0h$yf1nynz0!b=))^pQtjSt8g+L**w6LEmV-i1+- z-*}j4R?kNe$F_hQ5dw7=fB2>a4vlFxC|61oK~Yi^0l0d*8Q^HSQl}!&k(Z|wYSF0- zSac#J4Hp1VE%20ovyL09!@HT|R%Z z&Dv@{ZTF`vuqQAGO^Aie{F#}(^qC2| zvws3xlW?Hik|4O)K}gu())WUMgOU#Ai9vm!JR>Vzp&mztO%smC)L^5#sC zb1~>t13WR|%Hb9Se+DQNX;J9w(lbSu$&5QdqB#f7D}pInlTPMdoORsvk`3 z`^1WnJk7-OK#1r0JThlCnH+OL^~>V1K;X8O3uhs8=$CoZklp5Nbyrm5yC}UkTIk11 zA2jGmEgNbdhE|=J0(NT!a)vm@%W&>3{IXx!U+h;rGiYN_N**cGiE}>B_0VKgHy}hB zQcGJF-p^6Ef6#l-wTbM-tWYSOa5W=<6-MZrM5{9$4>7S^wnbN377OzKJp zE|w6VwGLe@u@k}RDn#|wQUNdOAiG67(KPdoOFcUke`LG>u(IVX6W%>t^KO@_3%>cjig` zRRG%ze~C~PsiW$_9+{vBm@DQz6*Qjg6^*6G_t06FP{BGX{FN_e3UoalRGlYznTkrT z*Xwcnz5xh3^9*u+J`TDnwjjhLWEOyC0Oz(U`joeaMoygB>G-Q%Bs|dPdhpF($*@z! zU#=yWsjpP-FBVo2>aUhqn~Wkn_hlLHjM~gqf0rMf!aT}-R#?~+g~D`D5eTY)N*!`Z ztb#(Sg4PJY)9KVH9p$zUu`opmAj_og!UCusC{ICf4PB`$1=H;+6&4S%z@h+D9pFcF2MLj4b5#R zf2qeGhHa(^=XDC;hm}2R<3uk;w=obPH1{IrM$7FU(KU{wfg6ttVO4;sIF%Us_ z6~$haKm|(GD~@s`02Rh3Ltud6^TL!4sQCZY<0A{CUt|UV)8pc5aG+ELIdr`;M3Ict zUJ=wO4FskNh;ko3_rpq0sl$6qUAAH-e_kn91exl6XklGd-k*A&U_gO=xu9)x z3^JX$%Tk%W%$_fm(OiI@OQiuYfZUUGya%ybs2;j1<*6tH5NTfbB)4{;oYFx`fBYY; z+YX)T9VdbwL(zBE#k&wtp(dfEOMx55xJH>;fmrW21iC^H3ZzBM!)R{r?UG^_Ic;@z zy+tPM%DK+|KU=To_FF1IiUs&r^_uiKf8qF8bIiQ2>*>Ex*KJVzLIoFD4LkKubszYg zU=81FbujgLC!Q`oS;7}F%n5LufBL~oYy4wfY$dU9EzRRzs?3N&afD=LqPoNIuroW_MlpVrO+m_T>tpU%oLUK%@a#eGp>Vkc zguM8iqHY5CA%f4KP%mVR;Om%L1zh25!^0%-@lxh?3BSVA+Gi+WS` z0XhM7uau?&x_rIDW-J$`B;o5H_9Gx6EdlVq?t`cQ{E0`bChpKhe*$TTzw4~~+p_uG z5d;|1q9%3T8QBn%@c}ba%N7BQ^p);v?s;tp06_@bN&)2l0L6_8#3x2L^pD5LWqj%? zE^LflLJPtv<|eN+X5^$Ne`n`sMH0tW}u(!9Ig< z$~VX3Y5$!Sq8jwG|AkC}w43)I>=t&;aRPHQYq}XB?tN!@SCr5shh3SD&paMQP2*8` z8jm9PAT9?`zHt~9cGI!hntb7Pn}zpYDS(V20Tb8` zN5MTOK1uXqu>;cwV>6BXEAcwF{iIF_-~9X%dcRyVe+U^rkCXm#KOZv|AmxPUi6SNI zn)^CuaO;jn-5SO6k)^@)j5Y5t2yUltoW(lmb*&M?ODFAw(R@n}SD7Y* z-B%p#k9swn@=Ly0jloqD%~2cBTL9$R7(lVsF3X#DIw->W$eDYHgLqu+gWJySUYD2T z76?hKfA+XZeR1I!{@QT@D6UUqo!nE_z5rJ;U24hbqA8eFb~UT)K!%~=bf3*jkHTh1 z6j~jF6+jue8vo1XhI&+!zk_)^t^{6OYdtDeF<`VRW~P2!fn3!Cq2!YByLmhb__&-| z;raY0ZPf*GFS~n*`)B|3ssx-gR@D?xx76n5es?1CzQ)S-bw)xB@$E9LfNEIRnTehNHrh`uhX ze_zD)*BIyAPx&Tr@n!0BPvdC;8Q-Wrb(D-RvDS!5`JRENevOIF-0&Dft(xw`BjBrZ zIr}6n8Ru7lhec0^sL-nifv zF5d*@x)twDS>&57tcpVNdPgX)F;rDie*q1Qilz`QX+Eg_;AHBYt1C_^0Q2UKlf5Ur(+bNO|}M*pU#UC=V(rGf@Kw$pb+k$(QK7MAf^6RdOo8=qIo7V_S6^11>S?2Smulfy=De zJdx#Q$FRF53na)Zq;Dbje_re`F2syv!BZ8?u~I7sn%9)5;s?4}tXPyCx9!yzX>&<@@O? z*+0ouVe97NI5Gh6;`*;+g!71uD{VR{_{zlh(_V2+Lg3FvFFnr#AcR<;!6^VLdA(R9 zap}bZ3@xw-Jq!V%j#)w-AG#^MxOS+`*W1_>d0y*uF?0we;4gq!e@Hg~*qJ_7TS6T> z0I;wo)bTtYc7P>xvDFc}c%BC(dE%!M>_R|f`_mIStp*AZ;tq!JNT}o8g{*(o0dD^N z0F*#$zYlc`Ly@1Tv3ODXa|;cW^ao+P6M zi65P+&VsK_mrg}e33?b-gybq;m^AdTjt8qETx1IfF9$`H_I~AAp?@*)W%08GYO4;Y zCLACbx*{=5kvuoSjol&IEMgf7Emi|e>%E38u?3nKDug=iYBEhYyB4_kXb}8O6Jw-z zY7}H@O2}e+5X%7KvMWehLi(W&`D7oyncY;40M!+rm(cCMsF(q{b4z)4FXz z=-o~t1YNRkt72w!>3?WqbydRSgeNaz3c@U>)u||22~JcnCd{G%xS1L*lgJak3F0{N zI(_1WKP)dD6caxy$lb&<8sfyel5(DDup5qKkp6JBw}~jWW@305!)HOLn-5iI;w52c zwYUY)n0_nBoruiT+koTrd$=;WG+QLLs>51i( z3;_~C8EKJ3_*?}uFmAy_=&33_Q=lK>RyICm3Wl~w45Y*j;mZn=R(8W8{bHJ@e>VWc zD6CsRs7Rp`kXaBCb|NMH54m@e;#dM{9BN43k;nTiYQxFLqBs3XlT6NCnKQNW>i}KYCL)F9rif*9Z)l>x& zYdh7iw*8B9!TchRXC{#lqxcjHP$E3<#9#|^T>$XanS_iJKuYrV51g>|&+GI$jjRXE z>-jksfT(#^=^}4tKmj+M|0=DIBJKd7LmUYxRR<}V=zmkd6RK8lQ`p1-ezp*_Wc!3@ z6zPc|#)Kdi31OWAjOqJS4+$OnsXs1ZkH5HC)gX+hJu|<7RjBHXka(_2tF@5=&md&2 zflz|nL^FVucotzceFHt+2tEC;W%3}A<^E@RK*%iASYc!!4-qail7tqb$Hb+Rb0pAv zu_Kf)!haJ{3{W)HhatohYc)~B_=K-_o$#A+3b0-w!d92R%#->HFyTt4bxcew)+5y~1!Vhg~6kZvHD>#yW>pv6FNLpN@rTH^{zE#SyGWvSCS zq!zJ}#WEQy!N4cHF1}9X8ch6YB>$1xjYxUTlz%WG)X+>36N+r6h7&^V&dUj#hUewO z(8HM;&Iq+TFDF!5HdDma&!_sfuF#6(#pMM~Z z&%YAKS@}gAgFmpie@i)&vSGeB=ZSJQP#_4et{U2z%oQfaZl;I{r#4f=HDP9FN|>;9 z=6_5PdqWRrYWQpzNSc?E@{MPTm{88cwESimNS&9H@@=N&2BC646$4Gf4X-BGP^-(|P$1^-NtNP94a#y!h7$W7tl|$E<#-gP4sg*60j=LpNr7XEe4n4HV;NZ47z@~J?o zg2Jev(vn-5iA6o@)fY1=l@)2zT7YW&P&t56C<)}aUVkNP(Qgbb?f`xe*7SL*Du1q4 z1;AG|* z@K<#P3Q0Nlb0$VvB>$4WlROZr_kY*@DM?$z|Ip9sbUlP#{KLkM9jQ0lU(>YUTH^}* zfoffLPW3|o&rGv=nva#-0m+Sx3h<<_Ig`v^%ShPk0B ztiS5k-=F8^H)&0-`$h_=-}n0_{&35X^i_)F@mFPN{?tJP+JBAzMeV;v3V*Brw7yyu zRuft6KEMt$r6)5mi3Gq$-Vfo4hoQ5p_K#_?%xHf~>k30>4=l4e3tcm4@fmJ3`AAiynWA{boi>d$km*)@3i+Untw>kh`9g9@;K;c%FGOe3j zcf#{e{wBus`YRtxC{th$Z8_D&iCtC+CI2Vx*_&wC;EQ)B{cJ+G`^Ez-^=jF9)qk;^ zY#X2eZ2(K=eR8Zl!V}X|yXkr+X{Tka^Yom4 zo}Q`X4?SGWRPsy@-+yK*`9l=XRPrZXJLi9pp7`Hm^~DfRX*b^t@szgvB8#WAxi=HUznn9RZ-0h($}9iH5KsJXA$*a= z6aNdp8RCfwR=*hHIqmR87Efu>Oc4K4j%x755KrY&R$mM;DHlQcCW}d#sKpmUJf*F? zFNSzZtACTlQ`)v|NpMbb-)R6@-lzx^mPE#8$Tcep)JF@-Qwz){P^zE4qelZWRyB|@ z4~$@aI(Ojfnt#U%UlcL;GewMmP|Z~FYZ^an|3DNs(|Dk3pk`)<;Ws~=Ag|YFs+bdi zphm_gt<OQkN>co7Lr1u zf95%00cQ`8NeImjD9ucD%GpeLPUD?~5N^mPQ3H8j!31DM2;DCN*8KO0TK)fOyBN^F zZr27g4g5vNssKW=-k4CUghIw}^^bIL0lkTe?I1&;Hm~9e@6Y82$$$Sp^3Ei~W(}nTnmM)AyXunRGQV*n9|CnozVch78&7v?HW!^G(L; zfINDbC|GX#9U*-Gd&he^jxNc^ykD13`TL~&_jsmjlB1U?g+8Vl5(F&UsC^zxqRm1 z7k?ALQ@#2kqtQejKR*QUKi3a(zJM!PJ{{kz`x9~OlX6SQaaXdr{>mlc)FPek-Y0!t=2U(JagCgnd7HLOfz`Y@H_L=BVn z=(C@{)<1zK`IZ6d{%2_8?;6cRwUB`XPgh@8#n+2`m;rWboJT zM+kq&VD*Ot{%qf(gVhiD3w}u8A2N9OS&qM#Tg88u24??9o7*}r`vkSdQhz!IuCmlD&Qvr1VAI^z62e1|scvgMyjVg|_eK3? z=(*JhvY0V$S0@I}&g}cHErV#g%lGWF0Lr+o-naF;5SD|;QEPU!yc{kTy`z0&(;kOg zQzs|0tOgRF-2)48mJ%xt!R3WCSh zDX&&5Agg151MYNjQ#R~zVSg^@*KttNeVVcFCe04dG0UmE8WvWMQJrsJ2MQTb0XvPp zG~Bo_cghmIsU{jiuGx6a`{ra@{9t)$?K;uL+-BBV)M#)J8CX6G0H5cpdzFN@(?ak% zTjzCqP2{EAtmGl|ugg}uWZrP2YwX8YcO8juzuWIornJ41zh6SAV-$7Qr&c(GyAMj4thnaTiafCXuH1_s(o2;v>*s$ zHI&vu-5={`0e@^uA1}&D8D3B4p~#)kZ@o3-o805fyyDRC-q^v0rBk2r1Xw?Lmdkc+ z&aq=yJHh>~)y1Pe$`{ysGPjR`F1^C^DCCW(*5GcY>%p+pRCv92rKv2bMYDW2)~&JK z(p_yWMrg*XjjZ?Os9xJTKEM3_R0iW-MjEE7=-zXObANX`>-D4aD_{17@H$!`zBkmk zvoD9Je?{I|Yw%6$*qgr6cWeAGo+|_R=KXed04|$SUpqlPD#2xg=t4ns;~=`yo9n4) zf`02I=MWsk;+E%!LDLJi*@WjYmW3|d$s_gLof})dE#6twE|B>u%eO_B+gF*7kv3Kk zp9L4mo_{KxvDy^j4O~3vo&}0pmyMBh`>^c#(?j#@!~G;Z?YKYfUt4L3tyR3``?Ct# z%4C%AwhX1+LR1vFSfi{i+4uSWtPVT&uBglVkogT&b-njo1*3mP&UTr=3yS=PGke~= zoi=ZdZ zCptNI8pqTW+$!U`JG*-g^4z7e&iFCjwny)T1yLOLqbwf6ignm^n2@HK%eMivZVPtw z>$}SB)4c@F3)w5f{qEGt2Z@#9G08-Aon0Rl*O+~Pgws~7wd6K7@9GJzz5Y5bw)9Oj z-+!6z_Qq+&`e$1g&LP#i*OR4t_j}N1Yr}im$UjSVWhBY%c}kAAOs~NBJj%R-4z~-w zrM78hX6tgUyM!&B=@NQ-XSD`WcG*AfU0`r0u~g2Nu<2fm+O=g`30d*B%VYA22m5(^ z0mxR4iQ5GM@M2V75a-!5~m-QZ=|d+*z-BS&uhBH4VbowqN$_=|=U@xG(p8bvTYv_&j9k>V+=jK!1*9 zT)lLaKK5_n!0ocF5HdQX4=GKL+hr#@ha=s*n)IEhk+rUaNb=8)Do`;OIKr6^a?zKP zTVh==%-xXu2A2H@7gCK~u*6jCYCeC%RneZzeQ=WoAoIR?+Eqn$KZ9h)^+i?dwR@6p z2BPJ|?d0adUc1s7-g~_Tn{{$F$A4_t)6)5oRjqiq(3n-!WgB_7TU-_AlgaFSsxffl zSvOOY>8PpW)JtCC&159{6_--RD1E@48LZW!zOD?~TfMw`^(b~3>mF-V70=9t&wK3P zw({HNY_HzeHeB4fP*(NSZp4DFpYG$nKxZaCUH87Uf3Qe?Fengsmu_Ay6Mwu~ntrv$ zb_3vJ3G3ZeJX2V<%2)I>L#W$bK~)?6n2Nm3jrbXI zmACZH2Er0>M>o6VTPN^nBY(WmD@xm%mFvQQeJ|C=JKAozH)qkbu}RE^?sg8t(OW^e zQu_-(`mQ3fM^9Uyd6A3FCb3Sex!Jb&;|sJ8>dfW~o*GUYu%h=BKw5ExM>85KDH}c1vX$<$sU${hfz7Ti!eI z4b)qMLuugP);VsE9JPBp$L#pplNQvA!aDtV-D*)avEb-Pd@3qGE4pS|a=U#-g zx!v!Uha=c{ah_Auq6mcR+dQi}bx2^;E5mn6z`Paa!4G%{X?N4~U=7B4FA+d9dP#*x44@jDu+&2Xb%+(CEp zQ^gALowgZPEjP@0IB7inv@3?TFML-mByM3uh3FhLmOi&0er~CEb5`zK%6MOwDYs_M z4o@?0HsX3eYk${5`j)nVczAfv1>o%}lxExksyY!!X_jXDZ?|xMe_WkKb&wpx>H*iH=m<1KSh4aWPZ!*)| zaMb1%lRr=Jnx5(Ob}3qBEhtT@b_e6)>>vHp4_O?$r(C0Tscn?a?O;){cv%eY%R6jZ zOJVmmEyhk!H0xLN+C6eRTjYB-^m{3M>wBwL+DbXMyFoBiv>IOa(PAm>kn;*OsFk^H z%X%n#rGMYT8ZtJ$;4H(w)P~{poxOQA7FV08t!5$#cxQOk+G*B|eLpYR&E~?>o2|1$ z2ye1sW9vxRUeep~$uNB0cSnDp-=6DjPr)Z2+~v{2&&~dLx@8=9xU8~!+2tE%ao2BD zl2eO4J!aKt2hr_BRWS2*)s4R0PR;%)WaU*^Wq+%EVDjpQIz&A<)_hcDj===3lx(4> z9Zx}rI=_+cL0NdG;Se6%?J7R(-etL!^O4#grRZIRDffICY4+w8opE?Q!{kk6!S&KX zky;i|4ND4$`eN4*d0W+EK=_h&P;AXgkea zZGYO!B9zBH-LvB5r4|56CbKfm`?{`M>lD`u4JC5=a^nn`cGOFaV=cRNbl}$gl`Fys zyVQ<54JzfXbt=|rhN0#0b#UBh6E6ZA%6@KHqjHRWIn|#M`>JhjP7S z4BEUOFX(K!5d@JKfrb-q))gh%^6Bp#seeduT=Po*l&TYYv@G=9&gWk?W9{%h%GK*L zE6+{?q}ADk;8dY=QrU7?2=dk_-0{|CCa~@9+MRq>D_`wlKkj5tpvvB5gVQ+*yGXg& zs$L%Nw)L)u(yM$$ZNebCuF^|g@n?z}jg6!4eI`7u)OCQ?^fq4<&0}pQdN}CWW`ES& zv#>9X_HY`&DnH=H_4b#oQCuzt8f358>zbfl3%Ucv`*@4slcyKVpr0;GYKBkYROePC zzK*)lM%^Bz(91o?N9WLOgDj5C3#zmFI0(rIcls7>VGMVMUfbR+5Z%QEg{&`m!2$2b z#r3)6abN7Zo|f?fp#bP`c~6`MV}Dw{OCldQl6fEYf?_T?n4CSi%C;yr_k~`KcDGf= z6)l<%(RkeNM&pC>b~3B%SqY$Dc`Q@|d^l=H;{u1JU_H%zTxTKo;&=L0O7O8w^KG0| zDsn5O-dop$7_E&Ws~$V*>|8XpfwCCfd{}yW8mSMK7gwdAwqq%neBjFPu794dQGI*g zp3E_QQEyg0v3D&uFP6F3hKr_hrNOm*N5PSuwameawjA|pbkj-TTdY>AY9&0Mhod&~ zoM>DM<6dWnj>zz-B)Pnx%IwMpjTl<%&=r%-{=VK=&trT~mZzJ~Ut_owhUXO?*5M2E zTvSw`y}W8Rx|Z_7copNW!hf!KG}&FeI|as@JKR>G6bt`8?I)q{v9SHWO#3PNue4u! z_SMWrS6{TBJO%xmKROvC2Y>l#^hNu{Q;^B%xgiE&d|(b zVR`MwKCE3GK33NodVlSVzP`r7^0mg`Flt3Q-nR?AT%zT(5QQNeU-?kmxqjPUwNrFw z&IW#J2U*qlK0xIntoM4!?q0WWe~Rt~W-AmY$M?f+;@3hD&*KZa4C{mRwT|;#2Kk{i z+D&~eU$~-H;^wRz@%EjBD{0^ilh+<<+Fd!!oDKL=f_oVILVxHRHWJmKI?>f~mECtb z2h(>f(tC9rt_!6;dK}(W>r3RH>wWinmj{W)SLHY^mSbMv+{!enPpbr6(}P_^*7l@7 z{9)u9NqhCK+P?j380vf|1Wohu+><^QwNyCvHpo&jOT>$%+#KVL7n1q7S%E454z=Q{ zE$Q*~8aJoYDSx%>l2j+I9I=l>+V}RG)ODZY0j!+e?sC`wt2J2>vak2DsaZf>{p_*3 zH3Pd+Y#g&q+pI<{gt}+QyFpwS@C=P(*}b+}-@a=#tAhrv)wsXD%{sAN>Zn2?P z&s%BpsYxHLc)2`B_giqktc_wsK4nEGboT%b; zyg8#1q<_p(>x<5=Yvv{NAOzHXCwunAT_ZYz^fJk%+nwG?67!p_ zv{lb99p*jF(8tXX7VING?23i6)lOA|xy|O~_Z7~uRwwt1S7EEtB$Yu~4Lxz7?uF#$ zVgBe(Iuqzi&u+T?{`BarIA9LRvMAH=x^WiD*?(05Z52$k&NxA5>)q0-JR$cSd!*W^ z%=DKyaF-jXG}l}1c)MK>3f$Uf>ZM-yRmL_3RJOXeekZ#l+REap>omb_G`*FTXS3QX zPH@~Z$@0eD?S+~J*b8dKP~JW;9aG<1^7Da1$gAn)z$k#H$JgK&trS7}z2AmIr^_ZE zY<~mVTc0CeZ40@3@iFvUVb`kH^m4%I!9bn5+m-9oa$B!TEXS{0p7r4C!7R05TTpvs z^|Ze5;;U43_q+7cjOKn>S!X~khEdgRirK$+XKL(J#9mzMh_uzf zs>YI{f_x(j$KpiYuX3iWqx-tMOWDnQmVZDC(xv&7HwPZQl;(IU>-VZ>7K8shP?^i0 z7ON$7O;$^%r&wkESagRKOBp)LOHi+5SDsgK*L5p_wEk~2>qPK<0FSSHrsZNH`b0%I6hN1<4{-qWnh-hc6q zZzJ~`!&|&9qurome>tc7=jlXcnaKOwEvIcNhh|-8x#EmG7RTebqqrxEbEXTN4ZC%D zC)rTENUgSVHkNGb*!x&I59Nx^H@;sib3ae?eqWzTrtp|FUPbCzO5fMbiVj|jjsbjp z>BW9O7~Jmo5RW5NN;F;{k4@#g;(r6&j!MU%I;nwDQa#1{uJdLSQ+ie>HGHfpYf$9( zUG3d=8(z!38Hx*Q>1n|e_@m47Jad-x145eFa|Z&1AGz)3!D78D3)$){r#*Jqx(j%w zRT3E%jSSRSh7|znb&1<*p_D)az%2_D{!Z;codgBDb;1hR8CYP61;-e!3V&4s4Hn`~ zufQ`CR-M6uE)%8B0$|yG=$!eZ$P1u6N#Y&H34qu+*T3wVIJcNWSLxP8_s^wR_*z8h znS~h^HXiKD}<2%=7~>6D31@y?jd9#wQW= zKy)Rbm^oimay}$%Dfe*1%YUj{)OXniB6EdV__e)Qc8geInVxIQ>ax=wTur4+wIeTT z6@CHk@2kA%K2**7foxTVr&xyf_<0jMQ^ef@C6^JzO%;~~O3xI`S>6!K%I{iP?&3~wy|4=vo zp>F;|-Ta5T`44sTAD8bE5(R&MsGI-ashh=rsGEPEy6LHV|7558jS8p^HR4_tt%|$` zK7v!wn=BZ45vb;&=$E3&nI`Tv5e(*`FJ9(F9nD&3HP1aIvehI@w^+%36Th~WK>3|1 zx&U>l+?W07?4NHsKCc$pxd=S726THh@ad%N5#6kc<#P-{b4x)XP!NB%*5`f_QWOuw z?*IF5Gd0nwA)m;DI{0B~z%=4b#%`z0X}>cy@*D{1ccunh)W2YA4wqk;8fGWa5%@4Q zAdM!bM&c{{B!k68{0mbvjcAiVK~Q6ooBz_-JFaG`dVNsEuyB91=TTF~%0EPAke-)`^cI zi|pb~u~KkxnILBob#J#au~3Aq1>&!8wY>qhG}({v6V{TH$?_{aEbL;D#0e##YXjUm z0Gnc3sPbe`;JtsNP#YLgn*#(n@Ab%lK-B~<5JjjO=)H!)duaK5WFQG{1Q6Jg?G>18 zkJMft5pscq1UD;Pd$@@6SRH3LW*mJ zDSSdWU*<64A9AELy*W%|HcCASEB;Nt$e>f@IRq{V(VTsU%9UU+hg19tt5^oYzKDU!4bPhGah;d|s zA{3?K%)40?4h*_l6yTn5D1o!1Hli}C%}Fk;+-wSZ!_;ayclEIsA_IDfDG&}6id!>0 zn{9$+JIF(38inn|ngZ zo=Jf)p8bqr4^d%n6&PT%A<&*7UbK!`X`(E9Wv(-nBS!5 z--pYobmZ4?P)x?@gt=};e(6cgd(?F4bL`E1)$5jUJj#T1vlyg>3bw99n<|0Dmw|s> zv|>?6KFC=VAsYN`($0SN@35T zakB1=`TM@FGVD$NV&5h5Rwxw=BP?Zu{;y#`qF?;zS$=f?Hy!(-OtarKyL{N4!9|$y ze(UI9%mBSnb-d zR|eQw_g8*IIpcoMqbaXF4+s*tN*6(iIeu z&J-&H-)$ajz7FP|l|kk(;N&MSI%I9V4syT?PB+`+;-b5AYY@$Phffua(`D=Xv`4{`|eE zp#NNV&s(6AaCUzU-4L8~v%r6xe6;-`fdNL1WFc!jKpfJndy>uTlpd5zE*L4?3Og3+)D$jHS;X zlv~Exz!?9=a4M^Xl>d?8BtvUcnI!v;aQI&^oH*bdumB(zwhy@MWH2#?vCo@l8-R(W z3YG;64vH_sDGU~TWzK&$PYM8_wm4gI9$$7-k$@Gk&I&ZpWPmOd>AOAyo!#+%wGauf zvwR4>Fq|zw&?*e;PcN^({|PH#!CPjUk5CDIm`qRuRRy0nPox5HJoDH7{rdadonLKW z9`he>!q{;WG-o!-Y(9XUgH6CBOwx1n*@IoCA0HO;Pj%D&+1P&vp+6JB{rX+;83XJ4 zLd<%8!kzu%WpH+Ly@H#~P-hET4>a+#S;5I(uG5qJ47T~%Vt>Yk>;H(W&bZ=9S}+Vl z7K~Z&4JHq`h9MJ1588o|9k#JMO(QDG;{V~zpWi#O0`{aLnIPnYSY{b*0l@VAWo23; z2{nL-6zIc{>12O-2tp08N+=TSnM~2BW%T3@PnM@!0k7AN_9y+vPahhmLRV0&we0FA zHN|Nx(@d-G3pT^r6!7ewDI#9oKZ)uV-)rX z%Gg=U4~I#~reF)K#Ih@B^plJ7fMc9#_)R!YWyE~etI&UyVfPj6%PROr^kFzt5ZZ|T zJ67D8=EF~(yixnwRq!Pmo%BOCmM2LR21**Td@VbRdUAjMhq7eA$z@Jl`JoFZa!Sr4qA_4JbtJoEX>&af>LMa=Y@LS(xSe~ErM&A1RzCJuw< z7lX8M(Mf+Aw$%#w<+HAQGML+}O$_l_Uux!ZVE~Az508m6HdBQ`+e9a&=+1x8ifw#y ziNcweC;0~aN#^|z2N;ERF@ZaGayiV0XDVv@!_0NR_{3=Xm8YD#GtFe*E4yM>mOW$7 zZc@uH0@VEcKb>!zoz#@upVX9W@2o1zuahg=^e=zs)cHYK_{?1TZ5zp#rVL$nwFM(O z869!|Fr&YZot6Sg(|}^{%YGKoU)-d8es=OXU)oVQ%gqm&$3z;CPWFa%*W$aM|As*O z%|!OoR5T{w4Z*JZ^5koN^P2nqPwK+@!)bm9F2J9>R#3=%sGfY$Nvz7>vH(O1tY`kp z%(8#lXBqmJR3-B7x2b{V+z$9`=dV0E%ddOe6ud~H|yJTdZga)^`dG>(7JG!wYAGFHqJqfhQdh}?_uvn-W^rkNPy z*YcAc1g?DvqREk6`#i3`y7KNEe<{!pb!mN=O6S90uE2@T^1vwp6a6rj3>&aj-=DNw zIWgcRpLHVS|C)Es^2>0M_G=p_L3n;1bSM8Qo%`>c+}%GR-2MZRw|##5Bkl@(a|M4K zxu3svewE)q1Ltx@2!v|oD}$B!aOJAAP9f)UE$oUZ=B9_JFMXGv`CEJ$%#$|zGMbC_;nhBz?OS7+Vq(fd=Fe-BP<+_dQ2c)oy;}8Q z8&^#OZpeBd@4G*Yf|8#}vN=-u_1!+_)kZ&T=J;U{gn408Su; z925WIR1XcD1n%XFF2DBsa(>TGyc7P5{W!Tm>fh|=cR3KoUpGO#4V|2v0r=p< z_;S%Cb8F%JCx#Xq4fdBVYrnmy%s87<+Jj+mV4($dOnSXrP<@226 zMlbb;unR>w{ai;~gPrPv0gmsV$JP8BhiLyv9Bv$7*bkY9hN@QSh0N{h=lk}5T<0&c zkX@Pc-0_>soQg^OvcnFY&W0lruw-yBfmIDBrbpczvlDg565bE zG7_KcYJIYcA8s%`>C10AqLZ0CdE@S%GLGMLP(b_;#qM%)kmZ+&RQ_3+;gt{3X}4dZ zQ*R^D`OQE7Ej>arPu%(!D_VW7hk{c5CJAecJIf97M-02N=ee8xI|Hbt0Q!rURKEmh z6?~6X3P2zfi_ER(z2<+1`~1D^%v(1+uUS8_qH=Pce)m6Edi__~YX5aT-+yzd_2)dx zUH_i$M*oZQ6xs91X!p-q=fBFJeGwm{pR{w5$XZlVKH4rD}&V_=U`a&DY z+!p`A!*QAh$di}+dn`yl_m+Q;wT;)kbvUvRmuPAC7ub%fKe{AFbtPb;!e^XaJY1$ui4`GV=J4?2%WLVfuUjK17~b{Xy#S zCzA&#_5A(t!;F7^nBsq+M3ql{aewIG^FEso1m;{Gy|k23|HE?a{BGdb-o$0+ysl}0 z_|TUhN;4MD`w5v(&iiQmuk_*5hpZhL<1CNfmwgn^Y5&VUey6|e<9GVYK7Oaa@zOaR zeB-~<-}vwJH~u^Qi;4Vll|M}6hpUW#F_B;X@)r~Njr)ILB0q82FDCNKTK-}pzi~fI zoI&fhBhc@M`FpK1RyU22-l{QA85On*PGzwyO6-F)Le)9p9@Gu?gTKhw`I!}*J~ z{4$)sSj*+iXZ#)it4@DO%uoEi`tp{4oI9}J#N~&fEPjZ~ z4=wp{e>lvOmSo7^#N{V;`XMequ{rz0VSZxiFK_uL=ZXIimmgX(-+ze94=o9PbC@4m zlK&wtKe0viLtK7hx!)Y-C)WP*mVe}9>HZLxXTE>1;&0;e%ojQP;V?gWQn5e8fvF5}zkv```J?#Xg_cG5qA(AHs6te`rglaWuaF zGMImb4CFHUi4C1)Q`QfKNwVi7!E>}W(eVEwG20J$IoZHe$AW?JaAjn40|moRYtS$S z!#n@_A)RtEXy_3>$zV2>-Zd>@_DW&ede1lPkGLN7|Qk+TdSPh zqYN19%>NHx`wwjAKW_JP-2Xk-y8Hv*`A^a(f9k=9a~*!wqw4(svtNJD@A7|Fx)>nv zzt*#Xd9s>f@nvBpaL&Ka_L%JCcqREMVqohG1K1YWKEo8e1kSc^n^*sP z#!_E@7)kvA#9z!L|9iPz{APb9@n6d^$7}w@8+-pNd?fyVnU7@7cG+*2c~xBjV6@kN z#YcL7@R2WIx0wIsBXL1z(E>0m5Gor;#op_Mv!+I69t{_dL7%x!%5038)>an4ZN_>~ zvXqjSXdEN&HslR_GVw9WT#~>bpb&nBb+HU2?(oz#82jChjTm0Jma>14r*|>TP^cQ0 z6$PMQV%@0h^)iW*-fgTNLs5pxW=xXQGb69?Dv9jnj=f=eaN&#%fjnFlHzzA90ZfAq z+(kykXi%+Z7|`~Tl)+rp{o$d@vOKxq@gzwDE$&@`;6CwA)IhsI0hurlGJ7Mjxv|++ z(av>&QEew z`==wBtD5#N`u? zIrF|>4JL5!>2o#Ed%-W!Ey0J&mSH-1-7!s>_t2*_3$mZednIfR61dWC22ppz5r{hM z^00xb(NC9P);%rh&;ZZ7_=^7c}9q`^jA; zhnHE0e8lNqa9^g0>EkqPI76SCnd?zPjNdajxJ2joB{zRnX*SXqIeT|IWR)?#G_$d4 zmC!BNEnoUhagWNN?`pf)uD0n|mbOa?0w|M-9G!gRSshWtUF(~vp}U{++VgU_LHP1b z>5U3hR+a3Uq*eR6PE`?27P5<5p+C_FhtLmC6xo@Z)XgkJEY|Jk^L(QKu+n?G7=9r3 zL6h>UAYFf~4iBG!Z!qRj-Ga?SY>FBk${Za6LmcZ`sGgm#)F4bAxn3@65f#JL7BAv_ z)Xy>Y2X;p0mlqZoG4gtaHJ2De)3WF%^TKLt=&1885O1&h^}0k+Lpjv=&4w1mB{92zQ?B^s z3Q-2$@;&A+cNs|r(I#f}CKc+-m4`bv3RI%_;f_A-ZPN%PwS|b$&}?%Na!b5Nr5=K} zW8Q<07-Te*#$$-K)~QVBMBmtCZNg;mln$)Fa}HoBECi{}8^Q!J#< zY@lCRdffMi`&5KUa9-sgKLr(=>9l^EK2zDxJ5ud!0uKwOZ{}f*I~S~X>hNXmu4(>b zIa+ILEO^hg5Tsr2ZoB>Np;+RnA}@cF)`bJN%Sw`96vzdoA2^lV^!MNq3sv8i*=yZp z%eiZ!f60cR*0~v%T^UiJojSMxePS*{D5Wra=wLMH7CB2spqxMZYmJ0jFz;;r6v8k% zHp5J2gCO@oWz_OUkg+4GR-W1diOQZ7L6fjkTX0}(^pxV8uMBt8(1}zf8OMJnG;RPJ zrzMHgL?C~p-Fv8wHS8+htPu3MzcQf=E@53D;;oc?o7odj)qP~>FLK-nyy8h7Q{Zcd z>(7YuX?x(k+aL~x*5S*ruv;gQnZduPqwGSp1|e8#d$`*I( zZ-_U-X7b{wK_dFOl``q7HOhZ&=E-;7GtEA?(}7vZk}7pTJNYZSM9c&9tv1HW^rqEF z4FcxHKI$EbU#Y*oAAPR0bSwDBq0B_SYRQYD)LstD4c&)PJ%qL#=DqQD!v?!j9|1j2puYB0L6m9?qhdQtid;&iP(G!y;->kSG0flc6PqBp@ZgM zU2BpF`@mvfj13b$daC~Uivl&();LfvG~#m5G;ev|jq??V~h}?&Tuu90HWTq6!O&b%+%CBFR0F{TeLMF5RIb zEi_v%`@0{QY$3+dMUa1a-i(O;-6)`paJRbXH8U)nyQmIu5j_wn*#m~;AHL{F_7-|~ zO5hADYc8}K$)sd7@cC2*CwiTiK%<)ViVkAvwDZQLY_&wU-cv88l`BHhqwTAT8_n@| z7V-Ke@LYLKj&ivg>terI0mb%0QNe;pO3Tx}cQ{tglORbz+p~X|M|h?{;epvm)nPPPo+*EarDtqqPkDYMdy3LgclV$Y-I^VT61ft73s~Gp?D)E{JCj8#86uH4 zXIEt!1fK5)!S`E?SfomOqal$63+QmpM2JZdX$JTCcO3Mp%|>Edmqx>1?DOS7nO0^+ zX;&?&qKCtzN+TRLjD=Y4RQGPETooI=!uPSU==FxMy4HW$GvjEJSlUuUzj~%I?4bo8 zg%x=9400~)a&*TR*Rv%ZEltbsFiZsV<0#xKK}?O*Dto2}VYbR16E`Z4pP zSKZ=x=z3Z1b>gP=6HeQ)yWL)lzogiFA(J=bUT;(1o70S@Zr0$GtWWHad<@$@{LXld(ciI>4v(pxsQyrv?=~piCqe4Bb&Erul4?{FYri zE7J#&=?3OStGD85+qblkGLyqm1(X%RhyyHEZ9jh_-ar|Sl2O-D+$W>T(jmCNlKS*U zEksJY%i7Z_4#9W9ogoz8k!SIOg~VFgjlSFk*u6OxymoxN1fFs@vt=BF2gX}_B#G4A zv4UkvX(IW%o%r&u)dDOotYgbphGLEi2saOCJHr!`U7H=)XvY|@72!ettVEtkJppAdt0f(EDzPzpHF}Q^GJuloeIVIuHHjq)_0tu# zZ!Yj}9C5$s_3nCa3a4@0Ru1oo{FZJuYTbmSxpoR!LN8sndW{{m(I|RwNfGa8Ho`qj z&*rX+F{I8+7A*ny?eN}(-f{M#^32l31QLJWSGO4f>C=Sn!JDCI!vbQ6o(Lg&cyF+! z`VcVXTnoSn*?Z=WZ%%p;UEc{(i8fo@3;Kbo%jVcg2|J3s$^8L9)FkC6v=@3^?+Q=j*0@ zPYE?wWJ&3*nufPvB@-=%++scH$8OGyvs7t+CTV&%s!2M)Ic92cO^ZzxP8ppcsak83 z6fpQZ2nn`fd5xs8!?&Fto~~n(?aX`b*y52kR|7Svck8Llu)_~B?4(hcM7&ShH}u-$ zBz?>wEPKp)k=|v_u}}E8Z1fyGkoC$!a$+Z5XdTg8MzkTu7*4-Jj48h7>u%fxazyKY zcY*W`Dd3@-&^MWg6N1h`JsFV4N5_z?+r?`iY8sI%7RzB^Eh&m`Y3W{4CX_B1L%^{$ zzM*NCErbIv3wr0m`(1nHfwoA0*g>AXbxl#%O|80i!r&rTZxxLyH7wd)STPO3BwLTG zunDKs);}NN^Iqe5e8Xhl!BA9_`@O$^Q+~4mf)=B5Tw`KAJm3@3#Yy9R>A^!gSRSKa zT+`B4Ho{O#XnuFI5*4}|dq#!S<+zti6TlovThJrjTgkn3iNvm*Qd6zbszfl7FLHTr z+FkSRW8?B%J~ug82=!_OnyKSUx9m~`y@M^kQ?QcCxR)}xT^$&>yxkp!;{!8)Zb~*i zZhIAM`l!Y^VL-znImB=4jSKj*bw6JS9|DGEiR9C{@#ug@QLhBSnSwBV%9S1IAxg zrNr0nJ}0J>*PU0rUJ-x{81rd=2Y0u8U&jb@i6>07xl_P*GeZx@I@Sl9bg3LXuuXFd z?8*}vTE_D;!uv#}qy4Zejl?dJ4t%t<2|Q`D)Nx6 zc#joi&|xcwSv7fT2V}q4?k(>LVv~o=vp4a2Mf2h%s96&I{slQ4~c+u(h4@Uz5y zc*EDMcFHP3WkcSy8p>bxo<;DK8%o*nf*pdGn4aQmp)kk_JDa+$zC_0Yk-is}i0;Ux z3%T67iZ)C51JOnI*zC7jd18VSLzFSEkFqq<>!#$F@Uf=tve2|B%Nfzsb*P6XOpCp? zkz(qkBVD7E8;jSM2!&LC`9$uzB)83VqZlOf%>j4AgM<$o&b#695;s(qL)_1&YNKO# zv%~zM*Z1X#xjeZdIJGXwajnN?8Y8a_`8?eRhpQLoawm<6Lu@=lAGqjQLge53+stAQ zIJ{kE_!hjoY9`-W4Y4borH8p}1jW^mls(_i#{@@?*a7-*bZe4-FA_6DQD)7`26G1E zF=D-yYsfq#RN4Yk4k2k;uN2v2del=vo|^=omsq`6`4hcwwJA_8y%pT34IT}N1T<(8 z`oygVgO0<eHxE4}v zNDXLMlY3EIcPqPQ^lld3_PQOxhUqc1nxc?D-ld!!`dzk$w;Wq)UaV)=#$2gZUrWQA z#5gm>11?nS$JI31r`9&nz`%*@oKSfZxTvWEBC6rPpXUfFE*f#L@e z@b6>23(blQ_ouwlLfAbJmh)fnuJtc&;X>IbnMCS;>Y$Qk>9swo%mO!8ly-~*tP<5~ z6Ky3Ma!+qVxZVq$^sVe2<4A5=j0 zA?UY1*=4*piFAbn4J*QZ;n?Y6UYf{ZJ)KrrOG`GsZB!`+{hmBsVg?qC3RATpZ|`*( z>IDvecO7Z-kW^pVbl0Sqd!a$idc3_p1pmS*NEockkUthT1DE}7HtK5144Y)!VT{4L zDM+NWmx*JQ77N_NDjf-ppONbo{(DGCv@32+V9{c}Ets}TcH?MP9-@~}NSz7~15K2> z=TogEX&N(M@t9DJaFTy|u9apyb-#Iyqewe{B(H|bUC86AOa}<)NT@k}e+pFQn@+96 z!P~c1**herV5h+;?4VcdILE8GsxtjPu;zniUAu_3My`!`=X;c&T9bP!uAn_-@6P)+ z#rw`is>AB21M9k~(z>o1`~KV0%I848n=!`?lz5j1hs-2XPXwGf_0)2P%?R?;rueIW zzZA$TjuPt#F{H}lrZS19$_c+>1;6qHziNa2{d~c%v3O~qvl&%PumQr)X39~j@(8Q%{96bgR?Qo^VH1tr%{A2Oz>_9DWBl;N5Y`Z14sps}_c~78 z=n+fxb%^A7abx3|>oP>A?#gFwEKm=B>DI&t^b{hUfSPJIv6NWjn1P*p&h}#w{XhTv z6!Ja$-L7-`!2cbEtfPNY$V+VI-4BI~e5k_mY>?CJCB1m@4}~n4#lNDEjp<)f$anwW zQ^@_Yc=?PQH_~C z%7hl{ekkO5{gXmoSEBr(kOj;BlS01bWBfxQhwW$N*fS~O%g*%%|Dli>XZj};G9475 zEgnxVNbk-5&|`Ep#hVK!-kcCvx_~URl*p@yo5@&p-6EbYk~qg7`p-W+_7;M=BGcYY z?`Mfa*YSMT{>CU8Y6Feoy9aN7$?R4%Z85NHkTN^tE~$k=aL!QmC5DfT&6sO9OFT)n z6v2~=7j@InW-d-d%3zO#FPFSyW_{5G%}EHemub_Tp{Zfk8=_#A5}G%o1}9I= zbPr>7^T&8{Q7?MoEoFP0UQihoQAXzCY{5xmIioj?hH0i>;Mdg3<7h#QkuRPr@Hk4d zd_w`>D_pPVnhxVDnF=sN6P{w|NEe$lR>J~^|Ru02}0%nq* zurh(IkcH?S;mQ2k?=)|J^RkPEqPyE=lWd43P61xBB0gQ=`{4~VPG@i~$>}Ur>I>TU zerlrq)$@baR24SI^ZD+5#G-OCMA>~l2W7-ODIz-*M z!;-#}?MwiUqz$QN;N+FWuLa*xlFg3 zyu4?V(M{vl&-9Id!O?lVheTW(fuXJVy-ynr^KS`0H(pjFNkly33dXP{oy>z{x_dU` z{VQK>ep}@3IucVWhW+)5L%H!g4j(Ff8-iv&JsAd)s{NjhK#T(|Bkt(Psn$KDF;ENfL@)Q%#oIWV-5*le{8m^DUENjxZx~t_YCwx zR9zpNrs(Z|=!9OZtP;Vu@~CNv!#T}47#Rp`#zC6^T`Z=Q*~g=Gy(`1MTWv8TJ5*;C z$y`6-31YltWyb)Te+k@Maf3T|?CH>-&1ZYume_V9ZqR`M3c+>O&#D5+iiXDFbQ?nL zeaj-uyUp*B6AXT@shD|@9nv^AIgj|E>Tzid7i6h_0>)0^XO5wn3zoOFnS28?`A!iB zZa@hPKea4{EHU3HxPEW2`J{K3&uS|6%#q06b7$NvQ{LpUe05yc+)Vq`M$WwAkGEBu z8Z#+u35{v1N5R+$cit62`a@#HaHbn$6^lb6_UvM79>b{ z8;I$DDX+JyH#|@oYzM}}wDrtF@-Y}C*#vn9t6?EVI<1W^JTmaQlrqN`7NhQL$6Z3# z#9-};-Eie0)0e~S(w@1#O{ZY@OJ?&uggQXT*Xhck6)Oze$Q<%N0%cdKnlH|1=91}rc~L6j|8vny94vuVMuKm_pRh}T<^7Te&t7>_&72@jR& za&!u!M^o4vumNGb_7*-Y_=|O6@yGSR0 zUcmbi!Egc28gv98Lri`SV)P?HP0aSdGZl!v$Tgu#)j^_szWuFSfH{|=a!5`3Q%V2RkvfQEr!3%KQ13T|_ zh$i24RmO<_I`X;5_HJHy_7W(6z));QWdyIB)E!#h^J4{AyOK81f{bmR;V>p+SadbW zb-?P0XZbXw(>iusdn_HM*c7LX<>P$1zup&oL%27IWU%Nm=0)M$UL0>P5yQ|?61@xU z^@BIuWcS@T%s@&>Mds?GU!Gpz!?-xXuNL11d}Vc^oedzf1sXyQZm>juZ5fJyV6m0k zz+eFvTlG`y3><^arw&MSJjkg$6y+W~xQaYvNTRSd&%Ag6&+aZ8T@sMRCWBIy$9lm9 zcrt|-mB=r%E z0ZLbCy*#YhCm>^lN6~YC+xgyD5i)X^#K1^)SqK^(lo*O;M7xZr4U=znL!nofzRR4x zO>{58%XKdwq$K*JbMVQ2Yabiq>etzFvotCnW7gd6Rpk@=Fb`EG)%AQg9IS;fi*SaR z7jIDn#d_j4@k+H%?aJj7V+@B#+dK`U&MSx3tC&Ld1)GC9Wv4@b!5W93&9r}^_@y%Y zp38xsE2~;QsfMWZGB2ml!k+o$l$>3oQdd4z9akN--^8oR==$vjsSrsJw`YWmS?J^H zTd>Hm7B{%1*pYVq_xyHZ{2p_QmhH;qz~wWSMiqVfQOyHero=lIWzVtgwYqGM%($rH z0doi1MMb_BMmg&mbeh!W}j6Dd~`drw*n{uylKlUE~dWg|mX>RQ9?5N-yljPn+_?!A&bZ1Vt~M{BHq*M53ekU+(Sira(th43qW z)uY@PQ$|JYC2 zd7njY7Qy|}*#%v;4y%d&Lw&b-k-Pay_OsnXe5`ImC>R4~WMf-uKT$#sw@sW4dsME~j4BEOpmRreP z%zc67wj;nM+fJQ&R2|izFAT!(U0Z2{lwgoKfHy3EzZzL_oz$J~T&82$@^E30E;fD9 z|Dx_Wrenpphzh`}Mx)8^61Q0!2x`yh>)+uT=X~K zw(36Eea^5A0bG2pEX)PaSDr6yk`X692d=^>4gUh}h0%Q;wgZ=mY+)>J6;o$ox+Xa@ z;UctuW)8FWQdm3$hSDW`&ELP?(>#CgbK7lr-PY@swt;nmGym)lQY1<$?h7+VzpRy? z7it+CZEoTzQ22}kko*l5gw!ijNFd>37kv!%I62x`Z_-hKq9ztcnuU2IA9buLQ#d?u zSpGytoo`soE)W6_-&Vg~hgo5e_A(rzv(k!xp6)JK7<#Ix#2<&mZ!-_3U0UNc{g5-_ z9^T{f%II%lRBU%yus2)#elnQ;eH75!LJSs#2;&_?)BatEqkcX|_znfoL?TPk?PAJN zPhYc~hj%!-;-GrzAOdPwGPC^a_-bLK9AMQ)uo|8)%MYeVc*(AHDB_qV78L*rMCZ(Z zs~{TWwaj@AX*chb#sr2Gb@qLoQC5z|w|H%;b6vPDyTyq_hPSzGUE%2$HCsd?V52T7 zghF&{kxj~wXBr_ceZcOa%r;7|L-xLLhOf`-s&)~wVV%+igW?kHJ$t2>I}AnK7G*UI zud8kgeqswZ$5mfZq4szyW?Yb7@YMo;D}KT_I1FQOdk$QXoKp}?Jz?nvC$M{?AqrbA z8}?eRK6YbVL=h>5Qu9yOw8wRM+2>FXuPK=CB#Nx2l)Q+5@cp4@(y9uvDIFbY#A801cClee5oz+yMgH_=`EwKz0jW}I!mc(A3^o3WX9-q#L)<=rFn zjgIw|)}Mf4>y3LuL^V1GOgSmt&kvVLf~E(hlcZ0Vmv3Hgc12%WY+W2ptA!2G3Aqgq z`#X@saZ1*bGms{rg~gxf@i05u!#S=gBQoiXlDz28_WE}8usGkMi_oo`#Rb3jYJiw% zSbJQeMAHOwTz@GJq5CVUxhNi+k@U;zK@JSQ+3@&;KMGREI1Q7+ewoT-$QbDY!!sfJX%0y5f)pezx8{$`&&EIR8B zbg5cjebLCY+-&q?dP~;QR&UH5z-GHhq9R@zy2`X$`rPgIx zR{L<%h2_m7T9l{5)A1HgTR{&lSWHjDlXuCh&%l$uoL~oieQMVIqHF>Jm{0txbF^^xBFM&ox$ZQix33K3a$ zVuL09tBI;?hVpuUkUq`v1KyWBpZ0b>(gd;rudlZ3g8HJ`*+>8K+!Qk4G}jp<@atN* zrK>Z&^q_ty8WCfyqCA&iZXGA;s(zTpy>3cPTlR5?u+~y?WTMEqDb#R|<+J_k#?P`X zif_0;A+<0&agv-T{B*iIe@zA%GGM*Z`y#Z^!7G%Dk?HDxqF<}*TID=*v0Ub=v9~gU zJNv4M6MvAQkGmETUuEe!59^fg$2o5wWkk33q}V3K8bT66V$TM%kv@Edi}AS20&CPZ z{*Fo8`YPD?j`&D?G$lL**ej|nLK&|&CDEUCm06OGJe`*`LaD-5xiUH#d9(EOW+B&C zuF}%wno&=GM>o5$qQZ*qZ|m)vs+hby!q^udlZF>&($cPFH}``Z3BPt`jhl!&fxpCb zkSI$Fgis**mLMGQZUDhH=|Cpa8*NP zCNA4z7}djZP3_dsv}0nu`?J0myAJtth{4$^s*!|$FCG)T84@3Xon1CgH5z3q+qV~! zThAh~MF;!XPb|8mlcf_TV#IGZ5V@puS{-e%GFV3lvW;(LMZ|JrbI0-C@=2I!+Kukx-DA z_1d&8veVognvpN&OHH^C5>obzJ_6ww&<-PuC!b8}Lb!tD-FjH_|Ei&QS&oEf`+ zAxFx*T2JsaDy49bUZvI-Y`G8gZDhd8mwr`f9f{cfRhC>582U?3 z!m58)h!}@ayfplA7h_^+bry3w=bpxWP`a#8cF&Lz5&h4q&>D_tZgxQY=f4EPC znAW6<=i`|+W%%d?nYh*fLroW|iPpuR+>0!kBe#poJd1?YVxF|a3_-#LpmW+La0n=m zH_)wH?aWmhW7Sk2G*l1}Y+qOCuCF-M+xHxit@le{h8vX-&1+=e2MwFB`*apR=FxGc{l1CWQ*H?WE`{VyX~jqy}BEEE#L$B`Bd+)la#EeLRm=XNmIRSS#J#e6LyneOTHUB5TA*M8| z%zwlPdA7cxd161`^n8Z_gW@3MJ7e3Jv8nbPr!8XVocaD~jnDk)@BE&BXrUgrhy{L0 z=M{ZyL|6wp5&7Y?eJzTgT%q==ow71n_PBjZ#(`=G`JA42Qe@L;+vKO>9lB3HY`rKt z&9lFp4gXNS$A`<@yQ`4$^rQ>7dmPWEPMBu_Hip4k?=dX{);?{uy_T`m{c>y>W_;+6 z_;By0R$~M4-LU63U7rJg<&x%6LxYqh?{jV`iz&H;@*w-$_CZkG?Yb=Dn{X+CDEHOD z54Yjy8Qj3@ZGb)iSz5J6e^fQn zNH_Oj45`CD7WVVep3}?WTE2b%K91>-XnS~8JpZ}BHcJ`~?}Jo-zoa;E=x$S8rEY4) zSGPOrak5{SCrNMoqbJC_;T5@=`s|`)e~I5rWz16ZK5k|ay07EiK1S)$q^RdkJ?+9) zJvB#S3XSKn-n?@9lIN|z*>9G*4|dwc)XkW=_d7AY95rWiD7JH)`nRV$?5;gI#|hbK z!G*22n**_&$o+hOfk(9-;*qB6H}oxW4c1A<&;64b0!pJB`vRU96}g>^xED$cAU}|> zJ>N3JX-O&tCC3s(>!R&Y%Kdf7#-P#0oMMj8iAl^-9&EFDcMFr4jXK)T9W%}hYnI9| z-(I`^&9FxG4E9|QCZ`Yq&l}efG57oKm8DPqWtJ&%7TEWHJlb}@KLn~f6VVjt%xON> zyRe`y+&d15VjG&zd7e(fOFrc7LCEt-&J=yV41?~Oo9m`-s74Mn+LT-E;L&E*K7{gU zU0hBmng4AnA+Ftq1^_DTZ&bp1M;r%%UeyEbk5qzOW`Cs;plSULmC#;)qY@sp4~glU zN+523pc0^e;QdA=tarTe{)6ABgz6Al_%#ag-}CxPB?J<@VAmK&1_S^^;^&SxzQg^4 z`TkG|*Ugv#AXnKhDgjAv#Y6ybF)kJOX#PMYI0G&zO`yvR26zZP1U85@Fkl^wfa$c~)PnhkMObE9|22h!`5t9>0~6*<|Sm2O9)P;&vns4`kqSx2!l=sR4j&+4f50>OQbY zXqc~lGqc)AHY7!^Uty*IwX)E>!F%KyhW|q!xQPD>$zLWreNKFT z0f!dP!j(cmHu#VTUm@6qIkXY{3PqoXKHi^fE392217wH;Q|1OF;?{GRIvC8l3?>GKs3PxSyDb5 z8P{^q!YoQ+!Yq4ah)kp$!O9X1o=07OT(K9?8bo9x1mPGN{88r8JxesCl8K|GEY?ln zls*H^COz$s+&PcUMMeNFk9o%uf3i4OIuH?b5bN?_9XDQPYkcGt;bs)k_x7LvK zD}#`Hx5;nr!2Yd_zZr@e*`QGhyd7C!sx$7u?|V{Sr_^AlfrdoSJ98^KQ$9JW8ISP5hifk2`P`8F$TE8SU^4|miss#E?YB6D_@`XnVWrO@66QpLxrh4- zKTI*}XGrFN&I86n5STN_LJk6129^$9^Q6M;l!7Y_Ig2zwmVIW6+QPv~_Fx^#(X#f# zPhGwBGNBZg=Jj`i`=8gZ~^ZOiaGn_ zVIm2X3^BC(Eu<-F%90w2&G&bR~kl#(x4Oo@IXR-`{L^qRC5d_Pd z7Qib9sF!Pa{$B(`;cdWM->oEtwYGWm8Hc~ywqD>5i}_5K)^V`(*0!&lh6T*6>;&27 zr>p>?|58>bX6h@NGPu3qY^%}{vYnZ@n*~@J2NDis2?sOz#NRD{7+5+D>^9Z1=?v#< z0W6EdY!44fk(;WEH-{<*hWcNnM=NyzUz?fc34q7*l?QUrEbqSBl85P?b*|$Q-1j8q zo$Q2McEGhUX0hd=MM~Tx9mz@O1#3d4svp7J>8>TT(P=tDo<%|?&m0(3IHhC^bOmHM zMo7y)&y^^4^%$MYQIy$Z_2kaA2)R@FMc37eUP$vPjdbMQ(( z7#3=s^x$o%Op9N5-T@Fer(FR~qKKZQyn=%h1U54A`;* zvtFAfmdEW;C3QTm^MVgxo+T)=kO(HrBu3(hPPEu?DElxvvZHT^Gng|-S=D-8GC@!v zLR^>uOmzuz7UH!@f%y@`3Acs~b}KPa%DDad$hKf(AxE4@U7RWf0^HXW#Lb=oEKgba znQ5kb3q8SqWI+&?o36#<4JwKA#56w%(s(NC28ccb&UG<6CXGRs*sRks{51NaHgk!$}y<1wg>^IBOa3_-ae) z&6XCxLn+blK8Y$m>S?-JT-^Z~F7qj4j~2WFMLxW(9^!gW8e%0JYI%JQ6gsjf<27NS z_`HP#Zl#cP{ic#o%X9=RQ6?S{ zjxz{1?M-Os)Z*!`|H8i4y*R4Vf?(G_KPR0a_g(M#+5fOzSBILAX zfj^*Z`mO)F9)w$W)1^BL_9uGdTL4%hAc&BAXC@Cge18m`KQ!CA)>-Kt0ear&cR%BQ z{(3F2*1@F75Z4R2#jM)OTp_L{L8fD>Nx!=a;w_W^^n(x?(e(~2J*j}Fn$CH_;{mV` z&tI95i=URZgc3gf$7?%)G(!dW%uBy$pf3(6`(BqOUu$q(QZv0==cLjjm1&)eniJ5s z#@qT3V5LVIlPG`+bYI7(g$On&PK#l#do{!9_=?7~?Yk7+5SWtc?LgCJYe^Xb!e zsZFIc=c8u;g;VzGOIH~VtsMR2)GB=^Q4 zs8)vWi!l2f|95}-oge$0tEHK+(6t6lJ;c`)Kqu84tP~js!JJx&2-1az*~P1W@0Tmr zXHc?a{EP?!FKNpoG~Kr$f*Epg`^`=S1=0)}pYimM<&3ZW*0J?7zJAp^%q<0|>wB(r znatU(#$zrESnF{ZYyWOf`8jOC_x7dYKJ~ipkI$y{F?jcz;rp!~@%+2)ODTS>pAFY} zfgIT!eL0=11mbHbP~!WoE4LdBtFWg%=eV11i zidH4B>DHK8Vj^;~A{co6Z#mLuPI^a&?7i89l>#!I=RhWdN#{84WTw-9`_aF@+uEz+ zE?dIK1>tl2MFQ$*p%CD2MyNfTS`v*zsVVqlieI$YHzy{opE>-TN4Wd+Yiqk+H)en6 zE7Hk3NZbtEPk-Ov`3d(I`D+f{-5Bfsiey2-pEV$S-Npy)1t3leX_0%kch<7j^%o%z z+&(iK95oF9WI}E&PoR8%v026}yJ$x4+7>6R23EK-b3B|7Q6-XZg%a;=jc`{+z-kp9 zlx&UH5Sh;TfEJ87cwjE8-oH27^bioiz_FRO@D_`&`}EBMxYvZ&e=Pufvi+?85_~-o z2)XBDf)VAq^ur(0Z|XS7GM-aGMtfd*V7%9f0=8}mVB(I1?0}Jf#!tWCTj3DRKw*h# z$G*sha9RaH-3F7kl3W*J$Us+#TWbyB_8}nM5E%IU-yZpAuBym^>sf-}-KEWQD|>t% zYT%Vp$5+Dfry*PSHJw!UCG$(FfA&R?IpN3-;Ydf>(ro|`_j(Zvc!peJtxW;RMPj3s z;!12VvJsq+%2|_t%x5&zGqms)actK4OWhEJj0~V$!H1Qkj0FV(NOEL_g)*rd2qx%H z?1>nVBxVWcS`&hED-p?HVtkm`l!Q7-I#!zSH&)=1Y`^-==NfMQ^fUOmmiV;GEG$tL z%B~FS)Y2UYabH0kg0gar+daw#1SbMt`V0 z{v*79yl%zitiQkeLgKG|;J51;Wl2Z_v;qWA;YvfD1EGzXOX;2aDNi2|kl_i^38-=< zx9AdPvS@OnAOaE}Lxit~SCT84po^md0>B&ZpKH~X_51ytzs8P6{G0s%OFsEqb`oWL z9D=m&%jv#<_w{|gm0%t5v0$PQe=Z5BOfbF(*0W%J_4UuWvEc2k*E?TzK;g0gS4-s3 z0z@9B{)b=0WvxCXkCrBj1Gx-cD=hieSNPehV3sLj9!8v%iNG3fYFUZvi2k+zI>S5H z0pKv@>8T*H>D$-iS1VbIL!$mXk@fXc{&2K-a`^s#_xp!8_coFa; zm@^=MTkZz%8D~$(fF~N50Mlt6uVkDBmJkQ!#rdT$B^F4buP5E`dap>o=rsaZB$wiK zU&BJ)QV9I_(S%As4N#tp9ymH5O(4au7t81Yq`7)R1n35Mnp8};ay71kZh&xFKWFD} z_+y@31s<8P2jSnn<{pp(AfEIZ$P=$*uz?(Z;9i-Pfu4rbi)cZj5ywp8h)srHQ{ciA z$ImzT#O-Ui0il(`w6KXyCdJv`Q*b&CD7<9<`xKlhcs~@}7$W+=q~O-C4RUMf+10iE zHxyjiJLXQ=G#R)F9=>CW01aM$7GY-fh!c>(4UWfS9>?0+w5$PL#@nQU$u;Cj z#R4*!!<5p%33rJTS5j`sXdifTB5Q!NFq4w&5Rv#8z8SQ75i-Pm5lona;8h0r);3!y zy8&K!Yx&Mw#j>1wY;$OV`7~;Pb=`NHtXswhR!VC_7H<=&) ziw^uM`yt{2p&GcAlbgTaPu4IGh6%XO-%a@Ma5;bI2GtYt4FCR%6H`}0j=!t~oKb@~ z#G%N)C=nhRn2&M-4FHG1e~Wp$nHS0DBz*|En?s2-0E01+uwLH zzry5LfXfoINzj=xOx!`v&oOU*21&yTG$3K(1aojJ)f9gSGWf*<4nXf^(U~EoaF~O` zd12*;BJ;)Qtv}5Ph)@U_AgBfAT5?{N+pOp@`GdYll#3mR&=M)Z3H+=K9N{8s5O~`H zQxx$^_5E;nn7B)Z0o-|&!AIz^UsXX zH?_tU%feYSut=X%3vt22!B_jzFj?{EzQA+eB;Cr`01Fc}0m$sIEkg%0g&_OX8HfiS zlO7-w83+RZAGkN?pWk!fZ2Uf8eV?B)swc2`W$4-Hi)k{~{*M`>iPQlw1biIgqUV9L z(o`o*iGeDZE~0?67+BhWx_?aQ96v;ykYxj4OY2@w2pQ_y4moZ@+^78@XZpo~EgWW^ z$`8XP@FB;km@Gk#!O06Xp<>oBGX+H(1&%ZEfmhxzRkzHV=jWQY{YtZk9H%4dlP+MU z04lui7cTO0VAqSwawg>(03B@;m1&0(*Rf~dCG3SD~ zHkUC!=lpt&+b^!u4@381yzuXX7!!IoeEt$9<^I}s^nb%lq4m>wT(=ejz^TV;L*dhY zp*H}WUin_20Y@g~%#MMH`xNi2eTb7qZYRPF;>hf1f}B;UYq<(5PbK7krqBdTMtmr^ zGbSBA)Enk4=w|IYl%wPY;`^A1~K$J=K7df|Z`0F6y zIFgU6y1 z_lxF#lh^sstADXxI*Z%Q$@wjh22?zD|B2EX2cG^#wDliGPw_r=H2~-Z9G#sw9hVvR zd)ms>#dyy8&Kuq_iRZ+BqMetowiW#Ae*TLb)YozLwEXe7t>uTq!_;OLiqYTGs6TBG z0zB^j2Rs}6YrelF-?)EzKRi#Yq@Wi~0%?SQUOrlZpJgYZ`i&*ihkv zF+19#m8Si9kN;U8wwZtED=R(Bsm01B+Wd!2^!YGY*O17fP`tjzJoxhW{U==8=4G9K zUrwA(iwnwR6qYt7JQqBU3YTB!w=z?Q-&rbW<-LYpcJv>r4^y9i<@hSxU;H0_6R)#v z&qK>$GKzK_Q2TGv@j3=T0h*Js;MdbI@D=9cjCI_symB3<+qMkd;^+9>Vp-Y80g<>p zo<4M1h_|f|owjbZx6fbH+J}bwqSij- z+c&kgl5f!$wf4#PMXi1EeN$^|zUvpY_95iHsI^Z%b{H}MY`lgO0p5n6Z8QuS`ue?$ zx4(X`ulb#oX|vY+pKWi=|Je@L{GaXQL)m?3xb{QYebI0q%I=GhYd@6TCtv@6IsY#j z?sNWMG#vRk{|_Pe{G9(+zU4#2{Wfk)KTO=ptfAqHh+A2<Hn{&G0QrJ>Spm|&tsO^zKdaJnC(sZC;r=p!pE+;(rs4qi7bQ&4-g~W&8*QM4TLM?O&{% zkJ-3-27rD%fkfmnKxK?LfSflBED<34ADEg7(@gktNW|gK_tfvtC)d{yz!0>*(mDK% zis!F%9ur3MuXF^s5c0o&(glEIsQ*eA|CDZkI)v9>uT8*PJe5IyC5`~xe^E-!hg$l4 zz8OyVb5Q{F3=?7im=+j+5nbTFCd9b^ul*umfZz9Fkifa0t=ZAnp_|g09i?XU$dTk}NSWJF(Kcp9zfot}mx=c)2@t9cqpVjxL9@q1D ze-UoaFJi1*U$4}fj?ZgY$+oZ8TK0MW6Zp_&N;KZ}{~}xp^P@ z5mmrlk9#fu`xgoMNBZtB@B69ueRVWO-}rbw$2tAp{$BXJ?F{PwJM`RF zP)Yoa8Cu8f^wE|4;@tfM@X+hHgLv5M+D^j0%6>5&_kN`1yNaC6EJ1 z0agpS3F-KMdlGzxPT2KC7IJ{CLjfQk*dY`q6au?~VuoU1Pf)VZ4d7d^0Oi1up**1q zINldI2ciC<Ntkk4>?EwkaK+Xi=4B*(;@erZhAjt-QJqM)9>h4 zy7!ZAf4@J8rtfsi{U+@UsQ*fT|4O%h(hch;o%)r3p8TZ$ka;BfA@eNn*ZW&P?=SGb z(tqBcGp4^K_11LT1=FY8?t;#Pc}@4c+4!RTY!_{H>nq*$mS5@mPr3oV?q{b20Ra17 z@OSJHjF>hFf|MhZ>~gzHHRsLPezr!h_(~q5y^XcD46!L5gN(>oUxRFTzc( zJ=tTfoAeD(HdPMQ@tsF*PwZ~~@=WvV`>7g#{xU1`)o96gwC%*G>-@|Q!@$nC_+7DCse$iPsP}>8p-N@+K&DPD**6>tdvt4^BJry9aZ=9f=2s1jQXF zwJFo?C{BPKIWdId()nmkV*tRN-N5&&}<8sugxiI+y+iQ7v*Uf@%R`>KPGkfE50o_Dr(7 zwF`M?ukb}rix0q(%4q{Phi!5gm(aOd$KK)sS&7YsBd%TAxj}QfyESJKAw+ND)NMRp z^@}sT=1k)iOKw! zcZZQbHe($mFFo$^Mcc`efYjO)1#wGT=R9eyTE`}Tno&m% z70VazjUf?kC@#n zCNZ+XD<#B&JYF|OS%)uvE|=XdHT`!(^r20uwr*Ys>b42MpedquJ1+eJTm;~;ulJxW z2acklx|qCkJZxX^Y(K81JsqEE9of`%cRt1&l-A-bCzf<>xvQk23!emkKsX$BTX4vx znct{h^74-phpyq0m#2-I4|bGE1JCd54h&{?m1X~_?*eeg1_#YJPg5fMIog1u+}e-h z^xB;KhChL^8D%6jU|Zcw*+@>&wnS~4Zf?e4UamLCy1go+6fTVTbWWa;@&d4xyZpXO zm0b;i(sESBAV)Wzdrq8xTerL)pkfle$VY>*Y2iE6ww=PTiVRkGBOmX(be!$p3qS`em9>qE3GaiD<)+9A z7Z6_G5xaf>GHIuHcB@OXQv*cH&8I&!@k|Txc#WzeI2MQPB zlH6oQ%qj<>rd*qUt=@=jtjY58McGLX$elutM+V*H_y*MC5L@Mq*Qtpzb)mkBX{0~Z z#uoV{9O0NPZnkax*DRf@Vc2H8KX$=%>(pYpn8ViK`z+vRu^L}fvcK%8bIWOoT$UY& z3a*?;bg)Wz8i$>%P7dD;>b;h3vb>zHl~Lr&fz9|ro(Z*o5Z^bh7?=ZH%okf(mW%bM zjQhh4Yj-^x zEn)JcRtWDGT>)1QNRJK2WM7Wincly5 zCdWUl>=dbg_rMIG-yGQhdX`UKMbLAevT$Cg07XE$zZRkCp;qxwe(Mz5?i9#4PF|?h zD>K-nt&X502S$!-)B$%(Uv5rkh`cW7o4UU{7c;yxIvFuwH3r=d z41gx}Mtt|2CMq5V;4zX+UdsWQGo?y212v_%`%(CGhUHX=f17q%-VZyE&T>_1yK`Xp z3QDSOqAA60tCPMxjBxv zCQ3v~HgtR=QhVk{5~RA;Nbi38UiybU)2XUf>eeG!T^3mb_6sNQ%jW4@n+WMDx6$v` z-I>p#E&Ed z<$D`$bzwHuaEd)rX#-7>_2J$>x#A_Ey;+U)`4Ab_e*%y0@wqsgBASxY2&c>5J!ran zXXC2OtZc{%=kB!1()iV0BHcWcB-w7Sj+WR)oxFp-=HwUG3f=(N$=HHRQf1bC>#&3c z{Wfw`#!IggXNcRbk)I%oI!(R5Dc%dDC}!vG{^qV5^>ma~15%oFh`b{|TdF;FjC>ii zX-Z=de;5NrLG@Aj+>6ook^kIAOzcKG3T&;>c-Y6HSZHRsoGagaYptNyTWLQ`O5ky0 zZUj;)NZ_K@HZ8JEA%rrDW$Qwt;XA+Q6aG?yhp2M+dvIEW6n_qg5a?o zJp1I|k-=rP$g+BmAC}+haV^!s+biieSmNp=GD*2e7ny$VNxD27s+yD4VcZ2_Lb+8r ze;ZnB0yXR?wHhW@(5>UehL%bcX~#YmnJO7OF{XFZfw)|NJPHNRwqtft zSj{-^fpDllH360b5Rq~{Rs zySC5yW4%-45Xu5P9iG<9%gF6c(t)RwAmTQSk-NZ5(6Bp2X+{rxF~|vC*sD&X0?xhE z=kd)HT_ww;nO8J4-6P>eAA}`8l5-u^SmQHof_a#uus{o^V zv)gTX4>kNl-3F&9Mm0Obb}Z&|b!#2ZJy^DK_b%c~xj2zrHC1Np1E(ID^t@3P&g#q% zLEQ;MxptzBgu+pcm~si7 zdlWFneOPD|(BhnQ1jBaO{@gDs~C2%YBFX`?IzYqk>aF^MnXRtA3`9=Z@e?=y&+` z^~hJ9DWtq*KSleDym~{Ce_6diy5R7V?hi+}zP*tiC_q&+r_!X&C9m5y2PWDL?jpzF zL5nj*I$jyiFdeNee>^?==lwnu>`HR?Z>{HU){2+g(0P1v+YzS7}$vr zltPKz5g9S!=Yb6J?d(bO?#^y!I|vXV3=e6SZ=KUsCF}0wQO9?Ye-*Ds7g+nrD~emq zQxZDwwTI)^K}3rnxZcuf21!z>OJ~vS*}S>a#(76G5J|^48C)45b2Kt4AD<3w#btVuH=$ z^~S?LZ}!3avQekTe<^Xmc&ki#9S4RpPEv zkyxx3DNOxVS^)^T{c`uqBRe-;bm>L|HZ5|VwwgF$L!HA*OP9587SCI(A5(W*ED!P4 zy+@^fxa4kg-|XJ(whJ^Ql^U2ci9NGboxA$;mR*=>v#ZpGf7~b3gVB+rUuaLY+aeAm zWV4>CAOxNd7#_89I$if$^ngWl67Kw|B!E$+7vfqZ|BS_XEa`TymkzTPM7|DOXHXNh#19TT7$eC z!sEN<_+uvKpgo+CeXvqi!X> z9|Tut{V`2ndrnU-rrV}*Tira5(Au^c?DLHLs)gh(yZKKepC^xnThF7JcUX5y^_*U^ zB-9LOfB4-@%F?y!DP(b zLB7p|C_~Rl+N1b*H1{X8_L9;H)4FXRB9!+?WZ@rkYouN=g!XbN-N2QO~pUf;H( z!E{5)V0ZHNYTLuA(+9RihH-}Xn3qxb@R2lHA-g?Y18+yY4%heLz`&|aI@S!8(kAOG ze_7`B+uN(NbM$0G zv+aiVri?dTavtNIwmiH8OsRY9GSBXye^n)ui-(u`SdI;RQJT!l&V0yq|0*{#JyJ?i zc?aoj?67n<%HOIc2i)lP{Qn0Si!XCE3 zQ%E}U2;!-D)S+^OD|@}Yw{DY`sum^LgSdI`hd2dip1x5%I?)ZkB`b}wM=uMCf5Alw z!Mr3Awe>da)4CGfNMqY8yOGC@uyAFYUQBhr8;hpcc_OWokN(AP2jn<$^RE1vxC^Oa zo5=5^mspQka>AGWmlmZy`|2t85WBp8mJ_)-4QFltyfS63D*T2Or{^SH_Uy%BM9mK} zg_-$GW=rx=sc)TtVmR;Kx8NQ}f9jSE9jVuHUln`#CTmpqVixxLG?u4)r6ROFZ?Br- z>+^H6@1k}htiqU(S~;V_o4H=TGVYuV&y*z``PA-!7piA-3EJT~@1#@#Qb4e|p#1&P#9v zZahey+dVoTLfc6QGdb9d!#o}p90>B3yoaWm!jtN-iN86dW1`zG{d)0UI#i4J(4vD~ zTkuJmkF%ZX6J01-)lvEQcJ7A`81J1qRas8&_UuHuu+yJQh}j-9tkXEk8}d;4@f(JxvysuGRQ9y$s%I%ad_fjph>F*`jj z>1JP@t?1^HTcPiF<2fU0?lN>u@*F+G3;gO)vF}Hrq4R<4g~>g2e_^(_gATqQg`g$S zUPOWmu9s`jPp5WY-4^7ZeNH!Dmi3gEqa}K~)ZHt>{V*oi$M}$BF4K#qNAfw2SYC@B z<$chocds}V=?VL5dA=hXw*9%>j|F*Q+{&&Cg@i!hpO>_Xg6PSO-i8rp56;i`VZN(N zbXnv*Qr!!;92U|;fB7Sc2LKlliX-!yf?&EGyI$QBe6p=Jdx4dU>yA^dqF#?4Cu!X# zQ1^;b3y4?Z>vc6~MrS?0g_BYp+!kv6Bg~s?F^0y%QF?k&kVTj~f8To{l?BX9p6kIGyjaY< zL)T0P`7Jt(Dsqa®I7uICYr$Ov>XQ_avgE8|I8lD2DA>AbBTz&%Ixek_lw7{=Gi zzC^+sM#PdF0w<<2SBF+F@fv#!=7VV56IpjJbGt;W9UDIyrR)xAwUxP-{OW{U7^|y1-*)ftxsw7Cil<`xLfclqMz$CiXKP|N@>6k|=#ku}@%q-w{&ABfXEQ0~p?IhZanB5ZVJmIZ?Wns!43F#G3;ZqnjFmVt zHm^W8z~X5P7av~To?Mo$9m5}#W7VjkGJ4L7;4v)zeeCeS6av|MkvHotCuI z!_3L*)zg=(D90DATx{RWV()R}S|R02r`W!Q2Upz__gLuGlbsosDX8|ad(d@sbKINj zqnq;VI_;VE`$40OM|1?oYGaEI4q>T#>`ti7eQp|!@D<1g&# zYGtus>ZyEkh!Txd?`^qo>mw*Usz}`2H7n+*2pp!@oobfHyh9#?n1)H6Xm`-hyCi!= zjUHUtDs05)WK*qivzR^9whu;W{3ceyf29*c;aN_+zCiMs?xXW-f4yIxard&j1*&6g z*1fMW+k-8zNBk@rIzR8lE^fvXr;-M@(DeOU7lbOF#f)_9j%55=W&o4)Xq%`GcQ2tH zA-6!Wes~?pNjuS8)>^4zp&91a$Zlzv_%C2so^p7OL8;j>D_=dejvsgG1h=Ede@uQ! zf$At;%O*D(qMyc19hIAmTn2M8b5ly=O5R1_<|lL9vA2hNjp^OP;Dg#sjQp91WvF#I zrO-{gyu3e#wz@ zhWU=##qZL|Ny!F~x24}!Of^4}m@aN(bi@@XDB z{o0FhnD4_jEAst8rrju`&(h3ZV!l+bmYvv>2<<79CNIlRqW_jue}@Nq>LRJft*D_q zzB}C6eM<0A^l%Fnih3No+`si%C2LVz8hW2EDc~Qn+K|!gd7ianbtjbJp|GERr@2IP z;nk=q8hy@eX}GynJsP*Yl5NTTShif^03`)1!K;yozm+nf7{-E6N>6exH9(m#z8)ArTnetwhzAMLEiyLex; zUJT-+d7J8p7|y`pm-%%Ur^DX9y?8CxbyrL8+sr`+>Mq(G^{$u{oHrNcL>wPbeiE*8 z;z?ev?K2-XX&-f6cOCBMtUA*pFFh>EyPk-h=*g+R9NAmge;8ipbkotj)rqgzuMKZ> z=x_<5N;N|lsG_@frDbs^SyBS({NnG1=*owJXp7C<-LxjZPp4q2@7KD~G$G4DWZUrF z*O%Jf7kXD)R@0xo7tV2~%(J6W@^QLyP-KO6K8)L3(e;73PCTxxVB95%>&(}Cl8AvjI zZ*Z!PK(&`Z!IgAAid%{0?B@7#sDyhJ(zcm2!s8XW%!y4Q+O=gsN#0ww2#PYkT#NhF zJkQSyvvbBr+G?#QlwFa@n|zYC|rD`?2>PyTDF;Av(X?fy_@rzzu?H zzq-9q>Q+z3Ku-2jQy#>Zug8?Q&fNRgYL82y@^8~0mbqOOA?QhdB+k0;k^=ZLxhTD|B> z7kR1qmnVl3Dt~_aN@lm~1)!?Gy>}Nefl96QMqEQOAsvrfm(t`Bc4Q#Bvc_5G;%deh z1Q6N|#5Gi6fE;m*G#KnDPfCZl{d9Fut}_+J{#60LF%8!6>5EX}h`RHf7za+FH949% zqIp9*`SKN90~N!B{m_q#J0#?TO;X@xmS}|?RAP?kO@9sAimUPnve*qQ6WDNW-hSuh z{E`xJ3g&Apww@s$=uyYML4bBnE*!h5O>SleqJY_fQ2_bP1vHFV+0U%#*cl(i&q}@s zM$-rrXSg?=Rbe!=z%@3aIN@`~&SZnMvpEfq(#@|2x(W3Gf;W7~qi^#QHsX4;b5vHh>IM*oDzl%<7$zEi)&+^u$J^|eo zGWqQvGY)HU*F$n0mz!Q3wiVcL`sB~mh$2v^O99tht<@iuwj9BE@qVNCRDGc+Tq0T= zQ|WrNynSKmy}jaNKvD`FH1LI_Xda~PbrvO$O@CSP$qj(U69r}SGIWj#WNBWXM+&QX zoJ01-Mz4jsxLtEr_fylhc^4?ny8gTH6p|GNW{mhufQcRf>rZIGla8g@XG!{JspXe? zznwbXqejN*E_(jx_3J>0m~qOews_M`X`8cMipE#fI=6ru|{;Xo5=N&z2r7l~6G zd%9nnp~lPM2u?zeUd|DAw5yn@uZ~z%2*^A3P%Pk2!7cADPXOlw{o^S`fbLmU$}r>9 zd0mUMdplt!->ufu>BmEb_9bV$uB+RWynh&%zC4Nmw$P<%&~j=9UKihjDc_GyqWrW~ z>-UnRp3{>b+nZb!p|?qUR@!+M1Scs;r7p{EkrFX3cWg69VCTCk8 zZq%~$%Z_m7eX0tdBG%P%$7|9yK0nMBtX!un#?n3#12}MF!j*lIHm;p-q9hD}Qh&ac zbS5M>y(iM7a>#X*D5c-u2imo44IDCVwohDZ{}SVjXGa8oOT4|uXq#gl>Y%8zR|~=L zu{~4PKCH2}g$^E%I=edm(kbrrg^V7<(M&bL!L5<&-%rV9JxAznZ3JE0#5S+O$qa`D zAx)9y50SpBlK>D-VSXoEJ*%=6X@4D0cpoND;1o&&2EwG0o54jH0kx>Bu|O~71EMUSgX@=#%OnP(n(K>96&~p?r(PiUC%xX z;y?2CZ<1s%9H~-RM%f}??Hfj}{DAMYe#CjG0BU}L(_%&l#}%_}-r3+GEo#L+ zOo1Cm2DR%`Oh)jt`I~PnqN|J1T7mZPaRVFWG#CQ({YM>fD7ncBD`k$y7IXP{wC;iG zMj_=UNN{xA7aDAl9;2iz%YTBswVeLc1M`Y;Ty%juHjgM;7w8rgs<*KNQ=SMYqd z(fDQQdZTGn-|Kb|K#h{(ap943Tj(6vv)kfCbFwtGDrQ0A<*{#sEq@=vi|C#C`MXk} z?m!&p1`!~%c@214I(_HGat<~W^E0+GLX@||pgD6UYtD+bS_y}t)JoxWmnGl+Ku5Gs z@3);)1C06yP67YRSZ4jx1c^?a5{7rm?E6N2G;ndJrt;!S^#VSS^Wo#(i&^fIk6MaTre8 zdd`Aun1fQX==H+e2%}_)#1Bw`6p`~+At0)OB%t|c4o^eA@mEpp^R0e1Zn+x>)X3Mf z`a)xzW3{@+wtpiM`qULZxt*g}LprI&w+2MuK%%FD9zGzFJHvLCMY_1abx?9&c;Rpc zCn)U;_j!PF3(^MwT1m{iPNPERoS`$(ela>%`nM_3@Vw=EZa|v6_Y@+MBiE!Q)AAdEc^yS{0S)osWmO(5+<$D(k z9qnqT&FQk*)_HUfd(XK+{*o8%C_U$9y3 z2cMj>V}Fzclz(bGmQYOqET?hwTR(x~Wj2U;-%)9N^0l!LNW4x)7BapP`E@auT6j@R;moI1ON^Lpx074TA z;XKM@>ifb6#RHeE+vXx_mURor+^?TuN{{745`TNDk!)~@<1Ik~O`AN%N{d&$^6C0^ zT0j@#{K&7~E&_lp%b@AHqsc~(dCzbp>bZpruX&?~YOnY8t4f+UNhr;@^H}Oq2zWbc z6tup!w_4eRgymRNBYPtZ=4V)y#lKU=T$Po-4v#TDhTh<2^!kNnt&QCyiOSwK`;{t? z6n{oxcB6htZ-VC|i|IS81xAfH{ZRX1d|na7!-C9SI`EdnZ8*8>SiV=zH#fGxdFykk zTb@XxnjZV~u=uBHTy}O2X!5(UVxn&|+jV@|PKsH%!i9#9Z9LslVlZhhf%l=jG{NcX zE#4WDEqi-5q_(;d!ou!y?Vqp(Ca1q|Lx1RN$BRnWs&MD5dOs$G&I&>I?0<1g-Csor zc*MYbyblZ81IcE3K=^>X{cD^vW7Ebpd1~G=AD$)1y1vC8uiyGJybS@u@#5BHjGlAC zcN5T((WmCm@ED&;qj1c6GsH@!O5P3&TV9GB(F=eDYv7e;X@z6Icp^^gStr6yyMF|a z!uOs#DJEwJ9!zk8AQnh|_f*%--3w~45n_mA#qVfE()WF{!^tv>?OGqTAA^RgfwhGP z;Pj1d&Q*wbscSRa+cF{`oVAM)OG+qhd0fi~A=(>PJ zhkcTRFZpK&Jx#c~UtXdNI(@&2u1fxz@Kb`D{rA;N$VTl{*7t)|=^c2vx<=M*EBH;% zeN`2N)RzFMSbZYz0#T}j38cPTS$LR&5g1!m-rqi~3&LFzR1;>OIr2P&n176EuGB)) zevGrb%wi%nAr# zfIrf?wg*jL=|qKw)XM3X%Q4DqpLEuC zhjD$9eI(=|B-%Bfjy8yazp2f4Yl!GRs)@H6E9f*Z_$;-W%wUyrcN!l=OyW0Wei#Bo zvnW9U@Cpbf9OO3N9Z7V>9G!oW&7S1`VW|05eWZh^B^=tbEZMG3Y=7hi8jXiZi`V@oR4vHQnUuA?UoHXQg0Rx&op*hBji~t%RlX{u^LRK<;YqkrBp?|I9dr(uRs-N+ z`ZS_vX1j}t427q$O~(xoZ8dCJf~m6$apjsN$*k8>)LyH#7MRtv>wDfOPnuQ4h^7HD z*Z|6E=oZ_PGwGVs=aYr6zFeLgDM;mILau9_qt|a8%_1pYD}U0cdm){Ar0J%Aq49** ze1XCeuMF|==*#M6dTl2h^=3LN(kJ&Zcw#CY@>HM^VPz7;fzVRRA0XTB}B4qqMB zK@-Fafi}-zPk*+=<^{8fM!D@d>r+Fw6F?qht0tbY#69B? z#ICwjl)Hf1(MR|GFoxh=!qJ2fqV~>I7G1s?bQ$zjmJ+Xe$~wC+iC}@XM}B!%nCIR>1b;5-Gq3&Q_?1R1XgMf!IgN92 zkLiWso*$41^k((Iy%7s>XZ70lS4KJBJmU%;n_-Lew{CfdH=cC6fP{7UDI>_G;=c-* z)Nkw8;DnKy+GI%`KkNh0^5n%s$8R9LEC}-z<6b_Dn+h#WyaL>?(W~cauHqIc7w$tp zEBf$r+JCx0gca-^%oSQh)P6|!5Y_Wi(mH@WEslRYNolmv8@7W4l@H$lDJ3rG;%2_* zd_u*iIa4YOgg<;Kld;HZsT8)A%s%txkR>nsIeZ zB9NUFO|3+EGjnj>-k29#s)~+mFQRzBUY*{hzZ(Zd=>EaV73&RAJN6YM1m597^h8Mt zzki*c#vKsz_t}It~Gt(^png{J*dz^RN7w()|9zdZ# zLnEDAprGy?@r1vWY4a{vHuxjO0}`9bBWg7{7OV2;;zi&lc#pFRfuNS5&yA{Y zAACiy&*>ayN!0W(scSf|FUYwiFU#%(deD6RkK#;A@q?P=;M7v(%EHZS!< zg?@zbER?;W^VJ-f3Di+9eVQC2a_$t)OjGyZ;=H}N=hn5Hwa^%A${+cBMy)SZ?EGTQ zD3BlRPv_601*x{7$L~gj4;M=d`dhRnw!Wk1BtA~3N~l!?h^g*ucwd4LYhW+O?{S8T7P z`2B8-GKJ(+?}IhC<;HVgRHhyr90xcFLzGN{(B#JY!KLvV0#0T`s1?o!+yFiw&FQPp z04eoI-Kxsd=vt5KMn!iRKz}K0lx|nFfWs8MrUr3mMR?b_1C}|+3JT?;<-jU44zlWG z2Ts%B38Hl1i2{79ysm7SnkmXJ>E__@i|kxydedaSuD|DOxewErnKwLXo>7e++X5{4 zQh~L0!Aj@HFID!5C|QDY>irA7A}avT2vu7a+)WM_UoKZUH5Hi|V1H7jWrE=XvqwGw zzs3nNs5U^Vz9#CboFS1yxjin+y&;*4yWN^PUaN7p6iP^>GG%6llGHvvU5`VFpH8A< zDWdi*r0Iiz{`n3N1K(&j9~~k^@Q}NxvoY9Ppaim|LYJ52Z;4f?Ov4_ghrr;e<2~)r z+D=VyH{UFT!^=$Mgnujf?^_83wGglX-ik?jld=$?I@)X}RmCOYw!c;>M4w2HLpqS( z?+`}n6>dj5+$_+Oe?9RP{Z>pVhzSo1YS!%oE*MOTHorGV;oUjQYhc)61{(b#B(Q=!Yzj+%WpF4oIEXr$ogeuv%nb;my1S(&lYhfxB@x{-{3*Cvu!bN=#yb>hkMmAL zTV?xpE2HEml~mBFa(hi^w|7rO3t!ng>}9rKG};ZCxL+T3yPJFH<2jy?Syn+>XgUjd zseK8X#W{k~j!dd@3WsUhZ~g&%3ww~KYs#>xliYQfQ>`io?KDn^{} zR)3SBORn3gBk%%Mdq-WyU`ARu0oS!7%v!azywp|oc_zcnt*C+a1Glm-u0~QO#gcjU zpf{oSlO}Rq!Qoe%27z&x%Tf8kF+KO&jHBB&siKf?3+7>A^`n76N^J z*R%_pk@)Mrf`z6YuoR&6cy@Tu>o8@9g{7S%;|Q9<#}IF<0|& zr57nk?ZcMU_}6XP0Ikt4Yo{o;uhFooJ}czABvaLxx2v`!Ws9U1*res$3in zs1n{trSPrbMO8(24{?E>tlFDI+-0LQ^0N#ej$=6eGXjwj_*dQsm@zrCEPtK6Rt#*S zsqHikMd}#Bdd@~TirnJXA|KcUdo~Q!CS_R+6Pve`PpEY$7omq+CyRfQyqrA~PJ=A6 zaae{vvdMllHTpPp6Eg;H)io@HyA-tPMoEX22Horj%l;$6gduCfvm4mbH6!&v0w_E* z=s)7ti0G0{@Vm}c@#VTY2!HFDr5-S{egZ_0KaxnIfiP3~(`YZxM)|da@xCf;)Ox>v zy_(HshyKhitQ$|0b?56f^(z7B<29wXbVHI64&$oIU~(RQ=FEUju{WEkCiw=A0|wYF zhnKW<7<$!51O$Qo)@vEIGJOqE6S3D4?3XYVO-e_ls?;iPbl1h86n{Szxlxy5^i~U- zx(wL{b5`^r;pAXTsxqO1yE={#u$7+YmS}ij3CYRp%nGJh_z&;{-|-f)=ljhszJirT zJ;s(x4V4ozw$M<;B~bpBEPetk3Lp@c#m++Eg895Z-8hP<$P&ch+4nyXJ8I)&DP~Z* zq*XmjQH(B7`@?dAQGfIUQ};T;YEns^k|^+W!81082`7G_Gg7qB;i$n{ z6}ayGTIB)BM?&hg_-PyQK3|jNxh|Xheqs??DE??Lzn8im4D}e4RWZAnlCdiEnlOP0Ga=4+*^9U;_#5 zo_f+^{>Maqw|~oxVI5-PqX=2Hpn1j#Wd+L*`<*;ziIw5rMXJ(W&Kz5wedlFkx!=n4 z@~)5$4X?eJ*7(Y_N5o4`UZQtY*k5Vs|MS$5{{gE1ef$4*ZQTFv+7QKmx;7c%{4Wv* zYX>j&CyIamKW5}#>9cWz%lt+E1a^kut$(ej>F?8gIe)jkhy+ccf6&RlRw45br->SM z5K(+8r1CFz4cUKt`X{AeM$(aV{U4E?CxZVVC;ykLaZ=>L)4$EYVHghlD;2u6xU=i$ zE@>qGmj{HwCluS_QvFX3Wc7_*}|F%~+hBY0c%O~+noj=&+?;z5j*&=LG zTsIK=%Te=>*P;KcM8^C3v!!A01jYL&xc}{M|93K`e~m|V{56_XnR{6?5hEK_SnlAq zZ|l@faxsUK^s9n!pBiy&e(nM20&fd7g2+?yTYt)Sww~WN0hk~qMB1f4I^T9+Ysbp% z?eWOUu#QU|M2Q`_?|Fx-#u;G7WdSh=I89~POuk?h-cWCj>zn2FB;#I`t{SrX?3Hp@ zv*XCtH54yaNH96YOwKOzT;RfXR@`%fEzIab`BB1zhthyVE48efLaB$B9?iI_wI+tV zT7SFhm@%~mlPDOB+6)s$Rxj}H4U1-mei3U|LW?k9-2s>vOHyKcLY&z@HQvlH`O1+e z_~SW4Z4K8ora>r>u*1iOW`8X9 zj>tAdKqO_QyvmjIL7%46Hm{*8gt`QR+X1>4Z3UG^g9jfI*BE{1*4|D~o*yQHzt1+Gvp4W=kt` zZLyZT9y3&5T~9yoTm-a*eoYP~vZ}mDlUV98?M<^$5mde^{5;nj6QC3u$ zF}Bt((8(8~z8F1B>h>6@<>OX8c-h1dQj1#e++30mP0S!o&rK$Oc%c%S zbjLm?sCMhUH-uVrNg{cxV}DncZJUCiuukhWf?@I&9sh(NoJ*60b1}1AKuX3c3-xg( zXMOz{zVaJsLB1(Qq${N&zZ5JsLG%&-LZcxBQwvc!uN~dsVw6v$-jKwkn#z7@M*n=h z3xVPmhp(t7!RSm#JN&X#e@jC$rKJ#MI83*y7qS0z@DEJe@Zt!ZIDg$$mMl=YA0uDJ zWg8Klq>?7o=)fbw>8qn`C6hc2x?w@U<>+Ir<2t5A(q|XSVtiA#^k5U;G=FtSLX6P- zrwZ1t!%^(>q0{ziN6WAp1Ww8Tvob0u4tfF@Zie&HdcS-xT`{B$X;hZU;YYp!A z=h4gqE?Pq^Qf_S$$A7v!2SI|11feDeciD5f7U&*wMXWzcQt6R9uGCqo^MKBg|5I)#dZeGB#Ov!iXxcxUaEkqMNn zjDF^>0oFF|jFR^Lyq>e*BPv{ffR3WYkK};QGIxedNV*_~uzwQ)uv4lbGnl=6CSx|NFuTW=uwtnxJEn|isol2K;IkyPb-BQ7`L+$)PeOlK) zAETm|l`Ic%%YV0CeNf;vWD+0*l(PIwNI-7)*!)TjdgbV+l67w-9|Oe9-o@q}PS<2< zfvtSAAJG=e2p@tCCOjTC04fo%G*FPb7T5vn0ZdYdg+wSuBiG>&wwH-nwr5nRG!BzV zY7{I}d1&5eXveJQOc&uYjO4naLs=w;WaxdrK0Yb^O@E2>D>iFMhE7!{U~tc6V}>>9 zHiCo_?@f}XTle-grvtC&xNPt+({!1Po>zk-K>|2?@akbo!@|CEUXJ*QZ+W{upd?9D zfHI_o_&Jiyufno4h1IwcruT=cij7KEW3UvqLT?RW^KEEt^B}&y5Z_WnP!wBWn*LZG z$qrI_P=8C&m`E=GL{Z+)@g>Vst-pNt9(eSWagY^OvzG+5ut}pXf5LElS=TQV*z7kI zc&XdXFRB~-5Z>$G5WK7R3f_TcYN@uK*S|z%H!WIjn!F+I$8WwfnRHqbDC$})SMz|k zFTe1B;`S_G1rNEq*NvN$nLxem>>uuVmcr0AQb@)^qWhN8@$ouRX)x?>SoS<(tHDeg`?`SOX2T60~!FgGewulN|~S2RtGR z4n7Qaa#o?}K&7xRNMWKSim5#lH0PGLT3*T4C;gN!YcF2ke9r!a%w@|SV=KIrDJWe|=}O9{8? za=d$>)~T_(tv=t0Goy`%D;=qyiIn<)#}Y#!s{r_yD+CAX?m%#7;C2vUP}K;&wu2N+dw0O zrJXp~&WKpm$EC%U{BgL_O8{-ySqg|iexV~0>2>T9{dJ9<&nTU!`nQ5BSrPt3QBnp5 z^R0@%cFfu1AZBbh^UBK5(D~OfxAI)^i4gTn6I-Vnn>%`8d>`3cIE*9=-7nXLQ(kty z`#`MEd{RFlt-t&i^Dea$bAN8_VU@h9uq_dzI&QN_`_$@q0j_2OJSb=C9Vjzo2}URJ zXUbj3NHQ;=#J<+J59j3zTo=*xLuY!|yjDcqX>Y zu1aMcFv*F1O(iIM34gjyVD;bZPStCYiVlc8Jghz=*Zn$qP^}9jJ`>M0O*+C#`XDi& z=CEb9^Gcjf=YZ)XU|&Nga;SpI<&SYT^WHnv)auyL%Iz%?5(Qd2n!gK_k#mPYMjCk* z%!@XI672Ai7h+nKBg@3YhWzUhT_}sg?NarYe~;rKroKTEYi+^?%9Extsv+6>{(?e4oE?UF1&*4?b>`&C_SF-g$GlkrFxP3Wy066oT*? z8a_;o;)px}7kNXv53B~VM&<)yG*4u%oFRf^e=G691|f~=8g#oxIFJui#oJserti6umNecFE~oOIJ|}}19CED z*&L}i1_A#8+*uipwpwkIZa2L)j+o;Sa1m151i^|7jz6ubZI2hOm21Ey^E_^Bf#8IO z9Mje9H-8jN9?-UEXuvO75Td=S=gIr*d0n zAJBUu6uED{bG)snH0QRy)W1;_dt?GKKMr0;L`TDRE4frH(?M1)EGvI_K;~qJn&BGW ziBU119?Q^z<_ra|?>uX@^u(YFJwuqjO}cZgYG3a5ZLoc#9HwA#slcKpv(7qQ}ndQVA{igfH5C@1 z8U%`E5JMy`bmFeE(`OqE36w%Zh6il?T7}Lra;S30u6#p@G)=`o6H+ldH}pkS{eBL} z{={jX)gScmbS11g%}FRziwV+!(r}iSpS4l+=UH!(TVL;UyXGEx~{aNs}wQ&2-rA&SkQtC zu3bb;U+OXFNbDv#RC42mDio3&t|{(!A5woe+RE}|0r0zx!z(HC`q-NE46*@(>um+@ zEw35`9sk@PFZ|@s=&e9V)x&;=J>JQ3Hs8&~^B$4J*?=!<1rKFf2dBN0Ab$S+3G9Qo*pr-5%6;>E~F{k+;%O?+DAUmsJ}siWXK*=5+A1 zWPgtY?dg@brGK&?Hrx4_pY#Jz?jFc6>~f-I_pufV+8ScE8UjSPh4nfr;~{@h;Ix+7 z@%A8=&x|};i&~TfkZk^?*eq#0sP(tv=#kqYcRKOk=#eQPe}5B^h4;6C2?x zlL@O%wXRE>PrW<jmuGpe8wP>%F*(uv{UotAG@-PRAyeK!?IIjQUnpmWKl!yUUUd;))FKp}jPJJ6E` zGcc5!5{RMddZUpqi}@w4jQRsx-GgeDF^M>ft zA;u~Dc>SnC3HH&aKP}QLSyG+t8u#uZ>=tm}7ti0ymYe$e2)y=(`wWZt8*<2H=%~&5 z2D3P*iPeUKjjPTkMC^acuLA%aQE2GFYYYCBenF;X3APyo2(mnj$2u{kw^Cr^93re9 zORs)(Rdg>KQ3AW_NmuHYPoRb!QZ)#fG5?kecidm*zOyp2z$G!rsXDno59C1u4TOSZ@@_<_~|2- zfv;Kq9C)%F&~!@2CtJ+oofS^^-4F>3v4g|bHFi^`1$-QFrKEqPL>4ru%}F! zrMdOHg|$7*PfmZIp}C*u^+|~Qxq8TE`;n&tX_)n)uYT$;?&`w~`3@Y1or~#n2vnbV zN-JY1F;&X-e7=&%`{Hv^eH=US_s7IiN_E!-z$C3$;=?;B@K$tn16MU z-i(un6I;8%UZ07me-9Z1mSQa0p-YEilF9D0IIf*!9pp(K8g*VxqXq#V^tz6@qr>SA zC%J>;1N)t`ad(q3g%?xcC6JZyK+hdF=_cx~ekL$|f>iJ9W5I|;V z&-UlmYRKt4I3}#_cl6fvUTir*BJ~6@mTFtT=CnP z8r-|MuN877oPSB;2o+M09;Qb*l~Q#>{b*m2TeNm>I4XG0Wr$L`_#SOlGAx<%q$uqM z<*~16pMQfcL2zW_^K3lCLXgrt!@mwgt4;F}wOWz8?vd_#`xa@Z8Xm$JF8vMH&(%4Q z!*$%NY~i6>WD6;Q;g`zYSGDejUJkp*E6=l;b|%Ma-u6#qOweFNU-rME3?>8#tqkmyb^1;>msC{PlCb5nRzy7LbWaf(dp`?u2^BZ+srKNm-@au`xsXP zd4G|lRa0;0%&Vi$3OB;?1RwgqCEkymZg@pfcY~+$#*=)Q_5lR;dE3x=+q+?V5$3Et z^(_}D#V>nZe|GKYiREnbsip$Yr2;?l&^(}r{)I}_7PB)Rg3Ny3Zea2La=*MqQ9P9+ z^~d);TmMS3-t=*3YGkEIF7-jr2o~iIFMs2LK)UnKd3!2<;VpRvLs`Y_ zdM_Zj*^0Cc_;s@Elkw*8W#Qg1ocIEm{LUV2%->+4nO`cknBT_Z%!SHe*$s20o>4d%V9 zKT#VG#VY8X<9^GW+FqCsQ%6QhU@*(u_d^UT(-#&i4^U!t+^D-bEoP!p40Z}%%+i8!!E#{xDPxw3;*!j1jzNic%{6N0UE+t#KP(l2QGf8&YT&z_ zHd)9!Bt_2_&#hb(y_F--JGdrZ)wPshNSmd8uuW)H^)ucwXaU;A>td;Niv za4RaTbswnj&~I3_o8!>S7l|ZktrbMi%K9TBq8$y|=T*L9y(}W`tRD9x7O}nT+C-g0 z8`=(SHyw@aR?$sr^YXTE0 ztFB*NT8GtbG}H3PgsuwuAsen~-4go^H%^EMo~n4qfLaQ7k-L>bF$rqX-_%{H@a02@ zukP{o+|4}XDv$o3-?OZ|H_ z!pS}5JPI+WJgQN<;ms{6af+Pz>(%nm%VLP};sk!2#P?{Lifp{E0uRU$1?~2PYKb>B z1$NJ)m=PX#XRqFmSn99~H&0;A-tnl@{lQLq#PzUR1%hP02!-voFBrM2nxkf1B;)zw z_U^T#am6zAEy}ZfUVmqcbif(Haz%XjR!(laprh}}3J5QE!Pt?%(PqUoGCYAWj_)ku zr5ToEYXt~jaL9gTXbxACB~F2wfrWCi_6`B@Wm189HKH36Z@Nt&5OLc(3dZ@I-kLMZ z$o*=4`5o4K(ve7?itdY$5>Rd43k?+7<7Ms2UT*5(DaMrY-G5?49LY>ubUCGc+-6F( zHS|Ag?#-}&X;_Zl<{(>$-C?9q4tY;X=gGh9?=WX!`sVB2a3;K)S?)GahvRc|Z7I}j z3dq)5e#STD_5^*FCumYh!W=3;9<#ZndlD6c2RA;*Mq^mS)BcLWtzfIY#A#S6d+8D9 zPUgy-HCgc2et&Axu3>V}d)DAGB#{aCOVak&)dIozzBzD_0mLAKeH?*_cr} zt&uOrZmp1y1aWjmmoP;4b(#L7I{hjxh(I1#t7LOUnSZ)Oog>u>vWaV7?iIz2u~T2u zA{|ytUVCAN%o7~*_&Ju8#A?g@rp$XkcdiQ$jE*2&{f^4w__uO>jkM^AT3SiQHb+M_ z4N;#yPo!IB!K89)()MpO$Sm+H5>tV1hmdY+=C>a_NP5JNkBS5WCyIRZ9+iq~p0dk` z?NFGA>3=&+w>40|v2(8jLA?7pNnutgl?4^>ty$jAu&;aGCi!6+uL-X`D;|R|gH6>$ z9Jna`8+J6&&#r8)bx<$=YzbbsS+>g164;E%~8q)vxaP=9je!z zmKhp`jUXE;i-ItD9Is!QV9AxB!w}ogc0Fd6^?!BOq`d~Z+F-c0DR>~tP)Z@Jg0XQ} zF57K#r?XLkM;|1g8{6)SxyBqTOwPAp`yz@6^hQnTKIg)302N~(laW`N!5OaYK2>l~ zf{TBb%yE2SU@Q6TimyqizI(77NsYdeK8+s^uk0uOi}y$}cz3QU^Z@${jJtb)4M+U? zqJNha6o~%CWnaJUOl}U?&kAK|?{rv<2Os72oNmcMYpv))HaUj;!D0C0v{!Ywrz}j( zEm_}K5gLMh9%VRu62GzdHIDR&k)IEyLw-vw>D`n{4aVF|MnT;NC0V^lH5Y$-xOd~e z)>1VfOvm;a*AS0=$(v}`59;!sk_tDa&wnP0$_(Q!_Nzo&3tziow^=?dus4=uUp5G{ zvC**1cu-R&f2xu!ew(aLM0DC6@HQ*tf>3K{C|YHlI&$5M*7;fcL2ZlgwB1EXA`0Xn z*MB3l=C5m4mBY!m6-K^mH=nsXU2~rb_TVqQ(VCAAGrK~6-&etDXDp;R%Povei+`7e zqwS_8q*PDHhoZS31AVZo`T$q74B>nS+BZX(Z?HSND#_l=ydU4X`CHOdu6}K)M!l!3 ziofa}MOuS>*LjiE09Q0NWP9S6RLd)<6B!SwhUc&UdX5*y4O-C+2ycu9YxY|vKetQ) z{QBUKE+2Uqhg5KMf6DqHOyS7sfwG2h2ieKQUdOpFG)+mVc2Op-CV=JjO| zlK&0auaf}5kgdg??U%WY(q$)^&Wc~GR=6oy(ou%2QTqyKfyPjp2}381y?=0jJCz_| z@sfeR=?j#*qNTl2yN;sCXC7SqW z+rFmHH=K`8WUF-vC6f)>IDd?^tr`0+#?A3YcHlM&e#iRbnThtsPCnAT;|`g7yVf(i z6XZB{h^Z^5nYqC<*9AUJXV)+kI=|MgjVSwpQebpn^x5JZfx`tn1_6JwwU?~)ImydU zUuNquWbp|j0k7RpbT;wRQ*no{8o3uQE2BN27JTvoUfUW2Zc_pwr++zK2Psn@>-e_$ z>WD4(K>|>Uod}WVS8ENdj$uk5%3f|IK-A1v&)GCeQ?O_!x%r2)Y^n18GqJ^2fvZM7a}uBpsdJC=UbLNqb9*cgSnPB&v*+ZHub~yXN~V{ z8e!(t>uVvLHKQWq;~eU8?{n!kDV$LRpzMt$_Ujdi={QY=yLBk?Y@sMwUL&UIH+^zs z&8Q(5B2V;j^8<*4(8MsXKA|9oUnyKkyo6eT`YLnOx;7rt{eKJW`2K)eRq1j6B7 zkt)#7D~RkvLYd@5x!?E|VyMl8UF2EYx52ZN&%w=cYVhR%+&cQ|c8F;AR-k#^rmnLp z;se<_#fFtgKysCcwXXPDGu=g2Hmm3p+Qbo~(&rxv4DBee`1XErVbdbY*QM62oAa?d ze4cT}EVGpa-hb|aXfn`qLZGPRBckg1Sid(E0@djDICK?Wytmqojha?N-`odR@AwzJ-)A;t!dw)jR zhrx%VJ|pbpd%+mnXOp{XAba4pRvHLVwk=s?=pl|a8Gm6sp5>STa2vNwR!$c9;_CNj z>LU`1&2E8LR_gIW@!>{dp%cqKLtawGdVbfoj&?~5kuNwu36#b z8CD4q4G#sW39@!La<$iQSk319)4S28?8$bU&7WmG8xE;P7J;wow4lu02cV%YTWqQx^V&JYG}OdE9+3cIb{+FMf;0 z7|l>@?_LlH&ez8`(kU-I!VzM>L{THGmT-eja)D5i7@~m(!4*r|`^!+DMz9U&M7KwH zx2F_-2rhFhpNM<>N(<$@Z@*-nE~YO{m@z*^?hIP8`Ym?|f!zDO^)2!dAUcW#>1hZ4 zY<~&j7R;0F2vOEfAVTJj#)`B%Tr1=-Y`E%FNa7nf%_*D;fxU7M#EZw~yYW^-V!5GDz`6>d z|Iq0Uo8V3x)e>JjR%DtL+@rsKALs+BA%8D&T`&gB@O;c^db8nJSxB*|aVs^l*ZbO* zH3!B_?_Y`KQHQNeNs8{SQO!A>w$8GMhXPTgwJZK$flEhrSjJ4=Xgw`4oGa<@*#tqK z@F!OF_|R$tw~WklM$xV*bS9%zt_YTC7KCiMA>7~#I(JAS-`w#e`&pc<%7j{w9CCU=9US) zV~>R2p<+SO{^JL?j>xu7Xgf7x)Ep-eyM098S7(W4BG5~7@Fp}T3=mvK z%c=1-twioFMZ<^-qUJWj`ARo>uD4f3kxKbjRFZ3*+c;ZQqq_4}dwe?sjG=;Xs2DYo8R&9QWJ!NIV0{$* zRV)jty;^cDgn+;p>e$@HZBI9e+Qv2J0UPC!=~#^w+WRGpJ%^u$fbl&z0DUaH{hhW_7*k%MuJuduc1^Y{0 zf`qi6x%M56<73;Q=CLUlsM|rH6NLHd=qjH4>IW*C|4;L|8t!`Dut9zZV62j^7=5?=$ zVak}j&sW09SlJ--3f~C2K?r~fa?ZOAV)ybRY|@C!#^VaoNEir*giWftIdeIfd3qaV zEPku2%Q*pasgO9S~sDClP_&e0a&x>WGgv#O_ zejFrGzLANXUMGTe`hw5iszL2MZ43A1ku1aTygKKOh&lgmUr}D=uX<%{(h(6Q=$izJ zM#5}T+mb1gVQzjU+blQr{W?D*T#}uORS3iqMTMR=>!@HXj0AB}8f}Ip*U^t85+A}} zs$t~kS{CJFqklCShCe;bJHsHdnahfEq9!wnrOJC2k)_#O5`1cYg|2RzAE?(SlH6Cs z>90*)jfoT$oQ4tJV>yJBq05 zAGd@N=B+7^43Hb*Uk+3@U+nVU^y?++A7dxW&2d|(IhHTJy=(GSiD-!E-+gOxW(}4}mnrE!z;e)qj4%mk zp^8IChks`IeNOAwO_mYsy-fTuSnBOk-onaW`~4;pw6T|3xZE_=-=$e=OoyJjGZ0&= zrYQ>cjnxzhEBaTy6coYXNBoH*{Y!3E3~ku!l}OSCHTp;)Izn6)s|O#B!{7CX>VcOp zGr^J)hTEn6%3CPYt~0rArC0&gK+TV;-f z9(~BBg-8$N(A~=Vy{kFly}thUOsF2Vm49!HImu^hItXjNN-yNDG&j(5VtMi`Sxl}KO^ypVIa8-^oK9N4aifY=vp2i9G8J&sN z)~6N)95iE2`H`UeZ+<3=@@kKFfjd&VUM4Zjh9H?rqVAkorVuuauO}#6;S0U{IOW~; z8ZeR_lCao$Ywks0A#nnU3;%Km?|<6KV77UcQI+Kdqptxnn%$}lt}^{k(uTgscQ)gU z+zNLcDpGJ}^yd71-XzvoEoUE@ZTVF*EXnx;`YuX?511n$Ol7FOBmI_FG zV?M0D5GFd@JeG`-+Uw>uV?rbR_6B_=Skm3$d zUY1|U{v@Nue&;QwD{}6V7*ip06%or|!t5E;<(xZLCr^v-^EyjJ%*qdOkqfuP z$~kk@Xw9r48VZYTA2w0ajOLmv;U&MiisOJowwa=|jFgo=w|^IXjiQajr0!=-68d+% z?B7H3#e^s`tHv*9_X;dl+(dWS!R2!l*Q_d-c{3y~`6jQKd5^ENEQ<7s#nrUEu5drc zLY(lMnNYUS$bX`{nUwTmA9a;gAhCQ|nn}nD^g!|*bhX@%i0S9XBN!~vK6n~HW!_0u z@EQlbfet#R(thYAHs0w3N3CoLDI^Y`6hS;<@lT*;p*5C3MrfF4JQJKbfB8_B!s6<(T(yLtUt_s{+u0P^ytOu1yA2LXU& zKmU*V7A!=WWC|LN*{lCJt@_U*u}#Emd#v^+N5NJEDiU3QrTcTZmgTxs0etXu7Rw5P31fHG3h}RA*<$qdXLM0sa3QSRh*;BluRpwyvrEh|K;%GoW zzP5QY@9gw8D3`qka{o%cwgNl5wEBvPJz=CkS`#c(Lg$0q@9b_VpLfyv6Sqal?-_x4 zz8q)s{O08=rc*0{(mQ_s9T)S{Ia`#W^OKWEObU_q+J~L(e9y`C%1>i*vlc(Ow13V& zMQTSE2q^Z#*cAXYjt+Vi3v`8k<8o*Or726Ohp6vBJG8VkfVP;-I%D={uLz|ZdbKKo`8~p# z<!(8^w#mw;VX^NTTcgY27e22fV9454j|v5*VD%y_ z6e~mGR?to^@)vcl<|)FlXY|+$t<+AIiC4V)nxx$)I|7(fYM=!#;`Bg@eI&Smu|Qv` z1&2&mK8s}2w{_~CztusPJ7i%$Y=g~FL1u4z!>H>Dt1?Xmn!V?lbC<5K;PokUJo&0`kdYS z2|m##P*w9hKYFB5qH|7B6 zUGhB;KtIx$U)uD#+CG0QiGt0scBM@gGplqowbV3!i z5Jj}a%c&up=^{p?FOX4CsMHaqU#(aR^wo$jqq%GxY7VJl^x3YYD5rUt1F0pqn8nig zrYw@on2xNK;i_H_F@HxZ1uvH7aqS#AM>og|lT$Q`pIjm&XxfucIate!zwAM6>A7i` zKGgbS<6tu_*SfdcgH}&%1^kmzSwC>ii}LDc^PZ+6B0`D-?3h0YEGLYWPR_f+scBa# zi;(v#;{feHyRw0`0cOR8U}N*D!{A{O{UEP=p*6=;Fz6wdMt}R$;l=5l@1&DoRb8lp zHtBF5yxWWemW$xHG*PLNFuJORvTU%QPT_c)KLdTKUF-ek>#oeE<{tJ)N`!(t`y;+Z zB52t&!6UeZOIn{SMjjUFZx#y2ZxX1Wu) z6P`Dd>tdn9INjldIp{KT9Q%`9hE&?JZgF+utCX87MY>15>fM~eeH-VaM<2Tpx-eOn z&T6v#Q?zf>p)E?Bqpj__)Tm3Ac;Z7CXu8MK>Cg~R4}TZe_aUA_=#jE`a|na5p%qbh z(nym{dOf8QIA3W?KSy`!vggENFUElv=x^V#A+@;IImY@@wzMsEd;$`*`PUBAbrz>7 zGN36>j$WAbG9c|EpQl^HG4$4A%j&Z2hIl7oUU3iNH{6r_!LiG+(RiTEj18$Z?wE0m zom+f1C4cikwpmARt-bn@97^UI85&aSN$GO|l!`YK+G*`ca=d59xlUIj7KpLi2%60H z9RLo+XL-Aa*(4(sa!%BRU2NLjE>^{k%*^>MH(lzJXuM#7|nAI)+`X^lW6Bq7abKdirG%& zpgtqm^O~2MK+VAnsaB~_W!AtVmfXmU$)DPq_WT~VkXxbLc}%DNBvoDt;PR?qd81u- zh>j6U7<-Je{!=C-@buTbT>I-ysvdBk3>B7w-OteK@P!eAByuxW6}kr@LQZN5LZuul z%76F9^YT6GGg#*3&zmW8A#bH>>`ceQmaK5J;dIDwx^p<-ZCUWF(Y=7d7*oOb%KZN!D(AgE=E9tLJr?5f(a00cK z+L$roJ7MfrUVWo2*7OxBv|JPG?$+#FR?l6~GEXo*$Y+R?t#$s68aD4C|%gWaibENbZpt+{;ogMZ$O z%{-Cmr!Q;WdGsyk;o)Y9H5Z%}`h-4Xv<{tFrI#>+KxH((SQ{CAZNDuC5`KArLlm)22aGCQRsBACUuZSb7HtD+hNb?MP&bm5CBF-(W%ORXLC|1e zTW-L@<#@~tYGUVpvQ>IPu!>2xe(mo3Q;yjZ5+G6}+`0&97OB~E1!wBeOQ;B5O`?x^^nVN$;7({ob z56l@%cXiDfn=OgD%1~lh6!j7$Dh&VduIqvYB3wRMHz(VA*q zz>9@GH>6I`8IY~%u`+{Z1b^5q17R-tB<^6Yg!zkYO1UFZmM0s%3(y=X~Nmj znz@E&M0#=-$|94?uHM8;q8xIYp@T0!maQUQ1sZgh4p-@gg(r!);v+yeC2y{FPkvWu zyAZs+L`*MjW&y1J(Z5KgTrM~5>lL6xXZMgco2v?kxiCJ}k(+h}H-DNxo1l3EMP!us zk1v~mb*!pmhPaA(xt9c5DIiF{Se*k3XIAGcGWqWMdRl9~{Iis1h#>FTWsjE$t>5#O z*pqhm3VZ(XZ@G#Lx>E$XiU+#GE$8M>?y5CRngH8bd9AVHsns{VN%ROZAS&^|oe}SV zt?%NIuGmR9QCZ}Rxqngx#|Nm0Ycg{gm7mI1XE&fK+G^t|o6?^g%VE>Ze@ZVD)Dkqh zP8JM5OkQ&XG7(eK%zfhXIA3^Y;`1|$2))jATcSNSZ|t-0FYhZMp8RAUXYqwcSo|jz z>#Rv$9vpvnrGXp+7hGNU_JHeqPkC<08gc%6Pcph}bX&o`M}NhiHZ;HD<@biYkDm@H z{5Yjl4Ndb44etc<>YZ51yoFW-$+8yBSf|fIiGS1F+h9M)S}9tvAMVLF>RY;na%^na zhBBH)cV)Gri#j^fv{i&_+cbp&y>!eS+umD#4RX1?)jKhjN_Y}+d%u@!EvtysFgHQY zAfFJXe?dO~vVRLXz|5>n_zRE=ZLM1t42V!iGq&ju8uBEf40DyY=E0_)!fU84I#8ZG zI+YHH^9;!g8kDEUv|P3W%Px<{YoD1)2X~kH^k~&BvAbREJouyyuDPZcX?*Al9^hkq zo0~bj+!1;n*SvCdH~z)TL>7{TdOc*smY-Ned+2P=EC7UvKD+`oz&?5HkhS%tf~Z zXC*sMKMQ{ME-asy1JCzeA7S%6%Sm0|sSCXj48zP4SU5>io9#Z&gPq;+{hFx|yj~3&Sn_m2lUJa_9 z)6=X>_#V7OkQ>4ZY7sp29t)>2#qd6*R@;kllY5Z%crZue=kEG7tM#(V?LEnjhU?Wz zEVW7uS~;#B)2*zTVesk6AoUr5c58FTHmz0*^?$jeq4bdOta=@vNtf0}Cjb!aPJjJ} z5`q9A|G#uP|9@mU_x*n=o&UN>!1$AvJn70sw6E6r>W_m1-u+XFjD(-+CiADV2>|T} zznS{g=A%`C!3xU?fY|)CKwi8nF(|w4f372d>#2bh+3(M5S>j8 z$(};-^aQ%*4({ifx+*1lip85_L{_SRP2Rl<3`|!t_=O`Rg~Tl=WzT|9{J*;D7V#?B8|Xnon1e&$HSiR+-O(OptEPXDjL7 z^*0%;KUOPUo&l=gGHH+>3)2~Clp7D0{@=F$or?@45u7J&SDdXfXUw0`SARV%j;SPB z1s%se)YJa=gti^4pp$=(+V{y#i_CN#1 z_jQ0=V)lyG0AGQq0?LlOGq*z+5oot^a^9b9TMyRn>=)-ge`NuGMSu3E&;R;C8s*0u ze{m+aKcD?A5BMuX5?ad4K#BJ^W)0^5uEj zBVl?aIwCy4#%5f}XRlZndG;>L={@@Q^<0k)TEOT|4#>PH4VMd}! zma$Kl0Ui4rTa2u@NIff2uOnqoC41y~JN!Io-I z+SPD38_*SV;Xi;G+f44I8VFM$~_jUKN6~4 zQTUhR)qiZ?W`lJ9&wiNRdN8=9L3b@ouK=dE+J7+t(B~gU{C}JG=g0ry2c|Q>{YWBR zTOr?SxBf(}d|NbgYANq{y;xP6SbKd~Sv065gFPjNt%1Q?91ZMtJWw~qC*SxcXhNMBi? zJ<<=`{^I_hPqOhJ_D17h?)>(1XM_G&qkK!PRAoPG3xEISSK1)o(hsYBp3Z{byofz9 zu8wl4k$mX_+V6{UvCjDt(So(92^LU6(fQ)`t7M8ozsh3%$WcezTsxX!U))B{=Z z=QEpgS$~kR^izr8J|xXLWyL-%@sd2Z-XtBIs_ueKvf)#{q*er%=&_Ry99R|_9A$-@GP{%@WA!~g%qdh{=*+X&MgNsw+; zm*`1judOm&&Sa`7=N+(slM_Lm{azidXlr{Z4}Ud?R!56qb6A^ZloO|a$DM!pmwLaa zJJs*F9N>P(<$2~WAN~)wp8w>_U%k!#=PUk-=-r7-8o;rKAc@?9lsPR^t|X3&wh+|t z8Uqp$Es5uU-!I7fTXQUrwxpZ}k-{J!il=+m5oN&qs=h^s&W2q7pjXYN0r#u- z!+$T1{>9#Z>~a1lZ(Hj>=jXxl9q+8wjqItZBClsI%S+q^e6H&Bsy+qfo|xjSRHnum zn)Y5_^fofDx>#BYn6pl&8>|u1{}>hxI12=oD1oX1X$6Gs$pH9aQ2SS#MV__>wH~6g)}XWApt~s|(HR4N_|u#h4JLu3E0Y#DTKheAIysZSfXsmN zl1_~VKb$#9i?5`=X_e(Triv^9kZc6Bi={ggSY*h7PGVWE1t&)XOJW#WGejGfIDZRP z!Ku-L{j*=@@YcrY7MIW!F0-`d@IUodJEFmLwE+4|#lqwf60>R(OO9`8T6?)}S~e;kQcY{}*=nD%%e@K+{y zJfKwO=(>vNp0<)}O1%Cz)_<%ef?jrndg(w_-OR5lS`@aDVu~ zSm1{bq4XbI0BrW(eTj4hM7qZ<{*Q-EB2z~OZ%H)t8#f&J@zft7Q*O5Qe74dgx{I_V#O=AQpx;Qwm1^vAP)Il-UNUmf{Rt^U=ZpY7Y_ z?slPy-YS26y((JhHakKtk#tz=ly;J7x2MfVGgwQuMZl>l@@a}xvViq(I*Kcq!DB1t zdIwl{9C#?lR=+r)e|SxwmHY+FpFa}z&mVvM8~G2f_=iXJkE3MvKYz^fJo@H_|M31_ zt_$>;$XtaBY;74dix12Rb6jO93>zVIEd|i7g^P)VkHc^s%D zhPFkDIyF(GCzWw45s)GPm6rec|97`&i4hp^-z}@K-In-=U;lU!`Y&Ik|Kqa%YVt3K z;JCXSPkMBrcFqlBG!&b8jX`<&CG`=-|zdrt>2MpCIh zo_ea-RMJviQn$zI63ab*{;%4uwmETJS${vjf;&}PHFIWQ{Qd&FQza8Zn7quAgvpXG zw~R5y!Pty#;t=Zo_ggI)uxt~d%sJHV1k0_brB?SNBe`uf!gB7_6}+c6m)mSM2wCUw zecXlb=j8j{W`l-#xewp-aw0!~?~g>Dd?)hcJCP^fH_#4*ZXu3;`yTWQ-g^w~SmAac zs|Rwn5B=)patZ1G2+Jj;|L0h4Li%qJIfe9}$UTD({oPFtym5ix`pvl1Ed+I)JMgRAWb4BE1no#D1C#hCewvHf*6#C?U|YUuN&ZdoAj zb%}h^8^l+Wu7$C8vV#F&A7r7;_j}nM#@!U`I2Zx;*PUJF(C#xMw@dee$mcd0c!0f^ z^`YJOd)Z6ej=gM>Bwj*(cV3}>Umx_4cXJMFGcad8j0SOkg62Vjt{0;V|8VViCm>j> z+Q6E}uxC95%M9)2Jh}UV(6pOu@EY;2WQZa)SC=uqRUO@4^%iJ6zlHhJKlfGGw|?y1ycZeP&rf!-^S9}g|O{yc>DJ@lu>gK=6w`5N`F;eNOT%J1^yE05s+jVu}Z zKh4k@#&e9@?IGKP@qv9(>lxbhn4!t9YkOO|x&!FB@9>=fy_FdrPMXtvq{UPle zW!87t)EmJ57-`qKqPJJY#r|?WCVqZ@aSi9O`~VTZ#QuyT=x_8^`vy3E$1ol%I6od* zKK#Y~x^Qy)ctno0%Yyfwk$QNa^OE?DB|!Usc<33PN9cgx68qw0D+k7(v@?PBO)_f( zad`#zUG>H=KSX)lWxx|x8F~i!F_y#eEx_^BE^bl5U%Cdm&#XS!X}>@F_vf>J{r+t2 zr`yf947)fb4R6tnLHhvaoi5&m%wQ|^2SNr+$+vi4K7+dgMwoi=yoZM}fv!(scku%3 z+{O8>Pp}XC9rkg%4(xBPV5{yj+wJqBXNddjA=&_c+#n>PvoXAOT z(RqX(GRr~d0{X|ZW_Z$udliP)=*jDzp&JE>S+9p~Jfy%o2JmY#&nD0)d3g4SZVU@@ z(9tlmx*5FtU9ir)-rr!E-mrIvuEr?MdiUsdcrkrYSJ!7qzV)W!WVy(H;AP>K{Bv#J zqs{fJlb`>7_G6fzU))%`^Fv{9ek+})lZRd|cdg!_pCh~fQy32)ug4Ff|Jd(!ANq^# zsJ}QLNkIs*KmV~_?T44=iFY%!`-`*D#my?m0l)=6`g|SUFZz>*U_JfkHXvmK;ra>Z zqgY?R8t5$h>#t9ZSrjkr^y7y#e_p%kX61hP^na~|ow_gESrn%2hS-usciFhK@7;g? z_uBTuFZce^O&T|D*0_z9cK9W6V}I5O{LpRAUEiChZLuYFQuifo`r*_K)3!j4VmF++ zu^)PEd!0tPn#RAk&1KS@``*0iL}41+N!qlhPwOOYhf(PMkH1_+acYNYe`gZKQ#Wop zQ4qy#XC23Gn0Db{2SxrgpWQg|qp)qyQa9cLC^X`O&=R$lBz`DKFYR=`47N_-uiCNe zq#woA%a_9cpK`rzp|k7W>ZKDsyYVcDvi97cPTjDxvZqt%nb289iJxNilOzh(soTl? zX*!2mCC&468GLJnZU(m7e@^{m71*2hB#4~FUUFh5?sg9uyMdkh&u%B1`>ESZR<`4| z!zhdGRj#GEj9!~dKWygR&Wj5r6!e$fL6XMnbl(O)#KTE91;%hP!*0>ch$VDEpicWK zIh~}Nz-{}f9rz9<5HE?q%wGDzrhV`#N-CMvNZr|CatE$PiKc0df9U;YxLtQ^T(r`w-U5AMPX#piLnY{^IW9Uf~Oq0!W5}n?aqgeJbiI-of z_DsG|13sxyg0AnVJ~*{j?b?Cw#rA4mGRZ8q7nBYtsHuM*+lzEf4owjHGi#sg=#(Pd9qH&bb z5~cHw8w8yy-#)u>>N|GO1fLGulPFE2B@Of~S=k}Y;=2h$f2bnX)TIGxDYziqS`o@F zPeg?8g|c0x?b41tKWxUhMNO$N8xfJD&Jtw2{8txGdN7L+d};MkZV2O;GgEyhnX1cPAAA;No@I`mn zRa@FGc`t>Ee>IoMdX=mvTYNo3`W?8lLa5JT+bN7@LC$eQicLVW(Sm=Qc>d2FpO`U^ zkt7QIX{XL5I(0^v+Q}S5_LiWJ$}0CKz=xpV@QpqnuY<$E8ZP39^!_X90l3HE5~*M_ zxJ-H5v7NwP?KK3&gE+8Pbr(RTfdPPTEk6w1n8GQS2E~W6ZRJXrgfbH_738*X)+ zsODTQbKFb*gMMC%1t*uuG7~caF1P(M6KVkqmqrSU9+V0_ocvptax@bz0kW5&G!rcX z{YID2G!vo$6_;~06Ivc_HdusVR0)IkXPChv45LaIColCFEXFXZjG>u)7nkcb6C(@< zu)!n_qe>jWPkg_Z8a5Mt1-EPn%-ffpHWN$%vX|;M6Cz0xG{-QRTFer8iMphTGM^*A zd83w4Lo$5mC&WC>Euv{^q9W<1UOtHITQ$WRZV?1iQ#4tSxU@)2EmDcWL{(SSlevX` z184OPmsK|tLI*c}R`2@6*O#6*6D0z>RhPv#6FvbumjpNyHUX!XJ2(@S0R)%PI1>{A zz?bDX6EhXg`Act?=(Ecl4c!i8>E(*{#{`!gmn}IH91p~SFO}knJBwoX@Far6BbQh? z6Hfu|m#8@tCJ0@?7xQCh_z9QRITJS_Gx@A96zyk+Ua$io3R;3<7W#$Hq|vHeUejN? zp)EnC@0Tt*6Fd*|g}E+ndcoS=Rs+YG!V{NyIukq@7eGVk9E;6e-EaAGb!do6t(s@M z)B}AhJeSTo6CPR*``K5YLH(UU@-QMq0W?t_Xnc9Qb7K5stPmQFy)Xay6*I=Se>EC^ zex1Aa6n}%t@TJlCbLu}g{OMmGAlvuDw?Ds5{bwRfWd8h$d+@LS2Xm%QNS)9o0MWC+ AUjP6A delta 889922 zcmV(}K+wO<_Dsh1On`&|gaU*Ev;-7)f8gWlERBn+Gr4kRhRHf7Md{4jbBxm}Aur_;}=sE7FUUcUu(j0zKf7%`z znfnBaS`P%ELABJ+a@6VqL8`Ne$>iB;QhBG}`%}0(gU^_5Xr#n^9IPO6C)hS5Y$lRT!E*<06 z*Ts~+x9ScDx)ol7LtJ#{zrFh7e+$P%H#x&ki_%#XC0LtJ3fDnb69?Sp2TtQX_`w?2 zNmfjrERw3$GE8cPhLLz_><3{kwwpJVeYjhrCZm^!3Z&^i!mRDoWI>e z)Xy%Lx=eAL>_ZXs4)F)ety93n3wu7g-Qk@RjoW?YfKwF|xt*_)GRbSLe`i3&SonHc zl%<~5*%i6bJGJo;{p2H6>SzuKj25+tu6|;*F6p1sF*JhysfFn8BPcGM!Y~NwTVNs->Q$Cs)p3 zzxVBho~+U*x(h|@x5%w~e+l-DD1weBwziN{O-rrw>a?h<_pjuuVxc$CStM7;T+>o- zd|g2Q3hpYvT*1%e@RJv=Qx!0YEA)afJ`XwQJ&Qcos1YLrqx;{Y{JTUz^;g089yA^> z={|#B|Kq^tbdkVuXUCe@v$7mPL`T(pyR)ivbvwQ^J}5;w@50?Sf5UYC!`-)7#QCy- z?v0a+9nW==>I~ud{Q2iV0?jdW%QFUwqQWW7#-N$zNmfkHi1S~?%0bDi((nvez6XwB z-VKs$9(SEwCnYuxFn_sylKh7~Eg(18n83g|wMe*$3Irg)P1X|nI9Cs$Bx02Ps|#@W zLFYJFo4UpzRj6Gu9bSMPVL9$>x%2VOLsp~iUU~_hO7lj;em0&TT|zkNA5MlB#nI(4 zC_le$e}6LmUGp3Ci!U%$`h%}ehG(PEb&~0_9=^qjWhk*cf4sh2JBt+*G}Mk6|8481 zVjNY{lWs}zvyRJ(JrcUl$1hfki7tmPamciKkw5{pckdqj?gamR**P`7ZxZ=xq04l- znFW09_<1{*L-Rp@xXTAkiXziVzWqgwy9>JX4B!4aKg%)f#EO%^IIyoXeTDtK9j#L+ z{0YTF-K+6@fBexUx_m(RapfM(kK@brjxaMzldPEU=+_p-3=8481N`NI59*+wDE~&4 zj(KBJTsjvXV^0{jdO2~q&mrJcyvMHr%<2B1cZxSCJGh%(3G{=DS={QFPB+n=ryk?w z);a!0J)YGd+O9Tz8=X~EmmrNg%hH@nrDN1f9#ojHe?QxRvCU{-PfZxs+`Zg53hz~T z_9I0bJtC?H&06I}xkx}2DkUtL_5Fuek}0T`1~F7Uy4-UA&xV&HcVCp`oZ!~Eb}vV_ zMd>YOv0a$HCeMb1^u1Ybjk;{Yj0SfMk~0ee9t(~3*JHhvJt~gvH?-4VZEbrCov)l* zam^0p=&q8Kj^D?4*JT*PvQu#jyD?At0A_GtcWxx z`0v_18}&wD{SlV$9ig*BeDGkrKiK@n!5%#Lf7Tx)M%$eFeCJ5fb!4$I0CUnFRlu#o z6lQ(eX^lI-hu800b&yfdIt*azSkr_SNs4T`4NP3rz2i>Hv+;lf(ho?*&IEkMbon0; z;5|*US9MXM>Z_-ry?KjCY+%D(LcIDX^jXjU2V|kTXg1dON7Zlosk<@xAnXn97VdBL zf7qLjmff{V&Npk&tzEx79Cpx5TNJA|K+7#krRz3vPP2q~W843&Tp-UYVTau3v$kjD zv7c7w+CHDYd#AsX+Z2TIbnP#1t!W~59$sG*NBa2PI}-%7e!KhK1yuPdwmtujyKA@j zkhN<(f4&h%dfQUE|7=#=60L~m3um)Yf8yr+p6J;AdTyOgm+w2f6@BvA1A8>^IQVx1 z1#s;4K%6nA-W%gQCwKIWydhHhMl`!iSoB6L+{k0Nz`Mv;w6T5f4{1h0;O2e*{V~o( z&!%MUXPtv`r~ZDo z@cJLK6Ac_o&Z%wvv-^LiC5j0e|k@~w%gTLe-J%Y)E>&jRc+{U!-(4R)~RPSc+N%h z!#aAPw7L;tjWCQB?T%mE&eDF}nb7c|vf=Ue`t;D<*5(-3cnF2S#rv;w8&&kODC*&j zu6gJ!ID(o|5cN5L%gt@ZT;nBfwo|sUik1+1)H<+Kre<^ha;@wrI zt5dD(fio?Os)AycY5vi#-FTu;ic-6Aa#HK^+VkoQ%jm(keF_O*E@Out?7MN*-av03 zQ0;ae-n4C=D+^FgwU^bIzUaJxmz#&HX_8@wtb2XcUW4oXW$bJ&uG)(u4|?qU^5o>CbJMn#^T`krU<(MQ_MImVQKQ{1%zhVRO21!He@bcRksI#a+H6LL z{1scbC)uiOzHfrg{|1((1qIclsylGGLdyIYZl#xL(p-Q* z&`#S1( z?Y1nr!kFxS>oQ66f1O5?Dau*LaWl+4qq%1^_l)MA(fmM+W=F1>6qnF9pmw8VFD?)4 zPlKvfd{%HL*(0V6Z-|7nvJ_%Jv%6AKod(J3yM?1Mfml z&eGa5`mgjG?Ikl%>&A@IUmbXh;v26{CT1Drfmd{|IzRDre|wTF((Ec;rCycfl~?I9 zJ#me5n72yZPUGO?DU8wUd;50#<;l`JP3NZ>29}tGMGkIU;nxr&d%Z}?d78)cpu2u) zeyYo~UZ0lNf3^iu0Koh9P6I7)#WXNw%gc2#nUwLxsV?>K2=&~@qw5|RW_ntf^jvY9 zYsHxrb**Oy?|Wc<%G^`#o(lI=x~IxL^{-6^Hh@zzEo`~ye?)TKpZfMd&2tk>FO$!& zyPZr>8L-A=8E2E33m$fm^7ncRFCb_YIJ$OUefoKke;3|MJzr%>>3**BtZ+Xs@?=`L zhiE@cD))Ihv3bfIc-lQIR%NOo80W%myZTN@uvr_X7i;t{=&V;rf3>bw6SrC|JFS8G zRSym5LA&dfIzw;hgC+;1mtYb~(9Xas(A%}~29KdK5{$W8Vl0wJ;Hn1G{57sUL|Nta zh70V)f2<~3V%+>#8`YDGS`4>X)|iUdt#qcRwc98#dvNJt#m+kR?(SL0-hH!@z&ZAy z*go2U%o+7Yuxgw8AKOyjGMQm42yH&oe3w};BwEl7$Dn}fEksLhV4*NKjz@cLp?ksJ zI~Asr?NBOs+kvaoRL7>n1K3FBsD9h!BHf$me*&TpaE>uDSd0|~Y2mt9)v!o0r~vHE zRSFwilBF}VXP6xr-(Fg8%j=u4EsmSnV#ZX=oa)(i+1x8ub^l(&&2Sd>^L0{APmlL~ zJ*+(%)(-6KtymO$e0kZKg6X2eQucLbR;7LCB+az9T7u(EGq;=9A5gAkQ`Yi&Jxi-) zf0kUu5CwxO;x#H=|FJ4+?anfHHgk74rh8htMjCYu7i;iuZ48LIzTSh88AP1St~U)u z=B_dq6tveky4l9z3-=+k@n`6~-qLERk*pdZuMKhYL*sm zdQv6}?druu&)fp@-n5`Z0B&j(;Z8Y%5mcZk@sE4*~yvpk| z^G<+Jk4K|+OoNx840DgLT0(uQ7y9^mhvv_da_D&{NtVGaZ^GQ-ESsJVP5Xq6f1a?> z=wSb|)Bdwle0Kex)oO`3q|m=|Xip~QN%|`|B8*1YI5o3_d({-?7Nb0K(CL2}JHM0> z(|FL?v`%B^moqt=^qVU@5=k}S&eN39;{3har_M6ST?Xhd~ z?JM(@NyPij`yanZaOP9aXZ_~=k85)V^-qGW=FQ{l|6J%0aesz!7d&IYKDKOKX{8T>U+wDhb>g+%aOjk*CkU`8XBOl)hn1+;pre`+(QW$eg^Fjx&i zp;mCSP*v>sky&%WBS%2RV{D(sP86COlu>_kCyAZFH_%|A+^lOJJAy;mc?GX<8TbxY ziUgGTNZ`2vmy9BW7h@_^4p$rsPqF}z0{8^%_jbX7KSVxZOtH2}#7@MCWPSvFP)0+C zMFG?!Rw{xArm@2Ve-hFl#92fNK34f8ltHBUcjUr9G&VBGyT}Z%V3I`e3RS!V z3VVYZEEn<~%Y8)bT)KE7wr_YC@{&!Ca{#-~X&#MLHz9j zNs={FgbQgteE$SRM)^BjIo+IGhww!U29GLJ}b^NxyBcVke|T82L#$0VP4hd;|~(qF&gx z0}L7A0RlGIz#w)QCjo*W=zySmbiFHc-r(bV*z-MN-XrGsf|y-ys27m(aP|}9h5s~+ z`3}$yeB%2YS3@W%gu`HWQMlY7&@&^G|xTQxyC}0Fmsqlp*)b?RG@F}90K|8@|z#Q{01(8{sJm ziS)5MN?AaChs%J3B8)iOqn4Y#ugfj!qx(pY?jt;QSJv4e?@yF*-ed8q;b07T`RHB8T}P!-wH_ zA%MH!FDSz7e#v>bnKkOigm0`1N(Rc`nl$2g|y-3dWenTMYvMKCgsJjS!&X~hP*jAKJBa6>0B!TZ6(VI$?zB^AW zjUhuy&de%BRG^}s(ny42qX6@j6o_rY=MLuq5h0hfMaO{n)Sxospm7QK&)}pKe{jMX zMG-(nipi~oLFEY8D(1w6FXr@s6>&&Z&xKp8--GMRAp2s5jZ^biK0Q|2MwwR zBoYQ{G-Qz@C?i6JLN=yo&Pd1^(n8208EygoeO(}L^FhMEvl{1tK>ijA=2Ow|5Kw$k zhMLdF2i+QvC|Ke{DJ>rieL@wAe_f0zDnf^W{*FsO2pZONdnd>^6;Xnjc`z23mExQS z4Zpg*3*@na1k`|vGa(|d_WL5@DrmUeEq&Z|SJ}|x?bgbB!n`NUd&2xa2(!BzmbxyB z8*Qnuu9QR^X+a7qIUbn8kVdTiV#bLwG$}+Ba)-$X)L;QhAoxK{5W)P=e<~3}>8Owh z77E$S7Bv6HSR|qlym|2W_+!z7ISPlqZ%|ZHFjsNn`_e}#3%CXc>_OzF)aOz-L~$Pa zyw&Cf(iqqkSKzpyB0?kS)6nviTiypD4un63LL668DMxV;+W0#OnreDEkhJn>beIJ_S!-$#qoQs{WxPY7~4ju}9G>aXdlYpXcAM>M# z?Vwrg_&!nSUckVv#QR8wlC+`RC~7gRFRjXbJX`J;RplN*Repe(f7a05dyTaB1b9z? z_XPMs1lZHWT}^bkT;*rCXyQPSh)~*NzApoZQAtE&#YVeJh`I_T;BejC{Gp9-ENuPY zRJa+|3b3J}RX_uuh}NkN^tpcH{miK}5}0+X6IPEaz~zonBE@%5YWz5y;m1gYVYrVZ zNLrjm?SjStCnPBFe|g@lCc+F+g0sNiETwU)Fa!qPZGJ=qYpB0*%nPCpez6;F^oW9P z69nBQ2D(KEceY1w=;IC=y?a8rCzPKZp>%hP^CZ(y4*WK|4IFWmafIbE6w(nQAk^n! zv>^n^LlcU?BftNE^@U>iY`Mf7UlzM6z*we^YRDL&qp{*0o5=l`y`x zGV&P)M1(vH@%;BAN{t{hSOSy#_jLBtPWQ_vrGRLE>MaSFR zH-RpJ45{p|xfP(OLNbs-NbU#~g7aJ`T1`78JfLRD15UXkgij&`LH8Rh_d|)CiTeTQ zrn=Y#_RD|>e`;nCGA5VUfiK| z1E!JS97A40!0w-b0_0y_r5_M!oO2`?nEh5Vj))%#f1@myegOQ+CqWQNX7wpjkqBRu z;RakuE3tDf9436Qr^_JPs7D z0RtMrb;Oto!p0-VC!7lV0OS1!erWA*C`jPjFh=Hsg19rdbqbA?AyOXDAaIx;62Sx3 zTK<>>o1jrJh%jEj6q0&ju{cY^sVd3ziJ%n-H8HxU_zq+Rtj5f-ADU2A zPNh_6Gg3;n#w7+0jwW>&7m{*^OCSa6Z&J+sSX$b^sGEO2r|3`--^db7$_P?)g~XSt ziOP_aFfJn#*vfq$m9;*F;4(`Al=z_uaiw7Zme(G5G2wvZCeDF!&bOe8k!aNCe@5a8 zD2@8S1p<&E2^4Br7!Q<)`Y86JDQhF7;t$k7ApyRB-?^v6drG{g#BWE5z1*$ENqU=8 zG*cup0RZ6Y&Z#0%iXyxgCiyjdnq7jS8ciZ-M@9`RF2$GB&ZcQo=Jm9 zG)Gd}3loe;6E)CTV`L3X7_(t_fzcL>Mp!0PXL&Jj!8fi+?c)9bacbYj zJ)+(t>JNgbU2eHds#<6NgoJ$kY&A>k*~g4)AN8Xkas=~97>E{y0@zoKe@-?EMdW~_ z5(><7Xc0e@M)+ZzD!@UML%83ZY{AvTH?H1H1|Yidkg!lh^yhZ6U81UhhTI5H64E|) zm(%#(1ilf{BG4jtR}&>kPK`;^r##$RO9ZKqh!`{w5Rctm$$eCMd$tFGE~?CL{O}oiLK9r|J-aDcwErh2f7-jp|9kxZ0r0=4{ZH~bsjhy4nM}ffm}86pTMSo(Hs{lb zlECDd6v1>f;NTtzp=eb1jgRQ!l<*xF(h-0 z%;Af77C3f-#wUWx&FbEopZ$g$6W_g9L4&zJ;any6DEx1V!d(t~f033NOE=!cV|O=s z$WOpo*LiHKDcNnx&5|Nln}!_ocwJ+f;Fin+l={M2<(sw4g))sum8` zZGnEY7mYp0WD=t+!Xz{_BmvllSbK1UY>^ReO^RU>fQWD*!chHFqGS$Flk99#B<0MV z7K?Q1K&fqKQMrRnf6eWIbr1_7v$^aXRJU4^ot7Luhbj2w-hAeiV1J2voCj2|GU&1& zsR&1q5y1opr|v#@Vn>0Sw9$kGehB8gK6uc06f(>>K%*$~qiyIIZCZp!Z>|E=3U`3B zbpqYT0r>voC*PON=L00{4QafOiQm)6J&pX#Xr!0NI!mTEe-y4+8f$wX{GKy$uO-=O z$#+`5M?{Z^P{#HzIj78FU_kKU|Fsz4P3<@Om^?qAw z#fcHqd_2c(GE?^Z{l1V89E>A$8h|n#pq1^f+=n3HUBe;2y$w_hBe}oU#FOzeV1(NlkVqA04K2B8t zx$hMeBvh2OQH7Aw*kob}BVRDg#Nsm_A%;SZ1ht`!ED@pa@KBKulCeacJa^1+5-K6j zouDD%wi-Lqb40?(R@V?A3-JK~J|B%6M*z_zfyrnieBZ|mI}vSShR|q5eJl^iI17Ss z8?#Vge`7X4GvXmgtbN2XZhky4ervwsatp7F=EEjC3w)u14O-pV)M-fW{oBPo?cLMf zJ?-6%_Ifoj%VfIBZgKAf*@0SjV5)!&f`BLr>AIQ~kx;0KZDm#; zb4&U_TvG5&kbNkrA(IGvrXhi27z!GxO(hb)e=Ri{avwc>R4Ajys|*sB$r3}kiM$ULl&9*!(j<(D ze~gbAmryaK2TH7hBBe4M5k$g1Br!M=xB_nin85F%hm0PLdz;uDP`$NZ;p$^v;BA_2Q=S@;n0lB>!Jh*cq zPfMS72tB?h)O$j`C)DqqPD|ig(+2z0LMJ)^C0Z#vbhX|&7ti_kS-HExmi~iBB_zoB=ZBv`Ah;9DEYHUe;EL& zH;sq0fSk%(z(eD(4A2dSQy!>r3wEg3<~)pySrWwDKIqUFGQjefdv)VL6h1fzngG#A z%4i3Yu_JvF;Q@#F90bHIIGPkX!h{Q`D3oFgjHsBQCaEZ=A;cxMP`q=7L7 z7%Bk=F>r1np|3)}Q;FRV6|?9bMGzPg@W@B4gpJS%15+)B22g4NRKb*oEb359D6NXc zxIj>|(pV+73F5Ib9P>@sB1C@poQV#}P^-xSl;IA#QAmO5R*+;O+Gafqe>n!u{~Q!2 z5|&#S4;9AzJ^(Sw#y7r)%#RpU;pxH;<69&*@Qoyxn|t8H2pi#>Os66cf$TvK(+7a| zt8ZxaBf>d4)6ggY@6s)JV(?HD8Kem*gRBvKc%q?|h9*48ml2J&z=?i2CZCyP29|e@ zku$_}m&_dZRCP~P|L#=Pf3K5WrMa%ETa(t;PY$ zfOEx&3`8VYM_gk@TR@czLHmtzlw@tT0jNr8(w_;TBqUQILBtda4c75dHpR+0q!d<| zgpJD$&2XZ)(F^ksz1Ag#-u!42PAZ{foV1k5AmNQqROGXbHbD^?f7!TK!6bv!qL3>x z2s$()xT!6~BeWPqpw3n__z7S>BLNk@@o=G_4(>q$WiJ}ppxi(L?)Lx@J3ho?p)$b_ z0`B+0fVzFFNMV9+0m0@o6ADt^bWooI8ru||3b3OO7DQ}ruXt)8yPEQ2!ZTe(J%-%iUnn~p$g)CI46J0vjMDs-sGX z3^-MYuoOkor(=d!<&7=4Sb^Bl60^_J13Q#Iw&!p$0@ zY!%uHHyxy>CY>ecL3d4&jhPz|XS`WZ#2d^A%^{~iyP7861GU4E=q+VmAl{GHHw3hO z$M{1efV-&Hf9@&ao)Z2wDWT_krx!(@%=GO%KcvRd6Z2_uTZ7V~d>9!g8wL)Z?LCY8 zHodak+*z6|m`xlOq*4*m#D5t9McH&)++mE7O)-F=tw2r9$_3rDXeM?G;>Ku5{Eoeo z8)Bo8bU3%k0ve?6++!*r=os007j^DYa1v>;zTc-Jf8o!;rl~?3w*~dN8-So}5}150 zeg_(N4&=jVXXx1^qh%275WeL%tC9@3X983v8R3*iqI98zk>0{b zI5Q|N9&P~%58aWpQ6hc_MX0+#!ecjWQ>#KsI@w32#`vggQJRoK=&=Y0R}4$3ZHH*1 z8>lqF6@n{B+S!LWI_Pl8jblrsRM-c(vABk4>Sh7t^J^h~*FDFO;2l!l-qYeeE#A}O zf48H>o`+d+a-#JumKX|)X0vZfI81Ulx`%%2pbqIbCF52P427?{Pc$d{C^BRu17BeT zmhwd;Hj#Frxla`&r;ngGjg(EApfCg{KO) z*3)iyg%LX#$MRnxVzrb#Y0Ht5C$aR;H%il zSSNi`n~f_U=$(-;U?vgs&%mK3ihSdrDMG#l$k-8*NUKGU(Eqgy$k>sbfDzHuP%~=i zE*z1IDQ1#JLmExd2P2XhSWJyC6^ANETj)47v}`L$L_CNjL_aVd30p`4qamcNf8pwj z5Mrf<*^|xC4~&%sL+d{@u~RH+UGkrwT3mN#12qmcMRI3i9ae(K3z(YqcAlQh@t_Ggj z`aV!J#^KZ_EQCCEJw?2|`gq>-A_H*V4Z z^?KZgq9aBG4J9?y$AZAIJ268Eo;G9xPx!z2_HSDndDsf zTj*wdywU3cqXmNvf3n><6gOSmArZwrh2B%>J%xVH6xy>27Uk@g6Ci`V-iV(}Nw700 ziRJ=akEXz#u(ojqXH71_c_v_CC&{jixXptu6^4jOKcXz?MPZou2=Kd%9DyosY19^t zl|cB`%M~l}2*`CJsNc;ELnAxYAd~@EKp@6hbUl~a6f_JSe`Se~%;X9A5q+0bM5Hhb zDF`5by1%9v#NZpb5K<3A#h%`h8sAZ0+9Ygf{pl{LO*Lj#T&O0~7!#~bZXA(_8#M;= zA+iExJQ;E2Z$c{=53KbFs!oAPAP`K6s0}hDJhDCq6zUd2_L(3IWvDjUC2t#L8asj$ zDQs;N9^oj4f7*oKiRYjOs6NM9?tRRNF&&KIx`7PQbl*b^>tt>G$ zoxExA2y;4f z5gB7P76$JuC~DW0Fz#4E;KOQc*3(3iN74lD`6gtdvz#D8+Kka?oc0?Rg@A6iqN<Rt^MTKSQFSNmkbft}R1gOcEh<(Fd)) zZ%p?g*xk1vYjgJ*RuzO$UAP0+Z71V>)DzyFdOpP8-V)Xw4uJO*c28mV6xN|Il4QE9 zuV-ns%#y2kUZ%5yi*#0>#-QsxfNL*V)rEJGe-#&TSzH_>Svt>=26bM?Cz-zVsyZoa zdn2v&qKc;)ED{G_uc|sdx#~C0lVyx5iCSOQ9=_GwvRG%Qc%GEQ-dDS*${00|*?Ai5 zyl7CiSSHi7zJi3owgtI??d>GVvPm*M^CsLa&a&z0(6mqZ=!yBY)6-})I#?#NS(?w| ze>%HME(#zeqSpO|%-hz8`T-4se zG+m9>|GCh!G;xNDG`CCa_>wL!N7oivwvb|4WUEDPP^H$3WtP;yqoJ{i;W1~C=8NQV z2)|u+QjWIp!k|wF8?a%$CdqkHCnZ^Cf5}vz7THXfAI;M97FhxJZo|3;M@{pZAThLW zr+Jp<+WW9h4(c+=s}$MjvMiRmtgm8RMt6H%-7faI_;oWqElR_KP3uXTWo}iM#hG?3 zD!7y4(yLCBS#jawU?)YnfCp<`y55DJoTaq~W6jE9>6!L_@~nQ+v$@Nf)#)P5f9KxG zDxbo2JY7xFsW;Kzq`Dju=~9<)myPcH3f=d{Y-K(xhJE)`!>K!NVzIEcN0ok~V-g0~ zg1EW$lp#iMQGDZ_q;-Rtwz=D4xs{jclR75K7w~q|SYn?|>#ZCY=*6`<;eQ6#HO=#0 z=li+WZA@=7z*=~iN{bJti{MturJ7PU-DU>v<0Jp2J93^l zHxh532eDlSbb|!=9!{tiC_a)6*Ss)yd`b_t_YZi}tmsW__`?}`j|tkPXpc%R9E?Wa zdjGp^cl*JF&icZR6+0BBfA8?+r9+)^J{eMQ^>ChmUb6S3nXaDo-)P6^J_HNad`#Rv zQNl_WCnuGzn-9gIen}~%Be(zDZ4a1xql02qBbwZGJ@(4MABOK8*0gV*+P^mfB*a% zn46h3d=@%%9gZBoWtyFF91Hs8=o-77m&HZQ%mdCzI$xFgTOYNc50WmDx%N(0S=Q4p z?A(nf`lKlJ9@zZqgQj_$u7Fp*ZScvk8EN>l0Eee+W9z6SUw+aRVM8EMW+e z9sT{@-}{LBBpWa!0s8K9jx^HMnHV)uRhd6k}2PuVX5&$3^W>(f4ovpml-lGM0K1JBnrYDStVhJ5BSm#m{oCrTd=vH5n&hf6s^~y^Z^xe4O=DH5JiOz4Xqp zKeG(h)ne8%%w$#b>v>A1Y&;2HO)L5J^>S(k=Fp8~gS?cO^gmkgZ;@~-BM7@8NCTw| za4!C>h^!DEoo5-6WrauP`L+XHz!Gf|m$s-2Ws~J{0r7rfi>44K?WZ^popF~D?dL#- zl2LnJe|<;)`Cs81>TC2}bk}lzW9ZL6m3Vh8=c38*%a_uZTs%plpBqCr-Cw>ao@c)n zi$M^)1f>_tk&6%j03n7iKb6FvO1#Z_N-h*H{qiM>eE20N{PM-f0l8AHlPe|v1#)Dr zlnW2w@OSA;F@_D%{8SSE_4M^G)mSxrC@Unlf82P;3gIJt{8X(vzc`-$d+x6^PT+w6 zWDbhD4Te{R|ITrGw6V!b5ltbpKZxeF^AePth(SC8UfLUu$yav<@C?JXeE~Wvd39F+ z)4M|W|Fb>e6v;m`?T>2gg&hIMU+|yKwu8UI5dAGT?Qn_2x8%31*s)OH746b<;bnnG ze^gwOAAKi0JU*njuH@s2yz?_cfse{8kFO6mpBZk5z^*+roXBck!u`$Cb=G#pNILEb zPw`(T{>xi>$a&gNZ)??6IaLqAqMy2W1if*kh^{~3EiHOQRa8L`PhU|TD{x=Ki$lWB z<^U=EaP@j#I@0P$J>Wkqx=7la$o>!Cf4^Q5)?Chxtf>1nxtxZ{Igxe6 zg5`T>qUlWTW77O&TZojgjy?R3Ws7EdT7=)n&sbg-t*8`HbjuZ=-tH`h_%@VYf1{@u zBGs{r<9N^&d;TDXc!-axW*=!|Op0I62LFUOw+h@K*G;gC0JjM9Ki`5aOQip~Iop#h zGd+{f1T3k%RPy_a;IM({+}mT1`YL;NIXyy=T=lp}_egK-UeMX@l@iKCJZ*x)RR!^q z9RKT~!IHWWg-bkGw3sdX6?kMDe*nMU)Ze-$T?0?0YZvD7wW4WRa7qX4l48&Ph%0RN zJKG^;n@(~0P^q)Pi3+<`E$1MUIc}zFCG{b!l?e_bwxOp>k_ zc!f1(@w7?duQ)AS2tOCXrSz1(5Ix%S6GSndeNoSn<+RK>-V0KP`C&u(`kgqOY|Y2* z`Nnn??T#0A>(mmC@Fsn!lT$eJ{is@ z)px9-kezVnn7`9FS+-yWf98#2I62y)N0cK#Qc=c2;n!hioV_X>ONXz_ zQT>S~)6>%h_;arOu$qorMZt1ZSv6lh%TClJLQVR@X0JO%HYr7i zW9s(lDd$*T1}6S~ddj()7#x$sufirtW=_&WJW;FWsf@VY`yA#pe<##+_(uWu9|hR& z!*C=^Es`sqExZg%ESkceuTF}E_s|Pa$&+DpXq{n*7~+@-JX^ZaEjEQe`u<1-$+2^z zim=J48e&X$st8fVI4|rqDP*2yU&nN4RE+WzQ(P)LAa@Dze5Ls9aRCm^I97Ko9A15I z)wDmAlKfb=Ha;|Ne}cuHB`Pju*}|3MRY$ z_1`ah<*yjg^D*bBC6?$AU%FLZlw$(lR~I0yX7KX86cDot2BG9N)B;N>OMg(wyp*lK z^xc)rZ&e^?)WKcfwk^kVb3ezJaBRw&77s_EcYBee+|U{eR_IJ_5G|J7l1#h zwfLL^pPq6!@KaK6mZ0BUPln2;b-ky^Vp`sTG;tNNhp{_^xYCZ4ox#)7r8#mNB31@} zk#*K}57RIDI?fIV{^{#J@oZYcKgBAVvSPX3zU^0(tuM@t3k1lbx;HP+7v9UETZM#m zj(*#Y8foo+f7Bqs&emYxG==!w+`rMzN7ln>gR||8p0++R?LUU@J*(h*+{Vq*4Lqyt zduy-zksCPlqc-r*?t53XgAd~NH1RHJ37-`FOU#&$G85S=CU|ne1pm_4%flmFVrCTc zd?}-(1B>)@G+H_sDjmx`#c#9+tEI^5DV{qYsNvMcf9a&?^-A7OM9LqQY1};<{w8$# z%!vkuCt6mHvlrpy_APkbUwV_Ag26hP=p0m!V|#??8kBgGc)M>0@E+UoWlMgmvDX7i z^{yR0S#Y8kh@yF-TD;d{j&{O~ehBz&QscCF$wRNdN6Rhn)NBhnF=vJETAn_v_>o>m zKhj(@f1z-E)|kG*h5nWgvrIM3a9C~CEO5T(S>|(GykH2ALcAHQBdgI>{rtwO*ZXp-&^PsB-=3og{kyj8 z0@|x>+-vS%ZfTNxZA#v3$G)`*)WhUF%=5D`K56TPkh+Rg@8f^8nXy ze`?PCn6HP-DWBN(Ck@rroUcD!cx?+5)e{R5xA1COQDxclA`w!j5AaOR#mdgOxtk1S z%-uab-$inlXMDS&!;|(o^ATS}N4xP0`amfMH7VVor(Ep5=9u!W?B2mk!-0F=UlmL% zw1n5KKoq{2r+k=|E*<sM!BwmvH9k?lPHqcN4;k7XG?^b-I2SmX zvubmRlb<~c9qIW*5BP>9u=*p)Sn=~s%-8})BeaP31(F5z3S>2|()ym^Y)QT&^eJtu zqX}S?bJ082!G1gS(tJxhYOhm@x6IHK#oSJ0D8tD%!{T<(| zW7KR!ge()Bl4B%m=Tx2*o$KMDe01*`3h}-}N1mSEP7P{evT>|akbTb+olq$`ve1bh z3~pOgar~cT28H7|ew1|mp*BxWs1W;hvDYCBbK2)e`gWBtg{ZQEs|dlPe@1Ml!~3Ym z&>Ngl(&rQP!$?eSPhFK=yy;!K1BTK>=;Z_c2(u#Izm_~VjVZPR z4SN&r5@82K)BCv$3!?m-YEKRZetPD8C;UxDCQ>l))6?@Q99W6~PYa%6%Cz9`(yQ&S|?4NCl^ruA5ol$iavh#g2-;vOXr(S<&x zk(ct~|D1cl(Tob#9X-20Ov5g%)%BJteYQK=Nm*0$J%`nAmNhdfg&)&4OZ?V;yQ*dB z4^p)(eMr^vnnvA6`~K%DNuBk=6?w}PzP%rAregP7`*8=|e|bMxJ``|%V<*^5t?ajU z;tskCgW>bM=(^uJ6n>U?{x=WCJ%r~yQBBGE#%`$Dqfs@#wHtR4o_8Z)9aA;sZ|q5s zoe04}DfKkgz-j{5jOPK0NB!9v0Hwz~KlX#pNI{}O+W zq2%v^ePi~>fBXTe+PD2DU)r1>R9?Sv&_8g#n|pogW0@b|jq-Nq&lp`WVPaw{fwvO} z+KG$mYu>W=*?)pl?DPdG)q(Wd`UYEvx5sbb4?I$4E^~if3df|$>|PHO-Sj5LI&m0cdH@@$3 zG~7V{f60U4qdVq(8@J58;SpQ6%NLCgF!P$Yl^oU(-P}>?_y+v+_OP%KdwO~=UjP7K z0O;x6v(7U@-ad*nktlz~Q4TU{#BWy~d#|fofBk^6aRxr}S$IVidGF8gys;^@@ei~$ zQ`cEb&suV;pVyS+kAX-|XGO{cEyHDBi%9Ykf7c`mZ-03Bx1<5QW!NkqIwC(tu1H7Y zeCS>AED!)QK+L~Jb|kM3-x*<()tX4_Q@4y*Um%}_&h@T|+7tis<-`{RLVt+lpqC_yE@!x~bc$YgCX$3u? zGf+5J8h=3GpPti5atQo_;}`S>&Oy;rQ1l9DQf;Pe`9$TQ4BVWBiJCUP9?Ubw5 z)4aX;@7*{)AGXK7R?~xq##!&Zf7Y?EZ$xC_ly&%>ZCKhV@^uf&4}&D^YPS5J)cFGI zI9BlMY4_vUa_2FKveRZCh$tP0=oo=1rQrXXE5tFnj;j5yxo9K&`#Jw(b^TA&sRm*9 zx_>Q8q=Fpk7F}=#_fn1nhaJL1ojFEB;h(}II4w$s;%Jv&FUDp(OetIU-S0ia{`)9j ziV6H6ee?giO|%o2;-NP8)8kIdpT7-HvOz_5tatsw-#~dX<|kf2-fuf`zm7MXm}t6R ztL()4ev)~1Af82h_>R#JGS{;UO2%hu%6~~Pw~Ly17oriIIn+OG)WLr9)Uo~p1#dan ze+2v%;K%&!e*)`0zx#)=*?W}$Pw4*f&=9AC@z6ukZ;6TUEqJev zQu?I0z*4}c90dgZgkWCnhd#KCc4v32(9A>=^cM?Po3aPn3(@M*YBd zl-q_^sYZO-`+K}Rbk+4z?v=uRhA+_tTje#;32o$6K2*Se%Ls=Lp0f(4V(yP=VWfKS zoc>5A*QuQN>nAGt`iI?@(_Knk<$n*)o=x=2xbVq$%JSfnkBcAtdhjW+^{omr z)B0grmhaH=H4b(ZMWso9OY+UG-|Y3x z4&UzUe^obsy}uat@JC@{cto(K5MI#5!kX26@hy5wj$C78olE_CaLIJ;w0|k>R1P?C zCm$Xq*g^z(KJN6H{=?>53S6LYlpw9l?+$5&lTk3|acFY*0Fx7hF& za?xPvQb>`ntZRpH?<3K~E0`Dyz?`zMop695X?}ng0{u-h-WXo0%Fd;y+882DO-Qh&bWG;uptjVSvo@KQ$M zuVB&y@CE)Jjfju>R4cv~K;&xy#=p7lVmz?Phn)P4;?mY04Wj$c9Q|jG{&{osaF#@_ zR?SeZ^3MkQ&*c4QgMa;Jj{Y-8zwZVsSf2a+cGqv2s;QY+&P>G8mz(6|Q=cmdseJ%fP1X3wC@J^PLu_&>|wpJniuUk1lB!haCn(sG%zXfL8+d)ueC zKStvi+&lA4IjPz|41_|kbD!I{$3N>o{#n0+f7Yn0k;`$8)k7y{-#H*vvKXE-v5;Qkhr>Uhrq}OX zljw1`ACC@Fjv zh2ccmIUVsTey6*w{v7GrBuX=pGX1^a1IG%kt1y53w14@YI^&m#o~X5chV++0btI#4 zM#ETr@F_hGKk@v%t*&fe_BzD)7(hn~D4NQ(c;TZqMD1|`2Ko z?&}bCAiCB@WnTBUk}JBfDZ|hhJap5VRf|8O&3_%E*W3(ca9%D6fN-7>pAf0^3;N=~ zUb^tN)nQEsq4+f#%>Tr@`|nTxll@GTd80t8n}Q>T@ifHHJ4@z7i4>P&DZ{^y9#&7F zo}M!2QU)e=#GWhM06uAUa-9E1P@P{KPyhY-@GJN4r%&|>Oxx!kCTD1$dk84?o-SfL zmVfMsu3Ok^U0pNQ-MkjS(Cb?gI^&uPgjD9AWat{V;NV0c`n=>p;E%E@j~@VH@92kZ zBWl$L?8%R0j@^Wp4{pNmIF!#g7ylsCo|dtCeiuD z?MBd~M;p3#j}#slsYnGsC>qQ+#WC8m=jk@2=G!GEi}|?_MQSO8Qg1!7!`g*sH-7-c ztbLNYj{)+6#H72rlbucPz0cy*Vv1Hp5Z3yk|91{u=B)NY5yY9#xAbg%0iiI!=82QHOC>VS-ckjC(BoSY@#-S$_*bxsirb&E zd`dcWn(&Y9)86*!gkqO$l}>1Qy?@Amfj!e%cW(FOPAC4f_V1&6J)ed9I;4c##P2VM z5Ov0{`{<)7$0S&0d%p-=@P|`~7&G-N(|q8ukqMDVL8}{3}7v;(X*netI)= zHBlEsW8=%W*jEq+e!Ajt>NPx*-;XiQzR@UjkLItt+Z_mT+};kII#2p%&wtzRd1rLH zw56R<@>Ze$f}QU|r|$LVZdd;BUZge(&kDV_UU78$f*+NqfXn`CQ_*nKyIc|i@KfRX zz(WYZKTCd=toyhPQH-*;o3@jOvIl8f{C&q^XX?zjPN)ks`Z1ac=fijC&RNc7IpXI+ zGaiU{!P(_^*TzU4KUZ5-%zr^Q0~Fo>EI^+QZFqtFS%Ul*H+|pWTLRJ_9wFmqSdW8v zwsbub=oRtiMe+Jx!gfE(>tVz5Y-FKdIp<+Tp zp^ufmO3%BFU6c5~j?DcG!EkQ>|9!~5Oo*N5(AVzM({mDF<`h*L!haE?sIr16J7O3! z6$t&gKHaG%0XhZ;3~LMDi6gjp?1h{ZB^u2J$ zc0D$#H1s+XyPjE#j;Hdh9!F_EYTNOa?i=Q%lDV|cnxh+>*e^~h2FUXz*ni&yNuhqb z7V`tR672){_J4RN_j-G5>A4f2E4{r1a-p{u+74W8HSn!*QI0KEEJxj0rguIM94ok7 z@H5jO1^uasaDBKxLze3k<$jMJoeG&Ae{eEfpZd3^$>qVmKlMYUOi#6M1PjRcm?{(! zfWy;XEQJPQxpX9qPA+Llxlot;IT!JUhuW^25MFY2wSNm87mAzLqxs(98o2l9`!)Ub z8X1ZH9GV#yJk>s{_W9+|HtW!oOZ$KPIvo~@w(*D=3LisLn%9ZRW46o5t@=>!-r3kg zRb51NtM2%Gjp71*$0$Js({hY(@}Hwhyj>>jV3_|LDVaQ_@Pp^bW89A()2H586On=- zO02JYcYi!;p6|(z_=k&Ve4-2qWEA=jZs8tBe&!@DM0DRb-03#*qx{=_q@K_FyE`+k zh_!$1+`(pSufhA2x-@(>*i9nXJt{h6)$gH{7Wm&(!FRuf#{(Kq%SW|D5}0seKMLYm zcBVX#&d5}L7)V1E1YNv7mN@u=g#Xa=h?bzs>VNy^`r0q}$#}lte~SS|J(d7{1rs3y2+zrjm?ISo@o?<0 znyx^en}?pjvZ{WOPkjN9h}AjTD5M~st}~2@&C=mb^dE-{e@BN4f*8*C7w|?*r4QT7 zzJH%gq4S$h$`CH2;~BqmXKcMLUa+1LIUKN8G6k|?y0gpIaZ83GrOm9-XUP1Y$RiUw zyn`c>cnQ7058o`VsiI^yh^(5)`8KfcXQTiU_8}o$lAKC8^!RWO3vV6!?jIBv&W9d_ z4fhIgb_2Kv8!Hcdw8jMLqgAo)&nMC4_J1bRw~BnUZIKS#{%ZaEQ>KT{cS^-QN=ZW# zdF4^ZB)dwo7xh#7?bFNig{S1~#7cW!xc2)KxBb*9bAILjP@}K4?_Osv$PLnsx-UG~ zYuH>8R3Ac4rZV?w?=So9>wH6df7!DTm*Q^44+(E44-4-v33md#?eDK4MePFcmVaCw z!;jzogNnv)zeh=EC{x%KK;Z2lB?7_(_?Q4k3BUaZ{O$K+q4yWZip&Ph0`w+3s!#}H z7KPtAC|AtIKi*%A0{FJ~kCg~Vj7_1Okr+gxj*QR%+!qX~`*z=c|NX__4i)WZ&Y?c8 z0G(zXa>^$;;64L6%{anfCs|OOp?@E%IoifY>F3>je=(e6gOdlreS4iFBIjtP6J?a& zF44(50z}Ti;C*sAoqQ;arBk7OYMRvhHT?b(zeAp!gE1%jmSt?;UqT7^Au;@Te+ehX zAu0TLe~GuOljk<20PLUifKalc6rl1x0f}0u6b=bsl7PIOh{luxc0dyy$$w$eZ~qAy z*WW_Ub*&U|(fIA@x8L`r_B`!c$rW;VbSFDP;2{7Q66g#Bk0GMW2qp{$erQhgUaJ&b z-VsGJmAY2mU!sTNxOnQ#I0gv{AoyDmgwC%0VODv68;trD6v~H$eG0q>1WQQ@lJ%1p zrW~Mmic2adb1lj$ zWwWqU7E(^-B<+lIe)|s&0QTm$-%|ymlyIh5LHq_2{tEY}Sm0Sm1mb5=6r(+hw{$n$ z*(B$G1CsOG@9|FNiH05BcjL!A_Z5_!93bt1LT?O((n--U@iq2__#q9MR*I zOo=4z{pH|rCzMMb-0$r9>!{)=>sT_W@3iQ?&V4#3dOu@J(jz z!=4zXKyk`3p&TW|vw#2ah8t~T2Zt>2CU$(_&1Uakh+jO%H~COU@YC*H!sGBDO?Z3E z6fHcg<8lS)01}BD4$8QPB3x*}4FSO04^@DF&Vi9@0Zxu91P>Md0PDEZ%AikH_K^1d zX=SBPR#wVZR=QVNk`nS)XVtxc?=Rs*&)x^aH0Ukktn&TE4u5&3j7|}Fu2B@P1bdq* zxAdkEi;S-1wxv=yzX@B{SDmuBI7(5?PC2a}Mtkg5y%h_E#)h%7l6a}FR7xzS|E7m77-X45G zjEZmEbny$NqoPp}&eDara1iIA0FY$GqKHgp`H<;~hI$5zgxOIGqP3EVWcXb*%J&N; zN)sK&az0Rr=wvDpoja9;+piNbuBU>r$iLtr`~79VXn!H#E#;QtkX%kBmygLvDj9iy ziREdDca?)1g7ASq)My4Wiia=Y({qv;>MIsb6%${;NQ~$q3R#3ld3mbPIo`{~ z(#n@d(SEp+<2!KLf;gAxi)PE+DiG&A?THetu@l0CuHJGm5VaCI?LGuw?f)s@Eq>8bn9)3Ie-bOM;7 z>!uRJjmTkLFO?)a>^6EIN~I~jxPO&8CaPwrY>-NDSud1Gr^~pat+J`_NW6V$cuKDK z+kZ*`V7Q?O<$`SflMW#-%F0354KF{A7W!e#DSzN|>H|uDl+qvN^aq^&KvEx(o74S= zmHyDxH&)kQqf3XEs^50Rw>0l-J7VPK_*BlF6?3P&GlX8&5-+Y<`>$ga7ezIDUH$ON z95cr`6v6Dx)7wkvo3dbqDj4hZ%TjbB^b=8V?n)^zQRX&OQ$4#N?lTetwv!bdy3;|f z?0?nIiQaiwrT{&@oHIDDFXjyDKXonh3hV8!|7SPyD3kNK&ZdO!wa2HzG#FEQ{GFfEPehuRh+rE5w!iT<4ufxRU z6W@Z6R4SFw6lGukmxP4>yN96Q`kd=!(SO{9;Vr`m`NISXfe{hC52cCQz;Z&{$U7~H z?ENjL;IZ=E!iMOvAH*+M{3Dp%Sk7{{T&Ov@rvw1P2oAkJ0un{!*u!B!sf1#8;Y`*L z@3tolW_7oa=^LErJU$faeX_26%X^@8d-0ja%KP4PC=3JO`@@alLwlVMlHB#eO@Hog z^BhqYH}-wHYz@xR?-pB!M!%s8`-iFWFodp}+T+8Qo}+LX#R1{{k%jem1-2{tg7v{M z;e1v1{x>$DZp|3 z9>k3UJJXOm{6M&Jd}PdlFa zJkP&BzWZcP{Kv?Cdf*Ep3V))=-5uoQyhjnfj05+eK6c!nETc=O{a#>WJE9~yZU+7} z7ajahZx>L9$KZE8Gh>7wIx}OC&tIrh@Q>c7(;yFDr_(?;j<-y6?i`-!vX32}X^88G zXSNMj56^V5$1c*R5D(s?(_q&Q&vcnE{Pud$*eT4oVA&KVUo>Uf zm%nE2ad(|vs4PsrSXq>D+0ZD=x^7Ps=AK%>fOjllz|aCF0%Yw178zeey3p_cccPum ziy1xbHaiyQ|9_k4j=rY$z{D{6L>u|5>+r9>qenicUc<~{U`Z16HB=lz%FhCTH@Y%W zA;NzR$R&t!^fSK*zris<)QDUEDh5F)`jF}p24AhzDAMXLIjTbnL=7U;VO1$v9fh!n z3Q5$}CehT?T2~(NTpeRfsKazu&3sAT3Pof=Qr}_cl_o zXu`r|;}M429$ISO;fqatxNX%uSA`l=(%#i2T3M_*3|DQBJrmGJfjbV$!z$V{&7p>r zOG{TUPtqR{};~TISqCRJ89~D}T^oL4noR+|2NsMGgmxp2*Ku%c8wv z`8>#wmeiv(dPr5~C}Mjn%-8sxunXiaIhoBYOXzBi;i4pYh`DXA0t#)5bzcOFqvph8 z`@t$e$8sI^Orb?u3@H=EJOZt{C7H1(nd!3z4q>p4&!vslEUJD9YdRcVZ?{HPx3#Ra zmVcyN8|CxG{+bD^m1Y7(qS?OPs1$#)N2aaDF?Z z*csa|wFGy9HmEHJ47ieQjU2nJGL@Z#5^a2>PdC*fM&t!n7f55^@U0z-Z7`?Uu^Y0w zF~%*YF3;gc5mN}EgOcky)`#;&b+)FeWPb@p(H3B^mclW>tWFc8wZ&G$fDi+$W$yA? zb>if)jRe-w8N!-l!;=l5H{6e#aGcn1MwxyF}8t+@6EC3H5lS{rZy+J6dK z!4zm-QA$wIm08K(mDXj=91Z<>O)V=8-51NXb{+D#VNpTZ?v!sW7i)+EmRe1&uN}av z2?|ojDWOy}r7>7UF}dC<(oMLi>qLL7D?L-%)l9Umw^|U^vPNreYhZP_p@_pJ(^iL) zy!02OMo3jb16%&8M2$FL432|ZhktcB4jq6qPmrse0rtWAj@2#4Wqb$?DqE6hFeFh~ zQN1c@EEK9$pA6bdre}9#l$~3oyyDzdtJEgP9m6RNWu;e}b%j;2jBt2em~D`nx7hVs zrm%xre1BEMP;0b8EY%%)`idVaYE5QDu{-5W9OV_gqyR%p+l?0r#qx9;aDSz?U76Sb z(%aODX>aa{5?quzi)Fstk?IT87`4?(cT0&%zTZFw<@rXlEUM}Zv;l*8Z4RzU^$lw? z70Tl@hOh@9&>F3XQqOA3?Ri7>8$*$4SW8c?7CR1HL@5~q?D>iqbtmhN$VsiiV8_hc z7^2a&kw~=2vhNc8s)N_coqr`brqSY@Lwj_iTjx4t9vzG-U};MuASem#!NLcNoQxy- zS|@5Ft**E6+N5uHm(z{rR#*AC2f+lg*e*t>A+@TAz~k%bR}fx_t{F#w&(S&yv6I|RdeEr zItQVSI;xJ9dIz-gFou{@Vb;QJ6z1|XrMOtRQ{O8!9ef4CTW~O&4pg(anXw#PG=N@> z#a&b>BDSwaBt8yK31~qg@&AYDHmJHHQIVcj>ZUQN~qvRjsWC z2pW)3kDkmMYiI+N>4`}gWTlFtK^NLBT%ljw;YMfDtI$5toqrDp;+*L8^YnVuaj-gT zOSZ7Vo7-*~rEJ9Sb{iU_D$SkglLLyXA>5D{X`M-9(i%=QW=E41u}ExatEyKPc~}6- z7Dj1|x2oZav{8moIvRGqVww1FEjp6l;UdtYK{p+6q(jI0PA~Xt0HC2y3kK9k;iZ`H3(# z=SI_^Lo4TSm4JH!X7B`SF0@@9)Pw7*BHGdb3)u_fS77bU1x;x@BZd3EFf zU6vN+j@=(uYYl&1QMlT=X2|Ox-~_Q{(M5XHMMl+rTVUnp5DHe##wOY%vfZ{u4WX9r zv*VIiu76ieQrz{Mfu?qJ+MmqwMkio2vDlEi-autq^F;|L4*3mGEVUSG(q_79Frw@x zH|n6%#*#IOL5Xu~ZE0yD!n)t`-Qud*kT+ci4<4axq7-B`s6Q04~#><@q)YT~;YRx9>sq9sbz8M2_-+sUH7tWxd> z^t*v(S#U7cs*?_Av%qMkQ^l6rFR3lI=`VJ=?9xD+n%LF4(QDL;L#Wy{9C{&w08CkR zxqs^0LqH|{dfw`<^L=todczJ=tjybEOY`~@hZ-}}W{t$fIs$as&1rqwpsPi9+jmJ5 zlYMeDYeCz_P#MZ(wL$r<8MdTGn7rJ;b;;Z|l`&P5ZM`v>ZDpgipc$iEac0c0CeM7k zH7l7xTh>hyE_E9m+OHxiIqo)hKxHfF`G3Kdt@Ok)?X=Lqw_U^U`XiR6v_`YcbmdLG zqO_}ZtUCxc#U(+y@TxXsI%~=DDvaImD#B>wBD%seba&#c)+EEzzSsAfGy%y=c-zoC zPRG1*emS;Ula`E-L*L}kwJYjPp`(uklv%(dcxulKjopwn&+WE~RAo1JyA^0rtA8=e z_5xD0YPpKowFM~6RB&O|N^p&`Djwb%1SCz>IH*q>O(-ai?YVCjun-w9rk>})y?qY|~LQEeOU z#3ffNQC&>I%4k`hK@$i7+lV=)YdsNdZ)96$U{+qNjqSQoWO>-GSGuNEsej54yz4Ub zsMVYJTs^O^fYGenc?>E5Kj9O&YFUEKsYzvhU|u;=CRom^Q`Xmx6@38Sly zYJua`3{DuJq_e3TD^OR|0Dq}d&S+aSF&P0XEMbotV+vNM_-MIlRp`cY>mnttS5ix5 zdkD3H9=Be4s{ozvhA;1QiiM&UYz@XOl4HU3uV1pyvCDv_} z(V<%hCMZH}tL+}u>3>nKf-RL5OOKYFNjsnK+l_tDhthOZ%!l@Zlw&&8kNcCX23j^v@83?PcSVAk< zZe!IuST{fE>WZ=Lj5~nlAzX*b16>V=cI%q3YRE(0@2XC#L4OS^;HHWw{<=r2<1I;{ z<3YdxGP>#VL#ADYtXc&U7z@%`T;Em8qls&chV3$<%b2Uxf`w4SI^t^9G%aXrQqb02 zRMu0!0qnM{$Zo5?-s?;kyyi`|99^C6_zEeIw!6h!ReQ;ySa&LU-C?E4_m*>mF$UfK z1h3<^NsR`u!G9Ks9@}B0om$il7={hJ@6CIpIMX&TyIb{JV|NT}QFV!tFphS6UA?Zi z`%6u2N-6^^>m3LfNHcuA7^0T2aT+FStp;{+3~;77Ui(ATuQs6`sBj>>8!t*LU@kRx zTg1h|i7gl-s3fd+MMm;cz>w(Y9brpb01WVa6_-812c~lY9D3%qIcIlMyw_#`R~`<~sZtp-0X{}?j1neh zW7|`!6@Qu_T$Smm=FVC+C`DF*TBX+W^Wz2yki$+_H-yomuPVC9!6H;%QftfVRWOpG zWX$Rg3Av$6&?@3fl-5`Eeq9wEtvst&G-BRu!t$_%G|?qD$_-C>|r=ITzg z8)U0e8`nu1-t7h!*q)RXum~f@s#af?P)0G(DSy#rOC<>(lvp^w*xAU+TbXP}Sk~~l z4z(Iy9n%_Uy9bZfNUH?matW0N7_F~K(O>M^vzZByfK3PW$xv-L$c$fFr5(Pl zDNNTkOv?496Na?tRnx!%IS{37dA%WLG%*t;jla)ARZis1U9}&c>CV*O>Y))&cybm44Cx0JUiWh4P(?Mlwiy8{&ZI!OVv?nM3+^HZgwHH*sl(?UOCU85VYts zV6f^dwozpV1JVaM8zhO_*@m&a{#jDPhy{Khs= zd#r(DVL1@AfElkcYS32}7eT*)@Wun%~apT_c|#s0~)`bM2-m4_$JW7xZ!-#U_0vU!z5( zYEWyqu@>E$%8c_8K-F2ZN-sO;s89CF%EV}b(?&z2>NGvm%XYE8B!7`6x~*=h9#Nbt zvj(ShdN@{I`gOb9S7B&@7@}Ba`T@L=+6)d?H&X;?sN_OHMiq?F3`Mn&IgQmcQ<)lL zfvYayT@}NqI;iZ7d1Wv~W=a>s6sc2oc4p114v-o%!SSHN=SM4wo>XQMJ+5qAwr^Pj znWfhwy}eVHwMA!%ZhsipAOUeGux-;5dE12S$;O|u3 znyTCCa%QZ7)mFl0)&8a^Ff4MUfu853xbESFpJ`f6;69$18sb;H(+^~y$Riip%VE7P6dlc`?YXif-C z9TycC00mw|McQ508rA+><~Q?|X0@A^56^kOH-UOp(wmTDLR_viYiM{<-*<3lVFzLf zFg1C!DM2s>iGNmL8n|p1Ro3$1W~U*vO7K)QwOV;1bi0!wMRCi9Pq-#9@uqY!aL9_X z@mdlh4?U{nEJ<#Gb~kggJDzoi^UZ9oEQ%N+lOiirid=izs4awzCV?QT(8kLHHw9C|{6L6Ku4%dpUR;k)9 z=YO>*j;ewejH)W0OLq%-V!<&_IT@lvMc2+L&&E?jdk>J*y_{O9$<6S@;nm94rj@(^; zCXhj~THcMQ+N>S)8f%j4kj*u|gI$inmg1CdkO65kraQ?C=Aenxz{MaiR2-I}&Umb^ zIDchc#A$l0HP98llItA@t&PU{$s|ykylN`KTCV~~&@T^gdR=Qe^Fdo?YL3v5Dph!H zS|aDv%z3`S)OYA`IL=pbc^K^UQnl`4COgEu;nrD!M7Pq@Sh8GaCX_H)1N6Aw zSDk7FkZGX;N!z;b}mVKnpJLh z5Sdu(!3>$MX>h<(JD#Xm1Fcfuk`CM2B8YEksO6CL`cU0%#`s{;))Uo-~1lD_fJG(JHoJm0@*Yj4j(;a6M@)CO}PXw5TnJq9VT>EC+xp zO(}0=jrm|@wbXEpAzHm$tG9O?R(~DU2fW29NN#i;Z}(Qt zf?DO78dO^c7!{12Z67GYv$|%M%1%ouPljD6FQD|KUKcugQ^(x8;nBgO3u;slxDZO# zrg&+N$3cbd1$pWpqfqc^y8d`B=&jK@O1)FY#Xmwd{JyJz}SCu;* z#RY8N9N=BN(N*hXTrUFf0DmW~Qjr7MIkBbJ{FZm;lWA>Db$0y;S~q%I*o2})w8(Y^9NB+>xxBNA}Y9U(C4k`!+n9E{u z3QuUnuE`sCK!cQwQVvcEixEvwL|a9enO5l zVJsLfb%xd!c&9BH=D@EFCf3jgdnGrhSRF}$SJk$^Y7eV58LKgBQLcIBax(0i;7Dx_ z-KA}N%BtMrhNOua^?w;W2jRBsxHDfHZu3ETI4!L?QsrnvXf-%kVKBeifE$!m-|3^~ zgkCL|Rbnw~7*eaOi<`l01apkvrQ0Tgj%|q@@$PcHhKFso0-+egi)EtSY54d=E)vRe z*cQp@&?G^WJs)}?Gs_hQzgS@Hr2Ib<~-VxlSs=I_SHfeob7lyOqaEuIPsPEwmUxUhI zdo6lpT9owx#kgwS9{E6rA5UsZwFc@O>|i?&YRN^?QQLsE6^m=S-WXJ?2H>hPDh=9O zvP@8A%JZuR2!E=kDNRcdAZzv(SxXhz1ePk>))xT=h)q^lEDRm1X|K5ee`O@ST^&Sban=U6UfI*#Z;vFDAq`{^?M_CeKgH3H< z>ENnb9W5(P$6EP(kEIkTKgKtcF&-%OJZU%4EIP+Z*wyN2TI z+&Uqb*x7dKxbhUyr2rdga7VAS7N(2xt!Ug|d_ZmyKxpG8!wS>zZx#3Qi1YlyG zQGf5ZOFpz6b0v$J?%H;@RQHzwI*060YrxSA&#J?=W%$5g$IkS&0VuQnBq-WzyJ>*s zd1ce9L2kV-ppXbn0a%ei2CWXv1#fzERrjQ>J8#Toq~y@Fua^+Nw&RCRjb$bV$gLW^ zjzc<(Q}3@80q}Xw!0>X1L=1LUGA6a!aDOg~o0;p(+QpgbjaIgl?+knL45|-VzSOL_ znoCY;u8r1PeIbx|gw675s+^R`fE#R7rhO&cMU1~P@=^TbZv8BPzt}d4N*_veeZhvRN z%M!5}*ntmVMpwl|akUwheG=lh;hbyPGe?__w**%!t`;Ok&ta29rihJ+4Ai6AlqNw< zpxSZCj!$RtQ%#eb;m)?lo* zBKr<8+khD13^kq+mb`~dutmGgH+Ffbv{>+pf|zB*8CB(#TSFy+Ah*3;%U-OifHcv4*Hp<`%s#^g#w5{H#2q`rU z8i~`cy`3%z9zo@*(x=*ZM}KL}eUT`!1F1E^d0Z`zp}J+Djis(h_HNT!OtgX{F{5Ze zwFd*#qUCwhF$FE^@MgpAs_oqd1sSs4nDh~*hEdkM1-D^tJIAP{sQ2NvGT*q|R-D?_ zEYPhoPVAOU4eM~ztxfsFQk$rrCkozB8Gs$lx7iuhbf*JmI`es)X$EG!DVsaiF zh`oTF(r{yG6o1R5MIP*cbr%7|CeXk@hL~_m4DC1`Hk_%g2}rPXMRTX~-PRm9dfw*H z0XLHgL<%gwY++Mo?(+R%x9h7&bF@+W9tNwlZo7Paux8|L9vsgHRoazQS^^Qn=ngjZ z&7$8O2lWAKQ(LG)0oWFsSIH?huQuCQ6W!{<(g*6eIe%Z&RsyA!763QeFl&A4!4sev z1n!0sS#}pu4dCLw}mZ-FaMWI z@dO)x+MD=#8mqpPxS39pHJx^0bs@-2RrB$d(C-EuUWOA}N_GEEv%TUs&V=jo4DIma zb(aEIiEM236sz~uQ!l4#U*}wT*33wSn0X#SThq%R78=5WZX(q6M9Y~b5a|uBn#te8 zz(xgDwPYGB5&41yo-b$YuZcz`>bm{Csxn%ChoAV9#=|p_D2^lF>s@xp3ooN=T1>}G zkB)m2t^7jODRzC?AB@n%XA~jc3KH_0cu$lGNXv9*lnJ>{OK0B|=vWym5)yG1-VyNxC}exv_b?f}0KpI1r0crQ5oxuqvgRR_@8{SU zvj?Jp1H(34AF+w!3wnf4GrylR zKQ%>jshs%j!l%t(+Q6hU6d26GQYx1r{zUXOHi#$-QB1Zk4Djn-O)fY?H1<=jvQI09 zuU&p-)8ow#KD>NvXr7ANxLkmZYaGR!HJ0#!gR8Sx9rHUN>o(|u$Jv?!+PS}fimQGp z`l>*Q<`+L3iRMwHW})+uc=5dGJ=UsCm+N|cJKRQ}*0Uk>@Q+jA2^-6cM7b1|)w)9q z`Ij(Eu;8=^oCBn41P<#{UGR-Iv=!}2M+Tarn9OLesF2*+EkjD?cf%Ds{EC63+h;Lcl0J`aa_*Q{Q=t`s zU$ZRrmECp+QN2~8DO$J+!SHK+4Che?+fiM4@#$hVEKMt9@e66ERxlQS;owQdqi-?G z03XzY?J;VN_A1R;)BJ!6UH!22hn5(eHdN$s>o_Qcc=<<>!{vssyyF58pISZhTS0#x zWyh;cB}n*2%nX`vRB_~tz1Rv@k&%k6M_W2+^p2c>{RB*``nZ5xOU`%ENzlG7DWQLc zu+wclxKU84=frRvrfB?s78Y=7U|J7D*Dj}@K(H2z*|rZFJL34-?VZ&Kj4OxY@3>q) zO6+UIaBRc4CgFTB4C`~q#LT#yMw9`YeWx^_Y3#;_Kn`@?wLcI)d7p*iZJ*y=pVgH) z`6c5^qKoE7XKPfruq)68}S$mxAI&CQ478BcHzG^FdG>Gtz~xJuUsfn+$u@Ofqt z6SG$lRF=~&DB%sT-h0S(z`j>Hj*?P40MI82nq>-MVanCpQsyS!K^2dbI$@IYAbFDa zq{=ZcuO4_iKKB92Hn+8LE$o&T^6OeEUrUJQ^c4=?=iSGB`j8{sA|LzFLZi}JKn-OB zpSymKDLY-DF0B-QrOyY{mkFfx<&i_BOIQVoPSf!lj`ybC*M8|kqL*sjzUQ3Hp2|O# z95JqSPxogsf{eyAzhxstUfHtLnYBou-J5`$I!#V#_Ndn-t z?7a{lilSVVd7XmFyh{K$K*zs94I@$4@@GKyp&&(0w!CB`F_mOCGo_CERI6)a{R~=~|jNhDME{0(%jsdm~5Tj)y zlIU^U<;mCWIf4L9fAu)*hI~@XYz-K*f$lPNH-*B)HzRprR5w7N_2(dK`IaSH3w0dF)S@-_x_~6>ySGRU-?%U$yaZ=KuGboG z4n)#}VJ>e_f6@<2ar+CzwVd?OGnU5xj>|~va;#zrRIm_O{I90P=(ixDpLe8U;52T4 zt-%k`iEwjja58!(+|H1nWH{lyQ*4neU%OzjHrFnnn``lU*y(oy2+*SdQRB&gLDaL=DhQJtRCTuV6jLu_j3!1--)tvy`{KJsR~ ze|fj(6xCyKMWoAx$iy&$4ba*sZ9tb~a*>#;#C?V0Sa)4k(G9smR!Vx>RE%sWBUfAnWD0n^O@w%0V$CDDsUwcxezX^_hY4Fjm4= zk|C4!qaj1%U`FogGfuFF7^6P zHIWkayB>=|ewfoTrO1f&oF7^}NL4&c(?BxY-~=J%xglU#;>;Faz%=kelji;(REl*T zE=51_W!wcHFQ;y9=me@V)q*XeIMlO{R-`Sb?+w)I4nNqGz?6aC>MbUie?-V35=Thj zT-r*3;bB9)0Qf>`#eE8|WB|XS9L^++-) z1oBI$NpVAD;U5~QqS2dzfB$vcSB*OgtgG6Gn8bEQHemMnTaq-=y|Ji}C$IMR7a3cT z%Io354E}B;>Pu7gV{7wlR>iG z=<4uMqdX7A@vp^rJ{?WZ1Qo^LxLkVA%l#Q(o_Wxfx>a@ne;XMa?)~N(x>iWhnuiqm z&{o$+eTV@HE^hx_%G^Bw+UDQNRnMrQ2n|8%gd~o&pTxK;LO$7_TyvZoN{x5|b6%6P z=bJj(v_XlO{-_}8ZLL*=P$u97l~AQAJ!eGk+o(~!&RbA4dS%}n3r}}-MN=p;4|6$L zLa(r?9F)UWf4!#QG#4j36a^daKoucMr@EGZ`ZbKlIXPjVoat>s-{vqarAf%{=cE+J zZ=>&m4hG3th`gxHA1wtPp42;VJ%HqXmv3C1W>}5+V^zLcG_IVo)d2U}N(pobLXJqK zsrHE-_RV~X$tLZ?DVYXOux+gbve>?Q&buO~0RA8#f1pW+-01@O9n=;6tg`~(veO8A zA*X&^sA%0~u*91H{#G%|BlxjxErdt? zWGe!qRmE#Hg0C3b&I~cz3DDTzFgX|*_HCyUdXB1$^~A&;^w=pGb=y!osUEj$4TjIP z(~l?7e_L=tk~t0;guvB0K%ZJ;G3h@VF&jUg%S_^NL;=>#d^u28$RF)O;3myLuS_BD zasV4nj&3g}ht+3)Pza+1t<*rJZU%Vsz9in*wt8Q&PgWH-x%%7I;q9FfpJT~c zK?2*>)$`#LZ<#1;bq|Taijhs;$&*g7e{5^UjxaK1h9CHxt9Nc?kgIfQEFL%!(eh#CCCvZr%1QMkyaA|%Q=+QTGSr&^q%d@urhtEqlu5FaRvsfmi$7?@~;}|e`-;Q5p1); zwmSnLS#uWkIsl<$05nl07mnb|KqdHx_nghNmT!eeAN3kC0M^4YWEC-{*nr(@nRri5 z$^{eY>?`=^^vx7h!eo@-u1>nO;f}aOOJBF7^XguAA>gEPc8!Bv%}224u4-ur+^7vi zb{Ez}rh-7&Y?wJOcN zLp4f{q`dj(N1rP;0UoeOkfu!k=%p7X{@J~`t!T1sWxfmY^!TmTLxm9 z`6p{1gFFdRDfx^;Ch{9MoW=8U=O@Uh6HxvrM5ZCEZ>2{%8yC;OfAA_s8kaA*eRiy* zNbSVioW*WR%+Pm;7A>4ffdo>^I5%6_F5XAhUV>5301G0mTbj(EbpaTW4I#yTe?>Yzg+Fu&ndn5=1wNWf_-J{`x}mFS~#2Q1l%+%dZu zm-U~^RvQpEf4?kxo3a`*wD{lTA88&HT_F55GJ}h(i_#JNi z_Q%E?pl|);e{iW{TRlMniJ_?cxui2L2L8e9B$C0w8h{_a*#QjDxy5L;ymCq3D|DB% z)$Y?{VQ`DBTe5{yq1gjXw?Uq14)H;OGIuscSz@_f1cF+C4=AUyo`hCnW&`6m~Vdi z9p)Hjm$SK^((n$x2>H0x^@NB#E-RyjWf5zM1~UJc^rmaTKZ2A_ycnm0f^W8Y6i7WG zMm-K54&y+>DtQE&-;IWI1xJ4a9%w-;A{Xude#f?;k1zKVJ3;}y#3uII$`m((e{QD1 zpII>Fe=P4(a_?!OEA$jb(HB*q%1e2;3nU&mz?&Tivx|I-A!x-yXB}pxyGhr-90rFy z&YqtDg*CIJTH6*1!ZQT6ly|wU7wSh;$`%P#Ws}}yqnGMUl9rm7wQ+}I3|FK_A4xju z1*;$Si{j3Gm_D_UaxSH?$7rBXGo1>D3Aa}5e@}&%mY9?nGkU%GReS}jNJl!R@;yOS zhGv#5VTu!zNWjgaJMe%%Y0}Qy&3D!$l0lF-Y4)3UKd(yR?2?Vxb)!0a_olv^u$Z=u z8qY&w$9d^wN>5_MKZA~?7*MucrK`|XXy^1uC3Gy59C~fd>ChyAtMJyM!$>x2))y%i ze;R6FvLx0;swVswRZ_V60!CWXe7p>bc4;ahWMK6QMfX3IxA}Jo>b>?`s#cUQ?X=+V z7x_~8%c?lW2&LE%+6ZK&UfFT4xB88dWl&vfd2oJLFt2i<&X$ON4)9JT@6nl27N9!s za(zK2V&hm(t!11+Kxchx6}_ip(R59ve-4mmxk2{*_6mJh#D?{3%o%w}8>AL1E?-74 z`tqncx;>HZk&Bw2Ofxx?KAtXncJ2F4!nqVU^h@W|0iX|%ABJ1v=k<(E$8BrOB%Zac zVI_i&Kp6^h4Ad6^g89i~%hfth->AWqrXV5Uf5zn7M&X@;hRqB^A?v0@fn26rf1*+N zE*8kp-g)~#3#~C3X0t#&{01_ux~qMsxKFoiq{jcj+lT6vQ3)6UP|dNlZfA%Ta?Sj7 zvRCAc>UPV&moA3wnl1W4fDS8*4&hOd(^pT$r)Z6ND*SM=Q7oV?_r(@J8+4l*Zc3wy zTvT$s=ul&*s}p(EBdWNaABjL=e;+Ve#R9+58;cqzZU54cVFakyUhC_3a09#oY5e>~ zAP0PwT4*>u28EgneZq@#r^Cn~LBI^4%PjU|$YrA6V{`g3Jvy$WgsU0pSh$ylp8biy zW$iMJD=r(R!EQBPIS#`*)utZ=BNIexQ&Mv)rW55eLwxg>fdLV2N;ilke@CV|zc96R zxAvw&Zr-Mo#p;UJ%GlDI#NPr2E7|Q)B)Adfzjq!nFdi(oCg6BkuRQw6gYaq0(!w!l z0Nd;%gF*+np!%edLt%=V>SrUTL%5E$t!1~PJGngG^>NmdOfu6J1#Anx_lgEvYl^w} zqkAo#6dt1Gd)}?Tm@W6Aetn)Y%%C6vU z@I`@oGKQ9z!@i-DFpYG-xH)~D&k+#y+x)y~{dBd*flVQn(la!$Ajxe5(Dp3$KoCp- zBgPAhh7@;dcPh!y) zLl#Kl=V&y;+WAR1TU?fhGUt@nkBV}sjk={*0lE{qsu!4l^YNMo+dG2J<4Zq%Y=2;J zA6-8)%+I=`dFR*PFK=EFtQ$JiBBj8_hlzjWrS^*mMzw%Te^}Wq-MW&cU!ZddOs7HT zkwMwR@!)Maf#&^XwisUi{aXC7@MgPmAOd$8HXSsKka;r7vtzWx_In_&D5MaGNer|G04G>t1# zUYDS+p^5e3E2+nu0$B~Hcmz&wrfY^R(^OGXf9}ZIz=T|C+W2W4KjWC}bCie)#UK$s z{v;B+u;#FP)z0UY3q|VpRVeMdv}?rB+%#jC=@qY)6a_Ym=trUG?WOhoaPrT_w5AW? zth?Pl=66tH8k?rRQ8Ev2hzLL}0*)87-ncgM#!(WdfZQ*8B4X_}*Qz^dv#Ite>J4p>o1KEZU0$GtnDNlZ*a}6ME-)l;Wb-yM@Ta2 zPo*bUY8Yz;{#~LrRU*h$yW=U$MtK!zr7ne7sG4^fx40sq|32YyRe60}~_1N8_aswD5$&Ww*u;^RwW7512EVQMHq$ z>z?0o^LR}2FQLi;ZvA-}$8|>szOpNZ`@kxA%TYgXi-X7UM6L0cC%>D8CM(hEbWEI( z*c=Bn{qSpe8?b*BZnuSve~{;^lhG(uJm)Ot*Mg)p#C-{%9HdRDy2R^m|2AGD1=l6n z6g3cfLYc7NUp1|3X&s((FLc&Ow3LJ$U>Nno*NO1MJ$zJU;t6s2@X4N@9`x&N@CCJ5 zb!2#w7lD&6?=^+$xM(=<#ax4`T6u8M^)=?65o^e}LKB`uFYOUS0Py z(Yd#%Xi1C;TXBQjUSM{y$sE1ufH{Lr88>))!wpT#1!?ykzm-{fFD)W;NoEE!I4J!qhpvGUgtTNQ<5YiEoWUK zCHm%I03Z8SXicme z=W8)%U|*Lgd(Q-fiQn-m@*pj*(9dtx{R7C~gz>*Rcqe}Xf13=rNT)|kl4DukDIMI{F_Rm#oCg4f5T2$&nQf`pdS?OMHX^Vz2;jekMrqe<E4`RC2&@bHQ1>NM#^oCuO&Lx$W%(8q<437Mz}E-$2y=2C@z>UDr}{<&bW;t-~I}If13YT=jq8a9u)wS&n@<#X=(>m zv>h$U6cOB>ygauQ5{dURUN+yNCZ|eTaoCn>>+r1)Fw-M-zBWLE-KD$}ki*aj?0h+8 zr>wjRzi7bd&UII;=i@U7!pNa5e)2`~)0`W5hPC9j1!%Wq}%(Ls&O2jn`hpa z`V(C{fA5Z57W_RQC>PY>s`vTnmb^bQJ}w6%8yRb(_g)Ot1yt^~W~)eF{$>U3BImUm z!Z?Af#N|5y{m5OE!!5#=xzWkRK42R-Oz|6$E-QC522Y7=-%vBZmA!_Leu0;AS!A_;afAVREOa~giZsReM)r6A0Wx|QnhUc;U z$Vpw>ddB;#O4i?N{nF=f;acZY$>3A^{cmYA~M9cjDw-CI6H+6fy9Zh zBpDMAEL0ehTpfhSMoZu)?mKoDD*@pd-Exa)f_M&t1qHOK)gZ?F7_O~yIh%!zNs1p3 zf0gZw=c-NI<1jf;&UXP@6bQix^W}olmIS zPusa%%!ky-^Ar87jL1!+LJe~(>K zLq;6`Q)65*X_DK`AjdRUDz%Sz%a}S}984h6xIDgkv$#(RfGMEo2M#w=z$@>*UP2YO zj;L^Pb5%vc+DX1wTZQ?=LmyU&KuYfCA^u}j!~Yu&C({z;zG_Bf(ah2W^;d1_L`@u$ zp|leEkgf%6zv;Y=jl4jUwz%(He@mP)O{Eq;+L(KOtFGFX=gPi-S{v>M1A!FYqD)-c z0sGz5vw-J>uj|e=f;@M%Oq3!%Ri7rx8V7ynlw)9VzG`rwUjq>a#?*crkUd#z6HO&I zqW1Jkqj~ks_j(toq{-9IG)_eR9M%?)m}d#p3Nc!b;!=q-_$~|$+fDt+e`Zi!1o?Cf zl*nMvws`xg!d6=9+z{V4I^Naf?_A~97*IOXY=ASc_|c;pd6E3w6Nm+RX#MrIws&O6 z|4O=Agc`GYZp$Ua+B;*Dp5}GokE6LcYnHdHE zFFf;Gxls6rCJGrGooWfGf9L=xO-JQ#Llkc8<>S>ZbUJH^o_J9JRk496O2K|ks;F%$ zI@=9F$?Z^|)noF@$(c+n+m#2c%fTy%Zq6`h4?G&PNY{ZC@tO9EM4XtNUgCu#(b#SE z)VO;!n<@yVvJls@q z{uobL%!796{oi+%E5y=Q ztZ0c9&GZy&-oneW>!CfqKUlll>m?4PR&#RFUwbKCVh}lf{9bH|e71-T_LU*$4{-x& z{cUVJ2bBo@Y~1qSpJuNYwt8Ie}_|rLhtbu z#dp3JMucy|+`bd|xz;mmPd%YYyU1#JTOe|pi>_1nh1?XvJtaY*f(%ya7p?S3riB)V z?e+ONJjP{W!LR1gLY`(ZWV$Ek?UC1XzL%^!|3ZpUfBM%3lh4t1>aH^LEz-z{ry!ek zW4B2*6@R7;vR8!C08XDpL6>t1A1D=4RofpfOER!nSiVjz(bfRH3XfL?DzenUG5ma6 z(>VgxHs>}G21*Sv9jW*CyjOJAE$hi6EP2%@j$7@-zKi7o-WJ?hJ)m%qXZ5w?sQfB` zI9Cwuf4qvZEX5vApduiy-=$`qS*r2p!RLA7KI3lcnV$lyu-9E0%E=7-h=u+DNkqAN zyF6L{se%lno9-Y&81&b{r}V&uF5%87L|aWHbxen5S6bo)s%+(tugxyci(~J~K zXN|+A4%-^_xbzm!Ay@c|TY@#u9?1R8tGYYBC|(VI^}_iQivk|#U2J-yu_e|(1iCUVG* z8PcE@N(D~Wa}bWMT^HP_SR3sM+sRV`T85H`JZoclqQSl^YsY1iywa&N^Y$`?xD%N8>&tuMFTgP#>(j0e28IfL4(^@1>pv)MabRQzI8ff^wI~36Osm zIssucEyhSk!u-XJ=k>6ge@1QGotwOM*NxdXI;6zLi}GSBYRg*SadgX@Mi;rYS`P_R z+A!fUKYkn{=enZ3vrAgE`(IiD``GP@pZUVA%$lmdEkUA;CG3SZ;R{?Wk*%d?&dmB= zhr>+%so9v{0KD-;fAy{NwyeC$-gz#we-o8ce}qThUcR26k6pM9op__5aX~M{@F{n) zKoIH4H0w!rU@H3A^2;!?()$_8D9nMdA;IWA$VvRIGX?f`E ztbsDs!BOF;3g(53Srp&OAS9UuV5J3Q4uT)9{O&|;s74h|1501#0GsVga<#(FQt^RL zDMD#9?~F7v23%ZZh(WpeUcr7-s$YDo{2_fm5S!f`e-W9I*?b|(J*$V#v-^zS&;x!L z7zkT&O}RCiE;yprC5Gbkv)Eb*dlsGr-ppvC;HntF9EZX8FHK6or3z!mxXfWlD`{Uf z#mT!AIswF#i}WmiF2Ybh0QxG$Dr~?L97YXs^VyRXHco*2h-wffDr6!Fl*Bro1IN&u zKEzXZfAjni01WkNjc}m_ijNWy5=MUYnCcO?l$Uqh3;|kGz9$Nis`GHx%Wln37U)NbXP328BPl_SU)mqcp9E;YG@7-MdaroH|$VoEUOB48{ zw4M|p9f=3(6A23!6z036wBst=^ScsLFulm##o-~yOwM0vZwJg?$%*`}AI0n1J0XlO ze}1Ror+h%6(rFQ8DcRyf!uOrP4gRX|k|J7ki=l|y|vK&l~Ck9Gs6%Vu%r5YcP=G&brcP+Q#9b zHA-!6*!GkiB%I5H(y4sx2T{zFSqpHDe-AA8Z^`h$dYi3>BshxV(U;w>e= zhCqJUMy|7xtNz(#4wfhV*>9e{Y-WOLES-J#3PU)*&NoBv(_s);Mb2(9Lx2V4bVI00 zr5?#GYC1qvXHqXnr=#W1-NVYsG)o6^F^j+O3vT;2#bvj0SB8D3U=n+22~K9wBCJ?R zz#Wf)6;YAM2slIgm`5@M(G-c2QEsV48bXZBOZU!~ry&I^e-vfXUt2wSLLu!^Ot1J> z^bjL%rkMnXO(WssKly|kWHWw&)q(9$sQ7~GtHfcH-ekLGr}Sr~=dEExm1n^dN8owt zB2|r0twT=`_}b5{MKX#_>-M*3rTqfU+x(p>wCYt?fQTuHk}fYigeRXU{^YwUP?t&m zB9K1jP2>?pf1Dc~38S^e6K;-;;5)4MUCdT)CqE5S;x1^4eSQr?C8X-ZP-^zD2>v7S z`LGJPhibzUrVPHO)SI^1e8Xj{cYBt}_taO(#r8mud1|>g*i2zFqBE+~C`hTzaBrh4 zyoTIiCe*3!uY|1$ZjB&>OtW6ul-jsEK%!McCXJGieZ&Y2we_}GS=zIPw7?$rA{goU7+LoLi*EHDQVSx1j%2W48vC%ode(isD9=o-v%p02f9BT_1==q*Bcs36lLnq|C=mAEj0_|dcRmiq?Dxc|l5Z-Jc1HKASh`?EFJOH(gy zr-Y8XWoeC5YZBQx8?8Y#T+HFDOV1zQx6;9`4U>A{8~=$DC7zAKd%lSG%zN@=M@%`H z1^!tU5V1s!htj{?6iQq?2$!`#0d05b__imbyO!A=5yJs`-DO4g(g%+mN3Q186}Hl| zs*mIhk$FY2AK%_{jLhOH+M&RX=ewKArSnNcks@#^L3(5R_;u}D8KI!{J@ZoQ@xI1E9B$fP zzc>(IbO?SuJ^!L47%h-pe}M=@xn)?gOM_@9nFM$k8~T*-#dXk*$4O-q3;1SufC;x8w-vm?%i@av^D$IK{qsQn{?9)5f0(5-}TN z`by5fj~x-3euhrocKXTZ5wqVnpG9M%^G?dQMD70 z%WkM3b+?Wd*Og{=e{5=XcYT0~Zx}1rdUwG-?n?`x^l@CF2F+=z6#LO2$`+U*8fw-V zERS+ikCeexm8V&reVZ3ELLfeBu#N@4sy7X1qn2I#7N5$!}ykyKT2{+J{{DMmv zMQ9R9%YB#g%+*oX7O{J=1=+g57@3<)H>9Nf$_q>8heUkVf9smXi4m9aguH;{J9q+# zD7!T|&9@QUcMP?OvPqTpNW?y(NAWjZWO#Ir9?CRIOxO%k)S$yp(O|vXXUi&J4?$Nb zg!RU%C{sn}HBzQ0cTylOBOEe**D(WK|&tE4ul3!NE&5V_l9# ztP=VdceExlnmRO%t6h+p2E@wkN05mea2Q9L2+ET*D-KFb!&gOC@AuM78?Y>?9DKTsCjCV&-x ziE$VTX;J*c&&4ztc8jI(!>~*;Ia-cp+Ikw+hhDT9Z zf9w-|`){WB(3uF?lvoBsyJs23&-AVKzyo|$)WO0BeJS*_SwpWH_@i4cDpxz`!| z^@UNHx%)y^3qqN=TzS9Vj_B@PcSA(9i7X`plZ-$ndg=~`WCOqr2sZw@P%=y1O?g^) z13ht53ISVzWtOUQH#A^Hy1u-=MORTSe=4~XJ$i`^B1^U6sV$KPl8jinv%y0PlKk~4 zxCQ^R;=sEh)&)3Aw01V4To_uCDB6AMkYxupdL_`=4ql|3dYawy^Ii&uw}N)1w=&XN zD|k-aFg`HG;Q^knG8f^vCHW$@*xIB!-$;HLP+w1SV&Hh!mIE+O7P2xb-0A&df6Asb z$(WIdq6Q0^NQIWthFrF1mQ1w0D+t7j7~^B#$7vGD$D82^bTy0%riP=*A#bqcsd(hp zHc*;Z!%tH@Ry!N{>SH{4$5xb_;=&=oFTe0qzWe79r@|c`R`a)x3D>jc*Vk%C z*<>E?IULxH@cwvfyI8Q53}3SC48czG zYZu1#_|g;)HN;QJ#t}NHK;3SF1n7rhd~}KnXEjkUChQ}S|MfQ3*X3-Ge>uqzQO^VQ zGK^%rx0%R67YYQs4Pneff%j!8{^ZbOzhI{vW07(jiI$9TS%Xc!+9RFSC36%U6?!pZ zcEb6b`HK#_9Ymsq%WJX%Mu$(a=UjP&cHYX8>{!oiAjd|)7Ts6k94qS&=(at6m7vQd z$P3r@%uL@$r3y@bwKISIe|EvRZ?!o#o<-#hM8fTy0%@SbEN#d5L#)XV2T5${d1@18sA(wuLAmuh! zS>Z}(`c=%(jahLhID~@QMQg_7c=Rl&XGK2}aAm^<#v91LihcI-ebs764bZCU&2R9@|c1I$2;K zii$VV?Rzrh&xP5zij05Lp`%Nf6peN8HwXcOSYNvJkO1g z2cmtGa54Mzn@+BpYQP#BA7+Ay;4FXfM?5;$>$w5uK!Bqai8U5hccx?IX{d-$$`fmC5=eH^ z**LSf-9UNnV$GD0iqHT4^(-HxUHw6eSr_icFPkXq)M>m5!xdg zLHBt+m5?ny75s*e@odmr9nC7>;OJF}{;>6>9=pbBfAa-tL|n=8pef`8J-4JXR2Y4o zQ3IZpzr-pX_OT^p{b^m$ZzQz4T>AayiqN%lwxBb^*LnQO4YB%Yd34`JTld9-wQXRh zW%p=^%03Ya5bp8OD?DuK?#{0@f07U#!U8%mQov4gRTP}{S|ccX+8K#hjBpZD$KKb`BO%_ zz{OQUi8k;OM7_j&l|B6q)Uv>l`eIA-e_Y`!qRQsHH5p6KLlUC1*ScF+0Qxia>#oMs zR&QyQ-SDUUs#yOmtHvslIcl@CGvul*u3n1MZUO+zCyNhFfd;&ETE&NpX@Rd%A&DQZ zo4Yz5^>Q+L;ajI+>o#cQC6vjhWCR3PzgR>)_Z-beB??C5;o{iu=6}LKVDx3Pe^Q!D zoNc6S?r1%~44+3K9a2?2pY2;r(*mu=Q%+vyQ#H7FEApf5e^!8T{Yl^aoD4L=+Os9_ z9K+mF>^4Y}IXL;$TFi}3q*zww9GQ?&>$ka7-#Ba1Agk#eZg7`QBP>h(%*0(F?WHL! zI9yxjR)wpT(IU|F=jeJ(Jh3ttf0@sqqkq1%iTJ-g7%Nc-R&(fm8e}uEENgH=B zt$af#VNNiBF%0MK`U5nSWXR6*_-SawdL5DGwpdA~`wwdKJk9-C%i z1@!r?-Vgk)o1YtmFa}`9Y2H1r;Rxt^XwK{|93^8nx&KMtX;kmF-gLW?undLdoUC_8 zdiu(c4kMCUFWO521+TT`f9A?yWB!JVr1)QIjzusWUTp)XxI1s?qmjo_*D~W*J%y7_q(jS zH6Av!RDqequ@)6iHrz}ZbS{NX47nXq2?95@bRpb5)Q2@ zUy!@jozU&tx2(B1Ox1_!+DMY8A%8Adz#}0a2SBuTxXLq}wgNpPdLa%r}kCPN;EXM$pqRoiTi-Xoabb9rP_iG7RYCB=5`fZm8kpbcJ zcun!rj#J#+;}dLhZxJ_$DpN_49Gz8dVyxotx zCJ|(9H0|Fdf8XfFK4;zf?ZoBQ+f5-hh8xTR#G~XV$@A>m=F-ZSqdmX(Oz~nnvEbk7 z-w}(O0*~SdnLs{lDz>faN~bt&R*rX6927sxkXUj|-jfk@Wc=vKkzxBKV3LQMXH@;* zJBn`=T4PUcfG)q9T5CCsjgOx84#yPvV&}vfSz--He;!=YS^Dnt>S=k|dYnj^8uBb!x)%aoe4 z#WO5Nw{Z2dptQM1fKrJ3Szu2cVkARC*d3QlapdFDc&f>?vcY?c2l(za~$@lkxQjhKlpZ?vp(HyqRXjebT=$=(ZHUnRzc-&_YCx124rM?t(X8XEA!V_AEPn2*@ zBnK*TTMqPT%(fT^q)eIrHp4K?Ut!NAfO9_%@N~yROa=i6o^wm@$0MX7aAH@vtSrXI zUZN09edzH=t>m`(LU2K8FH+1R!!acnqn%y{dm8uw08n>01lCm-Ii26r?2w}-MlGz+ z9-6G#^?#}clN&rMN+PV&_(J2QBWe7GVEVFNOMXTMhKvZLFtqvvhU{%+2G-Bf%!r(< zZw{-dA2UwVW=x;72|}C{d3i|SFBxY$E36x$*Das#y!nA;*oZ+J1+K6e4yizOLvOW6 zTKoCCWyVTLPJe~fQxd8+&em6;-b@s$T!mjLiGT7p6kGjj->`DNcjnAK(sp(u1y>@A zfpG+d_?gVoC&|}O<9t0UkT7Rtn@W+xu$m(D$0YUA= zqw!Q=`M8f*q`xOsDdV@YSCjs}#RIv1GD>=P_K^7F03WD_pJPZ9ZDqk_z{W129UNb# z8GlFvd4MZ|$kO4~>p$2yydX1l;;{F`dKB&2#!&wh;%r@~3`LqWmq%d&Us58On+9tT zqaoq-N|*rALNlt6VYddO#evX%SQ_U1iQX42iWTNN(Z*!XfsJlq-GN)omE~w1LMDH| zUn`6NPz5^tRQ4~{`~4X>8h@qQkO`v7oPUog?aff;^>BR{9qXt8I z7-*WH-rSkKf{Ns`#{Kv_16y?2&I5}LInKF)0#44TY{&{-$F4*Vej0!GF>|Eu2>DY7 zj+7qA$GwMSN?PaasRR`fId)X0;|63dusyYGQxP~>25xPsy*`t{eLcx zwJ$h(5LqEolTP5oFNHzw#Ws=sVeJ~zWG$Z7@RVgpWYUkuJ2uvB+6ib$?l#IT3{Eig`JK;Jt6{rOd8aSDStK?jvlG-?epT z2w@lg8#|X?#g&VGd}>cWDN8z~DC?1Lus3gYn70ZTWHK2na4DROFS0l>%n2Fyn`*-|xbtU2Hd{G4IVb<)13G*DtJH3}S$- z^U@*U;m+j>hJm);*$pptoTFH%h86NTU}H?-H2DL$hPWRwQgVjr)_=VJc|2UNsvhLT z>Bm;yTULSt{l0?Ae=g^sPIg6F( zt|owvL@;+1ALemN+^BM3<^22 zDHFz&JoMcwNs6&tRe$a?X8W|3=)Hx-J#kTl>2VX}SFfy=Y%M>gszsok%(+&4F-wFR z`Q&^9_BGwiXc0s7v+J;pcG*cebHN%u1`Ow4IN&w=yJf?NYxm_ax_vaws{d|v9DlOj zwzCl5pNr~2xN6t&lAru=VGA@I?S_kYQmKEU9@2qJau2AT2o z0I9#jfsu{1aV)>=0< zl{LL+k~D91KUQ0Sj(|`kGK@BYG?3<3$m&CZ{7X59A|Bthx1wvrg6etWDun}HB^Gx4 z($2o~Eug9^e5XL$LusRpyyuedozz}dADyxrZV8%WDE7fUl4+gtiQ7VLFXfFkvjZX7 z^(B+?#D5YGhwI&PLfT?9)tRi@;_r4(LSY@*wPZ5?snWZOUpC;p$dkgi4YhIIh<3}O z-$I;oj<8EAJzX}XXA+HL9+`UzfK8^ysT>e70@yI%L%&vH1YwD#L>4h-75`s+c=A2s za-@YCkE7fVzAM?F_xWPK8*kUTY+u{sI}3fU<$sLQUm%s+?{)~589-{MF4us zd1Tim9*cq!-ystrP%5Mo3xiZIzst3|r<8iX_TBr zlZce;c(!1G67Q5S7obQmU*-=SUqoEHG#^`pNT39xGN#Y7nx-C!6A6; z5K}vtWfOz5Uj93iqK*H^*mQyd*xGDwxvUtM5qJ)?j;(W!^lq1`N~q-9ObkjyrGE}@ zovPZcVbsQY0m;$p%Bj0Pj<)6iBBZzv?v5&|`9S0*!QPUxTXvL{Ts?I!Nt++kK0xL+9%EbZE&APyxg!_N~^>6jEPucH3{{3F+8u|Bs{pY!l|50YY?7!+|{(*nJ z|D9(?MPUC~LbT*I49ol#ML!q@`+uhis0_n6%>PqZJ9w#=VOZ!NF;V&JYuts^^vL`r zF~cyj7cdON@eRW;uunev`z2@!{YlpKKOog#WiVC8lI~PU<&R-lWc@?h4~AhdBk4%8 zLdv#4|Iu&wljr^!`UlE5De~ayUo&tRMj|za;T>Wf2PFS3X(SE|!@l;PB!8d$)kyrR zJQKA`8VL*|JO6;l_aCSuxOZb`5)Az*Rm^|0lKxcTf!{tTpL*no*)j}!8~=QM`BO+M z>VZZlxUJ+rp#NJj<^Pmq&xVZiN&KY_!_@6xr}PIc6e2>_MtbL{2{8-{{>_y6{)P_i zGcwgGvv2y(`oaI`*Z;lJ%74+-h@vvLB67k8p)&%$s9N})uPRuDrR0MDQ>H9=#&pZZ zpPY~VUZU*-Uq8g0&gQkJdU%cR?|B!RBHW?R^Cw6C95=qcF=~)=%wN7uSp^CKXJwG? zNifK6d`5dUpe|QJwnPO6SVTcv7r=Z8FeJ@HnR!B2xw>CCZt|}~6Myg`p4f8WsW911 z_4Y9!$snWAh=42tp7th*-UlEhf0<&=sw~K zr7_ot-7td)g0&ZO0VYQ~uXt|9;oqLG5sCsi*tJs@)i#a4Mw?oSPgx&u@QMieW4-za zcS4ewG@{(hI1Z?#8-F<^=&3?xn9avuFO%f?Z)J2#yuZh1YQ|uS?nRN;aQG{VxyHjh zRuzVp;XGb-?*bLdj3?m=CmR&cBi*D}n!DLHH*AzLpzCninI^z(k(m%f-%9-1fzj%B zlJ3)37_U2tfLUk4F(zFF5oAdIi~`PynS^v0Mdv(pgW5(&?0@9R8+Z{4h1Q7x_Ly&& z)AfQ9ZHCKe8Se*IUqAqebWc$GYs>Cejz^fCaIuBOKIX&Tf{3yfbuPMl=uxW=#~$G^ zCCiEX+eYegH=~v!)42e^VezYf`7mg;j2`aC&-@uf$JZgH8dk@GAeKNz0>r_6N_tk| z&*0tq5B3551b^M5kdjS8;=2AoBs%#eDOZQ#c=pb`|fC58iz(Y=T0NieggF>;pp_&1r6Ip zx9UJmFVdG%N@#JAkJJ7vk4VrM#3y@|4`32|$yYn4(SJffaUvcr;m9}$Aq zIQD=y{C7kAfaleV2GoICjRy&pO-d7gg1HmWmGG=EN)Hzt;vgwsSa^+MzOG0=XJ-nc z{)uZz2s%Tj1q(v7N~>0XvB6wT?|;!x(omz0 zT_+=%4}bkA-ydnH3P$fM3gt-7Q@Xz6gktsT&N;3~UR5b_)gQ+z$1`XeZ=SQiSvwi*^dPhPiDy$heI^58P zJv{?F&##vkw?ew2d3olQBQxt-e)xpa5MRg4t$&R;`fYs=nIXh_qwcIE2>f7M7Lf_f z40@@x69DnwER*C199PJUF&2Mfz~#L+wcLrZTo1_Z&&xP4*vnn<)G2b%x$>y}b37x` zA&(0MMinvRrcJ{oHN6bl(+zBU_-=+ts>QxHuzD&QpbJpdS&iA~`m`p{Y?B!OyHi`s zFMnb^8sikUYMiSCyXL_Ur0*ugf0K!!?_ay6NqF4nA(GdlWF1v83t5p;qD}5 z4noTNf+uC1?mg$ng}SgKlLgzNI;{~3>d(5dZ2X~geHHR=iknf~$m{!Xuc2v1V(Goi zdeo(Y>^AL7s(le7s~~V&@_cMMy`Jz$VBfIg{&J z8AXv1i@1XZkckZ+^D`C?>Az+y)|%E91F0hB>ck%5k&<(2;X~UQNx?0pyv9ub|I2BG(K1*nna`siT`8!qrC4-Ezt; zn-QBl8Qa@(*fxP%$yo+=*){3eVt-J8gQ2N_kk6V|=@KwodQ}lSD#XP^s)y^PjE3N( zI_PT}k|i-YkqP*HY2-||Yjux;LdP^o6WjecnUS}WF}K1A3#95#0lCjTz5Xs>F?;H^ zLaR7`YolhkkB-Z{7#HNjAVm-7YuLVMs`CIOOLt7i8?4-`TL9~>lvb^N@PC6HiGYf; zO#pk%GtiGAIKiCfM9nWG)mP7UE7&!+HbQxBVqXBowV7!qV=%5 zDKqsy%ZMCNe;Lq6S8hrrb-SL=d)DBfpXMVz-3}^7*wcyQ05VcpRuiJiV*G6u>Xi$YXo^d;6JnN2nn7gzOSR>3l#Vnruuyvghv^ z(eW!0EbOWVbO&s;;MBIj#_7nw#QfYN+rcDnSEDR$;|@#YspKv)uYZ8X3S#Z?3gZV@ zoGiJ@-!kVeY8o)97ORe5R;4%`i*(rADlyNZQ`lsS^X!xyhG1GN(Ex;;6-*AXfu_i< zg1|0dRUKbiy74PD(p2TgRn$royvfNn5HJTk9LrXtMQ*SVROUC809lBZvAzkEu5;)3 zNGnjEXEw6W4b05Ru}?NmwAjh z%Lzp1u%EY<8LfDuDIH;GI@Q_`<%UM>1mvBkgvI@i4L$?`&r-}}9f%}v--u)C<<@C>| zad>LqU}rP}6TF~r7ze2;kB?rkMbp>Yx}wmibK|GLtvG{Pft%pWTs1&95m?)S=;MRm zTME$d0QD#t&SCt4P+yC2_ve)C*71m&`Iy&qTvyD9y?>17J{&A&y(2_%E4DKqjqxu{ zx3K7ZG+>d-edUtIuh-Shs9!#Z;N6_8lM>e>lut=lC0PofUD$PDQ7;W}yL#oo;1O^jW>k}x3v41HN{ASz9oQV(x_=FJ#(9$rJ9n@=T~9%__lXFOZT6dYN=AhMBz^2-a~2L5f}!!Z}Z3K zD1R;ec!XbKN-B;K(LA&_Lz2nDoQPlj>wuKD1jqB}KU#|p;L!~R>ijWlBg?mNj_Gw@ z0;_{gV~(g%y&MimFAd6STO8e==J)-=r$3XNBs@2N;f&5x8hp8*&kWPd(iOH?C0$$d zQB&gZTuTW0hBy{p0w{Q-16J0P1by5-?#${3D~cutcr!V*R2s}HQRBvl`VxNBa%=a z`anI!g*m!8NTzYii91=`8x%Q~OMFV}V4HBka zD98Ggp5C=z5+Z?(NMJAp%bP}g%vkJe zB-au*7Vgk7hR1yu1lDM}IICIGEE2E?_3k{*=-=4>USLb5K+2UKAzp>M=Q9e!ortnM zTwo3WP1s-cM`5~087AN7eXHGRHncvh?#LX4ehh7wqpm+#c+-5OI8Ajd>wh}`1`yl~ zXbxTpR+v#|Z+aj9?q;s7@(!)ya+DiaNpa4S3Y8>$pNNjl?lG5Wz`VB4mHgGAZt=(VLKik(5TpJg^|*?Z z$>JSZ>@YPfi0>b@z>P0i{C@)$w^{n_d^*eu+?!h)SUPe(Tq*ZghCcp>a>`DNlluyVwa`eBU zK>;x&C^4<T;K6NIcG3kXn)AQWGY@tqWY1~ z59(}aW@x(5DHN!aS>29SLH(H4o?%2>YRS{<+${`Sm7WFDgXm`xpz{6-)?eE1UUTrYX zC57<*NDKGQoe_a2MM^SgLuCa~^e5Y_IS9`4$ymSO;Fg!e|IO)w=<$ytHls z;;usmspT4x8HtXb1 ztaKT=O56l;=nE8ZfXuiaG`skP5ZEHNX6*41 VIdrz!6^YnzvkR7BNQ|P~RN01$%W9#<&)W9Wl#{X#E`J=o@aEy5x8AQdHx)y@!&yAWv9EP$zABM3@ul<}L^E%iD<0dG}&DW-`~>Q zN@Qi^neGBz=|*@pm!sRBF4Ndja%Y@;7r@KIC~`(tdZ4>9-XwWaVuqwuNqY3!`Y? zB>K|*C)KH__!^!?W>Xg2kEAI%b?|UXto}9jFMs{gKcmvEsw)Lp{le}5*`EVUzL3!; z0*HJf0_)>X4TSZO1t8cTqP$2C1nz|7p2@yn5RDCJbRw=hTSIea6sv`C%D z`>vq^34ygfd$|RaoK(F!p7nxa>OkCgyr;6y?Gh@-bhcY!p4ibP>T_YpS0H{*Pc9at zXMgU1?-Hz!-V+Dn2&@tP!B@5XUy9Be$zm7?q6c#D%@CTI*_oM{efq7vffA`kJ>C5@ zCD5CQa?BIL@yr|GX2U2^tnTR-VOj#MQ0b%F^LyeNi^=o*BZbzkD2oMns*| zIJaveQ~>SZJHIR~$)FTxQ6R|R$8;kc*lcg+AEdUhIP*|`VA z=Z>6d$u!%J6LX7Qapup3`{Zv9s243hnF97-(k8+hVavB0VlEASU8GcOsaIUFtdL9m zl?Jc4zG#d?S1Ov*Lf?$3Kdv&A>{?Q03EUVEi0aQ1(g%1wxO`(|3!krvAb%_w%5fW* z20{P(6XtGNw`5cRS?)n~Z{$rheBTB$*yhCWo$NkJ+cKGK|EFQ+Sb0kN8uR23)^kAF zE*4t5jX>AGH^ZQW*pW6gN^}EbQC;ghmJ+5?qCVgOFob`O7Q3fwP0U0pS}%rf=3Lhv zdrI8YOyryT1^lkJlo82&dVdn>*=8r}s#$d!@IVx%YmP}vVsUC%0cV9#X5|^n-&_-B zoqlLJJ40)Q_uyKEN&tvd%Huib10sEV2_{%^Ptt~TK!2DReM2rxsDRqgW1lchAMw61 zx`SG7JY;{9(`cpJieMiMCHM1Ox9?|sr+8q#`L#~d$nzgTW=BPB34hAIkM8tNeGv}l z7Qt%^@EG}@w;6;LvY&$bGd4oNpi(YPw&pogVBb<0D#`}hiPa6Sq!eJc(q2!e_B}_{BY5JRN zfY~|lu}59bEPsnf3`CJRqo+Xgc(fADmT00I!-buQaz13yb_8<=6ns7FiqWwD)obpy zCHRBPERXXahE~+i9xv3bng7V(N%Sg#8`eCTiH%%D- zYlOg(jDMP8`u)NZyb8}9rxJQjGVKb5erQ*1^0Tr+T+~z&zT;N_?GG+D%Bz&P;iLG# zi+g1FW{f}DmTHkWFNM+p#nET9^=m%yFt5{2+idTq1)xEXcAJ|dfAZL~o;3x$iy|%s z_*ntyK|ewgfH&egN6mEK2>WJ&>qR@NMW zW%GO@?@xwg$9}xDGCsS=_eA{wbzShJGb#@V4OF|A*sD`Nt>BqUxW41U zE`Q1XC00c}^DV)(4Gu9-_&+e2P@b+uRgwMfzG`Y)6B z*4%!rEe9rk45uAq{O~i~x=?1TNaIL;1%HzY$+m5-uYIxB&6$qD3&s$=9^{|}!NFzE zg0=o~_rbyO8&(|UT=4gb!FpW}{cE-Pv`Q^+gpq44HZ*QF5HgjiBpy(d9v46bC z=|&`?x)c>KK_f1l>=T}?j{?BLdm2-u(=zz@8HP96mcG{jqa&v7hUHqWFsn$zTyUJ1 z<%(LunQ}8Hrp&j8{GD6%;_=M@_@3;j*{A7}^=N*f38AEwz?n#cqdOicaBwI^{02d5 z0CCr_&scBnx*I`-1@B=f!3KK)7=M-?m-i|P*{)Z~f<{*&CUe-oL<*-G^Da;R68%W9 z4bew6*ob+xX;muQJ$ICe#~cc|XIl>?OYIY7CY`5j*DSAZvghS^Ru(?&XHatJ;wgJ` zwxO;lngq6z;3qhSa!cQit8ZdXv1uX@=Rr_n$YJY3*nW}A-8R;?{PY6`NPo9L*XA*r zQz59NbjFSE z+8%vIzEMH}g8(?@;s0JwX_e_2Apz8&g8wl#Uaibs!B~H99WK2|&zm-7AqX{56j{dj z7mEmWO?f&~gqdXL<#if}O!yVKL=phsg+p*irOtqD{tCWz%yW#bJdAJOQh$JD-rOfG{++;^u(BA*2$8Nb-lsGeE3Jfydqq&7 ze4i{Ui)%@3*=EjrHh)+%JZ16}fr^t9mjwBx6wnm(Un5-JS*63*EtDR5sMuLbS z1xdyzbcnubnN7>{$zuxnmMxK{Yli6w^Aal3AWW*NS`JoMW#Z58Zv0%wsqTX<3&VG@ zA3Y!b*L5b^DPBxO3uLfqJF$k>8KV=$%DV;6|1nn(Fz_h80e{kADww|L1jZ*$irh$# zrJBEok@QHTyC*A4-(J=p=+N{z-1jIB@t?nz9!3-#FnRk5UNOyC;(fwkZ`fBoAtH0U z3F~9mHQ`HO$|HiH9zayatv|JBPX%lEZGX%TPL#M=;uSVm0ljL}ntuF!hDlV9LcIW* zI>@Mnc~Z>!$$xF@TkadT=oz9O1;6#HuT15t37B(D-mBk6csq^0?fJV(DlN_n0mrFo zs|B(SW#1NJh%QbD-@A*ag=cB_N-SFrY25W&t~a-vsXDj(ukI@_0=O3j`~7rcfk0tX zo&KeM0x-Y@OdwYQkY#0!Ur3GnJ0{AV9w^XNyE9s^fPb?WOc^}m;8g(w5k4VhEngX_ zcG70>iva7dW4-hDliO*JU(RwYaV}?i4w%5ES)gL`{vF064N+#MB8KQ}r8W;#DE1M` zWUFynJ{5zXXUJfZiITfrhCU$j;GTqz`^&cI$6kQV6kO}bE>scu9@zj@@X41j!pUW<=EPf_C7rHoBN5fI7a^BTBZQu$W%R0RrhBHuQk)zByN| z)Q}NXT?+?ChM{E8hMHV$CJ^$Idw-iwj;txjG~!E>i+5f&V4TYw>{q{>2Is(S z3T_{|D!{2d-2bS4P`GeWM4{iZzO&`}MNY&7mFLcoMe(XtM4XTy(lV*rZ{Cor3z{F@HgX z@`~6?OXFJ3zgO72V68~q)Li+vX?ynAP>3P4H97#6zF?X1%XE{(uITwrLOBbT6XSi} zupOz?{5Rw-r6mw^TsEl#W^8ZQih^O~>cACRQ0nnA@kC}tV^@aNj|HKdT;RxG?YB6s zyLzt8Jt8V`a2VPCuH)BQRG(ez}sMK!@*whihr2@Ui7DoCL*>3y<@v%=F7ZZjjg91grV<;Y65vbPO=eT_HjN6EYc?(? z<)HC^@6~IRJQo5Q;@M0J0`~s4&+xKV=D3Oe@p0Sl?V_5rY8g0^z|avzfe5E7AgIJO z4M8Algn@X0nbTC~to_QeYkvR$rE!(KYaF@xQun&7DQyd&>QL~);x2$ypK`c32MsBJ zYs2?bpf(W2skb*N6$TOu)^+UZ6|6-#IAb?6EX6y3hAA0VrGKqr|1MjB&>I!nc%|rV*1qz21aPc0W@%U=$uk^BVV&H!5v$WAMo)GpTGlw1Q;It`dh>Dt&mwGnGHZo3cI4)f{S3UeZ)IFv; z>=!`x68cx?HUxdoXd4Af%rpkwbG+Z89^K)XOGb576#`n_&*-LRdz|fxpSTIkxO$y? zYjT6n0Y(EuwWbQ4ct7ob{&8|bJ|D!sL;)!|+kc43+pptWw13wsgy0-%UNSeB!Zf7J zsc9x`={~I%4RCu{AnA!LK||MB8h(_T?dJQV&r-h@%HN3{)kPYwqqE_Ws!otbx>R`VyVnb9y0utSB96FMANLxIjeqU4^TXXhFOV7eV zEO0k1b4IoDqkqN`UY>f|)=!w+ELZ*reuMg@t#50{^|n;bl?WSwKXBqSlEOTP7T0!_nG_c1i75_^XZj9i`r9-jgd>@&FmQa!KU{@ zU9Um*TVcr9LEOjKeT%xUI{!n_{1Q}qeL2Y~dQBH+T!PwpansOW%sxnsK%bYeXw;@& zOAn6cdW6?Rt6w0el~lx|wuYu37y`Cus(&-I9$0n;aKzeuJ3)_3HD1RdC_f~lC-6x>&&da+g3zudiXuREe>5ma3h z@Q!V5XDh?0_aqBw4cxZ#q%IXCF(_JYxYyQ9Mp0vlg1P@6n(8}o~ zPW!Ml053q5rEXtZz%u=MD_P{kN?ho0d}(x!xsb(_R0xX>(tMw`uUb`ELIY|BUH+sL z=sRqLf?5nB%&WX9%dVgGw3(IQe_QcCwiOd9UuiGTWTNbI)ax7B8h>cw6`uVLi!b1s z-{iqH5$W)~9a{_k^+Lf>g#qwTux{8?+?gDlf*VR~rc)hfgH6qofMF<_MFGR|Crs&K zPA;wPd-6RPPUY^TS5>~D0l^Z3dMsU^hld>&ll~N@%M?MFU+D!;K-htCiXiyJ%uA7+ zJ}?|^dX5{g+7E!rJ%3t6Rd&6_>dJXQTUjkP4@A|0Q@ksxKCkA9z#CTxU zGl`mD3J(VMBlIfVCK@5Tpy_P)sOMA?GfbMvW7$utvW<*635lqS_e}}xn#Ag^x(Zo6 z+Sk;>KvlpKua7XomVn>4XrhKKGZm|BJJqmI`#^WVvD0OC)qj$2Cbr&NxmDwhrw-Ti z_q=yT3Z+OgYT~ngUMJ$U4#y)C*wpD70a%zXj?n4j3E#1v z9qxKDd_++-;1rr{2309_z3sQ%2=nQ=GzIPH%E&X+YidqyhozcGifbh<;J~7#wH{Ew z?a{FIahcDK`F|cC>6$|4i

    *WC&HHREb?S4|ZAxFGe0VTG$M}Qo{HThdZSr_OU5J zsgMVVLIL7aAz{!9l@%Vjhz4{1KzvwCDTN5c%k`3&=e0ofufhX5Ez9gBcNc3QkOoL* zz2V`^%#>Vn2wZXGb-Q8>sJ{Q&hi!yNY}qc09*RV7$$z1&EP{&IBv|}YP5-nZfx@DO zbgPeb;12EWb2z%|+*Aj5(~B#4GnPk~PBj+$9<_$}7e1a|WTwr^N*F+S7#4m#Ss*qa z&4&ar?;&vvfi>L+L-vTEYt^Fbh-Ew)>k!?df)3`fgN)?05d|iXk|?4`4rSxEijY@k zl{mjv`hWctqz5gkK7KFfy4ya#l@Lgm)c-#T5F1oreeIUH+o5PMfZzYPYs!?QC~<(HcHiU^@sJS({52k^rj@Tsb=3dd%3qBFqX zry_pC=>&qfTSG2~;#;3?0q9(~`Y@$0^H$7XuYW9TamQ;k*vHkd>H21^1Efl7Z86VP z$a#F5rBjF$d=v8!)&bycCY;?Rd}*SqP|m=(vP6Q`>8VQHytUG8?<+C(8v^Us#-^!5 zHR|bbd@WNKYpKqSsg@aQ%(rlQLO||6+oR~O=O5o1mi;dW19pQcD-H|&4hQz-?|#OE zWPfna*%D-rJzdvvZ^94f6w++8=OIF1pNow!7vo+qT<|Oy{dX#F1S~8@U$`0iHNZ~1 zU&rhNi$ ziq?at#)#|7uP_Ijo#h;qdmt1|6I2wge}C0kCeLH0A;u~@yDoz8QVQb4F)FY(H4qQ& zWE0Axady00Tje<-kJIZ+d=B9r9-)*KnHZWnG(W2cTEdNyBy_PUk{pCV0NpWMthIFT zb5gL6PBx*Q5OP{fu}OOy{`+H1HeJRr?qVwtbUbbdXDH$z+p{&`5>VJA^?p5j4}VcE z(I2IYG}e(WgVMT=H##-2z2P>M^GR}qi7oxjVSr;>J?)h0ht2Gk?!VbnX0;C8Nw<-# zW$vZK;G^A9*KM5PS}>zz6373Ff%^rN6J)ONG9w>rnYhNy%HBXMsKSAIRf#NE1Lp-K zJ5mMKjdbIZzoiB0zzHFJeEgnw=}kqB=243hOBF8&2ynZ}4;I@+MsL^qt|4_5PsGJ_HCT|DVNpkcF`$}Zhv_`T|`mK zRZah5n6lQjecfSnh-?biDqh0D?SG*#wH+AzR5sMo{DiVh9bVR1<+y6TN769-;@4}5EQv$7 zt&>0Wc^}7-3-xLm1z6l8+(6Bqwrbft@^@-im(mgu5Ip{^Y&7r3(HrE8on_e=dwQt+ zfs7R61u>Gh_E$qeE_o8+Y5Am^7|GjyT^9&taACld-yLrfv4tcIKYxjbP)Us-CTl~F z`4zZe6rq-%ZZq<5{Px>3jv#NO?Qb`d(wZNH2_KPlo&)0&e@~}1d z97jf}aU4zCQMvK_)}|sbdw_6T&4F+|Hb2zQk~R@*aj6s0Y7D2G^3G!?xa}yi2twsy z4oUb~oJx~+6~TVDcYlzkbI=W(^J~?HSzS&mO%3;=r7ZF4dz!4TZSARyYYHYy=Co~4 zYT8Q}QKT@hWNtGDl1Bn<(%?3#Z)z*0j9y=wJR|ivX zcKB7sk=DwKQkcN|I5+S`h=|zDC{~&S71QpiPEIwS3<6v!-+wyHc+Dm5n11869a_UR zh}Y!@v*By_U_?>nz<1tJ${fE>aO&0)GTDwU9PimvvS%fo zic7q;fQ8G;{N!NIm0U6j&a_OV7ETTaWH8PWDMLUuA(KO$n^$oFo`UOYqmbM&&fvN8!WuRb8WN`axPMw*KeCtNFi`mK0Vz^#Kv&Kp z#QVhg(sX(H-V`IJLfU4rjElEMNYe$K;)T2o7aREBgzvp(W7f~e93+K#l6aSOt)0Gw z)wr}3D3&1jSAU8hod7XiC5rCcC9>i;wW}|=J-qeRVi8?XuEMwm+BRf4QKcF-j(oqX zsl=08Lw^8C>MJZqN~l>V3*xHpCww`5(7v{*ln*cd@5`#IZ7m8r*il|Dws+kY^;OCPW3`O)$j2U}TZIfflkao%Np`qS!-={wb<_39<_H!g#? z4SyZ2$-zPh^N?*^nJ|udo0;;Rr^!wjE>oS%+}_V~fU<2pUpYdU8$@#Z$@xs8xfQO zY8!A4+v>3AbC49Ip^5)Q%c7OQ!Hb`JAF6McOkO-R_rZ%!!k zx3xyByE+$}C!bO8TUNhy>3I`k5FZRs8Q0sdvck{@z1@}LQySKb111I=(3sV9OAD}? z!%zSm{1obXO~E(&jM~C@4b-*>)PJ;Isoh43abW>EmBtU50$VJQ3Z`o@O1WYb>S!vM zLR>7HBGF#J_inBg$1iC2e9_=4MDsh)p#7&a2fQ3-1IfOmix9dtx6;p5Pj2WsX=@#t z<40OnV$D$KmU-z0pP_@F+Uq^ZZKI+*!H8s9bX=MhMMD_ha)NUM3-=L}K7V7-#t+xr zJHT4E)m^M%$52x)DdM-?(xx8ss;>IOF5iW)4eNg#U(kK_D-0u#9q>RY^UVj26>H6tHjZM6}5> zh@ZMJ=qJz%@}+#+LK@6iRe!UUaEt;_;c!4-heF#Dp+laoZ7RL!6R)uAj#)Vmp5Nk5 zq#U|miwthE_wpD{#d@cq170w3)+Kxn1)h*DUM)fU9R!C_sUMtozEZ{1BqmP9$CeX1 ztRPQtctIA7N9zS}2mK}H-=I@RBi~d zGqU=yFDw>-;|<@ImvEw&F;gYI`v|w+_PB$Rg;peFeZJAbb5c9En02* zx^$z`#z^7|y0ipOqGGz2kHWB!$oB*0hDA1Er%r{=Ej*{XF1JVrlX-ty`x^s0>@cWB z;J-A%_a=p;3O@=NfE{8-+Q4($GRN;z!An?#u>B0hRDZdou`O!n) z76vD`YyW;k`#bpzo?(G z(t@%EjctKlTC&n|4%=FqCbuNQ6qHm%zeEk2O;JlCKRFE;l1X5MP4? zvRnradRuYGt6jTMYk>O z7q5uTK@GgOP{DsoM4tTB)EbP(3DObt(LEroXatX`?P7iK>@D*52}M=2lWIU~-&HG` zx7ABjVpbxP5xic16-a zGq1XeklJ7p6D*0}CzF;~P3Z(**`uo?8-1{eyXssOYcR$qoPe4~5nn3R`qKWhU8^0jiE? z!XJ0TrehWHX?!IKwydqqtKt>?mD|MK(NZtkRMvM*m^+k4}t3b!5oaWM* zB|iSm(Smwoz`awDI`?pgH!D8lv5L>JHg&v;gKB@mU&8VV0pGKi-{r<^iQx)QO`Yn?cx0lErC~3M(hQM(C^E<#3NN6jw?IJ%5BUZ0>}#2 zY)yZ-Xr-U#ko#hDlu=nipo}eOwOZdhJP2}9)Nz!GJE)vD3L~bU%#M_@d1lj(on;M= zG4onq?C=Ht?36=N@rA21VjzYvC0&J>m~Do55AG=+rk}eWWmc z9+$)o9iYh@)M0_RaLHl!L(p~UCobj1nNfc;$J%eJFKiA@1<-_l+XVJNC(O27@ypf^ zB_^uVEclA9bj!l6DQagZSa2zDCH5>m&(Lb`X<8ONRc8H`wMK*G&@;=g4RxL7 zMKvk(#m>g2Piyt~@Di)h$RP3)ewu%wSY@oY&tNHY#Fv*FPxut1qa3PmM7maOH#l0$ z`j4F_1&1sgxa01{ea%$!;RL8<4_7pMf}c5ZZQz(@$`T0)VUE> zM%NgG;8CGw2}Qi4TXoCu=eB!q;#pRG8t)_>$p2fq#z{E1n_t1HGVNA{bS%bM6r6eO zLs<&`vB)(oXBGqyF<$CTtXK#5K(T)xSCKEqXX_&|0pa*-r4^vcWHtUn2i31?oVHMY z!CTfvcKOlZ2X>c7e%zfwEscKzL^kc@Ile5F_D>g!gthBrxo?8K@*th;o;p+0MCsC{ z96@Cap6(&w#NwZ{${(^chtU%hhdu!};3ZO=10@X!<-}7zRe~d*+<=o^mQP)G)zbb@ z6UX0FnjOl=ZHB;EOsBz?t(Yvotc=GrC}7L>4sUTQT88jQJmtT$Da3y-;Il9#Db>DF z9^8%cfRp(j?wY^GHEYLsaP~}NN%Qf+QYB?1gI3267ATP;=}t)XOf|o_ z4|iT`py^!$4Y2>Bvzt7UJyw35=1wiH4KsC&(S^MGGEV3H$W{K+sVc)mxQo+ryCkS5 ztaoh&R1>G3Bmq6i^-X`Hyzq^J#cXqTvH`8*{!K_Hq5yS6z@j6ws}<8;|aw-#L=+@ z9x7??w<*C^6u2HDC>Ma3VDSPaK~99fKOO8j1%n2^a^p)q2&;b$c_maX0!=RKa>P}& zrw@^-SHopm4S(9h=d8(*@hEj+5rFbK|4;{US&5U8E&t3b5guWWvkX+m5PR8?X>U&9 zeRMNR@e7Oi>fL<%>_PtZY88l^{0=`4Vu5p8Wt{Z01Z)bm<=lp5^Y=GayT>)W2K~T~ zba#4dq}4IMs$hTEPbJkj)7?)vFR7hjr6nTgS=8`)e<0jZr5J#PMue~*G})z8A>}k1 z{j;3PY$zRy8xrSObwAq;t)>RG%whR=S(T8ti2(FXckapWpM0~4_m z2NeAPB8+q*v#l5aa5POd;EZ97wU{!TrZ9c&IIQ@{dGLQu;%693F0YUWw_691(Hv1C zSy~a)51S<#!!>=xTdp|4w$eeK4~1+8rv_W>Fzz8fdM;AYNVLDeo-dERa~*$^h_(7H za~+@bRAbsGm2F)Q*#lNwmC6x{Lb8cqfJbMYy^X~Wp>~fJx@qc_wKk!rdW+9>T+__h zFCJeFuO)xI%|?nm7tUnXVsU4glnnp_)HE2rw;}%0bBx4!^hG5{4+-~%+tcum1$)lc$yQ7+H2V~Z zF)RmqH^N5xuHid@+0shdWYTDES!kR4G%6hfUP}Gogc7Vvml6C>{;|G7Y+RsS`w!YP z&h3BAcqtuBkqbTuw$(KT2cyL1k|1JNF_nM3Lp zqw}*a0(I~vip8&qnj+;)09d>Pp!ZSxS540_AmbOe2xNhm3>%4iv>a?T_;==jGZx?G zU`_fPy!u+oWhhI0D`mG=6bf0$XH_SA59!<&FU{>T)0KlQy#qP4X&3YSMb=S1p};0B7wXKgO0 zvy(Bm+0X=MgDa=?wc{fgv=lP}G8#6#LyWSF>%&>=Mswx}P&HBro&OG4yt}AG*~))W z4RmttmAgN$zU8VZ;5IdTbILc3ZU5|zpIHh9gc3$DX$Tq3$ z-A0)^YuVX{kVv9;)J-&`hf(0dSqB!ykQON)>FrqJ2H3?42wjSE+h0=;G&Rc1b`Vf$ z{N!O{j(eRGl3v2^YQYz)V47Y4w={o-TZ@s3;jj9XriH(NV!4k{HoEKwEwDJcuSV0y zca9s%6_Iw2VlJlbw(a&uxSUk~8lzO2&-q$RdLNv>67-Vgx1zd$&WaY;eY4P?RYnADI&)ROdLzpz~e%fZMQJWU}+P#{?TgZO{kyQY=9 z#sL_&Z>i-|4BkwE57Z`9YV)AdnqYwr(<(Fvq>2P_Oh6_sE;4TwJgUiTUz1xP@3D^=ZI1HAd-i<c(nQT1r4^UzUhe6hPntk$jLTj~=m* zfsnzv!cUs(S@y@)RU~bU>%f-|g9RIZ?he3nfR(M29glWN1OF0-->|gUXtfa+RTMkF za=rk7wBI&omG+>|BZksR;h$ZwbnB!EpJw@X4P4tY?4fc}PVHF7%@4V*L#dB3EP z=XVV2?L|A4)yJI74(SmIbuue|AM*-13!y{T6Oe4@K^;Q{I)P)uMvqu;+H*8O?w=vb5^CAmxxz*+4w!evpb7>>rYUf5dCWo$6Vx zyH3_~2l~R0ajR$(9SA4fjqcEnywDH->c1NilAq_)Q)JQWIYQt46RwivvRXovv z)8jY% zh_pbUE!nG8y#jFliF)EUp4T@{+OPG2H(fV`yQ={_(Zv>KTTXt~G#2+2PsPf}!A6{C z;ft_0WBFn}0a3vOa4U~(^Wr90ygrP*iW|F3*Rk1X04}&SWj&F9B0SlH{UlY)OqR)f z*dfE3j>8LpRPw>AJ-cWqKFxo%(V3^OsDsW!> z3Sv_F^5Dm!0OF{R&xr-DJ@z!Vwc20VrojdKgeKoaDT^3?^2?clF#C;}Ru&N`jDaUu zEmC~SjiEDtw;7m7SDdG>4HCw~fytB3;oA%!*G|siE!HWre}|KH(u8(JBeLx0irJ8s zM{E7H6Bz=rVW#t^{J^3Akt-nQL0^Zzph3)qk|3N|We0xZ636l7peOyi_gCDOCI_{E z_I2*JE6U1$)SJM!SoYHRLzot3%5srZ3TdqI)japrL1Dv3=q*6#p2D#^yLWiyli=fM z5o*B%=jDmG;%|^uVBdLOVs{BxSkOVV!8SnfHk9TJ6h`qv?QV#jI(@7t_1U6}F?=1; zJiVP|foY~d8{$21s$bNyZ7z7zO)G9F+aU;1$H5JM)HUfG#4kA@2+cIj+LET(a;%~g zEWtJNI$`;-IqH#fLOx#-L>!Eed*)qdwh;nO8I}kV!5}Eaw#w_e)DK25I|5#0@5WI7 zqAgQZ918@zqk1DyTwYzfF$F&ntU5Opqk?fod66%kHRcd(9z^aF^($B!l{ONSiV&!T zHA{+r6k5^4ee=Owum*@O{4G2NQf92wd@>mC7}6NW%<+8(yij@Zu$eh=2;;1+>N=4l zWJ~^<4*1qZ0UyWyoLOs5gZ7DE2X}v9wsbKRgq<)F%Sc zrWuI7aQ+&Ca}I!P7d#L z9?;X)tC}TsYJ7K4KULoEVo*P|OL>)gvhGLffS-cdbz$aw&&QZ8qsYN`J13_p3XFfPFl{hA;2@X(&0j80p zp5RjTn=zB7Ij9khv5tuw)$l?)Ie@z+;T2SO57N+JFm1m5ne^b;q*7+o&(qTx-3@1y zn9jB@xR8oyFH_imm^RJ!ZoB1wE+l&~k>VV?)Y!PGUyl8J*<^&vd8alwG-&u*&qFqq`G--BUgt-_s3x zj4Ia=P+a{73E8cj8T1%lI(bKT79|Wk>RUZc!lP&WfG%wQG3a*@+CM;a=CVFf`Qgkv}v_w{s3 zSILlpKi$j#(bUiQO@Lf~#qH)jcmnxRziiW#puu7>aVy*OpQ_Kehpa#;-7_YuHR(1e zIe9lFZ9#oMB?kd3_n77zW7j;kb`F^pGba~8HxfOqj=S`rZskHtf<4V53%fcHoxC^j z(bsT#sS8lI*|r|j;cwYDU!vIJwuH9@ZJiAA7`EGgJg;wI-f^UhgW8}1 z$jNHn&7lfXPhwhFBV;WPyr>3O{?tp|iY6PMQ^cQ={xJr%=m$vE{Ku3)EN8^DHE_UI za4^eDh8m>rVcj9A9f;c#_)ZOwUG@Yx9E*~GYpUQcw9f!gaGiTF+n`G1}J^4nOh)S6!BWOOTZslFnG7<69)F5E2;~UaSxu^%cRuAr--OX;z_uUOWl=c`}qN zfl|=;ZO7$8)*Mc)E6*aa3{dCG0O5^E_W;Yyx8-6$GGgZ-4S9FZeGv*J9R76~*A}O{ z75=DFYEZZ3ifYBZD+brFDM;W~;HjB{@2S*<3$VWf)D`c4)Tm2%rRCCkkGoFQ|H(Ef z?i2^xr>(oim-I=)gW`9zXd2d%zUZ%!nkoGw&Wa+)%x}v0&=^<0FU`Ehx?Cgq((hz~ zfFjwv`nTOo`}-ZmX=qzRDYMSpSZ}I)3b1Jp&pPIt40nrY$}^nao66O{(2OibbZP7~ z5hQonXDF$EKBuiI<~=EiQ#eU3;(Wd5%u#!|YlW;ixyeVOZN_&?Sk6e!%4eHwW3PHT z3@e3IK}iUb3lM@{2bNq+@i|0G7)HVCwU<$hJEm2Gm&H*dUaG0d2_*%#Xzi)iah=40 z!CPjGY@&}mDf}2Pfs6bw-S1>n1qZDXHDW9%lEehZy*BWPDv;9T9Kz~lLu5BmXlwvfji3GPv5 zuim3eL2hK3W?p3c&57smDyx|<>49q>CO$rTUHExpfIi`!qn5AVhtpLl(m%BC3&NLVO{fQ z;;$9-fS}5Mo`Yj8@dA^w*lOGdFX0$}@gT-SG^^mUKYeLR;-?3AB|_@WIzC8M3ME(( zC2jB)_%#ZG(*0!2PLrKnAVP-fJ_%T`NXC;{l)AF%8u2#OjfGC`7%t8JLi2rl`rN`$ zV@`P#BtSdQcz0q$6GYIwZQ8E+no-!kUjtBjqTer}-|isM66{@BekOfM6@2`E6>a__ zZc87ILcZ}Z#L}iGXYTt`fm<$L1v?w<((?E7?BuGA9EmEh$P1+2GB!AE@;kn3FP;wj zj<-UQ1z^bA*(fs-sm6vfeu7p__O8y^a+jvYEG%|niS*4M`pY77&6^^!9Nv-!rc*-; z>ayBwJ;)QPUrLg(3qcyGoQH0IFh*g~g+K%T@Tqn@P?Fx>4x}52X{5b^dKCznAE)BL zKFSXy@c5{tmQS4%`E`GlCX&Ea(-*EMibmucBqkj%JSFxQ^7_{4RY%n6w76I=AQ-~C zs-u@{ME|LA;iO6DKtnCmav!sC8&5+u3H(`^ZJrjuTqkL$VR(5n)LcD((L&>x9JLB@ zl*xP!9sP*1?|%#vIc`ia&uc?0K+}joPzqaFCQXyj-)mY3sA}T}Dc3*tG%yEmc$CoO z?ORIyF-Z{5JzAp^oN_fNslC3luoS{F$W$Y`MMBb{SR^v4KFAf{dJ7mG$tvEoraLEw zRh*p%1bm}9fomi2schGO#V9#F+@Y@8AODOJjlTzdP3?EwMFX?I`sJTBz5Y~UReeTe z2wjp`3&VYgk^Nkr#d`ZHEmxOs15)O;NLH>H5P-4F-LK z3w#&JqI=y`Uv`hAh~8?r4y0xVIa-N02K(b`rU~ed70=5k{snY@U|Wy~Qlg^7MPudz zE5qY9qcwfNi2$9^g(mwbZ%fYVKTGEp+1%T}Cds8^(l`Y7oiiI@+tzgeW1+D6J|T$f z!GEO6s|!W?(Gr3LJFqc8bME0)=f$^^|9EVun4WxeuQHjvL)h`4m7N*qNSSkrqmOoP ziu3gOxA9|ce{LjyQ59O+r}25+oY%>B{{#>zsxjDeQM}|uV@WEDxzHFZspG&{E1si1 ztSYaJ8^p(bbi_d661Q7ZzVFAk+0LD1YANKD-SO9E==Ga8ZN)^1^rh4;?Tkq4^@Uea5duhR^hhX5Dj^&YyManc9&P`jZ zMCRR4Rl+!wS8t<^aK)srSi53rGbl`sw%k|)x}4=n8}sF(G_0>}Y2$tbdyy98>Lhon z-9{2Y`Q`|J=_Eg+_&9ubg4fzM*L9;AIN$wA!aNwC;Gy}RTeF-AI?Di3YrG@zbh8K; z5qt^uVVFgKuMJ{j&VRk8=;M2NQ53;h^n55N6$wC`IjkNOyg({Vyi;72!KuT=vL40v z`YWsqcXCx)5{2nC#}{;6THEf{a^zEJo5z`H?F&wS+g#FF1@~d`3{i-!hf_qG{{n;O z)fv_E2b7spyS9em1-f`6kj8@h;>AyuOKKA=+wh&*3Hb5X>=^ms9*|H}LHmxzE;#8) z;5C`bI!&m;xxbZxPjI0;1BP(IIqd{*j&w-*5C)%BK2E^5un$}FYU78! zXKzb?HL8DWMli237+@s)Sg+Y_Qp1&qB3bl<;qi6A=vHwrdYK=UGxGGwBvc#3Esf-4 zurd=ipC9%ka6DvO<@A9@*yb6#h$aQVn8#N4!^hA7B;^2p!JIP&W~93MJqIxyZFn`i z{#|Ox>*t2s^`%jKd4T|AT-F%LY&=exbsM*T)Vz+pDUu1q`c0keu`YSz<1sm=Z*u-k z8c14?(Lk!{V6fOBU&uor$r|`Fi80Yx@EEv25wkXlt~^JyAebdx>rE57bfgd?*xn9) zSWLg#VhCcB#fP=UZ5q#C>auniizx;AT=~Wdw(is{2jnw{B4uQ7!UCI*>y#I1NiNc zOX&Fu$7Q|>zv&Ci%l4>wYutYV0@orQ>VK1JoSlQ&Itb^`x~ea z2jOFRF%7pAX9yc!ST%q`MSfNoM_gck^4IlO>Ooj+8Zi!%Q$cb{A%2W7*PiQn?GIRf zIbJijZXRFbUoD49A;-)gZy27La`hpN;kToYNg3^53;j!lRJo>cUI*a&3p zF*j{sNpwqk8e}O0K>G3}0*9H$U_|05%nypKTy9i5qNF#S4EXh(Y zui@yDAn_glp?SNZHcW9f{3^$Yzo6A^Z2$Ik#h^X&!`1Ts1Oh;L4zR39|51$I4Ev;* zlycU%GA!RPZA?14`-5k$`ab-Bw(m&XJ~Y>X#;W?##;I3i21IxXMY$FWd>GQ(I(y5W zgqZQyR}2rMc_U*70~n_eLv~O9TLvZjISM9l7Fn<-;18}3D(Mk{Z|C(tfQp;4%_~+2 zHk}+$^EKc5FbQshmy-&jfR_!0Tk_4I@c14a&v z^m~~3FmcUWX5qRql_|`B?^JtNe(93v_DNQ0`hkF78L2SjMjIr+dqz=%(lbXf0fW-_ znb;7szJa!A#~1|+-0X$T1{Y%IK${{igHr4|>eLOqI=)ijKZB4pTj=OdC!inDW#zS3 z0h}QZ^E*ah<(Ggwm+9O>I&fdb6$BIYKQRfrMD%GCH~cqZY5~lD85Rmmk)Gy|8fPH# znmQKX)tY_R)EzHa?FGlZM!U72a#np^N$IT$>!!5OgKaOkKrEC;TMRPQMB-u>n9zIA zj=YcH(hDukro(aCZ8ZB+_xAHX4&!|ZRsF2#7-CEhRf9r+*y$u2;-hMXZ(ct1szRpU zPn>Z~>+9Bjv{!OWXuv{2-62iK$GB)H|te zdS$LmN-L_B5e;Z!ls-|*Ujq*mSv#KYFdhl=0qPZsFPYhpV{0THywrCkAp}0f=X@Wu zD8&_ClII}g3@n2XC@;Dcw3Y0=>;Y(QrNQL#Fv*CMF~m!McmIVl?QQ;ZgBXk^>Q@hU zL{#JnqI%m5Q6rihf{GN|g46`DP$m7Y>DsT`H$F5jS}3C!pA}xdm~oYoLf?KXhYxjr zjO!Q^k8ty~4kW9e+4fUhc$sm5o21`uA~Ty>luac* z;=jeZs9valWo5jErt)cWJudQ#6C~=7GUs}>i|sa_KO{Q^LA(B1*jPgRlP7^=q)GuC zuy(2wGVfee)^j;^wyn*TP#kYCC9@`aeqw$97L9ReObTH@ zN0|jk@LM7bA6*N`v{9Bs)5}Qsy5!VZ`VB#GNFHK;b}<{?9btSYU|(oz>j9t)s=5yd zKNQ)0Js7|Tw)n8lR~?Mh4Wjg6NOTJMrU8*7VzUH@lWOi zw)yh&j5rAw#Lok5jFLDt^FmZiX`5Bf?IA(yeUjfYhC`Z2m(~sTpWNC@beD3yS1r?j zgXYFmW{8`)lhgRi+hA>T<3rXnY{!C8yyo_`c+iK;S{^FS&y0iV`*oJ-!ipAKH!&Bx zJ^iCyoeEP0KD%v82)I_ZsZHb|7dB7UXWs4a@&o1y>wupqaYFmdX?Syzur(+M##f*^ zYA9pVLdf={?Zvh$-3d35KGSQ9p~Z=RQM&V~MD+F$KP^%tfP`eec97%q`n8*zHUM0} zc`HUZd%!rilyYcrI-#F^NAgCTmLy1Zj*y+rT?K+}kK24(Q=(b-LB|%{^R<4)zB{mU zrs1(WTK9DxrYn@}IF@NCq`Rd>m$5*OkkAl<{p!y%6uKKnmTXA@z2NQG`@$N3xqU;8 zj-wpFyLnPz+zsWFzFUaiOL$b&gH5GrNwB_BB3*&ZJgwp zh2(cTtH*1aCh;nQ0S4CD)ac@Wigfu|Cm$}sGv;$i$H7ld(KfhFDbp|V(c^-eskJQS zERN6*bh@8P|PI#kkVJ2Pf`(1Tv zn$y+7;l&y2aq|xDkd&Lor#o^@O_4)AbbwvX0}xMuCOR-xNtG8G7xF`Y7z`o%%APD2 zGvUq}({zNwC>JXyL6GX_6W64=axFcXAoz6^7o9&!FWh*>wGB$d4Ct-1r0dly8OObu zBD1}uobY2r97WG1DJ0ac#lZ3@>1J9$CQ);wX(9^}bMF#%pC_CIU%x;f3+?0)g{4N? zKl{9T{)YynT!iMybhk=><60*7SG(Q|jqT%88~4_xF@>5k#l1OB(`wj*N>5So$C?Ks zGh$V0hqq%XZ9og+XFm`pf{3UxBrFp&#qq_7}9xv_WH`z=+MoUmIB$R zTN>Vax2p!K`An{ws)56#7Zy;J54eX9kQ;170UFeOSe7pSy!ux)_8b`v3bB?XF=yOX z^(X#dDXG^SWxnK8`!hWxurK6Yb)UcQgGL4P;;79VK<917pNn?gF4x4kb3tylfxLaX zpYo_ie>M_Db0oWe2X1_oN!fJaw~%C?q4F2U*J)K%{5aE)6!A5rO%-)&oh{)UoFcF`iY~4xzQ3D*L4E@zf7m&@ zf-zL%Z9)79tF^mFziasSl)OU@1ua7rE7#_IjCg@|S5G2EtV|&0i&4U$=qf{)Qcr_C zs*hO#8`@WYDo32+!gFq=-Cgl>zoOQM$$FU^&mpFjA*#PxI+A2db&K*ChT+p!Qtu#v z3j_ubHjGUfQ{*MD#ATjq949M|dJkecO7?e*+yHPYRdKU`8MA|0H__lW!mXa-)89

    `$92=V5yU@#7S_F`^DV(=dOkKPDM( zYOlK}nDZ0eNUj*_m`XaV9xF@xF_l_^uLE9Z?1=TOp`YtHXyDGWz%XgkLvu_8~v5UFc zFkymcxu$`2+R-JGOTUJ7`x;*~^+W2>v#=fG3Z097MXTZ%9+230+#i2TpoG`$7!oCa zo}oQSdnsfUOY|XIiw-V}9d-Cga?)Lv3De-7d0R|4rBD@Otp5a?WzeF}&w`1=vH2~L zMO1q6;w#}2rAfM%?Sv09TXyZ3@TZ$NtL0lngriFvH@R`ts1 zjkwW~Vq5eguS}V;i@GkzOwBKU9vfRwf6YozuEXrR7p$pK;&gEbOPN?}sF`1Qxia#e z(yWV>0>CluZb%>Wc0oO2FwqSUR>xx#!UR+)Z{yNXf~N*@sIcbnSd@xSg)_0QNK{#2 zrWE$!BlYS6v(BdAXu@&_(4^o4q2c3pi~Yjh2Va;08N);f#-9*x)F-xo#+p7B@09wA zm_lzkQ}|AS{SR=#=x%MT=Xzzd?}JyS`m^V6~(N zURm>Rk{`avV1V*e8dlb=7?I`B)YkX~ASec<+36oRf1=D}Av&SQ2`V43n!+f5Z2#9!Z~*gpw7BM~ ztzz7SHLqzD{A(|#?Km2cNt$Ixky5v*6f_@yXf_>4id}c@gJvk?s-Cln!??7V! z&BZ4h$737KbeI{&SLt>szt42n!h0%^s(Ry&*|D9l&o!A?wwv>I`E}G41GI<@WczGk zsDM+gSdXXrE2*|(37yBvbAxmuse!?js*rMD*!%WenVk857TWLa{J@C=44IJIhIZgz zQB4`Jgz%A)c}2$X=VptI^=2OcZK`7^<#_Wi(-E9?9Ud)@lxi0BQcDln?&fhK1{-d} z+M+WN)dRzhIDCVJ#tFc=u4#6+w27k4J&<$Psk;M)W2TA7GEJ3KRHXY>*CsN8aI0+H zySBhpD+wQeN`kiFOVftLkZ>o?XmZgW{jOHrYCu;mA}7E>n|UtV=F2vkg}W0GHPRsF zuv@3czB>UDSU5IjFCVO|h9pJ$kjAD$dXJV6mGXlhIf!{W9YtSkFx8q3esmu81^5O(h?v=b zi3E7Iw|QY2Z0yFqCV@@tYwI`2-{3%N(k!;>X&LZlC}SUOo3P}fS(K*V4||x;4`8`R zgoj%2LABEH`m#k@PdcalNxzv4$VU`~rGo!bSa1&g&)9& zOB~7>8=K@8=?MZXlvzO6eN?vvuT^`0H;dEC0H~XW4UQo$B9L=%mjRaLPJHK)9&Dm9 zsQ~jw<8o!^&bAHYtb~G9Yd8jj`G*8(UbjfxDvHtGJ^#XFzN=x1WUW6 z-kuhf-bOv@;KYZ_F`+|&KuFDUFH&yIx$hLcBw|zPDS!DTH+cV0Uw}3B1TJ1`IdVoN zR(Y~^|0;AG7TMuga1D&sd74ar6m~B{5gB4#KBDo=%v68mF`xMD@7Pi_R_O+{T-#n?qdGU&yyB;Y zK~)lEYfLTjWNs_y4<|vIrYOH)`fchkJfu6CKJcO2%4;s;zFhLrLgcXg1{{jGrJTAIb@c17a@y@ zS=4krmB$slY4EyTwU~imZhNI_{C39)3n%9JDJL(A1Ka{*+F3mV?FHuG$t6Hd?`W{V zBW_R0SAk$^S& zGpq3{foP&ETt)b@664=fp6CQ6KZbiQ7_2ArQC}^8kR0sreSCZj-C)2&Y@1of^|g&9 z(_q7gl>s6TS*Oi^-y~I;SYG{?Jooi`mVyl6DRJe+j7y^26K5SXw!F|*>FiS6eSG~M z5tUO9KvBpNYP|?eU%R-R5U+HL_x1+N6}U)Js$@@~|>W$PS zTkr9!=^Z9VFaBInqD;L#GeU`4ViNxWgZTRP+R_p!)_gC2CRHm)Q!WlSr-G;S9c5I#Vs$+u1k%iX^Afz`Bs50Tjd^cuA}WO#s(X3@gDl%Em6 z=JHPUZ=YfqX4ZVKPQ!S<3A9c00w%M&tg;{quj9qW0aJxQG1%k_3}9rrERO8_b|e#q z^J-v{g8p z@p~I7mo3fw)p9g%VpC{8I!{ZfE~V^flg%U6)xLDMwk`a_Pypx(OA+(13)z)Ge;BNR z^-7$>k8yD%*Tk+ffOvpmcsz0%E?r5oh+I(FPC*gh-6^pRY_oy1-O!{5#O95(cqz+& ztZ})Rn!4Nlcaz^&Qlhs9n*}G^ny3=KJD$~V%?zXui23O9RmU*x_45AG%w|$2No|ow zGGB+&9fFPGOQ3xV9JX%4%mgmi#y+$AjWn>EL$YV36f|_aB>9=&_9Z`yzU(7{HfFCL zy*>7Jh|pRTB%M-vjq+WAZ_@+QE2aN`Qw<!$G=bO$Bnd%tEhb7u-a0?SS2vk z;Zw3`YEgZbs` z=E(M4Kb&!^n=_x>`qLxkHhD)e?;7{`hq0%JGlgjOWLKL6WcSm@Sw*)G&*n#qBNJ_I z=86v%$8P0`VE=2~K9_td>6)p3mRcn(o)whtI!m5QmDxMbn)j3KU*Wk(C%1jMHIyVn zK{c}=rN~B2yn?6ree?4qRVnkw)$XTiuw6dy8GApjq}> zO>>3i#|^KR=F9ueyEgRe#BG;$8=K=Dh(=?(&_05$Ks&_2qMr1p^%A&$4m;QWf{4M- zcg^#f;oMKeOOsLMul;F&@j2hyIj5=qV4074v&oUV%QExb(blgnOJ*?Y8&<2>iJ0Fu z3#m!)yK7rusPltCyJP^V!)1HPfwv=H`=P*?2X==wnOxv|#|>Gxmn0`III&kiXTrHl z(0>lR@FQf`E(;5B;9ahNwTLUqE&qZ;VJmDNfYmpi71;mDbS}Y>L#0Os41L*yv zqMjvz`T2BnNJ{^kpH4@tFX4wCmm)W5-4Q~HD*?s1_=|5w_&0k0V%sc&JiaN4b{Lxh zYUckLePn>)^a}&*|E9pd>%RoRkO}ck=gSRT%A4%MS4J`HF6{$cIynv`X_F2TPt(MR23 zM~I4FGeT(MNQz>Pb5}d_kBN4zx{T9_sl^>Sqf^Ad#}Cr+ADR! z?xR_#50~!oIFtV3=6=vB8D9^gJ>YHcc+8lz#jqYVS6^$!kNH4vO`INfpTwHV^Qnn; zz1_(opSc^h*C<;z*NqpFveu+UNcH+uJ?=f5WpLZ)hHJ;KbsFn@nm4ZG;wL57?$3sxsbL5SIJ0G0BWTd5svEO);$-94|qmMOg5_9Z| zE8p+5E49eLdYx{=dSvuo)%mzT)5M)W8!@tBYA1I$4;P{3%`S&VoG#Af^cV@0+pl-; zvh4*My25^X_x6Xa-gtN47~6C4X)Ytp&hpn)9^L_eHr&zRG#P#pf4`0TnReZu>bo&_ zCp;$4BMiy-{jnf6hn3lXr80Z;m^@MIb)75@5A%5GO;4wzcIIAf`zNjc4nEC$Zuy_r z%aI3m?@K?)*40h|fPLO2LsTRd>h;qer^!8XZaB(`fLcGty@&kWF*pIO_Pv<%v?5D+R8%hMNZQIBzt+4sfu$_yc0qv6U(v7ZrM zpPJolFEMCd-!?pYxfZX&L?QtWK8}Nh1oe7FJ6NVSLOAl#wtntLli~P%8sC!Fb#z|r zx3A$;{Uo(F9oGBJY+McOK_4CPkore*;&(BBr)#Yk+mjvcxioQK6Y=(4_ddaRonKaY z)z8Fie)orx9gOl&pL~b_NkF#0OB@yVeH~xfldqin5)Vba*u|@ta~*1dKH=n4I;NII zNpjS%pXh|<4mr4Yl3bAs5I&%CAGCQq=z8)X!S%ro%Llc4S4f;IJj^R>2UoT@T*=3H z)3|-Zf7L_N{JHI&`h)8>#=DifUhRe}?On8=Vt>0(TYU@tpgo%A*dM%(k9+W_oX2y! zd(I}$Q19h_DuxMTmwZJEcgq%3n_nMWakaL++)Bp#9e%x>&rlnZN%T0~Pi16T@#f4o zdK`=k8O5WRr%Xi0N<{0ERu`DnLB#6Dt?GDNe;M;dWz4Nq4mElE6j}b*?Kk)_Lc?NN zzktOLKVB?Wi`$BB?58+6cEg8Hx2orfv(wW-jqYN;H=6$E)4qC}lY8sW`-5m2&!W*_ zdkV(A#i>tOe`Y(!`s0lkRX-a>jp_6~-LBr}V;K6nJxwp8SGyQaiLn>gRWE*QoZBM3 zf8LgLdr7KoYOD=6S;Fh4FT?cWc#@V=+sO9UcmHzr@8{9@F&OVxlWox3Wydv|)%R?= zy{ttP^w+2#96#}Uq%E$4V0c zgKU$!`#Ly+=N}kQ(6z!%LB;Q*TD=aFf1xIq_U9$gTQt3NC!IIZb!)=EdzZz7Aq3+663AgYyT0dO!Ujjy|XI989A_GW-mm z=h>XyV2aPR@#!g{qU}$E{k(h~L$q5JC1k;Fe+voSMCI}-%FM@S(H57*x)@*0e-rQD zo6m}_pVdxghNl&)x%rN(`LkVK=aFt0MV3r+8#U>xa@^ zY0KgI92q+&sj9wVlmpRfhXp>f;o!XT-zVex+8$8t?}Xpq%W4sC+D}xM=ixe=tbMPK zPsS+eZC`$S-B&Oizl!z4``FfUe>L(DklQs1R$eiAOkrGKdQCmn9z~C!SB;EZA1B8G z_K7!pm*#UYA=^w45nn2OkKX0sT*v!5+v=a;ID60P$!ofw(33t{nD)8n>N(Ki#eyu; z$cA(Df~$W3X6N!zih|B*QO}Rn$M)Zs{4)ygQ1ftk$=pepTuW!9+50mDf93YMeQ#Q? ze;F^rcDc5qm=KS>w~K9Z*OK99Z!98zhc|M&23rgtW8b@b`eeJw6Q_@1Js1Ue{7l>V z>mF7p+wgOX2D8BROuR>*x*`az+Lz17BL`oQ5xfjfTt*~++NiO70}$x3H+?b6;TH$$Bk6adCbSecJBJ!$2QCW=^ns$0mm5`p&n` zh|`n{o$vX0>E4c0|KgN;E$~?w@czw_duLMhhW%kWjPJ!}l0F`ve+LQJx(b4%uh6zhiy4L&R;y;epvbbD-BkpPq8&gxBiTChbz+aAzKq%nh7*ba0N- z`}Q*BlI&=ehdnQOtj-qM@1DCd%+qXqY{$zmOJ}wdzt2UMH|0k5X2X4U7kqoPCX2Jj z>9z`-?W$+>=g+I`f6-@l&z@en99^&@PxLxVN6n>pHVdowyfY(Qk!vsxhO3KtKCa^8 z)QqafWPKQKn%G!a+U3}<&u$oAohafxIGs#!TH&?-YW(x3UFInx`oTK&A}c12ymXS7GmHYhS?0ti^`F*xsvh%y_`DP5!;(fiImr3Ez^UvIXZGY<>^RXWk zRT^db{8l`iV18=1lx)VH>)*or{D$w7+a;QB%J=7dOM6aM3U0X0YDm#t80% zUF}Y%f5+JBZJxB)M4R_$J$CHVXGNU-bMw5TzV0;U$okaR`+_GZ8O@UY@HJ}=k{&Om zV>2t;?B!_{4YENy8Aq#6WDFl7f2B-qRkbWx#MArhuGium-*?kkgZ(ltBfIeY@r^x_ z^9UvH?4sEhaKbs}>3r0BgSc6{`PB5Ceeisae*;ac6TPIPty={1wR7BCcz+*wmqeKF z*~#4u^U=qv`n#`^|MK@br0kHvQl;vCn5xZa4GAPWUr7Tg>28aAcc8n0ix!z!9az^*z{eD^HVX-o2>!Nkf zwx@~fu55iz8yD*dX%^3gXH+}RD<&s0xnqg%-fQT3=n>qdQF=oQ&CAtr@CX)>`L5di z?n7RL-FD7t`67qg@R7FmQPj_3zSLdIe@m;8ONYnB?4BF@hqt!319D8R`?z=Zu2CF% zy>yal%Vq#C%P&%^mo*#BXU|QnryIij{R8aRyc{~!dz$p3v44tYuzB_#wr|P6*K~B5 z%v|RcPuD*?W9 zj8x`wHd0>{<<3yD@z?YI2*58{kJkA%ApWgyC)@;Ml%qBM_D@$Nbc>hC6U9_r-eChFRUU~bE=}qR$U@hy-LpiHwy-{;v)^fm){{8Z4p3l{KwK5ZXkkn!L zc)v?LH3v-VmARR(KLbn78sUYjI2}%c&E)8cz!#QPot*i{g8p;lo|dxCCds`$@15|F z53U=N+a$sB=p(8=tftTFe{G~`(Kv)^XU*sneM-}dN1OL*rblBmYWLQ0|LlGAjp591 z8+=C1?XY)8#u<)D^&ooDtgrGi=GSs`9^K=?Es9Kga2j~G{<}AQ2IKK7XpALF0(+xZ zz4~=}RncO1o=)3m!_VZlZLQPzT(73Za(YkNk9n-CcB2p1N9G!}f5w&aU0Mm+)7c{X6dM=8mj~{q1nW zd_&vjb4@hPgPpxzI55}keX5haomb_=n~WbQpX+>k0uir#=7;a6k&T7?Y{blk<>Szw zhNll1J>C%PcIliLrfXJI~KFfkq?WsR~Gs#^G{#wWHzidhCPL zC*uX2&*$g$jlAXZz1G-FpNyw+v3Nd&@nNn@0i6EJ8Y+_N}-2 z>pI#$7TY(?+0&Wy#KR4%^6^k#HhI7AhUdrR@JXz!*jvtHdmU%H_3D)jU$<$`D96h- zpS<^P{dl`fe~#e@53us+;GWO$jx!tbOU%#p19J z@V?A8#x2Kbi0#2_sXjDs8f@#+>9ZOx>E+g2PMWpWuj?e6AJ3DMcerWuXB&mr=Y#u$ zd#z8JAX`6-NnTG3bY96wR`=d*k{+J#P1e5+Mp0@H^uaZv>#0?)U&~x3Y;A4AAx3A- zYRz2gf13L;Gi^CBOi@$5#e;1%bjWe+&YG(Xkj<{PG!b#Cy}TQ0}r<#0B7^gg%Odm4u$ ze{b|WnC*Ks9PeUOPtNPvyYIYq+rxgbL~j;PMi!Z!Zq@Fc(bG&Xt0R!WyDrT0%5$Af zql@nUzk2bctd4k|%n!BYLk{ZyYxI~71y5#(HhZ?Vb>wT_3AoaW?S5Z^|KI+vQ~A8^ zUjOyK$ntdhFaPbozn_=?df2^o|M#i9f4%yz*XjS+@2cZO|L;Fa^Rg3+B@)qqeOR%B zFxLRQOoU ztV5QLerGp^o_9BuQ~YwhuA}W+sf|ks;2DpdwaT{UEa#E{c)WXa-Zd+?YeE7ke|+`& z%Wmcql>1Gf58wRE1i-*hPvTGZ*8H4o?6H5)0X$n*N8;-sJKBKWV!8n^f?;<)dME3_ zWwU?HY5_3$sJ9x()S9;kw&nJN11#qS0627mn(YQ%0(X)E7!O;WuXG#%jk?(YT>&sa z5`fjDC1Kixv;^gAm(TX76&5+cXN3~aZ&adP{Q)+ zVnTYRd3yEQq-E!1$l7G&Mf~P8Y=uN>nE_v$b+R!#7c&HsI3P^ZL2!!eq>&5xtU_p;3fBOZoRmC9V=W0Oiz`OSw zdBM3iXlrH&hADEoVgxJU)G-pEZwx-OZ7^5;szBH% z@3p_Vs3zN5yad0!nTNWe+P-`xen3*2hKCbqdN*rt{pQM zxQc~fHB7c!CUqMEuqlT>cv=bf1(pOM@DALh!_)HqUS*@#Clo~CY2=+Nz01{>shm~U zv%CH%XZi2-Vr`<#75}6B?AFI@lM!v`N}W#vQ0K?2Wml(30s`bG*SfW^4MAg5QN#0h z%~7kce^ss9Hy16rd@7s+K~VSmUvG2zx7+`t$N&CgjxC@DaB~=%hf^>tkIV7_j^LI7 z!bX`MbmwN|h(t;aj~h5D#t0||LCG>IRQ@6wj{_`p%c<%2^#xfhGec$e-dLsyIk~y<|$k@{OP;@o-r&7j5Bb7 z&k*sa`f)o8AY&t_jAVvzqIqF*bsCyGIs3^W*!;xH ze>eZS8mf*-y1I#Ey6Uj@9BZuo#o**j&-{B#9dEk&aQ>6~U5=ZAXtZImnZ;c!srScP zI{1V6A|B8>U7L~(R6KR>p=KjhW0-XMHm4Lm({ukdGk_-ycHmkW@z z1d$p)xx4_8MSdVPmve?B@pfp&h^UB)e*;|8HNjg4D@^5*EFzw|gCw;@feDY@{#6@z zJi#Cm^;>@~k^4sBEWyPY@Ix3#aD;#m47vfFE{G(;)0`+WJ;Tch zBC^83nTKVpe%o+anI^;p%fiI@*ub1)RFCz8+}d^sYs|y_Rd8ovkR_!{jI-DVf5Ghq zl9_`jg_L2CxEY;)!Z_pFBWyj zDlW)X40Sxu@OW2my7vjYGK^xNz^R3!cnd;Xz+;UZsqli>Sj2*e4g*38v8(DlmJHj1 z+8_x+oG-CVSQPP;g#|GnMObqPQtzLaCJ1UaQ9S5s!?m!~Il$u%tCr`)eYsYgE3Rm@}Cf30Z`o^i z4a7>L1R3-R5&hS?-EsN9e=s9i5tOc3OBHUY`_+1@SWq>8f>6>j6%Ux@sP5MK8&hB0 z>0-ufS)(ukI*j_Q|H;N|`OOhT)%~?urv?&f(_u~w6L~ zoV@90vRsK6p}Ke6n8Zb~KyRsUjDRIbD-lTQnYmb~_%nFun;=ZM(-cEVP;d$3x-bz! z+z$n%>WcvV>Ba_fKw8y9t>PuA`;8H{BT{rZh0KU z>G*Y3yQRe{fupAQf5Q=fbbX1ai85+(02~(T4u6Ys1c8 z4&t3v?#407;<$^WmTuT@ZTZ6&|J1J;oRb;kiRKyc)ydO8fBl5{Z#9Nry)X%gk!oHZ z|0{lI&N=?pA0tU&T+bZfo+09dB8uTe3Dj)=tu2*{oYy}e=iDH5l_0Kr2nVxc6B8sP zLb5b}&PRx|$f`Luc!h6vbwThS)vi|TyIQk-Awdul*6N>se|dp%4gUDnI4-@Bhi;fB~tXGsmj##1gdt`CxlVO5pTg<1I%yT_T=ZB!G=A zl16b0v!z|n`LD6W&Pn5oc;sb-JB@CsxcF0hP&)J)f3`Mj9-EmYDmH)mP8CZkrZf-D zj52`>0&R(c!Z2j)B1jDc?8`kRH<08m;xohK)Wg!vPEGQ&?v)90W@5o@bsTUNBaDDd z2^O*c+e3~7vG+3Vo(~OYcK5swb!}tM0FJuzWNabW_a^=60|{;+&Ttnm0W7_5z}gS1RDRuWO%+>8+Q zzauCdtLAIOZv(GQjIM{~8nSpgF}fPP8PJTZEksoBxBc51U|ch^|C#pskh~W0g_3|T zF-rE%Sr4&D7gXNOnIIAiVS`W;mlA>C#2~HgzX&nv*nzsXFmyZ&6Y%G^S~pFY?M$wz ze?G*aihm&`*fU4v!4dEhoEbc1$G`H@dGpLJ`WEl_m&UrDDph!=F!;Cs{SR~D%c1zt z2Y&rmob>*E9am3W18EWbH|{m*dq;xr}x?ITegclDVx z38EtD>VlcYD>%ZIaMg9igvhwVT@$+ee@g!iS2@8X2UF>122{q)mc=(;#hyq;U|nsO zfsRNhA}pO=QtJc-tXSf#^tA)M@@i>_iWov8_Vr3Ye9C`fttmk$zE$0k(gg%vnDr_& zPtylPC4#ClNKFHf2{GiULOc$Tb!op?`@hTQzVU`+$qE|WEF(QYz;_FXF#+w{f5-4N zaI^8nH&^SI(3T*GVSuJ$)`Ids!&-#{(lvR3am|CKkn zy+~XLB*J3R)BDqt(n|kh9>3fie?vwLuAAnsE+Art3_3)`OMT6~*)L1&r_URDqU-Z8 ztmJ~_Md>bqBJN}O>cR+tQ249x0a4?CW37&On`98MMYk@zGXJX6ePygg^S4R~W&pM6fTU*6n+{zdHyM`GFK+-S&mh5VxpX2}8=k<`$ z@vtEYKxsUAhdV%ixP5w~-}POSEFF5XTiXEC>({(x>X?j#sxCm7Ub}w3o?*Sy;A|^t zNsVBuZ30OUuPAA_t#~^zf7tTNwYXEa@kios^4d8#(Utj_~Vy80X}<|`1Yx5r4j)wf35 zy#=@L2b&X+Do+!NGrq*phxb3T5AZ*{%R-WyAeCN3w?riBd5r#Rf8QGbu?^v zpGs||VjTvl-V$O%#TsNqhy+))BrJCbk~{ty@b=z;i2z{C^?dUc(h04sn_~be-A;b_!zPzS;1<(p43%H zM5XYTphbe%yb%?4oG^GjqFZg6xheV|N1RB! zhA>&0XHiCBe~~Q_pO~h;$9$Ic9|sCb_ebyef@UTNkHW6@P|%B3dGa=ZV9*3U#3eGK zI{(ni(a!E}cy(v_MLYVEmf?K4k2D$~kz}{C$v3eT^{f7JB^e>Bn^CWl0A#Sz4(LEu zd+TyZt|pHSWTxHq>xQ706hAr4z^wkJJ9w{TFy2N%fBDbr_w5sv2Jk=QMD2grJ2oAr zMGK*L3-5Gr?`IB0vskUqm*Xnz%GmX*Q~m0o-E%7q^>^m<)gQjP=^iHG9V=66c-PiV zg& z^8!0(e+J@%jBt|Jyz3RM_26zSnlgvYN=K-xPyfnIToY8U*l`oA`T`TmFE{BAdP+w{ zBB5C7E3>xx5D<$%i7~HO|2^$ZrCeZ6u@kbBtv_7gi5H&rb`2E48SRr65DLvD%9ut& zp<2GYuW5J!ZXa+7zC_)g#sdeRLw!pIbrcake_SAqp@=?Fp!&IPw)=NY=;q`z{G%WJ z%$ELK_hHB`$J$!0?9~!Ks(vt|3fMqwQaNqgm z?z&lmDq9lsJEvkl{lpJXkvjfax8`S-fB%!)KiosD!>N+A@mU&7Q_T{Fo79Xpc!b&YX()x8m*f{ zD}BKz^a)0Xg&T-1lEg|JU15@Ih}ZO>*!#)* zpZUbEKKauFf9I-S9z=iU+g_yNwma@S2m987@AL4)?l7f|HTcHP^$)iEf9fYP{n1#v zJUBwlO(##4mNH~5JM338iXkbTM(pg2EIQqZ6U*S5K@9m^fbOufW*dX8>7#LCLb9;v zMJ%v5dx$$23W|9VER2Al?0|hq`E|`-j9Zz_6E#>(X)YeX>$P>x%rFpVLn@9d@j}k> zL#|CY>i0fUCJ>|n61Kixe-|cjZAo~!fC{j|sbR+CLuHA*Mvsk<)K*r+%L_}akeoP? zz7a6TcoWZx_?U|sV&`g!P1bW8tsaO189WwtHl07~a=s8ThYKB(EZw`PxVSk#VonVV z^bupWU7l%Vx1<;4gi%Z~EV}hq;tD=?eJxcj<4W-fwf?KYHVEUbe?j@rygVo3kFP~0 z)}QriSU~<)be4<+Kis#Qv;XoqHAD9rhDgW)6UCMxsltUTY2&kcy#-LUXa9si^_`3u zGPQum{Y9iZ!0F0CNv9p(6MFm6ju{I13IuVM0~vI>Z>>Bg)U*S3!e@z=rq=Zrd!>aO zsyWVN1P_ToFb4+Ne;R=>5KnyG6epy5h1bk7MfrDaQF$g1@N$g=$so|pWuq@mL`+;$ zvNj8X27IUZ@lcR!|5aAOUUNXEwqZ~-jbfW*Lg@;;-_vld^hC3OgT$uN6@-1MHpc9K zq$hqo3iWqh@H0bJJnUx%P2cF-hy3cp-5Tyw&-{(T=m8pR&Q_bL7MA6rKX^JmWYW;{t8yb`42Yg|}@SO{W_ zfFdM#5^6F*5QjGIJSL8?w~riw`Smg|j2hJ+3Z?xKC{yu z&uXpJ@c=5<*stz@ohI3MhT`pf-T79sbmX=C26;DI`tg=Aie*VIBXRC@L$2)zmv~KP z*eYfr;(COEnn=bzTvr8%nYxWIkb+3N*YB=9F^L#k7Vp`pewPM0*cd7dC*39)iyy*l zK*Sqje+5K-T0GJ-#qNJUW!dML(fy=XNKF=e>f1dtAjs!KsWRK!Mr7Wd3a>DY4}~Y zFNS*Oe_7)?{?<=J$6bDRb2+tERPp_hBY!=FiP`b8uU{Y}SsW`}>t~(6eX`2kNabg0 zL^azH#iz7v|6Q{(G-M$6Jh2S1u3&8nkg0qIx_ALCv|;5ffG><1Y)2-jzj=)iuOAyhk%{7UDk`M(o}4?F zD-S96?pwrM%quU+VoUYdMCb@hjUefOgOdQs_-Fmg13pd7elcfnKp7 z=yS2%124Dz*Qqya;o}R_j3IHb6r{r^QJrvq`ssfJf^(c&wUz_l?(K~Dh$qAj`O40z zV|4QjWvpHxmG^rjVsA_rtpoKjoEgwp!%=e={%(AfACMP ztC@4>3rWMDo?!LI-}K`*Ra)`4cT4SGzi=0G0k?mT@qX(`)5XvEZ}_t@GZv$k{c^p5 z7qTzEtCPsXsyFC3z}l?ik*~f7Bme0&SEc%RKc(ZZ?16CGVIU%vEHy+vdbda{1FRMO9^%;m)0SR_XB8rXQ&TRD$f7+t@Ii|Al zx!UhzQ2>|1=+5OBxIfQ;Tc)`P>T`L(Qm|#eVN7E<=;m>hfGE~TXoew2^BYQch^Jel zbJs)k^^gDX2acwSKO8rQ`zGhM;tvqgzt0^1(z?GsA~m!4^&b4YSE)@}Uns7h4t}ue zFCX0b>o@thlAVoT&d=E#cM2Lvg9u^Aje+CB;ZUV2uhr*wF3Ub0}>lI#iqPnnu9D5e*(?^hZm{y2@s5V z;U||#FsN03_OJ5|W*eWW7=lnGB6tIsJX&a%?%#uzp>BtAB`q#g(^jC{B=x@HL$Nu6M*^j@g{_?Yb?V+J2KbouZXmCsQ zOsM9a{zf{Df2QN((T^5nqc4|VL6~fba0sq78>#sxr12HsxVI$6pMsdS1Ko+28JkI#2No6=YaK5#ni zi?!dLf9eNEzua!{gL}#!I$$bRS~_BN+~uf=(j#Ou%+!924kOdE7qRuO7jI$EUkvcI2>=51$S^pZx~pNf0S>pVSSGE4?;4+T!hIWVMxw>OL~VZ zFcU$-!%MRFZi5~uzk-S_!EDU+z+kH9()~rfWs6*6SXRuH%vcDrK=GaZ`rVQ8fn>Aw zI#Bg9_BzkbROPcemg0pVE=YdHNWbH@Kv5wXNS#Bj8_?*c!^5r8Uk$E!VZF1Zx*h+_ ze+^7Gl$?TOZ`dYk{UefjPB)+O=K+ivt~nfm5I75b?r`8fj1PlEAq@KZI)CRYKRNxk zJncAA^wXnd-MU-o*GCfkTmSyWf=(w*uMj3b@ug~SG9sb=%QpXd{caZE zo$22^rhn;XU5~H4=ik?D&EUHiNO{|-fBfkyBlhDpuska)YrNSFSfIOzC%d`3u1+J^ zA`!=-;j#vb7g+^pZ6_RY8q{UQXiiu<&<-{!eGqZL*ZW>iyn9%RC6Q`BS7LUgKVwUw z>+40V$3#-^`Fc24tPgx+gCd~E-F=JEH&L^_q%4bdpu8BqD1kpwke|9@2b=|IwabyZ&Q~Wq=YOyB`TjrIT zzf&*FM7AEWC9T=svQ~lk40MpJ?65{yK9U{oln9c$9aMrql6f#A2*n>SW}M`ji$v^L zVdWsD6B9-^qIxKw$$C(_3gcWbe@84#ov&Z6#2d3jh$jJ&fB`@I6BXw#0c);y5Gvna zRbbiq{|+Ibhg`+`)>;9bn!Jl<(Uzq11V_Zaw%#cz#CH@~q)<=;5MZ#)NFdCG2I zj|x+MwCj(*>35%@!mmFwqgcjlJA}wFO9QwdhFWK?d!|T`8QuNGN5&3E%4xg1owz2840)r+zN1E#t4+>dcQHUfKpzVkdeLJ3QF75 zH-$XFQ!G(Fb4@%`xM6OkaA-)@YK<4=g<*8+&EhhqA)UtvHmmd8G-QGejoYX&!Oi2n zS&JG3wYM^*5E2n_Vp50^e|&TtAoetkEeY%DPPiuYLC#j?^3)t`inPwwVmYaE9m(8H^H42d;ub`h*c=A~uhV0|r^Vi69i$f3h|Nvy4EIS0YGV z+?Xgo24^)9nZab5Q;0oV$_>Ym(VSzY(PCa6vGVrwgphzOu(<3&2!=&TVj__rC7LN` z?deR`hA8JuDnD8A3~J&mnwZ=6g&1`@1@)<_$q^f<;s#U*-3Ie+KM~mdnRE~i3nP*c z0_5z7~uIzUqDahfFMdkXdHtz2z5Dqra8|MX1Nzd z!=e0NzE>o^+T%as10+zj_4zokw^0LH_cwR)>HR6GX0bImsK*O`9M+A7ejay1upK2yks36X5&s67 zg+p+$C~g8IU$%@J1HJcq4N_1u;ye(}ZGRaS~v}Io=Ov%Coh*f0m9RXU=}D-Efe&aG?}&(}%cF zo_FQ3Q=VxIe3xDn-YZTnDJrqIae=t%eX>)31}}~(fdchWGwf%+f@C`K1^dPG-}|he zJ)ggPqGf-4AN0oykbdz*{?Rl?e|a(Jmouny1=&H9U?bEl%th(M$>t^Xt(kKdG!K~a zi)OsBf0a+Oh8Q8V_nIUyk-93@kK-+RSUc{D8R;QG0#k zz1isgktYjFNCz7E;3Lj3PfX>pKs>e8UL5C~&S(_tjK>yi#LVZ~4$H!Fb6K;ZObsBO zAw}S}MfXX$2EZMBju|L@WY;Ra{%4l1ICJ>be>u`C{HcpG7vXq`*0Y@j96ve1RFrrj zsK6h+ha`89;3OFV)fnQPk(rx!uqZ!{Lx*0dPLO+cJouk`&je4#~il$e~96!Gyw-Rv`|uF ze_pxWUcdSuwc-Hl*BxZau=Iv$4McfZQ2VkyjT$)AL*Qaz8)OMG@zg@G5F(t4j?4r&2f|)c?dc zM(g!hkSy_8Yq8`@9KP)Tm$h$eOI_KvmsjfrGJjll>l{w4cX-d~yCw|)V7=sM>eO8w zc<$&jzt$W3eCPe|TJtS0b~5+w?3eF;wOKbV-?i@fi@SAQ>+79-^O1;N0)w0h%-!wx zOXNmS-Mj`;k=4Fz{Chzut!pYv0;idn%Yq0QISld48tYX;GSVF5WRKPjPTQskfj67cEK&=(Nw<_NAIpXVJ?b^>rs{Ki1ix)42eSBCaL4`Fij@k;ZpoLin zqrb6{^i!F9WQtfao*6BK1Ti9p`*h}s7=Mo@V`w6(OG=+v!qkXV-$1ge@gOTD$fxHO z=_*D})eW!L}uZa>N2aH8>aar@V2N)N&OQJ>`aZ#YM!NUo2)448NG#dgx2YK40ijFbXRLYUdC?xT=W$XkC zig-3PLVx()dM@ZIa)BrwNsc)~gV7-5%2g{nPHGxmbq+clK#zBKP6d6XutbbHsK#J` z*r`N*lSd+J>Ls=ds+H7mF?Xx`ou*sT>Z+Jk`b4Shhr~~pFyR-fA;c%TUvh=weEr@Cz=UjWB{Y`dK#{Y2n^9FM~ zK~g$YN2MntLDJ8hsfQd^d_t6PsjmGg<9??PQm0hr6jbgA!j&?;^J}>6H6p4Hi5B`IIm~@N&U`xAde!_b$tHsDF~#bUT*$4D;2tPxks7bdCpb1Wv?S+{dgyJ9GcbZ@=JkL?^SCJ(0qq zi#1T2qTdr~WN@`tKSZ2O9riB1yl^-b3r^Y(a&P+?X}|cXJyOXJkhUBo-qqP)xOrO9 zw0z;`(MtyVXL4>|5VHm6*%6d~fTcNiyEp?uE}seJ9)AN}u)fO2S|I2D&%Qz)NuS?*%Y&jELlYl0d? z!6o03A%A}&TIo2&xD2lA#))<-1JV6+-KyW~*5%rZnZRfHb(Rr3@8jLwO6F9hbkD5^ zpW@eA)kpUfeYv*wt#srCr&cm9}i${4Uuj{be?Pg&<+lM)54>nI0`mrE?)?sKD)?_!{PIr9n zkrwMoJL|b*A6i&IZ3OcJB*mh?yMCXOig&r(p*W9NoXKm=e)elk_CodA4qdIOs~Md6 zZlwivspQ8h_m!oZ zB)HlH>w7%m(=lOb;>M3}JlhU2M(dt;u>_gbk5o6O<+&X(X&jCRa3UwdbAv_fPM^Y@ z6I0{5pnVrt+~DHMtL<VMFMmRjvd zh9}!|!S)D!+c5pC0x3O~D9>XcA-C~f4281PxxhS#tR21^ zY3?{EDAn120P+oCYW8s#ADsI*&2uiTcB(PO+yU?F{ygE=dwSRV&3AG(Iy$FPdT9~t z;14Y?`b)Qq{-xtZ|5@A1Uw@feDrz!S%7t$`3`m`XDql~wJRu-XlS7HNY81#8u_^>) z?Ap^O4I1mC$o-d5$f5(tp%anT2jgcWx6FNlPk|CAz>(j0`ms^u9QFGpN`9Ud?mK_9 zG&aaOl?(r_cX2g7-U)pNB!FcE+D1h|-vhptNC%%1kD&T^|0>;8LY`)79#DD(;WAUU@LJ z6mur}T+(c34<&Q6SrR>@>D%H}jvarm(hjNKC-w>J_p5!WS{a>c*U<&$KC|R(OEw#S5LL!p zD$l#*A-#KB$e|FNcUUWJ@8%DA z?&RVz^I*XY#<`@PZ1Ur48A0Vt7jL7GUh)mmDq*s8gW)*lbm(!NJ5(17Qq>m@pR4q~ z&Y`NSozr|pN#hb!u4OyxLxw3dP3LAZ^9+lAT^GN!7H&DdAMHdF_ua?8b^UrhMB@)! zYo90S^=B17_YL@;a!!99w2Gi5UcO++qn@V$eGqvj^x}xTGaeSajENpI zsG^~$Nt%zy)B6bytE%-t=Py)aZmxA46=gBV2~^{TfZ+UGhAbV=DahA)HlBp=VCUze z@L{~(&y_b(>!LbK4^)n^3#X^?dp=*@&1Q$U*Ry>t8RnL2Kiq$gulB&B!~C-k_(#^G zO>G$jogvGw9Gk;0u5w=|{)Jh%#+-;DM!f&e-ojesuAApR7v1~vKIXK$Ty$=7P(||e zuIovbi5yfkC3?sfh~XuZ(WcUi(r+f1KYW>i&(iiS(1japM*-VH=#kOH0li@m%?a}5 zbQ!ujXXNzx?pJ?w@gmFj_eER#>}}`j7J>g0B84Xei5=`F=Z`smd$!4?@fL65m4GTj zX**=AQk?Suc+P2ik3LX`b!xstRr{dEcO|ti8(!!f{n@xMrVl3Vo$r=TrvHw)TImCLAc_z2hH}@Mc{`@Bp{J zb03|!G`fPQA$g5;F-Nv5@6NK!LFFqVg&(q}0>wA}uBGqlGRUS6k-=7gW5HtN2T2h) zr&73S`PcK5<{sU2no|eL_w}DRwnrxI~AwNIH9NrVqgN9cy3B7-eb5!rYRKI*8(#6jO z#&cJHMqP7aecZip=s(1u*29x^Y#9U1pvXxjufBh@{aGmKENH!OPOuDALn?3`P@{lM z@iUDdB=*G7!OODd6moxi^{@N;v5&32=jB{KfsP)QTI^yv3IS<+P*A!ZJqYx#mB|~k z7J?V-bt=s~{}C&x-Y!;u#YRWmM}&2EdmpExkG+jquSMC-SEoam@}s}q{NxxkwyexK ziY|Xx$M3-geqyGj7Idln#5tAQX^Psq@-t2N!qJ6*%0AM~!=rUwJ~bnz;{9a7Z>M2jHGImbLNNtPs%4QkxP zL~|9%#)gZZyQ**}ZqE6zv56{HGbUIJ%}L<%I#X+^!cRPBF>2pR}g`D+Hu#q9ASUYW58$ZFig8%=d$wC-}@J_xbm*g3s-`chggpp z-NGxW6bY-?4RjkUsG>BjW>mEwNfmuTRl44m4PLE>C#D+?ML{Z6Ypk8&I6Vfwb^I8- zDjC2EBAUX9?j(!$az#|}SZJBecgQPv%Oe%pbjKCQGDa={-8sZTJw#b^nel%BRB7^P zxiucZmyLBa+|xbxn_qLYfRxOK;3O=6SleB(TuM^VSLrBB3ToJ9y7^fYbyPK*wQ(b z%~HPw!3ktwH^K6(&Qr9y3frQK5oxsh(Jd}7k@B*M3KP1Eb|jG@ z(Mhm7H(yj0E@NtH){+J!P&siA-N3DlhN{EpV2F({ou zgPf%@#yY-qkZ+iyu3=2<=Lvy4(&u8Ife?NQ3@S8UhL5ww`8_Ey$370Fq?p@u;)^u6 z{OcTt>V_ff^C~9)bL~k{^zJ_OVK3AWopX$D4m%UH7-p-zU5D{0JroX6{N(c~nWNZ>@Ia^g%%LAAo%|D&5mHBp7~H1bFk z73X~jQf&kjuITgCcUae|a+H;v8GFsT`drVyL!G}X&ZN&Tl-GFG{R@}X*8>f2#H?Qz zqnWiqzs-Na_v2Z^*RMJ&0_$G$vJ%ipM(m9(n~TtSPIcj=Z_tD%o-E(&FXY8LknbY> zF}RSi0;Q&0iYF)n%G6`9J>~8DV>^Ljd_6m-1a4=C%(d-hwjEHdSYj*EpCg|@caBh3 z#ch%oiuGPNIk9>RGbwO=q*()Phg9;*QgMe|&&_{n^mA?`iL3|CTJU#R?**E>#MDsw zIqQ8_Q%AYtQaI(4vX72=9-zx0#3# zc2R%hE8&_jOqL;oCde+Rd03zwq2w||d~6q=KX7)?_`GAxk0U;&(}nBrxLdr^+llDX z<+eLKjWFkYs-odv{(65;=7$Mr`T+asMD7On)2Fh}sm&ol+(%BGlgPz3VzaRfFT;|F zv&eacaWU%hKD}FCO?M}ylK5smj)UFzeOA zYy&IjC{g0@Ag=Wvn!X`U%a1eZ1fVp=Z%%Nw9g;@DL^f^6?H!Rh!9sLjFGUSg(~yZw zBE$iFW4?9z{{fFdn7kKBfoGsb`-Ax$QQ96&_{8KHlk$z|h%W4LfqREk)pVf+Bj)poezhRtlvcEplqt3D3Pa2P;-3i4`A` zbNfV9PMq1;OY-Q|)yk4m1h9YQ3%B!oM%J(;&k(=hq9#~>?s0(8F8>FAvBw2gc^|7H zz773j^oODW7(8WkFq(8-eF49(m=dV*b8)5Eb$2NGN@`LSGU1YQPQ)wh$`NvE>1gGk z7P&)4G~SgHb}WdfhB@x@wiRiu&!3O%M!`7XT+&PVCqye?OZU0dM|GO2{~Rj)Z?pY?xqhdQCkC5c#gTEy#p zB+b3zV(9EKc)@u_uT0)>xuVxixZx&$`nZ^!p}))Tdfu+FEjL}ntRSnMJTo;?SQDUt zTBSwfZwo~^NvPx>!DY>zi+txGQrJcZ;zC87s$kW8NF`VCZp=v4cnot76;DBa&Lk9x zoRx=Aj$M$2oDpU z~`|9k$^Gg27FJ@l&(T*6_5mw_Fxm|HLbpMeBf!U->D2wZ4i8qz$2}#=O)! z-UUNdu88$Cht|%$WYl=o{UELy@9>~wW<7;C2CGH=Xl{S7K;T@gM-_a-Y7E@mce2+} zftwbAe=fj3FH4HrPP(KaaL$xs1g6h6;!T&Wz%s3M5Zq|UB4%WTp)va5I4NNYYXu?7 z&mz8#_WG&k(!FVL|EY_#a4y%l_80dGF%{cCY9+f4Yhlh1wJUX+S9--DTJY1Siuibt zlj3D*%gcZKH0pnTT1B#%czJz_?h@bIJn-yLM!asmNi__Xm*_ZE`s{4&t#_6|oTzP> z#caidj`Q+URO6cN@rE&Y_R zyy6S{=Y<|Ch|h>PJ3>-Ehqq>Gy=V!lq!TFceu{qyIPNz024m#Z$~Jbe4zF!WR71C@ zy@UF{(vipdV{pdY1?Jx^tHr;2mY`6_6)5j`gemXrlY-xT1t7YsCq)D_3Yg@uiK)*;;_e%ORfUcxDF{4 z4%vS^0+Bo}JbQwBF~6G$UyN&zO_zrn+w`Es5vc0z!!@@tqNK-*_gG%RUcg_(T)#VVKv8&2}LF+kkMBXYO?E7J9@b!a>MHGL7uMq^8tmu3!%nRdw`F+>i zUu}kylz;M29>yM%jSm?%-x;HpvOs-1lnJJ?!Pw89sM4oeFj-GOn6`-r{3X=wk|BTX z5D>?lKTktfGkN!V+>Y)3XX9y?x4O3m4aubYjrD(_e&K+Bd}4DKM%{k@qt8y9$!CIk zK-hl>^<>e`zK&h&M`^S^y4T3NEJ6}z8G+U%nR|TO@qq5!!`lVl)AC`ADM8PAnvPvO zJ5S1p`CNHL3C~PV&Z{=!+)47^c^rQeqw-(l;tBS!{Dq6(<-=cj8>3OqGkLdFs5w~B z=uX|nKV^tN&W&A6bHgA0r}rSI1BEmnQ^n~c6d{>D4Xy9oUWIqh<~h@Y`*zV|P&}d_ zTwub7~kh=y$qj?m&+rhV#AyN5T^lBYd3aRsumK2(3eU|j;0 zfDS}|@8CX>m)<*^EQ&Wg@bQVwgMV#f*rICWA-nyrHi#rFF?MiH; zvp7!&A&p+#lvi%j)RU^+iKgzSoJ_UKv@L~9en2&!kHSdo1{lj;< z?ccuVovEw!eiZ5nh(oq-P^TrJN%NLcz7gyh#JQ|RHEt(zg!&FHsFN3Mn>UYf%cl`} zx%X2S``SqT8D!%ryQl}A2NL323GeW__sF2Pm6xMlI{?3(J-Nc$y=JQ;yG}LJrMd#6h^rl z`(8i&zSDOxzS&)jb(BtH;Rla3jF1qxkd*JeT)jGE6u_3P0aBf_VT&y(tDdeQde?z8!%CzF3~mMpE0Sv{Jv+2@Mw^u+ou zB*+mF7dVl+`3O_x@ryWysvzo&YF1UiRQt)boiEg=b_nv?L`Y>s>F|X75NDF)D=&F~ znbefXILZ|E2G+rHhdyH+5$#M6>qK=CQ~l}o9>mxAGn%t8_8R+B^XAvtFMsMAKiuX&?T6+E z|6=S6Y8BEGj0VxEeGzHCNw%on+3m+jqR%|yG^;-x!YKp!5h?#Uq_x}cS3!y9%vr<# z{Ih>A&s#h*5O^H;1ZugN%($d-6oyqExUwd0GPb>B6gk@yFsX;{nS_gnqP8`4i4ItKiB2D*YUjUeyxA+4tY`^dOe+H>nV93PpMjN3ykr0W$NzA zHOJVbl#)j>236eisvUY2*8GRNLKOjf9EV9|S6ON&WHp|^tc}T~a?%XrFpW8p6dQyD zq~38^I;|c%r1HT7)wpSt5FMdjmzQn%L?+Fa^zpm#lU_CYX>DIeyIl1OqbnxsreS~E z?@!FvRH!k?ul44`1RlSMJ${buvHPLN0Xg_Pj1S|JSj8th!#QUZD@&XIBI*!N%s-wmDr%=hD1GQ2!MS&f@mwIqK?=ZKnqaUfc&gelaKWY*miQ=eDjN5bTIf0LYjwLKGe-Nl!+|Eglry?@H_RQPI=lsAgw>ag{ zY1H?7Zy5PRE8?=1oys-qx zdOP|?Jr=Boi?bhP{hjl@oHXxpG_{)M3$-#RXnZ}+z@~psTErX*;*a)|IClk+g~L@K zwo*6@_cCzL@>EEZ{n1t2cc3{%*IeJW+`{dQzAizUIXp!wGC3&!91(`LMv=_=YWsa& z8Xm}hyC43)%;hqp5_RBm;wkPC9S-ilpA@MC_DEHO>3u^^^x2*B;(b-e9Gtswi@XR- zdWxN>Pf>q&2M(U=$L$+a)?m9Gok^Fk+lhJ@F&VxRb?0v-vdwa8N=ewGsHpBNy&2cg z`tH13fvj`7<0q8a>E=fKV9_3GLRAx!3qON4IE16nJT|yR{$Ua2n!N4|$iBO8dLRCB zt%3c*98`VJ2LyH+t+N|mY1Y+^%3nN7@v8>kaY}zoub^%}-(BMX{?m`u`{~!RZn#!n z#G;rY4o&7bsmet?b}hoSM7-C-kQ$5vEQJxtDmNl+B1W(8=e0xs@%_0^+@!pV9ICiC z*JC=VdqT8ybvoK=8++ouI3!1`Q{<3h|NAp; z0@r`fj0kC^2kHIwGn~|c6;GDmPi{t(ru6<$@vZBZ+xBSI!70kWY#A3puYJo@Ss*543Xz>05w$iU-t>D`hGR zNt9nL&UzAoTr}?yV$MCfc^pIijAd3y^cH`anW4%{|NW(LAuGtzFd}pIp*Xd$K2r%c z1nfgwZn@Mh;$>#1&kaq7QHJ#xD5i(E_Y(0x1Pbyp`y7bN2wQr{&Q6X`XeYIT5%9?9 z#6Z_aE;qQt9v|1&U#~Urt8Nzm;V?yAuJIzw9quU8LmA@(?1MnpZ~a_YkCT2b5~6>q zcPg7B=gQm#>sq(6dYyiOWL&Lw{tV~Wol~g-uH<=k<`nk!beEU(L%j^n$~l6}b0Q~S zAj5H|<5q<5EFD53PblogSU3|JFDiOsGu_>RS;F6_Y=~b80pFuXH&s*kP_IynZFgS3 z2R81Hvq0Hm)IQo{&Jz1;{lfnI!gYU^h!fo2bf|U)8+a!!vg2`GiaNUGoNx6%@1}PJ zrOPz-lQsC7&ZoO{y_JM^DY>*mD9?vb=Pp$*vpdor@$(c$bh69uOznU~=V>-x`O+CE zC1b<|)Y93Z1~Y1Dkc8J%Yra<8=tLJ6doV2?r-+TCaz1Hxx!w3<@~sC`Wq5zFIrg^l zpms`7<%Aa42fBzT6dc|vm`naNMJ|kVEfnXbsu4trIG7b{Pl|gY$AYNn+w;>La-V}L zB;)Y|9EXF9x7~g6iGEYGL%sZksS61Hmpz=oCF=A?sz0>Xe)RJV*7>GGa&NFJzxBlM z_sN^CcWv&YwC@?M_!0X$#|MAxjSz*?yk=6Jel=!@;{q*aa`dpFvRSz&RBsV3;p69B zN&;SCADw@*7L5yVs3Q&q-9R{p<;W$<_SoiWoxE$ZR2@Fn)fhi$@PL1W{{#ONxhuDP zGwKBVh zBK?IS-R1|M7Dr!&uZT;s7KKNZnm~gwnGRt-gT?3R;VAioQapdO@=-`3qFaUe?9KC9 znLbHm`PvY=ydRuKg3yoLW3OXQ0ClrpzL+c#7kO~baS~>qmk78U>oR`Q(sfk9pD>rK z3*zsW+sS>TK^C2z(h>dC!JFaBpq=Wq9B$76^5*yYz%i$R>+CqEg7@})>(igPnX7U} zm3YJ+WUzUvtj~Ye;j*pa_2YRg#WPb42c0u(LwSWsNZsO@DXlNWZ?_C~f>x^Zxt^k? zQBI`u_^NoyR36!%_k%6uF{yyV^Jp}2M;<*1`g%m35$Za6U){a`e8hV{0ce1Y49>gD zCFF-+7#++F*F6>vYv>cu z$&XX&V&CPHa3`}cvU@A1+adzI+cM`Z*uHNZ)yOm}hEs?b(M`^1JdRW!$K2uAd=9iVI#@Rx z6zUQtzj}YOTsirF``NdCdqvjAjrZ7F@8%m^d&BLcm=extT5Ta|#v$?@bQJS4{vMJ7 z`yiqISM1#_p#(DrUxB(KTvdF%;oOD1AX3Ke=JP$4sKqPJ6q=XmgQ46k zkzkMQ)^pOmiqCS<&BcKPo1srgyh=QY9C<+`-b;TxSnZZR)4Jm!omy(EutXdV)_43r zxkuf)8a(VxAK&=KU4N`wY(fk-n)SBs%8U8Y?YsO#jpL?$03St;d=E<3voRksU!%68 zJWBgs&(czWsw1w{clz|aCRKTOrQli1|K+{$0Sj8eb9h%4*pG^w3eWWJcFXUn97k}P zO(%bjKDr9R2FK-QpW|3X0(myFm<@o7kE2fT$9DcFC zmnY2oc+VO6pWZ|*dzzpy`uUGDJ^zkJQ^ z%rYI#f3)2M#?sfGryU;Qz4VzB%Qz|CCkP1}zMZdobv>FLme`L7?>yMcjxI8W_$O6N;!wuRL(a)~G^LWCR_Q4zF@J!)QB+BxdhyR7 zJ>>m>nm>LjKADVoLO~*qcbiqsz?V{ju6@qxrv=6P9VLl2 zl#RqE@)|82Nw3e{ogruKKL1jo4@&NLx$r?_f=Kd^NWm|!|jGPia4()tP)RNx$x7-k#y?LSa)%e|!rkw4_l*5mN}p-*7-z+l-JhDJp1e0;Pn$pp zrlQJI)5%4T$9Z>0z1qhF+Lkuc>1z8=l>W#a*^~yn= z^4I(){*xI0Nm$Ab=hEB0yFajcRsOf`GtTS3okM@uV>~)8Lx&wV+Gc-O&FN@Iq{-7< z`a7-r`Cd*At7sXK5lKz&Fa$~!4%w?$s`I3Bie-Fmm54Hj>HEp0`ZES(>fzZtjmcs^ znzDIyEKcRwRn0Dh*K)7jd)Pm$A@vJ6EEmWJ$f2MM(s%^w34CxyF?V?ImspFK%;h%R zMvIxJ6QZ0bL;YREcIJOSaUgOZQXBi_V$G!{PpleEC31gk2X!Y$&xlPC$H`2u67p8W zTAKX$Y&xEy{oDY2fpd*-OpoAmw7bpgO{473UeZ7N`KndJuUeviH>%l%00f z(ch<4sd?s|%d_jZQ??v5Tyig;k7AHK$gswf$a?!Wj@iRuP40i@ORn>bY3@ZA$q`S9 zxkj;#N3UQ{t*`tNX^GgJ2u?f`cMo4oI;-^q>H{I)zczSaNRrPS)QhW>RU~?9FsQP| z8XTbwSv%H4#$@>&YQ!ig7j(YyH21My9-a}Sr-kVelLDc#%O^TA7{XD*Qq%ZaH^14& zn;vnOH=A4<#07ucdcO5N*{}V9j(%n8LVxj&P0sPs*CVg}gKwCoTo0BqA&E)O0c~^ry=hnDG90Q7=a1UzIZWVS zdlZysEu()42S2q|nha1|U4J|eQA-=OAf;z__1&ZOp^%SMAgSaF^9{MY;^F_9clS$^ zIE!|DZvRh?M0UeI|L~l*{?DDYh-uOFh({k?@l}|W4Rx23S(S4_95-{@xliqS(Wr!g zOp@K}Q>Pt$XaY6=TCoW>-wtrm_kiKJ-tr$;&DpLj|qq& zmwYPg&mrnV750A0we+0_Uv=NtkJigq`#N8yPO>H+-u{qR`9ezMJL2zcB3Ehry?9o~ zfz~U=mz&_Co&wsArigp~dmcq`t^57vd;fzznd!6mqeXTW*M9bMxW5y(G4Msud@ z0IFh6RU>&arc~Ke2LuUD`OJJywvs~|c(#~3`v|q(BvUo&J_fXc#KZAH)c2L)G*N$0 zVxOF;{y(25cI~s;bAh{jKm2dc&6t^<-m-8>8jto$M*r)1%}NSVqJAGzWF5 z5NCS=uz@2FfS7z8MCanjljzQ14^G@AI%hoP3ED<3qwe37Cke`uW4-=34zRvwwS0zn zh}3%e4yI3EP=61ew2UBcNUBgsYx95nBGG<2pO9Pz55P<7D;s=izP7$1;%%OGrr)~J zw+vJG;J_qGHsk;-I`~FU=VnhGFyp~-IVd}dy`kwwHw@JGKuca*FF8%EyCG`Hq{<1X z#O9{MPaX;shjdtgxgWs#V7i|taJHK z&qLB2*R}qGU%^Z4%k2^ETZVt9YdsN2v*%AdrBu{zf<5Q|>)Zz5SJu)Ric`B3DPOv} z$NK|x{nt>}jv*#e9Q)oQGo$YXW#u5pQ5K$(k zv0B6H$GvTxaZi{&NT7k`xADclwfZ}qlk2!He{!Aw@H5eGMPGg9INiAYYUC4*{niDE zce&xQ)y=DmeOV~j1V(?9Y&^q1J-T;x3G?BXSxatz_)t5YKkdL2MAVIa`*M(VKy{62 zjI{3kZ13Jy(|g;RDqz+aeL7+E)g}NImZ(iX6YS&Jjn(aN|M`FOtobGGdbjsI8;s`8 zoMdt*@HLISMoQ$!eDEjEq1+d^|+fV;{(the$7^4`okzcskd)(`J3U&PnFmqlk_fAZaaFgAsUHlgcN zS$IzOHTKE!$o791#}NDHsG1zQF>_2D5yQc^L$aKw`(++8>+y?&@~9qpFfZ&5hsB$K zh<=WO1Bd&8wT#Fz!F~KBAjdx2txuuxcGzDUefSf2*5`Nv^7(l)7pEoe88Y7Z=Mw9j z40ieDw!h7D6Tj0_z3v0lpE3JjX3pTb9HQpD`TCOE;)j0&a@`Mq%$Jk>q4VVRJqG*z zF`_0f^VY~Gy3b)=8@sSqWyn-XmOE+A2BfOts-=(>LkgG&ahK*i-wp3dUG$t0e)h@d zrnnQa)s#$Bwi3#HW>Pgd73Mq*QMd1Y_+0FqBc6HhG(P&;>=7Dg5%o?GTi0d zh%92NOpTL>W|?#C4SaC!jiwFaH1AE_XN=##Yyn1o_}GkkgcdQUa-d?k&co?L6npf-cYkKzq`F%Rx9gBQ4B zE%{`EoQrKPt?$Ai$_|JO{4MZpwU-%eMl~$TCu?8+9;>NyfTOvm-2!OW7Ywp%|McIz zn7TS)oc^JaggArkRj>G`kFBW#@98XhbmV{NK2>TWQClw?ntoB%$8^xDC}I$iOgS+u zm6oaA@hX3x=+=$X+J&UOq)J5j1^XYij}#mcWAAAtT!y`klpqT`x&*s6BwO>@uPNq- z3}5tUh2Yt!li(UX&k<@e9kJj0wWeTn-_Wj}BQizYu(Ev*T9Wbh)zx$B8#Osk4^e*> z7J}%OdmWdb(9B>1T`t=f1~>g!8t5XpaYVbTrWRtv0` ze6CTWA2r@o&erY-F(E`A#`|@xbC8?C1U<%eK4m(LsC;V1s+Q>2(+xWkt13Yv|iaNq-LNgX!%O-CMlFKkkY5TuJv(7Va&K|_YB2yXfd@d&=9 zkt)3y{mo9R4DWxJ=+0+HyzAD)5}yu%lgzO;qLg?EL3HFlfa|}OxdLmg#sFtN)$<(XH zsWW{VF~#FWZO=M>e*C_gSFupJk396n47Fd!Q|mi^T^Qa~?TC8VtmaM6L!ZTq@2AUP z_t)TShs)itbNzbtTnIL|NGiyMnL*}QsN zLC;CAA71sv;7A>cRW{dj+YLe;G1`-alH z))0vl%>Zh;R^A8D0s^d$%fH8Z^DC~JXTSN#=4&u&hx=jhPB!(Clh}ad{AwCC9io;>jrD9 z=wqlrHE<|>M z6^w{`=9`EMN_x!&LoYpmKvEcgFo@6EasSCTH! zALIwbV1pkc2Z_`KDIu_UGr~>@fx!rzh+ltKqyXcpI;Z<|_jm8htyOCop+w4gh={#6 z@;y#k4;9^kOR#^VALwQ;JzqI7?>J|b(Q$2TU-S}dunvD-h>m*_IdEjJTNZrkUF`UK_>@#C zI5hLGH2OprY}OB7{blT}u0FzV+^i`B!SO>Razbhclzbj{Rp#VLUartzk8wgaIRffX zoL)xjbPRu5tclKS)Nln=G52lPQaNWl4}u`DYAQv@q5H~zINAM0gI{gn^Rq`|uG-p6xYx$G`u zve|3mGuOd*Gq;BGc{xEnn9W9t54XJ4-Cy{;!ZnS2`=qw=#7AMBP2#WKn0?ZmoRx%P ztH^(oqnH-L_!d8&mkv)~UQ!{R5Lc4x@@6rnC}AwI&1Y&Z3C{?^ae2$Hb{wVzaclV*tn}H7OXr-GW0oYu=E;|c#YbEuN0NOBrk)N((F;Z0d-R|<>}7BC z*{dpATL)U#{-=gt>O(=)#J3=iqls6$;Aw90ZFf9M_dF@>aFP~5o;7PxAsVfU=4^j> z8D}Q;svSyqjO~cpo5Ong9FjT#5_S>-MCxsw1O2>!3^1?cJ zqQFTy6CoQI48mSS)ralZHB(Qg)fqZHhh?0a`Fr-g?5qlMR4u9KP=q~r#`T(Yd2VV)Z_wN|F6=x zE#%Xo^5fyD)qgPB23nLvN(QJSi}obs`bcnJmbmo2zi)l@sL_}!q)mSyU`(Z~^^Tlw zoJ43(m8IJ{em%65(W0X+Bph@wB5M5H(t<&(B-PSh_mE4y$BE>7(`RK*5H)^yZ1va3 z1Npy6)A(sxeT%oG=pjxxR3y5@Em~nMyS}>GpzbjSk_*kdfs*`&$GXPMqpgQF@uKb| zeYr~mSFR5?PHt&pjQxLB_VC;t`mb$~flNmjJLo9>4mz>P7;S4?4iiA0=pn?xh=HE8 zj>Cr*?(-p~_U_}M@1%?<+ab~-$fe;Z!I2yCq!jAY5~{fOIqvUpOCWV6$>;N7?C73% z0z#A;k@;*6GKq=wUQ6vhLHhYhw3<r;!vVZ7XyDTUMFiMZ3c-by zT2^WMwX8KIP)XxYwEc&7}y!p=UdEoY5enBW5-2})dwp1!zS8tMugj1B6nke``rN4$8!$`)JHsXhBj?W5D*^( z?4JLC~Lo^sW*A<9K6+v^0l-Qc$$AK+84r9%8RH#(>MyL_fEUC(=N+?0)<@6RR4 zef>4~jppoCUhQ`-jV5SUfMUzdoU}|`3RMX2$+cXBY4LxBXOy$<(GUdM2weE;xk_`$ z&{v!b-dZAM%$9((h%Lhc@oH@d3|^%zwV--I*la36E}pj-=VQ(;Zv{weNL5PpBrH5F zY?k{oP>2^z$-J{|?9R9+*r&9sAOU85^SN9&M~%Acb{Qdju`~W_&goG$jWZ5%YT4>*qZ$`u89YXza&8u|M0EP_#L}@ zGG~qT-L-A6ZaBvq{^6&c<1SuZd7a_v*=gGnjcF0+s{Q&^&Dncr$s}Hv+f?z_G3}vm zl(A^Y7SD19YHJh7LKZ!5uawKk!3_~FJF5iEt5z^X!yU%P zCIM0m4JU6I*9oIKn|d=qE>H?mg`<@3n*G4!g5wNe4u)rGX5v6|(ftM2@EyB*jpI3| zxAVoF53Rvz{y9E;BkG{hf&aVMUM4zWoDJ=@9KA<}uOZ~CGR`S<9p z`8B~(bo^U;;t3>A!Kw~Y(LM{V9b`Q)Wm4(OfZEp+OL1q zQHa#s)SvAv~WOl3n^ zeYqsj7yJtCsDFbqBLHDQp1aJGfIT^^ym?}_JkIlk~E%}tp z$~sWx2ScoUQk_LqiOI~mwoPH5#Vbg}L`R>&`PM#&A%b7(eGAS2@s4-cMXXnGi3Pp4 zD+RMl9KiQ_|C4+2ssm7jlW*;1@vU)x+`iaTuA09%&$WN?Wn66u-B5WQ8$me+hebkJ zU)BXH3=(Fg>sNak^N#b;gcM>0$2_28Ll&Fo~*d!bnhO3ATfr8@q9KW(p1 z7~+PWUwvsS0@=4_({F3UH#g!1do&4t=at1cp|+{>MyKS8m0SJ9FdE+zVn%Lp_*cKg zm2(w75ld8Yx7FXgbzqDd4K`eHireCia>12!s1m|$zno%AdUg@~&)W>LflrMV(%c_y zl%)CcieYrw`EV3!Faw=T#_{uihl7~W_g+`<_SbFRY6nw@68m)`ZgUo5?Hrwl3wdo@ z@e<@)VmPgx?fJWX^_Rf71^#uzm3aV94WCF-xKublxj{3liM>Xy6`cRDz702%(TPSa zU-x8U&xSa4BYp~f$@a)RwZc2(+KBxY9)o(oKAok1Zl4^?gY<&<;-zP4jJm> z3+2+g;#tp<{+l0rUML##h$_4X5%7?X2}J2?y7ha28GsSN+ZWT=96 zurlkQ>DO?Qv)kUq@LXJf>I?VVMbPx!oFkt30dSr=r?y+j4?ekQF4b9KOVl|N2phpo5HWXu=uN)wXPq;N$traJubB{A^(OG+U zElp4Zq7`Vs7ZdrWAu1gC=KH$IeJbrXq3 zz4P=s7i#;P7qsE_RyV$tMla}BJ{H73H0x&+PBa@U)KEiTe~+O(8o8^$aD8XdSP49v zy%>IBvo0qaC^Lt>$V0eE2#(Rs4jgi_{4iLcoyDPY;h*!;L4H`oMpDpojNu^yIO_4) zY-baa?iqA{OM{cU_5bZG+Re*hWId5|nSGqmrjxvnPN$<=s}f=uvon8weN;^7a!B9% z6!LT)fC!lm-H7mbl0W@}cB}f!A}L=n&OUOMb}bfMS`IN-E5&HxI=;3?lgA#Rt*6*e zwsXX%3LT7wU|EZq70sxw?2~ndq~VDVHF-im^_v2JGPvYU3(cP-W%FrPLIMbGZ!L$TI-(8(axIlCvQ4%`OI~> zYINmRcB z8R``o{}`Sn8_u+Aj2hZY4Ch18#CTllHW`}vDE@&Pxx~Sl=YL%HA-8?u$@p#04d&!q z?-gE^OiFvbE+7?jM5f7OGEGi16$h$X<=z;7hHs1hxXQ$vYc&`jc}> znT#A?rZMtdnewULh~)XvWAen094Gi!4nb!V#MV9C8{&aIi=y~oC%*< z^c2aw>_m1vosXv}6vMn%D0@1d9-eIPq>@&-)W?k?ba0G@@bLMx2wQ;%PYH59>SW4) z5QqM$(aG=lMS6B18Br}q$$3iBICUHn5VD?fTM0r`+KMJ}=M+|em_P!Lc&iLM86OO*ts_y|dA&e0tl!bbuJy&(<-2{UlQG ztA?bZG9C{epslf8Cf|02o5s-1W(W|pOx&3aXe%hqgp08)a8BvL*uGOv#`OWu2gt8a zWH|Z5^Cx#Pc5u&!2)Ed%fQ>*fCh_yjpjO^1=tifiUfLpxaLC{JX38LR| zGs*85nLpx<;EGGR!!`XS4(Bg_@l2e7?Lo4231UZQmQ5w!!e;pVf&N^3(&~fz46A{0 zaoE`Zi0EF=mpsRP&2q(hHj$8q=qpj%U~B`%#qoKut&soc;px4KXaViyjSaghd z)=OM=IFT`~$ITomV#C9GJ?o+^Z4-=DVgpf@OYmr;2PJ&q4r)MqZz7?868)hYuh80t z_x`SRMB8_{gO-0Dw6KptV+VZ37;~|}_{u-WgHQh$!|cgF#VKFvN4n1#+0EolH)CpJ z+>hyt%Y>nwqKB@p#APz2B1&D%Z|gXIc#_;^jotYEIkRt^Z;}A+Z*cE&3M^5JDbKTR zxRzs3)mZfeG5TF!lf1Wo+X<|X-=8m*poS0CZhs#9S`P$U6)_CK+W1QEb-T3JA-Zx} zNA}QFXy;3)Z?H7h@Q*S0y5W6$M;+e-pkHib#%B{l{H?=MAn*Qd9*8gJkeaU~6aZ*3EHK|)*gVCBLd+-$&?f5o;yRg19G@~lEWQTXpi(+t(8lEkp6RUMlD|o)i@y9 z#28NDrTdVo_`lRhL1hFNMk00kihiu?PK?d{7Nq$>u2;KgBrx| z+VMehmQIz^Ju7j0EKqUDw&NtQ^~W4K`)^Y4&ID^o}V7uBz;284k{%X9pr1% z4y{i(;qmZWM;rS+Q-Nu}tFD24$|7>vtC&6HsfoqAadIPnuQHgL_Dz4rj2pv`*2G1g z8y$6DjTuqS7g3ypQuHWGCRnUyP`Lzk*kpFfXX4FP0jKdyf+`v4#X};^o<+dEbfM<; z^h52b3OG@J+7HrcG0c(kv|N`U5aUd>!q|9e;sFTQhx3WMVpk+apS!*%z<*>uH-WBjDiGBRr; zSNwtM0ecr5F&&P2(F|h=nsFyNdk^Rd?c~+iM-Cf*BFo_!DTtMzdZ35q?0QPb`!&ws zaX>vC957^+YpB2Z53+`vdvYJQ-^3H}ChqBSJ`mS+6**j+zPic?Q$vGmX%$-jh~p=L z!`>t%GUXKOSo$TCsY|p?7xlC=A-L8B3#u>Jk$rlL>2FPii@x8EM*b-rgIlLnB%X=8 zN%y;d$#|wbWpt|PJuYSlhByA-<*_t=Z$h$W?zqgauqz!C`>XA^b$BQRzO(%0{e zMjus|{&qlpApZezzavQ&mvv@lI1rV*o;SMnE|HUO<_ebpf^{myknYiP< zPOq9kA3yLGMi&QlY`g`BQZPAw9-wU#5}(Tj>aOR=bqiY5;&ghC3Vj|YV(@C?e2sR0 zKevdA?HN9-en@wMLrI4Av*WYiS*}joDel))h1@)jw+iU{q_VtHNh&#gkcg=%Syv4g zxHsm?@G&|RZ3}Px=QVj8zC=p-g&YJ?Fr;}PSTfB}yT+wu&dE3&mdm9>;pj4b;yUxV zR$(;NDI`|u#awcOKYo8`<9YgPi~IS1F7oD@xc-SeH}!mp8M^k9Si=8VJ-4?*7)_!d{=3Iqm8q{9Wi5jRLefxL(o00H;X*~D> ziRUj#NddWut)aaGY0hN0e;ZP$Y;n_7xba54-1L$DZl216|DbOE9ls?1$93|z^IN?A zXLW&b##?@y_ta&bw{grww-~Y?ysvz9Kfbb`K9NcKLr3jryo2E}`ofcw z*D6v2>BVUfYVjFi(ZQr7{g<4`Ue5Q<|AL!LWvx04ZpMQ*TN=fP{ zoiaDhe#{R9%12&o_fzB%lgizr-Bl$oW*XZ z(zKViioV;Gr(;gb4PzO99Vg^&?hg62zwn%|Kq-%?lM6EN{AqNZ^M~O|HGPH2&zOJ< z(tjbkoWR3?neq9udxA)WM4jXA``OIjroEK60ljr8Y!1j#Mxvi>3QEncZ8m0nLiBb* zp>4l_bVrf+^xDW@aMxFATpxlZK3~8qBPv+C&3A^>Bx&_fAAvo8X-JI@4xp*W=&u?r zB3sFu4upHhe+}%Wh2z{b;$>#E#;$7!QI~#FU%9Wng38Koy=Oy^?|tOFrE>CoGI-dG zj=nlQrd%8O%8t4>f-)H!-5;$H&(+1ub!JZ7S^@2jjUV=&yTT3i$ z)&U!yC(p~%c8ENGg2&6>J@V&ad?{EiQRAuE?`49eQ`8^_4)*xL=x01s7-wGqMSSd! z_|oQ_i}T-g9MB)}rGL<2OMZ>lz098%z0N|GiPkQWq6Iqh$XC}Pfb}C#&-)QnMVh+l z<67~CI#1D6uWeC{pj4Ddz0UXL=EbTg z@ueA1^4=-`Qe*5%;5oq~2%!dHGa_hPqV9>d=bK9$>ep?5A7g#}5}%A~W@Ech@Y0O& zf3@fE=0pX8!EKK(sOH4zP9w(c1pHr~YruU1G+OD@a3&!6slj7ZyxHsj-FZge&SPEN zFL8iVV=qX5&zD&AZ*8SKxt_mfS8j9?2N&+Mvg^h_*ZaGCQm#{U+f9hvG6I49eP%%o zVy}P_)r~rU zlm609KlrZi_sVOx_}2eG8;RTa+{i^gvC*AnS$@HPqMJFJY4H-Add#%#53bDHzMus~ zfb+tfAhusx@AHyqqSc4{)ca)85?ltZ=lBngD}A&9d5Dw5pJ?K2#1{MdiTu~vNXI|& zibu(hm|WUmD)DJ8-Z~AaF6Z;pme5Fg!33$UE}BRN8_|rLjpRmeESQL^7S#%*^=mt9 zO`Iry@Ndk$%J?0=6ZcN*d;aX(8mW@m|4h$|@Q`Ym_^b27qp8~GP)ztmPp{vZe0t@GwVphGB^)AEVD2Y6gg3P1 zBF~24IFpS})eNMh=hb(dvn?~N*sv?uvr>e^n%j{f}A#us3diDC zxSq4N&G%FXn<2zu; z^>0)9tOK8%`1ZX(9d=1@KND=!mP_S1rKlz+#1Z0nvpDk0@p4efeEhur)`&jZa@*)P zpXG9>^vvggu_}1hz34hG!k;?1Mi2L|b#iZZo?f%>r5cS?!O!>tqv32_b#d>{C+JVO z`f9&573BZ7{i34;_K7{D6I)*T1Vv7N863maesql2flNKBWoK&n(yA*gFC`-LOB*$}PunUy5oJTc}AeGjZU#JNCJFeWm}WeSzXUae(VQLGxeyzQ(U4 za(Vv4I{9(o4xs+ET`%XJ^EcQi_IHXRU$3rEaiAp4+Jg3E=xb>H^8qho)jL>PMl5E0)AAN!uM}75? zoCaeSNfRDQ2&ZPGgR?Y~(RRtffqU<4Jv@yj34AeL~Q4GJS{CrmD1M$!&gW`i8U4pD_Y3+5fL@j7p-{^5VoEy z^v6FasKB+5H{r$^j5r4{;*B*9v!1S4|Fc%_Dig;edf_0!*(`ixIq&3kn1x! z1+{y|cr-mCaKZqVh1(8)-kG)%jIBi-KFHIG7FKzkr)Zg2G*9{?mBa|_n=ttw&qb}z z9!A$)`owfV5T7TfR7&g*sOA0$Q>m#qt9KW+7~S-(@xIDE3RT;VtXDVJh;N&a;*gRk z!2YttJ{lx}4@Ka)WYD|x!^?U_l2%QU3Gn-r!-!Ef3GBJSE?+jx%elI)iN-Wc^^%CWd-R@9o z>H+x^ki%MyhK_iD;6zL~d(076(!@x9JP%L5l^@Tg4+Zi7oH+OjgIfN(c89;$Hdy`S z<^Rc}WX^r}XORDln}slKAHqq>iP1UZ)QfjofEh(v9DSN)v=|#_f!&A2P(#nI7oO>=2288_$Y?7!`Y(d`WVN50x0XSiRv zxmND&9qHd|Cj4#x9gxKYwH{MOp(0R}pUFB1+2EC9J+szR;-7o$w|2_ssr~u7VbXv8 zkGNpo&!~TY@G!owZ)oO%&1FsqOT;=<0?&&9!?xUpY^Z}&k=hma;j1-X!%d?Lr(CS2Rx zO1xr=G37&X(jg~-OH?r7kEdrBH9~b9m8U4=?L2L?L9;=uEXj50z$ut4*4yQkZ*q_4 z?zLrqbk=R$!xE#Td!4h?&8AuhBe?FZn`@(IMK)EPv;~1vMr_KrVN?<>Rl;HO{002O z-RwT89E|C#eQ7@m9VHZCvUU3I4KaMZW`*tJc6n(=?PZ$DponX=;2wm26MM+F;D)&cQ8 z9-TUNKN~qwT2d7<$$~cmX%gfEA5VMW>5&yzZkyy zC&Otg`ws)z`}Z{GC+Jhv2RI22bmDk_g&sQU?bYa{*LqK&@}RaV!x_E9wfl$N?*++< z|4sk$e^y89z(t)eSMoQbb8xdQ2OMJ7YrP@<@t@aCSN)Bjc`E*4oizRD1-u7%RIrf5 z#B&8InS8H-<_v4%It@1GcYWkPX;(WpJ2Q^YJM5GhgY|EE)@`OlgX^i2PFUQ3xGuX8 zXO2CYe}c5F3^Jc0r*IxTbLPGd6(2Jhwcg_1ahIkahM)7Z{&MoYv!y+osI7Be!Uv5E zZ*7QN1+lUR1S+%<526^RsXZ{b9cS&Xy?Fbzo(9~+O-#0Q3{L-u{4iQXZ)ax@U(OdK z+1Th}p0jSXg1M)9l*s`fvyGE~8~Zc*N-{d(*0;{}hgghqE}81NT@a+y@c4wQ;u3CK8^KZMcNy`h4ExBReToxc^>cY#C5*PJr&hrh?_Jp2)+b(E~#j^;F zbl=rHv9XZYXP#b}Tfeqv?TVW<@44>yOi_|U=+$|bTw>!AntuE*^sHZ|7MaG?o=G(% z%H2*kN5^0ZAl#(DAF)oMPBt&oEq|3-66ESWq~t@T4wG5fI<%cV*cj^%ajPGj`c-6V z>X&7;(Tyv9#g_iieMAg@StmGOxu*uqCYOC^<15Frz{^dCy+Cci8-ItzSMr7rawivk z_Rf|ZWkLh>pXsizq^M_hq4Ao3J#7FfYt3(aG+(!K(8(BN)eb%S4zH&f=NCfH^I3#+T z{Qdm#Dc9(K($02Rmp0LrV9F*iLmoTavsaGfq8oO1&!Dl5-mM9E>%7R!T($QGuTJ%< zd}TD?LyKmv;p&!umSA7c3MQ+EZnyi8JtG&Y;qb+^kic}?y5Rnr*al_uNMk+@}iipte#xjdb8}I9c$9@uOEg_!FKP5ufK1?bgnD8*%83;0q{3 zl0sCAk#2Z!j@XW84X>I!-*|AExqc(A=d0WNf5T~ZuDmIK$Y~SY?75fv|H-TJa`*XL z|MJ&!*bS#^xE9U^A6f_Gg0oxQ>s|)uWx_twf}8^6RbxYVcIJ`vo3mPeTF6pTNGLf4 zK0PG?L;RKud_o5Ai!L3l^dQ{KbNkq@LY7UqDE##!+8c^M4z;k6PmAjWwT%V@wjQC` zR>YwjtKlMln21@BLp1Bbgy<|@PQh8I-Yc)?U|m?ztpfbi4cBgl&d)1CKE!U<{HN5lV9>aK{a#p?_Tp`}f7LBcrM5U9rU zS-74{D@3No?UM}I!qK7-Hj&AY0dF5vZ)8O_;qK>x$ac=}o`R z@ZYKgeYMHW$8y!S)ANb3E4AT)D)HjT%mI~uO-3qfah2G7o6j9mCeh;9XR^yrR1}YI zXs_$7I2lmH9yF>N#5ZoPM+(vA81g(sH`)Od}gc18?; zo*wtUu0N5;F}u8IrB*I1TW_G!h{MH>GSz_FBj(Qmlk9aVEq%iyp1 zhi~om%Xs-^uKQ)4L*1co8`M~%ukTmfeu?~rv24Dzji-Tts% z{m`jxcsZN#cCiUhJ^Kt%@a~>Py3tpg;WAn%oU?a9So5WQwoX2^?8X)-h$!r23f8#h z1a_Vrv)0qhkF`S(pP_gmlAXuogC#knd3QH@dDy=Z9R8bgT^ z;Y4y!K?EwLUXe9+K?~bWj^EdR*-OK-J{ZD2q3=N6|3<&|y5`^8y4!|RmGe4;p|XvRDF=F1qc1Wn?Eun@?B#H*)rkh$-OKHfex^Pi444$5R*x||!T94u zLq6Na!^r#Kooe!WlAm^eR8Eh1`fP7twvQH(ed@iuMDMZPD$^n=GiNfGuksJ&6ykMx z`fNE`E#{{$7XfCAygm=d=`xNwt6@>T?el>%w37ENX${0MP){|F#iY#B*m1>MR!jALoRUb9_YZ!2$IetyvX zm#2fhj1L{}xmWSx#C>5^7H!-9ygZKTr$@gnzpA>`3D{OE#=Jo4(~LPE!FyJZ&)!3D zmgjmUkD;5khiq1cdDc;@#UiY$ZZUcP8V#X0IIllP%j)Tc?f0XO>bK5XjX4aC=XPuS z1`p2aaX8*(5A~{lUyo0N)(d;6Bnctotd!A|M5W`%Y)S1>wB*Ml{o;Nq+dH$;OGfR;Uh{r3v!i_Nd?wjr^wbvPqFXQPtoM}5Nrw?|zS(kM7GM%5M+hq2+Tmakb~Hs9Ahnraw<-h!(F z4+tl@gakKbMB}xh8@%5K;L?wQAYK3p*aV1p^?(X|o`gcfPDfMwg<1O%y02Qk=DEz&rL$BkNLs&J= zRO9n{u={Ealrw0x1m?F39|Lgp&;ky4XnVp+2=#aiOtB|SZ?2;kV2V#QG4(vcOwqtz zNO)}S4=v=gcn0X=HJ5n5hvn~?{D6}qDauyD4qiEY;WaJs`Q7jLWlTykX|6*8cJRx6 zlb_;$_I|E!`5Qsf!HDrt5z`(c5EZ63?|-#mfg0Gh#Op`Q&kME(GxgoY`VJBA#xm$C zvVdsO>VtziuH+ec3w9WPj_r}*@4HQT@b}->p}t+8zVEAB`5E4a`}(-vClkC*@V*^1 z*Y6Tyb$6||5Z~_K{hsPS{QmB~DQ|Os3b*HfCBM86n*M_MDR1?2SDs7x+!e5_sjqHn zu3LkWrmdC>2h>aS@zcfe5NbZVJm2KE#PqJofBXCWiar3Dz?Zr_ z?^2#!1&F`I{A_gd`^)?6P(qJPlIHie_)~tq%YWBCckSWd?(^#Q`%8XUu0UY>{OIq0 zroV;ggJEw58-1%kYT^41;#&W|U7P&(kr4~~yJS*=#`2B8)c4Q)-?hhGdw=)8yZ(~= z(ti^DSN-+3{xE^CZ~A}hK{x$jS3w}neMmH!;NRH%mAC6D7#`?m|MBqk5J>V$qi(1D z)f&>cY`*{X-?#+SRzzi8g7>#u`ekN+XiW>{A$S2vlujVUFF(XB4g2E2s{Y+`6K8JE zaao{DIXY0Svx1o#?;%453_5;Mm#xpI)hNw1>xkJOH7IY1e+}@)#+xP?3vjt0iUgQ) zrpO*i2X~SlQDM^6z5@r;G#@Xa1uBgBSj1T#6uc)Ao|n{##k|Tt3P*|8Am0~%&Pgn? z{8&3Jk;(jA4;Nx-MJRGn*i_6!ZWV4J+G1rDvTz+ysv<7ju~-Miw01MG zDT@Vl&*CF1RP6a;2gRoLa`Cw>_O;g*dsTc<-xgI?w5h)kM_%;fVC4pC&1fRsNUj|^ zt=(6?uIgFGU07>PUp!YOAH2?gl=oJBz7)F$Z>vAo#pjb(rJv^!xIHpdPOH^w7E?ly zzikV24PFlhz(1f$X%~IE&lVd{XbU;W-I~{8lIAjYq?n6*Q#fzp74mK2?8O4|L*YD$ zMK9OI&=JdR-oBF6M<> ziA}Fq7Vbc7Rk5nQSFy8-PwG`-FN!1eo9`AA#pZwYuG*2yEpH{`po0cskcu)Px0RG!9uYS%v!BhtJi9=R)5@j zZ$0;0-Ek}U>JM7e{&W0)Zj~+mH5e^h?e6LPPL}++n-^8-_-p#+>_@O7es`bW}J-4gIZxD97-S@?E5O%+Rx^La(({uN;i@#oW zRd?B1c9-3G_w^-%c<@8}Y zn~{wWyXnja>du^hM~cJD30I;c79w^u*c%NzSVg`-J1tAOI8?x^3Kjc`j)p?7L>%P) z0+>%pSs|CaBvkfcB5RBU@J^6NOz!y_tdWK?Ox`GNCd_A+&SFh$(yNeDrwoyP~6i39U8MM{OrzDr&wG3z$hO zxTSS!QVO3z;sbZrpS1@(l-6_=Ex?JUDz6||&!+BTM)^uaM{h2ip8DAaX{ro$eQIu-3QpT(#;6Q&jLy;D0{+U2e7`~?hsA6!`;%w)y~(m1u!=b^XQfb>+hfGLy6A;^PKNG!u~5dHjjKf&2=;@aQwqc z?t_p4uxLtB5mhTOEk8xv(X6gdpj^9hqCBF19nEq4%rG(%#?tyUeFi@Z%biz*uy~LC zP^pSQ(|-uBP_;JQylR*Q>If3j-gcEPAd($W#Rd_qd?kzEP!UnZ?I^8eR7Q4LR?uFH zt~yjCsN%Mr#_jNWZy}~-S&{H^UsjcbShlBW6^U44yC$CI@nK#8VK_$Qo{yoJ(kcRf z>L#9sD8>H3+$n-RILHpg`lH2mtBNAbD*~Sa$Ha_(Uq{$~2}}LUGm2^R{dA+56%Mid zQHN^S{N8tj4eU@>WBW$Q z-d{lK`<&Wwb*M;LRWdO5ouWPwV(u?|pWDrST~#tQ_lI$PA}4eI&G)%z?hmR;mgc@! z>(zJrk2Kj3e`>3hpidg6)S49H?NhuO!fD8l@G%9U8GK=wW??0{aI?)@!VrBz1rT z$MtBtQ&l8|*QdUBstWV0xIO^|EMMXKoCsPSDjHOk6bj4d()t9oxxe;(&YSyKze#hy zu1};e_qV=}dWv`->sOikUQwTaXl(8uQRD>M(_g@sPl>8Xt^^O7Htw6XSHp~Izu4kh zs(l~6ygq#4{1uq_D^q{p&tR2K)XJ>olngPVmX+bkjLCwUv0QPYW;1N7P>BxjIkjn@ z`mPV2n3~(zuk1aXygj!en%Xl)=1%Qm+Ov;_fcp6Frnh`#+iuF;#`j!*(?_$vC`TWN z*Z-SxHuu5aUk+njK8SO13V~0(f*&fDR)T32)qh`poIBEqI#evH{5%`@3+Q1RVLkut zI)O&Aj-w$7S0mcdBkV&8W^N2e!QV8P^-!$i^wW##aESBu%n=K-j>aLaR*FEBB-pN_ zqSioOwQbg0iEHAz`HgdbVnApG2j+EVo?iq?u-Y^4tAGEEppQdgWh>lU5S%-t2Dt}c z))lhlN!-Us+fFMj6^UU{X}KZMZZ1o+ZhB^Iwz5@JhqlY%HMn}9>W=&fF{X^a7qQ>XQjj-nVh*YZzb}ZU|19cbR$xA*`N;N4* zf1GbqoKG+xeQ}KTgk$#hqhb5@zS2FF~fdi-KC+L;@nL^MMXF?dF$|=zkmnd$G!?L!tylKGhA1i zwpsv8ZFLk(<6cvLLveQx>o%U@nsrU()JF;=)BDz>#+XrgOci%ppOpr^Jot*vYkt(?apTu2DA9-6aY_f z{*&=<>0+l%eot5Y-SaSiuUKxUB1<;2#UioSsbHtMzvmi%BDwb$g!w)^dG)VpVPW=` zKhtvGhrZX_VtLGXF?I8=%7cBae$oNX@jPkjuo-`wYrCfl2!tgr{lwAK3ihIgfuOzB z$iKbc@qO;$e&p^M|J@koZ+|mZaC}s%*zRy2&Si&`q7Sz0*k%t&c$hopnwb6LpU1sp z>eQ?+Os__Nq{;;)jtKp|!d`R# zs7V6;gP#+MpY7&Kr)*$-ZyfJu*eXT5#J`QBepXwo0GEUH5#I6nK9R$;y;x3tXU-7=2L4fcomZZRWl~XbEOT^fgkZXro|IQH-U_xaKfbd9YAES=g6ObTp9hs-scg9t$YD|wcr$Zd?qvL0FVCOP)P*{ z1uLKdHCx2xhJbd40zy;{< z`y8ksk~4q|$Ri8|yh`$c;>%0)e^3Q~sfHcq zBv63NpD;rFBoh#z--Q|bJ_jd(Op_hrDi;bA`g>Ayoxnno75+sQXy9dYO?yE9(R(?P z;@`a&oovL%^r<9AOeZD2hoipc8XODWB;M!fcUpt3;6)>SwuSzv81pknX;(+%?`x!? zSZH#Dp+t&_FU-$ellA5O$EKWr@PyZ~COP`G=8~jf5AeMqme-emdTxb(6a0RL@l2e< z1pU|4f<$3HGgwMeVLmfOW~Lq$Ib$5B$q&9?;`=#_(62v~umrNf^h-%BtiM;ldhXro zZHfM6+Jh0`?*j1Tf7iZEyP5Kue9bTUN;KAYiQgZ=31BXvGWB!0_EqzL{4tO&3^Q12 zIAXh`Cf^EP70F?!5Ep%^Z+HM&{=d*K{*oLcAOn*;*H2G2F16U6{EGKMXiT!&h z;T7{`i%b2O_Uo8_VCwC=UkM;1eC|Rv{pv6KkwWhw`s2wK>o?QCJ!gXXnJZ#p{!Kq~ z68wG#Lv%%Ae=2eOt+70RnfYse*T!p8uA#td(++dljMrE;{>zCbx*4~>y*K#nxm4r# zf7@?Od2HGA)4$3e1uRDg>*w_{9#VyWO}V<5eu?#A(lz;#l0=u^S-`f~Z?>25q4C<+ z@P+S7HP%mw{mH^>|C&#@YganJayv*;{%X=mjJtip?|qHw15CGnL$dEBkTZ_2X1>$+ z{r3y_0Oyb3JAR0@l)lCf{qY|maEVvKd7I*b2OFT7aUmmLzlWwXYGFNo;dl%r)Htp~ zKt6>ObwhfBK-iFJvPMF*0M#9+IS zY+55cXa> zvNOwq-m3_6H3uAlo-Ce$0Fg!Sy}J?Uz4sjG-G%=5UcC+=0qQHFzK%!}k(pIQL?p*V z72nc<#l+T6)M3UXG(Xe9^z&9`qXURZEB>TIn#nBwp+k+yX&$D-h{-P=qr;9VDxUl~ zL`sVX=9i%(9!m@iRg5MG3_N-bj&ebCAdF}y|;vb zjx=(vD!U~hbejEazBrvu%y9`HI)#|C62KpS=f5oROJ^E$(;`o25p!RnL}wlI zRHFLh%&#RHboTjiew)rE2D(I#&OHXM#E{M_2C>A1&bOaUGpBP*%maMZSccJPdx{4B zVNm}_T>8~mNzR3Siv_<;5xDvDxy@PsmeB=`&f#?E9*x!De{t#7xaeQJx2!+1{r8Wo;d%e&JoDQ( zkNa)gaRew0o{&LUy<@O)_o^<{4mixoM`mel%{_^fG|Epi-|MEX$=UMHSYws_A zf7@sO>ev42f2&_^{Ps_0emNxla_jo#++Y5GJ3p-btKa^Wx4B<=&iSqEZ%dB`{q_y|I}YoY>umIY z%YXf>wfSHE>*EsqSKj+8|LbQG{AWDxBBJ3^Vjje`mNvjmziJv9nb&ezxuzw z^Vdng^JlfzFV}zdOTXj)yAQ~J_km7k8N@zw0j_i5w}0&K{C67uD-Zq3vw!7v_Amc! zpZ&E@r~lf={)$r}JAUqWZFUy&muJ7+_{)F$XLG;(P_+Q)YP^1tKvu)n^S|Fs|gYrj|eGrm`A|8Sc%=-Rbg`sdBWrTy#q z{V&hYe|d)f_53e=x|{v!FRZ_k91s1L3Vs(26vlMb&%TFhR}N)EU&pBbXYIk(Z`=OQ z8SyWU*vBvL{@BiEf7@#5Kj$ZZs{QUA+4F}R-k-6{jQ;10>p1k!*ey5b|8Rr-<=CIS z#d0h3FRy<2pS5ARHS^2=oB%TY zjgDvk%4>h+|D0jnpEIoIHJ1Jt_q-SKOFz%oc7D@U(WDyB|rbifBbLdv<}Js-~Zdb9>xC5#ee&c z|NH+er*Z$^%OorQ*T0{JrhooZN!%j*zrIP|RBJNVP4++kq1*oS#ea@JYWjbDaurQg z@Bc|c(Er@l>_3mS|L=K!vM2xf1Ho%5OXPp1$yN~9|HvUq(5iohxc>Z7`*{WR^QZmX zAH^*D|6j-+Gtjh6SKBwfw4YM3+w07%#UXTKvE|Rpte=agqpo0(N~@eK<(`?hr$DD4 z`WxeE1wzMH?;;7W0mQexso;{9vje1$m@ClFMu&ZbGl>&>D?_h;mmY1AQF;zk*!fQg zc3(MpYe-x&;C0HzX|CzET3PP$aZPa&SK9V53{AqOuf@3$3Rr+fZ-RNO2Tv-oiIxHMob2=_6Z?~#$iMiri=T6U@R0MfSVdC81W*j z$puTBZ`M`scL4@}A)k~a@f`<9gIK@!!=A7)te@B$+i#pBq>#1>)56~`FVQGs>3+fD zux#i4Rm~qf%2AT0b3<`@3;%wh1wI>aQIH;maY0kj@=14`&b0)QAa$OnQyZt)KAut; z0XdN&2!yB_kn+A3CM}HD^f)b>-q@&>k#Yn_;B?XRa3wYrHDE=j8w7zF5>GkqV2qFU0hU3zDKu}w5x`}R~FJ3OE2z0PI z??go&6>+0*WKtH(HXvC|=SnNnr-Z6)oOp&UR!l5FQ|`opC!0WCk{Dr37_+RdZs#uL z0*I!iMEUR??KBET)7~fJHQiZMkFF?Zc#7cs<~HwtMxzy@$3v{f#~H+Jq&I*@JP0OC zU;d&Q(b6Q0OB^4snV)hzLlZa+lNz`RGk?0 zkBn#yF?aYpn84cMT<%>&vZH9G4c%DK5c1VA@_cHxf5lB_v*xJMVB?E?fhudBH|gkh z$$HG>B7!|=-Je>$ao1}4whHV7&60WPg$!t3kz@3*{R(YWij%3cNr$nAy>`C&h)q00 z)0ERv_I`Bs4vET;23P%7wu8~1giXJsw}OK9$#qTA$4RA|Y0Exy$tw=#4v3DNY{FNK zjd?_^f8m(d@%1ZJ4W9KZ6-NP1D@ry}j=H2>Olbdb@O810QEo<=- ztx2L6Z{nPLU6x9%$U^SsxC*Nj<8f6z6S)%A(cV^uH4TNTG`XsmkPWh>Z;Paoy?np# zWDtpJOMv9%!HsVOOjS&1lt}VM70%EP{DJgMe~so)#$Xy{*aD(CRT%1?Cyv3kT)}@4 zQ?_uz(JqDp=w;S1B?od|7rE9NN0H&GBXFv|3EnGDjVsnIiPHqA!x+@nB%Hnk#?$G? zXDrQn9Dyd~miFdWV!V6C$10mHPJIq?9DMGm+eBZ!(jHmN6@^IT<~ZOww=$%Y-@Av*j~A7zKVTrN{yS)+g~ zTZ?NsU<JtNND4XQ^!4ENG z(ndT)p1RMPV^m*dYzttt#7NMmu+VW-e;tBO{S2a9_tsPYT%~$issX*V5hP6bT>UTV!ZdyhWJy;c}|Hq0SVb}8sJLs<|RyFA6js&9xDmxjljG}Wbwhs z8$)wjnFgJNIdJ}ZWqVfkuFc#&hY#eQtxNH_AM5+0nq*a=8J|Cjga%s9FteGpe;QpU z#RQ-8i%k&$SFRZZ6L$;V$x{`9*WTPEy%<>d^nS78n=YTd-X&c){9u{s*o%$O)jX;~ z8`B!;`)xwkutwN)8R5t)uJ^XPR+XJz-rk&3TD>7m(<*P&@?A}hrx7Om#h%eWXIojr zk#f8qP;=M1U^Gk54s|6{Z{38FfAHl&28uL?%O3e0IKPy8qq)9T+NpDRmyP%`jHF7^ z*6BOVDxlK>h@AoZ5Qqu`_`%`OAcY_$&l7@ zlI0zOOVrtacdCYVUlcJK|6!vLb5q(vA-Aqp0A}yAOzvW>qMie>xpbSti&a zZVoTpcHh0eyx_=HfP%M+FccxopyG_xCU!!J7FYP;2S*SY&^iXWQl8V>mI`vE4jP|% zYf)~$30(GJuC|1QNkqX@r`;S9i{6bdB&ZOwWL@g9BCTJkWLjD;fM!GsY;9eE=3AL? zQ7S>l&iA8eQz)mPa#c*?fALWkT6u^^(}zpcJe@WzqLb^n@2Xx~m0xAN3p#XJsDRZ+ zMX2zjO=ZAQ`FlXN@5=zjwj!ag;Zz359K%LIUM6wrOOG?J z^$YaI;`5?NJQjiTA(5umXdKz?ZU)40}`a|?LUdOp``W+HrMCc+ufJftUtnoe-(C{ZjQ>Kjs#I2^Q~kO zEk$OqL|^xD_oD(|IMSNLrG?rQd#-MJ1+&ZT{z(+GQGUP~ABI(HC$p!!`fYMrZ>@wO zuhvnQz8jv@=JNt8T#BmSyw=g50^w+Ubsv+7A;7S$9vN?TZPzB?D^PYt@W*j7Z%1C5 zbX$1-j+Tw;e`%;1layjBYcYOJPl%xhV5V5QJ>FPkiM3qG(OH>pJKZd%dL zv*%T&>pfz~Q#U>)J=;g0Vn>X5_}x=m))7P_01}CaP45sA#1vP+m@~eTBJOQVr>Bte z7Vl~8L{~rsE9)|w_lfnrCgkLyt35)vz3ds$@#_myfBCpnvrmmnxIk+|PwdmIY9o7* zfptm3VeLzg?}B`jS)|czHS1VRstt7P3Fugw*rBPz?Z=4kDuh7`pQ3PgR?@WrblQ&j zKyOrp7GhYtypuR2tYMyWUjUj@O0qWy@a)&+$_4>e>yy;Q*c8c0jY4$a_6|KyoRO9^ zu3u!~e=gczuiR?4rOQ%w435B;0B>*^+M~Mk@ZkO7*pd=w+kGG7}b9dwAIdQNI;O zIOx|$vsTy40?JYh`*=PicP`RfR-`Mnts8A!E6S`5aSWZr1GifT=r ze@jsp^>lMW8X6nC9Qn9GWnRTZ?LaSme?LMgf&naj$TcXBLSPmu*0-D;`>piFb`LRtrrxh#42xz9LMDuHn(Py;0_9f{ z+`SlxboIRhTBHidQs)ULB*qi&>WR{tL`H-jj;hW|%O@09prdsvO*Gr&g<|7*e^*ay4in!{chR5zirbTiAk&8R^yoj+qo?|4BZhCpB%O^VwFhM;PJbXI=XhfS(} zRASkM>Xh2hH?zDwg z1DWZNt(R_WNVDwhH@K1(GP#lzT^cuu$y*lDR*|=Uu-sAAM_nw zfFNc6QDu`OkF(9n3i~vt*e!{dHhkpDTe}HJ(JTg%U?s`;?~7$F*9qjDqtG-gXu8v6 zuf8x3!@C3ry=yn45O*{tRM+xZ@>|7Oit&^t*W%HhfqDze-1{75EyGaz~+oe zMqK-0vui#B!wKpez97`)Xapflw#Z=e#7-w>K={f10`J&8O}Vbz=3Fh_B~m*ET)@(S!#rL#)dhA^3W~7 zF55YxR*cUApwlMQf6?d2jpJ<(_>S}L?4sFwzAyE9${jAPG=4M14h~M*#FHJ$gWK&d z|M}Uu(!D}IQT~=CBA#M~rJRv*z30awdgz4f@QR8}>eTEML3EYJSG;61E2C0;g71z( z#*R&-N3D|!CVW28+RzpI`8F!Js#FDfSH4kP7WHa4X62Ojf1>Ot<(pTr0(cpmS4D`E zj$}#`&=gCkWze~zIC>uC2xh{Ep1u!@+M^QzW_gP7*ALZ=J|A zd;sCOB}0BY!OVvT7;?P1MDhWrX?#YK@umzfSe?qPRC>G;B-82lqnq>ZD5#+W& zS%~U3&rV10`6gh1#l?4s?Hh1^;p?l;8X`H2C`hR5(yRrIO#C5>Z_CBH;x1eWjY|_= zqr#LN)ZybB1ztoEhp$cj;F3Kl^gNj5labxR$Ir$5`%oZkb0V1*{oddBRJM!xe3H2s zu@QRsf31^($`MciIuJl;i(}ChhYx8Aka~qKkU%xk_r!8H)8BGVR8L)34*7IPWXFuB zJN4_GM4q5uJzeZ)Dp`!PC~&c|jd@_d01kVHUJB?vUy?w-PBZTGp{Qg=P!Ght$IT8L z9yAE&7O7Bh-285*`0!)}1XMoXGKlzLS{~x`e$WIrkBPkSfth0(8e>Sl<5uR4EOb6rB~XE=2{9X|mtL6dLi zAPAs>>lU4RWgxDBI#86l*WqAg>uT&HOPk^;gdC>G5wQu&wk_>gX#~z^K6c)}E}(xE zf7o~WZFn|Pj)Ssa2qJ@fY&OEh>7!#%7F>M9!iV2=uen`M#m|$M^RRN?3GgT&Io}7G zktgaPrB9!I4nW9viH8E$kkfK*`y{+}Yij}KYbw8T*CrG-sKhvnqWqrHg`ZCU%@74Y z$3;~U&1RkHGcC@Yf+*_R=K%_y)QVBhe=3C2ze4I$+ahLZ{I$OzIrK{QZ+hKDcce|ewT2_Dqp8TP2FFI z4vwiCp^%DRcP~UI7+dC0qnw# zv2GFQQbG?9}|&Cbx2dwfYWxcbTPT}Tix60ULpi|X8AIi;X4fV@kb2t3HJ z4xGE%4QTkHTd^8l%m?}IhOYfwrR9##T<)yi)N;*EH?a4Xya5NM$2s&=AON;QUjxG{ zY~^-aP#at6@#a>? zRIsYvFxJ;GG;0f<%|NlM`aR`5 zFB6Cwcy%oEMHF=(CG{FJhDn4_+mPIN-GJwKK+R(=9vcO~fAJlmOjG)q({R3S0$A9= zGSyZYgu^xj1G6@4kOf4o2bOOm-NcEm`VpM+jc+XameIFDQ42P0N`Ab$4#-UK4N0~@ z=OHYAVPQ{#hY#D@>{IRnHX`o|%yHu2%G$w&oGzk6RxECLGjSs(2vSK7A+wte&30-D zcJ__m=Myg%f9Nxqv3!(4->DDkJfzkS6_E~LKFBG8@GmWKgrrUr^a)tE(jxSrj&e5f z_lr`AAh-URk)szG3JRgCudmaR^rr-5ci4AIQx3@&Ai?e3tD?%=Cq!^iD4+54HYO3# z^*3Lu6FbJyQrN|LM3>a0gZ56>~g!K%J zRdv{jUm8+d+ha9W*c|{t2ob?>B1*x3@^Zyu3zc?$p$Zqr!|iaU&(I{}qp!}?#dy4G zTk?4me^-aOoAo$GDhl#Op4O)Z8KNdb$hc)K6oDg!@;$rV;3)#TtrG4!(b3C zlj+d2+Lc*CQo@2B?QhzI3BC})vqwICFh^A2e;*#QX7el6hWhCkIMqrCBhMtnQhz?% zii*&9aUFT~dNn7AgJ<7P3jtLg%JS)Vt5GY{9LmCUCDN2jK*~t5p&}$6U961HVu|#d zroY(fJQgc_L)=eqEnt!>=G9`uH!n1aj|*e-ZzDc2 zjIlmYZ{dqkH<{|kJqxC@q(>^(MLSrpc}v>w+0 z!r?BkU?&aj8zZ(4eB%sC6&nc0U#(VHe^l#U>F8N@48tc?yVr3-_M20bKs1DeXu9=+ z;YMjkd^By_!!-qEsYnVJW65h_RtNGTfP<*8aufpn4grKStQ+$YMjk_FwT0)sMPL$T}Bpggu-)M#ty!@N#R9 z!0l~Hs>uYe3^VAd*hroSp@Rsbe`4`dm!ZHe5UP-p4fUiRmJc%ER-T_RJ-jJ%C@8aB z=t8#T32Cz&HcE)%cXVNs3Zty|lM`&}&l$d}}?{6NU}PLlm%w8gseTe^6OrMF;CC zn@Do5><}e)Zsf@7M~mgRPhxjb@5kwS#grD(em{HHfm?0*54Pd^!lyp@1{=_iSXG0+2c{+|4|E)@GSlcCg>kr$r4;>R4aCs6k3dHwB z@)5>PT93^H6ou1re@A*S%Mo%^b>2W@g$%zvm)(K1C2=C^x8V8$%UA$m1>`T| z=5|DrT6ddE6}A=-6;TyI81m_s3Z61zP8p}_9P^;p4WeEHG4moQw+!pQ21fr4Kl&#u z=)e8P|M-6g3;owvA^SI02e1fB%y!$5i&jKMJDn z(N9fn#`z~_PXAZA2)enrEka}HC$U2OrxxH(=FLxf(o)j8K4e1v6I|r|C}q}v;+C|i zA;|*C>kR#;f7#z!e*ct#wPNJPm-uHqKjjJOl5b-g!M7CAf37EX)&EkR zHW1dj`mu;09r!)AXj)|dDbU>hNRES-S6}!_QT?a*U=IJI@24xw zWMs>0qRbQjGk?cFt+fB78Rng}AX%Z$eL?#l1ZvPT$c3Tif8HR8BW;RXAyYF;KRPAN z%Og9r)7)yzEy~eo=>i3TX%hfO%{>YQh0;zx0_7~kK=Eb)qp6>L+gLyq)=AcgBt4SN<7a}R z5k#Usc|9Uke^;+PhB>W|1A9_K0w<~xIsx{ULGZnMs>~=$`Cxgy1-46eO`v@ai`eg% zo=}+w$1}v-++GN@o8i3fv-5r~wQt)dF`qg&s(f2RQ6CWntsz`5D7=KCbH~S&ImTk4 zTX)Mr1v)3wu8reCX!XHPmIRcz+ecy!tNr3ZNevtjy1u`Fw`zI0TClOC!+W^;F?HFqndzEP@KINdV8l=bdekx;s0eZQ;%&#Y zwJ(kue`+<9XW$c~;vtAluj9q-hlmFa`m=;69>Bo*-hwk2r>fnWpWH>ki~{w}M}gxlrhB;}FSo zBp>NAcNH#1-89pH`SD_S0h6-D6rZ0fe-=%7Jdu-;C;`{9nJ7x?#60r?3ES}YNJVLJ zZOWHe&Ab!n0}e3%D4}9?vf=c4Xu#Rs27kV@O2+}Ic{OMqjFQ#L3HZiNJ1cfJXm@8asq`{r<61HxFc0jR-icgaIsG#mJ_;rw}h)aGafe@DZ8 z?M=y2X=o=eZ1W@3aLYMi@Ta|JX2V=1Fq1s5pq{5D#Hk+agqjX^J2&{6@8P4mH0#G2?Dg%cjmHEQC_*Q)9Pv70+ zp48IS3?QOWZ@+ZZkJaf$9Ibg^J%oZfzVL(3%@@BXfG22L_*U{D3POI&tEXag*e z*;FzrANdV06@`<%&{k~Z(U;cXCR%s8m{O_Qqh{*~<@xF)_1SreZshO4=el-nvABx- z2b;^%u%CNv0Vt0&X6RQ(f6c`kN3todc?Jz~JSgoR83hvSUU4`x)|{VobY6R^eA|Zt zwz4+}_mPwc!w6As86*_Pc_On9^|2~=ut@zp@dp@>n_^04mc>aObyp9kdvP7daCD`f z%$~7ro;8_aD+{*sh{!}t&nA%wda6xH$s?Jd6zd-1K6US9LZE0%fAYZ>M`wJ68oR%j z(2!#;AXoXh3!{uy`yI1#|(J=5j!Bu4ls8Ocgk0*2L#Q* z#QV@C1I3N9Wlg=I@z>SgsqQEc_+%dEkj!i@Oq>7&C60tLf9=h%0M^XIDJR%DP0CJ$ z7Gvfx2rXuwQRhksO`iv6JnCfW4&KDuZj5+rYG@Hd)X%AL!;g-X+w+ngDlnA6_B%(c z)o|GaY%|9ZK}7qm??ljR0ap<^pIbqM?Qk87s|;;J8v4;;+|mjH{VW!v(s%^2XMv?< z_B(q=89HmGf4jhj3(;ocx_<)-;OihLkhY?{uoNaal_cYlolKmpRDIO7#(t5n<>vF( zzLMTDNAg4Ed=q=<0V{gKRr-0bM)V4KSUBa|pGYOEldR_tb~blU3n_*Vf%xIdc41#O zFDO-P`0v!#kS86*8wej#w=iuVsBu1pnucRrVJ{?Xe_S4ic^%e3%a__18 zzcF0kf4TB3@TIM%W3ME3_E?Pruhba!&11YsNv5yWEW(Z8H%-2&50u1I?~|TJdZlF==Kc{%b0;(NgKo#4Ztw+MfMvk4aWFjCZ*F? zf2|(t_(4euK3F8{?z?)_#Lr!UaatOtC}$Lm80F7l&^3!~)R!m2y;$GvEUAx3Uu2hm zNn~fi9K6TD&}U^#xJ`6WUe!X2x!k@hwhjsg1ek+T*CLA|^)iiRj{g9)Y*ggs*_M9= zlp$nvI72H(<1Dk`#bOQsy|G%ndf46GaII9kvw%AP~d7$jQp)Qe+!Y~%}r2$ zr;DL!-EhBhzZMus%?m>TtbXn|v*WI$*0pWY(CJAx!GT1%RzDfh6+;*YA>=N@fA8>_ z3s;WAmEU4tjm0crut0`U#abGBC>aWqx%1u)N&tcJH`@bScd6s6Gf7YTKvZmpffWHP z5ryy&pAP`kZBdvR_pX6<0H5hR(4|}B z4_|C4tdXFYB<5pm*Bi?jY@u+}e_+Wl^ zU&PJtHJ^?;%;H88b0*md^uSjP!I}|;2okC3oxwBbwLv{mOE8iMn14^OuBjsBmRuZF z%sAKZQ~lK7k&1rGB^7zR*b+iLQcju)!uBW(-0bcA<4Pe4d{?5V;wKsDe*%l5iyOD< z1_kDDg>Q{)3*toaLEd0^+6&SS>n>_51luvVAyIXp4nTX**;0 z206y6X)Of@}s7eEt5`dUDa&u_CCa^ec z>koLQY+;XuX5kRUxC3jaTNcvBcL{uqHjY$Ns)7OZ_BOEogNGcLfyZZFt}?>S6r+GD z1XjI}JkQW-Lb)CX8qp(8E?W-CD8?M@UA`k_?Tw`y^fGbP!mWe}e|BV0@^^Q6N%2{` zk?;)PAUYk5&iV=T3REwO4Pu85!o`AbRH)%ncUXfpNuUc-6)Cq#!B=Gg<3?75GsqNs zI0w(l>EbCpIP~)y=_mVHQWXDw}91JKx?Ju_E^b+b5u&6BGizX z96z+XF#L9xgzYLEe-ts3v4P;(@}zF`sS?g>Nsol799+oK#E%VpPAd(@`47ZO(96+|+vG%8n!So&3JUpkI zVnS>q`u^FHC_l^&K{^x4fxLn?hVA4pWW`L%Anf6+MY7Z2MPW)fe(k(CvtgVHRCJ2> zE+*%r66jZyeLYIHuU7ak6xxR^D zH%W$7G;M;ki|T}Mk$oSFDgn1EX3R|QWX313wTlcx@HBz(s|$+3`S#0vm8+{QR=oAk>1Ez)4An!3wlt@Z# zaHDY-yg&uY3cF(wkAsu|3MhR&s^I8zaR)upwLc%JAaxwtz|Xdi_ZA|L*|`(OrjV?? zqjCU#GN1uE0e>;*HMiz~Qea0?!2G{B}PISZ^fC?pW z(52U52V7as=B|kVW z9wwZk#ItjtRnt5sq7}Z(cR#v!M!;Nu9gJMS<;qJY7I*CW6j{FdV={6)P2sV)j9Qkc zUEp@MqMu`KUrf3bnLvtgWFmPHuvMykf8EKb8TFl-zB{^Pnlkq8=#PQ5v_nSzD0qYT z45_@c#tV!QM_+h_+Bn!LqwY|F=LxOBos*nBkyr~7b`8&6;~wOJV;MXUlEeIn39O#Q zH|bD{&C^RD>!MP^Dn}gn)wf#A1P&~4Q&A2ONInwg@UONYVJ`D2F9OcFgWjhj~Ybbz8WI*MEyVvGi zo|`k;8?z9_U>P_x#omu{VEI69_-60AA{D4f=xRAOWS6k%`;ot-qa~)DoA>14KjAA zXgirLtMSCx?$)BXYXy1g`n!6q4D!;$!EGpJwsN`#Y<3t}j?&U_nHPz3x2Ki9Xr^n! zpV)7=AvaXqAFGtal`%azf5p6rOdz7_^r}b!zw_DL0csV_NcLQR;vg!(Vsc@FOF;)oCZUo2h5}2^DX#ySuMA1>nA@#_I;uq zEHl74p2gmI7I1=2l^aIhK%?47Km@fjm@wA?Ez&Z%4P48>W^oBP)ozDD z4QdT5;obl-?Iw^4e+w0KdFBwRZ~<_J<~3MwrvOxuia`c%0!whlfSJ~B^C$cB3*Xx4 z3k@{lCY!}-aDA!I=2C$lV_1PT?ND9A-n$-CgC^PHt_k-J!ejasRq4J2MHQ~Wl=O^B zji(d?;u~WjsJmUL9V0rN&yCH+QiI0DS-l?Uy!GEmb%U0rf5z&5JQJe}mrwawtwx1* zXkn3l-iJ7xtusX_s_K@X>oo#WyaDSptVvOxO*9xR_^Djd#bMA6=BvR}MYoy(Ob4#h ztdEW7q#S!!pVM@QbVcqPXdTB)LuzB+sD)R7 z3yNxm>ZGfH4UEC*}CD~0*zEbHpXVOJT$v#k;_o=9c2x*~n4n~Q+ zGrapqbjo|gPje&YNcC>d@VyE5%X6%r@j-C?f3Z{8b2D}a+f8(bG*;+b><%P2`s^&W z^DP1v=d<95?3!yT@`S_QskLkAtsO>}RQKy}IBt*i=7RO%{)&xe=(@?o-q-ZN+t9|l z>#j-HmPSaMFFV^!%IEPIy!8&Ri!Te8h{QWx3vN+`55Ah|=bMgx+I3O49n~I`JMzD zV?n|6)a6&#{*uzpUHY_GV;C>>h8j){Fr z+rk*#MmJJl=ziUNg;l*?p)Pjn2~C@uK3~1Ov${!%u`Zsa`i`)%B|#D&iq~K>X;!7J zZqm(rqdg|9%+1bvncn=G+4Hh}-KHTZC*P{}(e9M_>G%{|=KL|UpQp<{o}Zv=e;?NR zHBY;$x|WfmthhODQ*+2p$60< zYmmIf+f{D|BRtaCZFOv8Al>YGf34qVOBE6pU7d_qWtw@9xNG!T^y07iD1Q3+yym3lbe6@k^&+ns?G!F_AZ{?ZQ}}U&Q}l2w&%mrf3@p4y%Z>l zPbyysd$+z_w#``{=uJoJu4RHY?K(WG=W4t$!b=*Rr>kTDl{LR^7O$^~ppiJV`5E8$ z+NoUwpWI3#+R$*PIm`2V_>M?j9%lDp$kO;xM$zu!7CR;L=yZY)yHQq9xSUp9G`=+( z=AP9c^sV|-&}R_1YT5aAf6y9qQ8$asdf)tVTD-~nGMybK)^Y6WS~(j&K@0TTe!6z~ zsXm`4S?9R7^)U2~)pa*EPMPvT8R?FrmAh>)dsJ_>u>9e42z4_|F5~Ibf1ReCr7OeP zWB3gG@oebhWICKq+2IT4#jd^2vs<7Tkaz>bSlkcpCCBDE>6c@8}=-Y1*16D|D8-u9U;MB>!h4ow)2UaT@# zyZL(;yQi{#wqchjo9S1y^W0=A+P=*0w5T7WV%s!N=b}A4 z5j@M0~p1Dci3?Ra@JL2f^2EjQ76tI+J8IrCTo{>EiKF@8uH`0UF|!yTO5v#%%IdCt7^)O)_S&)2maX|a8=&*5l17MB@c+4Y7qcH5N+ z*13hR6l~Xq#8uYmwKa|iL6=>H?nfFnl-nkY)9R}G2mR`He|7ubQR{qj-|yWr)1QS~ zPP;do85MCfci9*=Yv2}ccjb|7s!a}4ZCCKaM&CU)RiL!nyE_`)TgTU|QMrEKeKT9! zE^a%n-OXebK5=neJ6>0MmvUgId^A1A#c1c7XX`v%)$gntlH=jBym(skV2*#BnCMmK zr-k;mhr^|*f47=8nsPKdY`k*oJV&eQD;z5W(5Hi2@opOuE|!q(U%1@H%kHJ`jLUt2 z-ixnd($1$@+s+yP`l{h@h&E>MI=62tPV~cg@U?putLm*9(Nq~ex0W(H4~F@2JI>Oy&)tZ0~;dvAJpMzyxctP+2*Wmb0gKswFR z{}YhPf9;=dthK@#K}4@D_+ z6Jl)9@=~6J{kl!6>r;vDreVx{!Lw_~ig_J|t9>;2Qg$6XAMN7ViN=rK+GAijYBCuoY&?d=K2bUO+MJ=CqIt7?x5?7`a(}ZU2dB4 zSr5-f&3Df*-Ye>G9<_P2ZiebP-`!W{fAF~6O|*%>x=c$I#rr|BPs7=)xg`rY(@pj$ z=4bWxwcv*lQFey6?(pa_EtALFvelz;yj|<@LRZ4$6J|ife)b>m8@_e(Qd7Fc7f%du`!AmKHu9zg8B0FOggJ`zQ;q)9 z&G=p!_*ttcdJ%)-vNccV`*T$ff8^7p-1Y37Y>c?b4BuyRkt4cS{mR61zY?%t`3C^o zb06;d^>*#6iC1Y`4q&wXsloHDu{fr2abC-V+|jMC&y3H?Ke^IRu4XLy`skS^!&Byu zz+nJ(f533C&yB@YSN6%H2KtqXb}kn;_8U%je?Vh-Y%~%8 zZcE-I$loMzyVIG;V}atqzUg4wZ#;b&%;(yeIWKv@Kmd>$CeV}k`lz!rfUHV@-C_cb61X$8WnF&P4P zF|T5qr3f=O=d0vrY95Bf@A7Ain{d-lAD?v>Ro;^K|GC;uR*wp z?bmA;It%+0zgl|m1tExgTTM`;wd%qP>B$r+A&N2yGQVm6ciB!`Ef45&d3n(1Sz*@_#`Ii7j z(=toq-}>Oc#}O*s()x|3k^ZJ@y@(V_9uUS4~rc zn&f;Mfmm3_y&y)Rt}IA(3z~oxbpou~AVDwbdqN3V$WdV7hQ^GDX6=;G0y6_|ks9qr z`+?3VMlF4kf7T5HuCA%iOcKm9#OIYi+)Z`l-IoC$_E|G7MH93gv!8Nfdd7f0!e&H2 za{B%r+h`?dKL*Ov&$xg1=biTa^O4o`jZi=Ko`)}U!y>qhyQL7Y~ zl`sWaCpSQx5lwLqx}eKA^s~LtiBX(P;N0|P8`J_1NHxYBjf? z@-%yNcRS2UXA4qVo%_)(AG};FiN_vqbdOM;=)Y~ivFESyM{h~la+BY>P0C$QgFEnC zwpr*!-qQ!JRe-jY?RM<2c{81$bm;}u2IjA*hztY~MuKdqh_D%)LW>2db|h=e6_*70 ze>9ZJ8o8d%DP_ex!0gKHAxF@P7nxcg5T%} z54R78GOdVg3{7AZyaYiI0?IT@_*%Qh?bjY_o>}{pwPyUOa{?_{+e;d1YKe#pi&{-eZegCqvANrJT`NQA9=ZqG4 z936FPh*KN{bhj#m69i;yLd7HPs)ww(_zI1Yr~~rOQw_1nsw4&tm~M}h=Q>d}ZG0-T z3w1Cr64U2Y$0A;~d`$|)1V%fppazsgY)hAi2J@|0-8r24D5>);O&eDTgoUdie_pLf z>*m0j1tKaErf%pu-W#n|UbSd~ILUMjz8;9DUMPW9xP{K;0cj8o7N9kfFAe3xmq0qZ z{TXY|W8F6&nW$xN_ucc}=ucVIefd27^}NTImhE9)w>qrJ_ri*)I2+6IdJf9f)-qn4Z7MAdx1_5#$TxO-Tdm|roU8_3r+qe!*o z2~8-Q-+|@QPEf7z!xRKJgraK*o>m)a1e2})t|r>Msj%N{fU_kMr16gFmXHd)T59Rl z;*5b*w5TqDE^=V6#75DTaF%bDR~B`34UC$4Mk2q- ziA#Mj>{#~$pa`=nbXkLe5m$?J7eRyVR$Prvy4mJnBK_iLjqeQdIJiCn zp6D9cQ|2PfxvBZAL_+#GS<}Q|1FNpkQVruw|M-%WXb56MG-i1JVk;(WJ>V9SGiBO( zaOBG?+*Ak{7|mOpr9SXl_G2?X`1#hI^tTW9+h_a6;a|A>!}k2*f72g(*26j!>3m@z z(ov{uh)|A??tB@LvCdMHM0BHzDEEi))zxy$Rl39=I)kZ>&hw*<^BeK$*Z#CyEwS$D zEVl`szxDB4*E#E!B3&^*NYOqnFwa-Ud+y^RFA0l+NGl9es03I|kx&B4xa1wv91eGP zZGPCi#Z4=eiH_>6e+lzPKPiYWgPHKK)SHx}5}+Kex@wJq{!`BX_LHRy|Hc!Oam2UG z|KQ{kXZ*JN??2)bNX=fR%k=WiSElZ>b?=vd`*|O~hpt@!KS030I2+h^W6UEHiZo}f z>R=`1hR93@HM&tY_*OI;nV{U@=~6G|XIrTq;FOxATqT;oghwaR3V+q)O#plzYS@3< zWn_{gat)aKq;ogY-*tI~&4T78&omH~hJXj@IiU}oMtY}U0(^O!PA+Sqdir3h>nyic zI>m_1WA8XUtr(AqNHD@x zCkX1j#+yhN>dGZTsYp)rpl%UA#g?CHMq8U26HB6y+9-0vJQnk?%RO|e&N`>8RC0^H zPthLFnHC72u-qg?>gd?tdPgHn-V25}#V)Qg&96oT+z_FfYk#0@(@j=6OduTu0>7RT zaOpA@E-)no&bPgP`5LTHR7wA`lz3VDMB-)aY2Uom{)wC3r|$Xa+b{glgFSD;A92Lb zvu`>6+y1lkFMasU&zE0*dY8&)X9uM^MtP;Ndb=VJBOufd=n^yb6$CDZk=!yModS2M z&eD|$Inm^X)qj4qB`s4OKTU|JLR7jYE>bkDQip+h)mTLlOJxf$2O`5FnZa5&G_{B# z#Lf}R69mN2tvs@Rc30y7R z@G-kFM#gW|@<4FR2!@t_1<7bh{EczhK#76uNdJMQIe(b01}mf!EszI|!R8%e*3;p) zPxZ_8@1gDK($m3vv*B<0xYE}2Jotw$HLW)@|Dij&{I8hY3t1m8I32xeqV{Rn1i-4* zvYne4QjCC3la6l;t|ACerqz*0gsa-XEI@pyOIxa83p^%})Qf)U%AKY54U7SChf*|R;s=IdPD(paTbo6{|k(cPT^>;3=V ze$)p~(Vua!cZ|OC5`WmF-#WR+RWBdKkN@A}_J2P}$ zKEkkI9zi72S?xNZn*pm1ZsvKU;Cn(IkzFP<9%o~-u+qIx41%8NCLs95sE>HQGh1~Q zo*B`2fLYh*x;+`9I1`xHYqL~B4c8hGk!F^yn=wU02+f|l@9 z<$pnL`s!-|ii+>@z(JA&cA}YE$fJzuUH^!OwjhdMk3a_j3$E77(5i1Pje-W7%K-)q`oHdo79*4)#3?iuP+NJr^_P$m+|-?IheP5OgsHQ z+xS27hkd;CM-K6CIsKjE`h~ZUePVb19d{M^+vQX>ihLi*q@f=%|5RnJT1-kNRexH# zLYSwqk)ND+xIHkBW)rm!0SQ>GC;RmOa7lcwC6?vlT#$}XZ9%2eC3IJ=>KLw{m;*rR&8(FD^ff7O2oFXN@Jxi)0K;-v%h@lxLV zQKWE(GG3wLU-+6Ido*(87vA6Y>$k6!|M*2e^!TymA0Gd0pZedwc>Y6+i}VY0_WQlh zRycd)>fnPOBiH(8F3CN4?a2sF>d`_#fh)JAK|CbHK^Ls1z^Q}tr-L{~MSn9r9@Q8J z(N+)WmH4p2KExm-ZPN7M&jT+!jdV`19w|q{7KHf3Fc@*bk8EH(#6I&-kzWXRZC59m z!#KhP4NRZxbke5DhlFGyst@{S7Jz*s@+3JlBBdr)BI;9qC83F5f8W$$|zWF>#lAvG)j7 zksv0n(!vDhA)mEQQMIUmmDbjYGv>7!w=7dFM8mwcgnEimj%Wc(4S%@|LGEchO%0s- zxK&dfshQnHYtvUQ7`c|#rq>XO2M%10Se=qKeL_J9;n7-S(gq4jFx}{d%d`S+E>7<& z6KanMMhoR_6hq4pT;lmWm^E4>9C<{tl!piTD}T_(UL&gI-*MKj+<-w#*X(7bq#qsq zBQspeQqR|c{H4F28-FW5*K{MDF|o)X@iOeI2gK_X%4+5_N*JLd0ocbZ#qiqmZv0tRqA2@@p|S z#7LLn`v{ogX$WM@Qe&RV1BFI|FRp=*`8_7_FhN4%ftJ7H?0-M({9iV{{~!P1zklmy zw(jGp-_L&9`qt0%@@I~{&!OjNv=BH%*U4CHlxBln2d2amZGij&c$It%o%#eI??`I=Y zYBxCT34dc4MDC(X#%;c{Es(iIVq)&^C}UU~rz1`3uE5>sof{${UM{DO8-3M22oH&x zXvgM)I9XnH(F^lj*Maz%-&7a`UblQ~YO|eJsvC!^8V1!oW>+b*0r>e4M2>*X6a2WG z4W0(YD9?4qZ36?gux?@I;^18~LWae~$t)yNf`7`F-KLNYUYE)hIkA{(1XGGk#SJj; z?X?O(w6f1{&U?QbedpbOVh#7`{^)IKk4Jyt{U_G^jng(t^dd_Bu+jg?SNi_%{h~fE z`eQe~UMX-QHj_&X5I$%6bsL_@LEg^U>CB`Q%dk9ABM)Vh|@MKS4+r!dt3CUt5R;I_7K zkYX2+bjei@7z>P}q7}f9eXJM~jA=|Upye&~>r4)8spmdo_8o%yiclKrK5wxgSi@=L z3`uB1?YM;Pkzi4u+izKc1w9%l-y8lFc7H6_f%_=%ql&P`(EEAaaIgk?pf$A(VwQ;M)AnG@R3Tx$v=JdMvgjjhIcM#4ajYp2@D+LC3<3S{2aD*fBj(=Hw zhO!9=+S$n8DeG(zQqIrXXfiZKwX+JoG~Trhq58spes(6$13&9nd6-V;=Fp_%ZF7--O)KagYPp;@E&yoLOliltgerB6@?nk%& zmE-%^X6c`%Yxh_F<Sa%!RYPok)WpIb3IPyyJc|*X~z>Ju3NLsHT{YLEu3FxzA`dF zehMHwI5mIpS+h%ro9!?fZmwyiwuTQP6}TLd&+X*~S3+6B$! z0p~woX7s-X;C{0$gRAcEjk@TsFG@;5wy-8>$gLGLTnk`zL_{So7W-&h8zI}S&|5L@ zz^B-&tQ0O9KF;lmH94LM*MH@2M{}F+oxupe>fG3Fe9+$SA;4Ul{>#rvw87-C$Wc^j z!OzZJ{8!l&wC{t~Bjs}spq&|9;3>cc>m(SeDLOwaF2p5lWTaj=HH~gso%&Y zznkrsE;0Gre*vV+!hb$-Bv)m9(*Xk{9P`0o{;5w0)T_Z@@UZ4DL_YQ7N0vA zZ_0Yk7Kr13X3rpILw=-zTQn%sufeDo8&YH-04Y_vb^$(2)mJ?&FZ z$a)gb^|hc5*pXgv`J8^*vs2XYM$2Ry!aKt@)lXUCKLwPD(1>?z00oHy5X@v)*@ER@=*Ov5-th?x z(h~RWHU#k`*$7BH?mM4x;7Q-LWlwo9XvGWE@0j^q4O7Bnn4B@u6c+$Emb42YEg z_tW~A2B@MA@DI&k+h|DQmm4z#NnA;~vc!n927lZW9qHt!4_ff=+ib8e+kw8FUpP7o z=%Qm9`9zN0*}}4yf9&B#C@z4f%YGQ0TO>hUI9pgb4#7|*A~SU2Y=Jrb`++yG(1N?< z8O|G+AKB?F2=VNm53%&}KKw%h!5f6i><%dHu!;a3%hD?vq-!b0b}C^557TZWNZC5E zUVpa8@JSK`LQA6K9*+d+S|^GNCmMd2`As3q$>>wAnY(q_DIG^n%^Nw})o| zWGSGGal_`=itLsEJ+V6=z2^t_6zOFP3~8wY#ENXYEU`^Th9I~u=nPOA{Jxj`D{0z% z+Q3%wd7f-x#gx4qznMCK|H(V@T@xvH1b+<5dz#7_2mRFZVd}`Fhq; zq={Zid_{L>3v0(ANDrDvgLu<08r$7S7(v%U+m~+~hmcQs(=;sV%T3(1sOZagj(^@v|z|v#Alu;a}BoSIE z@uBo`&q~B4ZXP~Qfq@Ui!sQvaI)CtcLz}zNS+cFdaR^Y{JQ_@!h7m5uN4u_t66E+Q z$06FMJZc)o`f{u6S`_r<7snySr(EJU@5}9=YccN2Z;nH(Pr2GqtrL*Iqrvlr8dw-| zrnu{AtbJ@Rpwxko4%xzyLG z^rfU7rHb7xfgs&ylCD!}PxL#idmDJEUllk1HPiQeBbP!bt3Lm z`O)t@>0207v0DugZs4KW9TutdF{E9M%QwQHQ}lczoa)BuggU) zM)GQd@*}&0W^Dmm(T7$IpeVsAUtw3`4F6e19nh_8XL}6l{CDG{G_^FyxA_ zZ{AA&YdLpHms72?m)Py~?5g{_->ie@H|=@q08fq0zr^8h9)Cn2^}bVWB%d1%+G&Xn z+vs#ksn%xBoe1#R-Pt1aPdK`b^nbwNf7hP06TY;^(MITsmqx{dB{AnKy?QnwAYJNL z;x>&|-~R!N9>G6hS<`6Ey)1#RKJp@Q^FJJqg)@H%-f znntT`&K7yS-mIcyuczz!n10OCrxjpMjT`ma4) zC*c5n;D1EZ$UH&R!|i%F(0jOb*}Z|0Z@78WZ~$NWa0bY0^tmQY!^!t^DzCqvyFGj$ zX<@J){&OAm@Sp3nhyPsXvLABCc-fD-9~VPJ_G1&segODATrJ0izTxhl@#{U@_A`F; zfm1%?k9xT0XZ+uA(l67*{SUn%r2n?UPr+nelYdCs&+QpP5Z2$|RP0iD2fVZ+b`3>7G#P}bWpbdvX} zKqtULKDR#i@RUvePrSDUjs5vvRe!!Wknje9E&t?GQ@NhXV30BN0JDLbM%t^+{tym#$4`fdXaRc@>5Z)%KmWBYB?T9 zzCV`Z*Ey^O()#`Yx`L6*dW9U{roVT=k@PX=vYi~{ z_$rC7$DjR+xx_bxT*!9kpYdz`Jd(84vR{hCW7fYvlH+)NkMsEpzh)}&_>@fGaCif* z{ulY6r!SG?`SdIQ^h)q=!Sp^8L^6el~ydqa)A?ND3i*_-`k_B-d(i1iU1CDn9&~_wBoq zPu}u=3-J7&Oa8wBmT4b& z$o@J)$}N%Yy`?;O3`U^k@;l&(?C(;*95e+1-wn6tO1MOJhRF3%1AzJe`FmG_$Njl~ zbl3g=_P_r>|Cij6;Qu&m^8J7RzB-qu?mz5JxL^O@cylWGeSdQ;%KX3erOl%|?f>H< zn>l@+h<(Y6$A74b^52?!{vZE5LuvHqAxiGALzM3GzYbACx65BsX#WaRXKAWt6-Sgfm!RSe&@$!yt1nph6zdFhSiUGbm_VBzO zN3#NNZ@1CnG*T-4JSGz-?gDJvB|b*PS&Ywl?bSBBXMY*9n#bZ9TvnN9D3jsTvQF#w z9TpsHF=S}5f9;aIIf{cYmsfY4?`QFLlcRCS_6bVv$aV{(nPua#L2l^iW@{@tFD75k zaq~4)#u4~Mv$&q_`Egm5EKtlO_3F!rC3x%5ki2F$b(T&~%syT20eXJDM(1?(^7U4NO5nk^dYTicwqG7w4gMRa}4(sc@3 ziyHzUZL1-RK&zUq9---^;f22%jrOnR5DoU$*?-M>xq#f6RdEi?^$v^aDZ_p^v*ubp z9Ovd>X5XsW^AZZDLet4$JiEY~n7i9)Z%+V^29wqPh>ka-iweh`UxyoVu&l!jtD~0b z$K5WS-uC$>-Q{=B`?_)uuWyTNzZ=iGh=}gGoKCICn2Uk78ElKfe(fgjg>VJv`=@@h zb$@q?E?Z{St^YJ@lccj(2kAq7)JR*p$tiq(se?`&2^=Uo=ZfbXiWumAO*%$J8@K$~twr(~NOW393c| zeb}sm1hQ_r+Mmd@6k1d8oNm~$5w}ulf`9FB$Ab55j=SANLC@iNy!k@gW2G(K<1~5i zV`U~+eVi@6S(1jl&ZDm{Qr${Zud|i4xu6q?8sh?AzF69967AKc@amY6)08tz&x_6# zR~d`Wz*Tnwmb>#?DeH;-+=Lyq+V;9xsHfv{6O;9Z0zL+q@?A^H<{G)4jyy)J8h@{Y z=kkFUXuo-QRSKu$lJCQuxY0|w-C|33_Ahk`Ywf)oo7*5Tw%18?V_%E&!#zDyu=L0K za7xVGnlwHymsr`CYH*n>3^$ts)<`?%b4Z)_04F%t+9%+8$x&J=B3#s`XCz>hhznY(utzkv0+n|?YLXsg0(hO(%UNQoTDX(fyXGj&CS|y9(CX^kvNo_ zS2eEgD!I2~)4IY?CEH8342(S;+V&X=b@iwJvYn!_Gn6<~I*r7zTk;e1tWviBjxH{3Y``B}e=fQDL zUw7DttvFUM)dBPVI=IdTkDR#Nd)&l}HU>ByClJT0og(Ooq4oBCW8)L`9`_yJr|1+6 zCXsuj)nIL*yAuOnVhr&nsDJUW42y0++Y<>0$i3iBP#Gn);AS`?f}^Fw?1b6ItZSX6 zH{>7h$FH<_&h_LpOLs&}guT<#uiF!{)y7}W?yfp~>|cimn=ZMam+OKr8xt+))=JO& z+p;jYGThjO&RS(NeV7ah#oH`Pr?^b&Ly$2M3*mf`!j4zhauTM|F@LddYk%gXT7b>f zb?v3UrVU2@Nz8I234++@~lKXIDNgkSF2?h(xC0taZB z>ntr7+xuz0PocBg;sIa$kl*+Em&n{}U@fE7gt3Si3(uh7*x8rAG3mW!*~}V5u%&~Y zS__LD1EyVuuqt%-Eq~US-HWaX31iqFwf-vL-Y`+PsXm@ke8vEz9EWozdg4Xv(R+ z8w|_GSN`J5V%t_qaY^6f0M+C%Xdkz7i|oy*8ijd5E}rlzz<6U~bf zxJ*iSH@dBhoqt*EL#wXbP;tEDde%MT!QtsSrguIMHYoMYZd{ECBF9I(yqd{UIm~Dp zEjPj;R*r1{G(G3!4Rii-738kt$+fn+%Owcg^8Y35y>+$)+BM4qBEY+#1HAX14DUTW zn4=%P-@m8#9!*Ov5(-pQl}Iqxy_Vf-hytReT!7;b>VMYJTTl`q7G*xYDVKtrYZDK@ z1x08dGdw8j{%O)+9*3y@Mioqo@-UfQ1U_Ui72R`j%W*VFdbxnuMg|do-fcd2_fE4q zE!(2RYa(aNY2sa=j22=LOQ`UXqnX^!?UyDL+>c2lF*#+K*}2vj)K{~Z$C*}QSi^wm zu0Swa=zj~ZEeLjmU9r$;!zV;x&*0`Jr?eL#etr0+YU~c5AM4B5#`*uy+Tk`|ReuxPQZqxFe0a!;HEkjQXz)f%S6Ng=~{S zIf+iO;qA`k$a`eJau;N%>s(KsKS@=oaUC*G_XRs zPc3?^jqVi?Ns175n)x8|`Jw{VHo*!<&?hh3zt*0nRm&#cnT8$^@DjxQi|`ph^pCdL zLVuI)zt(jqiynT9D$KvjV@s%=+@RQ6gLJasPivkI;mp(kT8)EFTLfC(jWJ?H0&#$p z(T+m=q-BA$k@Js7S`n)D*fB14Gb!SZn_6d(-C(nuF#july8;jMU7x88LO4}6QPuvHK5{C^7;J`*dfTOOTuq1w%qw$?!ldhCX;Yen@y zFy?JQcPg8g!gm0ShFKgqZ3pZLI81HT!rlv}RP%okma1d&o!AkGK2J_|;S#Kz^ed(- z<--rOm9TEewcT|f==riUuG)A6n zwZ3f3MtU}enLp}ZU?oh^-3;(n7BXfktMlLrxRTlpH>_R~0<0L%1TqI==2A3PRiP?_ zJ6De^S+_6ROY*9jB-L)2H3z2)dN#^7n<9cTciYwoPZs&IH)EJ&+lk^oGJlC@YR|DY z^xjO5vJuBfr=COt;$~^UQcLAS0)FF{g5|lgrUzce#yRul@T7*^@RAGWAQYyRl&OIC z(c1DpB7^X*Qs^DVtq|@CZZcp1hpL#Ula-0JW2morqox6QvI^Sm_5Sh)8~(KKo<5R2 zJpPD|?73JJc7E)&_gC;*?|Y#8`NGmm9^At3W@!Yyb?Fl($0sZjvzP!14d7xP7R%aejTbluYWcoy|1!Q0ztih zs-V3sz}pOT`pX|9z~U(j;DAg=yYzuA>$}boJcc{Jvn#HKq(6`|S$Pl2DQI13R`5P8 zONfXm-S=_J9zJj6!{2S6BW~n*ZPzf4$Uzgp4fa4&Bd!dz-|dwEc$9|P zjxz#p5)hXM#d$6)wtpC{n%ajRb#N7E({7eV>MY;%oE%2$Xl|g+%Qr|Pw2~Q^h~`csNy8xVw$szROb=Ig`oL(I&lzkeoqh0|(AVO=Hk^=i`A z%V|gySH2-)98u##EN}03BW=;b8v&9qLsKdHbVe{qe4D$Bj65e<=F>E46NnwX5gl&v zZ|LC<|L_>_5 zLV7`xpZC9b_fM?*QPfT3%ng=|3nY3J9Yj0`7ZKF>keI&6&ibh3j2i`y49pOaSK32F zCfqwgB@-MF>U9MV{#ir$ON4V7G0Zq59d=7Ow$(Bw;D1QOeoi!@2-KCyTLeDL!A(HM zS2VzFI014q41ziP*Fw!_bJRsB2|dZ6o?bVldzC50^>1@78UUuK%9wbwOf9x^Z@Q+4 zhDVkf%@P91u$2VD&F!(N*}XKBWgo9ymvm4{VJ{j)=q}?RWx)Z&9j@vF%(DSLp^Rl3 z$fNaMH-CSzuN-BOoC6m|FyZ>JBdvpXIjS3rCE!Z0n}slZd@NuBh}-Sn?Tv3PwEKix zour`IAEC6^GLFV{m8W_pU6xn1nq0`{w}sh0M1EwIrHKsaJs7o|udsNt-w0RM6G6}X zLfqZ~wyV8;aL+zPDvLB$;!zZNTwf-)ha>GMAAbkxx#Uvq1o3+Tt$cteoxu=f)Ym~D zgA;Vui6C#XL}ppL?KVyRdWHDUtW=m8nZx$bidLK7J-~?l_=*ra;mxIGrDNqhmX(Rg zrLqO-^5gN+%FxThVsJc;DwMN4(*!bs#!=m@X#MeJ_rlHt7SQIB>tiT+v;NHAEc1?M z`+qd2RB8)frS=*AoB~s8UsHl+0Zi+ckFS_86igi3&t zurTYilsHT89kuvCCtoZ8;2+%F!Z&`Z(+ zAM_a1yI@OSg1SvFz1y7#Pi0fnU@U> zj4Y3p2xX-!X$8{$%1~=;k?2N+xJr-1>8(wERho^s5#iW`60lBf-&ty+oO`j9dVj_a zMn0B7#;+DCkQTKz2IQq2Q3VG$ZIHK)yOVL4HXY*FBD2th$i)s5s=~*T5~k{3bdn24 zt25V|qGL_=Q2V(*7H?c1E7MFCY?ZU@dleP7Ax~8uw;8W6H6ER*Bihc$w?H+QUbyOpJZ+6i9>VFDWz>GFjFr&;DkJ$}HUdKBdGM~ycs-)`5K7zA@ z=`ltGt~W-mJ5ez62aEy;o`Nwa+i-))O+Tb`e3OCWX}$NSjWEVja00i3<^!az+XAq0 z;Zqm|$>1q}HekTfO0iSyYo)5nBJTDVW{{pb^yn;~grmNn@77-| zmDx!3UkF$vKp15XT$T(MdQ)olB~W_=)^9&Lq55RH5{gwo{>1ob^j4v4WR_}hM;@D* zR^3&1PE?lh85w7$T1qCp(sxxKri7TGeYjci<&D@DhF+0TfibXji)*uy7GT!tC!T*C1@DOnzaQN|DWfsD`*YSgu07xlgez zlpo3bPBcH?k$sP(MSr5UN3Y3FrU*SpHmH9x{f@%=M$-Me!2oZLw~v1`_x3XRdnv2= zvL5o&ckYDU#v;lxUq}vTvz#qT_J0E=$m9$}o??5!9`@_+ZfeNGGsR1M9gKyAwS216 z9#g)!jzaEaqoeAju!Ez2J9sWQ?-uS#^KqC7YZ;sJk8E@IxPQ0~M3%IxJxCfq7l!4U zhodVK>uhua!^e6-qmX44X@f_9zMI8zhWsG9`W54Yl4_Hzp(<}Iv3d?Qnsmwe_8eEB zx}a|=V^`%KzO@teM@T!GCNz22tamHA0M6?>$fs!;5k&jM&EE#liW8sf^he#@LBJ%b z4Bd%%Iawix#A)@V@bJzz4VjMjX!y+_`1_d>*dalM9lKB)IZ@E(@<0R3J`IiLF6TWt1P z4MT-U)w7v93RVeEJs2YTFS>$gt6@l9Y}(RQ1Ek5k3x7)V!M5}$@j=2GCkEC$WV`ls zDz1UmB|IUJz*dZcy%#AORO4~LNV5MlzZUD~BFqiNlB_S|_y_v@+8sh6mitaP-4#HSILMgTBS&%4Gc;)%z9_+CJ*vdS zvyl>DWv0e;D>?^{ZWzynt>(kfwJP#sslWyt1ODW}@tD6h7Y)}?@1yHvBtj?QoJ0XOBGc>2Qw2%vE5wOE3#&;bzBIxu5RbdHT7_ z#D5w-aIW~ti%q{2Sj%)CFk$FC@VIV%nu_ElK9TG_)Aa((+#MQt$dHN`=p6280q|Aq z(;SR8FWt);jv!xE9&wca24U1&tSzzyb(uuY@~``i?CcsOkZp1n6Gq5Aw!xBsB9A5H zrC4}c5rZ2UybOW zb=aA2V&q%)3tMCo2^8wZ8AVVzW60|a7!1*o88-k@hPxsHqnEB<1v-CZf+?Uo4jpb~1j2F;CTz``J zQdIgE4-Ig4SQnSFpZI!q#vCULsY9rMKlsG5ug^FpSMl^W`ve7@I3Q2g*tUOUnfc~Dn7=Rw`46OqGF7wkvl9tbYuODA)$#3nyNp!+~ zygPC#W0;ae7}gj-wCujjBcDAp ze2EjTHGwEEEMD?i!C4S7v`eff=>aZ97i#<>XnHCnb=f_=A0(giwF=>DU4NNN%W+Zj z?Oo^c*^-bB!P73#NC{&tcW9jU(d?+qaKbPV?%GRwOfoF08HGb18rdC8es(cQhuwqQhFL(yL=%}sw zX)>{lX!P^)rrq=~w0{ZvF7bwIptgX1v8#RRucwo_&6%f6Bu-}>o=F$>P=@xTAV)!f zNIj(6R$uj4aah5ykBUqA>9R)(iIF8}*xI)8*2AN0OaXRX7g6*Spl zw?c(%;<$Q_8(-Zrf@^VqMn9jT))GB4;>?mEo(Al_?yA?1axZ3lyAbY0MtlEbem(wW z<&A}xc!OXtbUNYy;0`#?$<)N(YK07IeQMm%K4?g~_qLYD4v5EsPFLE1lj*pSALA2} z_M5Ndi8tcyXn!-D-Jcyae4dXT?o;pXxB{(uQRxCdWPq+QD(uml>z{cfICrKGh>4}W zg^8dinRk7XbiJ_YiK#ZUdG@;M&~BULuaG%ccSS5MZ*~&%6HO)}OVH`Xg#z=Z)%V@h zUhR&2+V8HX*u2z7IV^SY!9TbOp3dltLR1yaQ%jm^+U8f zX%}=yh}K;Sk((C-Kt@O=yw!;_f;iS0B4u^UNoHJS2ujxiPN_rD7(Royxv<}bS&hY0 zRP&=I6zT1TcR&EksQLInXlWE3OWk~Nb9-Xt$Toa7lJG{GYtBe89^cC{zseo#e<5R; z#p>7z#(zrqo@i36qE=`ntHtQ?GtpHbT>Z~g{DeDxa8=0;FcJNYvN|4|`b^L#MBDQgTcpaku zC{!5MP5voO{*f7+e@c}EO`$)?fJFbtHu9ee{(m)QWOyp1@-J8v^#3ADe^_YDNIH_N zkpEJuuxRt&d9Q!FPKrEu`nUTR5qTs+IqgES0&I5fdg7^~=7ax31}+ki6wZZgirq3$F7eP@MGTwFVvnAg}+i zdEht;kA~GzvS%Ont1X&3ULnbrlv@Itv41d^%eg*)#*&}GaXUVaaBq&mh+yc=^7&++>lCwG_<#;9ZC_@8+goz-I%JY@eqO#+)6UYQK zW~@aYp!6m2o)6F!z_n(}s5Ct>Ai`PD>kjuH5vQCaU`Tv^kHKRV)m z04RTwim-*>?V++4`!w;VpOIm9%|s;P4~4Bo_ze~(hd-TJ8bi~z!Sm;c#neDF+8W$| z*zSt%<$v(6){2Q{X%TMRDY&03WUK5 z;*gwh(Ig9~4B;qD3HByIN<DRaKMbU(|IbSufVnq?vfekCAKjcHy~da$^9 zeZ3Gw<f1?BHh+)o>S~$0Sq8t+ z_;GCuaep}tq<)W*!-lJJIf>ChDP8$(hQN6FNgWJB3X+eEHsI6gS3OZNS5*~m1yh0( zXE1_3;zN^8#jiMicD;9OTCf6Zx`sapD}%VuX1jC`gEkT^b*@*v-kVz? zi~=>znzqP0sB%=UQCZ6$k`mZ`0F8uBpLO;rrlSqNmlDEgJAc52Hv=`W0`~TcksBLi z^i|Qi5#({(@ZK-e8|l<-S0}@ds2KcB2aBO;kYTt=!?a357cF`(oN4J53&hpLhe$ADAb8H%kfo7n5P%PV zOE;#hz%zQQtbdxP4UCYia#nwgsTdl#2dP9Q9_}WTIQ3U7nugki6Q|T5SCWse2SMY8 za8Yv($n%(E1%>CA8&{a=cLoaHq>9$?H3w;G&w>u#nm1c^ga)+EK&BG1n21y#`Ww(h zN$I@e&#kh;Pxt-jn&K%3Jn;U_)@Gwlh zz`E0+#AvvbjPLptyGa7UR}EO;_wxt-X+5m$(p1eXMSmif6uw9+sz zIVneVJ{GsJd#`kgX@j9N& z0gpJ26e@ew zpMUf7SA^YtfLxu!Y;YhAU<{Svz)Pds37EH13(t9Go;v@ksy=0XCnc=PN^jk-D0M{8 zf`hBHA$wk1A-finS~aAZMNX%DK7*g=^jT45*$oIC?BnLK6b&WF*J#%sj{&o4_FrVS zC~=##>+WC4c`mren5x4If>J5JTUzdAMStqcQXOh&X-#N4_ryqanImEuk;am8v1{zz z1}JOlanO7XoNAnuq$xo4}qlWE(r2kv|wXCC-$Drv9l8cBkXa_jqLg>cEQ ze2ysH_f0L>-r@qQ0o!BBS-VyF%uA7R6C8?1$E_K0O5kxq*3UBmdEwg8v% zL}==#sMdulIeEnb9r6tRe04Lim6TF$`^lwjx}&WEJh`M=amoC{I$+Azf$(8`(%>}w zLv1n=vD9ocl*;dIgE%TZ-gm7&4}Wlr+~>0KdUx3eZVov})rYW-pAwBsim6mP!*$hw zdi)XpNx_7}0Y#37{S=brXYV$g-(4Yjt+W`d(B3-aEBt`GS#k)wzT6KmD_@!dWfsd4 zZal(;KqzE~e)&yzvj_W$3|BM!3ewGkJ36zhS9627LfQ)Nr)pZMIEay>mw#+>(nv4N zeWV}&R6wi0S~)o&l3vEP8&dD36=T+1z0c}H~msK&yTJSABsc3 zGwECr-RrW3qviint;)Sl`9YlU+`3^UY%=S*LC9UiGc;lekL8i*=T$TPjP}zD$IX%L zzTxF>s(BcCQ|(cYPSVsy+G61m5U>AcyM9~26Zi*JkFR zoZTLk1ZIjtGdCN!l^L98F?;=S(>2pr@Ziffb!-1T5s@FMhF6Dq%9xT3`;g?Q_XWkB zxWV!=(#xt_80=5IR}S_P**JeG^at6QG4UC*;3x@zt~XcQ`^I1$tdb!a1Xw)CF!PLf~Fc0InW1-MkHD?`W$Hv4mmNu)6)bu;3U zpRd3(>f;bHMBX25TA#SXcYXc%dx)$G#L`@BlQ~1}yfNLRp8en%T)BUiKd^Td3^s2B zN+E!x_q?RHDjw!H>D99+<_Nux6EGz51L)+cq;t&~|3?FlDANxLLN?Ldjua@U)dA$I z6_tSl`eOm)v9mYLr0PyQNz&e$Xx0Pg7?yQ#_6>mE&!j?HFkZy$dX_pP2$>LfLW#^k zdiqWig2i?NUL59nCCh&~$SPh4OGs&Jcmiq-gr8(gJUCe*j?-w1$v4tGwap=uI+a!A z6z)OQHY#s}AZtOEh?82#k< zyAM9wr4AC`2E(%l_RdiR*S`*WH#C;CGCAsuFBLkD*y@LtbQ^z~=*ZS%3r|lPtcXeP z`vli;ciS=g0e~`=-Gj$31%}V~iOi@UP6cno^|fg8L)28wEu>;<9q6h1c%z>9o;*z3 zpP_T7cFnY2)LQt^$@s)y8fb4m@Oz!NW#&fAL}AuVUr%a3y-8;Mn6td{Q=K&v- z0yrskz*isH9L~#>$SxU5x<;ecDK&0A z+|%2=b5DpbZGbf4%YGvnG;RlwWPBEXStfm7i+}*Rvg?H3t4_%EnFAPD*=@Hk`GCZ} zm<7_{LKiW9oHwoBCMB~t&=S_!rfR2Wxw)xp_GnsxUYK|5byuOKTxOnB{lb1cqY+g+N$ao^~2TFl|7ZQ_9AlN8HRH3M{LJT)4XPe z*Ja^|oc(vJ?wRLlUrLK2z~GK+wuhJ|_kw}&Rom|ozyOcZ@oK^N1PD8wqDAcdukVCD zKy?mHV+zZ1E=Neh-5%M;lr07NmlgF21X>K@KPrDGM!8TMexF05z7jMt?4iE_ zm(Ex@)V}$sq`S7%eCS)p5kkd{*V44JfEU#tp8lmw41KjtMAT(0kOiuu(+2sFItywa zufb;zUZ>s?#Tn=@jvYuVJhlO?;d#+F*)j54ReFJJkLad>!W8_0iw_7nM*Of1j&kC= z2&jL3BC@CrIv3Nm&8QR&cM$9IWY{ZygG&j3m!D*2s6U~bYsN;;t;4YqAZR7Hi{bqe zkP5}h{Suvkc`i9z z%?!`dx*TlnY?(lWUSL%HQS!ba`j>68H4`MMbXl)=ug4ePtQ|JYU4r(KY?SNQM`JEw z;FbD4u+^j$Zu(#*NY9EE>D8EO_oe|+>qeE05T%@(=**S9G39y++YSvl${ zzQc2?{1i)~S6v@*;NY6=19j`SKkN&nR}4P4xKEql(=?o2dT{3=v&YHOGF#9f7q=5` z5r`E}Bs3GCXn{AZ8IyXXNg#n@oxI`6XLPeU_L8#DA||v;L^T<2$Bff<$CL zVQxjWe*L1=CA&Y;AczIdUfFoa-?k(oR?n9qzg>6mJ zEu;RPO{oLY(^k?n6m!8zHIeMw4bzj~^bsE0vwEn-vpfCyycI>0Fh|Ei^~>qZ8BQbEYWT7^rzEiCdb4N0ApP1 z_gQkH+ww2l1;GsH;f?hp;OA=EYf5B&jro<{oN=nP?z!mM^S-R3X+*rE&8-uJ&<>4e!sheb z6pvUQq=XD2s1zgQt9L<3sW-ev^i%dYSGLJHx$~; z92G|fhQhokuNVJ~wAK+D!_EII(i;2G4=W4&Ka|SKS=lIU%R>ph?XXl8f6fnEn)U=K_3(w%K#4#4*Zx8Qd*%H4tQ`SqTb$xNH+iG zmmutrNK6EVpduL2OVvPM-*mz)LbnM_#b{~CUJz56YoC8edUo?2+oTcOWRspb6wg40 z-AXBSOT6+L9>cOrBAJ+O=nu{ThP>+2WbnIX&`f(O#R%imxha} zvQ8o~GH5aMKeoVs#$%g5+^)hV?65q|w8ShUv>(HXioXbLH_0pr(#_ae3SEN_NZBBP zvD@M>3;}-{`VsfpF6MD{o(A6{+Z9%!sAbGl^t+i<8bm|-Lplgr$KcM}jm9{Zt6U7dD;zmk`;(Vnl)^0xP zc1C^cF94VuS0Za#`=mO36|Z&37Z@C+8dAqu42YWZ^-C(m7b9NQlZ5dR(fp|LS|bX~dp#pTZ}SF|Rxow?2QKek?!7 zVO2zYfI>gDw(GWvv_6ii)K=x{D8^Prqa{C&Jar zNe65g5w2B>Qa6vUaphVbNv!gM@D$do8Burjz4wVP7q@`Qb*vDUBJXX%z|}66is)t> z>+W$OlDl_C8V^NFAj=KO&lBOy#U=Ft$&jKDtz}UNA(xf8{g=)y5f&2b!@W$R1^*L< zz{cM~daK<&f0y>KI}t|Z4MQufcBuq+XzxV<{;3pgnk zyeO>OADH2X-Dy~sWM9UgxYHH#tNvt@T2qbcp5I3J3^cmxPz8VGn7Ro%hN?`Rq;`#S z6;dPfKBpT0sN@}nwF@A~Ze;XHWY`&dvgk3A`j(kzE!myH;Tc)ZeLtNF)kVjrfSqSA zz9Y=BHsY4sv^)3`gxQNzF5S50MOh&5uID%PpwmU5qn{AEtnGVyN4MoMic3 zrM`+}^(DVHFi}rOt7vmKGisRt3Wf>Px39h4Ei-POYV3cLbMmRS3~dRMxn+ZJ#2Wh6 z%6OaeW4a4jR1mCuoLf1b*q^XzKr_jolVBq32xJD8S~L0w;I|Uo^;1mTUh(4-GY!6V zQjjTr$?Mki!CxROT}|$=?b1wnZlZ3(5OLf~9oB5A8I^Ju>Tx*eHgOql1p>N2*< z<^2`JzpfhQJ(5B%2c4?l=5|E;3GqyjJ+AhsA@UswUzaX05g8(nO@+5dY$HE7=NvwV zR$Z8YndN|^86__FHxjQZLNIy4mzQZU5iEaK0O!z6Z;P_WJp1eRwmnqQBJ!*<5(U5hiQV4%~ zz=m>_tN$h;vI!+OBO#&B6Wisfr)r9Z2R9)_#&rNQ@k?ejCRdCa{UXhYFt0Daert#D zp*CxikD%y1lZ~w{S>rfc*+rb#sNAYBnxQd~0maGsNIlCkZ@S*18}OV2ZCM&7!z#M9 zxEk;Be5ImJoAG_6m5Ntubj15Pq%?mBI&WBqq&Ww(`FYv(PBvr(L-1wW)R|bG+2BD> zz8eU#Kf24SNEd&*E2s1}8xOh|8E*R**Y>e)zMVODpq4ZU(T6Qky7Ht>u)Tcg01JIR zTeHa@r=z<)F0;R-7-{~~VAztyhvyNZVaYl(yIj8zdjNbC0vVA2xAuDu!KZ&MCar#4 zmxtE-6H95{Y4)Rm7Jn!reICkb0AOM2I>==2H$9&kMq-VEKzX(H@L^IwK=75T5IKQx zVdZEVq9g-({$D1iG8g#G1@rD1r>#V^a|B*vf>U>uC4hXl44sDCN{gKsLFydTP!8L~ zusPv=OeB8rucmKK?#iBJXls9{`gTY`q4+ZOZsBI6)@bt14yPfBI;7T$W9mz~3Ubs# z`c!#=_gxst#GGc?cV9*>awH1^+KmS3ylN1mq2H(ma3WVGQD4Er)2Z5VfMccIyL)3L892G27YBZ?Gz4*z+6o!ADGoF$pqDUtk^)s=sk@g?Cax`wq} z77=cvW*%9Q?nWxyn=95S@!2j^q&IS>LNcr-=aZobiAgzjRM()4(p@UOLjrlo;r4 z2x#BC4R2FyE<^)+$NPVwBRi!^ED)(}`D$R}Uo*dH3(=PSFf?sRo4Iaiir>kIPuCXD zo|s7^#WM^#bRT>6=oVCegP!%%g()C8zx+NE5tzr-E&Rk7atmtPFrRk0uNBPWDjCze z*_Fb}QufrwOp*Sojh3$EE#2P}82WR{KPJ4H+eNp%s6Tiq!9`S&^PTllbnA z!oJT&wcGi~eBnwm-|9ZW612b6`zBmYvT5L2ffW?KAF};FynS1f>N?Zjf5}~nN@2i& z`Erw%EZg!4Dpi@Av8}c&b+@`(EurUmfH8Cl zL`Cm$9dV?A24sQUC?=lkjk*!u{61o&6n1lHK*jT{Xk4rmr{sS%>+6d4T(`5Gx2(bxCpcd^ zMsnRJtJAV;M~ZTLd0g=e-gbVhY6XW<$z>WbpkN3o>$xrCCPU)jJY5w6+e%;r<+2?-mFEba<@Hq|8l!$cFO$>-*$3|M=dy_ui=| zzhAt!0|w{yGEk$uz+(YmK0ioQYjd{Gl7qV(Qvh^%zt7CiPCdCZPwqZkzTZcWEFB#3 z)t1N^v(w6Hb0`XP_S)ztr+H}rxc}aXx3c8UJoJCwi6`NWr2N1s=4=a z2VsBQ0#2#d79j}5(3xrw-c))k4?-0evyDRo(>a2AjyOeJ^?WL3h2MEYsLllOJ5E@U z>Wp}}Ava5xpm_ol^SM$(ijiPcLW%EJ&L-L`Di(!FKqVN#Qsz$AF$g$fW?1-4QHbt zAcX_}#eRUty+QWAz)UmC_|Sdi@&}ml7l<4$BmnZseuJF8!KXiF`QFGqh0 zHB-6-L2aWOOW?v$08Mr=Nn{bx>70^zP*{3(foSiYQ9$o50fTZQ#VtuN(-u%FE3-OV zI#(}%6rE$YLI&M_jjUY&m*&R+8fXo^{$^*8Rm^g!eU z3Ks!&O;8OpXr68?hVH;#gnQ3-Cyk5CyB>$JTJA6SXu>c63E=7V5Q1xn>KuQAME%Yh zCm?g9^%8qjHhFRvQNXtHu*vu4-lvMM5)_WqfZYV<=|CN{S&Yw z{eU`-{Jn<#(o_5>+k4pCqhElBnPE$C!f$}Xm-h{tI`DC9OYmmPL$N1hE{8a%VD62X22r$P@_(=(^0_5<5_IrtKrH9=0Yscl=f&^e;!(UNQZjP>HdaE{e*)ki({vshmyr(fi@H*84BP-*k;KtP< ztaJ6v90_vKnYy-_nqyHox=`-{;)REl6$s(zrK$r{c1cGwcYm1^q`Da8Bq4=rhix5v zykpq!fxeKyqlPEJTvdOay}?|6f~-$qpLN~`AUOU4|CM%skG_Gw-?k-N8vGmN()$H4 z{c~)6gS3Bvg%16fihb-MJ>;A~%;LSwn>2%R?Z|5=2y- zT`%xF5}Z>}yTDoM+WFdJs)d&&XlpG0~xq4-#QM-1SY)|YU#22`22o{AMc05Bs zOeB;_#;oKO&o)c%oK%Ya1{Lo2Y*0A9%0s1L^)i9HL4_s2`VU|8BX;~{LyvD=)qm@+ z5&UUi{8zF6`5x`Q%D8PA=6HY^rGAh_`aXZ|tPm(h?BR^~l(g|6YQ2St)SUqoN0D-{ z_F4g4y(9#jfSi|0%nV?0VwFma7>}K6jIWH^2s}bc&G+mi-Bm1GykaE~weD6~O?4WZ z%)>gP4n~<4p!k|Xrd(#cj~OwILG7VZeitfAHU}BMy#4ZB$T;mF!2z(F8b0>Xr@+j}VXjSg{}HJ-C? zY*ax+IE!`z@ZrUtjsXUQaC^p0W3_)uX>q3yT;0vh&G}4&Ih16JS&S??U+s=|fJ8k9 zCPD+^7PvpatX}9$LwHyV1I3IpD?O%TBmSucEzrV-!U%;!s0NBNcr@#Hw_c(w5w%Kuv zzTySg6~u4F-0kX>6_7y25$1lgUSe&oGfvGe&UH@NYA+T>t7ddi4_&4?M5m$NYPB82 z4;PbLGAytTY$koA9*`#i4I>d3m}I_loLNQ8uC8cokU)l9NsX3fm{ZR)G9Lsp458`3 zYG(3!Zgs&jUd=KV4pPj>OD}(kXSUoU^2de@ioT|jg1Ndp)E?dZ1U_-o@XJpxf9P2B z)}vqkpT2eQGj_e7G2}PzB=Gt`+RHDVesuQ>blvW6&VSj4Z!q>>bJACaPNl@X zX9nG~x!%SG)M~Wt!&XA->Ja;=Q%nSB6QgXUm$7eD@dmN;N*9U_oKSztsDaANz9?#$ z@`51FDeCGK!Dn9&+1L;HfijPp`@)1$rAWjE=?V{H-(W#);*#R0U^<9q_n6Yw-XDA; z^N!M+iJUpe}Ab6PV^ZNL{10F;1>xBc82rV@I}85NY!RqKLCz zs7E_>7q7yGKv`fY8cTo1=PW}g*pX&UIW9fv#||pf1qEzphw9UK1u2xE>x>sTphFuq1O@agWebd0cAI}W7{tS^_`~KLqmWFz zO3S^9W~5R)B(el?V(3TrH0prrCAXkWANM@SaXg!ptB{yFvw@xGPOI9`(m7`Ji2H7! zOW(&S0YxFBF`;{yDif6y$ zfgig6J*WDbKgoaC!pGU~c<;Aef4>ihO(u8uHI_kwsJ<_h4C#1AtF}pyOm~WU6CYI% zlB^5%d^jJnr4b)IBjAD4YBp}Q(g42m2BMcZTGhR5N+3F_2Zj#m^8=Xg1b35yXH zu}K0VE=O)t+9EqRZ+7v81jga6BXSE%=2}v8rR~fU5x#$S!^nA+R^XAGR}n?(+xR`N zYUO;eP~JJ9{a4IWe$TCb_%>ctUv^;ln;*rIe$T5ucHzxWQhGhu$H!msm&i<;)ebO% zjT^c_sow0W*}|!q*__&b!1Kjryf8S!X2hSM5I|Brk+ut!SlKPa&_+O#bSNwnc_Cr5(MaG`yUvfBq$)A1stl#$d zhwlBE$NZl2$~ga5E{Hq(+_)D}9A_9S$3>?IL#Tf-Fj4S5Yp7`~{Rs-Ja;Ez!AbmN` zSvqUr#szDVqpm1rLmuL);=_ZywQ?)4<2969LbC2RG=7=*7b2Y{? zH^P5}YoJ5sAK2Ht|Evehxk``~bn~8A{9Qj~T>6=_|Ip1Z-}cEnWwr8oE|9VW{|=pF1h8S;b6JbzJjUYi+udp1C1h`f>m4+|ek`x>I6#hhXsh{IMX2sCi_ zn359FY!C>HxgzHQ9#QH*56^ouI9GpGrS^5EPJL7dVM5oot~8GDT+HGW3(qB}$W3C0 zW}t;g0fm=t1+NXDlewHzGkd>g%EAE8asz+nW6;*t zh>mct!Pk!3ddVWnGXt30W68j^OG>ba=W)&HSMJsVe)i z{_!g(o|D*{EcAG-civ1F=e>W8iYj!yzQL6Zg7d?Gnjn!;qI)xgGlJo~)mt!@0Sk?? zeuBu`l@gB$r~$_8gj=uga?QpesUqmg6Yty%;o}RW;Z|afcZt zfzDiu&L62!GR$d?xO#uP?MH6XDGri;7M=PnV~)(TCGR|&!0)`}i~Y`hzlt7w;@)?= zzhdGq`(J$P&nH&fv%mP=cb@0luE^X3hCkxCSaCU(d}ytd+{{wOnVF46eotc}=N^U| z$e#hlX}~lDtvV{&hFERm8B8-#37!u$7p9u&B10hfm~c-=_HBP}aJgp4eZrLOyy6Qu z)giTfRvZHjIFY(!*ZKX4!80W~qs1~S>{%V*tcq6lP7h#XNA!*vHBEzs#EE+Si?_gB zHNyFN8S>2R-%mZQZIB_f&`f6I<-Xh|`e!Hg`y`mIkmKk44)j2_KPCY2=#k^H=v-KE_E8Aq;K}WbAwr|^ z0|PBUa2<(87;RlR!MR>!${hoDETArM`)JGcJn$}F34wou+8&niTiI?*!F&ob$zGRc zt7b+irU_^UB=<4db7~1{;V_PERBy0nlI??1;7ZI_|H7G$h`<>7jy9i9w<+Ek|#sa zub)@E|7SqoKLlCnA3FFt)G7S@G6cI2F;ssEYXS!q4}_mU?@V$LMJOePBJu%x7S>Yi z5^OYd#t$7XxD$wLfm_wRGr@JY@b2R4q%H>V%=-I!pe+`Q7<92X|KpEB7X7L}BBWqm z@^x|+{#|zd21`T{``iO~TR-po6CaqL=_|q0xYFI|3~9h(Q3A)mOQ51gwAI z9)~U5C{PGnKtlHy{R6ZD=G;*MQQecEl>^BSdn@as61?~1_NT%=0IKy7Kn_g}zT8ZM zG(3=b=1-Soe|2LwYr!~^X4kL!4{ z?Qx9v|M$MCSQdI;CzpN8fy(+EUGCI+KVb~>e!t&W`+dDXQvW!CtpR8x%yg2JMj8SS zryTTy;Wm&*>7f!ndI-lj@VvrKl2ih!q>Ews-%zcM>gHh}U5kWQ7bdxL+m4-t0P zfLs@NW@h&WXhS=)r{PdQCwT69x`AR~^;{6er?o2F#Rsr(6H^7<( zyjz13J;h#0J5w07um7SA8lHb+EQEmt!uhJHIMQR^$dpu|I?1R6+)fT*I^ZSk$htak zxdP2nYzk)7YwCg;8@z1vGEYO?0vqT->Kj=a3dcYhU6Zf+V&3SV_q_LeI>ZKl^SL?_ z-oS;WCdm5uyFR#Zv%|ta$U0z{p*xp=+2$0my#|jB=K>5nYXG_Lcd>sdbmFbUDQw^G zqp_QwN==rDl?+A*vgc>r}CGcMXP%kwIM`c52kr z1&a+()X3)G$O4b%<&^mERJa}@!q&!%Xegl*Z5&7z;`!^bnqbvIfIi_AL#iTv3)MM*&^D&*I>_4gSBcGZeP02R`-u9t)t`p>R7QVdAN>hjj0G|6V*|#Xx}1PHcj(X*B=$L_7fxLWs6zMd8bX)cQGi-LD~;kv~9lq*fcig&qHbuC8k z@}27v`(2KkM)LVv{te_j90YPs|7gJ2NdFdg1JG>*tk48Es$B4A10HpX4U z-Nhbn5USohox(Ab`hq>~-T)P4Q_`fqHy9-1^1^?a5`a31Qx`(2P#&ydF8QI$F_j`r zO)PmcFcC)*H=Wgvler|u_#vvUS<{28i!==bF8fETu9fSy?i%*4OEAT5KtCHTSxl;NRpJHay-vJ4~rBjfqo z_sV~<*I@N3eW`~#;dT$v&D(!Pvb ztS%RLDCSu3h2-2Pc*^R2eNMc|q*K^YVqz!ueTk`ga@#md}31ByHS$98FmAa=Gv}!NE*ZlpR5FUR) z=Ua3rV~cFhZo~uP2&{NF>LM9qNWU7Su16@u+?;yE=J6_x?YG~=we-!tMU`Tog}#}& zU`o6Ck2)rObN3y`1bSw&eFKS}r2$J_AZ$q1?|Vb~v{bn{2g;tzL`aQO{d^6 zWhJ0W1+Kr^kxKo|^?$UtL)d@XF&BTRmbRVT8i=L6mT?9CC!Ik!FF$b(s_7JT`CrCP zX6KdZAMGinEz(>Uz_`Iu+7>zPP?&M|Tb?f=^=9^yrh)UGRlq69_Q6!P%`fM8C>B*G zbyXOhj~nm~(yx4EWePBY%&kC^v8H_fyG-&nK1iSSO^d2jpfIsNeyOH%PLO{Mz5Tat z{p)e=J!l7fLC>o6`|cY0f42LgQ#}5XJ(95s&u{<4L+`f^5@@8258isd5{BYuF}jCo z!ui6eA58?vkoJ#|ztfd>_IEn^@9NuAJT~9FxxtbR%FrC3JjI};;iUC|#L~X9t`qaB z`713m2ln4+1Kf#u_i$u?-s6AeEP3<4>(4`=+Iuv*wDHLJ;^Alfean4l3LX3STTdFe z>rUtK1%?NGRlmP4eJN)LcKcDs|I-+jcIRJ?mG}6d^y*|G7Kr#f$8&*Vzs%n;WT0ok zL*Dau{P>UKNA`stf}`{=pwSm!(-3%2`PRE`>35Rj@BVdN3;Oxy-iUvIfJuH)8I#Pr zrhMz73#N<-=kBLmw%xsfkne_}94|2srti2_#@WaSSmfSd=u|@yR{xuF`E4LEnZiik zM^e_|`Q8xaZ8w9^lXlvaarF21-8GuKE;urNh-6$tw5H(zLDk$F1LQUOT$85Z`ixW8X@D? zH{JS82gaLjiMyBV`-e`EbO7<2ZtJ>ojrfbM>pB2r$ma00`tbHo5E<0`lhR<=qZIMAEG)*)MFQ-t~4r_()?}cmJC&rEIsZeenfbANntzP%i8KMO(>sN55#DcRPO3Hci8u&%DzG z)0|QQ+&5%yM*6HYrnd&lLjztNTh;H_F;qb2sydnTU<7~d4@UvtK5v9ufDLZm`}MD% zYas}*^R;1Mv6~kHCysL?fQd5!<(*@5-}kXId8gLi_ch2|T;P%#vNDi9vofl&{4HKi z!qx$k#!01Oz%+HgXpMT^gaxV$BYSYUoth9v%YE|o9*`iHd7E_Y%@L5fbK|J5rUKDn z3mlO~BsG8Hhy?I&Q4(q}83gk8EREHlaQXW{z+%_GLK%P2)KKtuo{-@8N&nyZ!}mP` zOyT^y97}|T_IJ4s1_kAJIRTv_`**nkgrekkx%roJA5IhX_SaaZVpL+Jbk73Qx zUjBDweWd=}V-^1IzWZL!`|({azw1i+jG@OTpDMpYAO~~+Q-Lj$(0DI9LoefR2)W!;m8T_vyd-i|-o_t(4Ck z-EKtce`RChhO(D|J1(#GyWU{?=cH4sSmFU+Ie+Z{SG!x4bP+*7sY#Kk|5jnMwT^ed&MG z`#USyE(~Ae#{0<2djGHG^UYgc?{;+W`?sFQvi?Yp_vqb^w>%D#KY#sh^tXCHdXUKP zTlCAWXdk%|L=bN!GDu{+9iLV{rw)mM9OIe zNRf1fyvG{2LdwaL{JBW`u>tvg^{aoK;-@}7WxoTCypR6LdfxWpDeEbu{(tmSyz7^G zWO?$wk@bDs8&{x()Pv~9dR|}kzSr~qg)7p&*3+o;e>LD3Y!0Wl5-_*|mZ>Yc({yFok;R2ngB^H73#eaaX8n|6rNvD-Y6qWke0 z>dLVzevVFa@A9}21;T4L9;s++?#iH=b1jKpJTv zc%?;~;c340Ycm|)d(VyZL@MSJuio>r{9y+Lz!SlQIr=S*x|HvUh&X?Zu0GGl z^|;rInWA-MWN(^yYSM1qEY_pTIlNt*b1{t5ez7c%_BF^Z_~~k`bq?4ttLomhzfH1B zK33YC-Y)~I*`&&dGux%xvelW|@9ze-+gpsCQDodMPoN{ye?bF&R(qlRZzgTda{A-(AYC^E73@zg)LyIkeQ-{;_-Y z#A`L0&tW^26Aprxj-STWc3WL{H z7?a?*T5p@{t1ZxGnY|qU8P86wSMdAMJ@&T@DWSEqufw^h+j;xSYKJ{;A?Wl_)|shz zbsd)X-i?3pu4f+>r=_AF{bDcH@ldg%o;mDAcMZE&E!9ucn>V)Hv7?{bZ2sIAvARmP zr|=k@mj}9@8Ef+}Ug6WCzIL%yYTIMDm>ZhHOwpjmpWplD&5cyG%S^SrguU_lQY}JV zFBZz6uD!|en2%SDzkNh?OixchQzaJXRT#VZ6R2FKOr9y8~)47btzvuPJH|6RIyfGiOv!nXI>oSEDiwLg{N|8JB+q2ZO*N;JMgH6;W8>IU3?kZ^ zyqx+p?*~?3c2Csn6NzjX(mh^o?!~kpUxQ2OakS{dE}4uk&FU<=Vx`0_AMl&naLp+u z_04u?SSHgrxx0h@X%TKWx~unEt1vRHyeJ14O?c3_ zM?-75T{P;LWDoYdtfSeGUe(ys*F%+zV^!owpN`^dN-w?Cs9M+c6C)v4ot=!p$wGgm zx_P!U<2W)Q%arfXYyNx|w?)`uc7emOSHGMc!37H#ShRjRnSD)KP^w?+R zCZ5vsL<#(JUFMTAbX_AkPXlyMm+^mdI5vm1KjXKd(`;sSy#vE1KAv6D)+(Nc?ZztqU(m5g`i2V%rKj8^f?efE6gG;LMyl05>k9bbguThDHKNJ&1x*V8p_ z-T7RvBCnrBYQU+8bI? z`|I-d@{Vyan9p&XyVifB6Q)0M+vAK4n$dW2DbxCVCf608RcoD($3lHsY)bD}-Lj|b zQWn)JoHt6oO(8``2pfU}K#mdPj0SAnKkq3r&k3^y^DX3F$crUHKVx>dqq$3oS~@i7 zpvPEnEaD>#JG&r{xf8%-aj|$UI&XYBBp|x;A-Nt!cpfC?^%Z|+mw0~d04YG$zekzP zJZBr5N_!T{brsA*+T_}k*xtc^8QvaIo*zs$Z;T5H_U3(1IP>`sZhP)N3GA{~9-g$j zx6^&NOSZ3J~9_v-=6)2zbivN)CN6L)1H^Xv*pKr6={z{Zxjy5U7tF) z?Ly_FQ~k0wR&1c>t!gKK(nvh4$H0QxO?VOYEJL|z9+|*=vxrXgC3hvGtTzw}E z%cWJS!zzFYTJ366E8>e4t@gn=LS%03BQ+hb=S8YJt4vK-m!ZFZ)jP6sr`CBEk1HUJ za+wK+DLmVH-Y&2U@R*ll7RsDLxy9ZN9TTW=Qq8lY&%2t%Jocd}oUI-Vi{)Vxy$?$< zPhL*0!Cbxf*kk*g_s6gCdAAqG`hw`z?rLRO+Gw?LySTcxkKSo;Um05?(P(~2$7N2g z(8hlI*DpcUg?>Lbx6KU)oxM2N_O>+2_}m*a!%)b`46}6R;r-EGc205y z7v?KaB32B}H~j?rN9#FB*3C!DA~m&~w8?2}D@t#2@!HwC5&e079gkO&Wzai~)~ob3 zAL+ehYLAuOVBMc&hpCazwteSS-5P56nANqt2<@Q6#df)WTTJ^ZT5i?l#+v5+`kC~D z{$P5`x0`+X6pYH1O0wX(nK3(H7y3y{oF`8&OQntI3 z&=>IV{y-yrs6)_vp4jDbJ%NxXBiiCI||~NGRt4*UOc%fhOTY< z-9?`yy~F%}JPj;lTpy!6*%pqRmMeRT5xuR^7^J0fKB426s>olU%ei`2GZHS-3fsvc51wtq7$~J9!=%0JX ztrabTpg!m6^y=)U#%R>v&LHWH>$#nUyLsA7l)!?2RTJtOM9v-T=`(o_cIq$)>~uId zPYQQhCT3|{^ENK9wMra(8Xus$@yW6|MOR^tSLDIG;p*9&D-&2N>21@?&FkrKT9qDp zZg1@+!?e zChaPJ85;{@m#NG3pdRTwT`G9Gz3AR;4(eoquKUGYbDF%r-j|?P!`;>yHsPu+v4NLa z}duQ5frp?A=Xxz7DkHW+$%VbS>lk`B}U?f=E1B zMWe8>^}+4g;k`Py%Bxwfv#?R|F1L{FRGa;OaIsh{PD)dy3F$o=ebBcrp_Uok8;sEL znx@|Iwm4=++rP}BUU|^#@pv~9Ydon!4wGK8!f0Ekr#N;+IP3=N6MU<~E&;msy}J*DPv( zYGo7Q-Xc^zLk+I8*}!{53$KaQ@==Y>$NbQa*6He$xc%7Z>tt>}vMydQ)e^Jr+zgX` zP=?R@0Q$Kh&f`)487TRhD##>hYJaEBl1wkg6GM$o3!5yTdPq*rZoL6| zy4=0VRZokCZG*z=ygY38!{PcM=poI2mM5f|_IYQD{{FmZpw}*UR^gF-uza=3(~5@2 zu=Xr39`Y&K=Ub=p?^)6hslUuNePwFjC&yDhIons39`mR1aK-Z2J$#tdH!FDZrymZ= zh{dtC?5?lHu?v$Xn&uZ&u8%o?Z6+`CzPy?KUSX4qCXr8@#7IwTEq)Yv*VD&;VztDl zbU(a|O_Hulds_^9i^kB!3G>Ri{s;4oVzv-R$}9duD-FN^4K z9=^8Aznq@c zgFIjPjXS5Z-*l%&Sv}Nl{yMB3#C(0X=?yo7o!7g=^W2=1rk>7U8%wk5k=o4e_gxhh zac7FYH%u?5YA`D_KQ)fM&4O>YZnB@qtk>Qgg+u)k9Lzb-qM~UP*Ft4~x*HrOrzK$u zpFR8clUAEbIn1JCuP0ss^q$>wF}9z!4V^aHR_nR}Hzl`9t zBincvzp_oco~#CnyAVrlY%g(boGzPbJ4}n!)jQ?EJ+yb@peqsLY&6cJ^I>mAsaLN1*KxqJ z(d@9@`@=d^X5#qp${lZKW{{bVHbl{+8JF0jTK`dE6wZy&YByYM8l}`TSG3!6S&}Qd zWLq~NPd*-x*V%rA`(w+$SFtu4c=j-yUhWPNx8o@9cQdDlgi~36udvi7&a4*0zN!}M zM;mUg)6q>mx@tI%QgzhNme^@T8CUtbGp=$c`(x-=jn$TH&Xl>on$EZ0U^YsV zC(0&#+z+u{sSAI9J-LhXs9CGUD%0I8-`v$5Lw*0QM)k|B?~4Xs7QE=4S9$-oT&~6^ zvLu`9t)4y}xBOy%66(J^at?5`yC&o8(akQUw^6&(pxUorx<+%9*?u9txxfy*JtW!w;EIdzphuOgU&Ac z8V

    -rcUR)grnK=jyUZ@oQHtd+~;$&KfkwcrsjbxAwh%p%R7Oyi_l9f4mIyZQSeY z6D>)qQH|E5-1fS#RnQ?L=i}g(x|4}Y2RCPQR}3|n9@q0KA!I*LHOz+E?Q&5AY|N_z z2^_wtW{=9v?!wQ5ya`NX%y56$>$kLHIA)tVKgrKNX6XS((9KIlDvHbl*qi=(rCZJhajalYTuYrC2t ze)M7*Q5Bz}qFyfR=CGQQUFN~g*sj*g$M)JLqh@LM56)}Q)U*A>IhM^Z&QUs`%hzeG z@k(hRUOQqv9;Cf!KkTOmvl%f%x8?_Yra^nfx2+cODM{wnW)}6>Mf8Ji#UBeLnNIrF z>ba?Zs#(;Vr*yVFZKh|7#!8VC(M#n^96W;DHN*D8h#L(mF*FZLQV=WITkYYz?VtBI zqdwM4FTSdSl9)71;*6{=^U50*$}%{v46%xdJq(RWMID9$&i- z@WokM<6AU%O=53L?(8**eg8Zl^T13tj};t$)aBylT&=KnIJO>>sIj}v{lNFr;V?I> zSsC>Ly&7E0yN_mmbY-vounK##GBL?2eGb#jW))j2ChXOyB8&F2C0PS;yW2jt%niys z5&8c}AB(cKzAes0VElO@ZBQ`{y>4ElG9TSrr^IqHYtj%7(O!9|+tgeH9 z;@ERFG}UKi_Z-z^+uPsM)X2=>@j3{692U$x&6h`Kw>=yB;mG-6a&yiXWu;$^@xdKE zx7&{O@zf3f7hC78tS!*2*9#)RyFdZnn->om-h0p0tM>OFozf{O9jcm!WGH%`wd$Sw zfT))koPahsT1(I8l{Ysd4>BPQ)Wh$8(d7}H*aCD7^cUypL#Ytp3hEPQR*53P(r4qu zPjZXXbdV`8y@orCG!_+<#(O#ObMOf|#fV6_{?Y2F^YMC2zq}ER3L%_p76YRh$ll|} zXUOZ%^(M_ZU|hQICeOx&w!cY+uaZ?XIPBfWN?~x?Bn8%VB6@DcYpEW@{$d$_Dhal! zHx1%n(k{n~eN3K22^y@@E+gv`2&%H_&ujiUJY6TshTj2q5;vLc4>%ie7(_tihv=&f(e8++ktiilGg~;`PbjudMnlMfaZ3X z+AyOGgZXf%@LE;sKdSZOrOD(GPnPY5vhVzU|{9_!4)Xv_oLI>5YQ$ z^tgjBFp;8i(NHE4$l7CPIk&^H5TbKPDrwk((_P-SlA2hbU~a}A?DKWvuu455?!&NH zGu>&s2sL5%O0?^9$vKICw14%y2#=eYCz_v0`c&BBLma&xXODsT-*vSz*yi`;cj#kJ zwvF-7uchz`9?8!3)=SJ8I;73hmeik}t$2yYG})p#fSxo>{nW-Ao4U^?*obC$k?`3_ zGuWDc=ICOP5&NkG&Acg7I^@?Os@zWmGMHQz%w?ND8{5!PFPGSVbkqj5>^ttj7P6qF zC7L_Pd(#`o88MZrVbjm8tBTofEe*)9xoqj4!3c{1D`5}6Q>ooaR|Gx54d0(<;H4>e zy@pf;g%okZG>;r2xP6@wxrVP8*)D6X;RjXWn&4nk%NHsz-GU-MEE>`GG=DS;IDiXb znEU9n*^{YbMiqB|MM;6`CZuYDfRLT^Y#Cddm|LWq-yk~^*jCR=u8cGGq}RrS-}Mr1jSv4^rOGR$>$NFX!x^Gx9jN@!Vfe3%Zo z<^$u%3O+sZy6fH0^oka5**Vy5N?ipxea&;YL7H9xXMQ$Pv@HQK%5^)JM6O|VUr$L< z{uYTb3h4rWDT*)a9VVK9d!Z2D7H+T#w+ztqDW)s(TsC6^l*@+i0rYVE9g{SZ(;H6M z>(^VkCH*bOS&5k1r<*C|+7PAksQ?ci9eMWp%1$|W8&6g)yqv+Cy zF6&HW2nIh9`I))>U;ud<)$E9CNFX7fn*fRd4AG!}rUM>);3pTEMUJSwENzq$klXgB z?$qdB&8DW-+_=kuXEiYcQc)bG6-bi*BtJxqizouB9(GOC?r3xgvyw>yrD>JLW=^Td zIQrCLM9kn4{f%1|qHNUaI_i$tZ6tx>w))X=gsH}vJc-o>Eb_RH_+lhVm)fM?p3Q4! zM;SVQ6_ac=3)j9^D!q8KP!~LYoPl|U!tsQr!Z*gFwh%jJpyCJ|-sB3ykbcd%LKc33 z@$#)6c8_N7h;Jow0Ltjxg>yAItRCa0Xt5mM;{h&S%(~RZt8$DXgV2^%_6E{wVOvbn zsGGkO1ZA$b$|12&@*w@eF5lllTSlsU^5cnrXNj0xkjThOMo@lih@93zI$b-*&Tyo! z<>p{>?R{ddJ^_5Ch1$+J#y)*;N^`#9BbJ8+?&64a>Ji*di#?-2oqL(Sg5k=p2Xer{ zrM(sGO5(+_$wLv2NGV_c9Z>mWJpvWHp(36Z~`M+v;J62#`CT2A0(B|juZ;Oleq1mph6SgsXiFi#sDkJeF~00pVKtuwh= z5U)vBe0mh<*$tjyW+yY7T<8$N{1@R{al#5P%Ec*&1@221o#!p{n?S`nOK~@UmNNXo z!^kY6y!2dZ_5k#phXDF_7qMfLjq7Z+;wg^jT6653L)2v*Fj~5(z z9PYCx!G|SU@I0i0kW15OL1;05fn>Nf+$VcSY87&F_ul^elvO#q6=0_eT;hyFCbhd$ zfUA)y3(oBaXC7x__O*5oJQR6<6Rz8i{20Cc^7g$Zr+_BBJ_@o$=fx<>ehx8`%{j@% z@*7dsAzcKZjbEKL-Yna~`n2^RKgmgA@0TUGcx#0)_M-FM6uq4m9k3jL7H}5Mvq0s} zxNWD#Ck}C)R51plk`r)-_+r(d=Oba9nk~a2WMj8EMYVy5^sbEntb8ti$Bn`flnW!q z?LMu}YO!{pQqppMj%a;>70PGs6;^?Wco8w9ENZG1p- zGYmnj?g2FDlzRVodL^drh>;H|*T~1I;RHpT4?Zfk^+Xli4-)-RXxwSB1%}SH6|@rv zXbkx;y!)tzxqVtX;hM(WZFE5WGd%nk*#~T2D0ajEmz7-YktzIT$ zE;t5I_yz*_qoBUM9H~fe%ktOSEU{`?o%`F8QU%~m=vQjMh$U+1*^v3TWt0|*RN^w_ zL<~Eu+UNw|*>@4p=0s zkW5t?tt08h6{H*hNKSMq8me?=$z|Y}8%0si2xOw^t>I^X@=i^fNxllkhrnWyV9jHH zRjpP(Og$e}m4b(Nev~eQXx}D7Qpi310~weQ$8Pa6V_sEHX!hYHH3eK@B-t#TD3(tG z*?a+OC#mQN>000k^-~=8;>Jm#ZzGfB^7B(pz{_1=Xt3yccWdWzLLHqjI(itBx$iyn zSP${r<%76?3Wrpvrri`M1BW9JMF;=K8M=u8Ecb?JwE5&94B)q7TFZdD0Iz%IxoT}< zEvE>aPosYNLx92GHc}+SP1EVD46~G%EQyohq2y{F71x`G%=H1 z&lR{2QIyhX^SNq&t#TqLztE6G?#B@3du1+Od?8avD6n!u&JUOo8!Six$8(w`Iw(?z z)j{MApQj-ZNa#cN3~UGkA?qBM-!SRv?Xg~YHF@ThS%THU^k-G&^zvW9xUW>OCD}$q zu2Kqr<@4xrKrA<79J54*LW4QPtHSt1g${hi5D2kH=R%xe~B=C%cWUTZ9J|mlwTVtuE@}8X@UhC z4MX1*65qcbs+f1SHKU)3em5lZrjdpv^gADakvAkgOuhbxM$viR((NmmSK_OhByjx@ zh>HSUFlowD@j06i%1j#+6f#rSxPQ#|t2DIUz94VvqJ zrq}_a^sj!YE?|=ZMeOc*u{uNNL>wF`Ycs57mL3@;Xm-u`OB=7|?S`fG;fn;ZAR70w zfUXK6A_cRDYi0c}a-IA~b!Nz{Bu>d4N~Y61s8g@Iale8UPUVXydro|L2;fU_saf!o ztc!3u$#{^rm_s60za~SP_=ln&qEd)|o7I~`I@kLvmrY+@TPv4L4PJWA0G*Q0qT~DW zWN)0Tr-Vd?o*p`fdUg`EyagJ+sWxB-QGy@JQ<9eY{M-lSstpU-6EYf-LEXj!P4>Ug%}Vg(ZEIZsCk1eSy-VQpuBSlZMPl+a-KR+4zq<{$v3OT&YiL(PC#T zPMPW!eOOcQlzOimPoF67&-N8sndhjoseV2G9lz7 z11wzdA(j|H@dA67hPtN_+`t&QNST!HN56RHC7Yg`Dta?gM zf^z@2i;whqJ_ROKWkeO1VE}X1xt`TYd4RW;7rnY2_au;aW!o;Z3<(@b$cWLH-mtUI zgC3;rkC>d=s#dvwBz``#DASdC^*U|l`gk|)vR49&cK6Lif4tOL&ZT;%mqdJhks?g{L@EYT>!q{4S>^Oo{0y_p9hK*=D3@j;QEH6y#Q8xTOgu;@ zgwS=oW>)F>D;E~RA>HA!G2z_GHq5XIugOG90ChaQvn?urTLYSBZ3MOMIc5iZq=h6+B!iXYce#0%JkKkryKR zPDNhM=R?$s{^{xm@e6a zqTfZH!fM?bdlP5vdX%S=z?yOz(PwE$wVc1%#-opHL`0iZO`O z84qtjOl2VgW36lK>x=& z`!9e>uih;g^S&I_+5T zBu_b&lN5^I3~;jn8WI?z(Q8qZNhaH>S9L7$7!X`{wN{PF9GuD+V}U)vVV}z#vv@fTw_*%lt{9R0z`_!DW(?`Q)(A4~#vFdwK*!{9I4W_G z`}b5S)1gIK&R!y%FO#KphM3nVYq^`9k-Q2oP) zH7YA7r6cSQce%Xs9#QXQICTWbkG!$-I`!gxS~MCiQ>VWuNic^QK+7}vv1I&I{ry&d z0nJMzA_jx7ZW*;xw;t5jP}pX)*i}fDO9R}?Q@iw+nrMhq-8%sd6|cz@r1Nj$_1iFh zp)p3gjh*LLab1{)40xT9NF9cE08|;4s@B??twiqAE-V1hz;csbo`FR24!tA{#YgF{ z=Z|%<2WTslMMnRQ-f?M`{%n^iet@ZeMJ}k7(|W#fWR*jh-&-_{x@TARS}rXZFWdQP zXSY20L?*o-QPhIALv&<*FTr6D%%*%X@JZyKXBJ3U)GV88TRcr38Cu?R?LwShKn!Ep zDVzmNS`j93UPKGUbd6U?yLtjBrWWv&JG%u;tI8A?lBONC% z+|u@ZK3u7;qI9Z+T`IQu<|ANKoNSw&D!HrtCQWiMY#d#_1b-r)_7=7|{&RdNSh6+w zjidS5Q_v<9FiNK%f&~n@=(b^81T*qCfcR-wQOeuvfW^SP?eyN!9}l?mAvgfM!7M&T z6A8#bPVm1n^<`2)%jn$2%x>_1$bWIuM&Mi7CUT)HaI1|dA-@;yw{Wn+6;XU6W65ke zTFpsLPNWWKCbr8ruW)xNK0NS#vz_-l4{rF~{Ot+4tVNv)qRqH&Llb8vgC+GVFiK~+hlT@xJ~34ErH{ZK zgNH*gJP?^1Mn*E!UOHZfd=vdRQ&-8qY;?@=sJMGeekcZW9L$#UC_nHW_?A$y`Qga~ zSZ@eTL07{)L`@b2llRTCEtCBf&GE8q79+)}W~Z8f817vn)&8I|O_WY03Wx|KyA8T8 zxrSa`hwDoR@WB*V^VnmZzh|kForqkd z8~R?gb}OLv*_b$T9N1{GgSbqDEm5-{9pH;jzHYNCUxst1?D?>`eC41`tg9EKldg~N z%XVS4fO;F3Ng#wq$fDm6qM>YIy4QM;&cfhUMwP)Q%H`dXg$q7^J~2PK84d)QDHh|K z*avYsL?Lw|Urfs2?r^HuW2uFbARn^SMF-1*VJ&ZmnrO{uk*T=|z$M;ROM9JX$M}Rw zyL*mPGI%iU=%8h;XzUH844WKe*y+ss$;@L>*pV2t?$(;y&4t(3(|!(JaVP~{(2KYf z2lx{WSb)dX=A!3+=)KDxfiM!d8(-R!pcVo328_K7j<~vj8_g0qD#Tt(>d#57)BZ~C zZuio$JHjR+!OCwW<2JvRgZ5va=;NF=3f<98U1UVqRvgX~McHI;pl}2_R#yiz#G^U{ zPkxuFN7Gvh@|Mj2*!*5!h4`eJ<2jFv;_=WE=TS< zpG3L?f`*gn62TA2Mng^tq)hPXYjuP*s1Uw^)GvN@-n-a0NX3*u$T06pk?A)$>dsdF z;`1R?_fg4z*hBwBg{YddmWy{FWr3yP+*lI&+Fn!a8(bu7tJGaHEL*&LP$A+wBX^Y(8C=jv>G35>*6HD z{<@{*C~DQmC;j%G7cgVCp+IWG?WBi3dBV;}JWq~)XG$d-oxihMGH25#d39%%eXO#T zMzz!?zi^ijA`@FHlx&@bQ)Q7H=78bXaws8H1%7#{QITOhpe!jkFp_O0@+*dNg^^+l zY@G>M)5BN`IATX%_AQXSJ$j<0H_Y<%LHikc+1w!4mhppd2D9j!qZhYGIz5hPP>0Ep zk-Iv7Q3KLCO`%lBwO~od2>N)K=%nzBKeCYi@)(-Hh2Rvov$2egcfms%+Reu$&HId_ z7@HCswwRXpcV0{=iHE13m5W>F46(b)#+qJIEJhzU!7FETVCwP?r|WC@c|KFx>&!(&@qprneHTO^=q!u(yL9$u0kEwWl9I!D!-sez=)4bSLld2>u^X6^J~ z5y4y$$8Z^?bPKMUtUGSufBjDxIP(90*uei*|L+{F^M7)*ynX+7j#gp*2bT8#lc2Tz zm-++bF8}n}V*IyO|D6G+#{cNU8E*WCdrSW6DBke>;y6YV=&!U9{>w18{|Lnl0x=yt z=2G#8{WFpCfBi8?4451u{nLvx4A)ctg%f5N?x!&I??@#WD*xyl>|fKu^w&yd z2@gNwdxn_JU%h|&a^pYr4zq7~H~f!ooZ&L+zrny9!}1JqXQO!hJ--qE-9+mCidLg^ zR8DYbsQLGPzJI0k`X5L-GJKbRK>zgPf93zvboX~jD`t0J^U_>{vJPqlZjs|80^Jz~ zL7kF*8IC6=4^<&~h-V7qlT8hE}6-+2;}(MckIX#q@7N=IkD z*}Tf&dgbMf$3I1X#=Dl`%${t1y6U}zc`ZIC_*(r5sS0WY5M$1Th4mI50i@nF#VFod zGQnZ71!97A19AmhUmKA`aahA)?`$&{#KZnwmEH`Uodt&vPwF}D}Gk_&>o zWJW~db7AWjiZRCb5;%v!&54+oksxtYHWnQ@vi`YcOOAWkkJnaz_lAEP!tO0epr5C> zy0upntNK0JmjDS#0|Aj!r@D>zt4KisYGLP3Hx(8rfn%K$Wj3x7bo|m^V)cG+PStV%@el-_wG_s3iMxSV|{0$cEZ-eXeLdj2w#zP?> z)?gZNN#B8~^wY^nu4f!pJj8VgwMWsix(~p3)JNDe{2ad`guCukL9oM)njT!cBvU4t zzdoe^O17{8z-&RQuR75dP>IjK8A6_2fcuVGu3id`3M$1n2==8j46S z;C=gMpOz(CtKOL1c34q?1o%w+UjQ(%cp(8n&s*4k4uBp%jX;AGnHVCdnCPLI%Fb?n zp8A|~F)(<|_G|UOnFT_A_-LTAWq~0EKR@W&G$==;L=e${#J)P`eM}hzrl9~Tasxz9 z2*I!HKZl!*hBC{Tp@z0*=JLLO8WO$dpcZlU$f9U3@nD!T$^h(#KfXaMT?oh9MFFGv z9EEa!wfR7P18DC!BPj{0b>t_u?J8uOdT<0=ap=gFhf9qJWgp_|Ai^THa=zB<$r)-e ziTV>?;+~3K*o?^_45pocJgy$&2s&%8@(QsN(00ne6Da1VpWE?cI zsMU#k6o40!RaoUgfQ^0_+~Q7hCNJTzy&7e zvFew|#`~4X$f@=;zZTc3CI-qtF|%BRX@Wd`7W_~feVvAQw55m^6Ks!(+Hr+{9td{Y zYWQOgtQ8zQet!>wyRp}BdAHAQ9k6FJZ~2=1u0_#Q!GS)~+;SSCy9!~4Vw}DfugVK3 zY?vTHRjLh9N*U31Y9Qdd)cl@%{opLrHN=XZtFWf$31Q&O#4x@miCmf891?*>*E{1m z#mL3pLuman!nwH-bG%duA~(2y;UdaIwh?u13DRG}nk0;jqI1fs+=Nk-4q^{AA%j9_>WR=NTMf0VJYA_DT&0g z^);bp<_#VBY40whmkv>aGB%bNhjCw?vIw|3tJ*@7O86sOvvDg8NbbUQgfTA6CkH1I&oNk(hbjG*)&V2joYLH zgvU;w7=6Zy>$bO4#8==Y2QvZ`(3P?(M#U689u-oDJSpUsA9b&}6t11ww_85Zc)KAK zI(x=$1;4BYD%jsJjH)SX5~wWhyvhM(GBq?Rff+21vZ2_ zvm!cCve_=rdHzPcL1JDO+k^~e+a`=nFho7~H)RdmJ>OSJKRSX}Blr0*jFC-wJoPZ~ zQM*)_%eFP3XmBN`8-pM&AmXjWk>8jhrJb zCKaCos$DX{RQ5%7VH{XPaBcTp9`sf;;2PswpR%e$w8Mk|v>Ckm>C|zytApZ|2_Scy zWS&31k@{n9T+8mQxLd4z3{=(G*deeM8~lOvG~@b&;mcJ>nIT{IxVp?0$&SvL3s0BbC2HYA}p4q2|Nm`e$+O7f$nR&JNN9b8;E-K7eqyB|nQ+jd5do?S72h>w=H z`vc`3nUL-#EFYqvwOx=^GS@4AE6&*URV?#>4$}US$j6gc!8_k4#g&QQ zIv!_e4c6Oa!$C_$y-2Fvcdg5{3WNo~ZRI@`g-Rs?e~o|q(Aa`3HuwQEUA6Jl6!9UD z<$U0j_e{y6z!cI)IR6d2rAJT^SClMcRr_+CYXW4x5MIUlUCwG=5VwK*@i1V(;=R01C3rh=HP!5pH49tE7i2XD2eDd0UlHi(D{^~J9?z( zLw(P9AxM@NkywK^YkrvGi3A_3Puu1Opvwy3`7<==h6-b_#ToYzv49=@l$N zLR0vL2Tnjpe_0Lba%>7%ZDd}^P-n7#NAzZ4`I9GQho8c~JnrCaR>97G37&sbbqPxO zoM`ozF+vy|v!V%72>W?+yG<<)!lD~KTLiv4aLl}f9(-e(qzPUbUuciX=c_Vur2U?(TPxLn*JFsQ zy{3z#j#OpLX`sR2^9~Vt*$9MZ1Dv9fOU|F05#CAP zZb8985qtxlp!}DGeIJ)67>-rLPPZSEjMjI|Fxu|hXm`%AFt_%m=%}psXWj!krG1w{ z;9CjHE6osC+$c#cf6C3!$CTPIN<0RPRq|(lc?wHL5;nac z@!9E(0Aml$sD`CTa(2sHiheg$PYC7qDHoi@pFYqB2>q4vBE~AxqJE70`wkZEdoORb zOGTB%s@7_KZK|h&4IWW^&{mAJ{5y~=FR}iN$u+Rf5s7#gSY1^MZor!`*m!e6xq3H0~h67;o9}OW!l)4qj2_~`*rFNPKJ!3Y9 zC84p&a`|z0xK`J}eF&8uRUXW@1b4df(T-k2HvIE0xDS2Olr0rQMqh>Jsc2W7y7QSL zrAhD#qBJfpK1&W<2kCoDu-@A_mfAVJ{W3;HGdXOxr%FDU4$16)7VK*dWVF;2d?WhE zFwsROw7FrB$ZFDK$Fi{_xjP(>ijoE$iQk|>IA~gxYRgHlP{MBpG0&VG&~kIQVC3Hf zk!ePoV>3t0^BxEP>}9RyfThO?)u(-8zI$R!vz$^eAxW#^NxzjSo2*A-d}xp(B1V8u zK#uG^HzSkH(B->--c~+Plq~jNz0q*q%I=_x;C%{P3BAu~l@pLEBY=Y94!k8mtMEZep4u_52Om06vWmvO^ijn>@-INaWA{tvpp~)OUQA= z&|jT{L=Vpop%~^jsN)3wq>yo%LA`!$*O%lM&bDJaHYLYg!9pffc~SHDb)+*>>A{A= zvuvmZrx^c#HFxNR%Lr!@NWJa(6B7#t+&$B}CW?;(;y==sX`Dea4SMTkBz57~Uxsj4|EPVyHhbhtnm8j9tdQE!*`=33x-i3Y;=9xpgmFA%!_MI zL9@VkqEW?Rb=~z~hMbwEaCY~UVDTs3TlHOkhbFu35RA{n>}ZZpZdBb`yF9QzQ`kKq z!Z%_ATx{!n#Tt&+3n6z`a#$>ZNxHmh8X9Gj_5%lAJp4`erWhS?TAE>-JpJyM&@q1S zb0b&>j{k0$cEz&|a9E5xyQNL?qT??Y7fwc^cD2|`7Kss;~`&ffT{TXZR#1PLCcuM` zTvOgkk;ue@a^q#RqEh@k8gUD1*1VBC?)xO(;ssaj?eVvOpJ|1z(NMYs))_y`i`Azu z@i4yWZNd(-GEe9AUbfRxGB=<7Jc0JJNmATGYRom!7n#Au=IjB%`a73NkLk02WTiy= zfvr_M+`vcMK*>j<1BY8SVy)?5y5AbM#L+v(TSr%airhtWtXODCFD6pxr&B}j`qoi$ z8g$YhaI?yZA2RU!*hBzt=s|$O(09{DvI+O;jiuF{KhQfX0*+c^((TY3{aH=M51oqx zq|&r=2mvGEC73a5r z*%;x`k1D+~+ks65Yi&M{?~+hJDFL3?o6eo7Op)5*)wl)rdR%$CAGjAvTVx!yPQR1r z+?;F_7`k~Riph!UXCoW=0gU>egPV#yy)B>vt49hXe{E23kp~*7Wd){xdWZ2`zC|Ap z*xf~3Xh9j0?leY~_a31hAD7Q79FB-NhjMxkbL2N`HUL0Du~5p3`R51FHwFL}Qg?Ia z$T0y-M)8P-%Ht&~yP&z&$-8@R6ys^}8(u#S`syuj(S0LI&&I{X!P>R0_A2bM>(s&9 zo~De1uN~hH30;=8^bH_?7V+~O#sGk>UQ&{&jN_jbn*}H%s(+;VbRTbWp*%WDUlKf* zd#-lZj2b$41B}PbV;N<>OK4ovS!5c7`1vuC3TVKe_&ek6JL4UpoD0^F60d1ls1BbN zXs1W=^D{qfg#xn4pJ+%`R6!JR+CeGoqIO9Ot-{zuhK^Cs5GjSr_e{#N-5OW z9{%P)FA)($D+Tz9q(NNFHPA7fguW>q#ZL$O0%;-ypNkms>qqmv z(F|1@3uJ5G8mHc$2o3y)Qi>i}4l>fc#?LstM$xwCI=y%RN1@fG5mr=taO1azvKOpu zo5EZwO$Gj-7i%E#>*t}N4mNvQA=+>EwT18Hb8&`$p?}lJ7B1wpug?E?spTnH^wgj- zmC0rVc=nBsZ#ncukW1}~7)eWE$&4A8kjs+(&6+~sqfP39C$7C{yi9gyG&}<$W@9a6 zP7vCoez0E&WV0i|pf9BcQ7e}?0@zTytxs6D4ZQg)W;tCh@0dX0aaO2t1P|%ppRXbN zY|)T^MTRFG4+>$dtF}oqQdj=tu<*91rL}sF^_nJ2LUV~_2L&~)H#vC)w*<-JJRa(R zNp?PVCv9+YT0F1QL3&cVC?tc=yz1mt8yVxhEsB#L-9{9DD^~ld389}}>Ux~4-YSwIrlzl-AnTme zrIvu8Nl)}BgoH64AG!!iLo9`ZLIEMTO4%gaIIQL-HJTS0?_{ArJ3k8=vaTL_y*)#^ z?oKxSs%OqcJSprD(M_@y=H??HY5fD-M`Rdl3=1Pj^*yDrK98kZk+zHzOcBZFH8S3R zeype97-me~iVH(SsFFN+gJru<#6v^MN^jM$NN3q{?34?CyQDp)&SBJXJ`yI-hrOLZnM^tAca_}d!AJ+) zoHCAC)x{SVlXLp0e{`d@!>KAVNjx?`3)g3nf7ND{Qe&+EGGvk$el0He}(+KD600h#2eqEp;h>^R*xZ zNvq34gH0IH-|`_IBa!N#qG=zg`>G{f(5>qDAe5PEu=p9P)8m5`PTH8^OnnJR4KeZqjH3Kn4?$MV-4)s?`&4~VyE{(lbGHO8!W+>7 zACpwp>w<5Av=bVspAkOkJ#ijIoA~OIe;C=@wT6VBpGmqD4@Ou{z6t9<+F$8Pw&=9T zkEf%%Xcj-qCH-(VCn;7mKfhwWwG3+1lHq@&t4|{&^<<}vluyV7TyTt+k8gAnK&QLOE|PAPpgErhVIt>U#82mx=u*;SNd-3h%sDhl2Nwgq*PRL zs7MwPoKCxD7ny8Bzu>*@)RWv;Xn@0 z&F1E#G3Qy=3)X^C4?unA5(Ybvn!|Y42*~%&%w!zML_2w&qgK(gD7f?q;XT^)Uh*9y z1a3>t^iEn3rn45c=T}vA=r!vA`Pl7wq>ZRN>E;rHHm29HUo&BcOb*nJf8E>0_lz(( zN-lOIPv9eMA%xM`FEW9zVIre_Q1euE&XuUC;w%)+S97^clekTkCEd<-azej~G~80uVxor$3Da3(fT?xHf^61&3@tiw~}ByheJN zHS{7MeI7l4M?O~4Vl~UvG>Jnn9yCNMl60Ba!2m<0F6B7<;dZHJe;(@Bi?ZYm`U5)6 zz7R1;`nzBRH zM-xEq=(lZurktKe8qu0y`uHEUum-+-zh|=V$094=qR|G?E7PBvPB|t%3|RINP>Q9P z`K39ffvyDYd>c&9f8p5WGvoR0&z^j}KiD|bj81z==~zo?6&Bqt2qK%qqJ}55km`ww zwEr@`BPA{N_{_s)?9-XPQmERV5nWD*!RhBaeXVI)BLk2%g?b%y_R49=3UdDGliT%X zi2lAFK+EQ@D?q#WJt9gnKLQCa%Jm-f_I?F1+1Z&vc{M{ce**nJ(PtL*Y3?Jq6;`Cl z^~T&IX^nu=s<4jr_hEgzcBJ*sHv)5?)Q3Oggr$c1woj}k)j7LI((W+lJfg8L{D$!O!(P9Y%TpP>j-5i%|1n9J#~8L6N@U+FK*)x_Rh4Q zq?sAwvu|-Jd$0R5y*`Huhx;ek86=bF70|QUxk#8%e}vjr;c4+92i|j}IjFFI~u8r$nrhO1rqRyk! z0N2>flo2++N8*r8&++p`utt2>RuG9K;g-FK5&Ym-!r;*MIol=Avz^XMsn*HuN2Ll; zt8vSKf6kPDaSkp%jIPx7U3DyE+>YL7NN5)?b(H3K9tzWG-lRbcUbS(>PBMf) z9FNct;5JA++mhxg(@VSp7jcaK(xu>F_*ZI0=n3#-v&_*s;=x+1$f{ne`uf6pgd};v zd>`Iv>f*r%FIznehas>qao%>jLVAq{?(&H&fA>&Nu$PEW8r)lckA5uVPS8_=`W)fI z6oX((VFjY0{lL^28?=Dj&@z_Uqw*UYp7L-WXx1%j^h-Y#i6FNy6#6JRM^T0N)H z2{**)3$@NPWJERyXq51!($iIDUaqi28`W(c!|^3v7DPqp00TDdK(7=n$R4!uEKG9y zf0i{l4G&98QsL4_@<2_34H0y+{5-_BtaYB&C*2>hkx+(&I3F+QL#|2Y_*rjhfJfr@ z;j#zOks@4^h@VqlpdT=eC*V;=va7uwK)|I4_lXY;8Pn01sqdx0RYn+9GaF$`D9Vv> zDH3&Fuw?!6tvW)oZvmHquL{SamQz`Af0+{>>1&$HQ@jWzRV|gWrry&xM3w*oLFq~z zcV$LCq$Eqzr4E9Vsc!r+P&mTaBFWn%q;O4g~Z zOg}K6>o}n_H~H1ziP)mA!Iag3ezU^$x(eOHa*DdLXd9guZU^0d=yQu08)1NI@P1xP%TWTzf9#S>NKh`=Qum1DR3{K!HRqE{9|<-$7%7~ebDQ** zS7yq$?B3qW06Xd)laYEat6I!uTC%-5*>hAZUD3jFLpQ!Dnh zYd9&t$mX}^(Ke1B98A9!H+af8ktJ^&*+ef)gW8~&K6s?16Kv0fmO5u8e{hB+E+q6r zJ{$YU{Ti~ZK6r`nE|{`DVWjccuKCY{kon%zl1tCInJn5hK-M0Y92^JgLq9m3EW|+S zM_DrfF#DpsL1oXHjnqL zE6|`fWu?CYG9N>GK^{CRe@3#s?=bA8^rvi6W2u3<3+EgXRvhl1cs*A;fX(IodX%p( zH;mhyn2$kji*-eTUszZ&*@gx$QDx9Ilza?rUNwv2o>M~vPmnu)_Y`-W4`2`7%FB;H zQusotvCFki^gBlx;LA%plk^}D)I!I)%cb&pzfiE7IL49Gv;DwXe`JS?osS>VldVil z)9;5+ujnY`Pk|^;<1qV#z~K&?%I(TwypHDu3A2f1x(9|7IkavY>8lz;Yu0fj+WK`B zT!xb40V2IM&+aJQ_=$SRGb<8zeMV)W+Aforn8S^@V z(^ti{#Rq>1+s1>>e}}yfq!jO=8d6)uU)1N(D!b@b~$GP`i~F^3{1cd2g3xt{8zou_ZqXs<+g1K%JU$!+}q`72EC3Yyr(ROZp63AB{%<-@vL;;zb^bn$_ltev2 zF3xAJD^xtS;nyD^whfKyQsa=V%O*l_xZ^>7 zO`w+#^flXl6=du8JI(mLdYL>tl?Vt-$Cz68>MABoadKB| z(IfaNexP@}h5}|JA$9g&9l{vWyLhBtL$oN-Xdkige~cFU{q5K;wpnS1&+iap)d|Pu zOAEr~-3#7_%DN$`&;>bQT?$=&nTYOK)Z$z3`oOdchp=kWLL83mSFCIn1G#S&HyKi; zVg7ye$`&ml@nv2@JIh7x6i+kvR$}9@cdu~G81v@Ykp)fjzJZ=E+kJQIsPlBBU4a5( zbFHjze-n~5F%svHN8OlF4I^#akbZC@a<2Go`&h8}l{suwp!ZH4%ZSI&XTXK_BeRxt z>{yqGIqckf>CKJ@y!K?;KHP`-_2}}#(>QLt)nH!HQd4hh9(vc)gdR;?M4t=fh_q2^ zp@3$X>8)u@UgmS-*+UV?J@7YzSt-Zq zx#KVnt7!OwrwGOOxff3>*@=#ij-H!Xg=zAseDkp!XE({<#RxaZes(e`w4xf;82a5g ze*=*4pF3&qBQ633n%}jt;NPP)B4CZ*xfp=!|2gg>1HJ<8{nfm$l{9Sdw0RGB6O#kc z02$Bhl@x*!XCVBFbHr8VG)xyUwX@Wi=?Z5IrW^U3b6j6DgR9ZmPex#ONk3|>jk zy#}n(Dcs&~?EOlEy{asc>)d`;oXs1Ne~l7c7n_hx*W81*U52T6sj^##(|LP$j6)#! zOn07)O!5=&Znbvu7wn~uWeSY!J9i6mJrB4#$b@(ePAArcF%EYpw*|zH0;fO)0*Nh$ghXAyKN7SI`NuWyq2jVm8@t1}X9PspXE!#F0uKmb*(Xm9L@!H>OyWBy~b zZ(S%f$^Cd53Y~B~y_-@W4>uOMsYP{>{6573ppzqu3~w{$S}Q03 z435^P;y4{^Gpz7B4s+8saaRWh#=(`kH6sf9aEc}*MV-pXyxmE9OflEMDvrFlj#(eP zm&{k3Z(6SQU!JcuJKorgaaMC_`eZj%*JBlEerNIros>*vmm0Or*i=;7nJ`yj(e87^TgG!}ZsAt32pHV=gsqEY9wU0ZET0`bh#P9^hf0iY}+-Aqo_Qhq}j6!s9 z;3ej~?Pi|5*0n6uw^f5Iw{3hgL+K4flzVA2lhS1^#xWxcN@j|CHcK%Th*TVw8okBY zS1*?meKgX;Yz_y9noX`)8Pgkd;O8(cJ03(;86{e&;H>M1d(7~GgQ z?@GPW90Gh`Uwb8d3fSIKOoy`6UqBqdGPM^k-fbOw*fT0)fF0ydw`J6+;&8^;HT#N{ z;*jS9@yA$jCJ(`mc1lN9K*~2Ou86LOgj!DWt2#WDe=Xj{8b=Dp^G9x;svb65+q#!< zvQhm|*oNVQLg3ul@7R}vCjvI>>w9ro(ODWjiMqTd(P@XJ;|K|`vGHq#C0Lfp+B7MY zl#0@rzpNiFJ!4l~Kl`Fvz^uf}CNILK*SxXqBHUD@dGF#o-y+UjJj=!UeCg5X)kJuV z>b#IOfA5)Z+9HPh%_O4MJ!9ZoDpDX_t3e`ZZ^lqL?7I1*$3S&a>al5Jc+2|lMaxnB z+60(=1&r)=?#svAXd}aq1wksEuu3z^V2Ao(faVZDYBR|&?Z8T*IA&f zuQA|h%!^UIHTdPTV#d2+@T$L1N~~6+;%S^-e<}AN>?W5^FPW2|Hp5vHSL!xA>E~rjcVz$r+*U z=o!{IFO3S6Nh1cI8AmtTL)=cRGq*|u?@x>k^t5BLn^ADjm`HKu+XF_RW ze=bYZCRTF*#|-s$$Ms@#xKBm=?S|KW>G=E-y;gqCR(k}1KcDaSbPWmz#S3Kb;tTbn zY4hMjlX$$e5<+eKS45#MBRo8}CE)p$>~r}hd?+Bb1t&18i4)@!oU8EkztxvQgU!jQ z^_=ui933EsHL`ov@pF)*^DVYzC@JUjf2w^0b2Y3CXJ)swzH?M% zeONmiUZ$>2K|I!sAM|)d?`jdvW3~=^E(tP<7l*ye>k)YaK6F?rdSmPlq!mJ-Q>b@J ze;>py`}^z22X-mB{)j+D%#G-$vR81LIb6u)^HcMBE{JEx!Kd;4;-l9EoES1mf3U9t zO==m8y>*{!OI;wk!GCi$yqFl4nkoCn!Bl&CzrNwlCYo5pk zI9HfUGSaJl8U`XcHGPf{&uF`Mfr)(rf0=|oZ%vV{to`XOn^qizM8LEn;fRr&^X%r!Ae{n2SGP|${LIV;-dU@$%&Pwl;buMNKP)b64Ixh0Z z<|7%r)AJ-R5yyO+rLI>+Fv%<}^;kyKa6=3rUcJg61m``HOC4l-549;N7{X@#uiZ&j zL0v_l_sz5&-WkFv`S1%v4^5RdQ1{Khf+iY{Y9I?Q31uexN@q_`AGFTae|6JF?qlrk z)=bK~c;L@UM2HOs8%d)1ME%NJC4#C>rUU z_$dI#ei*S=fLc22rCb||!O6Ymi4{Xv?!tyq@ zZX?Y&8)IxBHLg&y4QjM-q(K=FYr2vS;oH|}js~}WY|Uv2iTO@>e@}CliXf&W92|#| z74v+hMw4k9YIP3}gCQW0&|F`~%6qg4lJd(oSzw-Dp%BM~W-j_s69+{m$3fZE<1vBP z%TFcuI~rE$-Gz%j2_svkz|R3fax=q{0DImi=`r(P$?e}xdIEZNq8c{0c3h!>^eS0;~?kj;r{_!zWG4h~^kkT%I>*{8_D z0Tr(00LyhQeg-CSNNVSoPI0S+qsjmm9GRAZ!54H|kgD}fi>4q~6HauWO_vh?Fd50f zi}KZ=hcxC}LamxdTx7=)2*5N?lPy_TVlY4UU%t`qAXX~}f5-!uL0@Wvhp#lea0_C( zvvH-vU>OSV9x&+YPks`MT$QkO6N^cem^{DMIits(ak4-lcq#V4o^#S#Wi0pJN5G#$ zQv0IRRhp8B$sQLevl@+aM030z?LG}S#MD+NPC|IPB3?CRLJn1aV%T@{svi9^z=2G0r%X9`%9@dxdIIwd<-9YQcNkcUrzZ zxa%Cge?ibw$~~VL?i|wxq6hU`B#qc*@k`m%?>8(TwTvL6N3farNIMHn-=|$tf)h{= za!nt~!zN+rjUf-*y0#2WH{cW6`ufLTWKcnn!SS(<@_^zwoegXuZH5=$s4@UDdto({ z-TAEX-SCqRT^tX-L)XjhG@qA1>y7E_Sj$GCh1U9h$rm@k%~umb3^#(9Dh(W|TFE(qFRT5xd(3Pi0tRbRpGv#>o%Kp@bT@xIGxZxVVOU2mR^+ ze-Z@k!1|xPR{mG3mCxtjPAdTLcLBiswFlXm%0u+8-eq`T$pCQq@0+oBcTVAcol*b* zSn_Kp;P{@rc`f82*2l92y(Lb_ zpR!RV(rxS|vJH0taL(`dck$N=<=6QH004Wx-)FWsMEW>IXyG3J{XF12a*@6Y;S%_5 z1%Utj+Q-P!-T8$1bvgln5$OB()%Zmf2mh; z0F~{tzgTTg_c*gg*RFDBc>?#Jp1cX1&(HH>k=XlB) z|90Vp+7FmZAC=aJf6sAzpV{+Cf7tUkS9eKKPoh@sA#h@5RjOG!W0h@PXm=+@1d}2H zO5Sg??O=u=z_~yr0G9*XW;6h1%;{vvZ2SOR@&dd()Grxh&bSq7-)D~a(p#K0j$D*a zd1M;BxyNC&`8{vUXn*7Lxk{FVI*yFihm}Te_%j}vDh>oDzVu2*4bL9re^V9($7YtD zC(Njs0qlfK|6z3|Zp8{$MK4r+I5{KKbp5`|4PIIFu6j9~zWYc5G#=n8BeI#>6=ffP zqR<}1CLchkA7H<`UhHlWfVaSQnP_LM;TcEZGnL+fGjw-ccj3mk_*}s1**HJC=3NgI zJHLVlub#9r{iPIQ3t+kXe|7HxJ`iD`Q0!+1Dgf!l+(GeYUEtrm9b011Dg^c9D8PS# z%wt4mvSqufw%gt|D?)J0guB)N#s*^~I5qD@Gq&2X4cb(%;UNxco{lzHY zfdV83I1~G=Kyg2Z+yB@@zG952^6~Zw<;p|hvPXT7C8NzFrpiNM&j;m5DIIi-#o)2& zPH&^$8*T4)wz6T7e<1|v%psZaF;Gd2*~*LTRlE0Kegouh$F-SIB2hpqrmUW6;YZ8rY#dHocQI20UPe`NLob-*FAEaeD0Pn3K-nyF;}5gkr9u8wufMp{#Bg)%v)b-+e0dC+o=E>(`D-2I>n!KYz?lg z^G#)F{`H$LxI1sL3XtA_neO@OpyE&U&_~6N1HrK`f4z+_2^12|W9iIet{D>ORgUh3?B>`%IL4g$1N_A}W{*W?&qcwxF)@!oJwAlWWaQkI{w@1xKIy~`hkX3h&O_!0r?XHd zLcq!eem&G&l_&u%xeY%nwN2}8e5s5QS(lFnfA+oefI(&@BcP-Jw}SwH2H_7N0b9Ot zOj0htHf$af_Sn(eT#^sF*AISVs+`$}N`RJ%iF#dSyLIJJ@3ES(;45OOg83=g63^|Z z0DX4e>IVR^5)^*wodV(5f&mAa$fq}9E^>f;#e`r%Y z0Qw!b|6l`Zurc-@AHMa>AKv@(S^n3D@0jwaD6pRvtkiP<@s?3ZO3hVJ0UM`wIeDn&9jXno45Zm7-ZRU56TJ%6=?7E zlbzjW@AW(VfA9=5+Wf}$drYEyfBC3!#hCKL{{Pl7q8RkOn)7nnj~^x+X&R9UHmW95 z>^{169);5ylNB9O+)j~m)p|{R*3_XC)29)csh%DVQnABHYgljnhuu2!1XB6anf9&! zcKNUVbM-p*nMNKAM*oBR_gJ~4w%BR5*taY->mt?gqV7_8bAj4kkL%`xe{-eO>eu8D zh0YwnzxB_7ccV3z!(Zf&2vleDj`k+n=dy7Jy&;)W*q6H&25mqA7c2xoY|Qm0$FwJ8xc6 zAZP+<&ThBAvHQ(S1%Ia~f33V2H}3uME!e1a=?_o;6N|aT)V?nq{-4jU{J(W*G&}gB z9(bXitKl_{oOlKsbtkJhhiIHN?s)iH;cUPB*QR2jIsB&aeq&#SZK8H5JA1^sRIIaI z8C*)QbX6-moqzL-^nz+|t?UZGL*cJwPwIy0&9d$krpM%Ey8r;e+2RXcab^*c*teaE=)TxHkZ4h20D z=h6o~15)J8dHMlue*|xYejrqZ&F}iX&N%eoxqVkZciQSzOn&DXK}V~OJF$13q1eBC zV))k%SNz8Zo4A#K@rJXvzE-Xn`)_;vc<#bkz0S*X|a$X=i-l~v$4Ci~E#P%k6mx+S0?|aA2{cp__CdoQJ1Jjw zAOgcS2fejxfB4pk67&EpYO6iqx*Okju>W{6GTJ|RoM8WAV^H=VhTZ;Uf}d9ZVcTs} z6Z~T2r)e-@|C8taiI*4!UY9ge6`Bk8*A~yDQhgf8wg?A+shc?zqfww5k&CFsM3I zM03?2)G`!S%l%W0D4~bWRY3g!?xzn2$^J9XUwr?G_3+Oe{_!KdVfI*3uD?0zUu7Ie;0R_+fZyZLaGkyAvsdN_UZp@7R8liqeEjN#+ftrlIL(J1Z(V zxuc8sBm7`}78M+PTi9}SEetG|&CH)cI$B4aS& zZtK!fMOjS?cMSvZBZTi-0cHwVcb;(K&7CIzCyoR=ACxoy)a>RGAE@Y>9W`r^3d}6>^(Y#a z$?dMy92;}26L8+TaCEQAx&0Y95DK$crLI7@b+Eznf7Tfj>RkF8$0hi2(?5FR=g0Tw zf8)PcdHckT1Aq8?P)=O>>9O;lpZoE{Uw`@KsDI)Eh-lx)odb4LbU?10_M!!-?RK@w z)oUWKJD>4YsyN-EUi+amb|F=I!Pb+`SCtgQYGoi)Db~ReGqj_6%px+q&PjTwcXbWi z4<^Do#XGzuyYb;3>m!2&$*KcKV*E-te{x!8;~G!{8CO0i zw{EWgwZZuh=ife}Rq_vS`STA)-M+-#{KUlJe`#6iPwgjX3|5DHC$2SyX7dK8e>n-* z+BqdEF!o4ftpIL9>lFMpm`+`IJ1nfvoV(IuV7_7`>5^Wa#Yz`>8!osrHdI^jId*M1 zfa6kAPjVm2N>kgan|mUEC?QXaB^$V`_i0L4*c@(rERLKQ^)|3>?11ag6vWLE1W{el zS9nG@MaQ;z&>4)q|LPgQdNAP2f3wT|S{g_%?ePEd^B+I4D*4tz_guF-$2I=rFMn!b zx1Ze@QpbOCkyHMCZrnAHRzI!EMt&Omltt!;KSJ|vr$KRZTRv+>;PhQ`r$gNWRX^VX z*V~v_2Yl!TeAL5RR^z0;tBA@pzb^Gjm*V!}UHe%2yz?3#D|-fGs4qh(L&;6vYkdhrE4ayHx5 z<<@QbtrtBoaeMd2e{MYe*(R3cJ1@Te$;D&l#!{)$Fa74~KiqNq+F#E3;Srlu{Wls2 zR=xsWwsT8#ecW!U_sUG3e|C&C4)SLLU`J--D6VLfG*DbL1Kzk5oOs8#JHoXx zWji#-9}=A~GUe4>raETIz4=l;K7j=$+;mYzMw^#*&+7}}5GJ$9uDoTz>wfF~yGfTV{;fa!!HVf7UTEo1ny@G=Zoc z_#xE1D_%mRGmltpv`62?jDOj?1p*^+dQX2NZ`YxEc8^uD1q~bSZ0zKS%z#93;7r`{ zuid@}sFxp%tLtqif3=)Kq&j#xcCR{@O6$QXiZub0Mo)N0xwlJzU_k>oVRo(<9tZ%i zTA({_rT*$T_O0#ze`mMEUw?_F7Uj8)oe>B=pkCnF{82gy(pcWJ(NQ&T>!AQh z(epo>Ko}qC>|CzSU6IKi37Y7}z6V?b!uYB<5F8A9*-%(le_@0bs$9dd!0^5p!oAT4 zlI6Zk)6F*@?Ly(uE$*}*Ove}wtix`+7phwo%(QNtlgsKEzlv)5t21mC0_qBQ`PLad z_lJK(=B_vV^k#0}bNm|z;@{d!^uILw&sKjpS+MiJHS@pqiJxZieGpQIy{DdsimZI- zx_BO(CWT?qfAFE=_$e<423OUb?#Zc65^v9yBc`#hg{e{&u1*7XEMW6b&86bXUJnP( zd&BJ_(Y1}ak{$gBwBWQWRbX7n@iC4Anppd;?%+Px<&wq$QaguIbHEP|#V_j$^Sy5pt0aq?FO{KL_)fA~Ls=FA`Ly^rx%%WIYXkN^Cs zrB;7)p1+#&kH-Jyp+DP7cRb#@rn=+K_oeEN$DBW&hRrlP!B>?iHFje2yzHdYDKogo zGrTBPQ@}dB$^+P98F&|uoo5|?mmSdXM{EZ4`qA8O;dZ-Xj$IXPDiZ8{)*0P)swq=_ zdr`QUe>U5W(ila_nEcbi0-M(T=<^j?iF?6SQ?4*wDDwEA*gVf3d?~Pc>WRqr={Z2z zZG+{f#d2Z%i_N{}0F&GEf<0n|2eATn{Vk7zW1kFg8{Fh-9#xgcYq$`s}nP|f9GBNpS&R-f4+Rm|D8Wc|J2*`uy~=7oRrwOjSwrXa%yZoW5&#d*(>JYctfK*OkY3)6UwIecO+P zpcY^}i&Tt$+fS_i>^m6T?ce$J#}HDr$PWwt#=$=r z^V6+3C;!nC18^%{I9E@!SAnzfgg;|^XDW=PT(5*o!S4zbnxhvO|%~ig{0G$Ok&oC|HMLx^qu#Ko4e~$_S zF0!><2n^weEMG-6Wh!T^+f4G3PREqI_3lbZ+W`zqRmY3=Pe*vr$#d(Sukq`BgNOVH z6-(Xun{S01GY$72E&dljZ;kaYHs08Qv&{Kl+Uuw7|G@+Or^C40zRcge?#^ot+u@!3 zUw!(g2AgjT_ZN37iE8RKGa5#4e=7=-IGYd-+mT`WqceytH#`39KX5KE;J{nTR0|PfRgow8kj@3_=c}|HlU<)f6A`6QX8-LY{8J0Kbc5@9FG zWP(I=`BC6j3Is60wN=Uce*vTM?$_)5S_wd2D3?$9IUq;p~4n>o0!pu_V;_U03<5weG#qe}C#*Hzwrcf9>6r z;P}O8UL&qbh|pVGX0J{aP&r5^ z-?4Sk^}0v_6il?b$Gf*d){A4}n5`$4KXh7foD5Vv!tT@R@G%F_7zM-nmTLKF9dWF& zyU!o`@PF8Qx29!zf2L9Jzs%W`H6<$UzIC!XAwbv~w!+DjqOw&~RKThE@2i4d_kBO# z`@GLLt5(e%&2@5tX+o#dyL5Mg&Q_)B3R{H)#1Mo<+SM1sO?HS0FDp&XN4LH?skn{Z zEZd`1YqIqrxYMiDi|mv}45JcD4)lVM%&kDld>PfmUER#Qe<4HsLpH`Dvvs%$zHYQLKk@uO;#pQV%+;_C=D8TZHdm^bxpamu zRo;bOff~2aMe^39tq+!*#}V_Jjg_~)JGoZbdb;18uOlmxuj(Z`X#!-5b2MG2#NeAp zcS_r9HIGPce~xue7;F8iF@@Ha()2ESr3GoL`aVj6t+fw#wztcLUSZ9g`o6u9KY4cb zHO`?Eanr(@+n0@#eC+eb_WsadhS9-G^l);u2F zx;H23PA&uYpt1|sD=Iwb-V5Kl3p2%O%t&4>s51c%f9kitiguBZV*h<7`+nPON^=QU zYbvqTRzr2oeBq!~P%Me>Q?NvHY-h6nBLXBTRQ{$7nt3%vK@wJjv+I89Cn~sm)GFvD(U6v#0N39y^R| zJpXtIoGx>OvDBqq1?>x;ucjpVOOiRSe5Yme{TfB&r2KI^xSE{|rL)Z~5x{^k?HpSqk+ z9&3=75oD(Nq1msxhRN5@`xVatcWuP!Dm93?RyvxJpmjA=Vxw$i!V)no#nz4v6Z5^^ zNFJJMZx&;=%AI`mFpE@Fr8cD*tkv8+>O$O1RlmxkRU>z;EVs;*h4hwU=-rm$`@{jY ze@}WPtraU3W2GW@3r$X1H_?rSOBoS4nDZmB?z`wVVA=XvX6lluSXt@%3@sV$AasC#A~+ zPmFkeg8X|L{;Y?GxT>6e>jMUP`o_{ffBBl9*nkbc`EqJ~>m`hodg!Bzrq+At_X%9K zyJ20#eZjAPJ*+8bAKw1@+8oAeT2uNftreV&13$Yf3UZaaN%gn9rEa{^9EiYeuV$a@ zA8wMX6*{{eQmu@2H8v*>6W#~u>S!6bnsB^k9dZNLbleVdn)-y~tBSfMS-DN7e?Q=C z_QKQw8WPF0rFYFX61Cx6p4IN(*oxj?c|KdPer1~yR9ZjB5-L+l(o&fx7tfh2C8x|V zOD?j|Ead|E#so2=4>Tu#jbgu%bw|7qtVNq5xtV?7_LEEc`syOq+UTQC-k(?rkG0cu zs520T_&c9VUJEqDy#^g5G0_6U6PvO?pif%gkWG;a%`?c!Nhk17VSNp;;88hq>^Jpg z!k!n;L3&t?ZI@k1>0Eu4(P%4L1ee_3#H@rZ!|Nypwt)WO`4TxQ=m9tre?$uhxrxe& z@A*2dwFKTNo?ueRBzCc^p#v@ts!N9t;sKV%dTPD6ehpKeS1|Wnzy1TCf8!cS_$znv zp~vq}}4ak653bq4m4W^%AF! zy;U#u&eIK>1#L@=I+~bhe=QjhE!{G*vfeK|7JI&Xz8liSB2%^^9Wy9yHfBZca2B5` z`g>kvb#Cq4dZz2w;B~ut`y$BT%+_J#JXmf%U5~Vim+Un$_N=@{mRAQcZWKo_y%^U7 zvcxiWkBO7}pM1qE!bfJ)Lsp=OBthCJzA{o{O%^J&u*1$ao zE=ewK7FC2z94ScGe|iT~me&YF%LupQ*c+cxDTk*InA`C-Q3gJ16UXfQ9Cai#N4i;% zt6*FK_b^+Dj%mO&`Q*P~IX;2J;AwuJ_sNG(lZja*j+UDfYrbH4k!i;m@C(Waz4{m< z-}pQasN*}xA1fZ}RhpI!b#}mfRcrkVxGAD|HXXmBJv_#9e`UQtNW9KRmc}Az1{_je zJol*+$yyCQbt3BTdQD#6_+7Vm3Af3-?E*jffG*masoXsaDuCm-$s5@$O&xeMO-n;Z z?Mn>BVW+fAP@R}1xB?j;-bd`nq!kh*=*mW_$1>7hM(&HiV5@H-Wd-wxdL(&27C{x^ zZgM=!Pd!WHe+~64ojLo)64XHkKQcG_o%iv#gWl`7FU0*rFDdeM0-c@j1NxMh;4530 z5lOh+Jo=z+g8ag-AVNF2_d9cz<)NF*jf+&4XMuWU5=riZZItG!uvSqUvUS>ne!{E4 zj}6A-Gwf0{WiZ5KPEM8De@pA>1`%b0t6NF}U@5a&p+@%* zTB)llowQRG*MZ-&rz2&0_^$2}k2{>kXGx>$lY0=Shh98=CWje=auor~uv@jdF+rjEHYbGf^=>h? zn7ZWoetodZiSgB!+`stEmp^qd9M>P&fBEpoz|U6q$xER}i!Hfb`^?%UJG|>o#EG+9BG*y>es`vZVG#F-vSLT&MHH zVxLwfbIOg%KYp769*-jAwEC0R3#&tYnYfoyHt_F&&nJd=(b})9{p18c`vQi%f4-KI z&;Wn76G|Pq(S=m2t5T#gbfSfmV6V9qGpfiVxy<~^I@m%J5Y~>V!^*ASdY)ridP_am zO;2t=TBWsbfnD>um}{{k-?p*ZluyE%D|uH`trcT9>c+f7Wmc%7O<9i0wcVDDQ43z5U#K zH*>YpQnHKGFg*~3U6hm&e9R#-^YW^w8>GD)1Cs_BZpqYu&TTX&)Jcy4e~up;fhCe- zy(35tO95gn7j~QBv(5zK&A0}y!MBtTjMv@&F z=V(4Ow&JcZcIpkS;-?2zd5P2nk+j#^QdiuOCzICA;Kzsg>b6gPFx$$ndAI=*?xv0rw#OmS+Yc3T5h#N@_n)+!vcb>ml20vS zk$l!J9XV^i>KKy1>i+PpI_9Cy55N0E&Z`S?fDsA2WFpCaf3Bx>=_2OY;z-44uKI4f zG7BwsK-AqEd7(VHq*;9>zGwi8&bs8tC@UQ;Z*P%Q-rQ~6XgM|}9PbJ>9R%mVe9IHvtlsHChkYzvX8 zo*(WeYmwfQf0MmvuJ}B?@0BE@=N;Z1cyo8r7QY^OHqZm8cfxjpEZ z8gsjh?q>3>-@5$rzmP}sB3nxzm_a76C1a_ z-hprTe^+9;b>*QIi`w|0xiI0nxieLY2Jf1UU3$@7WzJKj!B7^TA zee}R*4hqXl-j?^(*Pa=mu`k2hjEGm0rXX`v(G6QfCQSu<*8z&>fiB}nhD`AHZ*q2@ zglbo(lCowa-Az>+cDP=~Go~fn07pQ$zfaS+zV7PHSASn`f7j;?woLgKUii{j&XSo0 za(`r^qh^#Mi7FQXIermm7BYP~H|;Dvs0f>dUz+HKTP@kQy2Ge1ZL=%&o2y2yjVZGb zWv0DN6JaM=jnO8BW)1r2%A4)VG z?1=x3Pk#@301n%=Stc6Fua(Ttalh&YwAAgo<_3ARmb)|rVZ!#3dDlXr+vZ|GCnb?_ zJ5qykZF*GL$SsncwC$7=z+E@?rj;fE@tro> z>tL$pRG2B9sc5X>$MG>qVEcV3J3gcWVG)Lls34ZA)TwmXYNgsOLU&?5OdgZSZ;be0 zOF7%?`2)7znmL)-@A~;sYuf;|jzL=*C|R=5*jm>daVMwEXp;+DtuQ4L*e{9Kj`FJ$=DW@qR2X`-&#O#`t0?mvTt5et~_ibP0nCIGw z$n4zN+bA;-PHZWU3i(V@1g0f-e#*zO*?+8ZNKI_CeBbi!t*7;8AK$Pacc_>7_|-!# z!B=j&zs&Ej=Gmn558D-C0MjI$>21UG=%qrUfheIjecEl>Y~Mmo3V9EyDMLmbCUz3U z?RcXq8tKzWKBt1>M!GCKD#uGG3!0OJE?Wi4*_Nbx`zY!b6AwXZN(iW;IDTc|>KCA; z-}<}0L`SAZd_R}Y>jB&-mEp%;(U*|72N{1WmGg6>2QAuSbH>k(B_ya#!cr#HYoZE` zS>&3m99q4&?e>|V;F830)|_=%t(ikanzl!~qbU=r&B2Kr{Z1eK84LN=$^Y`l5K329$OIV*^3YJS zh1OZ_VHHVTt8a^VbZgDmRf;_aBHe##RR_D~?)XkFibN99YZphV3s*5S#2U|N8_xM0zD}ii63VY9C2HxTtq|A>-S~YV zIUYMGj&B5oIR?QW;vXwTILum`9@!B4uoILq%k`NgM~=4T>un@)zmK%s?`ObkThV79 z&Bc0~;kOo-&*{I3R}L~X`{jR6^Sz0u#g^sH(z)~58I zYDuze54XzR6mg(ShVHCg^O>hUI*ZsM<~+Dm*Nyhr8;&1XPTL-3AgbHwwiksSF|nUR zCQAq3!108C`kHO`$xU_sT`n->OBggBzs7s+aor-=a6agh*|)};>!PoW5Bq0_Jc{i7 zx!-rFjpHz%8V~g6zMrpOK5k7Sci1Zqmf6>?T0NMwdJuFV zmcZ7G9Cj97-EX@IqGEG2Fr3J`< z`W7pYG_Yeb1U(82kpPC^cQSy1nY#o8ZJP*)H z@)@MCPJx1<0wRc?KLCt?lxjAAU)UlBr3gWPsxpuh?48DAo)_~$A%J~6^mk4m1la(f z$lHt`{6rASWCs)`of$B`?u1s$Kxy!HGaf<3AZS&-ITT~QpEoMOXf+F<#9O{OeI9|; zss=!cJq9aCX5$e^3qFD46eh3{!DgBP1W=wr1(69Fr!O&4&foxR9~5S6$K&QBZ%>|o zzd$==o%p)GM%Z$ROgATPdtFp--7$=N_r8Hn< z`bsRvoQ<(mU}mP{cM4%@RIs5b0N!j6V)qj>K0EhC?dUp(PrA zX@LT!{0OEsNGk|ibOg1A8bd6$+XC}{c^V>Za0W?$O^Xo70z?d_6cq&=O*$Tr$KrUL zj@RSy>@_}&*YEN4HJXT$cnG>l@WtmJ{`dy$Hf;a}AqirNk?{nuegD~_%G!kn2+9)b zFhplcuR&-4)0jM6xtwve}f1$E8L z*n?zXR5q>Y%M4+*1dsJ7jfmbs=1A;LDiG`zO3OcD+)8-3mZH#Yk&MJ9r5H*pWp~J8 zr_CDBB(|is%w()$+(Ky<>`py!ERI9b0AisM#7!2V;kkU;B7V|A|7>O7ifB&?x`&Vn z<}X;KM2jI^VCC(TVF;I-6IH{CT*mQ zg;;5Axr(VlkmOr(Bm`M(K<&$E*067XE^f`?IcSIHOvFOW>3q{87q_@VQG}Yb!D#3Y zpJ7i8m=1kNajHyF90ujS0a`WCNzsz)*}Mhv9G<0nFj@C78Z;lCSIb9#_Cx^D5==I= zMZ{1Z22~ja=N8;Y9J+Q3GK%7$imBqZGN)x#J{s^FO8m3|Rz)F88#qY5oVTdAa06+< zvL>Uvj}RUWM9r6`lHQ}TVTYXMxW$;m#&U?B9k7Yyg!b^i0XcO-yG3rDQ3ELJu}vma!sj~e<;}9 zv?P9dTYw_!GzAUH8+E6x)b|~vn%pQ(=HO{dxcr2WQUf+jxD7nRuc@HL4MLK8=X{u^ z4T2B?*X@f|?a`Vw3+PD)ZYM4_xa#=Pospu?y zG{|h+i5$5T>!)o`gR*Rp^=)TO!@y;eAZi@yGCr@q-V*l_H-IEABkWE#kO#PN0BgS9 zf)jCmkhB0r4I{v&n9~pEb)Dw9l{E-%l*>d&e&0Kse~}b_8Q(tg;PZQMZY9A|`S_hF zZsGi|yh9DJ2e|{3b3WvA!xR(`d^p7&UxQM?ZNax&kQP1PzwzbYzyRK^$$^}=B)2dn zVJqcari6ma5X({e=+cPSFaa`wl^Xugl@*Nlzb8J%&o=LyJssvG;`9pie%i`>ejWdA z>>)A?f@QaVc{HeO8fFgdO5S$}4efiwAo8cZZ5kF0?W)^%s2kdkhC%43ebqFq9NP81 zEq>3x-J&xx0y!-xUbGMhd+N$uZW{xeX@k&@T!s;O_#rk^;|?fsxQ;0w4Hg>Od0Cu$IXEm)4^21HH|i*o)TG>y0n?EOdq z0)*S6`kFQlqxupXde`JbUvA$420p&rFbJZ!c{JE;8b(t-J`MLB^h0}P7=(PER3^}O$}JG4Iy3@bjL#vWblJ*1Tz-E^n7RIaZ>h3*GC>RPxdQUs3g^Le%N zz6G>v<-y)EZWF7|GfF1`!VNqmeH!XUZl+Y~_2GT}zdl3pT%|%%aNh!P-N9O5cQ9Cg z-=vMgeGG0_<*4ENCtcV#EjuIU8nbe+oBCjzg`{wLE2Bc3HV{kPj{XggrQA0E_%VoB zX^ia_`BoNl`hp`x&XeD#gH785{u|n7wCDY>GZo^@MorPcRdVitwLR$Ui${cUi%R=% z^=-uU3CCjz3fJx6K3n7lSu^+*+TZbi4AIWy;C3=n92R1-r@b8i1s}P5G*a6% z0AYiYg5`rfl(ih5-}x*y66YJ?4;b8HuDda$PHw3KZc`!}dl(r8=T+EJL2scCxn23p z)gB;_(xSHB{lF^P9%Riw4|eIvQhs1l5%uPmx2k_H=e<~ePz7=0S$rg5tTt`P{j~mQ6CIOtd(}unOHpX9DrL z{$uXHz`4o<{VQDlPyKQHc6{7QgAP(C)4eZO5tX{wxRVwfb~pA&G7MPg{eOmq{oq^v z6_ywVEOoKw;~C^-pv`|Z9?olj{h~Fwjwk+yh9CJm|IB-|2a^i)%O^^U1@lgOSWzq% zw(;xzieV6)`y+q!@xLa+ZpI$z|3$mO-wh7)8Y$uTreVOG>srs{dNY=} zKS9Zk`_bZW{KTe#x?p!?aenmrz6U@GzxFY|mOs~S-vj&pp7ZPP?;m)TUk4xf&-DlX zbNzw;T=)HZ2%q)Mqmk2prr~h}V)F6FhGFsymc-%0ynpj}&j+sIZ~*=Qmly_*d&>jd z*5z=3K5$~wv;=+OP6l7A5bndFbh!41>&!4H`u?5|*Pq{u4}8k6%Mbi;U47t(>+=VG zxV{W{03sZZ9p>XA;BcBMCC7u?XAXB8#>KvHpYfv)T>BZnKfpbIKI4}Mxcz7RUpVe} z!by3o_XHt7_$iO=tpZg7dwv79!D*ebPe`bmB0MGql?-#tleZJrDUzuj)7yKdT z{R{U$bjNIXhx=?9$4MRddaW-J;_yfJ1K$wJVHZRHzhIlDVdNL=iR145f?W^&{wXu6 z$on4YO9TTv$6XSC?+qb$V~^nj|7=;_JQ_-P-@jl}j<@#4KG^@BT{{ntb5DG|30Nf5B=)-@jmMj>m7Q zmBNmp3n;^()G4orj9R#IOw1Ho{-n*|!RuvA9>ct`7?hEV)nr+{^I4gobDz6{RxW6&!=ek@9y{BNI(5=0L?&6c**gT&rdYq zo$@U$8K@~3eEcefxP=j~ulRqTKmHD{hSM>B<8+Mv75@2)FC>xI-8B3=0|(CMDuW%z zOQv7?4D&bT|Kovf(Pti2#__Uwzk7zNfle*TKpyBC@%)LCqMoVLwI@_A4rJ`Y#k9x;>; z^1;WYCLDg?p8@WJ7Xq1lpBv64(?9&e=?MJ(1$f2vRPp7fmcu1{-amLY9KU_b>7Rhn zq79!nkK_4~x0cJl!Rz7FLBI6_{zLzNoX@}V3V-kkUV+EuowR>Rca25!C!hNFe>{&i z*g(o{76BOc|LFh?7e$Mw_#~b;L_AN*QDmHjLSA>N0B6~nlj?)oAb*tCjaGmoUL{a- z`O;Ik^YoL!=L4_4azpt!bpl8}SSif79N7#OAT_W7o+sm5=n&$1z+rq`2~VznYka?^ z+aeQ)z|W`M@MN3Mkjwcp02FY32wKkHC4?#3^ZJi1BN&Vb9xai$a3V$&U92O1{oR*h9{1<*(O-}M z+yDN5|9AeCnE&InOOF57_v)N~SN(tJJMXytKjr(bd_3>}ZD?s8eRceg3-6-({L&ut zG<*DqClH3!vBtY;?dlux;P$uk)zmuadGrO8dUUP^} zr$Btd__gt;)7jG8NvouG%BHmiDOxUWxmzi8UQWA_MyyhQ%L|QtB;K4-uU%X2Awloa={XuJ-7JoQHY;H@xwr4x z<~6<}d%D@NrZ*moy^y@0RTu8xqy4nf=i;c;M~|Ja-mB|@5&_}fo;8c*s&C$pJ$1at z(o2H*^b+is&W!bc!B&3k!QaAb)|bKbHP?gyU9Auc5cte;0nECaGJFQ8nV_7X;lzMv z-ev7J?~8hFttaPQA=s4_*lwA1H6Sv|yYVivAVTNEJI0`+*_!d_)l)pJ(pfKHG_=>6Tbu3mc58ZO=xH|E6$r_~J$URCd+LcG;)>phuwsnH{NYrtR}{Isf?HB@nR8=Z};&{zr5!Dgw(j+ z1p#TXl5WS3MopGMa6c{*X@AR%QM(ikI)5zYuo%6Q5tR zvpc(=ui9lt?S6a6r|vZ+29a`aL#KLQI-u*AQ15NyuD;1}x?3o_aFUXmPLAcR-r}`Y zL_(H-We+{y&}I~e_s;dpeY83R#&LX^x+@aQKvOq&{aHWoEp*r`TQ!mEj~rqht^tN= zo#+Cr-h21u_;1fdquOzrF`R^R#HRY4;`Z53!hPc{_H$MmZDSTj)<2(X*Kvbm>}ug* z+oj^$JxZDtIFWo1n`|a4?s{>39pc28Z2DY(uV(kB>g>s*TgR94S=y6xzB9*ok&DcW zx_)L|-;wqTpVxaAT*CuP)ysVw9#!|k?rDu`SvX4AT0fsVvd<1HO<6MEKI)h4cB1>s z(pmP~I#8YQ`GFsy^3)fbRvd@-N+a1yvts>Agxs;kNZvNg-%HTEv@d)oaXu zIw35ncC3s>Ns9$AJOzzbixn9QZ2!{7_T{CkVR_s?g|>1~t~|__9jVgH59E7rQsQ+W zu4*`9BQ@y-9ZO4^+fl=g%KKf5!7Iy*C_Y1cLs|bARXQxYRF@=0pc1azI8cSTy~5*g z-aAC1>)P;HZtBE0=T27Ku2GGyp^v72_ZuaXiOCkbr>xIg{_P-K&ri6Bqbux%hq68F zlHz4Qu`+>i1?TAI`~hU*gd848;~MMyUwl`&{mPab_Fqt+9%H6z0o+c`Qv2 z=dtyiT%+RN=6*;|`C@i1uJ%n{y_>XJrpt9E`TFD{O>zvCTD6{+>uX28y^kw#n^es! z%ckrkKa__|^5Wg~1%aL3*`}A_zGOSWmYp#)9cz4zU4*Lao}2M{8PU-C>saS>WxT2hD#hyyG0N#0B)bxV#_T>FIhE=(FxK)0~74C*H`Oyj?a|JhDL$&d#BcH+DS=1mq)s>3xqqmKT#a zKhNEZzKX(hSR-EQx3 zSgJvCIW7^M7u)Abx9NF*^>UQ+GDM>hHEwPpV>3F1IAM+DZPu%a9mQD(;=@bZed({* zj*i>-cu05W@DV!WXco;qy(^IWp019o+H>ylxkCATJ`pdurM#|*zdXd}`-xmLO>Loh zyO+5WY2}`+r|0oO4vulPL~L~HwldrfClJ_r(ex*R1B=+R^UKeF_eXkL7Dsub9mNDc z$)!~(p%EOOEnIOSPGAR?{UA(BMt*)$_ows)22 zhii}2;~|YjKi!x~^;AJNs$R<9r#eExt#$Ud2>H7d3j&fKH8S>0QThxY($G7q+RYiy z@3^s>$h%PMg4d^iTCWVb>3U3qOd*qur2RRv>)ka|7P}TttoN*?x*F0b6Gf+%O|p=B z)zpI)d$yQmm!wVKPl45ITB0@DFs*R9W5kv(lO8r_)0~Aj`GoBB#KPz?9;4UT5=W<7 zanHh{dxYWRL9;krFJL+GZp%>*rjo(T=_g6n2>atZ*OuvjmT}p@>kO(!!{=?671geM zhj7>`4XuQ#y-lR+Rx7v89OUUd2!;7^J4xK_`@rbwK3AhqkT!PRzFkR(tLJ@k*?QLb zxXr5D-k;2-<#KmEIaH6VGiArX676-o7+dps7_XWl5o~Sr9^dozdC{*fGf?j4e7%iV zBgyHm{^F{C6*FsgajLr^%x0llila%qTZ_)C8owSV`*|~7%c$sgOLaA0Z~AWelGW?0 z-mYzJ6-?Kwm)~@ql{}66xPJ%u5T=t5Z{8K)xpZ{z?%Fg|biuV7yBYt4#7I54?|Z)9 zPp{X@37O=$dM}T9bRzL%vaXPE_6Q9ohi-o^tJC6txVucOgBLwYsl+I=sdt&qt4IiN zR@~XFZ&07C_wUtCl&w-R-ypiND&`Mw>@AIMQjGh`U3KL=E9Xy9G89KiEPkqPIITy% zIzQRR-Tmpq>oJ|2rcZbsi-Y}~8JDb8kNell4*bb@?L8l}*X$Y|;#9;BtL-axAp7xZ z`+UuR-hp&jP}E+>?Ijy)TUU@?OF?K8($2i}q+&A<fdw5Qk#c>>eC~W^2)tBNnpDnM*o2P*0OzhI`I`1c^ z_iC)BG-F5}x5xZ!EZS8zeU93CY7{T#3y(+gm>eu`^?pv0@kTt~=^USmcoCm%dAr%5 z_ho&Y@3Zlf3TZ5EU*fUxqRQHAqjRx1&7anI{i?E6eJh^RU3Rv$#i-ejMzdLXUv27t zIGr8}Z&uy)<4Z1QNnK4k5a#OUnM9M>ej&UTLOX%EZOHa!eals4PSr5;jYoCaUXNMk zE7{pxZ{)dMjjdUG*KZO%XN7Z8$x#j-7x4F?kK?!T>fPsd)2{a8=PTFhSlRA_|CpPR%U_aP~A;#NYqs2sW4Es-3iTq!)x+v zOmRM+p5qq;^T(^u&{0QcSxoeCHn*raZL86kobfxqXSc(-$n&|gD6%&{M=!mz8_xXE zVs6O2HcGelI~0nO=n7Ax$c&rSz9ss6duw9cR>k5qee~^fX%&kR8waa<-tX++olVbo8o!vmYq_rURWJ_+EVM(&GQwV!s8sB z)urR-3waF4(&Iz{G*hw*7uv_Ae`dMt=Z}M#-9|x#%IZYi|A3cEn!;S zP8W$C(4=lz+Pd^TAvF5b)au_dkD$(bI6bBhkOw>r#K`#PNp zQW558dhlMSE#AA3(8Xq+tu6Vmi>lXLvR^^}dRP+Y*Arb-OH$CS zxK#YdWJ(Qa>sXdHVPrgiR%hw>P6{fRXLQy303WX>wk@j7YA&wcYgHhRO%U78Y+28( z+sIe)rr55R%VL!*wR&#tdV4=ws<9yY22vk;^(JguU$IwKH{OI3I(_eoLphncx#>@o z#pw_nuT{Agil;xti&cZmVD|9RY@EmTE1agjo3!)odn!Ngp&aOcw?vF?{qa(D#q>54 zoXuVir&!oUYgFGt{kEW+-FfU?wDU5xW<`Rw3!^_?$lCRfMLa(#>xh|N6p;BNyt-FA{6 z_oMx7R~g4QZj%0gyff%-en(dPT+Q9Qh&q!PyVrr7(7GrxdSlC-;BO#SjOTSbfhx^z zmK@S`>y(H*Pm9B1H4b{3xb1GfjOxwy;ivLk$}4v>$;?pMKj*bv-nDFHVLFzd0+w^_ zWb}CD)#5Z#`t#vsZ&%mUW7efeLUj|5uhaADT*cOQ6ytV(DJ}GH^GF;G@|k&_&bP8) z(^q)hdF#@(V|D4hE%j(XqqI|5m13nip3btPNV-Z#-QlrG=~%N?W6P}iQ{g!AE)Tp> z^m1jR3NUoMzi_oOvI#! z_scfZPU~pxY3)dtr+0Q#S0lsPqp^RI)JH|-Og=}eTxmwDVtHQQ@jc4RopYWYC!_K7 z-QUrF&M?b);EdC{N9%*^T+q(aPJKJ_%Y#VBXx!oQLgc5bTH*HSj7(j$OiNtvC+pjC z{8;R^+RHQ_Wcs{$FSFiF7Q!4q)uLbAphj^+82~TCp^Fr7?U`-Y2?{C%6UPk4tH2Cx?Cz9& zN@x&pgWVc&%T8OI+q5K-P!R3pc!O*`=N`?fF+j@+MPh&7z6N!ZMsK39rM4C@I<>t< ze|79%_e(#D`DHlv!sr=)i_==XC*Tx6tq+4lO^$09`0`dd&X(4L4-5%^%nbFDB&6P+ z2UH)|8DRHl(YGPp8Swyh@^x#mq$BS3#?^&OpFxk|pyAew&_55go!=Nsy>SxueyiD( zjJbJUL8r)%9UPKIxTr>siy`*-5CPUjXBFP|s2)T_S-#30st1y}HojbNu8fEuZmW3_ zP5DcDk%kngPlo3$#l1g&)dQlOF`20=USKR``kFg;^RQ+=MnLs6V$(y=){wZC3GmKL zEg4tJ$y~#n-lX}LT%V{rxr>aaVp1ae`Ri5B8rR6-*;2-@@t}z8jYIAo=_s*JXTC7t zQY+Q4DfL^#kQe<{=2Sot`J~z_}r?pBY zi2I#_E@sc3m-o~84%Rz0ttb}AQ8+w^gSp01KXt&QxQs0x)qDnaXfz5!I?FCRepuqJ zH8$)>B}Fk@C*(PE?3VT=g~Da~q1$)8Ji(}1##fI>i7R<=Yz~?76!AKuc;dgB-Di69sGQ4AOQUN{n^w(PFQI+{c|`C?Qm*@sXBF==SeKZao{QEq~Gmv9zW3j#T zqf;#(xhjhW3)F<@FWlF{M}P_{0$zjkCH#CCsAA+>l~?3Y0>8WXn7#JU?0mi339P$s zzML&}5v#5iof}PGuN);$4<{#2%7*U(%ouM%TY48!&Zf;6Fj z&;_bbFp8E#K*Tt1%z(36q*=imJI=qQEofmm>I#W~3=Kl4DTzxqUIvOctbMv=t;Z=W zshkoxcFy!|)poH6yDW;rBg;kwH+u^UY~zKB>^W)lr9pD;I$d;YGIZmE%J&z@%y`E} z043|rrqYAA4>9y~{Jb^j^=5XprxH_ty9AX3+}h3phyvxk6JJu118&Dx!?sVRjL+}c zLYb!TIs0`85Yq<2Vk0YD=AZmSKyGc|ARl^JQ>0C62iLX`HnARn-A#0DCIZFzED@*d zQ(Q`w65S$y*R#Q$ zj&N&415PvN0#$~ZvVk8E8LiYFPBgW4-|zBxx@Qp0Lrvl&i~qR!-E5E!t?755QMzkI z05)W*`8U@6&A1zf-#|9nn1tacCrRRo!vvUq-dwJ#VLg5F^H4Q&`wsIQf5T%@kKv>b(qvP#&9-OZ*O;*Oq# z;lZOZbGR$tSovF?wS~NhvrcDvTw>%l_h=`>>Pf?|l$NRRfjD@6KMFg4=4vDlI%|nN zKw~s3ff4zwZ>IEeCnCu-q4iOD+8G(ea^+|@t5c-^P+7nR=n}FB5Sb{fpk^PAB>v@a z`NL0wZ33oQVmzS;+79P!mvZ$`kGWJY8#x$Hj)Cf?yk|gZI6}R3#9`Wp-ff%nil4yw zHy`JNJqEB5-WUv#r82lc2BE|5~HyWVA$w7 zUG#N6R1HE^k2K*zKWBkC5;%Q$#u6?#9M`od%R63KWaUmKuTDAj$gyWL)9Ja zw%00|PES{s{qqt;P~@DX4U!4FAL{JA5Ihd~Eo-w?n2yb2spS}2g5AbIpnYb8$^MG0 zV#R!M;%{G6fXV{u{LNIivYrC{>Uizj5a4zz+J%`+kd``Aerz@GDNdd?aD$@swd@Ye z4WfO%L?~Q`2(6-jAEA??wBKV;4b%P-RlY+4CBA*%TvYo(Q32*0K>&SLUf2PYX>D#i za(+*+qrkQ=O(SrhvJWAwN~hk=ugzBLDS7M0Yr(J2LSRqSO+eF}^+yK|kGW^SYoqSgX!FmSA5cY@k7~YPB9kgzpKi^J&)#G#}=zv&VK-yHb(_fmRZm}U&>ZD@=345o z{8iysfz_fEPO`Fye!6jSDbe9!uj2=phtMToCdax5F=Ru`JUOe(ZDB=9+(6lPX|L%X z#flNSa0K0duKQw38YssSRGg!udMpUN=rCo;(rIXWKG>+4xi+ded3^sU%PIoxO;uwn z7KF-$UI#54ah&j6liAEm1lBx(fW;DY@%EvnlQ>Nt43bZTwsbM$1`rnWC7eeq=^)8m z!1_IYiP!~qFZdE#S9vJ?y5Q~w02{lh<&~v;d9Ou(lA_cmzP7Na8|=xM1N6#G*^~az ztW<+=(Zk3k_WnZ?R*)6JF*RZSgd7I6FM$kJpLX_vA0mo)sDNJ^fQ;;)r5P)qi z~_@rmul4(n@@9R42K)T%6$x<+`%@KG-h4+K79f+j$#v~I_Czt!YB{m~Ds_=O4m*NF_13w)?JE8}E*#t}Jbyqo4dV*9b@ zk-PZ&8mzz&&^dtFcfX+;yRr<@-Vf!z_?bJTn@F0&^#WTDC_^?h3pl1< zfmz8jzM>RAM{1Frf_`QbD=9{kQX=br>JLocpvqqF5Qsx=wrJX1*8ulo1}$3IWAPKl zx8%mfh7MZEAKWngTmo3}8X|jPVW;hjiFC}l;uDjM&W3uNJe4W~c9E$N`-Do5mVVrN zDa}r)f2cn86$8F%4L6#R|AWejY>Z?os(rl31 zPOJY^L_(&df7CqN$E5ge4eBwn=q3m6k+?c@krBhQ>JNEZWeTvp1vkH%@Euxec;;|? zFo;kE&HYfZ*vk8?TbE$M@+O9V4hCrfX4^7g40M9Kro)eh_2dveKeNSS_!*b)ncyMh zC7y0y6uW3^NUQedn#Zy9F9=x?ubslT*bWxghswLQnQON$W;*4Hi+Sn|p*^+y`VX78 zLd}~M5@-VVe_)MBNIM~L=2KBMQ*`eXb&;h=82`eu)f|BBqtpKdW87hXM~whdN1|!3 z{Njxi?u|qb=&^X*dIH_oM13KIVh5UjyN$0DBlpD0We-{UoCo|Qwq;2vY9_?_cJO3s z%I()L#-+Y1pK@Jg2?Jh-viuq&{|m*|OOsN-Jo;I|3G|o{`{sDUH=VkGY5mb6hP)ec zo2wEeP)Eg++kmsAVzDoOLg#_LB4O5fed#uf7346Qk_vcQNH@b8+dy&X0N?R6i?rWn z#qB$X)6~H*G3VT&)-sj2?JOJmWcq3A6xFtOQUO6yekMdfeYa57ASg8y2-_EkY#i#1 z--2Fs3g_1Fm~+)&H<@Vd-KB85Qg6_fH2GGsTI7In28r@p;#oR>V@Y~3-E=hIF978m zzI^(LO;L5jZ??@v^AZ8cu_wDe-V_(T} zJ*XmRZ1kiCqNupz*Lv{iXoy8xK!7x-NN-Sq(M^-b%ApoK#(E^ofeQba98^gY+e@E7 zHA_;DY2D0Py`)78h_Vi?xTeBEZv#wAUY$6rdSI>%}&z2F>j z5ptfmu>kL`ion{3>FIKj;o!;dG@x1tCoi<3KX^6$P6!pIR2PlyldyYql@pM5h(G6K|+Ap;q1l@bZ7f?=g@ zB*|@)z=Gx;89`Dj->_`oR7ocT7dB2;1mbY{(&8kbuHaPu%R-AWEe^Abtd^54!uuZftH^MQJ zc{+Xh1{MSCp0CT8+pc$~>frt9099o5jC&O#wm8{D7<`T)Crk2v6UEe!kG!wkt*=h; z!SJzN@H&>rZKgR%Xy1-`oOylCgpOp+EBk5MA2Ey$C4a@2LPJFL$t@?&uah2i2|258 zzN(ggE70h%N2qO`JUk(=NFLq*m@kBnCigvX#uke%8u412^mvz4_wFUCgrTs4HZ2sA z*pFyrf)9IeelUxQy8wZYMa&vj}eMx;~DJ$3CI>o)i47hZwCJ~LTTzOWKm;FN(@3x9+ zF8icV)3_-|`ji@grF3q=Ume&^r*(!Yc?R(|Hf*i-t(1f{Y2~gKqRvNUU_^OFZGgIe z#b+cFHQQw+)TAq4#YJe*TT^WMNkx`j&(YZ+q!vp3`uz=@S~?JD8rcVCevTO1v@7Yf zqT;{oeko=qT%g1$wnx9BnU1nc9|vRVjZ8wkc#aC7+Za-vXzpi++I8nj`QFNtl?rLsMg7QFCDozvny@#`mpmki2>q5t!O!%f@uB;wOdRP8`tBHPg z$;HP*ZkO^wYi#roZRJ

    • From 776f9e59cc8a85e840d1d4af8540d199c77190ac Mon Sep 17 00:00:00 2001 From: xiaofei Date: Fri, 7 Mar 2025 06:58:25 +0800 Subject: [PATCH 090/188] cmake : fix undefined reference errors for std::filesystem in ggml (#12092) (#12094) Signed-off-by: Ray Lee Co-authored-by: Ray Lee --- ggml/src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index cfd4ac54ca..52817510f6 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -236,7 +236,7 @@ add_library(ggml target_link_libraries(ggml PUBLIC ggml-base) if (CMAKE_SYSTEM_NAME MATCHES "Linux") - target_link_libraries(ggml PRIVATE dl) + target_link_libraries(ggml PRIVATE dl stdc++fs) endif() function(ggml_add_backend_library backend) From d76a86d967ef491d530400b08bc8ef8a14807936 Mon Sep 17 00:00:00 2001 From: lhez Date: Thu, 6 Mar 2025 16:20:35 -0800 Subject: [PATCH 091/188] opencl: Noncontiguous `norm`, `rms_norm`, disable `fp16` for some ops (#12217) * opencl: support noncontiguous `norm` * opencl: support noncontiguous `rms_norm` * opencl: disable fp16 for `ADD`, `MUL`, `SCALE`, `RELU`, `GELU`, `SILU`, `CLAMP` --- ggml/src/ggml-opencl/ggml-opencl.cpp | 70 +++++++++++++-------- ggml/src/ggml-opencl/kernels/ggml-opencl.cl | 26 ++++++-- 2 files changed, 65 insertions(+), 31 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index bc2ea06b59..b85a895c45 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1007,17 +1007,18 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_ADD: case GGML_OP_SCALE: case GGML_OP_MUL: - return true; + return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { case GGML_UNARY_OP_GELU: case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_RELU: - return ggml_is_contiguous(op->src[0]); + return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; default: return false; } case GGML_OP_CLAMP: + return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_SOFT_MAX: case GGML_OP_NORM: case GGML_OP_RMS_NORM: @@ -2573,26 +2574,33 @@ static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const memcpy(&eps, dst->op_params, sizeof(float)); const int ne00 = src0 ? src0->ne[0] : 0; - const cl_ulong nb01 = src0 ? src0->nb[1] : 0; + const int ne01 = src0 ? src0->ne[1] : 0; + const int ne02 = src0 ? src0->ne[2] : 0; + const int ne03 = src0 ? src0->ne[3] : 0; - GGML_ASSERT(ggml_is_contiguous_1(src0)); + const cl_ulong nb01 = src0 ? src0->nb[1] : 0; + const cl_ulong nb02 = src0 ? src0->nb[2] : 0; + const cl_ulong nb03 = src0 ? src0->nb[3] : 0; const int nth = MIN(64, ne00); cl_kernel kernel = backend_ctx->kernel_norm; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(float)*nth, NULL)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth, NULL)); - const int64_t nrows = ggml_nrows(src0); - - size_t global_work_size[] = {(size_t)nrows*nth, 1, 1}; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; #ifdef GGML_OPENCL_PROFILING @@ -2630,16 +2638,19 @@ static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, c memcpy(&eps, dst->op_params, sizeof(float)); const int ne00 = src0 ? src0->ne[0] : 0; + const int ne01 = src0 ? src0->ne[1] : 0; + const int ne02 = src0 ? src0->ne[2] : 0; + const int ne03 = src0 ? src0->ne[3] : 0; + const cl_ulong nb01 = src0 ? src0->nb[1] : 0; + const cl_ulong nb02 = src0 ? src0->nb[2] : 0; + const cl_ulong nb03 = src0 ? src0->nb[3] : 0; GGML_ASSERT(ne00 % 4 == 0); - GGML_ASSERT(ggml_is_contiguous_1(src0)); const int nth = MIN(64, ne00); - const int64_t nrows = ggml_nrows(src0); - - size_t global_work_size[] = {(size_t)nrows*nth, 1, 1}; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; cl_kernel kernel = backend_ctx->kernel_rms_norm; @@ -2654,15 +2665,20 @@ static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, c sizeof(local_work_size), local_work_size, sizeof(size_t), &sgs, NULL)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); // This is local memory - the size depends on subgroup size. - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(float)*nth/sgs, NULL)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); #ifdef GGML_OPENCL_PROFILING cl_event evt; diff --git a/ggml/src/ggml-opencl/kernels/ggml-opencl.cl b/ggml/src/ggml-opencl/kernels/ggml-opencl.cl index 8882a8c9c6..1d43642a98 100644 --- a/ggml/src/ggml-opencl/kernels/ggml-opencl.cl +++ b/ggml/src/ggml-opencl/kernels/ggml-opencl.cl @@ -506,14 +506,23 @@ kernel void kernel_norm( global float * dst, ulong offsetd, int ne00, + int ne01, + int ne02, + int ne03, ulong nb01, + ulong nb02, + ulong nb03, float eps, local float * sum ) { src0 = (global void*)((global char*)src0 + offset0); dst = (global void*)((global char*)dst + offsetd); - global float * x = (global float *) ((global char *) src0 + get_group_id(0)*nb01); + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0); + + global float * x = (global float *) ((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01); // MEAN // parallel sum @@ -533,7 +542,7 @@ kernel void kernel_norm( // recenter and VARIANCE barrier(CLK_LOCAL_MEM_FENCE); - global float * y = dst + get_group_id(0)*ne00; + global float * y = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00; sum[get_local_id(0)] = 0.0f; for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) { y[i00] = x[i00] - mean; @@ -566,14 +575,23 @@ kernel void kernel_rms_norm( global float * dst, ulong offsetd, int ne00, + int ne01, + int ne02, + int ne03, ulong nb01, + ulong nb02, + ulong nb03, float eps, local float * sum // Note, the size depends on number of subgroups ) { src0 = (global void*)((global char*)src0 + offset0); dst = (global float*)((global char*)dst + offsetd); - global float4 * x = (global float4 *) ((global char *) src0 + get_group_id(0)*nb01); + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0); + + global float4 * x = (global float4 *) ((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01); global float * x_scalar = (global float *) x; float4 sumf = 0; float all_sum = 0; @@ -607,7 +625,7 @@ kernel void kernel_rms_norm( const float mean = sum[0]; const float scale = 1.0f/sqrt(mean + eps); - global float4 * y = (global float4 *) (dst + get_group_id(0)*ne00); + global float4 * y = (global float4 *) (dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00); global float * y_scalar = (global float *) y; for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { y[i00] = x[i00] * scale; From d6c95b0740510231b3797b80d6d3440d8fe188b6 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 7 Mar 2025 06:23:16 +0100 Subject: [PATCH 092/188] metal : fix default.metallib build (#12224) This commit updates the custom command to build the default.metallib file to use the correct path to ../ggml-common.h by using the variable METALLIB_COMMON. The motivation for this change is that currently when building and specifying GGML_METAL_EMBED_LIBRARY=OFF the following error is generated: ```console [ 11%] Linking CXX shared library ../../bin/libggml.dylib [ 11%] Built target ggml make[2]: *** No rule to make target `ggml/src/ggml-metal/ggml-common.h', needed by `bin/default.metallib'. Stop. make[1]: *** [ggml/src/ggml-metal/CMakeFiles/ggml-metal-lib.dir/all] Error 2 ``` With the above change the build could progress but there was a follow on error about not being able to find the ggml-common.h file in ggml-metal.metal where is was included as a relative path: ```console [ 11%] Compiling Metal kernels /Users/danbev/work/llama.cpp/build/bin/ggml-metal.metal:6:10: error: '../ggml-common.h' file not found, did you mean 'ggml-common.h'? ^~~~~~~~~~~~~~~~~~ "ggml-common.h" 1 error generated. ``` Removing the relative path then allowed the build to complete successfully. --- ggml/src/ggml-metal/CMakeLists.txt | 4 ++-- ggml/src/ggml-metal/ggml-metal.metal | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txt index 89fcde2faa..be3fb3fa95 100644 --- a/ggml/src/ggml-metal/CMakeLists.txt +++ b/ggml/src/ggml-metal/CMakeLists.txt @@ -27,12 +27,12 @@ configure_file(../ggml-common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY) configure_file(ggml-metal-impl.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h COPYONLY) +set(METALLIB_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/../ggml-common.h") if (GGML_METAL_EMBED_LIBRARY) enable_language(ASM) add_compile_definitions(GGML_METAL_EMBED_LIBRARY) - set(METALLIB_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/../ggml-common.h") set(METALLIB_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal") set(METALLIB_IMPL "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal-impl.h") @@ -93,7 +93,7 @@ else() COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal - DEPENDS ggml-metal.metal ggml-common.h + DEPENDS ggml-metal.metal ${METALLIB_COMMON} COMMENT "Compiling Metal kernels" ) diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index d092a16906..c46a130508 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -3,8 +3,7 @@ #if defined(GGML_METAL_EMBED_LIBRARY) __embed_ggml-common.h__ #else -// TODO: this should not be a relative path, but can't figure out how to set Metal include paths in Package.swift -#include "../ggml-common.h" +#include "ggml-common.h" #endif #include "ggml-metal-impl.h" From f1648e91cf6c52e9593810aa70857e412d474c09 Mon Sep 17 00:00:00 2001 From: David Huang <1969802+hjc4869@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:06:08 +0800 Subject: [PATCH 093/188] HIP: fix rocWMMA build flags under Windows (#12230) --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e4596ab2d..f2c81c0c26 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1238,7 +1238,7 @@ jobs: cmake -G "Unix Makefiles" -B build -S . ` -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" ` -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" ` - -DCMAKE_CXX_FLAGS="-Irocwmma/library/include/" ` + -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/rocwmma/library/include/" ` -DCMAKE_BUILD_TYPE=Release ` -DGGML_HIP=ON ` -DGGML_HIP_ROCWMMA_FATTN=ON ` @@ -1294,7 +1294,7 @@ jobs: cmake -G "Unix Makefiles" -B build -S . ` -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" ` -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" ` - -DCMAKE_CXX_FLAGS="-Irocwmma/library/include/" ` + -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/rocwmma/library/include/" ` -DCMAKE_BUILD_TYPE=Release ` -DAMDGPU_TARGETS=${{ matrix.gpu_target }} ` -DGGML_HIP_ROCWMMA_FATTN=ON ` From 5e2d57b2b2e43eadbe6d66ba3e873a824b95e725 Mon Sep 17 00:00:00 2001 From: BB-fat <45072480+BB-fat@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:35:57 +0800 Subject: [PATCH 094/188] metal : simplify kernel arguments using a struct (#3229) (#12194) * metal : refactor im2col parameters into a struct * metal: Change im2col offset types from int32_t to uint64_t to support larger memory offsets * metal : refactor sum_rows parameters into a struct * metal : refactor soft_max parameters into a struct * metal : refactor diag_mask_inf parameters into a struct * metal : refactor ssm_conv parameters into a struct * metal : refactor ssm_scan parameters into a struct * metal : refactor get_rows parameters into a struct * metal : refactor group_norm parameters into a struct * metal : refactor conv_transpose_1d parameters into a struct * metal : refactor upscale parameters into a struct * metal : refactor pad parameters into a struct * metal : refactor pad_reflect_1d parameters into a struct * metal : refactor arange parameters into a struct * metal : refactor timestep_embedding parameters into a struct * metal : refactor argsort parameters into a struct * metal : refactor leaky_relu parameters into a struct * metal : refactor pool_2d parameters into a struct * metal : fix trailing whitespace --------- Co-authored-by: alexju --- ggml/src/ggml-metal/ggml-metal-impl.h | 235 ++++++++++ ggml/src/ggml-metal/ggml-metal.m | 466 ++++++++++--------- ggml/src/ggml-metal/ggml-metal.metal | 627 ++++++++------------------ 3 files changed, 685 insertions(+), 643 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index e3dc25f168..a58c474eb0 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -285,4 +285,239 @@ typedef struct { float eps; } ggml_metal_kargs_rms_norm; +typedef struct { + int64_t ne00; + int64_t ne01; + int64_t ne02; + uint64_t nb00; + uint64_t nb01; + uint64_t nb02; + int32_t n_groups; + float eps; +} ggml_metal_kargs_group_norm; + +typedef struct { + int32_t IC; + int32_t IL; + int32_t K; + int32_t s0; + uint64_t nb0; + uint64_t nb1; +} ggml_metal_kargs_conv_transpose_1d; + +typedef struct { + uint64_t ofs0; + uint64_t ofs1; + int32_t IW; + int32_t IH; + int32_t CHW; + int32_t s0; + int32_t s1; + int32_t p0; + int32_t p1; + int32_t d0; + int32_t d1; + int32_t N; + int32_t KH; + int32_t KW; + int32_t KHW; // KH * KW, pre-computed on CPU to save GPU resources +} ggml_metal_kargs_im2col; + +typedef struct { + int64_t ne00; + int64_t ne01; + int64_t ne02; + int64_t ne03; + uint64_t nb00; + uint64_t nb01; + uint64_t nb02; + uint64_t nb03; + int64_t ne10; + int64_t ne11; + int64_t ne12; + int64_t ne13; + uint64_t nb10; + uint64_t nb11; + uint64_t nb12; + uint64_t nb13; + int64_t ne0; + int64_t ne1; + int64_t ne2; + int64_t ne3; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; +} ggml_metal_kargs_sum_rows; + +typedef struct { + int64_t ne00; + int64_t ne01; + int64_t ne02; + float scale; + float max_bias; + float m0; + float m1; + uint32_t n_head_log2; +} ggml_metal_kargs_soft_max; + +typedef struct { + int64_t ne00; + int64_t ne01; + int n_past; +} ggml_metal_kargs_diag_mask_inf; + +typedef struct { + int64_t ne00; + int64_t ne01; + int64_t ne02; + uint64_t nb00; + uint64_t nb01; + uint64_t nb02; + int64_t ne10; + int64_t ne11; + uint64_t nb10; + uint64_t nb11; + int64_t ne0; + int64_t ne1; + int64_t ne2; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; +} ggml_metal_kargs_ssm_conv; + +typedef struct { + int64_t d_state; + int64_t d_inner; + int64_t n_seq_tokens; + int64_t n_seqs; + uint64_t nb00; + uint64_t nb01; + uint64_t nb02; + uint64_t nb10; + uint64_t nb11; + uint64_t nb12; + uint64_t nb13; + uint64_t nb20; + uint64_t nb21; + uint64_t nb22; + uint64_t nb30; + uint64_t nb31; + uint64_t nb40; + uint64_t nb41; + uint64_t nb42; + uint64_t nb50; + uint64_t nb51; + uint64_t nb52; +} ggml_metal_kargs_ssm_scan; + +typedef struct { + int64_t ne00; + uint64_t nb01; + uint64_t nb02; + int64_t ne10; + uint64_t nb10; + uint64_t nb11; + uint64_t nb1; + uint64_t nb2; +} ggml_metal_kargs_get_rows; + +typedef struct { + int64_t ne00; + int64_t ne01; + int64_t ne02; + int64_t ne03; + uint64_t nb00; + uint64_t nb01; + uint64_t nb02; + uint64_t nb03; + int64_t ne0; + int64_t ne1; + int64_t ne2; + int64_t ne3; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; + float sf0; + float sf1; + float sf2; + float sf3; +} ggml_metal_kargs_upscale; + +typedef struct { + int64_t ne00; + int64_t ne01; + int64_t ne02; + int64_t ne03; + uint64_t nb00; + uint64_t nb01; + uint64_t nb02; + uint64_t nb03; + int64_t ne0; + int64_t ne1; + int64_t ne2; + int64_t ne3; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; +} ggml_metal_kargs_pad; + +typedef struct { + int64_t ne00; + int64_t ne01; + int64_t ne02; + int64_t ne03; + uint64_t nb00; + uint64_t nb01; + uint64_t nb02; + uint64_t nb03; + int64_t ne0; + int64_t ne1; + int64_t ne2; + int64_t ne3; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; + int32_t p0; + int32_t p1; +} ggml_metal_kargs_pad_reflect_1d; + +typedef struct { + uint64_t nb1; + int dim; + int max_period; +} ggml_metal_kargs_timestep_embedding; + +typedef struct { + float slope; +} ggml_metal_kargs_leaky_relu; + +typedef struct { + int64_t ncols; + int64_t ncols_pad; +} ggml_metal_kargs_argsort; + +typedef struct { + int64_t ne0; + float start; + float step; +} ggml_metal_kargs_arange; + +typedef struct { + int32_t k0; + int32_t k1; + int32_t s0; + int32_t s1; + int32_t p0; + int32_t p1; + int64_t IH; + int64_t IW; + int64_t OH; + int64_t OW; + int64_t parallel_elements; +} ggml_metal_kargs_pool_2d; + #endif // GGML_METAL_IMPL diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m index 1f45ebad14..1158b285c1 100644 --- a/ggml/src/ggml-metal/ggml-metal.m +++ b/ggml/src/ggml-metal/ggml-metal.m @@ -1945,34 +1945,38 @@ static void ggml_metal_encode_node( id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUM_ROWS].pipeline; - // TODO: add ggml_metal_kargs struct + + ggml_metal_kargs_sum_rows args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.ne03 =*/ ne03, + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.ne10 =*/ ne10, + /*.ne11 =*/ ne11, + /*.ne12 =*/ ne12, + /*.ne13 =*/ ne13, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, + /*.nb12 =*/ nb12, + /*.nb13 =*/ nb13, + /*.ne0 =*/ ne0, + /*.ne1 =*/ ne1, + /*.ne2 =*/ ne2, + /*.ne3 =*/ ne3, + /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; - [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9]; - [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10]; - [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11]; - [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12]; - [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16]; - [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20]; - [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24]; - [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; } break; @@ -2021,8 +2025,17 @@ static void ggml_metal_encode_node( const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - // TODO: add ggml_metal_kargs struct - // TODO: optimize (see https://github.com/ggml-org/llama.cpp/pull/10238/commits/7941b6b9ec29a2866fec6fa6c51612515ca509f6) + ggml_metal_kargs_soft_max args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.scale =*/ scale, + /*.max_bias =*/ max_bias, + /*.m0 =*/ m0, + /*.m1 =*/ m1, + /*.n_head_log2 =*/ n_head_log2, + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; if (id_src1) { @@ -2031,14 +2044,7 @@ static void ggml_metal_encode_node( [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1]; } [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; - [encoder setBytes:&scale length:sizeof(scale) atIndex:6]; - [encoder setBytes:&max_bias length:sizeof(max_bias) atIndex:7]; - [encoder setBytes:&m0 length:sizeof(m0) atIndex:8]; - [encoder setBytes:&m1 length:sizeof(m1) atIndex:9]; - [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:10]; + [encoder setBytes:&args length:sizeof(args) atIndex:3]; [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; @@ -2056,13 +2062,16 @@ static void ggml_metal_encode_node( pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF].pipeline; } - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_diag_mask_inf args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.n_past =*/ n_past, + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; - [encoder setBytes:&n_past length:sizeof(int) atIndex:4]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; if (ne00%8 == 0) { [encoder dispatchThreadgroups:MTLSizeMake(ne00*ne01*ne02/8, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; @@ -2081,27 +2090,30 @@ static void ggml_metal_encode_node( id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SSM_CONV_F32].pipeline; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_ssm_conv args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.ne10 =*/ ne10, + /*.ne11 =*/ ne11, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, + /*.ne0 =*/ ne0, + /*.ne1 =*/ ne1, + /*.ne2 =*/ ne2, + /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; - [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9]; - [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:11]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:12]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:14]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:15]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:16]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:17]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:18]; + [encoder setBytes:&args length:sizeof(args) atIndex:3]; [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne1, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; } break; @@ -2152,7 +2164,31 @@ static void ggml_metal_encode_node( id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SSM_SCAN_F32].pipeline; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_ssm_scan args = { + /*.d_state =*/ d_state, + /*.d_inner =*/ d_inner, + /*.n_seq_tokens =*/ n_seq_tokens, + /*.n_seqs =*/ n_seqs, + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, + /*.nb12 =*/ nb12, + /*.nb13 =*/ nb13, + /*.nb20 =*/ nb20, + /*.nb21 =*/ nb21, + /*.nb22 =*/ nb22, + /*.nb30 =*/ nb30, + /*.nb31 =*/ nb31, + /*.nb40 =*/ nb40, + /*.nb41 =*/ nb41, + /*.nb42 =*/ nb42, + /*.nb50 =*/ nb50, + /*.nb51 =*/ nb51, + /*.nb52 =*/ nb52, + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; @@ -2161,30 +2197,7 @@ static void ggml_metal_encode_node( [encoder setBuffer:id_src4 offset:offs_src4 atIndex:4]; [encoder setBuffer:id_src5 offset:offs_src5 atIndex:5]; [encoder setBuffer:id_dst offset:offs_dst atIndex:6]; - - [encoder setBytes:&d_state length:sizeof(d_state) atIndex:7]; - [encoder setBytes:&d_inner length:sizeof(d_inner) atIndex:8]; - [encoder setBytes:&n_seq_tokens length:sizeof(n_seq_tokens) atIndex:9]; - [encoder setBytes:&n_seqs length:sizeof(n_seqs) atIndex:10]; - - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:11]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:12]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:13]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16]; - [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17]; - [encoder setBytes:&nb20 length:sizeof(nb20) atIndex:18]; - [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:19]; - [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:20]; - [encoder setBytes:&nb30 length:sizeof(nb30) atIndex:21]; - [encoder setBytes:&nb31 length:sizeof(nb31) atIndex:22]; - [encoder setBytes:&nb40 length:sizeof(nb40) atIndex:23]; - [encoder setBytes:&nb41 length:sizeof(nb41) atIndex:24]; - [encoder setBytes:&nb42 length:sizeof(nb42) atIndex:25]; - [encoder setBytes:&nb50 length:sizeof(nb50) atIndex:26]; - [encoder setBytes:&nb51 length:sizeof(nb51) atIndex:27]; - [encoder setBytes:&nb52 length:sizeof(nb52) atIndex:28]; + [encoder setBytes:&args length:sizeof(args) atIndex:7]; [encoder dispatchThreadgroups:MTLSizeMake(d_inner, n_seqs, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; } break; @@ -3041,19 +3054,22 @@ static void ggml_metal_encode_node( default: GGML_ABORT("not implemented"); } - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_get_rows args = { + /*.ne00 =*/ ne00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.ne10 =*/ ne10, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4]; - [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5]; - [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6]; - [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7]; - [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8]; - [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9]; - [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10]; + [encoder setBytes:&args length:sizeof(args) atIndex:3]; [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)]; } break; @@ -3110,18 +3126,21 @@ static void ggml_metal_encode_node( id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GROUP_NORM].pipeline; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_group_norm args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.n_groups =*/ n_groups, + /*.eps =*/ eps, + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4]; - [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:5]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:6]; - [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:7]; - [encoder setBytes:&n_groups length:sizeof( int32_t) atIndex:8]; - [encoder setBytes:&eps length:sizeof( float) atIndex:9]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake(n_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; @@ -3279,8 +3298,8 @@ static void ggml_metal_encode_node( const int32_t CHW = IC * KH * KW; - const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4; - const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4; + const uint64_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4; + const uint64_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4; id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F32].pipeline; @@ -3302,27 +3321,30 @@ static void ggml_metal_encode_node( default: GGML_ABORT("fatal error"); }; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_im2col args = { + /*.ofs0 =*/ ofs0, + /*.ofs1 =*/ ofs1, + /*.IW =*/ IW, + /*.IH =*/ IH, + /*.CHW =*/ CHW, + /*.s0 =*/ s0, + /*.s1 =*/ s1, + /*.p0 =*/ p0, + /*.p1 =*/ p1, + /*.d0 =*/ d0, + /*.d1 =*/ d1, + /*.N =*/ N, + /*.KH =*/ KH, + /*.KW =*/ KW, + /*.KHW =*/ KH * KW, + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ofs0 length:sizeof(int32_t) atIndex:2]; - [encoder setBytes:&ofs1 length:sizeof(int32_t) atIndex:3]; - [encoder setBytes:&IW length:sizeof(int32_t) atIndex:4]; - [encoder setBytes:&IH length:sizeof(int32_t) atIndex:5]; - [encoder setBytes:&CHW length:sizeof(int32_t) atIndex:6]; - [encoder setBytes:&s0 length:sizeof(int32_t) atIndex:7]; - [encoder setBytes:&s1 length:sizeof(int32_t) atIndex:8]; - [encoder setBytes:&p0 length:sizeof(int32_t) atIndex:9]; - [encoder setBytes:&p1 length:sizeof(int32_t) atIndex:10]; - [encoder setBytes:&d0 length:sizeof(int32_t) atIndex:11]; - [encoder setBytes:&d1 length:sizeof(int32_t) atIndex:12]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; if (is_gt_mttpt) { - [encoder setBytes:&N length:sizeof(int32_t) atIndex:13]; - [encoder setBytes:&KH length:sizeof(int32_t) atIndex:14]; - [encoder setBytes:&KW length:sizeof(int32_t) atIndex:15]; - const uint64_t n_threads = MIN(pipeline.maxTotalThreadsPerThreadgroup, (uint64_t)N); const int64_t quotient = N / n_threads + (N % n_threads > 0 ? 1 : 0); @@ -3362,16 +3384,20 @@ static void ggml_metal_encode_node( default: GGML_ABORT("fatal error"); }; + ggml_metal_kargs_conv_transpose_1d args = { + /*.IC =*/ IC, + /*.IL =*/ IL, + /*.K =*/ K, + /*.s0 =*/ s0, + /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&IC length:sizeof( int32_t) atIndex:3]; - [encoder setBytes:&IL length:sizeof( int32_t) atIndex:4]; - [encoder setBytes:&K length:sizeof( int32_t) atIndex:5]; - [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:6]; - [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:7]; - [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:8]; + [encoder setBytes:&args length:sizeof(args) atIndex:3]; [encoder dispatchThreadgroups:MTLSizeMake(OL, OC, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; } break; @@ -3386,30 +3412,33 @@ static void ggml_metal_encode_node( const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_upscale args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.ne03 =*/ ne03, + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.ne0 =*/ ne0, + /*.ne1 =*/ ne1, + /*.ne2 =*/ ne2, + /*.ne3 =*/ ne3, + /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, + /*.sf0 =*/ sf0, + /*.sf1 =*/ sf1, + /*.sf2 =*/ sf2, + /*.sf3 =*/ sf3 + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; - [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12]; - [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16]; - [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17]; - [encoder setBytes:&sf0 length:sizeof(sf0) atIndex:18]; - [encoder setBytes:&sf1 length:sizeof(sf1) atIndex:19]; - [encoder setBytes:&sf2 length:sizeof(sf2) atIndex:20]; - [encoder setBytes:&sf3 length:sizeof(sf3) atIndex:21]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0); @@ -3421,26 +3450,29 @@ static void ggml_metal_encode_node( id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_PAD_F32].pipeline; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_pad args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.ne03 =*/ ne03, + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.ne0 =*/ ne0, + /*.ne1 =*/ ne1, + /*.ne2 =*/ ne2, + /*.ne3 =*/ ne3, + /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3 + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; - [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12]; - [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16]; - [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; const int nth = MIN(1024, ne0); @@ -3455,24 +3487,31 @@ static void ggml_metal_encode_node( id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_PAD_REFLECT_1D_F32].pipeline; + ggml_metal_kargs_pad_reflect_1d args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.ne03 =*/ ne03, + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.ne0 =*/ ne0, + /*.ne1 =*/ ne1, + /*.ne2 =*/ ne2, + /*.ne3 =*/ ne3, + /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, + /*.p0 =*/ p0, + /*.p1 =*/ p1 + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:6]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9]; - [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:11]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:12]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:13]; - [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:14]; - [encoder setBytes:&p0 length:sizeof(p0) atIndex:15]; - [encoder setBytes:&p1 length:sizeof(p1) atIndex:16]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; const int nth = MIN(1024, ne0); @@ -3490,12 +3529,15 @@ static void ggml_metal_encode_node( id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARANGE_F32].pipeline; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_arange args = { + /*.ne0 =*/ ne0, + /*.start =*/ start, + /*.step =*/ step + }; + [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:0]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:1]; - [encoder setBytes:&start length:sizeof(start) atIndex:2]; - [encoder setBytes:&step length:sizeof(step) atIndex:3]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:0]; + [encoder setBytes:&args length:sizeof(args) atIndex:1]; const int nth = MIN(1024, ne0); @@ -3512,13 +3554,16 @@ static void ggml_metal_encode_node( id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32].pipeline; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_timestep_embedding args = { + /*.nb1 =*/ nb1, + /*.dim =*/ dim, + /*.max_period =*/ max_period + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:2]; - [encoder setBytes:&dim length:sizeof(dim) atIndex:3]; - [encoder setBytes:&max_period length:sizeof(max_period) atIndex:4]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; const int nth = MIN(1024, half); @@ -3551,12 +3596,15 @@ static void ggml_metal_encode_node( default: GGML_ABORT("fatal error"); }; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_argsort args = { + /*.ncols =*/ ne00, + /*.ncols_pad =*/ ne00_padded + }; + [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; - [encoder setBytes:&ne00_padded length:sizeof( int64_t) atIndex:3]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00_padded, 1, 1)]; @@ -3570,11 +3618,14 @@ static void ggml_metal_encode_node( id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32].pipeline; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_leaky_relu args = { + /*.slope =*/ slope + }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&slope length:sizeof(slope) atIndex:2]; + [encoder setBytes:&args length:sizeof(args) atIndex:2]; const int64_t n = ggml_nelements(dst); @@ -4150,21 +4201,24 @@ static void ggml_metal_encode_node( const int64_t n_threads = MIN((int64_t)[pipeline maxTotalThreadsPerThreadgroup], parallel_elements); const int64_t n_tg = (parallel_elements + n_threads - 1) / n_threads; - // TODO: add ggml_metal_kargs struct + ggml_metal_kargs_pool_2d args_pool_2d = { + /* .k0 = */ k0, + /* .k1 = */ k1, + /* .s0 = */ s0, + /* .s1 = */ s1, + /* .p0 = */ p0, + /* .p1 = */ p1, + /* .IH = */ IH, + /* .IW = */ IW, + /* .OH = */ OH, + /* .OW = */ OW, + /* .parallel_elements = */ parallel_elements + }; + [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&k0 length:sizeof(int32_t) atIndex:2]; - [encoder setBytes:&k1 length:sizeof(int32_t) atIndex:3]; - [encoder setBytes:&s0 length:sizeof(int32_t) atIndex:4]; - [encoder setBytes:&s1 length:sizeof(int32_t) atIndex:5]; - [encoder setBytes:&p0 length:sizeof(int32_t) atIndex:6]; - [encoder setBytes:&p1 length:sizeof(int32_t) atIndex:7]; - [encoder setBytes:&IH length:sizeof(int64_t) atIndex:8]; - [encoder setBytes:&IW length:sizeof(int64_t) atIndex:9]; - [encoder setBytes:&OH length:sizeof(int64_t) atIndex:10]; - [encoder setBytes:&OW length:sizeof(int64_t) atIndex:11]; - [encoder setBytes:¶llel_elements length:sizeof(int64_t) atIndex:12]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&args_pool_2d length:sizeof(args_pool_2d) atIndex:2]; [encoder dispatchThreadgroups:MTLSizeMake(n_tg, 1, 1) threadsPerThreadgroup:MTLSizeMake(n_threads, 1, 1)]; } break; diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index c46a130508..ad9d42a3ea 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -947,45 +947,22 @@ kernel void kernel_cos( kernel void kernel_sum_rows( device const float * src0, device float * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant int64_t & ne03, - constant uint64_t & nb00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant uint64_t & nb03, - constant int64_t & ne10, - constant int64_t & ne11, - constant int64_t & ne12, - constant int64_t & ne13, - constant uint64_t & nb10, - constant uint64_t & nb11, - constant uint64_t & nb12, - constant uint64_t & nb13, - constant int64_t & ne0, - constant int64_t & ne1, - constant int64_t & ne2, - constant int64_t & ne3, - constant uint64_t & nb0, - constant uint64_t & nb1, - constant uint64_t & nb2, - constant uint64_t & nb3, + constant ggml_metal_kargs_sum_rows & args, uint3 tpig[[thread_position_in_grid]]) { int64_t i3 = tpig.z; int64_t i2 = tpig.y; int64_t i1 = tpig.x; - if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) { + if (i3 >= args.ne03 || i2 >= args.ne02 || i1 >= args.ne01) { return; } - device const float * src_row = (device const float *) ((device const char *) src0 + i1*nb01 + i2*nb02 + i3*nb03); - device float * dst_row = (device float *) ((device char *) dst + i1*nb1 + i2*nb2 + i3*nb3); + device const float * src_row = (device const float *) ((device const char *) src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03); + device float * dst_row = (device float *) ((device char *) dst + i1*args.nb1 + i2*args.nb2 + i3*args.nb3); float row_sum = 0; - for (int64_t i0 = 0; i0 < ne00; i0++) { + for (int64_t i0 = 0; i0 < args.ne00; i0++) { row_sum += src_row[i0]; } @@ -997,36 +974,29 @@ kernel void kernel_soft_max( device const char * src0, device const char * src1, device char * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant float & scale, - constant float & max_bias, - constant float & m0, - constant float & m1, - constant uint32_t & n_head_log2, + constant ggml_metal_kargs_soft_max & args, threadgroup float * buf [[threadgroup(0)]], uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint sgitg[[simdgroup_index_in_threadgroup]], uint tiisg[[thread_index_in_simdgroup]], uint ntg[[threads_per_threadgroup]]) { - const int64_t i03 = (tgpig) / (ne02*ne01); - const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01; - const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01); + const int64_t i03 = (tgpig) / (args.ne02*args.ne01); + const int64_t i02 = (tgpig - i03*args.ne02*args.ne01) / args.ne01; + const int64_t i01 = (tgpig - i03*args.ne02*args.ne01 - i02*args.ne01); - device const float * psrc0 = (device const float *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00); - device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00 : nullptr; - device float * pdst = (device float *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00); + device const float * psrc0 = (device const float *) src0 + (i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00); + device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*args.ne00 : nullptr; + device float * pdst = (device float *) dst + (i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00); float slope = 1.0f; // ALiBi - if (max_bias > 0.0f) { + if (args.max_bias > 0.0f) { const int64_t h = i02; - const float base = h < n_head_log2 ? m0 : m1; - const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + const float base = h < args.n_head_log2 ? args.m0 : args.m1; + const int exp = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; slope = pow(base, exp); } @@ -1034,8 +1004,8 @@ kernel void kernel_soft_max( // parallel max float lmax = -INFINITY; - for (int i00 = tpitg; i00 < ne00; i00 += ntg) { - lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)); + for (int i00 = tpitg; i00 < args.ne00; i00 += ntg) { + lmax = MAX(lmax, psrc0[i00]*args.scale + (pmask ? slope*pmask[i00] : 0.0f)); } // find the max value in the block @@ -1059,8 +1029,8 @@ kernel void kernel_soft_max( // parallel sum float lsum = 0.0f; - for (int i00 = tpitg; i00 < ne00; i00 += ntg) { - const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)) - max_val); + for (int i00 = tpitg; i00 < args.ne00; i00 += ntg) { + const float exp_psrc0 = exp((psrc0[i00]*args.scale + (pmask ? slope*pmask[i00] : 0.0f)) - max_val); lsum += exp_psrc0; pdst[i00] = exp_psrc0; } @@ -1090,7 +1060,7 @@ kernel void kernel_soft_max( const float inv_sum = 1.0f/sum; - for (int i00 = tpitg; i00 < ne00; i00 += ntg) { + for (int i00 = tpitg; i00 < args.ne00; i00 += ntg) { pdst[i00] *= inv_sum; } } @@ -1100,35 +1070,28 @@ kernel void kernel_soft_max_4( device const char * src0, device const char * src1, device char * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant float & scale, - constant float & max_bias, - constant float & m0, - constant float & m1, - constant uint32_t & n_head_log2, + constant ggml_metal_kargs_soft_max & args, threadgroup float * buf [[threadgroup(0)]], uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint sgitg[[simdgroup_index_in_threadgroup]], uint tiisg[[thread_index_in_simdgroup]], uint ntg[[threads_per_threadgroup]]) { - const int64_t i03 = (tgpig) / (ne02*ne01); - const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01; - const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01); + const int64_t i03 = (tgpig) / (args.ne02*args.ne01); + const int64_t i02 = (tgpig - i03*args.ne02*args.ne01) / args.ne01; + const int64_t i01 = (tgpig - i03*args.ne02*args.ne01 - i02*args.ne01); - device const float4 * psrc4 = (device const float4 *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4; - device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00/4 : nullptr; - device float4 * pdst4 = (device float4 *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4; + device const float4 * psrc4 = (device const float4 *) src0 + (i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00)/4; + device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*args.ne00/4 : nullptr; + device float4 * pdst4 = (device float4 *) dst + (i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00)/4; float slope = 1.0f; - if (max_bias > 0.0f) { + if (args.max_bias > 0.0f) { const int64_t h = i02; - const float base = h < n_head_log2 ? m0 : m1; - const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + const float base = h < args.n_head_log2 ? args.m0 : args.m1; + const int exp = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; slope = pow(base, exp); } @@ -1136,8 +1099,8 @@ kernel void kernel_soft_max_4( // parallel max float4 lmax4 = -INFINITY; - for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) { - lmax4 = fmax(lmax4, psrc4[i00]*scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))); + for (int i00 = tpitg; i00 < args.ne00/4; i00 += ntg) { + lmax4 = fmax(lmax4, psrc4[i00]*args.scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))); } const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3])); @@ -1162,8 +1125,8 @@ kernel void kernel_soft_max_4( // parallel sum float4 lsum4 = 0.0f; - for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) { - const float4 exp_psrc4 = exp((psrc4[i00]*scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))) - max_val); + for (int i00 = tpitg; i00 < args.ne00/4; i00 += ntg) { + const float4 exp_psrc4 = exp((psrc4[i00]*args.scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))) - max_val); lsum4 += exp_psrc4; pdst4[i00] = exp_psrc4; } @@ -1195,7 +1158,7 @@ kernel void kernel_soft_max_4( const float inv_sum = 1.0f/sum; - for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) { + for (int i00 = tpitg; i00 < args.ne00/4; i00 += ntg) { pdst4[i00] *= inv_sum; } } @@ -1211,27 +1174,23 @@ template [[host_name("kernel_soft_max_f32_4")]] kernel kernel_soft_max_4_t kerne kernel void kernel_diag_mask_inf( device const float * src0, device float * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int & n_past, + constant ggml_metal_kargs_diag_mask_inf & args, uint3 tpig[[thread_position_in_grid]]) { const int64_t i02 = tpig[2]; const int64_t i01 = tpig[1]; const int64_t i00 = tpig[0]; - if (i00 > n_past + i01) { - dst[i02*ne01*ne00 + i01*ne00 + i00] = -INFINITY; + if (i00 > args.n_past + i01) { + dst[i02*args.ne01*args.ne00 + i01*args.ne00 + i00] = -INFINITY; } else { - dst[i02*ne01*ne00 + i01*ne00 + i00] = src0[i02*ne01*ne00 + i01*ne00 + i00]; + dst[i02*args.ne01*args.ne00 + i01*args.ne00 + i00] = src0[i02*args.ne01*args.ne00 + i01*args.ne00 + i00]; } } kernel void kernel_diag_mask_inf_8( device const float4 * src0, device float4 * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int & n_past, + constant ggml_metal_kargs_diag_mask_inf & args, uint3 tpig[[thread_position_in_grid]]) { const int64_t i = 2*tpig[0]; @@ -1239,42 +1198,26 @@ kernel void kernel_diag_mask_inf_8( dst[i+0] = src0[i+0]; dst[i+1] = src0[i+1]; int64_t i4 = 4*i; - const int64_t i02 = i4/(ne00*ne01); i4 -= i02*ne00*ne01; - const int64_t i01 = i4/(ne00); i4 -= i01*ne00; + const int64_t i02 = i4/(args.ne00*args.ne01); i4 -= i02*args.ne00*args.ne01; + const int64_t i01 = i4/(args.ne00); i4 -= i01*args.ne00; const int64_t i00 = i4; for (int k = 3; k >= 0; --k) { - if (i00 + 4 + k <= n_past + i01) { + if (i00 + 4 + k <= args.n_past + i01) { break; } dst[i+1][k] = -INFINITY; - if (i00 + k > n_past + i01) { + if (i00 + k > args.n_past + i01) { dst[i][k] = -INFINITY; } } } // ref: ggml.c:ggml_compute_forward_ssm_conv_f32 -// TODO: optimize kernel void kernel_ssm_conv_f32( device const void * src0, device const void * src1, device float * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant uint64_t & nb00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant int64_t & ne10, - constant int64_t & ne11, - constant uint64_t & nb10, - constant uint64_t & nb11, - constant int64_t & ne0, - constant int64_t & ne1, - constant int64_t & ne2, - constant uint64_t & nb0, - constant uint64_t & nb1, - constant uint64_t & nb2, + constant ggml_metal_kargs_ssm_conv & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -1282,15 +1225,15 @@ kernel void kernel_ssm_conv_f32( const int64_t i2 = tgpig.y; const int64_t i3 = tgpig.z; - const int64_t nc = ne10; - //const int64_t ncs = ne00; - //const int64_t nr = ne01; - //const int64_t n_t = ne1; - //const int64_t n_s = ne2; + const int64_t nc = args.ne10; + //const int64_t ncs = args.ne00; + //const int64_t nr = args.ne01; + //const int64_t n_t = args.ne1; + //const int64_t n_s = args.ne2; - device const float * s = (device const float *) ((device const char *) src0 + ir*nb01 + i2*nb00 + i3*nb02); - device const float * c = (device const float *) ((device const char *) src1 + ir*nb11); - device float * x = (device float *) ((device char *) dst + ir*nb0 + i2*nb1 + i3*nb2); + device const float * s = (device const float *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02); + device const float * c = (device const float *) ((device const char *) src1 + ir*args.nb11); + device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); float sumf = 0.0f; @@ -1302,7 +1245,6 @@ kernel void kernel_ssm_conv_f32( } // ref: ggml.c:ggml_compute_forward_ssm_scan_f32 -// TODO: optimize kernel void kernel_ssm_scan_f32( device const void * src0, device const void * src1, @@ -1311,48 +1253,27 @@ kernel void kernel_ssm_scan_f32( device const void * src4, device const void * src5, device float * dst, - constant int64_t & d_state, - constant int64_t & d_inner, - constant int64_t & n_seq_tokens, - constant int64_t & n_seqs, - constant uint64_t & nb00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant uint64_t & nb10, - constant uint64_t & nb11, - constant uint64_t & nb12, - constant uint64_t & nb13, - constant uint64_t & nb20, - constant uint64_t & nb21, - constant uint64_t & nb22, - constant uint64_t & nb30, - constant uint64_t & nb31, - constant uint64_t & nb40, - constant uint64_t & nb41, - constant uint64_t & nb42, - constant uint64_t & nb50, - constant uint64_t & nb51, - constant uint64_t & nb52, + constant ggml_metal_kargs_ssm_scan & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { const int64_t ir = tgpig.x; const int64_t i3 = tgpig.y; - const int64_t nc = d_state; - //const int64_t nr = d_inner; - const int64_t n_t = n_seq_tokens; - //const int64_t n_s = n_seqs; + const int64_t nc = args.d_state; + // const int64_t nr = args.d_inner; + const int64_t n_t = args.n_seq_tokens; + // const int64_t n_s = args.n_seqs; for (int64_t i2 = 0; i2 < n_t; ++i2) { - device const float * s0 = (device const float *) ((device const char *) src0 + ir*nb01 + i3*nb02); - device const float * x = (device const float *) ((device const char *) src1 + ir*nb10 + i2*nb11 + i3*nb12); - device const float * dt = (device const float *) ((device const char *) src2 + ir*nb20 + i2*nb21 + i3*nb22); - device const float * A = (device const float *) ((device const char *) src3 + ir*nb31); - device const float * B = (device const float *) ((device const char *) src4 + i2*nb41 + i3*nb42); - device const float * C = (device const float *) ((device const char *) src5 + i2*nb51 + i3*nb52); - device float * y = (device float *) ((device char *) dst + ir*nb10 + i2*nb11 + i3*nb12); // TODO: do not use src1 strides - device float * s = (device float *) ((device char *) dst + ir*nb01 + i3*nb02 + nb13); + device const float * s0 = (device const float *) ((device const char *) src0 + ir*args.nb01 + i3*args.nb02); + device const float * x = (device const float *) ((device const char *) src1 + ir*args.nb10 + i2*args.nb11 + i3*args.nb12); + device const float * dt = (device const float *) ((device const char *) src2 + ir*args.nb20 + i2*args.nb21 + i3*args.nb22); + device const float * A = (device const float *) ((device const char *) src3 + ir*args.nb31); + device const float * B = (device const float *) ((device const char *) src4 + i2*args.nb41 + i3*args.nb42); + device const float * C = (device const float *) ((device const char *) src5 + i2*args.nb51 + i3*args.nb52); + device float * y = (device float *) ((device char *) dst + ir*args.nb10 + i2*args.nb11 + i3*args.nb12); // TODO: do not use src1 strides + device float * s = (device float *) ((device char *) dst + ir*args.nb01 + i3*args.nb02 + args.nb13); if (i2 > 0) { s0 = s; @@ -1545,22 +1466,15 @@ kernel void kernel_rms_norm( kernel void kernel_group_norm( device const float * src0, device float * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant uint64_t & nb00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant int32_t & n_groups, - constant float & eps, + constant ggml_metal_kargs_group_norm & args, threadgroup float * buf [[threadgroup(0)]], uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint sgitg[[simdgroup_index_in_threadgroup]], uint tiisg[[thread_index_in_simdgroup]], uint ntg[[threads_per_threadgroup]]) { - const int64_t ne = ne00*ne01*ne02; - const int64_t gs = ne00*ne01*((ne02 + n_groups - 1) / n_groups); + const int64_t ne = args.ne00*args.ne01*args.ne02; + const int64_t gs = args.ne00*args.ne01*((args.ne02 + args.n_groups - 1) / args.n_groups); int start = tgpig * gs; int end = start + gs; @@ -1624,7 +1538,7 @@ kernel void kernel_group_norm( } const float variance = tmp / gs; - const float scale = 1.0f/sqrt(variance + eps); + const float scale = 1.0f/sqrt(variance + args.eps); for (int j = start; j < end; j += ntg) { dst[j] *= scale; } @@ -2588,17 +2502,7 @@ template [[host_name("kernel_rope_neox_f16")]] kernel kernel_rope_neox_t kernel_ typedef void (im2col_t)( device const float * x, device char * dst, - constant int32_t & ofs0, - constant int32_t & ofs1, - constant int32_t & IW, - constant int32_t & IH, - constant int32_t & CHW, - constant int32_t & s0, - constant int32_t & s1, - constant int32_t & p0, - constant int32_t & p1, - constant int32_t & d0, - constant int32_t & d1, + constant ggml_metal_kargs_im2col & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tgpg[[threadgroups_per_grid]], uint3 tpitg[[thread_position_in_threadgroup]], @@ -2608,17 +2512,7 @@ template kernel void kernel_im2col( device const float * x, device char * dst, - constant int32_t & ofs0, - constant int32_t & ofs1, - constant int32_t & IW, - constant int32_t & IH, - constant int32_t & CHW, - constant int32_t & s0, - constant int32_t & s1, - constant int32_t & p0, - constant int32_t & p1, - constant int32_t & d0, - constant int32_t & d1, + constant ggml_metal_kargs_im2col & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tgpg[[threadgroups_per_grid]], uint3 tpitg[[thread_position_in_threadgroup]], @@ -2639,17 +2533,17 @@ kernel void kernel_im2col( const int64_t ioh = tgpig[1]; const int64_t iow = tgpig[2]; - const int64_t iiw = iow*s0 + ikw*d0 - p0; - const int64_t iih = ioh*s1 + ikh*d1 - p1; + const int64_t iiw = iow*args.s0 + ikw*args.d0 - args.p0; + const int64_t iih = ioh*args.s1 + ikh*args.d1 - args.p1; - const int64_t offset_dst = (in*OH*OW + ioh*OW + iow)*CHW + (iic*(KH*KW) + ikh*KW + ikw); + const int64_t offset_dst = (in*OH*OW + ioh*OW + iow)*args.CHW + (iic*(KH*KW) + ikh*KW + ikw); device T * pdst = (device T *) (dst); - if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { + if (iih < 0 || iih >= args.IH || iiw < 0 || iiw >= args.IW) { pdst[offset_dst] = 0.0f; } else { - const int64_t offset_src = in*ofs0 + iic*ofs1 + iih*IW + iiw; + const int64_t offset_src = in*args.ofs0 + iic*args.ofs1 + iih*args.IW + iiw; pdst[offset_dst] = x[offset_src]; } } @@ -2660,20 +2554,7 @@ template [[host_name("kernel_im2col_f16")]] kernel im2col_t kernel_im2col; typedef void (im2col_ext_t)( device const float * x, device char * dst, - constant int32_t & ofs0, - constant int32_t & ofs1, - constant int32_t & IW, - constant int32_t & IH, - constant int32_t & CHW, - constant int32_t & s0, - constant int32_t & s1, - constant int32_t & p0, - constant int32_t & p1, - constant int32_t & d0, - constant int32_t & d1, - constant int32_t & N, - constant int32_t & KH, - constant int32_t & KW, + constant ggml_metal_kargs_im2col & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tgpg[[threadgroups_per_grid]], uint3 tpitg[[thread_position_in_threadgroup]], @@ -2683,53 +2564,40 @@ template kernel void kernel_im2col_ext( device const float * x, device char * dst, - constant int32_t & ofs0, - constant int32_t & ofs1, - constant int32_t & IW, - constant int32_t & IH, - constant int32_t & CHW, - constant int32_t & s0, - constant int32_t & s1, - constant int32_t & p0, - constant int32_t & p1, - constant int32_t & d0, - constant int32_t & d1, - constant int32_t & N, - constant int32_t & KH, - constant int32_t & KW, + constant ggml_metal_kargs_im2col & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tgpg[[threadgroups_per_grid]], // tgpg[0] = D x IC x KH x KW, CHW = IC x KH x KW uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { // [M, 1, 1] - const int64_t KHW = KH * KW; // KHW == ntg[1] * ntg[2], KW == ntg[2] + const int64_t KHW = (int64_t)args.KHW; - const int64_t d = tgpig[0] / CHW; - const int64_t chw = tgpig[0] % CHW; + const int64_t d = tgpig[0] / args.CHW; + const int64_t chw = tgpig[0] % args.CHW; const int64_t tgpig_0 = chw / KHW; // 0 ~ (IC - 1) const int64_t HW = tgpig[0] % KHW; const int64_t tpitg_0 = (d * ntg[0]) + tpitg[0]; - if (tpitg_0 >= N) { + if (tpitg_0 >= args.N) { return; } - const int64_t tpitg_1 = HW / KW; - const int64_t tpitg_2 = HW % KW; + const int64_t tpitg_1 = HW / args.KW; + const int64_t tpitg_2 = HW % args.KW; - const int64_t iiw = tgpig[2] * s0 + tpitg_2 * d0 - p0; - const int64_t iih = tgpig[1] * s1 + tpitg_1 * d1 - p1; + const int64_t iiw = tgpig[2] * args.s0 + tpitg_2 * args.d0 - args.p0; + const int64_t iih = tgpig[1] * args.s1 + tpitg_1 * args.d1 - args.p1; const int64_t offset_dst = - (tpitg_0 * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * CHW + - (tgpig_0 * KHW + tpitg_1 * KW + tpitg_2); + (tpitg_0 * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * args.CHW + + (tgpig_0 * KHW + tpitg_1 * args.KW + tpitg_2); device T * pdst = (device T *) (dst); - if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { + if (iih < 0 || iih >= args.IH || iiw < 0 || iiw >= args.IW) { pdst[offset_dst] = 0.0f; } else { - const int64_t offset_src = tpitg_0 * ofs0 + tgpig_0 * ofs1; - pdst[offset_dst] = x[offset_src + iih * IW + iiw]; + const int64_t offset_src = tpitg_0 * args.ofs0 + tgpig_0 * args.ofs1; + pdst[offset_dst] = x[offset_src + iih * args.IW + iiw]; } } @@ -2740,12 +2608,7 @@ typedef void (conv_transpose_1d_t)( device const float * src0, device const float * src1, device char * dst, - constant int32_t & IC, - constant int32_t & IL, - constant int32_t & K, - constant int32_t & s0, - constant uint64_t & nb0, - constant uint64_t & nb1, + constant ggml_metal_kargs_conv_transpose_1d & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tgpg[[threadgroups_per_grid]]); @@ -2754,29 +2617,24 @@ kernel void kernel_conv_transpose_1d( device const T * src0, device const float * src1, device char * dst, - constant int32_t & IC, - constant int32_t & IL, - constant int32_t & K, - constant int32_t & s0, - constant uint64_t & nb0, - constant uint64_t & nb1, + constant ggml_metal_kargs_conv_transpose_1d & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tgpg[[threadgroups_per_grid]]) { float v = 0.0f; - for (int64_t c = 0; c < IC; c++) { - const int32_t kernel_offset = c * tgpg[1] * K + K * tgpig[1]; - const int32_t input_offset = c * IL; + for (int64_t c = 0; c < args.IC; c++) { + const int32_t kernel_offset = c * tgpg[1] * args.K + args.K * tgpig[1]; + const int32_t input_offset = c * args.IL; - for (int64_t i = 0; i < IL; i++) { - if (tgpig[0] >= i * s0 && tgpig[0] < i * s0 + K) { - v += src0[kernel_offset + tgpig[0] - i * s0] * src1[input_offset + i]; + for (int64_t i = 0; i < args.IL; i++) { + if (tgpig[0] >= i * args.s0 && tgpig[0] < i * args.s0 + args.K) { + v += src0[kernel_offset + tgpig[0] - i * args.s0] * src1[input_offset + i]; } } } - device float * dst_ptr = (device float *) (dst + tgpig[0] * nb0 + tgpig[1] * nb1); + device float * dst_ptr = (device float *) (dst + tgpig[0] * args.nb0 + tgpig[1] * args.nb1); dst_ptr[0] = v; } @@ -2786,12 +2644,7 @@ kernel void kernel_conv_transpose_1d( device const float * src0, device const float * src1, device char * dst, - constant int32_t & IC, - constant int32_t & IL, - constant int32_t & K, - constant int32_t & s0, - constant uint64_t & nb0, - constant uint64_t & nb1, + constant ggml_metal_kargs_conv_transpose_1d & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tgpg[[threadgroups_per_grid]]); @@ -2800,38 +2653,14 @@ kernel void kernel_conv_transpose_1d( device const half * src0, device const float * src1, device char * dst, - constant int32_t & IC, - constant int32_t & IL, - constant int32_t & K, - constant int32_t & s0, - constant uint64_t & nb0, - constant uint64_t & nb1, + constant ggml_metal_kargs_conv_transpose_1d & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tgpg[[threadgroups_per_grid]]); kernel void kernel_upscale_f32( device const char * src0, device char * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant int64_t & ne03, - constant uint64_t & nb00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant uint64_t & nb03, - constant int64_t & ne0, - constant int64_t & ne1, - constant int64_t & ne2, - constant int64_t & ne3, - constant uint64_t & nb0, - constant uint64_t & nb1, - constant uint64_t & nb2, - constant uint64_t & nb3, - constant float & sf0, - constant float & sf1, - constant float & sf2, - constant float & sf3, + constant ggml_metal_kargs_upscale & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -2840,15 +2669,15 @@ kernel void kernel_upscale_f32( const int64_t i2 = tgpig.y; const int64_t i1 = tgpig.x; - const int64_t i03 = i3/sf3; - const int64_t i02 = i2/sf2; - const int64_t i01 = i1/sf1; + const int64_t i03 = i3/args.sf3; + const int64_t i02 = i2/args.sf2; + const int64_t i01 = i1/args.sf1; - for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) { - const int64_t i00 = i0/sf0; + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + const int64_t i00 = i0/args.sf0; - device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00); - device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00); + device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); dst_ptr[0] = src0_ptr[0]; } @@ -2857,22 +2686,7 @@ kernel void kernel_upscale_f32( kernel void kernel_pad_f32( device const char * src0, device char * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant int64_t & ne03, - constant uint64_t & nb00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant uint64_t & nb03, - constant int64_t & ne0, - constant int64_t & ne1, - constant int64_t & ne2, - constant int64_t & ne3, - constant uint64_t & nb0, - constant uint64_t & nb1, - constant uint64_t & nb2, - constant uint64_t & nb3, + constant ggml_metal_kargs_pad & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -2885,12 +2699,12 @@ kernel void kernel_pad_f32( const int64_t i02 = i2; const int64_t i01 = i1; - device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01); - device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1); + device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); + device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1); - if (i1 < ne01 && i2 < ne02 && i3 < ne03) { - for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) { - if (i0 < ne00) { + if (i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) { + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + if (i0 < args.ne00) { dst_ptr[i0] = src0_ptr[i0]; } else { dst_ptr[i0] = 0.0f; @@ -2900,7 +2714,7 @@ kernel void kernel_pad_f32( return; } - for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) { + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { dst_ptr[i0] = 0.0f; } } @@ -2908,21 +2722,7 @@ kernel void kernel_pad_f32( kernel void kernel_pad_reflect_1d_f32( device const char * src0, device char * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant int64_t & ne03, - constant int64_t & ne0, - constant uint64_t & nb00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant uint64_t & nb03, - constant uint64_t & nb0, - constant uint64_t & nb1, - constant uint64_t & nb2, - constant uint64_t & nb3, - constant int32_t & p0, - constant int32_t & p1, + constant ggml_metal_kargs_pad_reflect_1d & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tgpg[[threadgroups_per_grid]], uint3 tpitg[[thread_position_in_threadgroup]], @@ -2936,17 +2736,17 @@ kernel void kernel_pad_reflect_1d_f32( const int64_t i02 = i2; const int64_t i01 = i1; - device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01); - device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1); + device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); + device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1); - if (i1 < ne01 && i2 < ne02 && i3 < ne03) { - for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) { - if (i0 < p0) { - dst_ptr[i0] = src0_ptr[p0 - i0]; - } else if (i0 < ne0 - p1) { - dst_ptr[i0] = src0_ptr[i0 - p0]; + if (i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) { + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + if (i0 < args.p0) { + dst_ptr[i0] = src0_ptr[args.p0 - i0]; + } else if (i0 < args.ne0 - args.p1) { + dst_ptr[i0] = src0_ptr[i0 - args.p0]; } else { - dst_ptr[i0] = src0_ptr[(ne0 - p1 - p0) - (p1 + 1 - (ne0 - i0)) - 1]; + dst_ptr[i0] = src0_ptr[(args.ne0 - args.p1 - args.p0) - (args.p1 + 1 - (args.ne0 - i0)) - 1]; } } } @@ -2954,44 +2754,40 @@ kernel void kernel_pad_reflect_1d_f32( kernel void kernel_arange_f32( device char * dst, - constant int64_t & ne0, - constant float & start, - constant float & step, + constant ggml_metal_kargs_arange & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { device float * dst_ptr = (device float *) dst; - for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) { - dst_ptr[i0] = start + step * i0; + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + dst_ptr[i0] = args.start + args.step * i0; } } kernel void kernel_timestep_embedding_f32( device const char * src0, device char * dst, - constant uint64_t & nb1, - constant int & dim, - constant int & max_period, + constant ggml_metal_kargs_timestep_embedding & args, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { int i = tgpig.x; - device float * embed_data = (device float *)(dst + i*nb1); + device float * embed_data = (device float *)(dst + i*args.nb1); - int half_ = dim / 2; + int half_ = args.dim / 2; for (int j = tpitg.x; j < half_; j += ntg.x) { float timestep = ((device float *)src0)[i]; - float freq = (float)exp(-log((float)max_period) * j / half_); + float freq = (float)exp(-log((float)args.max_period) * j / half_); float arg = timestep * freq; embed_data[j ] = cos(arg); embed_data[j + half_] = sin(arg); } - if (dim % 2 != 0 && tpitg.x == 0) { - embed_data[dim] = 0.f; + if (args.dim % 2 != 0 && tpitg.x == 0) { + embed_data[args.dim] = 0.f; } } @@ -2999,8 +2795,7 @@ kernel void kernel_timestep_embedding_f32( typedef void (argsort_t)( device const float * x, device int32_t * dst, - constant int64_t & ncols, - constant int64_t & ncols_pad, + constant ggml_metal_kargs_argsort & args, threadgroup int32_t * shared_values [[threadgroup(0)]], uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]]); @@ -3009,8 +2804,7 @@ template kernel void kernel_argsort_f32_i32( device const float * x, device int32_t * dst, - constant int64_t & ncols, - constant int64_t & ncols_pad, + constant ggml_metal_kargs_argsort & args, threadgroup int32_t * shared_values [[threadgroup(0)]], uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]]) { @@ -3018,9 +2812,9 @@ kernel void kernel_argsort_f32_i32( int col = tpitg[0]; int row = tgpig[1]; - if (col >= ncols_pad) return; + if (col >= args.ncols_pad) return; - device const float * x_row = x + row * ncols; + device const float * x_row = x + row * args.ncols; threadgroup int32_t * dst_row = shared_values; // initialize indices @@ -3028,21 +2822,21 @@ kernel void kernel_argsort_f32_i32( threadgroup_barrier(mem_flags::mem_threadgroup); - for (int k = 2; k <= ncols_pad; k *= 2) { + for (int k = 2; k <= args.ncols_pad; k *= 2) { for (int j = k / 2; j > 0; j /= 2) { int ixj = col ^ j; if (ixj > col) { if ((col & k) == 0) { - if (dst_row[col] >= ncols || - (dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ? + if (dst_row[col] >= args.ncols || + (dst_row[ixj] < args.ncols && (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]])) ) { SWAP(dst_row[col], dst_row[ixj]); } } else { - if (dst_row[ixj] >= ncols || - (dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ? + if (dst_row[ixj] >= args.ncols || + (dst_row[col] < args.ncols && (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]])) ) { @@ -3055,8 +2849,8 @@ kernel void kernel_argsort_f32_i32( } // copy the result to dst without the padding - if (col < ncols) { - dst[row * ncols + col] = dst_row[col]; + if (col < args.ncols) { + dst[row * args.ncols + col] = dst_row[col]; } } @@ -3066,9 +2860,9 @@ template [[host_name("kernel_argsort_f32_i32_desc")]] kernel argsort_t kernel_ar kernel void kernel_leaky_relu_f32( device const float * src0, device float * dst, - constant float & slope, + constant ggml_metal_kargs_leaky_relu & args, uint tpig[[thread_position_in_grid]]) { - dst[tpig] = src0[tpig] > 0.0f ? src0[tpig] : src0[tpig] * slope; + dst[tpig] = src0[tpig] > 0.0f ? src0[tpig] : src0[tpig] * args.slope; } // ref: https://arxiv.org/pdf/2307.08691.pdf @@ -6009,28 +5803,21 @@ kernel void kernel_get_rows_q( device const void * src0, device const void * src1, device float * dst, - constant int64_t & ne00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant int64_t & ne10, - constant uint64_t & nb10, - constant uint64_t & nb11, - constant uint64_t & nb1, - constant uint64_t & nb2, + constant ggml_metal_kargs_get_rows & args, uint3 tgpig[[threadgroup_position_in_grid]], uint tiitg[[thread_index_in_threadgroup]], uint3 tptg [[threads_per_threadgroup]]) { const int64_t i10 = tgpig.x; const int64_t i11 = tgpig.y; - const int64_t r = ((const device int32_t *) ((const device char *) src1 + i11*nb11 + i10*nb10))[0]; + const int64_t r = ((const device int32_t *) ((const device char *) src1 + i11*args.nb11 + i10*args.nb10))[0]; const int64_t i02 = i11; - for (int64_t ind = tiitg; ind < ne00/16; ind += tptg.x) { + for (int64_t ind = tiitg; ind < args.ne00/16; ind += tptg.x) { float4x4 temp; - dequantize_func(((device const block_q *) ((const device char *) src0 + r*nb01 + i02*nb02)) + ind/nl, ind%nl, temp); - *(((device float4x4 *) ((device char *) dst + i11*nb2 + i10*nb1)) + ind) = temp; + dequantize_func(((device const block_q *) ((const device char *) src0 + r*args.nb01 + i02*args.nb02)) + ind/nl, ind%nl, temp); + *(((device float4x4 *) ((device char *) dst + i11*args.nb2 + i10*args.nb1)) + ind) = temp; } } @@ -6039,27 +5826,20 @@ kernel void kernel_get_rows_f( device const void * src0, device const void * src1, device float * dst, - constant int64_t & ne00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant int64_t & ne10, - constant uint64_t & nb10, - constant uint64_t & nb11, - constant uint64_t & nb1, - constant uint64_t & nb2, + constant ggml_metal_kargs_get_rows & args, uint3 tgpig[[threadgroup_position_in_grid]], uint tiitg[[thread_index_in_threadgroup]], uint3 tptg [[threads_per_threadgroup]]) { const int64_t i10 = tgpig.x; const int64_t i11 = tgpig.y; - const int64_t r = ((const device int32_t *) ((const device char *) src1 + i11*nb11 + i10*nb10))[0]; + const int64_t r = ((const device int32_t *) ((const device char *) src1 + i11*args.nb11 + i10*args.nb10))[0]; const int64_t i02 = i11; - for (int ind = tiitg; ind < ne00; ind += tptg.x) { - (( device float *) (( device char *) dst + i11*nb2 + i10*nb1))[ind] = - ((const device T *) ((const device char *) src0 + i02*nb02 + r*nb01))[ind]; + for (int ind = tiitg; ind < args.ne00; ind += tptg.x) { + (( device float *) (( device char *) dst + i11*args.nb2 + i10*args.nb1))[ind] = + ((const device T *) ((const device char *) src0 + i02*args.nb02 + r*args.nb01))[ind]; } } @@ -6067,27 +5847,20 @@ kernel void kernel_get_rows_i32( device const void * src0, device const void * src1, device int32_t * dst, - constant int64_t & ne00, - constant uint64_t & nb01, - constant uint64_t & nb02, - constant int64_t & ne10, - constant uint64_t & nb10, - constant uint64_t & nb11, - constant uint64_t & nb1, - constant uint64_t & nb2, + constant ggml_metal_kargs_get_rows & args, uint3 tgpig[[threadgroup_position_in_grid]], uint tiitg[[thread_index_in_threadgroup]], uint3 tptg [[threads_per_threadgroup]]) { const int64_t i10 = tgpig.x; const int64_t i11 = tgpig.y; - const int64_t r = ((const device int32_t *) ((const device char *) src1 + i11*nb11 + i10*nb10))[0]; + const int64_t r = ((const device int32_t *) ((const device char *) src1 + i11*args.nb11 + i10*args.nb10))[0]; const int64_t i02 = i11; - for (int ind = tiitg; ind < ne00; ind += tptg.x) { - (( device int32_t *) (( device char *) dst + i11*nb2 + i10*nb1))[ind] = - ((const device int32_t *) ((const device char *) src0 + i02*nb02 + r*nb01))[ind]; + for (int ind = tiitg; ind < args.ne00; ind += tptg.x) { + (( device int32_t *) (( device char *) dst + i11*args.nb2 + i10*args.nb1))[ind] = + ((const device int32_t *) ((const device char *) src0 + i02*args.nb02 + r*args.nb01))[ind]; } } @@ -6689,98 +6462,78 @@ template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel void kernel_pool_2d_max_f32( device const float * src0, device float * dst, - constant int32_t & k0, - constant int32_t & k1, - constant int32_t & s0, - constant int32_t & s1, - constant int32_t & p0, - constant int32_t & p1, - constant int64_t & IH, - constant int64_t & IW, - constant int64_t & OH, - constant int64_t & OW, - constant int64_t & parallel_elements, + constant ggml_metal_kargs_pool_2d & args, uint gid[[thread_position_in_grid]]) { - if (gid >= parallel_elements) { + if (gid >= args.parallel_elements) { return; } const int idx = gid; - const int I_HW = IH * IW; - const int O_HW = OH * OW; + const int I_HW = args.IH * args.IW; + const int O_HW = args.OH * args.OW; const int nc = idx / O_HW; - const int cur_oh = idx % O_HW / OW; - const int cur_ow = idx % O_HW % OW; + const int cur_oh = idx % O_HW / args.OW; + const int cur_ow = idx % O_HW % args.OW; device const float * i_ptr = src0 + nc * I_HW; device float * o_ptr = dst + nc * O_HW; - const int start_h = cur_oh * s1 - p1; + const int start_h = cur_oh * args.s1 - args.p1; const int bh = MAX(0, start_h); - const int eh = MIN(IH, start_h + k1); - const int start_w = cur_ow * s0 - p0; + const int eh = MIN(args.IH, start_h + args.k1); + const int start_w = cur_ow * args.s0 - args.p0; const int bw = MAX(0, start_w); - const int ew = MIN(IW, start_w + k0); + const int ew = MIN(args.IW, start_w + args.k0); float res = -INFINITY; for (int i = bh; i < eh; i += 1) { for (int j = bw; j < ew; j += 1) { - res = MAX(res, i_ptr[i * IW + j]); + res = MAX(res, i_ptr[i * args.IW + j]); } } - o_ptr[cur_oh * OW + cur_ow] = res; + o_ptr[cur_oh * args.OW + cur_ow] = res; } kernel void kernel_pool_2d_avg_f32( device const float * src0, device float * dst, - constant int32_t & k0, - constant int32_t & k1, - constant int32_t & s0, - constant int32_t & s1, - constant int32_t & p0, - constant int32_t & p1, - constant int64_t & IH, - constant int64_t & IW, - constant int64_t & OH, - constant int64_t & OW, - constant int64_t & parallel_elements, + constant ggml_metal_kargs_pool_2d & args, uint gid[[thread_position_in_grid]]) { - if (gid >= parallel_elements) { + if (gid >= args.parallel_elements) { return; } const int idx = gid; - const int I_HW = IH * IW; - const int O_HW = OH * OW; + const int I_HW = args.IH * args.IW; + const int O_HW = args.OH * args.OW; const int nc = idx / O_HW; - const int cur_oh = idx % O_HW / OW; - const int cur_ow = idx % O_HW % OW; + const int cur_oh = idx % O_HW / args.OW; + const int cur_ow = idx % O_HW % args.OW; device const float * i_ptr = src0 + nc * I_HW; device float * o_ptr = dst + nc * O_HW; - const int start_h = cur_oh * s1 - p1; + const int start_h = cur_oh * args.s1 - args.p1; const int bh = MAX(0, start_h); - const int eh = MIN(IH, start_h + k1); - const int start_w = cur_ow * s0 - p0; + const int eh = MIN(args.IH, start_h + args.k1); + const int start_w = cur_ow * args.s0 - args.p0; const int bw = MAX(0, start_w); - const int ew = MIN(IW, start_w + k0); + const int ew = MIN(args.IW, start_w + args.k0); // const float scale = 1. / ((eh - bh) * (ew - bw)); - const float scale = 1. / (k0 * k1); + const float scale = 1. / (args.k0 * args.k1); float res = 0; for (int i = bh; i < eh; i += 1) { for (int j = bw; j < ew; j += 1) { - float cur = i_ptr[i * IW + j]; + float cur = i_ptr[i * args.IW + j]; res += cur * scale; } } - o_ptr[cur_oh * OW + cur_ow] = res; + o_ptr[cur_oh * args.OW + cur_ow] = res; } From 7cf64f6beecf54c6ac71503181f154667fd4228a Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Fri, 7 Mar 2025 09:33:37 +0000 Subject: [PATCH 095/188] sync: minja - support QwQ-32B (#12235) https://github.com/google/minja/commit/8a76f7815e8a3ae00bd233c2b5a8b7d4e86564ec --- common/minja/minja.hpp | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/common/minja/minja.hpp b/common/minja/minja.hpp index c58dd66e06..fa4c34d6e4 100644 --- a/common/minja/minja.hpp +++ b/common/minja/minja.hpp @@ -1378,13 +1378,27 @@ struct ArgumentsExpression { } }; -static std::string strip(const std::string & s) { - auto start = s.find_first_not_of(" \t\n\r"); +static std::string strip(const std::string & s, const std::string & chars = "", bool left = true, bool right = true) { + auto charset = chars.empty() ? " \t\n\r" : chars; + auto start = left ? s.find_first_not_of(charset) : 0; if (start == std::string::npos) return ""; - auto end = s.find_last_not_of(" \t\n\r"); + auto end = right ? s.find_last_not_of(charset) : s.size() - 1; return s.substr(start, end - start + 1); } +static std::vector split(const std::string & s, const std::string & sep) { + std::vector result; + size_t start = 0; + size_t end = s.find(sep); + while (end != std::string::npos) { + result.push_back(s.substr(start, end - start)); + start = end + sep.length(); + end = s.find(sep, start); + } + result.push_back(s.substr(start)); + return result; +} + static std::string capitalize(const std::string & s) { if (s.empty()) return s; auto result = s; @@ -1467,8 +1481,26 @@ public: } else if (obj.is_string()) { auto str = obj.get(); if (method->get_name() == "strip") { - vargs.expectArgs("strip method", {0, 0}, {0, 0}); - return Value(strip(str)); + vargs.expectArgs("strip method", {0, 1}, {0, 0}); + auto chars = vargs.args.empty() ? "" : vargs.args[0].get(); + return Value(strip(str, chars)); + } else if (method->get_name() == "lstrip") { + vargs.expectArgs("lstrip method", {0, 1}, {0, 0}); + auto chars = vargs.args.empty() ? "" : vargs.args[0].get(); + return Value(strip(str, chars, /* left= */ true, /* right= */ false)); + } else if (method->get_name() == "rstrip") { + vargs.expectArgs("rstrip method", {0, 1}, {0, 0}); + auto chars = vargs.args.empty() ? "" : vargs.args[0].get(); + return Value(strip(str, chars, /* left= */ false, /* right= */ true)); + } else if (method->get_name() == "split") { + vargs.expectArgs("split method", {1, 1}, {0, 0}); + auto sep = vargs.args[0].get(); + auto parts = split(str, sep); + Value result = Value::array(); + for (const auto& part : parts) { + result.push_back(Value(part)); + } + return result; } else if (method->get_name() == "capitalize") { vargs.expectArgs("capitalize method", {0, 0}, {0, 0}); return Value(capitalize(str)); From 8fad3c7a7c54a25a1ca38dfb08244df55288e675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Fri, 7 Mar 2025 11:15:33 +0100 Subject: [PATCH 096/188] server : Log original chat template parsing error (#12233) --- examples/server/server.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 2a526b0e7a..e1371dbf8c 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1900,6 +1900,7 @@ struct server_context { try { common_chat_format_example(chat_templates.get(), params.use_jinja); } catch (const std::exception & e) { + SRV_WRN("%s: Chat template parsing error: %s\n", __func__, e.what()); SRV_WRN("%s: The chat template that comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses\n", __func__); chat_templates = common_chat_templates_init(model, "chatml"); } From ea002810a209246d034d1b6ddac387f778751588 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 7 Mar 2025 12:19:31 +0200 Subject: [PATCH 097/188] ci : fix save-load test invocations (#12245) --- ci/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/run.sh b/ci/run.sh index 77c32ce005..9fc19c89d8 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -352,10 +352,10 @@ function gg_run_open_llama_7b_v2 { (time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -t 1 -ngl 99 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log - (time ./bin/llama-save-load-state--model ${model_q4_0} -ngl 10 -c 0 ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log - (time ./bin/llama-save-load-state--model ${model_q4_0} -ngl 10 -c 0 -fa ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log - (time ./bin/llama-save-load-state--model ${model_q4_0} -ngl 99 -c 0 ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log - (time ./bin/llama-save-load-state--model ${model_q4_0} -ngl 99 -c 0 -fa ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/llama-save-load-state --model ${model_q4_0} -ngl 10 -c 0 ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/llama-save-load-state --model ${model_q4_0} -ngl 10 -c 0 -fa ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/llama-save-load-state --model ${model_q4_0} -ngl 99 -c 0 ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/llama-save-load-state --model ${model_q4_0} -ngl 99 -c 0 -fa ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log function check_ppl { qnt="$1" From 68d0027f3d19eb579c1863814c91e37ffa699014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20O?= Date: Fri, 7 Mar 2025 12:54:22 +0100 Subject: [PATCH 098/188] ggml-cpu: faster AVX2 variant for IQ1_M (#12216) --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 2ae66591d2..8c7dbd1ccb 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -11718,9 +11718,12 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const #elif defined __AVX2__ - const __m256i mask = _mm256_set1_epi16(2 * 0x7); + const __m256i mask = _mm256_set1_epi16(0x7); const __m256i mone = _mm256_set1_epi16(1); const __m256i mone8 = _mm256_set1_epi8(1); + const __m256i mtwo8 = _mm256_set1_epi8(2); + // VPSHUFB cannot cross 128-bit lanes so odd shifts go to upper half. + const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0); __m256 accum1 = _mm256_setzero_ps(); __m256 accum2 = _mm256_setzero_ps(); @@ -11732,6 +11735,14 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const const uint16_t * sc = (const uint16_t *)x[i].scales; scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); + // Extract 3-bit scales (16 values) + __m256i scales = _mm256_set1_epi64x(*(const uint64_t*)sc); + scales = _mm256_srlv_epi64(scales, scales_shift); + scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone); + + // Indices to repeat each scale 8 times. + __m256i scales_idx1 = _mm256_set1_epi16(0x0100); + __m256i scales_idx2 = _mm256_add_epi8(scales_idx1, _mm256_set1_epi8(8)); __m256i sumi1 = _mm256_setzero_si256(); __m256i sumi2 = _mm256_setzero_si256(); @@ -11777,11 +11788,12 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const const __m256i dot3 = _mm256_maddubs_epi16(mone8, _mm256_sign_epi8(q8b_1, delta1)); const __m256i dot4 = _mm256_maddubs_epi16(mone8, _mm256_sign_epi8(q8b_2, delta2)); - __m256i scale1 = MM256_SET_M128I(_mm_set1_epi16(sc[ib/2] >> 2), _mm_set1_epi16(sc[ib/2] << 1)); - __m256i scale2 = MM256_SET_M128I(_mm_set1_epi16(sc[ib/2] >> 8), _mm_set1_epi16(sc[ib/2] >> 5)); + __m256i scale1 = _mm256_shuffle_epi8(scales, scales_idx1); + __m256i scale2 = _mm256_shuffle_epi8(scales, scales_idx2); + + scales_idx1 = _mm256_add_epi8(scales_idx1, mtwo8); + scales_idx2 = _mm256_add_epi8(scales_idx2, mtwo8); - scale1 = _mm256_add_epi16(_mm256_and_si256(scale1, mask), mone); - scale2 = _mm256_add_epi16(_mm256_and_si256(scale2, mask), mone); const __m256i p1 = _mm256_madd_epi16(dot1, scale1); const __m256i p2 = _mm256_madd_epi16(dot2, scale2); const __m256i p3 = _mm256_madd_epi16(dot3, scale1); From d6ae2fa06139e496880cbf65197c84341e9d98e7 Mon Sep 17 00:00:00 2001 From: vmobilis <75476228+vmobilis@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:11:40 +0300 Subject: [PATCH 099/188] ggml : ggml_compute_forward_concat() for arbitrary tensor type (ggml/1118) * ggml_compute_forward_concat() for arbitrary tensor type * Check that tensors' type match * ggml-cpu.c: check type of source tensors * ggml-cpu.c: move tensor type check to ggml_compute_forward_concat() * ggml.c: check concatenated tensor type * Remove tensor type check from ggml_compute_forward_concat() in ggml-cpu.c ..., as it was moved to ggml.c. --- ggml/src/ggml-cpu/ggml-cpu.c | 143 ++++++++++++++++++++++++++++++++++- ggml/src/ggml.c | 1 + 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index c67fdd0456..f2ab4c5d69 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -6648,6 +6648,135 @@ static void ggml_compute_forward_repeat_back( // ggml_compute_forward_concat +static void ggml_compute_forward_concat_any( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + const size_t len = ggml_type_size(src0->type); + + const int ith = params->ith; + const int nth = params->nth; + + GGML_TENSOR_BINARY_OP_LOCALS + + const int32_t dim = ggml_get_op_params_i32(dst, 0); + + GGML_ASSERT(dim >= 0 && dim < 4); + + int64_t o[4] = {0, 0, 0, 0}; + o[dim] = src0->ne[dim]; + + const char * x; + + // TODO: smarter multi-theading + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = ith; i2 < ne2; i2 += nth) { + for (int i1 = 0; i1 < ne1; i1++) { + for (int i0 = 0; i0 < ne0; i0++) { + if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { + x = (const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03; + } else { + x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13; + } + + char * y = (char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3; + + memcpy(y, x, len); + } + } + } + } +} + +static void ggml_compute_forward_concat_i8( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t)); + + const int ith = params->ith; + const int nth = params->nth; + + GGML_TENSOR_BINARY_OP_LOCALS + + const int32_t dim = ggml_get_op_params_i32(dst, 0); + + GGML_ASSERT(dim >= 0 && dim < 4); + + int64_t o[4] = {0, 0, 0, 0}; + o[dim] = src0->ne[dim]; + + const int8_t * x; + + // TODO: smarter multi-theading + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = ith; i2 < ne2; i2 += nth) { + for (int i1 = 0; i1 < ne1; i1++) { + for (int i0 = 0; i0 < ne0; i0++) { + if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { + x = (const int8_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); + } else { + x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); + } + + int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + + *y = *x; + } + } + } + } +} + +static void ggml_compute_forward_concat_f16( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t)); + + const int ith = params->ith; + const int nth = params->nth; + + GGML_TENSOR_BINARY_OP_LOCALS + + const int32_t dim = ggml_get_op_params_i32(dst, 0); + + GGML_ASSERT(dim >= 0 && dim < 4); + + int64_t o[4] = {0, 0, 0, 0}; + o[dim] = src0->ne[dim]; + + const ggml_fp16_t * x; + + // TODO: smarter multi-theading + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = ith; i2 < ne2; i2 += nth) { + for (int i1 = 0; i1 < ne1; i1++) { + for (int i0 = 0; i0 < ne0; i0++) { + if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { + x = (const ggml_fp16_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); + } else { + x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); + } + + ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + + *y = *x; + } + } + } + } +} + static void ggml_compute_forward_concat_f32( const struct ggml_compute_params * params, struct ggml_tensor * dst) { @@ -6655,7 +6784,7 @@ static void ggml_compute_forward_concat_f32( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(src0->nb[0] == sizeof(float)); + GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float)); const int ith = params->ith; const int nth = params->nth; @@ -6698,6 +6827,16 @@ static void ggml_compute_forward_concat( const struct ggml_tensor * src0 = dst->src[0]; switch (src0->type) { + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + case GGML_TYPE_I16: + { + ggml_compute_forward_concat_f16(params, dst); + } break; + case GGML_TYPE_I8: + { + ggml_compute_forward_concat_i8(params, dst); + } break; case GGML_TYPE_F32: case GGML_TYPE_I32: { @@ -6705,7 +6844,7 @@ static void ggml_compute_forward_concat( } break; default: { - GGML_ABORT("fatal error"); + ggml_compute_forward_concat_any(params, dst); } } } diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 084240331e..89409bb0e4 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -2332,6 +2332,7 @@ struct ggml_tensor * ggml_concat( struct ggml_tensor * b, int dim) { GGML_ASSERT(dim >= 0 && dim < GGML_MAX_DIMS); + GGML_ASSERT(a->type == b->type); int64_t ne[GGML_MAX_DIMS]; for (int d = 0; d < GGML_MAX_DIMS; ++d) { From 102ac1891db32c346a7b6b96145a2a23c1e4c352 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 7 Mar 2025 14:00:27 +0200 Subject: [PATCH 100/188] sync : ggml ggml-ci --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 040b53ca39..c7944d1d42 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -58ecf6b96d887e408b6869915863fa1126483d51 +c7dfe3d174f98b14801f9ed12f129179d3e7b638 From 7c7f3b7f435f41f2508e0e3010f0013cd8335156 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 7 Mar 2025 14:15:27 +0100 Subject: [PATCH 101/188] ggml : skip intermediate .air file when compiling .metallib (#12247) This commit updates the compilation of default.metallib to skip the intermediate .air (Apple Intermediate Representation) file. The motivation for this change is to simplify the custom command a little and avoid generating and then removing the .air file. --- ggml/src/ggml-metal/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txt index be3fb3fa95..e222327809 100644 --- a/ggml/src/ggml-metal/CMakeLists.txt +++ b/ggml/src/ggml-metal/CMakeLists.txt @@ -88,9 +88,8 @@ else() add_custom_command( OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib - COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air - COMMAND xcrun -sdk macosx metallib ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib - COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air + COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o - | + xcrun -sdk macosx metallib - -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal DEPENDS ggml-metal.metal ${METALLIB_COMMON} From 7ab364390f92b0b8d83f69821a536b424838f3f8 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 7 Mar 2025 20:54:30 +0200 Subject: [PATCH 102/188] server : infill gen ends on new line (#12254) --- examples/server/server.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index e1371dbf8c..8386f4eebb 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1312,7 +1312,7 @@ struct server_slot { return task_type == SERVER_TASK_TYPE_EMBEDDING || task_type == SERVER_TASK_TYPE_RERANK; } - bool can_batch_with(server_slot & other_slot) { + bool can_batch_with(server_slot & other_slot) const { return is_non_causal() == other_slot.is_non_causal() && are_lora_equal(lora, other_slot.lora); } @@ -2157,14 +2157,6 @@ struct server_context { } if (slot.has_new_line) { - // if we have already seen a new line, we stop after a certain time limit - if (slot.params.t_max_predict_ms > 0 && (ggml_time_us() - slot.t_start_generation > 1000.0f*slot.params.t_max_predict_ms)) { - slot.stop = STOP_TYPE_LIMIT; - slot.has_next_token = false; - - SLT_DBG(slot, "stopped by time limit, n_decoded = %d, t_max_predict_ms = %d ms\n", slot.n_decoded, (int) slot.params.t_max_predict_ms); - } - // require that each new line has a whitespace prefix (i.e. indentation) of at least slot.params.n_indent if (slot.params.n_indent > 0) { // check the current indentation @@ -2203,6 +2195,14 @@ struct server_context { // check if there is a new line in the generated text if (result.text_to_send.find('\n') != std::string::npos) { slot.has_new_line = true; + + // if we have seen a new line, we stop after a certain time limit, but only upon another new line + if (slot.params.t_max_predict_ms > 0 && (ggml_time_us() - slot.t_start_generation > 1000.0f*slot.params.t_max_predict_ms)) { + slot.stop = STOP_TYPE_LIMIT; + slot.has_next_token = false; + + SLT_DBG(slot, "stopped by time limit, n_decoded = %d, t_max_predict_ms = %d ms\n", slot.n_decoded, (int) slot.params.t_max_predict_ms); + } } // if context shift is disabled, we stop when it reaches the context limit From 6fefc05a7a4e676780ae10b0a4d0728e5281f367 Mon Sep 17 00:00:00 2001 From: "Jason C.H" Date: Sun, 9 Mar 2025 00:02:39 +0800 Subject: [PATCH 103/188] ggml-backend : make path_str compatible with C++20 (#12269) --- AUTHORS | 1 + ggml/src/ggml-backend-reg.cpp | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/AUTHORS b/AUTHORS index 6796b29413..ddcb156388 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1045,3 +1045,4 @@ zrm 蕭澧邦 <45505768+shou692199@users.noreply.github.com> 谢乃闻 Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com> +Jason C.H diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp index d0d68becd8..9bedeae78a 100644 --- a/ggml/src/ggml-backend-reg.cpp +++ b/ggml/src/ggml-backend-reg.cpp @@ -76,7 +76,14 @@ namespace fs = std::filesystem; static std::string path_str(const fs::path & path) { std::string u8path; try { +#if defined(__cpp_lib_char8_t) + // C++20 and later: u8string() returns std::u8string + std::u8string u8str = path.u8string(); + u8path = std::string(reinterpret_cast(u8str.c_str())); +#else + // C++17: u8string() returns std::string u8path = path.u8string(); +#endif } catch (...) { } return u8path; From 0fd7ca7a210bd4abc995cd728491043491dbdef7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 8 Mar 2025 18:26:00 +0200 Subject: [PATCH 104/188] authors : update (#12271) --- AUTHORS | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index ddcb156388..0af9f44ad4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,4 +1,4 @@ -# date: Tue Feb 4 13:04:05 EET 2025 +# date: Sat Mar 8 18:23:52 EET 2025 # this file is auto-generated by scripts/gen-authors.sh 0cc4m @@ -8,10 +8,12 @@ 3ooabkhxtn <31479382+3ooabkhxtn@users.noreply.github.com> 44670 <44670@users.noreply.github.com> 65a <10104049+65a@users.noreply.github.com> +708-145 <40387547+708-145@users.noreply.github.com> AN Long AT Aarni Koskela Aaron Miller +Aaron Teo <57927438+taronaeo@users.noreply.github.com> Aaryaman Vasishta Abheek Gulati Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> @@ -20,6 +22,7 @@ Adithya Balaji AdithyanI Adrian Adrian Hesketh +Adrian Kretz Adrien Gallouët Adrien Gallouët Ahmad Tameem <113388789+Tameem-10xE@users.noreply.github.com> @@ -28,15 +31,18 @@ AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> AidanBeltonS Aisuko Akarshan Biswas +Akarshan Biswas Akarshan Biswas Al Mochkin <14274697+amochkin@users.noreply.github.com> Albert Jin Alberto <57916483+albbus-stack@users.noreply.github.com> Alberto Cabrera Pérez Alberto Cabrera Pérez +Aleksei Nikiforov <103434461+AlekseiNikiforovIBM@users.noreply.github.com> Alex Alex Azarov Alex Azarov +Alex Brooks Alex Klinkhamer Alex Klinkhamer Alex Nguyen @@ -67,6 +73,7 @@ Andrew Minh Nguyen <40281306+amqdn@users.noreply.github.com> Andy Salerno Andy Tai Anthony Van de Gejuchte +Antoine Viallon Antonis Makropoulos Arik Poznanski Armen Kaleshian @@ -83,6 +90,7 @@ Atsushi Tatsuma Austin <77757836+teleprint-me@users.noreply.github.com> AustinMroz BADR +BB-fat <45072480+BB-fat@users.noreply.github.com> Bach Le Bailey Chittle <39804642+bachittle@users.noreply.github.com> BarfingLemurs <128182951+BarfingLemurs@users.noreply.github.com> @@ -101,6 +109,7 @@ Bert Wagner Billel Mokeddem Bingan <70050083+binganao@users.noreply.github.com> Bjarke Viksøe <164612031+bviksoe@users.noreply.github.com> +Bodhi <3882561+BodhiHu@users.noreply.github.com> Bodo Graumann Bono Lv Borislav Stanimirov @@ -128,6 +137,7 @@ CentricStorm Chad Brewbaker Changyeon Kim Chao Jiang +Charles Duffy Charles Xu <63788048+chaxu01@users.noreply.github.com> Charles Xu Chen Xi @@ -139,12 +149,14 @@ Chris Kuehl Christian Demsar Christian Demsar Christian Falch <875252+chrfalch@users.noreply.github.com> +Christian Fillion Christian Kastner Christian Kögler Christian Köhnenkamp Christian Zhou-Zheng <59622928+christianazinn@users.noreply.github.com> Christopher Nielsen <62156882+mascguy@users.noreply.github.com> Clark Saben <76020733+csaben@users.noreply.github.com> +Clauszy Clint Herron Conrad Kramer Corentin REGAL @@ -163,6 +175,7 @@ Daniel Hiltgen Daniel Illescas Romero Daniel Kleine <53251018+d-kleine@users.noreply.github.com> Daniele <57776841+daniandtheweb@users.noreply.github.com> +Danny Milosavljevic DannyDaemonic Dat Quoc Nguyen <2412555+datquocnguyen@users.noreply.github.com> Dave @@ -170,6 +183,7 @@ Dave Airlie Dave Airlie Dave Della Costa David Friehs +David Huang <1969802+hjc4869@users.noreply.github.com> David Kennedy David Pflug David Renshaw @@ -236,6 +250,7 @@ Felix Finn Voorhees Firat FirstTimeEZ <179362031+FirstTimeEZ@users.noreply.github.com> +Florent BENOIT Folko-Ven <71110216+Folko-Ven@users.noreply.github.com> Foul-Tarnished <107711110+Foul-Tarnished@users.noreply.github.com> Francisco Melo <43780565+francis2tm@users.noreply.github.com> @@ -254,6 +269,7 @@ Gary Mulder Gavin Zhao Genkagaku.GPT Georgi Gerganov +Gian-Carlo Pascutto Gilad S Gilad S. <7817232+giladgd@users.noreply.github.com> Giuseppe Scrivano @@ -267,7 +283,9 @@ Guspan Tanadi <36249910+guspan-tanadi@users.noreply.github.com> Gustavo Rocha Dias <91472747+gustrd@users.noreply.github.com> Haggai Nuchi Halalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com> +Hale Chan Hamdoud Hakem <90524568+hamdoudhakem@users.noreply.github.com> +Han Yin HanishKVC Haohui Mai Haoxiang Fei @@ -278,6 +296,7 @@ Haus1 Henk Poley Henri Vasserman Henrik Forstén +Henry Linjamäki Herman Semenov Hesen Peng HimariO @@ -307,6 +326,7 @@ Ivan Ivan Filipov <159561759+vanaka11@users.noreply.github.com> Ivan Komarov Ivan Stepanov +JC <43374599+MrSMlT@users.noreply.github.com> JFLFY2255 JH23X <165871467+JH23X@users.noreply.github.com> Jack Mousseau @@ -325,6 +345,7 @@ Jan Ploski Jannis Schönleber Jared Van Bortel Jared Van Bortel +Jason C.H Jason McCartney Jason Stillerman Jean-Christophe Hoelt @@ -342,6 +363,7 @@ Jiahao Li Jian Liao JidongZhang-THU <1119708529@qq.com> Jinwoo Jeong <33892306+williamjeong2@users.noreply.github.com> +Jinyang He Jiří Podivín <66251151+jpodivin@users.noreply.github.com> Jiří Sejkora Joan Fontanals @@ -379,6 +401,7 @@ Justine Tunney Juuso Alasuutari KASR Kamil Tomšík +Kante Yin Karol Kontny <82021046+kkontny@users.noreply.github.com> Karsten Weiss Karthick @@ -419,6 +442,7 @@ LoganDark Loïc Carrère LostRuins <39025047+LostRuins@users.noreply.github.com> LostRuins Concedo <39025047+LostRuins@users.noreply.github.com> +Lucas Moura Belo Luciano Luo Tian Lyle Dean @@ -463,6 +487,7 @@ Matthew Tejo Matvey Soloviev Max Krasnyansky Max Krasnyansky +Maxim Evtush <154841002+maximevtush@users.noreply.github.com> Maxime <672982+maximegmd@users.noreply.github.com> Maximilian Winter Meng Zhang @@ -494,6 +519,7 @@ Miwa / Ensan <63481257+ensan-hcl@users.noreply.github.com> Mohammadreza Hendiani Mohammadreza Hendiani Molly Sophia +MoonRide303 <130458190+MoonRide303@users.noreply.github.com> MorganRO8 <47795945+MorganRO8@users.noreply.github.com> Murilo Santana Musab Gultekin @@ -524,6 +550,7 @@ Nikolas <127742645+nneubacher@users.noreply.github.com> Nindaleth Nuno OSecret <135510162+OLSecret@users.noreply.github.com> +Oleksandr Kuvshynov <661042+okuvshynov@users.noreply.github.com> Oleksandr Nikitin Oleksii Maryshchenko Olivier Chafik @@ -533,6 +560,7 @@ PAB Pablo Duboue Pascal Patry Patrice Ferlet +Patrick Peng Paul Tsochantaris Pavel Zloi Pavol Rusnak @@ -549,6 +577,7 @@ Pieter Ouwerkerk Plamen Minev Prashant Vithule <119530321+Vithulep@users.noreply.github.com> Przemysław Pawełczyk +PureJourney Qin Yue Chen <71813199+chenqiny@users.noreply.github.com> Qingyou Meng Qu Zongfu <43257352+yancaoweidaode@users.noreply.github.com> @@ -564,14 +593,17 @@ Rand Xie Randall Fitzgerald Random Fly Reinforce-II +Rémy O Rémy Oudompheng Ren Xuancheng Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Reza Kakhki +Reza Rahemtola <49811529+RezaRahemtola@users.noreply.github.com> RhinoDevel Riccardo Orlando Riceball LEE Rich Dougherty +Richard Richard Kiss Richard Roberson Rick G <26732651+TheFlipbook@users.noreply.github.com> @@ -588,6 +620,7 @@ Robert Sung-wook Shin Robey Holderith Robyn Roger Meier +Rohanjames1997 Roland <14355895+rbur0425@users.noreply.github.com> Romain Biessy Romain D <90720+Artefact2@users.noreply.github.com> @@ -610,6 +643,7 @@ Ryan Landay Ryder Wishart Ryuei Rőczey Barnabás <31726601+An0nie@users.noreply.github.com> +SAMI SRHMorris <69468379+SRHMorris@users.noreply.github.com> SXX SakuraUmi @@ -634,6 +668,8 @@ Shane A Shangning Xu <32517059+xushangning@users.noreply.github.com> Shankar Shanshan Shen <467638484@qq.com> +Shelby Jenkins <47464908+ShelbyJenkins@users.noreply.github.com> +Sheldon Robinson Shijie <821898965@qq.com> Shintarou Okada Shouzheng Liu <61452103+lshzh-ww@users.noreply.github.com> @@ -713,18 +749,24 @@ Victor Nogueira Victor Z. Peng Viet-Anh NGUYEN (Andrew) Vinesh Janarthanan <36610342+VJHack@users.noreply.github.com> +Vitali Lovich +Vivian Vlad Vladimir Vladimir Malyutin +Vladimir Vuksanovic <109677816+vvuksanovic@users.noreply.github.com> Vladimir Zorin VoidIsVoid <343750470@qq.com> Volodymyr Vitvitskyi <72226+signalpillar@users.noreply.github.com> +Wagner Bruna Wang Qin <37098874+wangqin0@users.noreply.github.com> Wang Ran (汪然) WangHaoranRobin <56047610+WangHaoranRobin@users.noreply.github.com> Weird Constructor +Weizhao Ouyang Welby Seely Wentai Zhang +Wilken Gottwalt <12194808+wgottwalt@users.noreply.github.com> WillCorticesAI <150854901+WillCorticesAI@users.noreply.github.com> William Tambellini William Tambellini @@ -816,6 +858,8 @@ chaihahaha chiranko <96988916+chiranko@users.noreply.github.com> clibdev <52199778+clibdev@users.noreply.github.com> clyang +cmdr2 +cmdr2 cocktailpeanut <121128867+cocktailpeanut@users.noreply.github.com> codezjx coezbek @@ -835,6 +879,7 @@ deepdiffuser <112834445+deepdiffuser@users.noreply.github.com> devojony <61173062+devojony@users.noreply.github.com> ditsuke divinity76 +dm4 dm4 dotpy314 <33351922+dotpy314@users.noreply.github.com> drbh @@ -849,6 +894,7 @@ fairydreaming <166155368+fairydreaming@users.noreply.github.com> fengerhu1 <2748250768@qq.com> fj-y-saito <85871716+fj-y-saito@users.noreply.github.com> fraxy-v <65565042+fraxy-v@users.noreply.github.com> +fxzjshm <11426482+fxzjshm@users.noreply.github.com> github-actions[bot] gliptic gn64 @@ -873,6 +919,7 @@ hydai iSma iacore <74560659+iacore@users.noreply.github.com> icppWorld <124377669+icppWorld@users.noreply.github.com> +igardev <49397134+igardev@users.noreply.github.com> igarnier intelmatt <61025942+intelmatt@users.noreply.github.com> iohub @@ -880,6 +927,7 @@ issixx <46835150+issixx@users.noreply.github.com> jacobi petrucciani <8117202+jpetrucciani@users.noreply.github.com> jaime-m-p <167997752+jaime-m-p@users.noreply.github.com> jameswu2014 <545426914@qq.com> +jason_w jdomke <28772296+jdomke@users.noreply.github.com> jiahao su jiez <373447296@qq.com> @@ -891,6 +939,7 @@ jon-chuang <9093549+jon-chuang@users.noreply.github.com> jp-x-g jukofyork <69222624+jukofyork@users.noreply.github.com> junchao-loongson <68935141+junchao-loongson@users.noreply.github.com> +junchao-zhao <68935141+junchao-loongson@users.noreply.github.com> jwj7140 <32943891+jwj7140@users.noreply.github.com> k.h.lai kaizau @@ -925,6 +974,7 @@ ltoniazzi <61414566+ltoniazzi@users.noreply.github.com> luoyu-intel m3ndax maddes8cht <55592906+maddes8cht@users.noreply.github.com> +magicse mahorozte <41834471+mahorozte@users.noreply.github.com> makomk manikbhandari @@ -935,6 +985,7 @@ matt23654 matteo mdrokz mgroeber9110 <45620825+mgroeber9110@users.noreply.github.com> +midnight minarchist mj-shifu <77107165+mj-shifu@users.noreply.github.com> mmyjona @@ -958,10 +1009,12 @@ omahs <73983677+omahs@users.noreply.github.com> oobabooga <112222186+oobabooga@users.noreply.github.com> opparco ostix360 <55257054+ostix360@users.noreply.github.com> +pascal-lc <49066376+pascal-lc@users.noreply.github.com> pculliton peidaqi pengxin99 perserk +petterreinholdtsen piDack <104877312+piDack@users.noreply.github.com> pmysl postmasters @@ -983,6 +1036,7 @@ semidark serhii-nakon <57632032+serhii-nakon@users.noreply.github.com> sharpHL <132747147+sharpHL@users.noreply.github.com> shibe2 +simon886212 <37953122+simon886212@users.noreply.github.com> singularity <12184989+singularity-s0@users.noreply.github.com> sjinzh sjxx <63994076+ylsdamxssjxxdd@users.noreply.github.com> @@ -1000,10 +1054,12 @@ tarcey tc-mb <157115220+tc-mb@users.noreply.github.com> texmex76 <40733439+texmex76@users.noreply.github.com> thement <40525767+thement@users.noreply.github.com> +theraininsky <76763719+theraininsky@users.noreply.github.com> thewh1teagle <61390950+thewh1teagle@users.noreply.github.com> tjohnman toyer <2042519524@qq.com> tslmy +tv1wnd <55383215+tv1wnd@users.noreply.github.com> ubik2 uint256_t uint256_t @@ -1014,6 +1070,7 @@ valiray <133289098+valiray@users.noreply.github.com> vb vik viric +vmobilis <75476228+vmobilis@users.noreply.github.com> vodkaslime <646329483@qq.com> vvhg1 <94630311+vvhg1@users.noreply.github.com> vxiiduu <73044267+vxiiduu@users.noreply.github.com> @@ -1028,6 +1085,8 @@ wzy <32936898+Freed-Wu@users.noreply.github.com> xaedes xaedes xctan +xiaobing318 <71554036+xiaobing318@users.noreply.github.com> +xiaofei xloem <0xloem@gmail.com> yangli2 ymcki <84055651+ymcki@users.noreply.github.com> @@ -1045,4 +1104,3 @@ zrm 蕭澧邦 <45505768+shou692199@users.noreply.github.com> 谢乃闻 Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com> -Jason C.H From 1e2f78a00450593e2dfa458796fcdd9987300dfc Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 9 Mar 2025 19:08:20 +0200 Subject: [PATCH 105/188] server : add speculative decoding presets for FIM (#12287) --- common/arg.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/common/arg.cpp b/common/arg.cpp index 3e549ede0e..b96a5678f7 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2571,5 +2571,43 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_examples({LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"--fim-qwen-7b-spec"}, + string_format("use Qwen 2.5 Coder 7B + 0.5B draft for speculative decoding (note: can download weights from the internet)"), + [](common_params & params) { + params.hf_repo = "ggml-org/Qwen2.5-Coder-7B-Q8_0-GGUF"; + params.hf_file = "qwen2.5-coder-7b-q8_0.gguf"; + params.speculative.hf_repo = "ggml-org/Qwen2.5-Coder-0.5B-Q8_0-GGUF"; + params.speculative.hf_file = "qwen2.5-coder-0.5b-q8_0.gguf"; + params.speculative.n_gpu_layers = 99; + params.port = 8012; + params.n_gpu_layers = 99; + params.flash_attn = true; + params.n_ubatch = 1024; + params.n_batch = 1024; + params.n_ctx = 0; + params.n_cache_reuse = 256; + } + ).set_examples({LLAMA_EXAMPLE_SERVER})); + + add_opt(common_arg( + {"--fim-qwen-14b-spec"}, + string_format("use Qwen 2.5 Coder 14B + 0.5B draft for speculative decoding (note: can download weights from the internet)"), + [](common_params & params) { + params.hf_repo = "ggml-org/Qwen2.5-Coder-14B-Q8_0-GGUF"; + params.hf_file = "qwen2.5-coder-14b-q8_0.gguf"; + params.speculative.hf_repo = "ggml-org/Qwen2.5-Coder-0.5B-Q8_0-GGUF"; + params.speculative.hf_file = "qwen2.5-coder-0.5b-q8_0.gguf"; + params.speculative.n_gpu_layers = 99; + params.port = 8012; + params.n_gpu_layers = 99; + params.flash_attn = true; + params.n_ubatch = 1024; + params.n_batch = 1024; + params.n_ctx = 0; + params.n_cache_reuse = 256; + } + ).set_examples({LLAMA_EXAMPLE_SERVER})); + return ctx_arg; } From 8352cdc87b207a735d34c431a36c425728cb4586 Mon Sep 17 00:00:00 2001 From: tc-mb <157115220+tc-mb@users.noreply.github.com> Date: Mon, 10 Mar 2025 16:33:24 +0800 Subject: [PATCH 106/188] llava : fix bug in minicpm-v code (#11513) * fix bug in minicpm-v code * update readme of minicpm-v --- examples/llava/README-minicpmo2.6.md | 34 +++---- examples/llava/README-minicpmv2.5.md | 88 ++++------------- examples/llava/README-minicpmv2.6.md | 96 ++++--------------- examples/llava/clip.cpp | 1 + examples/llava/minicpmv-cli.cpp | 35 +++++-- .../minicpmv-convert-image-encoder-to-gguf.py | 1 - 6 files changed, 80 insertions(+), 175 deletions(-) diff --git a/examples/llava/README-minicpmo2.6.md b/examples/llava/README-minicpmo2.6.md index 8f591506db..48c4232383 100644 --- a/examples/llava/README-minicpmo2.6.md +++ b/examples/llava/README-minicpmo2.6.md @@ -5,13 +5,25 @@ Currently, this readme only supports minicpm-omni's image capabilities, and we w Download [MiniCPM-o-2_6](https://huggingface.co/openbmb/MiniCPM-o-2_6) PyTorch model from huggingface to "MiniCPM-o-2_6" folder. + +### Build llama.cpp +Readme modification time: 20250206 + +If there are differences in usage, please refer to the official build [documentation](https://github.com/ggerganov/llama.cpp/blob/master/docs/build.md) + Clone llama.cpp: ```bash -git clone git@github.com:OpenBMB/llama.cpp.git +git clone https://github.com/ggerganov/llama.cpp cd llama.cpp -git checkout minicpm-omni ``` +Build llama.cpp using `CMake`: +```bash +cmake -B build +cmake --build build --config Release +``` + + ### Usage of MiniCPM-o 2.6 Convert PyTorch model to gguf files (You can also download the converted [gguf](https://huggingface.co/openbmb/MiniCPM-o-2_6-gguf) by us) @@ -22,25 +34,15 @@ python ./examples/llava/minicpmv-convert-image-encoder-to-gguf.py -m ../MiniCPM- python ./convert_hf_to_gguf.py ../MiniCPM-o-2_6/model # quantize int4 version -./llama-quantize ../MiniCPM-o-2_6/model/ggml-model-f16.gguf ../MiniCPM-o-2_6/model/ggml-model-Q4_K_M.gguf Q4_K_M +./build/bin/llama-quantize ../MiniCPM-o-2_6/model/ggml-model-f16.gguf ../MiniCPM-o-2_6/model/ggml-model-Q4_K_M.gguf Q4_K_M ``` -Build llama.cpp using `CMake`: -https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md - -```bash -cmake -B build -cmake --build build --config Release -``` Inference on Linux or Mac -``` +```bash # run f16 version -./llama-minicpmv-cli -m ../MiniCPM-o-2_6/model/ggml-model-f16.gguf --mmproj ../MiniCPM-o-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" +./build/bin/llama-minicpmv-cli -m ../MiniCPM-o-2_6/model/ggml-model-f16.gguf --mmproj ../MiniCPM-o-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" # run quantized int4 version -./llama-minicpmv-cli -m ../MiniCPM-o-2_6/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-o-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" - -# or run in interactive mode -./llama-minicpmv-cli -m ../MiniCPM-o-2_6/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-o-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -i +./build/bin/llama-minicpmv-cli -m ../MiniCPM-o-2_6/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-o-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" ``` diff --git a/examples/llava/README-minicpmv2.5.md b/examples/llava/README-minicpmv2.5.md index b0e72a0fa7..6bfe7abd16 100644 --- a/examples/llava/README-minicpmv2.5.md +++ b/examples/llava/README-minicpmv2.5.md @@ -4,13 +4,26 @@ Download [MiniCPM-Llama3-V-2_5](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5) PyTorch model from huggingface to "MiniCPM-Llama3-V-2_5" folder. + +### Build llama.cpp +Readme modification time: 20250206 + +If there are differences in usage, please refer to the official build [documentation](https://github.com/ggerganov/llama.cpp/blob/master/docs/build.md) + Clone llama.cpp: ```bash git clone https://github.com/ggml-org/llama.cpp cd llama.cpp ``` -### Usage +Build llama.cpp using `CMake`: +```bash +cmake -B build +cmake --build build --config Release +``` + + +### Usage of MiniCPM-Llama3-V 2.5 Convert PyTorch model to gguf files (You can also download the converted [gguf](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf) by us) @@ -20,80 +33,15 @@ python ./examples/llava/minicpmv-convert-image-encoder-to-gguf.py -m ../MiniCPM- python ./convert_hf_to_gguf.py ../MiniCPM-Llama3-V-2_5/model # quantize int4 version -./llama-quantize ../MiniCPM-Llama3-V-2_5/model/model-8B-F16.gguf ../MiniCPM-Llama3-V-2_5/model/ggml-model-Q4_K_M.gguf Q4_K_M +./build/bin/llama-quantize ../MiniCPM-Llama3-V-2_5/model/model-8B-F16.gguf ../MiniCPM-Llama3-V-2_5/model/ggml-model-Q4_K_M.gguf Q4_K_M ``` -Build for Linux or Mac - -```bash -make -make llama-minicpmv-cli -``` Inference on Linux or Mac -``` +```bash # run f16 version -./llama-minicpmv-cli -m ../MiniCPM-Llama3-V-2_5/model/model-8B-F16.gguf --mmproj ../MiniCPM-Llama3-V-2_5/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" +./build/bin/llama-minicpmv-cli -m ../MiniCPM-Llama3-V-2_5/model/model-8B-F16.gguf --mmproj ../MiniCPM-Llama3-V-2_5/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" # run quantized int4 version -./llama-minicpmv-cli -m ../MiniCPM-Llama3-V-2_5/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-Llama3-V-2_5/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" - -# or run in interactive mode -./llama-minicpmv-cli -m ../MiniCPM-Llama3-V-2_5/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-Llama3-V-2_5/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -i -``` - -### Android - -#### Build on Android device using Termux -We found that build on Android device would bring better runtime performance, so we recommend to build on device. - -[Termux](https://github.com/termux/termux-app#installation) is a terminal app on Android device (no root required). - -Install tools in Termux: -``` -apt update && apt upgrade -y -apt install git make cmake -``` - -It's recommended to move your model inside the `~/` directory for best performance: -``` -cd storage/downloads -mv model.gguf ~/ -``` - -#### Building the Project using Android NDK -Obtain the [Android NDK](https://developer.android.com/ndk) and then build with CMake. - -Execute the following commands on your computer to avoid downloading the NDK to your mobile. Alternatively, you can also do this in Termux: - -```bash -mkdir build-android -cd build-android -export NDK=/your_ndk_path -cmake -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-23 -DCMAKE_C_FLAGS=-march=armv8.4a+dotprod .. -make -``` - -Install [termux](https://github.com/termux/termux-app#installation) on your device and run `termux-setup-storage` to get access to your SD card (if Android 11+ then run the command twice). - -Finally, copy these built `llama` binaries and the model file to your device storage. Because the file permissions in the Android sdcard cannot be changed, you can copy the executable files to the `/data/data/com.termux/files/home/bin` path, and then execute the following commands in Termux to add executable permission: - -(Assumed that you have pushed the built executable files to the /sdcard/llama.cpp/bin path using `adb push`) -``` -$cp -r /sdcard/llama.cpp/bin /data/data/com.termux/files/home/ -$cd /data/data/com.termux/files/home/bin -$chmod +x ./* -``` - -Download models and push them to `/sdcard/llama.cpp/`, then move it to `/data/data/com.termux/files/home/model/` - -``` -$mv /sdcard/llama.cpp/ggml-model-Q4_K_M.gguf /data/data/com.termux/files/home/model/ -$mv /sdcard/llama.cpp/mmproj-model-f16.gguf /data/data/com.termux/files/home/model/ -``` - -Now, you can start chatting: -``` -$cd /data/data/com.termux/files/home/bin -$./llama-minicpmv-cli -m ../model/ggml-model-Q4_K_M.gguf --mmproj ../model/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" +./build/bin/llama-minicpmv-cli -m ../MiniCPM-Llama3-V-2_5/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-Llama3-V-2_5/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" ``` diff --git a/examples/llava/README-minicpmv2.6.md b/examples/llava/README-minicpmv2.6.md index c4be5e5dd6..2df39cdbac 100644 --- a/examples/llava/README-minicpmv2.6.md +++ b/examples/llava/README-minicpmv2.6.md @@ -4,13 +4,25 @@ Download [MiniCPM-V-2_6](https://huggingface.co/openbmb/MiniCPM-V-2_6) PyTorch model from huggingface to "MiniCPM-V-2_6" folder. + +### Build llama.cpp +Readme modification time: 20250206 + +If there are differences in usage, please refer to the official build [documentation](https://github.com/ggerganov/llama.cpp/blob/master/docs/build.md) + Clone llama.cpp: ```bash -git clone git@github.com:OpenBMB/llama.cpp.git +git clone https://github.com/ggerganov/llama.cpp cd llama.cpp -git checkout minicpmv-main ``` +Build llama.cpp using `CMake`: +```bash +cmake -B build +cmake --build build --config Release +``` + + ### Usage of MiniCPM-V 2.6 Convert PyTorch model to gguf files (You can also download the converted [gguf](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) by us) @@ -21,87 +33,15 @@ python ./examples/llava/minicpmv-convert-image-encoder-to-gguf.py -m ../MiniCPM- python ./convert_hf_to_gguf.py ../MiniCPM-V-2_6/model # quantize int4 version -./llama-quantize ../MiniCPM-V-2_6/model/ggml-model-f16.gguf ../MiniCPM-V-2_6/model/ggml-model-Q4_K_M.gguf Q4_K_M +./build/bin/llama-quantize ../MiniCPM-V-2_6/model/ggml-model-f16.gguf ../MiniCPM-V-2_6/model/ggml-model-Q4_K_M.gguf Q4_K_M ``` -Build for Linux or Mac - -```bash -make -make llama-minicpmv-cli -``` Inference on Linux or Mac -``` +```bash # run f16 version -./llama-minicpmv-cli -m ../MiniCPM-V-2_6/model/ggml-model-f16.gguf --mmproj ../MiniCPM-V-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" +./build/bin/llama-minicpmv-cli -m ../MiniCPM-V-2_6/model/ggml-model-f16.gguf --mmproj ../MiniCPM-V-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" # run quantized int4 version -./llama-minicpmv-cli -m ../MiniCPM-V-2_6/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-V-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" - -# or run in interactive mode -./llama-minicpmv-cli -m ../MiniCPM-V-2_6/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-V-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -i -``` - -### Video -Install FFmpeg -``` -brew install ffmpeg -brew install pkg-config -``` - -### Android - -#### Build on Android device using Termux -We found that build on Android device would bring better runtime performance, so we recommend to build on device. - -[Termux](https://github.com/termux/termux-app#installation) is a terminal app on Android device (no root required). - -Install tools in Termux: -``` -apt update && apt upgrade -y -apt install git make cmake -``` - -It's recommended to move your model inside the `~/` directory for best performance: -``` -cd storage/downloads -mv model.gguf ~/ -``` - -#### Building the Project using Android NDK -Obtain the [Android NDK](https://developer.android.com/ndk) and then build with CMake. - -Execute the following commands on your computer to avoid downloading the NDK to your mobile. Alternatively, you can also do this in Termux: - -```bash -mkdir build-android -cd build-android -export NDK=/your_ndk_path -cmake -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-23 -DCMAKE_C_FLAGS=-march=armv8.4a+dotprod .. -make -``` - -Install [termux](https://github.com/termux/termux-app#installation) on your device and run `termux-setup-storage` to get access to your SD card (if Android 11+ then run the command twice). - -Finally, copy these built `llama` binaries and the model file to your device storage. Because the file permissions in the Android sdcard cannot be changed, you can copy the executable files to the `/data/data/com.termux/files/home/bin` path, and then execute the following commands in Termux to add executable permission: - -(Assumed that you have pushed the built executable files to the /sdcard/llama.cpp/bin path using `adb push`) -``` -$cp -r /sdcard/llama.cpp/bin /data/data/com.termux/files/home/ -$cd /data/data/com.termux/files/home/bin -$chmod +x ./* -``` - -Download models and push them to `/sdcard/llama.cpp/`, then move it to `/data/data/com.termux/files/home/model/` - -``` -$mv /sdcard/llama.cpp/ggml-model-Q4_K_M.gguf /data/data/com.termux/files/home/model/ -$mv /sdcard/llama.cpp/mmproj-model-f16.gguf /data/data/com.termux/files/home/model/ -``` - -Now, you can start chatting: -``` -$cd /data/data/com.termux/files/home/bin -$./llama-minicpmv-cli -m ../model/ggml-model-Q4_K_M.gguf --mmproj ../model/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" +./build/bin/llama-minicpmv-cli -m ../MiniCPM-V-2_6/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-V-2_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?" ``` diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 76d4a78520..3f558b7bdb 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -1378,6 +1378,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { LOG_INF("%s: vision_encoder: %d\n", __func__, new_clip->has_vision_encoder); LOG_INF("%s: llava_projector: %d\n", __func__, new_clip->has_llava_projector); LOG_INF("%s: minicpmv_projector: %d\n", __func__, new_clip->has_minicpmv_projector); + LOG_INF("%s: minicpmv_version: %d\n", __func__, new_clip->minicpmv_version); LOG_INF("%s: glm_projector: %d\n", __func__, new_clip->has_glm_projector); LOG_INF("%s: model size: %.2f MB\n", __func__, model_size / 1024.0 / 1024.0); LOG_INF("%s: metadata size: %.2f MB\n", __func__, ggml_get_mem_size(meta) / 1024.0 / 1024.0); diff --git a/examples/llava/minicpmv-cli.cpp b/examples/llava/minicpmv-cli.cpp index 53d902d616..23b3de4db2 100644 --- a/examples/llava/minicpmv-cli.cpp +++ b/examples/llava/minicpmv-cli.cpp @@ -148,19 +148,34 @@ static void process_image(struct llava_context * ctx_llava, struct llava_image_e process_eval_image_embed(ctx_llava, embeds, params->n_batch, &n_past, idx++); eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); if (num_image_embeds > 1) { - size_t num_image_embeds_col = clip_uhd_num_image_embeds_col(ctx_llava->ctx_clip); - eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); - for (size_t i = 0; i < (num_image_embeds-1)/num_image_embeds_col; ++i) { - for (size_t j = 0; j < num_image_embeds_col; ++j) { - eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); - process_eval_image_embed(ctx_llava, embeds, params->n_batch, &n_past, idx++); - eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); - if (j == num_image_embeds_col - 1) { - eval_string(ctx_llava->ctx_llama, std::string("\n").c_str(), params->n_batch, &n_past, false); + if (has_minicpmv_projector == 2) { + size_t num_image_embeds_col = clip_uhd_num_image_embeds_col(ctx_llava->ctx_clip); + eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); + for (size_t i = 0; i < (num_image_embeds-1)/num_image_embeds_col; ++i) { + for (size_t j = 0; j < num_image_embeds_col; ++j) { + eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); + process_eval_image_embed(ctx_llava, embeds, params->n_batch, &n_past, idx++); + eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); + if (j == num_image_embeds_col - 1) { + eval_string(ctx_llava->ctx_llama, std::string("\n").c_str(), params->n_batch, &n_past, false); + } + } + } + eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); + } + else if (has_minicpmv_projector == 3 || has_minicpmv_projector == 4) { + size_t num_image_embeds_col = clip_uhd_num_image_embeds_col(ctx_llava->ctx_clip); + for (size_t i = 0; i < (num_image_embeds-1)/num_image_embeds_col; ++i) { + for (size_t j = 0; j < num_image_embeds_col; ++j) { + eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); + process_eval_image_embed(ctx_llava, embeds, params->n_batch, &n_past, idx++); + eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); + if (j == num_image_embeds_col - 1) { + eval_string(ctx_llava->ctx_llama, std::string("\n").c_str(), params->n_batch, &n_past, false); + } } } } - eval_string(ctx_llava->ctx_llama, std::string("").c_str(), params->n_batch, &n_past, false); } LOG_INF("%s: image token past: %d\n", __func__, n_past); } diff --git a/examples/llava/minicpmv-convert-image-encoder-to-gguf.py b/examples/llava/minicpmv-convert-image-encoder-to-gguf.py index 9b196757f0..cfe0961f98 100644 --- a/examples/llava/minicpmv-convert-image-encoder-to-gguf.py +++ b/examples/llava/minicpmv-convert-image-encoder-to-gguf.py @@ -597,7 +597,6 @@ elif args.minicpmv_projector is not None: fname_middle = "mmproj-" has_text_encoder = False has_minicpmv_projector = True - minicpmv_version = 4 elif args.vision_only: fname_middle = "vision-" has_text_encoder = False From 2b3a25c212f8c4d8a49cec23e03343a7719d51c9 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Mon, 10 Mar 2025 09:44:42 +0000 Subject: [PATCH 107/188] `sampler`: fixes trigger tokens + lazy grammars (fix typo cast from token to string) (#12291) * Fix typo in lazy grammar handling (fixes trigger tokens) Co-authored-by: Georgi Gerganov --------- Co-authored-by: Georgi Gerganov --- examples/server/server.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 8386f4eebb..aec8b9eed0 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -384,8 +384,9 @@ struct server_task { SRV_DBG("Grammar trigger token: %d (`%s`)\n", token, word.c_str()); common_grammar_trigger trigger; trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN; - trigger.value = (llama_token) token; - params.sampling.grammar_triggers.push_back(trigger); + trigger.value = word; + trigger.token = token; + params.sampling.grammar_triggers.push_back(std::move(trigger)); } else { SRV_DBG("Grammar trigger word: `%s`\n", word.c_str()); params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word}); From 87c2630546cd8ccc836ae4c7d87d03fa597d2267 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Mon, 10 Mar 2025 09:45:07 +0000 Subject: [PATCH 108/188] allow missing content in message if tool_calls provided (#12293) --- common/chat.cpp | 29 ++++++++++++++++------------- tests/test-chat.cpp | 15 +++++++++++++++ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 1b10219cca..1b3f286afc 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -60,7 +60,9 @@ std::vector common_chat_msgs_parse_oaicompat(const json & messa } msg.role = message.at("role"); - if (message.contains("content")) { + auto has_content = message.contains("content"); + auto has_tool_calls = message.contains("tool_calls"); + if (has_content) { const auto & content = message.at("content"); if (content.is_string()) { msg.content = content; @@ -81,19 +83,8 @@ std::vector common_chat_msgs_parse_oaicompat(const json & messa } else if (!content.is_null()) { throw std::runtime_error("Invalid 'content' type: expected string or array, got " + content.dump() + " (ref: https://github.com/ggml-org/llama.cpp/issues/8367)"); } - } else { - throw std::runtime_error("Expected 'content' (ref: https://github.com/ggml-org/llama.cpp/issues/8367)"); } - if (message.contains("reasoning_content")) { - msg.reasoning_content = message.at("reasoning_content"); - } - if (message.contains("name")) { - msg.tool_name = message.at("name"); - } - if (message.contains("tool_call_id")) { - msg.tool_call_id = message.at("tool_call_id"); - } - if (message.contains("tool_calls")) { + if (has_tool_calls) { for (const auto & tool_call : message.at("tool_calls")) { common_chat_tool_call tc; if (!tool_call.contains("type")) { @@ -118,6 +109,18 @@ std::vector common_chat_msgs_parse_oaicompat(const json & messa msg.tool_calls.push_back(tc); } } + if (!has_content && !has_tool_calls) { + throw std::runtime_error("Expected 'content' or 'tool_calls' (ref: https://github.com/ggml-org/llama.cpp/issues/8367 & https://github.com/ggml-org/llama.cpp/issues/12279)"); + } + if (message.contains("reasoning_content")) { + msg.reasoning_content = message.at("reasoning_content"); + } + if (message.contains("name")) { + msg.tool_name = message.at("name"); + } + if (message.contains("tool_call_id")) { + msg.tool_call_id = message.at("tool_call_id"); + } msgs.push_back(msg); } diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 35a307c632..35c7ee34e3 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -480,6 +480,21 @@ static void test_msgs_oaicompat_json_conversion() { "]" ), common_chat_msgs_to_json_oaicompat({message_assist_call_python}).dump(2)); + + auto res = common_chat_msgs_parse_oaicompat(json::parse("[{\"role\": \"assistant\", \"tool_calls\": []}]")); + assert_equals(1, res.size()); + assert_equals(res[0].role, "assistant"); + assert_equals(true, res[0].content.empty()); + assert_equals(true, res[0].tool_calls.empty()); + + try { + common_chat_msgs_parse_oaicompat(json::parse("[{\"role\": \"assistant\"}]")); + throw std::runtime_error("Expected exception"); + } catch (const std::exception & e) { + if (std::string(e.what()).find("'content'") == std::string::npos) { + throw std::runtime_error("Expected exception about missing 'content'"); + } + } } static void test_tools_oaicompat_json_conversion() { From be421fc429795d135786f5a0e489709220a9c43a Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Mon, 10 Mar 2025 09:45:29 +0000 Subject: [PATCH 109/188] `tool-call`: ensure there's always a non-empty tool call id (#12292) --- examples/server/server.cpp | 5 ++++- examples/server/tests/unit/test_tool_call.py | 3 +++ examples/server/utils.hpp | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index aec8b9eed0..8cb8d0033f 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -751,7 +751,10 @@ struct server_task_result_cmpl_final : server_task_result { {"name", tc.name}, {"arguments", tc.arguments}, }}, - {"id", tc.id}, + // Some templates generate and require an id (sometimes in a very specific format, e.g. Mistral Nemo). + // We only generate a random id for the ones that don't generate one by themselves + // (they also won't get to see it as their template likely doesn't use it, so it's all for the client) + {"id", tc.id.empty() ? gen_tool_call_id() : tc.id}, }); } message["tool_calls"] = tool_calls; diff --git a/examples/server/tests/unit/test_tool_call.py b/examples/server/tests/unit/test_tool_call.py index 25bddbaee7..569c2a1f8e 100755 --- a/examples/server/tests/unit/test_tool_call.py +++ b/examples/server/tests/unit/test_tool_call.py @@ -92,6 +92,7 @@ def do_test_completion_with_required_tool_tiny(server: ServerProcess, tool: dict assert tool_calls and len(tool_calls) == 1, f'Expected 1 tool call in {choice["message"]}' tool_call = tool_calls[0] assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}' + assert len(tool_call.get("id", "")) > 0, f'Expected non empty tool call id in {tool_call}' expected_function_name = "python" if tool["type"] == "code_interpreter" else tool["function"]["name"] assert expected_function_name == tool_call["function"]["name"] actual_arguments = tool_call["function"]["arguments"] @@ -373,6 +374,7 @@ def do_test_weather(server: ServerProcess, **kwargs): tool_call = tool_calls[0] # assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}' assert tool_call["function"]["name"] == WEATHER_TOOL["function"]["name"], f'Expected weather tool call, got {tool_call["function"]["name"]}' + assert len(tool_call.get("id", "")) > 0, f'Expected non empty tool call id in {tool_call}' actual_arguments = json.loads(tool_call["function"]["arguments"]) assert 'location' in actual_arguments, f"location not found in {json.dumps(actual_arguments)}" location = actual_arguments["location"] @@ -596,6 +598,7 @@ def do_test_hello_world(server: ServerProcess, **kwargs): tool_call = tool_calls[0] # assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}' assert tool_call["function"]["name"] == PYTHON_TOOL["function"]["name"] + assert len(tool_call.get("id", "")) > 0, f'Expected non empty tool call id in {tool_call}' actual_arguments = json.loads(tool_call["function"]["arguments"]) assert 'code' in actual_arguments, f"code not found in {json.dumps(actual_arguments)}" code = actual_arguments["code"] diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 393e3927c7..36ad276fd3 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -435,6 +435,10 @@ static std::string gen_chatcmplid() { return "chatcmpl-" + random_string(); } +static std::string gen_tool_call_id() { + return random_string(); +} + // // other common utils // From 4e39a3c332f84b890e91353ec448502cb8373a6f Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Mon, 10 Mar 2025 10:59:03 +0000 Subject: [PATCH 110/188] `server`: extract tags from qwq outputs (#12297) * extract tags from qwq outputs * const for all static regexes in chat.cpp --- common/chat.cpp | 282 +++++++++++++++++++++++--------------------- common/chat.h | 1 + tests/test-chat.cpp | 13 ++ 3 files changed, 162 insertions(+), 134 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 1b3f286afc..62ca26ad76 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -445,6 +445,7 @@ std::string common_chat_format_name(common_chat_format format) { case COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2: return "Functionary v3.2"; case COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1: return "Functionary v3.1 Llama 3.1"; case COMMON_CHAT_FORMAT_HERMES_2_PRO: return "Hermes 2 Pro"; + case COMMON_CHAT_FORMAT_HERMES_2_PRO_EXTRACT_REASONING: return "Hermes 2 Pro (extract reasoning)"; case COMMON_CHAT_FORMAT_COMMAND_R7B: return "Command R7B"; case COMMON_CHAT_FORMAT_COMMAND_R7B_EXTRACT_REASONING: return "Command R7B (extract reasoning)"; default: @@ -878,9 +879,9 @@ static common_chat_params common_chat_params_init_command_r7b(const common_chat_ return data; } static common_chat_msg common_chat_parse_command_r7b(const std::string & input, bool extract_reasoning) { - static std::regex thought_regex("(<\\|START_THINKING\\|>([\\s\\S]*?)<\\|END_THINKING\\|>)([\\s\\S]*)"); - static std::regex action_regex("<\\|START_ACTION\\|>([\\s\\S]*?)<\\|END_ACTION\\|>"); - static std::regex response_regex("(?:<\\|START_RESPONSE\\|>)?([\\s\\S]*?)<\\|END_RESPONSE\\|>"); + static const std::regex thought_regex("(<\\|START_THINKING\\|>([\\s\\S]*?)<\\|END_THINKING\\|>)([\\s\\S]*)"); + static const std::regex action_regex("<\\|START_ACTION\\|>([\\s\\S]*?)<\\|END_ACTION\\|>"); + static const std::regex response_regex("(?:<\\|START_RESPONSE\\|>)?([\\s\\S]*?)<\\|END_RESPONSE\\|>"); std::smatch match; @@ -1012,10 +1013,10 @@ static common_chat_params common_chat_params_init_llama_3_1_tool_calls(const com } static common_chat_msg common_chat_parse_llama_3_1(const std::string & input, bool with_builtin_tools = false) { // TODO: tighten & simplify the parser, don't accept leading text context. - static std::regex function_regex( + static const std::regex function_regex( "\\s*\\{\\s*(?:\"type\"\\s*:\\s*\"function\"\\s*,\\s*)?\"name\"\\s*:\\s*\"([^\"]+)\"\\s*,\\s*\"parameters\"\\s*: "); - static std::regex close_regex("\\}\\s*"); - static std::regex builtin_call_regex("<\\|python_tag\\|>\\s*([^.(]+)\\s*\\.\\s*call\\s*\\(\\s*([\\w]+)\\s*=\\s*([\\s\\S]*?)\\)"); + static const std::regex close_regex("\\}\\s*"); + static const std::regex builtin_call_regex("<\\|python_tag\\|>\\s*([^.(]+)\\s*\\.\\s*call\\s*\\(\\s*([\\w]+)\\s*=\\s*([\\s\\S]*?)\\)"); if (with_builtin_tools) { std::smatch match; @@ -1105,34 +1106,42 @@ static common_chat_params common_chat_params_init_deepseek_r1(const common_chat_ data.format = inputs.extract_reasoning ? COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING : COMMON_CHAT_FORMAT_DEEPSEEK_R1; return data; } -static common_chat_msg common_chat_parse_deepseek_r1(const std::string & input, bool extract_reasoning) { - static std::regex function_regex("<｜tool▁call▁begin｜>function<｜tool▁sep｜>([^\n]+)\n```json\n"); - static std::regex close_regex("```[\\s\\r\\n]*<｜tool▁call▁end｜>"); - static std::regex reasoning_content_regex("((?:)?([\\s\\S\\r\\n]*?))?([\\s\\S\\r\\n]*)"); - static std::regex tool_calls_regex("[\\s\\r\\n]*(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>)([\\s\\S\\r\\n]*?)<｜tool▁calls▁end｜>"); - common_chat_msg msg; - msg.role = "assistant"; +static common_chat_msg handle_think_tag_prelude(const std::string & input, bool extract_reasoning, const std::function & rest_parser) { std::smatch match; + static const std::regex reasoning_content_regex("((?:)?([\\s\\S\\r\\n]*?))?([\\s\\S\\r\\n]*)"); if (std::regex_match(input, match, reasoning_content_regex)) { - std::string rest; + auto rest = match[3].str(); + auto msg = rest_parser(rest); + auto reasoning_content = string_strip(match[2].str()); if (extract_reasoning) { - msg.reasoning_content = string_strip(match[2].str()); - } else { - msg.content = match[1].str(); + msg.reasoning_content = reasoning_content; + } else if (!reasoning_content.empty()) { + std::ostringstream content; + content << "" << reasoning_content << "" << msg.content; + msg.content = content.str(); } - rest = match[3].str(); + return msg; + } + return rest_parser(input); +} +static common_chat_msg common_chat_parse_deepseek_r1(const std::string & input, bool extract_reasoning) { + return handle_think_tag_prelude(input, extract_reasoning, [](const std::string & input) { + static const std::regex function_regex("<｜tool▁call▁begin｜>function<｜tool▁sep｜>([^\n]+)\n```json\n"); + static const std::regex close_regex("```[\\s\\r\\n]*<｜tool▁call▁end｜>"); + static const std::regex tool_calls_regex("[\\s\\r\\n]*(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>)([\\s\\S\\r\\n]*?)<｜tool▁calls▁end｜>"); - if (std::regex_search(rest, match, tool_calls_regex)) { + common_chat_msg msg; + msg.role = "assistant"; + std::smatch match; + if (std::regex_search(input, match, tool_calls_regex)) { auto tool_calls = match[1].str(); auto msg2 = parse_json_tool_calls(tool_calls, std::nullopt, function_regex, close_regex); msg.tool_calls = std::move(msg2.tool_calls); } else { - msg.content += std::string(rest.begin() + rest.find_first_not_of(" \r\n"), rest.end()); + msg.content = input; } - } else { - msg.content = input; - } - return msg; + return msg; + }); } static common_chat_params common_chat_params_init_firefunction_v2(const common_chat_template & tmpl, const struct templates_params & inputs) { @@ -1237,8 +1246,8 @@ static common_chat_params common_chat_params_init_functionary_v3_2(const common_ } static common_chat_msg common_chat_parse_functionary_v3_2(const std::string & input) { - static std::regex function_regex(R"((?:>>>)?(?:assistant<|end_header_id|>\n)?(\w+)\n)"); - static std::regex close_regex(R"($|(?=>>>))"); + static const std::regex function_regex(R"((?:>>>)?(?:assistant<|end_header_id|>\n)?(\w+)\n)"); + static const std::regex close_regex(R"($|(?=>>>))"); std::string content; auto it = input.begin(); @@ -1327,7 +1336,7 @@ static common_chat_params common_chat_params_init_functionary_v3_1_llama_3_1(con } static common_chat_msg common_chat_parse_functionary_v3_1_llama_3_1(const std::string & input) { // This version of Functionary still supports the llama 3.1 tool call format for the python tool. - static std::regex python_tag_regex(R"(<\|python_tag\|>([\s\S\n]*)$)"); + static const std::regex python_tag_regex(R"(<\|python_tag\|>([\s\S\n]*)$)"); std::smatch match; if (std::regex_search(input, match, python_tag_regex)) { auto code = match[1].str(); @@ -1341,8 +1350,8 @@ static common_chat_msg common_chat_parse_functionary_v3_1_llama_3_1(const std::s }); return msg; } - static std::regex function_regex(R"()"); - static std::regex close_regex(R"()"); + static const std::regex function_regex(R"()"); + static const std::regex close_regex(R"()"); // TODO: tighten & simplify. return parse_json_tool_calls(input, std::nullopt, function_regex, close_regex); } @@ -1409,6 +1418,8 @@ static common_chat_params common_chat_params_init_hermes_2_pro(const common_chat "(?:```(?:json|xml)?\n\\s*)?(?:|||)?\\s*\\{\\s*\"", //name\"\\s*:\\s*\"" + escaped_name + "\"", }); data.preserved_tokens = { + "", + "", "", "", "" // match 2 (open_tag) - "|" - "|" - "|" - "|" - "|" - "|" - "|" - ")?" - "(\\s*\\{\\s*\"name\"\\s*:[\\s\\S]*)" // match 3 (named tool call + rest) - ")" - "|" - "(?:]+)>" // match 4 (function name) - "|)" // match 5 (function name again) - "([\\s\\S]*)" // match 6 (function arguments + rest)})" - ); +static common_chat_msg common_chat_parse_hermes_2_pro(const std::string& input, bool extract_reasoning) { + return handle_think_tag_prelude(input, extract_reasoning, [](const std::string & input) { + static const std::regex open_regex( + "(?:" + "(```(?:xml|json)?\\n\\s*)?" // match 1 (block_start) + "(" // match 2 (open_tag) + "|" + "|" + "|" + "|" + "|" + "|" + "|" + ")?" + "(\\s*\\{\\s*\"name\"\\s*:[\\s\\S]*)" // match 3 (named tool call + rest) + ")" + "|" + "(?:]+)>" // match 4 (function name) + "|)" // match 5 (function name again) + "([\\s\\S]*)" // match 6 (function arguments + rest)})" + ); - try { + try { + common_chat_msg msg; + msg.role = "assistant"; - common_chat_msg msg; - msg.role = "assistant"; + std::string::const_iterator it = input.begin(); + const std::string::const_iterator end = input.end(); + std::smatch match; - std::string::const_iterator it = input.begin(); - const std::string::const_iterator end = input.end(); - std::smatch match; + while (it != end) { + if (std::regex_search(it, end, match, open_regex)) { + // Add content before the match + msg.content += std::string(it, match[0].first); - while (it != end) { - if (std::regex_search(it, end, match, open_regex)) { - // Add content before the match - msg.content += std::string(it, match[0].first); + auto block_start = match[1].str(); + std::string block_end = block_start.empty() ? "" : "```"; - auto block_start = match[1].str(); - std::string block_end = block_start.empty() ? "" : "```"; + auto open_tag = match[2].str(); + std::string close_tag; - auto open_tag = match[2].str(); - std::string close_tag; + if (match[3].matched) { + close_tag = open_tag.empty() ? "" : ""; + // Start parsing from after the opening tags + auto json_it = match[6].first; + json arguments; + if (parse_json(json_it, end, arguments)) { + msg.tool_calls.emplace_back(process_tool_call({ + {"name", function_name}, + {"arguments", arguments}, + })); + it = json_it; // Move iterator past parsed JSON + + // Handle close tags + consume_spaces(it, end); + if (!close_tag.empty() && !parse_literal(it, end, close_tag)) { + throw std::runtime_error("Failed to parse closing tag"); + } + consume_spaces(it, end); + if (!block_end.empty() && !parse_literal(it, end, block_end)) { + throw std::runtime_error("Failed to parse block end"); + } + consume_spaces(it, end); + } else { + // Not a valid tool call, treat as content + msg.content += std::string(match[0].first, match[0].second); + it = match[0].second; + } } } else { - auto function_name = match[4].str(); - if (function_name.empty()) { - function_name = match[5].str(); - } - GGML_ASSERT(!function_name.empty()); - - close_tag = ""; - // Start parsing from after the opening tags - auto json_it = match[6].first; - json arguments; - if (parse_json(json_it, end, arguments)) { - msg.tool_calls.emplace_back(process_tool_call({ - {"name", function_name}, - {"arguments", arguments}, - })); - it = json_it; // Move iterator past parsed JSON - - // Handle close tags - consume_spaces(it, end); - if (!close_tag.empty() && !parse_literal(it, end, close_tag)) { - throw std::runtime_error("Failed to parse closing tag"); - } - consume_spaces(it, end); - if (!block_end.empty() && !parse_literal(it, end, block_end)) { - throw std::runtime_error("Failed to parse block end"); - } - consume_spaces(it, end); - } else { - // Not a valid tool call, treat as content - msg.content += std::string(match[0].first, match[0].second); - it = match[0].second; - } + // Add remaining content + msg.content += std::string(it, end); + break; } - } else { - // Add remaining content - msg.content += std::string(it, end); - break; } + return msg; + } catch (const std::exception & e) { + LOG_ERR("Failed to parse hermes 2 pro input: %s\n", e.what()); + common_chat_msg msg; + msg.role = "assistant"; + msg.content = input; + return msg; } - return msg; - } catch (const std::exception & e) { - LOG_ERR("Failed to parse hermes 2 pro input: %s\n", e.what()); - common_chat_msg msg; - msg.role = "assistant"; - msg.content = input; - return msg; - } + }); } static common_chat_params common_chat_params_init_without_tools(const common_chat_template & tmpl, const struct templates_params & inputs) { @@ -1609,6 +1621,11 @@ static common_chat_params common_chat_templates_apply_jinja( return common_chat_params_init_command_r7b(tmpl, params); } + // Hermes 2/3 Pro, Qwen 2.5 Instruct (w/ tools) + if (src.find("") != std::string::npos && params.json_schema.is_null()) { + return common_chat_params_init_hermes_2_pro(tmpl, params); + } + // Use generic handler when mixing tools + JSON schema. // TODO: support that mix in handlers below. if ((params.tools.is_array() && params.json_schema.is_object())) { @@ -1630,11 +1647,6 @@ static common_chat_params common_chat_templates_apply_jinja( return common_chat_params_init_without_tools(tmpl, params); } - // Hermes 2/3 Pro, Qwen 2.5 Instruct (w/ tools) - if (src.find("") != std::string::npos) { - return common_chat_params_init_hermes_2_pro(tmpl, params); - } - // Functionary v3.1 (w/ tools) if (src.find("<|start_header_id|>") != std::string::npos && src.find("I'm thinkingHello, world!\nWhat's up?", + COMMON_CHAT_FORMAT_HERMES_2_PRO)); + assert_msg_equals(message_assist_thoughts_unparsed_think, + common_chat_parse("I'm thinkingHello, world!\nWhat's up?", + COMMON_CHAT_FORMAT_HERMES_2_PRO)); + assert_msg_equals(message_assist_thoughts, + common_chat_parse("I'm thinkingHello, world!\nWhat's up?", + COMMON_CHAT_FORMAT_HERMES_2_PRO_EXTRACT_REASONING)); + assert_msg_equals(message_assist_thoughts, + common_chat_parse("I'm thinkingHello, world!\nWhat's up?", + COMMON_CHAT_FORMAT_HERMES_2_PRO_EXTRACT_REASONING)); + test_templates(tmpls.get(), end_tokens, message_assist, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false); test_templates(tmpls.get(), end_tokens, message_assist_call, tools, "\n" From 6ef79a67caf1159b0150b44ee80888c7ec98b83f Mon Sep 17 00:00:00 2001 From: marcoStocchi Date: Mon, 10 Mar 2025 12:34:13 +0100 Subject: [PATCH 111/188] common : refactor '-o' option (#12278) As discussed in PR 'llama-tts : add -o option' (#12042): * common_params : 'out_file' string is the only output file name parameter left in common_params. It's intended to be used in all example programs implementing an '-o' option. * cvector-generator, export-lora, imatrix : default output filenames moved from 'common_params' to the 'main()' of each example program. --- common/arg.cpp | 9 +-------- common/common.h | 8 +++----- examples/cvector-generator/cvector-generator.cpp | 4 +++- examples/export-lora/export-lora.cpp | 6 ++++-- examples/imatrix/imatrix.cpp | 5 ++--- 5 files changed, 13 insertions(+), 19 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index b96a5678f7..8531f0871d 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1867,16 +1867,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_PASSKEY})); add_opt(common_arg( {"-o", "--output", "--output-file"}, "FNAME", - string_format("output file (default: '%s')", - ex == LLAMA_EXAMPLE_EXPORT_LORA - ? params.lora_outfile.c_str() - : ex == LLAMA_EXAMPLE_CVECTOR_GENERATOR - ? params.cvector_outfile.c_str() - : params.out_file.c_str()), + string_format("output file (default: '%s')", params.out_file.c_str()), [](common_params & params, const std::string & value) { params.out_file = value; - params.cvector_outfile = value; - params.lora_outfile = value; } ).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA})); add_opt(common_arg( diff --git a/common/common.h b/common/common.h index 733f7f1c8d..1c0f199774 100644 --- a/common/common.h +++ b/common/common.h @@ -407,8 +407,6 @@ struct common_params { int32_t i_pos = -1; // position of the passkey in the junk text // imatrix params - std::string out_file = "imatrix.dat"; // save the resulting imatrix to this file - int32_t n_out_freq = 10; // output the imatrix every n_out_freq iterations int32_t n_save_freq = 0; // save the imatrix every n_save_freq iterations int32_t i_chunk = 0; // start processing from this chunk @@ -420,16 +418,16 @@ struct common_params { int n_pca_batch = 100; int n_pca_iterations = 1000; dimre_method cvector_dimre_method = DIMRE_METHOD_PCA; - std::string cvector_outfile = "control_vector.gguf"; std::string cvector_positive_file = "examples/cvector-generator/positive.txt"; std::string cvector_negative_file = "examples/cvector-generator/negative.txt"; bool spm_infill = false; // suffix/prefix/middle pattern for infill - std::string lora_outfile = "ggml-lora-merged-f16.gguf"; - // batched-bench params bool batched_bench_output_jsonl = false; + + // common params + std::string out_file; // output filename for all example programs }; // call once at the start of a program if it uses libcommon diff --git a/examples/cvector-generator/cvector-generator.cpp b/examples/cvector-generator/cvector-generator.cpp index 413b71d34c..c72528dac3 100644 --- a/examples/cvector-generator/cvector-generator.cpp +++ b/examples/cvector-generator/cvector-generator.cpp @@ -394,6 +394,8 @@ static int prepare_entries(common_params & params, train_context & ctx_train) { int main(int argc, char ** argv) { common_params params; + params.out_file = "control_vector.gguf"; + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_CVECTOR_GENERATOR, print_usage)) { return 1; } @@ -498,7 +500,7 @@ int main(int argc, char ** argv) { } // write output vectors to gguf - export_gguf(ctx_train.v_final, params.cvector_outfile, model_hint); + export_gguf(ctx_train.v_final, params.out_file, model_hint); llama_backend_free(); diff --git a/examples/export-lora/export-lora.cpp b/examples/export-lora/export-lora.cpp index 91238e4beb..e7d0fbfffe 100644 --- a/examples/export-lora/export-lora.cpp +++ b/examples/export-lora/export-lora.cpp @@ -413,20 +413,22 @@ static void print_usage(int, char ** argv) { int main(int argc, char ** argv) { common_params params; + params.out_file = "ggml-lora-merged-f16.gguf"; + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_EXPORT_LORA, print_usage)) { return 1; } g_verbose = (params.verbosity > 1); try { - lora_merge_ctx ctx(params.model, params.lora_adapters, params.lora_outfile, params.cpuparams.n_threads); + lora_merge_ctx ctx(params.model, params.lora_adapters, params.out_file, params.cpuparams.n_threads); ctx.run_merge(); } catch (const std::exception & err) { fprintf(stderr, "%s\n", err.what()); exit(EXIT_FAILURE); } - printf("done, output file is %s\n", params.lora_outfile.c_str()); + printf("done, output file is %s\n", params.out_file.c_str()); return 0; } diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp index 4edc0bfacf..91649c4506 100644 --- a/examples/imatrix/imatrix.cpp +++ b/examples/imatrix/imatrix.cpp @@ -206,9 +206,6 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * void IMatrixCollector::save_imatrix(int ncall) const { auto fname = m_params.out_file; - if (fname.empty()) { - fname = "imatrix.dat"; - } if (ncall > 0) { fname += ".at_"; @@ -583,6 +580,8 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params) { int main(int argc, char ** argv) { common_params params; + params.out_file = "imatrix.dat" ; + params.n_ctx = 512; params.logits_all = true; params.escape = false; From e128a1bf5b65741b485dd094a4201264d0580d68 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 10 Mar 2025 14:07:15 +0200 Subject: [PATCH 112/188] tests : fix test-quantize-fns to init the CPU backend (#12306) ggml-ci --- tests/test-quantize-fns.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index c77c8ed138..037c0582bb 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -120,13 +120,7 @@ int main(int argc, char * argv[]) { generate_data(0.0, test_data.size(), test_data.data()); generate_data(1.0, test_data2.size(), test_data2.data()); - // Initialize GGML, ensures float conversion tables are initialized - struct ggml_init_params ggml_params = { - /* .mem_size = */ 1*1024, - /* .mem_buffer = */ NULL, - /* .no_alloc = */ true, - }; - struct ggml_context * ctx = ggml_init(ggml_params); + ggml_cpu_init(); int num_failed = 0; bool failed = false; @@ -188,7 +182,5 @@ int main(int argc, char * argv[]) { printf("%d tests failed\n", num_failed); } - ggml_free(ctx); - return num_failed > 0; } From 89b2b56e8658800375a8314200870b1ad4208a0b Mon Sep 17 00:00:00 2001 From: John Bean <113509988+johnbean393@users.noreply.github.com> Date: Mon, 10 Mar 2025 22:13:09 +0800 Subject: [PATCH 113/188] readme: added Sidekick to available UIs (#12311) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e371c44ed1..1eec944f27 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,7 @@ Instructions for adding support for new models: [HOWTO-add-model.md](docs/develo - [eva](https://github.com/ylsdamxssjxxdd/eva) (MIT) - [iohub/collama](https://github.com/iohub/coLLaMA) (Apache-2.0) - [janhq/jan](https://github.com/janhq/jan) (AGPL) +- [johnbean393/Sidekick](https://github.com/johnbean393/Sidekick) (MIT) - [KanTV](https://github.com/zhouwg/kantv?tab=readme-ov-file) (Apache-2.0) - [KodiBot](https://github.com/firatkiral/kodibot) (GPL) - [llama.vim](https://github.com/ggml-org/llama.vim) (MIT) From 8acdacb3ea00697477eb019efbb6fc183371055c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henry=20Linjam=C3=A4ki?= Date: Mon, 10 Mar 2025 18:57:00 +0200 Subject: [PATCH 114/188] opencl: use OpenCL C standard supported by the device (#12221) This patch nudges the llama.cpp a bit to be supported on PoCL which doesn't support OpenCL C CL2.0. The issue is solved by querying the device for the supported OpenCL C versions and using the highest one available. --- ggml/CMakeLists.txt | 2 + ggml/src/ggml-opencl/CMakeLists.txt | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 176 ++++++++++++++++++++------- 3 files changed, 136 insertions(+), 43 deletions(-) diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 412d294dca..9a4ee4992d 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -195,6 +195,8 @@ option(GGML_OPENCL "ggml: use OpenCL" option(GGML_OPENCL_PROFILING "ggml: use OpenCL profiling (increases overhead)" OFF) option(GGML_OPENCL_EMBED_KERNELS "ggml: embed kernels" ON) option(GGML_OPENCL_USE_ADRENO_KERNELS "ggml: use optimized kernels for Adreno" ON) +set (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING + "gmml: OpenCL API version to target") # toolchain for vulkan-shaders-gen set (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen") diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 45328a6579..59a208fe9c 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -15,6 +15,7 @@ if (GGML_OPENCL_PROFILING) endif () add_compile_definitions(GGML_OPENCL_SOA_Q) +add_compile_definitions(GGML_OPENCL_TARGET_VERSION=${GGML_OPENCL_TARGET_VERSION}) if (GGML_OPENCL_USE_ADRENO_KERNELS) message(STATUS "OpenCL will use matmul kernels optimized for Adreno") diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index b85a895c45..14d9934fb1 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1,4 +1,4 @@ -#define CL_TARGET_OPENCL_VERSION 220 +#define CL_TARGET_OPENCL_VERSION GGML_OPENCL_TARGET_VERSION #define CL_USE_DEPRECATED_OPENCL_1_2_APIS // suppress warnings in CL headers for GCC and Clang @@ -25,6 +25,8 @@ #include #include #include +#include +#include #undef MIN #undef MAX @@ -62,6 +64,97 @@ enum ADRENO_GPU_GEN { X1E, }; +struct ggml_cl_version { + cl_uint major = 0; + cl_uint minor = 0; +}; + +// Parses a version string of form "XX.YY ". On an error returns ggml_cl_version with all zeroes. +static ggml_cl_version parse_cl_version(std::string_view str) { + size_t major_str_begin = 0; + size_t major_str_end = str.find(".", major_str_begin); + if (major_str_end == std::string::npos) { + return {}; + } + + size_t minor_str_begin = major_str_end + 1; + size_t minor_str_end = str.find(" ", minor_str_begin); + if (minor_str_end == std::string::npos) { + return {}; + } + + cl_uint version_major; + if (std::from_chars(str.data() + major_str_begin, str.data() + major_str_end, version_major).ec != std::errc{}) { + return {}; + } + + cl_uint version_minor; + if (std::from_chars(str.data() + minor_str_begin, str.data() + minor_str_end, version_minor).ec != std::errc{}) { + return {}; + } + return { version_major, version_minor }; +} + +// Returns OpenCL platform's version. On an error returns ggml_cl_version with all zeroes. +static ggml_cl_version get_opencl_platform_version(cl_platform_id platform) { + size_t param_size; + CL_CHECK(clGetPlatformInfo(platform, CL_PLATFORM_VERSION, 0, nullptr, ¶m_size)); + std::unique_ptr param_storage(new char[param_size]); + CL_CHECK(clGetPlatformInfo(platform, CL_PLATFORM_VERSION, param_size, param_storage.get(), nullptr)); + + auto param_value = std::string_view(param_storage.get(), param_size); + const std::string version_prefix = "OpenCL "; // Suffix: "XX.YY " + if (param_value.find(version_prefix) != 0) { + return {}; + } + param_value.remove_prefix(version_prefix.length()); + return parse_cl_version(param_value); +} + +// Return a version to use in OpenCL C compilation. On an error returns ggml_cl_version with all zeroes. +static ggml_cl_version get_opencl_c_version(ggml_cl_version platform_version, cl_device_id device) { + size_t param_size; + +#if CL_TARGET_OPENCL_VERSION >= 300 + if (platform_version.major >= 3) { + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size)); + if (!param_size) { + return {}; + } + + std::unique_ptr versions(new cl_name_version[param_size]); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr)); + unsigned versions_count = param_size / sizeof(cl_name_version); + + cl_version version_max = 0; + for (unsigned i = 0; i < versions_count; i++) { + version_max = std::max(versions[i].version, version_max); + } + + return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; + } +#else + GGML_UNUSED(platform_version); +#endif // CL_TARGET_OPENCL_VERSION >= 300 + + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size)); + if (!param_size) { + return {}; + } + + std::unique_ptr param_storage(new char[param_size]); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr)); + auto param_value = std::string_view(param_storage.get(), param_size); + + const std::string version_prefix = "OpenCL C "; // Suffix: "XX.YY " + if (param_value.find(version_prefix) != 0) { + return {}; + } + param_value.remove_prefix(version_prefix.length()); + + return parse_cl_version(param_value); +} + static ADRENO_GPU_GEN get_adreno_gpu_gen(const char *device_name) { if (strstr(device_name, "730") || strstr(device_name, "740") || @@ -470,16 +563,11 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { // A local ref of cl_device_id for convenience cl_device_id device = backend_ctx->device; - // Check device OpenCL version, OpenCL 2.0 or above is required - size_t device_ver_str_size; - clGetDeviceInfo(device, CL_DEVICE_VERSION, 0, NULL, &device_ver_str_size); - char *device_ver_buffer = (char *)alloca(device_ver_str_size + 1); - clGetDeviceInfo(device, CL_DEVICE_VERSION, device_ver_str_size, device_ver_buffer, NULL); - device_ver_buffer[device_ver_str_size] = '\0'; - GGML_LOG_INFO("ggml_opencl: device OpenCL version: %s\n", device_ver_buffer); + ggml_cl_version platform_version = get_opencl_platform_version(default_device->platform->id); - if (strstr(device_ver_buffer, "OpenCL 2") == NULL && - strstr(device_ver_buffer, "OpenCL 3") == NULL) { + // Check device OpenCL version, OpenCL 2.0 or above is required + ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, device); + if (opencl_c_version.major < 2) { GGML_LOG_ERROR("ggml_opencl: OpenCL 2.0 or above is required\n"); return backend_ctx; } @@ -516,8 +604,7 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { // If OpenCL 3.0 is supported, then check for cl_khr_subgroups, which becomes // optional in OpenCL 3.0 (cl_khr_subgroup is mandatory in OpenCL 2.x) - if (strstr(device_ver_buffer, "OpenCL 3") && - strstr(ext_buffer, "cl_khr_subgroups") == NULL && + if (opencl_c_version.major == 3 && strstr(ext_buffer, "cl_khr_subgroups") == NULL && strstr(ext_buffer, "cl_intel_subgroups") == NULL) { GGML_LOG_ERROR("ggml_opencl: device does not support subgroups (cl_khr_subgroups or cl_intel_subgroups) " "(note that subgroups is an optional feature in OpenCL 3.0)\n"); @@ -581,9 +668,12 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { const std::string kernel_src = read_file("ggml-opencl.cl"); #endif - std::string compile_opts = - "-cl-std=CL2.0 -cl-mad-enable -cl-unsafe-math-optimizations " - "-cl-finite-math-only -cl-fast-relaxed-math "; + auto opencl_c_std = + std::string("CL") + std::to_string(opencl_c_version.major) + "." + std::to_string(opencl_c_version.minor); + + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable -cl-unsafe-math-optimizations" + " -cl-finite-math-only -cl-fast-relaxed-math"; backend_ctx->program = build_program_from_source(context, device, kernel_src.c_str(), compile_opts); // Non matmul kernels. @@ -693,10 +783,10 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { CL_CHECK((backend_ctx->kernel_transpose_16 = clCreateKernel(backend_ctx->program_transpose_16, "kernel_transpose_16", &err), err)); // Gemv general - std::string CL_gemv_compile_opts = - " -cl-std=CL2.0 " - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); if (has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } @@ -713,12 +803,12 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_general = clCreateKernel(backend_ctx->program_CL_gemv_general, "kernel_gemv_noshuffle", &err), err)); // Gemv 2048, 16384 - CL_gemv_compile_opts = - " -cl-std=CL2.0 " - " -cl-mad-enable " - " -DLINE_STRIDE_A=2048 " - " -DBLOCK_STRIDE_A=16384 " - " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); + CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DLINE_STRIDE_A=2048 " + " -DBLOCK_STRIDE_A=16384 " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); if (has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } @@ -735,12 +825,12 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_4096 = clCreateKernel(backend_ctx->program_CL_gemv_4096_1_4096, "kernel_gemv_noshuffle", &err), err)); // Gemv 2048, 16384 - CL_gemv_compile_opts = - " -cl-std=CL2.0 " - " -cl-mad-enable " - " -DLINE_STRIDE_A=2048 " - " -DBLOCK_STRIDE_A=16384 " - " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); + CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DLINE_STRIDE_A=2048 " + " -DBLOCK_STRIDE_A=16384 " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); if (has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } @@ -750,12 +840,12 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_11008 = clCreateKernel(backend_ctx->program_CL_gemv_4096_1_11008, "kernel_gemv_noshuffle", &err), err)); // Gemv 5504, 44032 - CL_gemv_compile_opts = - " -cl-std=CL2.0 " - " -cl-mad-enable " - " -DLINE_STRIDE_A=5504 " - " -DBLOCK_STRIDE_A=44032 " - " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); + CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DLINE_STRIDE_A=5504 " + " -DBLOCK_STRIDE_A=44032 " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); if (has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } @@ -765,12 +855,12 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_11008_1_4096 = clCreateKernel(backend_ctx->program_CL_gemv_11008_1_4096, "kernel_gemv_noshuffle", &err), err)); // Gemv 16000, 128000 - CL_gemv_compile_opts = - " -cl-std=CL2.0 " - " -cl-mad-enable " - " -DLINE_STRIDE_A=16000 " - " -DBLOCK_STRIDE_A=128000 " - " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); + CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DLINE_STRIDE_A=16000 " + " -DBLOCK_STRIDE_A=128000 " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); if (has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } From 251364549fe4a78d4e2e66f1adfaf2bd53041d2c Mon Sep 17 00:00:00 2001 From: R0CKSTAR Date: Tue, 11 Mar 2025 01:18:25 +0800 Subject: [PATCH 115/188] musa: support new arch mp_31 and update doc (#12296) Signed-off-by: Xiaodong Ye --- Makefile | 2 +- docs/build.md | 48 +++++++++++++++++++++++-------- ggml/src/ggml-musa/CMakeLists.txt | 2 +- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 5339d490b4..1f9455eff0 100644 --- a/Makefile +++ b/Makefile @@ -836,7 +836,7 @@ ifdef GGML_MUSA else MUSA_PATH ?= /opt/musa endif - MUSA_ARCHITECTURES ?= 21;22 + MUSA_ARCHITECTURES ?= 21;22;31 MK_CPPFLAGS += -DGGML_USE_MUSA -DGGML_USE_CUDA MK_LDFLAGS += -L$(MUSA_PATH)/lib -Wl,-rpath=$(MUSA_PATH)/lib diff --git a/docs/build.md b/docs/build.md index 3d8333328f..2e3975c145 100644 --- a/docs/build.md +++ b/docs/build.md @@ -197,29 +197,53 @@ The following compilation options are also available to tweak performance: ## MUSA -This provides GPU acceleration using the MUSA cores of your Moore Threads MTT GPU. Make sure to have the MUSA SDK installed. You can download it from here: [MUSA SDK](https://developer.mthreads.com/sdk/download/musa). +This provides GPU acceleration using a Moore Threads GPU. Make sure to have the [MUSA SDK](https://developer.mthreads.com/musa/musa-sdk) installed. -- Using `CMake`: +#### Download directly from Moore Threads - ```bash - cmake -B build -DGGML_MUSA=ON - cmake --build build --config Release +You may find the official downloads here: [Moore Threads developer site](https://developer.mthreads.com/sdk/download/musa). + +### Compilation + +```bash +cmake -B build -DGGML_MUSA=ON +cmake --build build --config Release +``` + +#### Override Compute Capability Specifications + +By default, all supported compute capabilities are enabled. To customize this behavior, you can specify the `MUSA_ARCHITECTURES` option in the CMake command: + +```bash +cmake -B build -DGGML_MUSA=ON -DMUSA_ARCHITECTURES="21" +``` + +This configuration enables only compute capability `2.1` (MTT S80) during compilation, which can help reduce compilation time. + +#### Compilation options + +Most of the compilation options available for CUDA should also be available for MUSA, though they haven't been thoroughly tested yet. + +- For static builds, add `-DBUILD_SHARED_LIBS=OFF` and `-DCMAKE_POSITION_INDEPENDENT_CODE=ON`: ``` - - For static build: - - ```bash cmake -B build -DGGML_MUSA=ON \ -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON cmake --build build --config Release ``` -The environment variable [`MUSA_VISIBLE_DEVICES`](https://docs.mthreads.com/musa-sdk/musa-sdk-doc-online/programming_guide/Z%E9%99%84%E5%BD%95/) can be used to specify which GPU(s) will be used. +### Runtime MUSA environmental variables + +You may set the [musa environmental variables](https://docs.mthreads.com/musa-sdk/musa-sdk-doc-online/programming_guide/Z%E9%99%84%E5%BD%95/) at runtime. + +```bash +# Use `MUSA_VISIBLE_DEVICES` to hide the first compute device. +MUSA_VISIBLE_DEVICES="-0" ./build/bin/llama-server --model /srv/models/llama.gguf +``` + +### Unified Memory The environment variable `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1` can be used to enable unified memory in Linux. This allows swapping to system RAM instead of crashing when the GPU VRAM is exhausted. -Most of the compilation options available for CUDA should also be available for MUSA, though they haven't been thoroughly tested yet. - ## HIP This provides GPU acceleration on HIP-supported AMD GPUs. diff --git a/ggml/src/ggml-musa/CMakeLists.txt b/ggml/src/ggml-musa/CMakeLists.txt index 2c75abf61d..166970ca6b 100644 --- a/ggml/src/ggml-musa/CMakeLists.txt +++ b/ggml/src/ggml-musa/CMakeLists.txt @@ -21,7 +21,7 @@ if (MUSAToolkit_FOUND) message(STATUS "MUSA Toolkit found") if (NOT DEFINED MUSA_ARCHITECTURES) - set(MUSA_ARCHITECTURES "21;22") + set(MUSA_ARCHITECTURES "21;22;31") endif() message(STATUS "Using MUSA architectures: ${MUSA_ARCHITECTURES}") From 2c9f833d17bb5b8ea89dec663b072b5420fc5438 Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Mon, 10 Mar 2025 19:28:11 +0000 Subject: [PATCH 116/188] mat vec double buffer (#12188) --- .../vulkan-shaders/mul_mat_vec_q2_k.comp | 47 ++++++++++--------- .../vulkan-shaders/mul_mat_vec_q3_k.comp | 26 +++++----- .../vulkan-shaders/mul_mat_vec_q6_k.comp | 12 ++--- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp index 8cdc640e80..423ceb8a3d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp @@ -5,23 +5,24 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; -shared FLOAT_TYPE sccache1[BLOCK_SIZE/16][16]; -shared FLOAT_TYPE sccache2[BLOCK_SIZE/16][16]; +shared FLOAT_TYPE sccache1[2][BLOCK_SIZE/16][16]; +shared FLOAT_TYPE sccache2[2][BLOCK_SIZE/16][16]; FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; +uint csel = 0; void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint v_im, const uint ix, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) { const uint y_idx = i * QUANT_K + y_offset; [[unroll]] for (uint n = 0; n < num_rows; ++n) { const uint ib0 = a_offset / QUANT_K + (first_row+n)*num_blocks_per_row; + csel ^= 1; - barrier(); if (!all_threads) { // when we don't have enough blocks to use all threads if (i < num_blocks_per_row) { const uint32_t scale = uint32_t(data_a[ib0 + i].scales[itid]); - sccache1[ix][itid] = FLOAT_TYPE(scale & 0xF); - sccache2[ix][itid] = FLOAT_TYPE((scale >> 4) & 0xF); + sccache1[csel][ix][itid] = FLOAT_TYPE(scale & 0xF); + sccache2[csel][ix][itid] = FLOAT_TYPE((scale >> 4) & 0xF); } barrier(); @@ -29,8 +30,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, continue; } else { const uint32_t scale = uint32_t(data_a[ib0 + i].scales[itid]); - sccache1[ix][itid] = FLOAT_TYPE(scale & 0xF); - sccache2[ix][itid] = FLOAT_TYPE((scale >> 4) & 0xF); + sccache1[csel][ix][itid] = FLOAT_TYPE(scale & 0xF); + sccache2[csel][ix][itid] = FLOAT_TYPE((scale >> 4) & 0xF); barrier(); } @@ -57,22 +58,22 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, FLOAT_TYPE sum1 = FLOAT_TYPE(0.0); FLOAT_TYPE sum2 = FLOAT_TYPE(0.0); [[unroll]] for (int l = 0; l < 2; ++l) { - sum1 = fma(FLOAT_TYPE(b0[l]), sccache1[ix][ 8*v_im] * qs_u32_0[l ], - fma(FLOAT_TYPE(b16[l]), sccache1[ix][1 + 8*v_im] * qs_u32_0[l+2], - fma(FLOAT_TYPE(b32[l]), sccache1[ix][2 + 8*v_im] * qs_u32_2[l ], - fma(FLOAT_TYPE(b48[l]), sccache1[ix][3 + 8*v_im] * qs_u32_2[l+2], - fma(FLOAT_TYPE(b64[l]), sccache1[ix][4 + 8*v_im] * qs_u32_4[l ], - fma(FLOAT_TYPE(b80[l]), sccache1[ix][5 + 8*v_im] * qs_u32_4[l+2], - fma(FLOAT_TYPE(b96[l]), sccache1[ix][6 + 8*v_im] * qs_u32_6[l ], - fma(FLOAT_TYPE(b112[l]), sccache1[ix][7 + 8*v_im] * qs_u32_6[l+2], sum1)))))))); - sum2 = fma(FLOAT_TYPE(b0[l]), sccache2[ix][ 8*v_im], - fma(FLOAT_TYPE(b16[l]), sccache2[ix][1 + 8*v_im], - fma(FLOAT_TYPE(b32[l]), sccache2[ix][2 + 8*v_im], - fma(FLOAT_TYPE(b48[l]), sccache2[ix][3 + 8*v_im], - fma(FLOAT_TYPE(b64[l]), sccache2[ix][4 + 8*v_im], - fma(FLOAT_TYPE(b80[l]), sccache2[ix][5 + 8*v_im], - fma(FLOAT_TYPE(b96[l]), sccache2[ix][6 + 8*v_im], - fma(FLOAT_TYPE(b112[l]), sccache2[ix][7 + 8*v_im], sum2)))))))); + sum1 = fma(FLOAT_TYPE(b0[l]), sccache1[csel][ix][ 8*v_im] * qs_u32_0[l ], + fma(FLOAT_TYPE(b16[l]), sccache1[csel][ix][1 + 8*v_im] * qs_u32_0[l+2], + fma(FLOAT_TYPE(b32[l]), sccache1[csel][ix][2 + 8*v_im] * qs_u32_2[l ], + fma(FLOAT_TYPE(b48[l]), sccache1[csel][ix][3 + 8*v_im] * qs_u32_2[l+2], + fma(FLOAT_TYPE(b64[l]), sccache1[csel][ix][4 + 8*v_im] * qs_u32_4[l ], + fma(FLOAT_TYPE(b80[l]), sccache1[csel][ix][5 + 8*v_im] * qs_u32_4[l+2], + fma(FLOAT_TYPE(b96[l]), sccache1[csel][ix][6 + 8*v_im] * qs_u32_6[l ], + fma(FLOAT_TYPE(b112[l]), sccache1[csel][ix][7 + 8*v_im] * qs_u32_6[l+2], sum1)))))))); + sum2 = fma(FLOAT_TYPE(b0[l]), sccache2[csel][ix][ 8*v_im], + fma(FLOAT_TYPE(b16[l]), sccache2[csel][ix][1 + 8*v_im], + fma(FLOAT_TYPE(b32[l]), sccache2[csel][ix][2 + 8*v_im], + fma(FLOAT_TYPE(b48[l]), sccache2[csel][ix][3 + 8*v_im], + fma(FLOAT_TYPE(b64[l]), sccache2[csel][ix][4 + 8*v_im], + fma(FLOAT_TYPE(b80[l]), sccache2[csel][ix][5 + 8*v_im], + fma(FLOAT_TYPE(b96[l]), sccache2[csel][ix][6 + 8*v_im], + fma(FLOAT_TYPE(b112[l]), sccache2[csel][ix][7 + 8*v_im], sum2)))))))); } temp[j][n] = fma(dall, sum1, fma(-dmin, sum2, temp[j][n])); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp index 3116fad165..e91724a28d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp @@ -5,20 +5,21 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; -shared FLOAT_TYPE sccache[BLOCK_SIZE/16][2][8]; +shared FLOAT_TYPE sccache[2][BLOCK_SIZE/16][2][8]; FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; +uint csel = 0; void calc_superblock(const uint a_offset, const uint b_offset, const uint ix, const uint itid8, const uint v_im, const uint v_im4, const uint v_in, const uint32_t hm_m[4], const uint q_offset, const uint y_offset, const uint s_shift, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) { const uint y_idx = i * QUANT_K + y_offset; [[unroll]] for (uint n = 0; n < num_rows; ++n) { const uint ib0 = a_offset / QUANT_K + (first_row+n)*num_blocks_per_row; + csel ^= 1; if (!all_threads) { // when we don't have enough blocks to use all threads - barrier(); if (i < num_blocks_per_row) - sccache[ix][v_im][itid8] = FLOAT_TYPE(int8_t(((data_a[ib0+i].scales[itid8] >> v_im4) & 0xF) | (((data_a[ib0+i].scales[itid8%4+8] >> s_shift) & 3) << 4)) - 32); + sccache[csel][ix][v_im][itid8] = FLOAT_TYPE(int8_t(((data_a[ib0+i].scales[itid8] >> v_im4) & 0xF) | (((data_a[ib0+i].scales[itid8%4+8] >> s_shift) & 3) << 4)) - 32); barrier(); if (i >= num_blocks_per_row) @@ -40,8 +41,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ix, co const vec4 qs_u32_6 = vec4(unpack8((qs_u32 >> 6) & 0x03030303)); if (all_threads) { - barrier(); - sccache[ix][v_im][itid8] = FLOAT_TYPE(int8_t(((data_a[ib0+i].scales[itid8] >> v_im4) & 0xF) | (((data_a[ib0+i].scales[itid8%4+8] >> s_shift) & 3) << 4)) - 32); + sccache[csel][ix][v_im][itid8] = FLOAT_TYPE(int8_t(((data_a[ib0+i].scales[itid8] >> v_im4) & 0xF) | (((data_a[ib0+i].scales[itid8%4+8] >> s_shift) & 3) << 4)) - 32); barrier(); } @@ -59,14 +59,14 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ix, co FLOAT_TYPE sum = FLOAT_TYPE(0.0); [[unroll]] for (int l = 0; l < 2; ++l) { - sum = fma(FLOAT_TYPE( b0[l]) * sccache[ix][v_im][0], qs_u32_0[l ] - hmk_0[l ], - fma(FLOAT_TYPE( b16[l]) * sccache[ix][v_im][1], qs_u32_0[l+2] - hmk_0[l+2], - fma(FLOAT_TYPE( b32[l]) * sccache[ix][v_im][2], qs_u32_2[l ] - hmk_1[l ], - fma(FLOAT_TYPE( b48[l]) * sccache[ix][v_im][3], qs_u32_2[l+2] - hmk_1[l+2], - fma(FLOAT_TYPE( b64[l]) * sccache[ix][v_im][4], qs_u32_4[l ] - hmk_2[l ], - fma(FLOAT_TYPE( b80[l]) * sccache[ix][v_im][5], qs_u32_4[l+2] - hmk_2[l+2], - fma(FLOAT_TYPE( b96[l]) * sccache[ix][v_im][6], qs_u32_6[l ] - hmk_3[l ], - fma(FLOAT_TYPE(b112[l]) * sccache[ix][v_im][7], qs_u32_6[l+2] - hmk_3[l+2], sum)))))))); + sum = fma(FLOAT_TYPE( b0[l]) * sccache[csel][ix][v_im][0], qs_u32_0[l ] - hmk_0[l ], + fma(FLOAT_TYPE( b16[l]) * sccache[csel][ix][v_im][1], qs_u32_0[l+2] - hmk_0[l+2], + fma(FLOAT_TYPE( b32[l]) * sccache[csel][ix][v_im][2], qs_u32_2[l ] - hmk_1[l ], + fma(FLOAT_TYPE( b48[l]) * sccache[csel][ix][v_im][3], qs_u32_2[l+2] - hmk_1[l+2], + fma(FLOAT_TYPE( b64[l]) * sccache[csel][ix][v_im][4], qs_u32_4[l ] - hmk_2[l ], + fma(FLOAT_TYPE( b80[l]) * sccache[csel][ix][v_im][5], qs_u32_4[l+2] - hmk_2[l+2], + fma(FLOAT_TYPE( b96[l]) * sccache[csel][ix][v_im][6], qs_u32_6[l ] - hmk_3[l ], + fma(FLOAT_TYPE(b112[l]) * sccache[csel][ix][v_im][7], qs_u32_6[l+2] - hmk_3[l+2], sum)))))))); } temp[j][n] = fma(d, sum, temp[j][n]); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp index f05f96b5ef..d53d9ee0a2 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp @@ -6,20 +6,21 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; -shared FLOAT_TYPE sccache[BLOCK_SIZE/16][16]; +shared FLOAT_TYPE sccache[2][BLOCK_SIZE/16][16]; FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; +uint csel = 0; void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint ix, const uint ql_offset, const uint qh_offset, const uint s_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) { const uint y_idx = i * QUANT_K + y_offset; [[unroll]] for (uint n = 0; n < num_rows; ++n) { const uint ib0 = a_offset / QUANT_K + (first_row+n)*num_blocks_per_row; + csel ^= 1; if (!all_threads) { // when we don't have enough blocks to use all threads - barrier(); if (i < num_blocks_per_row) - sccache[ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]); + sccache[csel][ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]); barrier(); if (i >= num_blocks_per_row) @@ -51,8 +52,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const vec4 q3 = vec4(unpack8(q3_u32)) - 32; if (all_threads) { - barrier(); - sccache[ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]); + sccache[csel][ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]); barrier(); } @@ -71,7 +71,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, sum[2] = fma(FLOAT_TYPE(by64[l]), q2[l], sum[2]); sum[3] = fma(FLOAT_TYPE(by96[l]), q3[l], sum[3]); } - temp[j][n] = fma(fma(sum[0], sccache[ix][s_offset], fma(sum[1], sccache[ix][s_offset + 2], fma(sum[2], sccache[ix][s_offset + 4], sum[3] * sccache[ix][s_offset + 6]))), d, temp[j][n]); + temp[j][n] = fma(fma(sum[0], sccache[csel][ix][s_offset], fma(sum[1], sccache[csel][ix][s_offset + 2], fma(sum[2], sccache[csel][ix][s_offset + 4], sum[3] * sccache[csel][ix][s_offset + 6]))), d, temp[j][n]); } } } From 96e1280839561aaabb73851f94972a2cd37b2d96 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Tue, 11 Mar 2025 09:20:16 +0100 Subject: [PATCH 117/188] clip : bring back GPU support (#12322) * clip : bring back GPU support * use n_gpu_layers param * fix double free * ggml_backend_init_by_type * clean up --- examples/llava/clip.cpp | 149 ++++++++++++++++---------------- examples/llava/clip.h | 11 ++- examples/llava/minicpmv-cli.cpp | 6 +- 3 files changed, 89 insertions(+), 77 deletions(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 3f558b7bdb..7f892beb6e 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -4,31 +4,12 @@ // Note: Even when using identical normalized image inputs (see normalize_image_u8_to_f32()) we have a significant difference in resulting embeddings compared to pytorch #include "clip.h" #include "ggml.h" +#include "ggml-cpp.h" #include "ggml-cpu.h" #include "ggml-alloc.h" #include "ggml-backend.h" #include "gguf.h" -//#ifdef GGML_USE_CUDA -//#include "ggml-cuda.h" -//#endif -// -//#ifdef GGML_USE_SYCL -//#include "ggml-sycl.h" -//#endif -// -//#ifdef GGML_USE_METAL -//#include "ggml-metal.h" -//#endif -// -//#ifdef GGML_USE_CANN -//#include "ggml-cann.h" -//#endif -// -//#ifdef GGML_USE_VULKAN -//#include "ggml-vulkan.h" -//#endif - #define STB_IMAGE_IMPLEMENTATION #include "stb_image.h" @@ -600,18 +581,54 @@ struct clip_ctx { bool has_post_norm = false; bool has_patch_bias = false; - struct gguf_context * ctx_gguf; - struct ggml_context * ctx_data; + struct gguf_context * ctx_gguf = nullptr; + struct ggml_context * ctx_data = nullptr; std::vector buf_compute_meta; - // memory buffers to evaluate the model - ggml_backend_buffer_t params_buffer = NULL; + std::vector backend_ptrs; + std::vector backend_buft; - ggml_backend_t backend = NULL; - ggml_gallocr_t compute_alloc = NULL; + ggml_backend_t backend = nullptr; + ggml_backend_t backend_cpu = nullptr; + ggml_backend_buffer_t buf = nullptr; + + ggml_backend_sched_ptr sched; struct clip_image_size * load_image_size; + + clip_ctx(clip_context_params & ctx_params) { + backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + backend = ctx_params.use_gpu + ? ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr) + : nullptr; + + if (backend) { + LOG_INF("%s: CLIP using %s backend\n", __func__, ggml_backend_name(backend)); + backend_ptrs.push_back(backend); + backend_buft.push_back(ggml_backend_get_default_buffer_type(backend)); + } else { + backend = backend_cpu; + LOG_INF("%s: CLIP using CPU backend\n", __func__); + } + + backend_ptrs.push_back(backend_cpu); + backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu)); + + sched.reset( + ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), 8192, false) + ); + } + + ~clip_ctx() { + ggml_free(ctx_data); + gguf_free(ctx_gguf); + ggml_backend_buffer_free(buf); + ggml_backend_free(backend); + if (backend_cpu != backend) { + ggml_backend_free(backend_cpu); + } + } }; static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch * imgs, struct clip_image_size * load_image_size, bool is_inf = false) { @@ -1184,6 +1201,14 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 // read and create ggml_context containing the tensors and their data struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { + return clip_init(fname, clip_context_params{ + /* use_gpu */ true, + /* verbosity */ verbosity, + }); +} + +struct clip_ctx * clip_init(const char * fname, struct clip_context_params ctx_params) { + int verbosity = ctx_params.verbosity; struct ggml_context * meta = NULL; struct gguf_init_params params = { @@ -1277,7 +1302,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { } } - clip_ctx * new_clip = new clip_ctx{}; + clip_ctx * new_clip = new clip_ctx(ctx_params); // update projector type { @@ -1296,36 +1321,6 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { } } -//#ifdef GGML_USE_CUDA -// new_clip->backend = ggml_backend_cuda_init(0); -// LOG_INF("%s: CLIP using CUDA backend\n", __func__); -//#endif -// -//#ifdef GGML_USE_METAL -// new_clip->backend = ggml_backend_metal_init(); -// LOG_INF("%s: CLIP using Metal backend\n", __func__); -//#endif -// -//#ifdef GGML_USE_CANN -// new_clip->backend = ggml_backend_cann_init(0); -// LOG_INF("%s: CLIP using CANN backend\n", __func__); -//#endif -// -//#ifdef GGML_USE_VULKAN -// new_clip->backend = ggml_backend_vk_init(0); -// LOG_INF("%s: CLIP using Vulkan backend\n", __func__); -//#endif -// -//#ifdef GGML_USE_SYCL -// new_clip->backend = ggml_backend_sycl_init(0); -// LOG_INF("%s: CLIP using SYCL backend\n", __func__); -//#endif - - if (!new_clip->backend) { - new_clip->backend = ggml_backend_cpu_init(); - LOG_INF("%s: CLIP using CPU backend\n", __func__); - } - // model size and capabilities { int idx = get_key_idx(ctx, KEY_HAS_TEXT_ENC); @@ -1421,7 +1416,9 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { } // alloc memory and offload data - new_clip->params_buffer = ggml_backend_alloc_ctx_tensors(new_clip->ctx_data, new_clip->backend); + ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(new_clip->backend); + new_clip->buf = ggml_backend_alloc_ctx_tensors_from_buft(new_clip->ctx_data, buft); + ggml_backend_buffer_set_usage(new_clip->buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); for (int i = 0; i < n_tensors; ++i) { const char * name = gguf_get_tensor_name(ctx, i); struct ggml_tensor * cur = ggml_get_tensor(new_clip->ctx_data, name); @@ -1434,7 +1431,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { return nullptr; } int num_bytes = ggml_nbytes(cur); - if (ggml_backend_buffer_is_host(new_clip->params_buffer)) { + if (ggml_backend_buft_is_host(buft)) { // for the CPU and Metal backend, we can read directly into the tensor fin.read(reinterpret_cast(cur->data), num_bytes); } else { @@ -1720,14 +1717,21 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { // measure mem requirement and allocate { new_clip->buf_compute_meta.resize(GGML_DEFAULT_GRAPH_SIZE * ggml_tensor_overhead() + ggml_graph_overhead()); - new_clip->compute_alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(new_clip->backend)); clip_image_f32_batch batch; batch.size = 1; batch.data = nullptr; ggml_cgraph * gf = clip_image_build_graph(new_clip, &batch, nullptr, false); - ggml_gallocr_reserve(new_clip->compute_alloc, gf); - size_t compute_memory_buffer_size = ggml_gallocr_get_buffer_size(new_clip->compute_alloc, 0); - LOG_INF("%s: compute allocated memory: %.2f MB\n", __func__, compute_memory_buffer_size /1024.0/1024.0); + ggml_backend_sched_reserve(new_clip->sched.get(), gf); + for (size_t i = 0; i < new_clip->backend_ptrs.size(); ++i) { + ggml_backend_t backend = new_clip->backend_ptrs[i]; + ggml_backend_buffer_type_t buft = new_clip->backend_buft[i]; + size_t size = ggml_backend_sched_get_buffer_size(new_clip->sched.get(), backend); + if (size > 1) { + LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__, + ggml_backend_buft_name(buft), + size / 1024.0 / 1024.0); + } + } } return new_clip; @@ -2408,12 +2412,6 @@ ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx) { } void clip_free(clip_ctx * ctx) { - ggml_free(ctx->ctx_data); - gguf_free(ctx->ctx_gguf); - - ggml_backend_buffer_free(ctx->params_buffer); - ggml_backend_free(ctx->backend); - ggml_gallocr_free(ctx->compute_alloc); delete ctx; } @@ -2609,8 +2607,9 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima } // build the inference graph + ggml_backend_sched_reset(ctx->sched.get()); ggml_cgraph * gf = clip_image_build_graph(ctx, imgs, ctx->load_image_size, true); - ggml_gallocr_alloc_graph(ctx->compute_alloc, gf); + ggml_backend_sched_alloc_graph(ctx->sched.get(), gf); // set inputs const auto & model = ctx->vision_model; @@ -2775,11 +2774,13 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima } } - if (ggml_backend_is_cpu(ctx->backend)) { - ggml_backend_cpu_set_n_threads(ctx->backend, n_threads); - } + ggml_backend_cpu_set_n_threads(ctx->backend_cpu, n_threads); - ggml_backend_graph_compute(ctx->backend, gf); + auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); + return false; + } // the last node is the embedding tensor struct ggml_tensor * embeddings = ggml_graph_node(gf, -1); diff --git a/examples/llava/clip.h b/examples/llava/clip.h index 002c419653..47059ca1b9 100644 --- a/examples/llava/clip.h +++ b/examples/llava/clip.h @@ -39,8 +39,15 @@ struct clip_image_f32_batch { size_t size; }; -CLIP_API struct clip_ctx * clip_model_load (const char * fname, int verbosity); -CLIP_API struct clip_ctx * clip_model_load_cpu(const char * fname, int verbosity); +struct clip_context_params { + bool use_gpu; + int verbosity; +}; + +// deprecated, use clip_init +CLIP_API struct clip_ctx * clip_model_load(const char * fname, int verbosity); + +CLIP_API struct clip_ctx * clip_init(const char * fname, struct clip_context_params ctx_params); CLIP_API void clip_free(struct clip_ctx * ctx); diff --git a/examples/llava/minicpmv-cli.cpp b/examples/llava/minicpmv-cli.cpp index 23b3de4db2..12f536cf5c 100644 --- a/examples/llava/minicpmv-cli.cpp +++ b/examples/llava/minicpmv-cli.cpp @@ -86,7 +86,11 @@ static struct clip_ctx * clip_init_context(common_params * params) { if (prompt.empty()) { prompt = "describe the image in detail."; } - auto * ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1); + struct clip_context_params clip_params = { + /* use_gpu */ params->n_gpu_layers != 0, + /* verbosity */ params->verbosity, + }; + auto * ctx_clip = clip_init(clip_path, clip_params); return ctx_clip; } From 6ab2e4765a673abcd162258a2671560a76106d69 Mon Sep 17 00:00:00 2001 From: BB-fat <45072480+BB-fat@users.noreply.github.com> Date: Tue, 11 Mar 2025 19:45:02 +0800 Subject: [PATCH 118/188] metal : Cache the Metal library at the device context level (#12265) --- ggml/src/ggml-metal/ggml-metal.m | 279 ++++++++++++++++--------------- 1 file changed, 147 insertions(+), 132 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m index 1158b285c1..e51a4169a2 100644 --- a/ggml/src/ggml-metal/ggml-metal.m +++ b/ggml/src/ggml-metal/ggml-metal.m @@ -46,6 +46,7 @@ static struct ggml_backend_device g_ggml_backend_metal_device; static struct ggml_backend_metal_device_context { id mtl_device; int mtl_device_ref_count; + id mtl_library; bool has_simdgroup_reduction; bool has_simdgroup_mm; @@ -57,6 +58,7 @@ static struct ggml_backend_metal_device_context { } g_ggml_ctx_dev_main = { /*.mtl_device =*/ nil, /*.mtl_device_ref_count =*/ 0, + /*.mtl_library =*/ nil, /*.has_simdgroup_reduction =*/ false, /*.has_simdgroup_mm =*/ false, /*.has_residency_sets =*/ false, @@ -108,6 +110,11 @@ static void ggml_backend_metal_device_rel(struct ggml_backend_metal_device_conte ctx->mtl_device_ref_count--; if (ctx->mtl_device_ref_count == 0) { + if (ctx->mtl_library) { + [ctx->mtl_library release]; + ctx->mtl_library = nil; + } + if (ctx->mtl_device) { [ctx->mtl_device release]; ctx->mtl_device = nil; @@ -495,6 +502,139 @@ static void * ggml_metal_host_malloc(size_t n) { return data; } +// load library +// +// - first check if the library is embedded +// - then check if the library is in the bundle +// - if not found, load the source and compile it +// - if that fails, return NULL +static id ggml_metal_load_library(id device, bool use_bfloat) { + id metal_library = nil; + NSError * error = nil; + NSString * src = nil; + +#if GGML_METAL_EMBED_LIBRARY + GGML_LOG_INFO("%s: using embedded metal library\n", __func__); + + extern const char ggml_metallib_start[]; + extern const char ggml_metallib_end[]; + + src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding]; + +#else + +#ifdef SWIFT_PACKAGE + NSBundle * bundle = SWIFTPM_MODULE_BUNDLE; +#else + NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; +#endif + + NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"]; + if (path_lib == nil) { + // Try to find the resource in the directory where the current binary located. + NSString * current_binary = [[NSProcessInfo processInfo] arguments][0]; + NSString * bin_dir = [current_binary stringByDeletingLastPathComponent]; + NSString * default_metallib_path = [NSString pathWithComponents:@[bin_dir, @"default.metallib"]]; + if ([[NSFileManager defaultManager] isReadableFileAtPath:default_metallib_path]) { + GGML_LOG_INFO("%s: found '%s'\n", __func__, [default_metallib_path UTF8String]); + NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:default_metallib_path error:&error]; + if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) { + // Optionally, if this is a symlink, try to resolve it. + default_metallib_path = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:default_metallib_path error:&error]; + if (default_metallib_path && [default_metallib_path length] > 0 && ![[default_metallib_path substringToIndex:1] isEqualToString:@"/"]) { + // It is a relative path, adding the binary directory as directory prefix. + default_metallib_path = [NSString pathWithComponents:@[bin_dir, default_metallib_path]]; + } + if (!default_metallib_path || ![[NSFileManager defaultManager] isReadableFileAtPath:default_metallib_path]) { + // Link to the resource could not be resolved. + default_metallib_path = nil; + } else { + GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [default_metallib_path UTF8String]); + } + } + } else { + // The resource couldn't be found in the binary's directory. + default_metallib_path = nil; + } + path_lib = default_metallib_path; + } + + if (path_lib != nil) { + // pre-compiled library found + NSURL * libURL = [NSURL fileURLWithPath:path_lib]; + GGML_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]); + + metal_library = [device newLibraryWithURL:libURL error:&error]; + if (error) { + GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); + return NULL; + } + } else { + GGML_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__); + + NSString * path_source; + NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"]; + + GGML_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, path_resource ? [path_resource UTF8String] : "nil"); + + if (path_resource) { + path_source = [path_resource stringByAppendingPathComponent:@"ggml-metal.metal"]; + } else { + path_source = [bundle pathForResource:@"ggml-metal" ofType:@"metal"]; + } + + if (path_source == nil) { + GGML_LOG_WARN("%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\n", __func__); + path_source = @"ggml-metal.metal"; + } + + GGML_LOG_INFO("%s: loading '%s'\n", __func__, [path_source UTF8String]); + + src = [NSString stringWithContentsOfFile:path_source encoding:NSUTF8StringEncoding error:&error]; + if (error) { + GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); + return NULL; + } + } +#endif + + if (!metal_library) { + @autoreleasepool { + // dictionary of preprocessor macros + NSMutableDictionary * prep = [NSMutableDictionary dictionary]; + + if (use_bfloat) { + [prep setObject:@"1" forKey:@"GGML_METAL_USE_BF16"]; + } + +#if GGML_METAL_EMBED_LIBRARY + [prep setObject:@"1" forKey:@"GGML_METAL_EMBED_LIBRARY"]; +#endif + + MTLCompileOptions * options = [MTLCompileOptions new]; + options.preprocessorMacros = prep; + + //[options setFastMathEnabled:false]; + + metal_library = [device newLibraryWithSource:src options:options error:&error]; + if (error) { + GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); + return NULL; + } + +#if !__has_feature(objc_arc) + [options release]; +#endif + } + } + +#if GGML_METAL_EMBED_LIBRARY + [src release]; +#endif // GGML_METAL_EMBED_LIBRARY + + return metal_library; +} + static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t dev) { GGML_LOG_INFO("%s: allocating\n", __func__); @@ -522,136 +662,14 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); - id metal_library = nil; - // load library - // - // - first check if the library is embedded - // - then check if the library is in the bundle - // - if not found, load the source and compile it - // - if that fails, return NULL - { - NSError * error = nil; - NSString * src = nil; - -#if GGML_METAL_EMBED_LIBRARY - GGML_LOG_INFO("%s: using embedded metal library\n", __func__); - - extern const char ggml_metallib_start[]; - extern const char ggml_metallib_end[]; - - src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding]; - -#else - -#ifdef SWIFT_PACKAGE - NSBundle * bundle = SWIFTPM_MODULE_BUNDLE; -#else - NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; -#endif - - NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"]; - if (path_lib == nil) { - // Try to find the resource in the directory where the current binary located. - NSString * current_binary = [[NSProcessInfo processInfo] arguments][0]; - NSString * bin_dir = [current_binary stringByDeletingLastPathComponent]; - NSString * default_metallib_path = [NSString pathWithComponents:@[bin_dir, @"default.metallib"]]; - if ([[NSFileManager defaultManager] isReadableFileAtPath:default_metallib_path]) { - GGML_LOG_INFO("%s: found '%s'\n", __func__, [default_metallib_path UTF8String]); - NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:default_metallib_path error:&error]; - if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) { - // Optionally, if this is a symlink, try to resolve it. - default_metallib_path = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:default_metallib_path error:&error]; - if (default_metallib_path && [default_metallib_path length] > 0 && ![[default_metallib_path substringToIndex:1] isEqualToString:@"/"]) { - // It is a relative path, adding the binary directory as directory prefix. - default_metallib_path = [NSString pathWithComponents:@[bin_dir, default_metallib_path]]; - } - if (!default_metallib_path || ![[NSFileManager defaultManager] isReadableFileAtPath:default_metallib_path]) { - // Link to the resource could not be resolved. - default_metallib_path = nil; - } else { - GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [default_metallib_path UTF8String]); - } - } - } else { - // The resource couldn't be found in the binary's directory. - default_metallib_path = nil; - } - path_lib = default_metallib_path; - } - - if (path_lib != nil) { - // pre-compiled library found - NSURL * libURL = [NSURL fileURLWithPath:path_lib]; - GGML_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]); - - metal_library = [device newLibraryWithURL:libURL error:&error]; - if (error) { - GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); - return NULL; - } - } else { - GGML_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__); - - NSString * path_source; - NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"]; - - GGML_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, path_resource ? [path_resource UTF8String] : "nil"); - - if (path_resource) { - path_source = [path_resource stringByAppendingPathComponent:@"ggml-metal.metal"]; - } else { - path_source = [bundle pathForResource:@"ggml-metal" ofType:@"metal"]; - } - - if (path_source == nil) { - GGML_LOG_WARN("%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\n", __func__); - path_source = @"ggml-metal.metal"; - } - - GGML_LOG_INFO("%s: loading '%s'\n", __func__, [path_source UTF8String]); - - src = [NSString stringWithContentsOfFile:path_source encoding:NSUTF8StringEncoding error:&error]; - if (error) { - GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); - return NULL; - } - } -#endif - - if (!metal_library) { - @autoreleasepool { - // dictionary of preprocessor macros - NSMutableDictionary * prep = [NSMutableDictionary dictionary]; - - if (ctx_dev->use_bfloat) { - [prep setObject:@"1" forKey:@"GGML_METAL_USE_BF16"]; - } - -#if GGML_METAL_EMBED_LIBRARY - [prep setObject:@"1" forKey:@"GGML_METAL_EMBED_LIBRARY"]; -#endif - - MTLCompileOptions * options = [MTLCompileOptions new]; - options.preprocessorMacros = prep; - - //[options setFastMathEnabled:false]; - - metal_library = [device newLibraryWithSource:src options:options error:&error]; - if (error) { - GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); - return NULL; - } - -#if !__has_feature(objc_arc) - [options release]; -#endif - } - } - -#if GGML_METAL_EMBED_LIBRARY - [src release]; -#endif // GGML_METAL_EMBED_LIBRARY + if (ctx_dev->mtl_library == nil) { + ctx_dev->mtl_library = ggml_metal_load_library(device, ctx_dev->use_bfloat); + } + id metal_library = ctx_dev->mtl_library; + if (metal_library == nil) { + GGML_LOG_ERROR("%s: error: metal library is nil\n", __func__); + return NULL; } // print MTL GPU family: @@ -725,7 +743,6 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de [metal_function release]; \ if (error) { \ GGML_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \ - [metal_library release]; \ return NULL; \ } \ } else { \ @@ -1044,8 +1061,6 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_POOL_2D_MAX_F32, pool_2d_max_f32, true); } - [metal_library release]; - return ctx; } From ba7654380a3c7c1b5ae154bea19134a3a9417a1e Mon Sep 17 00:00:00 2001 From: jklincn <985765408@qq.com> Date: Tue, 11 Mar 2025 21:25:17 +0800 Subject: [PATCH 119/188] ggml-backend : fix backend search path (#12330) * Fix backend search path * replace .native() with '/' * reverted .native() --- ggml/src/ggml-backend-reg.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp index 9bedeae78a..405d8e3151 100644 --- a/ggml/src/ggml-backend-reg.cpp +++ b/ggml/src/ggml-backend-reg.cpp @@ -497,7 +497,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, search_paths.push_back(get_executable_path()); search_paths.push_back(fs::current_path()); } else { - search_paths.push_back(user_search_path); + search_paths.push_back(fs::u8path(user_search_path)); } int best_score = 0; @@ -511,9 +511,9 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied); for (const auto & entry : dir_it) { if (entry.is_regular_file()) { - auto filename = entry.path().filename().native(); - auto ext = entry.path().extension().native(); - if (filename.find(file_prefix) == 0 && ext == file_extension) { + auto filename = entry.path().filename(); + auto ext = entry.path().extension(); + if (filename.native().find(file_prefix) == 0 && ext == file_extension) { dl_handle_ptr handle { dl_load_library(entry) }; if (!handle && !silent) { GGML_LOG_ERROR("%s: failed to load %s\n", __func__, path_str(entry.path()).c_str()); @@ -544,7 +544,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, // try to load the base backend for (const auto & search_path : search_paths) { fs::path filename = backend_filename_prefix().native() + name_path.native() + backend_filename_extension().native(); - fs::path path = search_path.native() + filename.native(); + fs::path path = search_path / filename; if (fs::exists(path)) { return get_reg().load_backend(path, silent); } From 10f2e81809bbb69ecfe64fc8b4686285f84b0c07 Mon Sep 17 00:00:00 2001 From: uvos Date: Tue, 11 Mar 2025 20:16:03 +0100 Subject: [PATCH 120/188] CUDA/HIP: refractor mmqv to unify the calculation of nwarps and rows per block between host and device code. (#12177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refactor mmqv to unify the calculation of nwarps and rows per block between host and device code. --------- Co-authored-by: Johannes Gäßler --- ggml/src/ggml-cuda/common.cuh | 4 +- ggml/src/ggml-cuda/mmvq.cu | 197 ++++++++++++++++++++++++---------- 2 files changed, 142 insertions(+), 59 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 1832314ec1..4d4ac47c03 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -395,11 +395,11 @@ static __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, int c) { #if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) -#if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(RDNA2) +#if defined(CDNA) || defined(RDNA2) || defined(__gfx906__) c = __builtin_amdgcn_sdot4(a, b, c, false); #elif defined(RDNA3) c = __builtin_amdgcn_sudot4( true, a, true, b, c, false); -#elif defined(__gfx1010__) || defined(__gfx900__) +#elif defined(RDNA1) || defined(__gfx900__) int tmp1; int tmp2; asm("\n \ diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 4fb466ca00..a7d518a574 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -47,11 +47,89 @@ static constexpr __device__ int get_vdr_mmvq(ggml_type type) { 1; } +enum mmvq_parameter_table_id { + MMVQ_PARAMETERS_GENERIC = 0, + MMVQ_PARAMETERS_GCN, + MMVQ_PARAMETERS_RDNA2 +}; + +static constexpr __device__ mmvq_parameter_table_id get_device_table_id() { +#if defined(RDNA2) || defined(RDNA3) + return MMVQ_PARAMETERS_RDNA2; +#elif defined(GCN) || defined(CDNA) + return MMVQ_PARAMETERS_GCN; +#else + return MMVQ_PARAMETERS_GENERIC; +#endif +} + +static __host__ mmvq_parameter_table_id get_device_table_id(int cc) { + if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3(cc)) { + return MMVQ_PARAMETERS_RDNA2; + } + if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) { + return MMVQ_PARAMETERS_GCN; + } + return MMVQ_PARAMETERS_GENERIC; +} + +static constexpr __host__ __device__ int calc_nwarps(int ncols_y, mmvq_parameter_table_id table_id) { + if (table_id == MMVQ_PARAMETERS_GENERIC) { + switch (ncols_y) { + case 1: + case 2: + case 3: + case 4: + return 4; + case 5: + case 6: + case 7: + case 8: + return 2; + default: + return 1; + } + } else if (table_id == MMVQ_PARAMETERS_GCN) { + switch (ncols_y) { + case 1: + case 2: + case 3: + case 4: + return 2; + case 5: + case 6: + case 7: + case 8: + default: + return 1; + } + } + return 1; +} + +static constexpr __host__ __device__ int calc_rows_per_block(int ncols_y, int table_id) { + if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN) { + switch (ncols_y) { + case 1: + return 1; + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + return 2; + default: + return 1; + } + } + return 1; +} + template -#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) // tell the compiler to use as many registers as it wants, see nwarps definition below -__launch_bounds__((ncols_y <= 4 ? 4 : 2)*WARP_SIZE, 1) -#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) +__launch_bounds__(calc_nwarps(ncols_y, get_device_table_id())*ggml_cuda_get_physical_warp_size(), 1) static __global__ void mul_mat_vec_q( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int nrows_y, const int nrows_dst) { @@ -59,24 +137,20 @@ static __global__ void mul_mat_vec_q( constexpr int qk = ggml_cuda_type_traits::qk; constexpr int qi = ggml_cuda_type_traits::qi; constexpr int vdr = get_vdr_mmvq(type); + constexpr mmvq_parameter_table_id table_id = get_device_table_id(); + constexpr int nwarps = calc_nwarps(ncols_y, table_id); + constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_y, table_id); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type); -#if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3)) - constexpr int nwarps = 1; - constexpr int rows_per_cuda_block = 1; -#else - constexpr int nwarps = ncols_y <= 4 ? 4 : 2; - constexpr int rows_per_cuda_block = ncols_y == 1 ? 1 : 2; -#endif // defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3) - - const int tid = WARP_SIZE*threadIdx.y + threadIdx.x; + const int tid = warp_size*threadIdx.y + threadIdx.x; const int row0 = rows_per_cuda_block*blockIdx.x; const int blocks_per_row_x = ncols_x / qk; const int blocks_per_col_y = nrows_y / QK8_1; - constexpr int blocks_per_iter = vdr * nwarps*WARP_SIZE / qi; + constexpr int blocks_per_iter = vdr * nwarps*warp_size / qi; -// partial sum for each thread + // partial sum for each thread float tmp[ncols_y][rows_per_cuda_block] = {0.0f}; const block_q8_1 * y = (const block_q8_1 *) vy; @@ -96,7 +170,7 @@ static __global__ void mul_mat_vec_q( } } - __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_y][rows_per_cuda_block][WARP_SIZE]; + __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_y][rows_per_cuda_block][warp_size]; if (threadIdx.y > 0) { #pragma unroll for (int j = 0; j < ncols_y; ++j) { @@ -120,7 +194,7 @@ static __global__ void mul_mat_vec_q( for (int l = 0; l < nwarps-1; ++l) { tmp[j][i] += tmp_shared[l][j][i][threadIdx.x]; } - tmp[j][i] = warp_reduce_sum(tmp[j][i]); + tmp[j][i] = warp_reduce_sum(tmp[j][i]); } if (threadIdx.x < rows_per_cuda_block && (rows_per_cuda_block == 1 || row0 + threadIdx.x < nrows_dst)) { @@ -129,6 +203,13 @@ static __global__ void mul_mat_vec_q( } } +static std::pair calc_launch_params(const int ncols_y, const int nrows_x, const int warp_size, const mmvq_parameter_table_id table_id) { + const int64_t nblocks = (nrows_x + calc_rows_per_block(ncols_y, table_id) - 1) / calc_rows_per_block(ncols_y, table_id); + const dim3 block_nums(nblocks, 1, 1); + const dim3 block_dims(warp_size, calc_nwarps(ncols_y, table_id), 1); + return {block_nums, block_dims}; +} + template static void mul_mat_vec_q_cuda( const void * vx, const void * vy, float * dst, @@ -137,65 +218,67 @@ static void mul_mat_vec_q_cuda( GGML_ASSERT(ncols_x % ggml_blck_size(type) == 0); GGML_ASSERT(ncols_y <= MMVQ_MAX_BATCH_SIZE); - int id = ggml_cuda_get_device(); - - int64_t nwarps = 1; - int64_t rows_per_cuda_block = 1; - - if (ggml_cuda_info().devices[id].cc < GGML_CUDA_CC_RDNA2) { // NVIDIA and AMD older than RDNA2 - switch(ncols_y) { - case 1: - nwarps = 4; - rows_per_cuda_block = 1; - break; - case 2: - case 3: - case 4: - nwarps = 4; - rows_per_cuda_block = 2; - break; - case 5: - case 6: - case 7: - case 8: - nwarps = 2; - rows_per_cuda_block = 2; - break; - default: - GGML_ABORT("fatal error"); - break; - } - } - - const int64_t nblocks = (nrows_x + rows_per_cuda_block - 1) / rows_per_cuda_block; - const dim3 block_nums(nblocks, 1, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); + const int device = ggml_cuda_get_device(); + const int warp_size = ggml_cuda_info().devices[device].warp_size; + const mmvq_parameter_table_id table_id = get_device_table_id(ggml_cuda_info().devices[device].cc); switch (ncols_y) { case 1: - mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + { + constexpr int c_ncols_y = 1; + std::pair dims = calc_launch_params(c_ncols_y, nrows_x, warp_size, table_id); + mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); break; + } case 2: - mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + { + constexpr int c_ncols_y = 2; + std::pair dims = calc_launch_params(c_ncols_y, nrows_x, warp_size, table_id); + mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); break; + } case 3: - mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + { + constexpr int c_ncols_y = 3; + std::pair dims = calc_launch_params(c_ncols_y, nrows_x, warp_size, table_id); + mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); break; + } case 4: - mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + { + constexpr int c_ncols_y = 4; + std::pair dims = calc_launch_params(c_ncols_y, nrows_x, warp_size, table_id); + mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); break; + } case 5: - mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + { + constexpr int c_ncols_y = 5; + std::pair dims = calc_launch_params(c_ncols_y, nrows_x, warp_size, table_id); + mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); break; + } case 6: - mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + { + constexpr int c_ncols_y = 6; + std::pair dims = calc_launch_params(c_ncols_y, nrows_x, warp_size, table_id); + mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); break; + } case 7: - mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + { + constexpr int c_ncols_y = 7; + std::pair dims = calc_launch_params(c_ncols_y, nrows_x, warp_size, table_id); + mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); break; + } case 8: - mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + { + constexpr int c_ncols_y = 8; + std::pair dims = calc_launch_params(c_ncols_y, nrows_x, warp_size, table_id); + mul_mat_vec_q<<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); break; + } default: GGML_ABORT("fatal error"); break; From bf69cfe62f9ccc01112c0232a55820b95a8c1fda Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Wed, 12 Mar 2025 00:59:19 -0500 Subject: [PATCH 121/188] vulkan: fix bug in coopmat1 mul_mat_id (#12316) * tests: run mul_mat_id with a larger N * vulkan: fix bug in coopmat1 mul_mat_id --- ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp | 2 +- tests/test-backend-ops.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index a8fd93fdea..0d03411f24 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -777,7 +777,7 @@ void main() { [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor); - [[unroll]] for (uint col = 0; col < BN; col += storestride) { + [[unroll]] for (uint col = 0; col < TN; col += storestride) { const uint row_i = dc + cm_col * TN + col + store_c; if (row_i >= _ne1) break; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index b4e3631ed0..c86ffb64e9 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4113,7 +4113,7 @@ static std::vector> make_test_cases_eval() { for (int n_mats : {4, 8}) { for (int n_used : {1, 2, 4}) { for (bool b : {false, true}) { - for (int n : {1, 32}) { + for (int n : {1, 32, 129}) { int m = 512; int k = 256; test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, n_mats, n_used, b, m, n, k)); From 7841fc723e059d1fd9640e5c0ef19050fcc7c698 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 12 Mar 2025 09:30:24 +0100 Subject: [PATCH 122/188] llama : Add Gemma 3 support (+ experimental vision capability) (#12343) * llama : Add Gemma 3 text-only support * fix python coding style * fix compile on ubuntu * python: fix style * fix ubuntu compile * fix build on ubuntu (again) * fix ubuntu build, finally * clip : Experimental support for Gemma 3 vision (#12344) * clip : Experimental support for Gemma 3 vision * fix build * PRId64 --- convert_hf_to_gguf.py | 80 ++++ examples/llava/CMakeLists.txt | 7 + examples/llava/README-gemma3.md | 30 ++ examples/llava/clip.cpp | 210 ++++++++++- examples/llava/gemma3-cli.cpp | 341 ++++++++++++++++++ .../llava/gemma3_convert_encoder_to_gguf.py | 307 ++++++++++++++++ gguf-py/gguf/constants.py | 19 + src/llama-arch.cpp | 21 ++ src/llama-arch.h | 1 + src/llama-model.cpp | 49 +++ src/llama.cpp | 147 ++++++++ 11 files changed, 1202 insertions(+), 10 deletions(-) create mode 100644 examples/llava/README-gemma3.md create mode 100644 examples/llava/gemma3-cli.cpp create mode 100644 examples/llava/gemma3_convert_encoder_to_gguf.py diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 6358a94e9b..b5d95bd563 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -861,6 +861,9 @@ class Model: for token_id, token_data in added_tokens_decoder.items(): token_id = int(token_id) token: str = token_data["content"] + if token_id >= vocab_size: + logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}') + continue if toktypes[token_id] != SentencePieceTokenTypes.UNUSED: if tokens[token_id] != token.encode("utf-8"): logger.warning(f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token!r}') @@ -3322,6 +3325,83 @@ class Gemma2Model(Model): return [(self.map_tensor_name(name), data_torch)] +@Model.register("Gemma3ForCausalLM", "Gemma3ForConditionalGeneration") +class Gemma3Model(Model): + model_arch = gguf.MODEL_ARCH.GEMMA3 + has_vision: bool = False + + # we need to merge the text_config into the root level of hparams + def __init__(self, *args, **kwargs): + hparams = Model.load_hparams(kwargs["dir_model"]) + if "text_config" in hparams: + hparams = {**hparams, **hparams["text_config"]} + kwargs["hparams"] = hparams + super().__init__(*args, **kwargs) + if "vision_config" in hparams: + logger.info("Has vision encoder, but it will be ignored") + self.has_vision = True + + def write(self): + super().write() + if self.has_vision: + logger.info("NOTE: this script only convert the language model to GGUF") + logger.info(" for the vision model, please use gemma3_convert_encoder_to_gguf.py") + + def set_vocab(self): + self._set_vocab_sentencepiece() + + self.gguf_writer.add_add_space_prefix(False) + + def set_gguf_parameters(self): + hparams = self.hparams + block_count = hparams["num_hidden_layers"] + + # some default values are not specified in the hparams + self.gguf_writer.add_context_length(hparams.get("max_position_embeddings", 131072)) + self.gguf_writer.add_embedding_length(hparams["hidden_size"]) + self.gguf_writer.add_block_count(block_count) + self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"]) + self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 8)) + self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("rms_norm_eps", 1e-6)) + self.gguf_writer.add_key_length(hparams.get("head_dim", 256)) + self.gguf_writer.add_value_length(hparams.get("head_dim", 256)) + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 1_000_000.0)) # for global layers + # both attn_logit_softcapping and final_logit_softcapping are removed in Gemma3 + assert hparams.get("attn_logit_softcapping") is None + assert hparams.get("final_logit_softcapping") is None + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_head_count_kv(hparams.get("num_key_value_heads", 4)) + if hparams.get("rope_scaling") is not None: + assert hparams["rope_scaling"]["rope_type"] == "linear" + # important: this rope_scaling is only applied for global layers, and not used by 1B model + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR) + self.gguf_writer.add_rope_scaling_factor(hparams["rope_scaling"]["factor"]) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + del bid # unused + + if name.startswith("language_model."): + name = name.replace("language_model.", "") + elif name.startswith("multi_modal_projector.") or name.startswith("vision_tower.") \ + or name.startswith("multimodal_projector.") or name.startswith("vision_model."): # this is for old HF model, should be removed later + # ignore vision tensors + return [] + + # remove OOV (out-of-vocabulary) rows in token_embd + if "embed_tokens.weight" in name: + vocab = self._create_vocab_sentencepiece() + tokens = vocab[0] + data_torch = data_torch[:len(tokens)] + + # ref code in Gemma3RMSNorm + # output = output * (1.0 + self.weight.float()) + if name.endswith("norm.weight"): + data_torch = data_torch + 1 + + return [(self.map_tensor_name(name), data_torch)] + + @Model.register("Starcoder2ForCausalLM") class StarCoder2Model(Model): model_arch = gguf.MODEL_ARCH.STARCODER2 diff --git a/examples/llava/CMakeLists.txt b/examples/llava/CMakeLists.txt index 319effd199..f275ce1ccd 100644 --- a/examples/llava/CMakeLists.txt +++ b/examples/llava/CMakeLists.txt @@ -51,6 +51,13 @@ install(TARGETS ${TARGET} RUNTIME) target_link_libraries(${TARGET} PRIVATE common llava ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(${TARGET} PRIVATE cxx_std_17) +set(TARGET llama-gemma3-cli) +add_executable(${TARGET} gemma3-cli.cpp) +set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME llama-gemma3-cli) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llava ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) + set(TARGET llama-llava-clip-quantize-cli) add_executable(${TARGET} clip-quantize-cli.cpp) set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME llama-llava-clip-quantize-cli) diff --git a/examples/llava/README-gemma3.md b/examples/llava/README-gemma3.md new file mode 100644 index 0000000000..20bf73fb5c --- /dev/null +++ b/examples/llava/README-gemma3.md @@ -0,0 +1,30 @@ +# Gemma 3 vision + +> [!IMPORTANT] +> +> This is very experimental, only used for demo purpose. + +## How to get mmproj.gguf? + +```bash +cd gemma-3-4b-it +python ../llama.cpp/examples/llava/gemma3_convert_encoder_to_gguf.py . + +# output file is mmproj.gguf +``` + +## How to run it? + +What you need: +- The text model GGUF, can be converted using `convert_hf_to_gguf.py` +- The mmproj file from step above +- An image file + +```bash +# build +cmake -B build +cmake --build build --target llama-gemma3-cli + +# run it +./build/bin/llama-gemma3-cli -m {text_model}.gguf --mmproj mmproj.gguf --image your_image.jpg +``` diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 7f892beb6e..a1f050e39a 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -136,6 +136,8 @@ static std::string format(const char * fmt, ...) { #define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s" #define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s" #define TN_IMAGE_NEWLINE "model.image_newline" +#define TN_MM_INP_PROJ "mm.input_projection.weight" // gemma3 +#define TN_MM_SOFT_EMB_N "mm.soft_emb_norm.weight" // gemma3 #define TN_MINICPMV_POS_EMBD_K "resampler.pos_embed_k" #define TN_MINICPMV_QUERY "resampler.query" @@ -162,6 +164,7 @@ enum projector_type { PROJECTOR_TYPE_RESAMPLER, PROJECTOR_TYPE_GLM_EDGE, PROJECTOR_TYPE_MERGER, + PROJECTOR_TYPE_GEMMA3, PROJECTOR_TYPE_UNKNOWN, }; @@ -172,6 +175,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_RESAMPLER, "resampler"}, { PROJECTOR_TYPE_GLM_EDGE, "adapter"}, { PROJECTOR_TYPE_MERGER, "qwen2vl_merger"}, + { PROJECTOR_TYPE_GEMMA3, "gemma3"}, }; @@ -298,7 +302,7 @@ static projector_type clip_projector_type_from_string(const std::string & name) return kv.first; } } - return PROJECTOR_TYPE_UNKNOWN; + throw std::runtime_error(format("Unknown projector type: %s", name.c_str())); } #ifdef CLIP_DEBUG_FUNCTIONS @@ -555,6 +559,10 @@ struct clip_vision_model { struct ggml_tensor * mm_model_ln_kv_b; struct ggml_tensor * mm_model_ln_post_w; struct ggml_tensor * mm_model_ln_post_b; + + // gemma3 + struct ggml_tensor * mm_input_proj_w; + struct ggml_tensor * mm_soft_emb_norm_w; }; struct clip_ctx { @@ -569,7 +577,7 @@ struct clip_ctx { struct clip_vision_model vision_model; projector_type proj_type = PROJECTOR_TYPE_MLP; - int32_t max_feature_layer; + int32_t max_feature_layer; // unused in newer models like gemma3 float image_mean[3]; float image_std[3]; bool use_gelu = false; @@ -595,7 +603,7 @@ struct clip_ctx { ggml_backend_sched_ptr sched; - struct clip_image_size * load_image_size; + struct clip_image_size * load_image_size = nullptr; clip_ctx(clip_context_params & ctx_params) { backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); @@ -631,7 +639,159 @@ struct clip_ctx { } }; -static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch * imgs, struct clip_image_size * load_image_size, bool is_inf = false) { +static ggml_cgraph * clip_image_build_graph_siglip(clip_ctx * ctx, const clip_image_f32_batch * imgs) { + const auto & model = ctx->vision_model; + const auto & hparams = model.hparams; + + const int image_size = hparams.image_size; + int image_size_width = image_size; + int image_size_height = image_size; + + const int patch_size = hparams.patch_size; + const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); + const int hidden_size = hparams.hidden_size; + const int n_head = hparams.n_head; + const int d_head = hidden_size / n_head; + const int n_layer = hparams.n_layer; + const float eps = hparams.eps; + + GGML_ASSERT(imgs->size == 1); // batch_size == 1 + + struct ggml_init_params params = { + /*.mem_size =*/ ctx->buf_compute_meta.size(), + /*.mem_buffer =*/ ctx->buf_compute_meta.data(), + /*.no_alloc =*/ true, + }; + + struct ggml_context * ctx0 = ggml_init(params); + struct ggml_cgraph * gf = ggml_new_graph(ctx0); + + // input raw + struct ggml_tensor * inp_raw = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, image_size_width, image_size_height, 3); + ggml_set_name(inp_raw, "inp_raw"); + ggml_set_input(inp_raw); + + struct ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); + inp = ggml_reshape_2d(ctx0, inp, num_patches, hidden_size); + inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); + inp = ggml_add(ctx0, inp, model.patch_bias); + + // position embeddings + struct ggml_tensor * embeddings = ggml_add(ctx0, inp, model.position_embeddings); + + // loop over layers + for (int il = 0; il < n_layer; il++) { + struct ggml_tensor * cur = embeddings; // embeddings = residual, cur = hidden_states + + // layernorm1 + { + cur = ggml_norm(ctx0, cur, eps); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, model.layers[il].ln_1_w), model.layers[il].ln_1_b); + } + + // self-attention + { + + struct ggml_tensor * Q = + ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].q_w, cur), model.layers[il].q_b); + + Q = ggml_reshape_3d(ctx0, Q, d_head, n_head, num_patches); + Q = ggml_cont(ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3)); + + struct ggml_tensor * K = + ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].k_w, cur), model.layers[il].k_b); + + K = ggml_reshape_3d(ctx0, K, d_head, n_head, num_patches); + K = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3)); + + struct ggml_tensor * V = + ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].v_w, cur), model.layers[il].v_b); + + V = ggml_reshape_3d(ctx0, V, d_head, n_head, num_patches); + V = ggml_cont(ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3)); + + struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + KQ = ggml_scale_inplace(ctx0, KQ, 1.0f / sqrtf((float)d_head)); + KQ = ggml_soft_max_inplace(ctx0, KQ); + + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ); + KQV = ggml_reshape_3d(ctx0, KQV, d_head, num_patches, n_head); + KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3); + + cur = ggml_cont_2d(ctx0, KQV, hidden_size, num_patches); + } + + // attention output + cur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].o_w, cur), model.layers[il].o_b); + + // re-add the layer input, e.g., residual + cur = ggml_add(ctx0, cur, embeddings); + + embeddings = cur; // embeddings = residual, cur = hidden_states + + // layernorm2 + { + cur = ggml_norm(ctx0, cur, eps); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, model.layers[il].ln_2_w), model.layers[il].ln_2_b); + } + + cur = ggml_mul_mat(ctx0, model.layers[il].ff_i_w, cur); + cur = ggml_add(ctx0, cur, model.layers[il].ff_i_b); + + // siglip uses gelu + cur = ggml_gelu(ctx0, cur); + + cur = ggml_mul_mat(ctx0, model.layers[il].ff_o_w, cur); + cur = ggml_add(ctx0, cur, model.layers[il].ff_o_b); + + // residual 2 + cur = ggml_add(ctx0, embeddings, cur); + + embeddings = cur; + } + + // post-layernorm + if (ctx->has_post_norm) { + embeddings = ggml_norm(ctx0, embeddings, eps); + ggml_set_name(embeddings, "post_ln"); + + embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.post_ln_w), model.post_ln_b); + } + + if (ctx->proj_type == PROJECTOR_TYPE_GEMMA3) { + const int batch_size = 1; + const int mm_tokens_per_image = 256; // default value for gemma3 + const int tokens_per_side = sqrt(mm_tokens_per_image); + const int patches_per_image = sqrt(num_patches); + const int kernel_size = patches_per_image / tokens_per_side; + + embeddings = ggml_cont(ctx0, ggml_transpose(ctx0, embeddings)); + embeddings = ggml_reshape_4d(ctx0, embeddings, patches_per_image, patches_per_image, hidden_size, batch_size); + + // doing a pool2d to reduce the number of output tokens to 256 + embeddings = ggml_pool_2d(ctx0, embeddings, GGML_OP_POOL_AVG, kernel_size, kernel_size, kernel_size, kernel_size, 0, 0); + embeddings = ggml_reshape_3d(ctx0, embeddings, embeddings->ne[0] * embeddings->ne[0], hidden_size, batch_size); + embeddings = ggml_cont(ctx0, ggml_transpose(ctx0, embeddings)); + + // apply norm before projection + embeddings = ggml_rms_norm(ctx0, embeddings, eps); + embeddings = ggml_mul(ctx0, embeddings, model.mm_soft_emb_norm_w); + + // apply projection + embeddings = ggml_mul_mat(ctx0, + ggml_cont(ctx0, ggml_transpose(ctx0, model.mm_input_proj_w)), + embeddings); + } + + // build the graph + ggml_build_forward_expand(gf, embeddings); + + ggml_free(ctx0); + + return gf; +} + +static ggml_cgraph * clip_image_build_graph_legacy(clip_ctx * ctx, const clip_image_f32_batch * imgs, struct clip_image_size * load_image_size, bool is_inf = false) { if (!ctx->has_vision_encoder) { LOG_ERR("This gguf file seems to have no vision encoder\n"); return nullptr; @@ -1177,7 +1337,8 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 } else { GGML_ABORT("fatel error"); } - } else if (ctx->proj_type == PROJECTOR_TYPE_MERGER) { + } + else if (ctx->proj_type == PROJECTOR_TYPE_MERGER) { embeddings = ggml_reshape_3d(ctx0, embeddings, hidden_size * 4, num_positions / 4, batch_size); embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings); @@ -1199,6 +1360,15 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 return gf; } +static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch * imgs, struct clip_image_size * load_image_size, bool is_inf = false) { + if (ctx->proj_type == PROJECTOR_TYPE_GEMMA3) { + return clip_image_build_graph_siglip(ctx, imgs); + } else { + // TODO: we should have one build_* function per model + return clip_image_build_graph_legacy(ctx, imgs, load_image_size, is_inf); + } +} + // read and create ggml_context containing the tensors and their data struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { return clip_init(fname, clip_context_params{ @@ -1358,8 +1528,12 @@ struct clip_ctx * clip_init(const char * fname, struct clip_context_params ctx_p GGML_ASSERT(new_clip->has_vision_encoder); GGML_ASSERT(!new_clip->has_text_encoder); - idx = get_key_idx(ctx, KEY_USE_GELU); - new_clip->use_gelu = gguf_get_val_bool(ctx, idx); + try { + idx = get_key_idx(ctx, KEY_USE_GELU); + new_clip->use_gelu = gguf_get_val_bool(ctx, idx); + } catch (std::runtime_error & /*e*/) { + new_clip->use_gelu = false; + } try { idx = get_key_idx(ctx, KEY_USE_SILU); @@ -1567,11 +1741,17 @@ struct clip_ctx * clip_init(const char * fname, struct clip_context_params ctx_p } try { - vision_model.patch_embeddings_0 = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD); + vision_model.patch_embeddings_0 = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD); + } catch(const std::exception& /*e*/) { + vision_model.patch_embeddings_0 = nullptr; + } + + try { vision_model.position_embeddings = get_tensor(new_clip->ctx_data, format(TN_POS_EMBD, "v")); } catch(const std::exception& /*e*/) { - LOG_ERR("%s: failed to load vision model tensors\n", __func__); + vision_model.position_embeddings = nullptr; } + try { vision_model.patch_embeddings_1 = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD_1); } catch(const std::exception& /*e*/) { @@ -1682,6 +1862,10 @@ struct clip_ctx * clip_init(const char * fname, struct clip_context_params ctx_p vision_model.mm_1_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "weight")); vision_model.mm_1_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "bias")); } + else if (new_clip->proj_type == PROJECTOR_TYPE_GEMMA3) { + vision_model.mm_input_proj_w = get_tensor(new_clip->ctx_data, TN_MM_INP_PROJ); + vision_model.mm_soft_emb_norm_w = get_tensor(new_clip->ctx_data, TN_MM_SOFT_EMB_N); + } else { std::string proj_type = PROJECTOR_TYPE_NAMES[new_clip->proj_type]; throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str())); @@ -2223,7 +2407,7 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli return true; } - if (ctx->has_glm_projector) { + if (ctx->has_glm_projector || ctx->proj_type == PROJECTOR_TYPE_GEMMA3) { res_imgs->size = 1; res_imgs->data = new clip_image_f32[res_imgs->size]; clip_image_u8 resized_image; @@ -2748,6 +2932,9 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions)); free(positions_data); } + else if (ctx->proj_type == PROJECTOR_TYPE_GEMMA3) { + // do nothing + } else { struct ggml_tensor * positions = ggml_graph_get_tensor(gf, "positions"); @@ -2960,6 +3147,9 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { if (ctx->proj_type == PROJECTOR_TYPE_MERGER) { return ctx->vision_model.mm_1_b->ne[0]; } + if (ctx->proj_type == PROJECTOR_TYPE_GEMMA3) { + return ctx->vision_model.mm_input_proj_w->ne[0]; + } std::string proj_type = PROJECTOR_TYPE_NAMES[ctx->proj_type]; throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str())); diff --git a/examples/llava/gemma3-cli.cpp b/examples/llava/gemma3-cli.cpp new file mode 100644 index 0000000000..a07864d4e5 --- /dev/null +++ b/examples/llava/gemma3-cli.cpp @@ -0,0 +1,341 @@ +#include "arg.h" +#include "log.h" +#include "common.h" +#include "sampling.h" +#include "clip.h" +#include "stb_image.h" +#include "llama.h" +#include "ggml.h" +#include "console.h" + +#include +#include +#include + +#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) +#include +#include +#elif defined (_WIN32) +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#endif + +static bool g_is_generating = false; + +/** + * Please note that this is NOT a production-ready stuff. + * It is a playground for trying Gemma 3 vision capabilities. + * For contributors: please keep this code simple and easy to understand. + */ + +static void show_additional_info(int /*argc*/, char ** argv) { + LOG( + "Experimental CLI for using Gemma 3 vision model\n\n" + "Usage: %s [options] -m --mmproj --image -p \n\n" + " -m and --mmproj are required\n" + " --image and -p are optional, if NOT provided, the CLI will run in chat mode\n", + argv[0] + ); +} + +#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) +static void sigint_handler(int signo) { + if (signo == SIGINT) { + if (g_is_generating) { + g_is_generating = false; + } else { + console::cleanup(); + LOG("\nInterrupted by user\n"); + _exit(130); + } + } +} +#endif + +struct gemma3_context { + struct clip_ctx * ctx_clip = NULL; + common_init_result llama_init; + + llama_model * model; + llama_context * lctx; + const llama_vocab * vocab; + llama_batch batch; + + int n_threads = 1; + llama_pos n_past = 0; + + gemma3_context(common_params & params) : llama_init(common_init_from_params(params)) { + model = llama_init.model.get(); + lctx = llama_init.context.get(); + vocab = llama_model_get_vocab(model); + n_threads = params.cpuparams.n_threads; + batch = llama_batch_init(params.n_batch, 0, 1); + init_clip_model(params); + } + + void init_clip_model(common_params & params) { + const char * clip_path = params.mmproj.c_str(); + ctx_clip = clip_model_load(clip_path, params.verbosity > 1); + } + + ~gemma3_context() { + clip_free(ctx_clip); + } +}; + +struct decode_embd_batch { + std::vector pos; + std::vector n_seq_id; + std::vector seq_id_0; + std::vector seq_ids; + std::vector logits; + llama_batch batch; + decode_embd_batch(float * embd, int32_t n_tokens, llama_pos pos_0, llama_seq_id seq_id) { + pos .resize(n_tokens); + n_seq_id.resize(n_tokens); + seq_ids .resize(n_tokens + 1); + logits .resize(n_tokens); + seq_id_0.resize(1); + seq_id_0[0] = seq_id; + seq_ids [n_tokens] = nullptr; + batch = { + /*n_tokens =*/ n_tokens, + /*tokens =*/ nullptr, + /*embd =*/ embd, + /*pos =*/ pos.data(), + /*n_seq_id =*/ n_seq_id.data(), + /*seq_id =*/ seq_ids.data(), + /*logits =*/ logits.data(), + }; + for (int i = 0; i < n_tokens; i++) { + batch.pos [i] = pos_0 + i; + batch.n_seq_id[i] = 1; + batch.seq_id [i] = seq_id_0.data(); + batch.logits [i] = false; + } + } +}; + +static int eval_text(gemma3_context & ctx, std::string input, bool logits_last = false) { + llama_tokens tokens = common_tokenize(ctx.lctx, input, false, true); + common_batch_clear(ctx.batch); + for (llama_token & t : tokens) { + common_batch_add(ctx.batch, t, ctx.n_past++, {0}, false); + } + if (logits_last) { + ctx.batch.logits[ctx.batch.n_tokens - 1] = true; + } + // LOG("eval_text (n_tokens = %d): %s\n", (int)tokens.size(), input.c_str()); + if (llama_decode(ctx.lctx, ctx.batch)) { + LOG_ERR("Failed to decode text\n"); + return 1; + } + return 0; +} + +static int eval_image(gemma3_context & ctx, std::string & fname) { + std::vector image_embd_v; + int n_embd = llama_model_n_embd(ctx.model); + int n_tokens = 256; + image_embd_v.resize(n_tokens * n_embd); + + bool ok; + struct clip_image_u8 * img_u8 = clip_image_u8_init(); + ok = clip_image_load_from_file(fname.c_str(), img_u8); + if (!ok) { + LOG_ERR("Unable to load image %s\n", fname.c_str()); + clip_image_u8_free(img_u8); + return 2; // non-fatal error + } + + clip_image_f32_batch batch_f32; + ok = clip_image_preprocess(ctx.ctx_clip, img_u8, &batch_f32); + if (!ok) { + LOG_ERR("Unable to preprocess image\n"); + clip_image_f32_batch_free(&batch_f32); + clip_image_u8_free(img_u8); + return 1; + } + + int64_t t0 = ggml_time_ms(); + LOG("Encoding image %s\n", fname.c_str()); + ok = clip_image_batch_encode(ctx.ctx_clip, ctx.n_threads, &batch_f32, image_embd_v.data()); + if (!ok) { + LOG_ERR("Unable to encode image\n"); + clip_image_f32_batch_free(&batch_f32); + clip_image_u8_free(img_u8); + return 1; + } + LOG("Image encoded in %" PRId64 " ms\n", ggml_time_ms() - t0); + + clip_image_f32_batch_free(&batch_f32); + clip_image_u8_free(img_u8); + + // decode image embeddings + int64_t t1 = ggml_time_ms(); + eval_text(ctx, ""); + llama_set_causal_attn(ctx.lctx, false); + decode_embd_batch batch_img(image_embd_v.data(), n_tokens, ctx.n_past, 0); + if (llama_decode(ctx.lctx, batch_img.batch)) { + LOG_ERR("failed to decode image\n"); + return 1; + } + ctx.n_past += n_tokens; + llama_set_causal_attn(ctx.lctx, true); + eval_text(ctx, ""); + LOG("Image decoded in %" PRId64 " ms\n", ggml_time_ms() - t1); + return 0; +} + +static int generate_response(gemma3_context & ctx, common_sampler * smpl, int n_predict) { + for (int i = 0; i < n_predict; i++) { + if (i > n_predict || !g_is_generating) { + printf("\n"); + break; + } + + llama_token token_id = common_sampler_sample(smpl, ctx.lctx, -1); + common_sampler_accept(smpl, token_id, true); + + if (llama_vocab_is_eog(ctx.vocab, token_id)) { + printf("\n"); + break; // end of generation + } + + printf("%s", common_token_to_piece(ctx.lctx, token_id).c_str()); + fflush(stdout); + + // eval the token + common_batch_clear(ctx.batch); + common_batch_add(ctx.batch, token_id, ctx.n_past++, {0}, true); + if (llama_decode(ctx.lctx, ctx.batch)) { + LOG_ERR("failed to decode token\n"); + return 1; + } + } + return 0; +} + +int main(int argc, char ** argv) { + ggml_time_init(); + + common_params params; + params.sampling.temp = 0.2; // lower temp by default for better quality + + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_LLAVA, show_additional_info)) { + return 1; + } + + common_init(); + + if (params.mmproj.empty()) { + show_additional_info(argc, argv); + return 1; + } + + gemma3_context ctx(params); + printf("%s: %s\n", __func__, params.model.c_str()); + + bool is_single_turn = !params.prompt.empty() && !params.image.empty(); + + struct common_sampler * smpl = common_sampler_init(ctx.model, params.sampling); + int n_predict = params.n_predict < 0 ? INT_MAX : params.n_predict; + + // ctrl+C handling + { +#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) + struct sigaction sigint_action; + sigint_action.sa_handler = sigint_handler; + sigemptyset (&sigint_action.sa_mask); + sigint_action.sa_flags = 0; + sigaction(SIGINT, &sigint_action, NULL); +#elif defined (_WIN32) + auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL { + return (ctrl_type == CTRL_C_EVENT) ? (sigint_handler(SIGINT), true) : false; + }; + SetConsoleCtrlHandler(reinterpret_cast(console_ctrl_handler), true); +#endif + } + + if (eval_text(ctx, "")) { + return 1; + } + + if (is_single_turn) { + g_is_generating = true; + if (eval_text(ctx, "user\n")) { + return 1; + } + for (auto & fname : params.image) { + if (eval_image(ctx, fname)) { + return 1; + } + } + if (eval_text(ctx, params.prompt + "model\n", true)) { + return 1; + } + if (generate_response(ctx, smpl, n_predict)) { + return 1; + } + + } else { + LOG("\n Running in chat mode, available commands:"); + LOG("\n /image load an image"); + LOG("\n /clear clear the chat history"); + LOG("\n /quit or /exit exit the program"); + LOG("\n"); + + if (eval_text(ctx, "user\n")) { + return 1; + } + + while (true) { + g_is_generating = false; + LOG("\n> "); + console::set_display(console::user_input); + std::string line; + console::readline(line, false); + console::set_display(console::reset); + line = string_strip(line); + if (line.empty()) { + continue; + } + if (line == "/quit" || line == "/exit") { + break; + } + if (line == "/clear") { + ctx.n_past = 0; + llama_kv_cache_seq_rm(ctx.lctx, 0, 1, -1); // keep BOS + LOG("Chat history cleared\n\n"); + continue; + } + g_is_generating = true; + if (line.find("/image") == 0) { + std::string image = line.substr(7); + int res = eval_image(ctx, image); + if (res == 2) { + continue; // image not found + } + if (res) { + return 1; + } + continue; + } + if (eval_text(ctx, line + "model\n", true)) { + return 1; + } + if (generate_response(ctx, smpl, n_predict)) { + return 1; + } + if (eval_text(ctx, "user\n")) { + return 1; + } + } + } + + return 0; +} diff --git a/examples/llava/gemma3_convert_encoder_to_gguf.py b/examples/llava/gemma3_convert_encoder_to_gguf.py new file mode 100644 index 0000000000..241b526b9e --- /dev/null +++ b/examples/llava/gemma3_convert_encoder_to_gguf.py @@ -0,0 +1,307 @@ +import gguf +import argparse +import logging +import sys +import torch +import json +import os +import numpy as np +from typing import cast, ContextManager, Any, Iterator +from pathlib import Path +from torch import Tensor + +logger = logging.getLogger("gemma3-mmproj") + + +# (copied from convert_hf_to_gguf.py) +# tree of lazy tensors +class LazyTorchTensor(gguf.LazyBase): + _tensor_type = torch.Tensor + # to keep the type-checker happy + dtype: torch.dtype + shape: torch.Size + + # only used when converting a torch.Tensor to a np.ndarray + _dtype_map: dict[torch.dtype, type] = { + torch.float16: np.float16, + torch.float32: np.float32, + } + + # used for safetensors slices + # ref: https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/src/lib.rs#L1046 + # TODO: uncomment U64, U32, and U16, ref: https://github.com/pytorch/pytorch/issues/58734 + _dtype_str_map: dict[str, torch.dtype] = { + "F64": torch.float64, + "F32": torch.float32, + "BF16": torch.bfloat16, + "F16": torch.float16, + # "U64": torch.uint64, + "I64": torch.int64, + # "U32": torch.uint32, + "I32": torch.int32, + # "U16": torch.uint16, + "I16": torch.int16, + "U8": torch.uint8, + "I8": torch.int8, + "BOOL": torch.bool, + "F8_E4M3": torch.float8_e4m3fn, + "F8_E5M2": torch.float8_e5m2, + } + + def numpy(self) -> gguf.LazyNumpyTensor: + dtype = self._dtype_map[self.dtype] + return gguf.LazyNumpyTensor( + meta=gguf.LazyNumpyTensor.meta_with_dtype_and_shape(dtype, self.shape), + args=(self,), + func=(lambda s: s.numpy()) + ) + + @classmethod + def meta_with_dtype_and_shape(cls, dtype: torch.dtype, shape: tuple[int, ...]) -> Tensor: + return torch.empty(size=shape, dtype=dtype, device="meta") + + @classmethod + def from_safetensors_slice(cls, st_slice: Any) -> Tensor: + dtype = cls._dtype_str_map[st_slice.get_dtype()] + shape: tuple[int, ...] = tuple(st_slice.get_shape()) + lazy = cls(meta=cls.meta_with_dtype_and_shape(dtype, shape), args=(st_slice,), func=lambda s: s[:]) + return cast(torch.Tensor, lazy) + + @classmethod + def __torch_function__(cls, func, types, args=(), kwargs=None): + del types # unused + + if kwargs is None: + kwargs = {} + + if func is torch.Tensor.numpy: + return args[0].numpy() + + return cls._wrap_fn(func)(*args, **kwargs) + + +class Gemma3VisionTower: + hparams: dict + gguf_writer: gguf.GGUFWriter + fname_out: Path + ftype: gguf.LlamaFileType + + @staticmethod + def load_hparams(dir_model: Path): + with open(dir_model / "config.json", "r", encoding="utf-8") as f: + return json.load(f) + + @staticmethod + def get_model_part_names(dir_model: Path, prefix: str, suffix: str) -> list[str]: + part_names: list[str] = [] + for filename in os.listdir(dir_model): + if filename.startswith(prefix) and filename.endswith(suffix): + part_names.append(filename) + part_names.sort() + return part_names + + def __init__(self, + dir_model: Path, + fname_out: Path, + ftype: gguf.LlamaFileType, + is_big_endian: bool,): + hparams = Gemma3VisionTower.load_hparams(dir_model) + self.hparams = hparams + self.fname_out = fname_out + self.ftype = ftype + endianess = gguf.GGUFEndian.BIG if is_big_endian else gguf.GGUFEndian.LITTLE + self.gguf_writer = gguf.GGUFWriter(path=None, arch="clip", endianess=endianess) + + text_config = hparams["text_config"] + vision_config = hparams["vision_config"] + + assert hparams["architectures"][0] == "Gemma3ForConditionalGeneration" + assert text_config is not None + assert vision_config is not None + + self.gguf_writer.add_string ("clip.projector_type", "gemma3") + self.gguf_writer.add_bool ("clip.has_text_encoder", False) + self.gguf_writer.add_bool ("clip.has_vision_encoder", True) + self.gguf_writer.add_bool ("clip.has_llava_projector", False) # legacy + self.gguf_writer.add_uint32 ("clip.vision.image_size", vision_config["image_size"]) + self.gguf_writer.add_uint32 ("clip.vision.patch_size", vision_config["patch_size"]) + self.gguf_writer.add_uint32 ("clip.vision.embedding_length", vision_config["hidden_size"]) + self.gguf_writer.add_uint32 ("clip.vision.feed_forward_length", vision_config["intermediate_size"]) + self.gguf_writer.add_uint32 ("clip.vision.projection_dim", text_config["hidden_size"]) + self.gguf_writer.add_uint32 ("clip.vision.block_count", vision_config["num_hidden_layers"]) + self.gguf_writer.add_uint32 ("clip.vision.attention.head_count", vision_config["num_attention_heads"]) + self.gguf_writer.add_float32("clip.vision.attention.layer_norm_epsilon", vision_config.get("layer_norm_eps", 1e-6)) + # default values taken from HF tranformers code + self.gguf_writer.add_array ("clip.vision.image_mean", [0.5, 0.5, 0.5]) + self.gguf_writer.add_array ("clip.vision.image_std", [0.5, 0.5, 0.5]) + self.gguf_writer.add_bool ("clip.use_gelu", True) + + # load tensors + for name, data_torch in self.get_tensors(dir_model): + # convert any unsupported data types to float32 + if data_torch.dtype not in (torch.float16, torch.float32): + data_torch = data_torch.to(torch.float32) + self.add_tensor(name, data_torch) + + def get_tensors(self, dir_model: Path) -> Iterator[tuple[str, Tensor]]: + part_names = Gemma3VisionTower.get_model_part_names(dir_model, "model", ".safetensors") + tensor_names_from_parts: set[str] = set() + for part_name in part_names: + logger.info(f"gguf: loading model part '{part_name}'") + from safetensors import safe_open + ctx = cast(ContextManager[Any], safe_open(dir_model / part_name, framework="pt", device="cpu")) + with ctx as model_part: + tensor_names_from_parts.update(model_part.keys()) + + for name in model_part.keys(): + data = model_part.get_slice(name) + data = LazyTorchTensor.from_safetensors_slice(data) + yield name, data + + def add_tensor(self, name: str, data_torch: Tensor): + is_1d = len(data_torch.shape) == 1 + is_embd = ".embeddings." in name + old_dtype = data_torch.dtype + can_quantize = not is_1d and not is_embd + data_qtype = gguf.GGMLQuantizationType.F32 + + # this is to support old checkpoint + # TODO: remove this when we have the final model + name = name.replace("vision_model.vision_model.", "vision_tower.vision_model.") + name = name.replace("multimodal_projector.", "multi_modal_projector.") + + # filter only vision tensors + if not name.startswith("vision_tower.vision_model.") and not name.startswith("multi_modal_projector."): + return + # prefix + name = name.replace("vision_tower.vision_model.encoder.layers.", "v.blk.") + name = name.replace("vision_tower.vision_model.", "v.") + # projector and input embd + name = name.replace(".embeddings.patch_embedding.", ".patch_embd.") + name = name.replace(".embeddings.position_embedding.", ".position_embd.") + name = name.replace( + "multi_modal_projector.mm_input_projection_weight", + "mm.input_projection.weight" + ) + name = name.replace( + "multi_modal_projector.mm_soft_emb_norm.weight", + "mm.soft_emb_norm.weight" + ) + name = name.replace("post_layernorm.", "post_ln.") + # each block + name = name.replace(".self_attn.k_proj.", ".attn_k.") + name = name.replace(".self_attn.v_proj.", ".attn_v.") + name = name.replace(".self_attn.q_proj.", ".attn_q.") + name = name.replace(".self_attn.out_proj.", ".attn_out.") + name = name.replace(".layer_norm1.", ".ln1.") + name = name.replace(".layer_norm2.", ".ln2.") + name = name.replace(".mlp.fc1.", ".ffn_down.") + name = name.replace(".mlp.fc2.", ".ffn_up.") + + if can_quantize: + if self.ftype == gguf.LlamaFileType.ALL_F32: + data_qtype = gguf.GGMLQuantizationType.F32 + elif self.ftype == gguf.LlamaFileType.MOSTLY_F16: + data_qtype = gguf.GGMLQuantizationType.F16 + elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16: + data_qtype = gguf.GGMLQuantizationType.BF16 + elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0: + data_qtype = gguf.GGMLQuantizationType.Q8_0 + else: + raise ValueError(f"Unsupported file type: {self.ftype}") + + # corrent norm value ; only this "soft_emb_norm" need to be corrected as it's part of Gemma projector + # the other norm values are part of SigLIP model, and they are already correct + # ref code: Gemma3RMSNorm + if "soft_emb_norm.weight" in name: + logger.info(f"Correcting norm value for '{name}'") + data_torch = data_torch + 1 + + data = data_torch.numpy() + + try: + data = gguf.quants.quantize(data, data_qtype) + except Exception as e: + logger.error(f"Error quantizing tensor '{name}': {e}, fallback to F16") + data_qtype = gguf.GGMLQuantizationType.F16 + data = gguf.quants.quantize(data, data_qtype) + + # reverse shape to make it similar to the internal ggml dimension order + shape_str = f"{{{', '.join(str(n) for n in reversed(data_torch.shape))}}}" + logger.info(f"{f'%-32s' % f'{name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}") + + self.gguf_writer.add_tensor(name, data, raw_dtype=data_qtype) + + def write(self): + self.gguf_writer.write_header_to_file(path=self.fname_out) + self.gguf_writer.write_kv_data_to_file() + self.gguf_writer.write_tensors_to_file(progress=True) + self.gguf_writer.close() + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Convert Gemma 3 vision tower safetensors to GGUF format",) + parser.add_argument( + "--outfile", type=Path, default="mmproj.gguf", + help="path to write to", + ) + parser.add_argument( + "--outtype", type=str, choices=["f32", "f16", "bf16", "q8_0"], default="f16", + help="output format", + ) + parser.add_argument( + "--bigendian", action="store_true", + help="model is executed on big endian machine", + ) + parser.add_argument( + "model", type=Path, + help="directory containing model file", + nargs="?", + ) + parser.add_argument( + "--verbose", action="store_true", + help="increase output verbosity", + ) + + args = parser.parse_args() + if args.model is None: + parser.error("the following arguments are required: model") + return args + + +def main() -> None: + args = parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + dir_model = args.model + + if not dir_model.is_dir(): + logger.error(f'Error: {args.model} is not a directory') + sys.exit(1) + + ftype_map: dict[str, gguf.LlamaFileType] = { + "f32": gguf.LlamaFileType.ALL_F32, + "f16": gguf.LlamaFileType.MOSTLY_F16, + "bf16": gguf.LlamaFileType.MOSTLY_BF16, + "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0, + } + + logger.info(f"Loading model: {dir_model.name}") + + with torch.inference_mode(): + gemma3_vision_tower = Gemma3VisionTower( + dir_model=dir_model, + fname_out=args.outfile, + ftype=ftype_map[args.outtype], + is_big_endian=args.bigendian, + ) + gemma3_vision_tower.write() + + +if __name__ == '__main__': + main() + diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index ecac5b4bb7..19624eae04 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -253,6 +253,7 @@ class MODEL_ARCH(IntEnum): MINICPM3 = auto() GEMMA = auto() GEMMA2 = auto() + GEMMA3 = auto() STARCODER2 = auto() RWKV6 = auto() RWKV6QWEN2 = auto() @@ -440,6 +441,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.MINICPM3: "minicpm3", MODEL_ARCH.GEMMA: "gemma", MODEL_ARCH.GEMMA2: "gemma2", + MODEL_ARCH.GEMMA3: "gemma3", MODEL_ARCH.STARCODER2: "starcoder2", MODEL_ARCH.RWKV6: "rwkv6", MODEL_ARCH.RWKV6QWEN2: "rwkv6qwen2", @@ -1077,6 +1079,23 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_PRE_NORM, MODEL_TENSOR.FFN_POST_NORM, ], + MODEL_ARCH.GEMMA3: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.FFN_PRE_NORM, + MODEL_TENSOR.FFN_POST_NORM, + ], MODEL_ARCH.STARCODER2: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 97a1e7e5e0..28f2bbc8f7 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -36,6 +36,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_MINICPM3, "minicpm3" }, { LLM_ARCH_GEMMA, "gemma" }, { LLM_ARCH_GEMMA2, "gemma2" }, + { LLM_ARCH_GEMMA3, "gemma3" }, { LLM_ARCH_STARCODER2, "starcoder2" }, { LLM_ARCH_MAMBA, "mamba" }, { LLM_ARCH_XVERSE, "xverse" }, @@ -766,6 +767,26 @@ static const std::map> LLM_TENSOR_N { LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" }, }, }, + { + LLM_ARCH_GEMMA3, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_ATTN_POST_NORM, "blk.%d.post_attention_norm" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + { LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" }, + }, + }, { LLM_ARCH_STARCODER2, { diff --git a/src/llama-arch.h b/src/llama-arch.h index 122fdcebe0..2ec2e2362e 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -40,6 +40,7 @@ enum llm_arch { LLM_ARCH_MINICPM3, LLM_ARCH_GEMMA, LLM_ARCH_GEMMA2, + LLM_ARCH_GEMMA3, LLM_ARCH_STARCODER2, LLM_ARCH_MAMBA, LLM_ARCH_XVERSE, diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 1da4eae7e6..9f75589d80 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -864,6 +865,23 @@ void llama_model::load_hparams(llama_model_loader & ml) { default: type = LLM_TYPE_UNKNOWN; } } break; + case LLM_ARCH_GEMMA3: + { + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + switch (hparams.n_layer) { + case 26: type = LLM_TYPE_1B; break; + case 34: type = LLM_TYPE_4B; break; + case 48: type = LLM_TYPE_12B; break; + case 62: type = LLM_TYPE_27B; break; + default: type = LLM_TYPE_UNKNOWN; + } + + hparams.f_attention_scale = type == LLM_TYPE_27B + ? 1.0f / std::sqrt(float(hparams.n_embd / hparams.n_head(0))) + : 1.0f / std::sqrt(float(hparams.n_embd_head_k)); + } break; case LLM_ARCH_STARCODER2: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); @@ -2454,6 +2472,35 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0); + } + } break; + case LLM_ARCH_GEMMA3: + { + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); // same as tok_embd, duplicated to allow offloading + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0); + layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0); + layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); @@ -3650,6 +3697,7 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: f_clamp_kqv = %.1e\n", __func__, hparams.f_clamp_kqv); LLAMA_LOG_INFO("%s: f_max_alibi_bias = %.1e\n", __func__, hparams.f_max_alibi_bias); LLAMA_LOG_INFO("%s: f_logit_scale = %.1e\n", __func__, hparams.f_logit_scale); + LLAMA_LOG_INFO("%s: f_attn_scale = %.1e\n", __func__, hparams.f_attention_scale); LLAMA_LOG_INFO("%s: n_ff = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_ff(il); }, hparams.n_layer).c_str()); LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert); LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used); @@ -3923,6 +3971,7 @@ enum llama_rope_type llama_model_rope_type(const struct llama_model * model) { case LLM_ARCH_PHIMOE: case LLM_ARCH_GEMMA: case LLM_ARCH_GEMMA2: + case LLM_ARCH_GEMMA3: case LLM_ARCH_STARCODER2: case LLM_ARCH_OPENELM: case LLM_ARCH_GPTNEOX: diff --git a/src/llama.cpp b/src/llama.cpp index 607f278615..4a4e914901 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -4978,6 +4978,149 @@ struct llm_build_context { return gf; } + struct ggml_cgraph * build_gemma3() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); + + const int64_t n_embd_head_k = hparams.n_embd_head_k; + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); + + // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings) + if (ubatch.token) { + inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); + cb(inpL, "inp_scaled", -1); + } + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = build_inp_pos(); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + // gemma3 requires different mask for layers using sliding window (SWA) + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(true); + struct ggml_tensor * KQ_mask_swa = build_inp_KQ_mask_swa(true); + + // "5-to-1 interleaved attention" + // 5 layers of local attention followed by 1 layer of global attention + static const int sliding_window_pattern = 6; + + for (int il = 0; il < n_layer; ++il) { + const bool is_sliding = (il + 1) % sliding_window_pattern; + const float freq_base_l = is_sliding ? 10000.0f : freq_base; + const float freq_scale_l = is_sliding ? 1.0f : freq_scale; + struct ggml_tensor * KQ_mask_l = is_sliding ? KQ_mask_swa : KQ_mask; + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens); + Qcur = llm_build_norm(ctx0, Qcur, hparams, + model.layers[il].attn_q_norm, + NULL, + LLM_NORM_RMS, cb, il); + cb(Qcur, "Qcur_normed", il); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens); + Kcur = llm_build_norm(ctx0, Kcur, hparams, + model.layers[il].attn_k_norm, + NULL, + LLM_NORM_RMS, cb, il); + cb(Kcur, "Kcur_normed", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, lctx, kv_self, gf, + model.layers[il].wo, NULL, + Kcur, Vcur, Qcur, KQ_mask_l, n_tokens, kv_head, n_kv, hparams.f_attention_scale, cb, il); + } + + cur = llm_build_norm(ctx0, cur, hparams, + model.layers[il].attn_post_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_post_norm", il); + + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); + cb(sa_out, "sa_out", il); + + cur = llm_build_norm(ctx0, sa_out, hparams, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm", il); + + // feed-forward network + { + cur = llm_build_ffn(ctx0, lctx, cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + } + + cur = llm_build_norm(ctx0, cur, hparams, + model.layers[il].ffn_post_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "ffn_post_norm", -1); + + cur = ggml_add(ctx0, cur, sa_out); + cur = lctx.cvec.apply_to(ctx0, cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head + cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); + + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } struct ggml_cgraph * build_starcoder2() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); @@ -8298,6 +8441,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_gemma2(); } break; + case LLM_ARCH_GEMMA3: + { + result = llm.build_gemma3(); + } break; case LLM_ARCH_STARCODER2: { result = llm.build_starcoder2(); From 34c961b181836a4f06ab4c56d5ce61ce03fc478b Mon Sep 17 00:00:00 2001 From: uvos Date: Wed, 12 Mar 2025 10:14:11 +0100 Subject: [PATCH 123/188] CUDA/HIP: Fix fattn-vec-* when device warp size is not 32 (#12315) When fattn-wmma was ported over to warp64 various bits that also touch fattn-vec where converted to selectable warp size, however the fattn-vec kernels dont work with 64 wide warps for now, so we need to avoid launching them with parameters for warp64 --- ggml/src/ggml-cuda/fattn-common.cuh | 52 ++++++++++++---------------- ggml/src/ggml-cuda/fattn-wmma-f16.cu | 7 ++-- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 46de140935..4067fd41bc 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -52,12 +52,11 @@ typedef half (*vec_dot_KQ_f16_t)( typedef float (*vec_dot_KQ_f32_t)( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds); -template +template static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q4_0( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { const block_q4_0 * K_q4_0 = (const block_q4_0 *) K_c; - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); GGML_UNUSED(Q_v); T sum = 0.0f; @@ -93,12 +92,11 @@ static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q4_0( return sum; } -template +template static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q4_1( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { const block_q4_1 * K_q4_1 = (const block_q4_1 *) K_c; - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); GGML_UNUSED(Q_v); T sum = 0.0f; @@ -138,12 +136,11 @@ static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q4_1( return sum; } -template +template static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q5_0( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { const block_q5_0 * K_q5_0 = (const block_q5_0 *) K_c; - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); GGML_UNUSED(Q_v); T sum = 0.0f; @@ -186,12 +183,11 @@ static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q5_0( return sum; } -template +template static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q5_1( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { const block_q5_1 * K_q5_1 = (const block_q5_1 *) K_c; - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); GGML_UNUSED(Q_v); T sum = 0.0f; @@ -238,12 +234,11 @@ static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q5_1( return sum; } -template +template static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q8_0( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { const block_q8_0 * K_q8_0 = (const block_q8_0 *) K_c; - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); GGML_UNUSED(Q_v); T sum = 0.0f; @@ -272,12 +267,11 @@ static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_q8_0( return sum; } -template +template static __device__ __forceinline__ T vec_dot_fattn_vec_KQ_f16( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds_v) { const half2 * K_h2 = (const half2 *) K_c; - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); GGML_UNUSED(Q_q8); GGML_UNUSED(Q_ds_v); @@ -480,25 +474,25 @@ static __device__ __forceinline__ T dequantize_1_f16(const void * __restrict__ v return x[i]; } -template +template constexpr __device__ vec_dot_KQ_f16_t get_vec_dot_KQ_f16(ggml_type type_K) { - return type_K == GGML_TYPE_Q4_0 ? vec_dot_fattn_vec_KQ_q4_0 : - type_K == GGML_TYPE_Q4_1 ? vec_dot_fattn_vec_KQ_q4_1 : - type_K == GGML_TYPE_Q5_0 ? vec_dot_fattn_vec_KQ_q5_0 : - type_K == GGML_TYPE_Q5_1 ? vec_dot_fattn_vec_KQ_q5_1 : - type_K == GGML_TYPE_Q8_0 ? vec_dot_fattn_vec_KQ_q8_0 : - type_K == GGML_TYPE_F16 ? vec_dot_fattn_vec_KQ_f16 : + return type_K == GGML_TYPE_Q4_0 ? vec_dot_fattn_vec_KQ_q4_0 : + type_K == GGML_TYPE_Q4_1 ? vec_dot_fattn_vec_KQ_q4_1 : + type_K == GGML_TYPE_Q5_0 ? vec_dot_fattn_vec_KQ_q5_0 : + type_K == GGML_TYPE_Q5_1 ? vec_dot_fattn_vec_KQ_q5_1 : + type_K == GGML_TYPE_Q8_0 ? vec_dot_fattn_vec_KQ_q8_0 : + type_K == GGML_TYPE_F16 ? vec_dot_fattn_vec_KQ_f16 : nullptr; } -template +template constexpr __device__ vec_dot_KQ_f32_t get_vec_dot_KQ_f32(ggml_type type_K) { - return type_K == GGML_TYPE_Q4_0 ? vec_dot_fattn_vec_KQ_q4_0 : - type_K == GGML_TYPE_Q4_1 ? vec_dot_fattn_vec_KQ_q4_1 : - type_K == GGML_TYPE_Q5_0 ? vec_dot_fattn_vec_KQ_q5_0 : - type_K == GGML_TYPE_Q5_1 ? vec_dot_fattn_vec_KQ_q5_1 : - type_K == GGML_TYPE_Q8_0 ? vec_dot_fattn_vec_KQ_q8_0 : - type_K == GGML_TYPE_F16 ? vec_dot_fattn_vec_KQ_f16 : + return type_K == GGML_TYPE_Q4_0 ? vec_dot_fattn_vec_KQ_q4_0 : + type_K == GGML_TYPE_Q4_1 ? vec_dot_fattn_vec_KQ_q4_1 : + type_K == GGML_TYPE_Q5_0 ? vec_dot_fattn_vec_KQ_q5_0 : + type_K == GGML_TYPE_Q5_1 ? vec_dot_fattn_vec_KQ_q5_1 : + type_K == GGML_TYPE_Q8_0 ? vec_dot_fattn_vec_KQ_q8_0 : + type_K == GGML_TYPE_F16 ? vec_dot_fattn_vec_KQ_f16 : nullptr; } @@ -681,7 +675,8 @@ static void on_no_fattn_vec_case(const int D) { template void launch_fattn( ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, - const int nwarps, const size_t nbytes_shared, const bool need_f16_K, const bool need_f16_V + const int nwarps, const size_t nbytes_shared, const bool need_f16_K, const bool need_f16_V, + const int warp_size = WARP_SIZE ) { constexpr int ncols = ncols1 * ncols2; @@ -704,8 +699,6 @@ void launch_fattn( GGML_ASSERT(Q->ne[3] == 1); - const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - ggml_cuda_pool & pool = ctx.pool(); cudaStream_t main_stream = ctx.stream(); const int id = ggml_cuda_get_device(); @@ -805,7 +798,6 @@ void launch_fattn( const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); GGML_ASSERT(block_dim.x % warp_size == 0); - GGML_ASSERT(!GGML_CUDA_CC_IS_AMD(cc) || block_dim.x * block_dim.y <= 4 * (unsigned int)warp_size); fattn_kernel<<>>( (const char *) Q->data, K_data, diff --git a/ggml/src/ggml-cuda/fattn-wmma-f16.cu b/ggml/src/ggml-cuda/fattn-wmma-f16.cu index 622cf28576..dab1d5cbca 100644 --- a/ggml/src/ggml-cuda/fattn-wmma-f16.cu +++ b/ggml/src/ggml-cuda/fattn-wmma-f16.cu @@ -469,6 +469,7 @@ void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggm constexpr int frag_m = cols_per_block == 8 && D % 32 == 0 ? 32 : 16; const int blocks_num_pb1 = ((Q->ne[1] + cols_per_block - 1) / cols_per_block)*Q->ne[2]*Q->ne[3]; const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; float logit_softcap; memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); @@ -485,7 +486,7 @@ void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggm fattn_kernel = flash_attn_ext_f16< D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true, warp_size); return; } if (2*blocks_num_pb1 < 2*nsm) { @@ -500,7 +501,7 @@ void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggm fattn_kernel = flash_attn_ext_f16< D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true, warp_size); return; } constexpr int parallel_blocks = 1; @@ -514,7 +515,7 @@ void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggm fattn_kernel = flash_attn_ext_f16< D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true); + launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true, warp_size); } void ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { From 363f8c5d67dcf80e00c39580dfa86dc2774d74c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alberto=20Cabrera=20P=C3=A9rez?= Date: Wed, 12 Mar 2025 09:57:32 +0000 Subject: [PATCH 124/188] sycl : variable sg_size support for mmvq kernels (#12336) --- ggml/src/ggml-sycl/mmvq.cpp | 152 ++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 77 deletions(-) diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 221f65c21e..a96286d710 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -3,44 +3,42 @@ #include template -static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, - const sycl::nd_item<3> &item_ct1) { - const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + - item_ct1.get_local_id(1); +static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, + const int ncols, const int nrows, const sycl::nd_item<3> & item_ct1) { + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); if (row >= nrows) { return; } - const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; - assert(blocks_per_warp>0); + const int blocks_per_row = ncols / qk; + constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi; // Ensuring blocks_per_warp > 0 -// partial sum for each thread + assert(blocks_per_warp > 0); + + // partial sum for each thread float tmp = 0.0f; - const block_q_t * x = (const block_q_t *) vx; + const block_q_t * x = (const block_q_t *) vx; const block_q8_1 * y = (const block_q8_1 *) vy; - for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; - i += blocks_per_warp) { - const int ibx = row*blocks_per_row + i; // x block index + for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) { + const int ibx = row * blocks_per_row + i; // x block index - const int iby = i * (qk/QK8_1); // y block index that aligns with ibx + const int iby = i * (qk / QK8_1); // y block index that aligns with ibx - const int iqs = - vdr * - (item_ct1.get_local_id(2) % - (qi / vdr)); // x block quant index when casting the quants to int + for (size_t elem = 0; elem < qi / vdr; elem += WARP_SIZE) { + const int iqs = elem + vdr * (item_ct1.get_local_id(2) % + (qi / vdr)); // x block quant index when casting the quants to int - tmp += vec_dot_q_sycl(&x[ibx], &y[iby], iqs); + tmp += vec_dot_q_sycl(&x[ibx], &y[iby], iqs); + } } // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { - tmp += - dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } if (item_ct1.get_local_id(2) == 0) { @@ -62,7 +60,7 @@ static void mul_mat_vec_q_iq2_xxs_q8_1(const void *__restrict__ vx, } const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; + const int blocks_per_warp = vdr * WARP_SIZE / qi; assert(blocks_per_warp>0); // partial sum for each thread @@ -87,7 +85,7 @@ static void mul_mat_vec_q_iq2_xxs_q8_1(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -111,7 +109,7 @@ static void mul_mat_vec_q_iq2_xs_q8_1(const void *__restrict__ vx, } const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; + const int blocks_per_warp = vdr * WARP_SIZE / qi; assert(blocks_per_warp>0); // partial sum for each thread float tmp = 0.0f; @@ -135,7 +133,7 @@ static void mul_mat_vec_q_iq2_xs_q8_1(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -159,7 +157,7 @@ static void mul_mat_vec_q_iq2_s_q8_1(const void *__restrict__ vx, } const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; + const int blocks_per_warp = vdr * WARP_SIZE / qi; assert(blocks_per_warp>0); // partial sum for each thread float tmp = 0.0f; @@ -183,7 +181,7 @@ static void mul_mat_vec_q_iq2_s_q8_1(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -207,7 +205,7 @@ static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx, } const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; + const int blocks_per_warp = vdr * WARP_SIZE / qi; assert(blocks_per_warp>0); // partial sum for each thread float tmp = 0.0f; @@ -231,7 +229,7 @@ static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -255,7 +253,7 @@ static void mul_mat_vec_q_iq3_s_q8_1(const void *__restrict__ vx, } const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; + const int blocks_per_warp = vdr * WARP_SIZE / qi; assert(blocks_per_warp>0); // partial sum for each thread float tmp = 0.0f; @@ -279,7 +277,7 @@ static void mul_mat_vec_q_iq3_s_q8_1(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -303,7 +301,7 @@ static void mul_mat_vec_q_iq1_s_q8_1(const void *__restrict__ vx, } const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; + const int blocks_per_warp = vdr * WARP_SIZE / qi; assert(blocks_per_warp>0); // partial sum for each thread float tmp = 0.0f; @@ -327,7 +325,7 @@ static void mul_mat_vec_q_iq1_s_q8_1(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -351,7 +349,7 @@ static void mul_mat_vec_q_iq1_m_q8_1(const void *__restrict__ vx, } const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; + const int blocks_per_warp = vdr * WARP_SIZE / qi; assert(blocks_per_warp>0); // partial sum for each thread float tmp = 0.0f; @@ -375,7 +373,7 @@ static void mul_mat_vec_q_iq1_m_q8_1(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -399,7 +397,7 @@ static void mul_mat_vec_q_iq4_nl_q8_1(const void *__restrict__ vx, } const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; + const int blocks_per_warp = vdr * WARP_SIZE / qi; assert(blocks_per_warp>0); // partial sum for each thread float tmp = 0.0f; @@ -423,7 +421,7 @@ static void mul_mat_vec_q_iq4_nl_q8_1(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -448,7 +446,7 @@ static void mul_mat_vec_q_iq4_xs_q8_1(const void *__restrict__ vx, } const int blocks_per_row = ncols / qk; - const int blocks_per_warp = vdr * QK_WARP_SIZE / qi; + const int blocks_per_warp = vdr * WARP_SIZE / qi; assert(blocks_per_warp>0); // partial sum for each thread float tmp = 0.0f; @@ -472,7 +470,7 @@ static void mul_mat_vec_q_iq4_xs_q8_1(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -489,7 +487,7 @@ static void mul_mat_vec_q4_0_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK4_0 == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -497,7 +495,7 @@ static void mul_mat_vec_q4_0_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -513,7 +511,7 @@ static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK4_1 == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -521,7 +519,7 @@ static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -537,7 +535,7 @@ static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK5_0 == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -545,7 +543,7 @@ static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -561,7 +559,7 @@ static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK5_1 == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -569,7 +567,7 @@ static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -585,7 +583,7 @@ static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK8_0 == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -593,7 +591,7 @@ static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -609,7 +607,7 @@ static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -617,7 +615,7 @@ static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -633,7 +631,7 @@ static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -641,7 +639,7 @@ static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -657,7 +655,7 @@ static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -665,7 +663,7 @@ static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -681,7 +679,7 @@ static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -689,7 +687,7 @@ static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -705,7 +703,7 @@ static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { @@ -713,7 +711,7 @@ static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -730,13 +728,13 @@ static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq2_xxs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -751,13 +749,13 @@ static void mul_mat_vec_iq2_xs_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler & cgh) { cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq2_xs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -772,14 +770,14 @@ static void mul_mat_vec_iq2_s_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq2_s_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -794,14 +792,14 @@ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq3_xxs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -816,14 +814,14 @@ static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq3_s_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -838,14 +836,14 @@ static void mul_mat_vec_iq1_s_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq1_s_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -860,13 +858,13 @@ static void mul_mat_vec_iq1_m_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq1_m_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -881,14 +879,14 @@ static void mul_mat_vec_iq4_nl_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK4_NL == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq4_nl_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -903,14 +901,14 @@ static void mul_mat_vec_iq4_xs_q8_1_sycl(const void *vx, const void *vy, GGML_ASSERT(ncols % QK_K == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { stream->submit([&](sycl::handler &cgh) { cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [[intel::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq4_xs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); From 80a02aa8588ef167d616f76f1781b104c245ace0 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Wed, 12 Mar 2025 13:45:32 +0100 Subject: [PATCH 125/188] llama.swiftui : fix xcframework dir in README [no ci] (#12353) This commit fixes the path to the xcframework in the README file which I had forgotten to change after renaming the build directory. --- examples/llama.swiftui/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama.swiftui/README.md b/examples/llama.swiftui/README.md index 5b0ee94720..bd7ce37747 100644 --- a/examples/llama.swiftui/README.md +++ b/examples/llama.swiftui/README.md @@ -16,7 +16,7 @@ Open `llama.swiftui.xcodeproj` project in Xcode and you should be able to build a simulator or a real device. To use the framework with a different project, the XCFramework can be added to the project by -adding `build-ios/llama.xcframework` by dragging and dropping it into the project navigator, or +adding `build-apple/llama.xcframework` by dragging and dropping it into the project navigator, or by manually selecting the framework in the "Frameworks, Libraries, and Embedded Content" section of the project settings. From f08f4b3187b691bb08a8884ed39ebaa94e956707 Mon Sep 17 00:00:00 2001 From: Oscar Barenys Date: Wed, 12 Mar 2025 20:06:58 +0100 Subject: [PATCH 126/188] Update build.yml for Windows Vulkan builder to use Vulkan 1.4.304 SDK for VK_NV_cooperative_matrix2 support (#12301) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f2c81c0c26..1e24293645 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -774,7 +774,7 @@ jobs: env: OPENBLAS_VERSION: 0.3.23 SDE_VERSION: 9.33.0-2024-01-07 - VULKAN_VERSION: 1.3.261.1 + VULKAN_VERSION: 1.4.304.1 strategy: matrix: From 2048b5913d51beab82dfe29955f9008130b936c0 Mon Sep 17 00:00:00 2001 From: Ishaan Gandhi Date: Thu, 13 Mar 2025 06:10:05 -0400 Subject: [PATCH 127/188] server : fix crash when using verbose output with input tokens that are not in printable range (#12178) (#12338) * Fix DOS index bug * Remove new APIs * remove extra line * Remove from API * Add extra newline * Update examples/server/server.cpp --------- Co-authored-by: Xuan-Son Nguyen --- examples/server/server.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 8cb8d0033f..ce0195475d 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2040,6 +2040,18 @@ struct server_context { return ret; } + bool can_be_detokenized(const struct llama_context * ctx, const std::vector & tokens) { + const llama_model * model = llama_get_model(ctx); + const llama_vocab * vocab = llama_model_get_vocab(model); + const int32_t n_vocab = llama_vocab_n_tokens(vocab); + for (const auto & token : tokens) { + if (token < 0 || token >= n_vocab) { + return false; + } + } + return true; + } + bool launch_slot_with_task(server_slot & slot, const server_task & task) { slot.reset(); slot.id_task = task.id; @@ -2054,6 +2066,11 @@ struct server_context { slot.lora = task.params.lora; } + bool can_detokenize = can_be_detokenized(ctx, slot.prompt_tokens); + if (!can_detokenize) { + send_error(task, "Prompt contains invalid tokens", ERROR_TYPE_INVALID_REQUEST); + return false; + } SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str()); if (slot.n_predict > 0 && slot.params.n_predict > slot.n_predict) { From e0dbec0bc6cd4b6230cda7a6ed1e9dac08d1600b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 13 Mar 2025 12:35:44 +0200 Subject: [PATCH 128/188] llama : refactor llama_context, llama_kv_cache, llm_build_context (#12181) * llama : refactor llama_context, llama_kv_cache, llm_build_context ggml-ci * graph : don't mutate the KV cache during defrag ggml-ci * context : reduce virtuals + remove test function ggml-ci * context : move interface implementation to source file + factory ggml-ci * graph : move KV cache build functions to llama_context impl ggml-ci * graph : remove model reference from build_pooling ggml-ci * graph : remove llama_model reference ggml-ci * kv_cache : provide rope factors ggml-ci * graph : rework inputs to use only unique_ptr, remove attn input abstraction ggml-ci * context : remove llama_context_i abstraction ggml-ci * context : clean-up ggml-ci * graph : clean-up ggml-ci * llama : remove redundant keywords (struct, enum) ggml-ci * model : adapt gemma3 ggml-ci * graph : restore same attention ops as on master ggml-ci * llama : remove TODO + fix indent ggml-ci --- common/common.cpp | 6 +- common/speculative.cpp | 8 +- examples/batched-bench/batched-bench.cpp | 4 +- examples/batched.swift/Sources/main.swift | 2 +- .../cvector-generator/cvector-generator.cpp | 2 +- examples/embedding/embedding.cpp | 2 +- examples/gritlm/gritlm.cpp | 4 +- examples/imatrix/imatrix.cpp | 2 +- examples/infill/infill.cpp | 4 +- examples/llama-bench/llama-bench.cpp | 4 +- .../llama/src/main/cpp/llama-android.cpp | 8 +- .../llama.cpp.swift/LibLlama.swift | 8 +- examples/llava/gemma3-cli.cpp | 2 +- examples/lookahead/lookahead.cpp | 12 +- examples/lookup/lookup.cpp | 2 +- examples/main/main.cpp | 12 +- examples/parallel/parallel.cpp | 10 +- examples/passkey/passkey.cpp | 28 +- examples/perplexity/perplexity.cpp | 12 +- examples/quantize-stats/quantize-stats.cpp | 4 +- examples/retrieval/retrieval.cpp | 2 +- examples/run/run.cpp | 4 +- examples/save-load-state/save-load-state.cpp | 4 +- examples/server/server.cpp | 22 +- examples/server/tests/utils.py | 2 +- examples/simple-chat/simple-chat.cpp | 4 +- .../speculative-simple/speculative-simple.cpp | 2 +- examples/speculative/speculative.cpp | 24 +- include/llama.h | 106 +- src/CMakeLists.txt | 7 +- src/llama-adapter.cpp | 39 +- src/llama-adapter.h | 20 +- src/llama-batch.h | 4 +- src/llama-context.cpp | 3809 +++--- src/llama-context.h | 298 +- src/llama-graph.cpp | 1695 +++ src/llama-graph.h | 576 + src/llama-io.cpp | 15 + src/llama-io.h | 35 + src/llama-kv-cache.cpp | 1519 ++- src/llama-kv-cache.h | 302 +- src/llama-memory.cpp | 1 + src/llama-memory.h | 21 + src/llama-model.cpp | 7392 +++++++++++- src/llama-model.h | 19 +- src/llama.cpp | 10035 +--------------- 46 files changed, 13903 insertions(+), 12190 deletions(-) create mode 100644 src/llama-graph.cpp create mode 100644 src/llama-graph.h create mode 100644 src/llama-io.cpp create mode 100644 src/llama-io.h create mode 100644 src/llama-memory.cpp create mode 100644 src/llama-memory.h diff --git a/common/common.cpp b/common/common.cpp index 6448b7b03d..8487e3834b 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -955,8 +955,8 @@ struct common_init_result common_init_from_params(common_params & params) { return iparams; } - if (params.ctx_shift && !llama_kv_cache_can_shift(lctx)) { - LOG_WRN("%s: KV cache shifting is not supported for this model, disabling KV cache shifting\n", __func__); + if (params.ctx_shift && !llama_kv_self_can_shift(lctx)) { + LOG_WRN("%s: KV cache shifting is not supported for this context, disabling KV cache shifting\n", __func__); params.ctx_shift = false; } @@ -1060,7 +1060,7 @@ struct common_init_result common_init_from_params(common_params & params) { if (llama_model_has_decoder(model)) { llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch))); } - llama_kv_cache_clear(lctx); + llama_kv_self_clear(lctx); llama_synchronize(lctx); llama_perf_context_reset(lctx); } diff --git a/common/speculative.cpp b/common/speculative.cpp index 1bac3a1ce1..ccad70fa9e 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -173,7 +173,7 @@ llama_tokens common_speculative_gen_draft( result.reserve(params.n_draft); if (reuse_n == 0) { - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); prompt.clear(); } else { @@ -192,14 +192,14 @@ llama_tokens common_speculative_gen_draft( } if (reuse_i > 0) { - llama_kv_cache_seq_rm (ctx, 0, 0, reuse_i); - llama_kv_cache_seq_add(ctx, 0, reuse_i, -1, -reuse_i); + llama_kv_self_seq_rm (ctx, 0, 0, reuse_i); + llama_kv_self_seq_add(ctx, 0, reuse_i, -1, -reuse_i); prompt.erase(prompt.begin(), prompt.begin() + reuse_i); } if (reuse_n < (int) prompt.size()) { - llama_kv_cache_seq_rm (ctx, 0, reuse_n, -1); + llama_kv_self_seq_rm (ctx, 0, reuse_n, -1); prompt.erase(prompt.begin() + reuse_n, prompt.end()); } diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index 0659ab6f11..430e8be512 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -132,7 +132,7 @@ int main(int argc, char ** argv) { const auto t_pp_start = ggml_time_us(); - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); if (!decode_helper(ctx, batch, ctx_params.n_batch)) { LOG_ERR("%s: llama_decode() failed\n", __func__); @@ -141,7 +141,7 @@ int main(int argc, char ** argv) { if (is_pp_shared) { for (int32_t i = 1; i < pl; ++i) { - llama_kv_cache_seq_cp(ctx, 0, i, -1, -1); + llama_kv_self_seq_cp(ctx, 0, i, -1, -1); } } diff --git a/examples/batched.swift/Sources/main.swift b/examples/batched.swift/Sources/main.swift index 55c31166ca..514989e340 100644 --- a/examples/batched.swift/Sources/main.swift +++ b/examples/batched.swift/Sources/main.swift @@ -116,7 +116,7 @@ if llama_decode(context, batch) != 0 { } for i in 1 ..< n_parallel { - llama_kv_cache_seq_cp(context, 0, Int32(i), 0, batch.n_tokens) + llama_kv_self_seq_cp(context, 0, Int32(i), 0, batch.n_tokens) } if n_parallel > 1 { diff --git a/examples/cvector-generator/cvector-generator.cpp b/examples/cvector-generator/cvector-generator.cpp index c72528dac3..2a90715501 100644 --- a/examples/cvector-generator/cvector-generator.cpp +++ b/examples/cvector-generator/cvector-generator.cpp @@ -342,7 +342,7 @@ static bool cb_eval(struct ggml_tensor * t, bool ask, void * user_data) { } static bool get_hidden_layers(llama_context * ctx, std::vector & tokens) { - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); if (llama_decode(ctx, llama_batch_get_one(tokens.data(), tokens.size()))) { fprintf(stderr, "%s : failed to eval\n", __func__); return false; diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index 3dd9f2b07d..6f08904159 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -38,7 +38,7 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu const struct llama_model * model = llama_get_model(ctx); // clear previous kv_cache values (irrelevant for embeddings) - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); // run model LOG_INF("%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq); diff --git a/examples/gritlm/gritlm.cpp b/examples/gritlm/gritlm.cpp index 72eb462574..f7db7861c1 100644 --- a/examples/gritlm/gritlm.cpp +++ b/examples/gritlm/gritlm.cpp @@ -45,7 +45,7 @@ static std::vector> encode(llama_context * ctx, const std::ve } // clear previous kv_cache values (irrelevant for embeddings) - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); llama_set_embeddings(ctx, true); llama_set_causal_attn(ctx, false); @@ -102,7 +102,7 @@ static std::string generate(llama_context * ctx, llama_sampler * smpl, const std llama_token eos_token = llama_vocab_eos(vocab); - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); llama_set_embeddings(ctx, false); llama_set_causal_attn(ctx, true); diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp index 91649c4506..31b675e8f9 100644 --- a/examples/imatrix/imatrix.cpp +++ b/examples/imatrix/imatrix.cpp @@ -495,7 +495,7 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params) { const auto t_start = std::chrono::high_resolution_clock::now(); // clear the KV cache - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); llama_batch batch = llama_batch_init(n_batch, 0, 1); diff --git a/examples/infill/infill.cpp b/examples/infill/infill.cpp index 489a208b66..4e2f7b7270 100644 --- a/examples/infill/infill.cpp +++ b/examples/infill/infill.cpp @@ -332,8 +332,8 @@ int main(int argc, char ** argv) { LOG_DBG("context full, swapping: n_past = %d, n_left = %d, n_ctx = %d, n_keep = %d, n_discard = %d\n", n_past, n_left, n_ctx, params.n_keep, n_discard); - llama_kv_cache_seq_rm (ctx, 0, params.n_keep + 1 , params.n_keep + n_discard + 1); - llama_kv_cache_seq_add(ctx, 0, params.n_keep + 1 + n_discard, n_past, -n_discard); + llama_kv_self_seq_rm (ctx, 0, params.n_keep + 1 , params.n_keep + n_discard + 1); + llama_kv_self_seq_add(ctx, 0, params.n_keep + 1 + n_discard, n_past, -n_discard); n_past -= n_discard; diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index f518d02d38..cbcbfcee86 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -1578,7 +1578,7 @@ int main(int argc, char ** argv) { test t(inst, lmodel, ctx); - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); // cool off before the test if (params.delay) { @@ -1618,7 +1618,7 @@ int main(int argc, char ** argv) { } for (int i = 0; i < params.reps; i++) { - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); uint64_t t_start = get_time_ns(); diff --git a/examples/llama.android/llama/src/main/cpp/llama-android.cpp b/examples/llama.android/llama/src/main/cpp/llama-android.cpp index 0de61ce77c..9654cd53cf 100644 --- a/examples/llama.android/llama/src/main/cpp/llama-android.cpp +++ b/examples/llama.android/llama/src/main/cpp/llama-android.cpp @@ -194,7 +194,7 @@ Java_android_llama_cpp_LLamaAndroid_bench_1model( } batch->logits[batch->n_tokens - 1] = true; - llama_kv_cache_clear(context); + llama_kv_self_clear(context); const auto t_pp_start = ggml_time_us(); if (llama_decode(context, *batch) != 0) { @@ -206,7 +206,7 @@ Java_android_llama_cpp_LLamaAndroid_bench_1model( LOGi("Benchmark text generation (tg)"); - llama_kv_cache_clear(context); + llama_kv_self_clear(context); const auto t_tg_start = ggml_time_us(); for (i = 0; i < tg; i++) { @@ -223,7 +223,7 @@ Java_android_llama_cpp_LLamaAndroid_bench_1model( const auto t_tg_end = ggml_time_us(); - llama_kv_cache_clear(context); + llama_kv_self_clear(context); const auto t_pp = double(t_pp_end - t_pp_start) / 1000000.0; const auto t_tg = double(t_tg_end - t_tg_start) / 1000000.0; @@ -448,5 +448,5 @@ Java_android_llama_cpp_LLamaAndroid_completion_1loop( extern "C" JNIEXPORT void JNICALL Java_android_llama_cpp_LLamaAndroid_kv_1cache_1clear(JNIEnv *, jobject, jlong context) { - llama_kv_cache_clear(reinterpret_cast(context)); + llama_kv_self_clear(reinterpret_cast(context)); } diff --git a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift index ee7141a663..f6e31abc93 100644 --- a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift +++ b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift @@ -210,7 +210,7 @@ actor LlamaContext { } batch.logits[Int(batch.n_tokens) - 1] = 1 // true - llama_kv_cache_clear(context) + llama_kv_self_clear(context) let t_pp_start = DispatchTime.now().uptimeNanoseconds / 1000; @@ -223,7 +223,7 @@ actor LlamaContext { // bench text generation - llama_kv_cache_clear(context) + llama_kv_self_clear(context) let t_tg_start = DispatchTime.now().uptimeNanoseconds / 1000; @@ -242,7 +242,7 @@ actor LlamaContext { let t_tg_end = DispatchTime.now().uptimeNanoseconds / 1000; - llama_kv_cache_clear(context) + llama_kv_self_clear(context) let t_pp = Double(t_pp_end - t_pp_start) / 1000000.0 let t_tg = Double(t_tg_end - t_tg_start) / 1000000.0 @@ -292,7 +292,7 @@ actor LlamaContext { func clear() { tokens_list.removeAll() temporary_invalid_cchars.removeAll() - llama_kv_cache_clear(context) + llama_kv_self_clear(context) } private func tokenize(text: String, add_bos: Bool) -> [llama_token] { diff --git a/examples/llava/gemma3-cli.cpp b/examples/llava/gemma3-cli.cpp index a07864d4e5..c36bb2eda0 100644 --- a/examples/llava/gemma3-cli.cpp +++ b/examples/llava/gemma3-cli.cpp @@ -309,7 +309,7 @@ int main(int argc, char ** argv) { } if (line == "/clear") { ctx.n_past = 0; - llama_kv_cache_seq_rm(ctx.lctx, 0, 1, -1); // keep BOS + llama_kv_self_seq_rm(ctx.lctx, 0, 1, -1); // keep BOS LOG("Chat history cleared\n\n"); continue; } diff --git a/examples/lookahead/lookahead.cpp b/examples/lookahead/lookahead.cpp index b9e8de694c..7df20aee17 100644 --- a/examples/lookahead/lookahead.cpp +++ b/examples/lookahead/lookahead.cpp @@ -96,7 +96,7 @@ int main(int argc, char ** argv) { llama_decode(ctx, llama_batch_get_one(&inp.back(), 1)); for (int s = 1; s < W + G + 1; ++s) { - llama_kv_cache_seq_cp(ctx, 0, s, -1, -1); + llama_kv_self_seq_cp(ctx, 0, s, -1, -1); } const auto t_enc_end = ggml_time_us(); @@ -438,17 +438,17 @@ int main(int argc, char ** argv) { // KV cache management // if no verification token matched, we simply remove all cells from this batch -> no fragmentation - llama_kv_cache_seq_rm(ctx, -1, n_past, -1); + llama_kv_self_seq_rm(ctx, -1, n_past, -1); if (seq_id_best != 0) { // if a verification token matched, we keep the best sequence and remove the rest // this leads to some KV cache fragmentation - llama_kv_cache_seq_keep(ctx, seq_id_best); - llama_kv_cache_seq_cp (ctx, seq_id_best, 0, -1, -1); - llama_kv_cache_seq_rm (ctx, seq_id_best, -1, -1); + llama_kv_self_seq_keep(ctx, seq_id_best); + llama_kv_self_seq_cp (ctx, seq_id_best, 0, -1, -1); + llama_kv_self_seq_rm (ctx, seq_id_best, -1, -1); for (int s = 1; s < W + G + 1; ++s) { - llama_kv_cache_seq_cp(ctx, 0, s, -1, -1); + llama_kv_self_seq_cp(ctx, 0, s, -1, -1); } } } diff --git a/examples/lookup/lookup.cpp b/examples/lookup/lookup.cpp index dbd0444ec8..4ae93b2a5e 100644 --- a/examples/lookup/lookup.cpp +++ b/examples/lookup/lookup.cpp @@ -192,7 +192,7 @@ int main(int argc, char ** argv){ // KV cache management // clean the cache of draft tokens that weren't accepted - llama_kv_cache_seq_rm(ctx, 0, n_past, -1); + llama_kv_self_seq_rm(ctx, 0, n_past, -1); common_batch_clear(batch_tgt); common_batch_add(batch_tgt, draft[0], n_past, { 0 }, true); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 4e0c69473b..fd7410a646 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -354,7 +354,7 @@ int main(int argc, char ** argv) { } // remove any "future" tokens that we might have inherited from the previous session - llama_kv_cache_seq_rm(ctx, -1, n_matching_session_tokens, -1); + llama_kv_self_seq_rm(ctx, -1, n_matching_session_tokens, -1); } LOG_DBG("recalculate the cached logits (check): embd_inp.size() %zu, n_matching_session_tokens %zu, embd_inp.size() %zu, session_tokens.size() %zu\n", @@ -602,8 +602,8 @@ int main(int argc, char ** argv) { LOG_DBG("context full, swapping: n_past = %d, n_left = %d, n_ctx = %d, n_keep = %d, n_discard = %d\n", n_past, n_left, n_ctx, params.n_keep, n_discard); - llama_kv_cache_seq_rm (ctx, 0, params.n_keep , params.n_keep + n_discard); - llama_kv_cache_seq_add(ctx, 0, params.n_keep + n_discard, n_past, -n_discard); + llama_kv_self_seq_rm (ctx, 0, params.n_keep , params.n_keep + n_discard); + llama_kv_self_seq_add(ctx, 0, params.n_keep + n_discard, n_past, -n_discard); n_past -= n_discard; @@ -626,9 +626,9 @@ int main(int argc, char ** argv) { LOG_DBG("div: [%6d, %6d] / %6d -> [%6d, %6d]\n", ga_i + ib*bd, ga_i + ib*bd + ga_w, ga_n, (ga_i + ib*bd)/ga_n, (ga_i + ib*bd + ga_w)/ga_n); LOG_DBG("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", ga_i + ib*bd + ga_w, n_past + ib*bd, dd, ga_i + ib*bd + ga_w + dd, n_past + ib*bd + dd); - llama_kv_cache_seq_add(ctx, 0, ga_i, n_past, ib*bd); - llama_kv_cache_seq_div(ctx, 0, ga_i + ib*bd, ga_i + ib*bd + ga_w, ga_n); - llama_kv_cache_seq_add(ctx, 0, ga_i + ib*bd + ga_w, n_past + ib*bd, dd); + llama_kv_self_seq_add(ctx, 0, ga_i, n_past, ib*bd); + llama_kv_self_seq_div(ctx, 0, ga_i + ib*bd, ga_i + ib*bd + ga_w, ga_n); + llama_kv_self_seq_add(ctx, 0, ga_i + ib*bd + ga_w, n_past + ib*bd, dd); n_past -= bd; diff --git a/examples/parallel/parallel.cpp b/examples/parallel/parallel.cpp index be18909ed4..588632f043 100644 --- a/examples/parallel/parallel.cpp +++ b/examples/parallel/parallel.cpp @@ -202,7 +202,7 @@ int main(int argc, char ** argv) { // assign the system KV cache to all parallel sequences for (int32_t i = 1; i <= n_clients; ++i) { - llama_kv_cache_seq_cp(ctx, 0, i, -1, -1); + llama_kv_self_seq_cp(ctx, 0, i, -1, -1); } LOG_INF("\n"); @@ -234,9 +234,9 @@ int main(int argc, char ** argv) { if (batch.n_tokens == 0) { // all sequences have ended - clear the entire KV cache for (int i = 1; i <= n_clients; ++i) { - llama_kv_cache_seq_rm(ctx, i, -1, -1); + llama_kv_self_seq_rm(ctx, i, -1, -1); // but keep the system prompt - llama_kv_cache_seq_cp(ctx, 0, i, -1, -1); + llama_kv_self_seq_cp(ctx, 0, i, -1, -1); } LOG_INF("%s: clearing the KV cache\n", __func__); @@ -372,8 +372,8 @@ int main(int argc, char ** argv) { } // delete only the generated part of the sequence, i.e. keep the system prompt in the cache - llama_kv_cache_seq_rm(ctx, client.id + 1, -1, -1); - llama_kv_cache_seq_cp(ctx, 0, client.id + 1, -1, -1); + llama_kv_self_seq_rm(ctx, client.id + 1, -1, -1); + llama_kv_self_seq_cp(ctx, 0, client.id + 1, -1, -1); const auto t_main_end = ggml_time_us(); diff --git a/examples/passkey/passkey.cpp b/examples/passkey/passkey.cpp index fa85190518..ea3a6c1fca 100644 --- a/examples/passkey/passkey.cpp +++ b/examples/passkey/passkey.cpp @@ -133,11 +133,11 @@ int main(int argc, char ** argv) { const int ib = i/n_batch - 1; const int bd = n_batch_grp*(n_grp - 1); - llama_kv_cache_seq_add (ctx, 0, n_past - n_batch, n_past, ib*bd); - llama_kv_cache_seq_div (ctx, 0, n_past - n_batch + ib*bd, n_past + ib*bd, n_grp); - llama_kv_cache_update (ctx); + llama_kv_self_seq_add (ctx, 0, n_past - n_batch, n_past, ib*bd); + llama_kv_self_seq_div (ctx, 0, n_past - n_batch + ib*bd, n_past + ib*bd, n_grp); + llama_kv_self_update (ctx); - n_past = llama_kv_cache_seq_pos_max(ctx, 0) + 1; + n_past = llama_kv_self_seq_pos_max(ctx, 0) + 1; } common_batch_clear(batch); @@ -167,12 +167,12 @@ int main(int argc, char ** argv) { LOG_INF("%s: shifting KV cache with %d\n", __func__, n_discard); - llama_kv_cache_seq_rm (ctx, 0, n_keep , n_keep + n_discard); - llama_kv_cache_seq_add(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); - //llama_kv_cache_defrag (ctx); - llama_kv_cache_update (ctx); + llama_kv_self_seq_rm (ctx, 0, n_keep , n_keep + n_discard); + llama_kv_self_seq_add(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); + //llama_kv_self_defrag (ctx); + llama_kv_self_update (ctx); - n_past = llama_kv_cache_seq_pos_max(ctx, 0) + 1; + n_past = llama_kv_self_seq_pos_max(ctx, 0) + 1; common_batch_clear(batch); @@ -198,12 +198,12 @@ int main(int argc, char ** argv) { if (n_discard > 0) { LOG_INF("%s: shifting KV cache with %d to free space for the answer\n", __func__, n_discard); - llama_kv_cache_seq_rm (ctx, 0, n_keep , n_keep + n_discard); - llama_kv_cache_seq_add(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); - //llama_kv_cache_defrag (ctx); - llama_kv_cache_update (ctx); + llama_kv_self_seq_rm (ctx, 0, n_keep , n_keep + n_discard); + llama_kv_self_seq_add(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); + //llama_kv_self_defrag (ctx); + llama_kv_self_update (ctx); - n_past = llama_kv_cache_seq_pos_max(ctx, 0) + 1; + n_past = llama_kv_self_seq_pos_max(ctx, 0) + 1; } } diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 5d07421e82..8c413f7d66 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -361,7 +361,7 @@ static results_perplexity perplexity_v2(llama_context * ctx, const common_params const auto t_start = std::chrono::high_resolution_clock::now(); // clear the KV cache - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); llama_batch batch = llama_batch_init(n_batch, 0, 1); @@ -547,7 +547,7 @@ static results_perplexity perplexity(llama_context * ctx, const common_params & const auto t_start = std::chrono::high_resolution_clock::now(); // clear the KV cache - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); for (int j = 0; j < num_batches; ++j) { const int batch_start = start + j * n_batch; @@ -924,7 +924,7 @@ static void hellaswag_score(llama_context * ctx, const common_params & params) { return; } - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); // decode all tasks [i0, i1) if (!decode_helper(ctx, batch, batch_logits, n_batch, n_vocab)) { @@ -1203,7 +1203,7 @@ static void winogrande_score(llama_context * ctx, const common_params & params) return; } - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); // decode all tasks [i0, i1) if (!decode_helper(ctx, batch, batch_logits, n_batch, n_vocab)) { @@ -1575,7 +1575,7 @@ static void multiple_choice_score(llama_context * ctx, const common_params & par return; } - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); // decode all tasks [i0, i1) if (!decode_helper(ctx, batch, batch_logits, n_batch, n_vocab)) { @@ -1765,7 +1765,7 @@ static void kl_divergence(llama_context * ctx, const common_params & params) { } // clear the KV cache - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); llama_batch batch = llama_batch_init(n_batch, 0, 1); diff --git a/examples/quantize-stats/quantize-stats.cpp b/examples/quantize-stats/quantize-stats.cpp index bd2f734670..dd07ab9b37 100644 --- a/examples/quantize-stats/quantize-stats.cpp +++ b/examples/quantize-stats/quantize-stats.cpp @@ -1,6 +1,6 @@ #include "ggml.h" #include "llama.h" -#include "llama-context.h" +#include "llama-model.h" #include "common.h" #include @@ -328,7 +328,7 @@ int main(int argc, char ** argv) { } } - const auto & tensors = llama_internal_get_tensor_map(ctx); + const auto & tensors = llama_internal_get_tensor_map(model); // check layer tensors int included_layers = 0; diff --git a/examples/retrieval/retrieval.cpp b/examples/retrieval/retrieval.cpp index 2439022a22..0efe20d4b3 100644 --- a/examples/retrieval/retrieval.cpp +++ b/examples/retrieval/retrieval.cpp @@ -83,7 +83,7 @@ static void batch_add_seq(llama_batch & batch, const std::vector & toke static void batch_decode(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd) { // clear previous kv_cache values (irrelevant for embeddings) - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); // run model LOG_INF("%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq); diff --git a/examples/run/run.cpp b/examples/run/run.cpp index 38407d5190..437f2533e5 100644 --- a/examples/run/run.cpp +++ b/examples/run/run.cpp @@ -891,7 +891,7 @@ static int apply_chat_template(const struct common_chat_templates * tmpls, Llama // Function to tokenize the prompt static int tokenize_prompt(const llama_vocab * vocab, const std::string & prompt, std::vector & prompt_tokens, const LlamaData & llama_data) { - const bool is_first = llama_get_kv_cache_used_cells(llama_data.context.get()) == 0; + const bool is_first = llama_kv_self_used_cells(llama_data.context.get()) == 0; const int n_prompt_tokens = -llama_tokenize(vocab, prompt.c_str(), prompt.size(), NULL, 0, is_first, true); prompt_tokens.resize(n_prompt_tokens); @@ -907,7 +907,7 @@ static int tokenize_prompt(const llama_vocab * vocab, const std::string & prompt // Check if we have enough space in the context to evaluate this batch static int check_context_size(const llama_context_ptr & ctx, const llama_batch & batch) { const int n_ctx = llama_n_ctx(ctx.get()); - const int n_ctx_used = llama_get_kv_cache_used_cells(ctx.get()); + const int n_ctx_used = llama_kv_self_used_cells(ctx.get()); if (n_ctx_used + batch.n_tokens > n_ctx) { printf(LOG_COL_DEFAULT "\n"); printe("context size exceeded\n"); diff --git a/examples/save-load-state/save-load-state.cpp b/examples/save-load-state/save-load-state.cpp index cf7cbd8159..760ebbbf08 100644 --- a/examples/save-load-state/save-load-state.cpp +++ b/examples/save-load-state/save-load-state.cpp @@ -15,7 +15,7 @@ int main(int argc, char ** argv) { return 1; } - print_build_info(); + common_init(); if (params.n_predict < 0) { params.n_predict = 16; @@ -196,7 +196,7 @@ int main(int argc, char ** argv) { fprintf(stderr, "%s : seq 0 copied, %zd bytes\n", __func__, ncopy); // erase whole kv - llama_kv_cache_clear(ctx3); + llama_kv_self_clear(ctx3); fprintf(stderr, "%s : kv cache cleared\n", __func__); // restore kv into seq 1 diff --git a/examples/server/server.cpp b/examples/server/server.cpp index ce0195475d..71e053b202 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2113,7 +2113,7 @@ struct server_context { SRV_DBG("%s", "clearing KV cache\n"); // clear the entire KV cache - llama_kv_cache_clear(ctx); + llama_kv_self_clear(ctx); clean_kv_cache = false; } @@ -2655,8 +2655,8 @@ struct server_context { res->n_tasks_deferred = queue_tasks.queue_tasks_deferred.size(); res->t_start = metrics.t_start; - res->kv_cache_tokens_count = llama_get_kv_cache_token_count(ctx); - res->kv_cache_used_cells = llama_get_kv_cache_used_cells(ctx); + res->kv_cache_tokens_count = llama_kv_self_n_tokens(ctx); + res->kv_cache_used_cells = llama_kv_self_used_cells(ctx); res->n_prompt_tokens_processed_total = metrics.n_prompt_tokens_processed_total; res->t_prompt_processing_total = metrics.t_prompt_processing_total; @@ -2772,7 +2772,7 @@ struct server_context { // Erase token cache const size_t n_erased = slot->cache_tokens.size(); - llama_kv_cache_seq_rm(ctx, slot->id, -1, -1); + llama_kv_self_seq_rm(ctx, slot->id, -1, -1); slot->cache_tokens.clear(); auto res = std::make_unique(); @@ -2840,8 +2840,8 @@ struct server_context { SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard); - llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard); - llama_kv_cache_seq_add(ctx, slot.id, n_keep + n_discard, slot.n_past, -n_discard); + llama_kv_self_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard); + llama_kv_self_seq_add(ctx, slot.id, n_keep + n_discard, slot.n_past, -n_discard); if (slot.params.cache_prompt) { for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) { @@ -3032,8 +3032,8 @@ struct server_context { const int64_t kv_shift = (int64_t) head_p - (int64_t) head_c; - llama_kv_cache_seq_rm (ctx, slot.id, head_p, head_c); - llama_kv_cache_seq_add(ctx, slot.id, head_c, head_c + n_match, kv_shift); + llama_kv_self_seq_rm (ctx, slot.id, head_p, head_c); + llama_kv_self_seq_add(ctx, slot.id, head_c, head_c + n_match, kv_shift); for (size_t i = 0; i < n_match; i++) { slot.cache_tokens[head_p + i] = slot.cache_tokens[head_c + i]; @@ -3071,9 +3071,9 @@ struct server_context { } // keep only the common part - if (!llama_kv_cache_seq_rm(ctx, slot.id, slot.n_past, -1)) { + if (!llama_kv_self_seq_rm(ctx, slot.id, slot.n_past, -1)) { // could not partially delete (likely using a non-Transformer model) - llama_kv_cache_seq_rm(ctx, slot.id, -1, -1); + llama_kv_self_seq_rm(ctx, slot.id, -1, -1); // there is no common part left slot.n_past = 0; @@ -3313,7 +3313,7 @@ struct server_context { slot.cache_tokens.push_back(id); slot.cache_tokens.insert(slot.cache_tokens.end(), ids.begin(), ids.end() - 1); - llama_kv_cache_seq_rm(ctx, slot.id, slot.n_past, -1); + llama_kv_self_seq_rm(ctx, slot.id, slot.n_past, -1); for (size_t i = 0; i < ids.size(); ++i) { completion_token_output result; diff --git a/examples/server/tests/utils.py b/examples/server/tests/utils.py index ec2d8ec558..30aa866095 100644 --- a/examples/server/tests/utils.py +++ b/examples/server/tests/utils.py @@ -302,7 +302,7 @@ class ServerPreset: server.model_hf_repo = "ggml-org/models" server.model_hf_file = "tinyllamas/stories260K.gguf" server.model_alias = "tinyllama-2" - server.n_ctx = 256 + server.n_ctx = 512 server.n_batch = 32 server.n_slots = 2 server.n_predict = 64 diff --git a/examples/simple-chat/simple-chat.cpp b/examples/simple-chat/simple-chat.cpp index c5534cc13e..84f4159737 100644 --- a/examples/simple-chat/simple-chat.cpp +++ b/examples/simple-chat/simple-chat.cpp @@ -98,7 +98,7 @@ int main(int argc, char ** argv) { auto generate = [&](const std::string & prompt) { std::string response; - const bool is_first = llama_get_kv_cache_used_cells(ctx) == 0; + const bool is_first = llama_kv_self_used_cells(ctx) == 0; // tokenize the prompt const int n_prompt_tokens = -llama_tokenize(vocab, prompt.c_str(), prompt.size(), NULL, 0, is_first, true); @@ -113,7 +113,7 @@ int main(int argc, char ** argv) { while (true) { // check if we have enough space in the context to evaluate this batch int n_ctx = llama_n_ctx(ctx); - int n_ctx_used = llama_get_kv_cache_used_cells(ctx); + int n_ctx_used = llama_kv_self_used_cells(ctx); if (n_ctx_used + batch.n_tokens > n_ctx) { printf("\033[0m\n"); fprintf(stderr, "context size exceeded\n"); diff --git a/examples/speculative-simple/speculative-simple.cpp b/examples/speculative-simple/speculative-simple.cpp index 403ba2dd21..a5d2bc9d09 100644 --- a/examples/speculative-simple/speculative-simple.cpp +++ b/examples/speculative-simple/speculative-simple.cpp @@ -217,7 +217,7 @@ int main(int argc, char ** argv) { { LOG_DBG("clear kv cache from any extra tokens, n_past = %d\n", n_past); - llama_kv_cache_seq_rm(ctx_tgt, 0, n_past, -1); + llama_kv_self_seq_rm(ctx_tgt, 0, n_past, -1); } if ((params.n_predict >= 0 && n_predict > params.n_predict) || has_eos) { diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index c7ccea50db..bfddc67e03 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -420,14 +420,14 @@ int main(int argc, char ** argv) { { LOG_DBG("keeping sequence %d, n_past_tgt = %d, n_past_dft = %d\n", s_keep, n_past_tgt, n_past_dft); - llama_kv_cache_seq_keep(ctx_dft, s_keep); - llama_kv_cache_seq_cp (ctx_dft, s_keep, 0, -1, -1); - llama_kv_cache_seq_keep(ctx_dft, 0); + llama_kv_self_seq_keep(ctx_dft, s_keep); + llama_kv_self_seq_cp (ctx_dft, s_keep, 0, -1, -1); + llama_kv_self_seq_keep(ctx_dft, 0); - llama_kv_cache_seq_rm (ctx_tgt, s_keep, n_past_tgt, -1); - llama_kv_cache_seq_keep(ctx_tgt, s_keep); - llama_kv_cache_seq_cp (ctx_tgt, s_keep, 0, -1, -1); - llama_kv_cache_seq_keep(ctx_tgt, 0); + llama_kv_self_seq_rm (ctx_tgt, s_keep, n_past_tgt, -1); + llama_kv_self_seq_keep(ctx_tgt, s_keep); + llama_kv_self_seq_cp (ctx_tgt, s_keep, 0, -1, -1); + llama_kv_self_seq_keep(ctx_tgt, 0); } for (int s = 0; s < n_seq_dft; ++s) { @@ -444,7 +444,7 @@ int main(int argc, char ** argv) { common_batch_clear(batch_dft); common_batch_add (batch_dft, token_id, n_past_dft, { 0 }, true); - llama_kv_cache_seq_rm(ctx_dft, 0, n_past_dft, -1); + llama_kv_self_seq_rm(ctx_dft, 0, n_past_dft, -1); // LOG_DBG("dft batch: %s\n", LOG_BATCH_TOSTR_PRETTY(ctx_dft, batch_dft).c_str()); llama_decode(ctx_dft, batch_dft); @@ -503,8 +503,8 @@ int main(int argc, char ** argv) { if (n_seq_cur < n_seq_dft && cur_p->data[f].p > p_draft_split) { LOG_DBG("splitting seq %3d into %3d\n", s, n_seq_cur); - llama_kv_cache_seq_rm(ctx_dft, n_seq_cur, -1, -1); - llama_kv_cache_seq_cp(ctx_dft, s, n_seq_cur, -1, -1); + llama_kv_self_seq_rm(ctx_dft, n_seq_cur, -1, -1); + llama_kv_self_seq_cp(ctx_dft, s, n_seq_cur, -1, -1); // all previous tokens from this branch are now also part of the new branch for (int t = 0; t < batch_tgt.n_tokens; ++t) { @@ -585,9 +585,9 @@ int main(int argc, char ** argv) { // evaluate the target model on the drafted tokens { - llama_kv_cache_seq_keep(ctx_tgt, 0); + llama_kv_self_seq_keep(ctx_tgt, 0); for (int s = 1; s < n_seq_dft; ++s) { - llama_kv_cache_seq_cp(ctx_tgt, 0, s, -1, -1); + llama_kv_self_seq_cp(ctx_tgt, 0, s, -1, -1); } // LOG_DBG("target batch: %s\n", LOG_BATCH_TOSTR_PRETTY(ctx_tgt, batch_tgt).c_str()); diff --git a/include/llama.h b/include/llama.h index d62792c0a6..e5286f0616 100644 --- a/include/llama.h +++ b/include/llama.h @@ -60,6 +60,7 @@ extern "C" { struct llama_model; struct llama_context; struct llama_sampler; + struct llama_kv_cache; typedef int32_t llama_pos; typedef int32_t llama_token; @@ -469,7 +470,8 @@ extern "C" { DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); - LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); + LLAMA_API struct llama_kv_cache * llama_get_kv_self ( struct llama_context * ctx); + LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model); @@ -586,7 +588,7 @@ extern "C" { // KV cache // - // TODO: remove llama_kv_cache_view_* API + // TODO: start using struct llama_kv_cache // Information associated with an individual cell in the KV cache view. struct llama_kv_cache_view_cell { @@ -641,13 +643,19 @@ extern "C" { // Returns the number of tokens in the KV cache (slow, use only for debug) // If a KV cell has multiple sequences assigned to it, it will be counted multiple times - LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx); + LLAMA_API int32_t llama_kv_self_n_tokens(const struct llama_context * ctx); + + DEPRECATED(LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx), + "use llama_kv_self_n_tokens instead"); // Returns the number of used KV cells (i.e. have at least one sequence assigned to them) - LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx); + LLAMA_API int32_t llama_kv_self_used_cells(const struct llama_context * ctx); + + DEPRECATED(LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx), + "use llama_kv_self_used_cells instead"); // Clear the KV cache - both cell info is erased and KV data is zeroed - LLAMA_API void llama_kv_cache_clear( + LLAMA_API void llama_kv_self_clear( struct llama_context * ctx); // Removes all tokens that belong to the specified sequence and have positions in [p0, p1) @@ -655,7 +663,7 @@ extern "C" { // seq_id < 0 : match any sequence // p0 < 0 : [0, p1] // p1 < 0 : [p0, inf) - LLAMA_API bool llama_kv_cache_seq_rm( + LLAMA_API bool llama_kv_self_seq_rm( struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, @@ -665,7 +673,7 @@ extern "C" { // Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence // p0 < 0 : [0, p1] // p1 < 0 : [p0, inf) - LLAMA_API void llama_kv_cache_seq_cp( + LLAMA_API void llama_kv_self_seq_cp( struct llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, @@ -673,17 +681,17 @@ extern "C" { llama_pos p1); // Removes all tokens that do not belong to the specified sequence - LLAMA_API void llama_kv_cache_seq_keep( + LLAMA_API void llama_kv_self_seq_keep( struct llama_context * ctx, llama_seq_id seq_id); // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) // If the KV cache is RoPEd, the KV data is updated accordingly: // - lazily on next llama_decode() - // - explicitly with llama_kv_cache_update() + // - explicitly with llama_kv_self_update() // p0 < 0 : [0, p1] // p1 < 0 : [p0, inf) - LLAMA_API void llama_kv_cache_seq_add( + LLAMA_API void llama_kv_self_seq_add( struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, @@ -693,10 +701,10 @@ extern "C" { // Integer division of the positions by factor of `d > 1` // If the KV cache is RoPEd, the KV data is updated accordingly: // - lazily on next llama_decode() - // - explicitly with llama_kv_cache_update() + // - explicitly with llama_kv_self_update() // p0 < 0 : [0, p1] // p1 < 0 : [p0, inf) - LLAMA_API void llama_kv_cache_seq_div( + LLAMA_API void llama_kv_self_seq_div( struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, @@ -704,24 +712,76 @@ extern "C" { int d); // Returns the largest position present in the KV cache for the specified sequence - LLAMA_API llama_pos llama_kv_cache_seq_pos_max( + LLAMA_API llama_pos llama_kv_self_seq_pos_max( struct llama_context * ctx, - llama_seq_id seq_id); - - // TODO: the llama_kv_cache_defrag and llama_kv_cache_update API tightly couples llama_context with llama_kv_cache - // how to avoid this? + llama_seq_id seq_id); // Defragment the KV cache // This will be applied: // - lazily on next llama_decode() - // - explicitly with llama_kv_cache_update() - LLAMA_API void llama_kv_cache_defrag(struct llama_context * ctx); - - // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) - LLAMA_API void llama_kv_cache_update(struct llama_context * ctx); + // - explicitly with llama_kv_self_update() + LLAMA_API void llama_kv_self_defrag(struct llama_context * ctx); // Check if the context supports KV cache shifting - LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx); + LLAMA_API bool llama_kv_self_can_shift(const struct llama_context * ctx); + + // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) + LLAMA_API void llama_kv_self_update(struct llama_context * ctx); + + DEPRECATED(LLAMA_API void llama_kv_cache_clear( + struct llama_context * ctx), + "use llama_kv_self_clear instead"); + + DEPRECATED(LLAMA_API bool llama_kv_cache_seq_rm( + struct llama_context * ctx, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1), + "use llama_kv_self_seq_rm instead"); + + DEPRECATED(LLAMA_API void llama_kv_cache_seq_cp( + struct llama_context * ctx, + llama_seq_id seq_id_src, + llama_seq_id seq_id_dst, + llama_pos p0, + llama_pos p1), + "use llama_kv_self_seq_cp instead"); + + DEPRECATED(LLAMA_API void llama_kv_cache_seq_keep( + struct llama_context * ctx, + llama_seq_id seq_id), + "use llama_kv_self_seq_keep instead"); + + DEPRECATED(LLAMA_API void llama_kv_cache_seq_add( + struct llama_context * ctx, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + llama_pos delta), + "use llama_kv_self_seq_add instead"); + + DEPRECATED(LLAMA_API void llama_kv_cache_seq_div( + struct llama_context * ctx, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + int d), + "use llama_kv_self_seq_div instead"); + + DEPRECATED(LLAMA_API llama_pos llama_kv_cache_seq_pos_max( + struct llama_context * ctx, + llama_seq_id seq_id), + "use llama_kv_self_seq_pos_max instead"); + + DEPRECATED(LLAMA_API void llama_kv_cache_defrag(struct llama_context * ctx), + "use llama_kv_self_defrag instead"); + + DEPRECATED(LLAMA_API bool llama_kv_cache_can_shift(const struct llama_context * ctx), + "use llama_kv_self_can_shift instead"); + + DEPRECATED(LLAMA_API void llama_kv_cache_update(struct llama_context * ctx), + "use llama_kv_self_update instead"); + // // State / sessions diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e1b02e4c08..b340dae5b2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,18 +15,21 @@ add_library(llama llama-chat.cpp llama-context.cpp llama-grammar.cpp + llama-graph.cpp llama-hparams.cpp llama-impl.cpp + llama-io.cpp llama-kv-cache.cpp + llama-memory.cpp llama-mmap.cpp llama-model-loader.cpp llama-model.cpp llama-quant.cpp llama-sampling.cpp llama-vocab.cpp - unicode.h - unicode.cpp unicode-data.cpp + unicode.cpp + unicode.h ) target_include_directories(llama PUBLIC . ../include ../common) diff --git a/src/llama-adapter.cpp b/src/llama-adapter.cpp index 8a08004631..b448614e47 100644 --- a/src/llama-adapter.cpp +++ b/src/llama-adapter.cpp @@ -4,14 +4,13 @@ #include "llama-mmap.h" #include "llama-model.h" -#include #include #include #include // vec -struct ggml_tensor * llama_adapter_cvec::tensor_for(int il) const { +ggml_tensor * llama_adapter_cvec::tensor_for(int il) const { if (il < 0 || il < layer_start || il > layer_end || (size_t) il >= tensors.size()) { return nullptr; } @@ -19,7 +18,7 @@ struct ggml_tensor * llama_adapter_cvec::tensor_for(int il) const { return tensors[il]; } -struct ggml_tensor * llama_adapter_cvec::apply_to(struct ggml_context * ctx, struct ggml_tensor * cur, int il) const { +ggml_tensor * llama_adapter_cvec::apply_to(ggml_context * ctx, ggml_tensor * cur, int il) const { ggml_tensor * layer_dir = tensor_for(il); if (layer_dir != nullptr) { cur = ggml_add(ctx, cur, layer_dir); @@ -40,7 +39,7 @@ bool llama_adapter_cvec::init(const llama_model & model) { auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * { auto it = ctx_map.find(buft); if (it == ctx_map.end()) { - struct ggml_init_params params = { + ggml_init_params params = { /*.mem_size =*/ hparams.n_layer*ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, @@ -91,7 +90,7 @@ bool llama_adapter_cvec::init(const llama_model & model) { return true; } -int32_t llama_adapter_cvec::apply( +bool llama_adapter_cvec::apply( const llama_model & model, const float * data, size_t len, @@ -104,17 +103,17 @@ int32_t llama_adapter_cvec::apply( // disable the current control vector (but leave allocated for later) layer_start = -1; layer_end = -1; - return 0; + return true; } if (n_embd != (int) hparams.n_embd) { LLAMA_LOG_ERROR("%s: control vector n_embd does not match model\n", __func__); - return 1; + return false; } if (tensors.empty()) { if (!init(model)) { - return 1; + return false; } } @@ -130,12 +129,12 @@ int32_t llama_adapter_cvec::apply( } } - return 0; + return true; } // lora -llama_adapter_lora_weight * llama_adapter_lora::get_weight(struct ggml_tensor * w) { +llama_adapter_lora_weight * llama_adapter_lora::get_weight(ggml_tensor * w) { const std::string name(w->name); const auto pos = ab_map.find(name); @@ -146,11 +145,11 @@ llama_adapter_lora_weight * llama_adapter_lora::get_weight(struct ggml_tensor * return nullptr; } -static void llama_adapter_lora_init_impl(struct llama_model & model, const char * path_lora, struct llama_adapter_lora & adapter) { +static void llama_adapter_lora_init_impl(llama_model & model, const char * path_lora, llama_adapter_lora & adapter) { LLAMA_LOG_INFO("%s: loading lora adapter from '%s' ...\n", __func__, path_lora); ggml_context * ctx_init; - struct gguf_init_params meta_gguf_params = { + gguf_init_params meta_gguf_params = { /* .no_alloc = */ true, /* .ctx = */ &ctx_init, }; @@ -201,7 +200,7 @@ static void llama_adapter_lora_init_impl(struct llama_model & model, const char auto it = ctx_map.find(buft); if (it == ctx_map.end()) { // add a new context - struct ggml_init_params params = { + ggml_init_params params = { /*.mem_size =*/ n_tensors*ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, @@ -264,7 +263,7 @@ static void llama_adapter_lora_init_impl(struct llama_model & model, const char throw std::runtime_error("LoRA tensor '" + name + "' does not exist in base model (hint: maybe wrong base model?)"); } - struct ggml_context * dev_ctx = ctx_for_buft(ggml_backend_buffer_get_type(model_tensor->buffer)); + ggml_context * dev_ctx = ctx_for_buft(ggml_backend_buffer_get_type(model_tensor->buffer)); // validate tensor shape if (is_token_embd) { // expect B to be non-transposed, A and B are flipped; see llm_build_inp_embd() @@ -281,8 +280,8 @@ static void llama_adapter_lora_init_impl(struct llama_model & model, const char } // save tensor to adapter - struct ggml_tensor * tensor_a = ggml_dup_tensor(dev_ctx, w.a); - struct ggml_tensor * tensor_b = ggml_dup_tensor(dev_ctx, w.b); + ggml_tensor * tensor_a = ggml_dup_tensor(dev_ctx, w.a); + ggml_tensor * tensor_b = ggml_dup_tensor(dev_ctx, w.b); ggml_set_name(tensor_a, w.a->name); ggml_set_name(tensor_b, w.b->name); adapter.ab_map[name] = llama_adapter_lora_weight(tensor_a, tensor_b); @@ -308,7 +307,7 @@ static void llama_adapter_lora_init_impl(struct llama_model & model, const char { llama_file gguf_file(path_lora, "rb"); std::vector read_buf; - auto set_tensor = [&](struct ggml_tensor * orig, struct ggml_tensor * dev) { + auto set_tensor = [&](ggml_tensor * orig, ggml_tensor * dev) { size_t offs = gguf_get_data_offset(ctx_gguf.get()) + gguf_get_tensor_offset(ctx_gguf.get(), gguf_find_tensor(ctx_gguf.get(), orig->name)); size_t size = ggml_nbytes(orig); read_buf.resize(size); @@ -327,8 +326,8 @@ static void llama_adapter_lora_init_impl(struct llama_model & model, const char LLAMA_LOG_INFO("%s: loaded %zu tensors from lora file\n", __func__, adapter.ab_map.size()*2); } -struct llama_adapter_lora * llama_adapter_lora_init(struct llama_model * model, const char * path_lora) { - struct llama_adapter_lora * adapter = new llama_adapter_lora(); +llama_adapter_lora * llama_adapter_lora_init(llama_model * model, const char * path_lora) { + llama_adapter_lora * adapter = new llama_adapter_lora(); try { llama_adapter_lora_init_impl(*model, path_lora, *adapter); @@ -342,6 +341,6 @@ struct llama_adapter_lora * llama_adapter_lora_init(struct llama_model * model, return nullptr; } -void llama_adapter_lora_free(struct llama_adapter_lora * adapter) { +void llama_adapter_lora_free(llama_adapter_lora * adapter) { delete adapter; } diff --git a/src/llama-adapter.h b/src/llama-adapter.h index 603fa08f6d..65824e9727 100644 --- a/src/llama-adapter.h +++ b/src/llama-adapter.h @@ -15,11 +15,11 @@ // struct llama_adapter_cvec { - struct ggml_tensor * tensor_for(int il) const; + ggml_tensor * tensor_for(int il) const; - struct ggml_tensor * apply_to(struct ggml_context * ctx, struct ggml_tensor * cur, int il) const; + ggml_tensor * apply_to(ggml_context * ctx, ggml_tensor * cur, int il) const; - int32_t apply( + bool apply( const llama_model & model, const float * data, size_t len, @@ -36,7 +36,7 @@ private: std::vector ctxs; std::vector bufs; - std::vector tensors; // per layer + std::vector tensors; // per layer }; // @@ -44,8 +44,8 @@ private: // struct llama_adapter_lora_weight { - struct ggml_tensor * a = nullptr; - struct ggml_tensor * b = nullptr; + ggml_tensor * a = nullptr; + ggml_tensor * b = nullptr; // get actual scale based on rank and alpha float get_scale(float alpha, float adapter_scale) const { @@ -55,12 +55,12 @@ struct llama_adapter_lora_weight { } llama_adapter_lora_weight() = default; - llama_adapter_lora_weight(struct ggml_tensor * a, struct ggml_tensor * b) : a(a), b(b) {} + llama_adapter_lora_weight(ggml_tensor * a, ggml_tensor * b) : a(a), b(b) {} }; struct llama_adapter_lora { // map tensor name to lora_a_b - std::unordered_map ab_map; + std::unordered_map ab_map; std::vector ctxs; std::vector bufs; @@ -70,5 +70,7 @@ struct llama_adapter_lora { llama_adapter_lora() = default; ~llama_adapter_lora() = default; - llama_adapter_lora_weight * get_weight(struct ggml_tensor * w); + llama_adapter_lora_weight * get_weight(ggml_tensor * w); }; + +using llama_adapter_loras = std::unordered_map; diff --git a/src/llama-batch.h b/src/llama-batch.h index 773c3808b7..f1df40d270 100644 --- a/src/llama-batch.h +++ b/src/llama-batch.h @@ -42,9 +42,9 @@ struct llama_sbatch { bool logits_all; // TODO: remove once lctx.logits_all is removed too // sorted indices into the batch - std::vector ids; + std::vector ids; // batch indices of the output - std::vector out_ids; + std::vector out_ids; std::vector seq; const llama_batch * batch = nullptr; diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 671d2a81ad..0a43a3af8e 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1,551 +1,1555 @@ #include "llama-context.h" #include "llama-impl.h" +#include "llama-io.h" #include "llama-mmap.h" +#include "llama-model.h" +#include "llama-kv-cache.h" #include -#include #include #include +#include -void llama_set_k_shift(struct llama_context & lctx) { - const int64_t kv_size = lctx.kv_self.size; +// +// llama_context +// - assert(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer)); +llama_context::llama_context( + const llama_model & model, + llama_context_params params) : + model(model) { + LLAMA_LOG_INFO("%s: constructing llama_context\n", __func__); - int32_t * data = (int32_t *) lctx.inp_K_shift->data; + t_start_us = model.t_start_us; + t_load_us = model.t_load_us; - for (int i = 0; i < kv_size; ++i) { - data[i] = lctx.kv_self.cells[i].delta; - } -} + const auto & hparams = model.hparams; -void llama_set_s_copy(struct llama_context & lctx) { - const int64_t kv_size = lctx.kv_self.size; + cparams.n_seq_max = std::max(1u, params.n_seq_max); + cparams.n_threads = params.n_threads; + cparams.n_threads_batch = params.n_threads_batch; + cparams.yarn_ext_factor = params.yarn_ext_factor; + cparams.yarn_attn_factor = params.yarn_attn_factor; + cparams.yarn_beta_fast = params.yarn_beta_fast; + cparams.yarn_beta_slow = params.yarn_beta_slow; + cparams.defrag_thold = params.defrag_thold; + cparams.embeddings = params.embeddings; + cparams.offload_kqv = params.offload_kqv; + cparams.flash_attn = params.flash_attn; + cparams.no_perf = params.no_perf; + cparams.pooling_type = params.pooling_type; - assert(ggml_backend_buffer_is_host(lctx.inp_s_copy->buffer)); + cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx; + cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base; + cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale; - int32_t * data = (int32_t *) lctx.inp_s_copy->data; + cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx : + hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn : + hparams.n_ctx_train; - for (int i = 0; i < kv_size; ++i) { - data[i] = lctx.kv_self.cells[i].src; - } -} + cparams.cb_eval = params.cb_eval; + cparams.cb_eval_user_data = params.cb_eval_user_data; -// llama input - -static int32_t llama_relative_position_bucket(llama_pos x, llama_pos y, uint64_t n_buckets, bool bidirectional) { - // TODO move to hparams if a T5 variant appears that uses a different value - const int64_t max_distance = 128; - - if (bidirectional) { - n_buckets >>= 1; + auto rope_scaling_type = params.rope_scaling_type; + if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) { + rope_scaling_type = hparams.rope_scaling_type_train; } - const int64_t max_exact = n_buckets >> 1; + if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) { + cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none + } - int32_t relative_position = x - y; - int32_t relative_bucket = 0; - if (bidirectional) { - relative_bucket += (relative_position > 0) * n_buckets; - relative_position = abs(relative_position); + if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set' + cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f; + } + + cparams.yarn_attn_factor *= hparams.rope_attn_factor; + + if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { + if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { + cparams.pooling_type = LLAMA_POOLING_TYPE_NONE; + } else { + cparams.pooling_type = hparams.pooling_type; + } + } + + if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) { + cparams.causal_attn = hparams.causal_attn; } else { - relative_position = -std::min(relative_position, 0); + cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL; + } + + // with causal attention, the batch size is limited by the context size + cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch; + + // the batch has to be at least GGML_KQ_MASK_PAD because we will be padding the KQ_mask + // this is required by GPU kernels in order to avoid out-of-bounds accesses (e.g. ggml_flash_attn_ext) + // ref: https://github.com/ggerganov/llama.cpp/pull/5021 + // TODO: this padding is not needed for the cache-less context so we should probably move it to llama_context_kv_self + if (cparams.n_batch < GGML_KQ_MASK_PAD) { + LLAMA_LOG_WARN("%s: n_batch is less than GGML_KQ_MASK_PAD - increasing to %d\n", __func__, GGML_KQ_MASK_PAD); + cparams.n_batch = GGML_KQ_MASK_PAD; + } + + cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch); + + const uint32_t n_ctx_per_seq = cparams.n_ctx / cparams.n_seq_max; + + LLAMA_LOG_INFO("%s: n_seq_max = %u\n", __func__, cparams.n_seq_max); + LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx); + LLAMA_LOG_INFO("%s: n_ctx_per_seq = %u\n", __func__, n_ctx_per_seq); + LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch); + LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch); + LLAMA_LOG_INFO("%s: causal_attn = %d\n", __func__, cparams.causal_attn); + LLAMA_LOG_INFO("%s: flash_attn = %d\n", __func__, cparams.flash_attn); + LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base); + LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale); + + if (n_ctx_per_seq < hparams.n_ctx_train) { + LLAMA_LOG_WARN("%s: n_ctx_per_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n", + __func__, n_ctx_per_seq, hparams.n_ctx_train); + } + + if (n_ctx_per_seq > hparams.n_ctx_train) { + LLAMA_LOG_WARN("%s: n_ctx_pre_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n", + __func__, n_ctx_per_seq, hparams.n_ctx_train); + } + + logits_all = params.logits_all; + + if (!hparams.vocab_only) { + // GPU backends + for (auto * dev : model.devices) { + ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); + if (backend == nullptr) { + throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev))); + } + backends.emplace_back(backend); + } + + // add ACCEL backends (such as BLAS) + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) { + ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); + if (backend == nullptr) { + throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev))); + } + backends.emplace_back(backend); + } + } + + // add CPU backend + backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (backend_cpu == nullptr) { + throw std::runtime_error("failed to initialize CPU backend"); + } + backends.emplace_back(backend_cpu); + + // create a list of the set_n_threads functions in the backends + for (auto & backend : backends) { + ggml_backend_dev_t dev = ggml_backend_get_device(backend.get()); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (reg) { + auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + if (ggml_backend_set_n_threads_fn) { + set_n_threads_fns.emplace_back(backend.get(), ggml_backend_set_n_threads_fn); + } + } + } + + llama_set_abort_callback(this, params.abort_callback, params.abort_callback_data); + + // graph outputs buffer + { + // resized during inference when a batch uses more outputs + if ((uint32_t) output_reserve(params.n_seq_max) < params.n_seq_max) { + throw std::runtime_error("failed to reserve initial output buffer"); + } + + LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__, + ggml_backend_buffer_name (buf_output.get()), + ggml_backend_buffer_get_size(buf_output.get()) / 1024.0 / 1024.0); + } + } + + // init the memory module + // TODO: for now, always create a unified KV cache + if (!hparams.vocab_only) { + kv_self.reset(static_cast(model.create_memory())); + + LLAMA_LOG_DEBUG("%s: n_ctx = %u\n", __func__, cparams.n_ctx); + + cparams.n_ctx = GGML_PAD(cparams.n_ctx, kv_self->get_padding(cparams)); + + LLAMA_LOG_DEBUG("%s: n_ctx = %u (padded)\n", __func__, cparams.n_ctx); + + uint32_t kv_size = cparams.n_ctx; + ggml_type type_k = params.type_k; + ggml_type type_v = params.type_v; + + if (llama_model_is_recurrent(&model)) { + // Mamba needs at least as many KV cells as there are sequences kept at any time + kv_size = std::max((uint32_t) 1, params.n_seq_max); + // it's probably best to keep as much precision as possible for the states + type_k = GGML_TYPE_F32; // required by ggml_ssm_conv for Mamba's conv_states + type_v = GGML_TYPE_F32; // required by ggml_ssm_scan for Mamba's ssm_states + } + + GGML_ASSERT(hparams.n_embd_head_k % ggml_blck_size(type_k) == 0); + GGML_ASSERT(hparams.n_embd_head_v % ggml_blck_size(type_v) == 0); + + if (!kv_self->init(model, cparams, type_k, type_v, kv_size, cparams.offload_kqv)) { + throw std::runtime_error("failed to initialize self-attention cache"); + } + + { + const size_t memory_size_k = kv_self->size_k_bytes(); + const size_t memory_size_v = kv_self->size_v_bytes(); + + LLAMA_LOG_INFO("%s: KV self size = %7.2f MiB, K (%s): %7.2f MiB, V (%s): %7.2f MiB\n", __func__, + (float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f), + ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f), + ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f)); + } + } + + // init backends + if (!hparams.vocab_only) { + LLAMA_LOG_DEBUG("%s: enumerating backends\n", __func__); + + backend_buft.clear(); + backend_ptrs.clear(); + + for (auto & backend : backends) { + auto * buft = ggml_backend_get_default_buffer_type(backend.get()); + auto backend_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get())); + + if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model.devices.empty()) { + // use the host buffer of the first device CPU for faster transfer of the intermediate state + auto * dev = model.devices[0]; + auto * host_buft = ggml_backend_dev_host_buffer_type(dev); + if (host_buft) { + buft = host_buft; + } + } + + backend_buft.push_back(buft); + backend_ptrs.push_back(backend.get()); + } + + LLAMA_LOG_DEBUG("%s: backend_ptrs.size() = %zu\n", __func__, backend_ptrs.size()); + + const size_t max_nodes = this->graph_max_nodes(); + + LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes); + + // buffer used to store the computation graph and the tensor meta data + buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false)); + + // TODO: move these checks to ggml_backend_sched + // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary + bool pipeline_parallel = + model.n_devices() > 1 && + model.params.n_gpu_layers > (int) model.hparams.n_layer && + model.params.split_mode == LLAMA_SPLIT_MODE_LAYER && + cparams.offload_kqv; + + // pipeline parallelism requires support for async compute and events in all devices + if (pipeline_parallel) { + for (auto & backend : backends) { + auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get())); + if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU) { + // ignore CPU backend + continue; + } + auto * dev = ggml_backend_get_device(backend.get()); + ggml_backend_dev_props props; + ggml_backend_dev_get_props(dev, &props); + if (!props.caps.async || !props.caps.events) { + // device does not support async compute or events + pipeline_parallel = false; + break; + } + } + } + + sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, pipeline_parallel)); + + if (pipeline_parallel) { + LLAMA_LOG_INFO("%s: pipeline parallelism enabled (n_copies=%d)\n", __func__, ggml_backend_sched_get_n_copies(sched.get())); + } + } + + // reserve worst-case graph + if (!hparams.vocab_only) { + uint32_t n_seqs = 1; // TODO: worst-case number of sequences + uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); + + llama_token token = model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph + + // max number of outputs + n_outputs = n_tokens; + + LLAMA_LOG_DEBUG("%s: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs); + + int n_splits_pp = -1; + int n_nodes_pp = -1; + + int n_splits_tg = -1; + int n_nodes_tg = -1; + + // simulate full KV cache + kv_self->n = kv_self->size; + + cross.v_embd.clear(); + + // reserve pp graph first so that buffers are only allocated once + { + llama_ubatch ubatch_pp = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; + auto * gf = graph_init(); + graph_build(ctx_compute.get(), gf, ubatch_pp, LLM_GRAPH_TYPE_DEFAULT); + if (!ggml_backend_sched_reserve(sched.get(), gf)) { + throw std::runtime_error("failed to allocate compute pp buffers"); + } + + n_splits_pp = ggml_backend_sched_get_n_splits(sched.get()); + n_nodes_pp = ggml_graph_n_nodes(gf); + } + + // reserve with tg graph to get the number of splits and nodes + { + llama_ubatch ubatch_tg = { true, 1, 1, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; + auto * gf = graph_init(); + graph_build(ctx_compute.get(), gf, ubatch_tg, LLM_GRAPH_TYPE_DEFAULT); + if (!ggml_backend_sched_reserve(sched.get(), gf)) { + throw std::runtime_error("failed to allocate compute tg buffers"); + } + n_splits_tg = ggml_backend_sched_get_n_splits(sched.get()); + n_nodes_tg = ggml_graph_n_nodes(gf); + } + + // reserve again with pp graph to avoid ggml-alloc reallocations during inference + { + llama_ubatch ubatch_pp = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; + auto * gf = graph_init(); + graph_build(ctx_compute.get(), gf, ubatch_pp, LLM_GRAPH_TYPE_DEFAULT); + if (!ggml_backend_sched_reserve(sched.get(), gf)) { + throw std::runtime_error("failed to allocate compute pp buffers"); + } + } + + for (size_t i = 0; i < backend_ptrs.size(); ++i) { + ggml_backend_t backend = backend_ptrs[i]; + ggml_backend_buffer_type_t buft = backend_buft[i]; + size_t size = ggml_backend_sched_get_buffer_size(sched.get(), backend); + if (size > 1) { + LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__, + ggml_backend_buft_name(buft), + size / 1024.0 / 1024.0); + } + } + + if (n_nodes_pp == n_nodes_tg) { + LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, n_nodes_pp); + } else { + LLAMA_LOG_INFO("%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n", __func__, n_nodes_pp, n_tokens, n_nodes_tg); + } + + if (n_splits_pp == n_splits_tg) { + LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits_pp); + } else { + LLAMA_LOG_INFO("%s: graph splits = %d (with bs=%d), %d (with bs=1)\n", __func__, n_splits_pp, n_tokens, n_splits_tg); + } } - int32_t relative_position_if_large = floorf(max_exact + logf(1.0 * relative_position / max_exact) * (n_buckets - max_exact) / log(1.0 * max_distance / max_exact)); - relative_position_if_large = std::min(relative_position_if_large, n_buckets - 1); - relative_bucket += (relative_position < max_exact ? relative_position : relative_position_if_large); - return relative_bucket; } -void llama_set_inputs(llama_context & lctx, const llama_ubatch & ubatch) { +llama_context::~llama_context() = default; + +void llama_context::synchronize() { + ggml_backend_sched_synchronize(sched.get()); + + // FIXME: if multiple single tokens are evaluated without a synchronization, + // the stats will be added to the prompt evaluation stats + // this should only happen when using batch size 1 to evaluate a batch + + // add the evaluation to the stats + if (n_queued_tokens == 1) { + if (!cparams.no_perf) { + t_eval_us += ggml_time_us() - t_compute_start_us; + } + n_eval++; + } else if (n_queued_tokens > 1) { + if (!cparams.no_perf) { + t_p_eval_us += ggml_time_us() - t_compute_start_us; + } + n_p_eval += n_queued_tokens; + } + + // get a more accurate load time, upon first eval + if (n_queued_tokens > 0 && !has_evaluated_once) { + t_load_us = ggml_time_us() - t_start_us; + has_evaluated_once = true; + } + + n_queued_tokens = 0; + t_compute_start_us = 0; +} + +const llama_model & llama_context::get_model() const { + return model; +} + +uint32_t llama_context::n_ctx() const { + return cparams.n_ctx; +} + +uint32_t llama_context::n_ctx_per_seq() const { + return cparams.n_ctx / cparams.n_seq_max; +} + +uint32_t llama_context::n_batch() const { + return cparams.n_batch; +} + +uint32_t llama_context::n_ubatch() const { + return cparams.n_ubatch; +} + +uint32_t llama_context::n_seq_max() const { + return cparams.n_seq_max; +} + +uint32_t llama_context::n_threads() const { + return cparams.n_threads; +} + +uint32_t llama_context::n_threads_batch() const { + return cparams.n_threads_batch; +} + +llama_kv_cache * llama_context::get_kv_self() { + return kv_self.get(); +} + +const llama_kv_cache * llama_context::get_kv_self() const { + return kv_self.get(); +} + +ggml_tensor * llama_context::build_rope_shift( + ggml_context * ctx0, + ggml_tensor * cur, + ggml_tensor * shift, + ggml_tensor * factors, + ggml_backend_buffer * bbuf) const { + const auto & n_ctx_orig = cparams.n_ctx_orig_yarn; + const auto & freq_base = cparams.rope_freq_base; + const auto & freq_scale = cparams.rope_freq_scale; + + const auto & yarn_ext_factor = cparams.yarn_ext_factor; + const auto & yarn_attn_factor = cparams.yarn_attn_factor; + const auto & yarn_beta_fast = cparams.yarn_beta_fast; + const auto & yarn_beta_slow = cparams.yarn_beta_slow; + + const auto & hparams = model.hparams; + + const auto & n_rot = hparams.n_rot; + const auto & rope_type = hparams.rope_type; + + ggml_tensor * tmp; + + if (ggml_is_quantized(cur->type)) { + // dequantize to f32 -> RoPE -> quantize back + tmp = ggml_cast(ctx0, cur, GGML_TYPE_F32); + + if (bbuf) { + for (const auto & backend : backends) { + // Figure out which backend KV cache belongs to + if (ggml_backend_supports_buft(backend.get(), ggml_backend_buffer_get_type(bbuf))) { + ggml_backend_sched_set_tensor_backend(sched.get(), tmp, backend.get()); + break; + } + } + } + + tmp = ggml_rope_ext_inplace(ctx0, tmp, + shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow); + + tmp = ggml_cpy(ctx0, tmp, cur); + } else { + // we rotate only the first n_rot dimensions + tmp = ggml_rope_ext_inplace(ctx0, cur, + shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow); + } + + return tmp; +} + +class llm_graph_input_k_shift : public llm_graph_input_i { +public: + llm_graph_input_k_shift(const llama_kv_cache_unified * kv_self) : kv_self(kv_self) {} + virtual ~llm_graph_input_k_shift() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * k_shift; // I32 [kv_size] + + const llama_kv_cache_unified * kv_self; +}; + +void llm_graph_input_k_shift::set_input(const llama_ubatch * ubatch) { + GGML_UNUSED(ubatch); + + if (k_shift) { + assert(ggml_backend_buffer_is_host(k_shift->buffer)); + + int32_t * data = (int32_t *) k_shift->data; + + for (uint32_t i = 0; i < kv_self->size; ++i) { + data[i] = kv_self->cells[i].delta; + } + } +} + +llm_graph_result_ptr llama_context::build_kv_self_shift( + ggml_context * ctx0, + ggml_cgraph * gf) const { + auto res = std::make_unique(); + + const auto & hparams = model.hparams; + + const auto & n_layer = hparams.n_layer; + + const auto & n_embd_head_k = hparams.n_embd_head_k; + //const auto & n_embd_head_v = hparams.n_embd_head_v; + + //GGML_ASSERT(kv_self->size == n_ctx); + + auto inp = std::make_unique(kv_self.get()); + + inp->k_shift = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, cparams.n_ctx); + ggml_set_input(inp->k_shift); + + for (uint32_t il = 0; il < n_layer; ++il) { + const int64_t n_head_kv = hparams.n_head_kv(il); + const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); + + ggml_tensor * rope_factors = kv_self->cbs.get_rope_factors(n_ctx_per_seq(), il); + + ggml_tensor * k = + ggml_view_3d(ctx0, kv_self->k_l[il], + n_embd_head_k, n_head_kv, kv_self->size, + ggml_row_size(kv_self->k_l[il]->type, n_embd_head_k), + ggml_row_size(kv_self->k_l[il]->type, n_embd_k_gqa), + 0); + + ggml_tensor * cur = build_rope_shift(ctx0, k, inp->k_shift, rope_factors, kv_self->k_l[il]->buffer); + + ggml_build_forward_expand(gf, cur); + } + + res->add_input(std::move(inp)); + + return res; +} + +llm_graph_result_ptr llama_context::build_kv_self_defrag( + ggml_context * ctx0, + ggml_cgraph * gf) const { + auto res = std::make_unique(); + + const auto & hparams = model.hparams; + + const auto & ids = kv_self->defrag_info.ids; + +#if 0 + // CPU defrag // - // set input data + // TODO: optimizations are possible: + // - multiple threads + // - avoid copying to the host memory when already there + // + // likely not worth the effort, as we have ggml_graph based defrag // - const auto & hparams = lctx.model.hparams; - const auto & cparams = lctx.cparams; - const auto & kv_self = lctx.kv_self; + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); - if (ubatch.token) { - const int64_t n_tokens = ubatch.n_tokens; + const uint32_t kv_size = size; - ggml_backend_tensor_set(lctx.inp_tokens, ubatch.token, 0, n_tokens*ggml_element_size(lctx.inp_tokens)); - } + std::vector buf_k; + std::vector buf_v; - if (ubatch.embd) { - const int64_t n_embd = hparams.n_embd; - const int64_t n_tokens = ubatch.n_tokens; + for (uint32_t il = 0; il < n_layer; ++il) { + const size_t k_size_row = ggml_row_size(k_l[il]->type, n_embd_k_gqa); + const size_t k_size = ggml_row_size(k_l[il]->type, n_embd_k_gqa*kv_size); - ggml_backend_tensor_set(lctx.inp_embd, ubatch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd)); - } + const size_t v_size_el = ggml_type_size(v_l[il]->type); + const size_t v_size = ggml_row_size (v_l[il]->type, n_embd_v_gqa*kv_size); - if (ubatch.pos && lctx.inp_pos) { - const int64_t n_tokens = ubatch.n_tokens; - auto n_pos = lctx.n_pos_per_token; - ggml_backend_tensor_set(lctx.inp_pos, ubatch.pos, 0, n_tokens*n_pos*ggml_element_size(lctx.inp_pos)); - } + buf_k.resize(k_size); + buf_v.resize(v_size); - if (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) { - //GGML_ASSERT(lctx.inp_out_ids && "every model that can must skip unused outputs"); + ggml_backend_tensor_get(k_l[il], buf_k.data(), 0, buf_k.size()); + ggml_backend_tensor_get(v_l[il], buf_v.data(), 0, buf_v.size()); - if (!lctx.inp_out_ids) { - LLAMA_LOG_WARN("%s: 'lctx.inp_out_ids' is not created\n", __func__); - } else { - const int64_t n_tokens = ubatch.n_tokens; + // batch move [i, i+nm) to [id, id+nm) + // note: cells can move only to a lower index + for (uint32_t i = 0; i < n_kv; ++i) { + const uint32_t id = ids[i]; - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_out_ids->buffer)); - int32_t * data = (int32_t *) lctx.inp_out_ids->data; + if (i == id || id == n_kv) { + continue; + } - if (lctx.n_outputs == n_tokens) { - for (int i = 0; i < n_tokens; ++i) { - data[i] = i; + uint32_t nm = 1; + + while (i + nm < n_kv && ids[i + nm] == id + nm) { + nm++; + } + + // move keys + { + const int64_t os = i*k_size_row; + const int64_t od = id*k_size_row; + + memcpy(buf_k.data() + od, buf_k.data() + os, nm*k_size_row); + } + + // move values (note: they are transposed) + { + const int64_t os = i; + const int64_t od = id; + + for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { + memcpy(buf_v.data() + (od + j*kv_size)*v_size_el, buf_v.data() + (os + j*kv_size)*v_size_el, nm*v_size_el); } - } else if (ubatch.output) { - int32_t n_outputs = 0; - for (int i = 0; i < n_tokens; ++i) { - if (ubatch.output[i]) { - data[n_outputs++] = i; - } - } - // the graph needs to have been passed the correct number of outputs - GGML_ASSERT(lctx.n_outputs == n_outputs); - } else if (lctx.n_outputs == 1) { - // only keep last output - data[0] = n_tokens - 1; + } + + i += nm - 1; + } + + ggml_backend_tensor_set(k_l[il], buf_k.data(), 0, buf_k.size()); + ggml_backend_tensor_set(v_l[il], buf_v.data(), 0, buf_v.size()); + } +#else + for (uint32_t i = 0; i < ids.size(); ++i) { + const uint32_t id = ids[i]; + + if (i == id || id == ids.size()) { + continue; + } + + uint32_t nm = 1; + + while (i + nm < ids.size() && ids[i + nm] == id + nm) { + nm++; + } + + for (uint32_t il = 0; il < hparams.n_layer; ++il) { // NOLINT + const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); + const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(il); + + ggml_tensor * view_k_src = ggml_view_2d(ctx0, kv_self->k_l[il], + n_embd_k_gqa, nm, + ggml_row_size(kv_self->k_l[il]->type, n_embd_k_gqa), + ggml_row_size(kv_self->k_l[il]->type, n_embd_k_gqa*i)); + + ggml_tensor * view_k_dst = ggml_view_2d(ctx0, kv_self->k_l[il], + n_embd_k_gqa, nm, + ggml_row_size(kv_self->k_l[il]->type, n_embd_k_gqa), + ggml_row_size(kv_self->k_l[il]->type, n_embd_k_gqa*id)); + + ggml_tensor * view_v_src; + ggml_tensor * view_v_dst; + + if (cparams.flash_attn) { + // NOTE: the V cache is not transposed when using flash attention + view_v_src = ggml_view_2d(ctx0, kv_self->v_l[il], + n_embd_v_gqa, nm, + ggml_row_size(kv_self->v_l[il]->type, n_embd_v_gqa), + ggml_row_size(kv_self->v_l[il]->type, n_embd_v_gqa*i)); + + view_v_dst = ggml_view_2d(ctx0, kv_self->v_l[il], + n_embd_v_gqa, nm, + ggml_row_size(kv_self->v_l[il]->type, n_embd_v_gqa), + ggml_row_size(kv_self->v_l[il]->type, n_embd_v_gqa*id)); } else { - GGML_ASSERT(lctx.n_outputs == 0); + view_v_src = ggml_view_2d(ctx0, kv_self->v_l[il], + nm, n_embd_v_gqa, + ggml_row_size(kv_self->v_l[il]->type, kv_self->size), + ggml_row_size(kv_self->v_l[il]->type, i)); + + view_v_dst = ggml_view_2d(ctx0, kv_self->v_l[il], + nm, n_embd_v_gqa, + ggml_row_size(kv_self->v_l[il]->type, kv_self->size), + ggml_row_size(kv_self->v_l[il]->type, id)); + } + + ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_k_src, view_k_dst)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_v_src, view_v_dst)); + } + + i += nm - 1; + } + + //LLAMA_LOG_INFO("gf->n_nodes = %d\n", gf->n_nodes); +#endif + + return res; +} + +void llama_context::kv_self_update() { + auto & kv = kv_self; + + bool need_reserve = false; + + if (kv->has_shift) { + if (!kv->get_can_shift()) { + GGML_ABORT("The current context does not support K-shift"); + } + + LLAMA_LOG_DEBUG("%s: applying K-shift\n", __func__); + + // apply K-shift if needed + if (model.hparams.rope_type != LLAMA_ROPE_TYPE_NONE) { + ggml_backend_sched_reset(sched.get()); + + auto * gf = graph_init(); + + auto res = build_kv_self_shift(ctx_compute.get(), gf); + + ggml_backend_sched_alloc_graph(sched.get(), gf); + + res->set_inputs(nullptr); + + graph_compute(gf, false); + + need_reserve = true; + } + + { + kv->has_shift = false; + + for (uint32_t i = 0; i < kv->size; ++i) { + kv->cells[i].delta = 0; } } } - GGML_ASSERT( - // (!a || b) is a logical implication (a -> b) - // !hparams.causal_attn -> !cparams.causal_attn - (hparams.causal_attn || !cparams.causal_attn) && - "causal attention is not supported by this model" - ); + // defragment the KV cache if needed + if (kv->do_defrag) { + LLAMA_LOG_DEBUG("%s: defragmenting KV cache\n", __func__); - if (lctx.inp_KQ_mask || lctx.inp_KQ_mask_swa) { - // NOTE: hparams.causal_attn indicates the model is capable of generation and uses the kv cache. - if (cparams.causal_attn && !lctx.is_encoding) { - const int64_t n_kv = kv_self.n; - const int64_t n_tokens = ubatch.n_tokens; - const int64_t n_seq_tokens = ubatch.n_seq_tokens; - const int64_t n_seqs = ubatch.n_seqs; + if (kv->defrag_prepare(graph_max_nodes())) { + ggml_backend_sched_reset(sched.get()); + auto * gf = graph_init(); - float * data = nullptr; - float * data_swa = nullptr; + auto res = build_kv_self_defrag(ctx_compute.get(), gf); - if (lctx.inp_KQ_mask) { - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); - data = (float *) lctx.inp_KQ_mask->data; - } + ggml_backend_sched_alloc_graph(sched.get(), gf); - if (lctx.inp_KQ_mask_swa) { - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask_swa->buffer)); - data_swa = (float *) lctx.inp_KQ_mask_swa->data; - } + res->set_inputs(nullptr); - // For causal attention, use only the previous KV cells - // of the correct sequence for each token of the ubatch. - // It's assumed that if a token in the batch has multiple sequences, they are equivalent. - for (int h = 0; h < 1; ++h) { - for (int s = 0; s < n_seqs; ++s) { - const llama_seq_id seq_id = ubatch.seq_id[s][0]; + graph_compute(gf, false); - for (int j = 0; j < n_seq_tokens; ++j) { - const llama_pos pos = ubatch.pos[s*n_seq_tokens + j]; - - for (int i = 0; i < n_kv; ++i) { - float f; - if (!kv_self.cells[i].has_seq_id(seq_id) || kv_self.cells[i].pos > pos) { - f = -INFINITY; - } else { - if (hparams.use_alibi) { - f = -std::abs(kv_self.cells[i].pos - pos); - } else { - f = 0.0f; - } - } - - if (data) { - data[h*(n_kv*n_tokens) + s*(n_kv*n_seq_tokens) + j*n_kv + i] = f; - } - - // may need to cut off old tokens for sliding window - if (data_swa) { - if (pos - kv_self.cells[i].pos >= (int32_t)hparams.n_swa) { - f = -INFINITY; - } - data_swa[h*(n_kv*n_tokens) + s*(n_kv*n_seq_tokens) + j*n_kv + i] = f; - } - } - } - } - - if (data) { - for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) { - for (int j = 0; j < n_kv; ++j) { - data[h*(n_kv*n_tokens) + i*n_kv + j] = -INFINITY; - } - } - } - - if (data_swa) { - for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) { - for (int j = 0; j < n_kv; ++j) { - data_swa[h*(n_kv*n_tokens) + i*n_kv + j] = -INFINITY; - } - } - } - } - } else { - const int64_t n_tokens = ubatch.n_tokens; - const int64_t n_seq_tokens = ubatch.n_seq_tokens; - const int64_t n_seqs = ubatch.n_seqs; - // when using kv cache, the mask needs to match the kv cache size - const int64_t n_stride = hparams.causal_attn && !lctx.is_encoding ? kv_self.n : n_tokens; - - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); - - float * data = (float *) lctx.inp_KQ_mask->data; - - for (int h = 0; h < 1; ++h) { - for (int s1 = 0; s1 < n_seqs; ++s1) { - const llama_seq_id seq_id = ubatch.seq_id[s1][0]; - - for (int j = 0; j < n_seq_tokens; ++j) { - const int32_t tj = s1*n_seq_tokens + j; - - for (int s0 = 0; s0 < n_seqs; ++s0) { - for (int i = 0; i < n_seq_tokens; ++i) { - const int32_t ti = s0*n_seq_tokens + i; - float f = -INFINITY; - - for (int s = 0; s < ubatch.n_seq_id[s0]; ++s) { - if (ubatch.seq_id[s0][s] == seq_id) { - if (hparams.use_alibi) { - f = -std::abs(ubatch.pos[ti] - ubatch.pos[tj]); - } else { - f = 0.0f; - } - break; - } - } - - data[h*(n_tokens*n_tokens) + tj*n_stride + ti] = f; - } - } - - for (int i = n_tokens; i < n_stride; ++i) { - data[h*(n_tokens*n_tokens) + tj*n_stride + i] = -INFINITY; - } - } - } - } + need_reserve = true; } + + kv->do_defrag = false; } - if (cparams.embeddings && cparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) { - const int64_t n_tokens = ubatch.n_tokens; - const int64_t n_seq_tokens = ubatch.n_seq_tokens; - const int64_t n_seqs = ubatch.n_seqs; + // reserve a worst case graph if needed + if (need_reserve) { + LLAMA_LOG_DEBUG("%s: reserving a worst case graph\n", __func__); - GGML_ASSERT(lctx.inp_mean); - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_mean->buffer)); + // build worst-case graph + uint32_t n_seqs = 1; // TODO: worst-case number of sequences + uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); - float * data = (float *) lctx.inp_mean->data; - memset(lctx.inp_mean->data, 0, n_tokens * n_tokens * ggml_element_size(lctx.inp_mean)); + // simulate full KV cache + kv_self->n = kv_self->size; - std::vector sum(n_tokens, 0); + llama_token token = model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph + llama_ubatch ubatch = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; - for (int s = 0; s < n_seqs; ++s) { - const llama_seq_id seq_id = ubatch.seq_id[s][0]; + auto * gf = graph_init(); + graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_DEFAULT); - // TODO: adapt limits to n_seqs when ubatch.equal_seqs is true - GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == MEAN"); - - sum[seq_id] += ubatch.n_seq_tokens; - } - - std::vector div(n_tokens, 0.0f); - for (int i = 0; i < n_tokens; ++i) { - const uint64_t s = sum[i]; - if (s > 0) { - div[i] = 1.0f/float(s); - } - } - - for (int s = 0; s < n_seqs; ++s) { - const llama_seq_id seq_id = ubatch.seq_id[s][0]; - - for (int i = 0; i < n_seq_tokens; ++i) { - data[seq_id*n_tokens + s*n_seq_tokens + i] = div[seq_id]; - } - } - } - - if (cparams.embeddings && ( - cparams.pooling_type == LLAMA_POOLING_TYPE_CLS || - cparams.pooling_type == LLAMA_POOLING_TYPE_RANK)) { - const int64_t n_tokens = ubatch.n_tokens; - const int64_t n_seq_tokens = ubatch.n_seq_tokens; - const int64_t n_seqs = ubatch.n_seqs; - - GGML_ASSERT(lctx.inp_cls); - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer)); - - uint32_t * data = (uint32_t *) lctx.inp_cls->data; - memset(lctx.inp_cls->data, 0, n_tokens * ggml_element_size(lctx.inp_cls)); - - for (int s = 0; s < n_seqs; ++s) { - const llama_seq_id seq_id = ubatch.seq_id[s][0]; - - // TODO: adapt limits to n_seqs when ubatch.equal_seqs is true - GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == CLS or RANK"); - - for (int i = 0; i < n_seq_tokens; ++i) { - const llama_pos pos = ubatch.pos[s*n_seq_tokens + i]; - - if (pos == 0) { - data[seq_id] = s*n_seq_tokens + i; - } - } - } - } - - if (cparams.embeddings && cparams.pooling_type == LLAMA_POOLING_TYPE_LAST) { - const int64_t n_tokens = ubatch.n_tokens; - const int64_t n_seq_tokens = ubatch.n_seq_tokens; - const int64_t n_seqs = ubatch.n_seqs; - - GGML_ASSERT(lctx.inp_cls); - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer)); - - uint32_t * data = (uint32_t *) lctx.inp_cls->data; - memset(lctx.inp_cls->data, 0, n_tokens * ggml_element_size(lctx.inp_cls)); - - std::vector last_pos(n_tokens, -1); - std::vector last_row(n_tokens, -1); - - for (int s = 0; s < n_seqs; ++s) { - const llama_seq_id seq_id = ubatch.seq_id[s][0]; - - // TODO: adapt limits to n_seqs when ubatch.equal_seqs is true - GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == LAST"); - - for (int i = 0; i < n_seq_tokens; ++i) { - const llama_pos pos = ubatch.pos[s*n_seq_tokens + i]; - - if (pos >= last_pos[seq_id]) { - last_pos[seq_id] = pos; - last_row[seq_id] = s*n_seq_tokens + i; - } - } - } - - for (int i = 0; i < n_tokens; ++i) { - if (last_row[i] >= 0) { - data[i] = last_row[i]; - } - } - } - - if (kv_self.recurrent) { - const int64_t n_kv = kv_self.n; - - if (lctx.inp_s_mask) { - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_mask->buffer)); - float * data = (float *) lctx.inp_s_mask->data; - - // clear unused states - for (int i = 0; i < n_kv; ++i) { - const uint32_t cell_id = i + kv_self.head; - llama_kv_cell & kv_cell = lctx.kv_self.cells[cell_id]; - - data[i] = (float) (kv_cell.src >= 0); - - // only clear once - if (kv_cell.src < 0) { - kv_cell.src = cell_id; - } - } - } - - if (lctx.inp_s_copy) { - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_copy->buffer)); - int32_t * data = (int32_t *) lctx.inp_s_copy->data; - - // assuming copy destinations ALWAYS happen ONLY on the cells between head and head+n - for (uint32_t i = 0; i < n_kv; ++i) { - const uint32_t cell_id = i + kv_self.head; - llama_kv_cell & kv_cell = lctx.kv_self.cells[cell_id]; - - // prevent out-of-bound sources - if (kv_cell.src < 0 || (uint32_t) kv_cell.src >= kv_self.size) { - kv_cell.src = cell_id; - } - - data[i] = kv_cell.src; - - // ensure copy only happens once - if (kv_cell.src != (int32_t) cell_id) { - kv_cell.src = cell_id; - } - } - } - } - - if (lctx.inp_pos_bucket) { - const int64_t n_tokens = ubatch.n_tokens; - - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_pos_bucket->buffer)); - GGML_ASSERT(!ubatch.equal_seqs); // TODO: use ubatch.n_seqs instead of failing - - int32_t * data = (int32_t *) lctx.inp_pos_bucket->data; - - if (!lctx.is_encoding) { - const int64_t n_kv = kv_self.n; - for (int h = 0; h < 1; ++h) { - for (int j = 0; j < n_tokens; ++j) { - for (int i = 0; i < n_kv; ++i) { - data[h*(n_kv*n_tokens) + j*n_kv + i] = llama_relative_position_bucket(lctx.kv_self.cells[i].pos, ubatch.pos[j], hparams.n_rel_attn_bkts, lctx.is_encoding); - } - } - } - } else { - for (int h = 0; h < 1; ++h) { - for (int j = 0; j < n_tokens; ++j) { - for (int i = 0; i < n_tokens; ++i) { - data[h*(n_tokens*n_tokens) + j*n_tokens + i] = llama_relative_position_bucket(ubatch.pos[i], ubatch.pos[j], hparams.n_rel_attn_bkts, lctx.is_encoding); - } - } - } - } - } - - if (!lctx.is_encoding && lctx.inp_embd_enc) { - assert(lctx.inp_embd_enc->type == GGML_TYPE_F32); - assert((size_t) ggml_nelements(lctx.inp_embd_enc) == lctx.embd_enc.size()); - - ggml_backend_tensor_set(lctx.inp_embd_enc, lctx.embd_enc.data(), 0, ggml_nbytes(lctx.inp_embd_enc)); - } - - if (!lctx.is_encoding && lctx.inp_KQ_mask_cross) { - const int64_t n_output_enc = lctx.embd_enc.size() / hparams.n_embd; - const int64_t n_tokens = ubatch.n_tokens; - - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask_cross->buffer)); - GGML_ASSERT(!ubatch.equal_seqs); // TODO: use ubatch.n_seqs instead of failing - - float * data = (float *) lctx.inp_KQ_mask_cross->data; - - for (int h = 0; h < 1; ++h) { - for (int j = 0; j < n_tokens; ++j) { - for (int i = 0; i < n_output_enc; ++i) { - float f = -INFINITY; - for (int s = 0; s < ubatch.n_seq_id[j]; ++s) { - const llama_seq_id seq_id = ubatch.seq_id[j][s]; - if (lctx.seq_ids_enc[i].find(seq_id) != lctx.seq_ids_enc[i].end()) { - f = 0.0f; - } - } - data[h*(n_output_enc*n_tokens) + j*n_output_enc + i] = f; - } - } - - for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) { - for (int j = 0; j < n_output_enc; ++j) { - data[h*(n_output_enc*n_tokens) + i*n_output_enc + j] = -INFINITY; - } - } + // initialize scheduler with the worst-case graph + ggml_backend_sched_reset(sched.get()); + if (!ggml_backend_sched_reserve(sched.get(), gf)) { + LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__); } } } -// llama output +enum llama_pooling_type llama_context::pooling_type() const { + return cparams.pooling_type; +} -size_t llama_output_reserve(struct llama_context & lctx, size_t n_outputs) { - const auto & cparams = lctx.cparams; - const auto & hparams = lctx.model.hparams; - const auto & vocab = lctx.model.vocab; +float * llama_context::get_logits() { + // reorder logits for backward compatibility + output_reorder(); - const size_t n_outputs_max = std::max(n_outputs, (size_t) cparams.n_seq_max); + return logits; +} + +float * llama_context::get_logits_ith(int32_t i) { + int32_t j = -1; + + try { + if (logits == nullptr) { + throw std::runtime_error("no logits"); + } + + if (i < 0) { + j = n_outputs + i; + if (j < 0) { + throw std::runtime_error(format("negative index out of range [0, %d)", n_outputs)); + } + } else if ((size_t) i >= output_ids.size()) { + throw std::runtime_error(format("out of range [0, %zu)", output_ids.size())); + } else { + j = output_ids[i]; + } + + if (j < 0) { + throw std::runtime_error(format("batch.logits[%d] != true", i)); + } + if (j >= n_outputs) { + // This should not happen + throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, n_outputs)); + } + + return logits + j*model.vocab.n_tokens(); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what()); +#ifndef NDEBUG + GGML_ABORT("fatal error"); +#else + return nullptr; +#endif + } +} + +float * llama_context::get_embeddings() { + // reorder embeddings for backward compatibility + output_reorder(); + + return embd; +} + +float * llama_context::get_embeddings_ith(int32_t i) { + int32_t j = -1; + + try { + if (embd == nullptr) { + throw std::runtime_error("no embeddings"); + } + + if (i < 0) { + j = n_outputs + i; + if (j < 0) { + throw std::runtime_error(format("negative index out of range [0, %d)", n_outputs)); + } + } else if ((size_t) i >= output_ids.size()) { + throw std::runtime_error(format("out of range [0, %zu)", output_ids.size())); + } else { + j = output_ids[i]; + } + + if (j < 0) { + throw std::runtime_error(format("batch.logits[%d] != true", i)); + } + if (j >= n_outputs) { + // This should not happen + throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, n_outputs)); + } + + return embd + j*model.hparams.n_embd; + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what()); +#ifndef NDEBUG + GGML_ABORT("fatal error"); +#else + return nullptr; +#endif + } +} + +float * llama_context::get_embeddings_seq(llama_seq_id seq_id) { + auto it = embd_seq.find(seq_id); + if (it == embd_seq.end()) { + return nullptr; + } + + return it->second.data(); +} + +void llama_context::attach_threadpool( + ggml_threadpool_t threadpool, + ggml_threadpool_t threadpool_batch) { + LLAMA_LOG_DEBUG("%s: call\n", __func__); + + this->threadpool = threadpool; + this->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool; +} + +void llama_context::detach_threadpool() { + LLAMA_LOG_DEBUG("%s: call\n", __func__); + + this->threadpool = nullptr; + this->threadpool_batch = nullptr; +} + +void llama_context::set_n_threads(int32_t n_threads, int32_t n_threads_batch) { + LLAMA_LOG_DEBUG("%s: n_threads = %d, n_threads_batch = %d\n", __func__, n_threads, n_threads_batch); + + cparams.n_threads = n_threads; + cparams.n_threads_batch = n_threads_batch; +} + +void llama_context::set_abort_callback(bool (*abort_callback)(void * data), void * abort_callback_data) { + LLAMA_LOG_DEBUG("%s: call\n", __func__); + + this->abort_callback = abort_callback; + this->abort_callback_data = abort_callback_data; + + for (auto & backend : backends) { + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get())); + auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback"); + if (set_abort_callback_fn) { + set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data); + } + } +} + +void llama_context::set_embeddings(bool value) { + LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value); + + cparams.embeddings = value; +} + +void llama_context::set_causal_attn(bool value) { + LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value); + + cparams.causal_attn = value; +} + +void llama_context::set_adapter_lora( + llama_adapter_lora * adapter, + float scale) { + LLAMA_LOG_DEBUG("%s: adapter = %p, scale = %f\n", __func__, (void *) adapter, scale); + + loras[adapter] = scale; +} + +bool llama_context::rm_adapter_lora( + llama_adapter_lora * adapter) { + LLAMA_LOG_DEBUG("%s: adapter = %p\n", __func__, (void *) adapter); + + auto pos = loras.find(adapter); + if (pos != loras.end()) { + loras.erase(pos); + return true; + } + + return false; +} + +void llama_context::clear_adapter_lora() { + LLAMA_LOG_DEBUG("%s: call\n", __func__); + + loras.clear(); +} + +bool llama_context::apply_adapter_cvec( + const float * data, + size_t len, + int32_t n_embd, + int32_t il_start, + int32_t il_end) { + LLAMA_LOG_DEBUG("%s: il_start = %d, il_end = %d\n", __func__, il_start, il_end); + + return cvec.apply(model, data, len, n_embd, il_start, il_end); +} + +int llama_context::encode(llama_batch & inp_batch) { + if (inp_batch.n_tokens == 0) { + LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__); + return -1; + } + + // temporary allocate memory for the input batch if needed + // TODO: this is incorrect for multiple sequences because pos_max() is the maximum across all sequences + llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : kv_self->pos_max() + 1); + + const llama_batch & batch = batch_allocr.batch; + const int32_t n_tokens = batch.n_tokens; + + const auto & hparams = model.hparams; + + GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT + + if (batch.token) { + for (int32_t i = 0; i < n_tokens; ++i) { + if (batch.token[i] < 0 || (uint32_t) batch.token[i] >= model.vocab.n_tokens()) { + LLAMA_LOG_ERROR("%s: invalid token[%d] = %d\n", __func__, i, batch.token[i]); + return -1; + } + } + } + + // micro-batching is not possible for non-causal encoding, so we process the batch in a single shot + GGML_ASSERT(cparams.n_ubatch >= (uint32_t) n_tokens && "encoder requires n_ubatch >= n_tokens"); + + if (t_compute_start_us == 0) { + t_compute_start_us = ggml_time_us(); + } + + n_queued_tokens += n_tokens; + + const int64_t n_embd = hparams.n_embd; + + sbatch.from_batch(batch, n_embd, /* simple_split */ true, /* logits_all */ true); + + const llama_ubatch ubatch = sbatch.split_simple(n_tokens); + + // reserve output buffer + if (output_reserve(n_tokens) < n_tokens) { + LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens); + return -2; + }; + + for (int32_t i = 0; i < n_tokens; ++i) { + output_ids[i] = i; + } + + n_outputs = n_tokens; + + //batch_manager->prepare(ubatch); + + ggml_backend_sched_reset(sched.get()); + ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data); + + auto * gf = graph_init(); + auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_ENCODER); + + ggml_backend_sched_alloc_graph(sched.get(), gf); + + res->set_inputs(&ubatch); + + const auto compute_status = graph_compute(gf, n_tokens > 1); + switch (compute_status) { + case GGML_STATUS_SUCCESS: + break; + case GGML_STATUS_ABORTED: + return 2; + case GGML_STATUS_ALLOC_FAILED: + return -2; + case GGML_STATUS_FAILED: + default: + return -3; + } + + auto * t_embd = res->get_embd_pooled() ? res->get_embd_pooled() : res->get_embd(); + + // extract embeddings + if (t_embd) { + ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd); + GGML_ASSERT(backend_embd != nullptr); + + GGML_ASSERT(embd != nullptr); + + switch (cparams.pooling_type) { + case LLAMA_POOLING_TYPE_NONE: + { + // extract token embeddings + GGML_ASSERT(n_tokens*n_embd <= (int64_t) embd_size); + ggml_backend_tensor_get_async(backend_embd, t_embd, embd, 0, n_tokens*n_embd*sizeof(float)); + } break; + case LLAMA_POOLING_TYPE_MEAN: + case LLAMA_POOLING_TYPE_CLS: + case LLAMA_POOLING_TYPE_LAST: + { + // extract sequence embeddings + auto & embd_seq_out = embd_seq; + embd_seq_out.clear(); + + GGML_ASSERT(!ubatch.equal_seqs); // TODO: handle equal splits + + for (int32_t i = 0; i < n_tokens; i++) { + const llama_seq_id seq_id = ubatch.seq_id[i][0]; + if (embd_seq_out.find(seq_id) != embd_seq_out.end()) { + continue; + } + embd_seq_out[seq_id].resize(n_embd); + ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float)); + } + } break; + case LLAMA_POOLING_TYPE_RANK: + { + // TODO: this likely should be the same logic as in llama_decoder_internal, but better to + // wait for an encoder model that requires this pooling type in order to test it + // https://github.com/ggerganov/llama.cpp/pull/9510 + GGML_ABORT("RANK pooling not implemented yet"); + } + case LLAMA_POOLING_TYPE_UNSPECIFIED: + { + GGML_ABORT("unknown pooling type"); + } + } + } + + // Reset state for the next token before backend sync, to allow the CPU activities in the reset to + // overlap with device computation. + ggml_backend_sched_reset(sched.get()); + + // TODO: hacky solution + if (model.arch == LLM_ARCH_T5 && t_embd) { + //cross.t_embd = t_embd; + + cross.n_embd = t_embd->ne[0]; + cross.n_enc = t_embd->ne[1]; + cross.v_embd.resize(cross.n_embd*cross.n_enc); + memcpy(cross.v_embd.data(), embd, ggml_nbytes(t_embd)); + + // remember the sequence ids used during the encoding - needed for cross attention later + cross.seq_ids_enc.resize(n_tokens); + for (int32_t i = 0; i < n_tokens; i++) { + for (int s = 0; s < ubatch.n_seq_id[i]; s++) { + llama_seq_id seq_id = ubatch.seq_id[i][s]; + cross.seq_ids_enc[i].insert(seq_id); + } + } + } + + return 0; +} + +int llama_context::decode(llama_batch & inp_batch) { + if (inp_batch.n_tokens == 0) { + LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__); + return -1; + } + + // temporary allocate memory for the input batch if needed + // TODO: this is incorrect for multiple sequences because pos_max() is the maximum across all sequences + llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : kv_self->pos_max() + 1); + + const llama_batch & batch = batch_allocr.batch; + + const auto & vocab = model.vocab; + const auto & hparams = model.hparams; + + const int32_t n_vocab = vocab.n_tokens(); + + const int64_t n_tokens_all = batch.n_tokens; + const int64_t n_embd = hparams.n_embd; + + // TODO: remove this stuff + class batch_guard { + public: + batch_guard(llama_kv_cache_unified & kv_self) : kv_slot_restorer(kv_self) { + } + + ~batch_guard() { + if (!is_done) { + kv_slot_restorer.restore(); + } + } + + void done() { + is_done = true; + } + + void save(const llama_kv_cache_slot_info & slot_info) { + kv_slot_restorer.save(slot_info); + } + + private: + bool is_done = false; + + llama_kv_slot_restorer kv_slot_restorer; + }; + + batch_guard bg(*kv_self); + + GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT + + if (batch.token) { + for (int64_t i = 0; i < n_tokens_all; ++i) { + if (batch.token[i] < 0 || (uint32_t) batch.token[i] >= model.vocab.n_tokens()) { + LLAMA_LOG_ERROR("%s: invalid token[%" PRId64 "] = %d\n", __func__, i, batch.token[i]); + throw std::runtime_error("invalid token"); + } + } + } + + GGML_ASSERT(n_tokens_all <= cparams.n_batch); + + GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens"); + + if (t_compute_start_us == 0) { + t_compute_start_us = ggml_time_us(); + } + n_queued_tokens += n_tokens_all; + + // this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens + const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE; + + embd_seq.clear(); + + int64_t n_outputs_all = 0; + + // count outputs + if (batch.logits && !embd_pooled) { + for (uint32_t i = 0; i < n_tokens_all; ++i) { + n_outputs_all += batch.logits[i] != 0; + } + } else if (logits_all || embd_pooled) { + n_outputs_all = n_tokens_all; + } else { + // keep last output only + n_outputs_all = 1; + } + + const bool logits_all = n_outputs_all == n_tokens_all; + + sbatch.from_batch(batch, n_embd, + /* simple_split */ !kv_self->recurrent, + /* logits_all */ logits_all); + + // reserve output buffer + if (output_reserve(n_outputs_all) < n_outputs_all) { + LLAMA_LOG_ERROR("%s: could not reserve space for batch with %" PRId64 " outputs\n", __func__, n_outputs_all); + return -2; + }; + + int64_t n_outputs_prev = 0; + + while (sbatch.n_tokens > 0) { + llama_ubatch ubatch = llama_ubatch(); + + const auto & n_ubatch = cparams.n_ubatch; + + if (kv_self->recurrent) { + if (embd_pooled) { + // Pooled embeddings cannot be split across ubatches (yet) + ubatch = sbatch.split_seq(cparams.n_ubatch); + } else { + // recurrent model architectures are easier to implement + // with equal-length sequences + ubatch = sbatch.split_equal(cparams.n_ubatch); + } + } else { + ubatch = sbatch.split_simple(n_ubatch); + } + + // count the outputs in this u_batch + { + int32_t n_outputs_new = 0; + + if (n_outputs_all == n_tokens_all) { + n_outputs_new = ubatch.n_tokens; + } else { + GGML_ASSERT(ubatch.output); + for (uint32_t i = 0; i < ubatch.n_tokens; i++) { + n_outputs_new += (int32_t) (ubatch.output[i] != 0); + } + } + + // needs to happen before the graph is built + n_outputs = n_outputs_new; + } + + // non-causal masks do not use the KV cache + if (hparams.causal_attn) { + kv_self_update(); + + // if we have enough unused cells before the current head -> + // better to start searching from the beginning of the cache, hoping to fill it + if (kv_self->head > kv_self->used + 2*ubatch.n_tokens) { + kv_self->head = 0; + } + + const auto slot_info = kv_self->find_slot(ubatch); + if (!slot_info) { + LLAMA_LOG_ERROR("%s: failed to prepare ubatch\n", __func__); + return -3; + } + + bg.save(slot_info); + + if (!kv_self->recurrent) { + // a heuristic, to avoid attending the full cache if it is not yet utilized + // after enough generations, the benefit from this heuristic disappears + // if we start defragmenting the cache, the benefit from this will be more important + const uint32_t pad = kv_self->get_padding(cparams); + kv_self->n = std::min(kv_self->size, std::max(pad, GGML_PAD(kv_self->cell_max(), pad))); + } + } + + //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self->n, kv_self->used, kv_self->head); + + ggml_backend_sched_reset(sched.get()); + ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data); + + auto * gf = graph_init(); + auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_DECODER); + + // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); + + ggml_backend_sched_alloc_graph(sched.get(), gf); + + res->set_inputs(&ubatch); + + const auto compute_status = graph_compute(gf, ubatch.n_tokens > 1); + if (compute_status != GGML_STATUS_SUCCESS) { + switch (compute_status) { + case GGML_STATUS_ABORTED: + return 2; + case GGML_STATUS_ALLOC_FAILED: + return -2; + case GGML_STATUS_FAILED: + default: + return -3; + } + } + + // update the kv ring buffer + { + kv_self->head += ubatch.n_tokens; + + // Ensure kv cache head points to a valid index. + if (kv_self->head >= kv_self->size) { + kv_self->head = 0; + } + } + + // plot the computation graph in dot format (for debugging purposes) + //if (n_past%100 == 0) { + // ggml_graph_dump_dot(gf, NULL, "llama.dot"); + //} + + auto * t_logits = cparams.embeddings ? nullptr : res->get_logits(); + auto * t_embd = cparams.embeddings ? res->get_embd() : nullptr; + + if (t_embd && res->get_embd_pooled()) { + t_embd = res->get_embd_pooled(); + } + + // extract logits + if (t_logits && n_outputs > 0) { + ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(sched.get(), t_logits); + GGML_ASSERT(backend_res != nullptr); + GGML_ASSERT(logits != nullptr); + + float * logits_out = logits + n_outputs_prev*n_vocab; + + if (n_outputs) { + GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all); + GGML_ASSERT((n_outputs_prev + n_outputs)*n_vocab <= (int64_t) logits_size); + ggml_backend_tensor_get_async(backend_res, t_logits, logits_out, 0, n_outputs*n_vocab*sizeof(float)); + } + } + + // extract embeddings + if (t_embd && n_outputs > 0) { + ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd); + GGML_ASSERT(backend_embd != nullptr); + + switch (cparams.pooling_type) { + case LLAMA_POOLING_TYPE_NONE: + { + // extract token embeddings + GGML_ASSERT(embd != nullptr); + float * embd_out = embd + n_outputs_prev*n_embd; + + if (n_outputs) { + GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all); + GGML_ASSERT((n_outputs_prev + n_outputs)*n_embd <= (int64_t) embd_size); + ggml_backend_tensor_get_async(backend_embd, t_embd, embd_out, 0, n_outputs*n_embd*sizeof(float)); + } + } break; + case LLAMA_POOLING_TYPE_MEAN: + case LLAMA_POOLING_TYPE_CLS: + case LLAMA_POOLING_TYPE_LAST: + { + // extract sequence embeddings (cleared before processing each batch) + auto & embd_seq_out = embd_seq; + + for (uint32_t s = 0; s < ubatch.n_seqs; ++s) { + const llama_seq_id seq_id = ubatch.seq_id[s][0]; + if (embd_seq_out.find(seq_id) != embd_seq_out.end()) { + continue; + } + embd_seq_out[seq_id].resize(n_embd); + ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float)); + } + } break; + case LLAMA_POOLING_TYPE_RANK: + { + // extract the rerank score - a single float per sequence + auto & embd_seq_out = embd_seq; + + for (uint32_t s = 0; s < ubatch.n_seqs; ++s) { + const llama_seq_id seq_id = ubatch.seq_id[s][0]; + if (embd_seq_out.find(seq_id) != embd_seq_out.end()) { + continue; + } + embd_seq_out[seq_id].resize(1); + ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (seq_id)*sizeof(float), sizeof(float)); + } + } break; + case LLAMA_POOLING_TYPE_UNSPECIFIED: + { + GGML_ABORT("unknown pooling type"); + } + } + } + + n_outputs_prev += n_outputs; + } + + // finalize the batch processing + bg.done(); + + // set output mappings + { + bool sorted_output = true; + + GGML_ASSERT(sbatch.out_ids.size() == (size_t) n_outputs_all); + + for (int64_t i = 0; i < n_outputs_all; ++i) { + int64_t out_id = sbatch.out_ids[i]; + output_ids[out_id] = i; + if (out_id != i) { + sorted_output = false; + } + } + + if (sorted_output) { + sbatch.out_ids.clear(); + } + } + + // set to total number of outputs in the batch, for use in llama_get_logits_ith + n_outputs = n_outputs_all; + + // wait for the computation to finish (automatically done when obtaining the model output) + //synchronize(); + + // decide if we need to defrag the kv cache + if (cparams.causal_attn && cparams.defrag_thold > 0.0f) { + // - do not defrag small contexts (i.e. < 2048 tokens) + // - count the padding towards the number of used tokens + const float fragmentation = kv_self->n >= 2048 ? std::max(0.0f, 1.0f - float(kv_self->used + kv_self->get_padding(cparams))/float(kv_self->n)) : 0.0f; + + // queue defragmentation for next llama_kv_cache_update + if (fragmentation > cparams.defrag_thold) { + LLAMA_LOG_DEBUG("%s: fragmentation: %.2f - requesting defrag\n", __func__, fragmentation); + + kv_self->defrag(); + } + } + + // Reset state for the next token before backend sync, to allow the CPU activities in the reset to + // overlap with device computation. + ggml_backend_sched_reset(sched.get()); + + return 0; +} + +// +// output +// + +int32_t llama_context::output_reserve(int32_t n_outputs) { + const auto & hparams = model.hparams; + const auto & vocab = model.vocab; + + const int64_t n_outputs_max = std::max(n_outputs, n_seq_max()); const auto n_batch = cparams.n_batch; const auto n_vocab = vocab.n_tokens(); const auto n_embd = hparams.n_embd; // TODO: use a per-batch flag for logits presence instead - const bool has_logits = !cparams.embeddings; - const bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE); + bool has_logits = !cparams.embeddings; + bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE); - const size_t logits_size = has_logits ? n_vocab*n_outputs_max : 0; - const size_t embd_size = has_embd ? n_embd*n_outputs_max : 0; - - if (lctx.output_ids.empty()) { - // init, never resized afterwards - lctx.output_ids.resize(n_batch); + // TODO: hacky enc-dec support + if (model.arch == LLM_ARCH_T5) { + has_logits = true; + has_embd = true; } - const size_t prev_size = lctx.buf_output ? ggml_backend_buffer_get_size(lctx.buf_output.get()) : 0; + logits_size = has_logits ? n_vocab*n_outputs_max : 0; + embd_size = has_embd ? n_embd*n_outputs_max : 0; + + if (output_ids.empty()) { + // init, never resized afterwards + output_ids.resize(n_batch); + } + + const size_t prev_size = buf_output ? ggml_backend_buffer_get_size(buf_output.get()) : 0; const size_t new_size = (logits_size + embd_size) * sizeof(float); // alloc only when more than the current capacity is required // TODO: also consider shrinking the buffer - if (!lctx.buf_output || prev_size < new_size) { - if (lctx.buf_output) { + if (!buf_output || prev_size < new_size) { + if (buf_output) { #ifndef NDEBUG // This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark) LLAMA_LOG_INFO("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0); #endif - lctx.buf_output = nullptr; - lctx.logits = nullptr; - lctx.embd = nullptr; + buf_output = nullptr; + logits = nullptr; + embd = nullptr; } auto * buft = ggml_backend_cpu_buffer_type(); // try to use the host buffer of the device where the output tensor is allocated for faster transfer to system memory - auto * output_dev = lctx.model.dev_output(); + auto * output_dev = model.dev_output(); auto * output_dev_host_buft = output_dev ? ggml_backend_dev_host_buffer_type(output_dev) : nullptr; if (output_dev_host_buft) { buft = output_dev_host_buft; } - lctx.buf_output.reset(ggml_backend_buft_alloc_buffer(buft, new_size)); - if (lctx.buf_output == nullptr) { + buf_output.reset(ggml_backend_buft_alloc_buffer(buft, new_size)); + if (buf_output == nullptr) { LLAMA_LOG_ERROR("%s: failed to allocate output buffer of size %.2f MiB\n", __func__, new_size / (1024.0 * 1024.0)); return 0; } } - float * output_base = (float *) ggml_backend_buffer_get_base(lctx.buf_output.get()); + float * output_base = (float *) ggml_backend_buffer_get_base(buf_output.get()); - lctx.logits = has_logits ? output_base : nullptr; - lctx.embd = has_embd ? output_base + logits_size : nullptr; - - lctx.output_size = n_outputs_max; - lctx.logits_size = logits_size; - lctx.embd_size = embd_size; + logits = has_logits ? output_base : nullptr; + embd = has_embd ? output_base + logits_size : nullptr; // set all ids as invalid (negative) - std::fill(lctx.output_ids.begin(), lctx.output_ids.end(), -1); + std::fill(output_ids.begin(), output_ids.end(), -1); - ggml_backend_buffer_clear(lctx.buf_output.get(), 0); + ggml_backend_buffer_clear(buf_output.get(), 0); - lctx.n_outputs = 0; + this->n_outputs = 0; + this->n_outputs_max = n_outputs_max; return n_outputs_max; } -void llama_output_reorder(struct llama_context & ctx) { - std::vector & out_ids = ctx.sbatch.out_ids; +void llama_context::output_reorder() { + auto & out_ids = sbatch.out_ids; if (!out_ids.empty()) { - const uint32_t n_vocab = ctx.model.vocab.n_tokens(); - const uint32_t n_embd = ctx.model.hparams.n_embd; + const uint32_t n_vocab = model.vocab.n_tokens(); + const uint32_t n_embd = model.hparams.n_embd; - const int32_t n_outputs = ctx.n_outputs; GGML_ASSERT((size_t) n_outputs == out_ids.size()); // TODO: is there something more efficient which also minimizes swaps? @@ -559,842 +1563,156 @@ void llama_output_reorder(struct llama_context & ctx) { } if (j_min == i) { continue; } std::swap(out_ids[i], out_ids[j_min]); - if (ctx.logits_size > 0) { + if (logits_size > 0) { for (uint32_t k = 0; k < n_vocab; k++) { - std::swap(ctx.logits[i*n_vocab + k], ctx.logits[j_min*n_vocab + k]); + std::swap(logits[i*n_vocab + k], logits[j_min*n_vocab + k]); } } - if (ctx.embd_size > 0) { + if (embd_size > 0) { for (uint32_t k = 0; k < n_embd; k++) { - std::swap(ctx.embd[i*n_embd + k], ctx.embd[j_min*n_embd + k]); + std::swap(embd[i*n_embd + k], embd[j_min*n_embd + k]); } } } - std::fill(ctx.output_ids.begin(), ctx.output_ids.end(), -1); + std::fill(output_ids.begin(), output_ids.end(), -1); for (int32_t i = 0; i < n_outputs; ++i) { - ctx.output_ids[out_ids[i]] = i; + output_ids[out_ids[i]] = i; } out_ids.clear(); } } // -// interface implementation +// graph // -void llama_free(struct llama_context * ctx) { - delete ctx; +int32_t llama_context::graph_max_nodes() const { + return std::max(8192, 5*model.n_tensors()); } -uint32_t llama_n_ctx(const struct llama_context * ctx) { - return ctx->cparams.n_ctx; +ggml_cgraph * llama_context::graph_init() { + ggml_init_params params = { + /*.mem_size =*/ buf_compute_meta.size(), + /*.mem_buffer =*/ buf_compute_meta.data(), + /*.no_alloc =*/ true, + }; + + ctx_compute.reset(ggml_init(params)); + + return ggml_new_graph_custom(ctx_compute.get(), graph_max_nodes(), false); } -uint32_t llama_n_batch(const struct llama_context * ctx) { - return ctx->cparams.n_batch; +llm_graph_result_ptr llama_context::graph_build( + ggml_context * ctx, + ggml_cgraph * gf, + const llama_ubatch & ubatch, + llm_graph_type gtype) { + return model.build_graph( + { + /*.ctx =*/ ctx, + /*.arch =*/ model.arch, + /*.hparams =*/ model.hparams, + /*.cparams =*/ cparams, + /*.ubatch =*/ ubatch, + /*.sched =*/ sched.get(), + /*.backend_cpu =*/ backend_cpu, + /*.cvec =*/ &cvec, + /*.loras =*/ &loras, + /*.memory =*/ kv_self.get(), + /*.cross =*/ &cross, + /*.n_outputs =*/ n_outputs, + /*.cb =*/ graph_get_cb(), + }, gf, gtype); } -uint32_t llama_n_ubatch(const struct llama_context * ctx) { - return ctx->cparams.n_ubatch; -} +ggml_status llama_context::graph_compute( + ggml_cgraph * gf, + bool batched) { + int n_threads = batched ? cparams.n_threads_batch : cparams.n_threads; + ggml_threadpool_t tp = batched ? threadpool_batch : threadpool; -uint32_t llama_n_seq_max(const struct llama_context * ctx) { - return ctx->kv_self.size; -} - -const struct llama_model * llama_get_model(const struct llama_context * ctx) { - return &ctx->model; -} - -enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx) { - return ctx->cparams.pooling_type; -} - -void llama_attach_threadpool( - struct llama_context * ctx, - ggml_threadpool_t threadpool, - ggml_threadpool_t threadpool_batch) { - ctx->threadpool = threadpool; - ctx->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool; -} - -void llama_detach_threadpool(struct llama_context * ctx) { - ctx->threadpool = nullptr; - ctx->threadpool_batch = nullptr; -} - -void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch) { - ctx->cparams.n_threads = n_threads; - ctx->cparams.n_threads_batch = n_threads_batch; -} - -int32_t llama_n_threads(struct llama_context * ctx) { - return ctx->cparams.n_threads; -} - -int32_t llama_n_threads_batch(struct llama_context * ctx) { - return ctx->cparams.n_threads_batch; -} - -void llama_set_abort_callback(struct llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) { - ctx->abort_callback = abort_callback; - ctx->abort_callback_data = abort_callback_data; - - for (auto & backend : ctx->backends) { - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get())); - auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback"); - if (set_abort_callback_fn) { - set_abort_callback_fn(backend.get(), ctx->abort_callback, ctx->abort_callback_data); - } - } -} - -void llama_set_embeddings(struct llama_context * ctx, bool embeddings) { - ctx->cparams.embeddings = embeddings; -} - -void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn) { - ctx->cparams.causal_attn = causal_attn; -} - -void llama_synchronize(struct llama_context * ctx) { - ggml_backend_sched_synchronize(ctx->sched.get()); - - // FIXME: if multiple single tokens are evaluated without a synchronization, - // the stats will be added to the prompt evaluation stats - // this should only happen when using batch size 1 to evaluate a batch - - // add the evaluation to the stats - if (ctx->n_queued_tokens == 1) { - if (!ctx->cparams.no_perf) { - ctx->t_eval_us += ggml_time_us() - ctx->t_compute_start_us; - } - ctx->n_eval++; - } else if (ctx->n_queued_tokens > 1) { - if (!ctx->cparams.no_perf) { - ctx->t_p_eval_us += ggml_time_us() - ctx->t_compute_start_us; - } - ctx->n_p_eval += ctx->n_queued_tokens; + if (backend_cpu != nullptr) { + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); + auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); + set_threadpool_fn(backend_cpu, tp); } - // get a more accurate load time, upon first eval - if (ctx->n_queued_tokens > 0 && !ctx->has_evaluated_once) { - ctx->t_load_us = ggml_time_us() - ctx->t_start_us; - ctx->has_evaluated_once = true; + // set the number of threads for all the backends + for (const auto & set_n_threads_fn : set_n_threads_fns) { + set_n_threads_fn.second(set_n_threads_fn.first, n_threads); } - ctx->n_queued_tokens = 0; - ctx->t_compute_start_us = 0; + auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf); + if (status != GGML_STATUS_SUCCESS) { + LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status); + } + + // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(sched)); + + return status; } -float * llama_get_logits(struct llama_context * ctx) { - llama_synchronize(ctx); - - // reorder logits for backward compatibility - // TODO: maybe deprecate this - llama_output_reorder(*ctx); - - return ctx->logits; -} - -float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) { - int32_t j = -1; - - llama_synchronize(ctx); - - try { - if (ctx->logits == nullptr) { - throw std::runtime_error("no logits"); - } - - if (i < 0) { - j = ctx->n_outputs + i; - if (j < 0) { - throw std::runtime_error(format("negative index out of range [0, %d)", ctx->n_outputs)); - } - } else if ((size_t) i >= ctx->output_ids.size()) { - throw std::runtime_error(format("out of range [0, %zu)", ctx->output_ids.size())); +llm_graph_cb llama_context::graph_get_cb() const { + return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) { + if (il >= 0) { + ggml_format_name(cur, "%s-%d", name, il); } else { - j = ctx->output_ids[i]; + ggml_set_name(cur, name); } - if (j < 0) { - throw std::runtime_error(format("batch.logits[%d] != true", i)); - } - if (j >= ctx->n_outputs) { - // This should not happen - throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, ctx->n_outputs)); - } - - return ctx->logits + j*ctx->model.vocab.n_tokens(); - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what()); -#ifndef NDEBUG - GGML_ABORT("fatal error"); -#else - return nullptr; -#endif - } -} - -float * llama_get_embeddings(struct llama_context * ctx) { - llama_synchronize(ctx); - - // reorder embeddings for backward compatibility - // TODO: maybe deprecate this - llama_output_reorder(*ctx); - - return ctx->embd; -} - -float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) { - int32_t j = -1; - - llama_synchronize(ctx); - - try { - if (ctx->embd == nullptr) { - throw std::runtime_error("no embeddings"); - } - - if (i < 0) { - j = ctx->n_outputs + i; - if (j < 0) { - throw std::runtime_error(format("negative index out of range [0, %d)", ctx->n_outputs)); - } - } else if ((size_t) i >= ctx->output_ids.size()) { - throw std::runtime_error(format("out of range [0, %zu)", ctx->output_ids.size())); - } else { - j = ctx->output_ids[i]; - } - - if (j < 0) { - throw std::runtime_error(format("batch.logits[%d] != true", i)); - } - if (j >= ctx->n_outputs) { - // This should not happen - throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, ctx->n_outputs)); - } - - return ctx->embd + j*ctx->model.hparams.n_embd; - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what()); -#ifndef NDEBUG - GGML_ABORT("fatal error"); -#else - return nullptr; -#endif - } -} - -float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id) { - llama_synchronize(ctx); - - auto it = ctx->embd_seq.find(seq_id); - if (it == ctx->embd_seq.end()) { - return nullptr; - } - - return it->second.data(); -} - -// llama state API - -// deprecated -size_t llama_get_state_size(struct llama_context * ctx) { - return llama_state_get_size(ctx); -} - -// deprecated -size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { - return llama_state_get_data(ctx, dst, -1); -} - -// deprecated -size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { - return llama_state_set_data(ctx, src, -1); -} - -// deprecated -bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { - return llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out); -} - -// deprecated -bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { - return llama_state_save_file(ctx, path_session, tokens, n_token_count); -} - -// TODO: replace all non-fatal assertions with returned errors or exceptions -struct llama_data_write { - virtual void write(const void * src, size_t size) = 0; - virtual void write_tensor_data(const struct ggml_tensor * tensor, size_t offset, size_t size) = 0; - virtual size_t get_size_written() = 0; - virtual ~llama_data_write() = default; - - void write_string(const std::string & str) { - uint32_t str_size = str.size(); - - write(&str_size, sizeof(str_size)); - write(str.data(), str_size); - } - - void write_model_info(const struct llama_context * ctx) { - const std::string arch_str = llm_arch_name(ctx->model.arch); - write_string(arch_str); - // TODO: add more model-specific info which should prevent loading the session file if not identical - } - - //void write_rng(const std::mt19937 & rng) { - // std::ostringstream rng_ss; - // rng_ss << rng; - - // const std::string & rng_str = rng_ss.str(); - - // write_string(rng_str); - //} - - void write_output_ids(struct llama_context * ctx) { - llama_output_reorder(*ctx); - - const uint32_t n_outputs = ctx->n_outputs; - - std::vector output_pos; - - const size_t n_batch = ctx->cparams.n_batch; - const auto & output_ids = ctx->output_ids; - - GGML_ASSERT(n_outputs <= ctx->output_size); - - output_pos.resize(n_outputs); - - // build a more compact representation of the output ids - for (size_t i = 0; i < n_batch; ++i) { - // map an output id to a position in the batch - int32_t pos = output_ids[i]; - if (pos >= 0) { - GGML_ASSERT((uint32_t) pos < n_outputs); - output_pos[pos] = i; + if (!cparams.offload_kqv) { + if (strcmp(name, "kqv_merged_cont") == 0) { + // all nodes between the KV store and the attention output are run on the CPU + ggml_backend_sched_set_tensor_backend(sched.get(), cur, backend_cpu); } } - write(&n_outputs, sizeof(n_outputs)); - - if (n_outputs) { - write(output_pos.data(), n_outputs * sizeof(int32_t)); - } - } - - void write_logits(const struct llama_context * ctx) { - const uint64_t logits_size = std::min((uint64_t) ctx->logits_size, (uint64_t) ctx->n_outputs * ctx->model.vocab.n_tokens()); - - write(&logits_size, sizeof(logits_size)); - - if (logits_size) { - write(ctx->logits, logits_size * sizeof(float)); - } - } - - void write_embeddings(const struct llama_context * ctx) { - const uint64_t embeddings_size = std::min((uint64_t) ctx->embd_size, (uint64_t) ctx->n_outputs * ctx->model.hparams.n_embd); - - write(&embeddings_size, sizeof(embeddings_size)); - - if (embeddings_size) { - write(ctx->embd, embeddings_size * sizeof(float)); - } - } - - void write_kv_cache_meta(const llama_kv_cache & kv_self, const std::vector> & cell_ranges, llama_seq_id seq_id = -1) { - for (const auto & range : cell_ranges) { - for (uint32_t i = range.first; i < range.second; ++i) { - const auto & cell = kv_self.cells[i]; - const llama_pos pos = cell.pos; - const uint32_t n_seq_id = seq_id == -1 ? cell.seq_id.size() : 0; - - write(&pos, sizeof(pos)); - write(&n_seq_id, sizeof(n_seq_id)); - - if (n_seq_id) { - for (auto seq_id : cell.seq_id) { - write(&seq_id, sizeof(seq_id)); - } - } - } - } - } - - void write_kv_cache_data(const struct llama_context * ctx, const std::vector> & cell_ranges) { - const struct llama_kv_cache & kv_self = ctx->kv_self; - const struct llama_hparams & hparams = ctx->model.hparams; - - const uint32_t v_trans = kv_self.v_trans ? 1 : 0; - const uint32_t n_layer = hparams.n_layer; - - write(&v_trans, sizeof(v_trans)); - write(&n_layer, sizeof(n_layer)); - - std::vector tmp_buf; - - // Iterate and write all the keys first, each row is a cell - // Get whole range at a time - for (uint32_t il = 0; il < n_layer; ++il) { - const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il) + hparams.n_embd_k_s(); - - // Write key type - const int32_t k_type_i = (int32_t)kv_self.k_l[il]->type; - write(&k_type_i, sizeof(k_type_i)); - - // Write row size of key - const uint64_t k_size_row = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa); - write(&k_size_row, sizeof(k_size_row)); - - // Read each range of cells of k_size length each into tmp_buf and write out - for (const auto & range : cell_ranges) { - const size_t range_size = range.second - range.first; - const size_t buf_size = range_size * k_size_row; - write_tensor_data(kv_self.k_l[il], range.first * k_size_row, buf_size); - } - } - - if (!kv_self.v_trans) { - for (uint32_t il = 0; il < n_layer; ++il) { - const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s(); - - // Write value type - const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type; - write(&v_type_i, sizeof(v_type_i)); - - // Write row size of value - const uint64_t v_size_row = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa); - write(&v_size_row, sizeof(v_size_row)); - - // Read each range of cells of v_size length each into tmp_buf and write out - for (const auto & range : cell_ranges) { - const size_t range_size = range.second - range.first; - const size_t buf_size = range_size * v_size_row; - write_tensor_data(kv_self.v_l[il], range.first * v_size_row, buf_size); - } - } - } else { - // When v is transposed, we also need the element size and get the element ranges from each row - const uint32_t kv_size = kv_self.size; - for (uint32_t il = 0; il < n_layer; ++il) { - const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s(); - - // Write value type - const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type; - write(&v_type_i, sizeof(v_type_i)); - - // Write element size - const uint32_t v_size_el = ggml_type_size(kv_self.v_l[il]->type); - write(&v_size_el, sizeof(v_size_el)); - - // Write GQA embedding size - write(&n_embd_v_gqa, sizeof(n_embd_v_gqa)); - - // For each row, we get the element values of each cell - for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { - // Read each range of cells of v_size_el length each into tmp_buf and write out - for (const auto & range : cell_ranges) { - const size_t range_size = range.second - range.first; - const size_t src_offset = (range.first + j * kv_size) * v_size_el; - const size_t buf_size = range_size * v_size_el; - write_tensor_data(kv_self.v_l[il], src_offset, buf_size); - } - } - } - } - } - - void write_kv_cache(const struct llama_context * ctx, llama_seq_id seq_id = -1) { - const struct llama_kv_cache & kv_self = ctx->kv_self; - std::vector> cell_ranges; // ranges, from inclusive, to exclusive - uint32_t cell_count = 0; - - // Count the number of cells with the specified seq_id - // Find all the ranges of cells with this seq id (or all, when -1) - uint32_t cell_range_begin = kv_self.size; - for (uint32_t i = 0; i < kv_self.size; ++i) { - const auto & cell = kv_self.cells[i]; - if ((seq_id == -1 && !cell.is_empty()) || cell.has_seq_id(seq_id)) { - ++cell_count; - if (cell_range_begin == kv_self.size) { - cell_range_begin = i; - } - } else { - if (cell_range_begin != kv_self.size) { - cell_ranges.emplace_back(cell_range_begin, i); - cell_range_begin = kv_self.size; - } - } - } - if (cell_range_begin != kv_self.size) { - cell_ranges.emplace_back(cell_range_begin, kv_self.size); - } - - // DEBUG CHECK: Sum of cell counts in ranges should equal the total cell count - uint32_t cell_count_check = 0; - for (const auto & range : cell_ranges) { - cell_count_check += range.second - range.first; - } - GGML_ASSERT(cell_count == cell_count_check); - - write(&cell_count, sizeof(cell_count)); - - write_kv_cache_meta(kv_self, cell_ranges, seq_id); - write_kv_cache_data(ctx, cell_ranges); - } -}; - -struct llama_data_read { - virtual const uint8_t * read(size_t size) = 0; - virtual void read_to(void * dst, size_t size) = 0; - virtual size_t get_size_read() = 0; - virtual ~llama_data_read() = default; - - void read_string(std::string & str) { - uint32_t str_size; - read_to(&str_size, sizeof(str_size)); - - str.assign((const char *) read(str_size), str_size); - } - - // validate model information - void read_model_info(const struct llama_context * ctx) { - const std::string cur_arch_str = llm_arch_name(ctx->model.arch); - - std::string arch_str; - read_string(arch_str); - if (cur_arch_str != arch_str) { - throw std::runtime_error(format("wrong model arch: '%s' instead of '%s'", arch_str.c_str(), cur_arch_str.c_str())); - } - // TODO: add more info which needs to be identical but which is not verified otherwise - } - - //void read_rng(std::mt19937 & rng) { - // std::string rng_str; - // read_string(rng_str); - - // std::istringstream rng_ss(rng_str); - // rng_ss >> rng; - - // if (rng_ss.fail()) { - // throw std::runtime_error("failed to load RNG state"); - // } - //} - - void read_output_ids(struct llama_context * ctx) { - std::vector output_pos; - - uint32_t n_outputs; - read_to(&n_outputs, sizeof(n_outputs)); - - if (n_outputs > llama_output_reserve(*ctx, n_outputs)) { - throw std::runtime_error("could not reserve outputs"); - } - - if (n_outputs) { - output_pos.resize(n_outputs); - read_to(output_pos.data(), n_outputs * sizeof(int32_t)); - - for (int32_t i = 0; i < (int32_t) output_pos.size(); ++i) { - int32_t id = output_pos[i]; - if ((uint32_t) id >= ctx->cparams.n_batch) { - throw std::runtime_error(format("invalid output id, %d does not fit in batch size of %u", id, ctx->cparams.n_batch)); - } - ctx->output_ids[id] = i; - } - - ctx->n_outputs = n_outputs; - } - } - - void read_logits(struct llama_context * ctx) { - uint64_t logits_size; - read_to(&logits_size, sizeof(logits_size)); - - if (ctx->logits_size < logits_size) { - throw std::runtime_error("logits buffer too small"); - } - - if (logits_size) { - read_to(ctx->logits, logits_size * sizeof(float)); - } - } - - void read_embeddings(struct llama_context * ctx) { - uint64_t embeddings_size; - read_to(&embeddings_size, sizeof(embeddings_size)); - - if (ctx->embd_size < embeddings_size) { - throw std::runtime_error("embeddings buffer too small"); - } - - if (embeddings_size) { - read_to(ctx->embd, embeddings_size * sizeof(float)); - } - } - - bool read_kv_cache_meta(struct llama_context * ctx, uint32_t cell_count, llama_seq_id dest_seq_id = -1) { - struct llama_kv_cache & kv_self = ctx->kv_self; - - if (dest_seq_id != -1) { - // single sequence - - llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1); - - llama_ubatch batch = ctx->sbatch.reserve_ubatch(cell_count, /* has_embd */ false); - batch.n_tokens = cell_count; - batch.n_seq_tokens = cell_count; - batch.n_seqs = 1; - - for (uint32_t i = 0; i < cell_count; ++i) { - llama_pos pos; - uint32_t n_seq_id; - - read_to(&pos, sizeof(pos)); - read_to(&n_seq_id, sizeof(n_seq_id)); - - if (n_seq_id != 0) { - LLAMA_LOG_ERROR("%s: invalid seq_id-agnostic kv cell\n", __func__); - return false; - } - - batch.pos[i] = pos; - } - batch.n_seq_id[0] = 1; - batch.seq_id[0] = &dest_seq_id; - if (!llama_kv_cache_find_slot(kv_self, batch)) { - LLAMA_LOG_ERROR("%s: failed to find available cells in kv cache\n", __func__); - return false; - } - - // DEBUG CHECK: kv_self.head should be our first cell, kv_self.head + cell_count - 1 should be our last cell (verify seq_id and pos values) - // Assume that this is one contiguous block of cells - GGML_ASSERT(kv_self.head + cell_count <= kv_self.size); - GGML_ASSERT(kv_self.cells[kv_self.head].pos == batch.pos[0]); - GGML_ASSERT(kv_self.cells[kv_self.head + cell_count - 1].pos == batch.pos[cell_count - 1]); - GGML_ASSERT(kv_self.cells[kv_self.head].has_seq_id(dest_seq_id)); - GGML_ASSERT(kv_self.cells[kv_self.head + cell_count - 1].has_seq_id(dest_seq_id)); - } else { - // whole KV cache restore - - if (cell_count > kv_self.size) { - LLAMA_LOG_ERROR("%s: not enough cells in kv cache\n", __func__); - return false; - } - - llama_kv_cache_clear(kv_self); - - for (uint32_t i = 0; i < cell_count; ++i) { - llama_kv_cell & cell = kv_self.cells[i]; - - llama_pos pos; - uint32_t n_seq_id; - - read_to(&pos, sizeof(pos)); - read_to(&n_seq_id, sizeof(n_seq_id)); - - cell.pos = pos; - - for (uint32_t j = 0; j < n_seq_id; ++j) { - llama_seq_id seq_id; - read_to(&seq_id, sizeof(seq_id)); - - if (seq_id < 0 || (uint32_t) seq_id >= llama_n_seq_max(ctx)) { - LLAMA_LOG_ERROR("%s: invalid seq_id, %d is out of range [0, %u)\n", __func__, seq_id, llama_n_seq_max(ctx)); - return false; - } - - cell.seq_id.insert(seq_id); - - if (kv_self.recurrent) { - int32_t & tail = kv_self.cells[seq_id].tail; - if (tail != -1) { - LLAMA_LOG_ERROR("%s: duplicate tail for seq_id %d in cell %d and %d\n", __func__, seq_id, i, tail); - return false; + // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends + // FIXME: fix in ggml_backend_sched + const bool full_offload = model.params.n_gpu_layers > (int) model.hparams.n_layer; + if (ubatch.n_tokens < 32 || full_offload) { + if (il != -1 && strcmp(name, "norm") == 0) { + const auto & dev_layer = model.dev_layer(il); + for (const auto & backend : backends) { + if (ggml_backend_get_device(backend.get()) == dev_layer) { + if (ggml_backend_supports_op(backend.get(), cur)) { + ggml_backend_sched_set_tensor_backend(sched.get(), cur, backend.get()); } - tail = i; - } - } - } - - kv_self.head = 0; - kv_self.used = cell_count; - } - - if (kv_self.recurrent) { - for (uint32_t i = 0; i < cell_count; ++i) { - uint32_t cell_id = kv_self.head + i; - // make sure the recurrent states will keep their restored state - kv_self.cells[cell_id].src = cell_id; - } - } - - return true; - } - - bool read_kv_cache_data(struct llama_context * ctx, uint32_t cell_count) { - const struct llama_hparams & hparams = ctx->model.hparams; - struct llama_kv_cache & kv_self = ctx->kv_self; - uint32_t v_trans; - uint32_t n_layer; - read_to(&v_trans, sizeof(v_trans)); - read_to(&n_layer, sizeof(n_layer)); - - if (n_layer != hparams.n_layer) { - LLAMA_LOG_ERROR("%s: mismatched layer count (%u instead of %u)\n", __func__, n_layer, hparams.n_layer); - return false; - } - if (cell_count > kv_self.size) { - LLAMA_LOG_ERROR("%s: not enough cells in kv cache to restore state (%u > %u)\n", __func__, cell_count, kv_self.size); - return false; - } - if (kv_self.v_trans != (bool) v_trans) { - LLAMA_LOG_ERROR("%s: incompatible V transposition\n", __func__); - return false; - } - - // For each layer, read the keys for each cell, one row is one cell, read as one contiguous block - for (uint32_t il = 0; il < n_layer; ++il) { - const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il) + hparams.n_embd_k_s(); - - // Read type of key - int32_t k_type_i_ref; - read_to(&k_type_i_ref, sizeof(k_type_i_ref)); - const int32_t k_type_i = (int32_t)kv_self.k_l[il]->type; - if (k_type_i != k_type_i_ref) { - LLAMA_LOG_ERROR("%s: mismatched key type (%d != %d, layer %d)\n", __func__, k_type_i, k_type_i_ref, il); - return false; - } - - // Read row size of key - uint64_t k_size_row_ref; - read_to(&k_size_row_ref, sizeof(k_size_row_ref)); - const size_t k_size_row = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa); - if (k_size_row != k_size_row_ref) { - LLAMA_LOG_ERROR("%s: mismatched key row size (%zu != %zu, layer %d)\n", __func__, k_size_row, (size_t) k_size_row_ref, il); - return false; - } - - if (cell_count) { - // Read and set the keys for the whole cell range - ggml_backend_tensor_set(kv_self.k_l[il], read(cell_count * k_size_row), kv_self.head * k_size_row, cell_count * k_size_row); - } - } - - if (!kv_self.v_trans) { - for (uint32_t il = 0; il < n_layer; ++il) { - const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s(); - - // Read type of value - int32_t v_type_i_ref; - read_to(&v_type_i_ref, sizeof(v_type_i_ref)); - const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type; - if (v_type_i != v_type_i_ref) { - LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il); - return false; - } - - // Read row size of value - uint64_t v_size_row_ref; - read_to(&v_size_row_ref, sizeof(v_size_row_ref)); - const size_t v_size_row = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa); - if (v_size_row != v_size_row_ref) { - LLAMA_LOG_ERROR("%s: mismatched value row size (%zu != %zu, layer %d)\n", __func__, v_size_row, (size_t) v_size_row_ref, il); - return false; - } - - if (cell_count) { - // Read and set the values for the whole cell range - ggml_backend_tensor_set(kv_self.v_l[il], read(cell_count * v_size_row), kv_self.head * v_size_row, cell_count * v_size_row); - } - } - } else { - // For each layer, read the values for each cell (transposed) - for (uint32_t il = 0; il < n_layer; ++il) { - const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s(); - - // Read type of value - int32_t v_type_i_ref; - read_to(&v_type_i_ref, sizeof(v_type_i_ref)); - const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type; - if (v_type_i != v_type_i_ref) { - LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il); - return false; - } - - // Read element size of value - uint32_t v_size_el_ref; - read_to(&v_size_el_ref, sizeof(v_size_el_ref)); - const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type); - if (v_size_el != v_size_el_ref) { - LLAMA_LOG_ERROR("%s: mismatched value element size (%zu != %zu, layer %d)\n", __func__, v_size_el, (size_t) v_size_el_ref, il); - return false; - } - - // Read GQA embedding size - uint32_t n_embd_v_gqa_ref; - read_to(&n_embd_v_gqa_ref, sizeof(n_embd_v_gqa_ref)); - if (n_embd_v_gqa != n_embd_v_gqa_ref) { - LLAMA_LOG_ERROR("%s: mismatched GQA embedding size (%u != %u, layer %d)\n", __func__, n_embd_v_gqa, n_embd_v_gqa_ref, il); - return false; - } - - if (cell_count) { - // For each row in the transposed matrix, read the values for the whole cell range - for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { - const size_t dst_offset = (kv_self.head + j * kv_self.size) * v_size_el; - ggml_backend_tensor_set(kv_self.v_l[il], read(cell_count * v_size_el), dst_offset, cell_count * v_size_el); } } } } - return true; - } + }; +} - void read_kv_cache(struct llama_context * ctx, llama_seq_id seq_id = -1) { - uint32_t cell_count; - read_to(&cell_count, sizeof(cell_count)); +// +// state save/load +// - bool res = read_kv_cache_meta(ctx, cell_count, seq_id) && read_kv_cache_data(ctx, cell_count); - - if (!res) { - if (seq_id == -1) { - llama_kv_cache_clear(ctx); - } else { - llama_kv_cache_seq_rm(ctx, seq_id, -1, -1); - } - throw std::runtime_error("failed to restore kv cache"); - } - } -}; - -struct llama_data_write_dummy : llama_data_write { - size_t size_written = 0; - - llama_data_write_dummy() {} +class llama_io_write_dummy : public llama_io_write_i { +public: + llama_io_write_dummy() = default; void write(const void * /* src */, size_t size) override { size_written += size; } - void write_tensor_data(const struct ggml_tensor * /* tensor */, size_t /* offset */, size_t size) override { + void write_tensor(const ggml_tensor * /* tensor */, size_t /* offset */, size_t size) override { size_written += size; } - size_t get_size_written() override { + size_t n_bytes() override { return size_written; } + +private: + size_t size_written = 0; }; -struct llama_data_write_buffer : llama_data_write { - uint8_t * ptr; - size_t buf_size = 0; - size_t size_written = 0; - - llama_data_write_buffer(uint8_t * p, size_t len) : ptr(p), buf_size(len) {} +class llama_io_write_buffer : public llama_io_write_i { +public: + llama_io_write_buffer( + uint8_t * p, size_t len) : ptr(p), buf_size(len) {} void write(const void * src, size_t size) override { if (size > buf_size) { @@ -1406,7 +1724,7 @@ struct llama_data_write_buffer : llama_data_write { buf_size -= size; } - void write_tensor_data(const struct ggml_tensor * tensor, size_t offset, size_t size) override { + void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) override { if (size > buf_size) { throw std::runtime_error("unexpectedly reached end of buffer"); } @@ -1416,17 +1734,19 @@ struct llama_data_write_buffer : llama_data_write { buf_size -= size; } - size_t get_size_written() override { + size_t n_bytes() override { return size_written; } + +private: + uint8_t * ptr; + size_t buf_size = 0; + size_t size_written = 0; }; -struct llama_data_read_buffer : llama_data_read { - const uint8_t * ptr; - size_t buf_size = 0; - size_t size_read = 0; - - llama_data_read_buffer(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {} +class llama_io_read_buffer : public llama_io_read_i { +public: + llama_io_read_buffer(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {} const uint8_t * read(size_t size) override { const uint8_t * base_ptr = ptr; @@ -1443,40 +1763,44 @@ struct llama_data_read_buffer : llama_data_read { memcpy(dst, read(size), size); } - size_t get_size_read() override { + size_t n_bytes() override { return size_read; } + +private: + const uint8_t * ptr; + size_t buf_size = 0; + size_t size_read = 0; }; -struct llama_data_write_file : llama_data_write { - llama_file * file; - size_t size_written = 0; - std::vector temp_buffer; - - llama_data_write_file(llama_file * f) : file(f) {} +class llama_io_write_file : public llama_io_write_i { +public: + llama_io_write_file(llama_file * f) : file(f) {} void write(const void * src, size_t size) override { file->write_raw(src, size); size_written += size; } - void write_tensor_data(const struct ggml_tensor * tensor, size_t offset, size_t size) override { + void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) override { temp_buffer.resize(size); ggml_backend_tensor_get(tensor, temp_buffer.data(), offset, size); write(temp_buffer.data(), temp_buffer.size()); } - size_t get_size_written() override { + size_t n_bytes() override { return size_written; } + +private: + llama_file * file; + size_t size_written = 0; + std::vector temp_buffer; }; -struct llama_data_read_file : llama_data_read { - llama_file * file; - size_t size_read = 0; - std::vector temp_buffer; - - llama_data_read_file(llama_file * f) : file(f) {} +class llama_io_read_file : public llama_io_read_i { +public: + llama_io_read_file(llama_file * f) : file(f) {} void read_to(void * dst, size_t size) override { file->read_raw(dst, size); @@ -1489,89 +1813,78 @@ struct llama_data_read_file : llama_data_read { return temp_buffer.data(); } - size_t get_size_read() override { + size_t n_bytes() override { return size_read; } + +private: + llama_file * file; + size_t size_read = 0; + std::vector temp_buffer; }; -/** copy state data into either a buffer or file depending on the passed in context - * - * file context: - * llama_file file("/path", "wb"); - * llama_data_write_file data_ctx(&file); - * llama_state_get_data_internal(ctx, data_ctx); - * - * buffer context: - * std::vector buf(max_size, 0); - * llama_data_write_buffer data_ctx(buf.data(), max_size); - * llama_state_get_data_internal(ctx, data_ctx); - * -*/ -static size_t llama_state_get_data_internal(struct llama_context * ctx, llama_data_write & data_ctx) { - llama_synchronize(ctx); - - data_ctx.write_model_info(ctx); - - // copy outputs - data_ctx.write_output_ids(ctx); - data_ctx.write_logits(ctx); - data_ctx.write_embeddings(ctx); - - data_ctx.write_kv_cache(ctx); - - return data_ctx.get_size_written(); -} - -size_t llama_state_get_data(struct llama_context * ctx, uint8_t * dst, size_t size) { - llama_data_write_buffer data_ctx(dst, size); +size_t llama_context::state_get_size() { + llama_io_write_dummy io; try { - return llama_state_get_data_internal(ctx, data_ctx); - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what()); - return 0; - } -} - -// Returns the *actual* size of the state. -// Intended to be used when saving to state to a buffer. -size_t llama_state_get_size(struct llama_context * ctx) { - llama_data_write_dummy data_ctx; - try { - return llama_state_get_data_internal(ctx, data_ctx); + return state_write_data(io); } catch (const std::exception & err) { LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what()); return 0; } } -static size_t llama_state_set_data_internal(struct llama_context * ctx, llama_data_read & data_ctx) { - llama_synchronize(ctx); - - data_ctx.read_model_info(ctx); - - // set outputs - data_ctx.read_output_ids(ctx); - data_ctx.read_logits(ctx); - data_ctx.read_embeddings(ctx); - - data_ctx.read_kv_cache(ctx); - - return data_ctx.get_size_read(); +size_t llama_context::state_get_data(uint8_t * dst, size_t size) { + llama_io_write_buffer io(dst, size); + try { + return state_write_data(io); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what()); + return 0; + } } -// Sets the state reading from the specified source address -size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src, size_t size) { - llama_data_read_buffer data_ctx(src, size); +size_t llama_context::state_set_data(const uint8_t * src, size_t size) { + llama_io_read_buffer io(src, size); try { - return llama_state_set_data_internal(ctx, data_ctx); + return state_read_data(io); } catch (const std::exception & err) { LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what()); return 0; } } -static bool llama_state_load_file_internal(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { - llama_file file(path_session, "rb"); +size_t llama_context::state_seq_get_size(llama_seq_id seq_id) { + llama_io_write_dummy io; + try { + return state_seq_write_data(io, seq_id); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what()); + return 0; + } +} + +size_t llama_context::state_seq_get_data(llama_seq_id seq_id, uint8_t * dst, size_t size) { + llama_io_write_buffer io(dst, size); + try { + return state_seq_write_data(io, seq_id); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what()); + return 0; + } +} + +size_t llama_context::state_seq_set_data(llama_seq_id seq_id, const uint8_t * src, size_t size) { + llama_io_read_buffer io(src, size); + try { + return state_seq_read_data(io, seq_id); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what()); + return 0; + } +} + +bool llama_context::state_load_file(const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { + llama_file file(filepath, "rb"); // sanity checks { @@ -1601,28 +1914,20 @@ static bool llama_state_load_file_internal(struct llama_context * ctx, const cha { const size_t n_state_size_cur = file.size() - file.tell(); - llama_data_read_file data_ctx(&file); - const size_t n_read = llama_state_set_data_internal(ctx, data_ctx); + llama_io_read_file io( &file); + const size_t n_read = state_read_data(io); if (n_read != n_state_size_cur) { LLAMA_LOG_ERROR("%s: did not read all of the session file data! size %zu, got %zu\n", __func__, n_state_size_cur, n_read); return false; } } + return true; } -bool llama_state_load_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { - try { - return llama_state_load_file_internal(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out); - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: error loading session file: %s\n", __func__, err.what()); - return false; - } -} - -static bool llama_state_save_file_internal(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { - llama_file file(path_session, "wb"); +bool llama_context::state_save_file(const char * filepath, const llama_token * tokens, size_t n_token_count) { + llama_file file(filepath, "wb"); file.write_u32(LLAMA_SESSION_MAGIC); file.write_u32(LLAMA_SESSION_VERSION); @@ -1632,82 +1937,13 @@ static bool llama_state_save_file_internal(struct llama_context * ctx, const cha file.write_raw(tokens, sizeof(llama_token) * n_token_count); // save the context state using stream saving - llama_data_write_file data_ctx(&file); - llama_state_get_data_internal(ctx, data_ctx); + llama_io_write_file io(&file); + state_write_data(io); return true; } -bool llama_state_save_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { - try { - return llama_state_save_file_internal(ctx, path_session, tokens, n_token_count); - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: error saving session file: %s\n", __func__, err.what()); - return false; - } -} - -static size_t llama_state_seq_get_data_internal(struct llama_context * ctx, llama_data_write & data_ctx, llama_seq_id seq_id) { - llama_synchronize(ctx); - - data_ctx.write_kv_cache(ctx, seq_id); - - return data_ctx.get_size_written(); -} - -size_t llama_state_seq_get_size(struct llama_context * ctx, llama_seq_id seq_id) { - llama_data_write_dummy data_ctx; - return llama_state_seq_get_data_internal(ctx, data_ctx, seq_id); -} - -size_t llama_state_seq_get_data(struct llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id) { - llama_data_write_buffer data_ctx(dst, size); - try { - return llama_state_seq_get_data_internal(ctx, data_ctx, seq_id); - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: error saving sequence state: %s\n", __func__, err.what()); - return 0; - } -} - -static size_t llama_state_seq_set_data_internal(struct llama_context * ctx, llama_data_read & data_ctx, llama_seq_id dest_seq_id) { - llama_synchronize(ctx); - - data_ctx.read_kv_cache(ctx, dest_seq_id); - - return data_ctx.get_size_read(); -} - -size_t llama_state_seq_set_data(struct llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id dest_seq_id) { - llama_data_read_buffer data_ctx(src, size); - try { - return llama_state_seq_set_data_internal(ctx, data_ctx, dest_seq_id); - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: error loading sequence state: %s\n", __func__, err.what()); - return 0; - } -} - -static size_t llama_state_seq_save_file_internal(struct llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) { - llama_file file(filepath, "wb"); - - file.write_u32(LLAMA_STATE_SEQ_MAGIC); - file.write_u32(LLAMA_STATE_SEQ_VERSION); - - // save the prompt - file.write_u32((uint32_t) n_token_count); - file.write_raw(tokens, sizeof(llama_token) * n_token_count); - - // save the context state using stream saving - llama_data_write_file data_ctx(&file); - llama_state_seq_get_data_internal(ctx, data_ctx, seq_id); - - const size_t res = file.tell(); - GGML_ASSERT(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + data_ctx.get_size_written()); - return res; -} - -static size_t llama_state_seq_load_file_internal(struct llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { +size_t llama_context::state_seq_load_file(llama_seq_id seq_id, const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { llama_file file(filepath, "rb"); // version checks @@ -1737,8 +1973,8 @@ static size_t llama_state_seq_load_file_internal(struct llama_context * ctx, con // restore the context state { const size_t state_size = file.size() - file.tell(); - llama_data_read_file data_ctx(&file); - const size_t nread = llama_state_seq_set_data_internal(ctx, data_ctx, dest_seq_id); + llama_io_read_file io(&file); + const size_t nread = state_seq_read_data(io, seq_id); if (!nread) { LLAMA_LOG_ERROR("%s: failed to restore sequence state\n", __func__); return 0; @@ -1750,26 +1986,785 @@ static size_t llama_state_seq_load_file_internal(struct llama_context * ctx, con return file.tell(); } -size_t llama_state_seq_save_file(struct llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) { +size_t llama_context::state_seq_save_file(llama_seq_id seq_id, const char * filepath, const llama_token * tokens, size_t n_token_count) { + llama_file file(filepath, "wb"); + + file.write_u32(LLAMA_STATE_SEQ_MAGIC); + file.write_u32(LLAMA_STATE_SEQ_VERSION); + + // save the prompt + file.write_u32((uint32_t) n_token_count); + file.write_raw(tokens, sizeof(llama_token) * n_token_count); + + // save the context state using stream saving + llama_io_write_file io(&file); + state_seq_write_data(io, seq_id); + + const size_t res = file.tell(); + GGML_ASSERT(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes()); + + return res; +} + +size_t llama_context::state_write_data(llama_io_write_i & io) { + LLAMA_LOG_DEBUG("%s: writing state\n", __func__); + + // write model info + { + LLAMA_LOG_DEBUG("%s: - writing model info\n", __func__); + + const std::string arch_str = llm_arch_name(model.arch); + io.write_string(arch_str); + // TODO: add more model-specific info which should prevent loading the session file if not identical + } + + // write output ids + { + LLAMA_LOG_DEBUG("%s: - writing output ids\n", __func__); + + output_reorder(); + + const auto n_outputs = this->n_outputs; + const auto & output_ids = this->output_ids; + + std::vector w_output_pos; + + GGML_ASSERT(n_outputs <= n_outputs_max); + + w_output_pos.resize(n_outputs); + + // build a more compact representation of the output ids + for (size_t i = 0; i < n_batch(); ++i) { + // map an output id to a position in the batch + int32_t pos = output_ids[i]; + if (pos >= 0) { + GGML_ASSERT(pos < n_outputs); + w_output_pos[pos] = i; + } + } + + io.write(&n_outputs, sizeof(n_outputs)); + + if (n_outputs) { + io.write(w_output_pos.data(), n_outputs * sizeof(int32_t)); + } + } + + // write logits + { + LLAMA_LOG_DEBUG("%s: - writing logits\n", __func__); + + const uint64_t logits_size = std::min((uint64_t) this->logits_size, (uint64_t) n_outputs * model.vocab.n_tokens()); + + io.write(&logits_size, sizeof(logits_size)); + + if (logits_size) { + io.write(logits, logits_size * sizeof(float)); + } + } + + // write embeddings + { + LLAMA_LOG_DEBUG("%s: - writing embeddings\n", __func__); + + const uint64_t embd_size = std::min((uint64_t) this->embd_size, (uint64_t) n_outputs * model.hparams.n_embd); + + io.write(&embd_size, sizeof(embd_size)); + + if (embd_size) { + io.write(embd, embd_size * sizeof(float)); + } + } + + LLAMA_LOG_DEBUG("%s: - writing KV self\n", __func__); + kv_self->state_write(io); + + return io.n_bytes(); +} + +size_t llama_context::state_read_data(llama_io_read_i & io) { + LLAMA_LOG_DEBUG("%s: reading state\n", __func__); + + // read model info + { + LLAMA_LOG_DEBUG("%s: - reading model info\n", __func__); + + const std::string cur_arch_str = llm_arch_name(model.arch); + + std::string arch_str; + io.read_string(arch_str); + if (cur_arch_str != arch_str) { + throw std::runtime_error(format("wrong model arch: '%s' instead of '%s'", arch_str.c_str(), cur_arch_str.c_str())); + } + // TODO: add more info which needs to be identical but which is not verified otherwise + } + + // read output ids + { + LLAMA_LOG_DEBUG("%s: - reading output ids\n", __func__); + + auto n_outputs = this->n_outputs; + io.read_to(&n_outputs, sizeof(n_outputs)); + + if (n_outputs > output_reserve(n_outputs)) { + throw std::runtime_error("could not reserve outputs"); + } + + std::vector output_pos; + + if (n_outputs) { + output_pos.resize(n_outputs); + io.read_to(output_pos.data(), n_outputs * sizeof(int32_t)); + + for (int32_t i = 0; i < (int32_t) output_pos.size(); ++i) { + int32_t id = output_pos[i]; + if ((uint32_t) id >= n_batch()) { + throw std::runtime_error(format("invalid output id, %d does not fit in batch size of %u", id, n_batch())); + } + this->output_ids[id] = i; + } + + this->n_outputs = n_outputs; + } + } + + // read logits + { + LLAMA_LOG_DEBUG("%s: - reading logits\n", __func__); + + uint64_t logits_size; + io.read_to(&logits_size, sizeof(logits_size)); + + if (this->logits_size < logits_size) { + throw std::runtime_error("logits buffer too small"); + } + + if (logits_size) { + io.read_to(this->logits, logits_size * sizeof(float)); + } + } + + // read embeddings + { + LLAMA_LOG_DEBUG("%s: - reading embeddings\n", __func__); + + uint64_t embd_size; + io.read_to(&embd_size, sizeof(embd_size)); + + if (this->embd_size < embd_size) { + throw std::runtime_error("embeddings buffer too small"); + } + + if (embd_size) { + io.read_to(this->embd, embd_size * sizeof(float)); + } + } + + LLAMA_LOG_DEBUG("%s: - reading KV self\n", __func__); + kv_self->state_read(io); + + return io.n_bytes(); +} + +size_t llama_context::state_seq_write_data(llama_io_write_i & io, llama_seq_id seq_id) { + GGML_UNUSED(seq_id); + + kv_self->state_write(io, seq_id); + + return io.n_bytes(); +} + +size_t llama_context::state_seq_read_data(llama_io_read_i & io, llama_seq_id seq_id) { + GGML_UNUSED(seq_id); + + kv_self->state_read(io, seq_id); + + return io.n_bytes(); +} + +// +// perf +// + +llama_perf_context_data llama_context::perf_get_data() const { + llama_perf_context_data data = {}; + + data.t_start_ms = 1e-3 * t_start_us; + data.t_load_ms = 1e-3 * t_load_us; + data.t_p_eval_ms = 1e-3 * t_p_eval_us; + data.t_eval_ms = 1e-3 * t_eval_us; + data.n_p_eval = std::max(1, n_p_eval); + data.n_eval = std::max(1, n_eval); + + return data; +} + +void llama_context::perf_reset() { + t_start_us = ggml_time_us(); + t_eval_us = n_eval = 0; + t_p_eval_us = n_p_eval = 0; +} + +// +// interface implementation +// + +llama_context_params llama_context_default_params() { + llama_context_params result = { + /*.n_ctx =*/ 512, + /*.n_batch =*/ 2048, + /*.n_ubatch =*/ 512, + /*.n_seq_max =*/ 1, + /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default + /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, + /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, + /*.pooling_type =*/ LLAMA_POOLING_TYPE_UNSPECIFIED, + /*.attention_type =*/ LLAMA_ATTENTION_TYPE_UNSPECIFIED, + /*.rope_freq_base =*/ 0.0f, + /*.rope_freq_scale =*/ 0.0f, + /*.yarn_ext_factor =*/ -1.0f, + /*.yarn_attn_factor =*/ 1.0f, + /*.yarn_beta_fast =*/ 32.0f, + /*.yarn_beta_slow =*/ 1.0f, + /*.yarn_orig_ctx =*/ 0, + /*.defrag_thold =*/ -1.0f, + /*.cb_eval =*/ nullptr, + /*.cb_eval_user_data =*/ nullptr, + /*.type_k =*/ GGML_TYPE_F16, + /*.type_v =*/ GGML_TYPE_F16, + /*.logits_all =*/ false, + /*.embeddings =*/ false, + /*.offload_kqv =*/ true, + /*.flash_attn =*/ false, + /*.no_perf =*/ true, + /*.abort_callback =*/ nullptr, + /*.abort_callback_data =*/ nullptr, + }; + + return result; +} + +llama_context * llama_init_from_model( + llama_model * model, + llama_context_params params) { + if (!model) { + LLAMA_LOG_ERROR("%s: model cannot be NULL\n", __func__); + return nullptr; + } + + if (params.n_batch == 0 && params.n_ubatch == 0) { + LLAMA_LOG_ERROR("%s: n_batch and n_ubatch cannot both be zero\n", __func__); + return nullptr; + } + + if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) { + LLAMA_LOG_ERROR("%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n", __func__); + return nullptr; + } + + if (params.flash_attn && model->arch == LLM_ARCH_GROK) { + LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__); + params.flash_attn = false; + } + + if (params.flash_attn && model->hparams.n_embd_head_k != model->hparams.n_embd_head_v) { + LLAMA_LOG_WARN("%s: flash_attn requires n_embd_head_k == n_embd_head_v - forcing off\n", __func__); + params.flash_attn = false; + } + + if (ggml_is_quantized(params.type_v) && !params.flash_attn) { + LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__); + return nullptr; + } + try { - return llama_state_seq_save_file_internal(ctx, filepath, seq_id, tokens, n_token_count); + auto * ctx = new llama_context(*model, params); + return ctx; + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what()); + } + + return nullptr; +} + +// deprecated +llama_context * llama_new_context_with_model( + llama_model * model, + llama_context_params params) { + return llama_init_from_model(model, params); +} + +void llama_free(llama_context * ctx) { + delete ctx; +} + +uint32_t llama_n_ctx(const llama_context * ctx) { + return ctx->n_ctx(); +} + +uint32_t llama_n_batch(const llama_context * ctx) { + return ctx->n_batch(); +} + +uint32_t llama_n_ubatch(const llama_context * ctx) { + return ctx->n_ubatch(); +} + +uint32_t llama_n_seq_max(const llama_context * ctx) { + return ctx->n_seq_max(); +} + +const llama_model * llama_get_model(const llama_context * ctx) { + return &ctx->get_model(); +} + +llama_kv_cache * llama_get_kv_self(llama_context * ctx) { + return ctx->get_kv_self(); +} + +void llama_kv_self_update(llama_context * ctx) { + ctx->kv_self_update(); +} + +enum llama_pooling_type llama_pooling_type(const llama_context * ctx) { + return ctx->pooling_type(); +} + +void llama_attach_threadpool( + llama_context * ctx, + ggml_threadpool_t threadpool, + ggml_threadpool_t threadpool_batch) { + ctx->attach_threadpool(threadpool, threadpool_batch); +} + +void llama_detach_threadpool(llama_context * ctx) { + ctx->detach_threadpool(); +} + +void llama_set_n_threads(llama_context * ctx, int32_t n_threads, int32_t n_threads_batch) { + ctx->set_n_threads(n_threads, n_threads_batch); +} + +int32_t llama_n_threads(llama_context * ctx) { + return ctx->n_threads(); +} + +int32_t llama_n_threads_batch(llama_context * ctx) { + return ctx->n_threads_batch(); +} + +void llama_set_abort_callback(llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) { + ctx->set_abort_callback(abort_callback, abort_callback_data); +} + +void llama_set_embeddings(llama_context * ctx, bool embeddings) { + ctx->set_embeddings(embeddings); +} + +void llama_set_causal_attn(llama_context * ctx, bool causal_attn) { + ctx->set_causal_attn(causal_attn); +} + +void llama_synchronize(llama_context * ctx) { + ctx->synchronize(); +} + +float * llama_get_logits(llama_context * ctx) { + ctx->synchronize(); + + return ctx->get_logits(); +} + +float * llama_get_logits_ith(llama_context * ctx, int32_t i) { + ctx->synchronize(); + + return ctx->get_logits_ith(i); +} + +float * llama_get_embeddings(llama_context * ctx) { + ctx->synchronize(); + + return ctx->get_embeddings(); +} + +float * llama_get_embeddings_ith(llama_context * ctx, int32_t i) { + ctx->synchronize(); + + return ctx->get_embeddings_ith(i); +} + +float * llama_get_embeddings_seq(llama_context * ctx, llama_seq_id seq_id) { + ctx->synchronize(); + + return ctx->get_embeddings_seq(seq_id); +} + +// llama adapter API + +int32_t llama_set_adapter_lora( + llama_context * ctx, + llama_adapter_lora * adapter, + float scale) { + ctx->set_adapter_lora(adapter, scale); + + return 0; +} + +int32_t llama_rm_adapter_lora( + llama_context * ctx, + llama_adapter_lora * adapter) { + bool res = ctx->rm_adapter_lora(adapter); + + return res ? 0 : -1; +} + +void llama_clear_adapter_lora(llama_context * ctx) { + ctx->clear_adapter_lora(); +} + +int32_t llama_apply_adapter_cvec( + llama_context * ctx, + const float * data, + size_t len, + int32_t n_embd, + int32_t il_start, + int32_t il_end) { + bool res = ctx->apply_adapter_cvec(data, len, n_embd, il_start, il_end); + + return res ? 0 : -1; +} + +// +// kv cache view +// + +llama_kv_cache_view llama_kv_cache_view_init(const llama_context * ctx, int32_t n_seq_max) { + const auto * kv = ctx->get_kv_self(); + if (kv == nullptr) { + LLAMA_LOG_WARN("%s: the context does not have a KV cache\n", __func__); + return {}; + } + + return llama_kv_cache_view_init(*kv, n_seq_max); +} + +void llama_kv_cache_view_update(const llama_context * ctx, llama_kv_cache_view * view) { + const auto * kv = ctx->get_kv_self(); + if (kv == nullptr) { + LLAMA_LOG_WARN("%s: the context does not have a KV cache\n", __func__); + return; + } + + llama_kv_cache_view_update(view, kv); +} + +// +// kv cache +// + +// deprecated +int32_t llama_get_kv_cache_token_count(const llama_context * ctx) { + return llama_kv_self_n_tokens(ctx); +} + +int32_t llama_kv_self_n_tokens(const llama_context * ctx) { + return llama_kv_cache_n_tokens(ctx->get_kv_self()); +} + +// deprecated +int32_t llama_get_kv_cache_used_cells(const llama_context * ctx) { + return llama_kv_self_used_cells(ctx); +} + +int32_t llama_kv_self_used_cells(const llama_context * ctx) { + return llama_kv_cache_used_cells(ctx->get_kv_self()); +} + +// deprecated +void llama_kv_cache_clear(llama_context * ctx) { + llama_kv_self_clear(ctx); +} + +void llama_kv_self_clear(llama_context * ctx) { + llama_kv_cache_clear(ctx->get_kv_self()); +} + +// deprecated +bool llama_kv_cache_seq_rm( + llama_context * ctx, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1) { + return llama_kv_self_seq_rm(ctx, seq_id, p0, p1); +} + +bool llama_kv_self_seq_rm( + llama_context * ctx, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1) { + return llama_kv_cache_seq_rm(ctx->get_kv_self(), seq_id, p0, p1); +} + +// deprecated +void llama_kv_cache_seq_cp( + llama_context * ctx, + llama_seq_id seq_id_src, + llama_seq_id seq_id_dst, + llama_pos p0, + llama_pos p1) { + return llama_kv_self_seq_cp(ctx, seq_id_src, seq_id_dst, p0, p1); +} + +void llama_kv_self_seq_cp( + llama_context * ctx, + llama_seq_id seq_id_src, + llama_seq_id seq_id_dst, + llama_pos p0, + llama_pos p1) { + return llama_kv_cache_seq_cp(ctx->get_kv_self(), seq_id_src, seq_id_dst, p0, p1); +} + +// deprecated +void llama_kv_cache_seq_keep( + llama_context * ctx, + llama_seq_id seq_id) { + return llama_kv_self_seq_keep(ctx, seq_id); +} + +void llama_kv_self_seq_keep(llama_context * ctx, llama_seq_id seq_id) { + return llama_kv_cache_seq_keep(ctx->get_kv_self(), seq_id); +} + +// deprecated +void llama_kv_cache_seq_add( + llama_context * ctx, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + llama_pos delta) { + return llama_kv_self_seq_add(ctx, seq_id, p0, p1, delta); +} + +void llama_kv_self_seq_add( + llama_context * ctx, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + llama_pos delta) { + return llama_kv_cache_seq_add(ctx->get_kv_self(), seq_id, p0, p1, delta); +} + +// deprecated +void llama_kv_cache_seq_div( + llama_context * ctx, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + int d) { + return llama_kv_self_seq_div(ctx, seq_id, p0, p1, d); +} + +void llama_kv_self_seq_div( + llama_context * ctx, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + int d) { + return llama_kv_cache_seq_div(ctx->get_kv_self(), seq_id, p0, p1, d); +} + +// deprecated +llama_pos llama_kv_cache_seq_pos_max(llama_context * ctx, llama_seq_id seq_id) { + return llama_kv_self_seq_pos_max(ctx, seq_id); +} + +llama_pos llama_kv_self_seq_pos_max(llama_context * ctx, llama_seq_id seq_id) { + return llama_kv_cache_seq_pos_max(ctx->get_kv_self(), seq_id); +} + +// deprecated +void llama_kv_cache_defrag(llama_context * ctx) { + return llama_kv_self_defrag(ctx); +} + +void llama_kv_self_defrag(llama_context * ctx) { + llama_kv_cache_defrag(ctx->get_kv_self()); +} + +// deprecated +bool llama_kv_cache_can_shift(const llama_context * ctx) { + return llama_kv_self_can_shift(ctx); +} + +bool llama_kv_self_can_shift(const llama_context * ctx) { + return llama_kv_cache_can_shift(ctx->get_kv_self()); +} + +// deprecated +void llama_kv_cache_update(llama_context * ctx) { + llama_kv_self_update(ctx); +} + +// llama state API + +// deprecated +size_t llama_get_state_size(llama_context * ctx) { + return llama_state_get_size(ctx); +} + +// deprecated +size_t llama_copy_state_data(llama_context * ctx, uint8_t * dst) { + return llama_state_get_data(ctx, dst, -1); +} + +// deprecated +size_t llama_set_state_data(llama_context * ctx, const uint8_t * src) { + return llama_state_set_data(ctx, src, -1); +} + +// deprecated +bool llama_load_session_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { + return llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out); +} + +// deprecated +bool llama_save_session_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { + return llama_state_save_file(ctx, path_session, tokens, n_token_count); +} + +// Returns the *actual* size of the state. +// Intended to be used when saving to state to a buffer. +size_t llama_state_get_size(llama_context * ctx) { + return ctx->state_get_size(); +} + +size_t llama_state_get_data(llama_context * ctx, uint8_t * dst, size_t size) { + ctx->synchronize(); + + return ctx->state_get_data(dst, size); +} + +// Sets the state reading from the specified source address +size_t llama_state_set_data(llama_context * ctx, const uint8_t * src, size_t size) { + ctx->synchronize(); + + return ctx->state_set_data(src, size); +} + +bool llama_state_load_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { + ctx->synchronize(); + + try { + return ctx->state_load_file(path_session, tokens_out, n_token_capacity, n_token_count_out); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error loading session file: %s\n", __func__, err.what()); + return false; + } +} + +bool llama_state_save_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { + ctx->synchronize(); + + try { + return ctx->state_save_file(path_session, tokens, n_token_count); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error saving session file: %s\n", __func__, err.what()); + return false; + } +} + +size_t llama_state_seq_get_size(llama_context * ctx, llama_seq_id seq_id) { + return ctx->state_seq_get_size(seq_id); +} + +size_t llama_state_seq_get_data(llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id) { + ctx->synchronize(); + + return ctx->state_seq_get_data(seq_id, dst, size); +} + +size_t llama_state_seq_set_data(llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id seq_id) { + ctx->synchronize(); + + return ctx->state_seq_set_data(seq_id, src, size); +} + +size_t llama_state_seq_save_file(llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) { + ctx->synchronize(); + + try { + return ctx->state_seq_save_file(seq_id, filepath, tokens, n_token_count); } catch (const std::exception & err) { LLAMA_LOG_ERROR("%s: error saving sequence state file: %s\n", __func__, err.what()); return 0; } } -size_t llama_state_seq_load_file(struct llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { +size_t llama_state_seq_load_file(llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { + ctx->synchronize(); + try { - return llama_state_seq_load_file_internal(ctx, filepath, dest_seq_id, tokens_out, n_token_capacity, n_token_count_out); + return ctx->state_seq_load_file(dest_seq_id, filepath, tokens_out, n_token_capacity, n_token_count_out); } catch (const std::exception & err) { LLAMA_LOG_ERROR("%s: error loading sequence state file: %s\n", __func__, err.what()); return 0; } } -const std::vector> & llama_internal_get_tensor_map( - struct llama_context * ctx -) { - return ctx->model.tensors_by_name; +/// + +int32_t llama_encode( + llama_context * ctx, + llama_batch batch) { + const int ret = ctx->encode(batch); + if (ret != 0) { + LLAMA_LOG_ERROR("%s: failed to encode, ret = %d\n", __func__, ret); + } + + return ret; +} + +int32_t llama_decode( + llama_context * ctx, + llama_batch batch) { + const int ret = ctx->decode(batch); + if (ret != 0) { + LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret); + } + + return ret; +} + +// +// perf +// + +llama_perf_context_data llama_perf_context(const llama_context * ctx) { + llama_perf_context_data data = {}; + + if (ctx == nullptr) { + return data; + } + + data = ctx->perf_get_data(); + + return data; +} + +void llama_perf_context_print(const llama_context * ctx) { + const auto data = llama_perf_context(ctx); + + const double t_end_ms = 1e-3 * ggml_time_us(); + + LLAMA_LOG_INFO("%s: load time = %10.2f ms\n", __func__, data.t_load_ms); + LLAMA_LOG_INFO("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", + __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval); + LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", + __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval); + LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval)); +} + +void llama_perf_context_reset(llama_context * ctx) { + ctx->perf_reset(); } diff --git a/src/llama-context.h b/src/llama-context.h index a9268b2920..71d702e8ba 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -3,66 +3,210 @@ #include "llama.h" #include "llama-batch.h" #include "llama-cparams.h" -#include "llama-model.h" -#include "llama-kv-cache.h" +#include "llama-graph.h" #include "llama-adapter.h" #include "ggml-cpp.h" #include -#include #include -#include + +struct llama_model; +struct llama_kv_cache; + +class llama_io_read_i; +class llama_io_write_i; struct llama_context { - llama_context(const llama_model & model) - : model(model) - , t_start_us(model.t_start_us) - , t_load_us(model.t_load_us) {} + // init scheduler and compute buffers, reserve worst-case graphs + llama_context( + const llama_model & model, + llama_context_params params); - const struct llama_model & model; + ~llama_context(); - struct llama_cparams cparams; - struct llama_sbatch sbatch; // TODO: revisit if needed - struct llama_kv_cache kv_self; - struct llama_adapter_cvec cvec; + void synchronize(); - std::unordered_map lora; + const llama_model & get_model() const; - std::vector backends; - std::vector> set_n_threads_fns; + uint32_t n_ctx() const; + uint32_t n_ctx_per_seq() const; + uint32_t n_batch() const; + uint32_t n_ubatch() const; + uint32_t n_seq_max() const; - ggml_backend_t backend_cpu = nullptr; + uint32_t n_threads() const; + uint32_t n_threads_batch() const; - ggml_threadpool_t threadpool = nullptr; - ggml_threadpool_t threadpool_batch = nullptr; + llama_kv_cache * get_kv_self(); + const llama_kv_cache * get_kv_self() const; - bool has_evaluated_once = false; + void kv_self_update(); - mutable int64_t t_start_us; - mutable int64_t t_load_us; - mutable int64_t t_p_eval_us = 0; - mutable int64_t t_eval_us = 0; + enum llama_pooling_type pooling_type() const; - mutable int64_t t_compute_start_us = 0; - mutable int64_t n_queued_tokens = 0; + float * get_logits(); + float * get_logits_ith(int32_t i); - mutable int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1) - mutable int32_t n_eval = 0; // number of eval calls + float * get_embeddings(); + float * get_embeddings_ith(int32_t i); + float * get_embeddings_seq(llama_seq_id seq_id); - // host buffer for the model output (logits and embeddings) - ggml_backend_buffer_ptr buf_output; + void attach_threadpool( + ggml_threadpool_t threadpool, + ggml_threadpool_t threadpool_batch); + + void detach_threadpool(); + + void set_n_threads(int32_t n_threads, int32_t n_threads_batch); + + void set_abort_callback(bool (*abort_callback)(void * data), void * abort_callback_data); + + void set_embeddings (bool value); + void set_causal_attn(bool value); + + void set_adapter_lora( + llama_adapter_lora * adapter, + float scale); + + bool rm_adapter_lora( + llama_adapter_lora * adapter); + + void clear_adapter_lora(); + + bool apply_adapter_cvec( + const float * data, + size_t len, + int32_t n_embd, + int32_t il_start, + int32_t il_end); + + int encode(llama_batch & inp_batch); + int decode(llama_batch & inp_batch); + + // + // state save/load + // + + size_t state_get_size(); + size_t state_get_data( uint8_t * dst, size_t size); + size_t state_set_data(const uint8_t * src, size_t size); + + size_t state_seq_get_size(llama_seq_id seq_id); + size_t state_seq_get_data(llama_seq_id seq_id, uint8_t * dst, size_t size); + size_t state_seq_set_data(llama_seq_id seq_id, const uint8_t * src, size_t size); + + bool state_load_file( + const char * filepath, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + + bool state_save_file( + const char * filepath, + const llama_token * tokens, + size_t n_token_count); + + size_t state_seq_load_file( + llama_seq_id seq_id, + const char * filepath, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + + size_t state_seq_save_file( + llama_seq_id seq_id, + const char * filepath, + const llama_token * tokens, + size_t n_token_count); + + // + // perf + // + + llama_perf_context_data perf_get_data() const; + void perf_reset(); + +private: + // + // output + // + + // Make sure enough space is available for outputs. + // Returns max number of outputs for which space was reserved. + int32_t output_reserve(int32_t n_outputs); + + // make the outputs have the same order they had in the user-provided batch + // TODO: maybe remove this + void output_reorder(); + + // + // graph + // + + int32_t graph_max_nodes() const; + + // zero-out inputs and create the ctx_compute for the compute graph + ggml_cgraph * graph_init(); + + llm_graph_result_ptr graph_build( + ggml_context * ctx, + ggml_cgraph * gf, + const llama_ubatch & ubatch, + llm_graph_type gtype); + + // returns the result of ggml_backend_sched_graph_compute_async execution + ggml_status graph_compute( + ggml_cgraph * gf, + bool batched); + + llm_graph_cb graph_get_cb() const; + + // used by kv_self_update() + ggml_tensor * build_rope_shift( + ggml_context * ctx0, + ggml_tensor * cur, + ggml_tensor * shift, + ggml_tensor * factors, + ggml_backend_buffer * bbuf) const; + + llm_graph_result_ptr build_kv_self_shift( + ggml_context * ctx0, + ggml_cgraph * gf) const; + + llm_graph_result_ptr build_kv_self_defrag( + ggml_context * ctx0, + ggml_cgraph * gf) const; + + // TODO: read/write lora adapters and cvec + size_t state_write_data(llama_io_write_i & io); + size_t state_read_data (llama_io_read_i & io); + + size_t state_seq_write_data(llama_io_write_i & io, llama_seq_id seq_id); + size_t state_seq_read_data (llama_io_read_i & io, llama_seq_id seq_id); + + // + // members + // + + const llama_model & model; + + llama_cparams cparams; + llama_adapter_cvec cvec; + llama_adapter_loras loras; + llama_sbatch sbatch; + + llama_cross cross; // TODO: tmp for handling cross-attention - need something better probably + + std::unique_ptr kv_self; + + // TODO: remove + bool logits_all = false; // decode output (2-dimensional array: [n_outputs][n_vocab]) size_t logits_size = 0; // capacity (of floats) for logits float * logits = nullptr; - std::vector output_ids; // map batch token positions to ids of the logits and embd buffers - size_t output_size = 0; // capacity (of tokens positions) for the output buffers - int32_t n_outputs = 0; // number of actually-used outputs in the current ubatch or last logical batch - - bool logits_all = false; - // embeddings output (2-dimensional array: [n_outputs][n_embd]) // populated only when pooling_type == LLAMA_POOLING_TYPE_NONE size_t embd_size = 0; // capacity (of floats) for embeddings @@ -72,57 +216,47 @@ struct llama_context { // populated only when pooling_type != LLAMA_POOLING_TYPE_NONE std::map> embd_seq; - // whether we are computing encoder output or decoder output - bool is_encoding = false; + int32_t n_outputs = 0; // number of actually-used outputs in the current ubatch or last logical batch + int32_t n_outputs_max = 0; // capacity (of tokens positions) for the output buffers - // TODO: find a better way to accommodate mutli-dimension position encoding methods - // number of position id each token get, 1 for each token in most cases. - // when using m-rope, it will be 3 position ids per token to representing 3 dimension coordinate. - int n_pos_per_token = 1; + std::vector output_ids; // map batch token positions to ids of the logits and embd buffers - // output of the encoder part of the encoder-decoder models - std::vector embd_enc; - std::vector> seq_ids_enc; - - // memory buffers used to evaluate the model - std::vector buf_compute_meta; ggml_backend_sched_ptr sched; + ggml_backend_t backend_cpu = nullptr; + std::vector backends; + + ggml_context_ptr ctx_compute; + + ggml_threadpool_t threadpool = nullptr; + ggml_threadpool_t threadpool_batch = nullptr; + ggml_abort_callback abort_callback = nullptr; void * abort_callback_data = nullptr; - // input tensors - struct ggml_tensor * inp_tokens; // I32 [n_batch] - struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch] - struct ggml_tensor * inp_pos; // I32 [n_batch] - struct ggml_tensor * inp_out_ids; // I32 [n_outputs] - struct ggml_tensor * inp_KQ_mask; // F32 [kv_size, n_batch] - struct ggml_tensor * inp_KQ_mask_swa; // F32 [kv_size, n_batch] - struct ggml_tensor * inp_K_shift; // I32 [kv_size] - struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch] - struct ggml_tensor * inp_cls; // I32 [n_batch] - struct ggml_tensor * inp_s_copy; // I32 [kv_size] - struct ggml_tensor * inp_s_mask; // F32 [1, n_kv] - struct ggml_tensor * inp_s_seq; // I32 [n_kv, n_batch] - struct ggml_tensor * inp_pos_bucket; // I32 [n_batch|n_kv, n_batch] - struct ggml_tensor * inp_embd_enc; // F32 [n_embd, n_outputs_enc] - struct ggml_tensor * inp_KQ_mask_cross; // F32 [n_outputs_enc, n_batch] + std::vector> set_n_threads_fns; + + // buffer types used for the compute buffer of each backend + std::vector backend_ptrs; + std::vector backend_buft; + + // memory buffers used to evaluate the model + std::vector buf_compute_meta; + + // host buffer for the model output (logits and embeddings) + ggml_backend_buffer_ptr buf_output; + + bool has_evaluated_once = false; + + // perf + mutable int64_t t_start_us = 0; + mutable int64_t t_load_us = 0; + mutable int64_t t_p_eval_us = 0; + mutable int64_t t_eval_us = 0; + + mutable int64_t t_compute_start_us = 0; + mutable int64_t n_queued_tokens = 0; + + mutable int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1) + mutable int32_t n_eval = 0; // number of eval calls }; - -// TODO: make these methods of llama_context -void llama_set_k_shift(struct llama_context & lctx); - -void llama_set_s_copy(struct llama_context & lctx); - -void llama_set_inputs(llama_context & lctx, const llama_ubatch & ubatch); - -// Make sure enough space is available for outputs. -// Returns max number of outputs for which space was reserved. -size_t llama_output_reserve(struct llama_context & lctx, size_t n_outputs); - -// make the outputs have the same order they had in the user-provided batch -void llama_output_reorder(struct llama_context & ctx); - -// For internal test use -// TODO: remove -const std::vector> & llama_internal_get_tensor_map(struct llama_context * ctx); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp new file mode 100644 index 0000000000..1e3f2efc89 --- /dev/null +++ b/src/llama-graph.cpp @@ -0,0 +1,1695 @@ +#include "llama-graph.h" + +#include "llama-impl.h" +#include "llama-batch.h" +#include "llama-cparams.h" +#include "llama-kv-cache.h" + +#include +#include +#include + +static int32_t llama_relative_position_bucket(llama_pos x, llama_pos y, uint64_t n_buckets, bool bidirectional) { + // TODO move to hparams if a T5 variant appears that uses a different value + const int64_t max_distance = 128; + + if (bidirectional) { + n_buckets >>= 1; + } + + const int64_t max_exact = n_buckets >> 1; + + int32_t relative_position = x - y; + int32_t relative_bucket = 0; + + if (bidirectional) { + relative_bucket += (relative_position > 0) * n_buckets; + relative_position = abs(relative_position); + } else { + relative_position = -std::min(relative_position, 0); + } + + int32_t relative_position_if_large = floorf(max_exact + logf(1.0 * relative_position / max_exact) * (n_buckets - max_exact) / log(1.0 * max_distance / max_exact)); + relative_position_if_large = std::min(relative_position_if_large, n_buckets - 1); + relative_bucket += (relative_position < max_exact ? relative_position : relative_position_if_large); + + return relative_bucket; +} + +void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) { + if (ubatch->token) { + const int64_t n_tokens = ubatch->n_tokens; + + ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens*ggml_element_size(tokens)); + } + + if (ubatch->embd) { + const int64_t n_embd = embd->ne[0]; + const int64_t n_tokens = ubatch->n_tokens; + + ggml_backend_tensor_set(embd, ubatch->embd, 0, n_tokens*n_embd*ggml_element_size(embd)); + } +} + +void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) { + if (ubatch->pos && pos) { + const int64_t n_tokens = ubatch->n_tokens; + + ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_token*ggml_element_size(pos)); + } +} + +void llm_graph_input_pos_bucket::set_input(const llama_ubatch * ubatch) { + if (pos_bucket) { + const int64_t n_tokens = ubatch->n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(pos_bucket->buffer)); + GGML_ASSERT(!ubatch->equal_seqs); // TODO: use ubatch->n_seqs instead of failing + + int32_t * data = (int32_t *) pos_bucket->data; + + for (int h = 0; h < 1; ++h) { + for (int j = 0; j < n_tokens; ++j) { + for (int i = 0; i < n_tokens; ++i) { + data[h*(n_tokens*n_tokens) + j*n_tokens + i] = llama_relative_position_bucket(ubatch->pos[i], ubatch->pos[j], hparams.n_rel_attn_bkts, true); + } + } + } + } +} + +void llm_graph_input_pos_bucket_kv::set_input(const llama_ubatch * ubatch) { + if (pos_bucket) { + const int64_t n_tokens = ubatch->n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(pos_bucket->buffer)); + GGML_ASSERT(!ubatch->equal_seqs); // TODO: use ubatch->n_seqs instead of failing + + int32_t * data = (int32_t *) pos_bucket->data; + + const int64_t n_kv = kv_self->n; + + for (int h = 0; h < 1; ++h) { + for (int j = 0; j < n_tokens; ++j) { + for (int i = 0; i < n_kv; ++i) { + data[h*(n_kv*n_tokens) + j*n_kv + i] = llama_relative_position_bucket(kv_self->cells[i].pos, ubatch->pos[j], hparams.n_rel_attn_bkts, false); + } + } + } + } +} + +void llm_graph_input_out_ids::set_input(const llama_ubatch * ubatch) { + if (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) { + //GGML_ASSERT(out_ids && "every model that can must skip unused outputs"); + + if (!out_ids) { + LLAMA_LOG_WARN("%s: 'out_ids' is not created\n", __func__); + } else { + const int64_t n_tokens = ubatch->n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(out_ids->buffer)); + int32_t * data = (int32_t *) out_ids->data; + + if (n_outputs == n_tokens) { + for (int i = 0; i < n_tokens; ++i) { + data[i] = i; + } + } else if (ubatch->output) { + int32_t n_outputs = 0; + for (int i = 0; i < n_tokens; ++i) { + if (ubatch->output[i]) { + data[n_outputs++] = i; + } + } + // the graph needs to have been passed the correct number of outputs + GGML_ASSERT(n_outputs == n_outputs); + } else if (n_outputs == 1) { + // only keep last output + data[0] = n_tokens - 1; + } else { + GGML_ASSERT(n_outputs == 0); + } + } + } +} + +void llm_graph_input_mean::set_input(const llama_ubatch * ubatch) { + if (cparams.embeddings && cparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) { + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_seq_tokens = ubatch->n_seq_tokens; + const int64_t n_seqs = ubatch->n_seqs; + + GGML_ASSERT(mean); + GGML_ASSERT(ggml_backend_buffer_is_host(mean->buffer)); + + float * data = (float *) mean->data; + memset(mean->data, 0, n_tokens * n_tokens * ggml_element_size(mean)); + + std::vector sum(n_tokens, 0); + + for (int s = 0; s < n_seqs; ++s) { + const llama_seq_id seq_id = ubatch->seq_id[s][0]; + + // TODO: adapt limits to n_seqs when ubatch->equal_seqs is true + GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == MEAN"); + + sum[seq_id] += ubatch->n_seq_tokens; + } + + std::vector div(n_tokens, 0.0f); + for (int i = 0; i < n_tokens; ++i) { + const uint64_t s = sum[i]; + if (s > 0) { + div[i] = 1.0f/float(s); + } + } + + for (int s = 0; s < n_seqs; ++s) { + const llama_seq_id seq_id = ubatch->seq_id[s][0]; + + for (int i = 0; i < n_seq_tokens; ++i) { + data[seq_id*n_tokens + s*n_seq_tokens + i] = div[seq_id]; + } + } + } +} + +void llm_graph_input_cls::set_input(const llama_ubatch * ubatch) { + if (cparams.embeddings && ( + cparams.pooling_type == LLAMA_POOLING_TYPE_CLS || + cparams.pooling_type == LLAMA_POOLING_TYPE_RANK)) { + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_seq_tokens = ubatch->n_seq_tokens; + const int64_t n_seqs = ubatch->n_seqs; + + GGML_ASSERT(cls); + GGML_ASSERT(ggml_backend_buffer_is_host(cls->buffer)); + + uint32_t * data = (uint32_t *) cls->data; + memset(cls->data, 0, n_tokens * ggml_element_size(cls)); + + for (int s = 0; s < n_seqs; ++s) { + const llama_seq_id seq_id = ubatch->seq_id[s][0]; + + // TODO: adapt limits to n_seqs when ubatch->equal_seqs is true + GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == CLS or RANK"); + + for (int i = 0; i < n_seq_tokens; ++i) { + const llama_pos pos = ubatch->pos[s*n_seq_tokens + i]; + + if (pos == 0) { + data[seq_id] = s*n_seq_tokens + i; + } + } + } + } + + if (cparams.embeddings && cparams.pooling_type == LLAMA_POOLING_TYPE_LAST) { + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_seq_tokens = ubatch->n_seq_tokens; + const int64_t n_seqs = ubatch->n_seqs; + + GGML_ASSERT(cls); + GGML_ASSERT(ggml_backend_buffer_is_host(cls->buffer)); + + uint32_t * data = (uint32_t *) cls->data; + memset(cls->data, 0, n_tokens * ggml_element_size(cls)); + + std::vector last_pos(n_tokens, -1); + std::vector last_row(n_tokens, -1); + + for (int s = 0; s < n_seqs; ++s) { + const llama_seq_id seq_id = ubatch->seq_id[s][0]; + + // TODO: adapt limits to n_seqs when ubatch->equal_seqs is true + GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == LAST"); + + for (int i = 0; i < n_seq_tokens; ++i) { + const llama_pos pos = ubatch->pos[s*n_seq_tokens + i]; + + if (pos >= last_pos[seq_id]) { + last_pos[seq_id] = pos; + last_row[seq_id] = s*n_seq_tokens + i; + } + } + } + + for (int i = 0; i < n_tokens; ++i) { + if (last_row[i] >= 0) { + data[i] = last_row[i]; + } + } + } +} + +void llm_graph_input_s_copy::set_input(const llama_ubatch * ubatch) { + GGML_UNUSED(ubatch); + + const int64_t n_kv = kv_self->n; + + if (s_copy) { + GGML_ASSERT(ggml_backend_buffer_is_host(s_copy->buffer)); + int32_t * data = (int32_t *) s_copy->data; + + // assuming copy destinations ALWAYS happen ONLY on the cells between head and head+n + for (uint32_t i = 0; i < n_kv; ++i) { + const uint32_t cell_id = i + kv_self->head; + + ////////////////////////////////////////////// + // TODO: this should not mutate the KV cache ! + llama_kv_cell & kv_cell = const_cast(kv_self)->cells[i]; + + // prevent out-of-bound sources + if (kv_cell.src < 0 || (uint32_t) kv_cell.src >= kv_self->size) { + kv_cell.src = cell_id; + } + + data[i] = kv_cell.src; + + // TODO: do not mutate the KV cache + // ensure copy only happens once + if (kv_cell.src != (int32_t) cell_id) { + kv_cell.src = cell_id; + } + } + } +} + +void llm_graph_input_s_mask::set_input(const llama_ubatch * ubatch) { + GGML_UNUSED(ubatch); + + const int64_t n_kv = kv_self->n; + + if (s_mask) { + GGML_ASSERT(ggml_backend_buffer_is_host(s_mask->buffer)); + float * data = (float *) s_mask->data; + + // clear unused states + for (int i = 0; i < n_kv; ++i) { + const uint32_t cell_id = i + kv_self->head; + + ////////////////////////////////////////////// + // TODO: this should not mutate the KV cache ! + llama_kv_cell & kv_cell = const_cast(kv_self)->cells[i]; + + data[i] = (float) (kv_cell.src >= 0); + + // only clear once + if (kv_cell.src < 0) { + kv_cell.src = cell_id; + } + } + } +} + +void llm_graph_input_cross_embd::set_input(const llama_ubatch * ubatch) { + GGML_UNUSED(ubatch); + + if (cross_embd && !cross->v_embd.empty()) { + assert(cross_embd->type == GGML_TYPE_F32); + + ggml_backend_tensor_set(cross_embd, cross->v_embd.data(), 0, ggml_nbytes(cross_embd)); + } +} + +void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) { + if (kq_mask) { + if (cparams.causal_attn) { + const int64_t n_kv = ubatch->n_tokens; + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_seq_tokens = ubatch->n_seq_tokens; + const int64_t n_seqs = ubatch->n_seqs; + + GGML_ASSERT(ggml_backend_buffer_is_host(kq_mask->buffer)); + float * data = (float *) kq_mask->data; + + for (int h = 0; h < 1; ++h) { + for (int s1 = 0; s1 < n_seqs; ++s1) { + const llama_seq_id seq_id = ubatch->seq_id[s1][0]; + + for (int j = 0; j < n_seq_tokens; ++j) { + const int32_t tj = s1*n_seq_tokens + j; + + for (int s0 = 0; s0 < n_seqs; ++s0) { + for (int i = 0; i < n_seq_tokens; ++i) { + const int32_t ti = s0*n_seq_tokens + i; + float f = -INFINITY; + + for (int s = 0; s < ubatch->n_seq_id[s0]; ++s) { + if (ubatch->seq_id[s0][s] == seq_id && ubatch->pos[ti] <= ubatch->pos[tj]) { + if (hparams.use_alibi) { + f = -std::abs(ubatch->pos[ti] - ubatch->pos[tj]); + } else { + f = 0.0f; + } + break; + } + } + + data[h*(n_kv*n_tokens) + tj*n_kv + ti] = f; + } + } + } + } + } + } else { + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_seq_tokens = ubatch->n_seq_tokens; + const int64_t n_seqs = ubatch->n_seqs; + const int64_t n_stride = ubatch->n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(kq_mask->buffer)); + + float * data = (float *) kq_mask->data; + + for (int h = 0; h < 1; ++h) { + for (int s1 = 0; s1 < n_seqs; ++s1) { + const llama_seq_id seq_id = ubatch->seq_id[s1][0]; + + for (int j = 0; j < n_seq_tokens; ++j) { + const int32_t tj = s1*n_seq_tokens + j; + + for (int s0 = 0; s0 < n_seqs; ++s0) { + for (int i = 0; i < n_seq_tokens; ++i) { + const int32_t ti = s0*n_seq_tokens + i; + float f = -INFINITY; + + for (int s = 0; s < ubatch->n_seq_id[s0]; ++s) { + if (ubatch->seq_id[s0][s] == seq_id) { + if (hparams.use_alibi) { + f = -std::abs(ubatch->pos[ti] - ubatch->pos[tj]); + } else { + f = 0.0f; + } + break; + } + } + + data[h*(n_tokens*n_tokens) + tj*n_stride + ti] = f; + } + } + + for (int i = n_tokens; i < n_stride; ++i) { + data[h*(n_tokens*n_tokens) + tj*n_stride + i] = -INFINITY; + } + } + } + } + } + } +} + +void llm_graph_input_attn_kv_unified::set_input(const llama_ubatch * ubatch) { + if (self_kq_mask || self_kq_mask_swa) { + // NOTE: hparams.causal_attn indicates the model is capable of generation and uses the kv cache. + if (cparams.causal_attn) { + const int64_t n_kv = kv_self->n; + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_seq_tokens = ubatch->n_seq_tokens; + const int64_t n_seqs = ubatch->n_seqs; + + float * data = nullptr; + float * data_swa = nullptr; + + if (self_kq_mask) { + GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask->buffer)); + data = (float *) self_kq_mask->data; + } + + if (self_kq_mask_swa) { + GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask_swa->buffer)); + data_swa = (float *) self_kq_mask_swa->data; + } + + // For causal attention, use only the previous KV cells + // of the correct sequence for each token of the ubatch. + // It's assumed that if a token in the batch has multiple sequences, they are equivalent. + for (int h = 0; h < 1; ++h) { + for (int s = 0; s < n_seqs; ++s) { + const llama_seq_id seq_id = ubatch->seq_id[s][0]; + + for (int j = 0; j < n_seq_tokens; ++j) { + const llama_pos pos = ubatch->pos[s*n_seq_tokens + j]; + + for (int i = 0; i < n_kv; ++i) { + float f; + if (!kv_self->cells[i].has_seq_id(seq_id) || kv_self->cells[i].pos > pos) { + f = -INFINITY; + } else { + if (hparams.use_alibi) { + f = -std::abs(kv_self->cells[i].pos - pos); + } else { + f = 0.0f; + } + } + + if (data) { + data[h*(n_kv*n_tokens) + s*(n_kv*n_seq_tokens) + j*n_kv + i] = f; + } + + // may need to cut off old tokens for sliding window + if (data_swa) { + if (pos - kv_self->cells[i].pos >= (int32_t)hparams.n_swa) { + f = -INFINITY; + } + data_swa[h*(n_kv*n_tokens) + s*(n_kv*n_seq_tokens) + j*n_kv + i] = f; + } + } + } + } + + if (data) { + for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) { + for (int j = 0; j < n_kv; ++j) { + data[h*(n_kv*n_tokens) + i*n_kv + j] = -INFINITY; + } + } + } + + if (data_swa) { + for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) { + for (int j = 0; j < n_kv; ++j) { + data_swa[h*(n_kv*n_tokens) + i*n_kv + j] = -INFINITY; + } + } + } + } + } else { + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_seq_tokens = ubatch->n_seq_tokens; + const int64_t n_seqs = ubatch->n_seqs; + // when using kv cache, the mask needs to match the kv cache size + const int64_t n_stride = n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask->buffer)); + + float * data = (float *) self_kq_mask->data; + + for (int h = 0; h < 1; ++h) { + for (int s1 = 0; s1 < n_seqs; ++s1) { + const llama_seq_id seq_id = ubatch->seq_id[s1][0]; + + for (int j = 0; j < n_seq_tokens; ++j) { + const int32_t tj = s1*n_seq_tokens + j; + + for (int s0 = 0; s0 < n_seqs; ++s0) { + for (int i = 0; i < n_seq_tokens; ++i) { + const int32_t ti = s0*n_seq_tokens + i; + float f = -INFINITY; + + for (int s = 0; s < ubatch->n_seq_id[s0]; ++s) { + if (ubatch->seq_id[s0][s] == seq_id) { + if (hparams.use_alibi) { + f = -std::abs(ubatch->pos[ti] - ubatch->pos[tj]); + } else { + f = 0.0f; + } + break; + } + } + + data[h*(n_tokens*n_tokens) + tj*n_stride + ti] = f; + } + } + + for (int i = n_tokens; i < n_stride; ++i) { + data[h*(n_tokens*n_tokens) + tj*n_stride + i] = -INFINITY; + } + } + } + } + } + } +} + +void llm_graph_input_attn_cross::set_input(const llama_ubatch * ubatch) { + if (cross_kq_mask) { + const int64_t n_enc = cross_kq_mask->ne[0]; + const int64_t n_tokens = ubatch->n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(cross_kq_mask->buffer)); + GGML_ASSERT(!ubatch->equal_seqs); // TODO: use ubatch->n_seqs instead of failing + + float * data = (float *) cross_kq_mask->data; + + for (int h = 0; h < 1; ++h) { + for (int j = 0; j < n_tokens; ++j) { + for (int i = 0; i < n_enc; ++i) { + float f = -INFINITY; + for (int s = 0; s < ubatch->n_seq_id[j]; ++s) { + const llama_seq_id seq_id = ubatch->seq_id[j][s]; + if (cross->seq_ids_enc[i].find(seq_id) != cross->seq_ids_enc[i].end()) { + f = 0.0f; + } + } + data[h*(n_enc*n_tokens) + j*n_enc + i] = f; + } + } + + for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) { + for (int j = 0; j < n_enc; ++j) { + data[h*(n_enc*n_tokens) + i*n_enc + j] = -INFINITY; + } + } + } + } +} + +// +// llm_graph_context +// + +llm_graph_context::llm_graph_context(const llm_graph_params & params) : + arch (params.arch), + hparams (params.hparams), + cparams (params.cparams), + ubatch (params.ubatch), + n_embd (hparams.n_embd), + n_layer (hparams.n_layer), + n_rot (hparams.n_rot), + n_ctx (cparams.n_ctx), + n_ctx_per_seq (cparams.n_ctx / cparams.n_seq_max), + n_head (hparams.n_head()), + n_head_kv (hparams.n_head_kv()), + n_embd_head_k (hparams.n_embd_head_k), + n_embd_k_gqa (hparams.n_embd_k_gqa()), + n_embd_head_v (hparams.n_embd_head_v), + n_embd_v_gqa (hparams.n_embd_v_gqa()), + n_expert (hparams.n_expert), + n_expert_used (hparams.n_expert_used), + freq_base (cparams.rope_freq_base), + freq_scale (cparams.rope_freq_scale), + ext_factor (cparams.yarn_ext_factor), + attn_factor (cparams.yarn_attn_factor), + beta_fast (cparams.yarn_beta_fast), + beta_slow (cparams.yarn_beta_slow), + norm_eps (hparams.f_norm_eps), + norm_rms_eps (hparams.f_norm_rms_eps), + n_tokens (ubatch.n_tokens), + n_outputs (params.n_outputs), + n_ctx_orig (cparams.n_ctx_orig_yarn), + pooling_type (cparams.pooling_type), + rope_type (hparams.rope_type), + ctx0 (params.ctx), + sched (params.sched), + backend_cpu (params.backend_cpu), + cvec (params.cvec), + loras (params.loras), + memory (params.memory), + cross (params.cross), + cb_func (params.cb), + res (std::make_unique()) { + } + +int64_t llm_graph_context::n_pos_per_token() const { + return arch == LLM_ARCH_QWEN2VL ? 4 : 1; +} + +void llm_graph_context::cb(ggml_tensor * cur, const char * name, int il) const { + if (cb_func) { + cb_func(ubatch, cur, name, il); + } +} + +ggml_tensor * llm_graph_context::build_cvec( + ggml_tensor * cur, + int il) const { + return cvec->apply_to(ctx0, cur, il); +} + +ggml_tensor * llm_graph_context::build_lora_mm( + ggml_tensor * w, + ggml_tensor * cur) const { + ggml_tensor * res = ggml_mul_mat(ctx0, w, cur); + + for (const auto & lora : *loras) { + llama_adapter_lora_weight * lw = lora.first->get_weight(w); + if (lw == nullptr) { + continue; + } + + const float adapter_scale = lora.second; + const float scale = lw->get_scale(lora.first->alpha, adapter_scale); + + ggml_tensor * ab_cur = ggml_mul_mat( + ctx0, lw->b, + ggml_mul_mat(ctx0, lw->a, cur) + ); + + ab_cur = ggml_scale(ctx0, ab_cur, scale); + res = ggml_add(ctx0, res, ab_cur); + } + + return res; +} + +ggml_tensor * llm_graph_context::build_lora_mm_id( + ggml_tensor * w, // ggml_tensor * as + ggml_tensor * cur, // ggml_tensor * b + ggml_tensor * ids) const { + ggml_tensor * res = ggml_mul_mat_id(ctx0, w, cur, ids); + for (const auto & lora : *loras) { + llama_adapter_lora_weight * lw = lora.first->get_weight(w); + if (lw == nullptr) { + continue; + } + + const float alpha = lora.first->alpha; + const float rank = (float) lw->b->ne[0]; + const float scale = alpha ? lora.second * alpha / rank : lora.second; + + ggml_tensor * ab_cur = ggml_mul_mat_id( + ctx0, lw->b, + ggml_mul_mat_id(ctx0, lw->a, cur, ids), + ids + ); + + ab_cur = ggml_scale(ctx0, ab_cur, scale); + res = ggml_add(ctx0, res, ab_cur); + } + + return res; +} + +ggml_tensor * llm_graph_context::build_norm( + ggml_tensor * cur, + ggml_tensor * mw, + ggml_tensor * mb, + llm_norm_type type, + int il) const { + switch (type) { + case LLM_NORM: cur = ggml_norm (ctx0, cur, hparams.f_norm_eps); break; + case LLM_NORM_RMS: cur = ggml_rms_norm(ctx0, cur, hparams.f_norm_rms_eps); break; + case LLM_NORM_GROUP: + { + cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, cur->ne[1]); + cur = ggml_group_norm(ctx0, cur, hparams.n_norm_groups, hparams.f_norm_group_eps); + cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[2]); + } break; + } + + if (mw || mb) { + cb(cur, "norm", il); + } + + if (mw) { + cur = ggml_mul(ctx0, cur, mw); + if (mb) { + cb(cur, "norm_w", il); + } + } + + if (mb) { + cur = ggml_add(ctx0, cur, mb); + } + + return cur; +} + +ggml_tensor * llm_graph_context::build_ffn( + ggml_tensor * cur, + ggml_tensor * up, + ggml_tensor * up_b, + ggml_tensor * up_s, + ggml_tensor * gate, + ggml_tensor * gate_b, + ggml_tensor * gate_s, + ggml_tensor * down, + ggml_tensor * down_b, + ggml_tensor * down_s, + ggml_tensor * act_scales, + llm_ffn_op_type type_op, + llm_ffn_gate_type type_gate, + int il) const { + ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur; + cb(tmp, "ffn_up", il); + + if (up_b) { + tmp = ggml_add(ctx0, tmp, up_b); + cb(tmp, "ffn_up_b", il); + } + + if (up_s) { + tmp = ggml_mul(ctx0, tmp, up_s); + cb(tmp, "ffn_up_s", il); + } + + if (gate) { + switch (type_gate) { + case LLM_FFN_SEQ: + { + cur = build_lora_mm(gate, tmp); + cb(cur, "ffn_gate", il); + } break; + case LLM_FFN_PAR: + { + cur = build_lora_mm(gate, cur); + cb(cur, "ffn_gate", il); + } break; + } + + if (gate_b) { + cur = ggml_add(ctx0, cur, gate_b); + cb(cur, "ffn_gate_b", il); + } + + if (gate_s) { + cur = ggml_mul(ctx0, cur, gate_s); + cb(cur, "ffn_gate_s", il); + } + + } else { + cur = tmp; + } + + switch (type_op) { + case LLM_FFN_SILU: + { + cur = ggml_silu(ctx0, cur); + cb(cur, "ffn_silu", il); + } break; + case LLM_FFN_GELU: + { + cur = ggml_gelu(ctx0, cur); + cb(cur, "ffn_gelu", il); + if (act_scales != NULL) { + cur = ggml_div(ctx0, cur, act_scales); + cb(cur, "ffn_act", il); + } + } break; + case LLM_FFN_RELU: + { + cur = ggml_relu(ctx0, cur); + cb(cur, "ffn_relu", il); + } break; + case LLM_FFN_RELU_SQR: + { + cur = ggml_relu(ctx0, cur); + cb(cur, "ffn_relu", il); + + cur = ggml_sqr(ctx0, cur); + cb(cur, "ffn_sqr(relu)", il); + } break; + case LLM_FFN_SWIGLU: + { + // Project to 4h. If using swiglu double the output width, see https://arxiv.org/pdf/2002.05202.pdf + int64_t split_point = cur->ne[0] / 2; + ggml_tensor * x0 = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, split_point, cur->ne[1], cur->nb[1], 0)); + ggml_tensor * x1 = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, split_point, cur->ne[1], cur->nb[1], split_point * ggml_element_size(cur))); + + x0 = ggml_silu(ctx0, x0); + cb(cur, "ffn_silu", il); + + cur = ggml_mul(ctx0, x0, x1); + cb(cur, "ffn_mul", il); + } break; + } + + if (type_gate == LLM_FFN_PAR) { + cur = ggml_mul(ctx0, cur, tmp); + cb(cur, "ffn_gate_par", il); + } + + if (down) { + cur = build_lora_mm(down, cur); + } + + if (down_b) { + cb(cur, "ffn_down", il); + } + + if (down_b) { + cur = ggml_add(ctx0, cur, down_b); + } + + if (down_s) { + cur = ggml_mul(ctx0, cur, down_s); + cb(cur, "ffn_down_s", il); + } + + return cur; +} + +ggml_tensor * llm_graph_context::build_moe_ffn( + ggml_tensor * cur, + ggml_tensor * gate_inp, + ggml_tensor * up_exps, + ggml_tensor * gate_exps, + ggml_tensor * down_exps, + ggml_tensor * exp_probs_b, + int64_t n_expert, + int64_t n_expert_used, + llm_ffn_op_type type_op, + bool norm_w, + bool scale_w, + float w_scale, + llama_expert_gating_func_type gating_op, + int il) const { + int64_t n_embd = cur->ne[0]; + int64_t n_tokens = cur->ne[1]; + + ggml_tensor * logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens] + cb(logits, "ffn_moe_logits", il); + + ggml_tensor * probs = nullptr; + switch (gating_op) { + case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: + { + probs = ggml_soft_max(ctx0, logits); // [n_expert, n_tokens] + } break; + case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: + { + probs = ggml_sigmoid(ctx0, logits); // [n_expert, n_tokens] + } break; + default: + GGML_ABORT("fatal error"); + } + cb(probs, "ffn_moe_probs", il); + + // add experts selection bias - introduced in DeepSeek V3 + // leave probs unbiased as it's later used to get expert weights + ggml_tensor * selection_probs = probs; + if (exp_probs_b != nullptr) { + selection_probs = ggml_add(ctx0, probs, exp_probs_b); + cb(selection_probs, "ffn_moe_probs_biased", il); + } + + // select experts + ggml_tensor * selected_experts = ggml_top_k(ctx0, selection_probs, n_expert_used); // [n_expert_used, n_tokens] + cb(selected_experts->src[0], "ffn_moe_argsort", il); + cb(selected_experts, "ffn_moe_topk", il); + + ggml_tensor * weights = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected_experts); // [1, n_expert_used, n_tokens] + cb(weights, "ffn_moe_weights", il); + + if (norm_w) { + weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens); + + ggml_tensor * weights_sum = ggml_sum_rows(ctx0, weights); // [1, n_tokens] + cb(weights_sum, "ffn_moe_weights_sum", il); + + weights = ggml_div(ctx0, weights, weights_sum); // [n_expert_used, n_tokens] + cb(weights, "ffn_moe_weights_norm", il); + + weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens); + } + if (scale_w) { + weights = ggml_scale(ctx0, weights, w_scale); + cb(weights, "ffn_moe_weights_scaled", il); + } + + cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens); + ggml_tensor * up = build_lora_mm_id(up_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens] + cb(up, "ffn_moe_up", il); + + ggml_tensor * gate = build_lora_mm_id(gate_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens] + cb(gate, "ffn_moe_gate", il); + + switch (type_op) { + case LLM_FFN_SILU: + { + gate = ggml_silu(ctx0, gate); + cb(gate, "ffn_moe_silu", il); + } break; + case LLM_FFN_GELU: + { + gate = ggml_gelu(ctx0, gate); + cb(gate, "ffn_moe_gelu", il); + } break; + default: + GGML_ABORT("fatal error"); + } + + ggml_tensor * par = ggml_mul(ctx0, up, gate); // [n_ff, n_expert_used, n_tokens] + cb(par, "ffn_moe_gate_par", il); + + ggml_tensor * experts = build_lora_mm_id(down_exps, par, selected_experts); // [n_embd, n_expert_used, n_tokens] + cb(experts, "ffn_moe_down", il); + + experts = ggml_mul(ctx0, experts, weights); + + // aggregate experts + ggml_tensor * moe_out = nullptr; + for (int i = 0; i < n_expert_used; ++i) { + ggml_tensor * cur_expert = ggml_view_2d(ctx0, experts, n_embd, n_tokens, + experts->nb[2], i*experts->nb[1]); + + if (i == 0) { + moe_out = cur_expert; + } else { + moe_out = ggml_add(ctx0, moe_out, cur_expert); + } + } + + if (n_expert_used == 1) { + // avoid returning a non-contiguous tensor + moe_out = ggml_cont(ctx0, moe_out); + } + + return moe_out; +} + +// input embeddings with optional lora +ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { + const int64_t n_embd = hparams.n_embd; + + auto inp = std::make_unique(); + + ggml_tensor * cur = nullptr; + + if (ubatch.token) { + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens); + //cb(inp->tokens, "inp_tokens", -1); + ggml_set_input(inp->tokens); + + cur = ggml_get_rows(ctx0, tok_embd, inp->tokens); + + // apply lora for embedding tokens if needed + for (const auto & lora : *loras) { + llama_adapter_lora_weight * lw = lora.first->get_weight(tok_embd); + if (lw == nullptr) { + continue; + } + + const float adapter_scale = lora.second; + const float scale = lw->get_scale(lora.first->alpha, adapter_scale); + + ggml_tensor * inpL_delta = ggml_scale(ctx0, ggml_mul_mat( + ctx0, lw->b, // non-transposed lora_b + ggml_get_rows(ctx0, lw->a, inp->tokens) + ), scale); + + cur = ggml_add(ctx0, cur, inpL_delta); + } + } else { + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, ubatch.n_tokens); + ggml_set_input(inp->embd); + + cur = inp->embd; + } + + // For Granite architecture + if (hparams.f_embedding_scale != 0.0f) { + cur = ggml_scale(ctx0, cur, hparams.f_embedding_scale); + } + + cb(cur, "inp_embd", -1); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_inp_pos() const { + auto inp = std::make_unique(n_pos_per_token()); + + auto & cur = inp->pos; + + cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens*n_pos_per_token()); + ggml_set_input(cur); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_inp_out_ids() const { + auto inp = std::make_unique(hparams, cparams, n_outputs); + + auto & cur = inp->out_ids; + + cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs); + ggml_set_input(cur); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_inp_mean() const { + auto inp = std::make_unique(cparams); + + auto & cur = inp->mean; + + cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens); + ggml_set_input(cur); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_inp_cls() const { + auto inp = std::make_unique(cparams); + + auto & cur = inp->cls; + + cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(cur); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_inp_s_copy() const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + auto inp = std::make_unique(kv_self); + + const auto n_kv = kv_self->n; + + auto & cur = inp->s_copy; + + cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_kv); + ggml_set_input(cur); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_inp_s_mask() const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + auto inp = std::make_unique(kv_self); + + const auto n_kv = kv_self->n; + + auto & cur = inp->s_mask; + + cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_kv); + ggml_set_input(cur); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_inp_cross_embd() const { + auto inp = std::make_unique(cross); + + auto & cur = inp->cross_embd; + + // if we have the output embeddings from the encoder, use them directly + // TODO: needs more work to be correct, for now just use the tensor shape + //if (cross->t_embd) { + // cur = ggml_view_tensor(ctx0, cross->t_embd); + + // return cur; + //} + + const auto n_embd = !cross->v_embd.empty() ? cross->n_embd : hparams.n_embd; + const auto n_enc = !cross->v_embd.empty() ? cross->n_enc : hparams.n_ctx_train; + + cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_enc); + ggml_set_input(cur); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_inp_pos_bucket_enc() const { + auto inp = std::make_unique(hparams); + + auto & cur = inp->pos_bucket; + + cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_tokens, n_tokens); + ggml_set_input(cur); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_inp_pos_bucket_dec() const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + auto inp = std::make_unique(hparams, kv_self); + + const auto n_kv = kv_self->n; + + auto & cur = inp->pos_bucket; + + cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, n_tokens); + ggml_set_input(cur); + + res->add_input(std::move(inp)); + + return cur; +} + +ggml_tensor * llm_graph_context::build_pos_bias(ggml_tensor * pos_bucket, ggml_tensor * attn_rel_b) const { + ggml_tensor * pos_bucket_1d = ggml_reshape_1d(ctx0, pos_bucket, pos_bucket->ne[0] * pos_bucket->ne[1]); + cb(pos_bucket_1d, "pos_bucket_1d", -1); + + ggml_tensor * pos_bias = ggml_get_rows(ctx0, attn_rel_b, pos_bucket_1d); + + pos_bias = ggml_reshape_3d(ctx0, pos_bias, pos_bias->ne[0], pos_bucket->ne[0], pos_bucket->ne[1]); + pos_bias = ggml_permute (ctx0, pos_bias, 2, 0, 1, 3); + pos_bias = ggml_cont (ctx0, pos_bias); + + cb(pos_bias, "pos_bias", -1); + + return pos_bias; +} + +ggml_tensor * llm_graph_context::build_attn_mha( + ggml_cgraph * gf, + ggml_tensor * q, + ggml_tensor * k, + ggml_tensor * v, + ggml_tensor * kq_b, + ggml_tensor * kq_mask, + bool v_trans, + float kq_scale) const { + //const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); + //const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(il); + + //const int64_t n_head = hparams.n_head(il); + //const int64_t n_head_kv = hparams.n_head_kv(il); + + //const auto & n_embd_head_k = hparams.n_embd_head_k; + //const auto & n_embd_head_v = hparams.n_embd_head_v; + + const auto n_embd_head_v = v_trans ? v->ne[1] : v->ne[0]; + + const auto n_tokens = q->ne[1]; + const auto n_head = q->ne[2]; + const auto n_kv = k->ne[1]; + + ggml_tensor * cur; + + // TODO: replace hardcoded padding with ggml-provided padding + if (cparams.flash_attn && (n_kv % 256 == 0) && kq_b == nullptr) { + GGML_ASSERT(kq_b == nullptr && "Flash attention does not support KQ bias yet"); + + if (v_trans) { + v = ggml_transpose(ctx0, v); + } + + cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias, + hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f); + + ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); + + cur = ggml_reshape_2d(ctx0, cur, n_embd_head_v*n_head, n_tokens); + } else { + ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); + + // note: this op tends to require high floating point range + // while for some models F16 is enough, for others it is not, so we default to F32 here + ggml_mul_mat_set_prec(kq, GGML_PREC_F32); + + if (arch == LLM_ARCH_GROK) { + // need to do the following: + // multiply by attn_output_multiplyer of 0.08838834764831845 + // and then : + // kq = 30 * tanh(kq / 30) + // before the softmax below + + kq = ggml_tanh(ctx0, ggml_scale(ctx0, kq, 0.08838834764831845f/30.0f)); + kq = ggml_scale(ctx0, kq, 30); + } + + if (hparams.attn_soft_cap) { + kq = ggml_scale(ctx0, kq, 1.0f / hparams.f_attn_logit_softcapping); + kq = ggml_tanh (ctx0, kq); + kq = ggml_scale(ctx0, kq, hparams.f_attn_logit_softcapping); + } + + if (kq_b) { + kq = ggml_add(ctx0, kq, kq_b); + } + + kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias); + + if (!v_trans) { + // note: avoid this branch + v = ggml_cont(ctx0, ggml_transpose(ctx0, v)); + } + + ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq); + + ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3); + + cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_head_v*n_head, n_tokens); + + if (!cparams.offload_kqv) { + // all nodes between the KV store and the attention output are run on the CPU + ggml_backend_sched_set_tensor_backend(sched, cur, backend_cpu); + } + } + + ggml_build_forward_expand(gf, cur); + + return cur; +} + +llm_graph_input_attn_no_cache * llm_graph_context::build_attn_inp_no_cache() const { + auto inp = std::make_unique(hparams, cparams); + + // note: there is no KV cache, so the number of KV values is equal to the number of tokens in the batch + inp->kq_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); + //cb(inp_kq_mask, "KQ_mask", -1); + ggml_set_input(inp->kq_mask); + + inp->kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->kq_mask, GGML_TYPE_F16) : inp->kq_mask; + + return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); +} + +ggml_tensor * llm_graph_context::build_attn( + llm_graph_input_attn_no_cache * inp, + ggml_cgraph * gf, + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * q_cur, + ggml_tensor * k_cur, + ggml_tensor * v_cur, + ggml_tensor * kq_b, + float kq_scale, + int il) const { + GGML_UNUSED(n_tokens); + + // these nodes are added to the graph together so that they are not reordered + // by doing so, the number of splits in the graph is reduced + ggml_build_forward_expand(gf, q_cur); + ggml_build_forward_expand(gf, k_cur); + ggml_build_forward_expand(gf, v_cur); + + const auto & kq_mask = inp->get_kq_mask(); + + ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3); + //cb(q, "q", il); + + ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3); + //cb(k, "k", il); + + ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3); + //cb(k, "v", il); + + ggml_tensor * cur = build_attn_mha(gf, q, k, v, kq_b, kq_mask, false, kq_scale); + + cb(cur, "kqv_out", il); + + if (wo) { + cur = build_lora_mm(wo, cur); + } + + if (wo_b) { + //cb(cur, "kqv_wo", il); + } + + if (wo_b) { + cur = ggml_add(ctx0, cur, wo_b); + } + + return cur; +} + +llm_graph_input_attn_kv_unified * llm_graph_context::build_attn_inp_kv_unified( + bool causal, + bool swa) const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + auto inp = std::make_unique(hparams, cparams, kv_self); + + const auto n_kv = kv_self->n; + + inp->self_kq_mask = causal + ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)) + : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); + //cb(inp->self_kq_mask, "KQ_mask", -1); + ggml_set_input(inp->self_kq_mask); + + inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask; + + if (swa) { + GGML_ASSERT(hparams.n_swa > 0); + + inp->self_kq_mask_swa = causal + ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)) + : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); + //cb(inp->self_kq_mask_swa, "KQ_mask_swa", -1); + ggml_set_input(inp->self_kq_mask_swa); + + inp->self_kq_mask_swa_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask_swa, GGML_TYPE_F16) : inp->self_kq_mask_swa; + } + + return (llm_graph_input_attn_kv_unified *) res->add_input(std::move(inp)); +} + +ggml_tensor * llm_graph_context::build_attn( + llm_graph_input_attn_kv_unified * inp, + ggml_cgraph * gf, + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * q_cur, + ggml_tensor * k_cur, + ggml_tensor * v_cur, + ggml_tensor * kq_b, + float kq_scale, + int il) const { + // these nodes are added to the graph together so that they are not reordered + // by doing so, the number of splits in the graph is reduced + ggml_build_forward_expand(gf, q_cur); + ggml_build_forward_expand(gf, k_cur); + ggml_build_forward_expand(gf, v_cur); + + const llama_kv_cache_unified * kv_self = static_cast(memory); + const auto & n_ctx = cparams.n_ctx; + + const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); + const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(il); + + const auto n_tokens = q_cur->ne[2]; + + const bool v_trans = !cparams.flash_attn; + + // store to KV cache + { + GGML_ASSERT(!kv_self->recurrent); + + const auto kv_head = kv_self->head; + + GGML_ASSERT(kv_self->size == n_ctx); + + ggml_tensor * k_cache_view = ggml_view_1d(ctx0, kv_self->k_l[il], n_tokens*n_embd_k_gqa, ggml_row_size(kv_self->k_l[il]->type, n_embd_k_gqa)*kv_head); + //cb(k_cache_view, "k_cache_view", il); + + // note: storing RoPE-ed version of K in the KV cache + ggml_build_forward_expand(gf, ggml_cpy(ctx0, k_cur, k_cache_view)); + + assert(v_cur->ne[0] == n_embd_v_gqa && v_cur->ne[1] == n_tokens); + + ggml_tensor * v_cache_view = nullptr; + + if (!v_trans) { + v_cache_view = ggml_view_1d(ctx0, kv_self->v_l[il], n_tokens*n_embd_v_gqa, ggml_row_size(kv_self->v_l[il]->type, n_embd_v_gqa)*kv_head); + } else { + // note: the V cache is transposed when not using flash attention + v_cache_view = ggml_view_2d(ctx0, kv_self->v_l[il], n_tokens, n_embd_v_gqa, + ( n_ctx)*ggml_element_size(kv_self->v_l[il]), + (kv_head)*ggml_element_size(kv_self->v_l[il])); + + v_cur = ggml_transpose(ctx0, v_cur); + } + //cb(v_cache_view, "v_cache_view", il); + + ggml_build_forward_expand(gf, ggml_cpy(ctx0, v_cur, v_cache_view)); + } + + // TODO: improve + bool is_sliding = false; + + switch (arch) { + case LLM_ARCH_COHERE2: + { + const int32_t sliding_window_pattern = 4; + is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); + } break; + case LLM_ARCH_GEMMA2: + { + const int32_t sliding_window_pattern = 2; + is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); + } break; + case LLM_ARCH_GEMMA3: + { + const int32_t sliding_window_pattern = 6; + is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); + } break; + case LLM_ARCH_PHI3: + { + is_sliding = hparams.n_swa > 0; + } break; + default: + { + is_sliding = false; + } + }; + + const auto & kq_mask = is_sliding ? inp->get_kq_mask_swa() : inp->get_kq_mask(); + + const auto n_kv = kv_self->n; + + const int64_t n_head_kv = hparams.n_head_kv(il); + + const auto & n_embd_head_k = hparams.n_embd_head_k; + const auto & n_embd_head_v = hparams.n_embd_head_v; + + ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3); + //cb(q, "q", il); + + ggml_tensor * k = + ggml_view_3d(ctx0, kv_self->k_l[il], + n_embd_head_k, n_kv, n_head_kv, + ggml_row_size(kv_self->k_l[il]->type, n_embd_k_gqa), + ggml_row_size(kv_self->k_l[il]->type, n_embd_head_k), + 0); + //cb(k, "k", il); + + ggml_tensor * v = !v_trans ? + ggml_view_3d(ctx0, kv_self->v_l[il], + n_embd_head_v, n_kv, n_head_kv, + ggml_row_size(kv_self->v_l[il]->type, n_embd_v_gqa), + ggml_row_size(kv_self->v_l[il]->type, n_embd_head_v), + 0) : + ggml_view_3d(ctx0, kv_self->v_l[il], + n_kv, n_embd_head_v, n_head_kv, + ggml_element_size(kv_self->v_l[il])*n_ctx, + ggml_element_size(kv_self->v_l[il])*n_ctx*n_embd_head_v, + 0); + + ggml_tensor * cur = build_attn_mha(gf, q, k, v, kq_b, kq_mask, v_trans, kq_scale); + cb(cur, "kqv_out", il); + + if (wo) { + cur = build_lora_mm(wo, cur); + } + + if (wo_b) { + //cb(cur, "kqv_wo", il); + } + + if (wo_b) { + cur = ggml_add(ctx0, cur, wo_b); + } + + return cur; +} + +llm_graph_input_attn_cross * llm_graph_context::build_attn_inp_cross() const { + auto inp = std::make_unique(cross); + + const int32_t n_enc = !cross->v_embd.empty() ? cross->n_enc : hparams.n_ctx_train; + + inp->cross_kq_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_enc, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); + ggml_set_input(inp->cross_kq_mask); + + inp->cross_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->cross_kq_mask, GGML_TYPE_F16) : inp->cross_kq_mask; + + return (llm_graph_input_attn_cross *) res->add_input(std::move(inp)); +} + +ggml_tensor * llm_graph_context::build_attn( + llm_graph_input_attn_cross * inp, + ggml_cgraph * gf, + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * q_cur, + ggml_tensor * k_cur, + ggml_tensor * v_cur, + ggml_tensor * kq_b, + float kq_scale, + int il) const { + // these nodes are added to the graph together so that they are not reordered + // by doing so, the number of splits in the graph is reduced + ggml_build_forward_expand(gf, q_cur); + ggml_build_forward_expand(gf, k_cur); + ggml_build_forward_expand(gf, v_cur); + + const auto & kq_mask = inp->get_kq_mask_cross(); + + ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3); + //cb(q, "q", il); + + ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3); + //cb(k, "k", il); + + ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3); + //cb(k, "v", il); + + ggml_tensor * cur = build_attn_mha(gf, q, k, v, kq_b, kq_mask, false, kq_scale); + + cb(cur, "kqv_out", il); + + if (wo) { + cur = build_lora_mm(wo, cur); + } + + if (wo_b) { + //cb(cur, "kqv_wo", il); + } + + if (wo_b) { + cur = ggml_add(ctx0, cur, wo_b); + } + + return cur; +} + +ggml_tensor * llm_graph_context::build_copy_mask_state( + ggml_cgraph * gf, + ggml_tensor * s, + ggml_tensor * state_copy, + ggml_tensor * state_mask, + int32_t n_state, + int32_t n_seqs) const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + const auto n_kv = kv_self->n; + const auto kv_head = kv_self->head; + + ggml_tensor * states = ggml_reshape_2d(ctx0, s, n_state, kv_self->size); + + // copy states + // NOTE: assuming the copy destinations are ALL contained between kv_head and kv_head + n_kv + // this shrinks the tensors's ne[1] to n_kv + states = ggml_get_rows(ctx0, states, state_copy); + + // clear states of sequences which are starting at the beginning of this batch + // FIXME: zero-out NANs? + states = ggml_mul(ctx0, states, state_mask); + + // copy states which won't be changed further (between n_seqs and n_kv) + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, + ggml_view_1d(ctx0, states, n_state*(n_kv - n_seqs), (n_seqs )*n_state*ggml_element_size(states)), + ggml_view_1d(ctx0, s, n_state*(n_kv - n_seqs), (kv_head + n_seqs)*n_state*ggml_element_size(s)))); + + // the part of the states that will be used and modified + return ggml_view_2d(ctx0, states, n_state, n_seqs, states->nb[1], 0); +} + +ggml_tensor * llm_graph_context::build_rwkv_token_shift_load( + ggml_cgraph * gf, + ggml_tensor * state_copy, + ggml_tensor * state_mask, + const llama_ubatch & ubatch, + int il) const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + const auto token_shift_count = hparams.token_shift_count; + + const int64_t n_seqs = ubatch.n_seqs; + + ggml_tensor * token_shift_all = kv_self->k_l[il]; + + ggml_tensor * token_shift = build_copy_mask_state( + gf, token_shift_all, state_copy, state_mask, + hparams.n_embd_k_s(), n_seqs); + + token_shift = ggml_reshape_3d(ctx0, token_shift, hparams.n_embd, token_shift_count, n_seqs); + + return token_shift; +} + +ggml_tensor * llm_graph_context::build_rwkv_token_shift_store( + ggml_tensor * token_shift, + const llama_ubatch & ubatch, + int il) const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + const auto token_shift_count = hparams.token_shift_count; + const auto n_embd = hparams.n_embd; + + const int64_t n_seqs = ubatch.n_seqs; + + const auto kv_head = kv_self->head; + + return ggml_cpy( + ctx0, + ggml_view_1d(ctx0, token_shift, n_embd * n_seqs * token_shift_count, 0), + ggml_view_1d(ctx0, kv_self->k_l[il], hparams.n_embd_k_s() * n_seqs, hparams.n_embd_k_s() * kv_head * ggml_element_size(kv_self->k_l[il])) + ); +} + +void llm_graph_context::build_pooling( + ggml_cgraph * gf, + ggml_tensor * cls, + ggml_tensor * cls_b, + ggml_tensor * cls_out, + ggml_tensor * cls_out_b) const { + if (!cparams.embeddings) { + return; + } + + ggml_tensor * inp = res->t_embd; + + //// find result_norm tensor for input + //for (int i = ggml_graph_n_nodes(gf) - 1; i >= 0; --i) { + // inp = ggml_graph_node(gf, i); + // if (strcmp(inp->name, "result_norm") == 0 || strcmp(inp->name, "result_embd") == 0) { + // break; + // } + + // inp = nullptr; + //} + + GGML_ASSERT(inp != nullptr && "missing result_norm/result_embd tensor"); + + ggml_tensor * cur; + + switch (pooling_type) { + case LLAMA_POOLING_TYPE_NONE: + { + cur = inp; + } break; + case LLAMA_POOLING_TYPE_MEAN: + { + ggml_tensor * inp_mean = build_inp_mean(); + cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, inp)), inp_mean); + } break; + case LLAMA_POOLING_TYPE_CLS: + case LLAMA_POOLING_TYPE_LAST: + { + ggml_tensor * inp_cls = build_inp_cls(); + cur = ggml_get_rows(ctx0, inp, inp_cls); + } break; + case LLAMA_POOLING_TYPE_RANK: + { + ggml_tensor * inp_cls = build_inp_cls(); + inp = ggml_get_rows(ctx0, inp, inp_cls); + + // classification head + // https://github.com/huggingface/transformers/blob/5af7d41e49bbfc8319f462eb45253dcb3863dfb7/src/transformers/models/roberta/modeling_roberta.py#L1566 + GGML_ASSERT(cls != nullptr); + GGML_ASSERT(cls_b != nullptr); + + cur = ggml_add (ctx0, ggml_mul_mat(ctx0, cls, inp), cls_b); + cur = ggml_tanh(ctx0, cur); + + // some models don't have `cls_out`, for example: https://huggingface.co/jinaai/jina-reranker-v1-tiny-en + // https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/blob/cb5347e43979c3084a890e3f99491952603ae1b7/modeling_bert.py#L884-L896 + if (cls_out) { + GGML_ASSERT(cls_out_b != nullptr); + + cur = ggml_add (ctx0, ggml_mul_mat(ctx0, cls_out, cur), cls_out_b); + } + } break; + default: + { + GGML_ABORT("unknown pooling type"); + } + } + + cb(cur, "result_embd_pooled", -1); + res->t_embd_pooled = cur; + + ggml_build_forward_expand(gf, cur); +} + diff --git a/src/llama-graph.h b/src/llama-graph.h new file mode 100644 index 0000000000..b7a66d1898 --- /dev/null +++ b/src/llama-graph.h @@ -0,0 +1,576 @@ +#pragma once + +#include "llama-arch.h" +#include "llama-hparams.h" +#include "llama-adapter.h" + +#include +#include +#include +#include +#include + +struct ggml_cgraph; +struct ggml_context; +struct ggml_tensor; + +struct llama_ubatch; +struct llama_cparams; + +class llama_memory_i; +class llama_kv_cache_unified; + +// certain models (typically multi-modal) can produce different types of graphs +enum llm_graph_type { + LLM_GRAPH_TYPE_DEFAULT, + LLM_GRAPH_TYPE_ENCODER, + LLM_GRAPH_TYPE_DECODER, +}; + +enum llm_ffn_op_type { + LLM_FFN_SILU, + LLM_FFN_GELU, + LLM_FFN_RELU, + LLM_FFN_RELU_SQR, + LLM_FFN_SWIGLU, +}; + +enum llm_ffn_gate_type { + LLM_FFN_SEQ, + LLM_FFN_PAR, // ffn_gate is parallel to ffn_up +}; + +enum llm_norm_type { + LLM_NORM, + LLM_NORM_RMS, + LLM_NORM_GROUP, +}; + +// TODO: tmp - need something better to pass the data from the encoder to the decoder +struct llama_cross { + // the output embeddings from the encoder as a ggml tensor + // TODO: this needs more work to be correct, for now copy the embeddings data to host memory + // ref: https://github.com/ggml-org/llama.cpp/pull/11213#discussion_r1969892524 + //ggml_tensor * t_embd = nullptr; + + int64_t n_embd = 0; + int64_t n_enc = 0; + + // embeddings data copied to host memory (tmp) + std::vector v_embd; + + // needed to construct the cross-attention mask in the decoder + std::vector> seq_ids_enc; +}; + +// +// llm_graph_input +// + +class llm_graph_input_i { +public: + virtual ~llm_graph_input_i() = default; + + virtual void set_input(const llama_ubatch * ubatch) = 0; +}; + +using llm_graph_input_ptr = std::unique_ptr; + + +class llm_graph_input_embd : public llm_graph_input_i { +public: + llm_graph_input_embd() = default; + virtual ~llm_graph_input_embd() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * tokens = nullptr; // I32 [n_batch] + ggml_tensor * embd = nullptr; // F32 [n_embd, n_batch] +}; + +class llm_graph_input_pos : public llm_graph_input_i { +public: + llm_graph_input_pos(int64_t n_pos_per_token) : n_pos_per_token(n_pos_per_token) {} + virtual ~llm_graph_input_pos() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * pos = nullptr; // I32 [n_batch] + + const int64_t n_pos_per_token = 1; +}; + +class llm_graph_input_pos_bucket : public llm_graph_input_i { +public: + llm_graph_input_pos_bucket(const llama_hparams & hparams) : hparams(hparams) {} + virtual ~llm_graph_input_pos_bucket() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * pos_bucket = nullptr; // I32 [n_batch, n_batch] + + const llama_hparams & hparams; +}; + +class llm_graph_input_pos_bucket_kv : public llm_graph_input_i { +public: + llm_graph_input_pos_bucket_kv( + const llama_hparams & hparams, + const llama_kv_cache_unified * kv_self) : hparams(hparams), kv_self(kv_self) {} + virtual ~llm_graph_input_pos_bucket_kv() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * pos_bucket = nullptr; // I32 [n_kv, n_batch] + + const llama_hparams & hparams; + const llama_kv_cache_unified * kv_self; +}; + +class llm_graph_input_out_ids : public llm_graph_input_i { +public: + llm_graph_input_out_ids( + const llama_hparams & hparams, + const llama_cparams & cparams, + int32_t n_outputs) : hparams(hparams), cparams(cparams), n_outputs(n_outputs) {} + virtual ~llm_graph_input_out_ids() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * out_ids; // I32 [n_outputs] + + const llama_hparams & hparams; + const llama_cparams & cparams; + + const int32_t n_outputs; +}; + +class llm_graph_input_mean : public llm_graph_input_i { +public: + llm_graph_input_mean(const llama_cparams & cparams) : cparams(cparams) {} + virtual ~llm_graph_input_mean() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * mean; // F32 [n_batch, n_batch] + + const llama_cparams & cparams; +}; + +class llm_graph_input_cls : public llm_graph_input_i { +public: + llm_graph_input_cls(const llama_cparams & cparams) : cparams(cparams) {} + virtual ~llm_graph_input_cls() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * cls; // I32 [n_batch] + + const llama_cparams & cparams; +}; + +class llm_graph_input_s_copy : public llm_graph_input_i { +public: + llm_graph_input_s_copy(const llama_kv_cache_unified * kv_self) : kv_self(kv_self) {} + virtual ~llm_graph_input_s_copy() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * s_copy; // I32 [kv_size] + + const llama_kv_cache_unified * kv_self; +}; + +class llm_graph_input_s_mask : public llm_graph_input_i { +public: + llm_graph_input_s_mask(const llama_kv_cache_unified * kv_self) : kv_self(kv_self) {} + virtual ~llm_graph_input_s_mask() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * s_mask; // F32 [1, n_kv] + + const llama_kv_cache_unified * kv_self; +}; + +class llm_graph_input_cross_embd : public llm_graph_input_i { +public: + llm_graph_input_cross_embd( + const llama_cross * cross) : cross(cross) {} + virtual ~llm_graph_input_cross_embd() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * cross_embd; // F32 [n_embd, n_outputs_enc] + + const llama_cross * cross; +}; + +class llm_graph_input_attn_no_cache : public llm_graph_input_i { +public: + llm_graph_input_attn_no_cache(const llama_hparams & hparams, const llama_cparams & cparams) : + hparams(hparams), + cparams(cparams) { + } + ~llm_graph_input_attn_no_cache() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * get_kq_mask() const { return kq_mask_cnv; } + + ggml_tensor * kq_mask = nullptr; // F32 [n_tokens, n_batch] + ggml_tensor * kq_mask_cnv = nullptr; // [n_tokens, n_batch] + + const llama_hparams & hparams; + const llama_cparams & cparams; +}; + +class llm_graph_input_attn_kv_unified : public llm_graph_input_i { +public: + llm_graph_input_attn_kv_unified( + const llama_hparams & hparams, + const llama_cparams & cparams, + const llama_kv_cache_unified * kv_self) : + hparams(hparams), + cparams(cparams), + kv_self(kv_self) { + } + ~llm_graph_input_attn_kv_unified() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * get_kq_mask() const { return self_kq_mask_cnv; } + ggml_tensor * get_kq_mask_swa() const { return self_kq_mask_swa_cnv; } + + ggml_tensor * self_kq_mask = nullptr; // F32 [n_kv, n_batch] + ggml_tensor * self_kq_mask_cnv = nullptr; // [n_kv, n_batch] + ggml_tensor * self_kq_mask_swa = nullptr; // F32 [n_kv, n_batch] + ggml_tensor * self_kq_mask_swa_cnv = nullptr; // [n_kv, n_batch] + + const llama_hparams & hparams; + const llama_cparams & cparams; + + const llama_kv_cache_unified * kv_self; +}; + +class llm_graph_input_attn_cross : public llm_graph_input_i { +public: + llm_graph_input_attn_cross(const llama_cross * cross) : cross(cross) {} + ~llm_graph_input_attn_cross() = default; + + void set_input(const llama_ubatch * ubatch) override; + + ggml_tensor * get_kq_mask_cross() const { return cross_kq_mask_cnv; } + + ggml_tensor * cross_kq_mask = nullptr; // F32 [n_outputs_enc, n_batch] + ggml_tensor * cross_kq_mask_cnv = nullptr; // F32 [n_outputs_enc, n_batch] + + const llama_cross * cross = nullptr; +}; + +// +// llm_graph_result +// + +// these objects deliver the result from the graph build process back to the llama_context +// note that the input tensors created for the graph are referenced here - the goal is to be able to populate their +// specific data, by calling the set_inputs() method +// along with the input tensors, the object also provides commonly used outputs tensors, such as logits, embeddings, etc. +// these are used by the llama_context to extact the relevant data, based on the compute parameters + +class llm_graph_result_i { +public: + virtual ~llm_graph_result_i() = default; + + virtual ggml_tensor * get_logits() = 0; + virtual ggml_tensor * get_embd() = 0; + virtual ggml_tensor * get_embd_pooled() = 0; + + virtual void set_inputs(const llama_ubatch * ubatch) = 0; +}; + +using llm_graph_result_ptr = std::unique_ptr; + + +class llm_graph_result : public llm_graph_result_i { +public: + virtual ~llm_graph_result() = default; + + ggml_tensor * get_logits() override { return t_logits; } + ggml_tensor * get_embd() override { return t_embd; } + ggml_tensor * get_embd_pooled() override { return t_embd_pooled; } + + void set_inputs(const llama_ubatch * ubatch) override { + for (auto & input : inputs) { + input->set_input(ubatch); + } + } + + llm_graph_input_i * add_input(llm_graph_input_ptr input) { + inputs.emplace_back(std::move(input)); + return inputs.back().get(); + } + + // important graph nodes + ggml_tensor * t_logits = nullptr; + ggml_tensor * t_embd = nullptr; + ggml_tensor * t_embd_pooled = nullptr; + + std::vector inputs; +}; + +// +// llm_graph_context +// + +// callback that allows us to apply custom logic to each tensor (e.g. ggml-alloc, offloading, etc.) +using llm_graph_cb = std::function; + +struct llm_graph_params { + ggml_context * ctx; + + const llm_arch arch; + + const llama_hparams & hparams; + const llama_cparams & cparams; + const llama_ubatch & ubatch; + + ggml_backend_sched * sched; + ggml_backend * backend_cpu; + + const llama_adapter_cvec * cvec; + const llama_adapter_loras * loras; + const llama_memory_i * memory; + const llama_cross * cross; + + int32_t n_outputs; + + const llm_graph_cb & cb; +}; + +struct llm_graph_context { + const llm_arch arch; + + const llama_hparams & hparams; + const llama_cparams & cparams; + const llama_ubatch & ubatch; + + const int64_t n_embd; + const int64_t n_layer; + const int64_t n_rot; + const int64_t n_ctx; // user-specified context size (can be different from n_ctx_train) + const int64_t n_ctx_per_seq; + const int64_t n_head; + const int64_t n_head_kv; + const int64_t n_embd_head_k; + const int64_t n_embd_k_gqa; + const int64_t n_embd_head_v; + const int64_t n_embd_v_gqa; + const int64_t n_expert; + const int64_t n_expert_used; + + const float freq_base; + const float freq_scale; + const float ext_factor; + const float attn_factor; + const float beta_fast; + const float beta_slow; + const float norm_eps; + const float norm_rms_eps; + + const int32_t n_tokens; + const int32_t n_outputs; + const int32_t n_ctx_orig; // yarn + + const enum llama_pooling_type pooling_type; + const enum llama_rope_type rope_type; + + ggml_context * ctx0 = nullptr; + + ggml_backend_sched * sched; + + ggml_backend * backend_cpu; // TODO: needed by build_attn_mha, figure out a way to remove? + + const llama_adapter_cvec * cvec; + const llama_adapter_loras * loras; + const llama_memory_i * memory; + const llama_cross * cross; + + const llm_graph_cb & cb_func; + + std::unique_ptr res; + + llm_graph_context(const llm_graph_params & params); + + int64_t n_pos_per_token() const; + + void cb(ggml_tensor * cur, const char * name, int il) const; + + // + // common + // + + ggml_tensor * build_cvec( + ggml_tensor * cur, + int il) const; + + // do mat_mul, while optionally apply lora + ggml_tensor * build_lora_mm( + ggml_tensor * w, + ggml_tensor * cur) const; + + // do mat_mul_id, while optionally apply lora + ggml_tensor * build_lora_mm_id( + ggml_tensor * w, // ggml_tensor * as + ggml_tensor * cur, // ggml_tensor * b + ggml_tensor * ids) const; + + ggml_tensor * build_norm( + ggml_tensor * cur, + ggml_tensor * mw, + ggml_tensor * mb, + llm_norm_type type, + int il) const; + + ggml_tensor * build_ffn( + ggml_tensor * cur, + ggml_tensor * up, + ggml_tensor * up_b, + ggml_tensor * up_s, + ggml_tensor * gate, + ggml_tensor * gate_b, + ggml_tensor * gate_s, + ggml_tensor * down, + ggml_tensor * down_b, + ggml_tensor * down_s, + ggml_tensor * act_scales, + llm_ffn_op_type type_op, + llm_ffn_gate_type type_gate, + int il) const; + + ggml_tensor * build_moe_ffn( + ggml_tensor * cur, + ggml_tensor * gate_inp, + ggml_tensor * up_exps, + ggml_tensor * gate_exps, + ggml_tensor * down_exps, + ggml_tensor * exp_probs_b, + int64_t n_expert, + int64_t n_expert_used, + llm_ffn_op_type type_op, + bool norm_w, + bool scale_w, + float w_scale, + llama_expert_gating_func_type gating_op, + int il) const; + + // + // inputs + // + + ggml_tensor * build_inp_embd(ggml_tensor * tok_embd) const; + ggml_tensor * build_inp_pos() const; + ggml_tensor * build_inp_out_ids() const; + ggml_tensor * build_inp_mean() const; + ggml_tensor * build_inp_cls() const; + ggml_tensor * build_inp_s_copy() const; + ggml_tensor * build_inp_s_mask() const; + + ggml_tensor * build_inp_cross_embd() const; + ggml_tensor * build_inp_pos_bucket_enc() const; + ggml_tensor * build_inp_pos_bucket_dec() const; + ggml_tensor * build_pos_bias(ggml_tensor * pos_bucket, ggml_tensor * attn_rel_b) const; + + // + // attention + // + + ggml_tensor * build_attn_mha( + ggml_cgraph * gf, + ggml_tensor * q, + ggml_tensor * k, + ggml_tensor * v, + ggml_tensor * kq_b, + ggml_tensor * kq_mask, + bool v_trans, + float kq_scale) const; + + llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const; + + ggml_tensor * build_attn( + llm_graph_input_attn_no_cache * inp, + ggml_cgraph * gf, + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * q_cur, + ggml_tensor * k_cur, + ggml_tensor * v_cur, + ggml_tensor * kq_b, + float kq_scale, + int il) const; + + llm_graph_input_attn_kv_unified * build_attn_inp_kv_unified( + bool causal, + bool swa) const; + + ggml_tensor * build_attn( + llm_graph_input_attn_kv_unified * inp, + ggml_cgraph * gf, + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * q_cur, + ggml_tensor * k_cur, + ggml_tensor * v_cur, + ggml_tensor * kq_b, + float kq_scale, + int il) const; + + llm_graph_input_attn_cross * build_attn_inp_cross() const; + + ggml_tensor * build_attn( + llm_graph_input_attn_cross * inp, + ggml_cgraph * gf, + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * q_cur, + ggml_tensor * k_cur, + ggml_tensor * v_cur, + ggml_tensor * kq_b, + float kq_scale, + int il) const; + + // + // recurrent + // + + ggml_tensor * build_copy_mask_state( + ggml_cgraph * gf, + ggml_tensor * s, + ggml_tensor * state_copy, + ggml_tensor * state_mask, + int32_t n_state, + int32_t n_seqs) const; + + ggml_tensor * build_rwkv_token_shift_load( + ggml_cgraph * gf, + ggml_tensor * state_copy, + ggml_tensor * state_mask, + const llama_ubatch & ubatch, + int il) const; + + ggml_tensor * build_rwkv_token_shift_store( + ggml_tensor * token_shift, + const llama_ubatch & ubatch, + int il) const; + + // + // pooling + // + + void build_pooling( + ggml_cgraph * gf, + ggml_tensor * cls, + ggml_tensor * cls_b, + ggml_tensor * cls_out, + ggml_tensor * cls_out_b) const; +}; diff --git a/src/llama-io.cpp b/src/llama-io.cpp new file mode 100644 index 0000000000..7ad70d1633 --- /dev/null +++ b/src/llama-io.cpp @@ -0,0 +1,15 @@ +#include "llama-io.h" + +void llama_io_write_i::write_string(const std::string & str) { + uint32_t str_size = str.size(); + + write(&str_size, sizeof(str_size)); + write(str.data(), str_size); +} + +void llama_io_read_i::read_string(std::string & str) { + uint32_t str_size; + read_to(&str_size, sizeof(str_size)); + + str.assign((const char *) read(str_size), str_size); +} diff --git a/src/llama-io.h b/src/llama-io.h new file mode 100644 index 0000000000..ce9216b83b --- /dev/null +++ b/src/llama-io.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +struct ggml_tensor; + +class llama_io_write_i { +public: + llama_io_write_i() = default; + virtual ~llama_io_write_i() = default; + + virtual void write(const void * src, size_t size) = 0; + virtual void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) = 0; + + // bytes written so far + virtual size_t n_bytes() = 0; + + void write_string(const std::string & str); +}; + +class llama_io_read_i { +public: + llama_io_read_i() = default; + virtual ~llama_io_read_i() = default; + + virtual const uint8_t * read(size_t size) = 0; + virtual void read_to(void * dst, size_t size) = 0; + + // bytes read so far + virtual size_t n_bytes() = 0; + + void read_string(std::string & str); +}; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index feffdf0de5..14c8933b4d 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -6,86 +6,92 @@ #include "llama-model.h" #include +#include #include #include +#include static const llama_kv_cache_slot_info llama_kv_cache_slot_info_failed{false}; -uint32_t llama_kv_cache_get_padding(const struct llama_cparams & cparams) { - // the FA kernels require padding to avoid extra runtime boundary checks - return cparams.flash_attn ? 256u : 32u; +llama_kv_cache_unified::llama_kv_cache_unified(const llama_hparams & hparams, callbacks cbs) : hparams(hparams), cbs(std::move(cbs)) { } -bool llama_kv_cache_init( - struct llama_kv_cache & cache, - const llama_model & model, - const llama_cparams & cparams, - ggml_type type_k, - ggml_type type_v, - uint32_t kv_size, - bool offload) { - const struct llama_hparams & hparams = model.hparams; - +bool llama_kv_cache_unified::init( + const llama_model & model, + const llama_cparams & cparams, + ggml_type type_k, + ggml_type type_v, + uint32_t kv_size, + bool offload) { const int32_t n_layer = hparams.n_layer; - cache.has_shift = false; + has_shift = false; - cache.recurrent = llama_model_is_recurrent(&model); - cache.v_trans = !cache.recurrent && !cparams.flash_attn; - cache.can_shift = !cache.recurrent && model.arch != LLM_ARCH_DEEPSEEK2; // not supported due to MLA + recurrent = llama_model_is_recurrent(&model); + v_trans = !recurrent && !cparams.flash_attn; + can_shift = !recurrent && model.arch != LLM_ARCH_DEEPSEEK2; // not supported due to MLA LLAMA_LOG_INFO("%s: kv_size = %d, offload = %d, type_k = '%s', type_v = '%s', n_layer = %d, can_shift = %d\n", - __func__, kv_size, offload, ggml_type_name(type_k), ggml_type_name(type_v), n_layer, cache.can_shift); + __func__, kv_size, offload, ggml_type_name(type_k), ggml_type_name(type_v), n_layer, can_shift); - cache.head = 0; - cache.size = kv_size; - cache.used = 0; + head = 0; + size = kv_size; + used = 0; - cache.type_k = type_k; - cache.type_v = type_v; + this->type_k = type_k; + this->type_v = type_v; - cache.cells.clear(); - cache.cells.resize(kv_size); + cells.clear(); + cells.resize(kv_size); // create a context for each buffer type std::map ctx_map; auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * { auto it = ctx_map.find(buft); if (it == ctx_map.end()) { - struct ggml_init_params params = { + ggml_init_params params = { /*.mem_size =*/ size_t(2u*n_layer*ggml_tensor_overhead()), /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, }; + ggml_context * ctx = ggml_init(params); if (!ctx) { return nullptr; } + ctx_map[buft] = ctx; - cache.ctxs.emplace_back(ctx); + ctxs.emplace_back(ctx); + return ctx; } + return it->second; }; - cache.k_l.reserve(n_layer); - cache.v_l.reserve(n_layer); + k_l.reserve(n_layer); + v_l.reserve(n_layer); for (int i = 0; i < n_layer; i++) { const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(i) + hparams.n_embd_k_s(); const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(i) + hparams.n_embd_v_s(); - LLAMA_LOG_DEBUG("%s: layer %d: n_embd_k_gqa = %d, n_embd_v_gqa = %d\n", __func__, i, n_embd_k_gqa, n_embd_v_gqa); + const char * dev_name = "CPU"; ggml_backend_buffer_type_t buft; if (offload) { auto * dev = model.dev_layer(i); buft = ggml_backend_dev_buffer_type(dev); + + dev_name = ggml_backend_dev_name(dev); } else { buft = ggml_backend_cpu_buffer_type(); } - ggml_context * ctx = ctx_for_buft(buft); + LLAMA_LOG_DEBUG("%s: layer %3d: n_embd_k_gqa = %d, n_embd_v_gqa = %d, dev = %s\n", __func__, + i, n_embd_k_gqa, n_embd_v_gqa, dev_name); + + ggml_context * ctx = ctx_for_buft(buft); if (!ctx) { LLAMA_LOG_ERROR("%s: failed to create ggml context for kv cache\n", __func__); return false; @@ -95,8 +101,8 @@ bool llama_kv_cache_init( ggml_tensor * v = ggml_new_tensor_1d(ctx, type_v, n_embd_v_gqa*kv_size); ggml_format_name(k, "cache_k_l%d", i); ggml_format_name(v, "cache_v_l%d", i); - cache.k_l.push_back(k); - cache.v_l.push_back(v); + k_l.push_back(k); + v_l.push_back(v); } // allocate tensors and initialize the buffers to avoid NaNs in the padding @@ -111,280 +117,80 @@ bool llama_kv_cache_init( } ggml_backend_buffer_clear(buf, 0); LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0); - cache.bufs.emplace_back(buf); + bufs.emplace_back(buf); } return true; } -struct llama_kv_cache_slot_info llama_kv_cache_find_slot( - struct llama_kv_cache & cache, - const struct llama_ubatch & ubatch) { - const uint32_t n_tokens = ubatch.n_tokens; - const uint32_t n_seqs = ubatch.n_seqs; - const uint32_t n_seq_tokens = ubatch.n_seq_tokens; +int32_t llama_kv_cache_unified::get_n_tokens() const { + int32_t result = 0; - if (cache.recurrent) { - // For recurrent state architectures (like Mamba or RWKV), - // each cache cell can store the state for a whole sequence. - // A slot should be always be contiguous. - - // can only process batches with an equal number of new tokens in each sequence - GGML_ASSERT(ubatch.equal_seqs); - - int32_t min = cache.size - 1; - int32_t max = 0; - - // everything should fit if all seq_ids are smaller than the max - for (uint32_t s = 0; s < n_seqs; ++s) { - const uint32_t n_seq_id = ubatch.n_seq_id[s]; - for (uint32_t j = 0; j < n_seq_id; ++j) { - const llama_seq_id seq_id = ubatch.seq_id[s][j]; - - if (seq_id < 0 || (uint32_t) seq_id >= cache.size) { - // too big seq_id - // TODO: would it be possible to resize the cache instead? - LLAMA_LOG_ERROR("%s: seq_id=%d >= n_seq_max=%d Try using a bigger --parallel value\n", __func__, seq_id, cache.size); - return llama_kv_cache_slot_info_failed; - } - if (j > 0) { - llama_kv_cell & seq = cache.cells[seq_id]; - if (seq.tail >= 0) { - llama_kv_cell & cell = cache.cells[seq.tail]; - // clear cells from seq_ids that become shared - // (should not normally happen, but let's handle it anyway) - cell.seq_id.erase(seq_id); - seq.tail = -1; - if (cell.seq_id.empty()) { - cell.pos = -1; - cell.src = -1; - cache.used -= 1; - } - } - } - } - } - -#ifndef NDEBUG - { - std::vector tails_verif; - tails_verif.assign(cache.size, -1); - for (uint32_t i = 0; i < cache.size; ++i) { - llama_kv_cell & cell = cache.cells[i]; - for (llama_seq_id seq_id : cell.seq_id) { - if (tails_verif[seq_id] != -1) { - LLAMA_LOG_ERROR("%s: duplicate tail for seq_id %d in cell %d and %d\n", __func__, seq_id, i, tails_verif[seq_id]); - } - tails_verif[seq_id] = i; - } - } - for (uint32_t i = 0; i < cache.size; ++i) { - if (tails_verif[i] != cache.cells[i].tail) { - LLAMA_LOG_ERROR("%s: wrong tail for seq_id %d, (%d instead of %d)\n", __func__, i, cache.cells[i].tail, tails_verif[i]); - } - } - } -#endif - - // find next empty cell - uint32_t next_empty_cell = cache.head; - - for (uint32_t i = 0; i < cache.size; ++i) { - if (next_empty_cell >= cache.size) { next_empty_cell -= cache.size; } - llama_kv_cell & cell = cache.cells[next_empty_cell]; - if (cell.is_empty()) { break; } - next_empty_cell += 1; - } - - // find usable cell range - for (uint32_t s = 0; s < n_seqs; ++s) { - const llama_seq_id seq_id = ubatch.seq_id[s][0]; - llama_kv_cell & seq_meta = cache.cells[seq_id]; - bool has_cell = false; - if (seq_meta.tail >= 0) { - llama_kv_cell & cell = cache.cells[seq_meta.tail]; - GGML_ASSERT(cell.has_seq_id(seq_id)); - // does this seq_id "own" the cell? - if (cell.seq_id.size() == 1) { has_cell = true; } - } - if (!has_cell) { - llama_kv_cell & empty_cell = cache.cells[next_empty_cell]; - GGML_ASSERT(empty_cell.is_empty()); - // copy old tail into the empty cell - if (seq_meta.tail >= 0) { - llama_kv_cell & orig_cell = cache.cells[seq_meta.tail]; - empty_cell.pos = orig_cell.pos; - empty_cell.src = orig_cell.src; - orig_cell.seq_id.erase(seq_id); - empty_cell.seq_id.insert(seq_id); // will be overwritten - } - seq_meta.tail = next_empty_cell; - // find next empty cell - if (s + 1 < n_seqs) { - next_empty_cell += 1; - for (uint32_t i = 0; i < cache.size; ++i) { - if (next_empty_cell >= cache.size) { next_empty_cell -= cache.size; } - llama_kv_cell & cell = cache.cells[next_empty_cell]; - if (cell.is_empty()) { break; } - next_empty_cell += 1; - } - } - } - if (min > seq_meta.tail) { min = seq_meta.tail; } - if (max < seq_meta.tail) { max = seq_meta.tail; } - } - - // gather and re-order - for (uint32_t s = 0; s < n_seqs; ++s) { - int32_t dst_id = s + min; - int32_t src_id = cache.cells[ubatch.seq_id[s][0]].tail; - if (dst_id != src_id) { - llama_kv_cell & dst_cell = cache.cells[dst_id]; - llama_kv_cell & src_cell = cache.cells[src_id]; - - std::swap(dst_cell.pos, src_cell.pos); - std::swap(dst_cell.src, src_cell.src); - std::swap(dst_cell.seq_id, src_cell.seq_id); - - // swap tails (assuming they NEVER overlap) - for (const llama_seq_id seq_id : src_cell.seq_id) { - cache.cells[seq_id].tail = src_id; - } - for (const llama_seq_id seq_id : dst_cell.seq_id) { - cache.cells[seq_id].tail = dst_id; - } - } - } - - // update the pos of the used seqs - for (uint32_t s = 0; s < n_seqs; ++s) { - const llama_pos last_pos = ubatch.pos[n_seq_tokens * s + n_seq_tokens - 1]; - int32_t cell_id = s + min; - llama_kv_cell & cell = cache.cells[cell_id]; - - if (cell.pos >= 0 && last_pos != cell.pos + (llama_pos) n_seq_tokens) { - // What should happen when the pos backtracks or skips a value? - // Clearing the state mid-batch would require special-casing which isn't done. - LLAMA_LOG_WARN("%s: non-consecutive token position %d after %d for sequence %d with %u new tokens\n", - __func__, last_pos, cell.pos, ubatch.seq_id[s][0], n_seq_tokens); - } - cell.pos = last_pos; - cell.seq_id.clear(); - for (int32_t j = 0; j < ubatch.n_seq_id[s]; ++j) { - const llama_seq_id seq_id = ubatch.seq_id[s][j]; - cell.seq_id.insert(seq_id); - cache.cells[seq_id].tail = cell_id; - } - } - - // allow getting the range of used cells, from head to head + n - cache.head = min; - cache.n = max - min + 1; - cache.used = std::count_if(cache.cells.begin(), cache.cells.end(), - [](const llama_kv_cell& cell){ return !cell.is_empty(); }); - - // sanity check - return llama_kv_cache_slot_info(cache.n >= n_seqs); - } - // otherwise, one cell per token. - - if (n_tokens > cache.size) { - LLAMA_LOG_ERROR("%s: n_tokens=%d > cache.size=%d\n", __func__, n_tokens, cache.size); - return llama_kv_cache_slot_info_failed; + for (uint32_t i = 0; i < size; i++) { + result += cells[i].seq_id.size(); } - uint32_t n_tested = 0; - - while (true) { - if (cache.head + n_tokens > cache.size) { - n_tested += cache.size - cache.head; - cache.head = 0; - continue; - } - - bool found = true; - for (uint32_t i = 0; i < n_tokens; i++) { - if (cache.cells[cache.head + i].pos >= 0) { - found = false; - cache.head += i + 1; - n_tested += i + 1; - break; - } - } - - if (found) { - break; - } - - if (n_tested >= cache.size) { - //LLAMA_LOG_ERROR("%s: failed to find a slot for %d tokens\n", __func__, n_tokens); - return llama_kv_cache_slot_info_failed; - } - } - - for (uint32_t s = 0; s < n_seqs; s++) { - for (uint32_t i = 0; i < n_seq_tokens; ++i) { - uint32_t k = s*n_seq_tokens + i; - cache.cells[cache.head + k].pos = ubatch.pos[k]; - - for (int32_t j = 0; j < ubatch.n_seq_id[s]; j++) { - cache.cells[cache.head + k].seq_id.insert(ubatch.seq_id[s][j]); - } - } - } - - cache.used += n_tokens; - - return llama_kv_cache_slot_info(cache.head, cache.head + n_tokens); + return result; } -uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) { - for (uint32_t i = cache.size; i > 0; --i) { - const llama_kv_cell & cell = cache.cells[i - 1]; - - if (cell.pos >= 0 && !cell.is_empty()) { - return i; - } - } - - return 0; +uint32_t llama_kv_cache_unified::get_used_cells() const { + return used; } -void llama_kv_cache_clear(struct llama_kv_cache & cache) { - for (int32_t i = 0; i < (int32_t) cache.size; ++i) { - cache.cells[i].pos = -1; - cache.cells[i].seq_id.clear(); - cache.cells[i].src = -1; - cache.cells[i].tail = -1; +size_t llama_kv_cache_unified::total_size() const { + size_t size = 0; + for (const auto & buf : bufs) { + size += ggml_backend_buffer_get_size(buf.get()); } - cache.head = 0; - cache.used = 0; - for (auto & buf : cache.bufs) { + return size; +} + +llama_pos llama_kv_cache_unified::pos_max() const { + llama_pos pos_max = -1; + for (const auto & cell : cells) { + pos_max = std::max(pos_max, cell.pos); + } + + return pos_max; +} + +void llama_kv_cache_unified::clear() { + for (int32_t i = 0; i < (int32_t) size; ++i) { + cells[i].pos = -1; + cells[i].seq_id.clear(); + cells[i].src = -1; + cells[i].tail = -1; + } + head = 0; + used = 0; + + for (auto & buf : bufs) { ggml_backend_buffer_clear(buf.get(), 0); } } -bool llama_kv_cache_seq_rm( - struct llama_kv_cache & cache, - llama_seq_id seq_id, - llama_pos p0, - llama_pos p1) { - uint32_t new_head = cache.size; +bool llama_kv_cache_unified::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { + uint32_t new_head = size; - if (p0 < 0) p0 = 0; - if (p1 < 0) p1 = std::numeric_limits::max(); + if (p0 < 0) { + p0 = 0; + } + + if (p1 < 0) { + p1 = std::numeric_limits::max(); + } // models like Mamba or RWKV can't have a state partially erased - if (cache.recurrent) { - if (seq_id >= (int64_t) cache.size) { + if (recurrent) { + if (seq_id >= (int64_t) size) { // could be fatal return false; } if (0 <= seq_id) { - int32_t & tail_id = cache.cells[seq_id].tail; + int32_t & tail_id = cells[seq_id].tail; if (tail_id >= 0) { - const llama_kv_cell & cell = cache.cells[tail_id]; + const llama_kv_cell & cell = cells[tail_id]; // partial intersection is invalid if ((0 < p0 && p0 <= cell.pos) || (0 < p1 && p1 <= cell.pos)) { return false; @@ -402,48 +208,59 @@ bool llama_kv_cache_seq_rm( } } - for (uint32_t i = 0; i < cache.size; ++i) { - if (cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) { + for (uint32_t i = 0; i < size; ++i) { + if (cells[i].pos >= p0 && cells[i].pos < p1) { if (seq_id < 0) { - cache.cells[i].seq_id.clear(); - } else if (cache.cells[i].has_seq_id(seq_id)) { - cache.cells[i].seq_id.erase(seq_id); + cells[i].seq_id.clear(); + } else if (cells[i].has_seq_id(seq_id)) { + cells[i].seq_id.erase(seq_id); } else { continue; } - if (cache.cells[i].is_empty()) { + if (cells[i].is_empty()) { // keep count of the number of used cells - if (cache.cells[i].pos >= 0) cache.used--; + if (cells[i].pos >= 0) { + used--; + } - cache.cells[i].pos = -1; - cache.cells[i].src = -1; - if (new_head == cache.size) new_head = i; + cells[i].pos = -1; + cells[i].src = -1; + + if (new_head == size) { + new_head = i; + } } } } // If we freed up a slot, set head to it so searching can start there. - if (new_head != cache.size && new_head < cache.head) cache.head = new_head; + if (new_head != size && new_head < head) { + head = new_head; + } return true; } -void llama_kv_cache_seq_cp( - struct llama_kv_cache & cache, - llama_seq_id seq_id_src, - llama_seq_id seq_id_dst, - llama_pos p0, - llama_pos p1) { - if (p0 < 0) p0 = 0; - if (p1 < 0) p1 = std::numeric_limits::max(); +void llama_kv_cache_unified::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { + if (seq_id_src == seq_id_dst) { + return; + } - if (cache.recurrent) { - if ((uint32_t) seq_id_dst < cache.size && (uint32_t) seq_id_src < cache.size) { - llama_kv_cell & tail_src = cache.cells[seq_id_src]; - llama_kv_cell & tail_dst = cache.cells[seq_id_dst]; + if (p0 < 0) { + p0 = 0; + } + + if (p1 < 0) { + p1 = std::numeric_limits::max(); + } + + if (recurrent) { + if ((uint32_t) seq_id_dst < size && (uint32_t) seq_id_src < size) { + llama_kv_cell & tail_src = cells[seq_id_src]; + llama_kv_cell & tail_dst = cells[seq_id_dst]; if (tail_dst.tail >= 0) { // clear destination seq_id if it wasn't empty - llama_kv_cell & cell_dst = cache.cells[tail_dst.tail]; + llama_kv_cell & cell_dst = cells[tail_dst.tail]; cell_dst.seq_id.erase(seq_id_dst); tail_dst.tail = -1; @@ -451,11 +268,11 @@ void llama_kv_cache_seq_cp( cell_dst.pos = -1; cell_dst.delta = -1; cell_dst.src = -1; - cache.used -= 1; + used -= 1; } } if (tail_src.tail >= 0) { - llama_kv_cell & cell_src = cache.cells[tail_src.tail]; + llama_kv_cell & cell_src = cells[tail_src.tail]; cell_src.seq_id.insert(seq_id_dst); tail_dst.tail = tail_src.tail; @@ -464,59 +281,75 @@ void llama_kv_cache_seq_cp( return; } - // otherwise, this is the KV cache of a Transformer-like model - cache.head = 0; + // otherwise, this is the KV of a Transformer-like model + head = 0; - for (uint32_t i = 0; i < cache.size; ++i) { - if (cache.cells[i].has_seq_id(seq_id_src) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) { - cache.cells[i].seq_id.insert(seq_id_dst); + for (uint32_t i = 0; i < size; ++i) { + if (cells[i].has_seq_id(seq_id_src) && cells[i].pos >= p0 && cells[i].pos < p1) { + cells[i].seq_id.insert(seq_id_dst); } } } -void llama_kv_cache_seq_keep(struct llama_kv_cache & cache, llama_seq_id seq_id) { - uint32_t new_head = cache.size; +void llama_kv_cache_unified::seq_keep(llama_seq_id seq_id) { + uint32_t new_head = size; - for (uint32_t i = 0; i < cache.size; ++i) { - if (cache.recurrent && (llama_seq_id) i != seq_id) { - cache.cells[i].tail = -1; + for (uint32_t i = 0; i < size; ++i) { + if (recurrent && (llama_seq_id) i != seq_id) { + cells[i].tail = -1; } - if (!cache.cells[i].has_seq_id(seq_id)) { - if (cache.cells[i].pos >= 0) cache.used--; - cache.cells[i].pos = -1; - cache.cells[i].src = -1; - cache.cells[i].seq_id.clear(); - if (new_head == cache.size) new_head = i; + + if (!cells[i].has_seq_id(seq_id)) { + if (cells[i].pos >= 0) { + used--; + } + + cells[i].pos = -1; + cells[i].src = -1; + cells[i].seq_id.clear(); + + if (new_head == size){ + new_head = i; + } } else { - cache.cells[i].seq_id.clear(); - cache.cells[i].seq_id.insert(seq_id); + cells[i].seq_id.clear(); + cells[i].seq_id.insert(seq_id); } } // If we freed up a slot, set head to it so searching can start there. - if (new_head != cache.size && new_head < cache.head) cache.head = new_head; + if (new_head != size && new_head < head) { + head = new_head; + } } -void llama_kv_cache_seq_add( - struct llama_kv_cache & cache, - llama_seq_id seq_id, - llama_pos p0, - llama_pos p1, - llama_pos delta) { - uint32_t new_head = cache.size; +void llama_kv_cache_unified::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) { + if (delta == 0) { + return; + } - if (p0 < 0) p0 = 0; - if (p1 < 0) p1 = std::numeric_limits::max(); - // If there is no range then return early to avoid looping over the cache. - if (p0 == p1) return; + uint32_t new_head = size; - if (cache.recurrent) { + if (p0 < 0) { + p0 = 0; + } + + if (p1 < 0) { + p1 = std::numeric_limits::max(); + } + + // If there is no range then return early to avoid looping over the + if (p0 == p1) { + return; + } + + if (recurrent) { // for Mamba-like or RWKV models, only the pos needs to be shifted - if (0 <= seq_id && seq_id < (int64_t) cache.size) { - const int32_t tail_id = cache.cells[seq_id].tail; + if (0 <= seq_id && seq_id < (int64_t) size) { + const int32_t tail_id = cells[seq_id].tail; if (tail_id >= 0) { - llama_kv_cell & cell = cache.cells[tail_id]; + llama_kv_cell & cell = cells[tail_id]; if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) { cell.pos += delta; } @@ -525,19 +358,19 @@ void llama_kv_cache_seq_add( return; } - for (uint32_t i = 0; i < cache.size; ++i) { - if (cache.cells[i].has_seq_id(seq_id) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) { - cache.has_shift = true; - cache.cells[i].pos += delta; - cache.cells[i].delta += delta; + for (uint32_t i = 0; i < size; ++i) { + if (cells[i].has_seq_id(seq_id) && cells[i].pos >= p0 && cells[i].pos < p1) { + has_shift = true; + cells[i].pos += delta; + cells[i].delta += delta; - if (cache.cells[i].pos < 0) { - if (!cache.cells[i].is_empty()) { - cache.used--; + if (cells[i].pos < 0) { + if (!cells[i].is_empty()) { + used--; } - cache.cells[i].pos = -1; - cache.cells[i].seq_id.clear(); - if (new_head == cache.size) { + cells[i].pos = -1; + cells[i].seq_id.clear(); + if (new_head == size) { new_head = i; } } @@ -546,93 +379,968 @@ void llama_kv_cache_seq_add( // If we freed up a slot, set head to it so searching can start there. // Otherwise we just start the next search from the beginning. - cache.head = new_head != cache.size ? new_head : 0; + head = new_head != size ? new_head : 0; } -void llama_kv_cache_seq_div( - struct llama_kv_cache & cache, - llama_seq_id seq_id, - llama_pos p0, - llama_pos p1, - int d) { - if (p0 < 0) p0 = 0; - if (p1 < 0) p1 = std::numeric_limits::max(); - // If there is no range then return early to avoid looping over the cache. - if (p0 == p1) return; +void llama_kv_cache_unified::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { + if (d == 1) { + return; + } - if (cache.recurrent) { + if (p0 < 0) { + p0 = 0; + } + + if (p1 < 0) { + p1 = std::numeric_limits::max(); + } + + // If there is no range then return early to avoid looping over the cache. + if (p0 == p1) { + return; + } + + if (recurrent) { // for Mamba-like or RWKV models, only the pos needs to be changed - if (0 <= seq_id && seq_id < (int64_t) cache.size) { - const int32_t tail_id = cache.cells[seq_id].tail; + if (0 <= seq_id && seq_id < (int64_t) size) { + const int32_t tail_id = cells[seq_id].tail; if (tail_id >= 0) { - llama_kv_cell & cell = cache.cells[tail_id]; + llama_kv_cell & cell = cells[tail_id]; if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) { cell.pos /= d; } } } + return; } - for (uint32_t i = 0; i < cache.size; ++i) { - if (cache.cells[i].has_seq_id(seq_id) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) { - cache.has_shift = true; + for (uint32_t i = 0; i < size; ++i) { + if (cells[i].has_seq_id(seq_id) && cells[i].pos >= p0 && cells[i].pos < p1) { + has_shift = true; { - llama_pos p_old = cache.cells[i].pos; - cache.cells[i].pos /= d; - cache.cells[i].delta += cache.cells[i].pos - p_old; + llama_pos p_old = cells[i].pos; + cells[i].pos /= d; + cells[i].delta += cells[i].pos - p_old; } } } } -llama_pos llama_kv_cache_seq_pos_max(struct llama_kv_cache & cache, llama_seq_id seq_id) { +llama_pos llama_kv_cache_unified::seq_pos_max(llama_seq_id seq_id) { llama_pos result = 0; - for (uint32_t i = 0; i < cache.size; ++i) { - if (cache.cells[i].has_seq_id(seq_id)) { - result = std::max(result, cache.cells[i].pos); + for (uint32_t i = 0; i < size; ++i) { + if (cells[i].has_seq_id(seq_id)) { + result = std::max(result, cells[i].pos); } } return result; } -void llama_kv_cache_defrag(struct llama_kv_cache & cache) { - if (!cache.recurrent) { - cache.do_defrag = true; +void llama_kv_cache_unified::defrag() { + if (!recurrent) { + do_defrag = true; } } -int32_t llama_get_kv_cache_token_count(const struct llama_kv_cache & kv) { - int result = 0; +bool llama_kv_cache_unified::get_can_shift() const { + return can_shift; +} - for (uint32_t i = 0; i < kv.size; i++) { - result += kv.cells[i].seq_id.size(); +llama_kv_cache_slot_info llama_kv_cache_unified::find_slot( + const llama_ubatch & ubatch) { + const uint32_t n_tokens = ubatch.n_tokens; + const uint32_t n_seqs = ubatch.n_seqs; + const uint32_t n_seq_tokens = ubatch.n_seq_tokens; + + if (recurrent) { + // For recurrent state architectures (like Mamba or RWKV), + // each cache cell can store the state for a whole sequence. + // A slot should be always be contiguous. + + // can only process batches with an equal number of new tokens in each sequence + GGML_ASSERT(ubatch.equal_seqs); + + int32_t min = size - 1; + int32_t max = 0; + + // everything should fit if all seq_ids are smaller than the max + for (uint32_t s = 0; s < n_seqs; ++s) { + const uint32_t n_seq_id = ubatch.n_seq_id[s]; + for (uint32_t j = 0; j < n_seq_id; ++j) { + const llama_seq_id seq_id = ubatch.seq_id[s][j]; + + if (seq_id < 0 || (uint32_t) seq_id >= size) { + // too big seq_id + // TODO: would it be possible to resize the cache instead? + LLAMA_LOG_ERROR("%s: seq_id=%d >= n_seq_max=%d Try using a bigger --parallel value\n", __func__, seq_id, size); + return llama_kv_cache_slot_info_failed; + } + if (j > 0) { + llama_kv_cell & seq = cells[seq_id]; + if (seq.tail >= 0) { + llama_kv_cell & cell = cells[seq.tail]; + // clear cells from seq_ids that become shared + // (should not normally happen, but let's handle it anyway) + cell.seq_id.erase(seq_id); + seq.tail = -1; + if (cell.seq_id.empty()) { + cell.pos = -1; + cell.src = -1; + used -= 1; + } + } + } + } + } + +#ifndef NDEBUG + { + std::vector tails_verif; + tails_verif.assign(size, -1); + for (uint32_t i = 0; i < size; ++i) { + llama_kv_cell & cell = cells[i]; + for (llama_seq_id seq_id : cell.seq_id) { + if (tails_verif[seq_id] != -1) { + LLAMA_LOG_ERROR("%s: duplicate tail for seq_id %d in cell %d and %d\n", __func__, seq_id, i, tails_verif[seq_id]); + } + tails_verif[seq_id] = i; + } + } + for (uint32_t i = 0; i < size; ++i) { + if (tails_verif[i] != cells[i].tail) { + LLAMA_LOG_ERROR("%s: wrong tail for seq_id %d, (%d instead of %d)\n", __func__, i, cells[i].tail, tails_verif[i]); + } + } + } +#endif + + // find next empty cell + uint32_t next_empty_cell = head; + + for (uint32_t i = 0; i < size; ++i) { + if (next_empty_cell >= size) { next_empty_cell -= size; } + llama_kv_cell & cell = cells[next_empty_cell]; + if (cell.is_empty()) { break; } + next_empty_cell += 1; + } + + // find usable cell range + for (uint32_t s = 0; s < n_seqs; ++s) { + const llama_seq_id seq_id = ubatch.seq_id[s][0]; + llama_kv_cell & seq_meta = cells[seq_id]; + bool has_cell = false; + if (seq_meta.tail >= 0) { + llama_kv_cell & cell = cells[seq_meta.tail]; + GGML_ASSERT(cell.has_seq_id(seq_id)); + // does this seq_id "own" the cell? + if (cell.seq_id.size() == 1) { has_cell = true; } + } + if (!has_cell) { + llama_kv_cell & empty_cell = cells[next_empty_cell]; + GGML_ASSERT(empty_cell.is_empty()); + // copy old tail into the empty cell + if (seq_meta.tail >= 0) { + llama_kv_cell & orig_cell = cells[seq_meta.tail]; + empty_cell.pos = orig_cell.pos; + empty_cell.src = orig_cell.src; + orig_cell.seq_id.erase(seq_id); + empty_cell.seq_id.insert(seq_id); // will be overwritten + } + seq_meta.tail = next_empty_cell; + // find next empty cell + if (s + 1 < n_seqs) { + next_empty_cell += 1; + for (uint32_t i = 0; i < size; ++i) { + if (next_empty_cell >= size) { next_empty_cell -= size; } + llama_kv_cell & cell = cells[next_empty_cell]; + if (cell.is_empty()) { break; } + next_empty_cell += 1; + } + } + } + if (min > seq_meta.tail) { min = seq_meta.tail; } + if (max < seq_meta.tail) { max = seq_meta.tail; } + } + + // gather and re-order + for (uint32_t s = 0; s < n_seqs; ++s) { + int32_t dst_id = s + min; + int32_t src_id = cells[ubatch.seq_id[s][0]].tail; + if (dst_id != src_id) { + llama_kv_cell & dst_cell = cells[dst_id]; + llama_kv_cell & src_cell = cells[src_id]; + + std::swap(dst_cell.pos, src_cell.pos); + std::swap(dst_cell.src, src_cell.src); + std::swap(dst_cell.seq_id, src_cell.seq_id); + + // swap tails (assuming they NEVER overlap) + for (const llama_seq_id seq_id : src_cell.seq_id) { + cells[seq_id].tail = src_id; + } + for (const llama_seq_id seq_id : dst_cell.seq_id) { + cells[seq_id].tail = dst_id; + } + } + } + + // update the pos of the used seqs + for (uint32_t s = 0; s < n_seqs; ++s) { + const llama_pos last_pos = ubatch.pos[n_seq_tokens * s + n_seq_tokens - 1]; + int32_t cell_id = s + min; + llama_kv_cell & cell = cells[cell_id]; + + if (cell.pos >= 0 && last_pos != cell.pos + (llama_pos) n_seq_tokens) { + // What should happen when the pos backtracks or skips a value? + // Clearing the state mid-batch would require special-casing which isn't done. + LLAMA_LOG_WARN("%s: non-consecutive token position %d after %d for sequence %d with %u new tokens\n", + __func__, last_pos, cell.pos, ubatch.seq_id[s][0], n_seq_tokens); + } + cell.pos = last_pos; + cell.seq_id.clear(); + for (int32_t j = 0; j < ubatch.n_seq_id[s]; ++j) { + const llama_seq_id seq_id = ubatch.seq_id[s][j]; + cell.seq_id.insert(seq_id); + cells[seq_id].tail = cell_id; + } + } + + // allow getting the range of used cells, from head to head + n + head = min; + n = max - min + 1; + used = std::count_if(cells.begin(), cells.end(), + [](const llama_kv_cell& cell){ return !cell.is_empty(); }); + + // sanity check + return llama_kv_cache_slot_info(n >= n_seqs); } - return result; + // otherwise, one cell per token. + + if (n_tokens > size) { + LLAMA_LOG_ERROR("%s: n_tokens = %d > size = %d\n", __func__, n_tokens, size); + return llama_kv_cache_slot_info_failed; + } + + uint32_t n_tested = 0; + + while (true) { + if (head + n_tokens > size) { + n_tested += size - head; + head = 0; + continue; + } + + bool found = true; + for (uint32_t i = 0; i < n_tokens; i++) { + if (cells[head + i].pos >= 0) { + found = false; + head += i + 1; + n_tested += i + 1; + break; + } + } + + if (found) { + break; + } + + if (n_tested >= size) { + //LLAMA_LOG_ERROR("%s: failed to find a slot for %d tokens\n", __func__, n_tokens); + return llama_kv_cache_slot_info_failed; + } + } + + for (uint32_t s = 0; s < n_seqs; s++) { + for (uint32_t i = 0; i < n_seq_tokens; ++i) { + uint32_t k = s*n_seq_tokens + i; + cells[head + k].pos = ubatch.pos[k]; + + for (int32_t j = 0; j < ubatch.n_seq_id[s]; j++) { + cells[head + k].seq_id.insert(ubatch.seq_id[s][j]); + } + } + } + + used += n_tokens; + + return llama_kv_cache_slot_info(head, head + n_tokens); } -int32_t llama_get_kv_cache_used_cells(const struct llama_kv_cache & kv) { - return kv.used; +uint32_t llama_kv_cache_unified::get_padding(const llama_cparams & cparams) const { + // the FA kernels require padding to avoid extra runtime boundary checks + return cparams.flash_attn ? 256u : 32u; } -bool llama_kv_cache_can_shift(const struct llama_kv_cache & kv) { - return kv.can_shift; +uint32_t llama_kv_cache_unified::cell_max() const { + for (uint32_t i = size; i > 0; --i) { + const llama_kv_cell & cell = cells[i - 1]; + + if (cell.pos >= 0 && !cell.is_empty()) { + return i; + } + } + + return 0; +} + +size_t llama_kv_cache_unified::size_k_bytes() const { + size_t size_k_bytes = 0; + + for (const auto & k : k_l) { + size_k_bytes += ggml_nbytes(k); + } + + return size_k_bytes; +} + +size_t llama_kv_cache_unified::size_v_bytes() const { + size_t size_v_bytes = 0; + + for (const auto & v : v_l) { + size_v_bytes += ggml_nbytes(v); + } + + return size_v_bytes; +} + +bool llama_kv_cache_unified::defrag_prepare(int32_t n_max_nodes) { + const uint32_t n_layer = hparams.n_layer; + + const uint32_t n_kv = cell_max(); + const uint32_t n_used = used; + + assert(n_used <= n_kv); + + //const int64_t t_start = ggml_time_us(); + + // number of cells moved + uint32_t n_moves = 0; + + // each move requires 6*n_layer tensors (see graph_build_kv_self_defrag) + // - source view, destination view, copy operation + // - x2 for keys and values + //const uint32_t max_moves = max_nodes()/(6*n_layer); + // TODO: tmp fix https://github.com/ggerganov/llama.cpp/issues/6685#issuecomment-2057579516 + const uint32_t max_moves = (n_max_nodes - 2*n_layer)/(6*n_layer); + + // determine which KV cells to move where + // + // cell i moves to ids[i] + // + // if ids[i] == i || ids[i] == n_kv, then cell i is not moved + // + auto & ids = defrag_info.ids; + + ids.clear(); + ids.resize(n_kv, n_kv); + + for (uint32_t i0 = 0; i0 < n_used; ++i0) { + const auto & cell0 = cells[i0]; + + if (!cell0.is_empty()) { + ids[i0] = i0; + + continue; + } + + // found a hole - fill it with data from the end of the cache + + uint32_t nh = 1; + + // determine the size of the hole + while (i0 + nh < n_used && cells[i0 + nh].is_empty()) { + nh++; + } + + uint32_t nf = 0; + uint32_t is = n_kv - 1; + + // starting from the end, find nh non-empty cells + for (; is > i0; --is) { + const auto & cell1 = cells[is]; + + if (cell1.is_empty() || ids[is] != n_kv) { + continue; + } + + // non-empty cell which is not yet moved + nf++; + + if (nf == nh) { + break; + } + } + + // this can only happen if `n_used` is not accurate, which would be a bug + GGML_ASSERT(nf == nh && "KV defrag bug: nf != nh"); + + nf = 0; + + uint32_t i1 = is; + + // are we moving a continuous block of memory? + bool cont = false; + + // should we stop searching for the next move? + bool stop = false; + + // go back and move the nf cells to the hole + for (; i1 < n_kv; ++i1) { + auto & cell1 = cells[i1]; + + if (cell1.is_empty() || ids[i1] != n_kv) { + if (n_moves == max_moves) { + stop = true; + break; + } + + cont = false; + continue; + } + + // this cell goes to (i0 + nf) + ids[i1] = i0 + nf; + + // move the cell meta data + cells[i0 + nf] = cell1; + + // clear the old cell and move the head there + cell1 = llama_kv_cell(); + head = n_used; + + if (!cont) { + n_moves++; + cont = true; + } + + nf++; + + if (nf == nh) { + break; + } + } + + if (stop || n_moves == max_moves) { + break; + } + + //LLAMA_LOG_INFO("(tmp log) KV defrag: move [%u, %u) to [%u, %u)\n", is, i1 + 1, i0, i0 + nh); + + i0 += nh - 1; + } + + if (n_moves == 0) { + return false; + } + + LLAMA_LOG_DEBUG("(tmp log) KV defrag cell moves: %u\n", n_moves); + + LLAMA_LOG_DEBUG("expected gf nodes: %u\n", 6*n_moves*n_layer); + + return true; +} + +void llama_kv_cache_unified::state_write(llama_io_write_i & io, llama_seq_id seq_id) const { + std::vector> cell_ranges; // ranges, from inclusive, to exclusive + uint32_t cell_count = 0; + + // Count the number of cells with the specified seq_id + // Find all the ranges of cells with this seq id (or all, when -1) + uint32_t cell_range_begin = size; + for (uint32_t i = 0; i < size; ++i) { + const auto & cell = cells[i]; + if ((seq_id == -1 && !cell.is_empty()) || cell.has_seq_id(seq_id)) { + ++cell_count; + if (cell_range_begin == size) { + cell_range_begin = i; + } + } else { + if (cell_range_begin != size) { + cell_ranges.emplace_back(cell_range_begin, i); + cell_range_begin = size; + } + } + } + if (cell_range_begin != size) { + cell_ranges.emplace_back(cell_range_begin, size); + } + + // DEBUG CHECK: Sum of cell counts in ranges should equal the total cell count + uint32_t cell_count_check = 0; + for (const auto & range : cell_ranges) { + cell_count_check += range.second - range.first; + } + GGML_ASSERT(cell_count == cell_count_check); + + io.write(&cell_count, sizeof(cell_count)); + + state_write_meta(io, cell_ranges, seq_id); + state_write_data(io, cell_ranges); +} + +void llama_kv_cache_unified::state_read(llama_io_read_i & io, llama_seq_id seq_id) { + uint32_t cell_count; + io.read_to(&cell_count, sizeof(cell_count)); + + bool res = true; + res = res && state_read_meta(io, cell_count, seq_id); + res = res && state_read_data(io, cell_count); + + if (!res) { + if (seq_id == -1) { + clear(); + } else { + seq_rm(seq_id, -1, -1); + } + throw std::runtime_error("failed to restore kv cache"); + } +} + +void llama_kv_cache_unified::state_write_meta(llama_io_write_i & io, const std::vector> & cell_ranges, llama_seq_id seq_id) const { + for (const auto & range : cell_ranges) { + for (uint32_t i = range.first; i < range.second; ++i) { + const auto & cell = cells[i]; + const llama_pos pos = cell.pos; + const uint32_t n_seq_id = seq_id == -1 ? cell.seq_id.size() : 0; + + io.write(&pos, sizeof(pos)); + io.write(&n_seq_id, sizeof(n_seq_id)); + + if (n_seq_id) { + for (auto seq_id : cell.seq_id) { + io.write(&seq_id, sizeof(seq_id)); + } + } + } + } +} + +void llama_kv_cache_unified::state_write_data(llama_io_write_i & io, const std::vector> & cell_ranges) const { + const uint32_t v_trans = this->v_trans ? 1 : 0; + const uint32_t n_layer = hparams.n_layer; + + io.write(&v_trans, sizeof(v_trans)); + io.write(&n_layer, sizeof(n_layer)); + + std::vector tmp_buf; + + // Iterate and write all the keys first, each row is a cell + // Get whole range at a time + for (uint32_t il = 0; il < n_layer; ++il) { + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il) + hparams.n_embd_k_s(); + + // Write key type + const int32_t k_type_i = (int32_t)k_l[il]->type; + io.write(&k_type_i, sizeof(k_type_i)); + + // Write row size of key + const uint64_t k_size_row = ggml_row_size(k_l[il]->type, n_embd_k_gqa); + io.write(&k_size_row, sizeof(k_size_row)); + + // Read each range of cells of k_size length each into tmp_buf and write out + for (const auto & range : cell_ranges) { + const size_t range_size = range.second - range.first; + const size_t buf_size = range_size * k_size_row; + io.write_tensor(k_l[il], range.first * k_size_row, buf_size); + } + } + + if (!v_trans) { + for (uint32_t il = 0; il < n_layer; ++il) { + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s(); + + // Write value type + const int32_t v_type_i = (int32_t)v_l[il]->type; + io.write(&v_type_i, sizeof(v_type_i)); + + // Write row size of value + const uint64_t v_size_row = ggml_row_size(v_l[il]->type, n_embd_v_gqa); + io.write(&v_size_row, sizeof(v_size_row)); + + // Read each range of cells of v_size length each into tmp_buf and write out + for (const auto & range : cell_ranges) { + const size_t range_size = range.second - range.first; + const size_t buf_size = range_size * v_size_row; + io.write_tensor(v_l[il], range.first * v_size_row, buf_size); + } + } + } else { + // When v is transposed, we also need the element size and get the element ranges from each row + const uint32_t kv_size = size; + for (uint32_t il = 0; il < n_layer; ++il) { + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s(); + + // Write value type + const int32_t v_type_i = (int32_t)v_l[il]->type; + io.write(&v_type_i, sizeof(v_type_i)); + + // Write element size + const uint32_t v_size_el = ggml_type_size(v_l[il]->type); + io.write(&v_size_el, sizeof(v_size_el)); + + // Write GQA embedding size + io.write(&n_embd_v_gqa, sizeof(n_embd_v_gqa)); + + // For each row, we get the element values of each cell + for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { + // Read each range of cells of v_size_el length each into tmp_buf and write out + for (const auto & range : cell_ranges) { + const size_t range_size = range.second - range.first; + const size_t src_offset = (range.first + j * kv_size) * v_size_el; + const size_t buf_size = range_size * v_size_el; + io.write_tensor(v_l[il], src_offset, buf_size); + } + } + } + } +} + +bool llama_kv_cache_unified::state_read_meta(llama_io_read_i & io, uint32_t cell_count, llama_seq_id dest_seq_id) { + if (dest_seq_id != -1) { + // single sequence + + seq_rm(dest_seq_id, -1, -1); + + llama_sbatch sbatch; + llama_ubatch batch = sbatch.reserve_ubatch(cell_count, /* has_embd */ false); + + batch.n_tokens = cell_count; + batch.n_seq_tokens = cell_count; + batch.n_seqs = 1; + + for (uint32_t i = 0; i < cell_count; ++i) { + llama_pos pos; + uint32_t n_seq_id; + + io.read_to(&pos, sizeof(pos)); + io.read_to(&n_seq_id, sizeof(n_seq_id)); + + if (n_seq_id != 0) { + LLAMA_LOG_ERROR("%s: invalid seq_id-agnostic kv cell\n", __func__); + return false; + } + + batch.pos[i] = pos; + } + batch.n_seq_id[0] = 1; + batch.seq_id[0] = &dest_seq_id; + if (!find_slot(batch)) { + LLAMA_LOG_ERROR("%s: failed to find available cells in kv cache\n", __func__); + return false; + } + + // DEBUG CHECK: kv.head should be our first cell, kv.head + cell_count - 1 should be our last cell (verify seq_id and pos values) + // Assume that this is one contiguous block of cells + GGML_ASSERT(head + cell_count <= size); + GGML_ASSERT(cells[head].pos == batch.pos[0]); + GGML_ASSERT(cells[head + cell_count - 1].pos == batch.pos[cell_count - 1]); + GGML_ASSERT(cells[head].has_seq_id(dest_seq_id)); + GGML_ASSERT(cells[head + cell_count - 1].has_seq_id(dest_seq_id)); + } else { + // whole KV cache restore + + if (cell_count > size) { + LLAMA_LOG_ERROR("%s: not enough cells in kv cache\n", __func__); + return false; + } + + clear(); + + for (uint32_t i = 0; i < cell_count; ++i) { + llama_kv_cell & cell = cells[i]; + + llama_pos pos; + uint32_t n_seq_id; + + io.read_to(&pos, sizeof(pos)); + io.read_to(&n_seq_id, sizeof(n_seq_id)); + + cell.pos = pos; + + for (uint32_t j = 0; j < n_seq_id; ++j) { + llama_seq_id seq_id; + io.read_to(&seq_id, sizeof(seq_id)); + + // TODO: llama_kv_cache_unified should have a notion of max sequences + //if (seq_id < 0 || (uint32_t) seq_id >= llama_n_seq_max(ctx)) { + if (seq_id < 0) { + //LLAMA_LOG_ERROR("%s: invalid seq_id, %d is out of range [0, %u)\n", __func__, seq_id, llama_n_seq_max(ctx)); + LLAMA_LOG_ERROR("%s: invalid seq_id, %d is out of range [0, inf)\n", __func__, seq_id); + return false; + } + + cell.seq_id.insert(seq_id); + + if (recurrent) { + int32_t & tail = cells[seq_id].tail; + if (tail != -1) { + LLAMA_LOG_ERROR("%s: duplicate tail for seq_id %d in cell %d and %d\n", __func__, seq_id, i, tail); + return false; + } + tail = i; + } + } + } + + head = 0; + used = cell_count; + } + + if (recurrent) { + for (uint32_t i = 0; i < cell_count; ++i) { + uint32_t cell_id = head + i; + // make sure the recurrent states will keep their restored state + cells[cell_id].src = cell_id; + } + } + + return true; +} + +bool llama_kv_cache_unified::state_read_data(llama_io_read_i & io, uint32_t cell_count) { + uint32_t v_trans; + uint32_t n_layer; + io.read_to(&v_trans, sizeof(v_trans)); + io.read_to(&n_layer, sizeof(n_layer)); + + if (n_layer != hparams.n_layer) { + LLAMA_LOG_ERROR("%s: mismatched layer count (%u instead of %u)\n", __func__, n_layer, hparams.n_layer); + return false; + } + if (cell_count > size) { + LLAMA_LOG_ERROR("%s: not enough cells in kv cache to restore state (%u > %u)\n", __func__, cell_count, size); + return false; + } + if (v_trans != (bool) v_trans) { + LLAMA_LOG_ERROR("%s: incompatible V transposition\n", __func__); + return false; + } + + // For each layer, read the keys for each cell, one row is one cell, read as one contiguous block + for (uint32_t il = 0; il < n_layer; ++il) { + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il) + hparams.n_embd_k_s(); + + // Read type of key + int32_t k_type_i_ref; + io.read_to(&k_type_i_ref, sizeof(k_type_i_ref)); + const int32_t k_type_i = (int32_t) k_l[il]->type; + if (k_type_i != k_type_i_ref) { + LLAMA_LOG_ERROR("%s: mismatched key type (%d != %d, layer %d)\n", __func__, k_type_i, k_type_i_ref, il); + return false; + } + + // Read row size of key + uint64_t k_size_row_ref; + io.read_to(&k_size_row_ref, sizeof(k_size_row_ref)); + const size_t k_size_row = ggml_row_size(k_l[il]->type, n_embd_k_gqa); + if (k_size_row != k_size_row_ref) { + LLAMA_LOG_ERROR("%s: mismatched key row size (%zu != %zu, layer %d)\n", __func__, k_size_row, (size_t) k_size_row_ref, il); + return false; + } + + if (cell_count) { + // Read and set the keys for the whole cell range + ggml_backend_tensor_set(k_l[il], io.read(cell_count * k_size_row), head * k_size_row, cell_count * k_size_row); + } + } + + if (!v_trans) { + for (uint32_t il = 0; il < n_layer; ++il) { + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s(); + + // Read type of value + int32_t v_type_i_ref; + io.read_to(&v_type_i_ref, sizeof(v_type_i_ref)); + const int32_t v_type_i = (int32_t)v_l[il]->type; + if (v_type_i != v_type_i_ref) { + LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il); + return false; + } + + // Read row size of value + uint64_t v_size_row_ref; + io.read_to(&v_size_row_ref, sizeof(v_size_row_ref)); + const size_t v_size_row = ggml_row_size(v_l[il]->type, n_embd_v_gqa); + if (v_size_row != v_size_row_ref) { + LLAMA_LOG_ERROR("%s: mismatched value row size (%zu != %zu, layer %d)\n", __func__, v_size_row, (size_t) v_size_row_ref, il); + return false; + } + + if (cell_count) { + // Read and set the values for the whole cell range + ggml_backend_tensor_set(v_l[il], io.read(cell_count * v_size_row), head * v_size_row, cell_count * v_size_row); + } + } + } else { + // For each layer, read the values for each cell (transposed) + for (uint32_t il = 0; il < n_layer; ++il) { + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il) + hparams.n_embd_v_s(); + + // Read type of value + int32_t v_type_i_ref; + io.read_to(&v_type_i_ref, sizeof(v_type_i_ref)); + const int32_t v_type_i = (int32_t)v_l[il]->type; + if (v_type_i != v_type_i_ref) { + LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il); + return false; + } + + // Read element size of value + uint32_t v_size_el_ref; + io.read_to(&v_size_el_ref, sizeof(v_size_el_ref)); + const size_t v_size_el = ggml_type_size(v_l[il]->type); + if (v_size_el != v_size_el_ref) { + LLAMA_LOG_ERROR("%s: mismatched value element size (%zu != %zu, layer %d)\n", __func__, v_size_el, (size_t) v_size_el_ref, il); + return false; + } + + // Read GQA embedding size + uint32_t n_embd_v_gqa_ref; + io.read_to(&n_embd_v_gqa_ref, sizeof(n_embd_v_gqa_ref)); + if (n_embd_v_gqa != n_embd_v_gqa_ref) { + LLAMA_LOG_ERROR("%s: mismatched GQA embedding size (%u != %u, layer %d)\n", __func__, n_embd_v_gqa, n_embd_v_gqa_ref, il); + return false; + } + + if (cell_count) { + // For each row in the transposed matrix, read the values for the whole cell range + for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { + const size_t dst_offset = (head + j * size) * v_size_el; + ggml_backend_tensor_set(v_l[il], io.read(cell_count * v_size_el), dst_offset, cell_count * v_size_el); + } + } + } + } + + return true; +} + +// +// interface implementation +// + +int32_t llama_kv_cache_n_tokens(const llama_kv_cache * kv) { + if (!kv) { + return 0; + } + + return kv->get_n_tokens(); +} + +int32_t llama_kv_cache_used_cells(const llama_kv_cache * kv) { + if (!kv) { + return 0; + } + + return kv->get_used_cells(); +} + +void llama_kv_cache_clear(llama_kv_cache * kv) { + if (!kv) { + return; + } + + kv->clear(); +} + +bool llama_kv_cache_seq_rm( + llama_kv_cache * kv, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1) { + if (!kv) { + return true; + } + + return kv->seq_rm(seq_id, p0, p1); +} + +void llama_kv_cache_seq_cp( + llama_kv_cache * kv, + llama_seq_id seq_id_src, + llama_seq_id seq_id_dst, + llama_pos p0, + llama_pos p1) { + if (!kv) { + return; + } + + kv->seq_cp(seq_id_src, seq_id_dst, p0, p1); +} + +void llama_kv_cache_seq_keep(llama_kv_cache * kv, llama_seq_id seq_id) { + if (!kv) { + return; + } + + kv->seq_keep(seq_id); +} + +void llama_kv_cache_seq_add( + llama_kv_cache * kv, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + llama_pos delta) { + if (!kv) { + return; + } + + kv->seq_add(seq_id, p0, p1, delta); +} + +void llama_kv_cache_seq_div( + llama_kv_cache * kv, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + int d) { + if (!kv) { + return; + } + + kv->seq_div(seq_id, p0, p1, d); +} + +llama_pos llama_kv_cache_seq_pos_max(llama_kv_cache * kv, llama_seq_id seq_id) { + if (!kv) { + return 0; + } + + return kv->seq_pos_max(seq_id); +} + +void llama_kv_cache_defrag(llama_kv_cache * kv) { + if (!kv) { + return; + } + + kv->defrag(); +} + +bool llama_kv_cache_can_shift(const llama_kv_cache * kv) { + if (!kv) { + return false; + } + + return kv->get_can_shift(); } // // kv cache view // -struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_kv_cache & kv, int32_t n_seq_max) { - struct llama_kv_cache_view result = { +llama_kv_cache_view llama_kv_cache_view_init(const llama_kv_cache & kv, int32_t n_seq_max) { + llama_kv_cache_view result = { /*.n_cells = */ 0, /*.n_seq_max = */ n_seq_max, /*.token_count = */ 0, - /*.used_cells = */ llama_get_kv_cache_used_cells(kv), + /*.used_cells = */ llama_kv_cache_used_cells(&kv), /*.max_contiguous = */ 0, /*.max_contiguous_idx = */ -1, /*.cells = */ nullptr, @@ -642,7 +1350,7 @@ struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_kv_cache return result; } -void llama_kv_cache_view_free(struct llama_kv_cache_view * view) { +void llama_kv_cache_view_free(llama_kv_cache_view * view) { if (view->cells != nullptr) { free(view->cells); view->cells = nullptr; @@ -653,18 +1361,25 @@ void llama_kv_cache_view_free(struct llama_kv_cache_view * view) { } } -void llama_kv_cache_view_update(struct llama_kv_cache_view * view, const struct llama_kv_cache & kv) { - if (uint32_t(view->n_cells) < kv.size || view->cells == nullptr) { - view->n_cells = int32_t(kv.size); - void * p = realloc(view->cells, sizeof(struct llama_kv_cache_view_cell) * view->n_cells); +void llama_kv_cache_view_update(llama_kv_cache_view * view, const llama_kv_cache * kv) { + // TODO: rework this in the future, for now quick hack + const llama_kv_cache_unified * kvu = dynamic_cast(kv); + if (kvu == nullptr) { + LLAMA_LOG_ERROR("%s: the kv_cache_view currently works only with llama_kv_cache_unified\n", __func__); + return; + } + + if (uint32_t(view->n_cells) < kvu->size || view->cells == nullptr) { + view->n_cells = int32_t(kvu->size); + void * p = realloc(view->cells, sizeof(llama_kv_cache_view_cell) * view->n_cells); GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells"); - view->cells = (struct llama_kv_cache_view_cell *)p; + view->cells = (llama_kv_cache_view_cell *)p; p = realloc(view->cells_sequences, sizeof(llama_seq_id) * view->n_seq_max * view->n_cells); GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells sequences"); view->cells_sequences = (llama_seq_id *)p; } - const std::vector & kv_cells = kv.cells; + const std::vector & kv_cells = kvu->cells; llama_kv_cache_view_cell * c_curr = view->cells; llama_seq_id * cs_curr = view->cells_sequences; int32_t used_cells = 0; @@ -673,7 +1388,7 @@ void llama_kv_cache_view_update(struct llama_kv_cache_view * view, const struct uint32_t max_contig = 0; int32_t max_contig_idx = -1; - for (int32_t i = 0; i < int32_t(kv.size); i++, c_curr++, cs_curr += view->n_seq_max) { + for (int32_t i = 0; i < int32_t(kvu->size); i++, c_curr++, cs_curr += view->n_seq_max) { const size_t curr_size = kv_cells[i].seq_id.size(); token_count += curr_size; c_curr->pos = kv_cells[i].pos + kv_cells[i].delta; @@ -711,8 +1426,8 @@ void llama_kv_cache_view_update(struct llama_kv_cache_view * view, const struct view->max_contiguous_idx = max_contig_idx; view->token_count = token_count; view->used_cells = used_cells; - if (uint32_t(used_cells) != kv.used) { + if (uint32_t(used_cells) != kvu->used) { LLAMA_LOG_ERROR("%s: used cells mismatch. kv_cache says %d but we calculated %d\n", - __func__, kv.used, used_cells); + __func__, kvu->used, used_cells); } } diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index 1ce0850ec8..0a7ff8a4ea 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -1,12 +1,29 @@ #pragma once #include "llama.h" +#include "llama-io.h" +#include "llama-memory.h" #include "ggml-cpp.h" +#include #include #include -#include + +struct llama_cparams; +struct llama_hparams; +struct llama_ubatch; + +struct llama_kv_cache : public llama_memory_i { + using llama_memory_i::llama_memory_i; + + virtual int32_t get_n_tokens() const = 0; + virtual uint32_t get_used_cells() const = 0; // TODO: remove, this is too-specific to the unified cache + + virtual bool get_can_shift() const = 0; + + bool get_can_edit() const override { return get_can_shift(); } +}; struct llama_kv_cell { llama_pos pos = -1; @@ -29,11 +46,105 @@ struct llama_kv_cell { } }; +// a structure holds information about the slot found in llama_kv_cache_find_slot +struct llama_kv_cache_slot_info { + std::pair boundaries; // slot boundaries [begin, end) + bool found = false; // the slot was found + + explicit llama_kv_cache_slot_info(bool found_) : found{found_} {} + llama_kv_cache_slot_info(uint32_t begin, uint32_t end) : boundaries{begin, end}, found{true} {} + + operator bool() const { return found; } +}; + // ring-buffer of cached KV data -struct llama_kv_cache { +// TODO: pimpl +// TODO: add notion of max sequences +class llama_kv_cache_unified : public llama_kv_cache { +public: + // can be used to query data from the model if needed + struct callbacks { + std::function get_rope_factors; + }; + + llama_kv_cache_unified( + const llama_hparams & hparams, + callbacks cbs); + + virtual ~llama_kv_cache_unified() = default; + + // TODO: become constructor + bool init( + const llama_model & model, // TODO: do not reference the model + const llama_cparams & cparams, + ggml_type type_k, + ggml_type type_v, + uint32_t kv_size, + bool offload); + + int32_t get_n_tokens() const override; + uint32_t get_used_cells() const override; + + size_t total_size() const; + + // TODO: better data structures to reduce the cost of this operation + llama_pos pos_max() const; + + void clear() override; + void defrag() override; + + bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) override; + void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override; + void seq_keep(llama_seq_id seq_id) override; + void seq_add (llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) override; + void seq_div (llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) override; + + llama_pos seq_pos_max(llama_seq_id seq_id) override; + + bool get_can_shift() const override; + + // find an empty slot of size "n_tokens" in the cache + // updates the cache head + // returns a structure holding information about the slot found + // Note: On success, it's important that cache.head points + // to the first cell of the slot. + llama_kv_cache_slot_info find_slot(const llama_ubatch & batch); + + // TODO: maybe not needed + uint32_t get_padding(const llama_cparams & cparams) const; + + // find how many cells are currently in use + uint32_t cell_max() const; + + size_t size_k_bytes() const; + size_t size_v_bytes() const; + + // defrag + + struct { + std::vector ids; + } defrag_info; + + // return true if cells have been moved + bool defrag_prepare(int32_t n_max_nodes); + + // state save/load + + void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1) const; + void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1); + + // members + + const llama_hparams & hparams; + + callbacks cbs; + bool has_shift = false; bool do_defrag = false; + + // TODO: remove this and implement llama_kv_cache_recurrent instead bool recurrent = false; // with recurrent state models, a cell can hold the state for more than one past token + bool v_trans = true; // the value tensor is transposed bool can_shift = false; @@ -47,124 +158,30 @@ struct llama_kv_cache { // computed before each graph build uint32_t n = 0; + std::vector cells; + + std::vector k_l; // per layer + std::vector v_l; + +private: ggml_type type_k = GGML_TYPE_F16; ggml_type type_v = GGML_TYPE_F16; - std::vector cells; - - std::vector k_l; // per layer - std::vector v_l; - - std::vector ctxs; + std::vector ctxs; std::vector bufs; - size_t total_size() const { - size_t size = 0; - for (const auto & buf : bufs) { - size += ggml_backend_buffer_get_size(buf.get()); - } + void state_write_meta(llama_io_write_i & io, const std::vector> & cell_ranges, llama_seq_id seq_id = -1) const; + void state_write_data(llama_io_write_i & io, const std::vector> & cell_ranges) const; - return size; - } - - // TODO: better data structures to reduce the cost of this operation - llama_pos max_pos() const { - llama_pos max_pos = -1; - for (const auto & cell : cells) { - max_pos = std::max(max_pos, cell.pos); - } - - return max_pos; - } + bool state_read_meta(llama_io_read_i & io, uint32_t cell_count, llama_seq_id dest_seq_id = -1); + bool state_read_data(llama_io_read_i & io, uint32_t cell_count); }; -// a structure holds information about the slot found in llama_kv_cache_find_slot -struct llama_kv_cache_slot_info { - std::pair boundaries; // slot boundaries [begin, end) - bool found = false; // the slot was found - - explicit llama_kv_cache_slot_info(bool found_) : found{found_} {} - llama_kv_cache_slot_info(uint32_t begin, uint32_t end) : boundaries{begin, end}, found{true} {} - - operator bool() const { return found; } -}; - -// TODO: maybe not needed -uint32_t llama_kv_cache_get_padding(const struct llama_cparams & cparams); - -bool llama_kv_cache_init( - struct llama_kv_cache & cache, - const llama_model & model, - const llama_cparams & cparams, - ggml_type type_k, - ggml_type type_v, - uint32_t kv_size, - bool offload); - -// find an empty slot of size "n_tokens" in the cache -// updates the cache head -// returns a structure holding information about the slot found -// Note: On success, it's important that cache.head points -// to the first cell of the slot. -struct llama_kv_cache_slot_info llama_kv_cache_find_slot( - struct llama_kv_cache & cache, - const struct llama_ubatch & batch); - -// find how many cells are currently in use -uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache); - -void llama_kv_cache_clear(struct llama_kv_cache & cache); - -bool llama_kv_cache_seq_rm( - struct llama_kv_cache & cache, - llama_seq_id seq_id, - llama_pos p0, - llama_pos p1); - -void llama_kv_cache_seq_cp( - struct llama_kv_cache & cache, - llama_seq_id seq_id_src, - llama_seq_id seq_id_dst, - llama_pos p0, - llama_pos p1); - -void llama_kv_cache_seq_keep( - struct llama_kv_cache & cache, - llama_seq_id seq_id); - -void llama_kv_cache_seq_add( - struct llama_kv_cache & cache, - llama_seq_id seq_id, - llama_pos p0, - llama_pos p1, - llama_pos delta); - -void llama_kv_cache_seq_div( - struct llama_kv_cache & cache, - llama_seq_id seq_id, - llama_pos p0, - llama_pos p1, - int d); - -llama_pos llama_kv_cache_seq_pos_max( - struct llama_kv_cache & cache, - llama_seq_id seq_id); - -void llama_kv_cache_defrag(struct llama_kv_cache & cache); - -int32_t llama_get_kv_cache_token_count(const struct llama_kv_cache & kv); - -int32_t llama_get_kv_cache_used_cells(const struct llama_kv_cache & kv); - -bool llama_kv_cache_can_shift(const struct llama_kv_cache & kv); - -// -// kv cache view -// - -struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_kv_cache & kv, int32_t n_seq_max); - -void llama_kv_cache_view_update(struct llama_kv_cache_view * view, const struct llama_kv_cache & kv); +// TODO: temporary reusing llama_kv_cache_unified -- implement recurrent cache and simplify llama_kv_cache_unified +//class llama_kv_cache_recurrent : public llama_kv_cache_unified { +//public: +// using llama_kv_cache_unified::llama_kv_cache_unified; +//}; // // kv cache restore @@ -184,13 +201,15 @@ struct llama_kv_slot_restorer { bool do_restore = false; - explicit llama_kv_slot_restorer(const struct llama_kv_cache & cache) { + llama_kv_cache_unified & cache; + + explicit llama_kv_slot_restorer(llama_kv_cache_unified & cache) : cache(cache) { old_state.head = cache.head; old_state.n = cache.n; } // saves a slot information for future restoration - void save(const struct llama_kv_cache_slot_info & slot) { + void save(const llama_kv_cache_slot_info & slot) { if (slot) { do_restore = true; if (slot.boundaries.first != slot.boundaries.second) { @@ -201,19 +220,68 @@ struct llama_kv_slot_restorer { // must be explicitly called to restore the kv_cache state // and rollback changes from all llama_kv_cache_find_slot calls - void restore(struct llama_kv_cache & cache) { + void restore() { if (do_restore) { cache.head = old_state.head; cache.n = old_state.n; if (cache.recurrent) { // recurrent models like Mamba or RWKV can't have a state partially erased - llama_kv_cache_seq_rm(cache, -1, -1, -1); + cache.seq_rm(-1, -1, -1); } else { for (auto & slot : slot_boundaries) { - llama_kv_cache_seq_rm(cache, -1, slot.first, slot.second); + cache.seq_rm(-1, slot.first, slot.second); } } } } }; +// TODO: maybe become part of the public llama_kv_cache in the future +int32_t llama_kv_cache_n_tokens(const llama_kv_cache * kv); + +int32_t llama_kv_cache_used_cells(const llama_kv_cache * kv); + +void llama_kv_cache_clear(llama_kv_cache * kv); + +bool llama_kv_cache_seq_rm( + llama_kv_cache * kv, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1); + +void llama_kv_cache_seq_cp( + llama_kv_cache * kv, + llama_seq_id seq_id_src, + llama_seq_id seq_id_dst, + llama_pos p0, + llama_pos p1); + +void llama_kv_cache_seq_keep(llama_kv_cache * kv, llama_seq_id seq_id); + +void llama_kv_cache_seq_add( + llama_kv_cache * kv, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + llama_pos delta); + +void llama_kv_cache_seq_div( + llama_kv_cache * kv, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + int d); + +llama_pos llama_kv_cache_seq_pos_max(llama_kv_cache * kv, llama_seq_id seq_id); + +void llama_kv_cache_defrag(llama_kv_cache * kv); + +bool llama_kv_cache_can_shift(const llama_kv_cache * kv); + +// +// kv cache view +// + +llama_kv_cache_view llama_kv_cache_view_init(const llama_kv_cache & kv, int32_t n_seq_max); + +void llama_kv_cache_view_update(llama_kv_cache_view * view, const llama_kv_cache * kv); diff --git a/src/llama-memory.cpp b/src/llama-memory.cpp new file mode 100644 index 0000000000..10173253ed --- /dev/null +++ b/src/llama-memory.cpp @@ -0,0 +1 @@ +#include "llama-memory.h" diff --git a/src/llama-memory.h b/src/llama-memory.h new file mode 100644 index 0000000000..69e6e34ca4 --- /dev/null +++ b/src/llama-memory.h @@ -0,0 +1,21 @@ +#pragma once + +#include "llama.h" + +// general concept of LLM memory +// the KV cache is a type of LLM memory, but there can be other types +class llama_memory_i { +public: + virtual void clear() = 0; + virtual void defrag() = 0; + + virtual bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) = 0; + virtual void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) = 0; + virtual void seq_keep(llama_seq_id seq_id) = 0; + virtual void seq_add (llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) = 0; + virtual void seq_div (llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) = 0; + + virtual llama_pos seq_pos_max(llama_seq_id seq_id) = 0; + + virtual bool get_can_edit() const = 0; +}; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 9f75589d80..522219c012 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2,12 +2,17 @@ #include "llama-impl.h" #include "llama-mmap.h" +#include "llama-batch.h" +#include "llama-cparams.h" #include "llama-model-loader.h" +#include "llama-kv-cache.h" #include "ggml-cpp.h" #include #include +#include +#include #include #include #include @@ -244,6 +249,7 @@ static ggml_backend_buffer_type_t select_weight_buft(const llama_hparams & hpara return cur_buft; } } + return nullptr; } @@ -302,7 +308,7 @@ static buft_list_t make_cpu_buft_list(const std::vector & de } // GPU: split if LLAMA_SPLIT_MODE_ROW -> GPU -static buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, enum llama_split_mode split_mode, const float * tensor_split) { +static buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, llama_split_mode split_mode, const float * tensor_split) { buft_list_t buft_list; // add the device split buffer type if requested and available @@ -369,7 +375,7 @@ struct llama_model::impl { std::vector dev_layer; }; -llama_model::llama_model(const struct llama_model_params & params) : params(params), pimpl(std::make_unique()) { +llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique()) { } llama_model::~llama_model() {} @@ -391,7 +397,7 @@ void llama_model::load_hparams(llama_model_loader & ml) { // get metadata as string for (int i = 0; i < gguf_get_n_kv(ctx); i++) { - enum gguf_type type = gguf_get_kv_type(ctx, i); + gguf_type type = gguf_get_kv_type(ctx, i); if (type == GGUF_TYPE_ARRAY) { continue; } @@ -1444,7 +1450,10 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // skip unused tensors if (info.op == GGML_OP_NONE) { - LLAMA_LOG_WARN("model has unused tensor %s -- ignoring\n", tn.str().c_str()); + const size_t nbytes = ggml_nbytes(t_meta); + LLAMA_LOG_WARN("model has unused tensor %s (size = %zu bytes) -- ignoring\n", tn.str().c_str(), nbytes); + + ml.size_data -= nbytes; ml.n_created++; return nullptr; @@ -3631,8 +3640,8 @@ size_t llama_model::size() const { return pimpl->n_bytes; } -size_t llama_model::max_nodes() const { - return std::max(8192, tensors_by_name.size()*5); +size_t llama_model::n_tensors() const { + return tensors_by_name.size(); } size_t llama_model::n_devices() const { @@ -3745,7 +3754,7 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared); LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale); LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm); - LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((enum llama_expert_gating_func_type) hparams.expert_gating_func)); + LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func)); LLAMA_LOG_INFO("%s: rope_yarn_log_mul = %.4f\n", __func__, hparams.rope_yarn_log_mul); } @@ -3821,9 +3830,9 @@ ggml_backend_buffer_type_t llama_model::select_buft(int il) const { }); } -const struct ggml_tensor * llama_model::get_tensor(const char * name) const { +const ggml_tensor * llama_model::get_tensor(const char * name) const { auto it = std::find_if(tensors_by_name.begin(), tensors_by_name.end(), - [name](const std::pair & it) { + [name](const std::pair & it) { return it.first == name; }); if (it == tensors_by_name.end()) { @@ -3833,12 +3842,7309 @@ const struct ggml_tensor * llama_model::get_tensor(const char * name) const { return it->second; } +struct llm_build_llama : public llm_graph_context { + llm_build_llama(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // rope freq factors for llama3; may return nullptr for llama2 and other models + ggml_tensor * rope_factors = static_cast(memory)->cbs.get_rope_factors(n_ctx_per_seq, il); + + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, kq_scale, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + // For Granite architecture + if (hparams.f_residual_scale) { + cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + if (model.layers[il].ffn_gate_inp == nullptr) { + + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + // MoE branch + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, true, + false, 0.0, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(cur, "ffn_moe_out", il); + } + + // For Granite architecture + if (hparams.f_residual_scale) { + cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + // For Granite architecture + if (hparams.f_logit_scale) { + cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale); + } + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_deci : public llm_graph_context { + llm_build_deci(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + const int64_t n_head_kv = hparams.n_head_kv(il); + const int64_t n_head = hparams.n_head(il); + + if (n_head == 0) { + // attention-free layer of Llama-3_1-Nemotron-51B + cur = inpL; + } else { + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + } + + if (n_head > 0 && n_head_kv == 0) { + // "linear attention" of Llama-3_1-Nemotron-51B + cur = build_lora_mm(model.layers[il].wo, cur); + cb(cur, "wo", il); + } else if (n_head > 0) { + // self-attention + // rope freq factors for llama3; may return nullptr for llama2 and other models + ggml_tensor * rope_factors = static_cast(memory)->cbs.get_rope_factors(n_ctx_per_seq, il); + + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, kq_scale, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + // For Granite architecture + if (hparams.f_residual_scale) { + cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); + } + + // modified to support attention-free layer of Llama-3_1-Nemotron-51B + ggml_tensor * ffn_inp = cur; + if (n_head > 0) { + ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + } + + // feed-forward network + if (model.layers[il].ffn_gate_inp == nullptr) { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + // For Granite architecture + if (hparams.f_residual_scale) { + cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + // For Granite architecture + if (hparams.f_logit_scale) { + cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale); + } + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_baichuan : public llm_graph_context { + llm_build_baichuan(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = model.type == LLM_TYPE_7B ? build_inp_pos() : nullptr; + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + switch (model.type) { + case LLM_TYPE_7B: + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + break; + case LLM_TYPE_13B: + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd/n_head, n_head, n_tokens); + break; + default: + GGML_ABORT("fatal error"); + } + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_xverse : public llm_graph_context { + llm_build_xverse(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_falcon : public llm_graph_context { + llm_build_falcon(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * attn_norm; + + attn_norm = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(attn_norm, "attn_norm", il); + + // self-attention + { + if (model.layers[il].attn_norm_2) { + // Falcon-40B + cur = build_norm(inpL, + model.layers[il].attn_norm_2, + model.layers[il].attn_norm_2_b, + LLM_NORM, il); + cb(cur, "attn_norm_2", il); + } else { + cur = attn_norm; + } + + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + + // using mode = 2 for neox mode + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + attn_norm = ggml_get_rows(ctx0, attn_norm, inp_out_ids); + } + + ggml_tensor * ffn_inp = cur; + + // feed forward + { + cur = build_ffn(attn_norm, // !! use the attn norm, not the result + model.layers[il].ffn_up, NULL, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + cur = ggml_add(ctx0, cur, inpL); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + // norm + cur = build_norm(cur, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_grok : public llm_graph_context { + llm_build_grok(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // multiply by embedding_multiplier_scale of 78.38367176906169 + inpL = ggml_scale(ctx0, inpL, 78.38367176906169f); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + // Grok + // if attn_out_norm is present then apply it before adding the input + if (model.layers[il].attn_out_norm) { + cur = build_norm(cur, + model.layers[il].attn_out_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_out_norm", il); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + // MoE branch + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_GELU, true, + false, 0.0, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(cur, "ffn_moe_out", il); + + // Grok + // if layer_out_norm is present then apply it before adding the input + // Idea: maybe ffn_out_norm is a better name + if (model.layers[il].layer_out_norm) { + cur = build_norm(cur, + model.layers[il].layer_out_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "layer_out_norm", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + // Grok + // multiply logits by output_multiplier_scale of 0.5773502691896257 + + cur = ggml_scale(ctx0, cur, 0.5773502691896257f); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_dbrx : public llm_graph_context { + llm_build_dbrx(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + ggml_tensor * Qcur = nullptr; + ggml_tensor * Kcur = nullptr; + ggml_tensor * Vcur = nullptr; + + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); + cb(cur, "wqkv_clamped", il); + + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + // MoE branch + cur = build_norm(ffn_inp, + model.layers[il].attn_out_norm, NULL, + LLM_NORM, il); + cb(cur, "attn_out_norm", il); + + cur = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, true, + false, 0.0, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(cur, "ffn_moe_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_starcoder : public llm_graph_context { + llm_build_starcoder(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); + cb(pos, "pos_embd", -1); + + inpL = ggml_add(ctx0, inpL, pos); + cb(inpL, "inpL", -1); + + for (int il = 0; il < n_layer; ++il) { + cur = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // add the input + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + // FF + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = build_norm(inpL, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_refact : public llm_graph_context { + llm_build_refact(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + cb(Kcur, "Kcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + cb(Qcur, "Qcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_bert : public llm_graph_context { + llm_build_bert(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + ggml_tensor * inp_pos = nullptr; + + if (model.arch != LLM_ARCH_JINA_BERT_V2) { + inp_pos = build_inp_pos(); + } + + // construct input embeddings (token, type, position) + inpL = build_inp_embd(model.tok_embd); + + // token types are hardcoded to zero ("Sentence A") + ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0); + inpL = ggml_add(ctx0, inpL, type_row0); + if (model.arch == LLM_ARCH_BERT) { + inpL = ggml_add(ctx0, ggml_get_rows(ctx0, model.pos_embd, inp_pos), inpL); + } + cb(inpL, "inp_embd", -1); + + // embed layer norm + inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1); + cb(inpL, "inp_norm", -1); + + auto * inp_attn = build_attn_inp_no_cache(); + + // iterate layers + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * cur = inpL; + + ggml_tensor * Qcur; + ggml_tensor * Kcur; + ggml_tensor * Vcur; + + // self-attention + if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT_V2) { + Qcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wq, cur), model.layers[il].bq); + + if (model.layers[il].attn_q_norm) { + Qcur = build_norm(Qcur, + model.layers[il].attn_q_norm, + model.layers[il].attn_q_norm_b, + LLM_NORM, il); + } + + Kcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wk, cur), model.layers[il].bk); + + if (model.layers[il].attn_k_norm) { + Kcur = build_norm(Kcur, + model.layers[il].attn_k_norm, + model.layers[il].attn_k_norm_b, + LLM_NORM, il); + } + + Vcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wv, cur), model.layers[il].bv); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + } else { + // compute Q and K and RoPE them + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + cb(cur, "kqv_out", il); + + if (il == n_layer - 1 && pooling_type == LLAMA_POOLING_TYPE_NONE) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // re-add the layer input + cur = ggml_add(ctx0, cur, inpL); + + // attention layer norm + cur = build_norm(cur, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, il); + + if (model.layers[il].attn_norm_2 != nullptr) { + cur = ggml_add(ctx0, cur, inpL); // re-add the layer input + cur = build_norm(cur, model.layers[il].attn_norm_2, model.layers[il].attn_norm_2_b, LLM_NORM, il); + } + + ggml_tensor * ffn_inp = cur; + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + if (model.arch == LLM_ARCH_BERT) { + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + } else if (model.arch == LLM_ARCH_JINA_BERT_V2) { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_PAR, il); + } else { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + } + cb(cur, "ffn_out", il); + + // attentions bypass the intermediate layer + cur = ggml_add(ctx0, cur, ffn_inp); + + // output layer norm + cur = build_norm(cur, model.layers[il].layer_out_norm, model.layers[il].layer_out_norm_b, LLM_NORM, il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cb(cur, "result_embd", -1); + res->t_embd = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_bloom : public llm_graph_context { + llm_build_bloom(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + inpL = build_norm(inpL, + model.tok_norm, + model.tok_norm_b, + LLM_NORM, -1); + cb(inpL, "inp_norm", -1); + + for (int il = 0; il < n_layer; ++il) { + cur = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // Add the input + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + // FF + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = build_norm(inpL, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_mpt : public llm_graph_context { + llm_build_mpt(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * pos; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + if (model.pos_embd) { + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); + cb(pos, "pos_embd", -1); + + inpL = ggml_add(ctx0, inpL, pos); + cb(inpL, "inpL", -1); + } + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * attn_norm; + + attn_norm = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(attn_norm, "attn_norm", il); + + // self-attention + { + cur = attn_norm; + + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + if (model.layers[il].bqkv){ + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + } + + if (hparams.f_clamp_kqv > 0.0f) { + cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); + cb(cur, "wqkv_clamped", il); + } + + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + // Q/K Layernorm + if (model.layers[il].attn_q_norm) { + Qcur = build_norm(Qcur, + model.layers[il].attn_q_norm, + model.layers[il].attn_q_norm_b, + LLM_NORM, il); + cb(Qcur, "Qcur", il); + + Kcur = build_norm(Kcur, + model.layers[il].attn_k_norm, + model.layers[il].attn_k_norm_b, + LLM_NORM, il); + cb(Kcur, "Kcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } else { + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // Add the input + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + // feed forward + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + model.layers[il].ffn_act, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_stablelm : public llm_graph_context { + llm_build_stablelm(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + ggml_tensor * inpSA = cur; + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + cb(Qcur, "Qcur", il); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + cb(Kcur, "Kcur", il); + + if (model.layers[il].attn_q_norm) { + Qcur = build_norm(Qcur, + model.layers[il].attn_q_norm, + NULL, + LLM_NORM, il); + cb(Qcur, "Qcur", il); + } + if (model.layers[il].attn_k_norm) { + Kcur = build_norm(Kcur, + model.layers[il].attn_k_norm, + NULL, + LLM_NORM, il); + cb(Kcur, "Kcur", il); + } + + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + { + if (model.layers[il].ffn_norm) { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + } else { + // parallel residual + cur = inpSA; + } + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_qwen : public llm_graph_context { + llm_build_qwen(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 2*sizeof(float)*(n_embd))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + + // using mode = 2 for neox mode + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward forward + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_qwen2 : public llm_graph_context { + llm_build_qwen2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_qwen2vl : public llm_graph_context { + llm_build_qwen2vl(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + int sections[4]; + std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_multi( + ctx0, + ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_multi( + ctx0, + ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_qwen2moe : public llm_graph_context { + llm_build_qwen2moe(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self_attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // MoE branch + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + ggml_tensor * moe_out = + build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, false, + false, 0.0, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(cur, "ffn_moe_out", il); + + // FFN shared expert + { + ggml_tensor * cur_gate_inp = build_lora_mm(model.layers[il].ffn_gate_inp_shexp, cur); + cb(cur_gate_inp, "ffn_shexp_gate_inp", il); + + // sigmoid + ggml_tensor * cur_gate = ggml_div(ctx0, ggml_silu(ctx0, cur_gate_inp), cur_gate_inp); + cb(cur_gate, "ffn_shexp_gate", il); + + ggml_tensor * cur_ffn = build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur_ffn, "ffn_shexp", il); + + ggml_tensor * ffn_shexp_out = ggml_mul(ctx0, cur_ffn, cur_gate); + cb(ffn_shexp_out, "ffn_shexp_out", il); + + moe_out = ggml_add(ctx0, moe_out, ffn_shexp_out); + cb(moe_out, "ffn_out", il); + + cur = moe_out; + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_phi2 : public llm_graph_context { + llm_build_phi2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * attn_norm_output; + ggml_tensor * ffn_output; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + attn_norm_output = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(attn_norm_output, "attn_norm", il); + + // self-attention + { + ggml_tensor * Qcur = nullptr; + ggml_tensor * Kcur = nullptr; + ggml_tensor * Vcur = nullptr; + + if (model.layers[il].wqkv) { + cur = build_lora_mm(model.layers[il].wqkv, attn_norm_output); + cb(cur, "wqkv", il); + + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + } else { + Qcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wq, attn_norm_output), model.layers[il].bq); + Kcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wk, attn_norm_output), model.layers[il].bk); + Vcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wv, attn_norm_output), model.layers[il].bv); + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + // with phi2, we scale the Q to avoid precision issues + // ref: https://github.com/ml-explore/mlx-examples/blob/08e862336ade809bc37d1035f94b359e7d1a5152/phi2/phi2.py#L64-L66 + Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head))); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + attn_norm_output = ggml_get_rows(ctx0, attn_norm_output, inp_out_ids); + } + + // FF + { + ffn_output = build_ffn(attn_norm_output, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(ffn_output, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_output); + cur = ggml_add(ctx0, cur, inpL); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = build_norm(inpL, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output_no_bias", -1); + + cur = ggml_add(ctx0, cur, model.output_b); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_phi3 : public llm_graph_context { + llm_build_phi3(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, true); + + for (int il = 0; il < n_layer; ++il) { + auto * residual = inpL; + + // self-attention + { + // rope freq factors for 128k context + ggml_tensor * rope_factors = static_cast(memory)->cbs.get_rope_factors(n_ctx_per_seq, il); + + ggml_tensor* attn_norm_output = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM_RMS, il); + cb(attn_norm_output, "attn_norm", il); + + ggml_tensor * Qcur = nullptr; + ggml_tensor * Kcur = nullptr; + ggml_tensor * Vcur = nullptr; + + if (model.layers[il].wqkv) { + cur = build_lora_mm(model.layers[il].wqkv, attn_norm_output); + cb(cur, "wqkv", il); + + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0 * sizeof(float) * (n_embd))); + Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1 * sizeof(float) * (n_embd))); + Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa))); + } else { + Qcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wq, attn_norm_output), model.layers[il].bq); + Kcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wk, attn_norm_output), model.layers[il].bk); + Vcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wv, attn_norm_output), model.layers[il].bv); + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head))); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor* inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + residual = ggml_get_rows(ctx0, residual, inp_out_ids); + } + + cur = ggml_add(ctx0, cur, residual); + residual = cur; + + cur = build_norm(cur, + model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + // feed-forward network + if (model.layers[il].ffn_gate_inp == nullptr) { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SWIGLU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + } else { + // MoE branch + cur = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, true, + false, 0.0, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(cur, "ffn_moe_out", il); + } + + cur = ggml_add(ctx0, residual, cur); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = build_norm(inpL, + model.output_norm, + model.output_norm_b, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + if (model.output_b != nullptr) { + cb(cur, "result_output_no_bias", -1); + cur = ggml_add(ctx0, cur, model.output_b); + } + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_plamo : public llm_graph_context { + llm_build_plamo(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + ggml_tensor * attention_norm = cur; + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, nullptr, + n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, nullptr, + n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + ggml_tensor * sa_out = cur; + + cur = attention_norm; + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + sa_out = ggml_get_rows(ctx0, sa_out, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // feed-forward network + { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, sa_out); + cur = ggml_add(ctx0, cur, inpL); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_gpt2 : public llm_graph_context { + llm_build_gpt2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * pos; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); + cb(pos, "pos_embd", -1); + + inpL = ggml_add(ctx0, inpL, pos); + cb(inpL, "inpL", -1); + + for (int il = 0; il < n_layer; ++il) { + cur = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // add the input + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + // FF + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = build_norm(inpL, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_codeshell : public llm_graph_context { + llm_build_codeshell(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + cur = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + ggml_tensor * tmpq = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * tmpk = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(tmpq, "tmpq", il); + cb(tmpk, "tmpk", il); + cb(Vcur, "Vcur", il); + + ggml_tensor * Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // add the input + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + // FF + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = build_norm(inpL, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_orion : public llm_graph_context { + llm_build_orion(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + // if (model.layers[il].bq) { + // Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + // cb(Qcur, "Qcur", il); + // } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + // if (model.layers[il].bk) { + // Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + // cb(Kcur, "Kcur", il); + // } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + // if (model.layers[il].bv) { + // Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + // cb(Vcur, "Vcur", il); + // } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_internlm2 : public llm_graph_context { + llm_build_internlm2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_minicpm3 : public llm_graph_context { + llm_build_minicpm3(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + //TODO: if the model varies, these parameters need to be read from the model + const int64_t n_embd_base = 256; + const float scale_embd = 12.0f; + const float scale_depth = 1.4f; + const float kq_scale = 1.0f / sqrtf(float(hparams.n_embd_head_k)); + + const uint32_t n_embd_head_qk_rope = hparams.n_rot; + const uint32_t n_embd_head_qk_nope = hparams.n_embd_head_k - hparams.n_rot; + const uint32_t kv_lora_rank = hparams.n_lora_kv; + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // scale the input embeddings + inpL = ggml_scale(ctx0, inpL, scale_embd); + cb(inpL, "inp_scaled", -1); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + ggml_tensor * rope_factors = static_cast(memory)->cbs.get_rope_factors(n_ctx_per_seq, il); + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self_attention + { + ggml_tensor * q = NULL; + // {n_embd, q_lora_rank} * {n_embd, n_tokens} -> {q_lora_rank, n_tokens} + q = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur); + cb(q, "q", il); + + q = build_norm(q, + model.layers[il].attn_q_a_norm, NULL, + LLM_NORM_RMS, il); + cb(q, "q", il); + + // {q_lora_rank, n_head * hparams.n_embd_head_k} * {q_lora_rank, n_tokens} -> {n_head * hparams.n_embd_head_k, n_tokens} + q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q); + cb(q, "q", il); + + // split into {n_head * n_embd_head_qk_nope, n_tokens} + ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, + ggml_row_size(q->type, hparams.n_embd_head_k), + ggml_row_size(q->type, hparams.n_embd_head_k * n_head), + 0); + cb(q_nope, "q_nope", il); + + // and {n_head * n_embd_head_qk_rope, n_tokens} + ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, + ggml_row_size(q->type, hparams.n_embd_head_k), + ggml_row_size(q->type, hparams.n_embd_head_k * n_head), + ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "q_pe", il); + + // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens} + ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); + cb(kv_pe_compresseed, "kv_pe_compresseed", il); + + // split into {kv_lora_rank, n_tokens} + ggml_tensor * kv_compressed = ggml_view_2d(ctx0, kv_pe_compresseed, kv_lora_rank, n_tokens, + kv_pe_compresseed->nb[1], + 0); + cb(kv_compressed, "kv_compressed", il); + + // and {n_embd_head_qk_rope, n_tokens} + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_pe_compresseed, n_embd_head_qk_rope, 1, n_tokens, + kv_pe_compresseed->nb[1], + kv_pe_compresseed->nb[1], + ggml_row_size(kv_pe_compresseed->type, kv_lora_rank)); + cb(k_pe, "k_pe", il); + + // TODO: the CUDA backend used to not support non-cont. (RMS) norm, investigate removing ggml_cont + kv_compressed = ggml_cont(ctx0, kv_compressed); + kv_compressed = build_norm(kv_compressed, + model.layers[il].attn_kv_a_norm, NULL, + LLM_NORM_RMS, il); + cb(kv_compressed, "kv_compressed", il); + + // {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)} * {kv_lora_rank, n_tokens} -> {n_head * (n_embd_head_qk_nope + n_embd_head_v), n_tokens} + ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_compressed); + cb(kv, "kv", il); + + // split into {n_head * n_embd_head_qk_nope, n_tokens} + ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens, + ggml_row_size(kv->type, n_embd_head_qk_nope + hparams.n_embd_head_v), + ggml_row_size(kv->type, n_head * (n_embd_head_qk_nope + hparams.n_embd_head_v)), + 0); + cb(k_nope, "k_nope", il); + + // and {n_head * n_embd_head_v, n_tokens} + ggml_tensor * v_states = ggml_view_3d(ctx0, kv, hparams.n_embd_head_v, n_head, n_tokens, + ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)), + ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)*n_head), + ggml_row_size(kv->type, (n_embd_head_qk_nope))); + cb(v_states, "v_states", il); + + v_states = ggml_cont(ctx0, v_states); + cb(v_states, "v_states", il); + + v_states = ggml_view_2d(ctx0, v_states, hparams.n_embd_head_v * n_head, n_tokens, + ggml_row_size(kv->type, hparams.n_embd_head_v * n_head), + 0); + cb(v_states, "v_states", il); + + q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this + q_pe = ggml_rope_ext( + ctx0, q_pe, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(q_pe, "q_pe", il); + + // shared RoPE key + k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this + k_pe = ggml_rope_ext( + ctx0, k_pe, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(k_pe, "k_pe", il); + + ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0); + cb(q_states, "q_states", il); + + ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0); + cb(k_states, "k_states", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + q_states, k_states, v_states, nullptr, kq_scale, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + // scale_res - scale the hidden states for residual connection + const float scale_res = scale_depth/sqrtf(float(n_layer)); + cur = ggml_scale(ctx0, cur, scale_res); + cb(cur, "hidden_scaled", il); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + // scale the hidden states for residual connection + cur = ggml_scale(ctx0, cur, scale_res); + cb(cur, "hidden_scaled_ffn", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head scaling + const float scale_lmhead = float(n_embd_base)/float(n_embd); + cur = ggml_scale(ctx0, cur, scale_lmhead); + cb(cur, "lmhead_scaling", -1); + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_gemma : public llm_graph_context { + llm_build_gemma(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head_k = hparams.n_embd_head_k; + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); + cb(inpL, "inp_scaled", -1); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + + Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); + cb(Qcur, "Qcur_scaled", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); + cb(sa_out, "sa_out", il); + + cur = build_norm(sa_out, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + // feed-forward network + { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, sa_out); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_gemma2 : public llm_graph_context { + llm_build_gemma2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head_k = hparams.n_embd_head_k; + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); + cb(inpL, "inp_scaled", -1); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, true); + + for (int il = 0; il < n_layer; ++il) { + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + + // ref: https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e + switch (model.type) { + case LLM_TYPE_2B: + case LLM_TYPE_9B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); break; + case LLM_TYPE_27B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd / n_head))); break; + default: GGML_ABORT("fatal error"); + }; + cb(Qcur, "Qcur_scaled", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f, il); + } + + cur = build_norm(cur, + model.layers[il].attn_post_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_post_norm", il); + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); + cb(sa_out, "sa_out", il); + + cur = build_norm(sa_out, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + // feed-forward network + { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = build_norm(cur, + model.layers[il].ffn_post_norm, NULL, + LLM_NORM_RMS, -1); + cb(cur, "ffn_post_norm", -1); + + cur = ggml_add(ctx0, cur, sa_out); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + // final logit soft-capping + cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); + cur = ggml_tanh(ctx0, cur); + cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_gemma3 : public llm_graph_context { + llm_build_gemma3(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head_k = hparams.n_embd_head_k; + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings) + if (ubatch.token) { + inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); + cb(inpL, "inp_scaled", -1); + } + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + // TODO: is causal == true correct? might need some changes + auto * inp_attn = build_attn_inp_kv_unified(true, true); + + // "5-to-1 interleaved attention" + // 5 layers of local attention followed by 1 layer of global attention + static const int sliding_window_pattern = 6; + + for (int il = 0; il < n_layer; ++il) { + const bool is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); + + const float freq_base_l = is_sliding ? 10000.0f : freq_base; + const float freq_scale_l = is_sliding ? 1.0f : freq_scale; + + // norm + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens); + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens); + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); + cb(Kcur, "Kcur_normed", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, hparams.f_attention_scale, il); + } + + cur = build_norm(cur, + model.layers[il].attn_post_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_post_norm", il); + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); + cb(sa_out, "sa_out", il); + + cur = build_norm(sa_out, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + // feed-forward network + { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = build_norm(cur, + model.layers[il].ffn_post_norm, NULL, + LLM_NORM_RMS, -1); + cb(cur, "ffn_post_norm", -1); + + cur = ggml_add(ctx0, cur, sa_out); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +// TODO: move up next to build_starcoder +struct llm_build_starcoder2 : public llm_graph_context { + llm_build_starcoder2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_mamba : public llm_graph_context { + const llama_model & model; + + llm_build_mamba(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params), model(model) { + ggml_tensor * cur; + ggml_tensor * inpL; + + // {n_embd, n_tokens} + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * state_copy = build_inp_s_copy(); + ggml_tensor * state_mask = build_inp_s_mask(); + + for (int il = 0; il < n_layer; ++il) { + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + //cur = build_mamba_layer(gf, cur, state_copy, state_mask, il); + cur = build_mamba_layer(gf, cur, state_copy, state_mask, ubatch, il); + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // residual + cur = ggml_add(ctx0, cur, inpL); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + // final rmsnorm + cur = build_norm(inpL, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } + + // TODO: split + ggml_tensor * build_mamba_layer( + ggml_cgraph * gf, + ggml_tensor * cur, + ggml_tensor * state_copy, + ggml_tensor * state_mask, + const llama_ubatch & ubatch, + int il) const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + const auto kv_head = kv_self->head; + + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = hparams.ssm_d_inner; + const int64_t d_state = hparams.ssm_d_state; + const int64_t dt_rank = hparams.ssm_dt_rank; + const int64_t n_seqs = ubatch.n_seqs; + // Some variants of Mamba arch (e.g. FalconMamba do apply layer norm on B and Dt layers) + const bool ssm_dt_b_c_rms = hparams.ssm_dt_b_c_rms; + // Use the same RMS norm as the final layer norm + const float norm_rms_eps = hparams.f_norm_rms_eps; + + const int64_t n_seq_tokens = ubatch.n_seq_tokens; + + GGML_ASSERT(n_seqs != 0); + GGML_ASSERT(ubatch.equal_seqs); + GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs); + + ggml_tensor * conv_states_all = kv_self->k_l[il]; + ggml_tensor * ssm_states_all = kv_self->v_l[il]; + + // (ab)using the KV cache to store the states + ggml_tensor * conv = build_copy_mask_state( + gf, conv_states_all, state_copy, state_mask, + hparams.n_embd_k_s(), n_seqs); + conv = ggml_reshape_3d(ctx0, conv, d_conv - 1, d_inner, n_seqs); + ggml_tensor * ssm = build_copy_mask_state( + gf, ssm_states_all, state_copy, state_mask, + hparams.n_embd_v_s(), n_seqs); + ssm = ggml_reshape_3d(ctx0, ssm, d_state, d_inner, n_seqs); + + // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs} + cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs); + + // {n_embd, 2*d_inner} @ {n_embd, n_seq_tokens, n_seqs} => {2*d_inner, n_seq_tokens, n_seqs} + ggml_tensor * xz = build_lora_mm(model.layers[il].ssm_in, cur); + // split the above in two + // => {d_inner, n_seq_tokens, n_seqs} + ggml_tensor * x = ggml_view_3d(ctx0, xz, d_inner, xz->ne[1], xz->ne[2], xz->nb[1], xz->nb[2], 0); + ggml_tensor * z = ggml_view_3d(ctx0, xz, d_inner, xz->ne[1], xz->ne[2], xz->nb[1], xz->nb[2], d_inner*ggml_element_size(xz)); + + // conv + { + // => {d_conv - 1 + n_seq_tokens, d_inner, n_seqs} + ggml_tensor * conv_x = ggml_concat(ctx0, conv, ggml_transpose(ctx0, x), 0); + + // copy last (d_conv - 1) columns back into the state cache + ggml_tensor * last_conv = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, conv_x->nb[1], conv_x->nb[2], n_seq_tokens*(conv_x->nb[0])); + + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, last_conv, + ggml_view_1d(ctx0, conv_states_all, + (d_conv - 1)*(d_inner)*(n_seqs), + kv_head*(d_conv - 1)*(d_inner)*ggml_element_size(conv_states_all)))); + + // 1D convolution + // The equivalent is to make a self-overlapping view of conv_x + // over d_conv columns at each stride in the 3rd dimension, + // then element-wise multiply that with the conv1d weight, + // then sum the elements of each row, + // (the last two steps are a dot product over rows (also doable with mul_mat)) + // then permute away the ne[0] dimension, + // and then you're left with the resulting x tensor. + // For simultaneous sequences, all sequences need to have the same length. + x = ggml_ssm_conv(ctx0, conv_x, model.layers[il].ssm_conv1d); + + // bias + x = ggml_add(ctx0, x, model.layers[il].ssm_conv1d_b); + + x = ggml_silu(ctx0, x); + } + + // ssm + { + // {d_inner, dt_rank + 2*d_state} @ {d_inner, n_seq_tokens, n_seqs} => {dt_rank + 2*d_state, n_seq_tokens, n_seqs} + ggml_tensor * x_db = build_lora_mm(model.layers[il].ssm_x, x); + // split + ggml_tensor * dt = ggml_view_3d(ctx0, x_db, dt_rank, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], 0); + ggml_tensor * B = ggml_view_3d(ctx0, x_db, d_state, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], ggml_element_size(x_db)*dt_rank); + ggml_tensor * C = ggml_view_3d(ctx0, x_db, d_state, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], ggml_element_size(x_db)*(dt_rank+d_state)); + + // Some Mamba variants (e.g. FalconMamba) apply RMS norm in B, C & Dt layers + if (ssm_dt_b_c_rms) { + dt = ggml_rms_norm(ctx0, dt, norm_rms_eps); + B = ggml_rms_norm(ctx0, B, norm_rms_eps); + C = ggml_rms_norm(ctx0, C, norm_rms_eps); + } + + // {dt_rank, d_inner} @ {dt_rank, n_seq_tokens, n_seqs} => {d_inner, n_seq_tokens, n_seqs} + dt = build_lora_mm(model.layers[il].ssm_dt, dt); + dt = ggml_add(ctx0, dt, model.layers[il].ssm_dt_b); + + // Custom operator to optimize the parallel associative scan + // as described in the Annex D of the Mamba paper. + // => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs} + ggml_tensor * y_ssm = ggml_ssm_scan(ctx0, ssm, x, dt, model.layers[il].ssm_a, B, C); + + // store last states + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, + ggml_view_1d(ctx0, y_ssm, d_state*d_inner*n_seqs, x->nb[3]), + ggml_view_1d(ctx0, ssm_states_all, d_state*d_inner*n_seqs, kv_head*d_state*d_inner*ggml_element_size(ssm_states_all)))); + + ggml_tensor * y = ggml_view_3d(ctx0, y_ssm, d_inner, n_seq_tokens, n_seqs, x->nb[1], x->nb[2], 0); + + // TODO: skip computing output earlier for unused tokens + + // {d_inner, n_seq_tokens, n_seqs} * {d_inner} => {d_inner, n_seq_tokens, n_seqs} + y = ggml_add(ctx0, y, ggml_mul(ctx0, x, model.layers[il].ssm_d)); + y = ggml_mul(ctx0, y, ggml_silu(ctx0, ggml_cont(ctx0, z))); + + // {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs} + cur = build_lora_mm(model.layers[il].ssm_out, y); + } + + // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens} + cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs); + //cb(cur, "mamba_out", il); + + return cur; + } +}; + +struct llm_build_command_r : public llm_graph_context { + llm_build_command_r(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + const float f_logit_scale = hparams.f_logit_scale; + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM, il); + cb(cur, "attn_norm", il); + ggml_tensor * ffn_inp = cur; + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + if (model.layers[il].attn_q_norm) { + Qcur = ggml_view_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens, + ggml_element_size(Qcur) * n_embd_head, + ggml_element_size(Qcur) * n_embd_head * n_head, + 0); + cb(Qcur, "Qcur", il); + Kcur = ggml_view_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens, + ggml_element_size(Kcur) * n_embd_head, + ggml_element_size(Kcur) * n_embd_head * n_head_kv, + 0); + cb(Kcur, "Kcur", il); + + Qcur = build_norm(Qcur, + model.layers[il].attn_q_norm, + NULL, + LLM_NORM, il); + cb(Qcur, "Qcur", il); + + Kcur = build_norm(Kcur, + model.layers[il].attn_k_norm, + NULL, + LLM_NORM, il); + cb(Kcur, "Kcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids); + } + + ggml_tensor * attn_out = cur; + + // feed-forward network + { + cur = build_ffn(ffn_inp, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + // add together residual + FFN + self-attention + cur = ggml_add(ctx0, cur, inpL); + cur = ggml_add(ctx0, cur, attn_out); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + if (f_logit_scale) { + cur = ggml_scale(ctx0, cur, f_logit_scale); + } + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_cohere2 : public llm_graph_context { + llm_build_cohere2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + const float f_logit_scale = hparams.f_logit_scale; + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, true); + + // sliding window switch pattern + const int32_t sliding_window_pattern = 4; + + for (int il = 0; il < n_layer; ++il) { + // three layers sliding window attention (window size 4096) and ROPE + // fourth layer uses global attention without positional embeddings + const bool is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); + + // norm + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il); + cb(cur, "attn_norm", il); + ggml_tensor * ffn_inp = cur; + + // self-attention + { + // rope freq factors for 128k context + ggml_tensor * rope_factors = static_cast(memory)->cbs.get_rope_factors(n_ctx_per_seq, il); + + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + if (is_sliding) { + Qcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, + beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow); + cb(Kcur, "Kcur", il); + } else { + // For non-sliding layers, just reshape without applying RoPE + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + cb(Qcur, "Qcur", il); + + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + cb(Kcur, "Kcur", il); + } + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids); + } + + ggml_tensor * attn_out = cur; + + // feed-forward network + { + cur = build_ffn(ffn_inp, model.layers[il].ffn_up, NULL, NULL, model.layers[il].ffn_gate, + NULL, NULL, model.layers[il].ffn_down, NULL, NULL, NULL, LLM_FFN_SILU, LLM_FFN_PAR, + il); + cb(cur, "ffn_out", il); + } + + // add together residual + FFN + self-attention + cur = ggml_add(ctx0, cur, inpL); + cur = ggml_add(ctx0, cur, attn_out); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + if (f_logit_scale) { + cur = ggml_scale(ctx0, cur, f_logit_scale); + } + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +// ref: https://allenai.org/olmo +// based on the original build_llama() function, changes: +// * non-parametric layer norm +// * clamp qkv +// * removed bias +// * removed MoE +struct llm_build_olmo : public llm_graph_context { + llm_build_olmo(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + NULL, NULL, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (hparams.f_clamp_kqv > 0.0f) { + Qcur = ggml_clamp(ctx0, Qcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (hparams.f_clamp_kqv > 0.0f) { + Kcur = ggml_clamp(ctx0, Kcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (hparams.f_clamp_kqv > 0.0f) { + Vcur = ggml_clamp(ctx0, Vcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, nullptr, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_norm(ffn_inp, + NULL, NULL, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + NULL, NULL, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_olmo2 : public llm_graph_context { + llm_build_olmo2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + cur = inpL; + + // self_attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, + LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, + LLM_NORM_RMS, il); + cb(Kcur, "Kcur_normed", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur_rope", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur_rope", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + cur = build_norm(cur, + model.layers[il].attn_post_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_post_norm", il); + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_ffn(ffn_inp, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = build_norm(cur, + model.layers[il].ffn_post_norm, NULL, + LLM_NORM_RMS, -1); + cb(cur, "ffn_post_norm", -1); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +// based on the build_qwen2moe() function, changes: +// * removed shared experts +// * removed bias +// * added q, k norm +struct llm_build_olmoe : public llm_graph_context { + llm_build_olmoe(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self_attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, + LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, + LLM_NORM_RMS, il); + cb(Kcur, "Kcur_normed", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur_rope", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur_rope", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // MoE branch + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, false, + false, 0.0, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(cur, "ffn_moe_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_openelm : public llm_graph_context { + llm_build_openelm(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + const int64_t n_head = hparams.n_head(il); + const int64_t n_head_kv = hparams.n_head_kv(il); + const int64_t n_head_qkv = 2*n_head_kv + n_head; + + cur = inpL; + ggml_tensor * residual = cur; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_reshape_3d(ctx0, cur, n_embd_head_k, n_head_qkv, n_tokens); + + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, cur->nb[1], cur->nb[2], 0)); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*n_head)); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*(n_head+n_head_kv))); + cb(Vcur, "Vcur", il); + + Qcur = build_norm(Qcur, + model.layers[il].attn_q_norm, NULL, + LLM_NORM_RMS, il); + cb(Qcur, "Qcur", il); + + Kcur = build_norm(Kcur, + model.layers[il].attn_k_norm, NULL, + LLM_NORM_RMS, il); + cb(Kcur, "Kcur", il); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + Vcur = ggml_reshape_2d(ctx0, Vcur, n_embd_head * n_head_kv, n_tokens); + cb(Qcur, "Vcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + residual = ggml_get_rows(ctx0, residual, inp_out_ids); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, residual, cur); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = inpL; + + // norm + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_gptneox : public llm_graph_context { + llm_build_gptneox(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + cur = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // ffn + if (hparams.use_par_res) { + // attention and ffn are computed in parallel + // x = x + attn(ln1(x)) + ffn(ln2(x)) + + ggml_tensor * attn_out = cur; + + cur = build_norm(inpL, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, inpL); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, attn_out); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } else { + // attention and ffn are computed sequentially + // x = x + attn(ln1(x)) + // x = x + ffn(ln2(x)) + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + } + + cur = build_norm(inpL, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_arctic : public llm_graph_context { + llm_build_arctic(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + ggml_tensor * ffn_out = ggml_add(ctx0, cur, ffn_inp); + cb(ffn_out, "ffn_out", il); + + // MoE + cur = build_norm(inpSA, + model.layers[il].ffn_norm_exps, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm_exps", il); + + cur = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, true, + false, 0.0, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(cur, "ffn_moe_out", il); + + cur = ggml_add(ctx0, cur, ffn_out); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_deepseek : public llm_graph_context { + llm_build_deepseek(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // rope freq factors for llama3; may return nullptr for llama2 and other models + ggml_tensor * rope_factors = static_cast(memory)->cbs.get_rope_factors(n_ctx_per_seq, il); + + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, kq_scale, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + // MoE branch + ggml_tensor * moe_out = + build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, false, + false, hparams.expert_weights_scale, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(moe_out, "ffn_moe_out", il); + + // FFN shared expert + { + ggml_tensor * ffn_shexp = build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_deepseek2 : public llm_graph_context { + llm_build_deepseek2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + bool is_lite = (hparams.n_layer == 27); + + // We have to pre-scale kq_scale and attn_factor to make the YaRN RoPE work correctly. + // See https://github.com/ggerganov/llama.cpp/discussions/7416 for detailed explanation. + const float mscale = attn_factor * (1.0f + hparams.rope_yarn_log_mul * logf(1.0f / freq_scale)); + const float kq_scale = 1.0f*mscale*mscale/sqrtf(float(hparams.n_embd_head_k)); + const float attn_factor_scaled = 1.0f / (1.0f + 0.1f * logf(1.0f / freq_scale)); + + const uint32_t n_embd_head_qk_rope = hparams.n_rot; + const uint32_t n_embd_head_qk_nope = hparams.n_embd_head_k - hparams.n_rot; + const uint32_t kv_lora_rank = hparams.n_lora_kv; + + ggml_tensor * cur; + ggml_tensor * inpL; + + // {n_embd, n_tokens} + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self_attention + { + ggml_tensor * q = NULL; + if (!is_lite) { + // {n_embd, q_lora_rank} * {n_embd, n_tokens} -> {q_lora_rank, n_tokens} + q = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur); + cb(q, "q", il); + + q = build_norm(q, + model.layers[il].attn_q_a_norm, NULL, + LLM_NORM_RMS, il); + cb(q, "q", il); + + // {q_lora_rank, n_head * hparams.n_embd_head_k} * {q_lora_rank, n_tokens} -> {n_head * hparams.n_embd_head_k, n_tokens} + q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q); + cb(q, "q", il); + } else { + q = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + cb(q, "q", il); + } + + // split into {n_head * n_embd_head_qk_nope, n_tokens} + ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, + ggml_row_size(q->type, hparams.n_embd_head_k), + ggml_row_size(q->type, hparams.n_embd_head_k * n_head), + 0); + cb(q_nope, "q_nope", il); + + // and {n_head * n_embd_head_qk_rope, n_tokens} + ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, + ggml_row_size(q->type, hparams.n_embd_head_k), + ggml_row_size(q->type, hparams.n_embd_head_k * n_head), + ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "q_pe", il); + + // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens} + ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); + cb(kv_pe_compresseed, "kv_pe_compresseed", il); + + // split into {kv_lora_rank, n_tokens} + ggml_tensor * kv_compressed = ggml_view_2d(ctx0, kv_pe_compresseed, kv_lora_rank, n_tokens, + kv_pe_compresseed->nb[1], + 0); + cb(kv_compressed, "kv_compressed", il); + + // and {n_embd_head_qk_rope, n_tokens} + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_pe_compresseed, n_embd_head_qk_rope, 1, n_tokens, + kv_pe_compresseed->nb[1], + kv_pe_compresseed->nb[1], + ggml_row_size(kv_pe_compresseed->type, kv_lora_rank)); + cb(k_pe, "k_pe", il); + + // TODO: the CUDA backend used to not support non-cont. (RMS) norm, investigate removing ggml_cont + kv_compressed = ggml_cont(ctx0, kv_compressed); + kv_compressed = build_norm(kv_compressed, + model.layers[il].attn_kv_a_norm, NULL, + LLM_NORM_RMS, il); + cb(kv_compressed, "kv_compressed", il); + + // {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)} * {kv_lora_rank, n_tokens} -> {n_head * (n_embd_head_qk_nope + n_embd_head_v), n_tokens} + ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_compressed); + cb(kv, "kv", il); + + // split into {n_head * n_embd_head_qk_nope, n_tokens} + ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens, + ggml_row_size(kv->type, n_embd_head_qk_nope + hparams.n_embd_head_v), + ggml_row_size(kv->type, n_head * (n_embd_head_qk_nope + hparams.n_embd_head_v)), + 0); + cb(k_nope, "k_nope", il); + + // and {n_head * n_embd_head_v, n_tokens} + ggml_tensor * v_states = ggml_view_3d(ctx0, kv, hparams.n_embd_head_v, n_head, n_tokens, + ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)), + ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)*n_head), + ggml_row_size(kv->type, (n_embd_head_qk_nope))); + cb(v_states, "v_states", il); + + v_states = ggml_cont(ctx0, v_states); + cb(v_states, "v_states", il); + + v_states = ggml_view_2d(ctx0, v_states, hparams.n_embd_head_v * n_head, n_tokens, + ggml_row_size(kv->type, hparams.n_embd_head_v * n_head), + 0); + cb(v_states, "v_states", il); + + q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this + q_pe = ggml_rope_ext( + ctx0, q_pe, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor_scaled, beta_fast, beta_slow + ); + cb(q_pe, "q_pe", il); + + // shared RoPE key + k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this + k_pe = ggml_rope_ext( + ctx0, k_pe, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor_scaled, beta_fast, beta_slow + ); + cb(k_pe, "k_pe", il); + + ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0); + cb(q_states, "q_states", il); + + ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0); + cb(k_states, "k_states", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + q_states, k_states, v_states, nullptr, kq_scale, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + // MoE branch + ggml_tensor * moe_out = + build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + true, hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(moe_out, "ffn_moe_out", il); + + // FFN shared expert + { + ggml_tensor * ffn_shexp = build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_bitnet : public llm_graph_context { + llm_build_bitnet(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + if (model.layers[il].wq_scale) { + Qcur = ggml_mul(ctx0, Qcur, model.layers[il].wq_scale); + } + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + // B1.K + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + if (model.layers[il].wk_scale) { + Kcur = ggml_mul(ctx0, Kcur, model.layers[il].wk_scale); + } + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + // B1.V + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + if (model.layers[il].wv_scale) { + Vcur = ggml_mul(ctx0, Vcur, model.layers[il].wv_scale); + } + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + NULL, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + + cur = build_norm(cur, + model.layers[il].attn_sub_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_sub_norm", il); + + cur = build_lora_mm(model.layers[il].wo, cur); + if (model.layers[il].wo_scale) { + cur = ggml_mul(ctx0, cur, model.layers[il].wo_scale); + } + if (model.layers[il].bo) { + cur = ggml_add(ctx0, cur, model.layers[il].bo); + } + cb(cur, "attn_o_out", il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward forward + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_scale, + model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_scale, + NULL, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_sub_out", il); + + cur = build_norm(cur, + model.layers[il].ffn_sub_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_sub_norm", il); + + cur = build_lora_mm(model.layers[il].ffn_down, cur); + if (model.layers[il].ffn_down_scale) { + cur = ggml_mul(ctx0, cur, model.layers[il].ffn_down_scale); + } + cb(cur, "ffn_down", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + // FIXME: do not use model.tok_embd directly, duplicate as model.output + cur = build_lora_mm(model.tok_embd, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_t5_enc : public llm_graph_context { + llm_build_t5_enc(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * pos_bucket_enc = build_inp_pos_bucket_enc(); + + auto * inp_attn = build_attn_inp_no_cache(); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm_enc, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_enc, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_enc, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_enc, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc; + ggml_tensor * kq_b = build_pos_bias(pos_bucket_enc, attn_rel_b); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo_enc, nullptr, + Qcur, Kcur, Vcur, kq_b, 1.0f, il); + cb(cur, "kqv_out", il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm_enc, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + // T5 uses relu, flan-T5 uses gelu-gated + cur = build_ffn(cur, + model.layers[il].ffn_up_enc, NULL, NULL, + model.layers[il].ffn_gate_enc, NULL, NULL, + model.layers[il].ffn_down_enc, NULL, NULL, + NULL, + model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU, + model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ, + il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + cb(cur, "result_embd", -1); + + cur = build_norm(cur, + model.output_norm_enc, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_t5_dec : public llm_graph_context { + llm_build_t5_dec(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + //const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * embd_enc = build_inp_cross_embd(); + ggml_tensor * pos_bucket_dec = build_inp_pos_bucket_dec(); + + const int64_t n_outputs_enc = embd_enc->ne[1]; + + auto * inp_attn_self = build_attn_inp_kv_unified(true, false); + auto * inp_attn_cross = build_attn_inp_cross(); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b; + ggml_tensor * kq_b = build_pos_bias(pos_bucket_dec, attn_rel_b); + + cur = build_attn(inp_attn_self, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, kq_b, 1.0f, il); + cb(cur, "kqv_out", il); + } + + cur = ggml_add(ctx0, cur, inpSA); + cb(cur, "cross_inp", il); + + ggml_tensor * inpCA = cur; + + // norm + cur = build_norm(cur, + model.layers[il].attn_norm_cross, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm_cross", il); + + // cross-attention + { + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_cross, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_cross, embd_enc); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_cross, embd_enc); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_outputs_enc); + + cur = build_attn(inp_attn_cross, gf, + model.layers[il].wo_cross, nullptr, + Qcur, Kcur, Vcur, nullptr, 1.0f, il); + cb(cur, "kqv_out", il); + + //ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3); + //ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3)); + + //ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); + //cb(kq, "kq", il); + + //kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias); + //cb(kq, "kq_soft_max_ext", il); + + //ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc))); + //cb(v, "v", il); + + //ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq); + //cb(kqv, "kqv", il); + + //ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3); + //cb(kqv_merged, "kqv_merged", il); + + //cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens); + //cb(cur, "kqv_merged_cont", il); + + //ggml_build_forward_expand(gf, cur); + + //cur = build_lora_mm(model.layers[il].wo_cross, cur); + //cb(cur, "kqv_out", il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + // T5 uses relu, flan-T5 uses gelu-gated + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU, + model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ, + il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + cb(cur, "result_embd", -1); + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_jais : public llm_graph_context { + llm_build_jais(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + cur = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*cur->nb[0]*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*cur->nb[0]*(n_embd))); + ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*cur->nb[0]*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/float(n_embd_head), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // add the input + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + // FF + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + inpL = ggml_add(ctx0, cur, ffn_inp); + cb(inpL, "l_out", il); + } + + cur = build_norm(inpL, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_chatglm : public llm_graph_context { + llm_build_chatglm(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, + model.layers[il].attn_norm, + NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + ggml_tensor * Qcur = nullptr; + ggml_tensor * Kcur = nullptr; + ggml_tensor * Vcur = nullptr; + + if (model.layers[il].wqkv == nullptr) { + Qcur = build_lora_mm(model.layers[il].wq, cur); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + } + Kcur = build_lora_mm(model.layers[il].wk, cur); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + } + Vcur = build_lora_mm(model.layers[il].wv, cur); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + } + } else { + cur = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + if (model.layers[il].bqkv) { + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + } + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + //printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor); + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur_rope", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur_rope", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + // Add the input + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // FF + { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SWIGLU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + + } + + inpL = ggml_add(ctx0, cur, ffn_inp); + cb(inpL, "l_out", il); + } + + cur = build_norm(inpL, + model.output_norm, + NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_nemotron : public llm_graph_context { + llm_build_nemotron(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + //GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_RELU_SQR, LLM_FFN_SEQ, il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, model.output_norm_b, + LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_exaone : public llm_graph_context { + llm_build_exaone(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // rope freq factors for llama3; may return nullptr for llama2 and other models + ggml_tensor * rope_factors = static_cast(memory)->cbs.get_rope_factors(n_ctx_per_seq, il); + + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_rwkv6_base : public llm_graph_context { + const llama_model & model; + + llm_build_rwkv6_base(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params), model(model) { + } + + ggml_tensor * build_rwkv6_channel_mix( + const llama_layer * layer, + ggml_tensor * cur, + ggml_tensor * x_prev, + llm_arch arch) const { + ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur); + switch (arch) { + case LLM_ARCH_RWKV6: + { + ggml_tensor * xk = ggml_add(ctx0, ggml_mul(ctx0, sx, layer->channel_mix_lerp_k), cur); + ggml_tensor * xr = ggml_add(ctx0, ggml_mul(ctx0, sx, layer->channel_mix_lerp_r), cur); + + ggml_tensor * r = ggml_sigmoid(ctx0, build_lora_mm(layer->channel_mix_receptance, xr)); + ggml_tensor * k = ggml_sqr( + ctx0, + ggml_relu( + ctx0, + build_lora_mm(layer->channel_mix_key, xk) + ) + ); + cur = ggml_mul(ctx0, r, build_lora_mm(layer->channel_mix_value, k)); + } break; + default: + GGML_ABORT("fatal error"); + } + + return cur; + } + + ggml_tensor * build_rwkv6_time_mix( + ggml_cgraph * gf, + ggml_tensor * cur, + ggml_tensor * x_prev, + ggml_tensor * state_copy, + ggml_tensor * state_mask, + const llama_ubatch & ubatch, + int il) const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + const auto n_tokens = ubatch.n_tokens; + const auto n_seqs = ubatch.n_seqs; + const auto n_embd = hparams.n_embd; + const auto head_size = hparams.wkv_head_size; + const auto n_head = n_embd / head_size; + const auto n_head_kv = hparams.n_head_kv(il); + + const auto kv_head = kv_self->head; + + const auto & layer = model.layers[il]; + + bool is_qrwkv = layer.time_mix_first == nullptr; + + ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur); + ggml_tensor * xxx = ggml_add(ctx0, ggml_mul(ctx0, sx, layer.time_mix_lerp_x), cur); + + xxx = ggml_reshape_4d( + ctx0, + ggml_tanh( + ctx0, + ggml_mul_mat(ctx0, layer.time_mix_w1, xxx) + ), + layer.time_mix_w1->ne[1] / 5, 1, 5, n_tokens + ); + + xxx = ggml_cont(ctx0, ggml_permute(ctx0, xxx, 0, 1, 3, 2)); + + xxx = ggml_mul_mat( + ctx0, + ggml_reshape_4d( + ctx0, + layer.time_mix_w2, + layer.time_mix_w2->ne[0], layer.time_mix_w2->ne[1], 1, 5 + ), + xxx + ); + + ggml_tensor *xw, *xk, *xv, *xr, *xg; + if (layer.time_mix_lerp_fused) { + // fusing these weights makes some performance improvement + sx = ggml_reshape_3d(ctx0, sx, n_embd, 1, n_tokens); + cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens); + xxx = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xxx, layer.time_mix_lerp_fused), sx), cur); + xw = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], 0); + xk = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * sizeof(float)); + xv = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 2 * sizeof(float)); + xr = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 3 * sizeof(float)); + xg = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 4 * sizeof(float)); + } else { + // for backward compatibility + xw = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], 0); + xk = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * sizeof(float)); + xv = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 2 * sizeof(float)); + xr = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 3 * sizeof(float)); + xg = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 4 * sizeof(float)); + + xw = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xw, layer.time_mix_lerp_w), sx), cur); + xk = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xk, layer.time_mix_lerp_k), sx), cur); + xv = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xv, layer.time_mix_lerp_v), sx), cur); + xr = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xr, layer.time_mix_lerp_r), sx), cur); + xg = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xg, layer.time_mix_lerp_g), sx), cur); + } + + ggml_tensor * r = build_lora_mm(layer.time_mix_receptance, xr); + ggml_tensor * k = build_lora_mm(layer.time_mix_key, xk); + ggml_tensor * v = build_lora_mm(layer.time_mix_value, xv); + if (layer.time_mix_receptance_b) { + r = ggml_add(ctx0, r, layer.time_mix_receptance_b); + } + if (layer.time_mix_key_b) { + k = ggml_add(ctx0, k, layer.time_mix_key_b); + } + if (layer.time_mix_value_b) { + v = ggml_add(ctx0, v, layer.time_mix_value_b); + } + + ggml_tensor * g = build_lora_mm(layer.time_mix_gate, xg); + if (is_qrwkv) { + g = ggml_sigmoid(ctx0, g); + } else { + g = ggml_silu(ctx0, g); + } + + if (n_head_kv != 0 && n_head_kv != n_head) { + GGML_ASSERT(n_head % n_head_kv == 0); + k = ggml_reshape_4d(ctx0, k, head_size, 1, n_head_kv, n_tokens); + v = ggml_reshape_4d(ctx0, v, head_size, 1, n_head_kv, n_tokens); + ggml_tensor * tmp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, head_size, n_head / n_head_kv, n_head_kv, n_tokens); + k = ggml_repeat(ctx0, k, tmp); + v = ggml_repeat(ctx0, v, tmp); + } + + k = ggml_reshape_3d(ctx0, k, head_size, n_head, n_tokens); + v = ggml_reshape_3d(ctx0, v, head_size, n_head, n_tokens); + r = ggml_reshape_3d(ctx0, r, head_size, n_head, n_tokens); + + ggml_tensor * w = ggml_mul_mat( + ctx0, + layer.time_mix_decay_w2, + ggml_tanh( + ctx0, + ggml_mul_mat(ctx0, layer.time_mix_decay_w1, xw) + ) + ); + + w = ggml_add(ctx0, w, layer.time_mix_decay); + w = ggml_exp(ctx0, ggml_neg(ctx0, ggml_exp(ctx0, w))); + w = ggml_reshape_3d(ctx0, w, head_size, n_head, n_tokens); + + if (is_qrwkv) { + // k = k * (1 - w) + k = ggml_sub(ctx0, k, ggml_mul(ctx0, k, w)); + } + + ggml_tensor * wkv_state = build_copy_mask_state( + gf, kv_self->v_l[il], state_copy, state_mask, + hparams.n_embd_v_s(), n_seqs); + + ggml_tensor * wkv_output; + if (is_qrwkv) { + wkv_output = ggml_gated_linear_attn(ctx0, k, v, r, w, wkv_state, pow(head_size, -0.5f)); + } else { + wkv_output = ggml_rwkv_wkv6(ctx0, k, v, r, layer.time_mix_first, w, wkv_state); + } + cur = ggml_view_1d(ctx0, wkv_output, n_embd * n_tokens, 0); + wkv_state = ggml_view_1d(ctx0, wkv_output, n_embd * head_size * n_seqs, n_embd * n_tokens * sizeof(float)); + + ggml_build_forward_expand( + gf, + ggml_cpy( + ctx0, + wkv_state, + ggml_view_1d( + ctx0, + kv_self->v_l[il], + hparams.n_embd_v_s() * n_seqs, + hparams.n_embd_v_s() * kv_head * ggml_element_size(kv_self->v_l[il]) + ) + ) + ); + + if (!is_qrwkv) { + // group norm with head_count groups + cur = ggml_reshape_3d(ctx0, cur, n_embd / n_head, n_head, n_tokens); + cur = ggml_norm(ctx0, cur, 64e-5f); + + // Convert back to regular vectors. + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.time_mix_ln), layer.time_mix_ln_b); + } else { + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + } + + cur = ggml_mul(ctx0, cur, g); + cur = build_lora_mm(layer.time_mix_output, cur); + + return cur; + } +}; + +struct llm_build_rwkv6 : public llm_build_rwkv6_base { + llm_build_rwkv6(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_build_rwkv6_base(model, params) { + GGML_ASSERT(hparams.token_shift_count == 2); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1); + + ggml_tensor * state_copy = build_inp_s_copy(); + ggml_tensor * state_mask = build_inp_s_mask(); + + const auto n_embd = hparams.n_embd; + const auto n_seq_tokens = ubatch.n_seq_tokens; + const auto n_seqs = ubatch.n_seqs; + + for (int il = 0; il < n_layer; ++il) { + const llama_layer * layer = &model.layers[il]; + + ggml_tensor * token_shift = build_rwkv_token_shift_load( + gf, state_copy, state_mask, ubatch, il + ); + + ggml_tensor * att_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], 0); + ggml_tensor * ffn_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], n_embd * ggml_element_size(token_shift)); + + ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM, il); + cb(att_norm, "attn_norm", il); + + ggml_tensor * x_prev = ggml_concat( + ctx0, + att_shift, + ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0), + 1 + ); + + cur = build_rwkv6_time_mix(gf, att_norm, x_prev, state_copy, state_mask, ubatch, il); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + ggml_tensor * ffn_norm = build_norm(ffn_inp, layer->attn_norm_2, layer->attn_norm_2_b, LLM_NORM, il); + cb(ffn_norm, "ffn_norm", il); + + x_prev = ggml_concat( + ctx0, + ffn_shift, + ggml_view_3d(ctx0, ffn_norm, n_embd, n_seq_tokens - 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], 0), + 1 + ); + + cur = build_rwkv6_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV6); + cur = ggml_add(ctx0, cur, ffn_inp); + + token_shift = ggml_concat(ctx0, + ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm)), + ggml_view_3d(ctx0, ffn_norm, n_embd, 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(ffn_norm)), + 1 + ); + ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il)); + + if (hparams.rescale_every_n_layers != 0 && (il + 1) % hparams.rescale_every_n_layers == 0) { + cur = ggml_scale(ctx0, cur, 0.5F); + } + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + + cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +// ref: https://huggingface.co/recursal/QRWKV6-32B-Instruct-Preview-v0.1/blob/main/modeling_rwkv6qwen2.py +struct llm_build_rwkv6qwen2 : public llm_build_rwkv6_base { + llm_build_rwkv6qwen2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_build_rwkv6_base(model, params) { + GGML_ASSERT(n_embd == hparams.n_embd_k_s()); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * state_copy = build_inp_s_copy(); + ggml_tensor * state_mask = build_inp_s_mask(); + + const auto n_embd = hparams.n_embd; + const auto n_seq_tokens = ubatch.n_seq_tokens; + const auto n_seqs = ubatch.n_seqs; + + inpL = build_inp_embd(model.tok_embd); + + for (int il = 0; il < n_layer; ++il) { + const llama_layer * layer = &model.layers[il]; + + ggml_tensor * token_shift = build_rwkv_token_shift_load( + gf, state_copy, state_mask, ubatch, il + ); + + ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM_RMS, il); + cb(att_norm, "attn_norm", il); + + ggml_tensor * x_prev = ggml_concat( + ctx0, + token_shift, + ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0), + 1 + ); + + cur = build_rwkv6_time_mix(gf, att_norm, x_prev, state_copy, state_mask, ubatch, il); + + token_shift = ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm)); + ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il)); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + + cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +// ref: https://github.com/facebookresearch/chameleon +// based on the original build_llama() function, changes: +// * qk-norm +// * swin-norm +// * removed bias +// * removed MoE +struct llm_build_chameleon : public llm_graph_context { + llm_build_chameleon(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_unified(true, false); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + // norm + if (hparams.swin_norm) { + cur = inpL; + } else { + cur = build_norm(inpL, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + } + + // self-attention + { + // compute Q and K and RoPE them + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + if (model.layers[il].attn_q_norm) { + Qcur = ggml_view_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens, + ggml_element_size(Qcur) * n_embd_head, + ggml_element_size(Qcur) * n_embd_head * n_head, + 0); + cb(Qcur, "Qcur", il); + + Qcur = build_norm(Qcur, + model.layers[il].attn_q_norm, + model.layers[il].attn_q_norm_b, + LLM_NORM, il); + cb(Qcur, "Qcur", il); + } + + if (model.layers[il].attn_k_norm) { + Kcur = ggml_view_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens, + ggml_element_size(Kcur) * n_embd_head, + ggml_element_size(Kcur) * n_embd_head * n_head_kv, + 0); + cb(Kcur, "Kcur", il); + + Kcur = build_norm(Kcur, + model.layers[il].attn_k_norm, + model.layers[il].attn_k_norm_b, + LLM_NORM, il); + cb(Kcur, "Kcur", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, nullptr, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + + if (hparams.swin_norm) { + cur = build_norm(cur, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, il); + } + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + if (!hparams.swin_norm) { + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + } + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + if (hparams.swin_norm) { + cur = build_norm(cur, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, + model.output_norm, NULL, + LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output_with_img_logits", -1); + + // TODO: this suppresses the output of image tokens, which is required to enable text-only outputs. + // Needs to be removed once image outputs are supported. + int img_token_end_idx = 8196; + int img_token_start_idx = 4; + int num_img_tokens = img_token_end_idx - img_token_start_idx; + // creates 1d tensor of size num_img_tokens and values -FLT_MAX, + // which ensures that text token values are always at least larger than image token values + ggml_tensor * img_logits = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, num_img_tokens); + img_logits = ggml_clamp(ctx0, img_logits, -FLT_MAX, -FLT_MAX); + cb(img_logits, "img_logits", -1); + + cur = ggml_set_1d(ctx0, cur, img_logits, ggml_element_size(cur) * img_token_start_idx); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_wavtokenizer_dec : public llm_graph_context { + llm_build_wavtokenizer_dec(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, inpL)); + + cur = ggml_conv_1d_ph(ctx0, model.conv1d, cur, 1, 1); + cur = ggml_add(ctx0, cur, model.conv1d_b); + + // posnet + for (uint32_t il = 0; il < hparams.posnet.n_layer; ++il) { + const auto & layer = model.layers[il].posnet; + + inpL = cur; + + switch (il) { + case 0: + case 1: + case 3: + case 4: + { + cur = build_norm(cur, + layer.norm1, + layer.norm1_b, + LLM_NORM_GROUP, 0); + + cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur); + + cur = ggml_conv_1d_ph(ctx0, layer.conv1, cur, 1, 1); + cur = ggml_add(ctx0, cur, layer.conv1_b); + + cur = build_norm(cur, + layer.norm2, + layer.norm2_b, + LLM_NORM_GROUP, 0); + + cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur); + + cur = ggml_conv_1d_ph(ctx0, layer.conv2, cur, 1, 1); + cur = ggml_add(ctx0, cur, layer.conv2_b); + + cur = ggml_add(ctx0, cur, inpL); + } break; + case 2: + { + cur = build_norm(cur, + layer.attn_norm, + layer.attn_norm_b, + LLM_NORM_GROUP, 0); + + ggml_tensor * q; + ggml_tensor * k; + ggml_tensor * v; + + q = ggml_conv_1d_ph(ctx0, layer.attn_q, cur, 1, 1); + k = ggml_conv_1d_ph(ctx0, layer.attn_k, cur, 1, 1); + v = ggml_conv_1d_ph(ctx0, layer.attn_v, cur, 1, 1); + + q = ggml_add(ctx0, q, layer.attn_q_b); + k = ggml_add(ctx0, k, layer.attn_k_b); + v = ggml_add(ctx0, v, layer.attn_v_b); + + q = ggml_cont(ctx0, ggml_transpose(ctx0, q)); + k = ggml_cont(ctx0, ggml_transpose(ctx0, k)); + + ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); + + kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(hparams.posnet.n_embd)), 0.0f); + + cur = ggml_mul_mat(ctx0, kq, v); + + cur = ggml_conv_1d_ph(ctx0, layer.attn_o, cur, 1, 1); + cur = ggml_add(ctx0, cur, layer.attn_o_b); + + cur = ggml_add(ctx0, cur, inpL); + } break; + case 5: + { + cur = build_norm(cur, + layer.norm, + layer.norm_b, + LLM_NORM_GROUP, 0); + } break; + default: GGML_ABORT("unknown posnet layer"); + }; + } + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + + cur = build_norm(cur, + model.tok_norm, + model.tok_norm_b, + LLM_NORM, -1); + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + + inpL = cur; + + // convnext + for (uint32_t il = 0; il < hparams.convnext.n_layer; ++il) { + const auto & layer = model.layers[il].convnext; + + cur = inpL; + + cur = ggml_conv_1d_dw_ph(ctx0, layer.dw, cur, 1, 1); + cur = ggml_add(ctx0, cur, layer.dw_b); + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + + cur = build_norm(cur, + layer.norm, + layer.norm_b, + LLM_NORM, -1); + + cur = build_ffn(cur, + layer.pw1, layer.pw1_b, NULL, + NULL, NULL, NULL, + layer.pw2, layer.pw2_b, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, il); + + cur = ggml_mul(ctx0, cur, layer.gamma); + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + + inpL = ggml_add(ctx0, cur, inpL); + } + + cur = inpL; + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + + cur = build_norm(cur, + model.output_norm, + model.output_norm_b, + LLM_NORM, -1); + + // lm_head + cur = build_lora_mm(model.output, cur); + + cur = ggml_add(ctx0, cur, model.output_b); + + cb(cur, "result_embd", -1); + res->t_embd = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +llama_memory_i * llama_model::create_memory() const { + llama_memory_i * res; + + switch (arch) { + case LLM_ARCH_RWKV6: + case LLM_ARCH_RWKV6QWEN2: + case LLM_ARCH_MAMBA: + { + res = new llama_kv_cache_unified(hparams, { + /*.get_rope_factors =*/ nullptr + }); + } break; + default: + { + res = new llama_kv_cache_unified(hparams, { + /*.get_rope_factors =*/ [this](uint32_t n_ctx_per_seq, int il) { + // choose long/short freq factors based on the context size + if (layers[il].rope_freqs != nullptr) { + return layers[il].rope_freqs; + } + + if (n_ctx_per_seq > hparams.n_ctx_orig_yarn) { + return layers[il].rope_long; + } + + return layers[il].rope_short; + } + }); + } + } + + return res; +} + +llm_graph_result_ptr llama_model::build_graph( + const llm_graph_params & params, + ggml_cgraph * gf, + llm_graph_type type) const { + std::unique_ptr llm; + + switch (arch) { + case LLM_ARCH_LLAMA: + case LLM_ARCH_MINICPM: + case LLM_ARCH_GRANITE: + case LLM_ARCH_GRANITE_MOE: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_DECI: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_BAICHUAN: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_FALCON: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_GROK: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_STARCODER: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_REFACT: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_BERT: + case LLM_ARCH_JINA_BERT_V2: + case LLM_ARCH_NOMIC_BERT: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_BLOOM: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_MPT: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_STABLELM: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_QWEN: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_QWEN2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_QWEN2VL: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_QWEN2MOE: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_PHI2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_PHI3: + case LLM_ARCH_PHIMOE: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_PLAMO: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_GPT2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_CODESHELL: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_ORION: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_INTERNLM2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_MINICPM3: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_GEMMA: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_GEMMA2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_GEMMA3: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_STARCODER2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_MAMBA: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_XVERSE: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_COMMAND_R: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_COHERE2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_DBRX: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_OLMO: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_OLMO2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_OLMOE: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_OPENELM: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_GPTNEOX: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_ARCTIC: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_DEEPSEEK: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_DEEPSEEK2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_CHATGLM: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_BITNET: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_T5: + { + switch (type) { + case LLM_GRAPH_TYPE_ENCODER: + llm = std::make_unique(*this, params, gf); + break; + case LLM_GRAPH_TYPE_DEFAULT: + case LLM_GRAPH_TYPE_DECODER: + llm = std::make_unique(*this, params, gf); + break; + default: + GGML_ABORT("invalid graph type"); + }; + } break; + //case LLM_ARCH_T5ENCODER: + // { + // llm.build_t5_enc(gf); + // } break; + case LLM_ARCH_JAIS: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_NEMOTRON: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_EXAONE: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_RWKV6: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_RWKV6QWEN2: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_CHAMELEON: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_WAVTOKENIZER_DEC: + { + llm = std::make_unique(*this, params, gf); + } break; + default: + GGML_ABORT("fatal error"); + } + + // add on pooling layer + llm->build_pooling(gf, cls, cls_b, cls_out, cls_out_b); + + return std::move(llm->res); +} + // // interface implementation // -struct llama_model_params llama_model_default_params() { - struct llama_model_params result = { +llama_model_params llama_model_default_params() { + llama_model_params result = { /*.devices =*/ nullptr, /*.n_gpu_layers =*/ 0, /*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER, @@ -3861,59 +11167,59 @@ struct llama_model_params llama_model_default_params() { return result; } -const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model) { +const llama_vocab * llama_model_get_vocab(const llama_model * model) { return &model->vocab; } -void llama_free_model(struct llama_model * model) { +void llama_free_model(llama_model * model) { llama_model_free(model); } -void llama_model_free(struct llama_model * model) { +void llama_model_free(llama_model * model) { delete model; } -int32_t llama_model_n_ctx_train(const struct llama_model * model) { +int32_t llama_model_n_ctx_train(const llama_model * model) { return model->hparams.n_ctx_train; } -int32_t llama_model_n_embd(const struct llama_model * model) { +int32_t llama_model_n_embd(const llama_model * model) { return model->hparams.n_embd; } -int32_t llama_model_n_layer(const struct llama_model * model) { +int32_t llama_model_n_layer(const llama_model * model) { return model->hparams.n_layer; } -int32_t llama_model_n_head(const struct llama_model * model) { +int32_t llama_model_n_head(const llama_model * model) { return model->hparams.n_head(); } -int32_t llama_model_n_head_kv(const struct llama_model * model) { +int32_t llama_model_n_head_kv(const llama_model * model) { return model->hparams.n_head_kv(); } // deprecated -int32_t llama_n_ctx_train(const struct llama_model * model) { +int32_t llama_n_ctx_train(const llama_model * model) { return llama_model_n_ctx_train(model); } // deprecated -int32_t llama_n_embd(const struct llama_model * model) { +int32_t llama_n_embd(const llama_model * model) { return llama_model_n_embd(model); } // deprecated -int32_t llama_n_layer(const struct llama_model * model) { +int32_t llama_n_layer(const llama_model * model) { return llama_model_n_layer(model); } // deprecated -int32_t llama_n_head(const struct llama_model * model) { +int32_t llama_n_head(const llama_model * model) { return llama_model_n_head(model); } -enum llama_rope_type llama_model_rope_type(const struct llama_model * model) { +llama_rope_type llama_model_rope_type(const llama_model * model) { switch (model->arch) { // these models do not use RoPE case LLM_ARCH_GPT2: @@ -3992,11 +11298,11 @@ enum llama_rope_type llama_model_rope_type(const struct llama_model * model) { return LLAMA_ROPE_TYPE_NONE; } -float llama_model_rope_freq_scale_train(const struct llama_model * model) { +float llama_model_rope_freq_scale_train(const llama_model * model) { return model->hparams.rope_freq_scale_train; } -int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size) { +int32_t llama_model_meta_val_str(const llama_model * model, const char * key, char * buf, size_t buf_size) { const auto & it = model->gguf_kv.find(key); if (it == model->gguf_kv.end()) { if (buf_size > 0) { @@ -4007,11 +11313,11 @@ int32_t llama_model_meta_val_str(const struct llama_model * model, const char * return snprintf(buf, buf_size, "%s", it->second.c_str()); } -int32_t llama_model_meta_count(const struct llama_model * model) { +int32_t llama_model_meta_count(const llama_model * model) { return (int)model->gguf_kv.size(); } -int32_t llama_model_meta_key_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size) { +int32_t llama_model_meta_key_by_index(const llama_model * model, int i, char * buf, size_t buf_size) { if (i < 0 || i >= (int)model->gguf_kv.size()) { if (buf_size > 0) { buf[0] = '\0'; @@ -4023,7 +11329,7 @@ int32_t llama_model_meta_key_by_index(const struct llama_model * model, int i, c return snprintf(buf, buf_size, "%s", it->first.c_str()); } -int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size) { +int32_t llama_model_meta_val_str_by_index(const llama_model * model, int32_t i, char * buf, size_t buf_size) { if (i < 0 || i >= (int)model->gguf_kv.size()) { if (buf_size > 0) { buf[0] = '\0'; @@ -4035,15 +11341,15 @@ int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int3 return snprintf(buf, buf_size, "%s", it->second.c_str()); } -int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) { +int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size) { return snprintf(buf, buf_size, "%s", model->desc().c_str()); } -uint64_t llama_model_size(const struct llama_model * model) { +uint64_t llama_model_size(const llama_model * model) { return model->size(); } -const char * llama_model_chat_template(const struct llama_model * model, const char * name) { +const char * llama_model_chat_template(const llama_model * model, const char * name) { const auto key = name ? LLM_KV(model->arch, name)(LLM_KV_TOKENIZER_CHAT_TEMPLATE_N) : LLM_KV(model->arch)(LLM_KV_TOKENIZER_CHAT_TEMPLATE); const auto & it = model->gguf_kv.find(key); @@ -4054,11 +11360,11 @@ const char * llama_model_chat_template(const struct llama_model * model, const c return it->second.c_str(); } -uint64_t llama_model_n_params(const struct llama_model * model) { +uint64_t llama_model_n_params(const llama_model * model) { return model->n_elements(); } -bool llama_model_has_encoder(const struct llama_model * model) { +bool llama_model_has_encoder(const llama_model * model) { switch (model->arch) { case LLM_ARCH_T5: return true; case LLM_ARCH_T5ENCODER: return true; @@ -4066,22 +11372,26 @@ bool llama_model_has_encoder(const struct llama_model * model) { } } -bool llama_model_has_decoder(const struct llama_model * model) { +bool llama_model_has_decoder(const llama_model * model) { switch (model->arch) { case LLM_ARCH_T5ENCODER: return false; default: return true; } } -llama_token llama_model_decoder_start_token(const struct llama_model * model) { +llama_token llama_model_decoder_start_token(const llama_model * model) { return model->hparams.dec_start_token_id; } -bool llama_model_is_recurrent(const struct llama_model * model) { +bool llama_model_is_recurrent(const llama_model * model) { switch (model->arch) { - case LLM_ARCH_MAMBA: return true; - case LLM_ARCH_RWKV6: return true; - case LLM_ARCH_RWKV6QWEN2: return true; - default: return false; + case LLM_ARCH_MAMBA: return true; + case LLM_ARCH_RWKV6: return true; + case LLM_ARCH_RWKV6QWEN2: return true; + default: return false; } } + +const std::vector> & llama_internal_get_tensor_map(const llama_model * model) { + return model->tensors_by_name; +} diff --git a/src/llama-model.h b/src/llama-model.h index a7c3044478..55c26a92b0 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -2,7 +2,9 @@ #include "llama.h" #include "llama-arch.h" +#include "llama-graph.h" #include "llama-hparams.h" +#include "llama-memory.h" #include "llama-vocab.h" #include @@ -10,6 +12,8 @@ #include #include +struct llama_cparams; +struct llama_ubatch; struct llama_model_loader; // available models @@ -347,7 +351,7 @@ struct llama_model { std::string desc() const; size_t size() const; - size_t max_nodes() const; + size_t n_tensors() const; size_t n_devices() const; // total number of parameters in the model @@ -362,9 +366,22 @@ struct llama_model { const struct ggml_tensor * get_tensor(const char * name) const; + // TODO: move this to new llm_arch_model_i interface + llama_memory_i * create_memory() const; // TODO: params + + // TODO: move this to new llm_arch_model_i interface + llm_graph_result_ptr build_graph( + const llm_graph_params & params, + ggml_cgraph * gf, + llm_graph_type type) const; + private: struct impl; std::unique_ptr pimpl; }; const char * llm_type_name(llm_type type); + +// For internal test use +// TODO: remove +const std::vector> & llama_internal_get_tensor_map(const llama_model * model); diff --git a/src/llama.cpp b/src/llama.cpp index 4a4e914901..81e1dd1d08 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -2,9517 +2,28 @@ #include "llama-chat.h" #include "llama-mmap.h" -#include "llama-context.h" #include "llama-vocab.h" -#include "llama-sampling.h" -#include "llama-kv-cache.h" #include "llama-model-loader.h" #include "llama-model.h" #include "ggml.h" -#include "ggml-alloc.h" #include "ggml-backend.h" -#include "ggml-cpp.h" #include -#include -#include -#include -#include #include #include #include #include #include -#include #if defined(_MSC_VER) #pragma warning(disable: 4244 4267) // possible loss of data #endif -// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback -static int llama_model_load(const std::string & fname, std::vector & splits, llama_model & model, llama_model_params & params) { - // loading time will be recalculated after the first eval, so - // we take page faults deferred by mmap() into consideration - model.t_load_us = 0; - time_meas tm(model.t_load_us); - - model.t_start_us = tm.t_start_us; - - try { - llama_model_loader ml(fname, splits, params.use_mmap, params.check_tensors, params.kv_overrides); - - ml.print_info(); - - model.hparams.vocab_only = params.vocab_only; - - try { - model.load_arch(ml); - } catch(const std::exception & e) { - throw std::runtime_error("error loading model architecture: " + std::string(e.what())); - } - try { - model.load_hparams(ml); - } catch(const std::exception & e) { - throw std::runtime_error("error loading model hyperparameters: " + std::string(e.what())); - } - try { - model.load_vocab(ml); - } catch(const std::exception & e) { - throw std::runtime_error("error loading model vocabulary: " + std::string(e.what())); - } - - model.load_stats(ml); - model.print_info(); - - if (params.vocab_only) { - LLAMA_LOG_INFO("%s: vocab only - skipping tensors\n", __func__); - return 0; - } - - if (!model.load_tensors(ml)) { - return -2; - } - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what()); - return -1; - } - - return 0; -} - -// -// llm_build -// - -using llm_build_cb = std::function; - -enum llm_ffn_op_type { - LLM_FFN_SILU, - LLM_FFN_GELU, - LLM_FFN_RELU, - LLM_FFN_RELU_SQR, - LLM_FFN_SWIGLU, -}; - -enum llm_ffn_gate_type { - LLM_FFN_SEQ, - LLM_FFN_PAR, // ffn_gate is parallel to ffn_up -}; - -enum llm_norm_type { - LLM_NORM, - LLM_NORM_RMS, - LLM_NORM_GROUP, -}; - -static struct ggml_tensor * llm_build_inp_embd( - struct ggml_context * ctx, - struct llama_context & lctx, - const llama_hparams & hparams, - const llama_ubatch & ubatch, - struct ggml_tensor * tok_embd, - const llm_build_cb & cb) { - const int64_t n_embd = hparams.n_embd; - - struct ggml_tensor * inpL; - - if (ubatch.token) { - lctx.inp_tokens = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ubatch.n_tokens); - cb(lctx.inp_tokens, "inp_tokens", -1); - ggml_set_input(lctx.inp_tokens); - - inpL = ggml_get_rows(ctx, tok_embd, lctx.inp_tokens); - - // apply lora for embedding tokens if needed - for (auto & it : lctx.lora) { - struct llama_adapter_lora_weight * lw = it.first->get_weight(tok_embd); - if (lw == nullptr) { - continue; - } - const float adapter_scale = it.second; - const float scale = lw->get_scale(it.first->alpha, adapter_scale); - struct ggml_tensor * inpL_delta = ggml_scale(ctx, ggml_mul_mat( - ctx, lw->b, // non-transposed lora_b - ggml_get_rows(ctx, lw->a, lctx.inp_tokens) - ), scale); - inpL = ggml_add(ctx, inpL, inpL_delta); - } - } else { - lctx.inp_embd = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, ubatch.n_tokens); - inpL = lctx.inp_embd; - ggml_set_input(lctx.inp_embd); - } - - // For Granite architecture - if (hparams.f_embedding_scale != 0.0f) { - inpL = ggml_scale(ctx, inpL, hparams.f_embedding_scale); - } - - cb(inpL, "inp_embd", -1); - - return inpL; -} - -static void llm_build_kv_store( - struct ggml_context * ctx, - const llama_hparams & hparams, - const llama_cparams & cparams, - const llama_kv_cache & kv, - struct ggml_cgraph * graph, - struct ggml_tensor * k_cur, - struct ggml_tensor * v_cur, - int32_t n_tokens, - int32_t kv_head, - const llm_build_cb & cb, - int64_t il) { - const int64_t n_ctx = cparams.n_ctx; - - const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); - const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(il); - - GGML_ASSERT(kv.size == n_ctx); - - struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, kv.k_l[il], n_tokens*n_embd_k_gqa, ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa)*kv_head); - cb(k_cache_view, "k_cache_view", il); - - // note: storing RoPE-ed version of K in the KV cache - ggml_build_forward_expand(graph, ggml_cpy(ctx, k_cur, k_cache_view)); - - assert(v_cur->ne[0] == n_embd_v_gqa && v_cur->ne[1] == n_tokens); - - struct ggml_tensor * v_cache_view = nullptr; - - if (cparams.flash_attn) { - v_cache_view = ggml_view_1d(ctx, kv.v_l[il], n_tokens*n_embd_v_gqa, ggml_row_size(kv.v_l[il]->type, n_embd_v_gqa)*kv_head); - } else { - // note: the V cache is transposed when not using flash attention - v_cache_view = ggml_view_2d(ctx, kv.v_l[il], n_tokens, n_embd_v_gqa, - ( n_ctx)*ggml_element_size(kv.v_l[il]), - (kv_head)*ggml_element_size(kv.v_l[il])); - - v_cur = ggml_transpose(ctx, v_cur); - } - cb(v_cache_view, "v_cache_view", il); - - ggml_build_forward_expand(graph, ggml_cpy(ctx, v_cur, v_cache_view)); -} - -// do mat_mul, while optionally apply lora -static struct ggml_tensor * llm_build_lora_mm( - struct llama_context & lctx, - struct ggml_context * ctx0, - struct ggml_tensor * w, - struct ggml_tensor * cur) { - struct ggml_tensor * res = ggml_mul_mat(ctx0, w, cur); - for (auto & it : lctx.lora) { - struct llama_adapter_lora_weight * lw = it.first->get_weight(w); - if (lw == nullptr) { - continue; - } - const float adapter_scale = it.second; - const float scale = lw->get_scale(it.first->alpha, adapter_scale); - struct ggml_tensor * ab_cur = ggml_mul_mat( - ctx0, lw->b, - ggml_mul_mat(ctx0, lw->a, cur) - ); - ab_cur = ggml_scale(ctx0, ab_cur, scale); - res = ggml_add(ctx0, res, ab_cur); - } - return res; -} - -// do mat_mul_id, while optionally apply lora -static struct ggml_tensor * llm_build_lora_mm_id( - struct llama_context & lctx, - struct ggml_context * ctx0, - struct ggml_tensor * w, // struct ggml_tensor * as - struct ggml_tensor * cur, // struct ggml_tensor * b - struct ggml_tensor * ids) { - struct ggml_tensor * res = ggml_mul_mat_id(ctx0, w, cur, ids); - for (auto & it : lctx.lora) { - struct llama_adapter_lora_weight * lw = it.first->get_weight(w); - if (lw == nullptr) { - continue; - } - const float alpha = it.first->alpha; - const float rank = (float) lw->b->ne[0]; - const float scale = alpha ? it.second * alpha / rank : it.second; - struct ggml_tensor * ab_cur = ggml_mul_mat_id( - ctx0, lw->b, - ggml_mul_mat_id(ctx0, lw->a, cur, ids), - ids - ); - ab_cur = ggml_scale(ctx0, ab_cur, scale); - res = ggml_add(ctx0, res, ab_cur); - } - return res; -} - -static struct ggml_tensor * llm_build_norm( - struct ggml_context * ctx, - struct ggml_tensor * cur, - const llama_hparams & hparams, - struct ggml_tensor * mw, - struct ggml_tensor * mb, - llm_norm_type type, - const llm_build_cb & cb, - int il) { - switch (type) { - case LLM_NORM: cur = ggml_norm (ctx, cur, hparams.f_norm_eps); break; - case LLM_NORM_RMS: cur = ggml_rms_norm (ctx, cur, hparams.f_norm_rms_eps); break; - case LLM_NORM_GROUP: - { - cur = ggml_reshape_3d(ctx, cur, cur->ne[0], 1, cur->ne[1]); - cur = ggml_group_norm(ctx, cur, hparams.n_norm_groups, hparams.f_norm_group_eps); - cur = ggml_reshape_2d(ctx, cur, cur->ne[0], cur->ne[2]); - } break; - } - - if (mw || mb) { - cb(cur, "norm", il); - } - - if (mw) { - cur = ggml_mul(ctx, cur, mw); - if (mb) { - cb(cur, "norm_w", il); - } - } - - if (mb) { - cur = ggml_add(ctx, cur, mb); - } - - return cur; -} - -static struct ggml_tensor * llm_build_ffn( - struct ggml_context * ctx, - struct llama_context & lctx, - struct ggml_tensor * cur, - struct ggml_tensor * up, - struct ggml_tensor * up_b, - struct ggml_tensor * up_s, - struct ggml_tensor * gate, - struct ggml_tensor * gate_b, - struct ggml_tensor * gate_s, - struct ggml_tensor * down, - struct ggml_tensor * down_b, - struct ggml_tensor * down_s, - struct ggml_tensor * act_scales, - llm_ffn_op_type type_op, - llm_ffn_gate_type type_gate, - const llm_build_cb & cb, - int il) { - struct ggml_tensor * tmp = up ? llm_build_lora_mm(lctx, ctx, up, cur) : cur; - cb(tmp, "ffn_up", il); - - if (up_b) { - tmp = ggml_add(ctx, tmp, up_b); - cb(tmp, "ffn_up_b", il); - } - - if (up_s) { - tmp = ggml_mul(ctx, tmp, up_s); - cb(tmp, "ffn_up_s", il); - } - - if (gate) { - switch (type_gate) { - case LLM_FFN_SEQ: - { - cur = llm_build_lora_mm(lctx, ctx, gate, tmp); - cb(cur, "ffn_gate", il); - } break; - case LLM_FFN_PAR: - { - cur = llm_build_lora_mm(lctx, ctx, gate, cur); - cb(cur, "ffn_gate", il); - } break; - } - - if (gate_b) { - cur = ggml_add(ctx, cur, gate_b); - cb(cur, "ffn_gate_b", il); - } - - if (gate_s) { - cur = ggml_mul(ctx, cur, gate_s); - cb(cur, "ffn_gate_s", il); - } - - } else { - cur = tmp; - } - - switch (type_op) { - case LLM_FFN_SILU: - { - cur = ggml_silu(ctx, cur); - cb(cur, "ffn_silu", il); - } break; - case LLM_FFN_GELU: - { - cur = ggml_gelu(ctx, cur); - cb(cur, "ffn_gelu", il); - if (act_scales != NULL) { - cur = ggml_div(ctx, cur, act_scales); - cb(cur, "ffn_act", il); - } - } break; - case LLM_FFN_RELU: - { - cur = ggml_relu(ctx, cur); - cb(cur, "ffn_relu", il); - } break; - case LLM_FFN_RELU_SQR: - { - cur = ggml_relu(ctx, cur); - cb(cur, "ffn_relu", il); - - cur = ggml_sqr(ctx, cur); - cb(cur, "ffn_sqr(relu)", il); - } break; - case LLM_FFN_SWIGLU: - { - // Project to 4h. If using swiglu double the output width, see https://arxiv.org/pdf/2002.05202.pdf - int64_t split_point = cur->ne[0] / 2; - struct ggml_tensor * x0 = ggml_cont(ctx, ggml_view_2d(ctx, cur, split_point, cur->ne[1], cur->nb[1], 0)); - struct ggml_tensor * x1 = ggml_cont(ctx, ggml_view_2d(ctx, cur, split_point, cur->ne[1], cur->nb[1], split_point * ggml_element_size(cur))); - - x0 = ggml_silu(ctx, x0); - cb(cur, "ffn_silu", il); - - cur = ggml_mul(ctx, x0, x1); - cb(cur, "ffn_mul", il); - } break; - } - - if (type_gate == LLM_FFN_PAR) { - cur = ggml_mul(ctx, cur, tmp); - cb(cur, "ffn_gate_par", il); - } - - if (down) { - cur = llm_build_lora_mm(lctx, ctx, down, cur); - } - - if (down_b) { - cb(cur, "ffn_down", il); - } - - if (down_b) { - cur = ggml_add(ctx, cur, down_b); - } - - if (down_s) { - cur = ggml_mul(ctx, cur, down_s); - cb(cur, "ffn_down_s", il); - } - - return cur; -} - -static struct ggml_tensor * llm_build_moe_ffn( - struct ggml_context * ctx, - struct llama_context & lctx, - struct ggml_tensor * cur, - struct ggml_tensor * gate_inp, - struct ggml_tensor * up_exps, - struct ggml_tensor * gate_exps, - struct ggml_tensor * down_exps, - struct ggml_tensor * exp_probs_b, - int64_t n_expert, - int64_t n_expert_used, - llm_ffn_op_type type_op, - bool norm_w, - bool scale_w, - float w_scale, -llama_expert_gating_func_type gating_op, - const llm_build_cb & cb, - int il) { - int64_t n_embd = cur->ne[0]; - int64_t n_tokens = cur->ne[1]; - - ggml_tensor * logits = llm_build_lora_mm(lctx, ctx, gate_inp, cur); // [n_expert, n_tokens] - cb(logits, "ffn_moe_logits", il); - - ggml_tensor * probs = nullptr; - switch (gating_op) { - case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: - { - probs = ggml_soft_max(ctx, logits); // [n_expert, n_tokens] - } break; - case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: - { - probs = ggml_sigmoid(ctx, logits); // [n_expert, n_tokens] - } break; - default: - GGML_ABORT("fatal error"); - } - cb(probs, "ffn_moe_probs", il); - - // add experts selection bias - introduced in DeepSeek V3 - // leave probs unbiased as it's later used to get expert weights - ggml_tensor * selection_probs = probs; - if (exp_probs_b != nullptr) { - selection_probs = ggml_add(ctx, probs, exp_probs_b); - cb(selection_probs, "ffn_moe_probs_biased", il); - } - - // select experts - ggml_tensor * selected_experts = ggml_top_k(ctx, selection_probs, n_expert_used); // [n_expert_used, n_tokens] - cb(selected_experts->src[0], "ffn_moe_argsort", il); - cb(selected_experts, "ffn_moe_topk", il); - - ggml_tensor * weights = ggml_get_rows(ctx, - ggml_reshape_3d(ctx, probs, 1, n_expert, n_tokens), selected_experts); // [1, n_expert_used, n_tokens] - cb(weights, "ffn_moe_weights", il); - - if (norm_w) { - weights = ggml_reshape_2d(ctx, weights, n_expert_used, n_tokens); - - ggml_tensor * weights_sum = ggml_sum_rows(ctx, weights); // [1, n_tokens] - cb(weights_sum, "ffn_moe_weights_sum", il); - - weights = ggml_div(ctx, weights, weights_sum); // [n_expert_used, n_tokens] - cb(weights, "ffn_moe_weights_norm", il); - - weights = ggml_reshape_3d(ctx, weights, 1, n_expert_used, n_tokens); - } - if (scale_w) { - weights = ggml_scale(ctx, weights, w_scale); - cb(weights, "ffn_moe_weights_scaled", il); - } - - cur = ggml_reshape_3d(ctx, cur, n_embd, 1, n_tokens); - ggml_tensor * up = llm_build_lora_mm_id(lctx, ctx, up_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens] - cb(up, "ffn_moe_up", il); - - ggml_tensor * gate = llm_build_lora_mm_id(lctx, ctx, gate_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens] - cb(gate, "ffn_moe_gate", il); - - switch (type_op) { - case LLM_FFN_SILU: - { - gate = ggml_silu(ctx, gate); - cb(gate, "ffn_moe_silu", il); - } break; - case LLM_FFN_GELU: - { - gate = ggml_gelu(ctx, gate); - cb(gate, "ffn_moe_gelu", il); - } break; - default: - GGML_ABORT("fatal error"); - } - - ggml_tensor * par = ggml_mul(ctx, up, gate); // [n_ff, n_expert_used, n_tokens] - cb(par, "ffn_moe_gate_par", il); - - ggml_tensor * experts = llm_build_lora_mm_id(lctx, ctx, down_exps, par, selected_experts); // [n_embd, n_expert_used, n_tokens] - cb(experts, "ffn_moe_down", il); - - experts = ggml_mul(ctx, experts, weights); - - // aggregate experts - ggml_tensor * moe_out = nullptr; - for (int i = 0; i < n_expert_used; ++i) { - ggml_tensor * cur_expert = ggml_view_2d(ctx, experts, n_embd, n_tokens, - experts->nb[2], i*experts->nb[1]); - - if (i == 0) { - moe_out = cur_expert; - } else { - moe_out = ggml_add(ctx, moe_out, cur_expert); - } - } - - if (n_expert_used == 1) { - // avoid returning a non-contiguous tensor - moe_out = ggml_cont(ctx, moe_out); - } - - return moe_out; -} - -static struct ggml_tensor * llm_build_kqv( - struct ggml_context * ctx, - struct llama_context & lctx, - const llama_kv_cache & kv, - struct ggml_cgraph * graph, - struct ggml_tensor * wo, - struct ggml_tensor * wo_b, - struct ggml_tensor * q_cur, - struct ggml_tensor * kq_mask, - int32_t n_tokens, - int32_t n_kv, - float kq_scale, - const llm_build_cb & cb, - int il) { - const llama_model & model = lctx.model; - const llama_hparams & hparams = lctx.model.hparams; - const llama_cparams & cparams = lctx.cparams; - - const int64_t n_ctx = cparams.n_ctx; - const int64_t n_head = hparams.n_head(il); - const int64_t n_head_kv = hparams.n_head_kv(il); - const int64_t n_embd_head_k = hparams.n_embd_head_k; - const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); - const int64_t n_embd_head_v = hparams.n_embd_head_v; - const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(il); - - struct ggml_tensor * q = ggml_permute(ctx, q_cur, 0, 2, 1, 3); - cb(q, "q", il); - - struct ggml_tensor * k = - ggml_view_3d(ctx, kv.k_l[il], - n_embd_head_k, n_kv, n_head_kv, - ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa), - ggml_row_size(kv.k_l[il]->type, n_embd_head_k), - 0); - cb(k, "k", il); - - struct ggml_tensor * cur; - - if (cparams.flash_attn) { - GGML_UNUSED(model); - GGML_UNUSED(n_ctx); - - // split cached v into n_head heads (not transposed) - struct ggml_tensor * v = - ggml_view_3d(ctx, kv.v_l[il], - n_embd_head_v, n_kv, n_head_kv, - ggml_row_size(kv.v_l[il]->type, n_embd_v_gqa), - ggml_row_size(kv.v_l[il]->type, n_embd_head_v), - 0); - cb(v, "v", il); - - cur = ggml_flash_attn_ext(ctx, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias, - hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f); - - ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); - - cur = ggml_reshape_2d(ctx, cur, n_embd_head_v*n_head, n_tokens); - } else { - struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); - cb(kq, "kq", il); - - // note: this op tends to require high floating point range - // while for some models F16 is enough, for others it is not, so we default to F32 here - ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - - if (model.arch == LLM_ARCH_GROK) { - // need to do the following: - // multiply by attn_output_multiplyer of 0.08838834764831845 - // and then : - // kq = 30 * tanh(kq / 30) - // before the softmax below - - kq = ggml_tanh(ctx, ggml_scale(ctx, kq, 0.08838834764831845f/30.0f)); - kq = ggml_scale(ctx, kq, 30); - } - - if (hparams.attn_soft_cap) { - kq = ggml_scale(ctx, kq, 1.0f / hparams.f_attn_logit_softcapping); - kq = ggml_tanh(ctx, kq); - kq = ggml_scale(ctx, kq, hparams.f_attn_logit_softcapping); - } - - kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias); - cb(kq, "kq_soft_max_ext", il); - - GGML_ASSERT(kv.size == n_ctx); - - // split cached v into n_head heads - struct ggml_tensor * v = - ggml_view_3d(ctx, kv.v_l[il], - n_kv, n_embd_head_v, n_head_kv, - ggml_element_size(kv.v_l[il])*n_ctx, - ggml_element_size(kv.v_l[il])*n_ctx*n_embd_head_v, - 0); - cb(v, "v", il); - - struct ggml_tensor * kqv = ggml_mul_mat(ctx, v, kq); - cb(kqv, "kqv", il); - - struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3); - cb(kqv_merged, "kqv_merged", il); - - cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_v*n_head, n_tokens); - cb(cur, "kqv_merged_cont", il); - } - - ggml_build_forward_expand(graph, cur); - - if (wo) { - cur = llm_build_lora_mm(lctx, ctx, wo, cur); - } - - if (wo_b) { - cb(cur, "kqv_wo", il); - } - - if (wo_b) { - cur = ggml_add(ctx, cur, wo_b); - } - - return cur; -} - -static struct ggml_tensor * llm_build_kv( - struct ggml_context * ctx, - struct llama_context & lctx, - const llama_kv_cache & kv, - struct ggml_cgraph * graph, - struct ggml_tensor * wo, - struct ggml_tensor * wo_b, - struct ggml_tensor * k_cur, - struct ggml_tensor * v_cur, - struct ggml_tensor * q_cur, - struct ggml_tensor * kq_mask, - int32_t n_tokens, - int32_t kv_head, - int32_t n_kv, - float kq_scale, - const llm_build_cb & cb, - int il) { - const llama_hparams & hparams = lctx.model.hparams; - const llama_cparams & cparams = lctx.cparams; - - // these nodes are added to the graph together so that they are not reordered - // by doing so, the number of splits in the graph is reduced - ggml_build_forward_expand(graph, q_cur); - ggml_build_forward_expand(graph, k_cur); - ggml_build_forward_expand(graph, v_cur); - - llm_build_kv_store(ctx, hparams, cparams, kv, graph, k_cur, v_cur, n_tokens, kv_head, cb, il); - - struct ggml_tensor * cur; - - cur = llm_build_kqv(ctx, lctx, kv, graph, wo, wo_b, q_cur, kq_mask, n_tokens, n_kv, kq_scale, cb, il); - cb(cur, "kqv_out", il); - - return cur; -} - -static struct ggml_tensor * llm_build_copy_mask_state( - struct ggml_context * ctx, - struct ggml_cgraph * graph, - struct ggml_tensor * s, - struct ggml_tensor * state_copy, - struct ggml_tensor * state_mask, - int32_t n_state, - int32_t kv_size, - int32_t kv_head, - int32_t n_kv, - int32_t n_seqs) { - struct ggml_tensor * states = ggml_reshape_2d(ctx, s, n_state, kv_size); - - // copy states - // NOTE: assuming the copy destinations are ALL contained between kv_head and kv_head + n_kv - // this shrinks the tensors's ne[1] to n_kv - states = ggml_get_rows(ctx, states, state_copy); - - // clear states of sequences which are starting at the beginning of this batch - // FIXME: zero-out NANs? - states = ggml_mul(ctx, states, state_mask); - - // copy states which won't be changed further (between n_seqs and n_kv) - ggml_build_forward_expand(graph, - ggml_cpy(ctx, - ggml_view_1d(ctx, states, n_state*(n_kv - n_seqs), n_seqs*n_state*ggml_element_size(states)), - ggml_view_1d(ctx, s, n_state*(n_kv - n_seqs), (kv_head + n_seqs)*n_state*ggml_element_size(s)))); - - // the part of the states that will be used and modified - return ggml_view_2d(ctx, states, n_state, n_seqs, states->nb[1], 0); -} - -// TODO: split -static struct ggml_tensor * llm_build_mamba( - struct ggml_context * ctx, - struct llama_context & lctx, - const llama_ubatch & ubatch, - struct ggml_cgraph * graph, - struct ggml_tensor * cur, - struct ggml_tensor * state_copy, - struct ggml_tensor * state_mask, - int32_t kv_head, - int32_t n_kv, - const llm_build_cb & cb, - int il) { - const llama_model & model = lctx.model; - const llama_hparams & hparams = model.hparams; - const llama_kv_cache & kv = lctx.kv_self; - const int64_t d_conv = hparams.ssm_d_conv; - const int64_t d_inner = hparams.ssm_d_inner; - const int64_t d_state = hparams.ssm_d_state; - const int64_t dt_rank = hparams.ssm_dt_rank; - const int64_t n_seqs = ubatch.n_seqs; - // Some variants of Mamba arch (e.g. FalconMamba do apply layer norm on B and Dt layers) - const bool ssm_dt_b_c_rms = hparams.ssm_dt_b_c_rms; - // Use the same RMS norm as the final layer norm - const float norm_rms_eps = hparams.f_norm_rms_eps; - - const int64_t n_seq_tokens = ubatch.n_seq_tokens; - - GGML_ASSERT(n_seqs != 0); - GGML_ASSERT(ubatch.equal_seqs); - GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs); - - struct ggml_tensor * conv_states_all = kv.k_l[il]; - struct ggml_tensor * ssm_states_all = kv.v_l[il]; - - // (ab)using the KV cache to store the states - struct ggml_tensor * conv = llm_build_copy_mask_state(ctx, - graph, conv_states_all, state_copy, state_mask, - hparams.n_embd_k_s(), kv.size, kv_head, n_kv, n_seqs); - conv = ggml_reshape_3d(ctx, conv, d_conv - 1, d_inner, n_seqs); - struct ggml_tensor * ssm = llm_build_copy_mask_state(ctx, - graph, ssm_states_all, state_copy, state_mask, - hparams.n_embd_v_s(), kv.size, kv_head, n_kv, n_seqs); - ssm = ggml_reshape_3d(ctx, ssm, d_state, d_inner, n_seqs); - - // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs} - cur = ggml_reshape_3d(ctx, cur, cur->ne[0], n_seq_tokens, n_seqs); - - // {n_embd, 2*d_inner} @ {n_embd, n_seq_tokens, n_seqs} => {2*d_inner, n_seq_tokens, n_seqs} - struct ggml_tensor * xz = llm_build_lora_mm(lctx, ctx, model.layers[il].ssm_in, cur); - // split the above in two - // => {d_inner, n_seq_tokens, n_seqs} - struct ggml_tensor * x = ggml_view_3d(ctx, xz, d_inner, xz->ne[1], xz->ne[2], xz->nb[1], xz->nb[2], 0); - struct ggml_tensor * z = ggml_view_3d(ctx, xz, d_inner, xz->ne[1], xz->ne[2], xz->nb[1], xz->nb[2], d_inner*ggml_element_size(xz)); - - // conv - { - // => {d_conv - 1 + n_seq_tokens, d_inner, n_seqs} - struct ggml_tensor * conv_x = ggml_concat(ctx, conv, ggml_transpose(ctx, x), 0); - - // copy last (d_conv - 1) columns back into the state cache - struct ggml_tensor * last_conv = ggml_view_3d(ctx, conv_x, d_conv - 1, d_inner, n_seqs, conv_x->nb[1], conv_x->nb[2], n_seq_tokens*(conv_x->nb[0])); - - ggml_build_forward_expand(graph, - ggml_cpy(ctx, last_conv, - ggml_view_1d(ctx, conv_states_all, - (d_conv - 1)*(d_inner)*(n_seqs), - kv_head*(d_conv - 1)*(d_inner)*ggml_element_size(conv_states_all)))); - - // 1D convolution - // The equivalent is to make a self-overlapping view of conv_x - // over d_conv columns at each stride in the 3rd dimension, - // then element-wise multiply that with the conv1d weight, - // then sum the elements of each row, - // (the last two steps are a dot product over rows (also doable with mul_mat)) - // then permute away the ne[0] dimension, - // and then you're left with the resulting x tensor. - // For simultaneous sequences, all sequences need to have the same length. - x = ggml_ssm_conv(ctx, conv_x, model.layers[il].ssm_conv1d); - - // bias - x = ggml_add(ctx, x, model.layers[il].ssm_conv1d_b); - - x = ggml_silu(ctx, x); - } - - // ssm - { - // {d_inner, dt_rank + 2*d_state} @ {d_inner, n_seq_tokens, n_seqs} => {dt_rank + 2*d_state, n_seq_tokens, n_seqs} - struct ggml_tensor * x_db = llm_build_lora_mm(lctx, ctx, model.layers[il].ssm_x, x); - // split - struct ggml_tensor * dt = ggml_view_3d(ctx, x_db, dt_rank, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], 0); - struct ggml_tensor * B = ggml_view_3d(ctx, x_db, d_state, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], ggml_element_size(x_db)*dt_rank); - struct ggml_tensor * C = ggml_view_3d(ctx, x_db, d_state, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], ggml_element_size(x_db)*(dt_rank+d_state)); - - // Some Mamba variants (e.g. FalconMamba) apply RMS norm in B, C & Dt layers - if (ssm_dt_b_c_rms) { - dt = ggml_rms_norm(ctx, dt, norm_rms_eps); - B = ggml_rms_norm(ctx, B, norm_rms_eps); - C = ggml_rms_norm(ctx, C, norm_rms_eps); - } - - // {dt_rank, d_inner} @ {dt_rank, n_seq_tokens, n_seqs} => {d_inner, n_seq_tokens, n_seqs} - dt = llm_build_lora_mm(lctx, ctx, model.layers[il].ssm_dt, dt); - dt = ggml_add(ctx, dt, model.layers[il].ssm_dt_b); - - // Custom operator to optimize the parallel associative scan - // as described in the Annex D of the Mamba paper. - // => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs} - struct ggml_tensor * y_ssm = ggml_ssm_scan(ctx, ssm, x, dt, model.layers[il].ssm_a, B, C); - - // store last states - ggml_build_forward_expand(graph, - ggml_cpy(ctx, - ggml_view_1d(ctx, y_ssm, d_state*d_inner*n_seqs, x->nb[3]), - ggml_view_1d(ctx, ssm_states_all, d_state*d_inner*n_seqs, kv_head*d_state*d_inner*ggml_element_size(ssm_states_all)))); - - struct ggml_tensor * y = ggml_view_3d(ctx, y_ssm, d_inner, n_seq_tokens, n_seqs, x->nb[1], x->nb[2], 0); - - // TODO: skip computing output earlier for unused tokens - - // {d_inner, n_seq_tokens, n_seqs} * {d_inner} => {d_inner, n_seq_tokens, n_seqs} - y = ggml_add(ctx, y, ggml_mul(ctx, x, model.layers[il].ssm_d)); - y = ggml_mul(ctx, y, ggml_silu(ctx, ggml_cont(ctx, z))); - - // {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs} - cur = llm_build_lora_mm(lctx, ctx, model.layers[il].ssm_out, y); - } - - // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens} - cur = ggml_reshape_2d(ctx, cur, cur->ne[0], n_seq_tokens * n_seqs); - cb(cur, "mamba_out", il); - - return cur; -} - -static struct ggml_tensor * llm_build_rwkv6_time_mix( - struct llama_context & lctx, - struct ggml_context * ctx, - const struct llama_layer * layer, - struct ggml_tensor * cur, - struct ggml_tensor * x_prev, - struct ggml_tensor ** wkv_state, - size_t wkv_head_size, - size_t head_count_kv) { - size_t n_embd = cur->ne[0]; - size_t n_seq_tokens = cur->ne[1]; - size_t n_seqs = cur->ne[2]; - - size_t head_size = wkv_head_size; - size_t head_count = n_embd / head_size; - - size_t n_tokens = n_seqs * n_seq_tokens; - - bool is_qrwkv = layer->time_mix_first == nullptr; - - struct ggml_tensor * sx = ggml_sub(ctx, x_prev, cur); - - sx = ggml_reshape_2d(ctx, sx, n_embd, n_tokens); - cur = ggml_reshape_2d(ctx, cur, n_embd, n_tokens); - - struct ggml_tensor * xxx = ggml_add(ctx, ggml_mul(ctx, sx, layer->time_mix_lerp_x), cur); - - xxx = ggml_reshape_4d( - ctx, - ggml_tanh( - ctx, - ggml_mul_mat(ctx, layer->time_mix_w1, xxx) - ), - layer->time_mix_w1->ne[1] / 5, 1, 5, n_tokens - ); - - xxx = ggml_cont(ctx, ggml_permute(ctx, xxx, 0, 1, 3, 2)); - - xxx = ggml_mul_mat( - ctx, - ggml_reshape_4d( - ctx, - layer->time_mix_w2, - layer->time_mix_w2->ne[0], layer->time_mix_w2->ne[1], 1, 5 - ), - xxx - ); - - struct ggml_tensor *xw, *xk, *xv, *xr, *xg; - if (layer->time_mix_lerp_fused) { - // fusing these weights makes some performance improvement - sx = ggml_reshape_3d(ctx, sx, n_embd, 1, n_tokens); - cur = ggml_reshape_3d(ctx, cur, n_embd, 1, n_tokens); - xxx = ggml_add(ctx, ggml_mul(ctx, ggml_add(ctx, xxx, layer->time_mix_lerp_fused), sx), cur); - xw = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], 0); - xk = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * sizeof(float)); - xv = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 2 * sizeof(float)); - xr = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 3 * sizeof(float)); - xg = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 4 * sizeof(float)); - } else { - // for backward compatibility - xw = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], 0); - xk = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * sizeof(float)); - xv = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 2 * sizeof(float)); - xr = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 3 * sizeof(float)); - xg = ggml_view_2d(ctx, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 4 * sizeof(float)); - - xw = ggml_add(ctx, ggml_mul(ctx, ggml_add(ctx, xw, layer->time_mix_lerp_w), sx), cur); - xk = ggml_add(ctx, ggml_mul(ctx, ggml_add(ctx, xk, layer->time_mix_lerp_k), sx), cur); - xv = ggml_add(ctx, ggml_mul(ctx, ggml_add(ctx, xv, layer->time_mix_lerp_v), sx), cur); - xr = ggml_add(ctx, ggml_mul(ctx, ggml_add(ctx, xr, layer->time_mix_lerp_r), sx), cur); - xg = ggml_add(ctx, ggml_mul(ctx, ggml_add(ctx, xg, layer->time_mix_lerp_g), sx), cur); - } - - struct ggml_tensor * r = llm_build_lora_mm(lctx, ctx, layer->time_mix_receptance, xr); - struct ggml_tensor * k = llm_build_lora_mm(lctx, ctx, layer->time_mix_key, xk); - struct ggml_tensor * v = llm_build_lora_mm(lctx, ctx, layer->time_mix_value, xv); - if (layer->time_mix_receptance_b) { - r = ggml_add(ctx, r, layer->time_mix_receptance_b); - } - if (layer->time_mix_key_b) { - k = ggml_add(ctx, k, layer->time_mix_key_b); - } - if (layer->time_mix_value_b) { - v = ggml_add(ctx, v, layer->time_mix_value_b); - } - - struct ggml_tensor * g = llm_build_lora_mm(lctx, ctx, layer->time_mix_gate, xg); - if (is_qrwkv) { - g = ggml_sigmoid(ctx, g); - } else { - g = ggml_silu(ctx, g); - } - - if (head_count_kv != head_count) { - GGML_ASSERT(head_count % head_count_kv == 0); - k = ggml_reshape_4d(ctx, k, head_size, 1, head_count_kv, n_tokens); - v = ggml_reshape_4d(ctx, v, head_size, 1, head_count_kv, n_tokens); - struct ggml_tensor * tmp = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, head_size, head_count / head_count_kv, head_count_kv, n_tokens); - k = ggml_repeat(ctx, k, tmp); - v = ggml_repeat(ctx, v, tmp); - } - - k = ggml_reshape_3d(ctx, k, head_size, head_count, n_tokens); - v = ggml_reshape_3d(ctx, v, head_size, head_count, n_tokens); - r = ggml_reshape_3d(ctx, r, head_size, head_count, n_tokens); - - struct ggml_tensor * w = ggml_mul_mat( - ctx, - layer->time_mix_decay_w2, - ggml_tanh( - ctx, - ggml_mul_mat(ctx, layer->time_mix_decay_w1, xw) - ) - ); - - w = ggml_add(ctx, w, layer->time_mix_decay); - w = ggml_exp(ctx, ggml_neg(ctx, ggml_exp(ctx, w))); - w = ggml_reshape_3d(ctx, w, head_size, head_count, n_tokens); - - if (is_qrwkv) { - // k = k * (1 - w) - k = ggml_sub(ctx, k, ggml_mul(ctx, k, w)); - } - - struct ggml_tensor * wkv_output; - if (!layer->time_mix_first) { - wkv_output = ggml_gated_linear_attn(ctx, k, v, r, w, *wkv_state, pow(head_size, -0.5f)); - } else { - wkv_output = ggml_rwkv_wkv6(ctx, k, v, r, layer->time_mix_first, w, *wkv_state); - } - cur = ggml_view_1d(ctx, wkv_output, n_embd * n_tokens, 0); - *wkv_state = ggml_view_1d(ctx, wkv_output, n_embd * head_size * n_seqs, n_embd * n_tokens * sizeof(float)); - - if (!is_qrwkv) { - // group norm with head_count groups - cur = ggml_reshape_3d(ctx, cur, n_embd / head_count, head_count, n_tokens); - cur = ggml_norm(ctx, cur, 64e-5f); - - // Convert back to regular vectors. - cur = ggml_reshape_2d(ctx, cur, n_embd, n_tokens); - cur = ggml_add(ctx, ggml_mul(ctx, cur, layer->time_mix_ln), layer->time_mix_ln_b); - } else { - cur = ggml_reshape_2d(ctx, cur, n_embd, n_tokens); - } - - cur = ggml_mul(ctx, cur, g); - cur = llm_build_lora_mm(lctx, ctx, layer->time_mix_output, cur); - - return ggml_reshape_3d(ctx, cur, n_embd, n_seq_tokens, n_seqs); -} - -static struct ggml_tensor * llm_build_rwkv6_channel_mix( - struct llama_context & lctx, - struct ggml_context * ctx, - const struct llama_layer * layer, - struct ggml_tensor * cur, - struct ggml_tensor * x_prev) { - struct ggml_tensor * sx = ggml_sub(ctx, x_prev, cur); - struct ggml_tensor * xk = ggml_add(ctx, ggml_mul(ctx, sx, layer->channel_mix_lerp_k), cur); - struct ggml_tensor * xr = ggml_add(ctx, ggml_mul(ctx, sx, layer->channel_mix_lerp_r), cur); - - struct ggml_tensor * r = ggml_sigmoid(ctx, llm_build_lora_mm(lctx, ctx, layer->channel_mix_receptance, xr)); - struct ggml_tensor * k = ggml_sqr( - ctx, - ggml_relu( - ctx, - llm_build_lora_mm(lctx, ctx, layer->channel_mix_key, xk) - ) - ); - - return ggml_mul(ctx, r, llm_build_lora_mm(lctx, ctx, layer->channel_mix_value, k)); -} - -struct llm_build_context { - const llama_model & model; - llama_context & lctx; - const llama_hparams & hparams; - const llama_cparams & cparams; - const llama_ubatch & ubatch; - const llama_kv_cache & kv_self; - - const int64_t n_embd; - const int64_t n_layer; - const int64_t n_rot; - const int64_t n_ctx; // user-specified context size (can be different from n_ctx_train) - const int64_t n_head; - const int64_t n_head_kv; - const int64_t n_embd_head_k; - const int64_t n_embd_k_gqa; - const int64_t n_embd_head_v; - const int64_t n_embd_v_gqa; - const int64_t n_expert; - const int64_t n_expert_used; - - const float freq_base; - const float freq_scale; - const float ext_factor; - const float attn_factor; - const float beta_fast; - const float beta_slow; - const float norm_eps; - const float norm_rms_eps; - - const int32_t n_tokens; - const int32_t n_kv; // size of KV cache to consider (n_kv <= kv_self.size) - const int32_t n_outputs; - const int32_t n_outputs_enc; - const int32_t kv_head; // index of where we store new KV data in the cache - const int32_t n_ctx_orig; - - const bool flash_attn; - - const enum llama_pooling_type pooling_type; - const enum llama_rope_type rope_type; - - const llm_build_cb & cb; - - std::vector & buf_compute_meta; - - struct ggml_context * ctx0 = nullptr; - - // TODO: consider making the entire interface noexcept - llm_build_context( - llama_context & lctx, - const llama_ubatch & ubatch, - const llm_build_cb & cb, - bool worst_case) : - model (lctx.model), - lctx (lctx), - hparams (model.hparams), - cparams (lctx.cparams), - ubatch (ubatch), - kv_self (lctx.kv_self), - n_embd (hparams.n_embd), - n_layer (hparams.n_layer), - n_rot (hparams.n_rot), - n_ctx (cparams.n_ctx), - n_head (hparams.n_head()), - n_head_kv (hparams.n_head_kv()), - n_embd_head_k (hparams.n_embd_head_k), - n_embd_k_gqa (hparams.n_embd_k_gqa()), - n_embd_head_v (hparams.n_embd_head_v), - n_embd_v_gqa (hparams.n_embd_v_gqa()), - n_expert (hparams.n_expert), - n_expert_used (hparams.n_expert_used), - freq_base (cparams.rope_freq_base), - freq_scale (cparams.rope_freq_scale), - ext_factor (cparams.yarn_ext_factor), - attn_factor (cparams.yarn_attn_factor), - beta_fast (cparams.yarn_beta_fast), - beta_slow (cparams.yarn_beta_slow), - norm_eps (hparams.f_norm_eps), - norm_rms_eps (hparams.f_norm_rms_eps), - n_tokens (ubatch.n_tokens), - n_kv (worst_case ? kv_self.size : kv_self.n), - n_outputs (worst_case ? n_tokens : lctx.n_outputs), - n_outputs_enc (worst_case ? n_tokens : lctx.embd_enc.size() / hparams.n_embd), - kv_head (worst_case ? (kv_self.recurrent ? 0 : kv_self.size - n_tokens) : kv_self.head), - n_ctx_orig (cparams.n_ctx_orig_yarn), - flash_attn (cparams.flash_attn), - pooling_type (cparams.pooling_type), - rope_type (hparams.rope_type), - cb (cb), - buf_compute_meta (lctx.buf_compute_meta) { - // all initializations should be done in init() - } - - void init() { - struct ggml_init_params params = { - /*.mem_size =*/ buf_compute_meta.size(), - /*.mem_buffer =*/ buf_compute_meta.data(), - /*.no_alloc =*/ true, - }; - - ctx0 = ggml_init(params); - - lctx.inp_tokens = nullptr; - lctx.inp_embd = nullptr; - lctx.inp_pos = nullptr; - lctx.inp_out_ids = nullptr; - lctx.inp_KQ_mask = nullptr; - lctx.inp_KQ_mask_swa = nullptr; - lctx.inp_K_shift = nullptr; - lctx.inp_mean = nullptr; - lctx.inp_cls = nullptr; - lctx.inp_s_copy = nullptr; - lctx.inp_s_mask = nullptr; - lctx.inp_s_seq = nullptr; - lctx.inp_pos_bucket = nullptr; - lctx.inp_embd_enc = nullptr; - lctx.inp_KQ_mask_cross = nullptr; - } - - void free() { - ggml_free(ctx0); - ctx0 = nullptr; - } - - struct ggml_cgraph * build_k_shift() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - GGML_ASSERT(kv_self.size == n_ctx); - - lctx.inp_K_shift = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_ctx); - cb(lctx.inp_K_shift, "K_shift", -1); - ggml_set_input(lctx.inp_K_shift); - - for (int il = 0; il < n_layer; ++il) { - const int64_t n_head_kv = hparams.n_head_kv(il); - const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); - struct ggml_tensor * rope_factors = build_rope_factors(il); - struct ggml_tensor * k = - ggml_view_3d(ctx0, kv_self.k_l[il], - n_embd_head_k, n_head_kv, n_ctx, - ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k), - ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa), - 0); - - struct ggml_tensor * tmp; - if (ggml_is_quantized(k->type)) { - // dequantize to f32 -> RoPE -> quantize back - tmp = ggml_cast(ctx0, k, GGML_TYPE_F32); - cb(tmp, "K_f32", il); - for (auto & backend : lctx.backends) { - // Figure out which backend KV cache belongs to - if (ggml_backend_supports_buft(backend.get(), ggml_backend_buffer_get_type(kv_self.k_l[il]->buffer))) { - ggml_backend_sched_set_tensor_backend(lctx.sched.get(), tmp, backend.get()); - break; - } - } - tmp = ggml_rope_ext_inplace(ctx0, tmp, - lctx.inp_K_shift, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(tmp, "K_shifted_f32", il); - tmp = ggml_cpy(ctx0, tmp, k); - } else { - // we rotate only the first n_rot dimensions - tmp = ggml_rope_ext_inplace(ctx0, k, - lctx.inp_K_shift, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } - cb(tmp, "K_shifted", il); - ggml_build_forward_expand(gf, tmp); - } - - return gf; - } - - struct ggml_cgraph * build_defrag(const std::vector & ids) { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - for (uint32_t i = 0; i < ids.size(); ++i) { - const uint32_t id = ids[i]; - - if (i == id || id == ids.size()) { - continue; - } - - uint32_t nm = 1; - - while (i + nm < ids.size() && ids[i + nm] == id + nm) { - nm++; - } - - for (int il = 0; il < n_layer; ++il) { - const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); - const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(il); - - ggml_tensor * view_k_src = ggml_view_2d(ctx0, kv_self.k_l[il], - n_embd_k_gqa, nm, - ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa), - ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*i)); - - ggml_tensor * view_k_dst = ggml_view_2d(ctx0, kv_self.k_l[il], - n_embd_k_gqa, nm, - ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa), - ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*id)); - - ggml_tensor * view_v_src; - ggml_tensor * view_v_dst; - - if (flash_attn) { - // NOTE: the V cache is not transposed when using flash attention - view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il], - n_embd_v_gqa, nm, - ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa), - ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*i)); - - view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il], - n_embd_v_gqa, nm, - ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa), - ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*id)); - } else { - view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il], - nm, n_embd_v_gqa, - ggml_row_size(kv_self.v_l[il]->type, kv_self.size), - ggml_row_size(kv_self.v_l[il]->type, i)); - - view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il], - nm, n_embd_v_gqa, - ggml_row_size(kv_self.v_l[il]->type, kv_self.size), - ggml_row_size(kv_self.v_l[il]->type, id)); - } - - ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_k_src, view_k_dst)); - ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_v_src, view_v_dst)); - } - - i += nm - 1; - } - - //LLAMA_LOG_INFO("gf->n_nodes = %d\n", gf->n_nodes); - - return gf; - } - - struct ggml_tensor * build_inp_pos() { - lctx.inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); - cb(lctx.inp_pos, "inp_pos", -1); - ggml_set_input(lctx.inp_pos); - return lctx.inp_pos; - } - - struct ggml_tensor * build_rope_factors(int il) { - // choose long/short freq factors based on the context size - const auto n_ctx_pre_seq = cparams.n_ctx / cparams.n_seq_max; - - if (model.layers[il].rope_freqs != nullptr) { - return model.layers[il].rope_freqs; - } - - if (n_ctx_pre_seq > hparams.n_ctx_orig_yarn) { - return model.layers[il].rope_long; - } - - return model.layers[il].rope_short; - } - - struct ggml_tensor * build_inp_out_ids() { - lctx.inp_out_ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs); - cb(lctx.inp_out_ids, "inp_out_ids", -1); - ggml_set_input(lctx.inp_out_ids); - return lctx.inp_out_ids; - } - - struct ggml_tensor * build_inp_KQ_mask(bool causal = true) { - lctx.inp_KQ_mask = causal - ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)) - : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); - cb(lctx.inp_KQ_mask, "KQ_mask", -1); - ggml_set_input(lctx.inp_KQ_mask); - - return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_mask, GGML_TYPE_F16) : lctx.inp_KQ_mask; - } - - struct ggml_tensor * build_inp_KQ_mask_swa(bool causal = true) { - GGML_ASSERT(hparams.n_swa > 0); - - lctx.inp_KQ_mask_swa = causal - ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)) - : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); - cb(lctx.inp_KQ_mask_swa, "KQ_mask_swa", -1); - ggml_set_input(lctx.inp_KQ_mask_swa); - - return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_mask_swa, GGML_TYPE_F16) : lctx.inp_KQ_mask_swa; - } - - struct ggml_tensor * build_inp_mean() { - lctx.inp_mean = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens); - cb(lctx.inp_mean, "inp_mean", -1); - ggml_set_input(lctx.inp_mean); - return lctx.inp_mean; - } - - struct ggml_tensor * build_inp_cls() { - lctx.inp_cls = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); - cb(lctx.inp_cls, "inp_cls", -1); - ggml_set_input(lctx.inp_cls); - return lctx.inp_cls; - } - - struct ggml_tensor * build_inp_s_copy() { - lctx.inp_s_copy = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_kv); - cb(lctx.inp_s_copy, "inp_s_copy", -1); - ggml_set_input(lctx.inp_s_copy); - return lctx.inp_s_copy; - } - - struct ggml_tensor * build_inp_s_mask() { - lctx.inp_s_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_kv); - cb(lctx.inp_s_mask, "inp_s_mask", -1); - ggml_set_input(lctx.inp_s_mask); - return lctx.inp_s_mask; - } - - struct ggml_cgraph * append_pooling(struct ggml_cgraph * gf) { - // find result_norm tensor for input - struct ggml_tensor * inp = nullptr; - for (int i = ggml_graph_n_nodes(gf) - 1; i >= 0; --i) { - inp = ggml_graph_node(gf, i); - if (strcmp(inp->name, "result_norm") == 0 || strcmp(inp->name, "result_embd") == 0) { - break; - } else { - inp = nullptr; - } - } - GGML_ASSERT(inp != nullptr && "missing result_norm/result_embd tensor"); - - struct ggml_tensor * cur; - - switch (pooling_type) { - case LLAMA_POOLING_TYPE_NONE: - { - cur = inp; - } break; - case LLAMA_POOLING_TYPE_MEAN: - { - struct ggml_tensor * inp_mean = build_inp_mean(); - cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, inp)), inp_mean); - } break; - case LLAMA_POOLING_TYPE_CLS: - case LLAMA_POOLING_TYPE_LAST: - { - struct ggml_tensor * inp_cls = build_inp_cls(); - cur = ggml_get_rows(ctx0, inp, inp_cls); - } break; - case LLAMA_POOLING_TYPE_RANK: - { - struct ggml_tensor * inp_cls = build_inp_cls(); - inp = ggml_get_rows(ctx0, inp, inp_cls); - - // classification head - // https://github.com/huggingface/transformers/blob/5af7d41e49bbfc8319f462eb45253dcb3863dfb7/src/transformers/models/roberta/modeling_roberta.py#L1566 - GGML_ASSERT(model.cls != nullptr); - GGML_ASSERT(model.cls_b != nullptr); - - cur = ggml_add (ctx0, ggml_mul_mat(ctx0, model.cls, inp), model.cls_b); - cur = ggml_tanh(ctx0, cur); - - // some models don't have `cls_out`, for example: https://huggingface.co/jinaai/jina-reranker-v1-tiny-en - // https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/blob/cb5347e43979c3084a890e3f99491952603ae1b7/modeling_bert.py#L884-L896 - if (model.cls_out) { - GGML_ASSERT(model.cls_out_b != nullptr); - - cur = ggml_add (ctx0, ggml_mul_mat(ctx0, model.cls_out, cur), model.cls_out_b); - } - } break; - default: - { - GGML_ABORT("unknown pooling type"); - } - } - - cb(cur, "result_embd_pooled", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_tensor * llm_build_pos_bucket(bool causal) { - if (causal) { - lctx.inp_pos_bucket = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, n_tokens); - } else { - lctx.inp_pos_bucket = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_tokens, n_tokens); - } - - ggml_set_input(lctx.inp_pos_bucket); - cb(lctx.inp_pos_bucket, "pos_bucket", -1); - - return lctx.inp_pos_bucket; - } - - struct ggml_tensor * llm_build_pos_bias(struct ggml_tensor * pos_bucket, struct ggml_tensor * attn_rel_b) { - struct ggml_tensor * pos_bucket_1d = ggml_view_1d(ctx0, pos_bucket, pos_bucket->ne[0] * pos_bucket->ne[1], 0); - cb(pos_bucket_1d, "pos_bucket_1d", -1); - - struct ggml_tensor * pos_bias = ggml_get_rows(ctx0, attn_rel_b, pos_bucket_1d); - cb(pos_bias, "pos_bias", -1); - - pos_bias = ggml_view_3d(ctx0, pos_bias, pos_bias->ne[0], lctx.inp_pos_bucket->ne[0], lctx.inp_pos_bucket->ne[1], ggml_element_size(pos_bias) * pos_bias->ne[0], ggml_element_size(pos_bias) * pos_bias->ne[0] * lctx.inp_pos_bucket->ne[0], 0); - cb(pos_bias, "pos_bias", -1); - - pos_bias = ggml_permute(ctx0, pos_bias, 2, 0, 1, 3); - cb(pos_bias, "pos_bias", -1); - - pos_bias = ggml_cont(ctx0, pos_bias); - cb(pos_bias, "pos_bias", -1); - - return pos_bias; - } - - struct ggml_tensor * llm_build_inp_embd_enc() { - const int64_t n_embd = hparams.n_embd; - lctx.inp_embd_enc = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_outputs_enc); - ggml_set_input(lctx.inp_embd_enc); - cb(lctx.inp_embd_enc, "embd_enc", -1); - return lctx.inp_embd_enc; - } - - struct ggml_tensor * llm_build_inp_KQ_mask_cross() { - lctx.inp_KQ_mask_cross = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_outputs_enc, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); - ggml_set_input(lctx.inp_KQ_mask_cross); - cb(lctx.inp_KQ_mask_cross, "KQ_mask_cross", -1); - return lctx.inp_KQ_mask_cross; - } - - struct ggml_cgraph * build_llama() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // rope freq factors for llama3; may return nullptr for llama2 and other models - struct ggml_tensor * rope_factors = build_rope_factors(il); - - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, kq_scale, cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - // For Granite architecture - if (hparams.f_residual_scale) { - cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - if (model.layers[il].ffn_gate_inp == nullptr) { - - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } else { - // MoE branch - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - nullptr, - n_expert, n_expert_used, - LLM_FFN_SILU, true, - false, 0.0, - LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - cb, il); - cb(cur, "ffn_moe_out", il); - } - - // For Granite architecture - if (hparams.f_residual_scale) { - cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - // For Granite architecture - if (hparams.f_logit_scale) { - cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale); - } - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_deci() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - const int64_t n_head_kv = hparams.n_head_kv(il); - const int64_t n_head = hparams.n_head(il); - - if (n_head == 0) { - // attention-free layer of Llama-3_1-Nemotron-51B - cur = inpL; - } else { - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - } - - if (n_head > 0 && n_head_kv == 0) { - // "linear attention" of Llama-3_1-Nemotron-51B - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo, cur); - cb(cur, "wo", il); - } else if (n_head > 0) { - // self-attention - // rope freq factors for llama3; may return nullptr for llama2 and other models - struct ggml_tensor * rope_factors = build_rope_factors(il); - - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, kq_scale, cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - // For Granite architecture - if (hparams.f_residual_scale) { - cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); - } - - // modified to support attention-free layer of Llama-3_1-Nemotron-51B - struct ggml_tensor * ffn_inp = cur; - if (n_head > 0) { - ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - } - - // feed-forward network - if (model.layers[il].ffn_gate_inp == nullptr) { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - // For Granite architecture - if (hparams.f_residual_scale) { - cur = ggml_scale(ctx0, cur, hparams.f_residual_scale); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - // For Granite architecture - if (hparams.f_logit_scale) { - cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale); - } - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_baichuan() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = model.type == LLM_TYPE_7B ? build_inp_pos() : nullptr; - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - switch (model.type) { - case LLM_TYPE_7B: - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - break; - case LLM_TYPE_13B: - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd/n_head, n_head, n_tokens); - break; - default: - GGML_ABORT("fatal error"); - } - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_xverse() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, NULL, LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_falcon() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * attn_norm; - - attn_norm = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(attn_norm, "attn_norm", il); - - // self-attention - { - if (model.layers[il].attn_norm_2) { - // Falcon-40B - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm_2, - model.layers[il].attn_norm_2_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm_2", il); - } else { - cur = attn_norm; - } - - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - // using mode = 2 for neox mode - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - attn_norm = ggml_get_rows(ctx0, attn_norm, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = cur; - - // feed forward - { - cur = llm_build_ffn(ctx0, lctx, attn_norm, // !! use the attn norm, not the result - model.layers[il].ffn_up, NULL, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = ggml_add(ctx0, cur, inpL); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - // norm - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_grok() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // multiply by embedding_multiplier_scale of 78.38367176906169 - inpL = ggml_scale(ctx0, inpL, 78.38367176906169f); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - // Grok - // if attn_out_norm is present then apply it before adding the input - if (model.layers[il].attn_out_norm) { - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].attn_out_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_out_norm", il); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - // MoE branch - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - nullptr, - n_expert, n_expert_used, - LLM_FFN_GELU, true, - false, 0.0, - LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - cb, il); - cb(cur, "ffn_moe_out", il); - - // Grok - // if layer_out_norm is present then apply it before adding the input - // Idea: maybe ffn_out_norm is a better name - if (model.layers[il].layer_out_norm) { - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].layer_out_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "layer_out_norm", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - // Grok - // multiply logits by output_multiplier_scale of 0.5773502691896257 - - cur = ggml_scale(ctx0, cur, 0.5773502691896257f); - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_dbrx() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - struct ggml_tensor * Qcur = nullptr; - struct ggml_tensor * Kcur = nullptr; - struct ggml_tensor * Vcur = nullptr; - - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); - cb(cur, "wqkv_clamped", il); - - Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - // MoE branch - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].attn_out_norm, NULL, - LLM_NORM, cb, il); - cb(cur, "attn_out_norm", il); - - cur = llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - nullptr, - n_expert, n_expert_used, - LLM_FFN_SILU, true, - false, 0.0, - LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - cb, il); - cb(cur, "ffn_moe_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_starcoder() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - struct ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); - cb(pos, "pos_embd", -1); - - inpL = ggml_add(ctx0, inpL, pos); - cb(inpL, "inpL", -1); - - for (int il = 0; il < n_layer; ++il) { - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // add the input - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); - cb(ffn_inp, "ffn_inp", il); - - // FF - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_refact() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - cb(Kcur, "Kcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cb(Qcur, "Qcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_bert() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - struct ggml_tensor * inp_pos = nullptr; - - if (model.arch != LLM_ARCH_JINA_BERT_V2) { - inp_pos = build_inp_pos(); - } - - // construct input embeddings (token, type, position) - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // token types are hardcoded to zero ("Sentence A") - struct ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0); - inpL = ggml_add(ctx0, inpL, type_row0); - if (model.arch == LLM_ARCH_BERT) { - inpL = ggml_add(ctx0, ggml_get_rows(ctx0, model.pos_embd, inp_pos), inpL); - } - cb(inpL, "inp_embd", -1); - - // embed layer norm - inpL = llm_build_norm(ctx0, inpL, hparams, model.tok_norm, model.tok_norm_b, LLM_NORM, cb, -1); - cb(inpL, "inp_norm", -1); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(false); - - // iterate layers - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * cur = inpL; - - struct ggml_tensor * Qcur; - struct ggml_tensor * Kcur; - struct ggml_tensor * Vcur; - - // self-attention - if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT_V2) { - Qcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur), model.layers[il].bq); - cb(Qcur, "Qcur", il); - - if (model.layers[il].attn_q_norm) { - Qcur = llm_build_norm(ctx0, Qcur, hparams, - model.layers[il].attn_q_norm, - model.layers[il].attn_q_norm_b, - LLM_NORM, cb, il); - } - - Kcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur), model.layers[il].bk); - cb(Kcur, "Kcur", il); - - if (model.layers[il].attn_k_norm) { - Kcur = llm_build_norm(ctx0, Kcur, hparams, - model.layers[il].attn_k_norm, - model.layers[il].attn_k_norm_b, - LLM_NORM, cb, il); - } - Vcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur), model.layers[il].bv); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - } else { - // compute Q and K and RoPE them - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3); - struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3)); - - struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); - cb(kq, "kq", il); - - kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias); - cb(kq, "kq_soft_max_ext", il); - - struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens))); - cb(v, "v", il); - - struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq); - cb(kqv, "kqv", il); - - struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3); - cb(kqv_merged, "kqv_merged", il); - - cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens); - cb(cur, "kqv_merged_cont", il); - - ggml_build_forward_expand(gf, cur); - - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo, cur); - if (model.layers[il].bo) { - cb(cur, "kqv_wo", il); - } - - if (model.layers[il].bo) { - cur = ggml_add(ctx0, cur, model.layers[il].bo); - } - cb(cur, "kqv_out", il); - - if (il == n_layer - 1 && pooling_type == LLAMA_POOLING_TYPE_NONE) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // re-add the layer input - cur = ggml_add(ctx0, cur, inpL); - - // attention layer norm - cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, cb, il); - - if (model.layers[il].attn_norm_2 != nullptr) { - cur = ggml_add(ctx0, cur, inpL); // re-add the layer input - cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_norm_2, model.layers[il].attn_norm_2_b, LLM_NORM, cb, il); - } - - struct ggml_tensor * ffn_inp = cur; - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - if (model.arch == LLM_ARCH_BERT) { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - } else if (model.arch == LLM_ARCH_JINA_BERT_V2) { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_PAR, cb, il); - } else { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - } - cb(cur, "ffn_out", il); - - // attentions bypass the intermediate layer - cur = ggml_add(ctx0, cur, ffn_inp); - - // output layer norm - cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_out_norm, model.layers[il].layer_out_norm_b, LLM_NORM, cb, il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cb(cur, "result_embd", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_bloom() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - inpL = llm_build_norm(ctx0, inpL, hparams, - model.tok_norm, - model.tok_norm_b, - LLM_NORM, cb, -1); - cb(inpL, "inp_norm", -1); - - for (int il = 0; il < n_layer; ++il) { - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // Add the input - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); - cb(ffn_inp, "ffn_inp", il); - - // FF - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_mpt() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * pos; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - if (model.pos_embd) { - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); - cb(pos, "pos_embd", -1); - - inpL = ggml_add(ctx0, inpL, pos); - cb(inpL, "inpL", -1); - } - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * attn_norm; - - attn_norm = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(attn_norm, "attn_norm", il); - - // self-attention - { - cur = attn_norm; - - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - if (model.layers[il].bqkv){ - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - } - - if (hparams.f_clamp_kqv > 0.0f) { - cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); - cb(cur, "wqkv_clamped", il); - } - - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - // Q/K Layernorm - if (model.layers[il].attn_q_norm) { - Qcur = llm_build_norm(ctx0, Qcur, hparams, - model.layers[il].attn_q_norm, - model.layers[il].attn_q_norm_b, - LLM_NORM, cb, il); - cb(Qcur, "Qcur", il); - - Kcur = llm_build_norm(ctx0, Kcur, hparams, - model.layers[il].attn_k_norm, - model.layers[il].attn_k_norm_b, - LLM_NORM, cb, il); - cb(Kcur, "Kcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } else { - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // Add the input - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); - cb(ffn_inp, "ffn_inp", il); - - // feed forward - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - model.layers[il].ffn_act, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_stablelm() { - struct ggml_cgraph * gf = ggml_new_graph(ctx0); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - struct ggml_tensor * inpSA = cur; - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cb(Qcur, "Qcur", il); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - cb(Kcur, "Kcur", il); - - if (model.layers[il].attn_q_norm) { - Qcur = llm_build_norm(ctx0, Qcur, hparams, - model.layers[il].attn_q_norm, - NULL, - LLM_NORM, cb, il); - cb(Qcur, "Qcur", il); - } - if (model.layers[il].attn_k_norm) { - Kcur = llm_build_norm(ctx0, Kcur, hparams, - model.layers[il].attn_k_norm, - NULL, - LLM_NORM, cb, il); - cb(Kcur, "Kcur", il); - } - - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - { - if (model.layers[il].ffn_norm) { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - } else { - // parallel residual - cur = inpSA; - } - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_qwen() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 2*sizeof(float)*(n_embd))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - // using mode = 2 for neox mode - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward forward - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_qwen2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_qwen2vl() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - lctx.inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens * 4); - cb(lctx.inp_pos, "inp_pos", -1); - ggml_set_input(lctx.inp_pos); - struct ggml_tensor * inp_pos = lctx.inp_pos; - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_multi( - ctx0, - ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_multi( - ctx0, - ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_qwen2moe() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self_attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // MoE branch - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - ggml_tensor * moe_out = - llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - nullptr, - n_expert, n_expert_used, - LLM_FFN_SILU, false, - false, 0.0, - LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - cb, il); - cb(cur, "ffn_moe_out", il); - - // FFN shared expert - { - ggml_tensor * cur_gate_inp = llm_build_lora_mm(lctx, ctx0, model.layers[il].ffn_gate_inp_shexp, cur); - cb(cur_gate_inp, "ffn_shexp_gate_inp", il); - - // sigmoid - ggml_tensor * cur_gate = ggml_div(ctx0, ggml_silu(ctx0, cur_gate_inp), cur_gate_inp); - cb(cur_gate, "ffn_shexp_gate", il); - - ggml_tensor * cur_ffn = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up_shexp, NULL, NULL, - model.layers[il].ffn_gate_shexp, NULL, NULL, - model.layers[il].ffn_down_shexp, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur_ffn, "ffn_shexp", il); - - ggml_tensor * ffn_shexp_out = ggml_mul(ctx0, cur_ffn, cur_gate); - cb(ffn_shexp_out, "ffn_shexp_out", il); - - moe_out = ggml_add(ctx0, moe_out, ffn_shexp_out); - cb(moe_out, "ffn_out", il); - - cur = moe_out; - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_phi2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * attn_norm_output; - struct ggml_tensor * ffn_output; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - attn_norm_output = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(attn_norm_output, "attn_norm", il); - - // self-attention - { - struct ggml_tensor * Qcur = nullptr; - struct ggml_tensor * Kcur = nullptr; - struct ggml_tensor * Vcur = nullptr; - - if (model.layers[il].wqkv) { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, attn_norm_output); - cb(cur, "wqkv", il); - - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - - Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - } else { - Qcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, attn_norm_output), model.layers[il].bq); - Kcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, attn_norm_output), model.layers[il].bk); - Vcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, attn_norm_output), model.layers[il].bv); - } - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - // with phi2, we scale the Q to avoid precision issues - // ref: https://github.com/ml-explore/mlx-examples/blob/08e862336ade809bc37d1035f94b359e7d1a5152/phi2/phi2.py#L64-L66 - Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head))); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - attn_norm_output = ggml_get_rows(ctx0, attn_norm_output, inp_out_ids); - } - - // FF - { - ffn_output = llm_build_ffn(ctx0, lctx, attn_norm_output, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(ffn_output, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_output); - cur = ggml_add(ctx0, cur, inpL); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output_no_bias", -1); - - cur = ggml_add(ctx0, cur, model.output_b); - cb(cur, "result_output", -1); - ggml_build_forward_expand(gf, cur); - return gf; - } - - struct ggml_cgraph * build_phi3() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = nullptr; - if (hparams.n_swa == 0) { - // Phi-4 doesn't use sliding window attention - KQ_mask = build_inp_KQ_mask(); - } else { - KQ_mask = build_inp_KQ_mask_swa(); - } - - for (int il = 0; il < n_layer; ++il) { - auto residual = inpL; - - // self-attention - { - // rope freq factors for 128k context - struct ggml_tensor * rope_factors = build_rope_factors(il); - - struct ggml_tensor* attn_norm_output = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM_RMS, cb, il); - cb(attn_norm_output, "attn_norm", il); - - struct ggml_tensor * Qcur = nullptr; - struct ggml_tensor * Kcur = nullptr; - struct ggml_tensor * Vcur = nullptr; - - if (model.layers[il].wqkv) { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, attn_norm_output); - cb(cur, "wqkv", il); - - Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0 * sizeof(float) * (n_embd))); - Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1 * sizeof(float) * (n_embd))); - Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa))); - } else { - Qcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, attn_norm_output), model.layers[il].bq); - Kcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, attn_norm_output), model.layers[il].bk); - Vcur = ggml_add(ctx0, llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, attn_norm_output), model.layers[il].bv); - } - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head))); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor* inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - residual = ggml_get_rows(ctx0, residual, inp_out_ids); - } - - cur = ggml_add(ctx0, cur, residual); - residual = cur; - - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - // feed-forward network - if (model.layers[il].ffn_gate_inp == nullptr) { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SWIGLU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - } else { - // MoE branch - cur = llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - nullptr, - n_expert, n_expert_used, - LLM_FFN_SILU, true, - false, 0.0, - LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - cb, il); - cb(cur, "ffn_moe_out", il); - } - - cur = ggml_add(ctx0, residual, cur); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - if (model.output_b != nullptr) { - cb(cur, "result_output_no_bias", -1); - cur = ggml_add(ctx0, cur, model.output_b); - } - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - - struct ggml_cgraph * build_plamo() { - struct ggml_cgraph * gf = ggml_new_graph(ctx0); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - struct ggml_tensor * attention_norm = cur; - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, nullptr, - n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, nullptr, - n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - struct ggml_tensor * sa_out = cur; - - cur = attention_norm; - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - sa_out = ggml_get_rows(ctx0, sa_out, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // feed-forward network - { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, sa_out); - cur = ggml_add(ctx0, cur, inpL); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_gpt2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * pos; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); - cb(pos, "pos_embd", -1); - - inpL = ggml_add(ctx0, inpL, pos); - cb(inpL, "inpL", -1); - - for (int il = 0; il < n_layer; ++il) { - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // add the input - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); - cb(ffn_inp, "ffn_inp", il); - - // FF - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_codeshell() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - - struct ggml_tensor * tmpq = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * tmpk = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - - cb(tmpq, "tmpq", il); - cb(tmpk, "tmpk", il); - cb(Vcur, "Vcur", il); - - struct ggml_tensor * Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // add the input - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); - cb(ffn_inp, "ffn_inp", il); - - // FF - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_orion() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - // if (model.layers[il].bq) { - // Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - // cb(Qcur, "Qcur", il); - // } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - // if (model.layers[il].bk) { - // Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - // cb(Kcur, "Kcur", il); - // } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - // if (model.layers[il].bv) { - // Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - // cb(Vcur, "Vcur", il); - // } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_internlm2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_minicpm3() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - //TODO: if the model varies, these parameters need to be read from the model - const int64_t n_embd_base = 256; - const float scale_embd = 12.0f; - const float scale_depth = 1.4f; - const float kq_scale = 1.0f / sqrtf(float(hparams.n_embd_head_k)); - - const uint32_t n_embd_head_qk_rope = hparams.n_rot; - const uint32_t n_embd_head_qk_nope = hparams.n_embd_head_k - hparams.n_rot; - const uint32_t kv_lora_rank = hparams.n_lora_kv; - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // scale the input embeddings - inpL = ggml_scale(ctx0, inpL, scale_embd); - cb(inpL, "inp_scaled", -1); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - struct ggml_tensor * rope_factors = build_rope_factors(il); - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self_attention - { - struct ggml_tensor * q = NULL; - // {n_embd, q_lora_rank} * {n_embd, n_tokens} -> {q_lora_rank, n_tokens} - q = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur); - cb(q, "q", il); - - q = llm_build_norm(ctx0, q, hparams, - model.layers[il].attn_q_a_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(q, "q", il); - - // {q_lora_rank, n_head * hparams.n_embd_head_k} * {q_lora_rank, n_tokens} -> {n_head * hparams.n_embd_head_k, n_tokens} - q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q); - cb(q, "q", il); - - // split into {n_head * n_embd_head_qk_nope, n_tokens} - struct ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, - ggml_row_size(q->type, hparams.n_embd_head_k), - ggml_row_size(q->type, hparams.n_embd_head_k * n_head), - 0); - cb(q_nope, "q_nope", il); - - // and {n_head * n_embd_head_qk_rope, n_tokens} - struct ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, - ggml_row_size(q->type, hparams.n_embd_head_k), - ggml_row_size(q->type, hparams.n_embd_head_k * n_head), - ggml_row_size(q->type, n_embd_head_qk_nope)); - cb(q_pe, "q_pe", il); - - // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens} - struct ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); - cb(kv_pe_compresseed, "kv_pe_compresseed", il); - - // split into {kv_lora_rank, n_tokens} - struct ggml_tensor * kv_compressed = ggml_view_2d(ctx0, kv_pe_compresseed, kv_lora_rank, n_tokens, - kv_pe_compresseed->nb[1], - 0); - cb(kv_compressed, "kv_compressed", il); - - // and {n_embd_head_qk_rope, n_tokens} - struct ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_pe_compresseed, n_embd_head_qk_rope, 1, n_tokens, - kv_pe_compresseed->nb[1], - kv_pe_compresseed->nb[1], - ggml_row_size(kv_pe_compresseed->type, kv_lora_rank)); - cb(k_pe, "k_pe", il); - - // TODO: the CUDA backend used to not support non-cont. (RMS) norm, investigate removing ggml_cont - kv_compressed = ggml_cont(ctx0, kv_compressed); - kv_compressed = llm_build_norm(ctx0, kv_compressed, hparams, - model.layers[il].attn_kv_a_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(kv_compressed, "kv_compressed", il); - - // {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)} * {kv_lora_rank, n_tokens} -> {n_head * (n_embd_head_qk_nope + n_embd_head_v), n_tokens} - struct ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_compressed); - cb(kv, "kv", il); - - // split into {n_head * n_embd_head_qk_nope, n_tokens} - struct ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens, - ggml_row_size(kv->type, n_embd_head_qk_nope + hparams.n_embd_head_v), - ggml_row_size(kv->type, n_head * (n_embd_head_qk_nope + hparams.n_embd_head_v)), - 0); - cb(k_nope, "k_nope", il); - - // and {n_head * n_embd_head_v, n_tokens} - struct ggml_tensor * v_states = ggml_view_3d(ctx0, kv, hparams.n_embd_head_v, n_head, n_tokens, - ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)), - ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)*n_head), - ggml_row_size(kv->type, (n_embd_head_qk_nope))); - cb(v_states, "v_states", il); - - v_states = ggml_cont(ctx0, v_states); - cb(v_states, "v_states", il); - - v_states = ggml_view_2d(ctx0, v_states, hparams.n_embd_head_v * n_head, n_tokens, - ggml_row_size(kv->type, hparams.n_embd_head_v * n_head), - 0); - cb(v_states, "v_states", il); - - q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this - q_pe = ggml_rope_ext( - ctx0, q_pe, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(q_pe, "q_pe", il); - - // shared RoPE key - k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this - k_pe = ggml_rope_ext( - ctx0, k_pe, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(k_pe, "k_pe", il); - - struct ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0); - cb(q_states, "q_states", il); - - struct ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0); - cb(k_states, "k_states", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - k_states, v_states, q_states, KQ_mask, n_tokens, kv_head, n_kv, kq_scale, cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - // scale_res - scale the hidden states for residual connection - const float scale_res = scale_depth/sqrtf(float(n_layer)); - cur = ggml_scale(ctx0, cur, scale_res); - cb(cur, "hidden_scaled", il); - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - // scale the hidden states for residual connection - cur = ggml_scale(ctx0, cur, scale_res); - cb(cur, "hidden_scaled_ffn", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head scaling - const float scale_lmhead = float(n_embd_base)/float(n_embd); - cur = ggml_scale(ctx0, cur, scale_lmhead); - cb(cur, "lmhead_scaling", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_gemma() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head_k = hparams.n_embd_head_k; - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); - cb(inpL, "inp_scaled", -1); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - - Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); - cb(Qcur, "Qcur_scaled", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); - cb(sa_out, "sa_out", il); - - cur = llm_build_norm(ctx0, sa_out, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - // feed-forward network - { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, sa_out); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_gemma2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head_k = hparams.n_embd_head_k; - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); - cb(inpL, "inp_scaled", -1); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - // gemma 2 requires different mask for layers using sliding window (SWA) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(true); - struct ggml_tensor * KQ_mask_swa = build_inp_KQ_mask_swa(true); - - for (int il = 0; il < n_layer; ++il) { - // (il % 2) layers use SWA - struct ggml_tensor * KQ_mask_l = (il % 2 == 0) ? KQ_mask_swa : KQ_mask; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - - // ref: https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e - switch (model.type) { - case LLM_TYPE_2B: - case LLM_TYPE_9B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); break; - case LLM_TYPE_27B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd / n_head))); break; - default: GGML_ABORT("fatal error"); - }; - cb(Qcur, "Qcur_scaled", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask_l, n_tokens, kv_head, n_kv, 1.0f, cb, il); - } - - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].attn_post_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_post_norm", il); - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); - cb(sa_out, "sa_out", il); - - cur = llm_build_norm(ctx0, sa_out, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - // feed-forward network - { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].ffn_post_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "ffn_post_norm", -1); - - cur = ggml_add(ctx0, cur, sa_out); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - // final logit soft-capping - cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); - cur = ggml_tanh(ctx0, cur); - cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_gemma3() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head_k = hparams.n_embd_head_k; - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings) - if (ubatch.token) { - inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); - cb(inpL, "inp_scaled", -1); - } - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - // gemma3 requires different mask for layers using sliding window (SWA) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(true); - struct ggml_tensor * KQ_mask_swa = build_inp_KQ_mask_swa(true); - - // "5-to-1 interleaved attention" - // 5 layers of local attention followed by 1 layer of global attention - static const int sliding_window_pattern = 6; - - for (int il = 0; il < n_layer; ++il) { - const bool is_sliding = (il + 1) % sliding_window_pattern; - const float freq_base_l = is_sliding ? 10000.0f : freq_base; - const float freq_scale_l = is_sliding ? 1.0f : freq_scale; - struct ggml_tensor * KQ_mask_l = is_sliding ? KQ_mask_swa : KQ_mask; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens); - Qcur = llm_build_norm(ctx0, Qcur, hparams, - model.layers[il].attn_q_norm, - NULL, - LLM_NORM_RMS, cb, il); - cb(Qcur, "Qcur_normed", il); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens); - Kcur = llm_build_norm(ctx0, Kcur, hparams, - model.layers[il].attn_k_norm, - NULL, - LLM_NORM_RMS, cb, il); - cb(Kcur, "Kcur_normed", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask_l, n_tokens, kv_head, n_kv, hparams.f_attention_scale, cb, il); - } - - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].attn_post_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_post_norm", il); - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); - cb(sa_out, "sa_out", il); - - cur = llm_build_norm(ctx0, sa_out, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - // feed-forward network - { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].ffn_post_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "ffn_post_norm", -1); - - cur = ggml_add(ctx0, cur, sa_out); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_starcoder2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_mamba() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - // {n_embd, n_tokens} - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - struct ggml_tensor * state_copy = build_inp_s_copy(); - struct ggml_tensor * state_mask = build_inp_s_mask(); - - for (int il = 0; il < n_layer; ++il) { - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - cur = llm_build_mamba(ctx0, lctx, ubatch, gf, cur, - state_copy, state_mask, - kv_head, n_kv, cb, il); - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // residual - cur = ggml_add(ctx0, cur, inpL); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - // final rmsnorm - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_command_r() { - - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - const float f_logit_scale = hparams.f_logit_scale; - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - struct ggml_tensor * ffn_inp = cur; - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - if (model.layers[il].attn_q_norm) { - Qcur = ggml_view_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens, - ggml_element_size(Qcur) * n_embd_head, - ggml_element_size(Qcur) * n_embd_head * n_head, - 0); - cb(Qcur, "Qcur", il); - Kcur = ggml_view_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens, - ggml_element_size(Kcur) * n_embd_head, - ggml_element_size(Kcur) * n_embd_head * n_head_kv, - 0); - cb(Kcur, "Kcur", il); - - Qcur = llm_build_norm(ctx0, Qcur, hparams, - model.layers[il].attn_q_norm, - NULL, - LLM_NORM, cb, il); - cb(Qcur, "Qcur", il); - - Kcur = llm_build_norm(ctx0, Kcur, hparams, - model.layers[il].attn_k_norm, - NULL, - LLM_NORM, cb, il); - cb(Kcur, "Kcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids); - } - - struct ggml_tensor * attn_out = cur; - - // feed-forward network - { - cur = llm_build_ffn(ctx0, lctx, ffn_inp, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - // add together residual + FFN + self-attention - cur = ggml_add(ctx0, cur, inpL); - cur = ggml_add(ctx0, cur, attn_out); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - if (f_logit_scale) { - cur = ggml_scale(ctx0, cur, f_logit_scale); - } - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - - } - - struct ggml_cgraph * build_cohere2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - const float f_logit_scale = hparams.f_logit_scale; - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - // cohere2 requires different mask for layers using sliding window (SWA) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - struct ggml_tensor * KQ_mask_swa = build_inp_KQ_mask_swa(); - - // sliding window switch pattern - const int32_t sliding_window_pattern = 4; - - for (int il = 0; il < n_layer; ++il) { - // three layers sliding window attention (window size 4096) and ROPE - // fourth layer uses global attention without positional embeddings - const bool is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); - struct ggml_tensor * KQ_mask_l = is_sliding ? KQ_mask_swa : KQ_mask; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, model.layers[il].attn_norm, NULL, LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - struct ggml_tensor * ffn_inp = cur; - - // self-attention - { - // rope freq factors for 128k context - struct ggml_tensor * rope_factors = build_rope_factors(il); - - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - if (is_sliding) { - Qcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, - beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur", il); - } else { - // For non-sliding layers, just reshape without applying RoPE - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cb(Qcur, "Qcur", il); - - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - cb(Kcur, "Kcur", il); - } - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, - KQ_mask_l, n_tokens, kv_head, n_kv, 1.0f / sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids); - } - - struct ggml_tensor * attn_out = cur; - - // feed-forward network - { - cur = llm_build_ffn(ctx0, lctx, ffn_inp, model.layers[il].ffn_up, NULL, NULL, model.layers[il].ffn_gate, - NULL, NULL, model.layers[il].ffn_down, NULL, NULL, NULL, LLM_FFN_SILU, LLM_FFN_PAR, - cb, il); - cb(cur, "ffn_out", il); - } - - // add together residual + FFN + self-attention - cur = ggml_add(ctx0, cur, inpL); - cur = ggml_add(ctx0, cur, attn_out); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, NULL, LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - if (f_logit_scale) { - cur = ggml_scale(ctx0, cur, f_logit_scale); - } - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - // ref: https://allenai.org/olmo - // based on the original build_llama() function, changes: - // * non-parametric layer norm - // * clamp qkv - // * removed bias - // * removed MoE - struct ggml_cgraph * build_olmo() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - NULL, NULL, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (hparams.f_clamp_kqv > 0.0f) { - Qcur = ggml_clamp(ctx0, Qcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (hparams.f_clamp_kqv > 0.0f) { - Kcur = ggml_clamp(ctx0, Kcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (hparams.f_clamp_kqv > 0.0f) { - Vcur = ggml_clamp(ctx0, Vcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, nullptr, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - NULL, NULL, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - NULL, NULL, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_olmo2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - cur = inpL; - - // self_attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Qcur = llm_build_norm(ctx0, Qcur, hparams, model.layers[il].attn_q_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(Qcur, "Qcur_normed", il); - - Kcur = llm_build_norm(ctx0, Kcur, hparams, model.layers[il].attn_k_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(Kcur, "Kcur_normed", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur_rope", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur_rope", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].attn_post_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_post_norm", il); - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_ffn(ctx0, lctx, ffn_inp, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].ffn_post_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "ffn_post_norm", -1); - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - // based on the build_qwen2moe() function, changes: - // * removed shared experts - // * removed bias - // * added q, k norm - struct ggml_cgraph * build_olmoe() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self_attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Qcur = llm_build_norm(ctx0, Qcur, hparams, model.layers[il].attn_q_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(Qcur, "Qcur_normed", il); - - Kcur = llm_build_norm(ctx0, Kcur, hparams, model.layers[il].attn_k_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(Kcur, "Kcur_normed", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur_rope", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur_rope", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // MoE branch - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - nullptr, - n_expert, n_expert_used, - LLM_FFN_SILU, false, - false, 0.0, - LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - cb, il); - cb(cur, "ffn_moe_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_openelm() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - const int64_t n_head = hparams.n_head(il); - const int64_t n_head_kv = hparams.n_head_kv(il); - const int64_t n_head_qkv = 2*n_head_kv + n_head; - - cur = inpL; - struct ggml_tensor * residual = cur; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_reshape_3d(ctx0, cur, n_embd_head_k, n_head_qkv, n_tokens); - - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, cur->nb[1], cur->nb[2], 0)); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*n_head)); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*(n_head+n_head_kv))); - cb(Vcur, "Vcur", il); - - Qcur = llm_build_norm(ctx0, Qcur, hparams, - model.layers[il].attn_q_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(Qcur, "Qcur", il); - - Kcur = llm_build_norm(ctx0, Kcur, hparams, - model.layers[il].attn_k_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(Kcur, "Kcur", il); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - Vcur = ggml_reshape_2d(ctx0, Vcur, n_embd_head * n_head_kv, n_tokens); - cb(Qcur, "Vcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - residual = ggml_get_rows(ctx0, residual, inp_out_ids); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, residual, cur); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - inpL = cur; - } - - cur = inpL; - - // norm - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_gptneox() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // ffn - if (hparams.use_par_res) { - // attention and ffn are computed in parallel - // x = x + attn(ln1(x)) + ffn(ln2(x)) - - struct ggml_tensor * attn_out = cur; - - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, inpL); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, attn_out); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } else { - // attention and ffn are computed sequentially - // x = x + attn(ln1(x)) - // x = x + ffn(ln2(x)) - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); - cb(ffn_inp, "ffn_inp", il); - - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - } - - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_arctic() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - struct ggml_tensor * ffn_out = ggml_add(ctx0, cur, ffn_inp); - cb(ffn_out, "ffn_out", il); - - // MoE - cur = llm_build_norm(ctx0, inpSA, hparams, - model.layers[il].ffn_norm_exps, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm_exps", il); - - cur = llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - nullptr, - n_expert, n_expert_used, - LLM_FFN_SILU, true, - false, 0.0, - LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - cb, il); - cb(cur, "ffn_moe_out", il); - - cur = ggml_add(ctx0, cur, ffn_out); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_deepseek() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // rope freq factors for llama3; may return nullptr for llama2 and other models - struct ggml_tensor * rope_factors = build_rope_factors(il); - - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, kq_scale, cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - if ((uint32_t) il < hparams.n_layer_dense_lead) { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } else { - // MoE branch - ggml_tensor * moe_out = - llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - nullptr, - n_expert, n_expert_used, - LLM_FFN_SILU, false, - false, hparams.expert_weights_scale, - LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - cb, il); - cb(moe_out, "ffn_moe_out", il); - - // FFN shared expert - { - ggml_tensor * ffn_shexp = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up_shexp, NULL, NULL, - model.layers[il].ffn_gate_shexp, NULL, NULL, - model.layers[il].ffn_down_shexp, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(ffn_shexp, "ffn_shexp", il); - - cur = ggml_add(ctx0, moe_out, ffn_shexp); - cb(cur, "ffn_out", il); - } - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_deepseek2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - bool is_lite = (hparams.n_layer == 27); - - // We have to pre-scale kq_scale and attn_factor to make the YaRN RoPE work correctly. - // See https://github.com/ggerganov/llama.cpp/discussions/7416 for detailed explanation. - const float mscale = attn_factor * (1.0f + hparams.rope_yarn_log_mul * logf(1.0f / freq_scale)); - const float kq_scale = 1.0f*mscale*mscale/sqrtf(float(hparams.n_embd_head_k)); - const float attn_factor_scaled = 1.0f / (1.0f + 0.1f * logf(1.0f / freq_scale)); - - const uint32_t n_embd_head_qk_rope = hparams.n_rot; - const uint32_t n_embd_head_qk_nope = hparams.n_embd_head_k - hparams.n_rot; - const uint32_t kv_lora_rank = hparams.n_lora_kv; - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - // {n_embd, n_tokens} - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self_attention - { - struct ggml_tensor * q = NULL; - if (!is_lite) { - // {n_embd, q_lora_rank} * {n_embd, n_tokens} -> {q_lora_rank, n_tokens} - q = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur); - cb(q, "q", il); - - q = llm_build_norm(ctx0, q, hparams, - model.layers[il].attn_q_a_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(q, "q", il); - - // {q_lora_rank, n_head * hparams.n_embd_head_k} * {q_lora_rank, n_tokens} -> {n_head * hparams.n_embd_head_k, n_tokens} - q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q); - cb(q, "q", il); - } else { - q = ggml_mul_mat(ctx0, model.layers[il].wq, cur); - cb(q, "q", il); - } - - // split into {n_head * n_embd_head_qk_nope, n_tokens} - struct ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, - ggml_row_size(q->type, hparams.n_embd_head_k), - ggml_row_size(q->type, hparams.n_embd_head_k * n_head), - 0); - cb(q_nope, "q_nope", il); - - // and {n_head * n_embd_head_qk_rope, n_tokens} - struct ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, - ggml_row_size(q->type, hparams.n_embd_head_k), - ggml_row_size(q->type, hparams.n_embd_head_k * n_head), - ggml_row_size(q->type, n_embd_head_qk_nope)); - cb(q_pe, "q_pe", il); - - // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens} - struct ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); - cb(kv_pe_compresseed, "kv_pe_compresseed", il); - - // split into {kv_lora_rank, n_tokens} - struct ggml_tensor * kv_compressed = ggml_view_2d(ctx0, kv_pe_compresseed, kv_lora_rank, n_tokens, - kv_pe_compresseed->nb[1], - 0); - cb(kv_compressed, "kv_compressed", il); - - // and {n_embd_head_qk_rope, n_tokens} - struct ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_pe_compresseed, n_embd_head_qk_rope, 1, n_tokens, - kv_pe_compresseed->nb[1], - kv_pe_compresseed->nb[1], - ggml_row_size(kv_pe_compresseed->type, kv_lora_rank)); - cb(k_pe, "k_pe", il); - - // TODO: the CUDA backend used to not support non-cont. (RMS) norm, investigate removing ggml_cont - kv_compressed = ggml_cont(ctx0, kv_compressed); - kv_compressed = llm_build_norm(ctx0, kv_compressed, hparams, - model.layers[il].attn_kv_a_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(kv_compressed, "kv_compressed", il); - - // {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)} * {kv_lora_rank, n_tokens} -> {n_head * (n_embd_head_qk_nope + n_embd_head_v), n_tokens} - struct ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_compressed); - cb(kv, "kv", il); - - // split into {n_head * n_embd_head_qk_nope, n_tokens} - struct ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens, - ggml_row_size(kv->type, n_embd_head_qk_nope + hparams.n_embd_head_v), - ggml_row_size(kv->type, n_head * (n_embd_head_qk_nope + hparams.n_embd_head_v)), - 0); - cb(k_nope, "k_nope", il); - - // and {n_head * n_embd_head_v, n_tokens} - struct ggml_tensor * v_states = ggml_view_3d(ctx0, kv, hparams.n_embd_head_v, n_head, n_tokens, - ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)), - ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)*n_head), - ggml_row_size(kv->type, (n_embd_head_qk_nope))); - cb(v_states, "v_states", il); - - v_states = ggml_cont(ctx0, v_states); - cb(v_states, "v_states", il); - - v_states = ggml_view_2d(ctx0, v_states, hparams.n_embd_head_v * n_head, n_tokens, - ggml_row_size(kv->type, hparams.n_embd_head_v * n_head), - 0); - cb(v_states, "v_states", il); - - q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this - q_pe = ggml_rope_ext( - ctx0, q_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor_scaled, beta_fast, beta_slow - ); - cb(q_pe, "q_pe", il); - - // shared RoPE key - k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this - k_pe = ggml_rope_ext( - ctx0, k_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor_scaled, beta_fast, beta_slow - ); - cb(k_pe, "k_pe", il); - - struct ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0); - cb(q_states, "q_states", il); - - struct ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0); - cb(k_states, "k_states", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - k_states, v_states, q_states, KQ_mask, n_tokens, kv_head, n_kv, kq_scale, cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - if ((uint32_t) il < hparams.n_layer_dense_lead) { - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } else { - // MoE branch - ggml_tensor * moe_out = - llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - model.layers[il].ffn_exp_probs_b, - n_expert, n_expert_used, - LLM_FFN_SILU, hparams.expert_weights_norm, - true, hparams.expert_weights_scale, - (enum llama_expert_gating_func_type) hparams.expert_gating_func, - cb, il); - cb(moe_out, "ffn_moe_out", il); - - // FFN shared expert - { - ggml_tensor * ffn_shexp = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up_shexp, NULL, NULL, - model.layers[il].ffn_gate_shexp, NULL, NULL, - model.layers[il].ffn_down_shexp, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(ffn_shexp, "ffn_shexp", il); - - cur = ggml_add(ctx0, moe_out, ffn_shexp); - cb(cur, "ffn_out", il); - } - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = ggml_mul_mat(ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_bitnet() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - if (model.layers[il].wq_scale) { - Qcur = ggml_mul(ctx0, Qcur, model.layers[il].wq_scale); - } - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - // B1.K - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - if (model.layers[il].wk_scale) { - Kcur = ggml_mul(ctx0, Kcur, model.layers[il].wk_scale); - } - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - // B1.V - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - if (model.layers[il].wv_scale) { - Vcur = ggml_mul(ctx0, Vcur, model.layers[il].wv_scale); - } - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - NULL, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].attn_sub_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_sub_norm", il); - - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo, cur); - if (model.layers[il].wo_scale) { - cur = ggml_mul(ctx0, cur, model.layers[il].wo_scale); - } - if (model.layers[il].bo) { - cur = ggml_add(ctx0, cur, model.layers[il].bo); - } - cb(cur, "attn_o_out", il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward forward - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_scale, - model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_scale, - NULL, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_sub_out", il); - - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].ffn_sub_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_sub_norm", il); - - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].ffn_down, cur); - if (model.layers[il].ffn_down_scale) { - cur = ggml_mul(ctx0, cur, model.layers[il].ffn_down_scale); - } - cb(cur, "ffn_down", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - // FIXME: do not use model.tok_embd directly, duplicate as model.output - cur = llm_build_lora_mm(lctx, ctx0, model.tok_embd, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - return gf; - } - - struct ggml_cgraph * build_t5_enc() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - GGML_ASSERT(lctx.is_encoding); - struct ggml_tensor * pos_bucket_enc = llm_build_pos_bucket(false); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask_enc = build_inp_KQ_mask(false); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm_enc, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq_enc, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk_enc, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv_enc, cur); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3); - struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3)); - - struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); - cb(kq, "kq", il); - - struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc; - struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_enc, attn_rel_b); - struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias); - cb(kq_b, "kq_b", il); - - kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_enc, 1.0f, hparams.f_max_alibi_bias); - cb(kq, "kq_soft_max_ext", il); - - struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens))); - cb(v, "v", il); - - struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq); - cb(kqv, "kqv", il); - - struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3); - cb(kqv_merged, "kqv_merged", il); - - cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens); - cb(cur, "kqv_merged_cont", il); - - ggml_build_forward_expand(gf, cur); - - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo_enc, cur); - cb(cur, "kqv_out", il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm_enc, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - // T5 uses relu, flan-T5 uses gelu-gated - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up_enc, NULL, NULL, - model.layers[il].ffn_gate_enc, NULL, NULL, - model.layers[il].ffn_down_enc, NULL, NULL, - NULL, - model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU, - model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ, - cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - ggml_tensor * layer_dir = lctx.cvec.tensor_for(il); - if (layer_dir != nullptr) { - cur = ggml_add(ctx0, cur, layer_dir); - } - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - cb(cur, "result_embd", -1); - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm_enc, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_t5_dec() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - GGML_ASSERT(!lctx.is_encoding); - GGML_ASSERT(n_outputs_enc > 0 && "call llama_encode() first"); - - struct ggml_tensor * embd_enc = llm_build_inp_embd_enc(); - struct ggml_tensor * pos_bucket_dec = llm_build_pos_bucket(true); - - struct ggml_tensor * KQ_mask_dec = build_inp_KQ_mask(); - struct ggml_tensor * KQ_mask_cross = llm_build_inp_KQ_mask_cross(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - llm_build_kv_store(ctx0, hparams, cparams, kv_self, gf, Kcur, Vcur, n_tokens, kv_head, cb, il); - - struct ggml_tensor * k = - ggml_view_3d(ctx0, kv_self.k_l[il], - n_embd_head_k, n_kv, n_head_kv, - ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa), - ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k), - 0); - cb(k, "k", il); - - struct ggml_tensor * v = - ggml_view_3d(ctx0, kv_self.v_l[il], - n_kv, n_embd_head_v, n_head_kv, - ggml_element_size(kv_self.v_l[il])*n_ctx, - ggml_element_size(kv_self.v_l[il])*n_ctx*n_embd_head_v, - 0); - cb(v, "v", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3); - - struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); - cb(kq, "kq", il); - - struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b; - struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_dec, attn_rel_b); - struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias); - cb(kq_b, "kq_b", il); - - kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_dec, 1.0f, hparams.f_max_alibi_bias); - cb(kq, "kq_soft_max_ext", il); - - struct ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq); - cb(kqv, "kqv", il); - - struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3); - cb(kqv_merged, "kqv_merged", il); - - cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens); - cb(cur, "kqv_merged_cont", il); - - ggml_build_forward_expand(gf, cur); - - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo, cur); - cb(cur, "kqv_out", il); - } - - cur = ggml_add(ctx0, cur, inpSA); - cb(cur, "cross_inp", il); - - struct ggml_tensor * inpCA = cur; - - // norm - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].attn_norm_cross, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm_cross", il); - - // cross-attention - { - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq_cross, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk_cross, embd_enc); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv_cross, embd_enc); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc); - - struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3); - struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3)); - - struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); - cb(kq, "kq", il); - - kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias); - cb(kq, "kq_soft_max_ext", il); - - struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc))); - cb(v, "v", il); - - struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq); - cb(kqv, "kqv", il); - - struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3); - cb(kqv_merged, "kqv_merged", il); - - cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens); - cb(cur, "kqv_merged_cont", il); - - ggml_build_forward_expand(gf, cur); - - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo_cross, cur); - cb(cur, "kqv_out", il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - // T5 uses relu, flan-T5 uses gelu-gated - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU, - model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ, - cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - ggml_tensor * layer_dir = lctx.cvec.tensor_for(il); - if (layer_dir != nullptr) { - cur = ggml_add(ctx0, cur, layer_dir); - } - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - cb(cur, "result_embd", -1); - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_jais() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*cur->nb[0]*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*cur->nb[0]*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*cur->nb[0]*(n_embd + n_embd_gqa))); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/float(n_embd_head), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); - } - - // add the input - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); - cb(ffn_inp, "ffn_inp", il); - - // FF - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - } - - inpL = ggml_add(ctx0, cur, ffn_inp); - cb(inpL, "l_out", il); - } - - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_chatglm() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - struct ggml_tensor * Qcur = nullptr; - struct ggml_tensor * Kcur = nullptr; - struct ggml_tensor * Vcur = nullptr; - if (model.layers[il].wqkv == nullptr) { - Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - } - Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - } - Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - } - } else { - cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - if (model.layers[il].bqkv) { - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - } - Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - } - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - //printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor); - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur_rope", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur_rope", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - // Add the input - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // FF - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SWIGLU, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - - } - - inpL = ggml_add(ctx0, cur, ffn_inp); - cb(inpL, "l_out", il); - } - - cur = llm_build_norm(ctx0, inpL, hparams, - model.output_norm, - NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_nemotron() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - //GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - NULL, NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, - NULL, - LLM_FFN_RELU_SQR, LLM_FFN_SEQ, cb, il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_exaone() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // rope freq factors for llama3; may return nullptr for llama2 and other models - struct ggml_tensor * rope_factors = build_rope_factors(il); - - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - if (model.layers[il].bq) { - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - cb(Qcur, "Qcur", il); - } - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - if (model.layers[il].bk) { - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - cb(Kcur, "Kcur", il); - } - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - if (model.layers[il].bv) { - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - cb(Vcur, "Vcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - ggml_cgraph * build_rwkv6() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // Token shift state dimensions should be 2 * n_emb - GGML_ASSERT(n_embd == hparams.n_embd_k_s() / 2); - - const int64_t n_seqs = ubatch.n_seqs; - const int64_t n_seq_tokens = ubatch.n_seq_tokens; - const int64_t n_tokens = ubatch.n_tokens; - GGML_ASSERT(n_seqs != 0); - GGML_ASSERT(ubatch.equal_seqs); - GGML_ASSERT(n_tokens == n_seq_tokens * n_seqs); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - struct ggml_tensor * state_copy = build_inp_s_copy(); - struct ggml_tensor * state_mask = build_inp_s_mask(); - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - inpL = llm_build_norm(ctx0, inpL, hparams, model.tok_norm, model.tok_norm_b, LLM_NORM, cb, -1); - - for (int il = 0; il < n_layer; ++il) { - const llama_layer * layer = &model.layers[il]; - - // (ab)using the KV cache to store the states - struct ggml_tensor * token_shift = llm_build_copy_mask_state(ctx0, - gf, kv_self.k_l[il], state_copy, state_mask, - hparams.n_embd_k_s(), kv_self.size, kv_head, n_kv, n_seqs); - struct ggml_tensor * wkv_states = llm_build_copy_mask_state(ctx0, - gf, kv_self.v_l[il], state_copy, state_mask, - hparams.n_embd_v_s(), kv_self.size, kv_head, n_kv, n_seqs); - - cur = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); - token_shift = ggml_reshape_3d(ctx0, token_shift, n_embd, 2, n_seqs); - - struct ggml_tensor * att_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], 0); - struct ggml_tensor * ffn_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], n_embd * ggml_element_size(token_shift)); - - struct ggml_tensor * x_norm_att = llm_build_norm(ctx0, cur, hparams, layer->attn_norm, layer->attn_norm_b, LLM_NORM, cb, il); - struct ggml_tensor * x_prev = ggml_concat( - ctx0, - att_shift, - ggml_view_3d(ctx0, x_norm_att, n_embd, n_seq_tokens - 1, n_seqs, x_norm_att->nb[1], x_norm_att->nb[2], 0), - 1 - ); - - cur = ggml_add(ctx0, cur, llm_build_rwkv6_time_mix(lctx, ctx0, layer, x_norm_att, x_prev, &wkv_states, hparams.wkv_head_size, n_embd / hparams.wkv_head_size)); - ggml_build_forward_expand(gf, cur); - ggml_build_forward_expand( - gf, - ggml_cpy( - ctx0, - wkv_states, - ggml_view_1d( - ctx0, - kv_self.v_l[il], - hparams.n_embd_v_s() * n_seqs, - hparams.n_embd_v_s() * kv_head * ggml_element_size(kv_self.v_l[il]) - ) - ) - ); - - struct ggml_tensor * x_norm_ffn = llm_build_norm(ctx0, cur, hparams, layer->attn_norm_2, layer->attn_norm_2_b, LLM_NORM, cb, il); - x_prev = ggml_concat( - ctx0, - ffn_shift, - ggml_view_3d(ctx0, x_norm_ffn, n_embd, n_seq_tokens - 1, n_seqs, x_norm_ffn->nb[1], x_norm_ffn->nb[2], 0), - 1 - ); - cur = ggml_add(ctx0, cur, llm_build_rwkv6_channel_mix(lctx, ctx0, layer, x_norm_ffn, x_prev)); - ggml_build_forward_expand(gf, cur); - - struct ggml_tensor * last_norm_att = ggml_view_3d(ctx0, x_norm_att, n_embd, 1, n_seqs, x_norm_att->nb[1], x_norm_att->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(x_norm_att)); - struct ggml_tensor * last_norm_ffn = ggml_view_3d(ctx0, x_norm_ffn, n_embd, 1, n_seqs, x_norm_ffn->nb[1], x_norm_ffn->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(x_norm_ffn)); - - token_shift = ggml_concat(ctx0, last_norm_att, last_norm_ffn, 1); - - ggml_build_forward_expand( - gf, - ggml_cpy( - ctx0, - ggml_view_1d(ctx0, token_shift, n_embd * n_seqs * 2, 0), - ggml_view_1d(ctx0, kv_self.k_l[il], hparams.n_embd_k_s() * n_seqs, hparams.n_embd_k_s() * kv_head * ggml_element_size(kv_self.k_l[il])) - ) - ); - - if (hparams.rescale_every_n_layers != 0 && (il + 1) % hparams.rescale_every_n_layers == 0) { - cur = ggml_scale(ctx0, cur, 0.5F); - } - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - - cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, model.output_norm_b, LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - // ref: https://huggingface.co/recursal/QRWKV6-32B-Instruct-Preview-v0.1/blob/main/modeling_rwkv6qwen2.py - ggml_cgraph * build_rwkv6qwen2() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - GGML_ASSERT(n_embd == hparams.n_embd_k_s()); - - const int64_t n_seqs = ubatch.n_seqs; - const int64_t n_seq_tokens = ubatch.n_seq_tokens; - const int64_t n_tokens = ubatch.n_tokens; - GGML_ASSERT(n_seqs != 0); - GGML_ASSERT(ubatch.equal_seqs); - GGML_ASSERT(n_tokens == n_seq_tokens * n_seqs); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - struct ggml_tensor * state_copy = build_inp_s_copy(); - struct ggml_tensor * state_mask = build_inp_s_mask(); - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - for (int il = 0; il < n_layer; ++il) { - const llama_layer * layer = &model.layers[il]; - - // (ab)using the KV cache to store the states - struct ggml_tensor * token_shift = llm_build_copy_mask_state(ctx0, - gf, kv_self.k_l[il], state_copy, state_mask, - hparams.n_embd_k_s(), kv_self.size, kv_head, n_kv, n_seqs); - struct ggml_tensor * wkv_states = llm_build_copy_mask_state(ctx0, - gf, kv_self.v_l[il], state_copy, state_mask, - hparams.n_embd_v_s(), kv_self.size, kv_head, n_kv, n_seqs); - - cur = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); - token_shift = ggml_reshape_3d(ctx0, token_shift, n_embd, 1, n_seqs); - - struct ggml_tensor * x_norm_att = llm_build_norm(ctx0, cur, hparams, layer->attn_norm, layer->attn_norm_b, LLM_NORM_RMS, cb, il); - struct ggml_tensor * x_prev = ggml_concat( - ctx0, - token_shift, - ggml_view_3d(ctx0, x_norm_att, n_embd, n_seq_tokens - 1, n_seqs, x_norm_att->nb[1], x_norm_att->nb[2], 0), - 1 - ); - - struct ggml_tensor * last_norm_att = ggml_view_3d(ctx0, x_norm_att, n_embd, 1, n_seqs, x_norm_att->nb[1], x_norm_att->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(x_norm_att)); - ggml_build_forward_expand( - gf, - ggml_cpy( - ctx0, - ggml_view_1d(ctx0, last_norm_att, n_embd * n_seqs, 0), - ggml_view_1d(ctx0, kv_self.k_l[il], hparams.n_embd_k_s() * n_seqs, hparams.n_embd_k_s() * kv_head * ggml_element_size(kv_self.k_l[il])) - ) - ); - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, llm_build_rwkv6_time_mix(lctx, ctx0, layer, x_norm_att, x_prev, &wkv_states, hparams.wkv_head_size, hparams.n_head_kv())); - ggml_build_forward_expand(gf, ffn_inp); - ggml_build_forward_expand( - gf, - ggml_cpy( - ctx0, - wkv_states, - ggml_view_1d( - ctx0, - kv_self.v_l[il], - hparams.n_embd_v_s() * n_seqs, - hparams.n_embd_v_s() * kv_head * ggml_element_size(kv_self.v_l[il]) - ) - ) - ); - - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - - cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, model.output_norm_b, LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - // ref: https://github.com/facebookresearch/chameleon - // based on the original build_llama() function, changes: - // * qk-norm - // * swin-norm - // * removed bias - // * removed MoE - struct ggml_cgraph * build_chameleon() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - // mutable variable, needed during the last layer of the computation to skip unused tokens - int32_t n_tokens = this->n_tokens; - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - if (hparams.swin_norm) { - cur = inpL; - } else { - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "attn_norm", il); - } - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - if (model.layers[il].attn_q_norm) { - Qcur = ggml_view_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens, - ggml_element_size(Qcur) * n_embd_head, - ggml_element_size(Qcur) * n_embd_head * n_head, - 0); - cb(Qcur, "Qcur", il); - - Qcur = llm_build_norm(ctx0, Qcur, hparams, - model.layers[il].attn_q_norm, - model.layers[il].attn_q_norm_b, - LLM_NORM, cb, il); - cb(Qcur, "Qcur", il); - } - - if (model.layers[il].attn_k_norm) { - Kcur = ggml_view_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens, - ggml_element_size(Kcur) * n_embd_head, - ggml_element_size(Kcur) * n_embd_head * n_head_kv, - 0); - cb(Kcur, "Kcur", il); - - Kcur = llm_build_norm(ctx0, Kcur, hparams, - model.layers[il].attn_k_norm, - model.layers[il].attn_k_norm_b, - LLM_NORM, cb, il); - cb(Kcur, "Kcur", il); - } - - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, lctx, kv_self, gf, - model.layers[il].wo, nullptr, - Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - - if (hparams.swin_norm) { - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].attn_norm, NULL, - LLM_NORM_RMS, cb, il); - } - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - n_tokens = n_outputs; - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - if (!hparams.swin_norm) { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - } - - cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - if (hparams.swin_norm) { - cur = llm_build_norm(ctx0, cur, hparams, - model.layers[il].ffn_norm, NULL, - LLM_NORM_RMS, cb, il); - cb(cur, "ffn_norm", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "ffn_out", il); - - cur = lctx.cvec.apply_to(ctx0, cur, il); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, NULL, - LLM_NORM_RMS, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - cb(cur, "result_output_with_img_logits", -1); - - // TODO: this suppresses the output of image tokens, which is required to enable text-only outputs. - // Needs to be removed once image outputs are supported. - int img_token_end_idx = 8196; - int img_token_start_idx = 4; - int num_img_tokens = img_token_end_idx - img_token_start_idx; - // creates 1d tensor of size num_img_tokens and values -FLT_MAX, - // which ensures that text token values are always at least larger than image token values - struct ggml_tensor * img_logits = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, num_img_tokens); - img_logits = ggml_clamp(ctx0, img_logits, -FLT_MAX, -FLT_MAX); - cb(img_logits, "img_logits", -1); - cur = ggml_set_1d(ctx0, cur, img_logits, ggml_element_size(cur) * img_token_start_idx); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_wavtokenizer_dec() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, model.max_nodes(), false); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); - - cur = ggml_cont(ctx0, ggml_transpose(ctx0, inpL)); - - cur = ggml_conv_1d_ph(ctx0, model.conv1d, cur, 1, 1); - cur = ggml_add(ctx0, cur, model.conv1d_b); - - // posnet - for (uint32_t il = 0; il < hparams.posnet.n_layer; ++il) { - const auto & layer = model.layers[il].posnet; - - inpL = cur; - - switch (il) { - case 0: - case 1: - case 3: - case 4: - { - cur = llm_build_norm(ctx0, cur, hparams, - layer.norm1, - layer.norm1_b, - LLM_NORM_GROUP, cb, 0); - - cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur); - - cur = ggml_conv_1d_ph(ctx0, layer.conv1, cur, 1, 1); - cur = ggml_add(ctx0, cur, layer.conv1_b); - - cur = llm_build_norm(ctx0, cur, hparams, - layer.norm2, - layer.norm2_b, - LLM_NORM_GROUP, cb, 0); - - cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur); - - cur = ggml_conv_1d_ph(ctx0, layer.conv2, cur, 1, 1); - cur = ggml_add(ctx0, cur, layer.conv2_b); - - cur = ggml_add(ctx0, cur, inpL); - } break; - case 2: - { - cur = llm_build_norm(ctx0, cur, hparams, - layer.attn_norm, - layer.attn_norm_b, - LLM_NORM_GROUP, cb, 0); - - struct ggml_tensor * q; - struct ggml_tensor * k; - struct ggml_tensor * v; - - q = ggml_conv_1d_ph(ctx0, layer.attn_q, cur, 1, 1); - k = ggml_conv_1d_ph(ctx0, layer.attn_k, cur, 1, 1); - v = ggml_conv_1d_ph(ctx0, layer.attn_v, cur, 1, 1); - - q = ggml_add(ctx0, q, layer.attn_q_b); - k = ggml_add(ctx0, k, layer.attn_k_b); - v = ggml_add(ctx0, v, layer.attn_v_b); - - q = ggml_cont(ctx0, ggml_transpose(ctx0, q)); - k = ggml_cont(ctx0, ggml_transpose(ctx0, k)); - - struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); - - kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(hparams.posnet.n_embd)), 0.0f); - - cur = ggml_mul_mat(ctx0, kq, v); - - cur = ggml_conv_1d_ph(ctx0, layer.attn_o, cur, 1, 1); - cur = ggml_add(ctx0, cur, layer.attn_o_b); - - cur = ggml_add(ctx0, cur, inpL); - } break; - case 5: - { - cur = llm_build_norm(ctx0, cur, hparams, - layer.norm, - layer.norm_b, - LLM_NORM_GROUP, cb, 0); - } break; - default: GGML_ABORT("unknown posnet layer"); - }; - } - - cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); - - cur = llm_build_norm(ctx0, cur, hparams, - model.tok_norm, - model.tok_norm_b, - LLM_NORM, cb, -1); - - cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); - - inpL = cur; - - // convnext - for (uint32_t il = 0; il < hparams.convnext.n_layer; ++il) { - const auto & layer = model.layers[il].convnext; - - cur = inpL; - - cur = ggml_conv_1d_dw_ph(ctx0, layer.dw, cur, 1, 1); - cur = ggml_add(ctx0, cur, layer.dw_b); - - cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); - - cur = llm_build_norm(ctx0, cur, hparams, - layer.norm, - layer.norm_b, - LLM_NORM, cb, -1); - - cur = llm_build_ffn(ctx0, lctx, cur, - layer.pw1, layer.pw1_b, NULL, - NULL, NULL, NULL, - layer.pw2, layer.pw2_b, NULL, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - - cur = ggml_mul(ctx0, cur, layer.gamma); - - cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); - - inpL = ggml_add(ctx0, cur, inpL); - } - - cur = inpL; - - cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - - // lm_head - cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); - - cur = ggml_add(ctx0, cur, model.output_b); - cb(cur, "result_embd", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } -}; - -static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector & ids) { - llama_ubatch dummy = {}; - dummy.equal_seqs = true; - - llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { }; - - struct llm_build_context llm(lctx, dummy, cb, false); - - llm.init(); - - struct ggml_cgraph * result = llm.build_defrag(ids); - - llm.free(); - - return result; -} - -static struct ggml_cgraph * llama_build_graph_k_shift(llama_context & lctx) { - llama_ubatch dummy = {}; - dummy.equal_seqs = true; - - llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { }; - - struct llm_build_context llm(lctx, dummy, cb, false); - - llm.init(); - - struct ggml_cgraph * result = llm.build_k_shift(); - - llm.free(); - - return result; -} - -static struct ggml_cgraph * llama_build_graph( - llama_context & lctx, - const llama_ubatch & ubatch, - bool worst_case) { - const auto & model = lctx.model; - - // this callback allows us to apply custom logic to each tensor (e.g. ggml-alloc, offloading, etc.) - llm_build_cb cb = [&](struct ggml_tensor * cur, const char * name, int il) { - if (il >= 0) { - ggml_format_name(cur, "%s-%d", name, il); - } else { - ggml_set_name(cur, name); - } - - if (!lctx.cparams.offload_kqv) { - if (strcmp(name, "kqv_merged_cont") == 0) { - // all nodes between the KV store and the attention output are run on the CPU - ggml_backend_sched_set_tensor_backend(lctx.sched.get(), cur, lctx.backend_cpu); - } - } - - // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends - // FIXME: fix in ggml_backend_sched - const bool full_offload = lctx.model.params.n_gpu_layers > (int) lctx.model.hparams.n_layer; - if (ubatch.n_tokens < 32 || full_offload) { - if (il != -1 && strcmp(name, "norm") == 0) { - const auto & dev_layer = lctx.model.dev_layer(il); - for (auto & backend : lctx.backends) { - if (ggml_backend_get_device(backend.get()) == dev_layer) { - if (ggml_backend_supports_op(backend.get(), cur)) { - ggml_backend_sched_set_tensor_backend(lctx.sched.get(), cur, backend.get()); - } - } - } - } - } - }; - - struct ggml_cgraph * result = NULL; - - struct llm_build_context llm(lctx, ubatch, cb, worst_case); - - llm.init(); - - switch (model.arch) { - case LLM_ARCH_LLAMA: - case LLM_ARCH_MINICPM: - case LLM_ARCH_GRANITE: - case LLM_ARCH_GRANITE_MOE: - { - result = llm.build_llama(); - } break; - case LLM_ARCH_DECI: - { - result = llm.build_deci(); - } break; - case LLM_ARCH_BAICHUAN: - { - result = llm.build_baichuan(); - } break; - case LLM_ARCH_FALCON: - { - result = llm.build_falcon(); - } break; - case LLM_ARCH_GROK: - { - result = llm.build_grok(); - } break; - case LLM_ARCH_STARCODER: - { - result = llm.build_starcoder(); - } break; - case LLM_ARCH_REFACT: - { - result = llm.build_refact(); - } break; - case LLM_ARCH_BERT: - case LLM_ARCH_JINA_BERT_V2: - case LLM_ARCH_NOMIC_BERT: - { - result = llm.build_bert(); - } break; - case LLM_ARCH_BLOOM: - { - result = llm.build_bloom(); - } break; - case LLM_ARCH_MPT: - { - result = llm.build_mpt(); - } break; - case LLM_ARCH_STABLELM: - { - result = llm.build_stablelm(); - } break; - case LLM_ARCH_QWEN: - { - result = llm.build_qwen(); - } break; - case LLM_ARCH_QWEN2: - { - result = llm.build_qwen2(); - } break; - case LLM_ARCH_QWEN2VL: - { - lctx.n_pos_per_token = 4; - result = llm.build_qwen2vl(); - } break; - case LLM_ARCH_QWEN2MOE: - { - result = llm.build_qwen2moe(); - } break; - case LLM_ARCH_PHI2: - { - result = llm.build_phi2(); - } break; - case LLM_ARCH_PHI3: - case LLM_ARCH_PHIMOE: - { - result = llm.build_phi3(); - } break; - case LLM_ARCH_PLAMO: - { - result = llm.build_plamo(); - } break; - case LLM_ARCH_GPT2: - { - result = llm.build_gpt2(); - } break; - case LLM_ARCH_CODESHELL: - { - result = llm.build_codeshell(); - } break; - case LLM_ARCH_ORION: - { - result = llm.build_orion(); - } break; - case LLM_ARCH_INTERNLM2: - { - result = llm.build_internlm2(); - } break; - case LLM_ARCH_MINICPM3: - { - result = llm.build_minicpm3(); - } break; - case LLM_ARCH_GEMMA: - { - result = llm.build_gemma(); - } break; - case LLM_ARCH_GEMMA2: - { - result = llm.build_gemma2(); - } break; - case LLM_ARCH_GEMMA3: - { - result = llm.build_gemma3(); - } break; - case LLM_ARCH_STARCODER2: - { - result = llm.build_starcoder2(); - } break; - case LLM_ARCH_MAMBA: - { - result = llm.build_mamba(); - } break; - case LLM_ARCH_XVERSE: - { - result = llm.build_xverse(); - } break; - case LLM_ARCH_COMMAND_R: - { - result = llm.build_command_r(); - } break; - case LLM_ARCH_COHERE2: - { - result = llm.build_cohere2(); - } break; - case LLM_ARCH_DBRX: - { - result = llm.build_dbrx(); - } break; - case LLM_ARCH_OLMO: - { - result = llm.build_olmo(); - } break; - case LLM_ARCH_OLMO2: - { - result = llm.build_olmo2(); - } break; - case LLM_ARCH_OLMOE: - { - result = llm.build_olmoe(); - } break; - case LLM_ARCH_OPENELM: - { - result = llm.build_openelm(); - } break; - case LLM_ARCH_GPTNEOX: - { - result = llm.build_gptneox(); - } break; - case LLM_ARCH_ARCTIC: - { - result = llm.build_arctic(); - } break; - case LLM_ARCH_DEEPSEEK: - { - result = llm.build_deepseek(); - } break; - case LLM_ARCH_DEEPSEEK2: - { - result = llm.build_deepseek2(); - } break; - case LLM_ARCH_CHATGLM: - { - result = llm.build_chatglm(); - } break; - case LLM_ARCH_BITNET: - { - result = llm.build_bitnet(); - } break; - case LLM_ARCH_T5: - { - if (lctx.is_encoding) { - result = llm.build_t5_enc(); - } else { - result = llm.build_t5_dec(); - } - } break; - case LLM_ARCH_T5ENCODER: - { - result = llm.build_t5_enc(); - } break; - case LLM_ARCH_JAIS: - { - result = llm.build_jais(); - } break; - case LLM_ARCH_NEMOTRON: - { - result = llm.build_nemotron(); - } break; - case LLM_ARCH_EXAONE: - { - result = llm.build_exaone(); - } break; - case LLM_ARCH_RWKV6: - { - result = llm.build_rwkv6(); - } break; - case LLM_ARCH_RWKV6QWEN2: - { - result = llm.build_rwkv6qwen2(); - } break; - case LLM_ARCH_CHAMELEON: - { - result = llm.build_chameleon(); - } break; - case LLM_ARCH_WAVTOKENIZER_DEC: - { - result = llm.build_wavtokenizer_dec(); - } break; - default: - GGML_ABORT("fatal error"); - } - - // add on pooling layer - if (lctx.cparams.embeddings) { - result = llm.append_pooling(result); - } - - llm.free(); - - return result; -} - -// returns the result of ggml_backend_sched_graph_compute_async execution -static enum ggml_status llama_graph_compute( - llama_context & lctx, - ggml_cgraph * gf, - int n_threads, - ggml_threadpool * threadpool) { - if (lctx.backend_cpu != nullptr) { - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(lctx.backend_cpu)); - auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); - set_threadpool_fn(lctx.backend_cpu, threadpool); - } - - // set the number of threads for all the backends - for (const auto & set_n_threads_fn : lctx.set_n_threads_fns) { - set_n_threads_fn.second(set_n_threads_fn.first, n_threads); - } - - auto status = ggml_backend_sched_graph_compute_async(lctx.sched.get(), gf); - if (status != GGML_STATUS_SUCCESS) { - LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status); - } - - // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched)); - - return status; -} - -static int llama_prepare_sbatch( - llama_context & lctx, - const llama_batch & batch, - uint32_t & n_outputs) { - const auto & model = lctx.model; - const auto & hparams = model.hparams; - const auto & cparams = lctx.cparams; - - const uint32_t n_tokens_all = batch.n_tokens; - const int64_t n_embd = hparams.n_embd; - - // this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens - const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE; - - GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT - if (batch.token) { - for (uint32_t i = 0; i < n_tokens_all; ++i) { - if (batch.token[i] < 0 || uint32_t(batch.token[i]) >= model.vocab.n_tokens()) { - LLAMA_LOG_ERROR("%s: invalid token[%d] = %d\n", __func__, i, batch.token[i]); - return -1; - } - } - } - GGML_ASSERT(n_tokens_all <= cparams.n_batch); - GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens"); - - lctx.n_queued_tokens += n_tokens_all; - lctx.embd_seq.clear(); - - // count outputs - if (batch.logits && !embd_pooled) { - for (uint32_t i = 0; i < n_tokens_all; ++i) { - n_outputs += batch.logits[i] != 0; - } - } else if (lctx.logits_all || embd_pooled) { - n_outputs = n_tokens_all; - } else { - // keep last output only - n_outputs = 1; - } - - lctx.sbatch.from_batch(batch, n_embd, - /* simple_split */ !lctx.kv_self.recurrent, - /* logits_all */ n_outputs == n_tokens_all); - - // reserve output buffer - if (llama_output_reserve(lctx, n_outputs) < n_outputs) { - LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_outputs); - return -2; - }; - - return 0; -} - -static int llama_prepare_ubatch( - llama_context & lctx, - llama_kv_slot_restorer & kv_slot_restorer, - llama_ubatch & ubatch, - const uint32_t n_outputs, - const uint32_t n_tokens_all) { - GGML_ASSERT(lctx.sbatch.n_tokens > 0); - - auto & kv_self = lctx.kv_self; - const auto & cparams = lctx.cparams; - const auto & hparams = lctx.model.hparams; - - // this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens - const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE; - - if (lctx.kv_self.recurrent) { - if (embd_pooled) { - // Pooled embeddings cannot be split across ubatches (yet) - ubatch = lctx.sbatch.split_seq(cparams.n_ubatch); - } else { - // recurrent model architectures are easier to implement - // with equal-length sequences - ubatch = lctx.sbatch.split_equal(cparams.n_ubatch); - } - } else { - ubatch = lctx.sbatch.split_simple(cparams.n_ubatch); - } - - // count the outputs in this u_batch - { - int32_t n_outputs_new = 0; - - if (n_outputs == n_tokens_all) { - n_outputs_new = ubatch.n_tokens; - } else { - GGML_ASSERT(ubatch.output); - for (uint32_t i = 0; i < ubatch.n_tokens; i++) { - n_outputs_new += int32_t(ubatch.output[i] != 0); - } - } - - // needs to happen before the graph is built - lctx.n_outputs = n_outputs_new; - } - - // non-causal masks do not use the KV cache - if (hparams.causal_attn) { - llama_kv_cache_update(&lctx); - - // if we have enough unused cells before the current head -> - // better to start searching from the beginning of the cache, hoping to fill it - if (kv_self.head > kv_self.used + 2*ubatch.n_tokens) { - kv_self.head = 0; - } - - const auto slot = llama_kv_cache_find_slot(kv_self, ubatch); - if (!slot) { - return 1; - } - kv_slot_restorer.save(slot); - - if (!kv_self.recurrent) { - // a heuristic, to avoid attending the full cache if it is not yet utilized - // after enough generations, the benefit from this heuristic disappears - // if we start defragmenting the cache, the benefit from this will be more important - const uint32_t pad = llama_kv_cache_get_padding(cparams); - kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(llama_kv_cache_cell_max(kv_self), pad))); - //kv_self.n = llama_kv_cache_cell_max(kv_self); - } - } - - return 0; -} - -// decode a batch of tokens by evaluating the transformer -// in case of unsuccessful decoding (error or warning), -// the kv_cache state will be returned to its original state -// (for non-recurrent models) or cleaned (for recurrent models) -// -// - lctx: llama context -// - inp_batch: batch to evaluate -// -// return 0 on success -// return positive int on warning -// return negative int on error -// -static int llama_decode_impl( - llama_context & lctx, - llama_batch inp_batch) { - - lctx.is_encoding = false; - - if (inp_batch.n_tokens == 0) { - LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__); - return -1; - } - - // temporarily allocate memory for the input batch if needed - llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : lctx.kv_self.max_pos() + 1); - const llama_batch & batch = batch_allocr.batch; - - const auto & model = lctx.model; - const auto & vocab = model.vocab; - const auto & hparams = model.hparams; - const auto & cparams = lctx.cparams; - - if (lctx.t_compute_start_us == 0) { - lctx.t_compute_start_us = ggml_time_us(); - } - auto & kv_self = lctx.kv_self; - llama_kv_slot_restorer kv_slot_restorer(kv_self); - - const int64_t n_embd = hparams.n_embd; - const int64_t n_vocab = vocab.n_tokens(); - - uint32_t n_outputs = 0; - uint32_t n_outputs_prev = 0; - - { - const int ret = llama_prepare_sbatch(lctx, batch, n_outputs); - if (ret != 0) { - return ret; - } - } - - while (lctx.sbatch.n_tokens > 0) { - llama_ubatch ubatch; - { - const int ret = llama_prepare_ubatch(lctx, kv_slot_restorer, ubatch, n_outputs, batch.n_tokens); - if (ret != 0) { - return ret; - } - } - - const int n_threads = ubatch.n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch; - ggml_threadpool_t threadpool = ubatch.n_tokens == 1 ? lctx.threadpool : lctx.threadpool_batch; - - GGML_ASSERT(n_threads > 0); - - //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); - - ggml_backend_sched_reset(lctx.sched.get()); - ggml_backend_sched_set_eval_callback(lctx.sched.get(), lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data); - - ggml_cgraph * gf = llama_build_graph(lctx, ubatch, false); - - // the output is always the last tensor in the graph - struct ggml_tensor * res = ggml_graph_node(gf, -1); - struct ggml_tensor * embd = ggml_graph_node(gf, -2); - - if (lctx.n_outputs == 0) { - // no output - res = nullptr; - embd = nullptr; - } else if (cparams.embeddings) { - res = nullptr; // do not extract logits for embedding case - embd = nullptr; - for (int i = ggml_graph_n_nodes(gf) - 1; i >= 0; --i) { - if (strcmp(ggml_graph_node(gf, i)->name, "result_embd_pooled") == 0) { - embd = ggml_graph_node(gf, i); - break; - } - } - GGML_ASSERT(embd != nullptr && "missing embeddings tensor"); - } else { - embd = nullptr; // do not extract embeddings when not needed - GGML_ASSERT(strcmp(res->name, "result_output") == 0 && "missing result_output tensor"); - } - - // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); - - ggml_backend_sched_alloc_graph(lctx.sched.get(), gf); - - llama_set_inputs(lctx, ubatch); - - const auto compute_status = llama_graph_compute(lctx, gf, n_threads, threadpool); - if (compute_status != GGML_STATUS_SUCCESS) { - kv_slot_restorer.restore(kv_self); - switch (compute_status) { - case GGML_STATUS_ABORTED: - return 2; - case GGML_STATUS_ALLOC_FAILED: - return -2; - case GGML_STATUS_FAILED: - default: - return -3; - } - } - - // update the kv ring buffer - { - kv_self.head += ubatch.n_tokens; - - // Ensure kv cache head points to a valid index. - if (kv_self.head >= kv_self.size) { - kv_self.head = 0; - } - } - - // plot the computation graph in dot format (for debugging purposes) - //if (n_past%100 == 0) { - // ggml_graph_dump_dot(gf, NULL, "llama.dot"); - //} - - // extract logits - if (res) { - ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(lctx.sched.get(), res); - GGML_ASSERT(backend_res != nullptr); - GGML_ASSERT(lctx.logits != nullptr); - - float * logits_out = lctx.logits + n_outputs_prev*n_vocab; - const int32_t n_outputs_new = lctx.n_outputs; - - if (n_outputs_new) { - GGML_ASSERT( n_outputs_prev + n_outputs_new <= n_outputs); - GGML_ASSERT((n_outputs_prev + n_outputs_new)*n_vocab <= (int64_t) lctx.logits_size); - ggml_backend_tensor_get_async(backend_res, res, logits_out, 0, n_outputs_new*n_vocab*sizeof(float)); - } - } - - // extract embeddings - if (embd) { - ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(lctx.sched.get(), embd); - GGML_ASSERT(backend_embd != nullptr); - - switch (cparams.pooling_type) { - case LLAMA_POOLING_TYPE_NONE: - { - // extract token embeddings - GGML_ASSERT(lctx.embd != nullptr); - float * embd_out = lctx.embd + n_outputs_prev*n_embd; - const int32_t n_outputs_new = lctx.n_outputs; - - if (n_outputs_new) { - GGML_ASSERT( n_outputs_prev + n_outputs_new <= n_outputs); - GGML_ASSERT((n_outputs_prev + n_outputs_new)*n_embd <= (int64_t) lctx.embd_size); - ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_outputs_new*n_embd*sizeof(float)); - } - } break; - case LLAMA_POOLING_TYPE_MEAN: - case LLAMA_POOLING_TYPE_CLS: - case LLAMA_POOLING_TYPE_LAST: - { - // extract sequence embeddings (cleared before processing each batch) - auto & embd_seq_out = lctx.embd_seq; - - for (uint32_t s = 0; s < ubatch.n_seqs; ++s) { - const llama_seq_id seq_id = ubatch.seq_id[s][0]; - if (embd_seq_out.find(seq_id) != embd_seq_out.end()) { - continue; - } - embd_seq_out[seq_id].resize(n_embd); - ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float)); - } - } break; - case LLAMA_POOLING_TYPE_RANK: - { - // extract the rerank score - a single float per sequence - auto & embd_seq_out = lctx.embd_seq; - - for (uint32_t s = 0; s < ubatch.n_seqs; ++s) { - const llama_seq_id seq_id = ubatch.seq_id[s][0]; - if (embd_seq_out.find(seq_id) != embd_seq_out.end()) { - continue; - } - embd_seq_out[seq_id].resize(1); - ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (seq_id)*sizeof(float), sizeof(float)); - } - } break; - case LLAMA_POOLING_TYPE_UNSPECIFIED: - { - GGML_ABORT("unknown pooling type"); - } - } - } - n_outputs_prev += lctx.n_outputs; - } - - // set output mappings - { - bool sorted_output = true; - - GGML_ASSERT(lctx.sbatch.out_ids.size() == n_outputs); - - for (size_t i = 0; i < n_outputs; ++i) { - size_t out_id = lctx.sbatch.out_ids[i]; - lctx.output_ids[out_id] = i; - if (out_id != i) { - sorted_output = false; - } - } - - if (sorted_output) { - lctx.sbatch.out_ids.clear(); - } - } - - // set to total number of outputs in the batch, for use in llama_get_logits_ith - lctx.n_outputs = n_outputs; - - // wait for the computation to finish (automatically done when obtaining the model output) - //llama_synchronize(&lctx); - - // decide if we need to defrag the kv cache - if (cparams.causal_attn && cparams.defrag_thold > 0.0f) { - // - do not defrag small contexts (i.e. < 2048 tokens) - // - count the padding towards the number of used tokens - const float fragmentation = kv_self.n >= 2048 ? std::max(0.0f, 1.0f - float(kv_self.used + llama_kv_cache_get_padding(cparams))/float(kv_self.n)) : 0.0f; - - // queue defragmentation for next llama_kv_cache_update - if (fragmentation > cparams.defrag_thold) { - LLAMA_LOG_DEBUG("%s: fragmentation: %.2f - requesting defrag\n", __func__, fragmentation); - - llama_kv_cache_defrag(kv_self); - } - } - - // Reset state for the next token before backend sync, to allow the CPU activities in the reset to - // overlap with device computation. - ggml_backend_sched_reset(lctx.sched.get()); - - return 0; -} - -// encode a batch of tokens by evaluating the encoder part of the transformer -// -// - lctx: llama context -// - batch: batch to evaluate -// -// return 0 on success -// return positive int on warning -// return negative int on error -// -static int llama_encode_impl( - llama_context & lctx, - llama_batch inp_batch) { - - lctx.is_encoding = true; - - if (inp_batch.n_tokens == 0) { - LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__); - return -1; - } - - // temporary allocate memory for the input batch if needed - llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : lctx.kv_self.max_pos() + 1); - - const llama_batch & batch = batch_allocr.batch; - const uint32_t n_tokens = batch.n_tokens; - - const auto & model = lctx.model; - const auto & hparams = model.hparams; - const auto & cparams = lctx.cparams; - - GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT - - if (batch.token) { - for (uint32_t i = 0; i < n_tokens; ++i) { - if (batch.token[i] < 0 || (uint32_t) batch.token[i] >= model.vocab.n_tokens()) { - LLAMA_LOG_ERROR("%s: invalid token[%d] = %d\n", __func__, i, batch.token[i]); - return -1; - } - } - } - - // micro-batching is not possible for non-causal encoding, so we process the batch in a single shot - GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens"); - - if (lctx.t_compute_start_us == 0) { - lctx.t_compute_start_us = ggml_time_us(); - } - - lctx.n_queued_tokens += n_tokens; - - const int64_t n_embd = hparams.n_embd; - - lctx.sbatch.from_batch(batch, n_embd, /* simple_split */ true, /* logits_all */ true); - - const llama_ubatch ubatch = lctx.sbatch.split_simple(n_tokens); - - // reserve output buffer - if (llama_output_reserve(lctx, n_tokens) < n_tokens) { - LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens); - return -2; - }; - - for (uint32_t i = 0; i < n_tokens; ++i) { - lctx.output_ids[i] = i; - } - - lctx.inp_embd_enc = NULL; - lctx.n_outputs = n_tokens; - - int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch; - ggml_threadpool_t threadpool = n_tokens == 1 ? lctx.threadpool : lctx.threadpool_batch; - - GGML_ASSERT(n_threads > 0); - - ggml_backend_sched_reset(lctx.sched.get()); - ggml_backend_sched_set_eval_callback(lctx.sched.get(), lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data); - - ggml_cgraph * gf = llama_build_graph(lctx, ubatch, false); - - // the output embeddings after the final encoder normalization - struct ggml_tensor * embd = nullptr; - - // there are two cases here - if (llama_model_has_decoder(&lctx.model)) { - // first case is an encoder-decoder T5 model where embeddings are passed to decoder - embd = ggml_graph_node(gf, -1); - GGML_ASSERT(strcmp(embd->name, "result_norm") == 0 && "missing result_output tensor"); - } else { - // second case is an encoder-only T5 model - if (cparams.embeddings) { - // only output embeddings if required - embd = ggml_graph_node(gf, -1); - if (strcmp(embd->name, "result_embd_pooled") != 0) { - embd = ggml_graph_node(gf, -2); - } - GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0 && "missing embeddings tensor"); - } - } - - ggml_backend_sched_alloc_graph(lctx.sched.get(), gf); - - llama_set_inputs(lctx, ubatch); - - const auto compute_status = llama_graph_compute(lctx, gf, n_threads, threadpool); - switch (compute_status) { - case GGML_STATUS_SUCCESS: - break; - case GGML_STATUS_ABORTED: - return 2; - case GGML_STATUS_ALLOC_FAILED: - return -2; - case GGML_STATUS_FAILED: - default: - return -3; - } - - // extract embeddings - if (embd) { - ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(lctx.sched.get(), embd); - GGML_ASSERT(backend_embd != nullptr); - - if (llama_model_has_decoder(&lctx.model)) { - lctx.embd_enc.resize(n_tokens*n_embd); - float * embd_out = lctx.embd_enc.data(); - - ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float)); - GGML_ASSERT(!ubatch.equal_seqs); // TODO: handle equal splits - - // remember the sequence ids used during the encoding - needed for cross attention later - lctx.seq_ids_enc.resize(n_tokens); - for (uint32_t i = 0; i < n_tokens; i++) { - for (int s = 0; s < ubatch.n_seq_id[i]; s++) { - llama_seq_id seq_id = ubatch.seq_id[i][s]; - lctx.seq_ids_enc[i].insert(seq_id); - } - } - } else { - GGML_ASSERT(lctx.embd != nullptr); - - switch (cparams.pooling_type) { - case LLAMA_POOLING_TYPE_NONE: - { - // extract token embeddings - GGML_ASSERT(lctx.embd != nullptr); - float * embd_out = lctx.embd; - - GGML_ASSERT(n_tokens*n_embd <= (int64_t) lctx.embd_size); - ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float)); - } break; - case LLAMA_POOLING_TYPE_MEAN: - case LLAMA_POOLING_TYPE_CLS: - case LLAMA_POOLING_TYPE_LAST: - { - // extract sequence embeddings - auto & embd_seq_out = lctx.embd_seq; - embd_seq_out.clear(); - - GGML_ASSERT(!ubatch.equal_seqs); // TODO: handle equal splits - - for (uint32_t i = 0; i < n_tokens; i++) { - const llama_seq_id seq_id = ubatch.seq_id[i][0]; - if (embd_seq_out.find(seq_id) != embd_seq_out.end()) { - continue; - } - embd_seq_out[seq_id].resize(n_embd); - ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float)); - } - } break; - case LLAMA_POOLING_TYPE_RANK: - { - // TODO: this likely should be the same logic as in llama_decoder_internal, but better to - // wait for an encoder model that requires this pooling type in order to test it - // https://github.com/ggerganov/llama.cpp/pull/9510 - GGML_ABORT("RANK pooling not implemented yet"); - } - case LLAMA_POOLING_TYPE_UNSPECIFIED: - { - GGML_ABORT("unknown pooling type"); - } - } - } - } - - // Reset state for the next token before backend sync, to allow the CPU activities in the reset to - // overlap with device computation. - ggml_backend_sched_reset(lctx.sched.get()); - - return 0; -} - -// find holes from the beginning of the KV cache and fill them by moving data from the end of the cache -static void llama_kv_cache_defrag_impl(struct llama_context & lctx) { - auto & kv_self = lctx.kv_self; - - const auto & hparams = lctx.model.hparams; - - const uint32_t n_layer = hparams.n_layer; - - const uint32_t n_kv = llama_kv_cache_cell_max(kv_self); - const uint32_t n_used = kv_self.used; - - assert(n_used <= n_kv); - - //const int64_t t_start = ggml_time_us(); - - // number of cells moved - uint32_t n_moves = 0; - - // each move requires 6*n_layer tensors (see build_defrag) - // - source view, destination view, copy operation - // - x2 for keys and values - //const uint32_t max_moves = model.max_nodes()/(6*n_layer); - // TODO: tmp fix https://github.com/ggerganov/llama.cpp/issues/6685#issuecomment-2057579516 - const uint32_t max_moves = (lctx.model.max_nodes() - 2*n_layer)/(6*n_layer); - - // determine which KV cells to move where - // - // cell i moves to ids[i] - // - // if ids[i] == i || ids[i] == n_kv, then cell i is not moved - // - std::vector ids(n_kv, n_kv); - - for (uint32_t i0 = 0; i0 < n_used; ++i0) { - const auto & cell0 = kv_self.cells[i0]; - - if (!cell0.is_empty()) { - ids[i0] = i0; - - continue; - } - - // found a hole - fill it with data from the end of the cache - - uint32_t nh = 1; - - // determine the size of the hole - while (i0 + nh < n_used && kv_self.cells[i0 + nh].is_empty()) { - nh++; - } - - uint32_t nf = 0; - uint32_t is = n_kv - 1; - - // starting from the end, find nh non-empty cells - for (; is > i0; --is) { - const auto & cell1 = kv_self.cells[is]; - - if (cell1.is_empty() || ids[is] != n_kv) { - continue; - } - - // non-empty cell which is not yet moved - nf++; - - if (nf == nh) { - break; - } - } - - // this can only happen if `n_used` is not accurate, which would be a bug - GGML_ASSERT(nf == nh && "KV defrag bug: nf != nh"); - - nf = 0; - - uint32_t i1 = is; - - // are we moving a continuous block of memory? - bool cont = false; - - // should we stop searching for the next move? - bool stop = false; - - // go back and move the nf cells to the hole - for (; i1 < n_kv; ++i1) { - auto & cell1 = kv_self.cells[i1]; - - if (cell1.is_empty() || ids[i1] != n_kv) { - if (n_moves == max_moves) { - stop = true; - break; - } - - cont = false; - continue; - } - - // this cell goes to (i0 + nf) - ids[i1] = i0 + nf; - - // move the cell meta data - kv_self.cells[i0 + nf] = cell1; - - // clear the old cell and move the head there - cell1 = llama_kv_cell(); - kv_self.head = n_used; - - if (!cont) { - n_moves++; - cont = true; - } - - nf++; - - if (nf == nh) { - break; - } - } - - if (stop || n_moves == max_moves) { - break; - } - - //LLAMA_LOG_INFO("(tmp log) KV defrag: move [%u, %u) to [%u, %u)\n", is, i1 + 1, i0, i0 + nh); - - i0 += nh - 1; - } - - if (n_moves == 0) { - return; - } - - //LLAMA_LOG_INFO("(tmp log) KV defrag cell moves: %u\n", n_moves); - - //LLAMA_LOG_INFO("expected gf nodes: %u\n", 6*n_moves*n_layer); - -#if 0 - // CPU defrag - // - // TODO: optimizations are possible: - // - multiple threads - // - avoid copying to the host memory when already there - // - // likely not worth the effort, as we have ggml_graph based defrag - // - - const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); - const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); - - const uint32_t kv_size = kv_self.size; - - std::vector buf_k; - std::vector buf_v; - - for (uint32_t il = 0; il < n_layer; ++il) { - const size_t k_size_row = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa); - const size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_size); - - const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type); - const size_t v_size = ggml_row_size (kv_self.v_l[il]->type, n_embd_v_gqa*kv_size); - - buf_k.resize(k_size); - buf_v.resize(v_size); - - ggml_backend_tensor_get(kv_self.k_l[il], buf_k.data(), 0, buf_k.size()); - ggml_backend_tensor_get(kv_self.v_l[il], buf_v.data(), 0, buf_v.size()); - - // batch move [i, i+nm) to [id, id+nm) - // note: cells can move only to a lower index - for (uint32_t i = 0; i < n_kv; ++i) { - const uint32_t id = ids[i]; - - if (i == id || id == n_kv) { - continue; - } - - uint32_t nm = 1; - - while (i + nm < n_kv && ids[i + nm] == id + nm) { - nm++; - } - - // move keys - { - const int64_t os = i*k_size_row; - const int64_t od = id*k_size_row; - - memcpy(buf_k.data() + od, buf_k.data() + os, nm*k_size_row); - } - - // move values (note: they are transposed) - { - const int64_t os = i; - const int64_t od = id; - - for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { - memcpy(buf_v.data() + (od + j*kv_size)*v_size_el, buf_v.data() + (os + j*kv_size)*v_size_el, nm*v_size_el); - } - } - - i += nm - 1; - } - - ggml_backend_tensor_set(kv_self.k_l[il], buf_k.data(), 0, buf_k.size()); - ggml_backend_tensor_set(kv_self.v_l[il], buf_v.data(), 0, buf_v.size()); - } -#else - // ggml_graph defrag - - ggml_backend_sched_reset(lctx.sched.get()); - - ggml_cgraph * gf = llama_build_graph_defrag(lctx, ids); - - llama_graph_compute(lctx, gf, lctx.cparams.n_threads, lctx.threadpool); -#endif - - //const int64_t t_end = ggml_time_us(); - - //LLAMA_LOG_INFO("(tmp log) KV defrag time: %.3f ms\n", (t_end - t_start)/1000.0); -} - -static void llama_kv_cache_update_impl(struct llama_context & lctx) { - bool need_reserve = false; - - if (lctx.kv_self.has_shift) { - if (!llama_kv_cache_can_shift(&lctx)) { - GGML_ABORT("The current context does not support K-shift"); - } - - // apply K-shift if needed - if (lctx.model.hparams.rope_type != LLAMA_ROPE_TYPE_NONE) { - ggml_backend_sched_reset(lctx.sched.get()); - - ggml_cgraph * gf = llama_build_graph_k_shift(lctx); - - ggml_backend_sched_alloc_graph(lctx.sched.get(), gf); - - llama_set_k_shift(lctx); - - llama_graph_compute(lctx, gf, lctx.cparams.n_threads, lctx.threadpool); - - need_reserve = true; - } - - { - auto & kv_self = lctx.kv_self; - - kv_self.has_shift = false; - - for (uint32_t i = 0; i < kv_self.size; ++i) { - kv_self.cells[i].delta = 0; - } - } - } - - // defragment the KV cache if needed - if (lctx.kv_self.do_defrag) { - llama_kv_cache_defrag_impl(lctx); - - need_reserve = true; - - lctx.kv_self.do_defrag = false; - } - - // reserve a worst case graph again - if (need_reserve) { - // TODO: extract to a function - // build worst-case graph - uint32_t n_seqs = 1; // TODO: worst-case number of sequences - uint32_t n_tokens = std::min(lctx.cparams.n_ctx, lctx.cparams.n_ubatch); - llama_token token = lctx.model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph - llama_ubatch ubatch = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; - ggml_cgraph * gf = llama_build_graph(lctx, ubatch, true); - - // initialize scheduler with the worst-case graph - ggml_backend_sched_reset(lctx.sched.get()); - if (!ggml_backend_sched_reserve(lctx.sched.get(), gf)) { - LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__); - } - } -} - -int32_t llama_set_adapter_lora( - struct llama_context * ctx, - struct llama_adapter_lora * adapter, - float scale) { - ctx->lora[adapter] = scale; - return 0; -} - -int32_t llama_rm_adapter_lora( - struct llama_context * ctx, - struct llama_adapter_lora * adapter) { - auto pos = ctx->lora.find(adapter); - if (pos != ctx->lora.end()) { - ctx->lora.erase(pos); - return 0; - } - - return -1; -} - -void llama_clear_adapter_lora(struct llama_context * ctx) { - ctx->lora.clear(); -} - -int32_t llama_apply_adapter_cvec( - struct llama_context * ctx, - const float * data, - size_t len, - int32_t n_embd, - int32_t il_start, - int32_t il_end) { - return ctx->cvec.apply(ctx->model, data, len, n_embd, il_start, il_end); -} - // // interface implementation // -struct llama_context_params llama_context_default_params() { - struct llama_context_params result = { - /*.n_ctx =*/ 512, - /*.n_batch =*/ 2048, - /*.n_ubatch =*/ 512, - /*.n_seq_max =*/ 1, - /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default - /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, - /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, - /*.pooling_type =*/ LLAMA_POOLING_TYPE_UNSPECIFIED, - /*.attention_type =*/ LLAMA_ATTENTION_TYPE_UNSPECIFIED, - /*.rope_freq_base =*/ 0.0f, - /*.rope_freq_scale =*/ 0.0f, - /*.yarn_ext_factor =*/ -1.0f, - /*.yarn_attn_factor =*/ 1.0f, - /*.yarn_beta_fast =*/ 32.0f, - /*.yarn_beta_slow =*/ 1.0f, - /*.yarn_orig_ctx =*/ 0, - /*.defrag_thold =*/ -1.0f, - /*.cb_eval =*/ nullptr, - /*.cb_eval_user_data =*/ nullptr, - /*.type_k =*/ GGML_TYPE_F16, - /*.type_v =*/ GGML_TYPE_F16, - /*.logits_all =*/ false, - /*.embeddings =*/ false, - /*.offload_kqv =*/ true, - /*.flash_attn =*/ false, - /*.no_perf =*/ true, - /*.abort_callback =*/ nullptr, - /*.abort_callback_data =*/ nullptr, - }; - - return result; -} - struct llama_sampler_chain_params llama_sampler_chain_default_params() { struct llama_sampler_chain_params result = { /*.no_perf =*/ true, @@ -9571,6 +82,57 @@ int64_t llama_time_us(void) { return ggml_time_us(); } +// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback +static int llama_model_load(const std::string & fname, std::vector & splits, llama_model & model, llama_model_params & params) { + // loading time will be recalculated after the first eval, so + // we take page faults deferred by mmap() into consideration + model.t_load_us = 0; + time_meas tm(model.t_load_us); + + model.t_start_us = tm.t_start_us; + + try { + llama_model_loader ml(fname, splits, params.use_mmap, params.check_tensors, params.kv_overrides); + + ml.print_info(); + + model.hparams.vocab_only = params.vocab_only; + + try { + model.load_arch(ml); + } catch(const std::exception & e) { + throw std::runtime_error("error loading model architecture: " + std::string(e.what())); + } + try { + model.load_hparams(ml); + } catch(const std::exception & e) { + throw std::runtime_error("error loading model hyperparameters: " + std::string(e.what())); + } + try { + model.load_vocab(ml); + } catch(const std::exception & e) { + throw std::runtime_error("error loading model vocabulary: " + std::string(e.what())); + } + + model.load_stats(ml); + model.print_info(); + + if (params.vocab_only) { + LLAMA_LOG_INFO("%s: vocab only - skipping tensors\n", __func__); + return 0; + } + + if (!model.load_tensors(ml)) { + return -2; + } + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what()); + return -1; + } + + return 0; +} + static struct llama_model * llama_model_load_from_file_impl( const std::string & path_model, std::vector & splits, @@ -9691,460 +253,6 @@ struct llama_model * llama_model_load_from_splits( return llama_model_load_from_file_impl(splits.front(), splits, params); } -struct llama_context * llama_init_from_model( - struct llama_model * model, - struct llama_context_params params) { - - if (!model) { - LLAMA_LOG_ERROR("%s: model cannot be NULL\n", __func__); - return nullptr; - } - - if (params.n_batch == 0 && params.n_ubatch == 0) { - LLAMA_LOG_ERROR("%s: n_batch and n_ubatch cannot both be zero\n", __func__); - return nullptr; - } - - if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) { - LLAMA_LOG_ERROR("%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n", __func__); - return nullptr; - } - - if (params.flash_attn && model->arch == LLM_ARCH_GROK) { - LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__); - params.flash_attn = false; - } - - if (params.flash_attn && model->hparams.n_embd_head_k != model->hparams.n_embd_head_v) { - LLAMA_LOG_WARN("%s: flash_attn requires n_embd_head_k == n_embd_head_v - forcing off\n", __func__); - params.flash_attn = false; - } - - if (ggml_is_quantized(params.type_v) && !params.flash_attn) { - LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__); - return nullptr; - } - - llama_context * ctx = new llama_context(*model); - - const auto & hparams = model->hparams; - auto & cparams = ctx->cparams; - - cparams.n_seq_max = std::max(1u, params.n_seq_max); - cparams.n_threads = params.n_threads; - cparams.n_threads_batch = params.n_threads_batch; - cparams.yarn_ext_factor = params.yarn_ext_factor; - cparams.yarn_attn_factor = params.yarn_attn_factor; - cparams.yarn_beta_fast = params.yarn_beta_fast; - cparams.yarn_beta_slow = params.yarn_beta_slow; - cparams.defrag_thold = params.defrag_thold; - cparams.embeddings = params.embeddings; - cparams.offload_kqv = params.offload_kqv; - cparams.flash_attn = params.flash_attn; - cparams.no_perf = params.no_perf; - cparams.pooling_type = params.pooling_type; - - cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx; - cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base; - cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale; - - // this is necessary due to kv_self.n being padded later during inference - cparams.n_ctx = GGML_PAD(cparams.n_ctx, llama_kv_cache_get_padding(cparams)); - - // with causal attention, the batch size is limited by the context size - cparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch; - - // the batch has to be at least GGML_KQ_MASK_PAD because we will be padding the KQ_mask - // this is required by GPU kernels in order to avoid out-of-bounds accesses (e.g. ggml_flash_attn_ext) - // ref: https://github.com/ggerganov/llama.cpp/pull/5021 - if (cparams.n_batch < GGML_KQ_MASK_PAD) { - LLAMA_LOG_WARN("%s: n_batch is less than GGML_KQ_MASK_PAD - increasing to %d\n", __func__, GGML_KQ_MASK_PAD); - cparams.n_batch = GGML_KQ_MASK_PAD; - } - - cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch); - - cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx : - hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn : - hparams.n_ctx_train; - - cparams.cb_eval = params.cb_eval; - cparams.cb_eval_user_data = params.cb_eval_user_data; - - auto rope_scaling_type = params.rope_scaling_type; - if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) { - rope_scaling_type = hparams.rope_scaling_type_train; - } - - if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) { - cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none - } - - if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set' - cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f; - } - - cparams.yarn_attn_factor *= hparams.rope_attn_factor; - - if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { - if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { - cparams.pooling_type = LLAMA_POOLING_TYPE_NONE; - } else { - cparams.pooling_type = hparams.pooling_type; - } - } - - if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) { - cparams.causal_attn = hparams.causal_attn; - } else { - cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL; - } - - const uint32_t n_ctx_per_seq = cparams.n_ctx / cparams.n_seq_max; - - LLAMA_LOG_INFO("%s: n_seq_max = %u\n", __func__, cparams.n_seq_max); - LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx); - LLAMA_LOG_INFO("%s: n_ctx_per_seq = %u\n", __func__, n_ctx_per_seq); - LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch); - LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch); - LLAMA_LOG_INFO("%s: flash_attn = %d\n", __func__, cparams.flash_attn); - LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base); - LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale); - - if (n_ctx_per_seq < hparams.n_ctx_train) { - LLAMA_LOG_WARN("%s: n_ctx_per_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n", - __func__, n_ctx_per_seq, hparams.n_ctx_train); - } - - if (n_ctx_per_seq > hparams.n_ctx_train) { - LLAMA_LOG_WARN("%s: n_ctx_pre_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n", - __func__, n_ctx_per_seq, hparams.n_ctx_train); - } - - ctx->logits_all = params.logits_all; - - // build worst-case graph for encoder if a model contains encoder - ctx->is_encoding = llama_model_has_encoder(model); - - uint32_t kv_size = cparams.n_ctx; - ggml_type type_k = params.type_k; - ggml_type type_v = params.type_v; - - // Mamba only needs a constant number of KV cache cells per sequence - if (llama_model_is_recurrent(model)) { - // Mamba needs at least as many KV cells as there are sequences kept at any time - kv_size = std::max((uint32_t) 1, params.n_seq_max); - // it's probably best to keep as much precision as possible for the states - type_k = GGML_TYPE_F32; // required by ggml_ssm_conv for Mamba's conv_states - type_v = GGML_TYPE_F32; // required by ggml_ssm_scan for Mamba's ssm_states - } - - GGML_ASSERT(hparams.n_embd_head_k % ggml_blck_size(type_k) == 0); - GGML_ASSERT(hparams.n_embd_head_v % ggml_blck_size(type_v) == 0); - - if (!hparams.vocab_only) { - // GPU backends - for (auto * dev : model->devices) { - ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); - if (backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize %s backend\n", __func__, ggml_backend_dev_name(dev)); - llama_free(ctx); - return nullptr; - } - ctx->backends.emplace_back(backend); - } - - // add ACCEL backends (such as BLAS) - for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { - ggml_backend_dev_t dev = ggml_backend_dev_get(i); - if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) { - ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); - if (backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize %s backend\n", __func__, ggml_backend_dev_name(dev)); - llama_free(ctx); - return nullptr; - } - ctx->backends.emplace_back(backend); - } - } - - // add CPU backend - ctx->backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); - if (ctx->backend_cpu == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize CPU backend\n", __func__); - llama_free(ctx); - return nullptr; - } - ctx->backends.emplace_back(ctx->backend_cpu); - - // create a list of the set_n_threads functions in the backends - for (auto & backend : ctx->backends) { - ggml_backend_dev_t dev = ggml_backend_get_device(backend.get()); - ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; - if (reg) { - auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); - if (ggml_backend_set_n_threads_fn) { - ctx->set_n_threads_fns.emplace_back(backend.get(), ggml_backend_set_n_threads_fn); - } - } - } - - llama_set_abort_callback(ctx, params.abort_callback, params.abort_callback_data); - - if (!llama_kv_cache_init(ctx->kv_self, ctx->model, ctx->cparams, type_k, type_v, kv_size, cparams.offload_kqv)) { - LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__); - llama_free(ctx); - return nullptr; - } - - { - size_t memory_size_k = 0; - size_t memory_size_v = 0; - - for (auto & k : ctx->kv_self.k_l) { - memory_size_k += ggml_nbytes(k); - } - - for (auto & v : ctx->kv_self.v_l) { - memory_size_v += ggml_nbytes(v); - } - - LLAMA_LOG_INFO("%s: KV self size = %7.2f MiB, K (%s): %7.2f MiB, V (%s): %7.2f MiB\n", __func__, - (float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f), - ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f), - ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f)); - } - - // graph outputs buffer - { - // resized during inference when a batch uses more outputs - if (llama_output_reserve(*ctx, params.n_seq_max) < params.n_seq_max) { - LLAMA_LOG_ERROR("%s: failed to reserve initial output buffer\n", __func__); - llama_free(ctx); - return nullptr; - } - - LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__, - ggml_backend_buffer_name(ctx->buf_output.get()), - ggml_backend_buffer_get_size(ctx->buf_output.get()) / 1024.0 / 1024.0); - } - - // scheduler and compute buffers - { - // buffer types used for the compute buffer of each backend - std::vector backend_buft; - std::vector backend_ptrs; - for (auto & backend : ctx->backends) { - auto * buft = ggml_backend_get_default_buffer_type(backend.get()); - auto backend_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get())); - if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model->devices.empty()) { - // use the host buffer of the first device CPU for faster transfer of the intermediate state - auto * dev = model->devices[0]; - auto * host_buft = ggml_backend_dev_host_buffer_type(dev); - if (host_buft) { - buft = host_buft; - } - } - backend_buft.push_back(buft); - backend_ptrs.push_back(backend.get()); - } - - const size_t max_nodes = model->max_nodes(); - - // buffer used to store the computation graph and the tensor meta data - ctx->buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false)); - - // TODO: move these checks to ggml_backend_sched - // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary - bool pipeline_parallel = - model->n_devices() > 1 && - model->params.n_gpu_layers > (int)model->hparams.n_layer && - model->params.split_mode == LLAMA_SPLIT_MODE_LAYER && - params.offload_kqv; - - // pipeline parallelism requires support for async compute and events in all devices - if (pipeline_parallel) { - for (auto & backend : ctx->backends) { - auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get())); - if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU) { - // ignore CPU backend - continue; - } - auto * dev = ggml_backend_get_device(backend.get()); - ggml_backend_dev_props props; - ggml_backend_dev_get_props(dev, &props); - if (!props.caps.async || !props.caps.events) { - // device does not support async compute or events - pipeline_parallel = false; - break; - } - } - } - - ctx->sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, pipeline_parallel)); - - if (pipeline_parallel) { - LLAMA_LOG_INFO("%s: pipeline parallelism enabled (n_copies=%d)\n", __func__, ggml_backend_sched_get_n_copies(ctx->sched.get())); - } - - // initialize scheduler with the worst-case graph - uint32_t n_seqs = 1; // TODO: worst-case number of sequences - uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); - llama_token token = ctx->model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph - - llama_ubatch ubatch_pp = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; - ggml_cgraph * gf_pp = llama_build_graph(*ctx, ubatch_pp, true); - - // reserve pp graph first so that buffers are only allocated once - ggml_backend_sched_reserve(ctx->sched.get(), gf_pp); - int n_splits_pp = ggml_backend_sched_get_n_splits(ctx->sched.get()); - int n_nodes_pp = ggml_graph_n_nodes(gf_pp); - - // reserve with tg graph to get the number of splits and nodes - llama_ubatch ubatch_tg = { true, 1, 1, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; - ggml_cgraph * gf_tg = llama_build_graph(*ctx, ubatch_tg, true); - ggml_backend_sched_reserve(ctx->sched.get(), gf_tg); - int n_splits_tg = ggml_backend_sched_get_n_splits(ctx->sched.get()); - int n_nodes_tg = ggml_graph_n_nodes(gf_tg); - - // reserve again with pp graph to avoid ggml-alloc reallocations during inference - gf_pp = llama_build_graph(*ctx, ubatch_pp, true); - if (!ggml_backend_sched_reserve(ctx->sched.get(), gf_pp)) { - LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__); - llama_free(ctx); - return nullptr; - } - - for (size_t i = 0; i < backend_ptrs.size(); ++i) { - ggml_backend_t backend = backend_ptrs[i]; - ggml_backend_buffer_type_t buft = backend_buft[i]; - size_t size = ggml_backend_sched_get_buffer_size(ctx->sched.get(), backend); - if (size > 1) { - LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__, - ggml_backend_buft_name(buft), - size / 1024.0 / 1024.0); - } - } - - if (n_nodes_pp == n_nodes_tg) { - LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, n_nodes_pp); - } else { - LLAMA_LOG_INFO("%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n", __func__, n_nodes_pp, n_tokens, n_nodes_tg); - } - if (n_splits_pp == n_splits_tg) { - LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits_pp); - } else { - LLAMA_LOG_INFO("%s: graph splits = %d (with bs=%d), %d (with bs=1)\n", __func__, n_splits_pp, n_tokens, n_splits_tg); - } - } - } - - return ctx; -} - -struct llama_context * llama_new_context_with_model( - struct llama_model * model, - struct llama_context_params params) { - return llama_init_from_model(model, params); -} - -// -// kv cache -// - -// TODO: tmp bridges below until `struct llama_kv_cache` is exposed through the public API - -struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max) { - return llama_kv_cache_view_init(ctx->kv_self, n_seq_max); -} - -void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kv_cache_view * view) { - llama_kv_cache_view_update(view, ctx->kv_self); -} - -int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx) { - return llama_get_kv_cache_token_count(ctx->kv_self); -} - -int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx) { - return llama_get_kv_cache_used_cells(ctx->kv_self); -} - -void llama_kv_cache_clear(struct llama_context * ctx) { - llama_kv_cache_clear(ctx->kv_self); -} - -bool llama_kv_cache_seq_rm(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) { - return llama_kv_cache_seq_rm(ctx->kv_self, seq_id, p0, p1); -} - -void llama_kv_cache_seq_cp(struct llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { - if (seq_id_src == seq_id_dst) { - return; - } - llama_kv_cache_seq_cp(ctx->kv_self, seq_id_src, seq_id_dst, p0, p1); -} - -void llama_kv_cache_seq_keep(struct llama_context * ctx, llama_seq_id seq_id) { - llama_kv_cache_seq_keep(ctx->kv_self, seq_id); -} - -void llama_kv_cache_seq_add(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) { - if (delta == 0) { - return; - } - - llama_kv_cache_seq_add(ctx->kv_self, seq_id, p0, p1, delta); -} - -void llama_kv_cache_seq_div(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { - if (d == 1) { - return; - } - - llama_kv_cache_seq_div(ctx->kv_self, seq_id, p0, p1, d); -} - -llama_pos llama_kv_cache_seq_pos_max(struct llama_context * ctx, llama_seq_id seq_id) { - return llama_kv_cache_seq_pos_max(ctx->kv_self, seq_id); -} - -void llama_kv_cache_defrag(struct llama_context * ctx) { - llama_kv_cache_defrag(ctx->kv_self); -} - -void llama_kv_cache_update(struct llama_context * ctx) { - llama_kv_cache_update_impl(*ctx); -} - -bool llama_kv_cache_can_shift(struct llama_context * ctx) { - return llama_kv_cache_can_shift(ctx->kv_self); -} - -/// - -int32_t llama_encode( - struct llama_context * ctx, - struct llama_batch batch) { - const int ret = llama_encode_impl(*ctx, batch); - if (ret != 0) { - LLAMA_LOG_ERROR("%s: failed to encode, ret = %d\n", __func__, ret); - } - - return ret; -} - -int32_t llama_decode( - struct llama_context * ctx, - struct llama_batch batch) { - const int ret = llama_decode_impl(*ctx, batch); - if (ret != 0) { - LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret); - } - - return ret; -} - // // chat templates // @@ -10212,7 +320,6 @@ const char * llama_print_system_info(void) { static std::string s; s.clear(); // Clear the string, since it's static, otherwise it will accumulate data from previous calls. - for (size_t i = 0; i < ggml_backend_reg_count(); i++) { auto * reg = ggml_backend_reg_get(i); auto * get_features_fn = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features"); @@ -10231,43 +338,3 @@ const char * llama_print_system_info(void) { return s.c_str(); } - -// -// perf -// - -struct llama_perf_context_data llama_perf_context(const struct llama_context * ctx) { - struct llama_perf_context_data data = {}; - - if (ctx == nullptr) { - return data; - } - - data.t_start_ms = 1e-3 * ctx->t_start_us; - data.t_load_ms = 1e-3 * ctx->t_load_us; - data.t_p_eval_ms = 1e-3 * ctx->t_p_eval_us; - data.t_eval_ms = 1e-3 * ctx->t_eval_us; - data.n_p_eval = std::max(1, ctx->n_p_eval); - data.n_eval = std::max(1, ctx->n_eval); - - return data; -} - -void llama_perf_context_print(const struct llama_context * ctx) { - const auto data = llama_perf_context(ctx); - - const double t_end_ms = 1e-3 * ggml_time_us(); - - LLAMA_LOG_INFO("%s: load time = %10.2f ms\n", __func__, data.t_load_ms); - LLAMA_LOG_INFO("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", - __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval); - LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", - __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval); - LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval)); -} - -void llama_perf_context_reset(struct llama_context * ctx) { - ctx->t_start_us = ggml_time_us(); - ctx->t_eval_us = ctx->n_eval = 0; - ctx->t_p_eval_us = ctx->n_p_eval = 0; -} From be7c3034108473beda214fd1d7c98fd6a7a3bdf5 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Thu, 13 Mar 2025 12:34:54 +0100 Subject: [PATCH 129/188] arg : no n_predict = -2 for examples except for main and infill (#12364) --- common/arg.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/arg.cpp b/common/arg.cpp index 8531f0871d..fe6a1eece7 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -764,7 +764,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_env("LLAMA_ARG_CTX_SIZE")); add_opt(common_arg( {"-n", "--predict", "--n-predict"}, "N", - string_format("number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)", params.n_predict), + string_format( + ex == LLAMA_EXAMPLE_MAIN || ex == LLAMA_EXAMPLE_INFILL + ? "number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)" + : "number of tokens to predict (default: %d, -1 = infinity)", + params.n_predict), [](common_params & params, int value) { params.n_predict = value; } From 84d547554123a62e9ac77107cb20e4f6cc503af4 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 13 Mar 2025 19:08:07 +0200 Subject: [PATCH 130/188] llama : fix Gemma3 SWA KV cache shift (#12373) * llama : fix Gemma3 SWA KV cache shift ggml-ci * hparams : add comment [no ci] --- src/llama-context.cpp | 17 ++++++++++++++--- src/llama-context.h | 2 ++ src/llama-graph.cpp | 29 +---------------------------- src/llama-hparams.cpp | 8 ++++++++ src/llama-hparams.h | 3 +++ src/llama-model.cpp | 21 +++++++++------------ 6 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 0a43a3af8e..89fb33cbcd 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -442,10 +442,10 @@ ggml_tensor * llama_context::build_rope_shift( ggml_tensor * cur, ggml_tensor * shift, ggml_tensor * factors, + float freq_base, + float freq_scale, ggml_backend_buffer * bbuf) const { const auto & n_ctx_orig = cparams.n_ctx_orig_yarn; - const auto & freq_base = cparams.rope_freq_base; - const auto & freq_scale = cparams.rope_freq_scale; const auto & yarn_ext_factor = cparams.yarn_ext_factor; const auto & yarn_attn_factor = cparams.yarn_attn_factor; @@ -537,6 +537,17 @@ llm_graph_result_ptr llama_context::build_kv_self_shift( const int64_t n_head_kv = hparams.n_head_kv(il); const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); + float freq_base_l = cparams.rope_freq_base; + float freq_scale_l = cparams.rope_freq_scale; + + // TODO: improve + if (model.arch == LLM_ARCH_GEMMA3) { + const bool is_sliding = hparams.is_sliding(il); + + freq_base_l = is_sliding ? 10000.0f : cparams.rope_freq_base; + freq_scale_l = is_sliding ? 1.0f : cparams.rope_freq_scale; + } + ggml_tensor * rope_factors = kv_self->cbs.get_rope_factors(n_ctx_per_seq(), il); ggml_tensor * k = @@ -546,7 +557,7 @@ llm_graph_result_ptr llama_context::build_kv_self_shift( ggml_row_size(kv_self->k_l[il]->type, n_embd_k_gqa), 0); - ggml_tensor * cur = build_rope_shift(ctx0, k, inp->k_shift, rope_factors, kv_self->k_l[il]->buffer); + ggml_tensor * cur = build_rope_shift(ctx0, k, inp->k_shift, rope_factors, freq_base_l, freq_scale_l, kv_self->k_l[il]->buffer); ggml_build_forward_expand(gf, cur); } diff --git a/src/llama-context.h b/src/llama-context.h index 71d702e8ba..88df8950e4 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -168,6 +168,8 @@ private: ggml_tensor * cur, ggml_tensor * shift, ggml_tensor * factors, + float freq_base, + float freq_scale, ggml_backend_buffer * bbuf) const; llm_graph_result_ptr build_kv_self_shift( diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 1e3f2efc89..4a53e83929 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1403,34 +1403,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, ggml_cpy(ctx0, v_cur, v_cache_view)); } - // TODO: improve - bool is_sliding = false; - - switch (arch) { - case LLM_ARCH_COHERE2: - { - const int32_t sliding_window_pattern = 4; - is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); - } break; - case LLM_ARCH_GEMMA2: - { - const int32_t sliding_window_pattern = 2; - is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); - } break; - case LLM_ARCH_GEMMA3: - { - const int32_t sliding_window_pattern = 6; - is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); - } break; - case LLM_ARCH_PHI3: - { - is_sliding = hparams.n_swa > 0; - } break; - default: - { - is_sliding = false; - } - }; + const bool is_sliding = hparams.is_sliding(il); const auto & kq_mask = is_sliding ? inp->get_kq_mask_swa() : inp->get_kq_mask(); diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index ea87b2953d..58e98bf231 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -69,3 +69,11 @@ uint32_t llama_hparams::n_embd_v_s() const { // corresponds to Mamba's ssm_states size return ssm_d_state * ssm_d_inner; } + +bool llama_hparams::is_sliding(uint32_t il) const { + if (il < n_layer) { + return n_swa > 0 && n_swa_pattern > 0 && il % n_swa_pattern < (n_swa_pattern - 1); + } + + GGML_ABORT("fatal error"); +} diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 1fe4541037..e3091c8127 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -36,6 +36,7 @@ struct llama_hparams { uint32_t n_layer; uint32_t n_rot; uint32_t n_swa = 0; // sliding window attention (SWA) + uint32_t n_swa_pattern = 1; // by default, all layers use non-sliding-window attention uint32_t n_embd_head_k; // dimension of keys (d_k). d_q is assumed to be the same, but there are n_head q heads, and only n_head_kv k-v heads uint32_t n_embd_head_v; // dimension of values (d_v) aka n_embd_head uint32_t n_expert = 0; @@ -133,6 +134,8 @@ struct llama_hparams { // dimension of the recurrent state embeddings uint32_t n_embd_v_s() const; + + bool is_sliding(uint32_t il) const; }; static_assert(std::is_trivially_copyable::value, "llama_hparams must be trivially copyable"); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 522219c012..5647d2ad62 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -858,11 +858,13 @@ void llama_model::load_hparams(llama_model_loader & ml) { case LLM_ARCH_GEMMA2: { hparams.n_swa = 4096; // default value of gemma 2 + hparams.n_swa_pattern = 2; + hparams.attn_soft_cap = true; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping, false); ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false); - hparams.attn_soft_cap = true; switch (hparams.n_layer) { case 26: type = LLM_TYPE_2B; break; @@ -873,6 +875,8 @@ void llama_model::load_hparams(llama_model_loader & ml) { } break; case LLM_ARCH_GEMMA3: { + hparams.n_swa_pattern = 6; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -952,6 +956,8 @@ void llama_model::load_hparams(llama_model_loader & ml) { } break; case LLM_ARCH_COHERE2: { + hparams.n_swa_pattern = 4; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); @@ -7374,12 +7380,8 @@ struct llm_build_gemma3 : public llm_graph_context { // TODO: is causal == true correct? might need some changes auto * inp_attn = build_attn_inp_kv_unified(true, true); - // "5-to-1 interleaved attention" - // 5 layers of local attention followed by 1 layer of global attention - static const int sliding_window_pattern = 6; - for (int il = 0; il < n_layer; ++il) { - const bool is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); + const bool is_sliding = hparams.is_sliding(il); const float freq_base_l = is_sliding ? 10000.0f : freq_base; const float freq_scale_l = is_sliding ? 1.0f : freq_scale; @@ -7970,13 +7972,8 @@ struct llm_build_cohere2 : public llm_graph_context { auto * inp_attn = build_attn_inp_kv_unified(true, true); - // sliding window switch pattern - const int32_t sliding_window_pattern = 4; - for (int il = 0; il < n_layer; ++il) { - // three layers sliding window attention (window size 4096) and ROPE - // fourth layer uses global attention without positional embeddings - const bool is_sliding = il % sliding_window_pattern < (sliding_window_pattern - 1); + const bool is_sliding = hparams.is_sliding(il); // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il); From 081bee8c643b1f6302e9edfe789ce2d5f0be6c77 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 14 Mar 2025 09:03:24 +0200 Subject: [PATCH 131/188] hparams : add SWA rope parameters (#12374) ggml-ci --- src/llama-context.cpp | 14 +++++--------- src/llama-graph.cpp | 4 ++-- src/llama-hparams.cpp | 2 +- src/llama-hparams.h | 4 +++- src/llama-model.cpp | 22 +++++++++++++++------- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 89fb33cbcd..4df6b18ec1 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -537,16 +537,12 @@ llm_graph_result_ptr llama_context::build_kv_self_shift( const int64_t n_head_kv = hparams.n_head_kv(il); const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); - float freq_base_l = cparams.rope_freq_base; - float freq_scale_l = cparams.rope_freq_scale; + const bool is_swa = hparams.is_swa(il); - // TODO: improve - if (model.arch == LLM_ARCH_GEMMA3) { - const bool is_sliding = hparams.is_sliding(il); - - freq_base_l = is_sliding ? 10000.0f : cparams.rope_freq_base; - freq_scale_l = is_sliding ? 1.0f : cparams.rope_freq_scale; - } + // note: the swa rope params could become part of the cparams in the future + // if we decide to make them configurable, like the non-sliding ones + const float freq_base_l = is_swa ? hparams.rope_freq_base_train_swa : cparams.rope_freq_base; + const float freq_scale_l = is_swa ? hparams.rope_freq_scale_train_swa : cparams.rope_freq_scale; ggml_tensor * rope_factors = kv_self->cbs.get_rope_factors(n_ctx_per_seq(), il); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 4a53e83929..1041ba29fb 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1403,9 +1403,9 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, ggml_cpy(ctx0, v_cur, v_cache_view)); } - const bool is_sliding = hparams.is_sliding(il); + const bool is_swa = hparams.is_swa(il); - const auto & kq_mask = is_sliding ? inp->get_kq_mask_swa() : inp->get_kq_mask(); + const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask(); const auto n_kv = kv_self->n; diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 58e98bf231..90dfe7a7fc 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -70,7 +70,7 @@ uint32_t llama_hparams::n_embd_v_s() const { return ssm_d_state * ssm_d_inner; } -bool llama_hparams::is_sliding(uint32_t il) const { +bool llama_hparams::is_swa(uint32_t il) const { if (il < n_layer) { return n_swa > 0 && n_swa_pattern > 0 && il % n_swa_pattern < (n_swa_pattern - 1); } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index e3091c8127..dbb7abd317 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -79,7 +79,9 @@ struct llama_hparams { float rope_attn_factor = 1.0f; float rope_freq_base_train; + float rope_freq_base_train_swa; float rope_freq_scale_train; + float rope_freq_scale_train_swa; uint32_t n_ctx_orig_yarn; float rope_yarn_log_mul; @@ -135,7 +137,7 @@ struct llama_hparams { // dimension of the recurrent state embeddings uint32_t n_embd_v_s() const; - bool is_sliding(uint32_t il) const; + bool is_swa(uint32_t il) const; }; static_assert(std::is_trivially_copyable::value, "llama_hparams must be trivially copyable"); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 5647d2ad62..cce943df08 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -475,6 +475,10 @@ void llama_model::load_hparams(llama_model_loader & ml) { } hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale; + // by default assume that the sliding-window layers use the same scaling type as the non-sliding-window layers + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false); // non-transformer models do not have attention heads @@ -877,6 +881,9 @@ void llama_model::load_hparams(llama_model_loader & ml) { { hparams.n_swa_pattern = 6; + hparams.rope_freq_base_train_swa = 10000.0f; + hparams.rope_freq_scale_train_swa = 1.0f; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -1346,13 +1353,14 @@ bool llama_model::load_tensors(llama_model_loader & ml) { const int i_gpu_start = std::max((int) hparams.n_layer - n_gpu_layers, (int) 0); const int act_gpu_layers = devices.empty() ? 0 : std::min(n_gpu_layers, (int)n_layer + 1); auto get_layer_buft_list = [&](int il) -> llama_model::impl::layer_dev { + const bool is_swa = il < (int) hparams.n_layer && hparams.is_swa(il); if (il < i_gpu_start || (il - i_gpu_start) >= act_gpu_layers) { - LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s\n", il, ggml_backend_dev_name(cpu_dev)); + LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s, is_swa = %d\n", il, ggml_backend_dev_name(cpu_dev), is_swa); return {cpu_dev, &pimpl->cpu_buft_list}; } const int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + n_devices(), float(il - i_gpu_start)/act_gpu_layers) - splits.begin(); auto * dev = devices.at(layer_gpu); - LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s\n", il, ggml_backend_dev_name(dev)); + LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s, is_swa = %d\n", il, ggml_backend_dev_name(dev), is_swa); return {dev, &pimpl->gpu_buft_list.at(dev)}; }; @@ -7381,10 +7389,10 @@ struct llm_build_gemma3 : public llm_graph_context { auto * inp_attn = build_attn_inp_kv_unified(true, true); for (int il = 0; il < n_layer; ++il) { - const bool is_sliding = hparams.is_sliding(il); + const bool is_swa = hparams.is_swa(il); - const float freq_base_l = is_sliding ? 10000.0f : freq_base; - const float freq_scale_l = is_sliding ? 1.0f : freq_scale; + const float freq_base_l = is_swa ? hparams.rope_freq_base_train_swa : cparams.rope_freq_base; + const float freq_scale_l = is_swa ? hparams.rope_freq_scale_train_swa : cparams.rope_freq_scale; // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); @@ -7973,7 +7981,7 @@ struct llm_build_cohere2 : public llm_graph_context { auto * inp_attn = build_attn_inp_kv_unified(true, true); for (int il = 0; il < n_layer; ++il) { - const bool is_sliding = hparams.is_sliding(il); + const bool is_swa = hparams.is_swa(il); // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il); @@ -8007,7 +8015,7 @@ struct llm_build_cohere2 : public llm_graph_context { cb(Vcur, "Vcur", il); } - if (is_sliding) { + if (is_swa) { Qcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); From c522ce4143a2b5c277f1e5f65cd570dbd0626466 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 14 Mar 2025 10:47:44 +0200 Subject: [PATCH 132/188] graph : simplify attn input build for unified KV cache (#12381) ggml-ci --- src/llama-graph.cpp | 14 ++----- src/llama-graph.h | 4 +- src/llama-model.cpp | 93 +++++++++++++++++++++++---------------------- 3 files changed, 53 insertions(+), 58 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 1041ba29fb..e4af507780 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1311,29 +1311,23 @@ ggml_tensor * llm_graph_context::build_attn( return cur; } -llm_graph_input_attn_kv_unified * llm_graph_context::build_attn_inp_kv_unified( - bool causal, - bool swa) const { +llm_graph_input_attn_kv_unified * llm_graph_context::build_attn_inp_kv_unified() const { const llama_kv_cache_unified * kv_self = static_cast(memory); auto inp = std::make_unique(hparams, cparams, kv_self); const auto n_kv = kv_self->n; - inp->self_kq_mask = causal - ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)) - : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); + inp->self_kq_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); //cb(inp->self_kq_mask, "KQ_mask", -1); ggml_set_input(inp->self_kq_mask); inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask; - if (swa) { + if (hparams.n_swa_pattern > 1) { GGML_ASSERT(hparams.n_swa > 0); - inp->self_kq_mask_swa = causal - ? ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)) - : ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); + inp->self_kq_mask_swa = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD)); //cb(inp->self_kq_mask_swa, "KQ_mask_swa", -1); ggml_set_input(inp->self_kq_mask_swa); diff --git a/src/llama-graph.h b/src/llama-graph.h index b7a66d1898..c4328e6f9e 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -509,9 +509,7 @@ struct llm_graph_context { float kq_scale, int il) const; - llm_graph_input_attn_kv_unified * build_attn_inp_kv_unified( - bool causal, - bool swa) const; + llm_graph_input_attn_kv_unified * build_attn_inp_kv_unified() const; ggml_tensor * build_attn( llm_graph_input_attn_kv_unified * inp, diff --git a/src/llama-model.cpp b/src/llama-model.cpp index cce943df08..750a702ff7 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -784,9 +784,11 @@ void llama_model::load_hparams(llama_model_loader & ml) { hparams.n_swa = 2047; } else if (hparams.n_layer == 32 && hparams.n_head_kv(0) == 32 && hparams.n_ctx_train == 131072) { // default value for Phi-3-mini-128k-instruct + // note: this seems incorrect because the window is bigger than the train context? hparams.n_swa = 262144; } else if (hparams.n_layer == 40 && hparams.n_ctx_train == 131072) { // default value for Phi-3-medium-128k-instruct + // note: this seems incorrect because the window is equal to the train context? hparams.n_swa = 131072; } bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); @@ -3710,6 +3712,7 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: n_head_kv = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_head_kv(il); }, hparams.n_layer).c_str()); LLAMA_LOG_INFO("%s: n_rot = %u\n", __func__, hparams.n_rot); LLAMA_LOG_INFO("%s: n_swa = %u\n", __func__, hparams.n_swa); + LLAMA_LOG_INFO("%s: n_swa_pattern = %u\n", __func__, hparams.n_swa_pattern); LLAMA_LOG_INFO("%s: n_embd_head_k = %u\n", __func__, hparams.n_embd_head_k); LLAMA_LOG_INFO("%s: n_embd_head_v = %u\n", __func__, hparams.n_embd_head_v); LLAMA_LOG_INFO("%s: n_gqa = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_gqa(il); }, hparams.n_layer).c_str()); @@ -3871,7 +3874,7 @@ struct llm_build_llama : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; for (int il = 0; il < n_layer; ++il) { @@ -4034,7 +4037,7 @@ struct llm_build_deci : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; for (int il = 0; il < n_layer; ++il) { @@ -4192,7 +4195,7 @@ struct llm_build_baichuan : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = model.type == LLM_TYPE_7B ? build_inp_pos() : nullptr; - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -4310,7 +4313,7 @@ struct llm_build_xverse : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -4418,7 +4421,7 @@ struct llm_build_falcon : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * attn_norm; @@ -4543,7 +4546,7 @@ struct llm_build_grok : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -4697,7 +4700,7 @@ struct llm_build_dbrx : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -4821,7 +4824,7 @@ struct llm_build_starcoder : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); cb(pos, "pos_embd", -1); @@ -4924,7 +4927,7 @@ struct llm_build_refact : public llm_graph_context { inpL = build_inp_embd(model.tok_embd); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -5187,7 +5190,7 @@ struct llm_build_bloom : public llm_graph_context { inpL = build_inp_embd(model.tok_embd); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); inpL = build_norm(inpL, model.tok_norm, @@ -5292,7 +5295,7 @@ struct llm_build_mpt : public llm_graph_context { inpL = build_inp_embd(model.tok_embd); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); if (model.pos_embd) { // inp_pos - contains the positions @@ -5436,7 +5439,7 @@ struct llm_build_stablelm : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { // norm @@ -5587,7 +5590,7 @@ struct llm_build_qwen : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -5703,7 +5706,7 @@ struct llm_build_qwen2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -5818,7 +5821,7 @@ struct llm_build_qwen2vl : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); int sections[4]; std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); @@ -5938,7 +5941,7 @@ struct llm_build_qwen2moe : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -6087,7 +6090,7 @@ struct llm_build_phi2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { attn_norm_output = build_norm(inpL, @@ -6211,7 +6214,7 @@ struct llm_build_phi3 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, true); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { auto * residual = inpL; @@ -6357,7 +6360,7 @@ struct llm_build_plamo : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { @@ -6465,7 +6468,7 @@ struct llm_build_gpt2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); cb(pos, "pos_embd", -1); @@ -6573,7 +6576,7 @@ struct llm_build_codeshell : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { cur = build_norm(inpL, @@ -6686,7 +6689,7 @@ struct llm_build_orion : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -6807,7 +6810,7 @@ struct llm_build_internlm2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -6937,7 +6940,7 @@ struct llm_build_minicpm3 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -7141,7 +7144,7 @@ struct llm_build_gemma : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { // norm @@ -7251,7 +7254,7 @@ struct llm_build_gemma2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, true); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { // norm @@ -7386,7 +7389,7 @@ struct llm_build_gemma3 : public llm_graph_context { ggml_tensor * inp_pos = build_inp_pos(); // TODO: is causal == true correct? might need some changes - auto * inp_attn = build_attn_inp_kv_unified(true, true); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { const bool is_swa = hparams.is_swa(il); @@ -7515,7 +7518,7 @@ struct llm_build_starcoder2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -7828,7 +7831,7 @@ struct llm_build_command_r : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { @@ -7978,7 +7981,7 @@ struct llm_build_cohere2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, true); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { const bool is_swa = hparams.is_swa(il); @@ -8110,7 +8113,7 @@ struct llm_build_olmo : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -8232,7 +8235,7 @@ struct llm_build_olmo2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -8358,7 +8361,7 @@ struct llm_build_olmoe : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -8481,7 +8484,7 @@ struct llm_build_openelm : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { const int64_t n_head = hparams.n_head(il); @@ -8611,7 +8614,7 @@ struct llm_build_gptneox : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { cur = build_norm(inpL, @@ -8757,7 +8760,7 @@ struct llm_build_arctic : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -8889,7 +8892,7 @@ struct llm_build_deepseek : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; @@ -9054,7 +9057,7 @@ struct llm_build_deepseek2 : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -9274,7 +9277,7 @@ struct llm_build_bitnet : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -9532,7 +9535,7 @@ struct llm_build_t5_dec : public llm_graph_context { const int64_t n_outputs_enc = embd_enc->ne[1]; - auto * inp_attn_self = build_attn_inp_kv_unified(true, false); + auto * inp_attn_self = build_attn_inp_kv_unified(); auto * inp_attn_cross = build_attn_inp_cross(); for (int il = 0; il < n_layer; ++il) { @@ -9698,7 +9701,7 @@ struct llm_build_jais : public llm_graph_context { inpL = build_inp_embd(model.tok_embd); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { cur = build_norm(inpL, @@ -9794,7 +9797,7 @@ struct llm_build_chatglm : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -9926,7 +9929,7 @@ struct llm_build_nemotron : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -10049,7 +10052,7 @@ struct llm_build_exaone : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -10565,7 +10568,7 @@ struct llm_build_chameleon : public llm_graph_context { // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); - auto * inp_attn = build_attn_inp_kv_unified(true, false); + auto * inp_attn = build_attn_inp_kv_unified(); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; From add2a3aa5a1571211aa5c7303b8e80c8d1824b91 Mon Sep 17 00:00:00 2001 From: Victor <194116445+dodekapod@users.noreply.github.com> Date: Fri, 14 Mar 2025 11:21:17 +0100 Subject: [PATCH 133/188] server: fix "--grammar-file" parameter (#12285) --- examples/server/utils.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 36ad276fd3..58cdd6af92 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -621,7 +621,9 @@ static json oaicompat_completion_params_parse( llama_params["chat_format"] = static_cast(chat_params.format); llama_params["prompt"] = chat_params.prompt; - llama_params["grammar"] = chat_params.grammar; + if (!chat_params.grammar.empty()) { + llama_params["grammar"] = chat_params.grammar; + } llama_params["grammar_lazy"] = chat_params.grammar_lazy; auto grammar_triggers = json::array(); for (const auto & trigger : chat_params.grammar_triggers) { From 8fcb563613e20a04dd9791f0a9b8a41086428c09 Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:47:05 +0100 Subject: [PATCH 134/188] Load all MoE experts during warmup (#11571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * llama : introduce llama_set_warmup() API call that controls warmup mode; use all MoE experts during warmup * common : use new API to enable warmup mode during model warmup --------- Co-authored-by: Stanisław Szymczyk --- common/common.cpp | 3 +++ include/llama.h | 4 ++++ src/llama-context.cpp | 13 ++++++++++++- src/llama-context.h | 1 + src/llama-cparams.h | 1 + src/llama-graph.cpp | 2 +- 6 files changed, 22 insertions(+), 2 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 8487e3834b..18ffb4e738 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1033,6 +1033,8 @@ struct common_init_result common_init_from_params(common_params & params) { if (params.warmup) { LOG_WRN("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__); + llama_set_warmup(lctx, true); + std::vector tmp; llama_token bos = llama_vocab_bos(vocab); llama_token eos = llama_vocab_eos(vocab); @@ -1063,6 +1065,7 @@ struct common_init_result common_init_from_params(common_params & params) { llama_kv_self_clear(lctx); llama_synchronize(lctx); llama_perf_context_reset(lctx); + llama_set_warmup(lctx, false); } iparams.model.reset(model); diff --git a/include/llama.h b/include/llama.h index e5286f0616..6a44be404d 100644 --- a/include/llama.h +++ b/include/llama.h @@ -945,6 +945,10 @@ extern "C" { // If set to true, the model will only attend to the past tokens LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); + // Set whether the model is in warmup mode or not + // If true, all model tensors are activated during llama_decode() to load and cache their weights. + LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup); + // Set abort callback LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 4df6b18ec1..c2fcce42a7 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -39,6 +39,7 @@ llama_context::llama_context( cparams.flash_attn = params.flash_attn; cparams.no_perf = params.no_perf; cparams.pooling_type = params.pooling_type; + cparams.warmup = false; cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx; cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base; @@ -948,6 +949,12 @@ void llama_context::set_causal_attn(bool value) { cparams.causal_attn = value; } +void llama_context::set_warmup(bool value) { + LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value); + + cparams.warmup = value; +} + void llama_context::set_adapter_lora( llama_adapter_lora * adapter, float scale) { @@ -1594,7 +1601,7 @@ void llama_context::output_reorder() { // int32_t llama_context::graph_max_nodes() const { - return std::max(8192, 5*model.n_tensors()); + return std::max(65536, 5*model.n_tensors()); } ggml_cgraph * llama_context::graph_init() { @@ -2372,6 +2379,10 @@ void llama_set_causal_attn(llama_context * ctx, bool causal_attn) { ctx->set_causal_attn(causal_attn); } +void llama_set_warmup(llama_context * ctx, bool warmup) { + ctx->set_warmup(warmup); +} + void llama_synchronize(llama_context * ctx) { ctx->synchronize(); } diff --git a/src/llama-context.h b/src/llama-context.h index 88df8950e4..04facb544c 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -64,6 +64,7 @@ struct llama_context { void set_embeddings (bool value); void set_causal_attn(bool value); + void set_warmup(bool value); void set_adapter_lora( llama_adapter_lora * adapter, diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 252012f3d9..30e550f023 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -29,6 +29,7 @@ struct llama_cparams { bool offload_kqv; bool flash_attn; bool no_perf; + bool warmup; enum llama_pooling_type pooling_type; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index e4af507780..4e90873397 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -577,7 +577,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) : n_embd_head_v (hparams.n_embd_head_v), n_embd_v_gqa (hparams.n_embd_v_gqa()), n_expert (hparams.n_expert), - n_expert_used (hparams.n_expert_used), + n_expert_used (cparams.warmup ? hparams.n_expert : hparams.n_expert_used), freq_base (cparams.rope_freq_base), freq_scale (cparams.rope_freq_scale), ext_factor (cparams.yarn_ext_factor), From 774973b8f3d5e375b0b74d58638eeb1817e950a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Fri, 14 Mar 2025 16:57:05 +0100 Subject: [PATCH 135/188] main : add -sysf / --system-prompt-file (#12249) (#12250) * add system_prompt_file * add -sysf / --system-prompt-file * remove system_prompt_file --- common/arg.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/common/arg.cpp b/common/arg.cpp index fe6a1eece7..240c699a2c 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -853,6 +853,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_excludes({LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"-sysf", "--system-prompt-file"}, "FNAME", + "a file containing the system prompt (default: none)", + [](common_params & params, const std::string & value) { + std::ifstream file(value); + if (!file) { + throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str())); + } + std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.system_prompt)); + if (!params.system_prompt.empty() && params.system_prompt.back() == '\n') { + params.system_prompt.pop_back(); + } + } + ).set_examples({LLAMA_EXAMPLE_MAIN})); add_opt(common_arg( {"--in-file"}, "FNAME", "an input file (repeat to specify multiple files)", From 9f2250ba722738ec0e6ab684636268a79160c854 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Fri, 14 Mar 2025 16:41:20 +0000 Subject: [PATCH 136/188] Add CLI arg to llama-run to adjust the number of threads used (#12370) We default to 4, sometimes we want to manually adjust this Signed-off-by: Eric Curtin --- examples/run/run.cpp | 131 +++++++++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 43 deletions(-) diff --git a/examples/run/run.cpp b/examples/run/run.cpp index 437f2533e5..462a6d1519 100644 --- a/examples/run/run.cpp +++ b/examples/run/run.cpp @@ -79,6 +79,7 @@ class Opt { ctx_params = llama_context_default_params(); model_params = llama_model_default_params(); context_size_default = ctx_params.n_batch; + n_threads_default = ctx_params.n_threads; ngl_default = model_params.n_gpu_layers; common_params_sampling sampling; temperature_default = sampling.temp; @@ -104,6 +105,7 @@ class Opt { ctx_params.n_batch = context_size >= 0 ? context_size : context_size_default; ctx_params.n_ctx = ctx_params.n_batch; + ctx_params.n_threads = ctx_params.n_threads_batch = n_threads >= 0 ? n_threads : n_threads_default; model_params.n_gpu_layers = ngl >= 0 ? ngl : ngl_default; temperature = temperature >= 0 ? temperature : temperature_default; @@ -116,12 +118,12 @@ class Opt { std::string chat_template_file; std::string user; bool use_jinja = false; - int context_size = -1, ngl = -1; + int context_size = -1, ngl = -1, n_threads = -1; float temperature = -1; bool verbose = false; private: - int context_size_default = -1, ngl_default = -1; + int context_size_default = -1, ngl_default = -1, n_threads_default = -1; float temperature_default = -1; bool help = false; @@ -159,53 +161,94 @@ class Opt { return 0; } + int parse_options_with_value(int argc, const char ** argv, int & i, bool & options_parsing) { + if (options_parsing && (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--context-size") == 0)) { + if (handle_option_with_value(argc, argv, i, context_size) == 1) { + return 1; + } + } else if (options_parsing && + (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "-ngl") == 0 || strcmp(argv[i], "--ngl") == 0)) { + if (handle_option_with_value(argc, argv, i, ngl) == 1) { + return 1; + } + } else if (options_parsing && (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--threads") == 0)) { + if (handle_option_with_value(argc, argv, i, n_threads) == 1) { + return 1; + } + } else if (options_parsing && strcmp(argv[i], "--temp") == 0) { + if (handle_option_with_value(argc, argv, i, temperature) == 1) { + return 1; + } + } else if (options_parsing && strcmp(argv[i], "--chat-template-file") == 0) { + if (handle_option_with_value(argc, argv, i, chat_template_file) == 1) { + return 1; + } + use_jinja = true; + } else { + return 2; + } + + return 0; + } + + int parse_options(const char ** argv, int & i, bool & options_parsing) { + if (options_parsing && (parse_flag(argv, i, "-v", "--verbose") || parse_flag(argv, i, "-v", "--log-verbose"))) { + verbose = true; + } else if (options_parsing && strcmp(argv[i], "--jinja") == 0) { + use_jinja = true; + } else if (options_parsing && parse_flag(argv, i, "-h", "--help")) { + help = true; + return 0; + } else if (options_parsing && strcmp(argv[i], "--") == 0) { + options_parsing = false; + } else { + return 2; + } + + return 0; + } + + int parse_positional_args(const char ** argv, int & i, int & positional_args_i) { + if (positional_args_i == 0) { + if (!argv[i][0] || argv[i][0] == '-') { + return 1; + } + + ++positional_args_i; + model_ = argv[i]; + } else if (positional_args_i == 1) { + ++positional_args_i; + user = argv[i]; + } else { + user += " " + std::string(argv[i]); + } + + return 0; + } + int parse(int argc, const char ** argv) { bool options_parsing = true; for (int i = 1, positional_args_i = 0; i < argc; ++i) { - if (options_parsing && (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--context-size") == 0)) { - if (handle_option_with_value(argc, argv, i, context_size) == 1) { - return 1; - } - } else if (options_parsing && - (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "-ngl") == 0 || strcmp(argv[i], "--ngl") == 0)) { - if (handle_option_with_value(argc, argv, i, ngl) == 1) { - return 1; - } - } else if (options_parsing && strcmp(argv[i], "--temp") == 0) { - if (handle_option_with_value(argc, argv, i, temperature) == 1) { - return 1; - } - } else if (options_parsing && - (parse_flag(argv, i, "-v", "--verbose") || parse_flag(argv, i, "-v", "--log-verbose"))) { - verbose = true; - } else if (options_parsing && strcmp(argv[i], "--jinja") == 0) { - use_jinja = true; - } else if (options_parsing && strcmp(argv[i], "--chat-template-file") == 0){ - if (handle_option_with_value(argc, argv, i, chat_template_file) == 1) { - return 1; - } - use_jinja = true; - } else if (options_parsing && parse_flag(argv, i, "-h", "--help")) { - help = true; - return 0; - } else if (options_parsing && strcmp(argv[i], "--") == 0) { - options_parsing = false; - } else if (positional_args_i == 0) { - if (!argv[i][0] || argv[i][0] == '-') { - return 1; - } + int ret = parse_options_with_value(argc, argv, i, options_parsing); + if (ret == 0) { + continue; + } else if (ret == 1) { + return ret; + } - ++positional_args_i; - model_ = argv[i]; - } else if (positional_args_i == 1) { - ++positional_args_i; - user = argv[i]; - } else { - user += " " + std::string(argv[i]); + ret = parse_options(argv, i, options_parsing); + if (ret == 0) { + continue; + } else if (ret == 1) { + return ret; + } + + if (parse_positional_args(argv, i, positional_args_i)) { + return 1; } } - if (model_.empty()){ + if (model_.empty()) { return 1; } @@ -232,6 +275,8 @@ class Opt { " Number of GPU layers (default: %d)\n" " --temp \n" " Temperature (default: %.1f)\n" + " -t, --threads \n" + " Number of threads to use during generation (default: %d)\n" " -v, --verbose, --log-verbose\n" " Set verbosity level to infinity (i.e. log all messages, useful for debugging)\n" " -h, --help\n" @@ -260,7 +305,7 @@ class Opt { " llama-run file://some-file3.gguf\n" " llama-run --ngl 999 some-file4.gguf\n" " llama-run --ngl 999 some-file5.gguf Hello World\n", - context_size_default, ngl_default, temperature_default); + context_size_default, ngl_default, temperature_default, n_threads_default); } }; From 92a391327e9201b9b5b32fdd3afb452026c22d4c Mon Sep 17 00:00:00 2001 From: Chenguang Li <757486878@qq.com> Date: Sat, 15 Mar 2025 09:31:08 +0800 Subject: [PATCH 137/188] [CANN]MUL_MAT optimization (#12382) --- ggml/src/ggml-cann/aclnn_ops.cpp | 8 ++++++-- ggml/src/ggml-cann/ggml-cann.cpp | 5 ----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cann/aclnn_ops.cpp b/ggml/src/ggml-cann/aclnn_ops.cpp index b2d857e1e5..6bb5d08349 100644 --- a/ggml/src/ggml-cann/aclnn_ops.cpp +++ b/ggml/src/ggml-cann/aclnn_ops.cpp @@ -2790,10 +2790,14 @@ static void ggml_cann_mul_mat_quant(ggml_backend_cann_context& ctx, (char*)output_buffer + batch1 * output_stride, ACL_FLOAT16, output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND, output_ne_offset); + int64_t antiquantGroupSize = 0; + if (src0->ne[0] > QK8_0) { + antiquantGroupSize = QK8_0; + } ACL_CHECK(aclnnWeightQuantBatchMatmulV2GetWorkspaceSize( acl_input_tensor, acl_weight_tensor, acl_scale_tensor, nullptr, - nullptr, nullptr, nullptr, QK8_0, acl_output_tensor, + nullptr, nullptr, nullptr, antiquantGroupSize, acl_output_tensor, &workspaceSize, &executor)); if (workspaceAddr == nullptr) { workspaceAddr = workspace_allocator.alloc(workspaceSize); @@ -2833,7 +2837,7 @@ static void ggml_cann_mul_mat_quant(ggml_backend_cann_context& ctx, ACL_CHECK(aclnnWeightQuantBatchMatmulV2GetWorkspaceSize( acl_input_tensor, acl_weight_tensor, acl_scale_tensor, - nullptr, nullptr, nullptr, nullptr, QK8_0, + nullptr, nullptr, nullptr, nullptr, antiquantGroupSize, acl_output_tensor, &workspaceSize, &executor)); ACL_CHECK(aclnnWeightQuantBatchMatmulV2( workspaceAddr, workspaceSize, executor, ctx.stream())); diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index b8d272cda6..68cd9920d1 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -1689,11 +1689,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, case GGML_OP_MUL_MAT: { switch (op->src[0]->type) { case GGML_TYPE_Q8_0: - // Current groupsize should not be greater than k-1 in - // aclnnWeightQuantBatchMatmulV2GetWorkspaceSize - if (op->src[0]->ne[0] <= QK8_0) { - return false; - } case GGML_TYPE_F16: case GGML_TYPE_F32: case GGML_TYPE_Q4_0: From b19bd064c09822cb81efe4a38abafab3e979c9ce Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Sat, 15 Mar 2025 15:19:30 +0100 Subject: [PATCH 138/188] SYCL : support non-contiguous tensors in binary ops (add, sub, etc) (#12399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * sycl : support non-contiguous tensors in binary ops * sycl : silence unused variable warning --------- Co-authored-by: Stanisław Szymczyk --- ggml/src/ggml-sycl/common.hpp | 87 ++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index a92988b7db..fdd07d9caf 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -474,6 +474,7 @@ static void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst, int ne0, int ne1, int ne2, int ne3, int ne10, int ne11, int ne12, int ne13, /*int s0, */ int s1, int s2, int s3, + /*int s00,*/ int s01, int s02, int s03, /*int s10,*/ int s11, int s12, int s13, const sycl::nd_item<3> &item_ct1) { const int i0s = item_ct1.get_local_range(2) * item_ct1.get_group(2) + @@ -495,9 +496,9 @@ static void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst, const int i12 = i2 % ne12; const int i13 = i3 % ne13; - const size_t i_src0 = i3*s3 + i2*s2 + i1*s1; + const size_t i_src0 = i3*s03 + i2*s02 + i1*s01; const size_t i_src1 = i13*s13 + i12*s12 + i11*s11; - const size_t i_dst = i_src0; + const size_t i_dst = i3*s3 + i2*s2 + i1*s1; const src0_t * src0_row = src0 + i_src0; const src1_t * src1_row = src1 + i_src1; @@ -515,6 +516,7 @@ static void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t int ne0, int ne1, int ne2, int ne3, int ne10, int ne11, int ne12, int ne13, /*int s0, */ int s1, int s2, int s3, + /*int s00,*/ int s01, int s02, int s03, /*int s10,*/ int s11, int s12, int s13, const sycl::nd_item<3> &item_ct1) { @@ -534,9 +536,9 @@ static void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t const int i12 = i2 % ne12; const int i13 = i3 % ne13; - const size_t i_src0 = i3*s3 + i2*s2 + i1*s1; + const size_t i_src0 = i3*s03 + i2*s02 + i1*s01; const size_t i_src1 = i13*s13 + i12*s12 + i11*s11; - const size_t i_dst = i_src0; + const size_t i_dst = i3*s3 + i2*s2 + i1*s1; const src0_t * src0_row = src0 + i_src0; const src1_t * src1_row = src1 + i_src1; @@ -566,9 +568,11 @@ struct bin_bcast_sycl { int nr[4] = { nr0, nr1, nr2, nr3 }; // collapse dimensions until first broadcast dimension - int64_t cne0[] = {ne0, ne1, ne2, ne3}; + int64_t cne[] = {ne0, ne1, ne2, ne3}; + int64_t cne0[] = {ne00, ne01, ne02, ne03}; int64_t cne1[] = {ne10, ne11, ne12, ne13}; - size_t cnb0[] = {nb0, nb1, nb2, nb3}; + size_t cnb[] = {nb0, nb1, nb2, nb3}; + size_t cnb0[] = {nb00, nb01, nb02, nb03}; size_t cnb1[] = {nb10, nb11, nb12, nb13}; auto collapse = [](int64_t cne[]) { cne[0] *= cne[1]; @@ -583,32 +587,41 @@ struct bin_bcast_sycl { cnb[3] *= cne[3]; }; - for (int i = 0; i < 4; i++) { - if (nr[i] != 1) { - break; - } - if (i > 0) { - collapse_nb(cnb0, cne0); - collapse_nb(cnb1, cne1); - collapse(cne0); - collapse(cne1); + if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)) { + for (int i = 0; i < 4; i++) { + if (nr[i] != 1) { + break; + } + if (i > 0) { + collapse_nb(cnb, cne); + collapse_nb(cnb0, cne0); + collapse_nb(cnb1, cne1); + collapse(cne); + collapse(cne0); + collapse(cne1); + } } } { - int64_t ne0 = cne0[0]; - int64_t ne1 = cne0[1]; - int64_t ne2 = cne0[2]; - int64_t ne3 = cne0[3]; + int64_t ne0 = cne[0]; + int64_t ne1 = cne[1]; + int64_t ne2 = cne[2]; + int64_t ne3 = cne[3]; int64_t ne10 = cne1[0]; int64_t ne11 = cne1[1]; int64_t ne12 = cne1[2]; int64_t ne13 = cne1[3]; - size_t nb0 = cnb0[0]; - size_t nb1 = cnb0[1]; - size_t nb2 = cnb0[2]; - size_t nb3 = cnb0[3]; + size_t nb0 = cnb[0]; + size_t nb1 = cnb[1]; + size_t nb2 = cnb[2]; + size_t nb3 = cnb[3]; + + size_t nb00 = cnb0[0]; + size_t nb01 = cnb0[1]; + size_t nb02 = cnb0[2]; + size_t nb03 = cnb0[3]; size_t nb10 = cnb1[0]; size_t nb11 = cnb1[1]; @@ -625,6 +638,28 @@ struct bin_bcast_sycl { size_t s12 = nb12 / sizeof(src1_t); size_t s13 = nb13 / sizeof(src1_t); + size_t s00 = nb00 / sizeof(src0_t); + size_t s01 = nb01 / sizeof(src0_t); + size_t s02 = nb02 / sizeof(src0_t); + size_t s03 = nb03 / sizeof(src0_t); + + GGML_UNUSED(s00); + + GGML_ASSERT(nb0 % sizeof(dst_t) == 0); + GGML_ASSERT(nb1 % sizeof(dst_t) == 0); + GGML_ASSERT(nb2 % sizeof(dst_t) == 0); + GGML_ASSERT(nb3 % sizeof(dst_t) == 0); + + GGML_ASSERT(nb00 % sizeof(src0_t) == 0); + GGML_ASSERT(nb01 % sizeof(src0_t) == 0); + GGML_ASSERT(nb02 % sizeof(src0_t) == 0); + GGML_ASSERT(nb03 % sizeof(src0_t) == 0); + + GGML_ASSERT(nb10 % sizeof(src1_t) == 0); + GGML_ASSERT(nb11 % sizeof(src1_t) == 0); + GGML_ASSERT(nb12 % sizeof(src1_t) == 0); + GGML_ASSERT(nb13 % sizeof(src1_t) == 0); + GGML_ASSERT(s0 == 1); GGML_ASSERT(s10 == 1); @@ -661,8 +696,8 @@ struct bin_bcast_sycl { [=](sycl::nd_item<3> item_ct1) { k_bin_bcast_unravel( src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3, - ne10, ne11, ne12, ne13, s1, s2, s3, s11, s12, - s13, item_ct1); + ne10, ne11, ne12, ne13, s1, s2, s3, s01, s02, + s03, s11, s12, s13, item_ct1); }); } } else { @@ -680,7 +715,7 @@ struct bin_bcast_sycl { [=](sycl::nd_item<3> item_ct1) { k_bin_bcast(src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3, ne10, ne11, ne12, ne13, - s1, s2, s3, s11, s12, s13, + s1, s2, s3, s01, s02, s03, s11, s12, s13, item_ct1); }); } From 3d35d87b4113648e224b837bb88e6b2c4c7f29e5 Mon Sep 17 00:00:00 2001 From: aubreyli Date: Sat, 15 Mar 2025 22:49:03 +0800 Subject: [PATCH 139/188] SYCL: Delete redundant plus sign and space (#12391) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 6977b705e4..ef7d5fa01a 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -3113,8 +3113,8 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, const int64_t i2 = i12; src0_row.data = src0_original + i02*nb02; - src1_row.data = src1_original + + i11*nb11 + i12*nb12; - dst_row.data = dst_original + i1*nb1 + i2*nb2; + src1_row.data = src1_original + i11*nb11 + i12*nb12; + dst_row.data = dst_original + i1*nb1 + i2*nb2; ggml_sycl_mul_mat(ctx, &src0_row, &src1_row, &dst_row); } From f4c3dd5daa3a79f713813cf1aabdc5886071061d Mon Sep 17 00:00:00 2001 From: marcoStocchi Date: Sat, 15 Mar 2025 17:23:11 +0100 Subject: [PATCH 140/188] llama-tts : add '-o' option (#12398) * added -o option to specify an output file name * llama-tts returns ENOENT in case of file write error note : PR #12042 is closed as superseded with this one. --- common/arg.cpp | 2 +- examples/tts/tts.cpp | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 240c699a2c..b6bfe6f89b 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1889,7 +1889,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.out_file = value; } - ).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA})); + ).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_TTS})); add_opt(common_arg( {"-ofreq", "--output-frequency"}, "N", string_format("output the imatrix every N iterations (default: %d)", params.n_out_freq), diff --git a/examples/tts/tts.cpp b/examples/tts/tts.cpp index c658f3182f..d953cadd62 100644 --- a/examples/tts/tts.cpp +++ b/examples/tts/tts.cpp @@ -87,11 +87,11 @@ struct wav_header { uint32_t data_size; }; -static void save_wav16(const std::string & fname, const std::vector & data, int sample_rate) { +static bool save_wav16(const std::string & fname, const std::vector & data, int sample_rate) { std::ofstream file(fname, std::ios::binary); if (!file) { - LOG_ERR("%s: Failed to open file '%s' for writing", __func__, fname.c_str()); - return; + LOG_ERR("%s: Failed to open file '%s' for writing.\n", __func__, fname.c_str()); + return false; } wav_header header; @@ -108,7 +108,7 @@ static void save_wav16(const std::string & fname, const std::vector & dat file.write(reinterpret_cast(&pcm_sample), sizeof(pcm_sample)); } - file.close(); + return file.good(); } static void fill_hann_window(int length, bool periodic, float * output) { @@ -536,6 +536,7 @@ static std::string audio_data_from_speaker(json speaker, const outetts_version t int main(int argc, char ** argv) { common_params params; + params.out_file = "output.wav"; params.prompt = ""; params.n_predict = 4096; @@ -1060,8 +1061,6 @@ lovely<|t_0.56|><|code_start|><|634|><|596|><|1766|><|1556|><|1306|><|1285|><|14 } #endif - const std::string fname = "output.wav"; - const int n_sr = 24000; // sampling rate // zero out first 0.25 seconds @@ -1072,11 +1071,15 @@ lovely<|t_0.56|><|code_start|><|634|><|596|><|1766|><|1556|><|1306|><|1285|><|14 LOG_INF("%s: time for spectral ops: %.3f ms\n", __func__, (ggml_time_us() - t_spec_start) / 1000.0f); LOG_INF("%s: total time: %.3f ms\n", __func__, (ggml_time_us() - t_main_start) / 1000.0f); - save_wav16(fname, audio, n_sr); + int retval = 0; - LOG_INF("%s: audio written to file '%s'\n", __func__, fname.c_str()); + if (save_wav16(params.out_file, audio, n_sr)) { + LOG_INF("%s: audio written to file '%s'\n", __func__, params.out_file.c_str()); + } else { + retval = ENOENT; + } llama_backend_free(); - return 0; + return retval; } From 7b61bcc87cfdeab88350e82df1c4b7be64331ea6 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Sun, 16 Mar 2025 18:22:05 +0100 Subject: [PATCH 141/188] ci : add --symlinks to xcframework zip command (#12409) This commit adds the --symlinks option to the zip command used to create the xcframework zip file. This is necessary to create symlinks in the zip file. Without this option, the Versions symlink is stored as a regular directory entry in the zip file, rather than as a symlink in the zip which causes the followig error in xcode: ```console Couldn't resolve framework symlink for '/Users/danbev/work/ai/llama.cpp/tmp_1/build-apple/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current': readlink(/Users/danbev/work/ai/llama.cpp/tmp_1/build-apple/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current): Invalid argument (22) ``` Refs: https://github.com/ggml-org/llama.cpp/pull/11996#issuecomment-2727026377 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e24293645..03cde0a484 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1379,7 +1379,7 @@ jobs: id: pack_artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} run: | - zip -r llama-${{ steps.tag.outputs.name }}-xcframework.zip build-apple/llama.xcframework + zip --symlinks -r llama-${{ steps.tag.outputs.name }}-xcframework.zip build-apple/llama.xcframework - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} From dc079cfdffa1141a6caf5d41a33d73a1ef03da55 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 16 Mar 2025 19:29:36 +0200 Subject: [PATCH 142/188] context : fix init of n_outputs (#12397) ggml-ci --- src/llama-context.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index c2fcce42a7..abb7e526f6 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -285,11 +285,15 @@ llama_context::llama_context( // reserve worst-case graph if (!hparams.vocab_only) { - uint32_t n_seqs = 1; // TODO: worst-case number of sequences - uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); + const uint32_t n_seqs = 1; // TODO: worst-case number of sequences + const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); llama_token token = model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph + // restore later + // TODO: something cleaner + const auto n_outputs_save = n_outputs; + // max number of outputs n_outputs = n_tokens; @@ -341,6 +345,8 @@ llama_context::llama_context( } } + n_outputs = n_outputs_save; + for (size_t i = 0; i < backend_ptrs.size(); ++i) { ggml_backend_t backend = backend_ptrs[i]; ggml_backend_buffer_type_t buft = backend_buft[i]; From 8ba95dca2065c0073698afdfcda4c8a8f08bf0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 16 Mar 2025 18:46:36 +0100 Subject: [PATCH 143/188] llama : fix OLMo-2-0325-32B-Instruct K-norm size (#12400) --- src/llama-model.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 750a702ff7..4b288d8f66 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1005,6 +1005,7 @@ void llama_model::load_hparams(llama_model_loader & ml) { case 16: type = LLM_TYPE_1B; break; case 32: type = LLM_TYPE_7B; break; case 40: type = LLM_TYPE_13B; break; + case 64: type = LLM_TYPE_32B; break; default: type = LLM_TYPE_UNKNOWN; } } break; @@ -2726,6 +2727,8 @@ bool llama_model::load_tensors(llama_model_loader & ml) { } break; case LLM_ARCH_OLMO2: { + const int64_t n_embd_head = n_embd / n_head; + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); // output @@ -2740,7 +2743,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0); layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0); layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_head_kv * n_embd_head}, 0); layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); From b3c9a65673a63a6c9a75da24ee00d13499494e0c Mon Sep 17 00:00:00 2001 From: Akarshan Biswas Date: Mon, 17 Mar 2025 07:15:12 +0530 Subject: [PATCH 144/188] SYCL: set extras only on GGML_TYPE_Q4_0 (#12366) * SYCL: set extras only on GGML_TYPE_Q4_0 * release tensor_extras in reset buffer interface --- ggml/src/ggml-sycl/ggml-sycl.cpp | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index ef7d5fa01a..05984d8c5a 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -333,10 +333,11 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, assert(tensor->view_src->buffer->buft == buffer->buft); return GGML_STATUS_SUCCESS; } - - ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; - tensor->extra = extra; - ctx->tensor_extras.push_back(extra); //used to release it when destroy ctx. + if (tensor->type == GGML_TYPE_Q4_0) { + ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; + tensor->extra = extra; + ctx->tensor_extras.push_back(extra); //used to release it when destroy ctx. + } if (ggml_is_quantized(tensor->type)) { // initialize padding to 0 to avoid possible NaN values @@ -486,6 +487,22 @@ catch (sycl::exception const &exc) { std::exit(1); } +static void ggml_backend_sycl_buffer_reset(ggml_backend_buffer_t buffer) { + GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); + if (buffer == nullptr) { + return; + } + + ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *) buffer->context; + + if (ctx != nullptr) { + for (ggml_tensor_extra_gpu * extra : ctx->tensor_extras) { + release_extra_gpu(extra); + } + ctx->tensor_extras.clear(); // reset the tensor_extras vector + } +} + static const ggml_backend_buffer_i ggml_backend_sycl_buffer_interface = { /* .free_buffer = */ ggml_backend_sycl_buffer_free_buffer, /* .get_base = */ ggml_backend_sycl_buffer_get_base, @@ -495,7 +512,7 @@ static const ggml_backend_buffer_i ggml_backend_sycl_buffer_interface = { /* .get_tensor = */ ggml_backend_sycl_buffer_get_tensor, /* .cpy_tensor = */ ggml_backend_sycl_buffer_cpy_tensor, /* .clear = */ ggml_backend_sycl_buffer_clear, - /* .reset = */ NULL, + /* .reset = */ ggml_backend_sycl_buffer_reset, }; // sycl buffer type @@ -576,7 +593,6 @@ ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { static std::mutex mutex; std::lock_guard lock(mutex); - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_buffer_type\n"); auto dev_count = ggml_backend_sycl_get_device_count(); @@ -3761,7 +3777,6 @@ bool ggml_backend_is_sycl(ggml_backend_t backend) { } int ggml_backend_sycl_get_device_count() { - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_get_device_count\n"); return ggml_sycl_info().device_count; } From 374101fd742bb35cb9bf46c86836e54d51191ffd Mon Sep 17 00:00:00 2001 From: Christian Kastner Date: Mon, 17 Mar 2025 10:05:23 +0100 Subject: [PATCH 145/188] cmake : enable building llama.cpp using system libggml (#12321) * cmake: Factor out compiler flag function from ggml llama.cpps's build requires it, too, and we may want to make use of it without add_subdirectory(ggml). * cmake: Enable building against system ggml This facilitates package maintenance for Linux distributions, where the libggml library most likely will be shipped as an individual package upon which a llama.cpp package depends. --- CMakeLists.txt | 10 +++++++++- cmake/common.cmake | 2 ++ ggml/cmake/common.cmake | 26 ++++++++++++++++++++++++++ ggml/src/CMakeLists.txt | 28 +--------------------------- 4 files changed, 38 insertions(+), 28 deletions(-) create mode 100644 ggml/cmake/common.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b2a1845e5..23cfbce5ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,8 @@ else() set(LLAMA_STANDALONE OFF) endif() +option(LLAMA_USE_SYSTEM_GGML "Use system libggml" OFF) + if (EMSCRIPTEN) set(BUILD_SHARED_LIBS_DEFAULT OFF) @@ -145,7 +147,13 @@ endif() # 3rd-party # -if (NOT TARGET ggml) +if (LLAMA_USE_SYSTEM_GGML) + message(STATUS "Using system-provided libggml, skipping ggml build") + find_package(ggml REQUIRED) + add_library(ggml ALIAS ggml::ggml) +endif() + +if (NOT TARGET ggml AND NOT LLAMA_USE_SYSTEM_GGML) add_subdirectory(ggml) # ... otherwise assume ggml is added by a parent CMakeLists.txt endif() diff --git a/cmake/common.cmake b/cmake/common.cmake index 0f54871e41..a5bb787f15 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1,3 +1,5 @@ +include("ggml/cmake/common.cmake") + function(llama_add_compile_flags) if (LLAMA_FATAL_WARNINGS) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake new file mode 100644 index 0000000000..1976d0ae9b --- /dev/null +++ b/ggml/cmake/common.cmake @@ -0,0 +1,26 @@ +function(ggml_get_flags CCID CCVER) + set(C_FLAGS "") + set(CXX_FLAGS "") + + if (CCID MATCHES "Clang") + set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return) + set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi) + + if ( + (CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR + (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0) + ) + list(APPEND C_FLAGS -Wdouble-promotion) + endif() + elseif (CCID STREQUAL "GNU") + set(C_FLAGS -Wdouble-promotion) + set(CXX_FLAGS -Wno-array-bounds) + + if (CCVER VERSION_GREATER_EQUAL 8.1.0) + list(APPEND CXX_FLAGS -Wextra-semi) + endif() + endif() + + set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE) + set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE) +endfunction() diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 52817510f6..a797e2b187 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -1,4 +1,5 @@ include(CheckCXXCompilerFlag) +include("../cmake/common.cmake") add_compile_definitions(GGML_SCHED_MAX_COPIES=${GGML_SCHED_MAX_COPIES}) @@ -24,33 +25,6 @@ if (NOT MSVC) endif() endif() -function(ggml_get_flags CCID CCVER) - set(C_FLAGS "") - set(CXX_FLAGS "") - - if (CCID MATCHES "Clang") - set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return) - set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi) - - if ( - (CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR - (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0) - ) - list(APPEND C_FLAGS -Wdouble-promotion) - endif() - elseif (CCID STREQUAL "GNU") - set(C_FLAGS -Wdouble-promotion) - set(CXX_FLAGS -Wno-array-bounds) - - if (CCVER VERSION_GREATER_EQUAL 8.1.0) - list(APPEND CXX_FLAGS -Wextra-semi) - endif() - endif() - - set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE) - set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE) -endfunction() - if (GGML_FATAL_WARNINGS) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") list(APPEND C_FLAGS -Werror) From 2f21123c1deb3ce1be3c0578c5f6980fe19ed077 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Mon, 17 Mar 2025 04:35:00 -0500 Subject: [PATCH 146/188] vulkan: Adjust coopmat2 tile sizes and selection heuristic (#12258) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 42 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index ff53bdfbe1..e46007a52f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1476,26 +1476,26 @@ static void ggml_vk_load_shaders(vk_device& device) { // spec constants and tile sizes for quant matmul (non-Qi_K) l_warptile_mmq = { 256, 128, 256, 64 }; m_warptile_mmq = { 256, 128, 128, 64 }; - s_warptile_mmq = { 256, 128, 128, 64 }; + s_warptile_mmq = { 256, 32, 64, 128 }; l_mmq_wg_denoms = { 128, 256, 1 }; m_mmq_wg_denoms = { 128, 128, 1 }; - s_mmq_wg_denoms = { 128, 128, 1 }; + s_mmq_wg_denoms = { 32, 64, 1 }; // spec constants and tile sizes for quant matmul (Qi_K) - l_warptile_mmq_k = { 256, 128, 512, 16 }; - m_warptile_mmq_k = { 256, 128, 256, 16 }; - s_warptile_mmq_k = { 256, 32, 128, 64 }; - l_mmq_wg_denoms_k = { 128, 512, 1 }; - m_mmq_wg_denoms_k = { 128, 256, 1 }; - s_mmq_wg_denoms_k = { 32, 128, 1 }; + l_warptile_mmq_k = { 256, 64, 128, 64 }; + m_warptile_mmq_k = { 256, 32, 64, 64 }; + s_warptile_mmq_k = { 256, 32, 32, 128 }; + l_mmq_wg_denoms_k = { 64, 128, 1 }; + m_mmq_wg_denoms_k = { 32, 64, 1 }; + s_mmq_wg_denoms_k = { 32, 32, 1 }; // spec constants and tile sizes for quant matmul_id - l_warptile_mmqid = { 256, 128, 128, 16 }; + l_warptile_mmqid = { 256, 128, 64, 16 }; m_warptile_mmqid = { 256, 128, 64, 16 }; - s_warptile_mmqid = { 256, 64, 64, 16 }; - l_mmqid_wg_denoms = { 128, 128, 1 }; + s_warptile_mmqid = { 256, 128, 64, 16 }; + l_mmqid_wg_denoms = { 128, 64, 1 }; m_mmqid_wg_denoms = { 128, 64, 1 }; - s_mmqid_wg_denoms = { 64, 64, 1 }; + s_mmqid_wg_denoms = { 128, 64, 1 }; l_align = 128; m_align = 64; @@ -3850,10 +3850,14 @@ static vk_pipeline ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ")"); if (ctx->device->coopmat2) { - if ((ctx->device->mul_mat_l[src0_type] && (m % mmp->l->wg_denoms[0]) == 0 && (n % mmp->l->wg_denoms[1]) == 0) || (!ctx->device->mul_mat_m[src0_type] && !ctx->device->mul_mat_s[src0_type])) { + // Use large shader when the N dimension is greater than the medium shader's tile size + uint32_t crossover_large = mmp->m->wg_denoms[1]; + if ((ctx->device->mul_mat_l[src0_type] && (n > crossover_large)) || (!ctx->device->mul_mat_m[src0_type] && !ctx->device->mul_mat_s[src0_type])) { return aligned ? mmp->a_l : mmp->l; } - if ((ctx->device->mul_mat_m[src0_type] && (m % mmp->m->wg_denoms[0]) == 0 && (n % mmp->m->wg_denoms[1]) == 0) || !ctx->device->mul_mat_s[src0_type]) { + // Use medium shader when the N dimension is greater than the small shader's tile size + uint32_t crossover_medium = mmp->s->wg_denoms[1]; + if ((ctx->device->mul_mat_m[src0_type] && (n > crossover_medium)) || !ctx->device->mul_mat_s[src0_type]) { return aligned ? mmp->a_m : mmp->m; } return aligned ? mmp->a_s : mmp->s; @@ -3898,13 +3902,17 @@ static void ggml_vk_matmul( } static vk_pipeline ggml_vk_guess_matmul_id_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, bool aligned, ggml_type src0_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ")"); + VK_LOG_DEBUG("ggml_vk_guess_matmul_id_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ")"); if (ctx->device->coopmat2) { - if ((ctx->device->mul_mat_id_l[src0_type] && (m % mmp->l->wg_denoms[0]) == 0 && (n % mmp->l->wg_denoms[1]) == 0) || (!ctx->device->mul_mat_id_m[src0_type] && !ctx->device->mul_mat_id_s[src0_type])) { + // Use large shader when the N dimension is greater than the medium shader's tile size + uint32_t crossover_large = mmp->m->wg_denoms[1]; + if ((ctx->device->mul_mat_id_l[src0_type] && (n > crossover_large)) || (!ctx->device->mul_mat_id_m[src0_type] && !ctx->device->mul_mat_id_s[src0_type])) { return aligned ? mmp->a_l : mmp->l; } - if ((ctx->device->mul_mat_id_m[src0_type] && (m % mmp->m->wg_denoms[0]) == 0 && (n % mmp->m->wg_denoms[1]) == 0) || !ctx->device->mul_mat_id_s[src0_type]) { + // Use medium shader when the N dimension is greater than the small shader's tile size + uint32_t crossover_medium = mmp->s->wg_denoms[1]; + if ((ctx->device->mul_mat_id_m[src0_type] && (n > crossover_medium)) || !ctx->device->mul_mat_id_s[src0_type]) { return aligned ? mmp->a_m : mmp->m; } return aligned ? mmp->a_s : mmp->s; From 891c63956dbfbdf7ed2ecd0b5882cff49dbfe90f Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Mon, 17 Mar 2025 04:41:59 -0500 Subject: [PATCH 147/188] vulkan: Pad N dimension of B matrix for coopmat2 perf, to avoid bounds checking (#12273) * vulkan: Pad N dimension of B matrix for coopmat2 perf, to avoid bounds checking --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 43 +++++++++++-------- .../vulkan-shaders/mul_mm_cm2.comp | 13 +++--- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index e46007a52f..a837b0dda4 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -29,6 +29,7 @@ #include "ggml-vulkan-shaders.hpp" +#define ROUNDUP_POW2(M, N) (((M) + (N) - 1) & ~((N) - 1)) #define CEIL_DIV(M, N) (((M) + (N)-1) / (N)) #define VK_VENDOR_ID_AMD 0x1002 @@ -368,6 +369,7 @@ struct vk_mat_mat_push_constants { uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; uint32_t k_split; uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3; + uint32_t padded_N; }; struct vk_mat_vec_push_constants { uint32_t ncols; uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; @@ -380,6 +382,7 @@ struct vk_mat_mat_id_push_constants { uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11; + uint32_t padded_N; }; struct vk_mat_vec_id_push_constants { uint32_t ncols; uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; @@ -3882,18 +3885,19 @@ static void ggml_vk_matmul( vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, - uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3) { + uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, + uint32_t padded_n) { VK_LOG_DEBUG("ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << (split_k_buffer.buffer != nullptr ? split_k_buffer.buffer->buffer : VK_NULL_HANDLE) << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ")"); ggml_vk_sync_buffers(subctx); if (split_k == 1) { - const vk_mat_mat_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, k, ne02, ne12, broadcast2, broadcast3 }; + const vk_mat_mat_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, k, ne02, ne12, broadcast2, broadcast3, padded_n }; ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d }, sizeof(vk_mat_mat_push_constants), &pc, { m, n, batch }); return; } GGML_ASSERT(batch_stride_d == m * n); - const vk_mat_mat_push_constants pc1 = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, CEIL_DIV(k, split_k), ne02, ne12, broadcast2, broadcast3 }; + const vk_mat_mat_push_constants pc1 = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, CEIL_DIV(k, split_k), ne02, ne12, broadcast2, broadcast3, padded_n }; // Make sure enough workgroups get assigned for split k to work ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, sizeof(vk_mat_mat_push_constants), &pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, batch }); ggml_vk_sync_buffers(subctx); @@ -3937,14 +3941,15 @@ static void ggml_vk_matmul_id( vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& ids, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, - uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11) { + uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11, + uint32_t padded_n) { VK_LOG_DEBUG("ggml_vk_matmul_id(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), ids: (" << ids.buffer->buffer << ", " << ids.offset << ", " << ids.size << "), " << "m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", " << "batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", " << "n_as: " << n_as << ", nei0: " << nei0 << ", nei1: " << nei1 << ", nbi1: " << nbi1 << ", ne11: " << ne11 << ")"); ggml_vk_sync_buffers(subctx); const vk_mat_mat_id_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, - nei0, nei1, nbi1, ne11 }; + nei0, nei1, nbi1, ne11, padded_n }; ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids }, sizeof(vk_mat_mat_id_push_constants), &pc, { m, nei1, n_as }); } @@ -4106,15 +4111,17 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub // Not implemented GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT - const int x_ne = ne01 * ne00; - const int y_ne = ne11 * ne10; - const int d_ne = ne11 * ne01; - const uint32_t kpad = ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ctx, mmp, ne01, ne11, qx_needs_dequant ? GGML_TYPE_F16 : src0->type)); const bool aligned = ne10 == kpad && ne01 > 8 && ne11 > 8; vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline(ctx, mmp, ne01, ne11, aligned, qx_needs_dequant ? GGML_TYPE_F16 : src0->type); + // Reserve extra storage in the N dimension for the Y matrix, so we can avoid bounds-checking + uint32_t padded_n = qy_needs_dequant ? ROUNDUP_POW2(ne11, pipeline->wg_denoms[1]) :ne11; + const int x_ne = ne01 * ne00; + const int y_ne = padded_n * ne10; + const int d_ne = ne11 * ne01; + const uint32_t split_k = ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, pipeline); const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type); @@ -4237,7 +4244,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub { d_D, d_buf_offset, d_sz * ne12 * ne13 }, { ctx->prealloc_split_k, 0, d_sz * ne12 * ne13 * split_k }, ne01, ne11, ne10, ne10, ne10, ne01, stride_batch_x, stride_batch_y, ne20*ne21, - split_k, ne12*ne13, ne02, ne12, r2, r3 + split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n ); // NOLINT } @@ -4688,15 +4695,17 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& // Not implemented GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT - const uint64_t x_ne = ne01 * ne00; - const uint64_t y_ne = ne11 * ne10; - const uint64_t d_ne = ne21 * ne20; - const uint32_t kpad = ggml_vk_align_size(ne10, ggml_vk_guess_matmul_id_pipeline_align(ctx, mmp, ne01, nei1, qx_needs_dequant ? GGML_TYPE_F16 : src0->type)); const bool aligned = ne10 == kpad && ne01 > 8 && nei1 > 8; vk_pipeline pipeline = ggml_vk_guess_matmul_id_pipeline(ctx, mmp, ne01, nei1, aligned, qx_needs_dequant ? GGML_TYPE_F16 : src0->type); + // Reserve extra storage in the N dimension for the Y matrix, so we can avoid bounds-checking + uint32_t padded_n = qy_needs_dequant ? ROUNDUP_POW2(ne11, pipeline->wg_denoms[1]) :ne11; + const uint64_t x_ne = ne01 * ne00; + const uint64_t y_ne = padded_n * ne10; + const uint64_t d_ne = ne21 * ne20; + const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type); const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne; @@ -4815,7 +4824,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& { d_D, d_buf_offset, d_sz * ne22 * ne23 }, { d_ids, ids_buf_offset, ids_sz }, ne01, ne21, ne10, ne10, ne10, ne01, stride_batch_x, stride_batch_y, ne20*ne21, - n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11 + n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, padded_n ); // NOLINT } @@ -6775,7 +6784,7 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t ctx, subctx, p, ggml_vk_subbuffer(d_X), ggml_vk_subbuffer(d_Y), ggml_vk_subbuffer(d_D), ggml_vk_subbuffer(ctx->prealloc_split_k), m, n, k, k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1 + split_k, batch, batch, batch, 1, 1, n ); } ggml_vk_ctx_end(subctx); @@ -7120,7 +7129,7 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, ctx, subctx, p, ggml_vk_subbuffer(qx_buf), ggml_vk_subbuffer(y_buf), ggml_vk_subbuffer(d_buf), ggml_vk_subbuffer(ctx->prealloc_split_k), m, n, k, k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1 + split_k, batch, batch, batch, 1, 1, n ); } ggml_vk_ctx_end(subctx); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 66dd2c860d..5b7a4efe2c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -48,6 +48,8 @@ layout (push_constant) uniform parameter uint broadcast2; uint broadcast3; #endif + // N dimension for the B matrix can be >= p.N + uint padded_N; } p; @@ -202,18 +204,19 @@ void main() { #endif // Use end_k rather than p.K as the dimension because that's what - // we need to bound check against when using split_k + // we need to bound check against when using split_k. + // Bounds check B against padded_N, but bounds check D against N. tensorLayoutA = setTensorLayoutDimensionNV(tensorLayoutA, p.M, end_k); - tensorLayoutB = setTensorLayoutDimensionNV(tensorLayoutB, p.N, end_k); + tensorLayoutB = setTensorLayoutDimensionNV(tensorLayoutB, p.padded_N, end_k); tensorLayoutD = setTensorLayoutDimensionNV(tensorLayoutD, p.N, p.M); tensorLayoutAClamp = setTensorLayoutDimensionNV(tensorLayoutAClamp, p.M, end_k); - tensorLayoutBClamp = setTensorLayoutDimensionNV(tensorLayoutBClamp, p.N, end_k); + tensorLayoutBClamp = setTensorLayoutDimensionNV(tensorLayoutBClamp, p.padded_N, end_k); tensorViewNV<2, false, 1, 0> tensorViewTranspose = createTensorViewNV(2, false, 1, 0); #if !defined(MUL_MAT_ID) // Detect a fast path where all loads are entirely in bounds and no clamping is required - if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.N && (start_k % BK) == 0 && (end_k % BK) == 0 && + if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % BK) == 0 && (end_k % BK) == 0 && #if QUANT_K == 1 (stride_a % 8) == 0 && #endif @@ -263,7 +266,7 @@ void main() { #ifdef MUL_MAT_ID bool unclampedB = true; #else - bool unclampedB = (ic + 1) * BN <= p.N && block_k + BK <= end_k && (block_k % 8) == 0; + bool unclampedB = (ic + 1) * BN <= p.padded_N && block_k + BK <= end_k && (block_k % 8) == 0; #endif if (unclampedA && unclampedB) { coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, (block_k & ~7), BK) DECODEFUNCA); From f07690c930f74d82d4f108e567c7092544847f77 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Mon, 17 Mar 2025 04:43:35 -0500 Subject: [PATCH 148/188] vulkan: use fp32 in coopmat2 q4_k dequant function (#12309) --- .../ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp index 4ccbe613af..8efe4653ff 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp @@ -178,7 +178,7 @@ float16_t dequantFuncQ4_K(const in decodeBufQ4_K bl, const in uint blockCoords[2 uvec4 v = bl128.block.q4k[0]; - const f16vec2 loadd = unpackFloat2x16(v.x); + const vec2 loadd = vec2(unpackFloat2x16(v.x)); uint32_t sc; uint32_t mbyte; @@ -199,15 +199,15 @@ float16_t dequantFuncQ4_K(const in decodeBufQ4_K bl, const in uint blockCoords[2 sc &= 0x3F; mbyte &= 0x3F; - const float16_t d = loadd.x * float16_t(sc); - const float16_t m = loadd.y * float16_t(mbyte); + const float d = loadd.x * float(sc); + const float m = loadd.y * float(mbyte); uint qs = uint32_t(bl16.block.qs[((idx & 0xC0) >> 2) + ((idx & 0x1E) >> 1)]); qs = (qs >> (b * 4 + 8 * (idx & 1))) & 0xF; - float16_t ret = d * float16_t(qs) - m; + float ret = d * float(qs) - m; - return ret; + return float16_t(ret); } layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K { From cf2270e4d3685ac46f4a166d8718997ba7cbc45a Mon Sep 17 00:00:00 2001 From: Daniele Date: Mon, 17 Mar 2025 12:42:33 +0100 Subject: [PATCH 149/188] vulkan: subgroup size tuning (#12087) * vulkan: subgroup size test * Vulkan: Add device architecture enum and logic to recognize AMD generations * vulkan: use new architecture logic to specify subgroup size * Initial vulkan subgroup size tuning for RDNA3 * vulkan: commonize RDNA subgroup tuning * vulkan: override subgroup size if required_subgroup_size = 0 * vulkan: disable warp 32 for RDNA3 * vulkan: fine tuned RDNA1 subgroup sizes * vulkan: adjusted subgroup size map * vulkan: fixed RDNA2 subgroup map --------- Co-authored-by: 0cc4m --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 155 +++++++++++++++++++++++++-- 1 file changed, 145 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index a837b0dda4..aa7281acbf 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -150,6 +150,66 @@ static void ggml_vk_destroy_buffer(vk_buffer& buf); static constexpr uint32_t mul_mat_vec_max_cols = 8; +enum vk_device_architecture { + OTHER, + AMD_GCN, + AMD_RDNA1, + AMD_RDNA2, + AMD_RDNA3, +}; + +static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& device) { + vk::PhysicalDeviceProperties props = device.getProperties(); + + if (props.vendorID == VK_VENDOR_ID_AMD) { + const std::vector ext_props = device.enumerateDeviceExtensionProperties(); + + bool amd_shader_core_properties = false; + bool integer_dot_product = false; + bool subgroup_size_control = false; + + for (const auto& properties : ext_props) { + if (strcmp("VK_AMD_shader_core_properties", properties.extensionName) == 0) { + amd_shader_core_properties = true; + } else if (strcmp("VK_KHR_shader_integer_dot_product", properties.extensionName) == 0) { + integer_dot_product = true; + } else if (strcmp("VK_EXT_subgroup_size_control", properties.extensionName) == 0) { + subgroup_size_control = true; + } + } + + if (!amd_shader_core_properties || !integer_dot_product || !subgroup_size_control) { + return vk_device_architecture::OTHER; + } + + vk::PhysicalDeviceProperties2 props2; + vk::PhysicalDeviceShaderCorePropertiesAMD shader_core_props_amd; + vk::PhysicalDeviceShaderIntegerDotProductPropertiesKHR integer_dot_props; + vk::PhysicalDeviceSubgroupSizeControlPropertiesEXT subgroup_size_control_props; + + props2.pNext = &shader_core_props_amd; + shader_core_props_amd.pNext = &integer_dot_props; + integer_dot_props.pNext = &subgroup_size_control_props; + + device.getProperties2(&props2); + + if (subgroup_size_control_props.maxSubgroupSize == 64 && subgroup_size_control_props.minSubgroupSize == 64) { + return vk_device_architecture::AMD_GCN; + } + if (subgroup_size_control_props.maxSubgroupSize == 64 && subgroup_size_control_props.minSubgroupSize == 32) { + // RDNA + if (shader_core_props_amd.wavefrontsPerSimd == 20) { + return vk_device_architecture::AMD_RDNA1; + } + if (integer_dot_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated) { + return vk_device_architecture::AMD_RDNA3; + } + return vk_device_architecture::AMD_RDNA2; + } + } + return vk_device_architecture::OTHER; +} + struct vk_device_struct { std::mutex mutex; @@ -162,6 +222,7 @@ struct vk_device_struct { bool pipeline_robustness; vk::Device device; uint32_t vendor_id; + vk_device_architecture architecture; vk_queue compute_queue; vk_queue transfer_queue; bool single_queue; @@ -1448,6 +1509,73 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec return supported; } +struct GpuPipelineConfig { + // GPU architecture identifier. + // Example: vk_device_architecture::AMD_GCN + vk_device_architecture arch; + + // Mapping of pipeline names to their specific subgroup sizes. + // Example: {"soft_max_f32", 64} + std::unordered_map pipelines; + + // Default subgroup size for this GPU. + // Defaults to 0 if not explicitly provided. + uint32_t default_subgroup_size = 0; +}; + +// Pipeline configuration for RDNA1 GPUs. +static const std::unordered_map rdna1_pipelines = { + {"soft_max", 64}, {"im2col", 64}, + {"argmax", 64}, {"mul_mat_vec", 64}, + {"mul_mat_vec_f16", 32}, {"mul_mat_vec_f32_f16", 32} +}; + +// Pipeline configuration for RDNA2 GPUs. +static const std::unordered_map rdna2_pipelines = { + {"soft_max", 64}, {"im2col", 64}, +}; + +static constexpr uint32_t RDNA_DEFAULT_SUBGROUP_SIZE = 32; + +// Define configurations for different GPUs. +static std::vector gpu_pipeline_configs = { + { + vk_device_architecture::AMD_RDNA1, + { + rdna1_pipelines, + }, + RDNA_DEFAULT_SUBGROUP_SIZE + }, + { + vk_device_architecture::AMD_RDNA2, + { + rdna2_pipelines, + }, + RDNA_DEFAULT_SUBGROUP_SIZE + }, +}; + +static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch) { + for (const auto &config : gpu_pipeline_configs) { + if (config.arch == arch) { + auto pipIt = config.pipelines.find(pipeline_name); + if (pipIt != config.pipelines.end()) { + return pipIt->second; + } + std::vector> sorted_pipelines(config.pipelines.begin(), config.pipelines.end()); + std::sort(sorted_pipelines.begin(), sorted_pipelines.end(), + [](const auto &a, const auto &b) { return a.first.size() > b.first.size(); }); + for (const auto &entry : sorted_pipelines) { + if (pipeline_name.find(entry.first) != std::string::npos) { + return entry.second; + } + } + return config.default_subgroup_size; + } + } + return 0; // If no matching configuration is found +} + static void ggml_vk_load_shaders(vk_device& device) { VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")"); @@ -1574,6 +1702,10 @@ static void ggml_vk_load_shaders(vk_device& device) { uint32_t parameter_count, uint32_t push_constant_size, std::array wg_denoms, const std::vector& specialization_constants, uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { + if (!require_full_subgroups && required_subgroup_size == 0) { + required_subgroup_size = get_subgroup_size(name, device->architecture); + } + if (!pipeline) { pipeline = std::make_shared(); pipeline->name = name; @@ -2250,7 +2382,7 @@ static void ggml_vk_load_shaders(vk_device& device) { device->need_compiles = false; } -static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props); +static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch); static vk_device ggml_vk_get_device(size_t idx) { VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")"); @@ -2279,6 +2411,8 @@ static vk_device ggml_vk_get_device(size_t idx) { device->physical_device = physical_devices[dev_num]; const std::vector ext_props = device->physical_device.enumerateDeviceExtensionProperties(); + device->architecture = get_device_architecture(device->physical_device); + const char* GGML_VK_PREFER_HOST_MEMORY = getenv("GGML_VK_PREFER_HOST_MEMORY"); device->prefer_host_memory = GGML_VK_PREFER_HOST_MEMORY != nullptr; @@ -2291,7 +2425,6 @@ static vk_device ggml_vk_get_device(size_t idx) { bool coopmat2_support = false; device->coopmat_support = false; - // Check if maintenance4 is supported for (const auto& properties : ext_props) { if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) { maintenance4_support = true; @@ -2404,7 +2537,7 @@ static vk_device ggml_vk_get_device(size_t idx) { device->fp16 = !force_disable_f16 && fp16_storage && fp16_compute; - if (!ggml_vk_khr_cooperative_matrix_support(device->properties, driver_props)) { + if (!ggml_vk_khr_cooperative_matrix_support(device->properties, driver_props, device->architecture)) { device->coopmat_support = false; } @@ -2782,7 +2915,10 @@ static void ggml_vk_print_gpu_info(size_t idx) { subgroup_props.pNext = &driver_props; physical_device.getProperties2(&props2); - const size_t subgroup_size = subgroup_props.subgroupSize; + vk_device_architecture arch = get_device_architecture(physical_device); + uint32_t default_subgroup_size = get_subgroup_size("", arch); + const size_t subgroup_size = (default_subgroup_size != 0) ? default_subgroup_size : subgroup_props.subgroupSize; + const bool uma = props2.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; bool fp16_storage = false; @@ -2808,7 +2944,9 @@ static void ggml_vk_print_gpu_info(size_t idx) { } } - if (!ggml_vk_khr_cooperative_matrix_support(props2.properties, driver_props)) { + const vk_device_architecture device_architecture = get_device_architecture(physical_device); + + if (!ggml_vk_khr_cooperative_matrix_support(props2.properties, driver_props, device_architecture)) { coopmat_support = false; } @@ -8843,7 +8981,7 @@ static bool ggml_vk_instance_portability_enumeration_ext_available(const std::ve UNUSED(instance_extensions); } -static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props) { +static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch) { switch (props.vendorID) { case VK_VENDOR_ID_INTEL: // Intel drivers don't support coopmat properly yet @@ -8851,10 +8989,7 @@ static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDevicePrope case VK_VENDOR_ID_AMD: if (driver_props.driverID == vk::DriverId::eAmdProprietary || driver_props.driverID == vk::DriverId::eAmdOpenSource) { // Workaround for AMD proprietary driver reporting support on all GPUs - const std::string name = props.deviceName; - return name.rfind("AMD Radeon RX 7", 0) == 0 || name.rfind("AMD Radeon(TM) RX 7", 0) == 0 || // RDNA 3 consumer GPUs - name.rfind("AMD Radeon PRO W7", 0) == 0 || name.rfind("AMD Radeon(TM) PRO W7", 0) == 0 || // RDNA 3 workstation GPUs - name.rfind("AMD Radeon 7", 0) == 0 || name.rfind("AMD Radeon(TM) 7", 0) == 0; // RDNA 3 APUs + return arch == vk_device_architecture::AMD_RDNA3; } return true; default: From 484a8ab513bbd740cc49f30280c1acf52cb4e7e9 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Mon, 17 Mar 2025 09:26:18 -0500 Subject: [PATCH 150/188] vulkan: Add N/2 and N/4 optimized paths in coopmat2 shader (#12312) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 24 +++--- .../vulkan-shaders/mul_mm_cm2.comp | 79 ++++++++++++++----- 2 files changed, 72 insertions(+), 31 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index aa7281acbf..97398f071b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1597,33 +1597,33 @@ static void ggml_vk_load_shaders(vk_device& device) { uint32_t l_align, m_align, s_align; if (device->coopmat2) { // spec constants and tile sizes for non-quant matmul/matmul_id - l_warptile = { 256, 128, 256, 64 }; - m_warptile = { 256, 128, 128, 64 }; - s_warptile = { 128, 64, 64, 64 }; + l_warptile = { 256, 128, 256, 64, 1 }; + m_warptile = { 256, 128, 128, 64, 0 }; + s_warptile = { 128, 64, 64, 64, 0 }; l_wg_denoms = {128, 256, 1 }; m_wg_denoms = {128, 128, 1 }; s_wg_denoms = { 64, 64, 1 }; // spec constants and tile sizes for quant matmul (non-Qi_K) - l_warptile_mmq = { 256, 128, 256, 64 }; - m_warptile_mmq = { 256, 128, 128, 64 }; - s_warptile_mmq = { 256, 32, 64, 128 }; + l_warptile_mmq = { 256, 128, 256, 64, 1 }; + m_warptile_mmq = { 256, 128, 128, 64, 1 }; + s_warptile_mmq = { 256, 32, 64, 128, 0 }; l_mmq_wg_denoms = { 128, 256, 1 }; m_mmq_wg_denoms = { 128, 128, 1 }; s_mmq_wg_denoms = { 32, 64, 1 }; // spec constants and tile sizes for quant matmul (Qi_K) - l_warptile_mmq_k = { 256, 64, 128, 64 }; - m_warptile_mmq_k = { 256, 32, 64, 64 }; - s_warptile_mmq_k = { 256, 32, 32, 128 }; + l_warptile_mmq_k = { 256, 64, 128, 64, 1 }; + m_warptile_mmq_k = { 256, 32, 64, 64, 0 }; + s_warptile_mmq_k = { 256, 32, 32, 128, 0 }; l_mmq_wg_denoms_k = { 64, 128, 1 }; m_mmq_wg_denoms_k = { 32, 64, 1 }; s_mmq_wg_denoms_k = { 32, 32, 1 }; // spec constants and tile sizes for quant matmul_id - l_warptile_mmqid = { 256, 128, 64, 16 }; - m_warptile_mmqid = { 256, 128, 64, 16 }; - s_warptile_mmqid = { 256, 128, 64, 16 }; + l_warptile_mmqid = { 256, 128, 64, 16, 0 }; + m_warptile_mmqid = { 256, 128, 64, 16, 0 }; + s_warptile_mmqid = { 256, 128, 64, 16, 0 }; l_mmqid_wg_denoms = { 128, 64, 1 }; m_mmqid_wg_denoms = { 128, 64, 1 }; s_mmqid_wg_denoms = { 128, 64, 1 }; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 5b7a4efe2c..7649febb07 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -23,6 +23,10 @@ layout (constant_id = 1) const uint BM = 64; layout (constant_id = 2) const uint BN = 64; layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working with a quant +layout (constant_id = 4) const bool enable_smaller_matrices = false; +const uint BNover2 = enable_smaller_matrices ? (BN / 2) : BN; +const uint BNover4 = enable_smaller_matrices ? (BN / 4) : BN; + layout (push_constant) uniform parameter { uint M; @@ -168,15 +172,13 @@ void main() { const uint end_k = min(p.K, (ik + 1) * p.k_split); #endif - coopmat sum; - sum = coopmat(0.0); - #ifdef MUL_MAT_ID uint pos_a = (expert_idx * p.batch_stride_a) / QUANT_K; uint pos_b = 0; #else uint pos_a = (batch_idx_a * p.batch_stride_a) / QUANT_K; uint pos_b = batch_idx * p.batch_stride_b; + uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * gl_NumWorkGroups.z; #endif uint stride_a = p.stride_a / QUANT_K; @@ -197,6 +199,7 @@ void main() { tensorLayoutNV<2> tensorLayoutB = createTensorLayoutNV(2); tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutBClamp = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV); tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutD = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV); + tensorLayoutD = setTensorLayoutStrideNV(tensorLayoutD, p.stride_d, 1); #if QUANT_K > 1 tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K); @@ -232,16 +235,54 @@ void main() { tensorLayoutB = setTensorLayoutStrideNV(tensorLayoutB, stride_b, 1); uint k_iters = (end_k - start_k + BK - 1) / BK; + if (enable_smaller_matrices && ic * BN + BNover4 >= p.N) { + coopmat sum = coopmat(0.0); + for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { + coopmat mat_a; + coopmat mat_b; - coopmat mat_a; - coopmat mat_b; + coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); + coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose); - coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); - coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose); + sum = coopMatMulAdd(mat_a, mat_b, sum); + } + coopmat mat_d = coopmat(sum); - sum = coopMatMulAdd(mat_a, mat_b, sum); + coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover4, ir * BM, BM), tensorViewTranspose); + return; + } else if (enable_smaller_matrices && ic * BN + BNover2 >= p.N) { + coopmat sum = coopmat(0.0); + for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { + + coopmat mat_a; + coopmat mat_b; + + coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); + coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose); + + sum = coopMatMulAdd(mat_a, mat_b, sum); + } + coopmat mat_d = coopmat(sum); + + coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover2, ir * BM, BM), tensorViewTranspose); + return; + } else { + coopmat sum = coopmat(0.0); + for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { + + coopmat mat_a; + coopmat mat_b; + + coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); + coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose); + + sum = coopMatMulAdd(mat_a, mat_b, sum); + } + coopmat mat_d = coopmat(sum); + + coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose); + return; } } else #endif // !defined(MUL_MAT_ID) @@ -254,6 +295,9 @@ void main() { tensorLayoutBClamp = setTensorLayoutStrideNV(tensorLayoutBClamp, stride_b, 1); + coopmat sum; + sum = coopmat(0.0); + [[dont_unroll]] for (uint block_k = start_k; block_k < end_k; block_k += BK) { @@ -296,19 +340,16 @@ void main() { sum = coopMatMulAdd(mat_a, mat_b, sum); } } - } - // Convert from ACC_TYPE to D_TYPE - coopmat mat_d; - mat_d = coopmat(sum); + // Convert from ACC_TYPE to D_TYPE + coopmat mat_d; + mat_d = coopmat(sum); #ifdef MUL_MAT_ID - // Call callback to store each element, remapping row through shared memory - coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic); + // Call callback to store each element, remapping row through shared memory + coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic); #else - tensorLayoutD = setTensorLayoutStrideNV(tensorLayoutD, p.stride_d, 1); - - uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * gl_NumWorkGroups.z; - coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose); + coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose); #endif + } } From 01e8f2138b2e40902afe2983ecbf503a08d74b1d Mon Sep 17 00:00:00 2001 From: Guus Waals <_@guusw.nl> Date: Tue, 18 Mar 2025 00:35:43 +0800 Subject: [PATCH 151/188] ggml-vulkan: remove unused find_program(glslc) (#12416) It's already found by FindVulkan.cmake in the parent CMakeLists --- ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt b/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt index 074031087f..51c78b7d22 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +++ b/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt @@ -1,8 +1,4 @@ find_package (Threads REQUIRED) -find_program(GLSLC_EXECUTABLE glslc) -if(NOT GLSLC_EXECUTABLE) - message(FATAL_ERROR "glslc not found.") -endif() set(TARGET vulkan-shaders-gen) add_executable(${TARGET} vulkan-shaders-gen.cpp) From b1b132efcba216c873715c483809730bb253f4a1 Mon Sep 17 00:00:00 2001 From: Gaurav Garg <52341457+gaugarg-nv@users.noreply.github.com> Date: Mon, 17 Mar 2025 23:55:13 +0530 Subject: [PATCH 152/188] cuda : enable CUDA Graph on CUDA Toolkit < 12.x (#12394) * Enable CUDA Graph on CTK < 12.x `cudaGraphExecUpdate` API was changed on 12.x. For this reason CUDA graph support was disabled on older CUDA toolkit. This change enables CUDA support in CTK version < 12.x by using older API if CTK < 12.x. * Fix compilation errors with MUSA * Disable CUDA Graph for MUSA --- ggml/src/ggml-cuda/common.cuh | 2 +- ggml/src/ggml-cuda/ggml-cuda.cu | 12 +++++++----- ggml/src/ggml-cuda/vendors/hip.h | 2 +- ggml/src/ggml-cuda/vendors/musa.h | 3 ++- ggml/src/ggml-musa/CMakeLists.txt | 4 ---- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 4d4ac47c03..e78205e5d5 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -678,7 +678,7 @@ struct ggml_tensor_extra_gpu { }; -#if ((CUDART_VERSION >= 12000) && defined(GGML_CUDA_USE_GRAPHS)) || defined(GGML_HIP_GRAPHS) +#if (defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)) #define USE_CUDA_GRAPH #endif diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 497de37be8..9bba398ce6 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2610,13 +2610,15 @@ static bool is_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx, static void update_cuda_graph_executable(ggml_backend_cuda_context * cuda_ctx) { +#if CUDART_VERSION >= 12000 cudaGraphExecUpdateResultInfo result_info; -#ifdef __HIP_PLATFORM_AMD__ - hipGraphNode_t errorNode; - hipError_t stat = hipGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &errorNode, &result_info); -#else cudaError_t stat = cudaGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &result_info); -#endif +#else + cudaGraphNode_t errorNode; + cudaGraphExecUpdateResult result_info; + cudaError_t stat = cudaGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &errorNode, &result_info); +#endif // CUDART_VERSION >= 12000 + if (stat == cudaErrorGraphExecUpdateFailure) { #ifndef NDEBUG GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__); diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index 81964611c6..aace21e3a8 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -112,7 +112,7 @@ #define cudaGraphExecDestroy hipGraphExecDestroy #define cudaGraphLaunch hipGraphLaunch #define cudaErrorGraphExecUpdateFailure hipErrorGraphExecUpdateFailure -#define cudaGraphExecUpdateResultInfo hipGraphExecUpdateResult +#define cudaGraphExecUpdateResult hipGraphExecUpdateResult #define cudaGraphNodeType hipGraphNodeType #define cudaGraphNodeTypeKernel hipGraphNodeTypeKernel #define cudaGraphInstantiate hipGraphInstantiate diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 6cc1b69ee3..997f671431 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -119,7 +119,7 @@ #define cudaGraphExecDestroy musaGraphExecDestroy #define cudaGraphExec_t musaGraphExec_t #define cudaGraphExecUpdate musaGraphExecUpdate -#define cudaGraphExecUpdateResultInfo musaGraphExecUpdateResult +#define cudaGraphExecUpdateResult musaGraphExecUpdateResult #define cudaGraphGetNodes musaGraphGetNodes #define cudaGraphInstantiate musaGraphInstantiate #define cudaGraphKernelNodeGetParams musaGraphKernelNodeGetParams @@ -132,6 +132,7 @@ #define cudaGraph_t musaGraph_t #define cudaKernelNodeParams musaKernelNodeParams #define cudaStreamCaptureModeRelaxed musaStreamCaptureModeRelaxed +#define cudaStreamBeginCapture musaStreamBeginCapture #define cudaStreamEndCapture musaStreamEndCapture typedef mt_bfloat16 nv_bfloat16; diff --git a/ggml/src/ggml-musa/CMakeLists.txt b/ggml/src/ggml-musa/CMakeLists.txt index 166970ca6b..92f05d5558 100644 --- a/ggml/src/ggml-musa/CMakeLists.txt +++ b/ggml/src/ggml-musa/CMakeLists.txt @@ -67,10 +67,6 @@ if (MUSAToolkit_FOUND) add_compile_definitions(GGML_USE_MUSA) add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE}) - if (GGML_CUDA_GRAPHS) - add_compile_definitions(GGML_CUDA_USE_GRAPHS) - endif() - if (GGML_CUDA_FORCE_MMQ) add_compile_definitions(GGML_CUDA_FORCE_MMQ) endif() From 60c902926c928f9c2cd6390ce411876f92feeaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Mon, 17 Mar 2025 21:14:32 +0100 Subject: [PATCH 153/188] docs : bring llama-cli conversation/template docs up-to-date (#12426) --- examples/main/README.md | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/examples/main/README.md b/examples/main/README.md index f7c2497294..e4b3590b5d 100644 --- a/examples/main/README.md +++ b/examples/main/README.md @@ -27,12 +27,24 @@ Once downloaded, place your model in the models folder in llama.cpp. ##### Input prompt (One-and-done) ```bash -./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf --prompt "Once upon a time" +./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf -no-cnv --prompt "Once upon a time" ``` ##### Conversation mode (Allow for continuous interaction with the model) ```bash -./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf -cnv --chat-template gemma +./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf --chat-template gemma +``` + +##### Conversation mode using built-in jinja chat template + +```bash +./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf --jinja +``` + +##### One-and-done query using jinja with custom system prompt and a starting prompt + +```bash +./llama-cli -m models/gemma-1.1-7b-it.Q4_K_M.gguf --jinja --single-turn -sys "You are a helpful assistant" -p "Hello" ``` ##### Infinite text from a starting prompt (you can use `Ctrl-C` to stop it): @@ -44,12 +56,24 @@ Once downloaded, place your model in the models folder in llama.cpp. ##### Input prompt (One-and-done) ```powershell -./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --prompt "Once upon a time" +./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf -no-cnv --prompt "Once upon a time" ``` ##### Conversation mode (Allow for continuous interaction with the model) ```powershell -./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf -cnv --chat-template gemma +./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --chat-template gemma +``` + +##### Conversation mode using built-in jinja chat template + +```powershell +./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --jinja +``` + +##### One-and-done query using jinja with custom system prompt and a starting prompt + +```powershell +./llama-cli.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --jinja --single-turn -sys "You are a helpful assistant" -p "Hello" ``` #### Infinite text from a starting prompt (you can use `Ctrl-C` to stop it): @@ -77,6 +101,8 @@ The `llama-cli` program provides several ways to interact with the LLaMA models - `--prompt PROMPT`: Provide a prompt directly as a command-line option. - `--file FNAME`: Provide a file containing a prompt or multiple prompts. +- `--system-prompt PROMPT`: Provide a system prompt (will otherwise use the default one in the chat template (if provided)). +- `--system-prompt-file FNAME`: Provide a file containing a system prompt. - `--interactive-first`: Run the program in interactive mode and wait for input right away. (More on this below.) ## Interaction @@ -89,7 +115,10 @@ In interactive mode, users can participate in text generation by injecting their - `-i, --interactive`: Run the program in interactive mode, allowing users to engage in real-time conversations or provide specific instructions to the model. - `--interactive-first`: Run the program in interactive mode and immediately wait for user input before starting the text generation. -- `-cnv, --conversation`: Run the program in conversation mode (does not print special tokens and suffix/prefix, use default chat template) (default: false) +- `-cnv, --conversation`: Run the program in conversation mode (does not print special tokens and suffix/prefix, use default or provided chat template) (default: true if chat template found) +- `-no-cnv`: Disable conversation mode (default: false) +- `-st, --single-turn`: Only process a single conversation turn (user input) and then exit. +- `--jinja`: Enable jinja chat template parser, will use the model's built-in template or a user-provided one (default: false) - `--color`: Enable colorized output to differentiate visually distinguishing between prompts, user input, and generated text. By understanding and utilizing these interaction options, you can create engaging and dynamic experiences with the LLaMA models, tailoring the text generation process to your specific needs. @@ -125,6 +154,8 @@ When --in-prefix or --in-suffix options are enabled the chat template ( --chat-t Example usage: `--chat-template gemma` +`--chat-template-file FNAME`: Load a custom jinja chat template from an external file, useful if the model contains outdated or incompatible template, some examples can be found in models/templates. Up-to-date chat templates can be downloaded from Hugging Face using scripts/get_chat_template.py + ## Context Management During text generation, LLaMA models have a limited context size, which means they can only consider a certain number of tokens from the input and generated text. When the context fills up, the model resets internally, potentially losing some information from the beginning of the conversation or instructions. Context management options help maintain continuity and coherence in these situations. From 7dfad387e3f6ac98d383ded2d175eb59736a3993 Mon Sep 17 00:00:00 2001 From: Molly Sophia Date: Tue, 18 Mar 2025 07:27:50 +0800 Subject: [PATCH 154/188] llama: Add support for RWKV v7 architecture (#12412) * ggml: Add op l2_norm Signed-off-by: Molly Sophia * ggml: Add op rwkv_wkv7 Signed-off-by: Molly Sophia * llama: Add support for RWKV7 and ARWKV7 models Signed-off-by: Molly Sophia * llama: fix inference with RWKV6Qwen2 Signed-off-by: Molly Sophia * llama: add more (a)rwkv7 variants in size Signed-off-by: Molly Sophia * Apply code-format changes Signed-off-by: Molly Sophia * fix MUSA build Signed-off-by: Molly Sophia * llama: fix shape error with rwkv using llama-parallel Signed-off-by: Molly Sophia --------- Signed-off-by: Molly Sophia --- convert_hf_to_gguf.py | 229 ++++++- ggml/include/ggml.h | 24 + ggml/src/ggml-cpu/ggml-cpu.c | 255 +++++++- ggml/src/ggml-cuda/ggml-cuda.cu | 10 +- ggml/src/ggml-cuda/norm.cu | 116 ++++ ggml/src/ggml-cuda/norm.cuh | 2 + ggml/src/ggml-cuda/wkv.cu | 199 ++++++ ggml/src/ggml-cuda/{wkv6.cuh => wkv.cuh} | 2 + ggml/src/ggml-cuda/wkv6.cu | 89 --- ggml/src/ggml-metal/ggml-metal-impl.h | 7 + ggml/src/ggml-metal/ggml-metal.m | 122 ++++ ggml/src/ggml-metal/ggml-metal.metal | 221 +++++++ ggml/src/ggml-sycl/backend.hpp | 2 +- ggml/src/ggml-sycl/ggml-sycl.cpp | 14 + ggml/src/ggml-sycl/norm.cpp | 108 ++++ ggml/src/ggml-sycl/norm.hpp | 6 + ggml/src/ggml-sycl/wkv.cpp | 305 +++++++++ ggml/src/ggml-sycl/wkv.hpp | 10 + ggml/src/ggml-sycl/wkv6.cpp | 143 ----- ggml/src/ggml-sycl/wkv6.hpp | 9 - ggml/src/ggml-vulkan/ggml-vulkan.cpp | 206 ++++--- .../ggml-vulkan/vulkan-shaders/l2_norm.comp | 41 ++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 3 + ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp | 91 +++ ggml/src/ggml.c | 87 ++- gguf-py/gguf/constants.py | 126 +++- gguf-py/gguf/gguf_writer.py | 12 + gguf-py/gguf/tensor_mapping.py | 131 +++- src/llama-arch.cpp | 118 +++- src/llama-arch.h | 18 + src/llama-hparams.h | 4 + src/llama-model.cpp | 581 +++++++++++++++++- src/llama-model.h | 16 + src/llama-quant.cpp | 11 +- tests/test-backend-ops.cpp | 68 ++ 35 files changed, 2948 insertions(+), 438 deletions(-) create mode 100644 ggml/src/ggml-cuda/wkv.cu rename ggml/src/ggml-cuda/{wkv6.cuh => wkv.cuh} (62%) delete mode 100644 ggml/src/ggml-cuda/wkv6.cu create mode 100644 ggml/src/ggml-sycl/wkv.cpp create mode 100644 ggml/src/ggml-sycl/wkv.hpp delete mode 100644 ggml/src/ggml-sycl/wkv6.cpp delete mode 100644 ggml/src/ggml-sycl/wkv6.hpp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index b5d95bd563..d13d57c541 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -908,6 +908,40 @@ class Model: special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) special_vocab.add_to_gguf(self.gguf_writer) + def _set_vocab_rwkv_world(self): + assert (self.dir_model / "rwkv_vocab_v20230424.txt").is_file() + vocab_size = self.hparams.get("vocab_size", 65536) + + tokens: list[bytes] = [''.encode("utf-8")] + toktypes: list[int] = [gguf.TokenType.CONTROL] + + with open(self.dir_model / "rwkv_vocab_v20230424.txt", "r", encoding="utf-8") as f: + lines = f.readlines() + for line in lines: + parts = line.split(' ') + assert len(parts) >= 3 + token, token_len = ast.literal_eval(' '.join(parts[1:-1])), int(parts[-1]) + token = token.encode("utf-8") if isinstance(token, str) else token + assert isinstance(token, bytes) + assert len(token) == token_len + token_text: str = repr(token)[2:-1] # "b'\xff'" -> "\xff" + tokens.append(token_text.encode("utf-8")) + toktypes.append(gguf.TokenType.NORMAL) + remainder = vocab_size - len(tokens) + assert remainder >= 0 + for i in range(len(tokens), vocab_size): + tokens.append(f"[PAD{i}]".encode("utf-8")) + toktypes.append(gguf.TokenType.UNUSED) + + self.gguf_writer.add_tokenizer_model("rwkv") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False) + special_vocab.chat_template = "rwkv-world" + # hack: Add '\n\n' as the EOT token to make it chat normally + special_vocab._set_special_token("eot", 261) + special_vocab.add_to_gguf(self.gguf_writer) + def _set_vocab_builtin(self, model_name: Literal["gpt-neox", "llama-spm"], vocab_size: int): tokenizer_path = Path(sys.path[0]) / "models" / f"ggml-vocab-{model_name}.gguf" logger.warning(f"Using tokenizer from '{os.path.relpath(tokenizer_path, os.getcwd())}'") @@ -3412,38 +3446,7 @@ class Rwkv6Model(Model): model_arch = gguf.MODEL_ARCH.RWKV6 def set_vocab(self): - assert (self.dir_model / "rwkv_vocab_v20230424.txt").is_file() - vocab_size = self.hparams.get("vocab_size", 65536) - - tokens: list[bytes] = [''.encode("utf-8")] - toktypes: list[int] = [gguf.TokenType.CONTROL] - - with open(self.dir_model / "rwkv_vocab_v20230424.txt", "r", encoding="utf-8") as f: - lines = f.readlines() - for line in lines: - parts = line.split(' ') - assert len(parts) >= 3 - token, token_len = ast.literal_eval(' '.join(parts[1:-1])), int(parts[-1]) - token = token.encode("utf-8") if isinstance(token, str) else token - assert isinstance(token, bytes) - assert len(token) == token_len - token_text: str = repr(token)[2:-1] # "b'\xff'" -> "\xff" - tokens.append(token_text.encode("utf-8")) - toktypes.append(gguf.TokenType.NORMAL) - remainder = vocab_size - len(tokens) - assert remainder >= 0 - for i in range(len(tokens), vocab_size): - tokens.append(f"[PAD{i}]".encode("utf-8")) - toktypes.append(gguf.TokenType.UNUSED) - - self.gguf_writer.add_tokenizer_model("rwkv") - self.gguf_writer.add_token_list(tokens) - self.gguf_writer.add_token_types(toktypes) - special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False) - special_vocab.chat_template = "rwkv-world" - # hack: Add '\n\n' as the EOT token to make it chat normally - special_vocab._set_special_token("eot", 261) - special_vocab.add_to_gguf(self.gguf_writer) + self._set_vocab_rwkv_world() def set_gguf_parameters(self): block_count = self.hparams["num_hidden_layers"] @@ -3565,6 +3568,168 @@ class RWKV6Qwen2Model(Rwkv6Model): yield (new_name, data) +@Model.register("Rwkv7ForCausalLM", "RWKV7ForCausalLM") +class Rwkv7Model(Model): + model_arch = gguf.MODEL_ARCH.RWKV7 + + def set_vocab(self): + self._set_vocab_rwkv_world() + + def calc_lora_rank(self, hidden_size, exponent, multiplier): + return max(1, round(hidden_size ** exponent * multiplier / 32)) * 32 + + def set_gguf_parameters(self): + block_count = self.hparams["num_hidden_layers"] + try: + head_size = self.hparams["head_size"] + layer_norm_eps = self.hparams["layer_norm_epsilon"] + except KeyError: + head_size = self.hparams["head_dim"] + layer_norm_eps = self.hparams["norm_eps"] + hidden_size = self.hparams["hidden_size"] + intermediate_size = self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else (hidden_size * 4) + + # ICLR: In-Context-Learning-Rate + try: + lora_rank_decay = self.hparams["lora_rank_decay"] if self.hparams["lora_rank_decay"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8) + lora_rank_iclr = self.hparams["lora_rank_iclr"] if self.hparams["lora_rank_iclr"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8) + lora_rank_value_residual_mix = self.hparams["lora_rank_value_residual_mix"] if self.hparams["lora_rank_value_residual_mix"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3) + lora_rank_gate = self.hparams["lora_rank_gate"] if self.hparams["lora_rank_gate"] is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6) + except KeyError: + lora_rank_decay = self.hparams["decay_low_rank_dim"] if self.hparams["decay_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8) + lora_rank_iclr = self.hparams["a_low_rank_dim"] if self.hparams["a_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8) + lora_rank_value_residual_mix = self.hparams["v_low_rank_dim"] if self.hparams["v_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3) + lora_rank_gate = self.hparams["gate_low_rank_dim"] if self.hparams["gate_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6) + + # RWKV isn't context limited + self.gguf_writer.add_context_length(1048576) + self.gguf_writer.add_embedding_length(hidden_size) + self.gguf_writer.add_block_count(block_count) + self.gguf_writer.add_layer_norm_eps(layer_norm_eps) + self.gguf_writer.add_wkv_head_size(head_size) + self.gguf_writer.add_decay_lora_rank(lora_rank_decay) + self.gguf_writer.add_iclr_lora_rank(lora_rank_iclr) + self.gguf_writer.add_value_residual_mix_lora_rank(lora_rank_value_residual_mix) + self.gguf_writer.add_gate_lora_rank(lora_rank_gate) + self.gguf_writer.add_feed_forward_length(intermediate_size) + self.gguf_writer.add_file_type(self.ftype) + + # required by llama.cpp, unused + self.gguf_writer.add_head_count(0) + + lerp_weights: dict[int, dict[str, Tensor]] = {} + lora_needs_transpose: bool = True + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # unify tensor names here to make life easier + name = name.replace("blocks", "layers").replace("ffn", "feed_forward") + name = name.replace("self_attn", "attention").replace("attn", "attention") + name = name.replace("time_mixer.", "") + # lora layer names in fla-hub's impl + if "_lora.lora" in name: + self.lora_needs_transpose = False + name = name.replace("_lora.lora.0.weight", "1.weight") + name = name.replace("_lora.lora.2.weight", "2.weight") + name = name.replace("_lora.lora.2.bias", "0.weight") + + name = name.replace("feed_forward_norm", "ln2") + name = name.replace("g_norm", "ln_x") + + if "attention.v" in name and "value" not in self.map_tensor_name(name) and bid == 0: + # some models have dummy v0/v1/v2 on first layer while others don't + # ignore them all since they are not used + return + + wkv_has_gate = self.hparams.get("wkv_has_gate", True) + lerp_list = ["r", "w", "k", "v", "a", "g"] if wkv_has_gate else ["r", "w", "k", "v", "a"] + + if bid is not None and "attention.x_" in name: + if "attention.x_x" in name: + # already concatenated + new_name = f"blk.{bid}.time_mix_lerp_fused.weight" + data = data_torch.reshape(len(lerp_list), 1, 1, -1) + yield (new_name, data) + else: + try: + self.lerp_weights[bid][name] = data_torch + except KeyError: + self.lerp_weights[bid] = {name: data_torch} + if all(f"model.layers.{bid}.attention.x_{i}" in self.lerp_weights[bid].keys() for i in lerp_list): + new_name = f"blk.{bid}.time_mix_lerp_fused.weight" + data = torch.stack([self.lerp_weights[bid][f"model.layers.{bid}.attention.x_{i}"] for i in lerp_list], dim=0) + yield (new_name, data) + return + else: + data_torch = data_torch.squeeze() + new_name = self.map_tensor_name(name) + + if not (new_name.endswith(".weight") or new_name.endswith(".bias")): + new_name += ".weight" + + if self.lora_needs_transpose and any( + new_name.endswith(t) for t in [ + "time_mix_w1.weight", "time_mix_w2.weight", + "time_mix_a1.weight", "time_mix_a2.weight", + "time_mix_v1.weight", "time_mix_v2.weight", + "time_mix_g1.weight", "time_mix_g2.weight", + ] + ): + data_torch = data_torch.transpose(0, 1) + + if 'r_k' in new_name: + data_torch = data_torch.flatten() + + if bid == 0 and "time_mix_a" in new_name: + # dummy v0/v1/v2 on first layer + # easist way to make llama happy + yield (new_name.replace("time_mix_a", "time_mix_v"), data_torch) + + yield (new_name, data_torch) + + +@Model.register("RwkvHybridForCausalLM") +class ARwkv7Model(Rwkv7Model): + model_arch = gguf.MODEL_ARCH.ARWKV7 + + def set_vocab(self): + try: + self._set_vocab_sentencepiece() + except FileNotFoundError: + self._set_vocab_gpt2() + + def set_gguf_parameters(self): + block_count = self.hparams["num_hidden_layers"] + hidden_size = self.hparams["hidden_size"] + head_size = self.hparams["head_size"] + rms_norm_eps = self.hparams["rms_norm_eps"] + intermediate_size = self.hparams["intermediate_size"] + wkv_has_gate = self.hparams["wkv_has_gate"] + assert self.hparams["wkv_version"] == 7 + + # ICLR: In-Context-Learning-Rate + lora_rank_decay = 64 + lora_rank_iclr = 64 + lora_rank_value_residual_mix = 32 + lora_rank_gate = 128 if wkv_has_gate else 0 + + # RWKV isn't context limited + self.gguf_writer.add_context_length(1048576) + self.gguf_writer.add_embedding_length(hidden_size) + self.gguf_writer.add_block_count(block_count) + self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps) + self.gguf_writer.add_wkv_head_size(head_size) + self.gguf_writer.add_decay_lora_rank(lora_rank_decay) + self.gguf_writer.add_iclr_lora_rank(lora_rank_iclr) + self.gguf_writer.add_value_residual_mix_lora_rank(lora_rank_value_residual_mix) + self.gguf_writer.add_gate_lora_rank(lora_rank_gate) + self.gguf_writer.add_feed_forward_length(intermediate_size) + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_token_shift_count(1) + + # required by llama.cpp, unused + self.gguf_writer.add_head_count(0) + + @Model.register("MambaForCausalLM", "MambaLMHeadModel", "FalconMambaForCausalLM") class MambaModel(Model): model_arch = gguf.MODEL_ARCH.MAMBA diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 2e5076d36a..cb3edb10d4 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -454,6 +454,7 @@ extern "C" { GGML_OP_RMS_NORM, GGML_OP_RMS_NORM_BACK, GGML_OP_GROUP_NORM, + GGML_OP_L2_NORM, GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID, @@ -502,6 +503,7 @@ extern "C" { GGML_OP_ADD_REL_POS, GGML_OP_RWKV_WKV6, GGML_OP_GATED_LINEAR_ATTN, + GGML_OP_RWKV_WKV7, GGML_OP_UNARY, @@ -1095,6 +1097,18 @@ extern "C" { int n_groups, float eps); + // l2 normalize along rows + // used in rwkv v7 + GGML_API struct ggml_tensor * ggml_l2_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_l2_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + // a - x // b - dy GGML_API struct ggml_tensor * ggml_rms_norm_back( @@ -1890,6 +1904,16 @@ extern "C" { struct ggml_tensor * state, float scale); + GGML_API struct ggml_tensor * ggml_rwkv_wkv7( + struct ggml_context * ctx, + struct ggml_tensor * r, + struct ggml_tensor * w, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * state); + // custom operators typedef void (*ggml_unary_op_f32_t) (const int, float *, const float *); diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index f2ab4c5d69..75dc96b478 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -8548,6 +8548,69 @@ static void ggml_compute_forward_group_norm( } } +// ggml_compute_forward_l2_norm + +static void ggml_compute_forward_l2_norm_f32( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + GGML_ASSERT(ggml_are_same_shape(src0, dst)); + + GGML_ASSERT(src0->nb[0] == sizeof(float)); + + const int ith = params->ith; + const int nth = params->nth; + + GGML_TENSOR_UNARY_OP_LOCALS + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + + GGML_ASSERT(eps >= 0.0f); + + // TODO: optimize + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + for (int64_t i01 = ith; i01 < ne01; i01 += nth) { + const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03); + + ggml_float sum = 0.0; + for (int64_t i00 = 0; i00 < ne00; i00++) { + sum += (ggml_float)(x[i00] * x[i00]); + } + + float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); + + memcpy(y, x, ne00 * sizeof(float)); + + const float scale = 1.0f/fmaxf(sqrtf(sum), eps); + + ggml_vec_scale_f32(ne00, y, scale); + } + } + } +} + +static void ggml_compute_forward_l2_norm( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_l2_norm_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + // ggml_compute_forward_mul_mat static void ggml_compute_forward_mul_mat_one_chunk( @@ -13604,6 +13667,184 @@ static void ggml_compute_forward_gla( } } +// ggml_compute_forward_rwkv_wkv7 + +static void ggml_compute_forward_rwkv_wkv7_f32( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + const int64_t T = dst->src[1]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t HEADS = dst->src[1]->ne[1]; + const int64_t n_seqs = dst->src[6]->ne[1]; + const int64_t head_size = C / HEADS; + + float * dst_data = (float *) dst->data; + float * state = ((float *) dst->data) + C * T; + + const int ith = params->ith; + const int nth = params->nth; + + if (ith >= HEADS) { + return; + } + + const int h_start = (HEADS * ith) / nth; + const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ? + (HEADS * (ith + 1)) / nth : HEADS; + + float * r = (float *) dst->src[0]->data; + float * w = (float *) dst->src[1]->data; + float * k = (float *) dst->src[2]->data; + float * v = (float *) dst->src[3]->data; + float * a = (float *) dst->src[4]->data; + float * b = (float *) dst->src[5]->data; + + int64_t t_stride = HEADS * head_size; // Same to C + + int64_t h_stride = C / HEADS; + GGML_ASSERT(C % HEADS == 0); // C must be divisible by HEADS + int64_t h_stride_2d = head_size * head_size; + + #if defined(GGML_SIMD) + for (int64_t t = 0; t < T; t++) { + int64_t t_offset = t * t_stride; + int64_t state_offset = head_size * C * (t / (T / n_seqs)); + float * state_cur = state + state_offset; + float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset; + + for (int64_t h = h_start; h < h_end; h++) { + int64_t h_offset = h * h_stride; + int64_t t_h_offset = t_offset + h_offset; + int64_t h_2d_offset = h * h_stride_2d; + + for (int64_t ii = 0; ii < head_size; ii++) { + int64_t t_h_i_offset = t_h_offset + ii; + int64_t h_2d_i_offset = h_2d_offset + ii * h_stride; + + GGML_F32_VEC v_vec = GGML_F32_VEC_SET1(v[t_h_i_offset]); + + float sa = 0; + { + GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO }; + GGML_F32_VEC ax[GGML_F32_ARR]; + GGML_F32_VEC ay[GGML_F32_ARR]; + for (int64_t j = 0; j < head_size; j += GGML_F32_STEP) { + for (int64_t kk = 0; kk < GGML_F32_ARR; kk++) { + ax[kk] = GGML_F32_VEC_LOAD(&a[t_h_offset + j + kk * GGML_F32_EPR]); + ay[kk] = GGML_F32_VEC_LOAD(&state_prev[h_2d_i_offset + j + kk * GGML_F32_EPR]); + sum[kk] = GGML_F32_VEC_FMA(sum[kk], ax[kk], ay[kk]); + } + } + GGML_F32_VEC_REDUCE(sa, sum); + } + + GGML_F32_VEC sa_vec = GGML_F32_VEC_SET1(sa); + + int64_t j = 0; + GGML_F32_VEC result_vec[GGML_F32_ARR] = { GGML_F32_VEC_ZERO }; + for (; j < head_size; j += GGML_F32_STEP) { + for (int64_t kk = 0; kk < GGML_F32_ARR; kk++) { + int64_t t_h_j_offset = t_h_offset + j + kk * GGML_F32_EPR; + int64_t h_2d_i_j_offset = h_2d_i_offset + j + kk * GGML_F32_EPR; + + GGML_F32_VEC r_vec = GGML_F32_VEC_LOAD(&r[t_h_j_offset]); + GGML_F32_VEC w_vec = GGML_F32_VEC_LOAD(&w[t_h_j_offset]); + GGML_F32_VEC k_vec = GGML_F32_VEC_LOAD(&k[t_h_j_offset]); + GGML_F32_VEC b_vec = GGML_F32_VEC_LOAD(&b[t_h_j_offset]); + + k_vec = GGML_F32_VEC_MUL(v_vec, k_vec); + + GGML_F32_VEC state_vec = GGML_F32_VEC_LOAD(&state_prev[h_2d_i_j_offset]); + // kv + s * decay + sa * b + state_vec = GGML_F32_VEC_FMA(k_vec, state_vec, w_vec); + state_vec = GGML_F32_VEC_FMA(state_vec, sa_vec, b_vec); + GGML_F32_VEC_STORE(&state_cur[h_2d_i_j_offset], state_vec); + + result_vec[kk] = GGML_F32_VEC_FMA(result_vec[kk], state_vec, r_vec); + } + } + GGML_F32_VEC_REDUCE(dst_data[t_h_i_offset], result_vec); + + // There shouldn't be left-overs though. + for (; j < head_size; j++) { + int64_t t_h_j_offset = t_h_offset + j; + int64_t h_2d_i_j_offset = h_2d_i_offset + j; + + float r_val = r[t_h_j_offset]; + float w_val = w[t_h_j_offset]; + float k_val = k[t_h_j_offset]; + float b_val = b[t_h_j_offset]; + float kv_val = v[t_h_i_offset] * k_val; + + float prev_state_val = state_prev[h_2d_i_j_offset]; + state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val; + dst_data[t_h_i_offset] += state_cur[h_2d_i_j_offset] * r_val; + } + } + } + } + #else + for (int64_t t = 0; t < T; t++) { + int64_t t_offset = t * t_stride; + int64_t state_offset = head_size * C * (t / (T / n_seqs)); + float * state_cur = state + state_offset; + float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset; + + for (int64_t h = h_start; h < h_end; h++) { + int64_t h_offset = h * h_stride; + int64_t t_h_offset = t_offset + h_offset; + int64_t h_2d_offset = h * h_stride_2d; + + for (int64_t i = 0; i < head_size; i++) { + int64_t t_h_i_offset = t_h_offset + i; + int64_t h_2d_i_offset = h_2d_offset + i * h_stride; + + float v_val = v[t_h_i_offset]; + + float sa = 0, result = 0; + for (int64_t j = 0; j < head_size; j++) { + sa += a[t_h_offset + j] * state_prev[h_2d_i_offset + j]; + } + + for (int64_t j = 0; j < head_size; j++) { + int64_t t_h_j_offset = t_h_offset + j; + int64_t h_2d_i_j_offset = h_2d_i_offset + j; + + float r_val = r[t_h_j_offset]; + float w_val = w[t_h_j_offset]; + float k_val = k[t_h_j_offset]; + float b_val = b[t_h_j_offset]; + float kv_val = v_val * k_val; + float prev_state_val = state_prev[h_2d_i_j_offset]; + state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val; + result += state_cur[h_2d_i_j_offset] * r_val; + } + dst_data[t_h_i_offset] = result; + } + } + } + #endif +} + + +static void ggml_compute_forward_rwkv_wkv7( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_rwkv_wkv7_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + // ggml_compute_forward_map_unary static void ggml_compute_forward_map_unary_f32( @@ -14170,6 +14411,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_group_norm(params, tensor); } break; + case GGML_OP_L2_NORM: + { + ggml_compute_forward_l2_norm(params, tensor); + } break; case GGML_OP_MUL_MAT: { ggml_compute_forward_mul_mat(params, tensor); @@ -14357,6 +14602,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_gla(params, tensor); } break; + case GGML_OP_RWKV_WKV7: + { + ggml_compute_forward_rwkv_wkv7(params, tensor); + } break; case GGML_OP_MAP_UNARY: { ggml_unary_op_f32_t fun; @@ -14582,6 +14831,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_GROUP_NORM: case GGML_OP_CONCAT: case GGML_OP_MUL_MAT: @@ -14648,14 +14898,15 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_FLASH_ATTN_BACK: case GGML_OP_SSM_CONV: case GGML_OP_SSM_SCAN: + case GGML_OP_RWKV_WKV6: + case GGML_OP_GATED_LINEAR_ATTN: + case GGML_OP_RWKV_WKV7: { n_tasks = n_threads; } break; case GGML_OP_WIN_PART: case GGML_OP_WIN_UNPART: case GGML_OP_GET_REL_POS: - case GGML_OP_RWKV_WKV6: - case GGML_OP_GATED_LINEAR_ATTN: case GGML_OP_MAP_UNARY: case GGML_OP_MAP_BINARY: case GGML_OP_MAP_CUSTOM1_F32: diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 9bba398ce6..8fb063822c 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -36,7 +36,7 @@ #include "ggml-cuda/tsembd.cuh" #include "ggml-cuda/unary.cuh" #include "ggml-cuda/upscale.cuh" -#include "ggml-cuda/wkv6.cuh" +#include "ggml-cuda/wkv.cuh" #include "ggml-cuda/gla.cuh" #include "ggml.h" @@ -2196,6 +2196,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_GROUP_NORM: ggml_cuda_op_group_norm(ctx, dst); break; + case GGML_OP_L2_NORM: + ggml_cuda_op_l2_norm(ctx, dst); + break; case GGML_OP_CONCAT: ggml_cuda_op_concat(ctx, dst); break; @@ -2304,6 +2307,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_GATED_LINEAR_ATTN: ggml_cuda_op_gated_linear_attn(ctx, dst); break; + case GGML_OP_RWKV_WKV7: + ggml_cuda_op_rwkv_wkv7(ctx, dst); + break; case GGML_OP_CROSS_ENTROPY_LOSS_BACK: ggml_cuda_cross_entropy_loss_back(ctx, dst); break; @@ -3161,6 +3167,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g break; case GGML_OP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: return true; case GGML_OP_RMS_NORM_BACK: return ggml_is_contiguous(op->src[0]) && op->ne[0] % WARP_SIZE == 0; @@ -3215,6 +3222,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_LEAKY_RELU: case GGML_OP_RWKV_WKV6: case GGML_OP_GATED_LINEAR_ATTN: + case GGML_OP_RWKV_WKV7: return true; case GGML_OP_FLASH_ATTN_EXT: { #ifndef FLASH_ATTN_AVAILABLE diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index f127616edd..0020dbcec5 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -201,6 +201,85 @@ static __global__ void rms_norm_back_f32( } } +// template +// static __global__ void l2_norm_f32(const float * x, float * dst, const int ncols, const float eps) { +// const int row = blockIdx.x*blockDim.y + threadIdx.y; +// const int tid = threadIdx.x; + +// float tmp = 0.0f; // partial sum for thread in warp + +// for (int col = tid; col < ncols; col += block_size) { +// const float xi = x[row*ncols + col]; +// tmp += xi * xi; +// } + +// // sum up partial sums +// tmp = warp_reduce_sum(tmp); +// if (block_size > WARP_SIZE) { +// __shared__ float s_sum[32]; +// int warp_id = threadIdx.x / WARP_SIZE; +// int lane_id = threadIdx.x % WARP_SIZE; +// if (lane_id == 0) { +// s_sum[warp_id] = tmp; +// } +// __syncthreads(); +// tmp = s_sum[lane_id]; +// tmp = warp_reduce_sum(tmp); +// } + +// // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html +// const float scale = rsqrtf(fmaxf(tmp, eps * eps)); + +// for (int col = tid; col < ncols; col += block_size) { +// dst[row*ncols + col] = scale * x[row*ncols + col]; +// } +// } + +template +static __global__ void l2_norm_f32( + const float * x, float * dst, const int ncols, const int64_t stride_row, const int64_t stride_channel, + const int64_t stride_sample, const float eps) { + const int nrows = gridDim.x; + const int nchannels = gridDim.y; + + const int row = blockIdx.x; + const int channel = blockIdx.y; + const int sample = blockIdx.z; + const int tid = threadIdx.x; + + x += sample*stride_sample + channel*stride_channel + row*stride_row; + dst += ((sample*nchannels + channel)*nrows + row)*ncols; + + float tmp = 0.0f; // partial sum for thread in warp + + for (int col = tid; col < ncols; col += block_size) { + const float xi = x[col]; + tmp += xi * xi; + } + + // sum up partial sums + tmp = warp_reduce_sum(tmp); + if constexpr (block_size > WARP_SIZE) { + static_assert(block_size == 1024, "unexpected block_size"); + __shared__ float s_sum[32]; + const int warp_id = threadIdx.x / WARP_SIZE; + const int lane_id = threadIdx.x % WARP_SIZE; + if (lane_id == 0) { + s_sum[warp_id] = tmp; + } + __syncthreads(); + tmp = s_sum[lane_id]; + tmp = warp_reduce_sum(tmp); + } + + // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html + const float scale = rsqrtf(fmaxf(tmp, eps * eps)); + + for (int col = tid; col < ncols; col += block_size) { + dst[col] = scale * x[col]; + } +} + static void norm_f32_cuda( const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { @@ -248,6 +327,19 @@ static void rms_norm_back_f32_cuda(const float * grad, const float * xf, float * } } +static void l2_norm_f32_cuda( + const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { + const dim3 blocks_num(nrows, nchannels, nsamples); + if (ncols < 1024) { + const dim3 block_dims(WARP_SIZE, 1, 1); + l2_norm_f32<<>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + } else { + const dim3 block_dims(1024, 1, 1); + l2_norm_f32<1024><<>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + } +} + void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const float * src0_d = (const float *) src0->data; @@ -340,3 +432,27 @@ void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * d rms_norm_back_f32_cuda(grad_d, src0f_d, dst_d, ne00, nrows, eps, stream); } + +void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const float * src0_d = (const float *) src0->data; + float * dst_d = (float *) dst->data; + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + GGML_TENSOR_UNARY_OP_LOCALS; + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + GGML_ASSERT(eps >= 0.0f); + + const size_t ts0 = ggml_type_size(src0->type); + GGML_ASSERT(nb00 == ts0); + const int64_t s01 = nb01 / ts0; + const int64_t s02 = nb02 / ts0; + const int64_t s03 = nb03 / ts0; + + l2_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream); +} diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh index d63d34380b..706a5660a6 100644 --- a/ggml/src/ggml-cuda/norm.cuh +++ b/ggml/src/ggml-cuda/norm.cuh @@ -7,3 +7,5 @@ void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/wkv.cu b/ggml/src/ggml-cuda/wkv.cu new file mode 100644 index 0000000000..d2fced705e --- /dev/null +++ b/ggml/src/ggml-cuda/wkv.cu @@ -0,0 +1,199 @@ +#include "common.cuh" +#include "wkv.cuh" + +template +static __global__ void rwkv_wkv_f32(const int B, const int T, const int C, const int H, const float * k, const float * v, const float * r, const float * tf, const float * td, const float * s, float * dst) { + const int tid = threadIdx.x; + const int bid = blockIdx.x; + + const int head_size = block_size; + const int batch_i = bid / H; + const int head_i = bid % H; + const int state_size = C * head_size; + const int n_seq_tokens = T / B; + + float state[head_size]; + __shared__ float _k[head_size], _r[head_size], _tf[head_size], _td[head_size]; + + #pragma unroll + for (int i = 0; i < head_size; i++) { + state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid]; + } + + __syncthreads(); + _tf[tid] = tf[head_i * head_size + tid]; + __syncthreads(); + + for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) { + __syncthreads(); + _k[tid] = k[t]; + _r[tid] = r[t]; + _td[tid] = td[t]; + __syncthreads(); + + const float _v = v[t]; + float y = 0; + for (int j = 0; j < head_size; j += 4) { + const float4& k = (float4&)(_k[j]); + const float4& r = (float4&)(_r[j]); + const float4& tf = (float4&)(_tf[j]); + const float4& td = (float4&)(_td[j]); + float4& s = (float4&)(state[j]); + float4 kv; + + kv.x = k.x * _v; + kv.y = k.y * _v; + kv.z = k.z * _v; + kv.w = k.w * _v; + + y += r.x * (tf.x * kv.x + s.x); + y += r.y * (tf.y * kv.y + s.y); + y += r.z * (tf.z * kv.z + s.z); + y += r.w * (tf.w * kv.w + s.w); + + s.x = s.x * td.x + kv.x; + s.y = s.y * td.y + kv.y; + s.z = s.z * td.z + kv.z; + s.w = s.w * td.w + kv.w; + } + dst[t] = y; + } + + #pragma unroll + for (int i = 0; i < head_size; i++) { + dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i]; + } +} + +template +static __global__ void rwkv_wkv7_f32(const int B, const int T, const int C, const int H, const float * r, const float * w, const float * k, const float * v, const float * a, const float * b, const float * s, float * dst) { + const int tid = threadIdx.x; + const int bid = blockIdx.x; + + const int head_size = block_size; + const int batch_i = bid / H; + const int head_i = bid % H; + const int state_size = C * head_size; + const int n_seq_tokens = T / B; + + float state[head_size]; + __shared__ float _r[head_size], _w[head_size], _k[head_size], _a[head_size], _b[head_size]; + +#ifndef GGML_USE_MUSA + #pragma unroll +#endif + for (int i = 0; i < head_size; i++) { + state[i] = s[batch_i * state_size + head_i * head_size * head_size + tid * head_size + i]; + } + + for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) { + __syncthreads(); + _r[tid] = r[t]; + _w[tid] = w[t]; + _k[tid] = k[t]; + _a[tid] = a[t]; + _b[tid] = b[t]; + __syncthreads(); + + float sa = 0; + #pragma unroll + for (int j = 0; j < head_size; j += 4) + { + const float4& a = (float4&)(_a[j]); + const float4& s = (float4&)(state[j]); + sa += a.x * s.x; + sa += a.y * s.y; + sa += a.z * s.z; + sa += a.w * s.w; + } + + const float _v = v[t]; + float y = 0; + for (int j = 0; j < head_size; j += 4) { + const float4& r = (float4&)(_r[j]); + const float4& w = (float4&)(_w[j]); + const float4& k = (float4&)(_k[j]); + const float4& b = (float4&)(_b[j]); + float4& s = (float4&)(state[j]); + float4 kv; + + kv.x = k.x * _v; + kv.y = k.y * _v; + kv.z = k.z * _v; + kv.w = k.w * _v; + + s.x = s.x * w.x + kv.x + sa * b.x; + s.y = s.y * w.y + kv.y + sa * b.y; + s.z = s.z * w.z + kv.z + sa * b.z; + s.w = s.w * w.w + kv.w + sa * b.w; + + y += s.x * r.x; + y += s.y * r.y; + y += s.z * r.z; + y += s.w * r.w; + } + dst[t] = y; + } + + #pragma unroll + for (int i = 0; i < head_size; i++) { + dst[T * C + batch_i * state_size + head_i * head_size * head_size + tid * head_size + i] = state[i]; + } +} + +void ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const float * k_d = (const float *)dst->src[0]->data; + const float * v_d = (const float *)dst->src[1]->data; + const float * r_d = (const float *)dst->src[2]->data; + const float * tf_d = (const float *)dst->src[3]->data; + const float * td_d = (const float *)dst->src[4]->data; + const float * s_d = (const float *)dst->src[5]->data; + + const int64_t B = dst->src[5]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + float * dst_d = (float *)dst->data; + + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE || C / H == CUDA_WKV_BLOCK_SIZE * 2); + + if (C / H == CUDA_WKV_BLOCK_SIZE) { + rwkv_wkv_f32<<>>(B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d); + } else { + rwkv_wkv_f32<<>>(B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d); + } +} + +void ggml_cuda_op_rwkv_wkv7(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const float * r_d = (const float *)dst->src[0]->data; + const float * w_d = (const float *)dst->src[1]->data; + const float * k_d = (const float *)dst->src[2]->data; + const float * v_d = (const float *)dst->src[3]->data; + const float * a_d = (const float *)dst->src[4]->data; + const float * b_d = (const float *)dst->src[5]->data; + const float * s_d = (const float *)dst->src[6]->data; + + const int64_t B = dst->src[6]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + float * dst_d = (float *)dst->data; + + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(dst->src[6]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE || C / H == CUDA_WKV_BLOCK_SIZE * 2); + + if (C / H == CUDA_WKV_BLOCK_SIZE) { + rwkv_wkv7_f32<<>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d); + } else { + rwkv_wkv7_f32<<>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d); + } +} diff --git a/ggml/src/ggml-cuda/wkv6.cuh b/ggml/src/ggml-cuda/wkv.cuh similarity index 62% rename from ggml/src/ggml-cuda/wkv6.cuh rename to ggml/src/ggml-cuda/wkv.cuh index a7124ee517..9623dd7f8c 100644 --- a/ggml/src/ggml-cuda/wkv6.cuh +++ b/ggml/src/ggml-cuda/wkv.cuh @@ -3,3 +3,5 @@ #define CUDA_WKV_BLOCK_SIZE 64 void ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +void ggml_cuda_op_rwkv_wkv7(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/wkv6.cu b/ggml/src/ggml-cuda/wkv6.cu deleted file mode 100644 index bbdafbee58..0000000000 --- a/ggml/src/ggml-cuda/wkv6.cu +++ /dev/null @@ -1,89 +0,0 @@ -#include "common.cuh" -#include "wkv6.cuh" - -static __global__ void rwkv_wkv_f32(const int B, const int T, const int C, const int H, const float * k, const float * v, const float * r, const float * tf, const float * td, const float * s, float * dst) { - const int tid = threadIdx.x; - const int bid = blockIdx.x; - - const int head_size = CUDA_WKV_BLOCK_SIZE; - const int batch_i = bid / H; - const int head_i = bid % H; - const int state_size = C * head_size; - const int n_seq_tokens = T / B; - - float state[head_size]; - __shared__ float _k[head_size], _r[head_size], _tf[head_size], _td[head_size]; - - #pragma unroll - for (int i = 0; i < head_size; i++) { - state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid]; - } - - __syncthreads(); - _tf[tid] = tf[head_i * head_size + tid]; - __syncthreads(); - - for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) { - __syncthreads(); - _k[tid] = k[t]; - _r[tid] = r[t]; - _td[tid] = td[t]; - __syncthreads(); - - const float _v = v[t]; - float y = 0; - for (int j = 0; j < head_size; j += 4) { - const float4& k = (float4&)(_k[j]); - const float4& r = (float4&)(_r[j]); - const float4& tf = (float4&)(_tf[j]); - const float4& td = (float4&)(_td[j]); - float4& s = (float4&)(state[j]); - float4 kv; - - kv.x = k.x * _v; - kv.y = k.y * _v; - kv.z = k.z * _v; - kv.w = k.w * _v; - - y += r.x * (tf.x * kv.x + s.x); - y += r.y * (tf.y * kv.y + s.y); - y += r.z * (tf.z * kv.z + s.z); - y += r.w * (tf.w * kv.w + s.w); - - s.x = s.x * td.x + kv.x; - s.y = s.y * td.y + kv.y; - s.z = s.z * td.z + kv.z; - s.w = s.w * td.w + kv.w; - } - dst[t] = y; - } - - #pragma unroll - for (int i = 0; i < head_size; i++) { - dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i]; - } -} - -void ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const float * k_d = (const float *)dst->src[0]->data; - const float * v_d = (const float *)dst->src[1]->data; - const float * r_d = (const float *)dst->src[2]->data; - const float * tf_d = (const float *)dst->src[3]->data; - const float * td_d = (const float *)dst->src[4]->data; - const float * s_d = (const float *)dst->src[5]->data; - - const int64_t B = dst->src[5]->ne[1]; - const int64_t T = dst->src[0]->ne[2]; - const int64_t C = dst->ne[0]; - const int64_t H = dst->src[0]->ne[1]; - - float * dst_d = (float *)dst->data; - - cudaStream_t stream = ctx.stream(); - - GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); - GGML_ASSERT(C % H == 0); - GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE); // The current cuda kernel is designed for RWKV6, HEAD_SIZE == 64 - - rwkv_wkv_f32<<>>(B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d); -} diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index a58c474eb0..1e954b4cea 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -285,6 +285,13 @@ typedef struct { float eps; } ggml_metal_kargs_rms_norm; +typedef struct { + int32_t ne00; + int32_t ne00_4; + uint64_t nb01; + float eps; +} ggml_metal_kargs_l2_norm; + typedef struct { int64_t ne00; int64_t ne01; diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m index e51a4169a2..af65e7d9f5 100644 --- a/ggml/src/ggml-metal/ggml-metal.m +++ b/ggml/src/ggml-metal/ggml-metal.m @@ -184,10 +184,13 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, GGML_METAL_KERNEL_TYPE_RMS_NORM, + GGML_METAL_KERNEL_TYPE_L2_NORM, GGML_METAL_KERNEL_TYPE_GROUP_NORM, GGML_METAL_KERNEL_TYPE_NORM, GGML_METAL_KERNEL_TYPE_SSM_CONV_F32, GGML_METAL_KERNEL_TYPE_SSM_SCAN_F32, + GGML_METAL_KERNEL_TYPE_RWKV_WKV6_F32, + GGML_METAL_KERNEL_TYPE_RWKV_WKV7_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, @@ -810,10 +813,13 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, get_rows_iq4_xs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, has_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_L2_NORM, l2_norm, has_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, has_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SSM_CONV_F32, ssm_conv_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SSM_SCAN_F32, ssm_scan_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RWKV_WKV6_F32, rwkv_wkv6_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RWKV_WKV7_F32, rwkv_wkv7_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, has_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_BF16_F32, mul_mv_bf16_f32, has_simdgroup_reduction && use_bfloat); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_BF16_F32_1ROW, mul_mv_bf16_f32_1row, has_simdgroup_reduction && use_bfloat); @@ -1251,6 +1257,7 @@ static bool ggml_metal_supports_op(const struct ggml_backend_metal_device_contex case GGML_OP_GROUP_NORM: return has_simdgroup_reduction && ggml_is_contiguous(op->src[0]); case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: return has_simdgroup_reduction && (op->ne[0] % 4 == 0 && ggml_is_contiguous_1(op->src[0])); case GGML_OP_ARGMAX: return true; @@ -1288,6 +1295,8 @@ static bool ggml_metal_supports_op(const struct ggml_backend_metal_device_contex return has_simdgroup_mm; // TODO: over-restricted for vec-kernels case GGML_OP_SSM_CONV: case GGML_OP_SSM_SCAN: + case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: return true; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: @@ -2216,6 +2225,83 @@ static void ggml_metal_encode_node( [encoder dispatchThreadgroups:MTLSizeMake(d_inner, n_seqs, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; } break; + case GGML_OP_RWKV_WKV6: + { + const int64_t B = dst->src[5]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == 64); + + size_t offs_src3 = 0; + size_t offs_src4 = 0; + size_t offs_src5 = 0; + + id id_src3 = dst->src[3] ? ggml_metal_get_buffer(dst->src[3], &offs_src3) : nil; + id id_src4 = dst->src[4] ? ggml_metal_get_buffer(dst->src[4], &offs_src4) : nil; + id id_src5 = dst->src[5] ? ggml_metal_get_buffer(dst->src[5], &offs_src5) : nil; + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RWKV_WKV6_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2]; + [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3]; + [encoder setBuffer:id_src4 offset:offs_src4 atIndex:4]; + [encoder setBuffer:id_src5 offset:offs_src5 atIndex:5]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:6]; + + [encoder setBytes:&B length:sizeof(B) atIndex:7]; + [encoder setBytes:&T length:sizeof(T) atIndex:8]; + [encoder setBytes:&C length:sizeof(C) atIndex:9]; + [encoder setBytes:&H length:sizeof(H) atIndex:10]; + + [encoder dispatchThreadgroups:MTLSizeMake(B * H, 1, 1) threadsPerThreadgroup:MTLSizeMake(C/ H, 1, 1)]; + } break; + case GGML_OP_RWKV_WKV7: + { + const int64_t B = dst->src[6]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + GGML_ASSERT(dst->src[6]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == 64); + + size_t offs_src3 = 0; + size_t offs_src4 = 0; + size_t offs_src5 = 0; + size_t offs_src6 = 0; + + id id_src3 = dst->src[3] ? ggml_metal_get_buffer(dst->src[3], &offs_src3) : nil; + id id_src4 = dst->src[4] ? ggml_metal_get_buffer(dst->src[4], &offs_src4) : nil; + id id_src5 = dst->src[5] ? ggml_metal_get_buffer(dst->src[5], &offs_src5) : nil; + id id_src6 = dst->src[6] ? ggml_metal_get_buffer(dst->src[6], &offs_src6) : nil; + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RWKV_WKV7_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2]; + [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3]; + [encoder setBuffer:id_src4 offset:offs_src4 atIndex:4]; + [encoder setBuffer:id_src5 offset:offs_src5 atIndex:5]; + [encoder setBuffer:id_src6 offset:offs_src6 atIndex:6]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:7]; + + [encoder setBytes:&B length:sizeof(B) atIndex:8]; + [encoder setBytes:&T length:sizeof(T) atIndex:9]; + [encoder setBytes:&C length:sizeof(C) atIndex:10]; + [encoder setBytes:&H length:sizeof(H) atIndex:11]; + + [encoder dispatchThreadgroups:MTLSizeMake(B * H, 1, 1) threadsPerThreadgroup:MTLSizeMake(C/ H, 1, 1)]; + } break; case GGML_OP_MUL_MAT: { GGML_ASSERT(ne00 == ne10); @@ -3122,6 +3208,42 @@ static void ggml_metal_encode_node( const int64_t nrows = ggml_nrows(src0); + [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_L2_NORM: + { + GGML_ASSERT(ne00 % 4 == 0); + GGML_ASSERT(ggml_is_contiguous_1(src0)); + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_L2_NORM].pipeline; + + int nth = 32; // SIMD width + + while (nth < ne00/4 && nth < (int) pipeline.maxTotalThreadsPerThreadgroup) { + nth *= 2; + } + + nth = MIN(nth, ne00/4); + + ggml_metal_kargs_l2_norm args = { + /*.ne00 =*/ ne00, + /*.ne00_4 =*/ ne00/4, + /*.nb01 =*/ nb01, + /*.eps =*/ eps, + }; + + [encoder setComputePipelineState:pipeline]; + [encoder setBytes:&args length:sizeof(args) atIndex:0]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + + [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; + + const int64_t nrows = ggml_nrows(src0); + [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; } break; case GGML_OP_GROUP_NORM: diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index ad9d42a3ea..3cef81b797 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -1295,6 +1295,184 @@ kernel void kernel_ssm_scan_f32( } } +kernel void kernel_rwkv_wkv6_f32( + device const float * k, + device const float * v, + device const float * r, + device const float * tf, + device const float * td, + device const float * state_in, + device float * dst, + constant uint & B, + constant uint & T, + constant uint & C, + constant uint & H, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const uint head_size = 64; // TODO: support head_size = 128 + const uint batch_id = tgpig.x / H; + const uint head_id = tgpig.x % H; + const uint tid = tpitg.x; + + if (batch_id >= B || head_id >= H) { + return; + } + + const uint state_size = C * head_size; + const uint n_seq_tokens = T / B; + + threadgroup float _k[head_size]; + threadgroup float _r[head_size]; + threadgroup float _tf[head_size]; + threadgroup float _td[head_size]; + + float state[head_size]; + + for (uint i = 0; i < head_size; i++) { + state[i] = state_in[batch_id * state_size + head_id * head_size * head_size + + i * head_size + tid]; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + _tf[tid] = tf[head_id * head_size + tid]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; + const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; + + for (uint t = start_t; t < end_t; t += C) { + threadgroup_barrier(mem_flags::mem_threadgroup); + _k[tid] = k[t]; + _r[tid] = r[t]; + _td[tid] = td[t]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + const float v_val = v[t]; + float y = 0.0; + + for (uint j = 0; j < head_size; j += 4) { + float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + float4 tf_vec = float4(_tf[j], _tf[j+1], _tf[j+2], _tf[j+3]); + float4 td_vec = float4(_td[j], _td[j+1], _td[j+2], _td[j+3]); + float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); + + float4 kv = k_vec * v_val; + + float4 temp = tf_vec * kv + s_vec; + y += dot(r_vec, temp); + + s_vec = s_vec * td_vec + kv; + state[j] = s_vec[0]; + state[j+1] = s_vec[1]; + state[j+2] = s_vec[2]; + state[j+3] = s_vec[3]; + } + + dst[t] = y; + } + + for (uint i = 0; i < head_size; i++) { + dst[T * C + batch_id * state_size + head_id * head_size * head_size + + i * head_size + tid] = state[i]; + } +} + +kernel void kernel_rwkv_wkv7_f32( + device const float * r, + device const float * w, + device const float * k, + device const float * v, + device const float * a, + device const float * b, + device const float * state_in, + device float * dst, + constant uint & B, + constant uint & T, + constant uint & C, + constant uint & H, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const uint head_size = 64; // TODO: support head_size = 128 + const uint batch_id = tgpig.x / H; + const uint head_id = tgpig.x % H; + const uint tid = tpitg.x; + + if (batch_id >= B || head_id >= H) { + return; + } + + const uint state_size = C * head_size; + const uint n_seq_tokens = T / B; + + threadgroup float _r[head_size]; + threadgroup float _w[head_size]; + threadgroup float _k[head_size]; + threadgroup float _a[head_size]; + threadgroup float _b[head_size]; + + float state[head_size]; + + for (uint i = 0; i < head_size; i++) { + state[i] = state_in[batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i]; + } + + const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; + const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; + + for (uint t = start_t; t < end_t; t += C) { + threadgroup_barrier(mem_flags::mem_threadgroup); + _r[tid] = r[t]; + _w[tid] = w[t]; + _k[tid] = k[t]; + _a[tid] = a[t]; + _b[tid] = b[t]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + const float v_val = v[t]; + float y = 0.0, sa = 0.0; + + float4 sa_vec(0.0); + + for (int j = 0; j < head_size; j += 4) { + float4 a_vec = float4(_a[j], _a[j+1], _a[j+2], _a[j+3]); + float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); + sa_vec += a_vec * s_vec; + } + sa = sa_vec[0] + sa_vec[1] + sa_vec[2] + sa_vec[3]; + + for (uint j = 0; j < head_size; j += 4) { + float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + float4 w_vec = float4(_w[j], _w[j+1], _w[j+2], _w[j+3]); + float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + float4 b_vec = float4(_b[j], _b[j+1], _b[j+2], _b[j+3]); + float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); + + float4 kv = k_vec * v_val; + + s_vec = s_vec * w_vec + kv + sa * b_vec; + y += dot(s_vec, r_vec); + + state[j] = s_vec[0]; + state[j+1] = s_vec[1]; + state[j+2] = s_vec[2]; + state[j+3] = s_vec[3]; + } + + dst[t] = y; + } + + for (uint i = 0; i < head_size; i++) { + dst[T * C + batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i] = state[i]; + } +} + kernel void kernel_argmax( device const void * x, device int32_t * dst, @@ -1463,6 +1641,49 @@ kernel void kernel_rms_norm( } } +kernel void kernel_l2_norm( + constant ggml_metal_kargs_l2_norm & args, + device const char * src0, + device char * dst, + threadgroup float * shmem_f32 [[threadgroup(0)]], + uint tgpig[[threadgroup_position_in_grid]], + ushort tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort ntg[[threads_per_threadgroup]]) { + if (sgitg == 0) { + shmem_f32[tiisg] = 0.0f; + } + + device const float4 * x = (device const float4 *) (src0 + tgpig*args.nb01); + + float sumf = 0.0f; + + // parallel sum + for (int i00 = tpitg; i00 < args.ne00_4; i00 += ntg) { + sumf += dot(x[i00], x[i00]); + } + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_f32[tiisg]; + sumf = simd_sum(sumf); + + const float scale = 1.0f/sqrt(max(sumf, args.eps)); + + device float4 * y = (device float4 *) dst + tgpig*args.ne00_4; + for (int i00 = tpitg; i00 < args.ne00_4; i00 += ntg) { + y[i00] = x[i00] * scale; + } +} + kernel void kernel_group_norm( device const float * src0, device float * dst, diff --git a/ggml/src/ggml-sycl/backend.hpp b/ggml/src/ggml-sycl/backend.hpp index 577ff51fde..73d807cab0 100644 --- a/ggml/src/ggml-sycl/backend.hpp +++ b/ggml/src/ggml-sycl/backend.hpp @@ -26,7 +26,7 @@ #include "softmax.hpp" #include "tsembd.hpp" #include "im2col.hpp" -#include "wkv6.hpp" +#include "wkv.hpp" #include "outprod.hpp" #include "element_wise.hpp" #include "cpy.hpp" diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 05984d8c5a..477652ab28 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -2696,6 +2696,12 @@ static void ggml_sycl_rms_norm(ggml_backend_sycl_context & ctx, ggml_tensor * ds GGML_SYCL_DEBUG("call %s done\n", __func__); } +static void ggml_sycl_l2_norm(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + GGML_SYCL_DEBUG("call %s\n", __func__); + ggml_sycl_op_flatten(ctx, dst->src[0], dst->src[1], dst, ggml_sycl_op_l2_norm); + GGML_SYCL_DEBUG("call %s done\n", __func__); +} + static void ggml_sycl_group_norm(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(ctx, dst->src[0], dst->src[1], dst, ggml_sycl_op_group_norm); @@ -3410,6 +3416,9 @@ bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tens case GGML_OP_RMS_NORM: ggml_sycl_rms_norm(ctx, dst); break; + case GGML_OP_L2_NORM: + ggml_sycl_l2_norm(ctx, dst); + break; case GGML_OP_MUL_MAT: if (dst->src[0]->ne[3] != dst->src[1]->ne[3]) { return false; @@ -3487,6 +3496,9 @@ bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tens case GGML_OP_RWKV_WKV6: ggml_sycl_op_rwkv_wkv6(ctx, dst); break; + case GGML_OP_RWKV_WKV7: + ggml_sycl_op_rwkv_wkv7(ctx, dst); + break; case GGML_OP_GATED_LINEAR_ATTN: ggml_sycl_op_gated_linear_attn(ctx, dst); break; @@ -4012,6 +4024,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g return (op->src[0]->type == GGML_TYPE_F32); case GGML_OP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: case GGML_OP_GROUP_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_SCALE: @@ -4045,6 +4058,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_LEAKY_RELU: case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: case GGML_OP_GATED_LINEAR_ATTN: return true; default: diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp index 9cf2be1557..6439db21b2 100644 --- a/ggml/src/ggml-sycl/norm.cpp +++ b/ggml/src/ggml-sycl/norm.cpp @@ -180,6 +180,50 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, const floa } } +static void l2_norm_f32(const float* x, float* dst, const int ncols, const float eps, + const sycl::nd_item<3>& item_ct1, float* s_sum, int block_size) { + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + const int tid = item_ct1.get_local_id(2); + const int nthreads = item_ct1.get_local_range(2); + const int nwarps = nthreads / WARP_SIZE; + float tmp = 0.0f; // partial sum for thread in warp + + for (int col = tid; col < ncols; col += block_size) { + const float xi = x[row * ncols + col]; + tmp += xi * xi; + } + + // sum up partial sums + tmp = warp_reduce_sum(tmp, item_ct1); + if (block_size > WARP_SIZE) { + + int warp_id = item_ct1.get_local_id(2) / WARP_SIZE; + int lane_id = item_ct1.get_local_id(2) % WARP_SIZE; + if (lane_id == 0) { + s_sum[warp_id] = tmp; + } + /* + DPCT1118:3: SYCL group functions and algorithms must be encountered in + converged control flow. You may need to adjust the code. + */ + item_ct1.barrier(sycl::access::fence_space::local_space); + size_t nreduce = nwarps / WARP_SIZE; + tmp = 0.f; + for (size_t i = 0; i < nreduce; i += 1) + { + tmp += s_sum[lane_id + i * WARP_SIZE]; + } + tmp = warp_reduce_sum(tmp, item_ct1); + } + + const float scale = sycl::rsqrt(sycl::max(tmp, eps * eps)); + + for (int col = tid; col < ncols; col += block_size) { + dst[row * ncols + col] = scale * x[row * ncols + col]; + } +} + static void norm_f32_sycl(const float* x, float* dst, const int ncols, const int nrows, const float eps, queue_ptr stream, int device) { @@ -311,6 +355,48 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, } } +static void l2_norm_f32_sycl(const float* x, float* dst, const int ncols, + const int nrows, const float eps, + queue_ptr stream, int device) { + GGML_ASSERT(ncols % WARP_SIZE == 0); + // printf("%s ncols=%d, nrows=%d, WARP_SIZE=%d\n", __func__, ncols, nrows, WARP_SIZE); + if (ncols < 1024) { + const sycl::range<3> block_dims(1, 1, WARP_SIZE); + stream->submit([&](sycl::handler& cgh) { + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, + block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(WARP_SIZE)]] { + l2_norm_f32(x, dst, ncols, eps, item_ct1, + nullptr, WARP_SIZE); + }); + }); + } + else { + const int work_group_size = ggml_sycl_info().max_work_group_sizes[device]; + assert(work_group_size % (WARP_SIZE * WARP_SIZE) == 0); + const sycl::range<3> block_dims(1, 1, work_group_size); + /* + DPCT1049:19: The work-group size passed to the SYCL kernel may exceed + the limit. To get the device limit, query + info::device::max_work_group_size. Adjust the work-group size if needed. + */ + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor s_sum_acc_ct1(sycl::range<1>(work_group_size / WARP_SIZE), + cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, + block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(WARP_SIZE)]] { + l2_norm_f32(x, dst, ncols, eps, item_ct1, + get_pointer(s_sum_acc_ct1), work_group_size); + }); + }); + } +} + void ggml_sycl_op_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor* dst, const float* src0_dd, const float* src1_dd, float* dst_dd, @@ -376,3 +462,25 @@ void ggml_sycl_op_rms_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* sr (void)dst; (void)src1_dd; } + +void ggml_sycl_op_l2_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* src0, + const ggml_tensor* src1, ggml_tensor* dst, + const float* src0_dd, const float* src1_dd, + float* dst_dd, + const queue_ptr& main_stream) { + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + const int64_t ne00 = src0->ne[0]; + const int64_t nrows = ggml_nrows(src0); + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + + l2_norm_f32_sycl(src0_dd, dst_dd, ne00, nrows, eps, main_stream, ctx.device); + + (void)src1; + (void)dst; + (void)src1_dd; +} diff --git a/ggml/src/ggml-sycl/norm.hpp b/ggml/src/ggml-sycl/norm.hpp index a9ad9156fa..11e91680cc 100644 --- a/ggml/src/ggml-sycl/norm.hpp +++ b/ggml/src/ggml-sycl/norm.hpp @@ -32,4 +32,10 @@ void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* float* dst_dd, const queue_ptr& main_stream); +void ggml_sycl_op_l2_norm(ggml_backend_sycl_context& ctx, const ggml_tensor* src0, + const ggml_tensor* src1, ggml_tensor* dst, + const float* src0_dd, const float* src1_dd, + float* dst_dd, + const queue_ptr& main_stream); + #endif // GGML_SYCL_NORM_HPP diff --git a/ggml/src/ggml-sycl/wkv.cpp b/ggml/src/ggml-sycl/wkv.cpp new file mode 100644 index 0000000000..540f6fbf5f --- /dev/null +++ b/ggml/src/ggml-sycl/wkv.cpp @@ -0,0 +1,305 @@ +#include +#include "wkv.hpp" + +constexpr int WKV_BLOCK_SIZE = 64; // Matching CUDA_WKV_BLOCK_SIZE + +// Helper function for the main kernel +template +static void rwkv_wkv6_f32_kernel( + const int B, const int T, const int C, const int H, + const float* k, const float* v, const float* r, + const float* tf, const float* td, const float* s, + float* dst, const sycl::nd_item<3>& item_ct1, float* shared_mem) { + + const int tid = item_ct1.get_local_id(2); + const int bid = item_ct1.get_group(2); + + const int head_size = block_size; + const int batch_i = bid / H; + const int head_i = bid % H; + const int state_size = C * head_size; + const int n_seq_tokens = T / B; + + // Set up shared memory pointers + float* _k = shared_mem; + float* _r = _k + head_size; + float* _tf = _r + head_size; + float* _td = _tf + head_size; + + // Local state array + float state[block_size]; + + // Load initial state + #pragma unroll + for (int i = 0; i < head_size; i++) { + state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid]; + } + + // Sync threads before shared memory operations + item_ct1.barrier(sycl::access::fence_space::local_space); + + // Load time-mixing parameters + _tf[tid] = tf[head_i * head_size + tid]; + item_ct1.barrier(sycl::access::fence_space::local_space); + + // Main sequence processing loop + for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; + t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; + t += C) { + + item_ct1.barrier(sycl::access::fence_space::local_space); + + // Load current timestep data to shared memory + _k[tid] = k[t]; + _r[tid] = r[t]; + _td[tid] = td[t]; + + item_ct1.barrier(sycl::access::fence_space::local_space); + + const float _v = v[t]; + float y = 0; + + // Process in chunks of 4 for better vectorization + sycl::float4 k4, r4, tf4, td4, s4; + #pragma unroll + for (int j = 0; j < head_size; j += 4) { + // Load data in vec4 chunks + k4 = sycl::float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + r4 = sycl::float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + tf4 = sycl::float4(_tf[j], _tf[j+1], _tf[j+2], _tf[j+3]); + td4 = sycl::float4(_td[j], _td[j+1], _td[j+2], _td[j+3]); + s4 = sycl::float4(state[j], state[j+1], state[j+2], state[j+3]); + + // Compute key-value product + sycl::float4 kv4 = k4 * _v; + + // Accumulate weighted sum + y += sycl::dot(r4, tf4 * kv4 + s4); + + // Update state + s4 = s4 * td4 + kv4; + + // Store updated state + state[j] = s4.x(); + state[j+1] = s4.y(); + state[j+2] = s4.z(); + state[j+3] = s4.w(); + } + + dst[t] = y; + } + + // Save final state + #pragma unroll + for (int i = 0; i < head_size; i++) { + dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i]; + } +} + +template +static void rwkv_wkv7_f32_kernel( + const int B, const int T, const int C, const int H, + const float* r, const float* w, const float* k, const float* v, + const float* a, const float* b, const float* s, + float* dst, const sycl::nd_item<3>& item_ct1, float* shared_mem) { + + const int tid = item_ct1.get_local_id(2); + const int bid = item_ct1.get_group(2); + + const int head_size = block_size; + const int batch_i = bid / H; + const int head_i = bid % H; + const int state_size = C * head_size; + const int n_seq_tokens = T / B; + + float* _r = shared_mem; + float* _w = _r + head_size; + float* _k = _w + head_size; + float* _a = _k + head_size; + float* _b = _a + head_size; + + float state[block_size]; + + #pragma unroll + for (int i = 0; i < head_size; i++) { + state[i] = s[batch_i * state_size + head_i * head_size * head_size + tid * head_size + i]; + } + + for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; + t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; + t += C) { + + item_ct1.barrier(sycl::access::fence_space::local_space); + + _r[tid] = r[t]; + _w[tid] = w[t]; + _k[tid] = k[t]; + _a[tid] = a[t]; + _b[tid] = b[t]; + + item_ct1.barrier(sycl::access::fence_space::local_space); + + const float _v = v[t]; + float y = 0, sa = 0; + sycl::float4 a4, s4; + + #pragma unroll + for (int j = 0; j < head_size; j += 4) { + a4 = sycl::float4(_a[j], _a[j+1], _a[j+2], _a[j+3]); + s4 = sycl::float4(state[j], state[j+1], state[j+2], state[j+3]); + sa += sycl::dot(a4, s4); + } + + sycl::float4 r4, w4, k4, b4; + #pragma unroll + for (int j = 0; j < head_size; j += 4) { + r4 = sycl::float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + w4 = sycl::float4(_w[j], _w[j+1], _w[j+2], _w[j+3]); + k4 = sycl::float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + b4 = sycl::float4(_b[j], _b[j+1], _b[j+2], _b[j+3]); + s4 = sycl::float4(state[j], state[j+1], state[j+2], state[j+3]); + + sycl::float4 kv4 = k4 * _v; + + s4 = s4 * w4 + kv4 + sa * b4; + y += sycl::dot(r4, s4); + + state[j] = s4.x(); + state[j+1] = s4.y(); + state[j+2] = s4.z(); + state[j+3] = s4.w(); + } + + dst[t] = y; + } + + #pragma unroll + for (int i = 0; i < head_size; i++) { + dst[T * C + batch_i * state_size + head_i * head_size * head_size + tid * head_size + i] = state[i]; + } +} + +void ggml_sycl_op_rwkv_wkv6(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { + + const ggml_tensor *src0 = dst->src[0]; + const ggml_tensor *src1 = dst->src[1]; + + const float* k_d = (const float*)dst->src[0]->data; + const float* v_d = (const float*)dst->src[1]->data; + const float* r_d = (const float*)dst->src[2]->data; + const float* tf_d = (const float*)dst->src[3]->data; + const float* td_d = (const float*)dst->src[4]->data; + const float* s_d = (const float*)dst->src[5]->data; + float* dst_d = (float*)dst->data; + + const int64_t B = dst->src[5]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == WKV_BLOCK_SIZE || C / H == WKV_BLOCK_SIZE * 2); // The current sycl kernel is designed for RWKV6, HEAD_SIZE == 64 + + dpct::queue_ptr stream = ctx.stream(); + + // Calculate execution configuration + const size_t shared_mem_size = C / H * 4 * sizeof(float); // For k, r, tf, td + sycl::range<3> block_dims(1, 1, C / H); + sycl::range<3> grid_dims(1, 1, B * H); + + // Submit kernel + if (C / H == WKV_BLOCK_SIZE) { + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + rwkv_wkv6_f32_kernel( + B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d, + item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() + ); + }); + }); + } else { + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + rwkv_wkv6_f32_kernel( + B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d, + item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() + ); + }); + }); + } + + GGML_UNUSED(src0); + GGML_UNUSED(src1); +} + +void ggml_sycl_op_rwkv_wkv7(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { + + const ggml_tensor *src0 = dst->src[0]; + const ggml_tensor *src1 = dst->src[1]; + + const float* r_d = (const float*)dst->src[0]->data; + const float* w_d = (const float*)dst->src[1]->data; + const float* k_d = (const float*)dst->src[2]->data; + const float* v_d = (const float*)dst->src[3]->data; + const float* a_d = (const float*)dst->src[4]->data; + const float* b_d = (const float*)dst->src[5]->data; + const float* s_d = (const float*)dst->src[6]->data; + float* dst_d = (float*)dst->data; + + const int64_t B = dst->src[6]->ne[1]; + const int64_t T = dst->src[0]->ne[2]; + const int64_t C = dst->ne[0]; + const int64_t H = dst->src[0]->ne[1]; + + GGML_ASSERT(dst->src[6]->type == GGML_TYPE_F32); + GGML_ASSERT(C % H == 0); + GGML_ASSERT(C / H == WKV_BLOCK_SIZE || C / H == WKV_BLOCK_SIZE * 2); + + dpct::queue_ptr stream = ctx.stream(); + + // Calculate execution configuration + const size_t shared_mem_size = C / H * 5 * sizeof(float); // For r, w, k, a, b + sycl::range<3> block_dims(1, 1, C / H); + sycl::range<3> grid_dims(1, 1, B * H); + + // Submit kernel + if (C / H == WKV_BLOCK_SIZE) { + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + rwkv_wkv7_f32_kernel( + B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d, + item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() + ); + }); + }); + } else { + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + rwkv_wkv7_f32_kernel( + B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d, + item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() + ); + }); + }); + } + + GGML_UNUSED(src0); + GGML_UNUSED(src1); +} diff --git a/ggml/src/ggml-sycl/wkv.hpp b/ggml/src/ggml-sycl/wkv.hpp new file mode 100644 index 0000000000..9f34a1001f --- /dev/null +++ b/ggml/src/ggml-sycl/wkv.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_WKV_HPP +#define GGML_SYCL_WKV_HPP + +#include "common.hpp" + +void ggml_sycl_op_rwkv_wkv6(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +void ggml_sycl_op_rwkv_wkv7(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_WKV_HPP diff --git a/ggml/src/ggml-sycl/wkv6.cpp b/ggml/src/ggml-sycl/wkv6.cpp deleted file mode 100644 index b54c20964e..0000000000 --- a/ggml/src/ggml-sycl/wkv6.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include -#include "wkv6.hpp" - -constexpr int WKV_BLOCK_SIZE = 64; // Matching CUDA_WKV_BLOCK_SIZE - -// Helper function for the main kernel -static void rwkv_wkv_f32_kernel( - const int B, const int T, const int C, const int H, - const float* k, const float* v, const float* r, - const float* tf, const float* td, const float* s, - float* dst, const sycl::nd_item<3>& item_ct1, float* shared_mem) { - - const int tid = item_ct1.get_local_id(2); - const int bid = item_ct1.get_group(2); - - const int head_size = WKV_BLOCK_SIZE; - const int batch_i = bid / H; - const int head_i = bid % H; - const int state_size = C * head_size; - const int n_seq_tokens = T / B; - - // Set up shared memory pointers - float* _k = shared_mem; - float* _r = _k + head_size; - float* _tf = _r + head_size; - float* _td = _tf + head_size; - - // Local state array - float state[WKV_BLOCK_SIZE]; - - // Load initial state - #pragma unroll - for (int i = 0; i < head_size; i++) { - state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid]; - } - - // Sync threads before shared memory operations - item_ct1.barrier(sycl::access::fence_space::local_space); - - // Load time-mixing parameters - _tf[tid] = tf[head_i * head_size + tid]; - item_ct1.barrier(sycl::access::fence_space::local_space); - - // Main sequence processing loop - for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; - t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; - t += C) { - - item_ct1.barrier(sycl::access::fence_space::local_space); - - // Load current timestep data to shared memory - _k[tid] = k[t]; - _r[tid] = r[t]; - _td[tid] = td[t]; - - item_ct1.barrier(sycl::access::fence_space::local_space); - - const float _v = v[t]; - float y = 0; - - // Process in chunks of 4 for better vectorization - sycl::float4 k4, r4, tf4, td4, s4; - #pragma unroll - for (int j = 0; j < head_size; j += 4) { - // Load data in vec4 chunks - k4 = sycl::float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); - r4 = sycl::float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); - tf4 = sycl::float4(_tf[j], _tf[j+1], _tf[j+2], _tf[j+3]); - td4 = sycl::float4(_td[j], _td[j+1], _td[j+2], _td[j+3]); - s4 = sycl::float4(state[j], state[j+1], state[j+2], state[j+3]); - - // Compute key-value product - sycl::float4 kv4 = k4 * _v; - - // Accumulate weighted sum - y += sycl::dot(r4, tf4 * kv4 + s4); - - // Update state - s4 = s4 * td4 + kv4; - - // Store updated state - state[j] = s4.x(); - state[j+1] = s4.y(); - state[j+2] = s4.z(); - state[j+3] = s4.w(); - } - - dst[t] = y; - } - - // Save final state - #pragma unroll - for (int i = 0; i < head_size; i++) { - dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i]; - } -} - -void ggml_sycl_op_rwkv_wkv6(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { - - const ggml_tensor *src0 = dst->src[0]; - const ggml_tensor *src1 = dst->src[1]; - - const float* k_d = (const float*)dst->src[0]->data; - const float* v_d = (const float*)dst->src[1]->data; - const float* r_d = (const float*)dst->src[2]->data; - const float* tf_d = (const float*)dst->src[3]->data; - const float* td_d = (const float*)dst->src[4]->data; - const float* s_d = (const float*)dst->src[5]->data; - float* dst_d = (float*)dst->data; - - const int64_t B = dst->src[5]->ne[1]; - const int64_t T = dst->src[0]->ne[2]; - const int64_t C = dst->ne[0]; - const int64_t H = dst->src[0]->ne[1]; - - GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32); - GGML_ASSERT(C % H == 0); - GGML_ASSERT(C / H == WKV_BLOCK_SIZE); // The current sycl kernel is designed for RWKV6, HEAD_SIZE == 64 - - dpct::queue_ptr stream = ctx.stream(); - - // Calculate execution configuration - const size_t shared_mem_size = WKV_BLOCK_SIZE * 4 * sizeof(float); // For k, r, tf, td - sycl::range<3> block_dims(1, 1, C / H); - sycl::range<3> grid_dims(1, 1, B * H); - - // Submit kernel - stream->submit([&](sycl::handler& cgh) { - sycl::local_accessor shared_mem_acc(shared_mem_size, cgh); - - cgh.parallel_for( - sycl::nd_range<3>(grid_dims * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - rwkv_wkv_f32_kernel( - B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d, - item_ct1, (float*)shared_mem_acc.get_multi_ptr().get() - ); - }); - }); - - GGML_UNUSED(src0); - GGML_UNUSED(src1); -} diff --git a/ggml/src/ggml-sycl/wkv6.hpp b/ggml/src/ggml-sycl/wkv6.hpp deleted file mode 100644 index 8c596a9972..0000000000 --- a/ggml/src/ggml-sycl/wkv6.hpp +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef GGML_SYCL_WKV6_HPP -#define GGML_SYCL_WKV6_HPP - -#include "common.hpp" - -void ggml_sycl_op_rwkv_wkv6(ggml_backend_sycl_context & ctx, ggml_tensor * dst); - - -#endif // GGML_SYCL_WKV6_HPP diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 97398f071b..c0ee5dadef 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -304,6 +304,7 @@ struct vk_device_struct { vk_pipeline pipeline_group_norm_f32; vk_pipeline pipeline_rms_norm_f32; vk_pipeline pipeline_rms_norm_back_f32; + vk_pipeline pipeline_l2_norm_f32; vk_pipeline pipeline_gelu_f32; vk_pipeline pipeline_gelu_quick_f32; vk_pipeline pipeline_silu_f32; @@ -328,6 +329,7 @@ struct vk_device_struct { vk_pipeline pipeline_timestep_embedding_f32; vk_pipeline pipeline_pool2d_f32; vk_pipeline pipeline_rwkv_wkv6_f32; + vk_pipeline pipeline_rwkv_wkv7_f32; vk_pipeline pipeline_opt_step_adamw_f32; // [2][2][2] is for {f16acc,f32acc}x{large,small_rows}x{unaligned, aligned} @@ -629,6 +631,13 @@ struct vk_op_rwkv_wkv6_push_constants { uint32_t H; }; +struct vk_op_rwkv_wkv7_push_constants { + uint32_t B; + uint32_t T; + uint32_t C; + uint32_t H; +}; + // Allow pre-recording command buffers struct vk_staging_memcpy { vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {} @@ -2263,6 +2272,7 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); @@ -2374,6 +2384,8 @@ static void ggml_vk_load_shaders(vk_device& device) { ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv6_f32, "rwkv_wkv6_f32", rwkv_wkv6_f32_len, rwkv_wkv6_f32_data, "main", 7, sizeof(vk_op_rwkv_wkv6_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv7_f32, "rwkv_wkv7_f32", rwkv_wkv7_f32_len, rwkv_wkv7_f32_data, "main", 8, sizeof(vk_op_rwkv_wkv7_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); + ggml_vk_create_pipeline(device, device->pipeline_opt_step_adamw_f32, "opt_step_adamw_f32", opt_step_adamw_f32_len, opt_step_adamw_f32_data, "main", 5, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); for (auto &c : compiles) { @@ -5473,6 +5485,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return ctx->device->pipeline_rms_norm_back_f32; } return nullptr; + case GGML_OP_L2_NORM: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_l2_norm_f32; + } + return nullptr; case GGML_OP_UNARY: switch (ggml_get_unary_op(dst)) { case GGML_UNARY_OP_SILU: @@ -5612,6 +5629,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return ctx->device->pipeline_rwkv_wkv6_f32; } return nullptr; + case GGML_OP_RWKV_WKV7: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rwkv_wkv7_f32; + } + return nullptr; case GGML_OP_OPT_STEP_ADAMW: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_opt_step_adamw_f32; @@ -5859,6 +5881,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co case GGML_OP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_SOFT_MAX: case GGML_OP_SOFT_MAX_BACK: case GGML_OP_SUM_ROWS: @@ -6108,23 +6131,17 @@ static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const }, dryrun); } -static void ggml_vk_op_f32_rwkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, const vk_op_rwkv_wkv6_push_constants&& pc, bool dryrun = false) { - const ggml_tensor * k = dst->src[0]; - const ggml_tensor * v = dst->src[1]; - const ggml_tensor * r = dst->src[2]; - const ggml_tensor * tf = dst->src[3]; - const ggml_tensor * td = dst->src[4]; - const ggml_tensor * state = dst->src[5]; +static void ggml_vk_op_f32_wkv(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, const vk_op_rwkv_wkv6_push_constants&& pc, int version, bool dryrun = false) { + GGML_ASSERT(version == 6 || version == 7); + int num_srcs = version == 6 ? 6 : 7; + + for (int i = 0; i < num_srcs; i++) { + GGML_ASSERT(!ggml_is_quantized(dst->src[i]->type)); + } - GGML_ASSERT(!ggml_is_quantized(k->type)); - GGML_ASSERT(!ggml_is_quantized(v->type)); - GGML_ASSERT(!ggml_is_quantized(r->type)); - GGML_ASSERT(!ggml_is_quantized(tf->type)); - GGML_ASSERT(!ggml_is_quantized(td->type)); - GGML_ASSERT(!ggml_is_quantized(state->type)); GGML_ASSERT(dst->buffer != nullptr); - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, k, v, r, dst, GGML_OP_RWKV_WKV6); + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); GGML_ASSERT(pipeline != nullptr); if (dryrun) { @@ -6133,89 +6150,73 @@ static void ggml_vk_op_f32_rwkv6(ggml_backend_vk_context * ctx, vk_context& subc } ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; - ggml_backend_vk_buffer_context * k_buf_ctx = (ggml_backend_vk_buffer_context *)k->buffer->context; - ggml_backend_vk_buffer_context * v_buf_ctx = (ggml_backend_vk_buffer_context *)v->buffer->context; - ggml_backend_vk_buffer_context * r_buf_ctx = (ggml_backend_vk_buffer_context *)r->buffer->context; - ggml_backend_vk_buffer_context * tf_buf_ctx = (ggml_backend_vk_buffer_context *)tf->buffer->context; - ggml_backend_vk_buffer_context * td_buf_ctx = (ggml_backend_vk_buffer_context *)td->buffer->context; - ggml_backend_vk_buffer_context * state_buf_ctx = (ggml_backend_vk_buffer_context *)state->buffer->context; + ggml_backend_vk_buffer_context * src_buf_ctxs[7] = { nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr }; + for (int i = 0; i < num_srcs; i++) { + src_buf_ctxs[i] = (ggml_backend_vk_buffer_context *)dst->src[i]->buffer->context; + } ggml_vk_sync_buffers(subctx); - vk_buffer d_D = nullptr, d_K = nullptr, d_V = nullptr, d_R = nullptr, d_TF = nullptr, d_TD = nullptr, d_State = nullptr; - size_t k_offset = 0, v_offset = 0, r_offset = 0, tf_offset = 0, td_offset = 0, state_offset = 0, dst_offset = 0; - bool K_uma = false, V_uma = false, R_uma = false, TF_uma = false, TD_uma = false, STATE_uma = false, DST_uma = false; + vk_buffer d_D = nullptr, d_srcs[7] = { nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr }; + size_t dst_offset = 0, src_offsets[7] = { 0, 0, 0, 0, 0, 0, 0 }; + bool dst_uma = false, srcs_uma[7] = { false, false, false, false, false, false, false }; if (ctx->device->uma) { - ggml_vk_host_get(ctx->device, k->data, d_K, k_offset); - ggml_vk_host_get(ctx->device, v->data, d_V, v_offset); - ggml_vk_host_get(ctx->device, r->data, d_R, r_offset); - ggml_vk_host_get(ctx->device, tf->data, d_TF, tf_offset); - ggml_vk_host_get(ctx->device, td->data, d_TD, td_offset); - ggml_vk_host_get(ctx->device, state->data, d_State, state_offset); + for (int i = 0; i < num_srcs; i++) { + ggml_vk_host_get(ctx->device, dst->src[i]->data, d_srcs[i], src_offsets[i]); + srcs_uma[i] = d_srcs[i] != nullptr; + } + ggml_vk_host_get(ctx->device, dst->data, d_D, dst_offset); - - K_uma = d_K != nullptr; - V_uma = d_V != nullptr; - R_uma = d_R != nullptr; - TF_uma = d_TF != nullptr; - TD_uma = d_TD != nullptr; - STATE_uma = d_State != nullptr; - DST_uma = d_D != nullptr; + dst_uma = d_D != nullptr; } - if (!K_uma) { - d_K = k_buf_ctx->dev_buffer; - k_offset = vk_tensor_offset(k) + k->view_offs; + uint64_t src_sizes[7] = { 0, 0, 0, 0, 0, 0, 0 }; + for (int i = 0; i < num_srcs; i++) { + src_sizes[i] = ggml_nbytes(dst->src[i]); + if (!srcs_uma[i]) { + d_srcs[i] = src_buf_ctxs[i]->dev_buffer; + src_offsets[i] = vk_tensor_offset(dst->src[i]) + dst->src[i]->view_offs; + } } - if (!V_uma) { - d_V = v_buf_ctx->dev_buffer; - v_offset = vk_tensor_offset(v) + v->view_offs; - } - if (!R_uma) { - d_R = r_buf_ctx->dev_buffer; - r_offset = vk_tensor_offset(r) + r->view_offs; - } - if (!TF_uma) { - d_TF = tf_buf_ctx->dev_buffer; - tf_offset = vk_tensor_offset(tf) + tf->view_offs; - } - if (!TD_uma) { - d_TD = td_buf_ctx->dev_buffer; - td_offset = vk_tensor_offset(td) + td->view_offs; - } - if (!STATE_uma) { - d_State = state_buf_ctx->dev_buffer; - state_offset = vk_tensor_offset(state) + state->view_offs; - } - if (!DST_uma) { + + const uint64_t dst_size = ggml_nbytes(dst); + if (!dst_uma) { d_D = dst_buf_ctx->dev_buffer; dst_offset = vk_tensor_offset(dst) + dst->view_offs; } - const uint64_t k_size = ggml_nbytes(k); - const uint64_t v_size = ggml_nbytes(v); - const uint64_t r_size = ggml_nbytes(r); - const uint64_t tf_size = ggml_nbytes(tf); - const uint64_t td_size = ggml_nbytes(td); - const uint64_t state_size = ggml_nbytes(state); - const uint64_t dst_size = ggml_nbytes(dst); - std::array elements = { (uint32_t)(pc.B * pc.H), 1, 1 }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { - vk_subbuffer{ d_K, k_offset, k_size }, - vk_subbuffer{ d_V, v_offset, v_size }, - vk_subbuffer{ d_R, r_offset, r_size }, - vk_subbuffer{ d_TF, tf_offset, tf_size }, - vk_subbuffer{ d_TD, td_offset, td_size }, - vk_subbuffer{ d_State, state_offset, state_size }, - vk_subbuffer{ d_D, dst_offset, dst_size } - }, sizeof(vk_op_rwkv_wkv6_push_constants), &pc, elements); + if (version == 6) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { + vk_subbuffer{ d_srcs[0], src_offsets[0], src_sizes[0] }, + vk_subbuffer{ d_srcs[1], src_offsets[1], src_sizes[1] }, + vk_subbuffer{ d_srcs[2], src_offsets[2], src_sizes[2] }, + vk_subbuffer{ d_srcs[3], src_offsets[3], src_sizes[3] }, + vk_subbuffer{ d_srcs[4], src_offsets[4], src_sizes[4] }, + vk_subbuffer{ d_srcs[5], src_offsets[5], src_sizes[5] }, + vk_subbuffer{ d_D, dst_offset, dst_size } + }, sizeof(vk_op_rwkv_wkv6_push_constants), &pc, elements); + } else if (version == 7) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { + vk_subbuffer{ d_srcs[0], src_offsets[0], src_sizes[0] }, + vk_subbuffer{ d_srcs[1], src_offsets[1], src_sizes[1] }, + vk_subbuffer{ d_srcs[2], src_offsets[2], src_sizes[2] }, + vk_subbuffer{ d_srcs[3], src_offsets[3], src_sizes[3] }, + vk_subbuffer{ d_srcs[4], src_offsets[4], src_sizes[4] }, + vk_subbuffer{ d_srcs[5], src_offsets[5], src_sizes[5] }, + vk_subbuffer{ d_srcs[6], src_offsets[6], src_sizes[6] }, + vk_subbuffer{ d_D, dst_offset, dst_size } + }, sizeof(vk_op_rwkv_wkv7_push_constants), &pc, elements); + } else { + // shouldn't happen + GGML_ASSERT(false); + } } static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, bool dryrun = false) { @@ -6224,7 +6225,7 @@ static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, const size_t n_heads = dst->src[0]->ne[1]; const size_t n_seqs = dst->src[5]->ne[1]; - ggml_vk_op_f32_rwkv6( + ggml_vk_op_f32_wkv( ctx, subctx, dst, { (uint32_t)n_seqs, @@ -6232,6 +6233,26 @@ static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, (uint32_t)n_embed, (uint32_t)n_heads, }, + 6, + dryrun + ); +} + +static void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, bool dryrun = false) { + const size_t seq_length = dst->src[0]->ne[2]; + const size_t n_embed = dst->ne[0]; + const size_t n_heads = dst->src[0]->ne[1]; + const size_t n_seqs = dst->src[6]->ne[1]; + + ggml_vk_op_f32_wkv( + ctx, subctx, dst, + { + (uint32_t)n_seqs, + (uint32_t)seq_length, + (uint32_t)n_embed, + (uint32_t)n_heads, + }, + 7, dryrun ); } @@ -6533,6 +6554,11 @@ static void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& sub ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }, dryrun); } +static void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst, bool dryrun = false) { + float * op_params = (float *)dst->op_params; + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_L2_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }, dryrun); +} + static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst, bool dryrun = false) { ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }, dryrun); } @@ -7528,6 +7554,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: case GGML_OP_SOFT_MAX_BACK: @@ -7544,6 +7571,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_POOL_2D: case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: case GGML_OP_LEAKY_RELU: case GGML_OP_FLASH_ATTN_EXT: case GGML_OP_OPT_STEP_ADAMW: @@ -7590,6 +7618,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_UNARY: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: @@ -7707,6 +7736,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_RMS_NORM_BACK: ggml_vk_rms_norm_back(ctx, compute_ctx, src0, src1, node, dryrun); + break; + case GGML_OP_L2_NORM: + ggml_vk_l2_norm(ctx, compute_ctx, src0, node, dryrun); + break; case GGML_OP_UNARY: switch (ggml_get_unary_op(node)) { @@ -7797,6 +7830,11 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod break; + case GGML_OP_RWKV_WKV7: + ggml_vk_rwkv_wkv7(ctx, compute_ctx, node, dryrun); + + break; + case GGML_OP_OPT_STEP_ADAMW: ggml_vk_opt_step_adamw(ctx, compute_ctx, node, dryrun); @@ -7870,6 +7908,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_tensor * case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: case GGML_OP_SOFT_MAX_BACK: @@ -7889,6 +7928,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_tensor * case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_POOL_2D: case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: case GGML_OP_LEAKY_RELU: case GGML_OP_REPEAT: case GGML_OP_REPEAT_BACK: @@ -8806,6 +8846,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_NORM: case GGML_OP_GROUP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_ADD: case GGML_OP_SUB: @@ -8835,6 +8876,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_POOL_2D: case GGML_OP_RWKV_WKV6: + case GGML_OP_RWKV_WKV7: case GGML_OP_LEAKY_RELU: case GGML_OP_OPT_STEP_ADAMW: return true; @@ -9219,6 +9261,9 @@ static void ggml_vk_check_results_0(ggml_tensor * tensor) { tensor_clone = ggml_rms_norm_back(ggml_ctx, src_clone[0], src_clone[1], eps); } else if (tensor->op == GGML_OP_SILU_BACK) { tensor_clone = ggml_silu_back(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_L2_NORM) { + const float eps = ((float *) tensor->op_params)[0]; + tensor_clone = ggml_l2_norm(ggml_ctx, src_clone[0], eps); } else if (tensor->op == GGML_OP_SOFT_MAX) { if (src1 != nullptr) { tensor_clone = ggml_soft_max_ext(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); @@ -9338,6 +9383,9 @@ static void ggml_vk_check_results_0(ggml_tensor * tensor) { } else if (tensor->op == GGML_OP_RWKV_WKV6) { tensor_clone = ggml_rwkv_wkv6(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], src_clone[4], src_clone[5]); + } else if (tensor->op == GGML_OP_RWKV_WKV7) { + tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], + src_clone[4], src_clone[5], src_clone[6]); } else if (tensor->op == GGML_OP_OPT_STEP_ADAMW) { src_clone[0]->flags = src0->flags; tensor_clone = ggml_opt_step_adamw(ggml_ctx, src_clone[0], src_clone[1], diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp new file mode 100644 index 0000000000..deba8c3985 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp @@ -0,0 +1,41 @@ +#version 450 + +#include "generic_head.comp" +#include "types.comp" + +#extension GL_EXT_control_flow_attributes : enable +#define BLOCK_SIZE 512 + +layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; + +shared FLOAT_TYPE sum[BLOCK_SIZE]; + +void main() { + const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; + const uint tid = gl_LocalInvocationID.x; + + sum[tid] = FLOAT_TYPE(0.0f); // partial sum for thread in warp + + [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + const FLOAT_TYPE xi = FLOAT_TYPE(data_a[row*p.KX + col]); + sum[tid] += xi * xi; + } + + // sum up partial sums and write back result + barrier(); + [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) { + sum[tid] += sum[tid + s]; + } + barrier(); + } + + const FLOAT_TYPE scale = inversesqrt(max(sum[0], FLOAT_TYPE(p.param1))); + + [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + data_d[row*p.KX + col] = D_TYPE(scale * FLOAT_TYPE(data_a[row*p.KX + col])); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index ee1fec4e11..eb2ad63ff6 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -434,6 +434,7 @@ void process_shaders() { string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rms_norm_back_f32", "rms_norm_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); + string_to_spv("l2_norm_f32", "l2_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("cpy_f32_f32", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("cpy_f32_f16", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float16_t"}}); @@ -528,6 +529,8 @@ void process_shaders() { string_to_spv("rwkv_wkv6_f32", "wkv6.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); + string_to_spv("rwkv_wkv7_f32", "wkv7.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); + string_to_spv("opt_step_adamw_f32", "opt_step_adamw.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); for (auto &c : compiles) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp b/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp new file mode 100644 index 0000000000..88c1c02b32 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp @@ -0,0 +1,91 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : require + +#define BLOCK_SIZE 64 +layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; + +layout(push_constant) uniform Parameters { + uint B; + uint T; + uint C; + uint H; +}; + +layout(binding = 0) readonly buffer RBuf { A_TYPE r[]; }; +layout(binding = 1) readonly buffer WBuf { A_TYPE w[]; }; +layout(binding = 2) readonly buffer KBuf { A_TYPE k[]; }; +layout(binding = 3) readonly buffer VBuf { A_TYPE v[]; }; +layout(binding = 4) readonly buffer ABuf { A_TYPE a[]; }; +layout(binding = 5) readonly buffer BBuf { A_TYPE b[]; }; +layout(binding = 6) readonly buffer StateBuf { A_TYPE state_in[]; }; +layout(binding = 7) buffer DstBuf { A_TYPE dst[]; }; + +shared A_TYPE _r[BLOCK_SIZE], _w[BLOCK_SIZE], _k[BLOCK_SIZE], _a[BLOCK_SIZE], _b[BLOCK_SIZE]; + +void main() { + const uint head_size = BLOCK_SIZE; + const uint batch_id = gl_WorkGroupID.x / H; + const uint head_id = gl_WorkGroupID.x % H; + const uint tid = gl_LocalInvocationID.x; + + const uint state_size = C * head_size; + const uint n_seq_tokens = T / B; + + if (batch_id >= B || head_id >= H) { + return; + } + + A_TYPE state[BLOCK_SIZE]; + [[unroll]] for (uint i = 0; i < head_size; i++) { + state[i] = state_in[batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i]; + } + + const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; + const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; + + for (uint t = start_t; t < end_t; t += C) { + barrier(); + _r[tid] = r[t]; + _w[tid] = w[t]; + _k[tid] = k[t]; + _a[tid] = a[t]; + _b[tid] = b[t]; + barrier(); + + A_TYPE sa = 0.0; + [[unroll]] for (uint j = 0; j < head_size; j += 4) { + vec4 s_vec = vec4(state[j], state[j+1], state[j+2], state[j+3]); + vec4 a_vec = vec4(_a[j], _a[j+1], _a[j+2], _a[j+3]); + sa += dot(s_vec, a_vec); + } + + const A_TYPE v_val = v[t]; + A_TYPE y = 0.0; + + [[unroll]] for (uint j = 0; j < head_size; j += 4) { + vec4 r_vec = vec4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + vec4 w_vec = vec4(_w[j], _w[j+1], _w[j+2], _w[j+3]); + vec4 k_vec = vec4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + vec4 b_vec = vec4(_b[j], _b[j+1], _b[j+2], _b[j+3]); + vec4 s_vec = vec4(state[j], state[j+1], state[j+2], state[j+3]); + + vec4 kv = k_vec * v_val; + s_vec = s_vec * w_vec + kv + sa * b_vec; + y += dot(r_vec, s_vec); + + state[j] = s_vec.x; + state[j+1] = s_vec.y; + state[j+2] = s_vec.z; + state[j+3] = s_vec.w; + } + + dst[t] = y; + } + + [[unroll]] for (uint i = 0; i < head_size; i++) { + dst[T * C + batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i] = state[i]; + } +} diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 89409bb0e4..2e081d5910 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -929,6 +929,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "RMS_NORM", "RMS_NORM_BACK", "GROUP_NORM", + "L2_NORM", "MUL_MAT", "MUL_MAT_ID", @@ -977,6 +978,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "ADD_REL_POS", "RWKV_WKV6", "GATED_LINEAR_ATTN", + "RWKV_WKV7", "UNARY", @@ -996,7 +998,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "OPT_STEP_ADAMW", }; -static_assert(GGML_OP_COUNT == 83, "GGML_OP_COUNT != 83"); +static_assert(GGML_OP_COUNT == 85, "GGML_OP_COUNT != 85"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1026,6 +1028,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "rms_norm(x)", "rms_norm_back(x)", "group_norm(x)", + "l2_norm(x)", "X*Y", "X[i]*Y", @@ -1074,6 +1077,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "add_rel_pos(x)", "rwkv_wkv6(k, v, r, tf, td, s)", "gated_linear_attn(k, v, q, gate, s)", + "rwkv_wkv7(r, w, k, v, a, b, s)", "unary(x)", @@ -1093,7 +1097,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "adamw(x)", }; -static_assert(GGML_OP_COUNT == 83, "GGML_OP_COUNT != 83"); +static_assert(GGML_OP_COUNT == 85, "GGML_OP_COUNT != 85"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -2686,6 +2690,37 @@ struct ggml_tensor * ggml_group_norm_inplace( return ggml_group_norm_impl(ctx, a, n_groups, eps, true); } +// ggml_l2_norm + +static struct ggml_tensor * ggml_l2_norm_impl( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps, + bool inplace) { + struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); + + ggml_set_op_params_f32(result, 0, eps); + + result->op = GGML_OP_L2_NORM; + result->src[0] = a; + + return result; +} + +struct ggml_tensor * ggml_l2_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps) { + return ggml_l2_norm_impl(ctx, a, eps, false); +} + +struct ggml_tensor * ggml_l2_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps) { + return ggml_l2_norm_impl(ctx, a, eps, true); +} + // ggml_mul_mat static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { @@ -4720,6 +4755,54 @@ struct ggml_tensor * ggml_gated_linear_attn( return result; } +// ggml_rwkv_wkv7 + +struct ggml_tensor * ggml_rwkv_wkv7( + struct ggml_context * ctx, + struct ggml_tensor * r, + struct ggml_tensor * w, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * state) { + GGML_ASSERT(ggml_is_contiguous(r)); + GGML_ASSERT(ggml_is_contiguous(w)); + GGML_ASSERT(ggml_is_contiguous(k)); + GGML_ASSERT(ggml_is_contiguous(v)); + GGML_ASSERT(ggml_is_contiguous(a)); + GGML_ASSERT(ggml_is_contiguous(b)); + GGML_ASSERT(ggml_is_contiguous(state)); + + const int64_t S = k->ne[0]; + const int64_t H = k->ne[1]; + const int64_t n_tokens = k->ne[2]; + const int64_t n_seqs = state->ne[1]; + { + GGML_ASSERT(w->ne[0] == S && w->ne[1] == H && w->ne[2] == n_tokens); + GGML_ASSERT(k->ne[0] == S && k->ne[1] == H && k->ne[2] == n_tokens); + GGML_ASSERT(v->ne[0] == S && v->ne[1] == H && v->ne[2] == n_tokens); + GGML_ASSERT(a->ne[0] == S && a->ne[1] == H && a->ne[2] == n_tokens); + GGML_ASSERT(b->ne[0] == S && b->ne[1] == H && b->ne[2] == n_tokens); + GGML_ASSERT(ggml_nelements(state) == S * S * H * n_seqs); + } + + // concat output and new_state + const int64_t ne[4] = { S * H, n_tokens + S * n_seqs, 1, 1 }; + struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); + + result->op = GGML_OP_RWKV_WKV7; + result->src[0] = r; + result->src[1] = w; + result->src[2] = k; + result->src[3] = v; + result->src[4] = a; + result->src[5] = b; + result->src[6] = state; + + return result; +} + // ggml_unary static struct ggml_tensor * ggml_unary_impl( diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 19624eae04..cc48913d97 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -118,22 +118,26 @@ class Keys: TOKEN_SHIFT_COUNT = "{arch}.token_shift_count" class Attention: - HEAD_COUNT = "{arch}.attention.head_count" - HEAD_COUNT_KV = "{arch}.attention.head_count_kv" - MAX_ALIBI_BIAS = "{arch}.attention.max_alibi_bias" - CLAMP_KQV = "{arch}.attention.clamp_kqv" - KEY_LENGTH = "{arch}.attention.key_length" - VALUE_LENGTH = "{arch}.attention.value_length" - LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon" - LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon" - GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon" - GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups" - CAUSAL = "{arch}.attention.causal" - Q_LORA_RANK = "{arch}.attention.q_lora_rank" - KV_LORA_RANK = "{arch}.attention.kv_lora_rank" - REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count" - SLIDING_WINDOW = "{arch}.attention.sliding_window" - SCALE = "{arch}.attention.scale" + HEAD_COUNT = "{arch}.attention.head_count" + HEAD_COUNT_KV = "{arch}.attention.head_count_kv" + MAX_ALIBI_BIAS = "{arch}.attention.max_alibi_bias" + CLAMP_KQV = "{arch}.attention.clamp_kqv" + KEY_LENGTH = "{arch}.attention.key_length" + VALUE_LENGTH = "{arch}.attention.value_length" + LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon" + LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon" + GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon" + GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups" + CAUSAL = "{arch}.attention.causal" + Q_LORA_RANK = "{arch}.attention.q_lora_rank" + KV_LORA_RANK = "{arch}.attention.kv_lora_rank" + DECAY_LORA_RANK = "{arch}.attention.decay_lora_rank" + ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank" + VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank" + GATE_LORA_RANK = "{arch}.attention.gate_lora_rank" + REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count" + SLIDING_WINDOW = "{arch}.attention.sliding_window" + SCALE = "{arch}.attention.scale" class Rope: DIMENSION_COUNT = "{arch}.rope.dimension_count" @@ -257,6 +261,8 @@ class MODEL_ARCH(IntEnum): STARCODER2 = auto() RWKV6 = auto() RWKV6QWEN2 = auto() + RWKV7 = auto() + ARWKV7 = auto() MAMBA = auto() XVERSE = auto() COMMAND_R = auto() @@ -329,8 +335,20 @@ class MODEL_TENSOR(IntEnum): SSM_A = auto() SSM_D = auto() SSM_OUT = auto() + TIME_MIX_W0 = auto() TIME_MIX_W1 = auto() TIME_MIX_W2 = auto() + TIME_MIX_A0 = auto() + TIME_MIX_A1 = auto() + TIME_MIX_A2 = auto() + TIME_MIX_V0 = auto() + TIME_MIX_V1 = auto() + TIME_MIX_V2 = auto() + TIME_MIX_G1 = auto() + TIME_MIX_G2 = auto() + TIME_MIX_K_K = auto() + TIME_MIX_K_A = auto() + TIME_MIX_R_K = auto() TIME_MIX_LERP_X = auto() TIME_MIX_LERP_K = auto() TIME_MIX_LERP_V = auto() @@ -445,6 +463,8 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.STARCODER2: "starcoder2", MODEL_ARCH.RWKV6: "rwkv6", MODEL_ARCH.RWKV6QWEN2: "rwkv6qwen2", + MODEL_ARCH.RWKV7: "rwkv7", + MODEL_ARCH.ARWKV7: "arwkv7", MODEL_ARCH.MAMBA: "mamba", MODEL_ARCH.XVERSE: "xverse", MODEL_ARCH.COMMAND_R: "command-r", @@ -517,8 +537,20 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.SSM_A: "blk.{bid}.ssm_a", MODEL_TENSOR.SSM_D: "blk.{bid}.ssm_d", MODEL_TENSOR.SSM_OUT: "blk.{bid}.ssm_out", + MODEL_TENSOR.TIME_MIX_W0: "blk.{bid}.time_mix_w0", MODEL_TENSOR.TIME_MIX_W1: "blk.{bid}.time_mix_w1", MODEL_TENSOR.TIME_MIX_W2: "blk.{bid}.time_mix_w2", + MODEL_TENSOR.TIME_MIX_A0: "blk.{bid}.time_mix_a0", + MODEL_TENSOR.TIME_MIX_A1: "blk.{bid}.time_mix_a1", + MODEL_TENSOR.TIME_MIX_A2: "blk.{bid}.time_mix_a2", + MODEL_TENSOR.TIME_MIX_V0: "blk.{bid}.time_mix_v0", + MODEL_TENSOR.TIME_MIX_V1: "blk.{bid}.time_mix_v1", + MODEL_TENSOR.TIME_MIX_V2: "blk.{bid}.time_mix_v2", + MODEL_TENSOR.TIME_MIX_G1: "blk.{bid}.time_mix_g1", + MODEL_TENSOR.TIME_MIX_G2: "blk.{bid}.time_mix_g2", + MODEL_TENSOR.TIME_MIX_K_K: "blk.{bid}.time_mix_k_k", + MODEL_TENSOR.TIME_MIX_K_A: "blk.{bid}.time_mix_k_a", + MODEL_TENSOR.TIME_MIX_R_K: "blk.{bid}.time_mix_r_k", MODEL_TENSOR.TIME_MIX_LERP_X: "blk.{bid}.time_mix_lerp_x", MODEL_TENSOR.TIME_MIX_LERP_K: "blk.{bid}.time_mix_lerp_k", MODEL_TENSOR.TIME_MIX_LERP_V: "blk.{bid}.time_mix_lerp_v", @@ -1172,6 +1204,68 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.RWKV7: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_NORM_2, + MODEL_TENSOR.TIME_MIX_LERP_FUSED, + MODEL_TENSOR.TIME_MIX_W0, + MODEL_TENSOR.TIME_MIX_W1, + MODEL_TENSOR.TIME_MIX_W2, + MODEL_TENSOR.TIME_MIX_A0, + MODEL_TENSOR.TIME_MIX_A1, + MODEL_TENSOR.TIME_MIX_A2, + MODEL_TENSOR.TIME_MIX_V0, + MODEL_TENSOR.TIME_MIX_V1, + MODEL_TENSOR.TIME_MIX_V2, + MODEL_TENSOR.TIME_MIX_G1, + MODEL_TENSOR.TIME_MIX_G2, + MODEL_TENSOR.TIME_MIX_K_K, + MODEL_TENSOR.TIME_MIX_K_A, + MODEL_TENSOR.TIME_MIX_R_K, + MODEL_TENSOR.TIME_MIX_KEY, + MODEL_TENSOR.TIME_MIX_VALUE, + MODEL_TENSOR.TIME_MIX_RECEPTANCE, + MODEL_TENSOR.TIME_MIX_LN, + MODEL_TENSOR.TIME_MIX_OUTPUT, + MODEL_TENSOR.CHANNEL_MIX_LERP_K, + MODEL_TENSOR.CHANNEL_MIX_KEY, + MODEL_TENSOR.CHANNEL_MIX_VALUE, + ], + MODEL_ARCH.ARWKV7: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.TIME_MIX_LERP_FUSED, + MODEL_TENSOR.TIME_MIX_W0, + MODEL_TENSOR.TIME_MIX_W1, + MODEL_TENSOR.TIME_MIX_W2, + MODEL_TENSOR.TIME_MIX_A0, + MODEL_TENSOR.TIME_MIX_A1, + MODEL_TENSOR.TIME_MIX_A2, + MODEL_TENSOR.TIME_MIX_V0, + MODEL_TENSOR.TIME_MIX_V1, + MODEL_TENSOR.TIME_MIX_V2, + MODEL_TENSOR.TIME_MIX_G1, + MODEL_TENSOR.TIME_MIX_G2, + MODEL_TENSOR.TIME_MIX_K_K, + MODEL_TENSOR.TIME_MIX_K_A, + MODEL_TENSOR.TIME_MIX_R_K, + MODEL_TENSOR.TIME_MIX_KEY, + MODEL_TENSOR.TIME_MIX_VALUE, + MODEL_TENSOR.TIME_MIX_RECEPTANCE, + MODEL_TENSOR.TIME_MIX_LN, + MODEL_TENSOR.TIME_MIX_OUTPUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], MODEL_ARCH.MAMBA: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 080d2b9dce..af8b388dfa 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -767,6 +767,18 @@ class GGUFWriter: def add_kv_lora_rank(self, length: int) -> None: self.add_uint32(Keys.Attention.KV_LORA_RANK.format(arch=self.arch), length) + def add_decay_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.DECAY_LORA_RANK.format(arch=self.arch), length) + + def add_iclr_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.ICLR_LORA_RANK.format(arch=self.arch), length) + + def add_value_residual_mix_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.VALUE_RESIDUAL_MIX_LORA_RANK.format(arch=self.arch), length) + + def add_gate_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.GATE_LORA_RANK.format(arch=self.arch), length) + def add_relative_attn_buckets_count(self, value: int) -> None: self.add_uint32(Keys.Attention.REL_BUCKETS_COUNT.format(arch=self.arch), value) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 617791e240..8d4a2b0320 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -27,7 +27,8 @@ class TensorNameMap: "embedding.word_embeddings", # chatglm "transformer.token_embeddings", # openelm "shared", # t5 - "rwkv.embeddings", # rwkv + "rwkv.embeddings", # rwkv6 + "model.embeddings", # rwkv7 ), # Token type embeddings @@ -42,6 +43,9 @@ class TensorNameMap: "emb_ln", # nomic-bert "transformer.norm", # openelm "rwkv.blocks.0.pre_ln", # rwkv + "rwkv.blocks.0.pre_ln", # rwkv6 + "model.pre_ln", # rwkv7 + "model.layers.0.pre_norm", # rwkv7 "backbone.norm", # wavtokenizer ), @@ -81,7 +85,8 @@ class TensorNameMap: "encoder.final_layernorm", # chatglm "transformer.norm", # openelm "model.norm", # nemotron - "rwkv.ln_out", # rwkv + "rwkv.ln_out", # rwkv6 + "model.ln_out", # rwkv7 "backbone.final_layer_norm", # wavtokenizer ), @@ -122,14 +127,16 @@ class TensorNameMap: "transformer.blocks.{bid}.norm_attn_norm.norm_1", # dbrx "encoder.layers.{bid}.input_layernorm", # chatglm "transformer.layers.{bid}.attn_norm", # openelm - "rwkv.blocks.{bid}.ln1", # rwkv + "rwkv.blocks.{bid}.ln1", # rwkv6 + "model.layers.{bid}.ln1", # rwkv7 ), # Attention norm 2 MODEL_TENSOR.ATTN_NORM_2: ( "transformer.h.{bid}.ln_attn", # falcon40b "encoder.layer.{bid}.layer_norm_1", # jina-v2-code - "rwkv.blocks.{bid}.ln2", # rwkv + "rwkv.blocks.{bid}.ln2", # rwkv6 + "model.layers.{bid}.ln2", # rwkv7 ), # Attention query-key-value @@ -462,112 +469,174 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.out_proj", ), + MODEL_TENSOR.TIME_MIX_W0: ( + "model.layers.{bid}.attention.w0", # rwkv7 + ), + MODEL_TENSOR.TIME_MIX_W1: ( - "rwkv.blocks.{bid}.attention.time_maa_w1", # rwkv v6 - "model.layers.{bid}.self_attn.time_maa_w1", # rwkv6qwen2 + "rwkv.blocks.{bid}.attention.time_maa_w1", # rwkv6 + "model.layers.{bid}.self_attn.time_maa_w1", # rwkv6qwen2 + "model.layers.{bid}.attention.w1", # rwkv7 ), MODEL_TENSOR.TIME_MIX_W2: ( - "rwkv.blocks.{bid}.attention.time_maa_w2", # rwkv v6 - "model.layers.{bid}.self_attn.time_maa_w2", # rwkv6qwen2 + "rwkv.blocks.{bid}.attention.time_maa_w2", # rwkv6 + "model.layers.{bid}.self_attn.time_maa_w2", # rwkv6qwen2 + "model.layers.{bid}.attention.w2", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_A0: ( + "model.layers.{bid}.attention.a0", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_A1: ( + "model.layers.{bid}.attention.a1", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_A2: ( + "model.layers.{bid}.attention.a2", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_V0: ( + "model.layers.{bid}.attention.v0", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_V1: ( + "model.layers.{bid}.attention.v1", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_V2: ( + "model.layers.{bid}.attention.v2", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_G1: ( + "model.layers.{bid}.attention.g1", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_G2: ( + "model.layers.{bid}.attention.g2", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_K_K: ( + "model.layers.{bid}.attention.k_k", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_K_A: ( + "model.layers.{bid}.attention.k_a", # rwkv7 + ), + + MODEL_TENSOR.TIME_MIX_R_K: ( + "model.layers.{bid}.attention.r_k", # rwkv7 ), MODEL_TENSOR.TIME_MIX_LERP_X: ( - "rwkv.blocks.{bid}.attention.time_maa_x", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_x", # rwkv6 "model.layers.{bid}.self_attn.time_maa_x", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_K: ( - "rwkv.blocks.{bid}.attention.time_maa_k", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_k", # rwkv6 "model.layers.{bid}.self_attn.time_maa_k", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_V: ( - "rwkv.blocks.{bid}.attention.time_maa_v", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_v", # rwkv6 "model.layers.{bid}.self_attn.time_maa_v", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_R: ( - "rwkv.blocks.{bid}.attention.time_maa_r", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_r", # rwkv6 "model.layers.{bid}.self_attn.time_maa_r", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_G: ( - "rwkv.blocks.{bid}.attention.time_maa_g", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_g", # rwkv6 "model.layers.{bid}.self_attn.time_maa_g", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LERP_W: ( - "rwkv.blocks.{bid}.attention.time_maa_w", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_maa_w", # rwkv6 "model.layers.{bid}.self_attn.time_maa_w", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_FIRST: ( - "rwkv.blocks.{bid}.attention.time_faaaa", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_faaaa", # rwkv6 ), MODEL_TENSOR.TIME_MIX_DECAY: ( - "rwkv.blocks.{bid}.attention.time_decay", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_decay", # rwkv6 "model.layers.{bid}.self_attn.time_decay", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_DECAY_W1: ( - "rwkv.blocks.{bid}.attention.time_decay_w1", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_decay_w1", # rwkv6 "model.layers.{bid}.self_attn.time_decay_w1", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_DECAY_W2: ( - "rwkv.blocks.{bid}.attention.time_decay_w2", # rwkv v6 + "rwkv.blocks.{bid}.attention.time_decay_w2", # rwkv6 "model.layers.{bid}.self_attn.time_decay_w2", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_KEY: ( - "rwkv.blocks.{bid}.attention.key", # rwkv + "rwkv.blocks.{bid}.attention.key", # rwkv6 "model.layers.{bid}.self_attn.k_proj", # rwkv6qwen2 + "model.layers.{bid}.attention.key", # rwkv7 + "model.layers.{bid}.attention.k_proj", # rwkv7 ), MODEL_TENSOR.TIME_MIX_VALUE: ( - "rwkv.blocks.{bid}.attention.value", # rwkv + "rwkv.blocks.{bid}.attention.value", # rwkv6 "model.layers.{bid}.self_attn.v_proj", # rwkv6qwen2 + "model.layers.{bid}.attention.value", # rwkv7 + "model.layers.{bid}.attention.v_proj", # rwkv7 ), MODEL_TENSOR.TIME_MIX_RECEPTANCE: ( - "rwkv.blocks.{bid}.attention.receptance", # rwkv - "model.layers.{bid}.self_attn.q_proj", # rwkv6qwen2 + "rwkv.blocks.{bid}.attention.receptance", # rwkv6 + "model.layers.{bid}.self_attn.q_proj", # rwkv6qwen2 + "model.layers.{bid}.attention.receptance", # rwkv7 + "model.layers.{bid}.attention.r_proj", # rwkv7 ), MODEL_TENSOR.TIME_MIX_GATE: ( - "rwkv.blocks.{bid}.attention.gate", # rwkv - "model.layers.{bid}.self_attn.gate", # rwkv6qwen2 + "rwkv.blocks.{bid}.attention.gate", # rwkv6 + "model.layers.{bid}.self_attn.gate", # rwkv6qwen2 ), MODEL_TENSOR.TIME_MIX_LN: ( - "rwkv.blocks.{bid}.attention.ln_x", # rwkv + "rwkv.blocks.{bid}.attention.ln_x", # rwkv6 + "model.layers.{bid}.attention.ln_x" # rwkv7 ), MODEL_TENSOR.TIME_MIX_OUTPUT: ( - "rwkv.blocks.{bid}.attention.output", # rwkv + "rwkv.blocks.{bid}.attention.output", # rwkv6 "model.layers.{bid}.self_attn.o_proj", # rwkv6qwen2 + "model.layers.{bid}.attention.output", # rwkv7 + "model.layers.{bid}.attention.o_proj", # rwkv7 ), MODEL_TENSOR.CHANNEL_MIX_LERP_K: ( - "rwkv.blocks.{bid}.feed_forward.time_maa_k", # rwkv v6 + "rwkv.blocks.{bid}.feed_forward.time_maa_k", # rwkv6 + "model.layers.{bid}.feed_forward.x_k", # rwkv7 ), MODEL_TENSOR.CHANNEL_MIX_LERP_R: ( - "rwkv.blocks.{bid}.feed_forward.time_maa_r", # rwkv v6 + "rwkv.blocks.{bid}.feed_forward.time_maa_r", # rwkv6 ), MODEL_TENSOR.CHANNEL_MIX_KEY: ( - "rwkv.blocks.{bid}.feed_forward.key", # rwkv + "rwkv.blocks.{bid}.feed_forward.key", # rwkv6 + "model.layers.{bid}.feed_forward.key", # rwkv7 ), MODEL_TENSOR.CHANNEL_MIX_RECEPTANCE: ( - "rwkv.blocks.{bid}.feed_forward.receptance", # rwkv + "rwkv.blocks.{bid}.feed_forward.receptance", # rwkv6 ), MODEL_TENSOR.CHANNEL_MIX_VALUE: ( - "rwkv.blocks.{bid}.feed_forward.value", # rwkv + "rwkv.blocks.{bid}.feed_forward.value", # rwkv6 + "model.layers.{bid}.feed_forward.value", # rwkv7 ), MODEL_TENSOR.ATTN_Q_A: ( diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 28f2bbc8f7..9debb56cc8 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -59,6 +59,8 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_EXAONE, "exaone" }, { LLM_ARCH_RWKV6, "rwkv6" }, { LLM_ARCH_RWKV6QWEN2, "rwkv6qwen2" }, + { LLM_ARCH_RWKV7, "rwkv7" }, + { LLM_ARCH_ARWKV7, "arwkv7" }, { LLM_ARCH_GRANITE, "granite" }, { LLM_ARCH_GRANITE_MOE, "granitemoe" }, { LLM_ARCH_CHAMELEON, "chameleon" }, @@ -110,22 +112,26 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_EMBEDDING_SCALE, "%s.embedding_scale" }, { LLM_KV_TOKEN_SHIFT_COUNT, "%s.token_shift_count" }, - { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, - { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, - { LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" }, - { LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" }, - { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" }, - { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" }, - { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" }, - { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" }, - { LLM_KV_ATTENTION_GROUPNORM_EPS, "%s.attention.group_norm_epsilon" }, - { LLM_KV_ATTENTION_GROUPNORM_GROUPS, "%s.attention.group_norm_groups" }, - { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" }, - { LLM_KV_ATTENTION_Q_LORA_RANK, "%s.attention.q_lora_rank" }, - { LLM_KV_ATTENTION_KV_LORA_RANK, "%s.attention.kv_lora_rank" }, - { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" }, - { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" }, - { LLM_KV_ATTENTION_SCALE, "%s.attention.scale" }, + { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, + { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, + { LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" }, + { LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" }, + { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" }, + { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" }, + { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" }, + { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" }, + { LLM_KV_ATTENTION_GROUPNORM_EPS, "%s.attention.group_norm_epsilon" }, + { LLM_KV_ATTENTION_GROUPNORM_GROUPS, "%s.attention.group_norm_groups" }, + { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" }, + { LLM_KV_ATTENTION_Q_LORA_RANK, "%s.attention.q_lora_rank" }, + { LLM_KV_ATTENTION_KV_LORA_RANK, "%s.attention.kv_lora_rank" }, + { LLM_KV_ATTENTION_DECAY_LORA_RANK, "%s.attention.decay_lora_rank" }, + { LLM_KV_ATTENTION_ICLR_LORA_RANK, "%s.attention.iclr_lora_rank" }, + { LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, "%s.attention.value_residual_mix_lora_rank" }, + { LLM_KV_ATTENTION_GATE_LORA_RANK, "%s.attention.gate_lora_rank" }, + { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" }, + { LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" }, + { LLM_KV_ATTENTION_SCALE, "%s.attention.scale" }, { LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" }, { LLM_KV_ROPE_DIMENSION_SECTIONS, "%s.rope.dimension_sections" }, @@ -1238,6 +1244,74 @@ static const std::map> LLM_TENSOR_N { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_RWKV7, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" }, + { LLM_TENSOR_TIME_MIX_W0, "blk.%d.time_mix_w0" }, + { LLM_TENSOR_TIME_MIX_W1, "blk.%d.time_mix_w1" }, + { LLM_TENSOR_TIME_MIX_W2, "blk.%d.time_mix_w2" }, + { LLM_TENSOR_TIME_MIX_A0, "blk.%d.time_mix_a0" }, + { LLM_TENSOR_TIME_MIX_A1, "blk.%d.time_mix_a1" }, + { LLM_TENSOR_TIME_MIX_A2, "blk.%d.time_mix_a2" }, + { LLM_TENSOR_TIME_MIX_V0, "blk.%d.time_mix_v0" }, + { LLM_TENSOR_TIME_MIX_V1, "blk.%d.time_mix_v1" }, + { LLM_TENSOR_TIME_MIX_V2, "blk.%d.time_mix_v2" }, + { LLM_TENSOR_TIME_MIX_G1, "blk.%d.time_mix_g1" }, + { LLM_TENSOR_TIME_MIX_G2, "blk.%d.time_mix_g2" }, + { LLM_TENSOR_TIME_MIX_K_K, "blk.%d.time_mix_k_k" }, + { LLM_TENSOR_TIME_MIX_K_A, "blk.%d.time_mix_k_a" }, + { LLM_TENSOR_TIME_MIX_R_K, "blk.%d.time_mix_r_k" }, + { LLM_TENSOR_TIME_MIX_LERP_FUSED, "blk.%d.time_mix_lerp_fused" }, + { LLM_TENSOR_TIME_MIX_KEY, "blk.%d.time_mix_key" }, + { LLM_TENSOR_TIME_MIX_VALUE, "blk.%d.time_mix_value" }, + { LLM_TENSOR_TIME_MIX_RECEPTANCE, "blk.%d.time_mix_receptance" }, + { LLM_TENSOR_TIME_MIX_LN, "blk.%d.time_mix_ln" }, + { LLM_TENSOR_TIME_MIX_OUTPUT, "blk.%d.time_mix_output" }, + { LLM_TENSOR_CHANNEL_MIX_LERP_K, "blk.%d.channel_mix_lerp_k" }, + { LLM_TENSOR_CHANNEL_MIX_KEY, "blk.%d.channel_mix_key" }, + { LLM_TENSOR_CHANNEL_MIX_VALUE, "blk.%d.channel_mix_value" }, + }, + }, + { + LLM_ARCH_ARWKV7, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_TIME_MIX_W0, "blk.%d.time_mix_w0" }, + { LLM_TENSOR_TIME_MIX_W1, "blk.%d.time_mix_w1" }, + { LLM_TENSOR_TIME_MIX_W2, "blk.%d.time_mix_w2" }, + { LLM_TENSOR_TIME_MIX_A0, "blk.%d.time_mix_a0" }, + { LLM_TENSOR_TIME_MIX_A1, "blk.%d.time_mix_a1" }, + { LLM_TENSOR_TIME_MIX_A2, "blk.%d.time_mix_a2" }, + { LLM_TENSOR_TIME_MIX_V0, "blk.%d.time_mix_v0" }, + { LLM_TENSOR_TIME_MIX_V1, "blk.%d.time_mix_v1" }, + { LLM_TENSOR_TIME_MIX_V2, "blk.%d.time_mix_v2" }, + { LLM_TENSOR_TIME_MIX_G1, "blk.%d.time_mix_g1" }, + { LLM_TENSOR_TIME_MIX_G2, "blk.%d.time_mix_g2" }, + { LLM_TENSOR_TIME_MIX_K_K, "blk.%d.time_mix_k_k" }, + { LLM_TENSOR_TIME_MIX_K_A, "blk.%d.time_mix_k_a" }, + { LLM_TENSOR_TIME_MIX_R_K, "blk.%d.time_mix_r_k" }, + { LLM_TENSOR_TIME_MIX_LERP_FUSED, "blk.%d.time_mix_lerp_fused" }, + { LLM_TENSOR_TIME_MIX_KEY, "blk.%d.time_mix_key" }, + { LLM_TENSOR_TIME_MIX_VALUE, "blk.%d.time_mix_value" }, + { LLM_TENSOR_TIME_MIX_RECEPTANCE, "blk.%d.time_mix_receptance" }, + { LLM_TENSOR_TIME_MIX_LN, "blk.%d.time_mix_ln" }, + { LLM_TENSOR_TIME_MIX_OUTPUT, "blk.%d.time_mix_output" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + }, + }, { LLM_ARCH_GRANITE, { @@ -1397,6 +1471,12 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_SSM_OUT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_W1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_W2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_A1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_A2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_V1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_V2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_G1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_TIME_MIX_G2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_DECAY_W1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_DECAY_W2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_KEY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, @@ -1415,6 +1495,9 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_TIME_MIX_LN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_CHANNEL_MIX_LERP_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_CHANNEL_MIX_LERP_R, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_TIME_MIX_K_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_TIME_MIX_K_A, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_TIME_MIX_R_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_TIME_MIX_LERP_W, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_LERP_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_LERP_V, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, @@ -1422,6 +1505,9 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_TIME_MIX_LERP_G, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_LERP_FUSED, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_DECAY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_TIME_MIX_W0, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_TIME_MIX_A0, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_TIME_MIX_V0, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_TIME_MIX_FIRST, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_RWKV_WKV6}}, {LLM_TENSOR_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_ATTN_NORM_2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, diff --git a/src/llama-arch.h b/src/llama-arch.h index 2ec2e2362e..a28815d8a1 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -63,6 +63,8 @@ enum llm_arch { LLM_ARCH_EXAONE, LLM_ARCH_RWKV6, LLM_ARCH_RWKV6QWEN2, + LLM_ARCH_RWKV7, + LLM_ARCH_ARWKV7, LLM_ARCH_GRANITE, LLM_ARCH_GRANITE_MOE, LLM_ARCH_CHAMELEON, @@ -127,6 +129,10 @@ enum llm_kv { LLM_KV_ATTENTION_CAUSAL, LLM_KV_ATTENTION_Q_LORA_RANK, LLM_KV_ATTENTION_KV_LORA_RANK, + LLM_KV_ATTENTION_DECAY_LORA_RANK, + LLM_KV_ATTENTION_ICLR_LORA_RANK, + LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, + LLM_KV_ATTENTION_GATE_LORA_RANK, LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, LLM_KV_ATTENTION_SLIDING_WINDOW, LLM_KV_ATTENTION_SCALE, @@ -250,8 +256,20 @@ enum llm_tensor { LLM_TENSOR_SSM_A, LLM_TENSOR_SSM_D, LLM_TENSOR_SSM_OUT, + LLM_TENSOR_TIME_MIX_W0, LLM_TENSOR_TIME_MIX_W1, LLM_TENSOR_TIME_MIX_W2, + LLM_TENSOR_TIME_MIX_A0, + LLM_TENSOR_TIME_MIX_A1, + LLM_TENSOR_TIME_MIX_A2, + LLM_TENSOR_TIME_MIX_V0, + LLM_TENSOR_TIME_MIX_V1, + LLM_TENSOR_TIME_MIX_V2, + LLM_TENSOR_TIME_MIX_G1, + LLM_TENSOR_TIME_MIX_G2, + LLM_TENSOR_TIME_MIX_K_K, + LLM_TENSOR_TIME_MIX_K_A, + LLM_TENSOR_TIME_MIX_R_K, LLM_TENSOR_TIME_MIX_LERP_X, LLM_TENSOR_TIME_MIX_LERP_W, LLM_TENSOR_TIME_MIX_LERP_K, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index dbb7abd317..bb17ba86dc 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -76,6 +76,10 @@ struct llama_hparams { uint32_t time_decay_extra_dim = 0; uint32_t wkv_head_size = 0; uint32_t token_shift_count = 2; + uint32_t n_lora_decay = 0; + uint32_t n_lora_iclr = 0; + uint32_t n_lora_value_res_mix = 0; + uint32_t n_lora_gate = 0; float rope_attn_factor = 1.0f; float rope_freq_base_train; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 4b288d8f66..c571aa69b6 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -32,6 +32,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_109M: return "109M"; case LLM_TYPE_137M: return "137M"; case LLM_TYPE_160M: return "160M"; + case LLM_TYPE_190M: return "190M"; case LLM_TYPE_220M: return "220M"; case LLM_TYPE_250M: return "250M"; case LLM_TYPE_270M: return "270M"; @@ -48,6 +49,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_1_6B: return "1.6B"; case LLM_TYPE_2B: return "2B"; case LLM_TYPE_2_8B: return "2.8B"; + case LLM_TYPE_2_9B: return "2.9B"; case LLM_TYPE_3B: return "3B"; case LLM_TYPE_4B: return "4B"; case LLM_TYPE_6B: return "6B"; @@ -1250,6 +1252,36 @@ void llama_model::load_hparams(llama_model_loader & ml) { default: type = LLM_TYPE_UNKNOWN; } } break; + case LLM_ARCH_RWKV7: + case LLM_ARCH_ARWKV7: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps, false); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps, false); + ml.get_key(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size); + ml.get_key(LLM_KV_ATTENTION_DECAY_LORA_RANK, hparams.n_lora_decay); + ml.get_key(LLM_KV_ATTENTION_ICLR_LORA_RANK, hparams.n_lora_iclr); + ml.get_key(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix); + ml.get_key(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate, false); + ml.get_key(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count, false); + + switch (hparams.n_layer) { + case 12: type = LLM_TYPE_190M; break; + case 24: + switch (hparams.n_embd) { + case 1024: type = LLM_TYPE_450M; break; + case 2048: type = LLM_TYPE_1_5B; break; + default: type = LLM_TYPE_UNKNOWN; + } break; + case 28: + switch (hparams.n_embd) { + case 1536: type = LLM_TYPE_1_5B; break; + case 3584: type = LLM_TYPE_7B; break; + default: type = LLM_TYPE_UNKNOWN; + } break; + case 32: type = LLM_TYPE_2_9B; break; // RWKV-7-World + default: type = LLM_TYPE_UNKNOWN; + } + } break; case LLM_ARCH_GRANITE: case LLM_ARCH_GRANITE_MOE: { @@ -3366,6 +3398,146 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); } } break; + case LLM_ARCH_RWKV7: + { + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + // Block 0, LN0 + tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, 0); + tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + const int n_lora_decay = hparams.n_lora_decay; + const int n_lora_iclr = hparams.n_lora_iclr; + const int n_lora_value_res_mix = hparams.n_lora_value_res_mix; + const int n_lora_gate = hparams.n_lora_gate; + const int attn_hidden_size = n_embd; + const int ffn_size = hparams.n_ff_arr[0]; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0); + + layer.attn_norm_2 = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, 0); + layer.attn_norm_2_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, 0); + + layer.time_mix_w0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W0, "weight", i), {n_embd}, 0); + layer.time_mix_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W1, "weight", i), {n_embd, n_lora_decay}, 0); + layer.time_mix_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W2, "weight", i), {n_lora_decay, n_embd}, 0); + + layer.time_mix_a0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A0, "weight", i), {n_embd}, 0); + layer.time_mix_a1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A1, "weight", i), {n_embd, n_lora_iclr}, 0); + layer.time_mix_a2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A2, "weight", i), {n_lora_iclr, n_embd}, 0); + + if (i == 0) { + // actually not used + layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0); + layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_iclr}, 0); + layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_iclr, n_embd}, 0); + } else { + layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0); + layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_value_res_mix}, 0); + layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_value_res_mix, n_embd}, 0); + } + + layer.time_mix_g1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G1, "weight", i), {n_embd, n_lora_gate}, 0); + layer.time_mix_g2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G2, "weight", i), {n_lora_gate, n_embd}, 0); + + layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, "weight", i), {n_embd, 1, 1, 6}, 0); + + layer.time_mix_k_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_K, "weight", i), {attn_hidden_size}, 0); + layer.time_mix_k_a = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_A, "weight", i), {attn_hidden_size}, 0); + layer.time_mix_r_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_R_K, "weight", i), {attn_hidden_size}, 0); + + layer.time_mix_key = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, "weight", i), {attn_hidden_size, n_embd}, 0); + layer.time_mix_value = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, "weight", i), {attn_hidden_size, n_embd}, 0); + layer.time_mix_receptance = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, "weight", i), {attn_hidden_size, n_embd}, 0); + + layer.time_mix_ln = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "weight", i), {n_embd}, 0); + layer.time_mix_ln_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "bias", i), {n_embd}, 0); + layer.time_mix_output = create_tensor(tn(LLM_TENSOR_TIME_MIX_OUTPUT, "weight", i), {n_embd, attn_hidden_size}, 0); + + layer.channel_mix_lerp_k = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_LERP_K, "weight", i), {n_embd, 1, 1}, 0); + + layer.channel_mix_key = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_KEY, "weight", i), {n_embd, ffn_size}, 0); + layer.channel_mix_value = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_VALUE, "weight", i), {ffn_size, n_embd}, 0); + } + + } break; + case LLM_ARCH_ARWKV7: + { + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + const int n_lora_decay = hparams.n_lora_decay; + const int n_lora_iclr = hparams.n_lora_iclr; + const int n_lora_value_res_mix = hparams.n_lora_value_res_mix; + const int n_lora_gate = hparams.n_lora_gate; + const int attn_hidden_size = n_embd; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + layer.time_mix_w0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W0, "weight", i), {n_embd}, 0); + layer.time_mix_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W1, "weight", i), {n_embd, n_lora_decay}, 0); + layer.time_mix_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W2, "weight", i), {n_lora_decay, n_embd}, 0); + + layer.time_mix_a0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A0, "weight", i), {n_embd}, 0); + layer.time_mix_a1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A1, "weight", i), {n_embd, n_lora_iclr}, 0); + layer.time_mix_a2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A2, "weight", i), {n_lora_iclr, n_embd}, 0); + + if (i == 0) { + // actually not used + layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0); + layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_iclr}, 0); + layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_iclr, n_embd}, 0); + } else { + layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0); + layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_value_res_mix}, 0); + layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_value_res_mix, n_embd}, 0); + } + + layer.time_mix_g1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G1, "weight", i), {n_embd, n_lora_gate}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.time_mix_g2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G2, "weight", i), {n_lora_gate, n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + try { + layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, "weight", i), {n_embd, 1, 1, 6}, 0); + } catch(std::runtime_error & e) { + // ARWKV models may not have gate tensors + layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, "weight", i), {n_embd, 1, 1, 5}, 0); + } + + layer.time_mix_k_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_K, "weight", i), {attn_hidden_size}, 0); + layer.time_mix_k_a = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_A, "weight", i), {attn_hidden_size}, 0); + layer.time_mix_r_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_R_K, "weight", i), {attn_hidden_size}, 0); + + layer.time_mix_key = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, "weight", i), {attn_hidden_size, n_embd}, 0); + layer.time_mix_value = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, "weight", i), {attn_hidden_size, n_embd}, 0); + layer.time_mix_receptance = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, "weight", i), {attn_hidden_size, n_embd}, 0); + + layer.time_mix_ln = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.time_mix_ln_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.time_mix_output = create_tensor(tn(LLM_TENSOR_TIME_MIX_OUTPUT, "weight", i), {n_embd, attn_hidden_size}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } + + } break; case LLM_ARCH_CHAMELEON: { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); @@ -10212,6 +10384,7 @@ struct llm_build_rwkv6_base : public llm_graph_context { const auto n_tokens = ubatch.n_tokens; const auto n_seqs = ubatch.n_seqs; + const auto n_seq_tokens = ubatch.n_seq_tokens; const auto n_embd = hparams.n_embd; const auto head_size = hparams.wkv_head_size; const auto n_head = n_embd / head_size; @@ -10224,6 +10397,10 @@ struct llm_build_rwkv6_base : public llm_graph_context { bool is_qrwkv = layer.time_mix_first == nullptr; ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur); + + sx = ggml_reshape_2d(ctx0, sx, n_embd, n_tokens); + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + ggml_tensor * xxx = ggml_add(ctx0, ggml_mul(ctx0, sx, layer.time_mix_lerp_x), cur); xxx = ggml_reshape_4d( @@ -10366,7 +10543,7 @@ struct llm_build_rwkv6_base : public llm_graph_context { cur = ggml_mul(ctx0, cur, g); cur = build_lora_mm(layer.time_mix_output, cur); - return cur; + return ggml_reshape_3d(ctx0, cur, n_embd, n_seq_tokens, n_seqs); } }; @@ -10389,6 +10566,7 @@ struct llm_build_rwkv6 : public llm_build_rwkv6_base { for (int il = 0; il < n_layer; ++il) { const llama_layer * layer = &model.layers[il]; + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); ggml_tensor * token_shift = build_rwkv_token_shift_load( gf, state_copy, state_mask, ubatch, il @@ -10422,9 +10600,6 @@ struct llm_build_rwkv6 : public llm_build_rwkv6_base { 1 ); - cur = build_rwkv6_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV6); - cur = ggml_add(ctx0, cur, ffn_inp); - token_shift = ggml_concat(ctx0, ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm)), ggml_view_3d(ctx0, ffn_norm, n_embd, 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(ffn_norm)), @@ -10432,6 +10607,18 @@ struct llm_build_rwkv6 : public llm_build_rwkv6_base { ); ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il)); + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + ffn_inp = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens), inp_out_ids); + ffn_norm = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_norm, n_embd, n_tokens), inp_out_ids); + x_prev = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, x_prev, n_embd, n_tokens), inp_out_ids); + cur = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, cur, n_embd, n_tokens), inp_out_ids); + } + + cur = build_rwkv6_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV6); + cur = ggml_add(ctx0, cur, ffn_inp); + if (hparams.rescale_every_n_layers != 0 && (il + 1) % hparams.rescale_every_n_layers == 0) { cur = ggml_scale(ctx0, cur, 0.5F); } @@ -10444,12 +10631,6 @@ struct llm_build_rwkv6 : public llm_build_rwkv6_base { } cur = inpL; - - ggml_tensor * inp_out_ids = build_inp_out_ids(); - - cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM, -1); cb(cur, "result_norm", -1); @@ -10481,10 +10662,9 @@ struct llm_build_rwkv6qwen2 : public llm_build_rwkv6_base { const auto n_seq_tokens = ubatch.n_seq_tokens; const auto n_seqs = ubatch.n_seqs; - inpL = build_inp_embd(model.tok_embd); - for (int il = 0; il < n_layer; ++il) { const llama_layer * layer = &model.layers[il]; + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); ggml_tensor * token_shift = build_rwkv_token_shift_load( gf, state_copy, state_mask, ubatch, il @@ -10508,6 +10688,13 @@ struct llm_build_rwkv6qwen2 : public llm_build_rwkv6_base { ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); cb(ffn_inp, "ffn_inp", il); + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, cur, n_embd, n_tokens), inp_out_ids); + ffn_inp = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens), inp_out_ids); + } + // feed-forward network cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, @@ -10532,10 +10719,358 @@ struct llm_build_rwkv6qwen2 : public llm_build_rwkv6_base { } cur = inpL; - ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); + cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + +struct llm_build_rwkv7_base : public llm_graph_context { + const llama_model & model; + + llm_build_rwkv7_base(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params), model(model) { + } + + ggml_tensor * build_rwkv7_channel_mix( + const llama_layer * layer, + ggml_tensor * cur, + ggml_tensor * x_prev, + llm_arch arch) const { + ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur); + switch (arch) { + case LLM_ARCH_RWKV7: + { + ggml_tensor * xk = ggml_add(ctx0, ggml_mul(ctx0, sx, layer->channel_mix_lerp_k), cur); + + ggml_tensor * k = ggml_sqr( + ctx0, + ggml_relu( + ctx0, + build_lora_mm(layer->channel_mix_key, xk) + ) + ); + + cur = build_lora_mm(layer->channel_mix_value, k); + } break; + default: + GGML_ABORT("fatal error"); + } + + return cur; + } + + ggml_tensor * build_rwkv7_time_mix( + ggml_cgraph * gf, + ggml_tensor * cur, + ggml_tensor * x_prev, + ggml_tensor * state_copy, + ggml_tensor * state_mask, + ggml_tensor *& first_layer_value, + const llama_ubatch & ubatch, + int il) const { + const llama_kv_cache_unified * kv_self = static_cast(memory); + + const auto n_tokens = ubatch.n_tokens; + const auto n_seqs = ubatch.n_seqs; + const auto n_embd = hparams.n_embd; + const auto head_size = hparams.wkv_head_size; + const auto head_count = n_embd / head_size; + const auto n_seq_tokens = ubatch.n_seq_tokens; + + const auto kv_head = kv_self->head; + + const auto & layer = model.layers[il]; + + bool has_gating = layer.time_mix_g1 && layer.time_mix_g2; + + ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur); + ggml_tensor * dummy = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_embd, n_seq_tokens, n_seqs, has_gating ? 6 : 5); + sx = ggml_repeat(ctx0, sx, dummy); + + ggml_tensor * xxx = ggml_add(ctx0, ggml_mul(ctx0, sx, layer.time_mix_lerp_fused), cur); + + ggml_tensor * xr = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], 0); + ggml_tensor * xw = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * sizeof(float)); + ggml_tensor * xk = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 2 * sizeof(float)); + ggml_tensor * xv = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 3 * sizeof(float)); + ggml_tensor * xa = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 4 * sizeof(float)); + ggml_tensor * xg = has_gating ? ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 5 * sizeof(float)) : nullptr; + + ggml_tensor * r = build_lora_mm(layer.time_mix_receptance, xr); + ggml_tensor * w = ggml_add( + ctx0, + ggml_mul_mat(ctx0, layer.time_mix_w2, ggml_tanh(ctx0, ggml_mul_mat(ctx0, layer.time_mix_w1, xw))), + layer.time_mix_w0 + ); + w = ggml_exp(ctx0, ggml_scale(ctx0, ggml_sigmoid(ctx0, w), -0.606531)); + + ggml_tensor * k = build_lora_mm(layer.time_mix_key, xk); + ggml_tensor * v = build_lora_mm(layer.time_mix_value, xv); + if (first_layer_value == nullptr) { + first_layer_value = v; + } else { + // Add the first layer value as a residual connection. + v = ggml_add(ctx0, v, + ggml_mul(ctx0, + ggml_sub(ctx0, first_layer_value, v), + ggml_sigmoid(ctx0, ggml_add(ctx0, + ggml_mul_mat(ctx0, layer.time_mix_v2, ggml_mul_mat(ctx0, layer.time_mix_v1, xv)), + layer.time_mix_v0 + ) + ) + ) + ); + } + + ggml_tensor * g = nullptr; + if (layer.time_mix_g1 && layer.time_mix_g2) { + g = ggml_mul_mat(ctx0, layer.time_mix_g2, ggml_sigmoid(ctx0, ggml_mul_mat(ctx0, layer.time_mix_g1, xg))); + } + + ggml_tensor * a = ggml_sigmoid(ctx0, + ggml_add( + ctx0, + ggml_mul_mat(ctx0, layer.time_mix_a2, ggml_mul_mat(ctx0, layer.time_mix_a1, xa)), + layer.time_mix_a0 + ) + ); + + ggml_tensor * kk = ggml_reshape_3d(ctx0, ggml_mul(ctx0, k, layer.time_mix_k_k), head_size, head_count, n_tokens); + kk = ggml_l2_norm(ctx0, kk, 1e-12); + + ggml_tensor * ka = ggml_mul(ctx0, k, layer.time_mix_k_a); + k = ggml_add(ctx0, k, ggml_sub(ctx0, ggml_mul(ctx0, a, ka), ka)); + + r = ggml_reshape_3d(ctx0, r, head_size, head_count, n_tokens); + w = ggml_reshape_3d(ctx0, w, head_size, head_count, n_tokens); + k = ggml_reshape_3d(ctx0, k, head_size, head_count, n_tokens); + v = ggml_reshape_3d(ctx0, v, head_size, head_count, n_tokens); + a = ggml_reshape_3d(ctx0, a, head_size, head_count, n_tokens); + + ggml_tensor * wkv_state = build_copy_mask_state( + gf, kv_self->v_l[il], state_copy, state_mask, + hparams.n_embd_v_s(), n_seqs); + + ggml_tensor * wkv_output = ggml_rwkv_wkv7(ctx0, r, w, k, v, ggml_neg(ctx0, kk), ggml_mul(ctx0, kk, a), wkv_state); + cur = ggml_view_1d(ctx0, wkv_output, n_embd * n_tokens, 0); + wkv_state = ggml_view_1d(ctx0, wkv_output, n_embd * head_size * n_seqs, n_embd * n_tokens * sizeof(float)); + + ggml_build_forward_expand( + gf, + ggml_cpy( + ctx0, + wkv_state, + ggml_view_1d( + ctx0, + kv_self->v_l[il], + hparams.n_embd_v_s() * n_seqs, + hparams.n_embd_v_s() * kv_head * ggml_element_size(kv_self->v_l[il]) + ) + ) + ); + + if (layer.time_mix_ln && layer.time_mix_ln_b) { + // group norm with head_count groups + cur = ggml_reshape_3d(ctx0, cur, n_embd / head_count, head_count, n_tokens); + cur = ggml_norm(ctx0, cur, 64e-5f); + + // Convert back to regular vectors. + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.time_mix_ln), layer.time_mix_ln_b); + } else { + cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens); + } + + ggml_tensor * rk = ggml_sum_rows(ctx0, + ggml_mul(ctx0, ggml_mul(ctx0, k, r), ggml_reshape_2d(ctx0, layer.time_mix_r_k, head_size, head_count))); + cur = ggml_add(ctx0, cur, ggml_reshape_2d(ctx0, ggml_mul(ctx0, v, rk), n_embd, n_tokens)); + + if (has_gating) { + cur = ggml_mul(ctx0, cur, g); + } + cur = build_lora_mm(layer.time_mix_output, cur); + + return ggml_reshape_3d(ctx0, cur, n_embd, n_seq_tokens, n_seqs); + } +}; + +struct llm_build_rwkv7 : public llm_build_rwkv7_base { + llm_build_rwkv7(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_build_rwkv7_base(model, params) { + GGML_ASSERT(hparams.token_shift_count == 2); + + ggml_tensor * cur; + ggml_tensor * inpL; + ggml_tensor * v_first = nullptr; + + inpL = build_inp_embd(model.tok_embd); + inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1); + + ggml_tensor * state_copy = build_inp_s_copy(); + ggml_tensor * state_mask = build_inp_s_mask(); + + const auto n_embd = hparams.n_embd; + const auto n_seq_tokens = ubatch.n_seq_tokens; + const auto n_seqs = ubatch.n_seqs; + + for (int il = 0; il < n_layer; ++il) { + const llama_layer * layer = &model.layers[il]; + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); + + ggml_tensor * token_shift = build_rwkv_token_shift_load( + gf, state_copy, state_mask, ubatch, il + ); + + ggml_tensor * att_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], 0); + ggml_tensor * ffn_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], n_embd * ggml_element_size(token_shift)); + + ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM, il); + cb(att_norm, "attn_norm", il); + + ggml_tensor * x_prev = ggml_concat( + ctx0, + att_shift, + ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0), + 1 + ); + + cur = build_rwkv7_time_mix(gf, att_norm, x_prev, state_copy, state_mask, v_first, ubatch, il); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + ggml_tensor * ffn_norm = build_norm(ffn_inp, layer->attn_norm_2, layer->attn_norm_2_b, LLM_NORM, il); + cb(ffn_norm, "ffn_norm", il); + + x_prev = ggml_concat( + ctx0, + ffn_shift, + ggml_view_3d(ctx0, ffn_norm, n_embd, n_seq_tokens - 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], 0), + 1 + ); + + token_shift = ggml_concat(ctx0, + ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm)), + ggml_view_3d(ctx0, ffn_norm, n_embd, 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(ffn_norm)), + 1 + ); + ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il)); + + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + ffn_inp = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens), inp_out_ids); + ffn_norm = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_norm, n_embd, n_tokens), inp_out_ids); + x_prev = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, x_prev, n_embd, n_tokens), inp_out_ids); + } + + cur = build_rwkv7_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV7); + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + } +}; + + +struct llm_build_arwkv7 : public llm_build_rwkv7_base { + llm_build_arwkv7(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_build_rwkv7_base(model, params) { + GGML_ASSERT(n_embd == hparams.n_embd_k_s()); + + ggml_tensor * cur; + ggml_tensor * inpL; + ggml_tensor * v_first = nullptr; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * state_copy = build_inp_s_copy(); + ggml_tensor * state_mask = build_inp_s_mask(); + + const auto n_embd = hparams.n_embd; + const auto n_seq_tokens = ubatch.n_seq_tokens; + const auto n_seqs = ubatch.n_seqs; + + for (int il = 0; il < n_layer; ++il) { + const llama_layer * layer = &model.layers[il]; + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs); + + ggml_tensor * token_shift = build_rwkv_token_shift_load( + gf, state_copy, state_mask, ubatch, il + ); + + ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM_RMS, il); + cb(att_norm, "attn_norm", il); + + ggml_tensor * x_prev = ggml_concat( + ctx0, + token_shift, + ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0), + 1 + ); + + cur = build_rwkv7_time_mix(gf, att_norm, x_prev, state_copy, state_mask, v_first, ubatch, il); + + token_shift = ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm)); + ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il)); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, cur, n_embd, n_tokens), inp_out_ids); + ffn_inp = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens), inp_out_ids); + } + + // feed-forward network + cur = build_norm(ffn_inp, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM_RMS, -1); cb(cur, "result_norm", -1); @@ -10883,9 +11418,11 @@ llama_memory_i * llama_model::create_memory() const { llama_memory_i * res; switch (arch) { + case LLM_ARCH_MAMBA: case LLM_ARCH_RWKV6: case LLM_ARCH_RWKV6QWEN2: - case LLM_ARCH_MAMBA: + case LLM_ARCH_RWKV7: + case LLM_ARCH_ARWKV7: { res = new llama_kv_cache_unified(hparams, { /*.get_rope_factors =*/ nullptr @@ -11132,6 +11669,14 @@ llm_graph_result_ptr llama_model::build_graph( { llm = std::make_unique(*this, params, gf); } break; + case LLM_ARCH_RWKV7: + { + llm = std::make_unique(*this, params, gf); + } break; + case LLM_ARCH_ARWKV7: + { + llm = std::make_unique(*this, params, gf); + } break; case LLM_ARCH_CHAMELEON: { llm = std::make_unique(*this, params, gf); @@ -11245,6 +11790,8 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_JAIS: case LLM_ARCH_RWKV6: case LLM_ARCH_RWKV6QWEN2: + case LLM_ARCH_RWKV7: + case LLM_ARCH_ARWKV7: case LLM_ARCH_WAVTOKENIZER_DEC: return LLAMA_ROPE_TYPE_NONE; @@ -11399,6 +11946,8 @@ bool llama_model_is_recurrent(const llama_model * model) { case LLM_ARCH_MAMBA: return true; case LLM_ARCH_RWKV6: return true; case LLM_ARCH_RWKV6QWEN2: return true; + case LLM_ARCH_RWKV7: return true; + case LLM_ARCH_ARWKV7: return true; default: return false; } } diff --git a/src/llama-model.h b/src/llama-model.h index 55c26a92b0..a9da1215ab 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -29,6 +29,7 @@ enum llm_type { LLM_TYPE_109M, LLM_TYPE_137M, LLM_TYPE_160M, + LLM_TYPE_190M, LLM_TYPE_220M, LLM_TYPE_250M, LLM_TYPE_270M, @@ -45,6 +46,7 @@ enum llm_type { LLM_TYPE_1_6B, LLM_TYPE_2B, LLM_TYPE_2_8B, + LLM_TYPE_2_9B, LLM_TYPE_3B, LLM_TYPE_4B, LLM_TYPE_6B, @@ -260,6 +262,20 @@ struct llama_layer { struct ggml_tensor * time_mix_receptance_b = nullptr; struct ggml_tensor * time_mix_gate = nullptr; + // rwkv7 + struct ggml_tensor * time_mix_w0 = nullptr; + struct ggml_tensor * time_mix_a0 = nullptr; + struct ggml_tensor * time_mix_a1 = nullptr; + struct ggml_tensor * time_mix_a2 = nullptr; + struct ggml_tensor * time_mix_v0 = nullptr; + struct ggml_tensor * time_mix_v1 = nullptr; + struct ggml_tensor * time_mix_v2 = nullptr; + struct ggml_tensor * time_mix_g1 = nullptr; + struct ggml_tensor * time_mix_g2 = nullptr; + struct ggml_tensor * time_mix_k_k = nullptr; + struct ggml_tensor * time_mix_k_a = nullptr; + struct ggml_tensor * time_mix_r_k = nullptr; + struct ggml_tensor * time_mix_ln = nullptr; struct ggml_tensor * time_mix_ln_b = nullptr; struct ggml_tensor * time_mix_output = nullptr; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index fb7982655a..09eb570779 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -756,10 +756,19 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: // NOTE: can't use LLM_TN here because the layer number is not known quantize &= name.find("ssm_conv1d.weight") == std::string::npos; - // do not quantize RWKV's time_mix_first tensors + // do not quantize RWKV's small yet 2D weights quantize &= name.find("time_mix_first.weight") == std::string::npos; + quantize &= name.find("time_mix_w0.weight") == std::string::npos; quantize &= name.find("time_mix_w1.weight") == std::string::npos; quantize &= name.find("time_mix_w2.weight") == std::string::npos; + quantize &= name.find("time_mix_v0.weight") == std::string::npos; + quantize &= name.find("time_mix_v1.weight") == std::string::npos; + quantize &= name.find("time_mix_v2.weight") == std::string::npos; + quantize &= name.find("time_mix_a0.weight") == std::string::npos; + quantize &= name.find("time_mix_a1.weight") == std::string::npos; + quantize &= name.find("time_mix_a2.weight") == std::string::npos; + quantize &= name.find("time_mix_g1.weight") == std::string::npos; + quantize &= name.find("time_mix_g2.weight") == std::string::npos; quantize &= name.find("time_mix_decay_w1.weight") == std::string::npos; quantize &= name.find("time_mix_decay_w2.weight") == std::string::npos; quantize &= name.find("time_mix_lerp_fused.weight") == std::string::npos; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c86ffb64e9..adb749bd5e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1916,6 +1916,40 @@ struct test_gla : public test_case { } }; +// GGML_OP_RWKV_WKV7 +struct test_rwkv_wkv7 : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + + std::string vars() override { + return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); + } + + test_rwkv_wkv7(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t n_tokens = n_seq_tokens * n_seqs; + ggml_tensor * r = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * w = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * a = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + // Outputs may become NaN with long seqlen without these normalization + a = ggml_l2_norm(ctx, a, 1e-7F); + b = ggml_l2_norm(ctx, b, 1e-7F); + ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector{ head_size * head_size * head_count, n_seqs }.data()); + ggml_tensor * out = ggml_rwkv_wkv7(ctx, r, w, k, v, a, b, s); + return out; + } +}; + // GGML_OP_MUL_MAT struct test_mul_mat : public test_case { const ggml_type type_a; @@ -2972,6 +3006,32 @@ struct test_group_norm : public test_case { } }; +// GGML_OP_L2_NORM +struct test_l2_norm : public test_case { + const ggml_type type; + const std::array ne; + const float eps; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_l2_norm(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 64, 320, 1}, + float eps = 1e-12f) + : type(type), ne(ne), eps(eps) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_l2_norm(ctx, a, eps); + ggml_set_name(out, "out"); + + return out; + } +}; + // GGML_OP_ACC struct test_acc : public test_case { const ggml_type type; @@ -4006,8 +4066,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, v, eps)); } test_cases.emplace_back(new test_rms_norm_back(GGML_TYPE_F32, {64, 5, 4, 3}, eps)); + test_cases.emplace_back(new test_l2_norm (GGML_TYPE_F32, {64, 5, 4, 3}, eps)); } + test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, {64, 5, 4, 3}, 1e-12f)); + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {4, 1536, 1, 1}, {4, 1536, 1, 1})); test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {8, 1536, 1, 1}, {4, 1536, 1, 1})); test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {4, 1536, 4, 1}, {4, 1536, 1, 1})); @@ -4019,6 +4082,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 4)); test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 128, 4)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 1, 1)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 1)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 4)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 128, 4)); + test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 1, 1)); test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 32, 1)); test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 32, 4)); From a53f7f7b8859f3e634415ab03e1e295b9861d7e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= <112692748+lslusarczyk@users.noreply.github.com> Date: Tue, 18 Mar 2025 01:51:25 +0100 Subject: [PATCH 155/188] fixed compilation warnings in ggml-sycl (#12424) --- ggml/src/ggml-sycl/convert.cpp | 2 +- ggml/src/ggml-sycl/dmmv.cpp | 25 +++++---- ggml/src/ggml-sycl/element_wise.cpp | 80 ++++++++++++++--------------- ggml/src/ggml-sycl/getrows.cpp | 3 +- ggml/src/ggml-sycl/ggml-sycl.cpp | 43 ++++++++-------- ggml/src/ggml-sycl/mmq.cpp | 1 - ggml/src/ggml-sycl/mmvq.cpp | 39 +++++++------- ggml/src/ggml-sycl/norm.cpp | 12 ++--- ggml/src/ggml-sycl/softmax.cpp | 2 +- 9 files changed, 101 insertions(+), 106 deletions(-) diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index 86b200e070..76ac6a4dd1 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -138,7 +138,7 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * sycl::range<3>(1, 1, WARP_SIZE), sycl::range<3>(1, 1, WARP_SIZE)), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]]{ + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ dequantize_block_q4_0_reorder(vx, y, k, item_ct1); }); diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index 99d3859de8..04a85fa35f 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -210,7 +210,7 @@ static void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec<1, 1, convert_f16>(vx, y, dst, ncols, nrows, item_ct1); }); @@ -879,7 +879,7 @@ static void dequantize_mul_mat_vec_q4_0_sycl_reorder(const void *vx, const dfloa stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec_reorder( vx, y, dst, ncols, nrows, item_ct1); }); @@ -902,7 +902,7 @@ static void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -923,7 +923,7 @@ static void dequantize_mul_mat_vec_q4_1_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -944,7 +944,7 @@ static void dequantize_mul_mat_vec_q5_0_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -965,7 +965,7 @@ static void dequantize_mul_mat_vec_q5_1_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -986,7 +986,7 @@ static void dequantize_mul_mat_vec_q8_0_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec( vx, y, dst, ncols, nrows, item_ct1); }); @@ -1004,7 +1004,7 @@ static void dequantize_mul_mat_vec_q2_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q2_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1020,7 +1020,7 @@ static void dequantize_mul_mat_vec_q3_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q3_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1036,7 +1036,7 @@ static void dequantize_mul_mat_vec_q4_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q4_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1049,7 +1049,7 @@ static void dequantize_mul_mat_vec_q5_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, 1, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q5_k(vx, y, dst, ncols, item_ct1); }); } @@ -1065,7 +1065,7 @@ static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y, const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { dequantize_mul_mat_vec_q6_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1143,7 +1143,6 @@ void ggml_sycl_op_dequantize_mul_mat_vec( default: printf("ggml_sycl_op_dequantize_mul_mat_vec unsupported GGML_TYPE %d\n", src0->type); GGML_ABORT("fatal error"); - break; } GGML_UNUSED(src1); diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cpp index 4bcd74376e..1e12cb220e 100644 --- a/ggml/src/ggml-sycl/element_wise.cpp +++ b/ggml/src/ggml-sycl/element_wise.cpp @@ -1,7 +1,7 @@ #include "common.hpp" #include "element_wise.hpp" -void acc_f32(const float * x, const float * y, float * dst, const int ne, +static void acc_f32(const float * x, const float * y, float * dst, const int ne, const int ne10, const int ne11, const int ne12, const int nb1, const int nb2, int offset, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + @@ -20,7 +20,7 @@ void acc_f32(const float * x, const float * y, float * dst, const int ne, } } -void gelu_f32(const float * x, float * dst, const int k, +static void gelu_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const float GELU_COEF_A = 0.044715f; const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; @@ -37,7 +37,7 @@ void gelu_f32(const float * x, float * dst, const int k, sycl::tanh(SQRT_2_OVER_PI * xi * (1.0f + GELU_COEF_A * xi * xi))); } -void silu_f32(const float * x, float * dst, const int k, +static void silu_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -48,7 +48,7 @@ void silu_f32(const float * x, float * dst, const int k, dst[i] = x[i] / (1.0f + sycl::native::exp(-x[i])); } -void gelu_quick_f32(const float *x, float *dst, int k, +static void gelu_quick_f32(const float *x, float *dst, int k, const sycl::nd_item<3> &item_ct1) { const float GELU_QUICK_COEF = -1.702f; const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + @@ -59,7 +59,7 @@ void gelu_quick_f32(const float *x, float *dst, int k, dst[i] = x[i] * (1.0f / (1.0f + sycl::native::exp(GELU_QUICK_COEF * x[i]))); } -void tanh_f32(const float *x, float *dst, int k, +static void tanh_f32(const float *x, float *dst, int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -69,7 +69,7 @@ void tanh_f32(const float *x, float *dst, int k, dst[i] = sycl::tanh((float)(x[i])); } -void relu_f32(const float * x, float * dst, const int k, +static void relu_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -80,7 +80,7 @@ void relu_f32(const float * x, float * dst, const int k, dst[i] = sycl::fmax((float)(x[i]), (float)0); } -void sigmoid_f32(const float * x, float * dst, const int k, +static void sigmoid_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -91,7 +91,7 @@ void sigmoid_f32(const float * x, float * dst, const int k, dst[i] = 1.0f / (1.0f + sycl::native::exp(-x[i])); } -void sqrt_f32(const float * x, float * dst, const int k, +static void sqrt_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -102,7 +102,7 @@ void sqrt_f32(const float * x, float * dst, const int k, dst[i] = sycl::sqrt(x[i]); } -void sin_f32(const float * x, float * dst, const int k, +static void sin_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -113,7 +113,7 @@ void sin_f32(const float * x, float * dst, const int k, dst[i] = sycl::sin(x[i]); } -void cos_f32(const float * x, float * dst, const int k, +static void cos_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -124,7 +124,7 @@ void cos_f32(const float * x, float * dst, const int k, dst[i] = sycl::cos(x[i]); } -void hardsigmoid_f32(const float * x, float * dst, const int k, +static void hardsigmoid_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -135,7 +135,7 @@ void hardsigmoid_f32(const float * x, float * dst, const int k, dst[i] = sycl::fmin(1.0f, sycl::fmax(0.0f, (x[i] + 3.0f) / 6.0f)); } -void hardswish_f32(const float * x, float * dst, const int k, +static void hardswish_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -146,7 +146,7 @@ void hardswish_f32(const float * x, float * dst, const int k, dst[i] = x[i] * sycl::fmin(1.0f, sycl::fmax(0.0f, (x[i] + 3.0f) / 6.0f)); } -void exp_f32(const float * x, float * dst, const int k, +static void exp_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -157,7 +157,7 @@ void exp_f32(const float * x, float * dst, const int k, dst[i] = sycl::exp(x[i]); } -void log_f32(const float * x, float * dst, const int k, +static void log_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -173,7 +173,7 @@ void log_f32(const float * x, float * dst, const int k, } } -void neg_f32(const float * x, float * dst, const int k, +static void neg_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -184,7 +184,7 @@ void neg_f32(const float * x, float * dst, const int k, dst[i] = -x[i]; } -void step_f32(const float * x, float * dst, const int k, +static void step_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -195,7 +195,7 @@ void step_f32(const float * x, float * dst, const int k, dst[i] = x[i] > 0.0f; } -void leaky_relu_f32(const float *x, float *dst, const int k, const float negative_slope, +static void leaky_relu_f32(const float *x, float *dst, const int k, const float negative_slope, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -206,7 +206,7 @@ void leaky_relu_f32(const float *x, float *dst, const int k, const float negativ sycl::fmin((float)(x[i]), 0.0f) * negative_slope; } -void sqr_f32(const float * x, float * dst, const int k, +static void sqr_f32(const float * x, float * dst, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -217,7 +217,7 @@ void sqr_f32(const float * x, float * dst, const int k, dst[i] = x[i] * x[i]; } -void upscale_f32(const float *x, float *dst, const int nb00, const int nb01, +static void upscale_f32(const float *x, float *dst, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int ne13, const float sf0, const float sf1, const float sf2, const float sf3, const sycl::nd_item<1> &item_ct1) { @@ -240,7 +240,7 @@ void upscale_f32(const float *x, float *dst, const int nb00, const int nb01, dst[index] = *(const float *)((const char *)x + i03 * nb03 + i02 * nb02 + i01 * nb01 + i00 * nb00); } -void pad_f32(const float *x, float *dst, const int ne0, const int ne00, const int ne01, const int ne02, +static void pad_f32(const float *x, float *dst, const int ne0, const int ne00, const int ne01, const int ne02, const sycl::nd_item<3> &item_ct1) { int nidx = item_ct1.get_local_id(2) + item_ct1.get_group(2) * item_ct1.get_local_range(2); @@ -262,7 +262,7 @@ void pad_f32(const float *x, float *dst, const int ne0, const int ne00, const i -void acc_f32_sycl(const float *x, const float *y, float *dst, +static void acc_f32_sycl(const float *x, const float *y, float *dst, const int n_elements, const int ne10, const int ne11, const int ne12, const int nb1, const int nb2, const int offset, queue_ptr stream) { @@ -277,7 +277,7 @@ void acc_f32_sycl(const float *x, const float *y, float *dst, }); } -void gelu_f32_sycl(const float *x, float *dst, const int k, +static void gelu_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_GELU_BLOCK_SIZE - 1) / SYCL_GELU_BLOCK_SIZE; stream->parallel_for( @@ -289,7 +289,7 @@ void gelu_f32_sycl(const float *x, float *dst, const int k, }); } -void silu_f32_sycl(const float *x, float *dst, const int k, +static void silu_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SILU_BLOCK_SIZE - 1) / SYCL_SILU_BLOCK_SIZE; stream->parallel_for( @@ -301,7 +301,7 @@ void silu_f32_sycl(const float *x, float *dst, const int k, }); } -void gelu_quick_f32_sycl(const float *x, float *dst, const int k, +static void gelu_quick_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_GELU_BLOCK_SIZE - 1) / SYCL_GELU_BLOCK_SIZE; stream->parallel_for( @@ -313,7 +313,7 @@ void gelu_quick_f32_sycl(const float *x, float *dst, const int k, }); } -void tanh_f32_sycl(const float *x, float *dst, const int k, +static void tanh_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_TANH_BLOCK_SIZE - 1) / SYCL_TANH_BLOCK_SIZE; stream->parallel_for( @@ -325,7 +325,7 @@ void tanh_f32_sycl(const float *x, float *dst, const int k, }); } -void relu_f32_sycl(const float *x, float *dst, const int k, +static void relu_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_RELU_BLOCK_SIZE - 1) / SYCL_RELU_BLOCK_SIZE; stream->parallel_for( @@ -337,7 +337,7 @@ void relu_f32_sycl(const float *x, float *dst, const int k, }); } -void hardsigmoid_f32_sycl(const float *x, float *dst, const int k, +static void hardsigmoid_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_HARDSIGMOID_BLOCK_SIZE - 1) / SYCL_HARDSIGMOID_BLOCK_SIZE; stream->parallel_for( @@ -349,7 +349,7 @@ void hardsigmoid_f32_sycl(const float *x, float *dst, const int k, }); } -void hardswish_f32_sycl(const float *x, float *dst, const int k, +static void hardswish_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_HARDSWISH_BLOCK_SIZE - 1) / SYCL_HARDSWISH_BLOCK_SIZE; stream->parallel_for( @@ -361,7 +361,7 @@ void hardswish_f32_sycl(const float *x, float *dst, const int k, }); } -void exp_f32_sycl(const float *x, float *dst, const int k, +static void exp_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_EXP_BLOCK_SIZE - 1) / SYCL_EXP_BLOCK_SIZE; stream->parallel_for( @@ -373,7 +373,7 @@ void exp_f32_sycl(const float *x, float *dst, const int k, }); } -void log_f32_sycl(const float *x, float *dst, const int k, +static void log_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_EXP_BLOCK_SIZE - 1) / SYCL_EXP_BLOCK_SIZE; stream->parallel_for( @@ -385,7 +385,7 @@ void log_f32_sycl(const float *x, float *dst, const int k, }); } -void neg_f32_sycl(const float *x, float *dst, const int k, +static void neg_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_NEG_BLOCK_SIZE - 1) / SYCL_NEG_BLOCK_SIZE; stream->parallel_for( @@ -397,7 +397,7 @@ void neg_f32_sycl(const float *x, float *dst, const int k, }); } -void step_f32_sycl(const float *x, float *dst, const int k, +static void step_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_NEG_BLOCK_SIZE - 1) / SYCL_NEG_BLOCK_SIZE; stream->parallel_for( @@ -409,7 +409,7 @@ void step_f32_sycl(const float *x, float *dst, const int k, }); } -void sigmoid_f32_sycl(const float *x, float *dst, const int k, +static void sigmoid_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SIGMOID_BLOCK_SIZE - 1) / SYCL_SIGMOID_BLOCK_SIZE; stream->parallel_for( @@ -421,7 +421,7 @@ void sigmoid_f32_sycl(const float *x, float *dst, const int k, }); } -void sqrt_f32_sycl(const float *x, float *dst, const int k, +static void sqrt_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SQRT_BLOCK_SIZE - 1) / SYCL_SQRT_BLOCK_SIZE; stream->parallel_for( @@ -433,7 +433,7 @@ void sqrt_f32_sycl(const float *x, float *dst, const int k, }); } -void sin_f32_sycl(const float *x, float *dst, const int k, +static void sin_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SIN_BLOCK_SIZE - 1) / SYCL_SIN_BLOCK_SIZE; stream->parallel_for( @@ -445,7 +445,7 @@ void sin_f32_sycl(const float *x, float *dst, const int k, }); } -void cos_f32_sycl(const float *x, float *dst, const int k, +static void cos_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SIN_BLOCK_SIZE - 1) / SYCL_SIN_BLOCK_SIZE; stream->parallel_for( @@ -457,7 +457,7 @@ void cos_f32_sycl(const float *x, float *dst, const int k, }); } -void leaky_relu_f32_sycl(const float *x, float *dst, const int k, +static void leaky_relu_f32_sycl(const float *x, float *dst, const int k, const float negative_slope, queue_ptr stream) { const int num_blocks = (k + SYCL_RELU_BLOCK_SIZE - 1) / SYCL_RELU_BLOCK_SIZE; @@ -470,7 +470,7 @@ void leaky_relu_f32_sycl(const float *x, float *dst, const int k, }); } -void sqr_f32_sycl(const float *x, float *dst, const int k, +static void sqr_f32_sycl(const float *x, float *dst, const int k, queue_ptr stream) { const int num_blocks = (k + SYCL_SQR_BLOCK_SIZE - 1) / SYCL_SQR_BLOCK_SIZE; stream->parallel_for( @@ -482,7 +482,7 @@ void sqr_f32_sycl(const float *x, float *dst, const int k, }); } -void upscale_f32_sycl(const float *x, float *dst, const int nb00, const int nb01, +static void upscale_f32_sycl(const float *x, float *dst, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int ne13, const float sf0, const float sf1, const float sf2, const float sf3, queue_ptr stream) { @@ -496,7 +496,7 @@ void upscale_f32_sycl(const float *x, float *dst, const int nb00, const int nb01 }); } -void pad_f32_sycl(const float *x, float *dst, const int ne00, +static void pad_f32_sycl(const float *x, float *dst, const int ne00, const int ne01, const int ne02, const int ne0, const int ne1, const int ne2, queue_ptr stream) { int num_blocks = (ne0 + SYCL_PAD_BLOCK_SIZE - 1) / SYCL_PAD_BLOCK_SIZE; diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp index 51c19f6b3b..b9cf8767cb 100644 --- a/ggml/src/ggml-sycl/getrows.cpp +++ b/ggml/src/ggml-sycl/getrows.cpp @@ -207,7 +207,7 @@ static void get_rows_sycl_reorder(ggml_backend_sycl_context & ctx, const ggml_te const size_t nrows = ne01; const sycl::half* src0_dq = (const sycl::half*)(src0_q + nrows * ncols / 2); stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]]{ + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ k_get_rows_reorder( src0_dd, src0_dq, src1_dd, dst_dd, ne00, ne12, s1, s2, s3, nb01, nb02, nb03, s10, s11, s12, item_ct1); @@ -302,7 +302,6 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, const ggml_tensor *s // TODO: k-quants GGML_LOG_ERROR("%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type)); GGML_ABORT("fatal error"); - break; } } diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 477652ab28..207c0b440a 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -95,7 +95,7 @@ const ggml_sycl_device_info & ggml_sycl_info() { return info; } -void print_device_detail(int id, sycl::device &device, std::string device_type) { +static void print_device_detail(int id, sycl::device &device, std::string device_type) { dpct::device_info prop; SYCL_CHECK(CHECK_TRY_ERROR( @@ -118,7 +118,7 @@ void print_device_detail(int id, sycl::device &device, std::string device_type) global_mem_size, device.get_info().c_str()); } -void print_device_opt_feature(int device_count) { +static void print_device_opt_feature(int device_count) { GGML_LOG_INFO("SYCL Optimization Feature:\n"); GGML_LOG_INFO( "|ID| Device Type|Reorder|\n"); @@ -401,7 +401,7 @@ catch (sycl::exception const &exc) { std::exit(1); } -void dev2dev_memcpy(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst, +static void dev2dev_memcpy(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst, const void *ptr_src, size_t size) { char *host_buf = (char *)malloc(size); q_src.memcpy(host_buf, (const char *)ptr_src, size).wait(); @@ -620,7 +620,7 @@ ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { return &ggml_backend_sycl_buffer_types[device]; } -ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(ggml_backend_sycl_context * ctx) { +static ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(ggml_backend_sycl_context * ctx) { GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_buffer_type\n"); int device = ctx->device; @@ -1682,7 +1682,7 @@ static void quantize_row_q8_1_sycl(const float *x, void *vy, const int kx, stream->parallel_for( sycl::nd_range<3>(num_blocks * block_size, block_size), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { quantize_q8_1(x, vy, kx, kx_padded, item_ct1); }); } @@ -1703,7 +1703,7 @@ static void ggml_mul_mat_p021_f16_f32_sycl(const void *vx, const float *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_p021_f16_f32(vx, y, dst, ncols_x, nrows_x, nchannels_x, nchannels_y, item_ct1); }); @@ -1723,7 +1723,7 @@ static void ggml_mul_mat_vec_nc_f16_f32_sycl( stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_nc_f16_f32(vx, y, dst, ncols_x, nrows_x, row_stride_x, channel_stride_x, nchannels_y / nchannels_x, item_ct1); @@ -1764,7 +1764,7 @@ static void sum_rows_f32_sycl(const float *x, float *dst, const int ncols, const sycl::range<3> block_nums(1, nrows, 1); stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { k_sum_rows_f32(x, dst, ncols, item_ct1); }); } @@ -2920,7 +2920,7 @@ inline bool ggml_sycl_supports_mmq(enum ggml_type type) { return false; } -bool ggml_sycl_supports_dmmv(enum ggml_type type) { +static bool ggml_sycl_supports_dmmv(enum ggml_type type) { switch (type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -3293,7 +3293,7 @@ static void ggml_sycl_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * dst) } -void ggml_sycl_set_main_device(const int main_device) try { +static void ggml_sycl_set_main_device(const int main_device) try { if (dpct::get_current_device_id() == static_cast (main_device)) { return; } @@ -3314,7 +3314,7 @@ catch (sycl::exception const &exc) { std::exit(1); } -bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tensor * dst) { +static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tensor * dst) { if (!g_sycl_loaded) return false; if (dst->src[0] != nullptr && ggml_backend_buffer_is_sycl_split(dst->src[0]->buffer)) { @@ -3638,7 +3638,7 @@ catch (sycl::exception const &exc) { std::exit(1); } -void reorder_qw(char *data_device, const int ncols, const int nrows, +static void reorder_qw(char *data_device, const int ncols, const int nrows, size_t size, size_t offset, dpct::queue_ptr stream) { auto tmp_buf = sycl::malloc_shared(size, *stream); SYCL_CHECK( @@ -3652,7 +3652,7 @@ void reorder_qw(char *data_device, const int ncols, const int nrows, stream->parallel_for( size / sizeof(block_q4_0), - [=](auto i) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { const block_q4_0* x = (const block_q4_0*)tmp_buf; const int ib = i; @@ -3666,7 +3666,7 @@ void reorder_qw(char *data_device, const int ncols, const int nrows, sycl::free(tmp_buf, *stream); } -void reorder_qw(ggml_tensor * src0, dpct::queue_ptr stream) { +static void reorder_qw(ggml_tensor * src0, dpct::queue_ptr stream) { char*data_device = (char*)src0->data; size_t ncols = src0->ne[0]; size_t nrows = src0->ne[1]; @@ -3675,7 +3675,7 @@ void reorder_qw(ggml_tensor * src0, dpct::queue_ptr stream) { reorder_qw(data_device, ncols, nrows, size, 0, stream); } -void opt_for_reorder(ggml_tensor * dst, dpct::queue_ptr stream) { +static void opt_for_reorder(ggml_tensor * dst, dpct::queue_ptr stream) { ggml_tensor *src0 = dst->src[0]; ggml_tensor *src1 = dst->src[1]; @@ -3688,7 +3688,7 @@ void opt_for_reorder(ggml_tensor * dst, dpct::queue_ptr stream) { } } -void optimize_graph_once(ggml_cgraph * cgraph, ggml_backend_sycl_context * ctx) { +static void optimize_graph_once(ggml_cgraph * cgraph, ggml_backend_sycl_context * ctx) { dpct::queue_ptr stream = ctx->stream(); if (ctx->optimized_graph) { return; @@ -3878,7 +3878,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g return true; } return false; - } break; + } case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { case GGML_UNARY_OP_NEG: @@ -3896,7 +3896,6 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g default: return false; } - break; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: { @@ -3927,7 +3926,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g return false; } return true; - } break; + } case GGML_OP_OUT_PROD: return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->ne[2] == 1 && op->ne[3] == 1; case GGML_OP_GET_ROWS: @@ -3944,7 +3943,7 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g default: return false; } - } break; + } case GGML_OP_CPY: { ggml_type src0_type = op->src[0]->type; @@ -3995,12 +3994,12 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g return true; } return false; - } break; + } case GGML_OP_CONCAT: { ggml_type src0_type = op->src[0]->type; return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; - } break; + } case GGML_OP_DUP: case GGML_OP_ARGMAX: case GGML_OP_NONE: diff --git a/ggml/src/ggml-sycl/mmq.cpp b/ggml/src/ggml-sycl/mmq.cpp index 8ea82c940c..ffb272aa28 100644 --- a/ggml/src/ggml-sycl/mmq.cpp +++ b/ggml/src/ggml-sycl/mmq.cpp @@ -3017,7 +3017,6 @@ void ggml_sycl_op_mul_mat_q( break; default: GGML_ABORT("fatal error"); - break; } GGML_UNUSED(src1); diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index a96286d710..1b92ba2d60 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -495,7 +495,7 @@ static void mul_mat_vec_q4_0_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -519,7 +519,7 @@ static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -543,7 +543,7 @@ static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -567,7 +567,7 @@ static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -591,7 +591,7 @@ static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -615,7 +615,7 @@ static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -639,7 +639,7 @@ static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -663,7 +663,7 @@ static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -687,7 +687,7 @@ static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -711,7 +711,7 @@ static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q( vx, vy, dst, ncols, nrows, item_ct1); @@ -734,7 +734,7 @@ static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq2_xxs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -755,7 +755,7 @@ static void mul_mat_vec_iq2_xs_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq2_xs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -777,7 +777,7 @@ static void mul_mat_vec_iq2_s_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq2_s_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -799,7 +799,7 @@ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq3_xxs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -821,7 +821,7 @@ static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq3_s_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -843,7 +843,7 @@ static void mul_mat_vec_iq1_s_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq1_s_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -864,7 +864,7 @@ static void mul_mat_vec_iq1_m_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq1_m_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -886,7 +886,7 @@ static void mul_mat_vec_iq4_nl_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq4_nl_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -908,7 +908,7 @@ static void mul_mat_vec_iq4_xs_q8_1_sycl(const void *vx, const void *vy, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_iq4_xs_q8_1( vx, vy, dst, ncols, nrows, item_ct1); }); @@ -1003,7 +1003,6 @@ void ggml_sycl_op_mul_mat_vec_q( break; default: GGML_ABORT("fatal error"); - break; } } GGML_UNUSED(src1); diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp index 6439db21b2..d9678da8f0 100644 --- a/ggml/src/ggml-sycl/norm.cpp +++ b/ggml/src/ggml-sycl/norm.cpp @@ -235,7 +235,7 @@ static void norm_f32_sycl(const float* x, float* dst, const int ncols, sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { norm_f32(x, dst, ncols, eps, item_ct1, nullptr, WARP_SIZE); }); @@ -258,7 +258,7 @@ static void norm_f32_sycl(const float* x, float* dst, const int ncols, sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { norm_f32(x, dst, ncols, eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); }); @@ -277,7 +277,7 @@ static void group_norm_f32_sycl(const float* x, float* dst, sycl::nd_range<3>(sycl::range<3>(1, 1, num_groups) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { group_norm_f32( x, dst, group_size, ne_elements, eps_ct4, item_ct1, nullptr, WARP_SIZE); @@ -304,7 +304,7 @@ static void group_norm_f32_sycl(const float* x, float* dst, sycl::nd_range<3>(sycl::range<3>(1, 1, num_groups) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { group_norm_f32(x, dst, group_size, ne_elements, eps_ct4, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); @@ -325,7 +325,7 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { rms_norm_f32(x, dst, ncols, eps, item_ct1, nullptr, WARP_SIZE); }); @@ -347,7 +347,7 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) - [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { rms_norm_f32(x, dst, ncols, eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); }); diff --git a/ggml/src/ggml-sycl/softmax.cpp b/ggml/src/ggml-sycl/softmax.cpp index eb20bd251e..7563d9ceda 100644 --- a/ggml/src/ggml-sycl/softmax.cpp +++ b/ggml/src/ggml-sycl/softmax.cpp @@ -132,7 +132,7 @@ static void soft_max_f32_submitter(const float * x, const T * mask, float * dst, cgh.parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { soft_max_f32(x, mask, dst, ncols_par, nrows_y, scale, max_bias, m0, m1, n_head_log2, item_ct1, From fd123cfead49eb32e386e26b8ef7a6d41554dda5 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Tue, 18 Mar 2025 07:21:40 +0100 Subject: [PATCH 156/188] Vulkan: Default to 1GB allocations instead of 4GB to avoid fragmentation and driver issues (#12434) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index c0ee5dadef..dd680aa522 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -2524,13 +2524,9 @@ static vk_device ggml_vk_get_device(size_t idx) { if (GGML_VK_SUBALLOCATION_BLOCK_SIZE != nullptr) { device->suballocation_block_size = std::stoul(GGML_VK_SUBALLOCATION_BLOCK_SIZE); -#if defined(_WIN32) - } else if (device->vendor_id == VK_VENDOR_ID_NVIDIA) { + } else { // Limit batching of allocations to 1GB by default to avoid fragmentation issues device->suballocation_block_size = 1024*1024*1024; -#endif - } else { - device->suballocation_block_size = device->max_memory_allocation_size; } device->suballocation_block_size = std::min(device->suballocation_block_size, device->max_memory_allocation_size); From d9a14523bb9074eef42d1b43ae4a1e149f81b7e2 Mon Sep 17 00:00:00 2001 From: fj-y-saito <85871716+fj-y-saito@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:14:39 +0900 Subject: [PATCH 157/188] ggml : add SVE support for q6_K_q8_K (#12361) --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 151 +++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 8c7dbd1ccb..4e0ae05724 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -8158,7 +8158,156 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi const int nb = n / QK_K; -#ifdef __ARM_NEON +#ifdef __ARM_FEATURE_SVE + const int vector_length = ggml_cpu_get_sve_cnt()*8; + float sum = 0; + svuint8_t m4b = svdup_n_u8(0xf); + svint32_t vzero = svdup_n_s32(0); + svuint8_t mone = svdup_n_u8(0x30); + svint8_t q6bytes_1, q6bytes_2, q6bytes_3, q6bytes_4; + svuint8_t q6h_1, q6h_2, q6h_3, q6h_4; + + for (int i = 0; i < nb; ++i) { + const float d_all = GGML_FP16_TO_FP32(x[i].d); + + const uint8_t * GGML_RESTRICT q6 = x[i].ql; + const uint8_t * GGML_RESTRICT qh = x[i].qh; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + const int8_t * GGML_RESTRICT scale = x[i].scales; + + const svbool_t pg16_8 = svptrue_pat_b16(SV_VL8); + const svint16_t q8sums_1 = svld1_s16(pg16_8, y[i].bsums); + const svint16_t q8sums_2 = svld1_s16(pg16_8, y[i].bsums + 8); + const svint16_t q6scales_1 = svunpklo_s16(svld1_s8(svptrue_pat_b8(SV_VL8), scale)); + const svint16_t q6scales_2 = svunpklo_s16(svld1_s8(svptrue_pat_b8(SV_VL8), scale + 8)); + const svint64_t prod = svdup_n_s64(0); + int32_t isum_mins = svaddv_s64(svptrue_b64(), svadd_s64_x(svptrue_b64(), svdot_s64(prod, q8sums_1, q6scales_1), + svdot_s64(prod, q8sums_2, q6scales_2))); + int32_t isum = 0; + + switch (vector_length) { + case 128: + { + const svbool_t pg32_4 = svptrue_pat_b32(SV_VL4); + const svbool_t pg8_16 = svptrue_pat_b8(SV_VL16); + svint32_t isum_tmp = svdup_n_s32(0); + for (int j = 0; j < QK_K/128; ++j) { + svuint8_t qhbits_1 = svld1_u8(pg8_16, qh); + svuint8_t qhbits_2 = svld1_u8(pg8_16, qh+16); + qh += 32; + svuint8_t q6bits_1 = svld1_u8(pg8_16, q6); + svuint8_t q6bits_2 = svld1_u8(pg8_16, q6+16); + svuint8_t q6bits_3 = svld1_u8(pg8_16, q6+32); + svuint8_t q6bits_4 = svld1_u8(pg8_16, q6+48); + q6 += 64; + svint8_t q8bytes_1 = svld1_s8(pg8_16, q8); + svint8_t q8bytes_2 = svld1_s8(pg8_16, q8+16); + svint8_t q8bytes_3 = svld1_s8(pg8_16, q8+32); + svint8_t q8bytes_4 = svld1_s8(pg8_16, q8+48); + q8 += 64; + + q6h_1 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_1, 4)); + q6h_2 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_2, 4)); + q6h_3 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_1, 2)); + q6h_4 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_2, 2)); + q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_1, m4b), q6h_1)); + q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_2, m4b), q6h_2)); + q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_3, m4b), q6h_3)); + q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_4, m4b), q6h_4)); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale[0]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale[1]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale[2]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale[3]); + + scale += 4; + q8bytes_1 = svld1_s8(pg8_16, q8); + q8bytes_2 = svld1_s8(pg8_16, q8+16); + q8bytes_3 = svld1_s8(pg8_16, q8+32); + q8bytes_4 = svld1_s8(pg8_16, q8+48); + q8 += 64; + + q6h_1 = svand_u8_x(pg16_8, mone, qhbits_1); + q6h_2 = svand_u8_x(pg16_8, mone, qhbits_2); + q6h_3 = svand_u8_x(pg16_8, mone, svlsr_n_u8_x(pg16_8, qhbits_1, 2)); + q6h_4 = svand_u8_x(pg16_8, mone, svlsr_n_u8_x(pg16_8, qhbits_2, 2)); + q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_1, 4), q6h_1)); + q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_2, 4), q6h_2)); + q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_3, 4), q6h_3)); + q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_4, 4), q6h_4)); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale[0]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale[1]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale[2]); + isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale[3]); + scale += 4; + } + isum += svaddv_s32(pg32_4, isum_tmp); + sum += d_all * y[i].d * (isum - 32 * isum_mins); + } + break; + case 256: + case 512: + { + const svbool_t pg8_2 = svptrue_pat_b8(SV_VL2); + const svbool_t pg32_8 = svptrue_pat_b32(SV_VL8); + const svbool_t pg8_32 = svptrue_pat_b8(SV_VL32); + svint32_t isum_tmp = svdup_n_s32(0); + for (int j = 0; j < QK_K/128; j++) { + svuint8_t qhbits_1 = svld1_u8(pg8_32, qh); + qh += 32; + svuint8_t q6bits_1 = svld1_u8(pg8_32, q6); + svuint8_t q6bits_2 = svld1_u8(pg8_32, q6+32); + q6 += 64; + svint8_t q8bytes_1 = svld1_s8(pg8_32, q8); + svint8_t q8bytes_2 = svld1_s8(pg8_32, q8+32); + svint8_t q8bytes_3 = svld1_s8(pg8_32, q8+64); + svint8_t q8bytes_4 = svld1_s8(pg8_32, q8+96); + q8 += 128; + q6h_1 = svand_u8_x(pg8_32, mone, svlsl_n_u8_x(pg8_32, qhbits_1, 4)); + q6h_2 = svand_u8_x(pg8_32, mone, svlsl_n_u8_x(pg8_32, qhbits_1, 2)); + q6h_3 = svand_u8_x(pg8_32, mone, qhbits_1); + q6h_4 = svand_u8_x(pg8_32, mone, svlsr_n_u8_x(pg8_32, qhbits_1, 2)); + q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svand_u8_x(pg8_32, q6bits_1, m4b), q6h_1)); + q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svand_u8_x(pg8_32, q6bits_2, m4b), q6h_2)); + q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svlsr_n_u8_x(pg8_32, q6bits_1, 4), q6h_3)); + q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svlsr_n_u8_x(pg8_32, q6bits_2, 4), q6h_4)); + + svint8_t scale_lane_1_tmp = svld1_s8(pg8_2, scale); + scale_lane_1_tmp= svzip1_s8(scale_lane_1_tmp, scale_lane_1_tmp); + scale_lane_1_tmp= svzip1_s8(scale_lane_1_tmp, scale_lane_1_tmp); + svint8_t scale_lane_2_tmp = svld1_s8(pg8_2, scale+2); + scale_lane_2_tmp = svzip1_s8(scale_lane_2_tmp, scale_lane_2_tmp); + scale_lane_2_tmp = svzip1_s8(scale_lane_2_tmp, scale_lane_2_tmp); + svint8_t scale_lane_3_tmp = svld1_s8(pg8_2, scale+4); + scale_lane_3_tmp = svzip1_s8(scale_lane_3_tmp, scale_lane_3_tmp); + scale_lane_3_tmp = svzip1_s8(scale_lane_3_tmp, scale_lane_3_tmp); + svint8_t scale_lane_4_tmp = svld1_s8(pg8_2, scale+6); + scale_lane_4_tmp = svzip1_s8(scale_lane_4_tmp, scale_lane_4_tmp); + scale_lane_4_tmp = svzip1_s8(scale_lane_4_tmp, scale_lane_4_tmp); + svint32_t scale_lane_1 = svunpklo_s32(svunpklo_s16(scale_lane_1_tmp)); + svint32_t scale_lane_2 = svunpklo_s32(svunpklo_s16(scale_lane_2_tmp)); + svint32_t scale_lane_3 = svunpklo_s32(svunpklo_s16(scale_lane_3_tmp)); + svint32_t scale_lane_4 = svunpklo_s32(svunpklo_s16(scale_lane_4_tmp)); + + isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale_lane_1); + isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale_lane_2); + isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale_lane_3); + isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale_lane_4); + scale += 8; + } + isum += svaddv_s32(pg32_8, isum_tmp); + sum += d_all * y[i].d * (isum - 32 * isum_mins); + } + break; + default: + assert(false && "Unsupported vector length"); + break; + } + } + + *s = sum; + +#elif __ARM_NEON float sum = 0; const uint8x16_t m4b = vdupq_n_u8(0xF); From eba92d64c3f6d86de2e6b4dd3a540d2805a22b0c Mon Sep 17 00:00:00 2001 From: Prajwal B Mehendarkar Date: Tue, 18 Mar 2025 15:07:33 +0530 Subject: [PATCH 158/188] cmake : fix PowerPC build (#12241) Closes #12240 --- ggml/src/ggml-cpu/CMakeLists.txt | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index d6c4a9c299..6aa078a93e 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -287,17 +287,25 @@ function(ggml_add_cpu_backend_variant_impl tag_name) endif() endif() endif() - elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64") + elseif ("${CMAKE_SYSTEM_PROCESSOR} " STREQUAL "ppc64le " OR "${CMAKE_SYSTEM_PROCESSOR} " STREQUAL "powerpc ") message(STATUS "PowerPC detected") - execute_process(COMMAND bash -c "grep POWER /proc/cpuinfo | head -n 1" OUTPUT_VARIABLE POWER_M) - if (${POWER_M} MATCHES "POWER10") - list(APPEND ARCH_FLAGS -mcpu=power10) - elseif (${POWER_M} MATCHES "POWER9") - list(APPEND ARCH_FLAGS -mcpu=power9) + if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64") + file(READ "/proc/cpuinfo" POWER10_M) + elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "powerpc") + execute_process(COMMAND bash -c "prtconf |grep 'Implementation' | head -n 1" OUTPUT_VARIABLE POWER10_M) + endif() + + string(REGEX MATCHALL "POWER *([0-9]+)" MATCHED_STRING "${POWER10_M}") + string(REGEX REPLACE "POWER *([0-9]+)" "\\1" EXTRACTED_NUMBER "${MATCHED_STRING}") + + if (EXTRACTED_NUMBER GREATER_EQUAL 10) + list(APPEND ARCH_FLAGS -mcpu=power10 -mpowerpc64) + elseif (EXTRACTED_NUMBER EQUAL 9) + list(APPEND ARCH_FLAGS -mcpu=power9 -mpowerpc64) elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le") list(APPEND ARCH_FLAGS -mcpu=powerpc64le -mtune=native) else() - list(APPEND ARCH_FLAGS -mcpu=powerpc64 -mtune=native) + list(APPEND ARCH_FLAGS -mcpu=native -mtune=native -mpowerpc64) endif() elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64") message(STATUS "loongarch64 detected") From 810e0af3f50379682dd46b7967c4aadf3f8286f6 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 18 Mar 2025 12:05:42 +0200 Subject: [PATCH 159/188] server : fix warmup draft cache type (#12446) ggml-ci --- examples/server/server.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 71e053b202..c2f1afeca4 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1872,6 +1872,10 @@ struct server_context { params_dft.n_gpu_layers = params_base.speculative.n_gpu_layers; params_dft.n_parallel = 1; + // force F16 KV cache for the draft model for extra performance + params_dft.cache_type_k = GGML_TYPE_F16; + params_dft.cache_type_v = GGML_TYPE_F16; + llama_init_dft = common_init_from_params(params_dft); model_dft = llama_init_dft.model.get(); @@ -1892,10 +1896,6 @@ struct server_context { cparams_dft = common_context_params_to_llama(params_dft); cparams_dft.n_batch = n_ctx_dft; - // force F16 KV cache for the draft model for extra performance - cparams_dft.type_k = GGML_TYPE_F16; - cparams_dft.type_v = GGML_TYPE_F16; - // the context is not needed - we will create one for each slot llama_init_dft.context.reset(); } From 35cae5ba05a5292dc3108636a71ec59fa2f80ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= <112692748+lslusarczyk@users.noreply.github.com> Date: Tue, 18 Mar 2025 11:16:31 +0100 Subject: [PATCH 160/188] SYCL: using graphs is configurable by environment variable and compile option (#12371) * alberto changes * enable sycl graphs by env variable * fixed compilation warnings in ggml-sycl.cpp * renamed graph variables * fix markdown in docs/backend/SYCL.md Co-authored-by: Romain Biessy * fix markdown in docs/backend/SYCL.md again * compiling graphs by default, renamed graph_enable to graph_disable --------- Co-authored-by: Romain Biessy --- docs/backend/SYCL.md | 4 ++- ggml/CMakeLists.txt | 1 + ggml/src/ggml-sycl/CMakeLists.txt | 3 ++ ggml/src/ggml-sycl/common.hpp | 5 ++++ ggml/src/ggml-sycl/ggml-sycl.cpp | 47 +++++++++++++++++++++++++++++-- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 5da439e94e..184cd41955 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -660,8 +660,9 @@ use 1 SYCL GPUs: [0] with Max compute units:512 |--------------------|---------------------------------------|---------------------------------------------| | GGML_SYCL | ON (mandatory) | Enable build with SYCL code path.
    FP32 path - recommended for better perforemance than FP16 on quantized model| | GGML_SYCL_TARGET | INTEL *(default)* \| NVIDIA \| AMD | Set the SYCL target device type. | -| GGML_SYCL_DEVICE_ARCH | Optional (except for AMD) | Set the SYCL device architecture, optional except for AMD. Setting the device architecture can improve the performance. See the table [--offload-arch](https://github.com/intel/llvm/blob/sycl/sycl/doc/design/OffloadDesign.md#--offload-arch) for a list of valid architectures. | +| GGML_SYCL_DEVICE_ARCH | Optional (except for AMD) | Set the SYCL device architecture, optional except for AMD. Setting the device architecture can improve the performance. See the table [--offload-arch](https://github.com/intel/llvm/blob/sycl/sycl/doc/design/OffloadDesign.md#--offload-arch) for a list of valid architectures. | | GGML_SYCL_F16 | OFF *(default)* \|ON *(optional)* | Enable FP16 build with SYCL code path. | +| GGML_SYCL_GRAPH | ON *(default)* \|OFF *(Optional)* | Enable build with [SYCL Graph extension](https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/experimental/sycl_ext_oneapi_graph.asciidoc). | | CMAKE_C_COMPILER | `icx` *(Linux)*, `icx/cl` *(Windows)* | Set `icx` compiler for SYCL code path. | | CMAKE_CXX_COMPILER | `icpx` *(Linux)*, `icx` *(Windows)* | Set `icpx/icx` compiler for SYCL code path. | @@ -671,6 +672,7 @@ use 1 SYCL GPUs: [0] with Max compute units:512 |-------------------|------------------|---------------------------------------------------------------------------------------------------------------------------| | GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG | | GGML_SYCL_DISABLE_OPT | 0 (default) or 1 | Disable optimize features based on Intel GPU type, to compare the performance increase | +| GGML_SYCL_DISABLE_GRAPH | 0 or 1 (default) | Disable running computations through SYCL Graphs feature. Disabled by default because graph performance isn't yet better than non-graph performance. | | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
    Recommended to use when --split-mode = layer | diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 9a4ee4992d..740f9f69cf 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -186,6 +186,7 @@ option(GGML_OPENMP "ggml: use OpenMP" option(GGML_RPC "ggml: use RPC" OFF) option(GGML_SYCL "ggml: use SYCL" OFF) option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF) +option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON) set (GGML_SYCL_TARGET "INTEL" CACHE STRING "ggml: sycl target device") set (GGML_SYCL_DEVICE_ARCH "" CACHE STRING diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index 3ad044432a..271413ca41 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -66,6 +66,9 @@ if (WIN32) find_package(MKL REQUIRED) target_link_libraries(ggml-sycl PRIVATE IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL) else() + if (GGML_SYCL_GRAPH) + add_compile_definitions(GGML_SYCL_GRAPH) + endif() if (GGML_SYCL_TARGET STREQUAL "INTEL") target_link_libraries(ggml-sycl PRIVATE sycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread) elseif (GGML_SYCL_TARGET STREQUAL "NVIDIA") diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index fdd07d9caf..7cc5e14f9a 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -301,6 +301,7 @@ inline optimize_feature check_gpu_optimize_feature(syclex::architecture &arch) { return opt; } +namespace sycl_ex = sycl::ext::oneapi::experimental; struct ggml_backend_sycl_context { int device; std::string name; @@ -392,6 +393,10 @@ struct ggml_backend_sycl_context { return pool(device); } +#ifdef GGML_SYCL_GRAPH + std::unique_ptr> exec_graph = nullptr; +#endif + ggml_sycl_pool & host_pool(int device) { if (host_pools[device] == nullptr) { host_pools[device] = new_pool_for_host(stream(device, 0), device); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 207c0b440a..360e3f166c 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -46,6 +46,7 @@ static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; int g_ggml_sycl_disable_optimize = 0; +int g_ggml_sycl_disable_graph = 0; static ggml_sycl_device_info ggml_sycl_init() { ggml_sycl_device_info info = {}; @@ -191,10 +192,12 @@ static void ggml_check_sycl() try { if (!initialized) { g_ggml_sycl_debug = get_sycl_env("GGML_SYCL_DEBUG", 0); g_ggml_sycl_disable_optimize= get_sycl_env("GGML_SYCL_DISABLE_OPT", 0); + g_ggml_sycl_disable_graph = get_sycl_env("GGML_SYCL_DISABLE_GRAPH", 1); GGML_SYCL_DEBUG("[SYCL] call ggml_check_sycl\n"); GGML_LOG_INFO("Running with Environment Variables:\n"); GGML_LOG_INFO(" GGML_SYCL_DEBUG: %d\n", g_ggml_sycl_debug); GGML_LOG_INFO(" GGML_SYCL_DISABLE_OPT: %d\n", g_ggml_sycl_disable_optimize); + GGML_LOG_INFO(" GGML_SYCL_DISABLE_GRAPH: %d\n", g_ggml_sycl_disable_graph); GGML_LOG_INFO("Build with Macros:\n"); #if defined(GGML_SYCL_FORCE_MMQ) GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: yes\n"); @@ -3699,10 +3702,9 @@ static void optimize_graph_once(ggml_cgraph * cgraph, ggml_backend_sycl_context if (ctx->opt_feature.reorder) opt_for_reorder(cgraph->nodes[i], stream); } } -static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; - ggml_sycl_set_main_device(sycl_ctx->device); +static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { + ggml_sycl_set_main_device(sycl_ctx->device); if (!g_ggml_sycl_disable_optimize) optimize_graph_once(cgraph, sycl_ctx); for (int i = 0; i < cgraph->n_nodes; i++) { @@ -3724,7 +3726,46 @@ static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_ } GGML_ASSERT(ok); } +} +static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { + auto * sycl_ctx = static_cast(backend->context); + +#ifdef GGML_SYCL_GRAPH + if (!g_ggml_sycl_disable_graph) { + if (!sycl_ctx->exec_graph && !dpct::get_device(sycl_ctx->device).has(sycl::aspect::ext_oneapi_graph)) { + GGML_SYCL_DEBUG("[SYCL-GRAPH] can not use graphs on device:%d\n", sycl_ctx->device); + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + return GGML_STATUS_SUCCESS; + } + + sycl_ex::command_graph model_sycl_graph(*(sycl_ctx->stream())); + model_sycl_graph.begin_recording(*(sycl_ctx->stream())); + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + model_sycl_graph.end_recording(); + + if (!sycl_ctx->exec_graph) { + auto exec_graph = model_sycl_graph.finalize({sycl_ex::property::graph::updatable{}}); + sycl_ctx->exec_graph = std::make_unique< + sycl_ex::command_graph>(exec_graph); + } else { + try { + sycl_ctx->exec_graph->update(model_sycl_graph); + GGML_SYCL_DEBUG("[SYCL-GRAPH] update success\n"); + } catch (sycl::exception const & e) { + GGML_SYCL_DEBUG("[SYCL-GRAPH] Exception when updating graph, %s\n", e.what()); + auto exec_graph = model_sycl_graph.finalize({sycl_ex::property::graph::updatable{}}); + sycl_ctx->exec_graph = std::make_unique< + sycl_ex::command_graph>(exec_graph); + } + } + + sycl_ctx->stream()->ext_oneapi_graph(*(sycl_ctx->exec_graph)); + } else +#endif + { + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + } return GGML_STATUS_SUCCESS; } From 8551c44d840a7db50adb958ccaf464dc3ded82e7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 18 Mar 2025 13:05:49 +0200 Subject: [PATCH 161/188] context : always use non-causal attention for encoder graphs (#12447) * context : always use non-causal attention for encoder graphs ggml-ci * context : move the change to llama_context::encode() ggml-ci --- src/llama-context.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index abb7e526f6..42332acf1e 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1057,6 +1057,13 @@ int llama_context::encode(llama_batch & inp_batch) { ggml_backend_sched_reset(sched.get()); ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data); + const auto causal_attn_org = cparams.causal_attn; + + // always use non-causal attention for encoder graphs + // TODO: this is a tmp solution until we have a proper way to support enc-dec models + // ref: https://github.com/ggml-org/llama.cpp/pull/12181#issuecomment-2730451223 + cparams.causal_attn = false; + auto * gf = graph_init(); auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_ENCODER); @@ -1064,6 +1071,8 @@ int llama_context::encode(llama_batch & inp_batch) { res->set_inputs(&ubatch); + cparams.causal_attn = causal_attn_org; + const auto compute_status = graph_compute(gf, n_tokens > 1); switch (compute_status) { case GGML_STATUS_SUCCESS: From 99aa304fb900654ec338749f64e62895b9a88afd Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Tue, 18 Mar 2025 17:24:33 +0100 Subject: [PATCH 162/188] llama : add support for EXAONE tied word embeddings (#12451) --- src/llama-model.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index c571aa69b6..9171585bd9 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -3264,7 +3264,12 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; From c6af2161b200284d55633cf184a07406ca89908e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 18 Mar 2025 19:35:11 +0200 Subject: [PATCH 163/188] speculative : fix seg fault in certain cases (#12454) --- examples/speculative/speculative.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index bfddc67e03..627d01bbcb 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -331,11 +331,11 @@ int main(int argc, char ** argv) { } active_seqs.erase(s); - for(int i = 0; i < n_seq_dft; i++) { + for (int i = 0; i < n_seq_dft; i++) { if (i == s) { continue; } - if (drafts[i].tokens[i_dft] == drafts[s].tokens[i_dft]) { + if (drafts[i].active && drafts[i].tokens[i_dft] == drafts[s].tokens[i_dft]) { // synchronize active status for sequences with the same drafted token drafts[i].active = drafts[i].active && accept; if (!drafts[i].active) { From 29fff308c704c1c752cdb5153361e545e2bac09d Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Tue, 18 Mar 2025 19:16:19 +0100 Subject: [PATCH 164/188] llama : support converting Mistral Small text-only (#12450) --- convert_hf_to_gguf.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index d13d57c541..7a2ef4c7e3 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -1747,6 +1747,25 @@ class LlamaModel(Model): raise ValueError(f"Unprocessed experts: {experts}") +@Model.register("Mistral3ForConditionalGeneration") +class Mistral3Model(LlamaModel): + model_arch = gguf.MODEL_ARCH.LLAMA + + # we need to merge the text_config into the root level of hparams + def __init__(self, *args, **kwargs): + hparams = Model.load_hparams(kwargs["dir_model"]) + if "text_config" in hparams: + hparams = {**hparams, **hparams["text_config"]} + kwargs["hparams"] = hparams + super().__init__(*args, **kwargs) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None): + name = name.replace("language_model.", "") + if "multi_modal_projector" in name or "vision_tower" in name: + return [] + return super().modify_tensors(data_torch, name, bid) + + @Model.register("DeciLMForCausalLM") class DeciModel(Model): model_arch = gguf.MODEL_ARCH.DECI From bb115d2bf7ed2cdd7dccd7ae74cc9cfe4b0adb71 Mon Sep 17 00:00:00 2001 From: R0CKSTAR Date: Wed, 19 Mar 2025 02:28:26 +0800 Subject: [PATCH 165/188] musa: override warp_size of musa device to 32 (#12445) Signed-off-by: Xiaodong Ye --- ggml/src/ggml-cuda/ggml-cuda.cu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 8fb063822c..5cb56df9a8 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -262,6 +262,8 @@ static ggml_cuda_device_info ggml_cuda_init() { id, prop.name, prop.gcnArchName, info.devices[id].cc & 0xffff, device_vmm ? "yes" : "no", prop.warpSize); #elif defined(GGML_USE_MUSA) + // FIXME: Ensure compatibility with varying warp sizes across different MUSA archs. + info.devices[id].warp_size = 32; // TODO: refine the .cc to reflect MUSA's actual CC capabilities info.devices[id].smpbo = prop.sharedMemPerBlockOptin; info.devices[id].cc = 100*prop.major + 10*prop.minor; From 75422e8bc42646005be0754f7aa438b97a5e777e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 18 Mar 2025 21:35:19 +0200 Subject: [PATCH 166/188] graph : normalize Q, K, V shapes + sync cross attention (#12449) * graph : normalize Q, K, V shapes and add comments ggml-ci * context : synchronize before getting cross attention data * model : fix command-r attention norm check --- src/llama-context.cpp | 2 + src/llama-graph.cpp | 2 +- src/llama-graph.h | 24 +- src/llama-model.cpp | 682 ++++++++++++++++++++++++++---------------- 4 files changed, 433 insertions(+), 277 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 42332acf1e..664703a896 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1143,6 +1143,8 @@ int llama_context::encode(llama_batch & inp_batch) { if (model.arch == LLM_ARCH_T5 && t_embd) { //cross.t_embd = t_embd; + synchronize(); + cross.n_embd = t_embd->ne[0]; cross.n_enc = t_embd->ne[1]; cross.v_embd.resize(cross.n_embd*cross.n_enc); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 4e90873397..0bd4017443 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1378,7 +1378,7 @@ ggml_tensor * llm_graph_context::build_attn( // note: storing RoPE-ed version of K in the KV cache ggml_build_forward_expand(gf, ggml_cpy(ctx0, k_cur, k_cache_view)); - assert(v_cur->ne[0] == n_embd_v_gqa && v_cur->ne[1] == n_tokens); + v_cur = ggml_reshape_2d(ctx0, v_cur, n_embd_v_gqa, n_tokens); ggml_tensor * v_cache_view = nullptr; diff --git a/src/llama-graph.h b/src/llama-graph.h index c4328e6f9e..bdf19ed015 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -487,9 +487,9 @@ struct llm_graph_context { ggml_tensor * build_attn_mha( ggml_cgraph * gf, - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, + ggml_tensor * q, // [n_embd_head_q, n_tokens, n_head_q] + ggml_tensor * k, // [n_embd_head_k, n_tokens, n_head_k] + ggml_tensor * v, // [n_embd_head_v, n_tokens, n_head_v] (v_trans == false) ggml_tensor * kq_b, ggml_tensor * kq_mask, bool v_trans, @@ -502,9 +502,9 @@ struct llm_graph_context { ggml_cgraph * gf, ggml_tensor * wo, ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * k_cur, - ggml_tensor * v_cur, + ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens] + ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] + ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] ggml_tensor * kq_b, float kq_scale, int il) const; @@ -516,9 +516,9 @@ struct llm_graph_context { ggml_cgraph * gf, ggml_tensor * wo, ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * k_cur, - ggml_tensor * v_cur, + ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens] + ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] + ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] ggml_tensor * kq_b, float kq_scale, int il) const; @@ -530,9 +530,9 @@ struct llm_graph_context { ggml_cgraph * gf, ggml_tensor * wo, ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * k_cur, - ggml_tensor * v_cur, + ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens] + ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] + ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] ggml_tensor * kq_b, float kq_scale, int il) const; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 9171585bd9..d286176c1f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -4093,19 +4093,25 @@ struct llm_build_llama : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -4267,19 +4273,25 @@ struct llm_build_deci : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -4396,28 +4408,32 @@ struct llm_build_baichuan : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + switch (model.type) { case LLM_TYPE_7B: Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); break; case LLM_TYPE_13B: - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd/n_head, n_head, n_tokens); break; default: GGML_ABORT("fatal error"); } + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -4514,19 +4530,25 @@ struct llm_build_xverse : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -4632,25 +4654,26 @@ struct llm_build_falcon : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // using mode = 2 for neox mode Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -4762,19 +4785,25 @@ struct llm_build_grok : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -4907,24 +4936,26 @@ struct llm_build_dbrx : public llm_graph_context { Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -5031,12 +5062,14 @@ struct llm_build_starcoder : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -5128,11 +5161,13 @@ struct llm_build_refact : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - cb(Kcur, "Kcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -5267,21 +5302,21 @@ struct llm_build_bert : public llm_graph_context { Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Kcur, "Kcur", il); - - Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); } cb(Qcur, "Qcur", il); @@ -5397,12 +5432,14 @@ struct llm_build_bloom : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -5534,20 +5571,19 @@ struct llm_build_mpt : public llm_graph_context { model.layers[il].attn_k_norm_b, LLM_NORM, il); cb(Kcur, "Kcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - - cur = build_attn(inp_attn, gf, - model.layers[il].wo, model.layers[il].bo, - Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); - } else { - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - cur = build_attn(inp_attn, gf, - model.layers[il].wo, model.layers[il].bo, - Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); } + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, gf, + model.layers[il].wo, model.layers[il].bo, + Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); } if (il == n_layer - 1) { @@ -5656,9 +5692,8 @@ struct llm_build_stablelm : public llm_graph_context { } Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cb(Qcur, "Qcur", il); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - cb(Kcur, "Kcur", il); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); if (model.layers[il].attn_q_norm) { Qcur = build_norm(Qcur, @@ -5667,6 +5702,7 @@ struct llm_build_stablelm : public llm_graph_context { LLM_NORM, il); cb(Qcur, "Qcur", il); } + if (model.layers[il].attn_k_norm) { Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, @@ -5675,20 +5711,21 @@ struct llm_build_stablelm : public llm_graph_context { cb(Kcur, "Kcur", il); } - Qcur = ggml_rope_ext( ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -5792,25 +5829,26 @@ struct llm_build_qwen : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 2*sizeof(float)*(n_embd))); - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // using mode = 2 for neox mode Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -5901,33 +5939,36 @@ struct llm_build_qwen2 : public llm_graph_context { { // compute Q and K and RoPE them ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); cb(Kcur, "Kcur", il); ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6019,35 +6060,36 @@ struct llm_build_qwen2vl : public llm_graph_context { { // compute Q and K and RoPE them ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); cb(Kcur, "Kcur", il); ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_multi( - ctx0, - ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_multi( - ctx0, - ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6136,33 +6178,36 @@ struct llm_build_qwen2moe : public llm_graph_context { { // compute Q and K and RoPE them ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); cb(Kcur, "Kcur", il); ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6307,23 +6352,27 @@ struct llm_build_phi2 : public llm_graph_context { Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); // with phi2, we scale the Q to avoid precision issues // ref: https://github.com/ml-explore/mlx-examples/blob/08e862336ade809bc37d1035f94b359e7d1a5152/phi2/phi2.py#L64-L66 Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head))); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6433,22 +6482,27 @@ struct llm_build_phi3 : public llm_graph_context { Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head))); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f, il); @@ -6564,17 +6618,25 @@ struct llm_build_plamo : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); + ext_factor, attn_factor, beta_fast, beta_slow + ); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + ext_factor, attn_factor, beta_fast, beta_slow + ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -6679,7 +6741,9 @@ struct llm_build_gpt2 : public llm_graph_context { cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6773,27 +6837,29 @@ struct llm_build_codeshell : public llm_graph_context { cur = ggml_add(ctx0, cur, model.layers[il].bqkv); cb(cur, "bqkv", il); - ggml_tensor * tmpq = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - ggml_tensor * tmpk = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); - cb(tmpq, "tmpq", il); - cb(tmpk, "tmpk", il); - cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - ggml_tensor * Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); - - ggml_tensor * Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -6904,19 +6970,25 @@ struct llm_build_orion : public llm_graph_context { // cb(Vcur, "Vcur", il); // } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -7025,19 +7097,25 @@ struct llm_build_internlm2 : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -7311,7 +7389,7 @@ struct llm_build_minicpm3 : public llm_graph_context { struct llm_build_gemma : public llm_graph_context { llm_build_gemma(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { - const int64_t n_embd_head_k = hparams.n_embd_head_k; + const int64_t n_embd_head = hparams.n_embd_head_v; ggml_tensor * cur; ggml_tensor * inpL; @@ -7345,20 +7423,26 @@ struct llm_build_gemma : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - - Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); - cb(Qcur, "Qcur_scaled", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head))); + cb(Qcur, "Qcur_scaled", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -7421,7 +7505,7 @@ struct llm_build_gemma : public llm_graph_context { struct llm_build_gemma2 : public llm_graph_context { llm_build_gemma2(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { - const int64_t n_embd_head_k = hparams.n_embd_head_k; + const int64_t n_embd_head = hparams.n_embd_head_k; ggml_tensor * cur; ggml_tensor * inpL; @@ -7455,27 +7539,33 @@ struct llm_build_gemma2 : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); // ref: https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e switch (model.type) { case LLM_TYPE_2B: - case LLM_TYPE_9B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); break; - case LLM_TYPE_27B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd / n_head))); break; + case LLM_TYPE_9B: + case LLM_TYPE_27B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head))); break; default: GGML_ABORT("fatal error"); }; cb(Qcur, "Qcur_scaled", il); - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, Qcur, Kcur, Vcur, nullptr, 1.0f, il); @@ -7552,7 +7642,7 @@ struct llm_build_gemma2 : public llm_graph_context { struct llm_build_gemma3 : public llm_graph_context { llm_build_gemma3(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) { - const int64_t n_embd_head_k = hparams.n_embd_head_k; + const int64_t n_embd_head = hparams.n_embd_head_k; ggml_tensor * cur; ggml_tensor * inpL; @@ -7593,7 +7683,10 @@ struct llm_build_gemma3 : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); @@ -7601,9 +7694,7 @@ struct llm_build_gemma3 : public llm_graph_context { ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); @@ -7611,7 +7702,10 @@ struct llm_build_gemma3 : public llm_graph_context { ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -7733,19 +7827,25 @@ struct llm_build_starcoder2 : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -8046,24 +8146,25 @@ struct llm_build_command_r : public llm_graph_context { cb(Vcur, "Vcur", il); } - if (model.layers[il].attn_q_norm) { - Qcur = ggml_view_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens, - ggml_element_size(Qcur) * n_embd_head, - ggml_element_size(Qcur) * n_embd_head * n_head, - 0); - cb(Qcur, "Qcur", il); - Kcur = ggml_view_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens, - ggml_element_size(Kcur) * n_embd_head, - ggml_element_size(Kcur) * n_embd_head * n_head_kv, - 0); - cb(Kcur, "Kcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + if (model.layers[il].attn_q_norm) { Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM, il); cb(Qcur, "Qcur", il); + } + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + if (model.layers[il].attn_k_norm) { Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, @@ -8071,19 +8172,15 @@ struct llm_build_command_r : public llm_graph_context { cb(Kcur, "Kcur", il); } - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -8198,25 +8295,28 @@ struct llm_build_cohere2 : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + if (is_swa) { - Qcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, - beta_fast, beta_slow); - cb(Qcur, "Qcur", il); + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); - Kcur = ggml_rope_ext(ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur", il); - } else { - // For non-sliding layers, just reshape without applying RoPE - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - cb(Qcur, "Qcur", il); - - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - cb(Kcur, "Kcur", il); + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); } + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -8328,19 +8428,25 @@ struct llm_build_olmo : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, nullptr, @@ -8442,22 +8548,25 @@ struct llm_build_olmo2 : public llm_graph_context { LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_ext( ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur_rope", il); Kcur = ggml_rope_ext( ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Kcur, "Kcur_rope", il); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -8572,22 +8681,25 @@ struct llm_build_olmoe : public llm_graph_context { LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_ext( ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur_rope", il); Kcur = ggml_rope_ext( ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Kcur, "Kcur_rope", il); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -8687,7 +8799,7 @@ struct llm_build_openelm : public llm_graph_context { cur = ggml_reshape_3d(ctx0, cur, n_embd_head_k, n_head_qkv, n_tokens); - ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, cur->nb[1], cur->nb[2], 0)); + ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, cur->nb[1], cur->nb[2], 0)); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*n_head)); @@ -8707,18 +8819,19 @@ struct llm_build_openelm : public llm_graph_context { cb(Kcur, "Kcur", il); Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Qcur, inp_pos, NULL, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, NULL, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ctx0, Kcur, inp_pos, NULL, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Kcur, "Kcur", il); - Vcur = ggml_reshape_2d(ctx0, Vcur, n_embd_head * n_head_kv, n_tokens); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); cb(Qcur, "Vcur", il); cur = build_attn(inp_attn, gf, @@ -8815,24 +8928,26 @@ struct llm_build_gptneox : public llm_graph_context { ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); @@ -8963,19 +9078,25 @@ struct llm_build_arctic : public llm_graph_context { ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, @@ -9112,19 +9233,25 @@ struct llm_build_deepseek : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -9502,19 +9629,25 @@ struct llm_build_bitnet : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, NULL, NULL, @@ -9906,7 +10039,9 @@ struct llm_build_jais : public llm_graph_context { cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -10019,29 +10154,30 @@ struct llm_build_chatglm : public llm_graph_context { Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + //printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor); + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - //printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor); - Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur_rope", il); - - Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur_rope", il); - cur = build_attn(inp_attn, gf, model.layers[il].wo, NULL, Qcur, Kcur, Vcur, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); - } if (il == n_layer - 1) { @@ -10145,19 +10281,25 @@ struct llm_build_nemotron : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -10270,19 +10412,25 @@ struct llm_build_exaone : public llm_graph_context { cb(Vcur, "Vcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, model.layers[il].bo, @@ -11166,19 +11314,25 @@ struct llm_build_chameleon : public llm_graph_context { cb(Kcur, "Kcur", il); } + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + Qcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(Qcur, "Qcur", il); Kcur = ggml_rope_ext( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); + + cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); cur = build_attn(inp_attn, gf, model.layers[il].wo, nullptr, From d84635b1b085d54d6a21924e6171688d6e3dfb46 Mon Sep 17 00:00:00 2001 From: lhez Date: Tue, 18 Mar 2025 12:54:55 -0700 Subject: [PATCH 167/188] opencl: improve profiling (#12442) * opencl: more profiling timing * opencl: generate trace for profiling * opencl: reduce profiling overhead * Populate profiling timing info at the end rather than after each kernel run * opencl: fix for chrome tracing --- ggml/src/ggml-opencl/ggml-opencl.cpp | 115 ++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 21 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 14d9934fb1..efaf7f4790 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -297,8 +297,27 @@ static int ggml_backend_opencl_n_devices = 0; struct ProfilingInfo { std::string op_name; std::string kernel_name; - // Kernel execution time in nanoseconds. - cl_ulong duration_ns; + + cl_kernel kernel; + cl_event evt; + + cl_ulong cmd_queued; + cl_ulong cmd_submit; + cl_ulong cmd_start; + cl_ulong cmd_end; + cl_ulong overhead_start; + cl_ulong overhead_end; + // For the times below, see spec for clGetEventProfilingInfo + // The time kernel spent in cmd queue - SUBMIT - QUEUED + cl_ulong cmd_queued_duration_ns; + // The time kernel spent for submission - START - SUBMIT + cl_ulong cmd_submit_duration_ns; + // Kernel execution time in nanoseconds - END - START + cl_ulong cmd_duration_ns; + // The time for the kernel to complete - COMPLETE - END + cl_ulong cmd_complete_duration_ns; + // Total time to finish the kernel - COMPELTE - QUEUED + cl_ulong cmd_total_duration_ns; // Global and local work sizes. size_t global_size[3]; size_t local_size[3]; @@ -903,12 +922,56 @@ static void ggml_cl2_free(void) { return; } + // Populate profiling info + for (ProfilingInfo & info : g_profiling_info) { + cl_ulong cmd_queued; + cl_ulong cmd_submit; + cl_ulong cmd_start; + cl_ulong cmd_end; + cl_ulong cmd_complete; + + CL_CHECK(clWaitForEvents(1, &info.evt)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_QUEUED, sizeof(cl_ulong), &cmd_queued, NULL)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_SUBMIT, sizeof(cl_ulong), &cmd_submit, NULL)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_START, sizeof(cl_ulong), &cmd_start, NULL)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &cmd_end, NULL)); + CL_CHECK(clGetEventProfilingInfo( + info.evt, CL_PROFILING_COMMAND_COMPLETE, sizeof(cl_ulong), &cmd_complete, NULL)); + CL_CHECK(clReleaseEvent(info.evt)); + + char kernel_name[512]; + CL_CHECK(clGetKernelInfo(info.kernel, CL_KERNEL_FUNCTION_NAME, + sizeof(kernel_name), kernel_name, NULL)); + info.kernel_name = kernel_name; + + info.cmd_queued = cmd_queued; + info.cmd_submit = cmd_submit; + info.cmd_start = cmd_start; + info.cmd_end = cmd_end; + + info.cmd_queued_duration_ns = cmd_submit - cmd_queued; + info.cmd_submit_duration_ns = cmd_start - cmd_submit; + info.cmd_duration_ns = cmd_end - cmd_start; + info.cmd_complete_duration_ns = cmd_complete - cmd_end; + info.cmd_total_duration_ns = cmd_complete - cmd_queued; + } + + // Dump a csv float total_kernel_time = 0; - fprintf(fperf, "op name, kernel name, duration (ms), global size, local size, output size\n"); + fprintf(fperf, "op name, kernel name, queued duration (ms), submit duration(ms), exec duration (ms), complete duration (ms), total duration (ms), global size, local size, output size\n"); for (const ProfilingInfo & info : g_profiling_info) { - total_kernel_time += info.duration_ns/1.e6f; - fprintf(fperf, "%s,%s,%f,%zux%zux%zu,%zux%zux%zu,%zux%zux%zux%zu\n", - info.op_name.c_str(), info.kernel_name.c_str(), info.duration_ns/1.e6f, + total_kernel_time += info.cmd_duration_ns/1.e6f; + fprintf(fperf, "%s,%s,%f,%f,%f,%f,%f,%zux%zux%zu,%zux%zux%zu,%zux%zux%zux%zu\n", + info.op_name.c_str(), info.kernel_name.c_str(), + info.cmd_queued_duration_ns/1.e6f, + info.cmd_submit_duration_ns/1.e6f, + info.cmd_duration_ns/1.e6f, + info.cmd_complete_duration_ns/1.e6f, + info.cmd_total_duration_ns/1.e6f, info.global_size[0], info.global_size[1], info.global_size[2], info.local_size[0], info.local_size[2], info.local_size[2], info.output_size[0], info.output_size[1], info.output_size[2], info.output_size[3]); @@ -916,6 +979,27 @@ static void ggml_cl2_free(void) { fclose(fperf); GGML_LOG_INFO("ggml_opencl: total kernel time: %f\n", total_kernel_time); + + // Dump a simple chrome trace + FILE* ftrace = fopen("cl_trace.json", "w"); + if (!ftrace) { + GGML_LOG_ERROR("Failed to open cl_trace.json\n"); + return; + } + + fprintf(ftrace, "[\n"); + for (const ProfilingInfo & info : g_profiling_info) { + fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"B\", \"ts\": %lu, \"pid\": \"\", \"tid\": \"Host\"},\n", + info.kernel_name.c_str(), info.cmd_queued/1000); + fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"E\", \"ts\": %lu, \"pid\": \"\", \"tid\": \"Host\"},\n", + info.kernel_name.c_str(), info.cmd_submit/1000); + + fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"B\", \"ts\": %lu, \"pid\": \"\", \"tid\": \"Device\"},\n", + info.kernel_name.c_str(), info.cmd_start/1000); + fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"E\", \"ts\": %lu, \"pid\": \"\", \"tid\": \"Device\"},\n", + info.kernel_name.c_str(), info.cmd_end/1000); + } + fclose(ftrace); #endif } @@ -2062,25 +2146,14 @@ static void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tenso // Profiling utility //------------------------------------------------------------------------------ #ifdef GGML_OPENCL_PROFILING -void populateProfilingInfo( +static void populateProfilingInfo( ProfilingInfo& info, cl_event evt, cl_kernel kernel, size_t global_size[3], size_t local_size[3], const ggml_tensor * tensor) { - cl_ulong start; - cl_ulong end; - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clGetEventProfilingInfo( - evt, CL_PROFILING_COMMAND_START, sizeof(cl_ulong), &start, NULL)); - CL_CHECK(clGetEventProfilingInfo( - evt, CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &end, NULL)); + info.op_name = tensor->name; + info.kernel = kernel; + info.evt = evt; - char kernel_name[512]; - CL_CHECK(clGetKernelInfo(kernel, CL_KERNEL_FUNCTION_NAME, - sizeof(kernel_name), kernel_name, NULL)); - - info.duration_ns = end - start; - info.op_name = tensor->name; - info.kernel_name = kernel_name; info.local_size[0] = local_size[0]; info.local_size[1] = local_size[1]; info.local_size[2] = local_size[2]; From c446b2edd2a9fe2772a1a18923c3e54a6749c364 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Wed, 19 Mar 2025 02:26:26 -0500 Subject: [PATCH 168/188] vulkan: Submit once enough matmul work has been recorded (#12406) I've been seeing significantly worse performance for tg with flash attention enabled vs disabled, and it seems to be related to the submit heuristic. Change the heuristic to check how many bytes worth of weight matrix are used and flush every 100MB, and ramp up after the first few submits. This seems to resolve the issue, and also increases perf for non-FA a bit. --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 32 ++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index dd680aa522..d450fe9a2f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -8436,8 +8436,12 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)"); ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + uint64_t total_mat_mul_bytes = 0; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_vk_build_graph(ctx, cgraph->nodes[i], i, nullptr, 0, true, false, false); + if (cgraph->nodes[i]->op == GGML_OP_MUL_MAT || cgraph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { + total_mat_mul_bytes += ggml_nbytes(cgraph->nodes[i]->src[0]); + } } if (ctx->device->need_compiles) { ggml_vk_load_shaders(ctx->device); @@ -8458,17 +8462,27 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg bool first_node_in_batch = true; // true if next node will be first node in a batch int submit_node_idx = 0; // index to first node in a batch - // Submit work every nodes_per_submit nodes to overlap CPU cmdbuffer generation with GPU execution. - // Start with a smaller count to get work submitted right away, and increase it after each submit. - int nodes_per_submit = 20; + // Submit after enough work has accumulated, to overlap CPU cmdbuffer generation with GPU execution. + // Estimate the amount of matmul work by looking at the weight matrix size, and submit every 100MB + // (and scaled down based on model size, so smaller models submit earlier). + // Also submit at least every 100 nodes, in case there are workloads without as much matmul. + int nodes_per_submit = 100; int submitted_nodes = 0; int submit_count = 0; + uint64_t mul_mat_bytes = 0; + uint64_t mul_mat_bytes_per_submit = std::min(uint64_t(100*1000*1000), total_mat_mul_bytes / 40u); for (int i = 0; i < cgraph->n_nodes; i++) { if (first_node_in_batch) { submit_node_idx = i; } - bool submit = (submitted_nodes >= nodes_per_submit) || (i == last_node); + if (cgraph->nodes[i]->op == GGML_OP_MUL_MAT || cgraph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { + mul_mat_bytes += ggml_nbytes(cgraph->nodes[i]->src[0]); + } + + bool submit = (submitted_nodes >= nodes_per_submit) || + (mul_mat_bytes >= mul_mat_bytes_per_submit) || + (i == last_node); bool enqueued = ggml_vk_build_graph(ctx, cgraph->nodes[i], i, cgraph->nodes[submit_node_idx], submit_node_idx, false, i == last_node, submit); @@ -8485,13 +8499,9 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg if (submit) { first_node_in_batch = true; submitted_nodes = 0; - switch (submit_count) { - case 0: - nodes_per_submit = 50; - break; - default: - nodes_per_submit = 100; - break; + mul_mat_bytes = 0; + if (submit_count < 3) { + mul_mat_bytes_per_submit *= 2; } submit_count++; } From a686171ea71ed8cb8a324850d146cb65a001e141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 19 Mar 2025 08:58:13 +0100 Subject: [PATCH 169/188] convert : Support chat_template.json (#12460) --- gguf-py/gguf/vocab.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index 2ef7d14ab1..cca0979862 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -154,7 +154,12 @@ class SpecialVocab: return True with open(tokenizer_config_file, encoding = 'utf-8') as f: tokenizer_config = json.load(f) - chat_template = tokenizer_config.get('chat_template') + chat_template_alt = None + chat_template_file = path / 'chat_template.json' + if chat_template_file.is_file(): + with open(chat_template_file, encoding = 'utf-8') as f: + chat_template_alt = json.load(f).get('chat_template') + chat_template = tokenizer_config.get('chat_template', chat_template_alt) if chat_template is None or isinstance(chat_template, (str, list)): self.chat_template = chat_template else: From 108e53c2f1b57b79c8bcf9773fba5a82e2f1eeb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 19 Mar 2025 09:08:49 +0100 Subject: [PATCH 170/188] llama : add support for GPT2, Bloom and CodeShell tied word embeddings (#12456) * Add support for GPT2, Bloom and CodeShell tied word embeddings * Deduplicate tied word embeddings weights * Workaround for incorrect weight map It appears transformer.wte.weight is in the weight map even though the weights are not there, remove it if output weights are encountered first. * check++ * fatfingers-- --- convert_hf_to_gguf.py | 37 ++++++++++++++++--------------------- src/llama-model.cpp | 21 ++++++++++++++++++--- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 7a2ef4c7e3..7574218e24 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -180,7 +180,8 @@ class Model: extra = sorted(tensor_names_from_parts.difference(self.tensor_names)) missing_files = sorted(set(weight_map[n] for n in missing if n in weight_map)) if len(extra) == 0 and len(missing_files) > 0: - raise ValueError(f"Missing or incomplete model files: {missing_files}") + raise ValueError(f"Missing or incomplete model files: {missing_files}\n" + f"Missing tensors: {missing}") else: raise ValueError("Mismatch between weight map and model parts for tensor names:\n" f"Missing tensors: {missing}\n" @@ -1099,13 +1100,6 @@ class BloomModel(Model): tensors.append((self.map_tensor_name(name), data_torch)) - if name == "word_embeddings.weight": - assert self.tensor_names is not None - - # TODO: tie them at runtime, don't duplicate in the model file - if all(s not in self.tensor_names for s in ("lm_head.weight", "output.weight")): - tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch)) - return tensors @@ -2423,10 +2417,6 @@ class GPT2Model(Model): tensors.append((new_name, data_torch)) - # note: GPT2 output is tied to (same as) wte in original model - if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD): - tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch)) - return tensors @@ -2756,21 +2746,26 @@ class CodeShellModel(Model): self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR) self.gguf_writer.add_rope_scaling_factor(1.0) + _has_tok_embd = False + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: del bid # unused + output_name = self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT) + tok_embd_name = self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD) + new_name = self.map_tensor_name(name) - tensors: list[tuple[str, Tensor]] = [(new_name, data_torch)] + # assuming token_embd.weight is seen before output.weight + if not self._has_tok_embd and new_name == self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT): + # even though the tensor file(s) does not contain the word embeddings they are still in the weight map + if self.tensor_names and "transformer.wte.weight" in self.tensor_names: + logger.debug(f"{tok_embd_name} not found before {output_name}, assuming they are tied") + self.tensor_names.remove("transformer.wte.weight") + elif new_name == tok_embd_name: + self._has_tok_embd = True - if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD): - assert self.tensor_names is not None - - if all(s not in self.tensor_names for s in ("lm_head.weight", "output.weight")): - # copy tok_embd.weight to output.weight - tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch)) - - return tensors + return [(new_name, data_torch)] @Model.register("InternLM2ForCausalLM") diff --git a/src/llama-model.cpp b/src/llama-model.cpp index d286176c1f..17af8cc30b 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2020,7 +2020,12 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; @@ -2381,7 +2386,12 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; @@ -2407,7 +2417,12 @@ bool llama_model::load_tensors(llama_model_loader & ml) { } break; case LLM_ARCH_CODESHELL: { - tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + + // if tok embd is NULL, init from output + if (tok_embd == NULL) { + tok_embd = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); From 0fd8487b142b2b92565bc95b39ddc440955a237c Mon Sep 17 00:00:00 2001 From: Guus Waals <_@guusw.nl> Date: Wed, 19 Mar 2025 10:15:23 +0000 Subject: [PATCH 171/188] Fix visionOS build and add CI (#12415) * ci: add visionOS build workflow Add a new GitHub Actions workflow for building on visionOS with CMake and Xcode. * ggml: Define _DARWIN_C_SOURCE for visionOS to fix missing u_xxx typedefs * ci: remove define hacks for u_xxx system types --------- Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com> --- .github/workflows/build.yml | 29 +++++++++++++++++++++++++++++ build-xcframework.sh | 8 ++++---- ggml/src/CMakeLists.txt | 4 ++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03cde0a484..7db8552865 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -676,6 +676,35 @@ jobs: -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO + macOS-latest-cmake-visionos: + runs-on: macos-latest + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + + - name: Dependencies + id: depends + continue-on-error: true + run: | + brew update + + - name: Build + id: cmake_build + run: | + sysctl -a + cmake -B build -G Xcode \ + -DGGML_METAL_USE_BF16=ON \ + -DGGML_METAL_EMBED_LIBRARY=ON \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DCMAKE_SYSTEM_NAME=visionOS \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \ + -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO + macOS-latest-swift: runs-on: macos-latest diff --git a/build-xcframework.sh b/build-xcframework.sh index 37833dc4ea..2ce3939c43 100755 --- a/build-xcframework.sh +++ b/build-xcframework.sh @@ -432,8 +432,8 @@ cmake -B build-visionos -G Xcode \ -DCMAKE_SYSTEM_NAME=visionOS \ -DCMAKE_OSX_SYSROOT=xros \ -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \ - -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 -Du_int=unsigned\ int -Du_char=unsigned\ char -Du_short=unsigned\ short ${COMMON_C_FLAGS}" \ - -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 -Du_int=unsigned\ int -Du_char=unsigned\ char -Du_short=unsigned\ short ${COMMON_CXX_FLAGS}" \ + -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}" \ + -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}" \ -S . cmake --build build-visionos --config Release -- -quiet @@ -445,8 +445,8 @@ cmake -B build-visionos-sim -G Xcode \ -DCMAKE_SYSTEM_NAME=visionOS \ -DCMAKE_OSX_SYSROOT=xrsimulator \ -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \ - -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 -Du_int=unsigned\ int -Du_char=unsigned\ char -Du_short=unsigned\ short ${COMMON_C_FLAGS}" \ - -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 -Du_int=unsigned\ int -Du_char=unsigned\ char -Du_short=unsigned\ short ${COMMON_CXX_FLAGS}" \ + -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}" \ + -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}" \ -S . cmake --build build-visionos-sim --config Release -- -quiet diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index a797e2b187..c1c7498694 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -325,6 +325,10 @@ if (CMAKE_SYSTEM_NAME MATCHES "Android") target_link_libraries(ggml-base PRIVATE dl) endif() +if(CMAKE_SYSTEM_NAME MATCHES "visionOS") + target_compile_definitions(ggml-base PUBLIC _DARWIN_C_SOURCE) +endif() + if (BUILD_SHARED_LIBS) foreach (target ggml-base ggml) set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON) From a9b59288e222f39fc0311dc66944ed5a86c815fa Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Wed, 19 Mar 2025 13:56:23 -0500 Subject: [PATCH 172/188] vulkan: optimize iq1 coopmat2 dequant functions (#12427) --- .../vulkan-shaders/dequant_funcs_cm2.comp | 18 ++++++++++++------ ggml/src/ggml-vulkan/vulkan-shaders/types.comp | 7 +++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp index 8efe4653ff..b3fad35e21 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp @@ -311,8 +311,8 @@ float16_t dequantFuncIQ1_S(const in decodeBufIQ1_S bl, const in uint blockCoords const float16_t d = bl.block.d; const uint idx = coordInBlock[1]; - const uint ib32 = idx / 32; - const uint ib8 = idx / 8; + const uint ib32 = (idx & 0xE0) >> 5; + const uint ib8 = (idx & 0xF8) >> 3; const uint qh = bl.block.qh[ib32]; const uint qs = bl.block.qs[ib8]; @@ -330,14 +330,20 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1 block_iq1_m block; }; +layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufIQ1_M_packed64 { + block_iq1_m_packed64 block; +}; + float16_t dequantFuncIQ1_M(const in decodeBufIQ1_M bl, const in uint blockCoords[2], const in uint coordInBlock[2]) { - const u16vec4 scales = u16vec4(bl.block.scales[0], bl.block.scales[1], bl.block.scales[2], bl.block.scales[3]) >> 12; - const float16_t d = uint16BitsToHalf(scales.x | (scales.y << 4) | (scales.z << 8) | (scales.w << 12)); + decodeBufIQ1_M_packed64 bl64 = decodeBufIQ1_M_packed64(bl); const uint idx = coordInBlock[1]; - const uint ib8 = idx / 8; - const uint ib16 = idx / 16; + uvec2 scales = unpack32(bl64.block.scales); + const float16_t d = uint16BitsToHalf(uint16_t(((scales.x & 0xF000) >> 12) | ((scales.x & 0xF0000000) >> 24) | ((scales.y & 0xF000) >> 4) | ((scales.y & 0xF0000000) >> 16))); + + const uint ib8 = (idx & 0xF8) >> 3; + const uint ib16 = (idx & 0xF0) >> 4; const int i8 = int(idx % 8); const uint sc = bl.block.scales[ib8 / 8]; const uint qs = bl.block.qs[ib8]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp index f01179326e..789776816b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp @@ -2,6 +2,7 @@ #if !defined(GGML_TYPES_COMP) #define GGML_TYPES_COMP +#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #extension GL_EXT_shader_explicit_arithmetic_types_int8 : require @@ -312,6 +313,12 @@ struct block_iq1_m { uint16_t scales[QUANT_K_IQ1_M/64]; }; +struct block_iq1_m_packed64 { + uint64_t qs[QUANT_K_IQ1_M/8/8]; + uint64_t qh[QUANT_K_IQ1_M/16/8]; + uint64_t scales; +}; + #if defined(DATA_A_IQ1_S) #define QUANT_K QUANT_K_IQ1_S #define QUANT_R QUANT_R_IQ1_S From 517b5ddbf002b91fd6d6daf5d8db8c88a0173039 Mon Sep 17 00:00:00 2001 From: Gaurav Garg <52341457+gaugarg-nv@users.noreply.github.com> Date: Thu, 20 Mar 2025 01:22:06 +0530 Subject: [PATCH 173/188] CUDA: Improve flash decoding kernel GPU occupancy for BS=1 case (#12183) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Find out active blocks per SM using cudaOccupancyMaxActiveBlocksPerMultiprocessor API. Use this value to determine the optimal parallel_blocks value. - Prefer vector flash attention kernels over MMA kernel for BS=1 Fixes Issue: #12182 --------- Co-authored-by: Johannes Gäßler --- ggml/src/ggml-cuda/fattn-common.cuh | 88 +++++++++++++++++++--------- ggml/src/ggml-cuda/fattn-mma-f16.cuh | 3 +- ggml/src/ggml-cuda/fattn-tile-f16.cu | 63 ++++++++------------ ggml/src/ggml-cuda/fattn-tile-f32.cu | 63 ++++++++------------ ggml/src/ggml-cuda/fattn-vec-f16.cuh | 73 +++++++++-------------- ggml/src/ggml-cuda/fattn-vec-f32.cuh | 73 +++++++++-------------- ggml/src/ggml-cuda/fattn-wmma-f16.cu | 65 +++++--------------- ggml/src/ggml-cuda/fattn.cu | 18 +++--- ggml/src/ggml-cuda/ggml-cuda.cu | 3 + ggml/src/ggml-cuda/vendors/hip.h | 1 + ggml/src/ggml-cuda/vendors/musa.h | 1 + tests/test-backend-ops.cpp | 29 ++++++--- 12 files changed, 214 insertions(+), 266 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 4067fd41bc..1c2a2a138f 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -606,48 +606,47 @@ static __global__ void flash_attn_stream_k_fixup( *dst = dst_val / rowsum; } -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(D, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) static __global__ void flash_attn_combine_results( const float * __restrict__ VKQ_parts, const float2 * __restrict__ VKQ_meta, - float * __restrict__ dst) { - VKQ_parts += parallel_blocks*D * gridDim.y*blockIdx.x; - VKQ_meta += parallel_blocks * gridDim.y*blockIdx.x; - dst += D * gridDim.y*blockIdx.x; + float * __restrict__ dst, + const int parallel_blocks) { + VKQ_parts += parallel_blocks*D * gridDim.z*blockIdx.x; + VKQ_meta += parallel_blocks * gridDim.z*blockIdx.x; + dst += D * gridDim.z*blockIdx.x; const int tid = threadIdx.x; __builtin_assume(tid < D); - __shared__ float2 meta[parallel_blocks]; + extern __shared__ float2 meta[]; if (tid < 2*parallel_blocks) { - ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.y*(2*parallel_blocks) + tid]; + ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.z*(2*parallel_blocks) + tid]; } __syncthreads(); float kqmax = meta[0].x; -#pragma unroll for (int l = 1; l < parallel_blocks; ++l) { kqmax = max(kqmax, meta[l].x); } float VKQ_numerator = 0.0f; float VKQ_denominator = 0.0f; -#pragma unroll for (int l = 0; l < parallel_blocks; ++l) { const float diff = meta[l].x - kqmax; const float KQ_max_scale = expf(diff); const uint32_t ftz_mask = 0xFFFFFFFF * (diff > SOFTMAX_FTZ_THRESHOLD); *((uint32_t *) &KQ_max_scale) &= ftz_mask; - VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.y*D + blockIdx.y*D + tid]; + VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.z*D + blockIdx.z*D + tid]; VKQ_denominator += KQ_max_scale * meta[l].y; } - dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator; + dst[blockIdx.z*D + tid] = VKQ_numerator / VKQ_denominator; } static void on_no_fattn_vec_case(const int D) { @@ -671,12 +670,10 @@ static void on_no_fattn_vec_case(const int D) { } } -// parallel_blocks == 0 is stream-k decomposition -template +template void launch_fattn( - ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, - const int nwarps, const size_t nbytes_shared, const bool need_f16_K, const bool need_f16_V, - const int warp_size = WARP_SIZE + ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared, + const int KQ_row_granularity, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const int warp_size = WARP_SIZE ) { constexpr int ncols = ncols1 * ncols2; @@ -748,12 +745,14 @@ void launch_fattn( nb23 = nb23*bs*sizeof(half)/ts; } + int parallel_blocks = 1; + const int ntiles_x = ((Q->ne[1] + ncols1 - 1) / ncols1); const int ntiles_total = ntiles_x * (Q->ne[2] / ncols2) * Q->ne[3]; const dim3 block_dim(warp_size, nwarps, 1); dim3 blocks_num; - if (parallel_blocks == 0) { + if (stream_k) { // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup. const int max_blocks = 2*nsm; const int tiles_nwaves = (ntiles_total + max_blocks - 1) / max_blocks; @@ -769,9 +768,43 @@ void launch_fattn( dst_tmp_meta.alloc(blocks_num.x*ncols * (2*2 + D) * sizeof(float)); } else { - blocks_num.x = parallel_blocks*ntiles_x; - blocks_num.y = Q->ne[2]; - blocks_num.z = Q->ne[3]; + GGML_ASSERT(K->ne[1] % KQ_row_granularity == 0); + const int ntiles_KQ = K->ne[1] / KQ_row_granularity; // Max. number of parallel blocks limited by tensor size. + + int max_blocks_per_sm = 1; // Max. number of active blocks limited by occupancy. + CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, fattn_kernel, block_dim.x * block_dim.y * block_dim.z, nbytes_shared)); + + // parallel_blocks should be at least large enough to achieve max. occupancy for a single wave: + parallel_blocks = std::max((nsm * max_blocks_per_sm) / ntiles_total, 1); + + // parallel_blocks must not be larger than what the tensor size allows: + parallel_blocks = std::min(parallel_blocks, ntiles_KQ); + + // If ntiles_total % blocks_per_wave != 0 then some efficiency is lost due to tail effects. + // Test whether parallel_blocks can be set to a higher value for better efficiency. + const int blocks_per_wave = nsm * max_blocks_per_sm; + int nwaves_best = 0; + int efficiency_percent_best = 0; + for (int parallel_blocks_test = parallel_blocks; parallel_blocks_test <= ntiles_KQ; ++parallel_blocks_test) { + const int nblocks_total = ntiles_total * parallel_blocks_test; + const int nwaves = (nblocks_total + blocks_per_wave - 1) / blocks_per_wave; + const int efficiency_percent = 100 * nblocks_total / (nwaves*blocks_per_wave); + + // Stop trying configurations with more waves if we already have good efficiency to avoid excessive overhead. + if (efficiency_percent_best >= 90 && nwaves > nwaves_best) { + break; + } + + if (efficiency_percent > efficiency_percent_best) { + nwaves_best = nwaves; + efficiency_percent_best = efficiency_percent; + parallel_blocks = parallel_blocks_test; + } + } + + blocks_num.x = ntiles_x; + blocks_num.y = parallel_blocks; + blocks_num.z = Q->ne[2]*Q->ne[3]; if (parallel_blocks > 1) { dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); @@ -803,7 +836,7 @@ void launch_fattn( K_data, V_data, mask ? ((const char *) mask->data) : nullptr, - (parallel_blocks) > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, + !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], K->ne[0], K->ne[1], K->ne[2], K->ne[3], @@ -815,7 +848,7 @@ void launch_fattn( ); CUDA_CHECK(cudaGetLastError()); - if constexpr (parallel_blocks == 0) { + if (stream_k) { if (ntiles_total % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles. const dim3 block_dim_combine(D, 1, 1); const dim3 blocks_num_combine = {blocks_num.x, ncols1, ncols2}; @@ -824,13 +857,14 @@ void launch_fattn( <<>> ((float *) KQV->data, dst_tmp_meta.ptr, Q->ne[1], Q->ne[2], K->ne[1]); } - } else if constexpr (parallel_blocks > 1) { + } else if (parallel_blocks > 1) { const dim3 block_dim_combine(D, 1, 1); - const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); + const dim3 blocks_num_combine(Q->ne[1], 1, blocks_num.z); + const size_t nbytes_shared_combine = parallel_blocks*sizeof(float2); - flash_attn_combine_results - <<>> - (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); + flash_attn_combine_results + <<>> + (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data, parallel_blocks); } CUDA_CHECK(cudaGetLastError()); } diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 718ee5402d..024032f622 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -970,7 +970,8 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml fattn_kernel = flash_attn_ext_f16; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, true, true); + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, FATTN_KQ_STRIDE, true, true, true); } diff --git a/ggml/src/ggml-cuda/fattn-tile-f16.cu b/ggml/src/ggml-cuda/fattn-tile-f16.cu index ef3569fab2..77455d8e4f 100644 --- a/ggml/src/ggml-cuda/fattn-tile-f16.cu +++ b/ggml/src/ggml-cuda/fattn-tile-f16.cu @@ -4,7 +4,7 @@ #define FATTN_KQ_STRIDE_TILE_F16 64 -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(nwarps*WARP_SIZE, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) @@ -58,18 +58,17 @@ static __global__ void flash_attn_tile_ext_f16( //In this kernel Q, K, V are matrices while i, j, k are matrix indices. - const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on. const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0); - const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio)); - const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.z + nb01*ic0); + const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.z / gqa_ratio)); + const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.z / gqa_ratio)); // K and V have same shape const half * maskh = (const half *) mask + ne11*ic0; const int stride_KV2 = nb11 / sizeof(half2); - const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slopef = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); const half slopeh = __float2half(slopef); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); @@ -105,8 +104,7 @@ static __global__ void flash_attn_tile_ext_f16( __syncthreads(); - const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F16; - for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F16) { + for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE_TILE_F16; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE_TILE_F16) { // Calculate KQ tile and keep track of new maximum KQ values: half kqmax_new[ncols/nwarps]; @@ -271,16 +269,16 @@ static __global__ void flash_attn_tile_ext_f16( const int i0 = i00 + 2*threadIdx.x; half2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)]; - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val /= __half2half2(kqsum_j); } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; - dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = __low2float(dst_val); - dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = __high2float(dst_val); + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; + dst[j_dst*D*gridDim.z + D*blockIdx.z + i0 + 0] = __low2float(dst_val); + dst[j_dst*D*gridDim.z + D*blockIdx.z + i0 + 1] = __high2float(dst_val); } - if (parallel_blocks != 1 && threadIdx.x == 0) { - dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); + if (gridDim.y != 1 && threadIdx.x == 0) { + dst_meta[((ic0 + j_VKQ)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); } } #else @@ -288,7 +286,7 @@ static __global__ void flash_attn_tile_ext_f16( #endif // defined(FLASH_ATTN_AVAILABLE) && defined(FP16_AVAILABLE) } -template +template void launch_fattn_tile_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * Q = dst->src[0]; switch (Q->ne[0]) { @@ -296,15 +294,17 @@ void launch_fattn_tile_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * constexpr int D = 64; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; - fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, FATTN_KQ_STRIDE_TILE_F16, true, true, false); } break; case 128: { constexpr int D = 128; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; - fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, FATTN_KQ_STRIDE_TILE_F16, true, true, false); } break; default: { GGML_ABORT("FlashAttention without tensor cores only supports head sizes 64 and 128."); @@ -324,37 +324,22 @@ void ggml_cuda_flash_attn_ext_tile_f16(ggml_backend_cuda_context & ctx, ggml_ten if (Q->ne[1] <= 16) { constexpr int cols_per_block = 16; - constexpr int parallel_blocks = 4; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - launch_fattn_tile_f16_64_128(ctx, dst); + launch_fattn_tile_f16_64_128(ctx, dst); } else { constexpr bool use_logit_softcap = true; - launch_fattn_tile_f16_64_128(ctx, dst); - } - return; - } - - if (Q->ne[1] <= 32) { - constexpr int cols_per_block = 32; - constexpr int parallel_blocks = 4; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - launch_fattn_tile_f16_64_128(ctx, dst); - } else { - constexpr bool use_logit_softcap = true; - launch_fattn_tile_f16_64_128(ctx, dst); + launch_fattn_tile_f16_64_128(ctx, dst); } return; } constexpr int cols_per_block = 32; - constexpr int parallel_blocks = 1; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - launch_fattn_tile_f16_64_128(ctx, dst); + launch_fattn_tile_f16_64_128(ctx, dst); } else { constexpr bool use_logit_softcap = true; - launch_fattn_tile_f16_64_128(ctx, dst); + launch_fattn_tile_f16_64_128(ctx, dst); } } diff --git a/ggml/src/ggml-cuda/fattn-tile-f32.cu b/ggml/src/ggml-cuda/fattn-tile-f32.cu index 04b69c83be..85fea4404d 100644 --- a/ggml/src/ggml-cuda/fattn-tile-f32.cu +++ b/ggml/src/ggml-cuda/fattn-tile-f32.cu @@ -4,7 +4,7 @@ #define FATTN_KQ_STRIDE_TILE_F32 32 -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(nwarps*WARP_SIZE, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) @@ -58,18 +58,17 @@ static __global__ void flash_attn_tile_ext_f32( // In this kernel Q, K, V are matrices while i, j, k are matrix indices. - const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on. const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0); - const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio)); - const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.z + nb01*ic0); + const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.z / gqa_ratio)); + const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.z / gqa_ratio)); // K and V have same shape const half * maskh = (const half *) mask + ne11*ic0; const int stride_KV2 = nb11 / sizeof(half2); - const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slope = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); @@ -103,8 +102,7 @@ static __global__ void flash_attn_tile_ext_f32( __syncthreads(); - const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F32; - for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F32) { + for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE_TILE_F32; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE_TILE_F32) { // Calculate KQ tile and keep track of new maximum KQ values: float kqmax_new[ncols/nwarps]; @@ -269,17 +267,17 @@ static __global__ void flash_attn_tile_ext_f32( const int i0 = i00 + 2*threadIdx.x; float2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)]; - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val.x /= kqsum_j; dst_val.y /= kqsum_j; } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; - dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = dst_val.x; - dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = dst_val.y; + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; + dst[j_dst*D*gridDim.z + D*blockIdx.z + i0 + 0] = dst_val.x; + dst[j_dst*D*gridDim.z + D*blockIdx.z + i0 + 1] = dst_val.y; } - if (parallel_blocks != 1 && threadIdx.x == 0) { - dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); + if (gridDim.y != 1 && threadIdx.x == 0) { + dst_meta[((ic0 + j_VKQ)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); } } #else @@ -287,7 +285,7 @@ static __global__ void flash_attn_tile_ext_f32( #endif // FLASH_ATTN_AVAILABLE } -template +template void launch_fattn_tile_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * Q = dst->src[0]; switch (Q->ne[0]) { @@ -295,15 +293,17 @@ void launch_fattn_tile_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * constexpr int D = 64; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; - fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, FATTN_KQ_STRIDE_TILE_F32, true, true, false); } break; case 128: { constexpr int D = 128; constexpr int nwarps = 8; constexpr size_t nbytes_shared = 0; - fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, true, true); + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, FATTN_KQ_STRIDE_TILE_F32, true, true, false); } break; default: { GGML_ABORT("FlashAttention without tensor cores only supports head sizes 64 and 128."); @@ -320,37 +320,22 @@ void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_ten if (Q->ne[1] <= 16) { constexpr int cols_per_block = 16; - constexpr int parallel_blocks = 4; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - launch_fattn_tile_f32_64_128(ctx, dst); + launch_fattn_tile_f32_64_128(ctx, dst); } else { constexpr bool use_logit_softcap = true; - launch_fattn_tile_f32_64_128(ctx, dst); - } - return; - } - - if (Q->ne[1] <= 32) { - constexpr int cols_per_block = 32; - constexpr int parallel_blocks = 4; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - launch_fattn_tile_f32_64_128(ctx, dst); - } else { - constexpr bool use_logit_softcap = true; - launch_fattn_tile_f32_64_128(ctx, dst); + launch_fattn_tile_f32_64_128(ctx, dst); } return; } constexpr int cols_per_block = 32; - constexpr int parallel_blocks = 1; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - launch_fattn_tile_f32_64_128(ctx, dst); + launch_fattn_tile_f32_64_128(ctx, dst); } else { constexpr bool use_logit_softcap = true; - launch_fattn_tile_f32_64_128(ctx, dst); + launch_fattn_tile_f32_64_128(ctx, dst); } } diff --git a/ggml/src/ggml-cuda/fattn-vec-f16.cuh b/ggml/src/ggml-cuda/fattn-vec-f16.cuh index b7686c1ec3..32c52ebe33 100644 --- a/ggml/src/ggml-cuda/fattn-vec-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-vec-f16.cuh @@ -1,7 +1,7 @@ #include "common.cuh" #include "fattn-common.cuh" -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(D, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) @@ -55,17 +55,16 @@ static __global__ void flash_attn_vec_ext_f16( constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16; constexpr dequantize_1_f16_t dequantize_1_v = get_dequantize_1_f16(type_V); - const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on. const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - Q += nb02* blockIdx.y + nb01*ic0; - K += nb12*(blockIdx.y / gqa_ratio); - V += nb22*(blockIdx.y / gqa_ratio); + Q += nb02* blockIdx.z + nb01*ic0; + K += nb12*(blockIdx.z / gqa_ratio); + V += nb22*(blockIdx.z / gqa_ratio); const half * maskh = (const half *) mask + ne11*ic0; - const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slopef = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); const half slopeh = __float2half(slopef); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); @@ -172,8 +171,7 @@ static __global__ void flash_attn_vec_ext_f16( half2 VKQ[ncols] = {{0.0f, 0.0f}}; - const int k_start = parallel_blocks == 1 ? 0 : ip*D; - for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) { + for (int k_VKQ_0 = blockIdx.y*D; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*D) { // Calculate KQ tile and keep track of new maximum KQ values: // For unknown reasons using a half array of size 1 for kqmax_new causes a performance regression, @@ -283,29 +281,29 @@ static __global__ void flash_attn_vec_ext_f16( kqsum[j_VKQ] = warp_reduce_sum((float)kqsum[j_VKQ]); half dst_val = (__low2half(VKQ[j_VKQ]) + __high2half(VKQ[j_VKQ])); - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val /= kqsum[j_VKQ]; } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; - dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; + dst[j_dst*D*gridDim.z + D*blockIdx.z + tid] = dst_val; } - if (parallel_blocks != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { - dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); + if (gridDim.y != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { + dst_meta[((ic0 + tid)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = make_float2(kqmax[tid], kqsum[tid]); } #else NO_DEVICE_CODE; #endif // defined(FLASH_ATTN_AVAILABLE) && defined(FP16_AVAILABLE) } -template +template void ggml_cuda_flash_attn_ext_vec_f16_case_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { constexpr int nwarps = D/WARP_SIZE; - fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16; constexpr bool need_f16_K = D != 128; constexpr bool need_f16_V = D != 128 && D != 64; constexpr size_t nbytes_shared = 0; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, need_f16_K, need_f16_V); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false); } template @@ -325,65 +323,48 @@ void ggml_cuda_flash_attn_ext_vec_f16_case(ggml_backend_cuda_context & ctx, ggml memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); if (Q->ne[1] == 1) { - constexpr int cols_per_block = 1; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 1; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } return; } if (Q->ne[1] == 2) { - constexpr int cols_per_block = 2; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 2; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } return; } if (Q->ne[1] <= 4) { - constexpr int cols_per_block = 4; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 4; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } return; } - if (Q->ne[1] <= 8) { - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 4; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); - } else { - constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); - } - return; - } - - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 1; + constexpr int cols_per_block = 8; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f16_case_impl(ctx, dst); } } diff --git a/ggml/src/ggml-cuda/fattn-vec-f32.cuh b/ggml/src/ggml-cuda/fattn-vec-f32.cuh index c1d2dd8d19..336c136d19 100644 --- a/ggml/src/ggml-cuda/fattn-vec-f32.cuh +++ b/ggml/src/ggml-cuda/fattn-vec-f32.cuh @@ -1,7 +1,7 @@ #include "common.cuh" #include "fattn-common.cuh" -template // D == head size +template // D == head size #if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(D, 1) #endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) @@ -55,16 +55,15 @@ static __global__ void flash_attn_vec_ext_f32( constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16; constexpr dequantize_1_f32_t dequantize_1_v = get_dequantize_1_f32(type_V); - const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on. const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - Q += nb02* blockIdx.y + nb01*ic0; - K += nb12*(blockIdx.y / gqa_ratio); - V += nb22*(blockIdx.y / gqa_ratio); // K and V have same shape + Q += nb02* blockIdx.z + nb01*ic0; + K += nb12*(blockIdx.z / gqa_ratio); + V += nb22*(blockIdx.z / gqa_ratio); // K and V have same shape const half * maskh = (const half *) mask + ne11*ic0; - const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slope = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); constexpr int nwarps = D / WARP_SIZE; @@ -167,8 +166,7 @@ static __global__ void flash_attn_vec_ext_f32( float VKQ[ncols] = {0.0f}; - const int k_start = parallel_blocks == 1 ? 0 : ip*D; - for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) { + for (int k_VKQ_0 = blockIdx.y*D; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*D) { // Calculate KQ tile and keep track of new maximum KQ values: float kqmax_new_arr[ncols]; @@ -268,29 +266,29 @@ static __global__ void flash_attn_vec_ext_f32( kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]); float dst_val = VKQ[j_VKQ]; - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val /= kqsum[j_VKQ]; } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; - dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; + dst[j_dst*D*gridDim.z + D*blockIdx.z + tid] = dst_val; } - if (parallel_blocks != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { - dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); + if (gridDim.y != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { + dst_meta[((ic0 + tid)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = make_float2(kqmax[tid], kqsum[tid]); } #else NO_DEVICE_CODE; #endif // FLASH_ATTN_AVAILABLE } -template +template void ggml_cuda_flash_attn_ext_vec_f32_case_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { constexpr int nwarps = D/WARP_SIZE; - fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f32; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f32; constexpr bool need_f16_K = D != 128; constexpr bool need_f16_V = D != 128 && D != 64; constexpr size_t nbytes_shared = 0; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, need_f16_K, need_f16_V); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false); } template @@ -307,65 +305,48 @@ void ggml_cuda_flash_attn_ext_vec_f32_case(ggml_backend_cuda_context & ctx, ggml memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); if (Q->ne[1] == 1) { - constexpr int cols_per_block = 1; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 1; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } return; } if (Q->ne[1] == 2) { - constexpr int cols_per_block = 2; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 2; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } return; } if (Q->ne[1] <= 4) { - constexpr int cols_per_block = 4; - constexpr int parallel_blocks = 4; + constexpr int cols_per_block = 4; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } return; } - if (Q->ne[1] <= 8) { - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 4; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); - } else { - constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); - } - return; - } - - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 1; + constexpr int cols_per_block = 8; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } else { constexpr bool use_logit_softcap = true; - ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); + ggml_cuda_flash_attn_ext_vec_f32_case_impl(ctx, dst); } } diff --git a/ggml/src/ggml-cuda/fattn-wmma-f16.cu b/ggml/src/ggml-cuda/fattn-wmma-f16.cu index dab1d5cbca..5c214ea310 100644 --- a/ggml/src/ggml-cuda/fattn-wmma-f16.cu +++ b/ggml/src/ggml-cuda/fattn-wmma-f16.cu @@ -18,7 +18,7 @@ namespace wmma = rocwmma; #endif // FP16_MMA_AVAILABLE // D == head size, VKQ_stride == num VKQ rows calculated in parallel: -template +template __launch_bounds__(nwarps*ggml_cuda_get_physical_warp_size(), 1) static __global__ void flash_attn_ext_f16( const char * __restrict__ Q, @@ -67,8 +67,7 @@ static __global__ void flash_attn_ext_f16( constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - const int ic0 = ncols*(blockIdx.x / parallel_blocks); // Index of the first Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + const int ic0 = ncols*blockIdx.x; // Index of the first Q/QKV column to work on. static_assert(D <= FATTN_KQ_STRIDE, "D must be <= FATTN_KQ_STRIDE."); static_assert(ncols == 8 || ncols % 16 == 0, "ncols must be 8 or a multiple of 16."); @@ -91,16 +90,16 @@ static __global__ void flash_attn_ext_f16( constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half); const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const float * Q_f = (const float *) (Q + nb02* blockIdx.y + nb01*ic0); - const half * K_h = (const half *) (K + nb12*(blockIdx.y / gqa_ratio)); - const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const float * Q_f = (const float *) (Q + nb02* blockIdx.z + nb01*ic0); + const half * K_h = (const half *) (K + nb12*(blockIdx.z / gqa_ratio)); + const half * V_h = (const half *) (V + nb12*(blockIdx.z / gqa_ratio)); // K and V have same shape const half * maskh = (const half *) mask + (nb31/sizeof(half))* ic0; const half2 * mask2 = (const half2 *) mask + (nb31/sizeof(half))*(ic0/2); const int stride_Q = nb01 / sizeof(float); const int stride_KV = nb11 / sizeof(half); - const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const float slopef = get_alibi_slope(max_bias, blockIdx.z, n_head_log2, m0, m1); const half slopeh = __float2half(slopef); const half2 slope2 = make_half2(slopef, slopef); @@ -176,7 +175,7 @@ static __global__ void flash_attn_ext_f16( __syncthreads(); // Iterate over ne11 == previous tokens: - for (int k_VKQ_0 = ip*FATTN_KQ_STRIDE; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE) { + for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE; k_VKQ_0 < ne11; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE) { // Calculate tile of KQ: #pragma unroll for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) { @@ -395,7 +394,7 @@ static __global__ void flash_attn_ext_f16( if (ic0 + j_VKQ >= ne01) { return; } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; + const int j_dst = (ic0 + j_VKQ)*gridDim.y + blockIdx.y; float KQ_rowsum_j; if (std::is_same::value) { @@ -411,13 +410,13 @@ static __global__ void flash_attn_ext_f16( break; } float dst_val = VKQ[j_VKQ*D_padded + i]; - if (parallel_blocks == 1) { + if (gridDim.y == 1) { dst_val /= KQ_rowsum_j; } - dst[j_dst*gridDim.y*D + blockIdx.y*D + i] = dst_val; + dst[j_dst*gridDim.z*D + blockIdx.z*D + i] = dst_val; } - if (parallel_blocks == 1 || threadIdx.x != 0) { + if (gridDim.y == 1 || threadIdx.x != 0) { continue; } @@ -428,7 +427,7 @@ static __global__ void flash_attn_ext_f16( dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]); } dst_meta_val.y = KQ_rowsum_j; - dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = dst_meta_val; + dst_meta[((ic0 + j_VKQ)*gridDim.z + blockIdx.z) * gridDim.y + blockIdx.y] = dst_meta_val; } #else NO_DEVICE_CODE; @@ -462,60 +461,26 @@ static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed."); template void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * KQV = dst; - const ggml_tensor * Q = dst->src[0]; constexpr int nwarps = 4; constexpr int frag_m = cols_per_block == 8 && D % 32 == 0 ? 32 : 16; - const int blocks_num_pb1 = ((Q->ne[1] + cols_per_block - 1) / cols_per_block)*Q->ne[2]*Q->ne[3]; - const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; float logit_softcap; memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); - if (4*blocks_num_pb1 < 2*nsm) { - constexpr int parallel_blocks = 4; - fattn_kernel_t fattn_kernel; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; - } else { - constexpr bool use_logit_softcap = true; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; - } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true, warp_size); - return; - } - if (2*blocks_num_pb1 < 2*nsm) { - constexpr int parallel_blocks = 2; - fattn_kernel_t fattn_kernel; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; - } else { - constexpr bool use_logit_softcap = true; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; - } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true, warp_size); - return; - } - constexpr int parallel_blocks = 1; fattn_kernel_t fattn_kernel; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; + D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>; } else { constexpr bool use_logit_softcap = true; fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t, use_logit_softcap>; + D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>; } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, true, true, warp_size); + launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, FATTN_KQ_STRIDE, true, true, false, warp_size); } void ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 2e72fc8fd3..973541893e 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -281,13 +281,13 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst if (!fp16_mma_available(cc)) { if (prec == GGML_PREC_DEFAULT) { - if (Q->ne[1] <= 8) { + if (Q->ne[1] <= 8 || Q->ne[0] == 256) { ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); } else { ggml_cuda_flash_attn_ext_tile_f16(ctx, dst); } } else { - if (Q->ne[1] <= 8) { + if (Q->ne[1] <= 8 || Q->ne[0] == 256) { ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); } else { ggml_cuda_flash_attn_ext_tile_f32(ctx, dst); @@ -296,17 +296,17 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst return; } - const int gqa_ratio = Q->ne[2] / K->ne[2]; - const bool mma_fast_for_bs1 = fp16_mma_available(cc) && gqa_ratio % 2 == 0 && - K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && mask; - if (Q->ne[1] == 1 && Q->ne[0] % (2*warp_size) == 0 && !mma_fast_for_bs1) { + const bool gqa_opt_applies = ((Q->ne[2] / K->ne[2]) % 2 == 0) && mask; // The mma-based kernels have GQA-specific optimizations + const bool mma_needs_data_conversion = K->type != GGML_TYPE_F16 || V->type != GGML_TYPE_F16; + const bool mma_faster_for_bs1 = new_mma_available(cc) && gqa_opt_applies && cc < GGML_CUDA_CC_ADA_LOVELACE && !mma_needs_data_conversion; + const bool can_use_vector_kernel = (Q->ne[0] % (2*warp_size) == 0) && (prec == GGML_PREC_DEFAULT || Q->ne[0] <= 128); + if (Q->ne[1] == 1 && can_use_vector_kernel && !mma_faster_for_bs1) { if (prec == GGML_PREC_DEFAULT) { ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); - return; - } else if(Q->ne[0] <= 128) { + } else { ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); - return; } + return; } // The MMA implementation needs Turing or newer, use the old WMMA code for Volta: diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 5cb56df9a8..b783310ef7 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3230,6 +3230,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g #ifndef FLASH_ATTN_AVAILABLE return false; #endif // FLASH_ATTN_AVAILABLE + if (op->src[0]->ne[3] != 1) { + return false; + } if (op->src[1]->type == GGML_TYPE_BF16 || op->src[2]->type == GGML_TYPE_BF16) { return false; } diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index aace21e3a8..a4c717a321 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -129,6 +129,7 @@ #define cudaGraph_t hipGraph_t #define cudaStream_t hipStream_t #define cudaSuccess hipSuccess +#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor #define __trap() do { abort(); __builtin_unreachable(); } while(0) #define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS #define CUBLAS_STATUS_NOT_INITIALIZED HIPBLAS_STATUS_NOT_INITIALIZED diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 997f671431..f2d55796e7 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -134,5 +134,6 @@ #define cudaStreamCaptureModeRelaxed musaStreamCaptureModeRelaxed #define cudaStreamBeginCapture musaStreamBeginCapture #define cudaStreamEndCapture musaStreamEndCapture +#define cudaOccupancyMaxActiveBlocksPerMultiprocessor musaOccupancyMaxActiveBlocksPerMultiprocessor typedef mt_bfloat16 nv_bfloat16; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index adb749bd5e..d48cd21723 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -259,6 +259,10 @@ static std::string var_to_str(ggml_type type) { return ggml_type_name(type); } +static std::string var_to_str(ggml_prec prec) { + return prec == GGML_PREC_F32 ? "f32" : "def"; +} + static std::string var_to_str(ggml_op_pool pool) { switch (pool) { case GGML_OP_POOL_AVG: return "avg"; @@ -3206,11 +3210,12 @@ struct test_flash_attn_ext : public test_case { const float max_bias; // ALiBi const float logit_softcap; // Gemma 2 + const ggml_prec prec; const ggml_type type_KV; std::array permute; std::string vars() override { - return VARS_TO_STR10(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, type_KV, permute); + return VARS_TO_STR11(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, prec, type_KV, permute); } double max_nmse_err() override { @@ -3225,9 +3230,9 @@ struct test_flash_attn_ext : public test_case { } test_flash_attn_ext(int64_t hs = 128, int64_t nh = 32, int64_t nr = 1, int64_t kv = 96, int64_t nb = 8, - bool mask = true, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_type type_KV = GGML_TYPE_F16, - std::array permute = {0, 1, 2, 3}) - : hs(hs), nh(nh), nr(nr), kv(kv), nb(nb), mask(mask), max_bias(max_bias), logit_softcap(logit_softcap), type_KV(type_KV), permute(permute) {} + bool mask = true, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32, + ggml_type type_KV = GGML_TYPE_F16, std::array permute = {0, 1, 2, 3}) + : hs(hs), nh(nh), nr(nr), kv(kv), nb(nb), mask(mask), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec), type_KV(type_KV), permute(permute) {} ggml_tensor * build_graph(ggml_context * ctx) override { const int64_t hs_padded = GGML_PAD(hs, ggml_blck_size(type_KV)); @@ -3261,6 +3266,7 @@ struct test_flash_attn_ext : public test_case { } ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hs), max_bias, logit_softcap); + ggml_flash_attn_ext_set_prec(out, prec); ggml_set_name(out, "out"); return out; @@ -4376,11 +4382,16 @@ static std::vector> make_test_cases_eval() { for (int kv : { 512, 1024, }) { if (nr != 1 && kv != 512) continue; for (int nb : { 1, 3, 32, 35, }) { - for (ggml_type type_KV : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { - test_cases.emplace_back(new test_flash_attn_ext(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, type_KV)); - // run fewer test cases permuted - if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) { - test_cases.emplace_back(new test_flash_attn_ext(hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, type_KV, {0, 2, 1, 3})); + for (ggml_prec prec : {GGML_PREC_F32, GGML_PREC_DEFAULT}) { + if (hs != 128 && prec == GGML_PREC_DEFAULT) continue; + for (ggml_type type_KV : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { + test_cases.emplace_back(new test_flash_attn_ext( + hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, prec, type_KV)); + // run fewer test cases permuted + if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) { + test_cases.emplace_back(new test_flash_attn_ext( + hs, nh, nr, kv, nb, mask, max_bias, logit_softcap, prec, type_KV, {0, 2, 1, 3})); + } } } } From 568013d0cd3d5add37c376b3d5e959809b711fc7 Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Wed, 19 Mar 2025 21:01:57 +0100 Subject: [PATCH 174/188] context : clear sets containing encoder output sequence ids before storing new values (#12470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stanisław Szymczyk --- src/llama-context.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 664703a896..5bec63e2e7 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1153,6 +1153,7 @@ int llama_context::encode(llama_batch & inp_batch) { // remember the sequence ids used during the encoding - needed for cross attention later cross.seq_ids_enc.resize(n_tokens); for (int32_t i = 0; i < n_tokens; i++) { + cross.seq_ids_enc[i].clear(); for (int s = 0; s < ubatch.n_seq_id[i]; s++) { llama_seq_id seq_id = ubatch.seq_id[i][s]; cross.seq_ids_enc[i].insert(seq_id); From 732b5fbf5e7f9cf069942f0c5850ee959ef321ba Mon Sep 17 00:00:00 2001 From: Bartowski Date: Thu, 20 Mar 2025 02:36:37 -0400 Subject: [PATCH 175/188] convert : avoid calls to tokenizer.added_tokens_decoder (#12473) tokenizer.added_tokens_decoder returns a fresh dict every time relatively slowly (~0.04s on average) which results in massive slowdowns when we have a huge number of added tokens --- convert_hf_to_gguf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 7574218e24..d21edce16b 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -529,6 +529,8 @@ class Model: reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} added_vocab = tokenizer.get_added_vocab() + added_tokens_decoder = tokenizer.added_tokens_decoder + for i in range(vocab_size): if i not in reverse_vocab: tokens.append(f"[PAD{i}]") @@ -538,13 +540,13 @@ class Model: if token in added_vocab: # The tokenizer in llama.cpp assumes the CONTROL and USER_DEFINED tokens are pre-normalized. # To avoid unexpected issues - we make sure to normalize non-normalized tokens - if not tokenizer.added_tokens_decoder[i].normalized: + if not added_tokens_decoder[i].normalized: previous_token = token token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) if previous_token != token: logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer") - if tokenizer.added_tokens_decoder[i].special or self.does_token_look_special(token): + if added_tokens_decoder[i].special or self.does_token_look_special(token): toktypes.append(gguf.TokenType.CONTROL) else: # NOTE: this was added for Gemma. From 3d82dbcbce2c677fc35fbf99574ccd107d95a1f8 Mon Sep 17 00:00:00 2001 From: Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com> Date: Thu, 20 Mar 2025 17:05:34 +0530 Subject: [PATCH 176/188] ggml : block interleaving support for Q4_K quantization for x86 AVX2 architecture (#12332) * Add block interleaving support for Q4_K quantization * Remove whitespaces and fix CI/CD issues * Update pointer of bsums from int16_t to const int16_t * Add vector version of quantize_q8_K_4x8 function * Update code formatting based on review comments --- ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp | 1505 +++++++++++++++++++++++- 1 file changed, 1493 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp b/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp index c24fd56e20..e852c8253b 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp @@ -45,6 +45,24 @@ using block_q4_0x8 = block<4, 8>; using block_q8_0x4 = block<8, 4>; using block_q8_0x8 = block<8, 8>; + +struct block_q4_Kx8 { + ggml_half d[8]; // super-block scale for quantized scales + ggml_half dmin[8]; // super-block scale for quantized mins + uint8_t scales[96]; // scales and mins, quantized with 6 bits + uint8_t qs[1024]; // 4--bit quants +}; + +static_assert(sizeof(block_q4_Kx8) == sizeof(ggml_half) * 16 + K_SCALE_SIZE * 8 + QK_K * 4, "wrong q4_K block size/padding"); + +struct block_q8_Kx4 { + float d[4]; // delta + int8_t qs[QK_K * 4]; // quants + int16_t bsums[QK_K / 4]; // sum of quants in groups of 16 +}; + +static_assert(sizeof(block_q8_Kx4) == sizeof(float) * 4 + QK_K * 4 + (QK_K / 4) * sizeof(int16_t), "wrong q8_K block size/padding"); + struct block_iq4_nlx4 { ggml_half d[4]; // deltas for 4 iq4_nl blocks uint8_t qs[QK4_NL * 2]; // nibbles / quants for 4 iq4_nl blocks @@ -60,6 +78,13 @@ static_assert(sizeof(block_iq4_nlx4) == 4 * sizeof(ggml_half) + QK4_NL * 2, "wro #define UNUSED GGML_UNUSED +static inline int nearest_int(float fval) { + assert(fabsf(fval) <= 4194303.f); + float val = fval + 12582912.f; + int i; memcpy(&i, &val, sizeof(int)); + return (i & 0x007fffff) - 0x00400000; +} + // Functions to create the interleaved data layout formats // interleave 4 block_q4_0s in blocks of blck_size_interleave @@ -534,6 +559,270 @@ static void quantize_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRIC #endif } +static void quantize_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(QK_K == 256); + assert(k % QK_K == 0); + const int nb = k / QK_K; + + block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy; + +#if defined(__AVX2__) + float iscale[4]; + __m256 srcv[4][32]; + __m256 iscale_vec[4]; + + for (int i = 0; i < nb; i++) { + for (int row_iter = 0; row_iter < 4; row_iter++) { + // Load elements into 4 AVX vectors + __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 ); + __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 8 ); + __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 16 ); + __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 24 ); + + // Compute max(abs(e)) for the block + const __m256 signBit = _mm256_set1_ps( -0.0f ); + __m256 abs0 = _mm256_andnot_ps( signBit, v0 ); + __m256 abs1 = _mm256_andnot_ps( signBit, v1 ); + __m256 abs2 = _mm256_andnot_ps( signBit, v2 ); + __m256 abs3 = _mm256_andnot_ps( signBit, v3 ); + + __m256 maxAbs = _mm256_max_ps( abs0, abs1 ); + maxAbs = _mm256_max_ps( maxAbs, abs2 ); + maxAbs = _mm256_max_ps( maxAbs, abs3 ); + + __m256 mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ ); + __m256 mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ ); + __m256 mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ ); + __m256 mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ ); + + __m256 maskAbs = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3)); + + srcv[row_iter][0] = v0; + srcv[row_iter][1] = v1; + srcv[row_iter][2] = v2; + srcv[row_iter][3] = v3; + + for (int sb = 1; sb < 8; sb++) { + // Temporarily stores absolute quant values + __m256 tempAbs = maxAbs; + + // Load elements into 4 AVX vectors + __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32); + __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 8 ); + __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 16 ); + __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 24 ); + + // Compute max(abs(e)) for the block + __m256 abs0 = _mm256_andnot_ps( signBit, v0 ); + __m256 abs1 = _mm256_andnot_ps( signBit, v1 ); + __m256 abs2 = _mm256_andnot_ps( signBit, v2 ); + __m256 abs3 = _mm256_andnot_ps( signBit, v3 ); + + maxAbs = _mm256_max_ps( maxAbs, abs0 ); + maxAbs = _mm256_max_ps( maxAbs, abs1 ); + maxAbs = _mm256_max_ps( maxAbs, abs2 ); + maxAbs = _mm256_max_ps( maxAbs, abs3 ); + + __m256 mask_prev = _mm256_cmp_ps( tempAbs, maxAbs, _CMP_EQ_OQ ); + maskAbs = _mm256_and_ps( maskAbs, mask_prev ); + + mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ ); + mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ ); + mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ ); + mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ ); + + __m256 mask_curr = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3)); + maskAbs = _mm256_or_ps(maskAbs, mask_curr); + + srcv[row_iter][sb * 4] = v0; + srcv[row_iter][sb * 4 + 1] = v1; + srcv[row_iter][sb * 4 + 2] = v2; + srcv[row_iter][sb * 4 + 3] = v3; + } + + __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) ); + max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); + max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); + const float maxScalar = _mm_cvtss_f32( max4 ); + + __m256 maxScalarVec = _mm256_set1_ps(maxScalar); + + __m256 mask_next = _mm256_cmp_ps( maxScalarVec, maxAbs, _CMP_EQ_OQ ); + __m256 finalMask = _mm256_and_ps(maskAbs, mask_next); + + const int mask = _mm256_movemask_ps(finalMask); + iscale[row_iter] = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; + + if(mask) { + iscale[row_iter] = ( maxScalar != 0.0f ) ? -127.f / maxScalar: 0.0f; + } + + y[i].d[row_iter] = maxScalar ? 1/iscale[row_iter] : 0; + iscale_vec[row_iter] = _mm256_set1_ps(iscale[row_iter]); + } + + __m256i quants_interleaved[32]; + for (int j = 0; j < 32; j++) { + // Apply the multiplier + __m256 v0 = _mm256_mul_ps(srcv[0][j], iscale_vec[0]); + __m256 v1 = _mm256_mul_ps(srcv[1][j], iscale_vec[1]); + __m256 v2 = _mm256_mul_ps(srcv[2][j], iscale_vec[2]); + __m256 v3 = _mm256_mul_ps(srcv[3][j], iscale_vec[3]); + + // Round to nearest integer + v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST ); + v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST ); + v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST ); + v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST ); + + // Convert floats to integers + __m256i i0 = _mm256_cvtps_epi32( v0 ); + __m256i i1 = _mm256_cvtps_epi32( v1 ); + __m256i i2 = _mm256_cvtps_epi32( v2 ); + __m256i i3 = _mm256_cvtps_epi32( v3 ); + + // Convert int32 to int16 + i0 = _mm256_packs_epi32( i0, i1 ); + i2 = _mm256_packs_epi32( i2, i3 ); + // Convert int16 to int8 + i0 = _mm256_packs_epi16( i0, i2 ); + + // Permute and store the quantized weights in the required order after the pack instruction + const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 ); + i0 = _mm256_permutevar8x32_epi32( i0, perm ); + + _mm256_storeu_si256((__m256i *)(y[i].qs + 32 * j), i0); + quants_interleaved[j] = i0; + } + + // Masks to shuffle the quants of corresonding sub blocks for rearraning quants for vectorized bsums computation + __m256i shuffle_mask_sb2 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 0, 1, 4, 5, 6, 7, 8, 9, 8, 9, 12, 13, 14, 15)); + shuffle_mask_sb2 = _mm256_permute2f128_si256(shuffle_mask_sb2, shuffle_mask_sb2, 0); + __m256i shuffle_mask_sb3 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 0, 1, 6, 7, 8, 9, 10, 11, 8, 9, 14, 15)); + shuffle_mask_sb3 = _mm256_permute2f128_si256(shuffle_mask_sb3, shuffle_mask_sb3, 0); + __m256i shuffle_mask_sb4 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 4, 5, 0, 1, 8, 9, 10, 11, 12, 13, 8, 9)); + shuffle_mask_sb4 = _mm256_permute2f128_si256(shuffle_mask_sb4, shuffle_mask_sb4, 0); + + for (int k = 0; k < 4; k++) { + // Quants from four different sub blocks are taken + __m256i q0 = quants_interleaved[k * 8 + 0]; + __m256i q1 = quants_interleaved[k * 8 + 1]; + __m256i q2 = quants_interleaved[k * 8 + 2]; + __m256i q3 = quants_interleaved[k * 8 + 3]; + __m256i q4 = quants_interleaved[k * 8 + 4]; + __m256i q5 = quants_interleaved[k * 8 + 5]; + __m256i q6 = quants_interleaved[k * 8 + 6]; + __m256i q7 = quants_interleaved[k * 8 + 7]; + + + // The below code block has the first half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time + __m256i sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2); + __m256i sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34); + __m256i sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3); + sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68); + __m256i sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4); + sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136); + + __m256i one = _mm256_set1_epi8(1); + __m256i bsums_r1 = _mm256_maddubs_epi16(one, sb_h1_interleaved); + + for (int l = 0; l < 3; l++) { + // Quants value shifted to process next two values from each sub block + q0 = _mm256_srli_epi64(q0, 16); + q2 = _mm256_srli_epi64(q2, 16); + q4 = _mm256_srli_epi64(q4, 16); + q6 = _mm256_srli_epi64(q6, 16); + + sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2); + sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34); + sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3); + sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68); + sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4); + sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136); + + bsums_r1 = _mm256_add_epi16(bsums_r1, _mm256_maddubs_epi16(one, sb_h1_interleaved)); + } + + // The below code block has the second half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time + __m256i sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2); + __m256i sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34); + __m256i sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3); + sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68); + __m256i sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4); + sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136); + + __m256i bsums_r2 = _mm256_maddubs_epi16(one, sb_h2_interleaved); + + for (int l = 0; l < 3; l++) { + // Quants value shifted to process next two values from each sub block + q1 = _mm256_srli_epi64(q1, 16); + q3 = _mm256_srli_epi64(q3, 16); + q5 = _mm256_srli_epi64(q5, 16); + q7 = _mm256_srli_epi64(q7, 16); + + sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2); + sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34); + sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3); + sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68); + sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4); + sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136); + + bsums_r2 = _mm256_add_epi16(bsums_r2, _mm256_maddubs_epi16(one, sb_h2_interleaved)); + } + + // Overall bsums in interleaved fashion computed by adding results of both halves + __m256i bsums_r = _mm256_add_epi16(bsums_r1, bsums_r2); + _mm256_storeu_si256((__m256i *)(y[i].bsums + 16 * k), bsums_r); + } + } + +#else + + // scalar + const int blck_size_interleave = 8; + float srcv[4][QK_K]; + float iscale[4]; + + for (int i = 0; i < nb; i++) { + for (int row_iter = 0; row_iter < 4; row_iter++) { + float amax = 0.0f; // absolute max + float max = 0; + + for (int j = 0; j < QK_K; j++) { + srcv[row_iter][j] = x[row_iter * k + i * QK_K + j]; + // Update the maximum value of the corresponding super block + if(amax < fabsf(srcv[row_iter][j])) { + amax = fabsf(srcv[row_iter][j]); + max = srcv[row_iter][j]; + } + } + + iscale[row_iter] = amax ? -127.f/max : 0; + + y[i].d[row_iter] = amax ? 1/iscale[row_iter] : 0; + } + + for (int j = 0; j < QK_K / 4; j++) { + y[i].bsums[j] = 0; + } + + // Quants values are interleaved in sequence of eight bytes from corresponding super blocks + // Bsums values are interleaved in sequence of four bsums from each super block taken for interleaving + // i.e first four bsums from the first super block, followed by first four bsums from second super block and so on + for (int j = 0; j < QK_K * 4; j++) { + int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave; + int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave; + src_offset += (j % blck_size_interleave); + int index = (((j & 31) >> 3) << 2) + ((j >> 8) << 4) + ((j >> 6) & 3); + + float x0 = srcv[src_id][src_offset] * iscale[src_id]; + y[i].qs[j] = nearest_int(x0); + y[i].bsums[index] += y[i].qs[j]; + } + } +#endif +} + static void quantize_mat_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row, int64_t blck_size_interleave) { assert(nrow == 4); UNUSED(nrow); @@ -546,6 +835,16 @@ static void quantize_mat_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRIC } } +static void quantize_mat_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row, int64_t blck_size_interleave) { + assert(nrow == 4); + UNUSED(nrow); + if (blck_size_interleave == 8) { + quantize_q8_K_4x8(x, vy, n_per_row); + } else { + assert(false); + } +} + static void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -994,6 +1293,281 @@ static void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, c } } +static void ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK_K; + const int nb = n / qk; + const int ncols_interleaved = 8; + const int blocklen = 8; + static const uint32_t kmask1 = 0x3f3f3f3f; + static const uint32_t kmask2 = 0x0f0f0f0f; + static const uint32_t kmask3 = 0x03030303; + + assert (n % qk == 0); + assert (nc % ncols_interleaved == 0); + + UNUSED(s); + UNUSED(bs); + UNUSED(vx); + UNUSED(vy); + UNUSED(nr); + UNUSED(nc); + UNUSED(nb); + UNUSED(ncols_interleaved); + UNUSED(blocklen); + +#if defined(__AVX2__) + // Lookup table to convert signed nibbles to signed bytes + __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0)); + signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0); + // Shuffle masks to rearrange delta and scale values to multiply with appropriate scales + __m128i deltamask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0); + __m128i scalemask = _mm_set_epi8(7, 7, 3, 3, 6, 6, 2, 2, 5, 5, 1, 1, 4, 4, 0, 0); + // Permute mask used for easier vector processing at later stages + __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); + + // Mask to extract nibbles from bytes + const __m256i m4b = _mm256_set1_epi8(0x0F); + + int64_t b_nb = n / QK_K; + + const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 *)vx; + const block_q8_K * a_ptr_start = (const block_q8_K *)vy; + + // Process Q8_K blocks one by one + for (int64_t y = 0; y < nr; y++) { + + // Pointers to LHS blocks of block_q8_K format + const block_q8_K * a_ptr = a_ptr_start + (y * nb); + + // Take group of eight interleaved block_q4_K structures at each pass of the loop and perform dot product operation + for (int64_t x = 0; x < nc / 8; x++) { + + // Pointers to RHS blocks + const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb); + + // Master FP accumulators + __m256 acc_row = _mm256_setzero_ps(); + __m256 acc_min_rows = _mm256_setzero_ps(); + + for (int64_t b = 0; b < nb; b++) { + + // Load and convert to FP32 scale from block_q8_K + const __m256 row_scale_f32 = _mm256_set1_ps((a_ptr[b].d)); + + // Load the scale values for the 8 blocks interleaved in block_q4_Kx8 + // col_scale_f32 rearranged so as to multiply with appropriate quants + const __m256 col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, deltamask); + const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin); + + __m256i iacc_b = _mm256_setzero_si256(); + __m256i iacc_min_b = _mm256_setzero_si256(); + + const __m256i q8sums = _mm256_loadu_si256((const __m256i * )(a_ptr[b].bsums)); + __m256i q8s = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(q8sums), _mm256_extracti128_si256(q8sums, 1))); + q8s = _mm256_permute2f128_si256(q8s, q8s, 0); + + // Processes two sub blocks from each Q4_K in each iteration + for (int sb = 0; sb < QK_K / 64; sb++) { + + // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7 + const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); + const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); + const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); + const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); + const __m256i rhs_raw_vec_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); + const __m256i rhs_raw_vec_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); + const __m256i rhs_raw_vec_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); + const __m256i rhs_raw_vec_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); + + // 4-bit -> 8-bit + // Values of the first sub block of eight block_q4_K structures for the sb loop + const __m256i rhs_vec_0123_00 = _mm256_and_si256(rhs_raw_vec_0123_0, m4b); + const __m256i rhs_vec_4567_00 = _mm256_and_si256(rhs_raw_vec_4567_0, m4b); + const __m256i rhs_vec_0123_01 = _mm256_and_si256(rhs_raw_vec_0123_1, m4b); + const __m256i rhs_vec_4567_01 = _mm256_and_si256(rhs_raw_vec_4567_1, m4b); + const __m256i rhs_vec_0123_02 = _mm256_and_si256(rhs_raw_vec_0123_2, m4b); + const __m256i rhs_vec_4567_02 = _mm256_and_si256(rhs_raw_vec_4567_2, m4b); + const __m256i rhs_vec_0123_03 = _mm256_and_si256(rhs_raw_vec_0123_3, m4b); + const __m256i rhs_vec_4567_03 = _mm256_and_si256(rhs_raw_vec_4567_3, m4b); + + // Values of the second sub block of eight block_q4_K structures when sb = 1 + const __m256i rhs_vec_0123_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m4b); + const __m256i rhs_vec_4567_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m4b); + const __m256i rhs_vec_0123_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m4b); + const __m256i rhs_vec_4567_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m4b); + const __m256i rhs_vec_0123_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 4), m4b); + const __m256i rhs_vec_4567_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 4), m4b); + const __m256i rhs_vec_0123_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 4), m4b); + const __m256i rhs_vec_4567_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 4), m4b); + + uint32_t utmp_0[4], utmp_1[4]; + + // Scales and Mins of corresponding sub blocks from different Q8_K structures are stored together + // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12); + utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp_0[1] & kmask1; + utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4); + utmp_0[2] = uaux_0; + utmp_0[0] &= kmask1; + + // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12); + utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4); + const uint32_t uaux_1 = utmp_1[1] & kmask1; + utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4); + utmp_1[2] = uaux_1; + utmp_1[0] &= kmask1; + + // Scales of first sub block in the sb loop + const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]); + __m128i scales_rearrange_0 = _mm_shuffle_epi8(mins_and_scales_0, scalemask); + __m256i scales_0 = _mm256_cvtepu8_epi16(scales_rearrange_0); + + // Scales of second sub block in the sb loop + __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]); + __m128i scales_rearrange_1 = _mm_shuffle_epi8(mins_and_scales_1, scalemask); + __m256i scales_1 = _mm256_cvtepu8_epi16(scales_rearrange_1); + + // Mins of first and second sub block of Q4_K block are arranged side by side + __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78))); + + // Load the two sub block values corresponding to sb in block_q8_K in batches of 16 bytes and replicate the same across 256 bit vector + __m256i lhs_vec_00 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + sb * 64))); + __m256i lhs_vec_01 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16 + sb * 64))); + __m256i lhs_vec_10 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 32 + sb * 64))); + __m256i lhs_vec_11 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 48 + sb * 64))); + + lhs_vec_00 = _mm256_permute2f128_si256(lhs_vec_00, lhs_vec_00, 0); + lhs_vec_01 = _mm256_permute2f128_si256(lhs_vec_01, lhs_vec_01, 0); + lhs_vec_10 = _mm256_permute2f128_si256(lhs_vec_10, lhs_vec_10, 0); + lhs_vec_11 = _mm256_permute2f128_si256(lhs_vec_11, lhs_vec_11, 0); + + // Dot product done within 32 bit lanes and accumulated in the same vector + // First done for first sub block and thenn for second sub block in each sb + // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3) + // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7) + // ........................................................................... + // B0(28-31) B4(28-31) B1(28-31) B5(28-31) B2(28-31) B6(28-31) B3(28-31) B7(28-31) with A0(28-31) + + + __m256i iacc_0 = _mm256_setzero_si256(); + __m256i iacc_1 = _mm256_setzero_si256(); + + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_00 ,_mm256_shuffle_epi32(rhs_vec_4567_00, 177), 170), _mm256_shuffle_epi32(lhs_vec_00, 0))); + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_00, 177) ,rhs_vec_4567_00, 170), _mm256_shuffle_epi32(lhs_vec_00, 85))); + + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_01 ,_mm256_shuffle_epi32(rhs_vec_4567_01, 177), 170), _mm256_shuffle_epi32(lhs_vec_00, 170))); + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_01, 177) ,rhs_vec_4567_01, 170), _mm256_shuffle_epi32(lhs_vec_00, 255))); + + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_02 ,_mm256_shuffle_epi32(rhs_vec_4567_02, 177), 170), _mm256_shuffle_epi32(lhs_vec_01, 0))); + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_02, 177) ,rhs_vec_4567_02, 170), _mm256_shuffle_epi32(lhs_vec_01, 85))); + + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_03 ,_mm256_shuffle_epi32(rhs_vec_4567_03, 177), 170), _mm256_shuffle_epi32(lhs_vec_01, 170))); + iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_03, 177) ,rhs_vec_4567_03, 170), _mm256_shuffle_epi32(lhs_vec_01, 255))); + + iacc_0 = _mm256_madd_epi16(iacc_0, scales_0); + + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_10 ,_mm256_shuffle_epi32(rhs_vec_4567_10, 177), 170), _mm256_shuffle_epi32(lhs_vec_10, 0))); + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_10, 177) ,rhs_vec_4567_10, 170), _mm256_shuffle_epi32(lhs_vec_10, 85))); + + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_11 ,_mm256_shuffle_epi32(rhs_vec_4567_11, 177), 170), _mm256_shuffle_epi32(lhs_vec_10, 170))); + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_11, 177) ,rhs_vec_4567_11, 170), _mm256_shuffle_epi32(lhs_vec_10, 255))); + + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_12 ,_mm256_shuffle_epi32(rhs_vec_4567_12, 177), 170), _mm256_shuffle_epi32(lhs_vec_11, 0))); + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_12, 177) ,rhs_vec_4567_12, 170), _mm256_shuffle_epi32(lhs_vec_11, 85))); + + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_13 ,_mm256_shuffle_epi32(rhs_vec_4567_13, 177), 170), _mm256_shuffle_epi32(lhs_vec_11, 170))); + iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_13, 177) ,rhs_vec_4567_13, 170), _mm256_shuffle_epi32(lhs_vec_11, 255))); + + iacc_1 = _mm256_madd_epi16(iacc_1, scales_1); + + // Accumulate the iacc value for one sb + __m256i iacc_sb = _mm256_add_epi32(iacc_0, iacc_1); + + // Broadcast the bsums of the two sub blocks of the iteration of Q8_K across the vector + // Multiply-Add with corresponding mins of Q4_Kx8 with bsums + __m256i q8s_sb = _mm256_shuffle_epi32(q8s, 0); + __m256i iacc_min_sb = _mm256_madd_epi16(q8s_sb, mins_01); + q8s = _mm256_bsrli_epi128(q8s, 4); + + // Accumulate for the complete block + iacc_b = _mm256_add_epi32(iacc_b, iacc_sb); + iacc_min_b = _mm256_add_epi32(iacc_min_b, iacc_min_sb); + } + + // Multiply-Add with scale values for the complete super block + acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_b), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row); + acc_min_rows = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_min_b), _mm256_mul_ps(col_dmin_f32, row_scale_f32), acc_min_rows); + + } + + // Accumulated output values permuted so as to be stored in appropriate order post accumulation + acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask); + _mm256_storeu_ps(s + (y * nr + x * 8), _mm256_sub_ps(acc_row, acc_min_rows)); + } + } + +#else + + float sumf[8]; + float sum_minf[8]; + uint32_t utmp[32]; + int sumi1; + int sumi2; + int sumi; + + const block_q8_K * a_ptr = (const block_q8_K *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb); + + for (int j = 0; j < ncols_interleaved; j++) { + sumf[j] = 0.0; + sum_minf[j] = 0.0; + } + for (int l = 0; l < nb; l++) { + for (int sb = 0; sb < 8; sb++) { + memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12); + utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1; + utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4); + utmp[sb * 4 + 2] = uaux_0; + utmp[sb * 4 + 0] &= kmask1; + } + for (int k = 0; k < (qk / (2 * blocklen)); k++) { + uint8_t *scales_0 = (uint8_t*) utmp + (k / 4) * 32; + uint8_t *scales_1 = (uint8_t*) utmp + (k / 4) * 32 + 16; + for (int j = 0; j < ncols_interleaved; j++) { + sumi1 = 0; + sumi2 = 0; + sumi = 0; + for (int i = 0; i < blocklen; ++i) { + const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF); + const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4); + sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 64 + (k % 4) * blocklen + i]); + sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 64 + (k % 4) * blocklen + i + 32]); + sumi1 = sumi1 * scales_0[j]; + sumi2 = sumi2 * scales_1[j]; + sumi += sumi1 + sumi2; + } + sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d; + } + } + for (int sb = 0; sb < 8; sb++) { + uint8_t *mins = (uint8_t*) utmp + 8 + sb * 16; + for (int j = 0; j < ncols_interleaved; j++) { + sum_minf[j] += mins[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) * GGML_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d; + } + } + } + for (int j = 0; j < ncols_interleaved; j++) { + s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j]; + } + } +#endif +} + + static void ggml_gemv_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -3480,6 +4054,781 @@ static void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, c } } +static void ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK_K; + const int nb = n / qk; + const int ncols_interleaved = 8; + const int blocklen = 8; + static const uint32_t kmask1 = 0x3f3f3f3f; + static const uint32_t kmask2 = 0x0f0f0f0f; + static const uint32_t kmask3 = 0x03030303; + + assert (n % qk == 0); + assert (nr % 4 == 0); + assert (nc % ncols_interleaved == 0); + + UNUSED(s); + UNUSED(bs); + UNUSED(vx); + UNUSED(vy); + UNUSED(nr); + UNUSED(nc); + UNUSED(nb); + UNUSED(ncols_interleaved); + UNUSED(blocklen); + +#if defined(__AVX2__) + const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 * ) vx; + const block_q8_Kx4 * a_ptr_start = (const block_q8_Kx4 * ) vy; + int64_t b_nb = n / QK_K; + int64_t y = 0; + + // Mask to mask out nibbles from packed bytes + const __m256i m4b = _mm256_set1_epi8(0x0F); + // Permute mask used for easier vector processing at later stages + __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4); + + int anr = nr - nr % 16;; // Used to align nr with boundary of 16 + // Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation + for (; y < anr / 4; y += 4) { + + const block_q8_Kx4 * a_ptrs[4]; + + a_ptrs[0] = a_ptr_start + (y * nb); + for (int i = 0; i < 3; ++i) { + a_ptrs[i + 1] = a_ptrs[i] + nb; + } + + // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation + for (int64_t x = 0; x < nc / 8; x++) { + + const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb); + + // Master FP accumulators + __m256 acc_rows[16]; + for (int i = 0; i < 16; i++) { + acc_rows[i] = _mm256_setzero_ps(); + } + + __m256 acc_min_rows[16]; + for (int i = 0; i < 16; i++) { + acc_min_rows[i] = _mm256_setzero_ps(); + } + + // For super block + for (int64_t b = 0; b < nb; b++) { + + // Scale values - Load the eight scale values of block_q4_kx8 + const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d); + + // dmin values - Load the eight dmin values of block_q4_kx8 + const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin); + + // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration + for (int sb = 0; sb < QK_K / 64; sb++) { + + // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 + const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); + const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); + const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); + const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); + const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); + const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); + const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); + const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); + + // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values + const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240); + const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240); + const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240); + const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240); + + // 4-bit -> 8-bit + // First sub block of the two sub blocks processed in the iteration + const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) + const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) + + const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) + const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) + + const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) + const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) + + const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) + const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) + + // Second sub block of the two sub blocks processed in the iteration + const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) + const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) + + const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) + const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) + + const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) + const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) + + const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) + const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) + + // Shuffle pattern one - right side input + const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) + const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) + + const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) + const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) + + const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) + const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) + + const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) + const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) + + const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) + const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) + + const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) + const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) + + const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) + const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) + + const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) + const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) + + + // Shuffle pattern two - right side input + const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) + const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) + + const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) + const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) + + const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) + const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) + + const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) + const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) + + const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) + const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) + + const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) + const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) + + const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) + const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) + + const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) + const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) + + uint32_t utmp_0[4], utmp_1[4]; + + // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together + // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12); + utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp_0[1] & kmask1; + utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4); + utmp_0[2] = uaux_0; + utmp_0[0] &= kmask1; + + // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12); + utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4); + const uint32_t uaux_1 = utmp_1[1] & kmask1; + utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4); + utmp_1[2] = uaux_1; + utmp_1[0] &= kmask1; + + // Scales of first sub block in the sb loop + const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]); + const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0)); + + // Scales of second sub block in the sb loop + const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]); + const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1)); + + // Mins of first and second sub block of Q4_K block are arranged side by side + const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78))); + + const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68); + const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238); + + const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68); + const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238); + + for (int rp = 0; rp < 4; rp++) { + + // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 + // Loaded as set of 128 bit vectors and repeated into a 256 bit vector + __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 * sb))); + __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0); + __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17); + __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 256 * sb))); + __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0); + __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17); + __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 256 * sb))); + __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0); + __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17); + __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 256 * sb))); + __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0); + __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17); + __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 256 * sb))); + __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0); + __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17); + __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 256 * sb))); + __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0); + __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17); + __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 256 * sb))); + __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0); + __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17); + __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 256 * sb))); + __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0); + __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17); + + // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks + __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].bsums + 16 * sb))); + __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1))); + lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0); + + // Shuffle pattern one - left side input + const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) + const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) + + const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) + const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) + + const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) + const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160); //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) + + const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) + const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) + + const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) + const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) + + const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) + const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) + + const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) + const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160); //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) + + const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) + const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) + + // Shuffle pattern two- left side input + const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) + const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) + + const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) + const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) + + const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) + const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) + + const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) + const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) + + const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) + const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) + + const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) + const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) + + const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) + const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) + + const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) + const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) + + // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane + __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1)); + __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1)); + __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1)); + __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1)); + __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1)); + __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1)); + __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1)); + __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1)); + + __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2)); + __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2)); + __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2)); + __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2)); + __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2)); + __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2)); + __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2)); + __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2)); + + // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block + __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); + __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); + __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); + __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); + + __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); + __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); + __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); + __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); + + // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block + iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0); + iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0); + iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0); + iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0); + + iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1); + iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1); + iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1); + iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1); + + // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step) + __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204); + __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204); + __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204); + __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204); + __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204); + __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204); + __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204); + __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204); + + __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1); + __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1); + __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1); + __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1); + + // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes + const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d); + const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);//GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask); + + // Multiply with appropiate scales and accumulate (for both d and dmin) below + acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]); + acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]); + acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]); + acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]); + + __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0), mins_01); + __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85), mins_01); + __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170), mins_01); + __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255), mins_01); + + acc_min_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]); + acc_min_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]); + acc_min_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]); + acc_min_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]); + + } + } + } + // Store the accumulated values + for (int i = 0; i < 16; i++) { + _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i])); + } + } + } + for (; y < nr / 4; y++) { + + const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb); + + for (int64_t x = 0; x < nc / 8; x++) { + + const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb); + + // Master FP accumulators + __m256 acc_rows[4]; + for (int i = 0; i < 4; i++) { + acc_rows[i] = _mm256_setzero_ps(); + } + + __m256 acc_min_rows[4]; + for (int i = 0; i < 4; i++) { + acc_min_rows[i] = _mm256_setzero_ps(); + } + + for (int64_t b = 0; b < nb; b++) { + + // Scale values - Load the eight scale values of block_q4_Kx8 + const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d); + + // dmin values - Load the eight dmin values of block_q4_Kx8 + const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin); + + // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration + for (int sb = 0; sb < QK_K / 64; sb++) { + + // Load the eight block_q4_k for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 + const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); + const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); + const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); + const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); + const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); + const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); + const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); + const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); + + // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values + const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240); + const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240); + const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240); + const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240); + const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240); + + // 4-bit -> 8-bit + // First sub block of the two sub blocks processed in the iteration + const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) + const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) + + const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) + const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) + + const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) + const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) + + const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) + const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) + + // Second sub block of the two sub blocks processed in the iteration + const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) + const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) + + const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) + const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) + + const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) + const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) + + const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) + const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) + + // Shuffle pattern one - right side input + const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) + const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) + + const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) + const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) + + const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) + const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) + + const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) + const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) + + const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) + const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) + + const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) + const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) + + const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) + const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) + + const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) + const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) + + // Shuffle pattern two - right side input + const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) + const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) + + const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) + const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) + + const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) + const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) + + const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) + const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) + + const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) + const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) + + const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) + const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) + + const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) + const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) + + const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) + const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) + + uint32_t utmp_0[4], utmp_1[4]; + + // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together + // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop + memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12); + utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp_0[1] & kmask1; + utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4); + utmp_0[2] = uaux_0; + utmp_0[0] &= kmask1; + + // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures when sb = 1 + memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12); + utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4); + const uint32_t uaux_1 = utmp_1[1] & kmask1; + utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4); + utmp_1[2] = uaux_1; + utmp_1[0] &= kmask1; + + // Scales of first sub block in the sb loop + const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]); + const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0)); + + // Scales of second sub block in the sb loop + const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]); + const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1)); + + // Mins of first and second sub block of Q4_K block are arranged side by side + const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78))); + + const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68); + const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238); + + const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68); + const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238); + + // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 + // Loaded as set of 128 bit vectors and repeated into a 256 bit vector + __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 * sb))); + __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0); + __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17); + __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 256 * sb))); + __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0); + __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17); + __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 256 * sb))); + __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0); + __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17); + __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 256 * sb))); + __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0); + __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17); + __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 256 * sb))); + __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0); + __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17); + __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 256 * sb))); + __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0); + __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17); + __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 256 * sb))); + __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0); + __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17); + __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 256 * sb))); + __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0); + __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17); + + // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks + __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].bsums + 16 * sb))); + __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1))); + lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0); + + // Shuffle pattern one - left side input + const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) + const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) + + const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) + const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) + + const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) + const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160); //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) + + const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) + const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) + + const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) + const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) + + const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) + const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) + + const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) + const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160); //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) + + const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) + const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) + + // Shuffle pattern two- left side input + const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) + const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) + + const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) + const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) + + const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) + const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) + + const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) + const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) + + const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) + const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) + + const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) + const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) + + const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) + const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) + + const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) + const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) + + // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane + __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1)); + __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1)); + __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1)); + __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1)); + __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1)); + __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1)); + __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1)); + __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1)); + + __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2)); + __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2)); + __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2)); + __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2)); + __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2)); + __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2)); + __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2)); + __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2)); + + // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block + __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); + __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); + __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); + __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); + + __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); + __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); + __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); + __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); + + // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block + iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0); + iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0); + iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0); + iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0); + + iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1); + iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1); + iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1); + iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1); + + // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step) + __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204); + __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204); + __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204); + __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204); + __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204); + __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204); + __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204); + __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204); + + __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1); + __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1); + __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1); + __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1); + + // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes + const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d); + const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); //GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask); + + // Multiply with appropiate scales and accumulate (for both d and dmin) below + acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]); + acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]); + acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]); + acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]); + + __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0), mins_01); + __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85), mins_01); + __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170), mins_01); + __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255), mins_01); + + acc_min_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]); + acc_min_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]); + acc_min_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]); + acc_min_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]); + } + } + + // Store the accumulated values + for (int i = 0; i < 4; i++) { + _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i])); + } + } + } + +#else + + float sumf[4][8]; + float sum_minf[4][8]; + uint32_t utmp[32]; + int sumi1; + int sumi2; + int sumi; + + for (int y = 0; y < nr / 4; y++) { + const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb); + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + sumf[m][j] = 0.0; + sum_minf[m][j] = 0.0; + } + } + for (int l = 0; l < nb; l++) { + for (int sb = 0; sb < 8; sb++) { + memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12); + utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4); + const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1; + utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4); + utmp[sb * 4 + 2] = uaux_0; + utmp[sb * 4 + 0] &= kmask1; + } + for (int k = 0; k < (qk / (2 * blocklen)); k++) { + uint8_t *scales_0 = (uint8_t*) utmp + (k / 4) * 32; + uint8_t *scales_1 = (uint8_t*) utmp + (k / 4) * 32 + 16; + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + sumi1 = 0; + sumi2 = 0; + sumi = 0; + for (int i = 0; i < blocklen; ++i) { + const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF); + const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4); + sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 256 + (k % 4) * 4 * blocklen + m * blocklen + i]); + sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 256 + (k % 4) * 4 * blocklen + m * blocklen + i + 128]); + sumi1 = sumi1 * scales_0[j]; + sumi2 = sumi2 * scales_1[j]; + sumi += sumi1 + sumi2; + } + sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m]; + } + } + } + for (int sb = 0; sb < 8; sb++) { + uint8_t *mins = (uint8_t*) utmp + 8 + sb * 16; + for(int m = 0; m < 4; m++) { + const int16_t *bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) * 6); + for(int j = 0; j < ncols_interleaved; j++) { + sum_minf[m][j] += mins[j] * (bsums[0] + bsums[1]) * GGML_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m]; + } + } + } + } + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j]; + } + } + } + } +#endif +} + static void ggml_gemm_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -3660,6 +5009,82 @@ static block_q4_0x8 make_block_q4_0x8(block_q4_0 * in, unsigned int blck_size_in return out; } +static block_q4_Kx8 make_block_q4_Kx8(block_q4_K * in, unsigned int blck_size_interleave) { + block_q4_Kx8 out; + //Delta(scale) and dmin values of the eight Q4_K structures are copied onto the output interleaved structure + for (int i = 0; i < 8; i++) { + out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d; + } + + for (int i = 0; i < 8; i++) { + out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin; + } + + const int end = QK_K * 4 / blck_size_interleave; + + // Interleave Q4_K quants by taking 8 bytes at a time + for (int i = 0; i < end; ++i) { + int src_id = i % 8; + int src_offset = (i / 8) * blck_size_interleave; + int dst_offset = i * blck_size_interleave; + + uint64_t elems; + memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t)); + memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t)); + } + + // The below logic is designed so as to unpack and rearrange scales and mins values in Q4_K + // Currently the Q4_K structure has 8 scales and 8 mins packed in 12 bytes ( 6 bits for each value) + // The output Q4_Kx8 structure has 96 bytes + // Every 12 byte is packed such that it contains scales and mins for corresponding sub blocks from Q4_K structure + // For eg - First 12 bytes contains 8 scales and 8 mins - each of first sub block from different Q4_K structures + uint8_t s[8], m[8]; + + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 8; j++) { + s[j] = in[j].scales[i] & 63; + m[j] = in[j].scales[i + 4] & 63; + } + + out.scales[i * 12] = (s[0] & 63) + ((s[4] & 48) << 2); + out.scales[i * 12 + 1] = (s[1] & 63) + ((s[5] & 48) << 2); + out.scales[i * 12 + 2] = (s[2] & 63) + ((s[6] & 48) << 2); + out.scales[i * 12 + 3] = (s[3] & 63) + ((s[7] & 48) << 2); + out.scales[i * 12 + 4] = (m[0] & 63) + ((m[4] & 48) << 2); + out.scales[i * 12 + 5] = (m[1] & 63) + ((m[5] & 48) << 2); + out.scales[i * 12 + 6] = (m[2] & 63) + ((m[6] & 48) << 2); + out.scales[i * 12 + 7] = (m[3] & 63) + ((m[7] & 48) << 2); + out.scales[i * 12 + 8] = (s[4] & 15) + ((m[4] & 15) << 4); + out.scales[i * 12 + 9] = (s[5] & 15) + ((m[5] & 15) << 4); + out.scales[i * 12 + 10] = (s[6] & 15) + ((m[6] & 15) << 4); + out.scales[i * 12 + 11] = (s[7] & 15) + ((m[7] & 15) << 4); + + } + + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 8; j++) { + s[j] = ((in[j].scales[i] & 192) >> 2) | (in[j].scales[i+8] & 15); + m[j] = ((in[j].scales[i + 4] & 192) >> 2) | ((in[j].scales[i+8] & 240) >> 4); + } + + out.scales[i * 12 + 48] = (s[0] & 63) + ((s[4] & 48) << 2); + out.scales[i * 12 + 49] = (s[1] & 63) + ((s[5] & 48) << 2); + out.scales[i * 12 + 50] = (s[2] & 63) + ((s[6] & 48) << 2); + out.scales[i * 12 + 51] = (s[3] & 63) + ((s[7] & 48) << 2); + out.scales[i * 12 + 52] = (m[0] & 63) + ((m[4] & 48) << 2); + out.scales[i * 12 + 53] = (m[1] & 63) + ((m[5] & 48) << 2); + out.scales[i * 12 + 54] = (m[2] & 63) + ((m[6] & 48) << 2); + out.scales[i * 12 + 55] = (m[3] & 63) + ((m[7] & 48) << 2); + out.scales[i * 12 + 56] = (s[4] & 15) + ((m[4] & 15) << 4); + out.scales[i * 12 + 57] = (s[5] & 15) + ((m[5] & 15) << 4); + out.scales[i * 12 + 58] = (s[6] & 15) + ((m[6] & 15) << 4); + out.scales[i * 12 + 59] = (s[7] & 15) + ((m[7] & 15) << 4); + + } + + return out; +} + static int repack_q4_0_to_q4_0_4_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) { GGML_ASSERT(t->type == GGML_TYPE_Q4_0); GGML_ASSERT(interleave_block == 4 || interleave_block == 8); @@ -3690,6 +5115,36 @@ static int repack_q4_0_to_q4_0_4_bl(struct ggml_tensor * t, int interleave_block GGML_UNUSED(data_size); } +static int repack_q4_K_to_q4_K_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_K); + GGML_ASSERT(interleave_block == 8); + constexpr int nrows_interleaved = 8; + + block_q4_Kx8 * dst = (block_q4_Kx8*)t->data; + const block_q4_K * src = (const block_q4_K*) data; + block_q4_K dst_tmp[8]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_K; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_K)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++ ) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q4_Kx8(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} static int repack_q4_0_to_q4_0_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) { GGML_ASSERT(t->type == GGML_TYPE_Q4_0); @@ -3807,6 +5262,10 @@ template <> int repack(struct ggml_tensor * t, const void * da return repack_q4_0_to_q4_0_8_bl(t, 8, data, data_size); } +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_q4_K_to_q4_K_8_bl(t, 8, data, data_size); +} + template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { return repack_iq4_nl_to_iq4_nl_4_bl(t, 4, data, data_size); } @@ -3832,6 +5291,10 @@ template <> void gemv(int n, float * s, size_t bs, const void ggml_gemv_q4_0_8x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_q4_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc); +} + template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemv_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc); @@ -3853,6 +5316,10 @@ template <> void gemm(int n, float * s, size_t bs, const void ggml_gemm_q4_0_8x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_q4_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc); +} + template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc); @@ -3863,16 +5330,16 @@ class tensor_traits_base : public ggml::cpu::tensor_traits { virtual int repack(struct ggml_tensor * t, const void * data, size_t data_size) = 0; }; -template class tensor_traits : public tensor_traits_base { +template class tensor_traits : public tensor_traits_base { bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override { // not realy a GGML_TYPE_Q8_0 but same size. switch (op->op) { case GGML_OP_MUL_MAT: - size = ggml_row_size(GGML_TYPE_Q8_0, ggml_nelements(op->src[1])); + size = ggml_row_size(PARAM_TYPE, ggml_nelements(op->src[1])); return true; case GGML_OP_MUL_MAT_ID: - size = ggml_row_size(GGML_TYPE_Q8_0, ggml_nelements(op->src[1])); + size = ggml_row_size(PARAM_TYPE, ggml_nelements(op->src[1])); size = GGML_PAD(size, sizeof(int64_t)); // + padding for next bloc. size += sizeof(int64_t) * (1+op->src[0]->ne[2]) * op->src[1]->ne[2]; return true; @@ -3925,16 +5392,23 @@ template class tensor_ // GGML_ASSERT(ggml_n_dims(op->src[1]) == 2); char * wdata = static_cast(params->wdata); - const size_t nbw1 = ggml_row_size(GGML_TYPE_Q8_0, ne10); + const size_t nbw1 = ggml_row_size(PARAM_TYPE, ne10); assert(params->wsize >= nbw1 * ne11); - const ggml_from_float_t from_float = ggml_get_type_traits_cpu(GGML_TYPE_Q8_0)->from_float; + const ggml_from_float_t from_float = ggml_get_type_traits_cpu(PARAM_TYPE)->from_float; int64_t i11_processed = 0; - for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) { - quantize_mat_q8_0((float *) ((char *) src1->data + i11 * nb11), (void *) (wdata + i11 * nbw1), 4, ne10, + if(PARAM_TYPE == GGML_TYPE_Q8_K) { + for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) { + quantize_mat_q8_K((float *) ((char *) src1->data + i11 * nb11), (void *) (wdata + i11 * nbw1), 4, ne10, INTER_SIZE); + } + } else { + for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) { + quantize_mat_q8_0((float *) ((char *) src1->data + i11 * nb11), (void *) (wdata + i11 * nbw1), 4, ne10, + INTER_SIZE); + } } i11_processed = ne11 - ne11 % 4; for (int64_t i11 = i11_processed + ith; i11 < ne11; i11 += nth) { @@ -3944,7 +5418,7 @@ template class tensor_ ggml_barrier(params->threadpool); const void * src1_wdata = params->wdata; - const size_t src1_col_stride = ggml_row_size(GGML_TYPE_Q8_0, ne10); + const size_t src1_col_stride = ggml_row_size(PARAM_TYPE, ne10); int64_t src0_start = (ith * ne01) / nth; int64_t src0_end = ((ith + 1) * ne01) / nth; src0_start = (src0_start % NB_COLS) ? src0_start + NB_COLS - (src0_start % NB_COLS) : src0_start; @@ -4098,12 +5572,13 @@ template class tensor_ }; // instance for Q4 -static const tensor_traits q4_0_4x4_q8_0; -static const tensor_traits q4_0_4x8_q8_0; -static const tensor_traits q4_0_8x8_q8_0; +static const tensor_traits q4_0_4x4_q8_0; +static const tensor_traits q4_0_4x8_q8_0; +static const tensor_traits q4_0_8x8_q8_0; +static const tensor_traits q4_K_8x8_q8_K; // instance for IQ4 -static const tensor_traits iq4_nl_4x4_q8_0; +static const tensor_traits iq4_nl_4x4_q8_0; } // namespace ggml::cpu::aarch64 @@ -4124,6 +5599,12 @@ static const ggml::cpu::tensor_traits * ggml_aarch64_get_optimal_repack_type(con return &ggml::cpu::aarch64::q4_0_4x4_q8_0; } } + } else if (cur->type == GGML_TYPE_Q4_K) { + if (ggml_cpu_has_avx2()) { + if (cur->ne[1] % 8 == 0) { + return &ggml::cpu::aarch64::q4_K_8x8_q8_K; + } + } } else if (cur->type == GGML_TYPE_IQ4_NL) { if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) { if (cur->ne[1] % 4 == 0) { From dbb3a4739e53226346c772dd72666e68b78dc583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Thu, 20 Mar 2025 12:49:59 +0100 Subject: [PATCH 177/188] llama : make Qwen2MoE QKV bias optional (#12477) --- src/llama-model.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 17af8cc30b..cd7e0a0c4d 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2210,9 +2210,9 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0); // optional bias tensors - layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0); - layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0); - layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0); + layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); + layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED); + layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED); layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); @@ -6193,16 +6193,25 @@ struct llm_build_qwen2moe : public llm_graph_context { { // compute Q and K and RoPE them ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); From e04643063b3d240b8c0fdba98677dff6ba346784 Mon Sep 17 00:00:00 2001 From: Woof Dog <197125663+woof-dog@users.noreply.github.com> Date: Thu, 20 Mar 2025 14:57:43 +0000 Subject: [PATCH 178/188] webui : Prevent rerendering on textarea input (#12299) * webui: Make textarea uncontrolled to eliminate devastating lag * Update index.html.gz * use signal-style implementation * rm console log * no duplicated savedInitValue set --------- Co-authored-by: Xuan Son Nguyen --- examples/server/public/index.html.gz | Bin 1260534 -> 1260624 bytes .../webui/src/components/ChatScreen.tsx | 78 ++++++++++++------ .../server/webui/src/utils/llama-vscode.ts | 12 ++- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/examples/server/public/index.html.gz b/examples/server/public/index.html.gz index c7a3c426b623c707bf17b900886abb3a3a57c8d6..d0e6da8e4a1e0646ad83f40bceea6462506ffc8a 100644 GIT binary patch delta 1176841 zcmV(+K;6Ig_e{|EOpqiGvXf4HDLYOj8&G2+_@ELB6$+F{^mFzBxcTKI%YB&usEJML!NNKBoCdN85U5bCn6p2EQ3<} zsgtDA;b9~l2@PT+VI=rK#xuzAId#VDjyr=9ONKlnj^MXkI5Ncw-QW0)bMe7-GCq(| znmAE{@nu=5GyT5I(h3P=fCfpgxas2LY+v>-0wnvAEPpVMp=yz4St_Dol)RfV;runGoW6UMBaz-apJx-^7a|oN_ZMAA zJ`AbHi6n_n-^td9WghqE^S{QPyT=k(PIo==G3S0#=JXCj{B{LrA#9zoGZE30N!VFD z4qcxly`%*@$K!q=TSG2A?)&6pwA@_JYK?^($3dE8(vdVw2l<#MvNaH#N#2h+ggiHnk~`OL z2_AonGTDu8J%8C~$X1q5V1s7%x2G3;B6ys#(Di-6WiFELZ7Mv~P6|i6XzH(=i`hAk z`9P*(Cyu@U39VfU9@AP?SO3%ZBQN#43BPxaa>=0gCpQ@vQ!c!ezI!a&h@p+tk8ZuV zh5L|FcQA_L5C-7-WyTCL#;`U#JWfNN`B_V{;V~QzN`Ku?AC5n|jYbAzhj6t=jYi~U z{=#pisu!La6?rTg3)b2s%WtIMJXmWl{H~dIC-G7<^)Y>1bk#&TX)8)3ABsrM8VxTl zL)1#~N!mgnCAc_E<7lArR-Y0<(^Ti*CGnz|1z6F?=+;YHd}8(reABgd*Gvw~RQHz4 z!N^NnBf)P;>@OC+rvSh&QwmF*-a5lLynBHTe`V1;1tx_f- zqHJ?=m1MdOVIZ(U#W>b(R*OB9uytPi{qKKs{_gx& z91VDqap#P)frJcX% zPB@t(~R07;tBhh8!+s-E!#UNeEZxh&x9gKAULgHqD_ryf~qX zPapRB$LEv61Sfx??7kn<4sH%k&u8PCG;YC>chz)o1=n1k$Q7wDw=4!Axgzmau%Rv+ ztw_$Khz9a{oQB->NxG_bk;0gT>yvnO@-zxDIa`s8_wxg^K3LVcRNONWUJHKf`XpS{ zzdW13dbmEBtVka7aq9YHwjwEJ-)F8*h9)VJfOVy!T!()L@31%yhm_JJkK_4VAEU!H z>ZW)k+#VzZ-XR?}8i$v|tNHx!(*3XhG8MX4#FU}bHMP6F!vMu6LL&# z$guAnk}-cLyZ$3=U|7>hMh^)F^a|23S7E_#GkUDPbL^;~yYu4J)pg-$vtb4VTPU7@5MxTvM~xYPI~pk?f9V4q!T1 z7o5=ylAS3N5xa?b255^Khm#72ImM*hkD|fIQ7eD+d=)L%@7nXxs;z|wi3F#(=%4FX zXXLhAcxzJhg4F7@Hi@a)waUA`wX|oj{%2)F)M~WM_36K31HeW^-nKxCZm2*GBpD8pY8bNYkJihQAugIoxnNIZlK zGT?vo@s7^`w-Njney*?af>d7!u>spDr{tIya{|2>N^*yFRt#8^_-U03mt?#EnRxMNO+(r+}@2K6y| z81LQ6%b~oYV-7zGy?Bt6i6NqOSu5trP>z3^8_>@ONh$64x3Y!Ep5O`H?eaL|PIT*a zwjZRA2r1ODt0&!ikUse;X}kMXZa>KGSGm6K@AAv9@~Uj)@~d3mxawBw<%{UnJ8U!# z4Uz!;&a4@FkhGdc9;B~FW6g|?d5~&+>Nr0B@{0lq0Y}&y91>XESmNRyOM)9XxRpWV zI=xT02)?xRc36~n+#T0I=czgo566nf94kJA<srl&fbo{gXZ zpWq4{OWGzwX^%J<%5^tzVX%k3Zy?J90Fwu~eW*cG_W?l4V=U?6<%7KHYUneRFk!I- zsL(x@>+~~*Z~}HYY@daJT)UhCDrc~;U2dac*zO_S)$sJte<#?1HP~y1t%N^F&-Ytl zn((e4OB&18l*KtG)h?{B`($M&3h=2ms@^>sB^h(7&%vQjG=8FPuFPSABa?5+;wW?& za1`)EK*agqD?My!*SO&;Pg?UW-9M~luei%jJOB}lGrJ`5WSZsx6v@T;gzHlbq`wz)~V+|R#0~~kchPd zbq+^EC3EOFlHnysK=I3w4TcT0Tz^r}(XBm$fjo}cBsk;bHWhs~7zLAO3u}KLBCn=Z zqT~AzJkaa}nn37O^9E>&p;04Kxb(uOq_;od*0)M zX!OtrFo$>%!TSdAM?exZwpbw^9x+fS8)8E?K)C{qT}4COR%i%?y#kts^ic%~*Vpih zBxHK;az(+|M93~*O@v^)H(T`KS6(}z!gqA{_qJx zWtaPp4^k5-yWH;@+`Ldgs$}hs=8E^VhMvx zK*R9EmZ`$(M@+6YaUXviLKagmJ+#JTPS-In;|>^dJn0H>c+6WjQ4)ICkHZ3*z-0>@ zfkT9cMMY=)mM*cws&^9R){MeR_QPe$Z9o(%6KLQSWnTcjGDG+ValjXRwvrjSp0F8+ zOM*vzwFb=&(HAGo)HMh%gG4Vq@F44ata_ajL#7VW@q`H;AX0y)PX@9V$6VExNhajy z;tF0h#W%cF-qGDk2)KGu%pMOte{mZnAa8!GW-#sKAn?11`E>zH^NjDYIKE+nyVb*S z0BEYEu`{bY3sc3@N1#?*W%LeF-wzQm%f`rPqA{Nh!>zaRN4^5FLG6KH9|8AnYkJik+*P^?Rh3M|DtoyMP z-NDSRl+l$EydMhbe#F~ODgO^J=hhN2M&f^$C7yW$5|W9!h-PF6?C=N04Zl;&@QIC( zJ^(&C@kxJJ$5b&PRCsDwM|kfjMk%BYxD8I|YbBD9%m53G>qzJfanQ@z6%8+ES3Y2n zgGOVp%z0MT(P|0_eJpGB!K$l%lS_4z4-hYWz?r)=K)D`YUVwnLW@< ze`NTDr}T@p*Z!QAr}PV)(l2mIe=ASvdpxCO1lE7>7lm>p(R=ZH0E_gsE`t{k`+F_+ z*YH`%Jg8iOs(|?4?}#5-)Be$#_CuKVLzwmg4s-vp$SS=XPA<4Q52t{aX_n@VW{+NQ zzsm#IdENH*X;)=Dq^Hf!yLaz81aY~`!zrxQ-f-&;pke)E**0iWcP4;w`)XbbmQ;d;d;k!c7QIvL75d0#zVQ@0s8ia zU&c~R$Ga3b)~CK7%k435G$7&_#Rff3P0)X_Bq~3kuQZl^pc8H}VW<;^6(+4+*QuH6 zdQG*t){}E#NoLZemCsTkKO`Z4XkJ(WL;k?TwR@+*4R1vuaB<*fW!V5OXG0m9P_3 znNL<5AI4)I0<6&rQDp???dqHfn?^%W<$!FBMiFdjkQXIG>xNpD+={Xf$t0JjJPH5p z+?rrskZ#0!1hOMj29wMinqSl9Qsaxjg?KzfW5ySvRB zZnqb6a{|@w0s%XGoO#FGC&wJP?6CWk3c)FTK(ZL7?*O1O>`^cDyThVIZ_OLlM*i*s zG&OP5R;PJ!lXwtdGf ziSjj9==i|w9JN9>%Tw3=MsK*e+Sb~*8!k8u`{E*P(@s+&iYU=r z(n`6=xY&);!CjOLi?SPbjsaY#@iJ5F22p>Y>H%L=3Z{vSlAQM+_#m&-@q6?6-F)s@ zpwO=rMoq2*!N&Zo!j^zm+wX4peBRl3xqZWXpjDN|yv3DNqgu{rzB!W=_ybcwi$RSGLHlpL5jhDoMVo@@5ZlR*95oZCsZB$RXg;EvNabmWN1QR{y; zPE+A^_~wfn?guX`LmF~Jd7j!Ja0O!VP%n27iAHn~3S`@2vOdt$MWqW91{}my>tD$x_+L5C zN4agTsH5yJ~d!v!#As2Gt?naJp#?@uo7wi)V*u2*Ond zgj{fPYVOM=V5~c=q2Z~G^qy)+-t$iBX)EM0n~_s$<w0F3E8XWq;DLo|C z==IIo>{yCD&Ia(@H(uJxKF&*s>$G2QzTWD*+1Rv`wlIlr)po>~)0cd6QAmF^=;`{H zqu4wOgW38zF+d@>;G}ZjgFw8vJA+eFz4JlEtU$toW@kZ8-yH&_W1(iJM0yh#CxH!5 z&CPtn@y4G36Xdw(*%(l83_wO*-KWh%-$xqBs#2qOQ0EG;*sC`l4V-I0`&EO_w(=j_j{ zm5SkuFdbws@P=xJ9L6bHU}@Nk5>%fC)4g>;a@y(U+jU`ZH(y`(**br98%zwd%PZ=x z^91U+IQvj28hE92H!t{g?opIPw-FDG>>fJc{110sth-%j8f6jKhNS53BbM*iG{Hs)}P^OvoJ({Jd2$Xk0-Zz?A$YvM9DB~ zxwum!MDotLBxywpKlXnFVg6%=KP2%)-A^2AufxQe06+1ZbgbGc#>s zecg}HdXHTSEo&dpHLqe*^v=!Q!rDPawTv~`b7@tvFysWwjUO?sC-lYtaT)A1f4^qU z?|0HV?3wt@hR5e|bS>z|b95w$lTgPJNg|#Xuht;rG_sfey|v*a;>r$usE* zzc+;awAK#1nfiYqYaMIj-ND8qIT<0S$P3yqHx`SAk+{~WUGYro;1*<((!$M965HX0ZJz|(a7;!-cHaLS?XRffvy)jjU{% z#+gAo^q`^1wT?kkbYzqrvty6>D*9DDdOq*GV~b)mr745yCYh2xInqWRwOORl;>LUq zKbz`jOBs1EDMfUXOO7_CqZN_agbg^Isua+hy2!XW`ZOnnO$mgt5~1R7+c5Q=F|Y{# z&M(eB(Y}ANG#RH;-XBloOzF|mBZOu9u+Lqi!sCQs1257%i)ZIten=87-hV#&9~~nn{lD^P>Jt2JatvHut?Zq;x5Z`qs=I&5Oh*~%U`H=ISM|QSgX%1~ z#0>=-4)u37;c<*p>=GCCsNJg{Ps;|jRz1K=4TBGbGY-;iXozd^`59M;O?C$36gcjp5g6%x370K%{YI zwj2C%;e-)(C^R3Z(EzUX4~hOT@n?LTN)9yv9u4Tl_&7xsCZOm?Q5f>XiPLoAB&o97 zb&@hYo$$n&#B7#*Na85rfV+k#pav@R>^Ofy&0y$cgLDEvoQ-2JxQ`?spF`Zgx1%?I zX6Xj%%(=uJ=dp|?xH98hVpN8P^gR5T#n~hvt6r|(AW3eR-Y3fP{>PPVtvrn2ULtde zE7E)T$)fLSU$f+A>U_1jPLbZ9Lk+}50`O;QdileNMa2A!zPS9O8C-dntoi-Qx2=D9 zK3Ug49OafwFD4UH*8+h|BAX+Qqzj9qA>y>n8*hzZirrM9EN(c+A8d7fqgo=5rhspSTm9bjp7PuB+k_ zf@IUg$Wb)7OVAkyFsGWS0FrdVk~HJZjx$MdEt`A_q?c3l370sLPBvl_yQ`T@C%Rww zX%+&kvhFyjXXnFF*^O#+*_TX;^o}<}mW`A(iWQOG-ez2y7#PHWP3$0;!EYv)4F6si6uN3}c$b`j_o)brEd-uiU=_h3fb%eplPq`u_a@GRKJY)hA2d!;SdV|5kx%|A?4o|h zVlx37$QG>_ed~S`xfz|eZ{NPa zwp<&)KaAr5pp#4RGiJ$9eJT!`=B zcY^Wi`xq@+_dMvul#C#dD_W6^=JTnhI(rWuQ%O23G-b47XfTiYfdHopRt$07_1yP^ zX`LYJiOXq)8oQ!J%~M0kZvxHjx`(_!eN`qS*sVo=7 z4^Mkfgp7~Hj_h{m4Z8e{5#=g@`Ef`1Ko_lbBt z9u&?1AIwK)m?grg4wC3eQug~L7%ln2K$54E%q5Q*m%)=2Eec(iGkyyMqAeoSQKeTA z{}KAP6zwr`d1YpPs6|mDI^>Jxtb$x#0h1k7V4yapSfqAwEGksIH6*LhswFQ=!nn#f z(7@r1#;DNq8*;TLC^fDMN`K+Xl$unH-Td=!|MW?pj+P^j=+xNKb;t$n7lCePn;wDK zlwLF%)5{~cf%=yhS6%<2*+gw?D@XUJ*(4XuCM>v>7rTA5Mrmh0zknYOYpO_I~A=k%7BXj7fYk@wWS@pWq+Q#U3hSNhZpBp z`n9akP9&8|e4l(>+>fG|d*uLqCD>go3ZWvLCi2ud$o269R&%dX`+P21fP@9E>-P{A z1g=~1UT=Y~ZY1;qJoRbLZ^6`sGKwXv*t(rpR*L%xL8VUFv{TLtD6Lc9u1Zzz167W8 zMK7RJ^gvL69qMuhRDVq93Og$@v=qTp$+xneHl$ZM?5=BT@JS6eMb$@uajLTlB>SuO zh*^O7d__@nl1xD(mk@dt2Hf<`=;ym?P+v^^sk-h}>|K%Fm_F--<07FndDcnyWs+Gj zmGF5f0T*jg9f5HvQEZ`_Ly~h-i&vW!ch4XK>uzD%shj>*mVYrOpwARGrlY#>H4jSC z1vW!*bZMmvM9k-&psZRm(~IT0L{JpRxIOovu@i=%@W(<295ua4xr(&OIP|Es;-GIB znU1^11|zfh<`%Ge?B`dl(=H4B`^(mOWBRerM<~tUlb<0bBBiM*X+Wp@8783)Ph(lg z&feH^wqxG2k8R*ur*~t+RC!X9=OG~hLX!z1M}G~Gdi5KP8pi+}?nI!W?=~7J{8ciH zoAkj|l2KQ%FiKrZHvZLu1=wt_kb4AL6h?AGL=3sK*bMtz`JYaOCu~=F7S_ysG6JJH zammm;W=$-!Dhfn0S7r<*1g+7EwWr+1do6+*Kc69_bIToBu+J7i;h)&(rU z$4fLdYDRGB+@#@5-^WEJj4ON77m5TnbD@V>SYd%@7DgVcx&XRgq~bju?$GAjG7J?s_XGOKQ^o6`sBF|~3< zZq@PJC$*l^$7)xD5z!H9+&R%A>=>IWJ&~8-uZUDIuvhD9Q0a=bIDcyX1SC}vf?qc3_6bq#u}m!tM^!haCFD}5WJC&04NTrfRkp0)wdPu0v**M&``yP5+z zY0{8!8Wd?5t|u239oR~i-+%}4`5Z%_b0A$f{dG2|z6cDhyI^%FZZh;j+{v0>>Icgi z?fnA8uw4LW#?IjTq?mJ~v{m0nh1)<;OQ36O^}N>F943m3uJojy8V_un=YL0FxHIKC zu6d)ia&8+945_(=>dfjGL%T8tSKoMMXLxO{KCbGp&2z%&f=O%B?(~1I%rqrTM-Ftzcg#w1_}+;4A(9 z!4w45?Dm$gSCeag`?;t(<$tx@x+N9WIY6}138ZA|@PmqC!PRPar`x$vTb+Wo?#*^b24|`Bia)!M9N^fo%fAvE%5}CsX9VsdzoobCV3h zB4q5TKDh4sgY^(z6-nWINZzWRRLmh{}IDkStn8LO;@&8fPaNG!_8JXwf4|_ z?t!w_^vW-C0D5XyZ<|Q|8WsFt)_OumZiNT!#CzNE{rZ(zxd>O?ltvpb_`zPY@5bM2 z*EKeRAe@#i(bPSzJyEjS4N7v$c^X?inhhUq*UyEQ`YP2cb4GzW`$M)L(L zCmGLoPTAtA`&3N^ZhsML4X!IAtln75P`aBJKo~b%j_^T)BkKl&UAO{~fZMHhj#nJw z-1o_78Q2#md}J-qucAUf@dv53B)bY?KE2G12GgL;+NvdBwP;0d^GA)V-s#W~cq@d* z8|`-c1!PeKA!tB6lN+VvV4;Tpjy}0{g7GQoG$>cv#N-FA)_<}TIq5SBcmA`L^d6Li zc6G*LIE@C9mbariHVarR;D`AM12-9lH_jS@%r zqE$pt)-6Evoqwcvaz20RMW8U6g1l8)h28CJI(+A@>oa+B>-nYUYJc6`eu3$P(L33? zMLR@*LtSA9e1qj-(6iH&3!5&hWrq`!getq82Gp+QhNP6{74W0dFn~rSI~T2(=~&Rk z2e!s%EyedARam1h=7L#N4l`xSra7UHEQ!Xb-1GruqkmDFC=Y~dZNf5eC3*&t5WGQs zun(Y+c9Kh==BttCpnzYC`wdj8RNTWs%pa`JLy_ha9X=5uh-8YWK^o`dvLh;(!RM{6 zYSf?oB(y@H&}j!joz8^! zu!q%m;D=O#pdPnx6!5B1N(HZsJ1Vs7%DAPv%(hsN z-x7KWkJsHrpawq%4cw#jyGz^?_o{3Aw!TkR>wk9rmMljrl5-uH9!%^=ukBY=Vb?_~ z%Wo9((d+o+J^>Z1`{Yu9uYBuBek(|wl6PFS3b9<(Yd)b1)O}v&ShMhWrhG%+Cv{vF zP>2kzLJueAgMw(d?IEZO3o#x4NIM!nPmv(CXBP)Y0SmFESimA4|;3MaRo+ibeP z^nYxVDgBmsIboOi6&V3C1Q$}AGUSBj)E#8m0S0}er&Cr;rhWjZG@UAk4~d|s-s6}j zIcS&T0zWc+yg*wHe}z8M@B^>Y;VBtmY&0I?4=`fkuMg}`98rHEj_x45U4d2Fjg|Ke= za);ptYZe~Li@Kw%lq=Mvvo1bmsMMn$0NKIFinh3p)9T!VrEn9J3hQyXh?e5ee}9r( ziUv$YM~(s5t+G_L966Ketn3nHRMPCUdgks2%vzduO!RJ)gmIMR= zLhx*oqKlx5%wRHrM$;!#fd^sV|2@$HMTSIz=LC%_{;plH3?ubBi)uanN6FLO%YZ#? zJh#27k(v zn(%{JL@&ivFpQ|MK%n*jGmI2e6Lc66MrRQmoLNN8AaP8Dp%=gv*0TQ)+EmsO=+w}C z&!$>-B=~5624zxF!V)$+OeVgK6#HGs+_u@=su7ZxrtMfC@rF^s1DKglV1MvH!(_;iM>`m{J<(QROkJ9Oin3Y*CCiU>jY<;k1a072d*G2`$#YY7Oy+=oM z%|jCICYLgW8(x%hLLE@77=IH292+h?*9VEp+Ni5t%9S7k{0YH9t1G|crI7Bq^S0R` z%!pI+l4+BxooOD&8K62M+VSJti>oDSToM;N+DB9FqWr9)ou?kV@rgDDXJOa}!w-;D z!*j--3l_vK;gWD!*xzewG=QPsmm0feJfb%1YtvD!HXNfY%fX;RG9XVI-Rs(e7zP?ooEe80=1 z`P_pcw_(V`imSEAW>5l=oTB7Wv{;NLn3DUN%GC^apV8tQiE}|&H%zfGYu(0dm^HTA zZ{Hw3X>!8(C`u_+JyE>s!9bY?Z?in8DjM7{b5rRooz0S61q#T~cOH`qEieHqlRPa$ ze{|F}p05^4yAtYFNG_$cufCL4E7Xs{ZeRH>>?6Zm4`Ciba3Q`PsOfBh$`uaEvMUiOQb%`8a(|y&J%O>!&xWo!*y#46DyqrN*X0*@T`yQ98U- z=LKu&D)xW%n0Apa*0W`wQ?#5-seIa%71oL?U*Ngq>+b8l}j%2$Xz9Ul~X&ynVqHTEsq>d%gXo3}(ATS|9l>3_DwB6Ai6C0Q1iDe$@4dQ2Iva0*q=cVB zO%uI8N{SBY2J!dJjb8Z~FhOLbf70S~NbZ^X{&c)i_MZuR7r%S=j<$(FZ_1k+!DgrF zN!V^L?%BNU&)eAEg`}^xND|Sl&fBfc_LgN}CT!V8qbUjxWx9cmaZRSf8#N~&TQz0H zH+7B1&MU18SyZv9>v(0f|2E!et-zPBvQZewfX%^TC$)|;2FUzzsazV5s%`YakS zbsJl4J$yBfx*MBs-n?p?N?yHw{d%LLYuVg<`EqM(Q#btTb*KI2&8w}l(#~ePy|KBq zv1Jr6w>CPjUvCt%Sw!ene|Ys~YxCvS%VKhi2(6X6`RdK<_S-`3vy3p>x!R9h(SG)xvfNjqef%RV~r^gvr{^1G)JUQS)<9w^d0Rho50SiM#JlyP2ppwOQ;SlG`674 zIn5iLSAZ13)5wy+t7dxW*rIQ)4wI;;1JoOs2H6y`7WMoB|6I z^89PsEi?v;#X3o~f0M4N89~+KV(G+Z9UKnoGi;8oIk5k~JEm;?CExsSJfh7`{b1J4 zVeQzhK4r~12d-)Rar%&!&e96@NcGf=YNrMcR@FJ9K3Vp;Y;X5926CkLP~RsPu!oMw zR|=+1-LJ^+?0IY6MI-m;^NR)=_(B>zNxchTdIev%!N$+ie=X`R%;|+s`VIQ83u`2j z)%b^Ye;UnVUS1;7)$WNvG|I{9S@YH|E8R~$!#BGqG<`4R_BQR1HSa6p;9c*B&6Dc-HL%b)m&&IPvtz4p3gTEDKxq(f zkEcL_Ya6rM-p&h(PK`JRyTH!&_O#0`r&qL1M)1=Fe@^N*Z_t1BZEh>8&PTO}bIQ6a z!D!bsJSrPTAUf(!8*DBvM^_Ep+LxoNIq1?syIL~ze>=R>S8AD5EO{HWDM)8q^tQG| zPn7ppuk$J>VZm0Q9C1G|@;MR{nD@Y%?HX{LB@lf-*GmWc;`uBrWPPW;E#7Xga zmq3|MDRsewkb@^72fy-(Bh!2^QZ#`22A3-Mp}eTD{9yoGO((pG)ra$z~CI$+804^se=!3nkivJM$cKKl{ zi?y=3!f~1{CB(_9+N_gwn%X?O6~^hH`GQ*QaInEdw&w}@cmwnr4^NVSkuV)&B0k-i z6NCWgvrO{w0T4aGh*&iQeWHX&pvb|Zgo`Y=oL()6tsV$69Ohe5pFt$<6X6G*1~FCx ze{(uQfK;!SOxH@qVI$O=`uc@p4d{VoV{x_4Ef!|6eOP__7Z@qUT9!jambI?f zR=TE2^(0QpdgJ7|dRMr0ypKz+;n&nNe~VUShG?)Jf&8lCjZ$$Hk^h5SMPQv`)D1&5 z*CL;26&sW?atq4G#<8{i+%Zc)^|8=lEna6wf0Ce=w8TTIGFn?vR@x<2Tg^)buw9GU zoF9_gRJmv|Z7=N7Ri@MTv|u_FS*na95a$&r&GJbA%E5>Z&Fmh)?Au3|ET1d_f6ib? z1p@dL`P`$@EiY0v8UPnmZkvT^2cjTPoF#Nba@|EV;sZVimi9(I(K2YY6ot;pUS_Ya zh6kiF-(;=RWi9me<{38pnuRt##0Gi8>4eoakkLB-yMSD(8~Lz&+T}Z(a(n?@zpX(s zX(wh967{{{ldzH)Vq$0#KgdGef9E0g;WSx6%Wtj5aO^Qsjdxe~3F(6%q>qA-XV`<@ z)9kd})VDeN-wF%SFYZ=Cw6I+JU4oB!R#Z@k^naIhN~(oy{Cnqr zt`DnK#Q^g)5b_!Ys>ch$)e>BF%nh!c)VE2#4b_OkJX3|>jn2}qXNkg4mT z$f-*b1}vOrXZurHpc9~Jg4z(YaKsku>dQpm$LN@RF*1f>6N6m-Y`}Ho0@)t;LgBDXSo@M!$K<) zw}MsK{fHO42Aj4Ah7tBqbW3&~9i?+sgk-LMyTdi=YB#axYwr9*R1>(}MKuWw_vt42k93Mz`CWjV-Akw1U_W(0D(LFlVk1GI|Zq(~EN5-(<5^1HjKFfa{pY4G%qPDjJHGBx;FnU_F zucDNwRoOvxHt|a7Z2Dk1#W1PUHgRNgV;ejUVC-g3*i7wnRz4wUMgNkP(H0dQiyDSr ze<$G@{SxA{1^Ow2YejHh;SvuXUXE6|kHT^tHZ> zwk@P<9lv+Zf?raitGDo_bD2&iJfu2tQP_>zPX8V!s&3t_s^7mTTk1jSudqI$U9f1( zTh;n{^$5=o3LhudP}ZoMCwB>4kXVa7O7Lg))Z~AiGhev{vBn+L-m!YJtF>~yd>P{` zJF70%-TdYzUh1u#2ByBxECzOX!bKKkmC0U^C_7crz&<=@3!fF+C;UBURK&fMq_Y`H=YD5j15JrNS@E(Nn!ocFj?ARZ4zalMV{gJf{Zg73-6lw zBw~LQO^&a@<~ZW_0azPL7R5n(;gccei@l*Nr~(lqkLq^*8nlV}`Ubs^ z!NvO5;tS#?Y{B!;Wo4??mw1A5@2NPh>Fj6tow1%W$vjKf}T-7OLo z02Zw>H~R0|1I7sa6NRUb5{>A2f2GhZHPg*KW5V-wpz$V-! z=U{q!!$jaZF8Rc7W#H}zk>10nOD;_&I(RX0$r+j&sxW@L1eEf13~KC?eI5tyKJx!u z@>`Sy?#a1Jj(Cy>#vdI7T(UEnWVNVsL>>cI{eDW}<(Q-N`>BW$WUs-&b1#x8lOa4U z0q2uXJQaT;p+z>@0X({vsh(`SE+=hs1E*_q6Y>thtblB8sc$t6n=hfo&?TF%0{49i zPxWlRuFUM_8*64a-&Ut}t6iDYt(SrOA<4Laq*vwqc3_ML9r)M`+=C7H*b3Z(P55{j zxCdMC@hWf+Uc$%gz&&^cA8!Kp;5B@_4cvn_Fjar;z&&^iA04Q$4PRI_)V%?9ccAjE zt-w9b$7()0&?|eXHa4~b_lQg8F35FGA9rFIxC(N*L{D_!YWNC!LbAZs0M#Y9D1mE$ zu4}2#AGJ#ia_M3%m=$$^+*^+FP|iiKb1?7et*rqTT=8J_u|x}8P!{P3@%J>M47{?S zQU{X#s6SiA|Y$$eJdO@+WOKuc^n z3fe?HrvY~fw3!RUu7GurfD?^VD>8}^e}P}XR^&Zkb#0<61yeu7r%pvQW@3iGe+Pm8 zedd#scDa`orzXRO;QISqgcZ{wEF)H|_R3wcV!`3=!K?SA?UN7f@@lf*9QcVF^)r7~ z@}cdM@)bMU|BdE7FS= ze@EOlmh4FF9-r*1&6E3NC#RQhNoSL6ye69)SL91d2aSfiS2DH`5<>ObV3endkc4c; z2gZ@6vQ4mn8h)pF_!-NOs!ByTi6CQ@g2{s9S!*l;a%TLO-7m+IFk)Vi!*1=vQP z+Vm0a-oc{*aJ0=Y5Zv(Xnm`k4ca=K@pqs%>LYgSQE6VkR@|cii@bD9SJc|>gNGHFPK(xfzkJzgKGr~T|2W>Q3Ozq#<#BhUG=fswHz>OnM!YM?NfjIQeClE z0jrqMTFM7mDRTP_glvqS=J4K4n(QgwyH83T4fUiBzjwx&SLPr_1fj)yAUzaHiAk|R zf}F(KWHk%e!n`u0%|Vard|autN+$5gr^N?NqjFmIV{NDO9<)-6LuBdV=B3{R<#SUF zra&E`p4-Bv7pomx-}lr_6E%O47R*N*!h4nDnm+3tse4_0U#F;!xY|?@9*m(H0$#Ut zvTe$<7}I+t6dWVmqhq)(R^B-C`P=Q)S5Y}7pFyuE0xh-(;jhtcanr;g6J{Mxd{`B| zKvoqe1=3!5fR-9wdZUp>-ZAsdGF@JEA2N@TV@3o)pA9|{i6>Kd9mIc9BPz6)mCG(A zFYI}rJ)f|MLV21K7Qs77&v#q<1uJf+M&q+*I|MFYNuW!jGHqwB?$R=Vm!8=-YIv6D zzqZRzK5AvQ^KxFzT(Nbp?ax}#bje$qQn*<|pEsQsNaMVKH;aW59|O^(_V`D$#k8S1 zdI%|>M;;3y3e>n=Eq8MqNPnD{dI`+GcDtSPyq3JceG}SD&q?&*0&uYNV>eYJblU zT&)<96^uI_R`0O^62JsEPFhbjLeN`V^PWruKkl7Z;A6YOq!8fd#4LrN8s}$J3eWE{ zCjyyxGQcg~0tY`I4g?z;@ZMGLBWWpx@w=m6a{f(m7Y+5_g6H>2hdkP_G5|e5!oSyk zDsmrC$Vy42wxU#Pus@;z=A{9DIgkW#gkvs_-jsCllvQ9+n`#yRKrV;mY z(l=6b42}Hmw9z=iorp*@`Y2SdV~Ky3dNuY~5vfz$+-39$0zgbqun>}Pujm3tMFEU0 z@*nB^KAr}q5josUfO-H1him4k7Ky1J@LTPUJdQDY=J}-mZqx%`9$~0zKt*6R4W8-& zDH`+x^%g*XMH;Ep;cHz}vlER@P%52?KLD9#ZFpoW2O@TvAXtAF5Ih}w$csfxu*_6}g znJvHX4xGg}kUTs9ec!dwEbhcHQrMKJRmb z?(|`QN#Vh3rQV44`6?(dLP&oy@_QrthzBVc)u005NoYs*lW+n4le|$KjE(9rZ28># zwL?xu;EdE9pkQR3CSsqUB1HvBafE_Vd7OxKgiO2&EqwBYHyRoDzHox`@rC2DKh3>U z4kV@O6OxpplB8r7NvU&~Nz^1I>YY|eN@kIiBaplswU9oq>Wo(~vB!T!;fCX~_Dszc zXo^-|y3Gssnd{<;y7&5~Qv#Mm)>FrvIf{mCt_udt%uxaaPe2$z)KAlvdukvB3P zVxKNiIFYr>sm=6c8MOg@QYtQ+PJJ$7Tw(>VTMH6_Yb){z9b5sA@{@d?*i}9a?cr4! zp7Xh<-*)+vx9kZj17v?ihGJHp&xVsw2K~x_>K!S|S0=>@S=M^r%>y!#1c{g4Iv7C0 z;!kcx3+=2ABT25@?LX%_w9&xiPqI=mg3^+tfI5GX^t<$2d*WQ53`I3IvNp(mtwg3b z!q)`jD?@jE@_T8z`CCd|c9SYY%spl3s0=R$EIEzY%=~^<&&huqV@OahHnxx;jV^5_ zL5mXq^|gH_O5)VlL!{C_ff6EqolJGwd1=KN=GuL-vbf~qiHvyY@MIun6Nz6T{0DbC z&M=jyN|>Wv(`jo?tz8Tjp8MM5^VFDpUQ8T#z7GF!)Ctg^AY%W=2*GVq*2aJDetUTsvBT`gGc0dgo|G|6l6H~2i=(=wix?;`x8|pTnP`mzy`b9d# zk%5Y`o-<{cs??GlrI3t2&}=3dyilXdG4NNv)eZ|yJ}99RmR=r2SLExAeoJcr{db7FY+uz}7Jo~%5@yHm_7m^x46Z77{r8!_=qKJU5`s0r z-A}xssKndgxu1CZ)lvo!_Y<$wqbD`jjw;!(zaaGh)uFBU6d8zVvJ0Bj82>eaa1DYJ zyh|-tjO2f^)K(^zrt+1Y6%XMLv+BKH)N3gBR7pzYhN~I|8aY}s0FV?N93|D%OIsFhFCHtq9r%Q zj98*dELF2>FUInOJ$os=TkfD$x7wv-$voh{tA2k4O{xtjn}Ywa1-x>50g|;P_)stWwcNlPKkC*&9!+ZLi zOV2g8GG4sqa>HA!0)XxEz;_ui>D^_%1$4!vN*F07&U|hZW}Ap`yIoOJwa&>uB5y>P z@hN`>x;87cu`Q2Um)<2Z-2uvS3QPx7R|57!f%$iNppDw})9UDj zy!1slD&OmFZ_^`qhub)88eJY;k+1ZkW!QaW$mvD3|6Oz+cu}-RzJi;9Mq|2CQP}f+ zX=PDkOOVtbJscWIJ}Wfs-3p0 z8?lYPu19mat&8e$w36wKjI4SGMhY~Ca)3N>f$mT0wU+^0?u91Hg zk3(L5%CF#csZTk)lSQKCXAg4!5nmUp!}~_V`*nsR1gAu&Z4a!y;UuU}I22&UZ;Ur8 zoCOio9vg$_LN@=cE1-Ju8;e}%@#qB$8TV6Yytzb%b zOpFnlPwDUQKHIc)6D6T%h;SI$RIh(OS(9=;N9{bzN=-jiR2nPk2!-^`=iU^x?MJ=o znx&|{ zhOb`KLm3wjQ2Nr<(wvIqt83trWJB~J2hx!F9DcS6$n=!4XjgkPozCYM_Evv-(`Z~+ z96`%GSGTa`E8t^Xw|0-JHo;%%k?D~4RrTmA7Hc#-NPD6`N56}&Oqr$>d8T!*7WS4)Jo7!@O4h|23q z+NE=!oN%x$&V91aL0GM?lO2Cfe^>rPl#;@+uKevBd|2gYjYvi^y zZz}1YU7PcZR92Hip?>!z3(Hnt+e&lnqp8p z`%hgu{rkq$|DJ8EWr3Yax}$UgXR^d5$w5NbX0p*x`Wr)l%$6hLZGU0SwjfLsZ zdvqqfkDN%p?kf08QmJ?94mT2rW-+Ts=`RmGnJ=?{i7NjU?V*jj?{=^pw*+U+uF} zAqybGuOTmW*ELmivOoh-`cK|-m}_gIyQ{sVz)I;;^9+t|)u!yckwf$)^^?a>B?8W* zlkQIxpM?+jdBK8;eHNeiVq2PKGW**lYHgFvrk0G+%> z8uQx16Uq#T0ltVl0dmkNw>U^w{&tT&^lX8k}xCzUy zf9zvRwS4LIw=i|e(&kAf^a-F4!$K`nD0iin=J&*AKbdLjQ)YQmAxAJzc`5qg6@q6>xHY=@nmV z?tDI4U+?C0x=?^I_q$j}IjAHpO)=EAK_%FVSwmgKV&16hDE;%zk|WM=THcD#nZ`b;ubx{ihoi3;6fWC-HaR z#Gd6v&wp~$vq#;jcBz5Jiuu-^Z8X^W`eH?AmfrCC5YwNKpAp8Wd#&ysd|K)kcl|?5 zuCpQ*q>-75-6``HE}vD(`*x9n@)unonikPUBBN7Snev_`Va!#0c}~9CUeYcZyHc{? z;-MS+%@Rv{THDAlnghRSKF^`3GMk~yx-~V#EPvjSwQk!=ihd;r_kfMMV1?K((J$(15@y5V@;OPc|Y5U!$7@m_Y zUVqU0vwis~z9Q;JU8JAlX4C4+NK3On#p~-PUs1l)T+n87Q3@KcGeJe_MX68z!Y|;3 z{GQS%EjxS~S0DVqcT_c%3db5>omwm2VvN$_L6Qm%1{&fmy}>^uxa;C0?P2yJ8MEw; zhX`V|!ymA>GIc&#@Mx)+Hurd3kUpM>zJG%@D$~n8=$-Zpz0;9?*3*7l`rqjtdpiGe zr_dzCGSU<^X8{8%FHR}Zw}9m%5zwfc#QRJ5XeQ8E4n_TrQXC41czW8D}PjGgHI)LqEYx47w3lcMol&l67|gB?M0Ygy{D5l z5s3~Nx*rHHfv*wwjhO&2Z5+3n&7Mgb0Ur|>ju~&Uh=0jDCI+!5&Ncu{2Y+Z%nuuLR zx`4a-Xa?~1UZdqp`BVEVtgN2N%+96aE_)>XNe_}dD!|gsw(qmknfEJY(F`}RK}(Os z_^c{S#$RKg`7g%2#aIO%LKooK;&s>-kUGl)2jW zS@PR&yvNEXyobGh{@hfyES^6vmkA6+xS0vwPLv7BBXWL!My#*Qx zo3-dM{d>#s*PQd$MaO?#@W00APoBIOUx*RD(6d!{zoTFNIP3TyXZ&S){^ZH0^g@iT zeEB4*zkgbPA6Lv+`lT)8-!sHJ>8Fm6JekvD_%!wQgr*DsYx$i*g`Xj-y>S%23qOTN z8;s;Y!N5#0Y>-_WQGaU2F=}u%ygbDB>APISyj_V@KM-gvw>v#CuIDyL4X{$u9cwS3 zOoM^&FtO4i`mwNW5};RDV>*%eCpsSqG?(^oobn7ua^^)OfZa#usfNUJ)+sw9lg|yA zdJri%~<%WbI@I{`YyT5yXH7hbyH*-aqi2l@Mn6D9ZcBr5u zkQ(G`ShqncER>wn;C!4KObHmmGmXoH4Zy~=n1rpsQT9YOBa!VJodCJf=3-2v*wrlJ z=>M3+(Mb71ew~eOHk!=_DE34lvvhHz`H;~A>R6=JX@4x|5Nd3VDH7kP<4YpQBC)~g z+=7>f=!+8ObT$lU-+-Q5?KL2FdJl=!ojJ3*(|+pv9SJ$%Xl2;nRdgublF(}03QA=w zdbIs2TBBNO5GB)csk9%Yij~}{GON}i<>R!A-!XoU^oukb7G+prA_$DcB89GI)fN+R z7y#<=dw=9CGmi(j!TYH?Yx*fNk^NLNk$51McckDk2jZs%k$N&ztY0y+a|nhD3G%KY zyGR6)OZ2LdY7F&LR<5$%x80S|dgd#dLWt<9mYLsu3XHd8wg7qnwD9^!BueEhzH&Q( zH|=P3ytQ>H^xmiNl!sls8F&%bD%74bl9cdh(SK2{r(GE(+CL930{D;jRxG0Rg#I5n#H5rjZGVLXY1Q6oq8MW(T2advDyeS$|-)G(_93X=`=L}vR<}z zIe(9CAOwZH@pJBE1VA?(u_{GPb8wk4TvZ1tggE!+i!GKy8iEUhk&VH6vB5n&XgxCw6t<_-chuK)5V zCUi!9S3l$PnQHe)R>cASQq>|;w30ePQGXts3QaZgO*ntLo|pxr@SJOZNq96@EuP2E z^B5;aPQx~KY##f8d(*ljQ(?n>)2bf$BM}@S+5nw>idO4eo$BjOv13z@es#IbW2OD< zdU?zrkGwg!T_VQ?uQ2UU$;EI6P^|RCKvkX+N^O5xx**MCi;*uuNjC)7L|qzT(|;D) z_$J&u5db;JiH$XnEop`GGS}8QsMVO^O5;*c=39>+i$X8}d9(R-?v*((k-2D?-v#GW z**(z|@IXcC`DtNRh`6UZehkBM9P{g;Cp#$HtSe!1xtVc&NTozXiXS|YyLi1O9-+bc z+_~ixPbYIRt{5yO%G>vFq5Sw;3x7pV=W=;|ao4IjvHHo%{LZdE#ioqMc2nT3f4ppu zVnPPY;RRn;r&qIy3vt7kgYl`f5&NdSk4N6d#%6BXR`cnO^9{Xh^n@y*d|k}N)anyb62X>;K>_2_e8I`m`Y-P0RMd&cDrAT_5yJFM?D{E$D zZq5AMnfZCm%xi)pgmguMH#RoKm)*+sl+PwU zGchAPzaeG`n~UqdFP0X}j`%{y`GTIEAa1Az3}gf3*X>z* za4pwyO*L{Kp5^@5*@tvs&#-bcsb#Jv^;k~ov652qf?#z_4YD1yw+vThnbYIS6Q=pnKzr$X$@BviCU3;4ZZ1< z<-&>bOjXu}-cEzF+xG2+U^W9+MQtrPholu$A5j9S7V(gGtHOaEKg?s zsyZ1&N0-;0J%V5oic{JKPe%c;*Aim#JmzIC1X^*J%jtHDw|!$2Jp=#xJN6WtPGOl% zg(;ORhQ@TdHJVLhoGYu;-`#1Gnb~R6sqi{hdu=j%@LrqD?%ivXS?yd;W^}G6Gk7|g znY|`^`j0l*9e=|#oyy%tJI$t3XTSX|0`c(uc6Y<*##`IG%qDO(TW>mbww}20Pe|W= zTd!Z*Td!ZFVi;>u2_*$efIA#bW$^HL}Xyz&tQLc&T`8? z>MYk>9R4;EHIt&WDkc=zPyJh5#b%swN=ol6XLX&R9Dnr)V>y*7S!aAc%m1^2H?$Q5 z@B14n+o!}9d~#=-WsQn{h}_ZY-dWDg$5wR8jZo?8B8ZGBbrLn7|8GzQj37c*b5f`h zQjDTpJ7;R$SmC&^*464g%YE@vtc+cdFxP1bcvVZ_6t^`3yhckPC)DWeX<53uJD ztz}BEtA9)++_RK}tNT?GSbxn}8rkQJ$Ud5UoZTh+s3XH+YY~mY5{eu~peg|UZK)G= zk3zO2q*>4Ne%HHW9@ebif~b5SKVUR{hU4f5t_1#SiuRcPnE#xEUk&>;$E&u_elNgV zP)PEBNI#dyWAXQk>)-#e<@vW?TKw^t%d&>Rzkel}+kP*U<5cJC)cYL~Q7`2kk#?21 zy~HDB{!wnzG>8VgtMrkF=1ghK29x-r8KI9s*Sfg`XL56FCiWxW57s=CmDRk0CW=D( zSCilx;><36EB^qu9x+0~(vQ9w;F=ImhpF%f)D~gk)}TD>^fA9R7OiPEy>A&}(AY)Y zRDT!mQ_Ipp#~B9fOA8V$_-I?ZmF}KF#*`%$%LNewW<=yKg85hHC0KL|Ud!0-w7{VI zc+4~QM`3l6-(nAf7NgAu-&iCWkH`G+7?~e#>}I$?y2#*5D?K}e9p?^T#DDe*~CH3%vy@|Zca!SM-5cVQw zZpovLvKgDDix{hBmJ3}L@#0yH(*WRQ+pF5Dky*7l3)ZbtveJ?o?R1Hi(1_6h`=QRK2%IqAVT-x6)xp+$2M+-had; zv|{nqRXofWfs>mml-^3vOVUCXS?$FXBVM$=5Bj>&e!1z)9k(+u-PUv2v206R=$2<& z@`{GP97Qx%O2ms{PP`1e4t_wO`%UDsF+MSGuu*C3)?ag z;PdNhr`yte!FVCr@otg5Ow8tiN3!Q$(cEo6+uh!6p_7xZMllk*6&xzHqZw{CRxEkK zx4(@6#w9q4?s(*T%tg>QD zUyJo72of$jBolh!L9P|NGOMrHQkP^6c0w&w0@_s&@=(lGkEuj1Dr`!%%zh;@uSDXc zz`LD-?%3px9b39PG*nwMakpfmwM&v};Mouj8O6 zo?OG{2l(bmY%SXpVh%n*TpPM<`*6c;G~~0u0A2Z^lejXV*KyF-U#?Wa2R^d>q3LLp zJvxd31yC)IPLH<$H4Qo`I@af*iZDEc&|+Nhvz+(MUO)~rX&xeK%YR|)-&T8vjUqc~ zmWvJ4!oAbC_D;XLclvdE$5(i60My2Aa^6>HZHM5>IRiEY(P6GNJ|bxAat>Hb@(WTZ znOLEv{_PP!!U!$pEN2LpVm#CEp#{+JXeXHD((zGUG@tD6?d?2Wwzm&NnO61dhcT~h zK)3SoU;?HrIdAIIM}M(M^=h|Id~G{`ZP8k-%+TpM5tC>;q%QgXE_|U>dmKL}>5RI^ z&-)j_p>)dW`@SGJv)Y+GjQ@?Pg(axYwrf~GfYUSJN3kD}^)$;_Z^;j1h2x56U7xby z)tzph3&8}|H0vWw47@G>ily%o4t@wJVrpo^O44YEzDTdU6My2}f{kWmhw%X`Yc_86 zV<+HTsy|21eG@rS@(3b7dP^j7(-_b@1#J=y0>I2jPE%+arE3)@aLeyx<0bDg$&2Ha z?QDG%({!(M%l9zSOGaNwIgSQRg0E;~5P&0!02~PfV6_ksSf@dz(L53rCm?`GwcTAt zhi7e!j4H)uN`L&7NJjI10TFVu2!#0e3!fb>)KyPi5owAdKfGVKpLn`!La&H#UpBJB z0+OlMTgZ^J!-Wi%nneu0nGyGRh0F8knE(lFw*x>^hE%k6Dk~p2$!HQU=Fu?jpS}s+ zhPOT(?M zq?>jo>7?*M*A@?ZoMb&3-Pqi)-811cl2g=G=|;#4Y|COz5PLL27)R}6F|u5=Tb=BA zG@=o5(daz8*t+~V%f7#X)GUbM{yEMT>1--fz<-O5=+}4)KlHJ1y|9nc#T*>5%V8bq0C*axjK+HTVYzZmJ%oz<+H zHrl$$>gyn{V+f*$!0V8V#$?H?@q#Z#2B3z8_}%H4htR=@Ir~9$(bgNtTk@b`AGMN` zn}3v|?qMTO{;M)bN=S+30XPmjUHG@5fV%+S0yBi$d(U}T(X)X8Io#xJ7?tWYj WYxIPc!^nR6Cw4lJk)ZmLh<~CA!9+`Jhf`-_78%ciMaCCWZp+w$H0dxE z;jJHy?}=#OAu+&qtnia+%LUDX5fprk41`+Sra-nuC5OcCE;1g$*2_97U@2Rpj%~v< ze8g}scN{-DUm|1raY_VgGq&=!kl}u)TX;cv+rKlwD?KShbGonqgH)7+B|t=8=6?r7 zh8~TKx*O(+!D048skhcCtJ|=NglK*Wis~#LoyMbL7-u?+_Tv;;H`DJB^2{2kI;4DD zX-S?v#j8s7lu%9{GNM^9qkpk{dkDewyweb8bN4u5KJ9dC&%mPCs%x|XPoaNoMYV(XdtMTJ?GaQD5g zRXE7Jgy~QE67*6?@Sg*)}!X*w&mMv|$xAUFyCT|3o$jg@AP0ytik zt2G;|s4BsFTn_+_Q|5)Nrwn{y~LTT5+(Z(?VNUFOD1~(q(>0@WaORBNooz? zo-!ZPoH0cIIGaZEMS`Hb(Q+9smkVR!TsX%fNCVBb48>ECGOgVMcC_%Gv7Y;#62we} zaE2D@V%_1l3juu7adW4&TpmIK7RaEkjSb{r*YZAzhTl2Zg)@_MCVz;SK;tY{EP_Md zG94IO1yHL+9-^i|8n^OU7E5ClvtL9mQ#=PNCRiM~x}im^E-eDOcVXwGp-9BgDM3Sd z!6G^rJYgq*?$>O-v0*z-=QaxFqk@98S?|zj5G$09dlKq85*gg9T%uWn1a|BzvNkTH@e zvSW;!4v7lhELdeL2N{H9;(n55Syc}o(X=&(Q`}lEZOzE;KU$c{*C;r*pq9mD7KMS zJJ@;F+TYt}2Y)QHbZ5!P=HrJRl6AM(UvszEF57m>yr?O&>y&vl*P4Smk-;~JT#y?b zpack&i-fn+N{U(37A{{fa=rDQk(_Old%r?ddC^dYLYc=7>Ai^X5*>OqxAIkGzGb?e zkE}!fv30J1n9yv_9qSh)w1z`9LpQz1^C@ao!R)dL zS&rWxNPicHrR^zlHIq4{3MqpnN(3t$(W}v>!j_|ZVaISl`=23$Ug7IZ2L&e%TEsU& zkj&G}^E=;CnK-ZX+3$jfOQVLkGf(E#m^TOMgj3Ta+;StmmubXtv9nk#?n8qSU@_ z@`eypnUq&c*VnDor-xwS^b%a@=`87VBA7YMC99oFI+@27Pf2uY20Hq1Dw6g|Qh|2} zifpI~o!v}_;V*%6cm_Q~Ak=;LwNB$Av|8cI{K1KHLBaIb^sSsk8nn;_PO1W%8_F-;o%zqLV5{=p2S}!PDRJC3!Nh&Kd2BX6wUSwP8D4yo& zRT^g>viK_f)gkD61z?9FOlSeno(KtQP(pnh;&ZcL5#Ih8lJx%)79JbzzH&XZ9s zIq%QwNa$bl3PFA`*JSg@oRYcGNNxIebF*>DRH>N#k@sdHi0|Ln&qjN$hU_&ksuhb~ z=HI)$MFY-T7fe)GE1V%Aui3<(=Ji|m{amiiZ}YonZvOwhnY()6%w7G5W=>m^yA_fj z{Nq`@`hRm)jl0L=RI6k#U4M{t*%Ba;w~_GfjC>goWu6&F4}b{IeDLC+iQDlv_;(nl z!DJyYI2I$K6R*HT7U&9rQ89&@fIu{e6*clavn4&tMVoLV5^Xyh^4oOLn-x=lo~)k5 z2sLRVeJ}iO3efT3RfL-(p5p0o7U#|yR7_^I3Wfcgp|MBE!XZD3&;#kC6O$cr9e)dP z?fbfory{%1f2^J0Al7F8eeHcU%Z4f*z~PM7G`0|pH428K$b*$7(|2vNsA0TMbps>a zKx|31fq+e`8i=y6**t&Dsb>=FSvdtvVgqI3*jgmB0>I%Xb4a|GOD|jMG_j_PmtggP z{;Z%G!7Uvw#Bh+f^R}{uc~#g9 zF{%WtQ~j30h`bS4zE&B}hRnu{j@%W*GxqOhq$vMZMvBm9G{A%y2Q$@8W`A;NsX6^4 z9gk1rVVst+1kAQEN!J#{gdkb(Su)(JAw6ECN--xK_Igx} zv7ssWF#4YtOeW^5oE8FL72K*Z0J5X9*wLkY=Nc|mQMH!C+O=$o;j#^Ve{GZC^Ve~H z6lZBE@4d=q6C5!{+CmMV@_$GSf$FTOm(1h3+G(W*UxM0zMFPhv2_mKICLPbqT1ek= z>+XWfr5CBJitLVe?D428;rZD-Ii)Lq;Q8JVTCv}?wcpliKBE> z(>xObBlSdNUwo4x>c&#P0eLGi>Y~`>S0D|dcZwL@!fyhNp)y*KXslumura9*0}pnuO9aujEA?w#jJy14Lz zVzw2HMjrf^by4QAc|7C4C(UIBL168jGF6F;;x zghHx{WV%RZIe#z36A+A~%c_;0Kt!j=d<6FKClV7#!P$hR$a{gX-Ri`G1fN5uuhrqk zwHS!`gyyX^dMV6x(Ir8JdwVkbPZt^F4 z+Xt;ZrZ5@WxI$e>wXS&FehOglVz0JwY*7#y&(N~g#(#{o%Dmn7nZ)0}uHf&VAQ(Q_ zu?fyYFQX1+B2$R=>;46L3FVRJ2mLWdHDMtqHP_S{N0A^GbnBXd%Hk_E13$*No*??o z1ZALLS8#|MATTM7Z&}v?aBu-1)PQcc_IA_+HTi#)>|G>mlwk1$XtJW&{M>?-9_dM8 zviE4Ilz+V^oyx3EQSWLN=>|ot5&(}SVQB{l)iFP z3hhWP1wbJCz|a&XS8uS0S>g3qg1Ol+`GNSh+JCQlinLb8Oic#p@!*nlZ=*>RU1h== z_Jn2C#xG+0ird2U3Y~h}EV6S8B|KzgR1hda=fc5Fu916$`-AS8ARY2mR_Xdz5}-tL ztxas|Z#PGTn<{9u@a~WlJS}i3K+Il}jz;lxL!{`7RCXWVRZb%M!aHeK8^X!*VKy5} zZ+|AOnN(ha$<+HiW<~0=B3<%AhpjZcqG>W1`NrkS$o^aUL(3v4v<*EX;5OV;!R}~h zc7Ho-0;ue!XH10C{d=QP+dc46vA&haxMrQmAmP?%)P|?|0R#^_6)8$2$yzCnVjMsM z=sdAfFC%5vluhKMk>P9vtxBXbDe#C^dw+DVy=`decI(;G_TDzl!9f?a+<&h#uW&?R z&C`J-9D%+@oexFjd`SK~&xd6F`B2oK4`Y|yVE0sC8?DY`Fl(!*b&<14buJ>BBuS&9 zpF#7VsZq4yAVL+2<-5q(~}bZVGM#lupK>^*@0V-$xj@@ zxDS>~lZte?oXxsk?T87j4=k6H7=PAIrBh4piM6OB&l34uXmzlFOc_l!M4}7M!EBJUXB-Do9 zvH-D>@czHo3{cZcPYIiLNL;ll4S{_fDR|b1qR)2w!Q<2u&sg8I{yK)u?SHT%WM?3#&Oj0!z|2`obUFZRB0o0E|2ft1|IgAK*r0+U zP@*%CG>28`4XdX&9SJq$QYY&hlYA}NndS9lXO;umsaEVWI4cO@ffWptXOgo4n3dXo z*E_k7`iS9w$S_t*uBAWKIe)DXAZqk;c{uV7yuKvySu9buj1gt87Wbou{R<01&X09Ce*>*=&$v0C=IyXAs)ES-O3G2r7V{z*ze_KvngL!Cn}7-Ky0#s!)c64ox&3m zezHN!D~a&4y|>SZa(__w5jyp2WMwUu;;%_uEL6cdcX5k&JA5A*=a|1Gvy<4B`&1(D zro89MB)~y9`X|w~cY;FsM59@b55b%h2+1`F8Rl>DcFiH;v zx^>jjCR6YIl$Fi4uaPCyZu%3Y(5!Nt(?l$sro1&l|6Q%g>3?PvovEwTDF=bNGP)FV zOIRfXuSc~?8BeyZ6trCk+6k#o43%<*_4qB^36xTash5M^iZ#5X+4hNv96WoBPNxO# zKUBlk^MjDz3_)+mloMD?WwD`#H6qxr(!-@itn+7E%F z>4`)yWz|Q(%zp#m@{xJa(3=`+UPLWHSNf=q!Jtln`5 zN_itUydI9X0m~}Iq=~8%d+-x`@CPg+sZ8Z^NI(@^0{fF}EsC$O(_bM*WNH0uws-dr zTCM$EUk^k#gja!*f+NIu3(RP(84k$)$kRZElb%G<+zm>;Eg>-lrsjT7@&kHje^;VyZXbw??Y(_* zv47XzmXBK7vJeEJh^yV!Gg<7Q{Y;euToiG+v;A~mmJ}v#Q2=dSva0><_O2|wx4pZ4 zaG*Yh$&}Ui&_EzRiJB<+%E8XT!G24P;0e=yZmp~{N#RBtH$dm~^zuO8*Jl0#yCj7yr zESOhxMApA}DUjw%4027-^B96?bgq?==eygw?QCleOheH$yPma&D;u2mEH4Z2786dQ zr_eMWsa+3;3E{-qBIVPWXRU6ZjZ+@6vnepJ{j_p0yiCtOrJI`<{4`?v`h;A6oPV)X z5NpyAJAuEmGS05Xv!CqaOgKU#Wlab%HL`E(*}etlFq#-h9xH!_jgD_6Z85nsAbX64 z+q5tg&T5QL4j*Um=UtZnHeI}khTnfi*=PYlk0@;Fs6+kpERQmQ72U#U!XROJl#YV} z|MY;ymaD)Ew@7u8PRbf2lK)najy1CoX^GEIy`0o+;=+Jv zIAinDaI${gLmL@3vyov%JZBXd7F~EW4!KA{RErIpx!ADehN*yfA7w5stbbmp=p8O` zVUs3rLpy{tm{>fWU&nWd4$IS_6i36x)hxrswSj|PXx3O{is%$3r3@!dMr6+;f-r&A z1||}mI+O@f@RD$drT%~-Qxh^4_Q$w2l`4m&5w4iN;y4iNSb_+)5>0JMImKyhKD&mVae<6P(0TpU9$~{-fin=3YYZuSiq8Y3JVJI{B z3?iINuZMyan&ULPmaO&6EJm|1>+2*n*$a2=M3owkL~h22Bxu+P78B6nfL_hwQ-YU( zvSx8{0fYw#r0`}F9R}#*-5?v8XoqT+K)bG1piTgE;Js9O!O}J7Ne|rw^0TqTP8(b1 zyEoLvkssL!kX?1PQ<`^GTyoV}vn%PGIbND=zfO43D~+tTu2%a6wbf)?FLFlQpyXr* zLKvud(dyeX(KmOgBd<=6H{{q=PxblRpz z*-m&epfQ=S%MqJT*ksa4Cfp91Gq~L9TrVtotlU}m032TCKJ>G#Xb!s#1D8CKdyL=% zLE#1_#4H?=P}m1~fqS`nkQe407RPu|dyE(EF<#JPT*f{K?^Qg-3-=UPPbf)Qt=OgI z2k7EDC=bvL)jEtegVUj~06(2~HtT|8zPp@{OS6VJ0M#L|i2xxu0Lq z!)pS-mgWi}n+{TGG5dK=4#OfXM%3Yj%@7~h=5`J3r9&R&1&y;B}m8;S|nHx|Ww4oLqf%}w)glwvUUA5k_H zNlEQuuaVrQv#EbuAS&|NtJ$zvz+ld5#-JN6mmbU-Dvm5*D_SeCnKCF#Hj>F zMdYWE05~{Mfr3t~9d7XH=mQ*eLvF&p9Q(@PL(PKf$Wd?94Eu69Xf_9R_iLuZOyR&E z)JU_doIiv7I4;q*-;#Mw=~jfMCDs*X^Yzz1ry>((fw1S={7Bi9ie%@`5^i0O{ID6U9q zoL{Zjq>g`j!%3GoMU7eW-9jTEG|r~JB+;1`*d?u&=9@r-y`*E3=|cD$>*F-4$@LDR zolo zZJHqV-LF|KIvmk6=zCHW(1Hdlz@#Ny6e3?Ibk~2fzY!k`m1=6SED5W}lI1$9mUwh! z-z$xVp(Hjt-2M3Zf9KQWi|an5BbMlUfI&xf&43V1hPS*+U8J~+-=!SaMjFQ`~w(9c@WR3lL%AZ z{}?K8ZvcRa`AB3|rHZ4wqYc>KHz{OHcZ7d>&dIEWV-1mxbm~=?bgY@pV5`(+=TUS4 zwyE>W#Ra55;6}v&NIr$=cto?UMQa0(5h=`bE|V0bqC9M2Xp{GvoPy~?6V zyn!oRIj)Ky@#HIzT6hz}VBHtXbQI5I(Wn@uGob*3L`voJ?+cabsNCH2%c=K$ab8|H zHHbieXz>M%X0EhtaxFiQPNHko$Z}xx~n!LgM7(qKP@B$xNl?x z6dHj43xT@~t1O`6_`~zwMW;^ zX6ZcN=z1Br%_G9U%$w+KjM8!gXmhIM3BK`Ml>hDd|NSp{wITDsPiAE-^e-~8IsWkV z3QM-q1zFV9&lT)8g~Z9}Vue=5%5$H8i582gYBs$dfZ^zk-0kU0-e@83_ZQe!dw9Xc zZ{PQWO4aYr5JHz_l744MzY~^-Gpf->uSa^yWROo+8PIu_30RrC$a`jycia}qKP2(h zm;VWw2jm&MmKsuD)6R#H`;3zNYAv-Szvg3_?4!=7)LJKt3TM_%R-4!DoZR4l+1=+t zLeGVDnyNd#ZXFo1(Mn}i|C(_gNW@`MLTpTlCrKz&5}>1S>1C)fx%=ng1sa;ADB67q zUmk#q(! z&$IMvk;mQ!KE%WGSMJHOzF)t-4G|=(8BOv686=cyv=7i?y9_ixNE9@{XWF7_X}cUb zao6c%@6@Y)kFwhSgl-tol7L85%;jJ zk49c2Zte#HsTw5k38Y zJ&{X7L}_PyoEpDys*o~9N@NN3o2&p}10oSQrb}l?9>j0nQTZyYVYMEC?ak)3ilDH0 zEno5npWhSiIHIsUZ-=$qJ$lUIohieRnyg^*Fi?v7Y6UVtFX2ud#&p zET{=fg0zK8I;D{vLILXPg4G$+L%gs>bO(tiq;*wg-{=f~LaC8=W?-N44Inl|AWaq}nPA)iu`+P^q-P=Ny!ks10`t zHCny0Cp8o??>&=kiX?yHtg_rTad^~?P&kH~WZt>w|Mt5@OJL;!lVz5#B8Soy+aykC zBtD)ZUkdQN&*;%ot*&tJR*H3-ps+T9Xhq9czO9;k84p(jQ3R2uX1zC?U|_4Ni}C2# z{$J4dCv_gRI3UnG6ZrP`2B>2>@^(>mA?#PKr(zG&APWCoX5fFYbopMOODZqlo@Q08 zxrDro;)dlm9I0{BmcV)^;*W?+YKH(7mUpl4JM9**wt@oPvHZZR96MEqq8Gx`y*@&P z-FKS0wp2JreZWrE;fqDuv?fY4$tIr7{Yo8v;2uTW`p6{r_UOHp>z%kS?OrrQTBdee zYaLr}z=E?z*zkWc$)u<@^Q`#Oh+ThZjzdwhVYna_MQ`Dg0N8CI z6J82*+r+0WInEFBV?REio0?@F~A}pSc=*@6CS_#Q04sHwX!JybOA^}nSoW?(w%b)3V zH2bMjQv!eRf~pP5(%&NIPy!NG1`4{D0?&xI0N66wil)P4mI*v@G_tB&Cs$VxG=_n9 zpy$9MiTWj;Vpc~Z;j&O)$}D>YaoTtkuLJ}Gcm+LavjO~f4Y467@ZS;ST{@I;0XZs_ zJ00VsImz zAHsb=z`}!M0QcoLX{eLhAwrRHgitv6{Vf2ioTC0ay?E! zOL>wr;}GvT$6vc8#Um3U;FD*M|U``&dNKbaqoqle$$rhn#D({$ zk*-?A(Zxw6rabMGbna+)9R=hLq#Y4N{5VD`9_iR>b{zY{;lgl~840FRYf2Tzr6qr& zX<*svpeJzvPI7)dVR$aENlsEqFI>+fA|VLLUkTCOWXnxQo;Yb^fwID0?~Gs3KHVa_{Wegw+xW<8 ze8J&m+W4Zc1pd9{G7-@(_Uh|e){KL0 zK^Tv%6w}v<0|?1VP}JP>oGO33h|^jyYqRxE0Upv3YFudKP38+NlZ+dm&G3}MqfBMp zS}r4H_tPlAUPMiSCT1rQ@j6~+fK0Tt38q{Rmp#Rr2dRB-M*Hv|j&|Lc+N1E`9vt4& zgWBiFvClKHS%0LKXtbEAK8ZlrqN4Ntse-et_#&f)n2<2e7rA!lf02KK8Fx0Br1`7# zYn=6`6Bh>*g~2uDybn2qf5|l8%Ghfi2S2mr@_6iF%(Y}b#JtSGANfbxQC-%3WOGnn z|44bs<|N|uM+)9d^GtM*B+0VYct*Dg<%|kH$ebH~Fn4Z|5HmZx?;!a5P5IICXlod@ zfbx9JCKSQ^Zg3L%!H<8I2bHdC#crTS4aTTg$6Mm+Y1IuVt<7b{cQs26#O-G$eD>EF zDu9PFtcqc=z<W@A$&d`6Y{+bSKw}blo+d-l8 ziu}wVb=}X*hX}-44XT;;pBYd5N}Y-;)Az|q^~>VA93DF&Y`x1$bgIl@!AbOq#f`*2 zAN>-5fwL@(hKWg*q+=aV?bI-d-zhMKOoG8#m_i1O_aa$LiJ2Q|KVGLcQoZH@E$x>H zU}<%d=Y{g8OE!NuRWpk7s!Qcp7B0efr6@2CzeM$ zPMNx$Y`?MlH{MwFd0%l;>6JIn*7=x#V)f?)LDov77vvNY{%8a*p))XaU_jYe-vpZ= ze<@b}wU8N0hS=__m2mC1)DhbUd%HLtkv3U#-H@Ux(FYeJ7?|flp zhxj{9*=ui`L<`L(5;_%G>`3}G9l7&u9j3msU6%n7I&z$QC4?FoGPtZ^f&U>5fi_S@GJ8+ilDE5Ps zyMsGVa;u#2(mYlVUU?;8^&iE!p%mq@{TDw^u_J%=Zi|;$<)M!778wXi+o5)g_|tao zc*bcAh5}R%C1}JfH$I9bqa^_X9GnypL7YCo_=KwCVoeQU#cwgFE2Q1Z&o6?F4fv?P zDn($=gSX}W8X|#f@m_bnZ9tm0c1AV35FR;3IsP}=wo}_Sy6cNOLd$T!{26%_6yUo) zZ*hO9)~6lx54%|gvsA@Kf*lqkam#wnrX+`6x_FH?ejc($xLlSs(Vpa9ZGFq7cQQgt zw=V|?*&m1XO0NL)f9>#TyS2rkUB`k&JBgX<^v?_-!riGs+w-jmU$3`#GrFiuzcKu{ zaT24UTzE$UaRU7+7p0QcZ4`p~PJB%N6W`0|`veir>U0gFt*b)~pxA`Za#l5uz4UMIQa++EZ3 zn(ukevIh)dcf0LX`&L;#hxN{x=vfI*AktjAs4eRyE$48h>EuAJ>H^{w>; z;))jYadfl6Hm0*FxV5K~`7Fz$X};kH;g)>N#Zy1<`u4{M@Y8V=#&-0$cH6H2FX)f@ z-GTao(3|(g+xoMAPDu;F?f*`Sx>A3t9zBZt&M3n0(e@K+eI0iBQ?`r5A8-1zf$g5KGwqxkBULxWn{?=l!5F&p-YH4l@ zNgxggpplwK2hpgU=!{;8vWPs2R|F!cMg~*7Tw+O)%2~kB%0#p3pzV-cIZHG~@Xn6H z{Xrdl2NjNxx}jsS3JM+;T?LZ0B4uCJf{It$UFB*E1}0w@TSU$Mnu$0aH;TDgX2 zm<`d3yr=Pfh{!qMea$EI*S!BT>-70-s053jm=t+7x;{3r=$Yn`U`{id#_-yD0;#GpVifX5duYuTrNlqOLpl+krF)9hzmTs{J z`5%COdOY;@n2WBbEX@{&qUzsmHXA=PU*d-$?MkZYv^?oquCiO#dL6jJu5m$dp7@=l zCa;WFvZ^NX0UitfN`ZeKP;uJi$G(IlC7~TSFZ~*NRr`C7I`S0?PtrP!^lXR|i4r*N z#7+j=nv0qOt4oo{s_<=oOzcW$+KetzPawEtskDY`fFGnc=06Kv8)%Q@(4Qc1pBOIV#&K#q~s%A^d}FVpp*3WrXU_cT?z z=fxg=#km;~*5GrrvR>!!J1H_cGZS;>1*o@@={bj%h*H&y#lx|l=r;*9l1 z^?BTTseAtgyzz22y-Kgy+Z+&WqygZybWfRNa5LpC_IZEIvzgcO+1n{^vA>kF$rzpw zd5fKnc#FM|w~u8tCvfxkF>n8?#Xdyh-Vi;U-$*x|VkZ8M@a|Clx`G;CLFmqrESC(y zlXX7iEn^%%%{=Y9@-}bocR{5SB*O+eVd0X8{ z)_gL_f^mOI^0;V*W2&tEO1644q8O&AaMe82M|MPPGqplx{26JJEJ6`3!i z3k$G}%4cxk43~h&h5?4V&1M4DzzKMIe3l3Yg)o0nzs*Ho?VK^ck8wyh_gXFXqi8lG z7zHxISP3!93}rqMBnNM1CQDqx`%qLgusg|umalv@TE5Y`9YW?#X4j(>Z|dHMES+U( ze#1?+C2sAJ$n`!1Rd$gq!2gvRz$nMbuU>FJgC$Jhf*a_c@uom2m>dl!WR9-cIJTFYh9 z%(miR^E6YbIhK(h@yQUEvKaV?C(`ZyZQg%*n=kpc6p?!q&+m8z?3Qe~#Qh(^8dsEx zuJuR+RQR3I>=B+4rQ+#~ng0+*tM5yf1prX^*z$a~7-~ny5U6a)UTcaH@mAm|qKm{f z;%}$oJQ48&@?_aYvsx|`=)!^9Lr^0AZiHC6zFNa#VhaA(LNvPNGu-2e+~KJAW`%#^ zO!j^GRL8=ieqJ<~h`0?LaF9|jORvNzZ8gkHH87M740QuT)xgj+@WCzA=G&`unl6%f z^v5jw&P!%)h~Ur!mZ-(f1aABgP)Z3&ID=0fXY51dNI0;jWHTWD+)k^ty|cT$>&t+F zh@XsLDLlatxpwQh1YN)t+Y$%`qCS5)hIY;v=9SIF7qHE6aDne;I=xoQwo<;Sa}R9g z=_H;N`Riyp8pr%`%bSfYCkfFREyu3b^l;bdAw(%|@3ydi1P2>5UwCvuJRsx(dD8B* zo+llpAK7U?`I-aVR><4WpZCBs0&JhX-%4DfJy62&{|Z+kXA(d>LwqIVVDo?4+pXV{ zZhN~G>>j|??m;VPx9}4F1np;{z_aaEuoLh6mUMUGonUtd@4}y;{cOMWzs^c8`4?RI zP;a~S9FVXdI!ehsqqJAEKd|GBTAj$q0FkR7*Fpnzk6Ec#&GaNQgTx;-`G1{`82l-Q%q2NXWCjRTYV!ZW zU_5ccB4P^?g-+{v)KN1k=m9$2wVyvPI@x(~0jvUiB%#jGD4_z@!aDW=*o2N)Lz1Y< zd~GCE8`)*V{Y)>=eBhDXSWVe+j9#5u@3|9mGkQL%&N_=0X16Y4}M%BZ!z{9id zpwXMxI=AW$Rre%|7$H8)>8lyazbtYgxtcbvboFhJVXx9@G#=mFzRh{T6gywf!IODP z$1qRBnckPfnc|8;7?2DunJfK$(VSjNBqx^eFS*)+))T?tOY-Q}6SbkSa+iZ=I=;@`ODXEzQnaCySz4PY}GtIdz`2 zSzaPuqh|4Z;kf;JT4OCzHX;)uE_@qFh&XuEV$?AbB?wxPxhZt_@%TKtKtJIWbmikX zD&r9e7_;shPuEg9II|EUDn#rJLL*t4QgC@jH)!z(xFCOV(Oc=ac(RJeR>B~12p>wh zOS)0R@===zMT(Ck27AcfT2`>bsiVnFDrT^)A9(s&nPpV7Wab#z&(+%kK$g-gZwdpw z$oR)8Ka`vIbOAb>6VOfcMpjU`PW@%2OA!O~cFNl@iIGw=Nq#I2h_kIun$qV58}LX- zE0c`%6_bC1IVPELNF|_z5^mGnLvP`#y74;Z18|w?Gojj+;UW$1BqkaUw2!x>vAx|^ z7kkpidx)uGHNvFEK?-NG~08#6!`NQ_S znwqhArRnl*!+SNQNMJD8>W*Zh^NIIrYQ|^K^1h9|0qgsw9jlNJxV*~{Mrv1}c5VBh z4wZjg^os>Q4=;ij3z204Ev!9=l8~f7v{3eRwp?r%a8geqg?gm2U5;Y@1%_q-e9swN zL3$+1VZ0sm`2Y*&F&F-?Yz|kJg1+H}Hu{>}H93Y+AH`W(%H{Pcn@w=g=~h}U=drs9 z;kY82`7L_91_rKaTlF=yG3o{y4U#L|LAtE=H5*)d_fK zNR>^QLs)y610h1#03wULo^o?Q5mzlitF(s4kZe#U;cCmcQTQlUkf?yBLuN3ah5;e^ zu)J?Q0sOXH;@=(l7ZtZ}r+|Z07{IuOmyMe$dF>vkX1roZZY-C{%qvj4YrY~`Wt)F@ z+?Lw6`!r&%7VM>~1uLZ%tfX2nb!s6WN`@jKT$0U4Z-Xm|8azuq_I65aUZp2ZUb>!+ ze>(AAPf342RSPwnjVplcsFXb=Q84c&G|rf1tVsQi8<}gz-=cmCC-wv?3SNV7=$wq^ zV<$A^rlU`K$5NXSIZylkJl#K#_V$0Dr3iSUwnItaq1aPc+C|5(Sp@XKI!7}XGR+O2 z76R_uDc{$}wlYrXlx+MfwgyIuG{|WIkyc5F`)ka|Cl3z(h5nD3xdpt%tnUVUZyQiF#lu+1Kej} zt5&vK`@4dAskmymv0RRZ%cTc~!4~^4@#2UPON$(YKNMWk%7-IW*U;X3_N*n7bU1n( zc(PoE{%z=ps?`-D^2LD!3rl~cBaFF{WbRgYUW73!eD)noFPYD7V((SUN2r#@ixzv; z;(%@>hgH5jvy@TrQ9-cha2;CWw7^gt@##lhCs!Ni_oF*e$r0R zhiIhxc4MBZ+oA=M6?iq}H@S8kF_#kZBt0P zWVFg?BI+}^&1%ZLd*Ii|EVUE+jD;-Jv5!({38^p4$c*twk8w!6VU$1MqZoT;ImF~K z*#mMl=JsmrskMNBta|8sbz^^{#Tb7cTkf*+m<~_R{RZZ;Omd}}5JN=Vt@?Kk(|G7T z1Di1sN#0DoK2yFytjtk84Y7HEXEhE4fPvJ5Mbs1ow1`ZjMF3{1;-(U+?=75jFVf}=;w69cMmv5-Aa)l* zL-A#fcQL8+c&4r+E-bA^tV3^^wpQmy)OAII6v~X}3INB)55mDt2u{?njXA39?HdeP z)-m^*H̯R+;OHLw8K1}Ye^{jiKtet?QorwX=V9-9!vEevqzzgYl zS4q#}p=HURWXeq$p2(;|o zr1Q>AxGHun!8dheb<9RcrS;|VhY^UrWdNRFbdM5&yIJ^^9&lL5Ji7MFKJK`cMtZS?l+rr*mmCvL}VJpzkK6?u9L4@`7{;rz~qIIGVr?z zIR(S`VeCch#`LlC18x)$fLlmMgoq_9W2MS=8}cEjSnX5@OWsxyfhAS0m#)@NWv92P zPr3E@u_$2NASF3$v${kNj;2_+qe6Z+2vKab*keq05x$Ks)xmLNZ<1~qVE;>>eHCb2iP!)(Cig$Bb3Cckxf8?zP?ZIs6tXo) zEIS9FgK<;q)+E9xd8j_m!ns%!g96v2u)N7E_wIj-!jH0N-nHZZk`yog`Sis>lI(s&W{w$?slhP8uoW;JN%R?r`A0p5ZYXRR{iE4z5I(SW8=0LSYVSa^f=z#v zYvhw)TB2q%_^!65&8X;%&aH1{dC`DdLeL!OH00<;jGfYQ<$xbW-jg;fnokO!4e-IX ziZ%{ilRb$5yvFFiMG|H4sAIwbm3NCqKIjgF6U!j*)V~_iC*-K)=M6@#N_targ+WUFUsi8Azl!~0Ww9-^##E$o=SE6r`SCW5pZf&-n zZ0^>OO{9qS$9#EP7V%kDrIu}zz+@uH7?!S3O*eJfKc(1;Wmm$E6k;+W#GqiP#^=vD za&e`vWDJxO90?;c4b*dRWglk~qegPPb0zKN^1~M5xETRqg~i`WuE!rXjyX}%kWfHn?u~!a z88Bu0BO22P8n$ef&v3yuyhgTMHnOdBA)-E&qL6AmOX-#^rFctxSIU15{6@DOU?Rn= zK{H2YTE0i_=rUICwA&q-TJqG~+f{UA+q7UlTZ81N9YA!tx`M;VAcQQqs(+7fB>tAq z3Ty3T^cO3{3dMRxbGg)r5b3kB!K3ntJVR;0;&_ z3Hq$n%D+QgfSB!&W#Z+Z&dWREW%t#eHLrehUVT>o>Zh7lhdM#k%@8t~i%u%ttvz;H zFPJ&FABI54G5j;=@wS6)dj@0n%)WOC_u39@?kV2fwQo*yxVg^~+c@SNdOIzkP(ol; z$*eP5u>=FuQ{#VQpY55^r$u_4B9i7~Tp`&w-cJF=7&Uh~nJn@7yBXHMwY_a#qpcX` zv>rmpZWTUY&}gsd7anU=d|LlL%Y@jvu3bwl|L)k1R6S0h32W$II9)gUD~-7xq;*oKPVmAG#rGkKN*9D}%P$7jK#CvEj? znn%;?aZH&I+dE9KIYkamKBsX+@6ym}GdR`_+c0PK@g3!LR6Ahetj4FKc$%kIX`Fq? z;;ZzRVzz&(&PTjtw5?~eA{)kGbRB0pqxeyjM-DUfdPGkO^yVIr-2jZ0UN-PZgeil8 zuZ`YQ4Gb7t@&mNwL6Ys$)Y}o;$Lt{!;!SheW|dU3>vXJzRI!RBKs2RzW4*!^`wFIv z&-rJ0xc!a-T*d8s%wwtN+xmJu)e$nvs} zDoMIPdA@j#o2W%l83ysYqhN8u7j)Fhqeez%xu?zLUNoP z(XxL+8ze__1Okc7X+}chCUK9md%-=&WJPpoxZk^@pf1gQpuq5(Z;&Gr(~0y#DMcy^V&nfN zy4nK=**^<$?fa}pQY#JRZpg?;H{=;>x&CVQstT)SqQ@O~b)F@;M&hg>71GDPP-kGe$=kxzb;VXgAh3_q-;Fu&Q94@j_jXgUji+{y%gJgVNK zUlO#bTAf1Ws8K@hww?h|6Z=6QKK_bX-!CL&vef`}llV^y7uWg*m^6Mh{ndWaAQjq~ z4bw5j!YPLed4V+6XVLj#bRkY?GQ593hYuUxV!FVi|8g-*;!!b<^}%lqFLXi;@?TT* zxVim0o(;dpqZ7$h=23PX=RwBilEVt3RmXt4f|K+2sW~denvI@+pPm;NTk0Kjf9`yr zZY}2HbQpUDYoRS$I1Bw8Big0uiQNotem; z166~B+vgjP@N|2EYv;*)mQM30YWyElKz4QT_quBD z;D`-`y3uC?l6z%HCs@UU)HDJ^qjgL)8J)6jbjnWYj?O@?0XaB`^*2O7K8!Jw#+1KA z+>ziS)TrQ#t6z~t7;3!TgtMEK0#X3J-VeQ^PP$jXAyS#g@5n>W`y+n><^HlQ{Ut@5 zg+?&2z!)Tyngi`=&j|AJ=segK$XHH>Q*h@gC|P(G116sG90`eMOk8ga=g ztXAx=DX)eFeaTnsDzkq)1Xbf@@P;l3iX9tvqFAlim$>3ysALE`+e&)neV%&K8?3;L zGD98cvK+Jj#U{LJy6nTFO0oKktliAtLG<+@$n=sTz3ee&MjA)}#nEWjfBy3r& z*dXSAjo4w%Q^AD1DEV!%06u=4Lh9pbp51(alL~y6LeP~gg7cRPb~I+M3ibzk8MD7e z7c16Z@Y}0&IzoRJ^%pm?Buu(h+*-e7{C zi?a{W@OyL}zmFzyup!RM(QLA@Vy{YmORDHEw#+YGU4OA9FKa98FScrK>!Jd*la`V& zo}e(v(oy_+Hv4{Nd>t#dkI=LBLzE|L3Y^BU&Z-57i;sVGYh8u-(5-9{3-()R%QAga zDLCx=lGhXX)#BdJMLj)@uY&cEs4&-bl3(25%v;>Q7KeLCkTT1vK@+^Pbt|Wjnzgz| z%jJi%(ul>Y88kbK7m3}``kS*^uE$B1u!iY6Rvj086Wn(^oK1)6ILQ4ZY_#Ah++$DF~*R5$T<|7f;GW=RYD*N4~Hst zMCmqu&$ccv&%&3d;it~j7?+@RdT^^lWzCVBYblE=(FHcS{|2*km{_*DByUQ2h z<*PTR;SqmPvfneABKremC>hrt5!-j_%C>8DW!q9$2HpIOplVJ;d#hz4+NE+%`fILC zqfq3rn zpc8)_Al&EhPE|om-vUidwFII-B4gXO8QTJWqWVa%tJVy>%*ltQ}bs+?#nKS5&|m`WC2=5KwQrBiQ% zZTQRpARIt!jbZVDD4~1iTC&74(fe_+NX~z5rbAvs+Ys9ei%Ax?E{LM>Yk1H4{jNp( z2$Y6}Rb^!jt7_|3@osC+S9ri0k+)D)jf94Ov0REYV6f0bSm-@i=oG})EACq4B`)&f z&P5IqOm&eL_bhTiDq6G1QAm};)MkZLgqE<(QOj7j%mJPflb`ceCwrbKmMz=d^b3EU zoM#shyRmh79p{poL!Ml(*t%TcK*6TG&o-iTq4$+LFIagI+*?lI*`S{^aa8#^RGYD) z`}(KLtgV)rRAy;kmczVmr#DbPQdw>O-9mHMv*j(-0@;j4+kzmd6}v@IqvQh78FCVW zSr%I_SLs*eOk^&P1#q?WsTj61;Yfe-W}45;2kfDgVByf+qJA(9*#iZS8_?oVF%ynp zO@sIp5NI(6l5yqt@)GR#OeixZMho^wDFW^X6GdqUBQP`qAJ7?0I7xpVi;|6pu_M22 z6ecY4?xI`6Bp!Z0dKoq<>7O6?q8BR|q1;s*Ls~D{v)J>Q=^g*Qw0`5zd?9~B^QRn| zFM4RshMg}Hi9HejKg7NJew#?PF#7*I1t!@Vh|5^_3vR4!OV*^j9Xsu~dv>?tu@Mw9 zV+3kIkrjz2IDeh<0^g&YSNa~|{MM=}C?q+Tnf;yJnMp)Z)OFRWbzc@N-AVpNCK{;p zKt4DnWK88J^1)#{^2vhU$lHI!T4&@JX%K(#LHv>r;%##B%gm|~er>tb^Fx}aVtbnq z_5PkF?;;>mRkB8I&OE4Q&^lbz3AGaVnb6p`1OGF%<`7XSNfl}x|6}}dQLWZ%NAOSM z4F4YB-xpZaxwt!geN569YKl5nds2GEuJg zb*T3Z)!w{5_KBbOU6uPD%6&)TQZIznY4icPa}rpZrdy|LHs@zqV3CRKd@eo|EwLTt znz!dn*YskO6788h>Ab}ODzO7{Wj%{4!}qkSAL?QQ2Yk7QmiEMN0Cw!nC&Q%<7|?At ztCN7H%8gY8jVY^tX+3{4E39izQ!SO;AUe6UAp9i{a?2`?+4Qi+ zwX`=sT$5Y6EdqWRlZv`11lOA%^OLr^=K<@Jp1Zgh?^qcz(4CbLFJNT^o{|hhwP!p! z(zW^@lWx3X0b`TVydDqUiekkBn>WP|;RQ5t`QwxRydfG>wEXVJ+rKDpj)5xvX=bVO zXpTY^wV<|s`I9-lSOGnglD&cf?~@h2eF2k`lfEzkPm{#H$N_JYjlYHouOHsBZZ54q zJ(C>3ECOWblS;r*HhV@}l}9_!hU}Q$$d_Og`%@p006>V^-+5(j-&+#2s4GwdDB8>HfobA?y^{sOCIT>#lP$p~AJdFX zb>dn>f&#jT+w<+O#Ee#n1Qx>r$!ifA>f+_ElXJm&0YQ`P!Oj8Ilf1%)e}Q*(@WAoj zELDXB5t?JM5?yG@!M4T;7A{SYnA7zL+bANA;8B2$4V%0*UWd4#%ajPVuvyTZ6iwh$ z1Y@)x;=3VbZXMsj9*;vH#Qi%fC6N^xG;}%q_f?$)R#;q9ANz1mvF7~U=9H*E4v4DW znd9J&BbBhJ`P`b~Sk-|Ge;wb6MRVA4Bat_==bCGsy0}jLz#Ip}y^}uFNdzmEf1ndb zd_#8+EpJ9WR!vuCm|L(s)JUQ@)qwNUt<%)NZ3gpJ_N&kTJ>qXnr=_JvvG3TeZ=L!^ z38&~J{Dw+83zr$UtmA)}t3m9RRVPu)s*#{&)v6?Ee_2O$lGZG0og&Tl zMiPH4T8x7eJVmk9#J9rSN`d2eBZ+2IOEGzpe`qBwouuri-GXs@KDhK~k#%~I*0f01 zjhMO%hib^3Tc^>X7PpvqI?(%m%M951MlCaF>44-5{Yy(`EfrcoNt#=3(OhV7RMWwK zHeP9(Pa@qq^`YW8f4bBAbS%C-s@)x>qT1L|iMB@rl71I85s z?1=*fdCQ$2{NfgKg#5!FJ9lJm!DZv;osyLmFmXgbHB+JN61b9Guv07Ltf*N=ms=N< zTfU$O4P9VuRyuhGoD35{ZOOD!nO7ncnylbX&)=Qf2{l(OZMzDT-{~Bun$MZO_3i-I zyHe2t>xEARagj*G#exCgt~3G|c#AC*-CCQ28e5KPrgMzw-&tNuW6}#kx0eo(q)s9$ zHD<_sG1BTFf7+p{0xKFElc&x#?3<5Qx zfRN+B&E|kgchnmd^r2G0@f%eq4>u)xfM%+)^n*Z|pu8FE*c3Zad{6OHWl3X-m85$P z%m=WgOjRb-u@?zNQtRWxn~x6)NPuhyB<)DH*um0XC5jgXL9|qC<>SNg#|I(&=94p& z+eiWae=7FfTmlYqKQKm6=8VE|9_?^Sxy(Ete;R#ZbjE{_2#xPZWU2p959ON2Z$7@^ zcXmTkn+gFeK+V~$Ie?K1I9qN76RFCr~wXkv8+T{M84%9bl zrjzK<%0UBv9KTWfs%E^=U#q67_05kTk2T{^-+cUd3}w0*_M<~fg~!*biC9LhR0?bZ zFSWj@^{tN|k2kM%UDe4L7~LB>y?CB$5lMwkUiIO^y^~V=B84Gof#jk*ezj1yq@o=KTqfsrFl#bf zC>MUG?A8>RkqS!%S$ilGaB2CKzJpRje=C)byQ=pK(!dly35ZC!q&Cc9%CA&>w#`h~ zCkT;{UEq>2217r{m6sGsp%4Z5xo9~;WzrtuHwuAGx+7Nzfx;|Pto_iOwX~2hxi{|)Z=B3a@+D^Y)`3_6GWv`e)AK1dYYN(3Sn*dfDE`#HZ>t5iZ`9?dn~e}oh~ zs-a>59KQ%Sj-p~eEFjb@E#XgYA+KN~co^6rz$E8(6kGW#fa_l+eCme?nBf9Ezm#N7 z9kBg%m5&c;WB{and{8`}2Cg#R(d_toc4sJpTTCe6i4?&-TBHEM41r@qVcw9ToN;${ z{}T;X$|_nYGkXQ#|H*e5GWHfhf56U{4F`vH31j`?(hr}MG=k+b)KkR{-2!{Z?8-r7 zI~WB>=2p?dc#tI0h%E{hl9Az@G2vMUIqQLh2ym#yQ_fvWYBMKfK%E^bg~ENBa4B(7Fi&@rx^$|4`fl4^iXDef5}fS{7Y2v zS|utw3o40p{Tmy4#T$B6ZV1ki+ze|N8(}|RXhIC2Xh7$L9px;2F$ydIS`Qh|qMV={ zAmM^H#|l}vXlQ0C?E_*)AGEV2-LfGxv(Hu{A8WKhdmdNs`0+i_pN-_v_fkzK(<7^P zA`R?WzDKO-sa5T#_F%p}bPoB1)ek7A5c+Cg zRn4;8ps(tEidsKPCgtj{_Edkxs=rbXsT4Dzbe<2&$rzZ1b&;SRe*vtm)SXwi7#pSF zSMt{)0E1o{yL|u{aPP zfKbV0_-2}+9{D=d9v7NU)J4LSHacFp6RIr0*a+1}oPFZ)fl#Wg{J_BERLRpT87gAq zHJG??`YdEC8pR;~3Bj-?ANbtX$(qq@GCRuYtt7MNi>=`TcOK+ow46h5a#>J_a*vP% zAjj}!{bWJ!e>r&%oM4nyj<66#Kv{XT-hxYrCH-VnRF)SpS7`k^3rTjm&vlQ8UxKEk z{vIuqx?Ki!x(yD0fA|G$-2b(u>E?Y)lUjkN60BYo zv;FtrRiRr$NfMnDK%7cI9PR~Tahm--bCL|tqRax2A%V3;E5}U@0AWC$zidDf{`C#@ zw1Zsw1gnp_*CNm&dXDg(cuXx#bIBZzGm}02?rV8<5;S#kkU#20ibsB-xy2o*uS3v} zx}tL=M^={K)PG!pgGTyU3v?2Zc`n|B(+9Iu^%I)sJj##fsvvw;CE^T)w{T!nvXTUq zI??8WZ0P6e4wG>_y0#*7Zi6z{GcpDuI3N+BcTAQ)XuXWGMaJt^L6z_9m%-5VNnx9m zH+R8eooR!X@t*M`vGC=g8Ii5cu38-81!U}lc9L2psegLjWk2NtUUMyRnLF4pHv$r! zgkJ_fpMzht2mC@^a!-UNAXFcM9da7d)<-(=Nm}eU8B-EL(VaB^v$EP@f!vDpe$gzr zQOs9%A|Gr#r^&y64Hd^WA`DfT!57bOvTp$$D;X!nE)myq`cAPypqjvEzRI9^Se1%U zuF4b=4u2@km_}jE>}jTz$~4!VpEylzP7bD<&1hSuS}FkU>;aI!K-P*&6NiSb9j1Ie zl2b*CV!RfRVV9Qm5(P_4Rs`si(R!0ZtvIriJF3ao&$MD9M#H zwta|6t*CEQE~}bRJ49J7uZTjnR;qGFzcLWc4VFqTk3qz zUbgfAFkZ11Hycg6v_?4;=w*0 zsHk){Pgklkx2M#|ed&fRZCI%cQI= zK$Eaaaq)~^)b)-^R}n{etAC+46n{eQ>yhbyGoWGdQkSTwoBj^{WHFn z_lXAJE3f?T|Mh?WUySvA>3<`jdK^2<6#}-hL?RK?y2`Kr1y@wa)wwyBYtQEj+Q%je zPC0=|vIgLUW1VOjD(aq&;UuVJousP(>=TJvS_f4psuIO2QNQhD?4G~IZ{aReB0f%r zIKeFIUs>13x>%!ML;5_Y4ye`7<`8$4;aC=%W$*GnHu7Pvy(mmS;D1lmPwl{W;QWB= zMSst>#KrugjD$Lmzr|Rwo7OIXylb&Qx8%BIIf0!d3Vlw2v<1E+$KuTEI$og16xG+s zVou=)DUu^u16?sh2Y*UoWj|3OrwTr!%W~ z3TC)vc20}|Eq{zfEK}jX3Gvp@-c*}gr}KQ7(2}myN-!?o-swdg24E}QQ>#iI2DvE- zG>Y2!)Oo~nf;2T~wfP+f>SrwbahQPni5yZbkV2|~S4xx{y;TW%FwXoCym6WFhIBn>K+Vp#nWbnt)&BpMs9=ArLTv7k+u8nYXo2=U;&j zDD!40R(J(qNQRasR_-P@gY4f}Pv&#~#u}LOZGS$sf3r)4>{MbzN&YdPegDSVDlu;? zcNe?;*y1uob52SqT{n`tIF#-693G z6uxk~hh=Ihzq-sPPB~4BQ4TeI|3pj5&GK9is#&JPml;oWP`Hgg7Q_~?V3`n1G0875 z>VK%t0tGua<}noI1EmUtB}vTVV~n@J-#F3in1;s?)8&{ICC78$dGf>Qk zFKPp%TlG5BQi9_wIe5H(@msh0wQoOvNel6nZ$CX2-(ou?6vodq8K+yfnxtE}4z(~x zB7`kAuu~{qFGTG^)Pw>*p2I@1l||_PxqqOF>bvSg?qFf~u9QcoRS;*N9<^MW|D!8M zN&u!C<^KL&_1p9jSn~I+;&bj6OF~O-l$V|q79KfiYihog3c8?t$@=3^vJm}MXlfW> zfTHG-bu842Dh!5qE0u*`R@Q}IRMy$aDs;e`gKO-5_qDL3uFIc*WMksysL;~&!GCj@ zD?cOMw87Ltk;}*$+_3n{n<>aff=mZ>8AU}6tbJGmU--gPgsWuq^+5`RQf*A{Disa? z1q9CQTSHyXLW&Xpwbw04r}kPL2Q+jd8Ie@y zX9wzl#aNA~XIzwj3Q}c+UsMdFQGX7{b|R?|BCpVlaAZzOx&+XoU{sV~l{>jDV|&hc z^SrMxS2-o8h78niTUwzrW^oS+mcZgko&*lwmQ(rM8GoP33Viq3teJ+4i$H(~6oSn{ z3Fe0hibcW*^eB0`p-8p@Y}P16oiRNNVgQT~BLpTwHJ(^>uL+l9ZRrW6$$uw%ry-zX z4JjAK35mc;L)a{aCBKjmtQL86g+d2f#wJ0JgvJj$wA&f?_BiEI^B^!`dJ2ZMS_n^$usfq^3HzuwI?Cn=li6HH3ZdE z0vapsj4E8qnYv6UX27jA1AjFh*QpdH<+-eU%Q;oFPn3jN;O2nC)N#3_d>b9=*~37WM83tbAnc2$Ob$Zx&z4qHp^{jlKqF@@?X8FMAo&1#oP!jW*oA8& z$d-f?tgyAFggd!0t`ZWL{08KY7_u*H-MNA37{a5F=#F_?@&p!=Tz?KsC%{6%u;=q& zrG?}z)GPD*B*^vb*>zYn^Rq)9#;=8wlqk-%i>$z8|6tIF>@J6l95ogCZ{-ZUa&y}r z(R&F83KFaMJ_2-s7DHl6rw`0NS4;za+%}toooT6OpsLFqhy{Au^&ZZ#%G!6$yjgd% z2{=P~1h}Q-43>9R_J8@}^tgB{PI^7~7bau}8vVSBO&f zNcQtGd9D4g$riJfSKlfpGI3u_R4OE8p}eS4Dg`{XzuNW-^vvIE_O^1s^@jUK!EqS7 zTy6vpQEc4y3&Kp-sZ*M& z| zQ*>_p!DBYk1}{Q08dQ7>@--e85t-&S?4=xnRfin?oWC7GtXh4IMaEs;+M~e{(>!m>y4-k2Eycr@9{577GQU)c@k?yLd46>GNH zI4`cIr}>uKN5b^w1%<|nMRmS=xzW{7T5{@X&UfvI0}9{+IGBumM(TJ@kWN`~vTmAL z=wG;!_JX_4%t6;a&x{9CRZjvSnP}*04-onI`GKeqd4H)~x?-ga4CWBkguk7Wsh(jm zU*?JIjVebgPm|@PSz%0HsxEGFWTM zN3_Xc4;fgp#Z=ToTO0#elyY;SPI$1cdy#(vSe#r3_Jx;*pM03xkQs$+$NOUaYAk}+ z`#75U31ysQR*HVuknuccoepXy#ZYAY8CYs`S$~+bR!hD`0AC+s=L3Dg_XywnpJ`IO z&SO2YcK3Y6I?u_Jt!+;mLiWRyi)>^0Bn^mtOVP%OeHL-TW3DkgzD5$KV zpITZ#3b)_H2>kFV>GOXz-5lR#A_7iU7$VyZ%{RQjPPLjYoH1!bm%^h&Vja-}A$g=U zMSl;#t<`0o-Y?YegI{DsVj7t60bZplu@jS3QZsX3g%ZOW*Mv1Qks`o?mIwSs$Md_s zo39T9wi(a!pdk?Cf;gfs%0yQ=(+c&3dot$>Rn;)*(O8esQ3~txfVvA*z(f8v7h6=Q0-GU^fA% z9YHb5^>=wO#Ei;xaiHbnBcV(XvIr`i>eAT#h8)6`gry8&RA(`y^gNu9kMX(mbnM?* zbnF+t6-A47L-yL|oys|yU<|2FXn!8{g^P>}Fu?`iR}&h4qOo~=4Cdki!&S$sxqX*$ zz{`UnDCv%0PEnxVL&R_ckGLt7KfGQSPqODPbNtQliw9Y26*O|Q%NG@GNPo?sh#Lh| zTpo!AeS&NgB5vh}!dF$?-$KZk%lOq&AmHSs5lHH~e{lq83ijn;06JXm@!%B32FooU z&QWa7=rW3*ctuGt;Wl5G$nx=Dn+TUZxk4Pgclj1uM`o*IGWJPy$IDp|*n^r*#sNuo zOV?32ikwAaIwT6ugGFq*B!3FuMc}On?$zd8JYG`!6EGD~cpEJo2)7%jz+0r`S<=BV zA20UIeGoM+++~@DnH^cBl4y|Y4$D6IbP2eLvn5y7jdi-d$tA>>;B=>5^3*4BLLIOO z*E2mMLqr%waTH!p*x}Qz2t*C1$zv3_rqRmuB7sq^(;FHQtFGmRI)4kWIs_1KqA)~% z%?#8BV9&jGUcMN1LepD+MxzCPhf~Z*t(rqM%dQkqI~EnMSmU6NfJUB}Sw+OY9OO?m zoh0ygVU*Cv&^+J*AEqiq70)dD2^aZ5FK_Ta6K&4wNZb6+ zL|d_%(q{iN(Wc}{hksrUYOFs=T~M~Wgv*RpDiB46gd!UD`g8Eru?DfExCk#PQvhho1~g>`FVR<$w2qU9qt%Y)kYg0Tpc#EZGx|s~mT9FYL0gc4#i-ge$1p=q zBwwD0FxQ;^8h=Xk*TKzZb8u5o$eWFJ;IbH!BkOdO*R(8aC>ypF55UI*S*BfDf!uT+ z^Yn6;jNWXpOO`gAb=)6V2apgA2pXv;fQcs~fvk1gu}qaK3(1*WzgnrVnh=Bbimw_Y zeT{^s8qgXrf0g9C#LzlL51>+yNVX2GQ>m;lXw=F0s(;uU(aRkwVu3?*NlGXvpUHH4 zZ^MQ*Cy`FZa}wQQ{Ktkim(R^ttnybluqW$u*Hj(|K9wN#_q31O4dS)Sq_y2D=iy_G01e8UVm&r$k#&3kpjleK^pL{q|`5R{6bC* zVekez{(sy+CWRXni6gIRm55rFm!njar&amVyYhsfCQ7ZmW~HL-lm7fBvUOsn9yQPL zQTSdrWZMqOu7k63JBHank?1FWK1VU@qU4uPn#kS|7Ks{wXVW>0mDk-vK?&JmBs7t^ zmb?glpA}Xbx#5uurb#?>P7#&`VL#Dav=Bqlhht9Rcs*y3sIn9{Ww6gp{Yo5;xh&#)Y_dqe za~;w)h#O{4#j8q18(MSXHCY;0yql;c_{nV^5%U_5->rE$I37mFmWj#^;yMI#oz;$8$x%Gh_{w^-k-whViAE*p=7`b|^ zi1CK*>!-pxF##`4o<0n1W0sG;^T=bqiP(u~mCYQ(9DoDrDn}yP=K}K#v(Sn~D;U^~ z`+{FuA0YO45rQECL#7cV%nazAwP-Xpt-Nh)5Ad@e_ndXXS`pJAN z*c8A93zRf#yodu-JNE4(j#^bnN`C~cs#{hkA$mH1gBr#*#rTpClQ@EM?$8J4JlWnX zr@(bCtS?5A=E3FW3oVi0rwE{u0jgnJ`+fg$ z%ZL=ks2G8ZBY=bqC$q**khcO+Ot%q*w$4cdT95|Z!s2JheI}?92!C(3MHTotRS;F+T!*|ovKx?=7igQ?R~8k2xp znoY|Z6#GjC5Bg@4oPP{7n?hI|4m-DaHIa2`qxjMXw7g~8JM_IfAiL<$y#`0H4ZwTx z0Sv{)Ulkkg_g(W6_j30(y{S~_7jDx`Kbda9G}vWG;^9@}f6}97*8FCVn#PW23rL>U zoX|+~%?kNu`TKS#Dt0i~-mD}KlchKo1fnA@yJ+Xz;RS$A~8=;jiK%u(MAk_^qm~nJx`MJLwb2UAxL5xNuP!<|Qk}}kC7*YtR0!ZXU_Dv5& zw$FtB5Lw$8Tl_(ib!wt6U?XOC7X+G$7U|3ckjr;h)+rj#feoW$xHO<}7xH?<6~x)p zI-M3lx6v{rYkRfyZ2qLVOmH=Hzd)8gF(4G5HWW#bv%7eFOXx6k#tWz;@70$#*M=D)URze zntxxjY9#fnwf*P9HrJ{H(U6W3=aJ5+YO1asKY`58pJ70wSJfVMyat4+XGKToe+XQO z`l?!OHCiojK64@-0O?h)-S#F8l|W|AFg4Xzo=1DV9>f7!q_G{C>Z``2YdcMqV9xKO zTD1yoHJUBg?x@5My~tEw)m!Zbov0*PIDZaJ5>tIuYuk;ctrDJ(##CRqy-ugvR=FRx z8a)NhqtK&ue(AA7g0&pn#>RcNg_*0&bD9h&~3QD7|;(wyB&I)Ty z_#+A%tgzmOKcY}tOt%exO6$aX;!6TOc%Hl@)oK@eEtbUdJP%6vlDvW^9M{nihJU`I z?$IQAR!w!QEDd+dYqoA?*&^-fqum)nI$;wMJd(wT*U7-wT|qdZTOf>QKGfZ0EJ5 z?d6SDjYf~bOjGGpjZRy~qzb&+ZhxyiqtV4y+ugjr01)0}&8T;<$yP^cwT)I?m$sJG zsnv}Rf@-zXQtI8B(d+4DxZ?Gy)&EecN);g4nXR6_zE-cAt+R`Zi!vl%berAonFPtB z(Wu_2%Me$y;m?=HAi%MCcGewKWpM1)8}(iZ94{`qwML@=$BSNL)M&soqJMZEfo09N z?t5EoQN7+(>g}GfwUMtnlU{vNmuA=QI1{^Cnq9YM*I{dD>L6Yvw4d< z$IGa*=yW@W%&6^o!;hN$u6$Ia=KlSYm=R|&ckHKcSo~M*meX*$<)x-x&zm^XQd4Ss z4O?F7dau@KY@>M3_Bvi`3x7#ckJ_}og(PW*)?IJUg6HjSFKf-{$tyRBVh|Bv_`iM2 z*@4O#aaI!=&llm#M+(BejqI5ptW0&`AI+jLO6I^2d>Flp!szIRP8We4lXo->BJwT@ zZ6_iFG`zPHa_LVfivpuSZ8C@!u}@>=iY`gs)DEn7nnoRwR=wI&+J8o)wHIeKj79?y zOt-1jt4*U*-$6p1n$c}DWZ7vejat=cHg?fay@* z^SQM!Ue~MVw?=nD zd!8%b8m+2lcP1qwrsmmQMFx&uujkp=<9A&+d5ix&_kd@IshzC8^%8D$ zj3%S4yFH~*ZyDA4zVp*+aH6Z$t}FFs&1me>N1c|@V~4ET1p25`qK{tHyn3(k{ns)3 zDg(g|SkQg-1b@`%q}yt~#N05_Sq78Z@Vwi-kYE7{&dy6C?={->;r8+F)ateR%hy&$ zrgrSo@SCZjI^T$Tr>iuKZlm33SF4?7qt>g}dY%0gF#}_HI^BzAtu50`EvMrFXqPVX z3uGYn(M&zsq1&8LxlrDmYSpWHYDOMyh-gXo&2}%=?tgTQW)3Rts?n~$Fuz)()~>bp z0Aw=+88f%NMQrPgR0)lQ??={1^-R=e543vusp-#YR2QSH3i+QyBr z`L%zawQaYI4m(#piMK@v`BuPb_KX@f(Wrj$3T*d`4qkALdat+yzrYgpy8MG`t@`q{ z*)}?DL4ULCyEe}o&E^ZhHtYkjS)#1<9^?A{{XkWCyggb)B^S|MSsSe@K z7g$eeSS1v4R_oPzUtCM+%lu^KVuKwh-1(+h&s-|;5-qp0bpQIgwl7*2tv%~1#=wA| z!$HqzA;np%Hk5j`WdPrs9|sOlRinn}+g3xVwSSvNt*J{VgcHGCqrtfBMoa1Rj2_e} zi-WpGn}Hm#*haIXe;2n_{Sv!Y{Q|#M`|oAgg2nS9UVS6NdELs(K%~u%e{KlUUjlfw{~jnT6bISMeA;TA4}&tcB|>+*s(-o_yCv!K+RntSN$WFd(q362*Qwf$)A^pm zQYQ2GTDUd08|Fv7Ua!}Bf&T5*oAoWJt=)7cljckGulOQgnc<+@ zZB_T)iK4EA)i3JZ`q_SVMFNv&{_ArrVutGeg7k!CCKYpPYb?;{U{z zyAOXfUqTh)@#jAPsF zNe$E?#TwgQv)y?C$9h$_O}jf{GurW*d)Bz!s5_0xe~Zeb+F?M^*$d>~hMSgtOg-?; z{3c=A;L#4ACXpSx5L4H8lmt@K%zqN{SNL(1(Af9>Dlhk>3xBqx6}+rK*}R}wZaRTK z--EI#{?@xSlV#n^4T-RgadF2<<8gg)voqmTMal47Mu&_0HR8^162G?X%Wk8&qrXr%(~pnW(E+rDm_!=$2Ikz0L@fWPc@yY~>Ky ztSR-TQEQ_X57K=>J74GP2hEG_7Vq8}^r}5s{c&;D?lsHIZlm6BpM6hr0fziD*Yb=hku^`6ma_G;ZutJ`jMyVd4CQh%WK;*Qo{+RoaS z(P}_vQE0p0tSM^=l_!uawMNru@7>%E*FAzLyjH6j)!wcSvn6zH)f%o(XOH%vQRT7~ zP-51qUF>d;HXxS|)RkJD6CU!mcH2g+$27GarC#U!-uAZ2m1?by(QWNv3QLu$b)&Yk zujS5K*i1&k{gRX&D}RCR#Pzl=qHe7_7|9pWsM)EVm&oASMZMS9mit#nwGPn60=W#% zS{J=NWKu4S%&|^=IBcGeWUMo4o{w6R?tgUNsdihaZ1p_)4P@l-yjO3YmnT+lH!d0@ zS>jo3b=t$y#Ol>St-gabYwg--57FYZdS1Igr`_S;Y*3RY)_)tGcgpB=G#b`hBWYrT zUaMO}iUU37zx80Wg*L3!QtGXa(duk*$jx@+>`a!24r%B%}TT)Q1 zw~AYMPrHKN?b1n^25Z-;H*rhsf#+6R9hnBRX>S5&J&M0YkJP&8)&Ky?XZ@@;kcp#i zcTgRc#K6r#tKEKqak$84GSy}xzpLS zX}JsSzM-RqVz3~|g7^vX0XT3uUO)$^?kjB)Wl@i~DBJtfi6}3zr zgz9yoK_DO2s%}7cR#$dcO}bWt^g^-Kxxp^8zxiUIHJq^^F(o`DO+N&+%vx z>izyO6cu$n%Yp@(VE>)kPw(tGNy?9YQhM})(&M723;{g;=~=lSDdsMRaATn*`U@8N zVSjjX!ZO~02Y<+nrLR5h;V%zpl`EZV918P|rmr)dxwS(Dc8*<}I14g~$ZtLg60&NNPfdQ5uNtKz zi$OH=Q7vwIS`MlYazr1e?D1=aoJWet9t3$K+6eESCJaVb{)U|aZ zrV#$M2xZHn=}L<$;`%z$)>ezfWPfd7uz}0Sd4fo?Sss251_bHa{`Jx2NAnOn_s?l%(R$!9=r|v|d=T8^;^D_2R@td_hX`JYnMYwl6?hGyz*(Aiu{){7J~Zb3oZddh(PH|j30IUMMOPF?x9K}o1`Uix z$~^rvzGe`W^j$+sbsh7T%8^yT71zW>2%!{ztYAF6zPh+mb$#X2z*R8l=-iGIs)>8C zrR#aJ({hWPFwT#zZ|(WKt?MV-%i@cy=9`WBYxopGZfweOtz~@Et{bouu(o+@3d0Hf zIqEs`5Ziw8fjWy=xMJyh>KXuy6xI_q5W~2OwSOt412kpF)K>MA)Ij*PC`k>B5@5`K zBj)d9srDp^f<;Qz{9Qo3)KZUn_#c={=RL4Zjw$-gr8;~;1B}2})S=JR8AP+04Q_Q# zG+)7LNa=e^fxj`a_)QT9Yah=H;lCKEn_)aKw*{hk4|fW0q&MnyW= zigs3xW5>NTLkNqTWkFOrAw%7_ZES-4~B8Ts4*Ja^v62LQgO1(@0i{q^QY z9fu)(xEpOQUJ zv*%PLDV@#1PXO}38C~M8LK}+iV>_Huvsx6;XP(8WM$%}0|713+_+xGwy%w4IA^fVg z$aCu4v+z!S;HN9IiX~F~)f$8c@f|z3pPMysccylFAJ|EHADZo^D45fK&<@g-Sp$ds zgoX~yi>kzn>7NT4IxDHnjaT<9|4ZQ0m>0nIm^Gu*!Y5|@>E1Ku&8%Y~Z$XUUs77Y? z^L>h|4NuM@NuwET*?a}jW#=(XX?zjFbq%|akx`@A*bkJYPCtA9qw+IdDZa;?P-Adn zg%k-#NA}!5dZH^OnzPt{oKcKluvoRm4rJWIJGevOYkmu`JQ<9%NtDNE3Aq_)AhHQg zD5UvTfU_?{0wxAw|VcuWPjj8y?QrsBq{ zqf>;0qtkRXzn`KvU);PuxMtf_DV#ZkZ#LdJG~=teV8up%aYq{@_r z6vH1YfPR(9O0hBA9jpSpmq1K${e(p&!C>9&`)HA3q7d9iLwV~@A^D=k(krs=hyvh{ z7oz1Gb>VcE4}p5-y3$5Tvk+@&D6qb^3t5@*`h%cld|p!uLxLv3(m?u{@O`k|e=d?V zk!Ix61SSmVIGV42a7Jtz>Yn5HEu;G|iy--A0;{sKdND1n9~>D>I1BY=QId*aZIq3| zQ2F^+f2MTK^L#@5w|Oy2h8NUJf99tNFJ?;;B^-FuXi703CT66w6=yM9FlXt~;35wI z7Fdr|nX4SlY&BdSu^;-g#jMl_5`C;TPeuxtijNsaF?|Vt>zD7@R`>Zb1pX5ZR{0`q z<2m)eZU7hSokuo4>le1IB4gy1|0+hop zB+7r?4tW0sR{i&SXGa*|@Vv#olo^$nm)P0rCl8maE|9y>WfdqcPxe zWkORh!&icTXnG{~3oUTCH`J}mLA|t*VlUuHY$NV4B69yT=BuG|nj+DuZr>+ccqgxz zpE4z3cMDB0*Ukv$Ur=Oq$|L%6!I&FGJ1+)NOfeWdv3Is8U<`0c_|HF zZWi!F`8HICH@`FRV9Nv96B2p$ez=WsfzwoMCytVTLNBI zHjVBi_?Eoo|e8ornYFfY?k8h}@HKO;-4M(!;; zo+1yGh;Wa~AMQ_VYT< z6Z|A5e=?$F_%mH4`b%pt&^)pvun0Pjoc(0&6;c<2A23*i!~29kU2R#ZOtmF{!C;Yk z5p5mTq!ev)4bl3r7=$e55-`iQ*f}$~5PbP1fpkXhWx>%EoMo=DfUrqsKF4C2y#j`} zq|P%*kw*raRFSBX)=nj!PS#8#Z$29nKXfsj43R#7Y$>>mLkTV<7#PyTiT!yR#pK2s z-?76~tnH;Z=X}gJIcSijG~H2u34UgpC%v;NtOZE3yqCdUY60539+dE*&N7dFdFq-f-Dc=?9~ zzFS)3J92MT$!&r1?ti#FIXt{)ci6RsBN*S`=_gB;LhD*+C6y&^$!BYSR%p--ra=G9 z)zt9*t>ukp5O6d}W35U)>v=k`w`DVe-?XrttwZ0_*&OkdknR9e1JX~}Y(;gzv}_1JjF_jhCo zi^o!=;);^bT7WGwOQU#SYc3NliDITvYMK*LNoTWi3sIYHzIr4ff~yPTy<}0GzJ%KE7UO=3DZaEOp)d zYHu^Sw^^yT8T2;0vzF3UUg3!lgf!kkqVjJ0pJvWmnVZZ2&gOw!{UJd zULY#viSm1Yd`CQne4nYMp8L~Z{V;7e@unl^{`AM+0m5 zZsi(ZQmNKGEn5d!MT}dGA%cTto@94f5GMltEl=11y+>VtskTJabT8iCqt0Dh>iYT~ z(ICYnIB)~!hnrD@OPLxxJiOKK877<|t*jT>u|FVnVejpL8c zNS#}6Ez8;_cKhqhEMKQ@b$N(nsRM2B8P3Z6`173wgZT`D5!1^wFJEB7tmSSgkWWH$ zQLOrnyy~}qoC@`HuB7aeqZ&I$VV)C_DgKeRWIqi*1=bW)p8xru;SA5B zFM|Xkf}BFm!@18aZycs~dZjYs=h@RUbB7O6FhHSXe?ZHZxaH-TcBYf+57w+wDd8$C z!`b~x5gCb=-b+}D+>=`Xmu)`AvaA{cA|S^6*5&Je#xjLI`iW&(ztWt3lQIE|Y)u~C z+qagJLNjM;OU+?gA_%I(z>k&fvqsd4w)OTqX^iZxgh*m`FbtTneBmta!qK>4!{^~r8p6Y;6WM}#frx_ zr&wuIMSD-`4$o0VIAd=N`b6`r&F32N>;rRbMPVB~KGo!bazuaeed? z14HzB1b=fap*1V$t7%$7f!?8NFh(VRF%|Vt7N*Vg@&#RRNWi*ED(-Ce;zyiVTpBr+ zT>_1IpK+T3u)LLW6Q%`|i=&5CTf{W6W+Z>6F0j_tJuBfh=!0kxrq;4NXFgGh*Gjin z3C|BsUM1xozkMCR+FbtheDyxcM_d|9shXZ$Y3I5De6xll&<)1qYHP8ud#}W%54pBhuv7>`>*4;$>tF`EOF|oSf4aKW5Bg z{#gN+_u4JFmREd9Zu=am-%5#-wbE*-sz3v`Ir)0sfB1{?aCmKk0R`6f!;r?-^=5;Y znklu)3B5|?#b$9Z1WZzA)?pV)*u>m0@^w(xcMm8~Ba z_YI5$E;zhE?j#DyVutV9Za2D^W4`ro!4|qWd}%D-*iwM-xI6)i$C~zkEXzvcz9dvy z%CZ+A%q6Dr&#h%B4vx29%u8l3h<~NrwdwZAthcE=AOZI;baAqjgb)R7IIiAVOP%u` z2t`rI(HOZE1!&QCQhv)wb-4oLj+h|kh4~1A>Rud%qPg;8|IbjnC*?BTyw}NX@g?h2 zet@rTOJmZ_Tb*1Nxi+qUO_{m>E&S@5zO zuPx90*7DrTF!r~vt;do9+5L+^<}#L)v7=k7b6s8?`40ISRFUON5G4b|%WHMJvsM5W zUt6o|t+l$=$$cSSk|a%BCb%?R?H+`*QX1y?s}Lrz$o}~qfHgCJaDiXL4MK z$nbNyJ@{V8f_q8tgbPJ(cbTediK)8PPi}#!x(3Bkfo^UKri!_Dyu1?k23TG=}uqXIGVV4$nDdWd$ih=ld{EN5%~{S(e!2e`rAEZjjW~O-K}~ zL5<)sO43cO-uY;MRPlc`{g~yv+A{deH{=%|u9)|s#l#tZ#MU|^_pw#0R>?E9c6Ckt z@!j@rc|y@-#59S5Pl8Y9DY#I;#?({?>;dGjX+U#UnSh3fcy}z*p;fA6xc(F-pniCJ zcGRk2(DS8&Qa-kYpzszbdpsj^5K4G=nWh7m`V9JT;QyX~K}`-bdZyz#mb{KDmDe$D zFZq#Lzr>ZwFR?6UB2TGxaPUmYb-Jx~j+w+@>=C%dmtm!%p@-Ky--)BtPM);06v`p9 zPey)D!B}^RCcA$RK`{0^{ym7TG$vtet$o*2nF>-R>|f9#LCUQvS%i-$?sOA#d`F@% ze>UHyWImmLqs40Y$y{f|^U;Qfns3vL{6wuVCUFcstwO6xKKmd5$#?z_p?=}}1RyI3 z{0fasPGe~*-{&MxEs`%6nC|&D)CVheKuGs2r!I{X7=b97fL$}&W>)n6!_fMPO7P+x zl+H8AU=A$nk9hxxuA)@2m7;zLVc%oUAL(4z#6XyTm3{G|euw@^n*15VX+Oo>53Kl3 z*E6|ZaT)MtvEcAG0oLG+bz9(*YnVO zZQLV&PK1{xYXN^}x>-!*pi1t=dR&vgYxlZNK36I+y2z2AVo50p`cdYDcCD{}iWx@D zxThtPl1_aKHL^R1=O;>L)+hgDdzBe|hfRF84m^V$Dxks-EbAw_*`NSJK)k=Q%h$oaIyz~Sc)ychnED1AyvMD_Im6h827e@|TNwxAt;iY3OG+@Jg&a-D%ryz==0WtZjsf8@RUcH+p=C;BReRfkb-(iHCg zX4Z^wZl*5oO*hqr>7w0jXg)1$QnsoP@|J{EV0gB9r1J`AMdV!)F8997I&;oHHoB!! zd5MgS_{A@*Ax~rWE_H5foWXA~K4TiNkp8w7e<`YTTiK=1nqM@=ka@i@DNSLzJ7n)6)sanEigz zQKH`?AnX}~?NhtGUDHu+D=(*~aJAcNEg*W3&gOP)-*N34SMAhL0!e9Q;&eZ@hlC$n ze|?S`8gs1Xp7ogQa6guPO+3{Nqz5ANAzPx@RCWKI(^)}rofSSIyXjQ_n@{*%`?NvM zD*RC_U(svCvL>z8wTeP-wLDyoL2bTVgKxy1kbU>ClWyvi!O)w4_=A5`y<)$=HgWJ1 z(hy7i8sprq5&)j8oS4K zks=EqxLZ^cy+`8N>1i?d>_*Hj>yKQTrYVI6wC>`$ZBG12=OZgEl>tLJIDtE0YADNF z>J=CoeQfKd2U6T>#r7cjn0gb4fAojVyjax_U%e@}5J^w4F8Oy6MRmbsRcTCr0u_tmtDOb!n7hzka78pemPDkY8PdYh0edA3p0hfY3$c1!O0&S$hvI(W z=Q)8nNBkE&+iC?JnGeu@RO;@ltzYA-GsV{638rS(6@p9i*mf1)@ee1nXvBTNr0 zMA+YI#jbNAmv#oB9~z&0b!&0t%djp88zjCZ=DHx}s93%HJf|t7b^RANs^g*x>e?zG zcKx6|Elo7pEVDa%EoCFR(|QQ|yCahj0!gPr+6-(U{nRSbnlb%}NBogzUj?htf$k!q zLYmc-d*o^05xAlhf2>Grahwy-wg`QZW__@}fY1=m3P&w$w+Q_a_)&i3z$5jE=lO~+ zh47(GbEZlEWdr`W46zidz%q>jpz5CzDawn2*hf%_H<{lI)uy2-nL2g{Yh60_R4O+Y z5RbOVk~KVaY^^#yEkJ|?V!9G+>CR%k)spGW-CfEdM>Segf5b{48-Bh07q)m+ZWp+y zx$lQ=D9ds>vHBsX?S~QF)%C-u+$sah0XVQ#fB|`$0LX11WPWYcCd6q(uq^!|UX*5F z%qX2=<$L6ZUFGZvSwk|vOSuq!T^cR@?tvy?nVg_v&m-2QkMQR{V_gfUks64D*#7>+ z>8MOIqx)e5e}o`>owErY!KMcAhSL56n~zZ^TC#TgK9kEf9pU2rCGS7U72Th5Dt`hP zUZMP_{CSu^toc)%&SI&8UV(IjW;V;q@`x_hTRgMg>NVE%q+0m!4E#EPfj4fomfncQ zAc1h}Pe(>bI>Ja8P_QAy&dxqz?eA#7ZX$0)Lu_B!e_}XTN48BNu*@)#Sud{gPPE+b zJ^atw?PLu#yUIJFyXjTl5&hj}K}zv1QxF*46sD}9pJ}zc(^GvU3>l*ME-~dXeaBw{ z&=sr=(apkIcJ%|T7TU)w^FCwjN8b0~b7*)}y++GmT&p_crErakG4ZWU?VCi)j!8P3 zq?+z-f8Lv`wjgC`#q1^?N$rA=O^wrc(mW2p7KbY zA^UyF2NCxjV^+KU7*VT1u$jd{gLOqOmRpzUrk_Zk$4?w9Kfy*Pc5Dgp-vCu38dzH` z4>TV1M6Od{e=#~TpbeyBy=N*vM5(3L%;l ze_{z}vrPsg9P_5QUlJqR-Ceok27;Z1yDXz2qiD5o`hHBk7+z^tDhj~|f}0gD8)VO>Cy*KD?5rNl(7%+`WP0a&=^ zO^qA+DPmE|2Dw;E!!^@O0xcre{J;O#fB$OIan2Ov^aRLJ9@6A7TJmKFSX7L+4!nHZ z2OvbjXbc&GOiv&z0}z*}aGZ;)Cwmp4yvOT*?KJ7liZ#VP_=JCbK#dHj6u_@YzeKDV z1j`#RZe&f$7pw`G&IG}}KX7q`+*lUW?V$u0>c{;W<7$0R8WelEi~Z0S@bk-@e_>pr z|2;*n694q|&E|9QQ=YeBqP7{%8k`b_x8J}1qa&T{A?eped>rb6 z{Nh7V(5b#!`$e=$nUn`pgA%~f@%E-NUaysQq@Wf(Vjv7rw>EM0ubUs_G<>uglfc@aOmb= z_xqoC2X9pPL_e&!*oLg4O}+ z>RJ#B{<9N4ZaNgPU&9V^xi26F%ODTl@{S78@R}I^aMEl~+D#x|e*s2}92XWVxIyp5 zVsF@qmN%jyhN3|S=!xhl88u~S6JOqRf_3s3JphCiYN}8415|HCYxaXqAh8c#h{2S; zZD2CygYs<$Dmcgefa3oZaJ{Kg;Q)FJTzHlx2-<~aoQ|<^5)PlTyaCQ?E`5BR z$)E-7^A*n}Y36P-+4ukB zzX$Q+>xU$HVBbpz&Bm}1Zn$zVe9Sv`JWZBVA+i(H_=Px^5K9XIFN%FSvA8+$G)wVE5r@Ru^wCAK!b~j2o(Pfb1ll=@@RbrH}IvH<>ul- z_SLuEMS7+mf5m_sFR-pl3*5d-{(^=lokj}J_n$w-J=v09-BRXYMSrrSe zqu-&1Zw|X4wA+Sw9U^3>dJizrfFwu40~y3Oy<1glg4nJJ2V!y0Q&{UdVU%Xzyz(VT zzi*ltwKuJ{EBzJb7s^-+uq90*-HKw_@hqjf#&H2*f3m4~Qj5DUeCvmmF+Z`MxlfWW z$OaCvRAH2IL^G&IfUU-99{e^|^2;8L#KaXuN%Y^~(RZuGCgTd<)hmU1K6`s}W)?ol zpxSN{+MFwXqL^i#C^J6hAQoK{g9D($|GrtQG6>C>b?0*jFE18Cu{-0Lvxv$|kmqE` zx79eVe*w1000hPZESj)W;g3As)(+L~IU7LC)a4?-w>=F9VidzE>-MJS8ZOXoPYHga zx?LAAo6g0?k*HuzIba*zgAU&$oh(T$=LMqt6oG`UiZohHOD7Iw)}1RDwpys;#n&O< zHW(`+WfUm%b1FMRtYH_`?ilI57`aZlsTHO(e}w?|L!)T5K#Ou$C}7t5`-Z1mqZ%PO zv>1$02>Kjt0s~x)Mnsz@g{1ym^DxR!AQtIlandC1rs~8KZ0i#<0OQ&L7`UTn`StDg z<07_*QzlMx-{g1*<^#?AQI0nn!u@cOb2bDxa{~X*=5)w{*>p}b&Y*1+1e){xen{bO ze`DBx$Whj|TEh-9%9;J?Sl409uDmgw5_;>6Nvkz3!QdipIps68hWa%H;bl=4_;M3I zKhT**cFYMKu^rTZ$95J9V`NLrkHJk+2vS1bX+-gB8%?=E{Z)>ai4-`*tad&%sMe@T+4IlwQ znsIg{NVU{f@@}tOO1yZOB<~@QFe?3e~ zw3IKm^2*rT^REyOVI(-^+*T1dZ#WcTT!Tz&vF`XPon1=M12@|;ikfrMZUWKj8p8lA zeGIulH>*|C+UzFWlXzxZaho1w9qH79PjiJO?z?_Q&WR72w|%Dt7lO(soE*FfxHAZ? zcUnwr-Y!Tde*}(E9H^c6;;h0ze@5uUoT<|o7Iau`@AsO>aH z$OOYV!JS#c!Pv#7@VB%hGv+RP?XdeSbsR)l3M`h(qq`L9sv%NSr4tt5mXD5Dpw?9w z1J*_h0u^`cE#QSgDW{dahqNr!01V{jBi@K?0Al|!K-`_*TtEtLUwDjg-2_XZu&?n0wpm`i|VX!okrXMYvMyDa27$X5;SLlVv23ONrs(*yJREBVx z&0~pR#B2U70_s{(hm&Q(g-itc+C{X6c%T!Rmk}I9dRO_a3YMGVe_@mVA2jG5fFi~W zuln)#|I$Gx96{9u-myY_v!Sf`|GI`MWJ8IU$q9-?VwJBbtR}B-lfrEinS`vOKLb{m zRX|pM>Z@$3%$l!?ov#v%3{)f|xXf!l&2q{e`Ly^&pLVlZefe8SuYW0;!@puG3m#$ZRm)31uOP@ zZ=&MPfzl8$6yTDUx4pQ>cX|gLl;z6+G%yq}9F;LCV!%Yfd8;*X9T^q5DZkSrJ_yFl z*x@2+m;)D>$Q&RdE7vz@4uG2lGv}n$Qf|8jGn8P@zgGRqf27D3?Ehc1w;uC4U}BR+ z%q8u)jOLsgoFIO2t2O8gDD&GK)gxca)}U^B>?%w0L+CMq!XWSoKX}earXNYG4X`|^_U1f|oL$KH~Ie$hdC+%|ECRe*;W72Vg74bjkQ-N zWjBC^gm74Q1uIbD-abs?v1B*v?y#Ii$QF5i z&y{69Mt&XB^+8d*$-PJLj3I!-nO%lv(QCdFf&XG_ zVypJBUpBoB4kQsbd&|KGy5;l{vU>_FXC{mu@N*0y?BXYQUk_pqCY(*pr^&AQ)-;=1 z(@fMafAgPI_hBP{df>O2KdEB)hobE==ns3^K}Xrs3UPQAyhWQfBIu-z+eu^6i}?9Q z_fe$#DCN+h8$OHpoNhTA%|0R!$-C!d-rI7`oSJerp55~~l`PESU<{qgQVw(wwNpB_ zr?l!2dsb(>JgpOcWL}Stt7-3_+bUS`vM-*9f5G#+hM;cO5Y{NxJB8KUIlVbzJ6me* z*v{$Pi6b+6FKJuR0&d$*%1v7wY1y4@Rm*1etINZ-MhKKel?;>*=;O0pbQPi@&aE(x z>FF8iot{p6g!9=HGK`y$)TyBxn(gKtZHux4Ptb>bFlN+U)zbClV8LUaKM ze{Yc~Z)DM{pViPJX`1zsMY|RluJG8j&IrDrPHm&^&SH*{Dok&{2K)UoRf6P6e^!c2 zHn92dr);|44|1xn`L--bc*lB3dpn038(X>o#n%u)Cr~;Yc=8t=f+fSU@;&n!Rw}gY z-OY?YECbW^@L*n8dQ^JykR0V;cQK3Me^0^>V9#WAHdP}DIiC&Z!b_fULPj1q_h!R6 zc+$j=v$;$(i;}T{nNAlE&2Ok?3V47ykxztWS*l)R25-roKz~cZRtEaU9_X+pMjS}B zgkZJCdBo?vvD>)i?*>%Qio2*0dm3!Gl7j~uNB)D$EA|nyhybDFlNOaAp{T{_fBLKQ zNW%+dPeatfVs)3KQGOr8L6SoFLEH~JkZBO}iusLDq$u6~fX`T{#Ze-PRmCyDILqG- zv+9qDyVr0_?74?5r~yVl>&9;Z?o*4^}8>2as zeP_XJ=fDz+Pn00D2`7Z!c_TW6LPxXV9C)B&ReM6T5L6H`Ak3J-XLka@a1Vqg1)*;v z4?7-+3aK?8>5f{x2^cKwe`68VOy(Ms<~)xbV2S{amgh<_UHi7uwLJH%nTUx;%%;T) zyb-}u2fwf~RJlsgNScGS%M8=gmgAduHGZNIrlOA48*mBt;#R8>L*!o7YQ19vH_~$? zhQol+9RqGS_Zkibb&c%z6K^CMgdQ!O(48m5FpkKSdA}`eSix6*f4e!VlFpj{^M5|1 z$wQnhn{$EHS}d2E{!~*V!&#kCLh1Tm0$E7V#S(%V0g*<9L^Rp7g9R&7@_SOb56OPt zCGDbg^G$RI1op&=Arcu)ypXgDX|H!LKq-e>ueDlSbDqWqkp%p^__-;MlSf|q;()0O z;-dro(c)_>nuxaRe>Q?nw8os(8JcrtYxpxOTfs?CX^<1$Ag4j;#^)O2G&s^2r@`}$ z0fdg};-0UUrbz;mUZVh8TC7rkU#2{az7nRmjcbTZK?5r4l0aZS3Yb;dZ&40WCjC;e zv$J_Slv#iPN1)P?Ko}?eZDH*Me#0Py%BWRznK|>u+zt+af9=#!4_PO;&A*zBw4iV; zgx_9m$V4pV+6t_cH$vJV_g33`V1YH=3gO=F$V7DN6U+L<->Ofn>l4@1N9`qPbCukF z!Ngsq8ZDN~H%b^kTd~*(ZP22uL!<_AOO?w9Umj0()_$QTeO zpd0HPe`(~!7$_b)tyXEb3PSFf6Rh$gB?hsdz;SHe0|M$-Act&pwOpJ?fSp$O9HlK^ z085b46r~0iXbtWtVa3k?T8*Q%x1)i;%@Ju18!>Ile@`QuCsRG!ZbN|UffjUfG~i)m z8A{4c2Xfm1s2TYYs`ONyriBcn0Yfo3(HWIa&+Lu5&>A)HzCZG&vjQM9uy;H?BmQ(2 z%q8;F3@ND^LzWx}-?NJxxJBON9=rs2sv(kSLdais2N%*N@qSgX&{Ql_0|#mDG)En_ z@E;mW2biVG`K(q;myjm`BY$0Ytd71lJS|898%PHPo%9E2gr5+9AXTxQWpH=LsHMOO zLkZA<6wYTI9kV-R4=Z<14UF}OTnW#SU?7%{v{^`h1wPLHj|qsB3i)@1#LiBP0-6An z2LI3cr>BKKv8x<(7RyIKB6m;V!I-oXOQ`yZrk5=_dlp8AWW#uxzfkoB} zL1kG$i%DOIQD53m#7)V1i76VRWGx&$5^Gt#;5nt%w$qYlx5p}HA)_mL&X8|krtYf! zKWmsS!`rx5Op9sRdzi9DtX1bXp5lS+Ynv8xJ9q$&^_TjO&`uhaz{l7Cg7SZ2U~q-!>6 zMr8lMlt8y_jeqz~v;4U^rk5A-{6Da^JXd;Yd8wVU%P7mE^=)2{Zs5-+g9 z-++-+H2_y_kT#}z6E-WEys+qTWKtR@E}XcPB<*{*TwJ3U+KTC2f>|40(Ix&RSnV!=-^#jET~#}u zdoDMO16Ek#fDYNfMt(@aJ7vR8vdIx|1wP{;B(+Nyw=l#YMB=+{mkwEsftJYwIggAI=6MpnAU_&YFK|&iDK8x?O*0*|;+KB^tQ$Xh2MW zUU_sdP=KFcDL;KQsEZ3gU0iaOd0TAM^Ggl3xX@sWE6y(Y>;heYC}XyKcE#sJWT77e zWJjSljF^gr6~-gBtgny>;80LsbI(2XDHq&m-+u`Z0H{6kqsj6*C_{|`foA~8j)AL> z&@^r6WQjeBVt_+H7SM`dytKyT6-QE2{5#-m3W;;jK68zyKEq)B%+D`)uigGED;(v} zMxXhde#n{R9W1yP<%CM8ke~9|BX57^^FwbZ2**F<^g}KU_8)SUi^)^xWEj)6MEB_D ziGM%c?{5G~hvjpwpTL~x-M+VzkLdGs+G-6doIa^To{EBNb{=pHj z38Gcy0(!-(=9}EK{44M%p6qgyH}aY*Hdmct{Z$TJuDy(V`sH*7mY!pao!33zsXO=k zt($#VVIPjy-{7MLTz$um(_0XWX|(XS+J_BbD&$_Av9s4eC|TFMB7Gu$t<=N{u?kCltdROxPYZdkTdiFbdfoGJ6&xGpo_|RG zcp^kMaHu6~WL=BFDK6o$O**C#r&bnf{hbQ_RoW!E(66ex0j!ExaaADT9stl_0Ri^= zxFeIsd@m!H%ImcSQ>f^^_F^_+vE}$QcRpvRRo?X)K;X!V^sq{ME{es8_m~{Rqzy`- zGZ&Gj5x%<;bZuAa(sxK302l^|41a{!c@d?WHK~n49FSwg1_LBEprccZ@#J(Tx?4nP z$#AT%3+#S7dmBWw;#mK>s5kG#9g~yFBDI^8awrN)wGgusbp10cA4aLT^YF-xI}M%J zx&0i@t}I7RuFwvz4v8N^8XnjP$TW()@QVFjI}Zsa?Xl7wrzQJ`TVB;7e}D2fjywP= z0^Vkdy(qi)DLV8Ktbn9g+NdrdFJ%%Biy1)*l!zNAdh>J7jjGo1}j&*p8gslWiQ*uR)Fh{SL>NbFtL2vEmYY7aar z39#hmCqi3|<-@<{jWRq`=G_CBGvJ%iDB^87ozk%ysg15y=^tepyMGJL#-Py;du`Bx z@r9RG^{+VHaXR4in$ypmMwS}(!LofD9ZN6pgzj>7dPZO6tlN8)pMS`E?e;6_gSkuD zj?Z4@xC6dR8;o7!p_)>kL2ZK*bRf>3cl%dddi1?6eU+28xY8cz)35UO8S%}d*GC_{ zu79+9&UU$T^DZat9eoqVB?DF$&A1dRQD}G&D@q>4daRLL_tI;X`-GR@qcWLsgJ!*P{ zF}vXHGvCu^_WSPn4t7njZEm&T!lmJJ+klqVP+VA+-oy>KNq?$wK1d5Bk+T+n`?O3L zEErzDlMJt+lrOPG^IBNcu7RCMn485!lyX)oB@~;}EN4gPrK`{nrR zf9sd48dTqHkYTm9-U`FBuEfns>f>_afdGoW zFl!<4FQU@t+1jdM+=!}D^%!@k40X^1K`b?$GhOo|Na`fBx*rT#|5I-09jGfYuKpu8+#hp z*Vh=|<9`Pf__5Hgi)RF{%ao|YuGCjbPP;!h^}x+S<5_I-UVzpfJgWvns!BGpwk(}s zk=^n&@D%uACjf7-IHKmOWH!{Uf#Docm?co##A+d}T(5gxcvBh!+cBoyY_}&Eohel( zy$czD9-?uZ>JbS;+1chF3!;tpNbm_HP;t9=oqyy&y#ZyTcp&7%^$!m>hRcFtw?kdip5&Qk|1e#k(>%jH$0V{ z%*1XVweR@}vJ;))fp}czI>(z%^83YF{C{lHYJi&}5op9<-E?|TsK%T#$Z6Vf`Vy68 zVf&dl<{A+=t(t2no8ysV9yu8A;C{R3mC-AoD|^bemV@m4l3SbWE4YYmGQcc)_y&3< zn!SryML2hc=el!3p71~0c~@4A!~<;L;eUz`klszo7heu)Cw2iFwo49uEoR0eCx3Ly zMN()nc_BRL+4(wVU2xiQ-me=}-E$Z=G~u%`1}R!5Cg(0zx(!@_(<3F0%&pyLN-W(* zQ?QYMm-f%CWC^sUpj@3L+?6(sDQzH4uW<-GyTU;Qhiq#Z&pc793Gv|wscw2O zdL3ENw#xIYN7f~$ICOAfOTp}*)_=4yV@y|G&DuxqU2tnzYeeAZW>=iFT8$0*ya8e< zc7<0~?}$~ee4(o@JUX$PKz}jmZ>_z(zTK~$J+s!!d;YvK_or33lI*K6xW%^RwqLLE zm6wJR1%@tj7Sp(0L_k;UYw=k60FRtN=*CHrq+cZGigY6eLj3 zDv1tyvLD2plsOs*ZG^2>qaZTBkWRNP^9;FJ4X~W1?*uZJax4Tgd(Y2NmPPBkH7>nHeCJN0Vqm zG?FyK66(pJq8f*V`+vR4wSaEa{k|X_D{qUWK7}A{BzvX0r|8oXtdP+v%C|xa-rlE4 zY^O1KuWtYdMY0zDJX<_X$kS?#;BgpK$g3Ah!GvCtWWBga()`#n;`>2kQ%}oQdUJy= zBo&pWZv`bU?j&okiRCtl;k(1P$(G~J3(1rFc%VQkgf^w%x_<;Y-i$DKD~dXEALJ`nb+hQ?lIhYxoJrxq#Amq;pQu;XdY}!2(;gl^(4RQk4 z;1SjH&Lh3&C&jS{uX2TOy71f^SwXl1z|+w`@$jzvFsU2iZig^7!K|kCIb2%=63H1! zLUvn4S${$W5Lf011go|8H@6W*5WF{o|Io+Vm>d00K_~_q8(piTAfTSf~C{! zNyr!gLz`?gkfpJj6$Vb$kf2nPq=hd@l2$M~kOv?2Aj(!KrGns?+3U9WT);(R63GG@ z!`d!_8ij}-|@L4Ofz?gtd;CjALjDNy_o&C**RGQTu- z{bsA_H=CsG1^wn<1}7HmFn3E zD*>i0bq$VtI;)T7%Rs=E{Ea}6Sls>=g;m69qmgpr+qcEy1>%k|Z8SC<=pJt5F@HX@ zkq;iM9`hLBLo>#a*2s{4`s(z#(#w3M$yECI3WC!X8T@N5K&dk+j20j?e|Y&8v2mN z%YYLBto+=Y(a?>@jUj6&J z)%txso34%9roZFBQ|i6X;eRw0SBdYOC4_$Fdb|E1MzG5Tln!I+&EtN`EowQQD#u+j3$> zVS#6;w-`H#vk&=Pcc^y($Uxjui7SL)`Rqdu(RZc|KXY*` znm&TJ4mjxzxIn7O7k`Upeqj^;xfYh5yK2+Q%A=St^>xC*w1d^VYFA?;yZi7pE>Ab=N5rzC2f8*O>lw5HUE~DtSt^ni9 zK64oq^BI5Rq6@_js>ALYB%Ta15z!(20rv;80EqNK%Pf=>cz>UTQR&Jdc22qXB-+ZK z%@3qE?t9)21`R~B-SPa*YUO--P79e=S+Apn*!cE$VFM_xdJYwXPrxkV*N{L(!KQ7`AyVaHF+H@sw;$sxA2 zhLup;WF)oJ+r+N(&ns@%=6fp!TSiRmQm@F<-&*4;JVeP z(T@uQxN%=Q9tN=Dm&QW?(u!Vl;t!lrq0jd|JFS1kiGRDK*yd_8ROl|JuX1{qf)epr z51aRS-+P~Tf*^>Ntlj>7LEq<{B@ml6&OkuFHzPq9h~DQ2%mATc51(kO~_{nG*0Z-$Dpox2*%Z%MAhu?6;u0wr&tk=K%m5VgjuLO+&+7#OP zecpjCEq^R8YI5?jPY5X0O4Dsa)+m`nG z+S-Lqr4LefF>*BizE*k3Q6rI`h5868bh^{KHRT^~ts8zHt(PaD2%YeCo^G3}g7_uS zHoqmTcFta`DX3K!YiL_gkK9rHLqv4r{}Ac!)_;$KCioAHfxDDm;Nu5lN9T=tMf_sCC{zFP@>?Jmlh`A7F#vizvzD0q&bD9O2kgN9!J4~ic0bp&(L>`_W*S9GLlnIzRAaycWW^l91L`9_e zgPL_&)=-!st=0!RIK=?rkEOXmTU{frZGW_5>b%)cxn{2JQo|bIe_42AP{pMc^m^fC zvU5d~xgUCOGuq2Ba9?eY3qufbAwTUAe@e#eOFTf^@dd~VSl=&9%U!B=fk=1tZMTR* zQlEFBKj)ECQ;>hjOCNW^4VCnU&(?Tm&Mx_Z&{P1V}F{* zddni78k5o;gr=xA-7m7gp%UtM&j%XE(G-d)=mYMUgGfPuTdAPE_2MzQTjWW4-rO`> zt=FFx;UbM*cXFQPp1#=GWITOy2N1_rtMS@xs^!bS66 znszFd|AOj42qP0*<5tb)2m14mAA{k=wU6-PpWpnB2XDd4p)8tH%iZy&TYshwFFKZq zx93JO@a6lr^htawxFtSzy_imD06+zCw&IC!{SZ5eyP=U6c~1vG67IYa_{o4^*;H!^ z?D>I0AqOH&G}IM9#mnW27Yhh;FDOFpA2M)MiyVu1GqTu(s*@|-IZ?j&iOf3fums4b z+`tGsZ4^Ai7#kfR>}(?58h_dzuMwf)Y&5r`%0uBV13H7*Xf7TRPvVxvgnBQm;J9Zh z9?T~5UR=FFyYEd)dKe_GtN&ge@AnW({D2wCS}k?g!Q0UZ>3NO8e&5865pEB_^)E?U z6csq8LWTTl7Vg4$I`{kCiTnBHuKqiC$+Llnq8TUTKr{QMDS7%LP=AUODp_Zy6i_8H z@UR?+=!7YDF&Brwn@#32Mq0QqCv+sXjaHTpnH!^~feV9V4S>C+AjK@G?P*E00GI8u zzieDagHCW8-gRK3871q}*g3S4H!P~j8-nGHNN3S2D#}JjCe+Qm=Bt?J3t)U{Ogi`* z2QB_$Usvin4Hm#Z0Dm^!*&JiR4QfH`;#EP4Vi;umOcE^4-3Nf~6pcs;8be zWg}wYGYmbw`r5u+D)b37r8Q*-HOqS>YoaB*-{)Y-umCAA*?*2dk-xue5=vk`h<&6-hy;bTBAx;}=hQA-h@6uO1a%H|8FQjD?td)!9vRaR8PM*H3 zz$18#UWjXOL4W8Bl;TM0l%Nb;aG5l(z9e*AbW$z`rwf(%_2DjEEVw@( z8PK4YYpVx+bH!)Zd=8N&*Br=aen`9_oek?3ojIWwi|4M>46P?uST8X=*%g=WRWQB% z4Xt*H?}oQpKSdteK&R0Bt5s5z$Ne?$2~)2(hIJI4@vb?GQVYf;V6HnZX8tvdS|T=; znwv{rbbn|CEx7@f;|qS2^_t_-Mk`a(kPA}bbg4{EBt3u++;Zwd7%nj8zp{y~cGHy+ zE04&Or??kH31V1{teJoYhJUsvhtA?+2@#stCt$3o8|d z9k@Cml}vS1CSr*wAVzL7OSoAc#F4c;D3t&e+uDh4!Jj%0$-`y^7=Hv%kBD6v0p6vs z>VF&6?I9Bi->->h>1;Fx0Js@K|JPZ!0dYz33x*=x1rR%2uj)n>EHKt+wcqpltb2M! ztge=*K~y3_sYJvCR&Aubvg&IkX6(`lEYpR32a)R(BxuyR8WpMLh}l~dZrQeM|bkHSY5u+iiRr#onm07uyrMCo^dd%EN4&DSaVH$d+k$*+` zr~%{nS=L^Ays+|j(11NH`3hhri{%pjiDm=GCJulCc0D19XT^NK4{+s1v04ywIo6w$ zSuv+01~+5IRFyMK7}yHB-JWwl; zV|n5dgEZMyrw;4nAH0JgqU0Ct^^EQf$v zSK4hebL4f;zHc_26KVHVO_p$|Vz zOhi6KAkZ*ztEp;svvL-7~;|33iD2zC4UE)HPgMA#hpBjAos3HE$F@WX05->_kXniXp`4>KuAOC zF_VoEkba=5`JaExyv|?9AM;nQp^ECA?I!U9?HpJ^PKNy+H@mO@Xvtz?LSO&!7yF?7 zN823Ug5ba$Ns0$K&X^Z1oXA69!NxJ40o`>^=1XcJg`7V zB7;_6^+D2OMw0waIJPw)Z34g#5h>(?g8q$-q++L9sq(bhkqcr1I7 ze&I{NH#?&9#A~;SfHyacysa%67Z3|Jy*Pnscd}S7txs62Pk8*0Z%-5xCuE?W6)4yW z^zT4+59;&{ZhuEDFj(QlGA(a=;fDgF)r3$Hj%-;3@T`cGboxc^E1t<_QZeMQPk*U*>>6gh56zn_6nwOB;D`6B%oY}9fDK>8HXqu$f< z2F7&eH{oCN0I_J&W-qVU0W*KMypoqr|08(tIAzGHN@5PomhCF!ODo>hzD#9mtoIh#uv+$}S676&njXgTkF|7q!; zQFxyp(me6c&MSAset7VomOePiOb`FGaP`KU9M=|xWejJ9(lj6F^~sDlC0rVX_=qFx zpFzeu`7wDS=&Z^b=4C?!Y(4f^Sk{h=IQ#%vqWN(2Y}&nt0XTS)Yn)I zu`=PiILJlF+7rYSt^M$zqsjQ!+^=;XvuxPssE+j=BsPn!onrfVN>p_zYnFp)Cixg&SstIpJ`zN3u*v#y92h{l!;D}U`` zac51TYk*cxJpG+`dffm@K()U~VLzR*@2UHylxBYS&{O!t13vV<4=U>BK{oM|D6)Gv zSOfgi%8K`=0rNZ$8Td@r?YNS4JMJtofBLEC;i9v8Hbu9*nNHZ*dC+gBO+Pq4J?l4D z5YXF$=<=9NfX1RJ&ZSLSFhsC~F#bc>rAdED5Qq_n^E`lI2^Ymat`NWCIrYL>JZA;z zEZL_G&Ao*zkh>1tV(&#;ggsL(2;yLngD)oDP3x zqd9Br_Iu^!YG`xhe*XzESzrh@NvTPH=Lcv>S3bCdhqZ9YwYOObL$SayApY6OEFdu0 zf$>;mv9n^{YF%uwe6+~GV*S;7Yb#@(_bO$enfNv8)aOJpvMKFov0MV~ujbaD7V_5e z&6)pv&fu`NmrSWJ9}2_Ua=*yRH4uMgUaSN)m=JEE6ldg(jZ!CL7&a3TjQ?-85r9WYM8EnUF*Orn@48waM2kJFnB z?r(TlBw)q5#F`~HaxfSXfU_0sG)8gy#B)Ipx>&7<>&KAeMMQyklLaX|FoJ&#j^|)5 za*7ipP;njzd$=%c(*b+^pV9K>5C4zXf4s(xv;6xSv$TQbX3%PRBQ^l)s{tF#zME4h zelcrC%NzgoYi;o#(dKJo>>m&?n?O}$Ey$s$bvP(wzpHx&8)L3D3-Sz%^IcenMH>Kg zKH`}dBEUQ1nE-l|-spU%T4a9&E*)cI+QB@ZGE64H6<`JOz`+Oh5`I2Z4<TPQiMsMzlC zhs@w7MV8Mfd&{Hs9Yn(O<$fxP0l1VrqFFBqEbJuFcQ#uj3GExk%KkNP_gbmp`fz*fgMBjpM+@FKVFC!X?<8xm!V&n86yA6AzkNsgUEjG(hb-miXUdt)^f%H`tA)<^1kh)& zwN3f!Gi1>n!>WIxx~ncigghl8(M(SS!BCNVL<4K8FT3LG1BB965Hw;xYBHrjSd=7q zxOBSLgOns!+#xhU+FXb_Xc}%bAyCXnAomFB*+Tp~1;*Bph)D_^XJ7y_WT%89#P5PO zcZ^LRyn9~fCj!$}%IOSl*TyZdSZ$G`S1y%N=vCkZ5JP{6k=5gchcAZTA&~%t9Y&s7 z*aG7CG7X^5)$Xh~o&k5B7;xA5c>u6hV^j|{5Dt3r=J!z;`tmo*B>pw$>m^?z6>749 z6prAk6JEc2ezM-gH#|LgSY#Pr0$jmhFFevY5oH(AYIU+$F15c9h?g?7%0}AAyIueb z)rovq6`X%3NX5E92`o%{O<)+%Gk5~SNIuj)1Gx;^l{WzHFNC`6G+XlFYa}sISOW6B zWQG#B93mBW9&dk$Qn7}F$XMt+pqL8>T#z0BcOg`&QjmBSuer}adRQ0-64bv#-eD21 zW8x1)xQ=BQq<>EY5AY7~>0hG#{x0(n)+TUH0&Rb=Gq|BaP6(B0*$sfliO7(<@Jyv8 zl^aDF`ypvMe_elBC&gM)ZQ{I+JD0zleg8lHJMf)5olLvqK#*)3%4s@K7y=}u?$1X4hp)&wC?eXz+h1+ z`RIR;ndAcSG`Nyx0j)#@{k6Gqz>CCh)cC<*Nzh(9(2t+>+hIF+d>?IM@{ODIKrt5G zer@<+W@qCa8?B+X;tK%Y`N$gQ*n|4^OBBI>YJs`ThffrjtB!+U(Pu47bm>c zGOCo4sLHsE`8k~001+&I{-g#L&N0my$T=*svNM%>v%NhXS5L_z$Xt3ZkdKnifE;d)@aI2WGDO_{sYm=J zUuDhu8XO3XBJayuu0I(kU}wlUz)!%~WbHVH==#xzv6Jl;4JsocKMOVy_VCG(%&@R+ zK+LFR60z^F_VHqSJ@Tdy%s6oEH3NUNiT9`eRQs={HTZEcU|`Sh-+EKhSK^$YI7UV!>jD6Q1!VYC9$3~Z?9Pq{& zZ=cBxmYYb`%~vAQLO|tDAN5a>Wg%ueFR)^nH8=?nn|6rvL2Yb6~=~p%T1w%!a}iB#PCHh8^7-ec6BKLqeaNRMO*) z(}OY#G{|pNq#%hIv$1LhDpEgYn&|DU|D8IL=^#I@Abws@F=iI`Vn6JtK;W<=j!;Z7^e{ zCN&>{;>3E=+$=xD8NNVec5s+4>Z6kHZ5oBc0px1sRv89)?+~|b#7xCBOc0D?HN@dS zMeUEjQ$qETYy{O_3!G78yi4C#Fs~k@{-m&F(>5d4&f32HnN@vSdc|~uqtJrmoineFCx}D^Tj{#EeijT z09tm;N-7k;(J{?tkR)HSs!H?Szd@z3257G;fy)%O8eo5{!lV(6p^EL?C&?FtV_jc+``&k_k4VG>rh?EEpiBs9pp0VorpiLdT)#R0>Yecx4(Sf(W3| zl_^eu)wWbJ`hWh_?~x+`UN!hP6)p&2|21zgyA@9c%T*S(C+uiY3&miDcpFD638Wq} zK2W4u^do;>*><`_P%i0T!abp+sK%o>iBeA0n-9;dZ{72I-mz#>`^G$icB*^cQLL$$ zm@8Gr-1CU{pERNBY))N<-zr_Mk8Nb><3fE(`7E`Pr#b2oY!|tpiGYEQGiD-{v@`x6 zqHhnn!Nqj*Cf362s*WO<(ab)Kh8MEt4Zx{~!q_nRDT22;q|n|_iaeHhGl;hpM}$yOPrL(usO#_>&8n;rJmA#x#%xHO zTJ_5&ui2b{5-W*k)es%aR>6u!oZc>O@44TMVrW;0tU1&wN}7nk#ly=CbUOPS6~W&3 zn7)4ym=vF+!$|=Cv-F#AqDdxqsc zE@h~EGy(yf1e${s>0%!pSpw||JmXpvfN!DO+wpTf<9qG)PL7fRXWk{>?^8}@J26DI z9DX5}FDl5_+DqALncv<8-X@4t)i9Db$Rp34mO!5XkMwo>?)cmXZ-~UhX}Na6HBo=` zC3i&J6?bGDKUA%l?jF0+T^BY1p7}8ITX6o;l0<%#60xeBzS}6H5tvFrl3{vN7%)m` zhowx9=v_4?rWu-)0H4yEoI6jpS|j~1nwt=(gfjzBc(akn@z`p4(hgPh3+^cm4i!xM zfVSHsE3!~KTh%N2fsg}ELJK^Ds^5Q)Rn<@Ec&E?UtzMwn)#z{)cJ4r+#DfmXN22GC zlBR*eR=?YLfk;RY2+7(B@DVB>h0%DsO4i(DdzsiSoC$0=O{t}6IkGZN*`+hF(8xDS zoul$b=$sMpM+eADb*SW82<*b;A_Y`^%>b1#YKAxD3iN?wL8 z^;WCV2qhE=aTVe|{`FxM-A0&FQk;Q|e+qJemB1x5%AgOSijBCcoNJM7nOJ4Lq#_+k z9U67Brmy!Xq6M*3Nk)vf-SNEXt6$JV@(9fy^0zS-sPYma$p6+|d1cYP!C$76=(y+VAh2oe@a=$2D z+0JDIP`y?3$j8vICm87RHSe5^5!aQTWEdOd8<5IRTw=abVr6KxAsV8Fhcc(16%;Mz za{vITi84O#WCXh8Z5*l?R-WOW%j_<3*QLgb*mXI9V6E1$LIyH6zO#Rh;;#hruG9t$ z#nnznjsa!1wogxLc8g=n{Sk3o-8FF#D|=*O#~BkiN1su(fpves_rwo5P{c+W`8@;( zg()TQ5q{cv3io@@Jpr&#fCWSxILts7fM3%41IoZySV^zduX0X~{%kCC`ME!?sChGN zy9_Q#BY<4N{%f~Cd$WJ?c63ZCz2G#_0Z3`Yvvb&=?11DOz=g>64M@;y{kc+_so{1D z+&@UAE&;>Hz6}&ZS9T=<`(T!?f#zs=w*h4fNW4)x?)R6Kf0~! z;tWKWyvHXRLfd$sD@A>mBf0qRYwN`CYr(1miCB2w8*mwLW&?dzH^;yL@il2d8=w z*Su}|yFcXErd#d`x&TbrC1D6)d*t31EMPiWVsuC!A;pU3&TXVB%Jt48-IpG!> zN;}ukcItl&_0}Hz{-hQyfeGJh8&qo~ATlw@*ga>HjrSAagj=nT8%#e&`jLXSv!{c< z;Uf8(HdWL^7ys0fIzadAY=BFu_@S3nSUb03QM?ep)S`$!so8k$Nh(GGcHbzldlZ84 zhB2vwg5$|?0JmC=NM+9GUjSD&q?FT(fDj8{$7p|)9z<y>zHztv>rDn0wb$bkk^1~Q!f@_h)Ad*-avm4^MihFg|?*Ovg20n zRvS-5(b6ugtNN#lw*!i=5@%0du#kFQz+xg+9&I??@4di6G{~rGd5ZWVT+N(7vtw!-H2;et{rSk6YlO7hs@i#z>@ubN9;Q?gd&tV+J(1BE^K314raIn zM7(6kwoZVmremt^xm9-cBS;X=WNS+r>71fskw2s<#X7zaSoQq-aST!!n8+J zH-teXc!hcc7>$pd^dHOEe@i{RNtEo(sKpR)OMIwMWdsq5X@G>yy z_+_t0O+}u0+e=>IK+Vw&E8|uwjbnzux2i;X*fP&lY^N)x+f6*|i*Kqn>Pvr`!ub~E z>Xo+ty3awu`pvoyp08{5ozeqr6a}G-f9pM0K(&S)bB!i=4K@K|TF=mlGZywkfR&X{ zcaD4bCijLAXEnq)FO2%V-jr5ae{LqL) zU5dWzT%YnLon@5fTZZu4QD1*5En&*}4$--OX5;XFb}hLeu_f+JW(1Zl0K6d3Y)YVu z3t-^Ys|jO`Mr;nnRlPoltNPGX;+K*+Sx0DFYl7h#(7YpNM|67TW0KgQa?=@V;DKDz z2v!=)o3vUwYN;zYHSI^#wbL`h*;sBGYJEAiK3m6q|M?F&_8q5x{yTq8o$oktzr*QC zYV{GdI>;I(BRlNbi7l4)4y3xgc;xH+LzLxw%~P+5h<*oRfwo*a>bXV847vqyH`wR1 zXeTp%dJ`4`{ppeD_Q{dQZn!rB+9~=y|DLGWoF6yeu+zT-v*Ana^Dni);|Q<`A6)ad zxZJaX9xRXw@<4bak;Z=&+Vowr&eH^9AxI~S?$(PHov@nNb*qU4Z<>r&EtPMKKAfu{ ze-(XRd?WOnj_X41-EDTp32H$G5U#BG5G;Tq8hk%-l;zRw=plQ4Eth#I7#kcA+A-(^ z#-B8W1S_^xM{8Coe9t{Eu@sEU0s1LF*Qs-RDSWOfKxG2;FvWk`Eah{eaQnS2KiA2( zds}#Fn_=eCM-D0(k(LWus#~k&-2ja{hH37;5wq{Cy*TaU`m46I-E&PT4$&!2ll~)3 z+z?G{;RL#%TYjSfFZlVTZi)-L0To_E=N^WZyhQAXX~}DT{#ieM4Ua=Je#W!~K)jeK zOR(FNtri#^K*fK-#ShKGvZ7J;Lz?XP+S{Rt05gN~L>^K6_jjN}k7V_*CuAxAeDC;Z zP8Kgi{0IC1Bf}0h;s^p@mVa1fWraOBn3h#lP?r6ZRRJIFF{`icdQZR*=-{|KpLmTk zfTE+|@plJOdo!XPUV5|4V8ikLu5&-3gNj#B^Qb8~Qb>O;%5UH|gBp+ANKt7k1esv6 zG;p9@)j7><=QP}*1Q7gIto`T>Z@7Y1JK&{(e?%l3`Zr~0LFPJWsqgKWRRkTC+yJAJ z8=&F+*X}tRxtmFZ$AL~#+WuM0C&Ko8Af)dNO5y`|eE4d27EV^4dy3Fq$?i8LPu=sW zZG;hGu=IZl$Y|zG*_(~mr6aNNtgXcKC6z$OL>bCMp6cKF?c!iw*L;>-L9xmvEt|5i zj%s&u38%wy0OHiZGl%nj!hyNtyb99>~`67S&QWIzTlGcOXAigz(E5Rhv4kX+F z0om^xA2*&fF+-X$J&}Y#d!!Oxc|eq`X0jnFqUtYHStaNEnM6-~rky#6*UghPnZ9m5no?%z&Ts zo-ls`fwKY3GFZ#L`XQ`i9i4)dfJ;HV;*lRMQCe_P?t*75s34|atVwx)iEF9Yh>9#O za!)ERNOLD%Q7(y}H=ev|TkP}~;2|-P#tYt`T|j85APt%Ovu7&gHSzsmd=>BL+cosh zUU$h~ty+2wT`zR+n<|r5>z9|+Q?}$IpZ9-k=#Z}cv+ei7wxHMojSESfGCv%Uz0AxG z(A_z@p@#7PY!22+&VeCo4FA8M8=QHoHG~Wa(z7;_KYVY<%-kwTI@cL0y)IGcL192T z0}2j!tafH!p_K29rCOig^Ip51a^hMjHbS-TmCq{Ijd~+1@T`det70`R)65t`%u#=d z8lxaA;V%)94$bx?zh5;3l{%U~q?XR*t4)%?=*xyr`QQB!MRg6_X>{BlYkGU%8#CN4 zd1yG^@4c6-`0Po_0AB??DnIPg;Xsp{&!Q$jFCM5Pt^pdeSqniWr7k={NQrrCRFM-X zA=nr!M;llw#uPH50MpS!ErFJ4Vx@n%^2Cb_Q4p=+{csoD0BD1DL?K@tAJX4kmxd&t z___u5TgTZYch~mzd3n?9$Nn!oT){AcT!Z4KgERvK5%V&OqEIotIDF4wx<`865a?jf zJ7+FRZnAygUh{9!BVTKFnJB}YvWt_4B#V%DM)1uDdZDn)`bB_12+3!VuxEe27yQvk zdgih!)YFo`6WrrKfum|r-1GH`V%&iGfSa>P1dq7jFVcLM<{a2NE#7fM5Ds+V<=vu< zCABkOft~aKxi4L&YN#`oDrr@KCMQW6inc969 zz&9f`1$&+b(zJA|<=HQ=nIeB6E2hOjt@;GmGy0)N{4!e_TE)`aTDMwm7~$hRXmFUpHY8znu&y~IIeV2 z(u`L~BVW+(pLMPBOZI#_$?{0l`E)1#Yi>}t4c6ao1;p_Nx;ZSe(7LbC1GDsTbtkJH zV>2=P6Zt+M3-lX}0`MPDl?G!4?5k_O1_lQ!*=s{*0^LO3I<1MCL(;}WWnK!}s@f72 zvt!J@Mq&jTg@OWA5W9b|xYha^VV;az$bNK2EIBE{A2?{MpxuD4%i+eUl?Gu)P%fg|BS5ZIr(sI%qS=Wp??C@ani$^584kg7J}@1- zr)gm~XD{x1h5NCsR$HWS`Pg<0z zGs`cAJxr+~h40{))DLYc+mO|@s3Nzma)|3lZGB;{x{%;(M>P; z|73LtriXC`52OjGS-*C~-+5KJbhEhq!q?09Z`qN&=_oo%4@KFiBpxeINV81o!TUw_ zmh-ikD9;K*d$?N(D#Q{3TCF?jn$;J9P0f9Yle47qmim7hFSqMO96=D{R#JLBU!YOX z4l1N^$-`*PFVp0Kr}@@VwP{n(YBgyc?#*3qGAULxDP}oElbrxw-!zLMRlJk6{;ZtS z9*7U@Z?@cV)jo1^LYY*3m9zt3J3jyJ8zX&^Bqt#+j9h6NFN^f@rRiepO-|29@AP!i zBcs{m^lX1_UN@V}jjwP7sZ>2eXED8oi5Sceq#mf}Xu*N?!S?MHH7}@b87H3!J~%NZ z^axGb@KqlO3Hky!S%N+Od zxF=&H0Tr}>=IniQ9oYylR3oM1*Od>f=j{e&f-EK+`g z%)*%IN;_*%;f34?`#r2&d_hKxFGSu<*gQ5^GcB5}R@h)a@E3|xJ8BC&68=fE9g^ON z{lE_*lPLN-Wau7Wv|1y^pm^M73`)Ds&5f08N$);6kTIStPO{C-sr^1+c0Kv!;?4Wt ze)(~7fv*>TyVHk7ifv{`lx2A5=;Md$Ch3WxahO25&WgiI_9_lCE19zzI-H9U_M$Hk zJ=1J+-frT?T^i7VKenct9aFahrg?$fGLsHmeB_8z5K4;BL$Rhh40K``C`7^<*9S0! zyAQ+<@dE{osF6E>w}Sgos1P16Ak+vR)c&%-enZ05+Tzl^ zvhQWnI;pzV`uH(d6X6p)i@cIz+4cw^a!yZ=mD~4fS!6$Q!9AREP)$#kH%&B#2@S5Q zPM)EE6;bW6j}VVqJ&EH$ChHC{xcnecpaB2;8(7zRI@r9``YA%DaaNmJTQ~xGKsz2u z{(%%a8WEdY-Usk!ly+{w_>8X=QNH(5QF=taC!$FBfIx(Q&21V)tq$WOX;Qcs;l;xQ z*dt)G1NL9l1$h2YZ(#l(?%s8~ZDUy&{69}Y;dr=# zs6mms*#-^s*pd}Hkz_|wlqfVKgFur+1fm%L6iJIbZ#HWlXBmKX zBU$Y1-bLZv8TZrfO&|3RFrw>bGVV*lWw6_=mLVGb7FXIpGaf1Rp01oP8W`Y(N&^8z zAPD|v%p>l-uzg;EN3l)R<~{j?Sip!|jweY$rytuUS=ex-zcs{a+{`ZL8%n?m4+YDi zDnZ>ER(Uuwa5nB= z_?jPh`>{Ng!)eZc(8Y}(DQ;{Q2iD8RvRR@WX`x>BS4nxnN`z$Sm9csSQ^TuYGHTI^ zNRqPqc*2KQDER?HN6--VgK^&AmSIXqH{i|J3=0bjGKM?=otl@#G2YvPln;G8mlk$F zijuU3C(y?gn5ObZ;O*%-94lePs!i$F1V%qQ4?2;Q82!8g_1~|TfN%j6e+~FIOZ_zF zY`ZmKlW7uw4vi&_;TEZwl_$bfJb6Il)m|4ZOZQ|&%BZ!v>he$QM`FkZyN?z3A$bZ0 zr>lDuH`g?iVv}iET8=I@40Ks8*ZCVz8vLbR2h&pyQtdV55so^l=Cq|_8zZ-kzwnlG zMt)f?f3{8Yk7&JFb$yzuXU>Z}o8!=xj>Dgj;@};DrY1i<*;o9ibduSB6vlzrFunv^ z_JeT!>R+|FaqtfXG!b7y@n?^~NB!bwO5%w|Cu2|>5U>B?Ui&sI=KM(_C(LjIn^81L zhwYBjAzCYv9%hTVFYKvgmwIslEPrR71&oI!P3l?}36dep;$k%Bicf{|UEM6;r}8#G{PAq&o}Hf-4#!DYDLpla-Z} zne($FDCEky46;pgDUBe@$$T%Cb20aSF_!Yr{98EB&BZE*NdnKhdT0B&*MF&mF_D+X zoDX=AMg_g@Uj-~{BIelZzR5_D^@y}H=yiWgF@dy6=yk7CP6%`#jWfUfS@rY-d%UOI z5}Pt=r?=Xu?|w<6&cqt?CattS+qrMcwJED%ap!~VCnB!7F>3^~hQf|msVXnAi z_U*R{YsdLUtkp~Nt@ak}KZrdOR6Yz(*PMt*C)c2f!G)MyC&TpgT%i`fjc7)Xk^t@r z?LSnT^5RQ0Bj#6O(6&Fw8hQR%P-lFYghsmg8`c~t*)Nj03&#^7Tn_S(ra%k)exZ0u z1uN^ZnQwtmE!vUgjemprV`=dfDF=Zy0}upAE<61~`5f?~SV45x7Lz9 z=0{9T!n%u{5*{>P{M<*lz)J#jd=mpli8O98!KZd z6@zIG;+8pvU^P>RUi@tM73*w}w8)#C4c*`#>|~8=gCrK+rU8(dYW+oRZ6rr50pwR& zGnQdbTeo$m)pS(PVFCP17yfHzmrh!Ws#e8m{`wACQf8Vp)?{zLkeQ*B`uGlLuFKL@ zn{Q6ZtTbD~Lx0E54w=Vmngh9RTCU`2GvT}hwTx%=`c|&QXTpQks&XGwF6L3sRzKsN z99R9wsr|0$7=ft~I5uN0%)t95zd2z1B*qoPkh+#{l?;-~=_M%gSlYL>f^(%`r+Sz% zDAte6o{1L+Am9RhY3TydR5TMcYSl#Z>LJKFKZ1%JzigODroRr40T=SH21#OXO z9owwVx(o-a+I$&=rGa7Ml~fIjCq4s9hHC12O%&l7<18UhGGEB|E*IlIOI}!0 zt6sO5Fm#ttP`%|*9sbkMah~;`|7oNor%m6!NDN_;K802f|+0;zqbB(SpW- zC!B{1hJSxdBuC@T01eGryHB#-BbS|!&=4$LpbOqg8~nivj%H_xDXvMhxMsOb8e{hP zof)3d{#{&@K;!(~MgdP^k{u(jA@va&Fy$5??_w6jSrpZmd2aXx2Y&4q*VJiq-PpbV zBYK=CZz0(qAP}PHD~WL_jA1-$9Omh zhE%*8D4nmt#pJ3DH$OpE@GGDo1FsaE9Ds8*Qr0;VEqRrpo-g^uFRifK#Qp>2)r;Jw zp!5Lb4!GQSO>s#o`M0;;*6U}+w8HZ<7=$SfMX5iZH$B^glX%HrgizxzQk!39zUR{Z zwtvvNe+^zA7t~OL_uqHkQ04S0&K&OzU5VfWTt#ImxOxCzG7o(0yeTE}S(SXmAl@|q z<0$MidIMo%oIm{`RWsA;l6q+V44vWn3rgN{sQ?zw#{jjhaC9CqKMXf*_tc38_E1=u z7Bsl6o-W(yJlI(-e_k$a^fm?7#(#!4?tkP}wk}(T`MJUy4EYPqL^Z5xyIZ7N(W-bO zPKr2ZJkZncY8DlR=7q4|wp_klF3W9syIfv1gau&n!J7dM(Z&VTi{_kyCs1ZsaJ~hV zj%$!fY{2K)irm6}^H}_NMYy}DAog!#O}NejttG``?t(3b30z-y+`An*^ctY)3V(@V zEcVB%)!YtZ`O@i0>W~M{#cguN0E-}qL2+A%%)6kaAoY1x1m*hqx98k=#b)}Uka}pK z=ppV{tg<&`9WN|CP0O0EuSCzC0>!(e=~dLteJ10 znLZM${W8_9OB{k$_vEUb8!VF;Nq@FLxeKt*8|F5+K5rCE3ny*IcsOY^!YLbU))#R4 zn8g`_F`2ChSNl0if;{&g_pJb97%6p=6^Vx_%S>Ko6Cyl*b>(;iz@hti;=N@g$QNn2 zsQ{xr(pT^p?koqJ86?M4)a87|c>NZ0>=_d3u@uK@CS{PK5v>);C&A3@mw&eE&2O8G zhRnzq+@t9h+^QDbuD9UUY{9K;!Ijy9EA;v^%|OIrj`AZW!{q34sn}{oKtOe3x^F7^ zd@&y|O9rge!XR#}&9Du#<}?aH`^L-VA>DU~nq>LW5Ov#Q#~ZKYe&qc@Nkse8;bUF+ zkb+&rqJ##Q7EM6>vwdVlAHyCC)=c;z$6y*Xj`G|S?MdM5;5DCb@p6P9{$Oq97} zZ=Q>cC4vvE4nTVZcj8iYumJ-Y zoT`$wxAUj0R10X!@!|+nhejJpT*EP{DiwQTa9(W8h_I?ql1d_iGbR|-RRob z7OXL0d2z<;<=rMvbua zLdv9ch(3DnFnZetSFUjgQ zVG4FGAF)}vbAMmyS-oz5ch|5Iha69BhnxrYLy#n1*>f!Vjl~&K?z3Z-@{2cr(=M|$ zSS|;q9OZQ-tTZ$$@XJM71alFyH4v{Hw|+0W{JRHsy~c>j8V1|ZpdnBgjQUvkng=NF zpk6oXR`gE$OuOJ^Ao+OZ!pI*8|+_iX4c%fQ@-nY23~TyZ*xqQ_Wq z$c536E~8_1PrsB@rhwED9R%=1^fo3)U17_7osu!lhTMKz?)}C1y>R~io@Plf@!m{V zc&?B+SAPzHKP&sU&~eCbA*dy?tjLiAeY@xXy=OaZK-Qx6+rW>e_S-#&u$C{xPq45! z@>&l0CG>|0>j%lzD9*dA1@Fqr0ROiEsz3fVXz;yj%lCf?#rK*+sZUokR*T}nfTG%{ z)UBzp0ap#KB`gq{r67RMgE6fIlZjV*@}MeM)qjFCsy%s7wX8mg{ybc4!r88phfSasvGLYBf1VGEALrHtGP!Mo=DyT}9Sq_ZNo{JQg((6&yK zV1FEhlgV$iKaJrXNGczn+AT*tA2rL)J*yr^qz0~ic(38+LM(0>$?GGvF8|n$qaLXN zbD9^ycw((2QjJsp0S_3pAuV1d$&p!+UjG4bt+l`Q)JscAKT!(c5Su5sh09f=fJVW% z>c8Xghlw^Nh2y_08fN2NS_;B?l`VRoQh&2WHm$U1t-=}HPTE!sw*&O=X^`YLMHziw z*iNUz--f8-#parCDpwAuC{oX$qkFXy+8eJIr!*LxvxvTnpo=7R9{BjiaG}mQi?d*Y z8zGMfVHr_}HF8?mVIz)6=pgPTK0~Q{!5_^g&I3S0d^(_VM?81bP*eIctk>sQ>R{!g?5oca?u{CUC`;kGY)wdAf`1W zk>kG%b;YjP2olRN=R*GKl=40m{xMv{q-Zeve=W8jo_w`W!<9RDYHdG0ic|2sej-qe4_~ynIm2-4^*AL2s5w;T_5j9@lKg^PAPae`$MSt)06VJ)k zbAI426-52DcgHnAfJ%1Qc*1HGpPcq1laJ~`)K1Am35^V-Q}_gW8LLfrBFZSpgP!0< zqrn<7NFZx(^(c-t8VxGZq$p#_${|V1P8Ip%I~A;te|hE~84>v!a#L&9sh$#UVmsmH zfu%=ldtPKN=+u8n?Nl7#8h^Yjr=5y7?OjTWo9xQIT(Dc&H=YbV|P;$ z_ZbF$ZZS@?2i~tHxkMOmfE$gBFPnYk{8+d%T6BY)^B^Ncr zqIy@*F_t=rhZa~VdRVDvQGfw6MgR|1DAExHQ|F;nOkOF5V0~PY5`PaCHrU^Vc3q9= znCgvsb5jaPge?IDU<>Qz89`4M&3FU6Q_9VUs#JitxrscMfamHq$6Y0y2bKZI+~#(T z<6#DZ`78GB__gJfXkQJDjq5EBm%CYvQ}C;@{I|Fm@kw+or(PHKyT3L+h12Qa#5iqf z#fSUr`IlE4Y`KNsG=JKj^6i@A4(u~=c`IX@eM?q;N^(B|b4Q_{G=jp0{T7B$+D6`8 zF1PaKax3Ho0vndFq5bLdE}l%_^jaZ`@ru7(d-5QQSV_v7b8Z|RqOSyN;pcKo>U1nS zWh$*>wZE{hZtRw#sL0{qCLJl>bAr8W#F4w(etb)ws4e-C{eO7!efx3F2{g_Q%+$hf zwihoFW$hF!!t(YF{iM6-wI8SCL(!rFHGFk5RnvyhYte`W*zoO4yrjEuTRmj>WSbx= zQAY7Q-JaJ?x{Ga01sg-Ly-i9peJ0k;_s#Tba~Zg^mgcht4ya0N4Ro#475*oOY^DR{ zc`;!uh&awz)Nxai>aCV`e#NhU z$*?tw~Uf5KD`df5eav9W+5i=ND7k+6hNNvE%WHhW&^z-@c=M zT)sk$f$Rihu-9jT@2J=TNjD)m_~HS*AAHzn^T%u5kbgvFIE+wtLKYjKLozQzd9UR@ z<&f?jIoepuIXN!Fw^~k{hu;hapsk8z?1-lqi}glK(vYu?og>T=GOA47H`;vBUss9p}<1HoK zo1S+G{(q})H9F0x&tp~dgsR|-c5ixQ2G7>WsV;Q!Z!7e~zwuvgZG`{{C?7zqhTKw# zdcRWg0jT?leN);0X4EfFDKg7nS{gUbx>N8YeoI!S6vC+20L*q-YFx3t4F}qdi|WRK zetVEA)En*FeOAF5_s z^?>_(w`X5~!OR)FL`GeVs}We|D&D6T^zVAz{?R|)vww7ypoaPnip>2;@*<c}h$d6fqctEY z*yQ#O3x^L3PGoQbmu@_WA23~Djzgr70WvQ_=t*DPK|*8&gh`4~1~U01k_i+5#C|j= zZh;NCO(D7jkkJ#G1_>{}3?Y+@<}?kdnSZ1TBLmbkO2dVjY7I!DY;lWjd?pEKBEhJY zAeaDDBxDqPBT(2!AZ}I(8KY!N9 zKaqzqEk-QztPk%muPid6K?Jz_U>)(*ks%H4QSF)t)@4B&$y=Y+9d13$JpP8dFEAlBI=E_Tt9)U&9fjn1b#2-DXoV zh6Bl=yly~SK*y~@-E8|;>JN%C1Aj0mgt)W}{*_A35)Np;rgtfur8WM(M{-ZY!oS$F zUnAGbg21e4%nF0WTnPIAU=~!DbXl0*XK$63BVs_mvb2 z3~siQ0!?7!4%w%Q5S|y?Nujv^IhHC=unn4Rn^6yolIi>hPUvn^Z;lU6@PC_#k8y55 z$2lzKC=Lr6$>b_%M9EA#ymE-x0LU%KJA?+xk2uB0wp&vDIFzC5e3sj0@2XyQOvrvXZ32a*!D%SIBs9egnC% zqW9&T2jB~5p2OUr^+M)$LR}{_$#2BMX?89nB&M6MbIBjS- z*o{Nx!17DB`tQbH`P`oGH$G(SK8|Q64A6TIasDHS@AwnY6LKHZS)2}!MnT~vWWwKq zNp{r>Ne(^dLC^xq3x75-@?XmfLy_cF8i`m6dQM@G-Um4-P84Yl zkP3q5iAuHc%6}W!aB^L!q35X=4oU5Wv)XVJ#5a-ch=r(1*nfaM6!N_)iY)F=3u;@S z_TmaTVBwIjCJkA_G8Z2oQj6I3!XNH;i(X}OT6C>a!XD5`6jeGa0?Ea(y@U!$2^DhD z-_TENJFggT#shgHjvOR~B*F=R1r>sbt$Z9?_>bjo&*C&VA*hIx7lp&V47JRf$!MXA@I=ga8&T`L7XI> zB{XG^U@4W`5!d>$n}7)#Tr{uQ+_RdsX03HtgY4DjfOyOz4^ZmG6!ryrPg52}7KuFT zY`1YxYwZS&gT_G(3#l~@+{VGE)ihzvTI=ne0e^5Y(YWdE|F6zzaoa;gbZ!kxhJYME zh27nGQX8Q9VR5MX1uh$>V7-eh%%c&+mT$)B3 z@UK5D3OE?SqiU4ZdQGu=%a{rVj@_0;q~9Me{GUNSIJ#3eVny!W)2skNz<1#yAl`F8 zynnr*1wS~&ty;_7gI@&!*j}wUZn_8m&U7Ud%57_!l*fO=G`fO|s*PznHAzYht?1Lp z?X!99v)gU;YBTp>w>EKiYxyXMK+$oX=)@LzVCfN1OSmej3{qejH`Ug$Aj+1B_E>_^ zPDFTNT|yQdeXU^bCl!kA;s1}Wse&;Oe}B}(uiIxu!NwL53`9J!JWKK_>vOJ9yo|}Z zp|EGh=yI!^HUD3P%|jYwUqRpQI{rSZ?Khqt><4>|-TlKF|I5i%vv%0nf4*BQ$KZc1 z#+)5M&f2qJ7vpwS-d1fNb2e+`7@&S4#?tdtT1-!$AJ$rpz5Tt=Z5-@Abz6<6&wt&$ zM)UAMejY&f#BDU6?>F{Z&x6LZr+e^El;3LX9vn0db`!U;`+V1Jv<^eJ)i~UD8_%Ba zx_gcNy}ics=B~TjIM}bjpH|Xn9XyA`)uI{)yEQB>aiJ!tF~nA=mesIjwPp>QmB>_; zf@)?#twX3P#OBnXVU6d_Rt?%Enty|T6VdpCYX54ge{ujqXid#%jrio_1gYE&2YkSAyw*#Dat^Uf z#F}R~X3!L9o^BncJbb#Z5pcaM7u&?!mvHoIh0rF^IK(m+o5Y)kBi7=`wO*R;=b>6; z&q7zW%f)tyFFt6l#<9)C?SJwn%9%P0U1%CK5z2rjV)OQSBNK^dtyi1I?fL#e?OEgb zo`FOmPNJ)3))iyNTd0w8LwSIgwf)B4)2FyJU9mQyX}pclv_n3jypf4Qx6L>rI{Vqe zz37WP-nZ;WcsyR)jlP*x(*ard+Frm5`Yyihe}q|^1PLt)y8GQ6T7R?FH~>dLxW9N+ zV$y5P(LtGQZyekoGy9dt_cfZkW z9n|(e<6zylb{qT84?d5ZHU4$o+WY*kaGR3R-1`6 zGLdfOkYD%k_1(gM`K2buDR>u^Qjsi(VrB^s+9=m@<&c)v1&<2VwRi2idPuAwivyQS z4!i|7*XJ!rIHF+T20OeUQWe4*`Y(r#0-q@~M3Q=)fy~ZXk%c38Z^?sea~MCGa>Mfj zc$y}}hoB-^5+8!gp)P+T$p*y{?GmIa9(+VmN2t3P${yhEq$}Z|Ub$8jZ)NUlZMwLm zf3W^U;TyD0K#Se}s-8)XdL})!y#8*SWWf+*Po}nS1zX~1P0qs+2<8JVelUEWv-rB$14V5iOB5B_Z8mG;ICV$vV45T~+2q<};qKQab9cS*-zmnE>#R?U z84S+2@9x*eg}Z4~M!S__O-YZL!78DiaeVsUR`eHS8JoHNAafHs__qiFTPOuQSQED0 zyDd`!2m9iEYp;L09iO$DwP)*lG;ICJF0chP_EIxF1}0FI@&d*y2P`rejs z$z{s2ageOH3e;hJ-?v%$LV>}p$Xa{Voh;;9eR8;g+X{bUAn|t+R3tS;HU)!p)JU5P zGaoAgB|2Z=(?KdF);506!4G5*#>K+jZ^FukR@BDxFDzKpFoQcfbb-*%*es{nKXR?u zYRRp$w0lll=JSGXkEA|#dt#$|QT}Qx_EYWag@9KDzey5*i$B_GZzXN#Ax1YI@R;6D z#YH+KX-a>Mh}=&bIeLd(n+@pX!2^*rh{{$g=z+i7DGL>E9+I#$i{EX&!ph_%h(rxa9N z{H<2UvOKG1IkM5>y8dR*-}$rq{M{jvux zfN+08R=5Ca<0+rMY#J6UvkQ`vSXgSC2!B+*FoVihxu23)x?RjCK}h}BS*^xX!x){) z_^@i1B18#QFfRrzcB>BD?N@ONp6sn~JuvxRvAw;`UMM%o(m68dV-8b0CA7E zIgh(P{II=E`(kPSKJd#%1~&U17~d?ef}zx0vCddDP2jr3A(`qD*p0V;A2@A}t9PRsx5~2# zZAnZRe;1s%R}WIjCs({A0CR-HkTkL{m@Zpe5f!r}er`yG74WC8aI;{=*6^!o8oRa`JS} zX>M-24=a*N+7csSiQzq%B4jbm!Qx)g=LUpKIEfuu4)e{3ZrmdY>O|a)fYyKT2VVo0 zjcuw)K)zf8VRHy0Vr{{LvC8^@S7#NFHnA6jmPfL_q`>-XH}3ffNxSLPj-8$#^Up+A z55QHK@&N+Krih`u-1n!s{eUpfnkR93XAz@2dhg?rp0!;FUgN;2&`P35$a|Qd#_8Rw zu$Y1(6s{%?$-)Z|QEJHBj1GT{U?mn*kc{9}T$G|g$yb^gC%5IkjFU%uGER(s?0vZ( z-Rp#Ra34p9$W8l%gSsj0XNIQVTp21;Vl~eId<)+A4I?tYAeEA!l}cT zlOWYW6u#x&L}cejUR@P55lpQ{uNN}SF}_I7jWkjXLbTyHV-qfyl37r{S1J&yd~fEH z0A#5$-$?=r%rei9S&Ul3{(()V6P^7n9{y##a2utgf2NDe5zC5jS{Q9T1!+Si zn3%mgMc3eBcuv7KoJxGgq9Bp=oxxLt(>SFiz$FC>ifGbUfZt~^O$)ACuS$A9OVyBn z0L~TOUnq}60A%^Q6il{=hJ46qp6kH?Z#L*BMTHZ3UqYh``uBeneJ9BjETMcZ(Vn=h z{DRF&=yJhkC3Lx9GZS5S_jIY3w785vQx(lJqY2mCe~K!LhvT#qK4-I1_zH|ObAK+9 z3c^}U`{wEgQBisE2VqDjML2>76|f#L=!&qA-Ees{ep&>I7=W~C0J0qr5y61nb7_%_ z-&M*{*@L{UCFFm|7O}*`6pEh(f^sodRNexf()=(1USVyDM%PgvZloq zkQXS@6L|cTPR7{Nuj>MOaU*DWf2s)Omh-uEYII(@h}!8!*!+l zqF{knglNdnX^seSyHh7%)NnvE{D{R|MsbFX7Xd*JA%##I0!BB=g5gj$m!_fGxe3Z< zex{4JvFv{!zW(`;#o+B|AwYnK>g$z(6%c-7mIr_{{y0d5&)GE)NZ>_I3WRW@O+1Bs zJTytAAoK|(T)Ya&NAYlk5XGs$m*xtj@D{Qc#pDgHr3u0 zn&H^u<-i#eu2V>y7Idh`AeQ;_vtWYB3p#!$D=B~I7*;1%T+s0a9mvnikWHvekK;5+ z$z7a^oQd8Q64fX8nyL*86g|Zk3p7?F;3n>*APJ{q;~F5t0QX+kcf70$O{)KjC-1_P?TCDg}Qj(X(IE5g!3HB09#Tan~%Fjw^%s8~5a06ZM#?>NSs6lB6COd*;Gn45YDWHJn399QMb zsP0K74;Ci{$$-nrXvU{Z#wXCQOr9<<8QcKHn?tS~H(`G)2R|`O zPJ=Nz499PAynt_1rnz9O0OQoiHI<;CHa!dqnk!CJ$7%K<9|fvPaZt%M7m_DTj8raL zJE2K}M{3~F@+=h5S$NF!x8{3tgN4BhLoB4A6A4;KTvDn=PbevjIV!X`hg-2%xL7E5 z4!+K&dBJz&lY#&V3Pc@TF8Y60D(d5|Cr zM{wW;7Jn`|t$q;R4N>Vygok(^E~Ud+#3j4@s}u$LdS^j4RVHV^+MCoW}zrXmZ6QbGK> z9DyrA{QNEBd@sK?lPj1FVu?h;n1xua_u>O8)J@H%iCO~qI8c5A(T4c|+%I6^ie;Zq z$KVV?V52&s7i^*nMLd7h1265l^S~>OaQ7PF?$uD@5fwcG;G0-UZurSF%T!mpT!}rCWu^5=snk@@GJh2Q)wuZYkkVU#Y4rNv1wj+2GZuNq*0N1*(TgMoBt;HwBUS&S$*K!`|DWFTRX z(L5xh7KthnzB%|BWOS)4@ScZw)NcaYFlHvuZI~=3Bk_M25I$WvwFp8E*lebOz^m8~Rr?B7)&h!?+QeQ>)zk3#bxQ}?&(ApAO#&3FpS{Z86a@e0z4G} z#kWBnS}^0+)qE1c7W~txH89MnH7$rHsU4DdnFQ z7nL9+l_4sB!y+Cf2LO@JJNd!GV(9=r6y#m1U>rLT`koItf*BuoL&{>YbQI(x-9|2N zu~;f}rM1unVfbkovKNb`cXYv*vmt`vV6efJTS^qefm0kWl)PAszicf_Uj&p!uyI=J zau^jq77(i>h7r+frldT^Qtn`^k_ZOi5RiQ^WHB;-N@12Em634@hqQb~MuF7>%M{ee z2r&;@V51`Lje7WqA)JpHLiuT4M$8~&8NwZlrHp@CTE+vwRKk>`$6~4WG-F771e`l* zk&+(-=7mT(%>`u2DS#w6lG;c)kO1L6V~B$Q%qHl<*D}RI02kj@Sv+L16qIx{o?f5@ zIE3SWm!*V62*S%ZB`M+{WKxv8$YLo!c34^p1Tg%v3|Wu>iarRD5CY)~VPe$eh)J;! zfcU3nNQV%LpOzL90T90|LsW!7{ItpvBMa0f$CTSOx3ugC;Nq2nk`q}hovKwp*U!;=yFHJ9;-*J1U81P1UJU6oZc9J zz`V%V9X$^65jwZPl@Bu32UAY-I3g*jI2j&hlVj6~6rrDa&)rW)0!EtCelO2ZMp z+yLeB00TVYn1(D9$3={c6o4l%rWxaZjagQA@A6!y0IodT!Ex^Wz_YlDo<-ngUS&}z zkTJaqW=yS4XH4rfjZwA~M=_+pfZLCwSVeIU$|^47D|nj51sLIbGsdVQcPFq!#uf-d zCZnKFi#YVGdpA%ai`>5r(jkt-y*uKc7P)^HQ8vti$%to&bPvr{k#>xq^JM&^kYBVj2N|R#ON?jKNRgG!jZMN@63_Dxf}$k+LDJCZIrCXcl~q z$5So~><4MYCAvZkYl?gn=tLS$)u0ciG_Vh%cm73{zC%9_P|78xdts1-T4b)JMV$;TELUw7Lnuc%(rP2R1fd>c-m*kgO+R=73(!-Txd04x$CkCcunfRKqP^_xI`%eH4sYV(L`8;AxCTc8s* z+{M32h$aJ@MQ1$YV*)ZAdm+_#hVV#RdH~j%MG|ZbI==wTQ~>vXC{P80%P3)KxRksI zO-e8_#^F)zAtjH4a!nyzIeY_QrBY<tMcULPJWvGnz zgpxb_)55>I_;(Ng?&DvYr8GGUiY%VPhQho`G%O2$5QS2bDKr$Iam+y}eU-v-1n#DoLIW*GG=gVwDQGBCaKKw*5;??} zVGj6B?^WXaIhLS||PP%vo%F7yydv8W83s~a;Z3YCF!CaEN0q9va?Xr`Qw--ktTFENOr z?`frQ!jBhHwX3B0C1s(HYJzQ4^HQ7Qf3yD9|6f(GzFQlMOA*bWpxRREG72jy}?7|MRZK>V`G1Tfj^#+{=Ao;fSy;eRxMIuS`{pv z)iw)dQGafu_*KN{hJt9){MPcO*!({)?$itvV^g&mEZ+UsF@VHn`$e!VU&(=^FG+9tuv@_Ie)bL%Mu_d^0^>4NN4Ve>?; zC7YD^#&dIN>15O*N#AM5qO|Qq=5E5<<6kEe`Sy4_5t&=giY7T-D|Dax#v8SGYZ8YW zN67s|RTlIK>pSgS*Rh2ytk>-vez$*fu!<*IlCM_GzZ;qF?wMUSH^Tbb+z4c^S{zXc zlYYrAiKxC+>7sZcp7;F+sVvMj9N!&pRsa-_#w`3je)Rh8`0KZK$6vd=JN@$A-RW1a z?#_&xyXt*c$-n?NP9Q>7+0=lD%R+pyZWtZl5%6glwg))$LZ{_d>j=HX+jW1#5&;L} zw`Eu)5MUf{<8eKIn*Y0anSZhG-|gGo+kJAmPtNzrt9_F8dxSsmb;zW-7M(#fXxkdL z1|}sVQziRS(h!vLdRjDcxUaFLnWF6Dptrp4mfb&B?Vhvk z?2sJnMzufYy&rZ+h)vjDe$2O@>h74^}PA9y7zoc0y1*X@gK|4N!! za(Drl3VZ;*6IQ|^r+w=G*9Gm&lYZ+@phlkne2POU;OG?P3>R=+H65qksWP~! z2Ifkm+7)fcTWU0|l7MqShn!OzJZG&8x`+|n{t`0m{)}?#OEiCt*6ZX^{KVoPi;q=8 zOO}oBEajgY9+S*5)Bf;e{%y;%wlDDS>9*z7tnC@aNx1@hTtUuKA02LiS5ykJR8=zo zp^5$leDPNLr|>PBcuuyC8#!bv1ae$a-T9DKCX>!-XrGI^dBg?vuV}Z~lcRkC4nxVg ze=at?3pL&4frWpeeh8_Z8k^9aSs$9unL)M7Ww+P9qW+ETw4Ez(BDU2+KYVCu?n=8_ zo*8Yw(U<{Pf;enXw)}IwwJw(nBiwQv=K<;z-Q8^ujo2#+=5ZEf$Z|O;0k&I?({{GE zuP7g(ZGV>xecw#twxs{u+O=)HNh5HKD3*0?9OpS-abkaD!68@F52+oii3N9hwnMhS$e-C>$%M(4*8buBL$@}h zc7kr)R15C3SmMY=G*O%mB}2y%PpU3x=g7WTkM@Qu)#s3cO~JG}l&M z1e}4c8c3JIs{t2(oU4Y0WC-n4n-FeULxUhjVy7Cao$4lO5A{xUlho@sN#pi59#2?S zh(qRbsmNBW>W%C5xN&<+bN>Fk6MGNRRzkd5wKv-tyGe+p^H|cn7nac?%}tNDY2rufY(N%%#4Q=N!w$12WI-awTYz0FKS2XoNq5ro z7Y-a^l9iecoU=jShpCdj%UXUkwFiBNjQakfC3~@4E?VUMzW;9Dwm?nMA#X+);M%vw zu)W(ituShTcwwiKgNpqWjx2H`GQYdDhg+=P#$!&%m1#6U1%SM4*6S*Xpzx(c9QIN= z%j8=Y&nx#WnMmGfsq%QZTpowLbXy(y?Qv_YD`RO+`X;0JW@J-vQvV*t1p8F4r?Nd< zpjU7u2&0JKb3K4tRG}N)(v~d2o~KeagHz*ji5xqBes}J{uBD82JTXNg8BCr^<-jYD zl*%|h7(Duq0Y+Q8T?mmA+i%$vtg1nF2X|-*;e{BR+r4Q>6AFP;*d}Kij%E@>)5yrx zFPYy|Q*9tZhk7kxIr1)3*7%Tv*kq8Co8VK%_5fML>g6V1N$T_IbcBUW%Oc2t=RWtd z3KsHz)NrNsx-nd7C-EPUEPCn67;s1>2E39)&K80}#s-?gxz|m5L?lu(xbhNo)tjzV z;|tyR4)Y6dD`|vqXH>5jcwpXY66P1udo2;MKKOc18t_i+T~x1cC4w2!>qr+XaX9j0 zZN=8JoOXm(ayzmZKlW@{jvp&2MC1^iLO>&b;1X6qLz4#fv0hIas_UhIuH683jc;ul zw;iGNJ}eL@qC|UfVh?>)nVuE_>WE*CjGhR!xrtF(li7mhl4A&GcKMi!v8_aLaujw# z@#&edgP;f!zXXXB!CZ?e^BE5!>4RdBc$ry{-Tgh6T?hBjEw5$qNIn`Os@yceH=C7z zi3mPha@xwd#01Nku0$3=c<18`N-X5~B*G$rM%fbFvqYlxD7&_mw+9eh!eM9?qB)b&8GM=!qN=Jtk0#aKw*mIvQYjZd|y)bjbeZ|0@rGPpJawl zAc=3ddz0pxzUGh8nQom9yA%2bsG39nZ)9*bHot&9es z&`yEF4&c^Ix!0AD&KksaB)OJIy7Z`5}vDD{4(!vfy?lO~ZR`OG24CdkDQg(Rzq|5=#i<+YHlmP2y?mFO?DBJ__&JS4ht z--j*BR(U`|75ZU=Yjc+l(PeJ{k->y){j_IC4vGUD@&VeAlLVZ9qu-9~oP?D&vx)8K z9)UG#fwl!nNKQ(^JiQMU4Y&~k-w^QIpkM@npLHae&;wsegYL8q;bM&QNsNr@OJk@V z1K%V9&MKSYxN44AweZb%wNPDBtk;bSXTjJm<#yVS*L?Xc_=eouzYV=qdFVfdDjMSn zyh{=3nt*g?d|b$X8p8cM(c^Y)2$a*0+ASD57j-*kA-dTlE|n%$w8C>P1=8^!1c^kK z6yoH&Wt@D|E#KFL!^F!y1w2$VQWl|i3f=mVfjZ*;xv6iq6|B`MyaEsy?aDVKIS1?V zhHY*&6cWufHy7Ishc1+bPkg*NZNxeL+PNy!>7-tv#HF2oiOutQ_+i#}ULLHpBI|?t zWNl%kwnW!PB9QCE`o>STlsxd$!8vzE5OpGPTyR?^FMw|?(3lf7QW^(5z0>KY3wmfe zib39!FqHc;;Hrv>wS5Lx=(5>Xy?#03*7wbHggIIX-~uXP#EkE<9&RX!NY%|r*z=P% z-OL2k>p41q0R)hxcKpzkwe+jRyA%7Nmv-}>AFe<=5bFk|oIhDEZSmzltekezrJMN{ zs5;DZt0SI?Vk`KwPi&-R8td0%d$vBdFs5R8){xcJhL>Pu`ZOK!$7cPaJ0Yx*1C&g* zTh2|p*NE1 ztGBNZGURpM_2Dj(bVW6U^~Bh8>7$2MJRRVvRWVSVyx-N$?k$%(;KiE0k+f%QJMY&? zW05T$Mt!>=v9k(M$ANK~3*B&%)tJjj&V9LLwsjPwDJyCdFh*bdgx&g7Q!TKFArLoz zxGaK_GA|)q#^uTJ5Nguaqavb})8>K+E@<`XnjD9qWYIs~-?siE#KyVUIBuNGHJX;M zR_ojY@+wlNjkR+n2H)jcY+Q}4qN3-HbE)12U)#A-mIGEaSNE)oJ~{0>4#6EVG3Ms8 zI{hjs)K==mzHpIy3R+$GNFT2~ixsDTYRI@L-#1pnGQ96d%_V@grTZe1x0N#@R;5Ux zAGU1PO?pm8RQoQXX%P=%cubFk0%BL*=0;&$JNI*$`;5Z+G!EU;3T3V@)I${YprP`u zEjy{#Lq)D+@ebi)r6BRPS}QMH#eU97eGCmQoO*q0=TEM)DK#V??LhHZ>x)%?yTl8q z+KtVTZx`~gKa%c`w0w#%r(=e@tmm;6()4dSf|jMV$5N(N)Q>s|vL$$I;y7rG3xROdnmE>Dtod&31J~xhtCO@>Z)+ zvWT&Lyz7@eRw0I`h}>ND)=~3+LvRx)y2n86dp_-^Jn8FVjh_Hx%h3c0*5{H;? zOw%c-!UDu_r=WZpbotr7#~&gY<@KZ?>8^DT&#|o@noRCm!4y4!TR<@2CLu;Ey6`a` zs`e3!yQ6^*L`A%c2*CK9EN(?U_e;JguPPke=RS$mo-QfpJ9rBIDuH`g;j_-_tVMeK z8n%-4Z81(ZW`hWCt0@tG=QWUb&_K}>|Jsyyz6LD7FAIqbUfXL*A&HWZnQWz?G$7nv z3QYtnNVr-6aN9vr=*&Su?^`7Y~m5vLN1am88793*3PY;DR$dTd}Fray~Li zhgG+>>9*Cj+VfT;PE(q_y*fMf3w2R62SxXJAztPw$C#)Lcf=z4s!+=NZVRZw!Si|@ z1n5>y$Fa3sroQhNovoHAm9n`yN~SxCxUEOwv$3nlYQ6H8%qa&$`QvWXSuW?2x1MA_(eB0 z4s{7Sfy|+-#CecKFp@(q2Haq`=cvmw>INHq+F8U`C0*a)HsWjye*ACz!nN61`pd~i;hq)AMv;jfhkFya`7JodY^D9KvW z7ti?uXtkDq%k*VyuU=1I?Cs+B?jZzPhfN3_9PBRA8J|NVbi9zUhwT7bohW;RONy1&I6Jp=D_xw?N1 zwEYPx^Q)*oYs(sFxZ`^PbF<#~%;Yrlub7P&@5rKmI%F;&A0rKW(&w=MaR2ec_D9!! z?ss0ffBfnGS8vB4T|b3pW4HL>7-X_Irm#h{cAqNe$^$I&W3=Otzx%q%lfWH7m2U-X z%iZzZ|LnS-9xAoXijB3=7w)d1U7bGGq$Y3C-mWp)H>Dy$`$)9gZOPQ4_2VYA4tU%P zI(##KpV-#NkCTTFt0nwgt}J3rEyp3Fgm*GG8-B&UptUG5Q#Cbx368=kKRTHBA_zPj z@r`8`=4BVOAteV*CnWT0!c3B2n#4)b@_ zSulCg+I{BibhrNFN!@Mx9rush!2SGR@>sNgsc(q`1St0(iPJ^P6`JrBs?UsTO>HMH z4i5KroArA5^5Af<^}JpWd8W=|6~Pq@^IMLW`?zfR@$Ru!wR61AKsrUmwiM&m2DV)2 zuNUu*UT1BsAaAyeK?w?cUNed{6X}eKUu@; zc(8YHA2Tpfp!NLWqAYEIIdeY(o#BY&rHb+mL&H0#E7Kh|8~lYKoE5+w*c-=x%q%uR zJTVgG3Qw+0Jo&M|B5zs-;5QR-ek>slIi zt2GIO9e}4lG>qa0wA<*{U0gsHMP8021SW>eOR0uNszTD)B>24O!ae=WcJL0l z=f_)mph)h^Nd9Dj9F@v{oR`Yn)M~kM-n7UaIV!jN&d3cbL|ooU_ z&(|7xY?sI-Uvj8U^0ju)3j3r)dQTj(u`~y{L=!&+K?zGl`NScA=SEYH?GqB{`xh!7 zefTq!Mxek-Bb#nQNPac#v z)7^7;#7FKZx#6vUIbS2a=Z})!^T$bVJh1Pm^Kj!=6Ke{5NBynl3M$1Vf9;2~ef%DfcDyJ1S7%S zHo-j>MoFlf5bCUCk^YGx@3x%>i8@_CzaAi&+%44uiLdT|tVS<>t7;S>ulH3JRSE$y zYl$%=kLBRbzZ%eN<2nrv6Y!in1+Uyrhquq$H`ZlG30yMF6CCDV2&i}bruW#vp@=D3$wdC91UaaE?KAN^8{~L>c z+0wJ}l^R~QI=h~fNDh(O8I`QaT!4(6tM+Bf9uhZl$Ysl(!z;90T!%+5TOBlak`C2W zdKMjn4ZJv4AmL$ih>tR>Rjx?*PnEXbPAq;7%JOUF;ekt+#m|stImQs^Mw*4M2wN}K zDAFu{E;kzlfKc?*sK+lSH0$zp7Bca>J@SocP_Nq)|9xyvNVh9iw9HD19_jMsXn=Z- zL)LOIk(P&YGiP3kO0Wc3z_4i-8z_35f)zaQDcj(7$fZzcy9LCsD7OG>NlLYZW&=tA z#_Rt^#P8S(f0t|+pLBhEQawK2!fz6kzQ=BVi==M6y+NQk#5^0H~+?h3X8ISv&0 zmx=ASkX5^sh}&e+8coJS=8E#rBU`De;tZ`k=Y05Mkz8=2g!w@(539*nL}ZB{Y1V(e z2V7!Sk(GfWoA3LE;Vn~4)|MJ>l0V;6>H?Y=Ib`T>EtX5r4BM^O*>aic4K{Mf+}~n< z%O&Hni&CuCDV}gT4<}I&@$T<-hTagg!IsOdIbfpL7mNgRZw`h!mdh=$FM-lPL+`%D zr$Rnp-1b4IT#$VHzU@y31Dc^Nnr&Zh^ndi#Mn5eW-s$U|{TFj*|D|8jM$%$T);2_I zck7}xRj_eflQ51EB@x!07Eo1PAQ)H)`ahln8rBm1)$hN#YxE|W? zr1l-Ck>8cXa5^)j+TU9)@%KRd9xj)2UTEyMNNC?V)gr=55eLsI4S@4r%5{t4}aJ(n|^$DFgURs)gg;5ZntRd7~PBg5}^x=kSS_oYE;%Ev&DI?yD zgE+DNn}&E^>H~z3os=kKqR!!-q|HY@kMGRAVS9}L-Z|t*X+1*mf;bGP{=+25K&lU_ zE{ZPvWYUG7zWBx?F)t|+r-R3THC7hQ+lk{bJ1L3i6NA=El$v1)@W)FbxoG8+egDA} z_%SqFGa(3p0N=Y-!*-u-XU%#W$8ftuZsrk-0_k}2Ud$Np6p}0 z?b3^;^bn*8>-|?6PJi#q4LnqiE%Gp!nE0KU_?;bo+j=!btL1VHoTCWYfNzPO(n=b( zd6mEI!`ikVs}v0mKUTwkqiHurbCfa($lLWHya1!({D`MUF*Zjr-i&5kM$>%snrrxl z;nj7P+k~&=$7+%DmiT*c zFF()7!X7#$?x5W2XU~?)A?mj@zoGeW={$&N7>|SGd%IB0`VC!wI3&6McOSTiZ|m|? zlKa05sy&7vT38b*giNB_+<&~yqEK~&@_m1v)$3d5BRj4NsOApvWWz7F@}I`I$CKb9 zl)FxD51o3QZEwFwOa^R->>jjEF4WuBW`2YnuEYi6(vP{7c>c1rwze7SzbHC{WN$Lg{`3a&2R zI*`_rH@UYhat>E)P>Z=h$|bx@2h6J8L~Z# zck|B|fJ1(_Hsp>6&3k@Wuk#6(^JvWQymWv3c)x}c1vZW<_z3#$Q9UY;>e0V#R7Gos z-!=kAr3>YM>kX_PBLOAHPHFWVmr{ON@#*>z#itANx=$CEfGI^|DV>&i++@3j%jvY# zp3^Uk-i8%YoyvsGVu^?aH{#sN>zymwt}beSKF@LvuwdL<;(K z^+kFA6f9pBd5x^q@)p>>O;K7 z!qy6Zeu=v5Eg_5Mdh){+Ne(x2L{pQQ%D;UmCL2ydB~&Gwj^hW8Jt=j0BCA?_3qSS$ zviElDZ5z$H;PI@hps=#J(2Gr4RAfm84RcwNohVi$MY5eZG(VXFniN6^U=TLT|E{io{;RHfYf-N+)|{mvrhJxK*hi^X{%$vO zQQA1MM8>)|6FIOUq?_gL1eQy=@W*>Jqak!+!+IIG6Jkk|%cUd7PByHU&?IZUoPgli zya7%smrH~GP=Ueb_kV`XALYYYL9X2(duhKvk(Mrc{=Z|WI{|4@trvw!r2wQaO!Z3~Nggq}ZivRN|5 zw}_E=g5&MREn`&JGK?aTex6PCNl_NHX^hfF1FrWl>!R+akt#;vUW1j#mGg;rRv>{- z_H68L;lU(k!`#oiy;iH0gB7HIfIJ5GV1BQ^0@NA-Y6doF`72-L60@VBGe{kD4R(rN z+63CEfL0g;0n5O4Gi3AH7pIXo97H}H)cC}%@#l5`mjbt|ki>mLoek%U7Tx79i#fU~ zpqKYOoxN8oshKy5nVW4SM2&yBS}*+3X05nUubm@f`O*C}OQ)a*ZDc`zth`mGjE1#O zfNpoKX_P!KZ0K=I78wyQe^s51w?HH;kt8aSgg%9CVCD%<^O#G#QfmlW8w_&yPt|V} zO;*~_+O#Q40Bfm0aBu+F%BS~90G{785~9(B$(lB#bln(eqx*XOU8b3pMu_z;+quH= zUBd}$%ci>LoIeDLRC=!%3!@zMuDqEr6T>Bdbf22n2qMv z0IPHY#o7hv>LO3T@>C3~*GnVm`#!8>FSkXUks6{1Dz}%WQ$bmabF$>S^_*dMDlt1{ zfW;;2IB?_Q(tJwAI9yh}#?|CqW-1+TcMR-GL_ZJ)5U@8G;07Ik)AHR&BBpOb69hFN zaNM*23w~RNEMO};OMfc}$}@rUnpS>ZXw7Zj6a*CLkc2}7pf%J2vDOkolX^X>HbAJ? zMWSY#*oW0DKgrphy0o~Vk40$H7-hu;uaz%D#b;qJUUTj}j0edW%8DEQ4%wJV$Y&iw z6tYxx;w>X_hm?wcQ3i_80}**0WhV)Gb`$U$=fhLcU2uyLHyUX95?Six1bc6Yjq7#S z#!bxW-JZLkfc?e?flfg8LK!=A@+ln+K5ns}s8f0;7E0%t7`d3xZ0%Jl+eBlM6?EZU zFi_^Drh1^u`HTw9>S*f6x+huj1|)SU@G{kbsv=V}*sNE7=m-b?Ld=pBRA&PP`w+YF9Z?x$#e<$ymJtWdVX{=4=WQgbj zU(qG20Hch%?vsij-mw6cp8LnL?{io|1dm_(ftU-&y?LJ=_vVdupU_iq`U3enp|Hg~ zrKgQceZn_TcTVXJy(CEHQ16_`zDV;>(R|W7YJ{BbeP8@+_vul?7eAXKw%Jw+%u+{i z&wr5esy_^Qf7Q+2iZ0g3jAy1S>km_sfm9(A2WYH?abwU49o30X%M4FCW4dAlHqF8x+mx(;|{UQRZs4%DfuOki3i$P}1Kl z+Vt!41m3_`5y1j406-}$gQmuTYxZYk!Rx#Hr$oOMe{$cV$X6(5DuKDO6-7nI&eJ8r zTYvh&tzQ4zgvm_nDY#Yb0pn(x(7d2Q_)Y=D>%*xA^bMEGjc_Fk{VZzuJ0H21`V|co zytSAs(nFf+pRqZlmug63&=>G7)gWC8krn*kZKKxsC1@yJ8nQ5>f=vGmzE)Y_%)PW# zFbZ8VeQ|%`R{vb9#&)a5)va2uxe$PE$t5Y$6g2SRjZ@Ro=nf=!vC}>B z#+LF#afn4pVsu_uW(k*sj)@1YHB}!}30vT)(ryuqZUBuTkM~IY$}B2lFBdN-^*R=} zU1-jt(V%CX47=@OT(c?TD%ftj$aZB?9ecJVf4)ts-EG~)N>ZH>4~Sz-(vQ|8mD4lR zN~HlP177ax?SB*!0#(yOIDbLaa#~xmSTbUiA~p2ZI%mjkj=Zuc+DX)Vk}6W++#8eSGH5h_sOva>fwn{i^O)t*4gIZ8PynH0&Q5c0RI~vxnxHD)kqszX z5H%X;+ZRneGGxJ5Z-L?ldyL=8CHCMhifjW1WTK$6b!W5m811nouL(>EH&Dp9>*EJG&GvnPROa zW9t7Ll{LsW9G(&RfZt);6x_p8Z46oJ!K=0V_`9h)F~S8(GdVntut-Jt%Fa8c|lih zTxm{N+RgMHTNn(~o)$Yh7f6LlxP==QA0k8haij53V2)oP#xKQNb+Yza{vCHSVvXPX zD36&_3aJ4d zZylLS%hA_bZxzce*ifx8$Ks}zYQ)QhK)h{5^;hF+T;Ix_`4 zTA2xyV7@i=PKx?C#vp*e8lNPR=FY3Aj2vSpO`4Fj41+NLo;TaXK|n}4fT2yAyQ1Xg z&bqP*FFb0rAy;?D+a)xCmyFT@9e)z=5eOd9UJJAln-COn8Rx}!>! zgjmYLTobBm#`(421{@2q?gWK($GW%__zDho3tJhgs;ve^LPNJ|Jq64S)PF9JcZe6g zfuy2!oSb$GCTb!1anJ8LgD?cIC+H<2j{;oH2Y}zA9ZSK!}?`K8RwdDvSnkI>H~gOGJObU)AZyeM~2GEHlp3Ev=>dsS9_ zKoO4hD7KgPsdsE_FsFMfdcNkM{?EM?Jy{cXVQx&v6*{*3buVu=0dWhJREwBpwt}o` zDCuaWE)J2E7d8@0EUeUUIS7+ZEmrzo-e_-%SIQU79lB$$*QXoqCQ>|HDQ7y=6W%?v z4yA_`rw-L|S_%{ThJSkY9}+q;kFL7BLs(EO>BS)mj3In?zYre7_3w;F?iDkopc-u& z`gs?`JDs|Rq>!*0VEmBKW7q%=t+TS-(B0>yI(I-SdWa&KQrvS2GQFefR;2!Jxw&@r zxfQ7jJmE?;IV!e*e1Xbc_;5xrtjBG#bTL20W2az<;yryJp@|vC1=}$$yRZ z$50aZ$WeO_X9@lvWYhsbDCvqK9ZX zbx4T!L|?(NSLLhlArly!#WyZxa_llGMV|D9d#>dfk7ATaYSsk8Zc-fle54l!0 zvc;hg%;NtSg^EJ|kOpf=^bt+m%ti#o{)DCC5YTPRcz?C(aC=C2TMCPYX^?lu*7i`h zzA(Ah_FPQ*E)GDU)-$u#acR_zg@Y3X#-YH?pB|eISEUJXRoy*K8=Ul&V^A*Wy3D8b zIzr49vL?CM0JnL3k@W%@zr*8q3-RB{{gX!*x6uo+or&n;mh*w3fc%i)rPufOI1b*v zDM~1v0)J3};(U+ZF|Y!~g|JAfb-8C^-Lx8UT`)j3k0-pclRYpSy~D~Zse{8$I5Zj0`ar4KPQ}BL5Z&)ek4Kwyvjcqkrw|Umg zO>>l!GE?yi%)K9!*o2Pm!ISr>ine|zQu)QVkAG6;;<2eLrop;dRxa_o4MV+-d+(~a z_dBD|y8=gAg`kk&&=js-{28sBzi5CN3(Lm8Vuh?7-fk3brca)n>mG=-%zSju{Td39 z!=aX4=65j{v8h|(Tk}@<_L1vT<$LKx^#!SDCMUy67o>{|{u zK%>XApH_5dxwHyW?LVz<|EYEH*hHPugc&fS__x)?7tl%8^!BaQtJq9eHCa9uhxKEL zIq{xSl`&ve&cLiobtYi4H=puyDR61Pc3UU-r?udW$z@576Q0*ENOuhJg?^1o#eWu8 zGdSs9h#T64z6+M>{bUT)cw_Iz<=%a&=-h?49u_N~S66;zll`j13dObZ+GMV)jV-=H z)D)}yYqox7Y>WSGJpvc<&L~(NZe16pJL3gNq%IJoMYZgDW6PiucF6m}vD|KDTB=TH z&R+_)(KAxDV8*Iv-mC!mvW_0QuYc9_G3m07Yd7@qvdqFy;e6!4!dhY1Vx9!8^MV~L zfi5rXE5SwOiPA1->4Q}lBoUa>y>Hvk!35buLM2R^V(M=sIL46=yx z-|Jg>{)<5s<_$cr7~W34-!mQ>els2#-tDdEhdsj^-p9Qa{l*OVZf`|@H-7_uHcyk^ z8*dECVkt44^X6qAFOe@E2f)frIOKv55XY&<1n32NDR;ow$qGJ&2cuFbblLD@SyfVZ z*W2ktrcVHH<(#fo1;@>Xcd~k271+;x-;f;{?z34bHE3V$=zSzRM9m`1IX6&?5SQ!d9%tB0Yge8PRs-3;OZ zU`>H-C0G<{>pY#3JVYAr1RIf-M-c3>{6tzdghFwE6-_!-sa;Mt&xx>+mK&0|=+1-( zGaF5WI}@2m6@{4VoIRnO%*ej5vX?!g#Eq!2Q{@#UGoQUyD|PwuwSQ_uXsI?#K!Je< z;Cdra4nX2Yn?e=e5+b8?{snjKA;Q1XwFs2DOS9&;+`WiVII2@X*_8JJ|FyNI#l)&s zmY0Fo-%>H5@O~1ZB9xj%fi>0MZ2~MV=0U!z%^TQR0WaI8lWo?sJO0r=xap-u&Y5Nx&gQyYR-hJS5_A}*z}wA)6ecHu~9 z7sdo2fnp1dC@vt~?6$iAe&eC0C_Ip>ifld6icqa3x;|2AB~~Qi8ln{uQbqjR9b&aT ztL|oq-CV2%T}rH&5026w_|L<;hQR{L7T9>kMQs>h8F7?gUw=A`3Qr-MTMbbri zl-$3{FU1IXgnu4cb`nn)vj&xV+HkOOrW#>h|2D`aLLOC3{ko;@YqYgZb=6AkR<6jZ zT{C{IYU;EK@P4`hbv!mlsF*BW?>t8Fz3Qs9d7^j!E3;$H@53fE0PgxBt=H3a9)B6q3WNT%D=|}=Y=~UgVQZ0A5qVQtZM0!6B!L$d& zAL|-f8SWpc1@5Ti?uyWg_L1_!VBuPEgEB>p=L(MF6PUtj!!PLmmZ7XB(!Y4T{8Eg> zSAPa$Gs|UKr5PGhs$Oq2vX|}cZqe_ZS7=oBat9P=){wosX-LMx+?>0yGp2vB-4kt< zwb6!LW!!9+S4y>}f&O|%Qz}5CdrP4(@fN0E?|nL_1>ZPh;}BdZ9j&_tF~VYntDrPO|_s{r8uw=S#9rbD9AO)*PcZjXs!B>zODENAl|$ex4$0{r&Gb)_sQ{YU~Mkw1i{8pHU z!EC?!9SRLltrw`$gO7d&s=~i~0GEv40SOe*bTGN)(DR9ST3T0By{@c!651x0sow!C z3fsQi4Wv4Ff!VH?-`@cme}5$;!-R9gqVzU0;ta76457GtvEH-5hBV6LYfT!>Nh7}^ ziOopXi^(R_5j&b={*WAqB)4rMR6GWVF%Hl@6D=GkGZ(6&Y6RWi=9mwo;s0PL{WV`{|S= z02j&|FAo|?L6Z9y+VpsH9%+df=;6LXxdgG`r(Kt<=u<}wF}0Z0t1?ZN;g3f6_I*d&&Td9^t@R_rDcT7+kexq-Nyzi57Lwn{8{5{h@^ zr@aLi*C5&{xKNc))bp34qDsx!ANA&_PqU?gT9-@xTfW^ye=3BiD&=z9HWA4`8Dn0{ ztx*JQfBx#;Y|GO{qmjN;N;4f2YsWQ7FYVXsN#RKy$%I>c3T`(*k^o7TOEck?Oo-Mv zTfQHoPntyUpnlRz4WBfL-i~)@+Kij%gC+)7G;zrlO_yO;yu|4B-M)>(uHeqK0W~TU zB)+dg%Fc*ve@{CAxzR3n-qeN?31C84_{kBG0z0{PXFzifrH5^c4Zz)rpY~#80;nh* z!rq|fHljIa{zP_TVrpY&%{D;*!43grwXE6fpn?!rHG(>o))4GB%^ia83!ate;O^3sbCqs9{|;FtjfUDPBzp0_|FFB`{XC_+-{Rlr z`1b(+f4+df&$sb!8~&?ze`{2c)#zi=p3tgf8O^psAgCE{}%s$`>pqBkMiWZJKhgN zD8BdW4L#iJ(-&>;*F7kucfCh{-|f>E-+Hfaq0%3QP-y_LXZ!vGEVN$lhanyA_gD0d z3fbMEFP`^T^h$;FSM++{7b7;DGWJg<(f0W(^~E%uuxY+*ocFWx7erRUg}-#(=ZM0?^u$1qkRfTAMb|$m#P2Qr&>A78}Cy-qrKTa zg?-nX@2}|9;_fDmTY1VK+usjH^x+SXmwL{RX?PzqXGM(t{HX9oWKXPSTTY;DK4X%4 zJ;4dHT(;$$!7%Ua=U;y>_c7`D10emSPKe5k)et*meMTs4J zd?ZwCpWZ3Ox#7L#4=V#k)!`pg(g*zw(sCs z?~b8z+YWVXw&(pmZ;fgGBbIubnV@9OOxKKa(@d?q@0sMAOSbYOL7P9B-?S5ghNitM z#;=P06%(9FKe?9059j|Y#<$W)e{2*lv*PLMn)x96;nC&>d(Dg;77Donr|3K#jjU}6 zyxh2UkiSh-rM~Z9F}Oopd9L-Tq2w4*x(v z40LKVQ0NGBLbwP{+J=7xN0#Qd-5swD^aFDc-0H)0Y!7<>PezR zGJId2F**U2RvtVv*d{Itgt^^MI$g*Dq|iSPGq=SaDAbEAAOI zk%pf*(as2jvn2C-a9J0l8_}5bapUBUHkfjpmJ%Ahg6pjGumS$8I%3xk^rqiMKUdPv zO+>t`B^*tzf8tl$01zRpn^IZlHjOpRQD(5YC3#S$RFOP3r&<Y8f55z6Pl-^IVx7!)YBE*GhJ*V!_o^YYVzNs8e;8#AAhQA@W{PZ4>$fyYT*q{m z^4btrN51y3nt5+`-QM*+>(hej&=g65^}Z#P6-4MNyP)a5L7 zKi&|2KTF+UpHO&`fO~|vjiv7GKH(R#)Sc{;RRvJEsnlx;G4-4F(LOLG_&GOb$B)=G z9R4kIe=pQA{3#3bzqbFZ_j&ts{yF(P?LRXdom}pVV=nXCGzYX44l-V;-H8Tkl8T{Z zy$MC1VRYQ#9^>*x1&4L@_0uIDI1EHfMFw}UshcgALj2YWF$lnS>B!#}O#(@vdAi*O zan#=Po&6W}`l!*MiIU!pUdEdJqoMn{-HHcUf6kAEBRcV4w_9w^0(8vcPhhXa0^LGqyK|Mpu*&@kp&?S+TjvGCEWYQUn{ ze`U{h$$#!rS@RC*utwwMqNCc|CT0n3T}5-3Q0BJ_vor#xVSalx0(WC8?PeZ2X!I{v zUpbMrqkJ30=Ym_4>W-ij>o>!0xTZF@yJ&ql_)I@vcpe~GuIY!VLw=AQ4<<9+kKr=%XRL=q+1MkCZB z?|u8+@Ly}jwn3DPxKuma92@H3K+dHPyWoG13!ve?L}=V??|JQQwQ`-^x(Qh><+S}} z*MC^GZU@=bs!D*bDArzy4ZIU;{LerzJ?zyxqi7sfB3_L z6lu-G30aCOejlD5bVQ=mR5f4xRyk6}7o)C(7 zxjzA3t{21Eo)(4r*TlfgQ7!LkM0RBBmhN+hoLw{uXK!i%g5zJ0A zUH~k1UhICKn*m^_4ONEXKb9G%GgM{72&L}e#cn<&bOVyUm(fap2UfUYoaALxj*=I< zZ>EG6Fwni&{Vg{_rMO*&3zQ}=c0Xb_P3$LVytGEQ5}4galUy%$ze`}df4pI_mr-Nj zrEZKDyBCT6Bg0l z?x-+rKG~Bn1b0+w28Qk>E6l%7eENMdCmT?0(^MO|a|2+{KlzKSzzly8_?7nM`%I~D5PvBES9-7`k$T7?-eC}hCSew6bP8feeT5H7Ht zUw7S{%1gjqILPuh)=hvY{ECNFrK6v@z`8hLQXOrInBl=~6o*rmz{8O!W1Rb7l^u+| z-U;h-jfgA$-G}?ee=r((x!4i|zFbTwzq#_I2wpuFoOhEIJzw znVz%WyS!giL6l|`%Vl-EVX&afuyh2^h5z_>!vDCGs3tzdpq%qv?x*gY(OE*@<=km2 zZ)>rcI&a=nb@e;H2m{kuLhV+PdVQAk?65vhYYVR)iVrkAe@jWNbw$TH(UlN9*bC7Z zj*)a+xox!7O3h(=MxNWZpK1`Q$&$S~GsUESZlod6`DsSoIpBl9j=9}R_JNJZ< z@1h>K(7H_e{@FfZjr3@ProioXzxAn+(s9)nCcVABj&94$x!Plm4H0FO{h_i zWk>ocfAObNr8cuw(NslKJ4*BF6B8K2y*{bnp)y@8D*(b(epbl9Tc48U@@V?#oUcb+#I zbeuODz@;1cQ8LAJssMOI0*8v$dRuLwKU8Wfe=l0No`(fe%#F^^Fm^^`WkdNh;B;DU zDLqo2(vzvbwY^fHQga3e07Qxr0c>vbk`$pb1_;yHxx--1f|(rzAIEUihseCslLMsaG-Ysn&b4RZ#EFS}M1 ze`f5N{OuE~kix+}GDyYFc)}`1$-LjZC>p}(oaeo5rNTJpkIePDd&0WbaLpOTl*Dt-bNkp}$BWnf{C1ko zY9L#EGo7YW*Et#_Nt)Nt?y2_0X^5&Cf6kYh>ok7JBrwpN86LLgjN2YG9OBiY{D1w= z|0}CODd~hI4(W_@H3B%p`OyMquW7lcH9>COB6TeLM8Q6Aj%~5|1f20CqqosC%TLW6 z52lQvkOJcgQo5&PmE!~lluewv)9*+=@`cB6d__G;gerQ$_6eA_y)78`(e{^^e{!sz z{pO-!{7~=0y!9WMQ_#|Qkc|8}t5C7|vnAfKd|;J$W(^O#ZXDs=8r02}OHLJjCviJn|_Jx1p6$GHjdWlaQ2v~U&QPba79U}r5nU!NYwo$_-5i)4j|As{Ggv%8z`@4QKUhO_u?Nek+E_{ZjGN0CDgt&}!5fj`)IFv(cx zSgM)-pC3T!+7=J*ZT^7pI)gIPb#w<-@Lv9Y+L% z%}=CGx4rGP_ss6De`&uE1bgSXe|8HqQ9KsCdwe+FK-Zpe<#^2g6YkDheqypY1~ z-uaqSElid*v^RK|{Vnx?r{FkGJ*V+#jlbkdP$2|grk`a~^riEBlN9s7q1>Tx9p`31 zaE<1_gpHMBWdFWYWt4!h_XrEA$dFHq84M5%s;8ajIO2sDf2+pTg#Wys#&(f!N=0gY zo$5~iUh1^c<-rS!x4kCgZ)FN2OrLqTDwmD^petzcT`8%6buaan4&XK8K8{T>V5!^O zwRAaibziQgh8)olCp%QWgNya2_<|wi%iKnc3!UdoihI{oc9B$9#%0%4rLbVkLSfEU zS>#VqvHyk|N>+82UrQtt+FEPUUzgJ3ov&&ve<~-nfE-N?*HLN!4B#Urvn+I#0hf5N zWJ00b?8$CHe_E+a=VkvBIjBq9_8}4JVi6JX{<)8rlJx;Ef7j*Xj}-h_5MNfxx$TD# zD2u8xOugea?==@Yv+f4L^BTFcDRwrc&VyPQ4RK@4YbZjmxnG<{E(@VGvyj{;!9%8#@*_$#$!c@)Q@E@Lw*!J~AfUj=^+I_xPZcZ0!rj)w$f%6C6@cy*_- z&ssx)nT9w7(sUXrjY1DUrfdk`4L5kz?mA7U z5qR)!1XDyJhhMTG;4BZO(cqY6!8E#I;o-vTIkI_8?iQ%O+B_S@kYogArRJ1S_Ghxb zKiU0c{S*tU*0o8I|4dx6cTeU0TUeX$jQ@8}fsdjIfG((*dJv;#YcgWf^gjDOzl%$4 zf1HBH5Kd3A);_Uj15v}?PFhgymv7dtFA%aB6(mC8#EM&~k zEb-e*TlWdKKdh>%=d3iGV%kfqF+vN~e`HeZ5P7MkLShyjt*ll5y8`n)VWoyj;S(hKnH*6d-!=>ZfeT^*3L^xT`Z1%ZVsi za=d=RM0X==qNa}=KjI>%#>O1^xj0Qle{$##frG^$);93+OSL{hu_p#2`&qrsf6Kj1 zL~nDux6#vkn>^CnsMy=ce7LJ#luS+%GhRQj%NKTdU#~gDTvn4Ow7+zovewn)tGn>a z|9h;BTbjk&|4ke{SjW#BR{o7#{cCZYWmhy?|2MGeU>&1w=*Gse{;7nxs+Q%{Pr5&V zg`M}qE?=g>6c|+WwfS}45;7amf5S}Em<{ zPIu%lJa@F=kiO|!L?g8|jKK1?`1lOQ>PFzEZKUR`tO5#YE7p)?a-Z9c6M|Ks&rLda zlypncOd@23faYW?Dbm=wWaWcw+yu#4GXl|Be}zaUh6E^h85Jc!K>ncLe{s|0GT8F{ z?d5Wzlo(|DF^qR%3kQHGbN)4#M?#-4jNmV`n4^&(@F2Jf2b zav4@MHT>G9K7JsLBWuVEp=0*7htVJQwUgK_#Im&Is18_LgR(71I+x2W?MIf`N+^*U zRsa@gG@_Rpax(XS*maX$f7DmTw#GZj54)zcMpmUR&fRWF8{<`5=Vk|m^=w7;TF{YD znAk8i_qm>-?L0i7x#^sGC5TOJCG)sJ%`B5*$H6^Rytjy%BIKmJ;V)eQ$ z;56g(1R{1O9burQ7iJhi1XW`O3UIZosS-NU_2ufZ%_naO*?QMWe`CjU?&I~VIEt8} zHL|6R8&ZDo#M{v1Zx;k4)m0bl#tQPYudZeK)DBAV{@>ios8GM!C?h|Sh$u7_`(vcp zdu=;&cz=3K+}MlyoEuatwSLuhA54-{1s#G`{zbE{O%1q7!^L9_NU(Yu_;0Mb+kHYq z|9saB%PZ8qNGGV*e_>V6N@{We3Zlc6*@Z_BayjgnkF`1dGZScVP&^7H-9+_B=*TkN=j8BG<|9U=x?^ICyKs+*Myi~^&$`m^%qtNN7u--oP;@Wsi>f+fu=L@z`(wVAAFU;De{7O=Zr_AJ`g^1`Q}_fJ zX{qao9)X7_lf)2&&~-4rM{;}@8^(9~*x*jq4DO;hxF$k&`qafE5C5)(gI9Ig_FvHR zHsB6xjgA9zrxw>MMrh(2S3z+YNI239p2xS9?4<1qTxB<{kW?-l#hjSnW8N4Y#moX6P>h6F8MT^G-8oFaFl0e>HkJVKs@`t zf7xCx;s1HNL%^3^IV~=5)di&h_EK3?+HBIQ)ba$CzKXg`fn}uxUU;>6(ZQp)`S`$9 z7i0|i`kn3msmyu%_0K|U*fKcqAM~4-_jaG+VaEHge~-ul-Z%R~(a-yRU)%kEzfXr+ zoGh>qe%q%)k-~eokGA{X$9-yx2fgn^i1#x<{YqsK@6$e7>+5vC@Br`EeQmq{6aVe8 z1H(W+KcH~F@R9>*5kEbkxYT-)_;JMI*h|I7WjgZ)2OPaBJ1CgW-yhICy^Fo0fG!67 zzk#H1e-x-*ADf{D1l>{Kk_SOrwgc9`i*0|K{*=K3z_!0bf4ud-rI)wI@t$ek2R z`0-Y^3m(Uh0m~6iL!yP3x1xpzfxi&YVLTXRUbym=!zG^el8xTXCvHGP$j*yDzcm{O zf2n>POxCrMKk2w3q0rD`OHWEgU9o%(!3JSiNw0iI_6FT21@w+(8KPp8dUM?F1%21( zN@9%QVKf3$Fij_ihd4S3SDer(byB=z9ipj}V-x>Fl;j8E3PyxA70pC8Sb9E2Y$la7 z9s~kJu<}cJJH;`@oZy%kT-p}<8TWZle>9NAik`N8=D!LL)R-~iZWH=T3>a(=628qF z9<1P~^e|?N%)32U2_s+nF@x+{=RrJa;uf~0C+2dw#pH{}Vw>iDb(T9h|lB^^SrQ3pFgM?*fFM`VM`Cp4eL5#ZcZvqX;HY;3qqD;b#U!AQupD!m2KEttHVz z4_0WRZC>H!PN@0ZC$MeR^0byo#m}142uWvw0%f!8`$WJws%e?JJ+=e4!<7yQ=mJC3 z0v-*a6WslZT0}yizv6=KyfjC}^d=srG}i>UW|XHip{d&oDF7PJ`@8|-e-6;=(~++m zW~_&Ez8)?FaG*Qa3*ty`AlUxGXcU$+tlAuszFL+Jg}9oJ2RD`#A3Slf5Io$S4|y4I z6wT{(Cr^L4IzIz|=uEOy41z2Sc8QchKHAya10WUP_H4ZmT;RdpyiW)L*($dp2P)8* zhxd!*(7Pd40S!30Ixls)f4nO$#I86p_7N=-W)gdt2d`(zCU8GT;8oYAEX95O}w1O zX_hU#1U41DFd?w5^q_R$Zn^n#nGm|Ql~cJpMFcK>(Te8V>IjIKf3Eja79PcE!p!Bd zD-M5oz3w_tGcaDhTsl}ievs~I+r8jgjl?UItg7~jg{8+DJl7o&3F+Giq4onyI&fy>!!wRW1A8Jm+1s9?XHy4U9xo6!| z7uMDLZ`W7?p@>W=fA0*?$RnO&xx70-v3gaSQ?7I|I>4)QF@=bAq&iRL;H|bY*?Ug+ z<8WW(e095ld21r0E4^jL{Zj$q|Df2-E`8VSk4nejH;XpCR_ zXC3lzWIP@ot$lC82Scp#R5Dd!uutM8Zb-*OqHCY}Cc5cqm000)Md;BZ(`uc5SguvV z<_&i&X2Q6n5(Zp{c^p&=L#FUV6CZbumD{sCK)cXLqA zDFN=Ke-lhn0?K)V$So|a$=1*s&q!= zi+i-FDNyIBKekblt3f1NtyU@pwYr?xuv38^_ToO5PI5!JRJ&{JXm^d;fY_H%1Ogd< zmhVILvO;-n+%nN^kj@Xfnw%6Ieabte@!DfLX+kDg=U$SgGHPqf9Ur)WKW1VPB-B`l z#RGdclCMXp!kRZ#Lgxp{9LYo^7@Pet6p~7;3HO=B1l`?8^zP{nY@BZn5E}g!01cxZ z-^s?&#L*o(zEziW?le>GbGbP;RE^1hmUX0!{DdZcPQ295)dR=k9-bUBKk7EyUb_>x zOhQcd`ouK`B5*@Gp$q*W7OHweLy#JJ4=ZCZ1|~%=S9)o<Lx$$7#LE2oZe%9 zx@UE%PcX>cOVT-e1&E7hFH5OA zdcW6ky)!&89QlEJ3LPgdp+~BJ8`9H6=S+_05jY^%6A3=Ovqq+5ik4B>l{sNb>$Dnv zW)#?ab3j?&6B|W$y5sejVQBNmePM>sm}Xx!OQdgCn)x}X<`ECPRKQy^JnIpPb1 z3YM+Op%ua=y@`QsF!C2&t(v7?v_s-qfGf4k4e1CTQ^k^y-Gc7B*a+$OkitbW zUoNlV=A#7Z>Q}6OW$0)$H~>2_5Yr`L%%&M!K^_Ki>`kb8$nwq(sI6`4Mbx}+cqz5d z`Cd$|%Lf1y%No3lrU~5j#U;s0C?2=G_XiY?4c?Cj>fVpzs166hy|MOrtBsnawjD}0 zOOHp6Ual&0cOx&Qb9wiFS**YNbVex8oRj5pgaXew4iuq5$MoY&M+(iFNgcKOp>+{~YX`+~-!7w0_ z+6&GRfqSQQhe0h2ZW!^;w){y5-I!CJmx*lKnSW$z5_Qh%^|MkvXS$v<^}1cGraLmO zK;B5{dZ@*9WV*qB5-WqoY=B1qts6ACaV^xPvjfxS89kXWI4Qx!#Z+2GNQzt7ol$(k zKaWKJP*lQ$w9vyi4j7kRf{z3xswL_?5SiHEnQt+Qr7#4B$m`$oqjh(cY_&=%?!a-; zq&YEE8fm_b&W7y9p!-_2F{kR_m)bh$YEGr&oeAAd$iXQLkck44F6-&V%Wi$cz_wlS#c8aVZj zW}u05oEGVuR1UnYB~FhPVc74twF2@s(2BZCkpEhvPBZ8d$WtjGN`wt0G63K+y2AY1)0nAYxx^g~;JZ{X)}t&4{4hql~_?}wcaZNJ@t z#7yE-ql<1JMe%Hwms7|GF`D?5w-g%5@Da<6vbg2wBu}R_)k2a<(R-6M5=Jp+ZqX*)EC|NkBc9gq3c$=mp@JOjc zz=xWD@u^9rZslv@sS{-v^5cBy;3i)vdTUXyZ;kAh&2d7`K`5bhTLk&$ubg{lxl~P` z6EM8dlz{7Q0xw&Mkcle)>`oV>F@ip@1i+h6CkiBc=#b-Hee;>JXeEq$ok8E;B63B% zNskxP_jM9DjN8qC^RL|kf=WlCu3TLcNmp7J91R4}IehOCEr1 z%1Dh|j4{?n;Iq+d!5(1lG=!f_2$$=?|yA^Aj zye)4T+B3klhD23a!{o-YW6Ua$cZ!pLeeSC<0R_*(!nRyi=g*0(5Y8!AsC=Oo!S{m^ zmkk4=Je~3g1N!+uLF6%zjOG5r^gi(tdY^%baHlNoONzo!Wu2_y)aA>s#Y&Z0 zAtfO%NOZwDWKHrIMWA!10Z;zumXKoowosteY8i3?)@w9kVW|f#@@=MB$t$sd2i&vqMU#iLi8P-Zm=TBehzsOuNigXSK{y+y|S7 zOjgjzr;CSBYQ=NuE^1QXla3*Ooxyu`y`G4UF0?`dn9D7fF3RBYTR6>25hs3HJ1h@E zMNluSMM;(qk^s^o!MZ63cw$g7H2v6QFcJZINR~?%eW&I!h?q_sGV+k}fyZz; zH-n7fvT<=h!)S^>|9JpdKlt+W83+?+{f4!0!gp=A7ytVM1 z&eSW1WoCCDI}?+K#7`U2$*Ay>@)GV`F?y$kNX3rP%k8e?c-lKN*LH`7Nkv1);lL%( zfxy=w)t$f6piDO?!vIV`v%dz>e*opMRpsPXP|$3WZh+Y%KP8?E_NX-SGre2DH6!Xq ze&$6YouVSOda5rm2V&E}L8HtV3rU3*Cj+`zO7u8JoD9uoQbC5qAVUO(n29|q%cTp> zU-dcc<#v~qfy#IHyUsJmb6SoE1JlN{VP&FcdMLN4b%DYTYL?7(4-!s4f3TcrwrT8d z(~S2iH#`Fc1xtpUOXGC+GH`{St5KG!6{`eCAcKKUAf8anwAW2vZg*2Z_hMnS8T)PG zrJM<1ztSp%FSnP=1pdT)Zp54h;2$<=iXTy?xpP!q9I_{hF;jx^dd zBOa_}W&kXu&9vF>e~3DHF?`b2>+!2*yBjwe9!${+u^qB{;q<*}A9XGU`E6$hmw@3j^0t$thE+HxOxjs}$` z6w$!sf{%LLnFuejDqIG@@4-DzQgIG7u*wb!^e`m8j4Lf)f1bkXpq!auL$$s^;n?y1 zmerieAiu3Rd%&?}F=2v+pnNe%)IkhlKU|&~l>PD_4S;HNq)G?hW_{<)<>mFs*~izX zC&$;XFTcBf|LNk5L-Q5xZnDoH0zm&1{pV2`{DBR`s(QW zH$P&CQmWmM71wu&LjlU%hu|w9+d)j1aV_vk5X4xT1Vq_;w#%NSMJ5T*;qr z8z8HN1UtALN51dxZ!eedzrDR_2d%;8(8Z)Yk1>kx5+2~~*kaW4I_q*E3a zfH*+19}bI;d$Au5dr`lGibe3)!qtl81x9@G#SL$Ly}0kg9;&F_1)~T^*Z1HrS#jkV zs58YvNjTJxrzsKcs(NYPPkL#etVS%a$&qr4KtfpT4-no^PzeStm0rcm2me)O7MGKU z+o78qtU&0n7@+G22HYx`j|t}x0Ca-WFEmpGJ++lGd37UIEP42*Etd}x0y2Lh@_pee z2VyGJW>J1y`_lTt<7;gdALr2|28iK2bI5A*P|MlF?i!rP zgwqu=7(_MZXwUr6YAHxS%mr^X@E485euG7VP|lIKMj68k%6#>@RUXP-1nWVo{`~CV z)d(tC!;IysX@bk7oFd}mDbs(W;hGf)(hUa@qHUWGQRads;?!O+9OMJM#HXt$8KzIk zDsOcnuT(5t*~zS}YdkGvR{cXvHhhgxK@G=q8a$AuEjViEI6(XmO>Wx@wUWxED`m8m z+x){d67+4)@ysC=v1&|pAYy2vTY20Kr>^qJNm_h^bai4=C^;NzkWYU&W2!k7fD5Wg z)ucild^M%;M2%ZuEsR3kOLoSovF7RPi9IY&f$hs)GCXMJ~IDO;<9 z74H0%4{oU+^;qA08dQJ$kqQ}WOx25v)&vkQ1DX9Y5QAPoyrMrJl(pK#`R4SthN$IX zR{QS2I_i*z?0&+gt`nq_g`Dqj#)@L5eC1-`1xHOcg9dp%2yWR_A5Ma*oWm(&|73V; z1tEY4OaI9dD;dl)^pC90LX&cp3U423g{O=aG~BhX4yU(Or)7Vu#TDKLtawm;$y@gs zR=3tEC~P>TOme00@+Vg0=b{!nc*RH4)t+BPc4L9-Sb+OdK2~6b%>&X)KzJYZ zi^EbZg+o&HH zn5K6TlCfjF-@t>6k`VK*qg6c8C?K5(RaHV77|N^gj?N$Om&>iua=A5^TnPG*9jOpM zqDOvY$bH6iPCCCHfZpF+?6ybVTwJ33BMMea@V4AB)hj^(6|3oH?t*x67oMoSMD}e2 z%7TIIg`opBQQWXx=AA`1fKkl4V8rc3vP$5L6?T7?kD*3`+G^9(8bwKvnA28daVV-} z!9+|;&3)TOun#|=5&E9j9ElZGQ8eOxqp81Xz-7aLOH`o_3_BmIl&cED+J);~vToOt z^{CYzR#(_9?qPMYHw&QyVj$^SW@I6UewM;WBaO5ovlXdw(uGOmW$++W@S|S!SbmSd zvo(J%4Z#DcI0!n{+1?z4i77sv6k0n0vz%gcCq{Flp<7h_l6%zxh&t-~lMdHZ7Y~yF z`SWLcQany)QhUsGCh`3!N-_uaZb0Efd#J%6vfi%Ftg3c%pVy{RggZ!tz;ZHW8Ov#H zb{hq^;8YJ_@uD`U-9=eG7&A{K%HV%!6TvpRs0HaA6IXZ9zAv>w64tWxj^)uE z%S5DX>KBUj!w}VO2U#s)QGUy&4yb%gTA~Pm-IE#cekMPBegM+}MndzwKLDz6)Dx~0 zGSOUH8ccCcS-hMS4~HWkw7NR5;;)`Cpo1EKeSwr(^keO)hFk?rs;$tuabg`AK(l{% z#u@ebnPx&B=Ke^$jz&Xw?)&h_DG+3O6JW-L!OIKd83-_9#*pBs?=L!1jXvtl`@MF* zlia#zq+U;M-7&YY;$%LfV*&tt^S#x2C8VV7QsKf{d zQQ)!)u6DpW>3t42JJ=2_65jot<;#Ed)v94+)KH#V(wEgBj#YV}(^TUNWle0e>9qAZ zIhwLT4mWAYuOe#0Xqx4;cC)}z;7%LmUhN)ycBw=z`GOBHY=B5xI^s*qZ0qk*9d`?r z*Ob~H6KK?%U(gf>gFzpyjWmhOI=)vxiC9nN-5$vR#z5IEObAd;Zq+_;FJymsTNiyX z%RLJWo3&pKxS<*pJJ_+FYkqm!{awgoL|#ObJY z7drH}Hr&DAwb9b7;JXFE+1Y>0fA$X-Yw9keX^?f7+(YMbem9wrZ=OXopZUE9a7^Yo zgDk%g+YDO2PuX4i2W#Hkk4B3o03qpyi31_Oig zUn~UrXU#zpHUW*^2y-a-AH|S2!UV%EoQ3asy1!gL3q=i?u0p5&8)|>if#r%=77Qlr z&yrcl0!6WSZ{@}OyC_E_5LPT`WLe4$!(?RTyeV@+H!)i?PdAqaz?ts{(-F(9oMpwOpw)hZM! z59AxAsM+{d#qg4?GI)Ow79b6bcy{?!$Xsf0MQhbv>;$k&KTIQ$V-SYT`)RDAc}@4z z_`M1#mvI^;8 z!1DH3wTRArbd0Qsvs_MRbnJgRa0yD44)^*0kpH_ppcDQp07PoO4|dAdeugP!WP*vE zq7ui92$?Y#3%iV-!VT^Yj(3jJ;2#b5w3Vk_=brD+@L_JzavIj6+#!u04jQMeD5SPL z?aUtwQOyz83Os-I$1Qn{KBGs(Fn)56{ihqoX)u@6u#wXcJ7{LrJvM9>j{$J&k9+qR zpk`(_?lFCMI~|OmBJcN>{7E(IcrlGeNjhaG$sdC_3a|17H!t}un|r^ve#_?YE8{;i z3TDCY0CRI|8-K&nhxR6#)i2#s`j!s(Nle4{fXlRceV~7j3~#mhy%bz6x&y<>MK(co zs<*UFJR`jKwqFUjB&<^C+m2PQG6Cfp{EUN%FnFI&hj6#Bc_bau7%U0w&{NY`GuC=V zAAK{X7qdPMn4b>eSH}&Q(ytGUXQxwXi;74N#RB}flwp44&cb3wPXQ)mW3T+YyOmc} z0aBjIaPEKPWGiO%dQvqepgzj{twfF{G~}`I8v$OE9H%v73CqT*;ZbJIh|{+wqQkVX zd`hqb#{(5kGob5mZ&oVc-yQBrG&1xCm5c;0kIN;fNP>oB{wiHA6ZMgB?MVI+EthQ8 z0`MKzR1Jn>x`{acv0dqW%fbZ8pL`WjibD?RXkdT##Bg6Aw8ycT9CzJ{r)fhfN@-nn z!(L5@HD9LF78%MRT5tLJ^TWy|z0duBeAD>PZmSQj(LR6vrtT1(s>uWAJJv-g z-Lc&`q<|&kCScV?T70PjH^E?;I+#h{Hklzh$X>=ubbzQDVwngm8=8XXLCYN$Mfu0r zU-W-QeR|~2yYr$u2^iGW9n+)kk)runDjCqZ#AE5qeJ-q!6NQUDN}>=2Dq-Tj(77M? zG7x{zG5^)>3nvf~9q-s5m4QGeoHUmpy*&kk=(*OP^|vwZtv~0k1193toRi_L>idBC zZ+U?94T-kNy~55Z}U*Y4HC?YXkr2f7H6Q z6$|^vl_%xnAMu^5Yj$M78g0)#+VXu^noieR6pJ2;nR^5)+KNzH4EB+9PZboxN8)`{ z1%8B6$bdE88h#fp!Qy~!&08GbU1(*ONGe~;F1zm0OW6uu#8FD_Uff6 zo2eon)a@ekdUvxvWl7Ga-b0Mmjqz(ys29v=A?OWAH@ur+G5q^HpHu|@Jezqp!xd%o zC@&X3SyTM&tg`rUR#p6PX71Gb9jp(P15|t2>D~UqGquzqu8JSjsiWSz)7YH{QUXZ6N>5^6bX>$ zxGEWpvRo55yP$tb4=wlZr!fG2T8) zHTSSOH$u0VIV>HC7H%Cq{nLLy0OkIQDLUokqC)T4Gl#2F;L>;{!PPA}6 ze2@lls582E2wlwjefq!;MqbY`RvGQw89-aLM1Ll+%&0PJb{u{g*x_2r&;Buc7V)z# z>TiO@SweHLqA^B+(5*jaZbC@W3Ow^~W^U4mI6g2K_ZL$H{wDU08+ijE0XCwH@-!=r zHvq=}UBLb>r-i6h-wS^XAnBP#bQ6t>cev-imt`8mAqfAfsfAR*nYwzqLYa%r-M z^8NpE8p&f#$3)fw@W-s^{xObnQ2Uh!>@7Dp%(n8Lwjc!~yJy#;u~r0GF1Mcb2F-uI zZvOf?>{lisR$)8q3fpTHAn`iu}#1DsC zx%9{(QE2)d<{Nh{;S@_~x8=Yfk#W~?h@(|8@EYd0;@T98Z%D!YZi#DM`YP@a)-a6D zA~3DgR%rNHtcAvnMn|nIoouZFHJRF+n+XRJOx1gd1hv#%@BE%R4|tg{cQ% zkZ>d2-6yMTr!{|JZ0!Fy)5nFGv6G79!Dx8*w(Rd{JfoEnh#D=ts=Tq%906?KQ7K5I zeLs`uh3fjQpHSabxxUxy>N~Rz5c5Y45IuYQ?HLC$apa=T)L0)4IKhVsXHl243ddR5 zj@hE|nhm@HJ&t9z*6{;{gp$NffXz? z!p~`9fED^bM{e~_OKh}u-J0tSyXVm(6w>?E8UoowdB1@uXaBz<%Igh8$#(QUWN2sN zTlxFxAQ-bLU^DYoENy`(+I4#WP)l6~-v5xA6?6n6aTTrEUHb`a$Rd*#%po@cbh+%Xy6i>xv(;-M)n;Gzw1XmW1XHMFuR=n z?F2A^I!y}jiT~f!d$Z>rxUn#>`^ui1x4VE}@1O~@a6Y?hmcciiBtMN3mI25gx9VZj zw=rCAsnI4v&u2tXUz7EGCdFumtaH1|JQr>7oLYaaR&z*0;zXa!>|1KcvOG$V5U2by zX9r#mX4VQy9Xt{J1l|bc$LmQ7uKgyrU}|_T_O3%??>aUX@n8VUHT3aK8fW020Nocl z>e36h_c6;kB&m0yb{{+ZpVlsM`p7QvFMF3THi^Pz?PO-{5W=ctXbYfN7k?I z|J;ADJlsENc>ok%$>bJ294Q`^tog5hY5vdP+@l!lLi8P+;#4zzCYk>GN$`mQ??=nVFu8{tOPuaNR_T|zl)+@i8@VC*3HE_T-{{3d^@AyUX`L% z26#WAp#+1R_}g@0$eR`hW`zP(?Ew&($Z&rt6SLkziv3l%o^(hIm`~reQQMA~FsWx4N;;*)XRCja zhQyi!@y3RyV@#)tfDGbw)N$w~+&Rwx&hkaqJ@!G8atVKlcZOEi2VD&0U+vv`-R053 zNlS^6Tr%ADejMbtt-Hb8ZPTzf>Nk(&zJ95OOH%u|;WQn1j$nh_Jtd9HPWAX8%A72x zaoV_4&~&e$3LKG9-#-N)cK&}3|KESEGAXBRQ+N!Zf%!cqoEf!$8_kzlnJ;08>9G$( z+%af=xrW@Q{#kF-hX+I<=m#G$Ox*J~vti79q?~(*;mwu76k=Y)9WZOFk8x zB_g^^2A9<(0wV#Fm-!_E9e>%$%zf|fxqW;rq8p8Vt?sU_uDZ$xcedhvy_#6Iw@-Tr zbbu(j6#8z3?Fzagi;Avjl&bgSApG)hq^_uEM=~X(8d{LpTUN32OY@@|1?q|sS zf+obd^!2Rn-D-_TZO5|gu6ilO=!X>b@h=~BTSQ~c<`DUo)s=UfDu3ylPL+Qy!xNWj z9Npc~FJqs7N4S7{d;PsV1Zn#2ImN(=(AoJOx+gtpF$$z$hyKj}%FS_RHx>+8YX(MbCh#c~DsGwyVzld7IuIagpxIsK>Ngl4(>wWyo$L;NZ-T@O2=Qu}8Cu-0h)P}hb zSOqQOXKdczymB9h!@qpgRWG8_m&-a9_ZkmvAEWnsB zL+c~DCHwfedb<2tF+Qg4`f6N_xJK z7^!L|4H5f*+=r7yq69ra1-?lIDkp2y1quT}(0>+cy22&WaHm-t&Ubdcd##qwc^caN zx~I2tP10gy{K5!I!kwq#dL>_erjjo$8-YHn3I~bK7nj6ZNHQqIg4wcGDs=@M%@tIx2k+hzgC0z!Is8YJ_*GV&3IX-AebG0X9_ zOcqoiR=HG{3FN=f^Y`V2EcluW=JkoKAPrO>Ntf+--p;{Ca}Y+5*P;b&m`)dG0;N* zH|822{!_(fEm=l-2%`FPN0!mCL4QNL@&p}>>wh!(+odA^DRW^sF0c~l#6!N4qIY6@ zH;JXgb+I(;Kay`-8T|1Qcx2$f8g-iH7RV5rBn$z@Sa|$uDi(QRqRbA#*@zj399^R$ z@l&&&^^Fu%#m0>jZ}36B0PNeU)(VkB)&LfjGc6n1FI3IGsaYDQYW9lk0DpcwQjJsYzAq20Ny8cqs0;Shs*duuef~?AZk5^w>ahN~E?>+EcAo3Vtx-+8*d zoiv+H08mzZ%n}{$`%+fArMRb*k&!W& z0>fI3OZJvrryf-KkPau5&(1Y>op9#j(gUSYDkEWJe*^u?5r6z?w3?$vSE%WV1Y5Lk zym(o)bJ(jTC*43J-Q~Zt$5t*t;OGFLlXlnmz_k@a+fAhQiOoG3Woi$TOkLwM2O_?X zOw2tkD+%vpF0iNJUZWP2$x3Guus%BHCe~o010a z5;BuX$iAFO*neDEViNp^nS|`mF$r0TN$?FO!QaRvWMw8H`*J2>V@qQ!-^ky#BttC4 zH;d8{l1~b75f56bt6M%JU2R6*{xP<})b;=OxdxMIc!-9-at~j?J-CZ%#m7$@r1{tL zFl6}#QDj3rG`~b98RTsCkU)}(S`$ZpY;c)mvsxw@FF^E~V>{3Y+syVIK$`6nN|JWP zZ#}X4R@3V8=0&1;k!W7@Cz}^-Xdc$%Gi`*5fQPIH(BnSx&Z^aUSK4R~`B9e>F9K@; zfVY1y0-FJUKC6!B>FRh(Ika$YNVEfR9X(P@7JUKknMpTHR?oH@lWzT{55}n*s@<4q z#sL09drjh|?j&`SsbyRaYF8lH2F5!uuDnRAkg^{y04HU@FNqv!b)c>s9cdqr z5|W@KhIFhT3WJCpA{ZeLyd^hNmDa|NHS-rgWef0s)ghnrWcu~*nmEEuC%x;NVY%&Lr9A-E) zO*SlRRP%pFUx0FWVDX{^U!W^+$B&RR?Fe~SafD13rQ2xU{hz&yZhXCs{({@+#((*J z^lY(z@jhCkM@_J~Hfd^7QczV7DWKTz!%+MEJ@CjxG?4|WdS zXl0j8_mtzpRuEl6uHex?*`_bb_Iwo|a6e?qe0))DJt!Lefc@azc(eWB-#ZUvd&t7@ zA**w&68U;IOsMzQnuEpzxiT>s4(o$|;jkFl?TK@-uoWoO7n@!asAuj}H|t_ym_?vm zsj&LFj^cD>(ghk4M!mcxoBp1RZ@|e6MZOObb^z1?o*)dnLfAEF zs(vL0-pZm?SO^YA&gIq6TDUy4YlT)g5vrleAv+O9FBXNx~l z7_gdUK?IkXN!f|0N6)&B{a zd=Z!BfnF~jgqW)*v(!X?0ry$Y6Cykm3xkS=0Jk>KAOVud0Fp##he$gQqxi(`My({y zkpF~csApjnE~7}o)QZtVgMk6Omd1<2`7CNQoDE+$8ctMt)8k;uZk7d;s164Gr4)c0 zwW{9g4;EFIq1rGxp|mrtAcgJiczZjp=x-_{slF)P-saLP zq4ENYj37#0QP>HBjTew1KT*oX(d(vo*ilN^aw(L5$rmNj|oTn?h?Idv3ZTR$( zd&aVnY4iB4S|$bnBk2G*lmjRm4)E;`ELIv=)%ikcXx^iTOc2Xt-2tF8I)Jd`0Dy*r zjFfYUAM>-%yG8F;K96r$6ntXcS9q|<1DAYl-0W)aUYqZZ^>-i4cNhA*N9MZ|y}aMe zcUQZ;`v~#V@9ypnB!r1f{(j*kDZ35gdDgw$T{(v`eZs*ED?a+9eRQ_Fa(;G~<~9N{ zfBoz_S=eYe88{vP_WOPo_MU&+#n2vr5@ca-?>l>c<_?}ch1B@;t$cbep7vzice|tB z;Ahu4+#Nx%v-8XdfBzi?ulVcEo(d~Wm9^tGdKyC;UMG~->^ieIk(Q^yv_1hCxr1F2 zw*{~g@Lscv;?=YswAd~BeeFl!VzADXf8f`^6tzJ0qxu!1JEAJa#B_%x>rumc^kMGD z5uRx#w&U5|37DWiE5-ZZ=x}22k18Yh;@R^)UqtFB3uoyhHq;#7d|MRoyZ3ETEK)2I zy*!umUmE!*TK-1L7{2v&Zcil)aiKd+>sp6GLvTIa)I%FygC|_*J8HLPe^@>Ae?r?% zccZ9euz-8;{de2j@Z0I^6~)%&7Y%;)=LY|myE}YK&zm|B(RaHszWdbDYzIxv(`9|i&=o<5~fe)^36 ze$RjR==WKN|L*c%o({S?!`aGzdHUx&{FmpuCw{->)U^9tADi3>ZBZ3DHW`ggX7_Wx zFBlEkOitLkwU%og`lx)RqB#Q&>Yj}Dc+YseXFT2`9`6y4_lU=P#N$2U@gDJic#n9z zCp_K*9`6CINuCYPf3bk?OYtGa?>#NW#}p4;?dkpIQNMZ6Z@5r)G=)UjdQn-asBiby8%!UY(HYN3!}ob`yuu2;DTD#n9XzXh{S|&)!)^zgzt(X0NqM0 z6^8YSJ>$ha2q=^(0SKxRgYko< z+ofXc30c`~Z*K*yt7|uO0g8h>X)9-~LF;NVo{3$F>!SbShBwSViSihKlXrv+q=XUp za%X#c3uGc}pe}=BNZ9_Q#+~ou{THGS@(OLE|h#+zj~G(T{CziXlN zPy6_<3kS?r!drG3cYg+#q&os%0ST7`JOV>kv%!N%V=*#qgW(0Zo3<<9 zR;1IG(aZ+xaE0Gv~&s?w4PtLC!KzAGPc#d2`=(e$fJMktUFie~`W8Yqx4oU| z=Yxj|f~32nGWEmffPJhqAfEL>iqnluQO@6_5@Xb>Gx8z}8e1VlKvs29pu@H4JwctK zLve^c2*YkHU5-VxE+BLR&^ul2-gkDNcY~FGLmK24i5qqAQI~WOj8=|R3nvoG#JkS! z^O3G37uPe#qpJIw#hI_6!^3Jd@Xf;%beC@&zKk|qu;1rsM9yYOoVw{!```*;o+L^K zm6-4&T{cK$LBQ3*XUQKehd#K96EJiW#j3+gQ$(5SC_V9^kuq6JY2O=7uda~g81>$N zkVx98^D|6|E;bxkj7n`dwrH)kSI(refJ-!o`Ai|3k7PzK+AC+#8_46#((&}e4dun- z=!j!PQL>5Sjli{oCpaKa=$U3MMdU4@^PI1`i;XKCe@Qh$MNCb z(00n{2BHp=QYHQMO0Gpg!$z2A<|NI3j^j6z##8XR;P;Y-#Ch8l(nX+coL1(H`onXl zZju~|Dyks3LzvBuH04c5GfmvwI!g-EddcxmOv|U z?2MY|vyvMGK=L)1ec|5FOd$7vDdFxTdbzW2N6lUb?*D-!$@%;eACi!*p$9u7RWO8i zAuMWOez#xB)z(t(NC9yYCw1Y0e&4=_2QFszN2l{Y_IGT08QGr z5S#&_p|tLz^7i8wOz$Tg42%JxGVWJ7@I^eLGivle@WO+;BkAv3npUoV;zqBtqD;x{ zG}dUOa~1hs3FossJo&nBp%^b!5NZM|8JjZDSYLKX7nzPN7V2(7Mc5$M<)R{;)Dq`B zYu^gl7$E1-iiI5c)Ud!!B~Ygg^5z%hF5SfR0=eky2ZM0b!#^PGh0^<$M-A&*X)u}Q z)@rr69p>fdTZ=4kT-sINyj`gloH-ip5BfVv;rtOohLaL#^Bzh)k9OFlB1v>Eq~Qj za$jQ=~#E#p%LEJq4o}W{s07OYzuFocDR*>3rE3i;Fk`!S`8}F zVIM7Z5i2n;&ZPdU;mdbY(Pu>=JKKZSD0M@g33Rlft??_Dg+l^Te=A8dj*wtPuVjXJ z@ZTVTGrJp_GxAKYgKGcLI)NS;-yN0e4iXrm;Ptw$?J5t{cRlSnb;Pr8c>(80Xo@ar z9=niLxpC)f9o*kU2CsuK#Ntjd`%*@yl^JPmIHejvFJx=W$9f_E$^LmwwEa+VuT8pp zT|MS5TOV}S><0{%Sytcvp`cZLJm4ZE-s<(jK6d4%D$12rz2iGUUNsJ(UVdb}`g!Bl zLS=YnPF__@{v*q*$JTqJUa_4}1GWD5D*AtItv+MsSe~&zvsh#6H>}n-tb3-b65Y42 zL;?o{f1i)9PR?GRoSj_$`cN4N3~|e9s0PAnlyA+=Y6$0J+43GC*KiS&&`{Mjw zbpX!?(Wti!(Gozx^-KXoLt6=2tyWBOfKbaPG~J5lgK*SxlO$Zy{cBw1SX?a|Q)Ahf zf}=}yZ>IyjwQ_fI=FWh-XS$`SZex`j-xBvCJ;TgkHlj}#u*9b2#Sz#lYD)A24$tkx zH$}mDBYmAqvH-|B#LaLq%#drQtv9Z zC6t*yx9dp)!T|&R<)fFdO9F5N<6k~@w{iUjf=MZ4@!if&BMF3dB~{)Z*2CZ z7M3KBTqlosUx++&P9DtO4|`cZ;}jun(YNC5?X7ToJKNsg%B9j13@hFmDm5rhFCb5a zwIJ(<4Uk*&e7d17*9ysMT+1xrOIuA}+m86OKyhGsRnsIwtt5>TmgWIi1l_Opv%4Vo zph|z7C9>~5H)C=G?qW#ovlNDHwaYz1!M?kXu~B@T%IGM*OWlOOfizvowZe-FNV zSIZ*zc?tPc`yh&c!B2Re-CSFV`FCuQyD4+^T2b!O3}639Q`{)AuBGm=ThqYzndb87 z#^5+L4jk%^8@v`X)nFr!H-5;OukG!s@q zW9zX4w=WYngsda7+@%g-tyc$!f4w+8xcKqv_~P*3{qbn!@SH4_)8o3;u!h6ou)Y}V zH1`gguifT&bicbQ=6d*<^ZlwA>@#?Qk(X!wU;rQ(6 z>f+$-@zue_)vMz-CuemrLiK)EmVOTIRO|L|SR38%IGyJ!dsuH{mhGWker2^+Hk{p! z-r@`IdT&{SMsw7!_q%}HxYOJl)%)GYeSET?42P|%XuDqwhpqK!NPEmim}7s{M{9AL zQx`jfzb{6-aNfJ}JrCPY>iw>&;>J{VUsZT>qQU!QQ_Z1(*Z5KAdGU0&c>b*D?0#?e zi~6hA#r4?kJTC^P;i#}JMLL+(aClwscXtM_u1ABNj>xnl+xK9NiX9R69fSs*s%m|h zL)__UzX-#=T|jv7k58k1p~ZhfWWRuDc!FAvoRo#^7O)<=J7~C}k;;t0hi*p)8Aq@D%yMxP}I3tTvB)Hm=>dX%s}08Ux8q z4V2GnN$O5!Ztdzl{RU%57mP7ES2(j)&W5$iYyoQp(Uhe@UQe8gMPh$tSH+4pzGyw^ zzih7@(4{`SdUg8l@UK^V4ZV8%?g$oyg}GGG37kGRmp|4zIfC!bkIf3g-CA`zSXQzp z!(rWW4toO&?_ym%#=}!Qh9^7?yODpze#cY&4F2ZvEby*8C?V9s@oWaSLn0i-Y~eAK zCBWJa=l>w546lPchGT!XaI=HkC;a3d_`|^l!*6`$-np2k7hmyB6`z*@3w?Z?B_aMs zyeQn3t~@u(i_VS+vs*Wm&x<4#&mbG6X8@XE=m=x)HVB6deo9?|Vl%>UBpfmkZZp*+~#T^a)Zhf>71EKNyplcd2d zNqqp{6=j|9JiiVD4`fAb9P*w_*(AtF29q*W9ZgcE4?)U)p9d*Qo8%Z~!6$|kmgV3h z1A zzQfPv5nSkEkI4s|1hUhuk#hv=K(ypj_KK+G+D+2{$M-sj+;n-Bv6-7pW0ZAW2YJSF zF%{W@a`n;l^a}TUs_c5?6&2*gbC`V-BUc}6mBe(p;uCTly7|@2O~fOWJD%qx76b?) ztRHqskduFjYkI|npaHGDS)MW%rs+S4N}%WXvIC;ZtI6C=eVk%p9#CQUNQKSjIW|!& zsPI_IH*viicf$NDxI4r!N;8u*o&_17S#cWV%d6k#3>ztaQ|8UnEVyM?q8JsE2Q%i& z_{@tFh6H|t;zk_9shcwYN_Gbii!>CH8Bnm4D@A|Opd-XPO+Of%+W;4rQd`aMWrxDl z5rdY`laSAm-{(wJhli-BdBnZ>@xj%?Y7k11)Mgb)Va{X16Z2?&yJPMRFXv9|zT$~m zHFC~@>aS_uxYxJ?dNKaMbqVf!S7NGwqYB*PT=BDuJ69OS9!j(WoXRZ4sf=kol4?_y zUYvhTiRyE{-O%2|B9wQtg*SC0bRb8%7O;olM8JBD!?B~urlyeLmYeDZ@yKr@JAzFR>S0(vYyVZrnS z2Sj`lJ5rF%lZ2;Rh-^Y?O6UT62Jb?jsz`s`6vh)=_h;@6yW*1vpNTxp(A79juVB}@ zO9R{i@SUJJ3=pYdh>y{X%{-L)!^14$56O}q_(YHWG5*}*l*Ro+c5yP#`C85UI%Ohn z(aS_7SrFld70FLC_4GUKck+wer@urxS?*I7xlg2$<*Db2hiS?mu=? zL&hH>`CTN_NAiQ@I_2|RWh?W@Xf3$IDM%!ryV$0jr896yk8?b}!m$jOG`;dzkVhU1 z0=QKadK+X&2)8Ombj!$b8P`KQ6TE)+(BVM5-GO!X;q+8vqkyG}iPw?L4Q|{c=XxH5 z`Be~g2Nvh^t&xMP($zG8bLi2^37K2LT?+{ZSfC`dDQK*;cJ{y=y5}p&h;!L>8;&aN zdVfo>kv0W`F@G4xDSskvh6j*0u-lHt!{@Cqj&EGJud27}LVi)$z_Hicw!LzG?hQf) zM&28b4}hXRay*w4RstmqahWpJL`FMaJ@iIP^lWO@&~p3WM(| z3|^{OXSa%VwyZ!smjbujLa@XIUuz1{`aFcW%nTd@RR0&ZjJmSbo9MTTCG$4#R+)2cHJN&jbH35?y9FesLaO z$2rf`a;!HOZ-0)LI#>c90$+lcTUY`i2Cwr-e*Np0f>;8^0RfjuSppn?KeLp3sHSYr znd=+5WGwQJ;8fs-@8dLoPt1N4gT^h3-bJS%z?o8a!_o4mc$wvack^nV=RAXo!t|6E zQfB1A#5oGGBy^YO2&Ldy6P8PYzsTKuo|RLc0tka+;pyTnD6wV?1Q{6#1>I#S4p*p4 zkSWaq;hlTJ@(U2G0v;WI-mFr$uXXi=;=-$Jy&R4CA}QI2s2NA&^Ozu<-rDPgNx{;U&`I^&!gf$eXge z0J2YU_gV5Z05$`%MlO?p#VGb}gjE-%Ifdc4U{QuE2#fqu4+`pkQJ&F#mb>2cM3||Y zl|CmSgqxSsu*gf7xJzAI zf*5}-h5wER%z4JnS;*WB&$e*Wdd3zxpXzWpJY)G?oZgVKo@@`2AdRIc3#soSVeSnP z0A&3h7ER1K$PRFShdPS6EP0->qtu;D80p>-y~mT73IZs2nhMaDzW*K=Lng$$vp8pR z(ob0aHui&Yz*6}3hA+gtheVI?AjtC%mApNnb@wDn@Q{mO0E)_RIWcFp)*Cb{iOf;e zFokpk}GnT)93i5X;kt=qwmQpA88Ek=> z7z{sk?+)WxLN^(Fqy*7~%uO%6l+g(tH-h6^+By*CZCDY`TLWze-Bh_Rtj7OGsdMCEbGC=d;JqZv~8ifPw9h_xv<6F$_+VyTG=)9i#?eB*|f7CZ&`L8nIMj3i$65V^w zGP;Jk)tECD`H~C?I)hYHVYi8SJhtnni+5)fFT2Inwgf)jSif-)`D!)iOCuLVqB#za z7hQ$3ObrzF*)l^}apGdf;PjigIlFV0C-3Yibnpf881R7}po<7l~Ae--@GinjZYwV*yI?NV-aCAPc_EM)51ec^9O9f>Z*6 zMF2R$@0fQ(pWQ_9wN9?Ya$N`cAxvvn@e<$7*~vM@idqX-oucE;-@JmOWiXk(;zt7F zviQ`wNQeY+AgGU{_##h9WJ?r>J%(3URP}5+&%x>ifAV#s2`3fW7Lbo+kuR!4q|N22 z0Uo!o2-*AC;&V2UjsNghEFu`^@8-1n{p^N#$%K(75z0eD=sZPOVi6A`x?pibtaK7Y zV(?jh^7i;dZ9y~|oJe`+Dt7w;;Zeex9_oxJKmp}`hun;jRzmPBxPWS zg98&pfB9w1`&teFuU`~{`84lxBq3tTNIpuVLF|a^RQ&3awpq3aGF)h{1$|8!U%#(| z{DS6hka}V=au!Ss0u@;K7?T)`B5azz|TppPb7ZLyWis!o$%Q_yCK#5bVk33e+nqN&~hwsEg$e1`Rd~0`-vqji=Ofses$!J#;H_Eg%Oye29Ung-oW|q@naFP4@ z#RVRWsBTPXBuEK2UDS{SpCtcu1x5E3Gk1Yq%vW0?NV*EJMw+Dh=;Y1G<-zII+vAIi ze}gy2aC|#Ce|vCwaCLEcescB(&Ua@Ar|;fqDbA1IzdOIY;-9JrN(f|d_Wr}=)%y>p zryqdWFODy--k+SIgbUwZ9-p5bobLYoE!a9-{CaVD{Pybn_~IB=sMF)4t5+XRPA_qY zMcU5K9ZGuq_p6J`gY!%69UM;oE+3DMe_tPbIEDA`K3u;4a4D(-@_>^!KSBxbez8U? zC+eYwAlUr2gmh+K8q=a$=m4Ms$pdz))gs5#BPVF_?&I%yh?4LCq!Pjso=gNOod`if zI`;N*{@_F{kp1@~4}^f4>f_-Cf7M5{ z-;m->t*;|PHmh#a*TnhS4aB3t@NVdT^TP;#uSQSW z4(Lt}hQqtz&>z|G{Agrb_R0xHD`86!n;MSuYuGtedvt;AXEYeYo>GZY8lkol&DDP^ zh8f7(zG*#q)E+r_nccnL?Mic$e-+xUh;DlEqS+s2PwM^N@UCIE37#P9-s|;vK$(#~ z!rJraDlM(iYE{w-Evaf*JP+&P4saqT9)W8akw|EpOZ9%<5AEU5YH!jkcCnTKOVn2ID1pSC)JVe$NHoD1y>y}EPG{t)P(+> zofTl)KXjSX9^MJM#+xwsgLh*0?akLIIze8ZT%I0N3-8ym1pU?0-4ZMesP{9)9Z0Bp zg%~*wxDtfaOpCW3&fxRX3Koc;OHOhl98BqMfivz}`p7I9v;f?mIzK zK~O<`*aq#$)>RWOwAw4@6cmYsKAmrBwQ>D7up2NG3+Ti$VR=la!6}Vjawj;?I0~1B z+7@tnxN{?C4NcF82Spu?86CCe5nQ5=oY$mPh&qMWv?1f_Oj;2xe-~tX^0*JTro@vm zZyMv?%#SHk7NXRW<${eQO#wPw&fH{7-7asGQ+4XcOA4raJWo-2l+AHKgo4U4VO*O8 zv0||PRBODA9^iQw&KXes32({`a$MI-8qx%rWXTiJK*JF{ouK2pjE6nq$nh~3Pd)>i z%WLrsv_zyw8$}Tie^!+6oVybdO$uu8geE5>ZAj@S68669tUvp190QIujoGZ_b zkq#%CVwoUH&VN%oCybc+X^?Tj2M81-=Oz);# zxOg{ZDOcD-Uj7sGEs@tJFfzrWKaF#!(I<+Y#XcvHPk4Of)9F)^qK@OIB!_-(?1DQ+fTWd01pX7=S@8lV@)S1kr$1iHzm~{FOg=mV`VZMq+T4xIQn$r(z*88 z%{Umx_z+P*49`$|4F;hA77Q_tT%m(NeXSMlF=e|E+59CdPNffN;`cR8-yxr^&N zX`JwM^GG%Vw2n~y60M>=zJm&dQ4sQ@Zmv%q=m#l(@@dq&jd{!pjcf9bw)|KG8p`Y*AZTDR<}G@_FH>oqqJ zz%jXh>O5Z!uc0+x&8+^jW>&Jl>pRzjozCM?1J$|&1KI4`!|VEB=szAc>cbX3pn(-V zHtc#m42#p#A}mhr`n>o6W=HU&MfZzw@w&J!URC70Gg$l`rrh|T`5NqoKndfq9wsZQ zf0h>Yq?F_dm{5VSQSyE?Xg0WoRKxDqv2d!?deK84%NqKkh@t<4B4mc_m@5+CkafPvtFwBWLhzQ8j%a%C80LL(Ofj;u` zN~0FCJC_ev^XPZ72U#~}QI;jbiloA3f8^>w+Gf!sW`1>m>nJ?EVG&E|@hq10xwEGbG&IqSPl)RmW-8ojg{xC@qm`>yWDs-p zD#+<3=IT&dmt37_b{HNd(X8%XZL#}-NJl_LA>ScJeQA@e>FZ* z#*~0^;wctdl7*gTv2J8jHbWEb;zmHqibV;NQ3_B)89DQR5~RHEvK*;%p_Yl?-_yL7@lE zHExhlLj z`}p}6>daSu@???X?y6;E- zy{hl}(b+gNA30oT2%3=lnzd!JHC|oTxWY=lbenzr4D~<2{{w^!M&H7~p!uACsD_ zlz3<|g?Yq}-Rgi$GK-A4A;}lOgN_?o;4pEZNnrH-UTyI)N%HMl(Ok>)OC(Fg&FhjM zT{(suqZ^_eA;AI34W0_d!Gig^>I@^}$C407mnUpTrmRV77CfBhQcK`S=Pvg#JxnQU2hy;Te&`$^xjtYJszX)GKVb#UT{qSBth^&UIHr%bOnHS!Sz$Wd%Qh(5jfBn*u^p12Eqm^@{wfoF$ z_n*u325e0i(k!w!ICBo2rNdeW@84Z;lW3Y)to64K?=FvxIDgXxk#nPEd4yW`e;`ZE zPPK|>E)DmH+i;%i=gGwRBMljWot~JPFBJ2)S(PD(YOv?6kOh3*)49L;3(L68ssmf; zz#lW-e)7`pf7ge@!EiXT`xc5AcolxIHr<<*4h!EHX9Z*-zR^zunWAtwr0KW(no@+& zir9GY(+S&;z1*!*GoXyB&Vkl&@$_)3*({pPm)a8eqTQEWy&9qJ1&sLV$=R_v;G5`R z!yT>l4bS$G^&h&Jg}bCUg2eoZ-_#=C0Om)lm2>X+e`~S{QgzXoEBz#?AMN!6eAaay z)S|)dv&Ol6$8%2qP_KEjnCgkL(NcXis+M92XH+Q6LI#n`vdlU48JW^+ygx^2Qu$6DY z{&@WN!*{3e&Sidr1s2*Z^*To*IB32GBd}*Hf4eZ_c2_pV+h88%y#8=@czN>f?CR*` z;{Czp;g4Ot>!nyp6fhXxHNoVlH>|@j9b8_XpS=12yYcbq@!{n=LOGDtqr*jU7v8~Xn)*sNfS!kZQ;n-#Zr#qG4Xodr>G ze>-y*g*Pi^vm%I!Brg)LNEU_9CdDLm{Q~|2br)GN%ZeXA6*oVAD*PlU{KT6Tev&MJ z6EFxHD){rk`N_en(_^D=jV@_62M5i6xXn*jBmS_{+`DQ5cEZYb&iCMiiV~6Qlz;;_ zx(0SRy6N=>H^J8eugr6!TRiM)j)OTue;i>nkr^}#ITTZXXIXz7N!P*3l`ulZj^QB4 zZ_f>1!qUUA?o{~C_!K60!U0M_Qc{u#W)nO$6VPFRM)fz>bpL+M*1Zo4pTjO*PnoWV zg{%ujYjKi`a%0gF^h+Fg{HF7Fykw`&&0M+yVH< za=M%kR4%Zb(D|Wb9lm!WG_fOU9Fx)=bAe@NHS?2S9mvcQdAg>x!HVAGA+Tb7EaI8CN;7*9}$ z#@>y`oEF(d!n}FtrbdaETqwgX=jru4Zyw61@|I>4iape@RBp$sWQq zo$rCk^t-*0bF@cFHqB{#%Ml=#O+*l*P?j(+Z>DaJs{`U0h~MVKoy~I@BFDHe>2Cv% zHA`ReSc$jB!vgD!!_75kw1$l1)MI{A!Y549z}L~*L}i2MmZiC#au!E1=wl&$$Dqju z^1(RL00f)YT(&7Aq)wY^e|VAt!fi$wHv#-&#*FAFPu(onqEhD1q0+NhO#9LcRPO?q zyT~KKrw+dC(JV)Q?_5Hja6AIhN3$T3{WWtJO+8-r47}ihQ45d|>fnF)7BIBxwsVT3 z?d1f&2t&=S{;xxqFy^F4)FW8qlIkHGn0yW40?iXJB1QUUGeXxOf2X8QH1`^Svqw zqErY(=JI0U8v|jt*&}l6<TZMNq319fc>ipmqonhgiPc{-ZV~|T(8rOf|*##pCLZl11Z3XqOsDORy4hUbOOsaf(}xsFX-&Eamwj7}x2t`XB@T(#SbMQ}MI9vBzV zM|vcTkcKtne1DYl6kDo_7l)xsET)RlYXRK}|I39?|3f#F-$*Leg3Wl& zb*_zAMPqs_I=Nqq6h|E+w$vKQq8Z_GlX$E_Z7Oba>T-{2Zz}6irZ_!zLn)OsXS~}~ zQq>owf3%z-(L&`~U|cCLj4S0-9&)k!>%nr~6j{ogB1^f0yqTj#^9XEt6%~s9s-nYE zQFOFYfpYyIs&cNui|UC6Z`MmGH%A5KtY0*7L+8&v2)SvK5;`Clt!>2+9t+S3dTa^_ z0z|-YXZ$%%ebDzX-qRU}2NKs4Wn@#=N3{nGe*~X+w?i$OlE=c3WUnX~L|i1qMKMho zOqm*b0Y=StH~4AtqbokAb8cv%9~j9V2Q?{3aI_xFcjZjJ1=PMk<<(3XDECrpN{2Hq z4(Bt)o$w&Xr{64(iUIpw%gKwbwMZ zf7tcc@@TjIN-PnZKHd02RS@5Bq;sd7ISQiQkKtTrve0-EZF($3`6Hc2n+8vZf=4wqIGrHx z8^b!4FGY~kiDVf0e8stX?PftH0($i@e}X$K5VKl7XVa9o5L9?ScS zhf*nE#vt+BB9I0%^O^`Ae^eO|m<3uuE9oXv_e${7ltKnAB4j-k5tn2>_*6K=vlj*!U!?A7|<&xr8XGS1{}j?_4xF{XGsKE9HmQRk({~twCS^i zS1q4P;S2>h>4D_mU?sJy01BA1x_pITerwz*Xe6Rut8W*k>??9_f1BL#5j=`}A$aB_ z1&@NB;F&Oy38_5-Q14LQt2W1hGAPjdGZq(h^w1<>sS5i+mU96hkKxA?5h`cZOk*yt zRJ@iTMYtn$aWRhPx$uHRPjb}|VO236AqGfjIVYQ@IODm-4v#bW&qOU7`n zQid0LV95i-Ho_A*f01imv#EO<$U2l_jp~%&8C591Tx^IZ<#V5_!Sg3sj1lGwiWw)C z5*aJjKM|oW_l$^AW~C)iM00`_K|!=4Osinx@_UvE-!|lNMvIcC8ZAy9wG(jq)fCi* z?%2eymKTInHY`QCc5qt?sU-;SH?*CS!eXDK5Z@*F#b*<3e@!yeL7EgJOX8F<#`zDe zX3M-KYC51*GF^Gn`jIm3Miv<4vBGdw%~Q8dd{P4MMoeBhzg|jC1p``TGtcF|I?L3| zW8};+!jj-BVmYN$y%FS6dM8NGjFyC06kSQzuQof*%RftT8cLMB8%Kal9C>mttaC_6QqDnDMcM)V2 z(OjxR(bujL$E%phoU`RBN^BO{)J-@~*#u~dY)UQ}C|(gY27xuAg@wZU0U=9>uf^Xc z9&)nO;~FZeP7qN0pmeTB4^40_Bt5yPD7p3VIA}L_e-@%5xQ!BznZd21@CU-<%H_&V z$lYtffuPz`Gjx~SnE_kmav)WpNf)VFL`aw66eHF%40-YF8pdg!2+F}|CBj{gk=<2} z=af;7&Tf#Utcq@uIB4w^=N-!N`84ouB2wnVcWjnSxg^R6n^lyt+KOquFbV|LEs{OK z4^|>*f1X4sN^vw8*69u-q)?hdhM)+e$bTm3`C$Y>IT(CZDZbnk&i2S>!Ir8B*+wxn zrA|Yf2%joVSIF^v!S!k_$QJQ2<(+&FjDqgsR32yhSfXNGc$bf8BPIjmIKP(jGa>0Io(~292;-1kFex zNHAmh!bp*-B{EuLG0^F5^V$tLXHQ|Re<(r6=eb1MKE*j6)fASbCX4Vi9)>BKpb1-$%ArYdMp;hk$PSh%? zkekSgA|nub2qk+%L%Y%5Y~LZj))9HZ1f^v)GV`;jwLzD@I>RJm=x%M@imO|<8mD{8 z7a$ALDpSyw%{p63RjZQW{oGPB_xX$>nL>pc^cNlmj-!I0h^70 z2EW;(Dohl^?1cJq&9FjYO&B0NKeTN!Od8zpk4!5nW&79OV520_BPiogGZxK0Th9&M zr2IR<5`Y9Q?YbRF2;HMSWhzt!e`?AbW3r4ClwT^ zdxUyjwT7B26zHStc9?{zGTPAPWWl3**W{o#d5Z=4l%@6dZ$JD{Ycz*`!x{RGT4UId zk3am-cG|xUpTNJGil&I+lSlBj*=%aBn$2eOQQIy7d~=VkRUT_aB~D_fJ<;k^84BOfqcnC_fmFYt7~)$T?lMA8C$&;ET?;t4Hnf2*H`ma47oQ zu+i4O*9}ks_S8t)+>mlT(=C-!ALI|(Acwjyn=)Rm>pv`48fvr zKA9>^jyYmLOybSZstS&Se{cpld?WcDr(cPWAuUBJz1VP@4(Sc1g#1GAnB)$Sp2>%b zP?a4^XDegrYbA*C@975}vIG$&2qJo|pj~*vxkxz|N#`Oh2YpdL1eTW0BWAemHXN=x zRuID^vZ5d~A=Ng*;GWg{VFeNGm zX_9mN-78Cfxo%ZT!5B{9a{QU%`EeUHv0BnEq&$=jCSqWz9YZ zuP|JN)vx+m$bNz@!Vvk}uV3|tzWpTVx2r8TWrn3Z8uj|HRezuifHPk&$L@cor1%}? zL}=m+z_KcKvFheYD|3(jHt9Pdpl2A>=zgKWUd{7T|pT?bMiF zzrzYpbv&r>n9-?#0~i3#En3k5dbN-p&#$=w4jr6n85Kr?f2d~WDjlR-R$8(J@G^*? zT>{c}CtMGTV`s(DL(3b&^B7(%S?foD8!4>}Rg>H}iEv|9Ll1#d=b$yhE!!mBst=^; z?iLtV54*$Oa4;MV?}x>3HM;L?%0r z?qy`cRSvT!f8DlS0&SG^I%p5}UpyYQoi@6iXzOJA`v$VDS}Ap=cpcxe4l}-C_Xk6N z)VLaeJC8Z`>9=DK9bX&sCKJCv;*bb0UZ1gPLUR z_uA!yf6{9MJd9NgezQjQ6RYhkd+o=A-yV;iJZ?Lap5(^SRV`p8N!p$`1S~Tgwh&sz znK+jHf5cj&_#U7+{PxW-8?|lQTuv|Fy*+hmZkWZjld}_jRn6nsre#%O?_8K|H_uxJ zEJb@bXviMZ$bI3gnX}`?PS(8_jy3cWJkh_S6>KWEZdzj?aU!1J3HJhGId8q*NAQC3 z=$>UO`=fL2Q6i3Dz?1S2d=N~&3>NGoAan6-f9v%}_iw$G4WL}$_=G~X+|+rp!T7a* zgnmyx%4UCD&qXDJ{OS}O4zKOLt$nwaxH3=qvH6x*tNNq+b8lq>C|bdv`_xzGSbf1p zJy3lDN8|2SWVhRc@DvkT6{S+{B88dN4&?z%*m@S2U~v74yv0Dmj*-rsVoLIPx?5*X zf9^y!$26)%y04G8R&TuVuIEnBdnr;jJGQMB{2wjk9dTs)@rj2U3MIHbYSX2RAjLQcF;%9AVWsf5>KY z!Y4=M$~7O{1`5m%RwJqcf77}3GIA|S$V_CvFM?S1H{jrGf`Kzm38i!LPwuU&Z#w_X z68)I$tF7k6uw8t!3B7DMd@>w9u^ekSeA2c|H0g?7xoiCJfWTdVn~ARF8}_JSAv~GL zh2~1_pYE-Tr(f^})zZ32P9{~*f9~oJf}1IrMgbfJPk)qWT74<*o(WM*D8%wiQ^4FW z0Bbh#vjV_l(X1mYB6cU?iK8!vCYDF!2zAt94z?Z6%Zsn%9xzZi_VxZ2EI)SJiEL4- z07vfC`&&JG4Ii1#ZclZt_uH<5u3f2O+5|-Q$D5;#!1CY5Vyi~8uY%GVf4~NVJ_ix+ z%W_!j!`adC`NiS8^W!Ut9tcbz$6GR5w7c${Q=fd*!s~x)kx0Ij7K6!u3{)$aIEjSS zawk2D&*(>2+}9irJh7aAqWOL%AdzwN!X4GzNY(C1yDy0iwQF|VSfq4TY@T5W|G8qP zf#S|rLaC9O|K9cObX#%D}phzMF0U7`$DgyJ{@7jCU z1Q3*!oYTGf-s|OMu_hD>g+dK`c%F>qST#`;Wh}o5r;N z(R|7+Z#inY*b2f&oDlAK7?bf-koa>hS48ZY>pV$mL!GfK3zJwS$jvYZ4tMQ0KXnwt z!|Pr9-LC!Tf39Xd^4pp-P-DRC2ej4Qd(W0uXP=jO*8zRT)`0c4-4=RUGD287OqrDF% zCt{~mQ%I~%?bWRP0xu70H$wSkxKdGb5o1bjC6>AAsQ*-abPaq%@vYf7z<;;XA zCjd##eaDo72aOmPqz9Y5AAiCn9hUOHDbKklLCj2-=FBR>oB)I-s{)sMOy*Z}@r|tD z5wVHntc`2r9e@b0%d5I2c~!htMAF>hg?H8!e@iZ1>@B7Esx1atTXCI;!V$)`ijg?6XW=l?CwgiLpz@wtv4ft%M9x?idvkzAr9ziR~mRi#z91?@*QYSSVuYDw`~ zOM3;zlOLVfwZMp~tj-l6=QSt(+LH~U%RkX@(w^8y_Jw_{u#h*L0L1MhSBLmN6>I*( zqbv*ls?bPWcxo*zz94x_+eDn(>NXazf7p3!g=Vb*K+pB(#z*exF$?(BK0GF|TCMqs z;Lh58Betv((+Rl<-dG3+LLqhFi>64IqKeQq8>X!>@U_}@XrYS6a=EDEF`)toDP-Oc zr9458zyeK@Qf=Q;?g3g9WT{&vU{hvb&H#=PQ11XfLzkpo4He!jz;rn2dj z*aN2QDs9s;EiQw(W1R;v#&mkhK5M|sd%dd;JX`|dwx-^!F<_njPa6*dw&d*apDfD` zSJa(ILb*Crlz65zVc$r~jq;bO*Q$NeAb-@QMh(|$TNar)hAXp;k_LmGemvX|A_J?zDTe(&e==P`;oDq>u=nup@apjGb^qw{;^y$|;(9M)_R;y_^>zR1 zR}Kr{jj5`Wfj7J>ES>g3JIzu&DItyMWO;=GRJ!jvQQQ91X@ zk+ELJzHU1SLQo|uhv~^`+u-C!)=U4r8y8o%7PQycB;rR`&RKcJq=moOoTSC#Kwhr= z12SI5OyWHZnR{7le?^#rRb*I2&EafyV#>5T<7J|=Y89C7(*E6EWyc94W`sk7^lpUzHeduIt2dBVE!VcBzB8|i|y&ssNYEOkP-MuP!to?=!kTe!&~ z7ixYim30LL@cN39#J;|SJm`#uvKMot3tpYP6Feg7_MLxA8K%t69p~C4`>DLne(F-% zPvvMob@nCuf2n`}^!Ma#Ik$YeEZI+8R_&+q$L*)|TApk_b?JYx&M(Gluin5SWGrxP*9Mr){Jx9EL0~`2E2_8WceW|HDln3@?roG7v}C zKZDE`48fLbgz+$37{ags*a3~fy`jyh2A*pSAi_zRZ*Uqn?BXzv8RTc+y<)&g0VfTE z*nOcJe-?o0Vt77r(JD{G*Y`OyuECJVI2wDYaTe#sc@mEdu0l0XePtY@iIMRk2?OIj zJlBQ5T@*7A&VB~YeFn~a%Zhz-#Z?gOL{OVu2tBmg--Q`)LrJmc8hW&n%{JM0!vX<* z$!m+PS*Q<8@KSz+M|7hk<0{)Nl^h2vM$J`me;_TOwXIlZ(!4SYmhF*IR%p7GP6`^? ziCbrdypFE4*6wH3u=(ki+t>nk`Ew%Wd9B$z8#gi`_wygIM{9KV0I0VR2iyYjRSSNY zEmG#Q@Rp?oZn7bX0)lfKH&A=rU~>HtvuiFZbiP%qxca^NlP%@cmMp9Dwpv=YW=l8g zfBgF=`@T~#OjA~AUo@3lTA;Pk2Fn^b_nNWyg=fJwE;=Kjj$KG!g5Hj+QMHa-d$cBN z;o5v#^J@a|W%|Zu0vhN^UOFD<|22EPryJ$fZ-Vaa*anmF;K@Ey{4?t{++t&u7_t5Y zmg^_+)uFZaT!9j-m7A=yt<)+}X9IV=e=el`iVuSQcIkAD>)a!BlK;TpE4LrFx0F8o z9ouQwy1uJxM(bW>MFqL;g~Q9v?A0m#-qno!s+Ns;>V;AoRr|hN>W^!Zr>_|QP1do! z8d$Z-pMTGJfPqzYe3E-|u1GMHaphz5!+Zm!c2sg3A~svv#=~6NYsS!%M$qHJe~7h% zsZ?Fh-g8mFN1gG?Y1r0-lj$9zeFaiG3gRU@aF=%W2yg52XYF~Gwg+L{X7R0}{uN#O z>3@D&YDR$n`ITGqE4Su@S@tfrDveznImqv?n(5jgx2lD|+R z`HLqy$=|4>+)^8V60FP8Y|DM+PH|s$P4&qE?Qdad2MzOuY1u*Jh55p=^Hriwfm=~# zf`qn7-^JBFB`+a#^pD*N(AaWm=ZcJSWX~F!3!!;%&^5ObduQ9KW@zu(#7t$}Z~oQ& z)cg3!-Ryoct;b0<6LKxu<89n~+=~gCs>m{{9wsnC!ORYSO0D=I2rs}z$A1(|Oup9) zP)a(hSPhK4uGYE)t!2%S1)!A^@K%n>p-u0M=u6fWPAC_Xug3v2a=FRegYp;A(G@69B)wH^T z0yitN`3zBiZ3+P{TfiED6LA5j>H?M!tQL$w>A^|>hrOR4mabX;kd;S|{;lgDcoop_ z5t1VC?M$>cEioYerJQD=2w;ws8@6=KHH}#;m-ba>6$=GE^w#seZyl$%?}~39A5h*a z;^Y0Hjqdank3Q{Y(jlq$s`FyQbU!^prtsuBZUVD^g?#g1(+F?m3lq#kX{hEkLvau1Uoi!>1tC3AJT#5YHd~R+<gwhx*j(?7wKgYTzK(76FC+l9V<_rsMQFn~u{nfn&#UT!^@) zqA1U?kK-B`AUo;k?#0a}pwDgP@RDqE!$kLBm1%APxQgbD9lB*cFss5oem*a8l+2=e zo|F(=-SOMMwVJ_f*4?teT0RmkB5e-0Kgt37}@U&}84N`L$ z5;Tj&#yrl$$ly!`3M8Wc#^|41d@%YSPOh%cE-#FJANRRNzaO(Z5rn<4(eL|S6hVyT z9v=UVem_rW?_>1)5sQs}f8-nez83^Wzn{$qPeB`AAfa(a)5XnG8aKIE9viKH$ReQpj7p9pJC0>Akusj5K zBMIr;h}i-FK%nRhp9v+WM(W+M&vP$=06>s23Tv)GXu^chMzKU)lKIyC{GP! zWq@a$&u+bF&Uiw`@^O+(%AGLciKPZ+ET<6y7%dFUaO8uGSRTg11mdFhktjW4IavB< z0HY#$JWLFILsfRx;e#|8|bfQ=vmiDM%R; zPsUha3{$w$XJKT35l?EQUSN2G%*Z~cIShuO;bmcLOyPeL7(@6IC8M2f!^^#R7qC^J zZj2cTfUbnO>`De;FW6VmD?o044JT?NWz*!A85s6KJ`2wWGlr2e{9y~`&jIs>BhbuJ zCI@U34)B*zl%<*BgW3#ywuj-!z;MPG3@1L&;47QY06qpjz|k;_1Ni<83?Sr{0oywx zm``WMh!Xgb)Bv_Am`J8xZj7L!D7ix%8a`y;b>Wvdf-qijYTH3N1P1b>?J+@y)T=vEzFn5;uL^K&day$6uRoD6G zY&n~V=vILVDnDIYl2?6y0dOXFtH3`s^s!EZT-*Uxdlf$VK`nTQ{sZ3;1)xpzt?_!aqa>67KXJa4;QDy@_*?b6iPhbm)AG{X?Jjn|Ja7qjp2JG65 zyD$oTFNN>bi}2I-1{wa+8h7WR>5P$$Sri#0;WUPkcbla0^9zT6u;9Z^0oNE{EHQ94 z#hKi!fXCscNWA-xkGyuo9q#RIb=_{qGOO4}Ivhg#qtkl#%jL!Idy-tXZV%M%IW6Z- z7Y3Ph86Z_x;E^g6p&xr6oy^_R>ersB3fTJAn=rChR$H?v*|tmsp}V`Lfp;Zg75BD; z%@!LPg0091hY$^ahcW%I&k$yx`Ti^pa65*w6r(8unFvUXF~Wy6<}nWlY>rzrlK2w6 zbB+i$i5=OQj+>ZG)^HIENe*L0jsz^7PBER8C<7J(3~ zPBe{AaG3?-sz$4DoY>mzI?m3g&7R?IYN&LLzj(R7O6C54s+RloNfH@+`m_a^8(QvP zfBnUC|N85%mE2FQt?noNp54yJhVk&-vT&T-vTD#rO)Z~S+<>-iRB`fFV8RCT2nn}j z7aIS=-E8zWx0-IzaGYQZ)+vjK1hy>2&AwF?A8K?qADteO2y8j-UVHtS?-59~5|qcI zC@$s0Fge+O3IHDk+p>z)LbS+&T}MO5*lGbh2Hdacc_T9m8(XGhm|Hj}EV{l!)k2bbk?+WdyfalO~&1xvj`wzQIU?|BfK5ijFr!n8G z7QmGDDkZ8?_db3{WJKw0I+?3#)Na^^+`g9Eud;&JqmJ74&UE<%0Gutf%==2_&$9|b z-@>wg?<+|oB9WSOda)e{`9^PM-`gX5;kJfJdgA%x##Lwk;YtC%si637CKgKZCnAoN z1g-@UEA}HwTpwm+#YmEUvE9mP|WDK(ZD1VE(Cp+vw3T1&J4m z=LiJ*uw(zQES~rNz0fJ3unVl8%K*yrwP%4UG8D>r4?h$~hsDuparCA*IxCLOi=(&2 z(M54|Q@s7Pc>At+d-H4Y_U62}xGt`bE{f}4i}z>6Z@(7*KC&F=ZVLfH+q}sKu4;RK zA7*;L^H4|Y)hmaf8fdJrKdK?=lqOjqFX3QHU;?KI0^R_K@@ucott@&U0e?JbQg8;S~g!(H-X;O=kC! z+>|3S(eL1%=MC;t?Cd0FcW58c@5@^HeO^z$FKpQQeXyX0sm#kr$9|yLw-0o7?gL5B zeLyUx2by8U0a>FR2;$QN!H#;MxKP?ejqtd4>&H<0m205+%}?*a~pL*8U}cc2=;S@vn10}hQ58lY8s~aiSmF# zS!tendddWN9^rlTFOrxOs`fADQN)KehWqmvj8ZwaquK9E!F?a2@2*a6Zho((O0^}% zN5WGj69L)Ed7ODe)}P1Pv&6E0HL+?;yxJB_LZ?LWAqtI;;JxMMeizsMi?f@v!}GKMb#g^{?=KEN9G;zn?v)a*PX6=#+11H)|N8wqQRwRA=Ka;hb^q+* z;^eCT?(*#7Mr55`fDVMhm70!E-d&v>!EGsia`FBxg)ZMgcU@ED`nR)x)0@?H9A00) ze|yruxV*S{e}3M7ee&k)qBiH`;#jou^z7o~xPN+iMV0FHUmU(YDa9{ue!05*jkokY zh?6M%%i+cGxp=yhi{n!K!{Pb+6Y33~aCvsjgkliYcGxY+gy?Ji{{XRTH2V0MELgvAv!d#<}LK3BpYp zjNypJpbCuXlZ|Ma3J~cUW<9tDn*jD$bUi0y9?%5x`ttG|=hb2xz6n6g28B|fS-0q` z)PY3y#EF5AWSr&_f)+ZE;GJYCRTWzhRv1{1AC>k=Q}Vs=>8B}wf48=zQwCfF@e==x zv=n&NHcKL)zoqtD6=|-@FKfh{Ps6%CsDYoZY_@6zjumvu5Obm70-%hqild}# zN8rY@h}3D}(BSeN6!b{Z8wuhN{e6~tqp4Ss|Bzg#o=M38EbR(BtrB2ANfKNUP*)_V zuUeY`Xh4_0-F@|#rB3|Ye{|?2Ar6gT#oZIMc+ah~TyY+X4XeEl3psciX0hCow$>qZ zm(NCO!|Sh+eD#2q05(&;&t?P!<)I;|M|ek|<$!_lX{S$Z6+6L>nhGjpK%q_Uv4$O(C8<_4uBCDqzJ} z?$eKW3=EFQ1sZ~}e>bHdJlx--+|MtRhOtf*6!ttaKSh)m5^WMl7E(a7Y(9 zvr#(Zp)^cVGMdF$k|%ltHAr3`?4onD3mcBChp=!w?4-Ki3Ik|&z2RU&>n1wTfu$+;H zmhi|Dzv$+rf9U>95Ws+g7H%;f2j{A(?5YsAN0a|Hyev~jYYf0 zQ{K(lcq-4_<9W^nRT+8CL<~O*sF5zi02q{Dgya-6?(_@+N)UqptPjxZhh8in*aE9B z%jW^13lHdThA6?={gl7q>>m0&q7Waa*?lG&exHdqai8S~@SNRe;#qRW{c)m*UQ!;C zBL>q^e~H5>dJN{|88HWm9y%}h_(y!ibM7>b8z%hai{?D$EhY08+MM$~2Qwtbe#kp{Mf*l%xS<(+lc*KJ)@T*$jS9@PS$|+&P2dIoxA!&$-*O zHVfy$x!ZD#ME4qV?p>};~N)$Fe=+C!P*79!wFwaDPMD>$<#f3rcw`|g(2EoE4hD~Zk?{`B3=YTiPV zvZ`QB!9j1kxdTOZcxHu^;=WVQqZ7FHsMBjd1B!_@{3+m1w{5pyv~4hpXg_be_M+1O z>&q)Je%vvw&OYDfo_DS~9ZKw8sTX`@e;->vS9y1J0^U8-zChMp`xrQ5_XLNY({`=R zZreVuO}(;1YiUn-TddChKdKvQv5z{vz^>D3Yh>~tJI)<2zxwmMZLb=_FD<*4ct?$| z`x<@0=icYLMGEw0oMT04v2?2%yS{u$JN1O}rK#WvhqdX8h_N^ri{|m=|U0zbWzfA8U6cWwRy{#}|@Y0BDNKCSKT#K2ak+HqPKZ*=!0bulJ%t44$Nai>@2mgcHA z*S_lX%zppq{P6mkt7nhTu6XG5?EFLo%PQQX%eU{&&QGrTf4>}_-}Em}Pp?mIcou2| zDERi|=9kN39svd8^}E9(k$H1 zi_b^`Puxme4>usH+KMDzXpG7BLYM?-LiU(xW+9qfJz#$1rDcN+l9vnBpc0icl9fw7 zl|Wt0;()}}e-sc>cF-bm3AwQGU~wQVShzWf5SAmSA41VI@zNEq6xNIXS2ACT1& z=qCwjNhGUr14 z{7cj>ihzrVdxV*XZsj43JTwazuzJ_!j$(>XCNje8e}mH{nq(j{=O`8=JG#}?x1zJT z;0Kiru6^F=nJ1vg!-qJWfl&7>CpKnCNJr?+d<59YC!DL~Fb(tZl;xqXQc~}}bXn@% zOOB^QbXg0NSQLNl1*d!ia^ywP$pSP`WPbD73*O>pvYK%O?ltj9y9ty4Z>>}GN6<5p z=`4(xf4CdAoxiqY1VK(xy|Kq3;CLkQ>-+!3(nMT!oB*$y-hvYt@7i@(vQ$6|pN0`5 zDWi79N=x5F$v}&q#X|r+Wu+t}sS)o%zlA8Jl|D!1fnMkw;EVJU$td*o0&nM09?l~D zo)^g#3b(bS%Xxk|d=22#nSL=QUjd3pCAIXFf0rD3M=wM23a!vPaI+Vkzf$ED1HkR@ zmQ_cZ%vc>CTGpC@B1_Z_3z@n8vd7Zu@~&WWU6*l_vT|jWav&0(mOA8`u?dW6tsbV2@hl!DDkeWoj4xPl$d{h$S@a7-<}Z%F-&J^x(Y1rABWZ}& zgW0^(fLbnCUE9c;ON_%z}$f#=-P1;-3 z8))v~z$v$QZx0%~w14kBtom@2;Vz^|#$%dfXu|BwzF_Oj&5rB~`-8i*f3I2j2I6JI zX}UC+hcOIbA?Lxt48!}XI=qWVM}1iOVm4PF99fgqOOia_;>N8oz6E$j-3sh?clq=0 z6>p1cCeV@ktOK3#rLM5=SZ76p_uT2B?6KQcV$;NqEK4UN=h7a0ALQy+N>>(fVm>_9 z%o%BWPbW3E*0fEykGbuCe_9ovv0JTHpSer=wbzQOXapqkbBIuOD@-U?@kVym48S|~ z1j&?CFPrKQ+Kiv%?%PC5ox&W?LeqXw?M#YNmXILKlY-$@6*mbbsCQK zv(|rRC0}7D@!JTgDRw<;y#?eIGz11|($z_w%NUpkW|aYvDK(nAe*_lwst{3498wv* z0lpfj_sr~T9Jg`?#sLiy^{j1&wd=>aiSmQIvy!{7%qto;Kx~xZ0oou7ui6|ch^&oo z9H-}+k3fdO0TS#aYYh^m+y|6U$MYrerj!^;XB_j08|xiG(%_)H$_0es&| zpD{#zM#+Bs-7oJ9MW(Xq(Y-o{zjC^b*?3lTtgcwMakkJef1QnIb&4ID5BWfcIS~MK z>BHb^;(=z{UClHHn%Y@rK3&|eDU!<4e2j?-E&Fe21fEo-Te3MG*Sv+u3uA+rkCmc| zGa=`T+;nYD*?c(96CgzPaY9j*&eo`^Vs>X-vC+w5)&ShO*~7ojj{DcJJ}mpLh#zrD z=Mq+{IqYnZe_LEiR|!@L1LR=gaSJOYT+MV>oRZ{pNT$Xp93?p?7nLr!X|*Q6B*A4x zTq)#~38$h_e9zNC;0ZY+`4FM~XR))btTM^u5}TEJA-7KCf*b zyxs!se|V=!dJh1SL~o*ObCr0xWSNVmxHA+FD>GZ^I4D4dQ#MPLNh=!Aa`rh6lrA4% zkko+vCJUr2LB1SrhKX_va#^*_d_E(DX|m|OCFZC5|16`u;*GDHMwW43YkZX2&)eu;uav9V=TT#?I%A_cbCurwy<311vFN2&6rwjeqp{pV*^MgukNNo(3l7f)3wAQ`ttpi@z3_w{<=>Kti|iuvl7F z7j)JbEM%hNv_M(^2_z#M51|Gs`l*5>e=Icg`sR1#Q`2r&G&UY4OUsffa7ym|@gHut z(d}$J94)O+vbR1_Y0TA9p*FPRRZmrHZ#DYeZEMr&MdxcO6E*{Q)sAaz^`i6j1(Qo& z{SRL%x%jte2)7k@w7hy?mG4$Djpf;1NP9E^)N{mA+-qB_4vl`z#$FUxL-4tne}4zg z>-H+59kk}EQwM~ny;v<*P3&lz4v5v2Eq+-R_{4Ltzj_P~>GVeS z13wnx)AU4n8`Nc-w#*mY~I}Mw5KlEtfSrf0U2=`}<%J zXt>}_t3IcXwA0eE$`3JGZ{WBG=0_a!vT$0xmH@!CWDq9QO{;+VpE&_Hf8>JYHknDO zscwsWD|l?PH+#n|n}V$nTo+c#!~L~FR=0N^x=<4ArqFE4X*t0r*i5xn%%xO$u&r$O zI?rD1@421ADV(DC_S<4_e{Zi_ylNJ&UNwv6{(jNiFPvh(=ry-Cop$%x#<#`ZtIqzx z$G_ad*>pDBXh|s^t=F*Jw%zVM0QP^|>7mhZujjZJb=r2@Zd+wH{VTH$I#59YNJ%C~ zWhV4y#2eO=6!ZglW^8Og1dqa?HsvyMT+b~UHSl}Sap9O;8atk=aUe*FpON*Oj zQ5;E^A((o11k~#xf6yXF|hMr@cv$Xm zh^p3wS&PhXFeC>nycCr#UQn3HwG6`meHyBl7G|xfm!~1#Jof1Cl+iC%3h$JD>0O3d zi%## zrwoC#r)(;mMNqC_WbAX_1H2Io0U~q8K0}8ZfJ%aY!XaWx<514Rcoq^IHONl_FAeya z-M}++m;+KYj6t%AL&$3!`MC#I%0}cVAEW56Y0P69>hoBZcz>TUGCDIB`7oVUFh-^4 z)pPJY+h~7T&DB0&3lGL#R$lP*t*e@Ao%rMP7lbN%AeN&o3+CDQW25`*`3uyxm(?sv zzaj1{7(8%)YSzHPKI$1s?T*zR>yva*`{X#Twe<-!8&wUNz3%blW3*KnxuFreI7chx zqENbK=TfgAq+*G`9)zhusOGZ;-R9w^^)|`#iz8A!lWM{Q2Pt;3WT6%IA9qQd8CRrk^NYxdEmz+>zqZhY@qx8rtUm-&}_P`r9o?C%$y zzlu($(=A?g_KV)DeYXQX0Gp1pS@fFit#3YlyZP6r!Yv%92OK6{%Pj4(LB4^r1a|{+ zw=C$xx_@Nu|5Nfc3hlP=;BKQ^P>sSJ1<{(zfaVBra&dHd3^Js?>Du*Jjl#Z;V>K!R z4}p6}Y9{hqu}uMYwnnj}f!8Fp_%t9XM_e=(b8|(~DRKA_P!S-NhmPPO)kG5TLPNOl z=Uiiy-j_5-gdR`>VLSMAJ~kCgt>dYHN4PZk$t;cf=+|I zhprWa-EH4PV@`ZJjDrQb!Px`4ufCqGyI$)Fq1X?UKaeT}}q@M&R`B|dh%WU$@W^KiVsJPHQ0$)4bhT}NF!;bxOX|3igdWX$Z zuQ}w%xVv>_=FUU6TW0l#n@AA$w{bJT=@)pHiR+f8Tj19K}i zI8MVajvg=3b*}61I9|Q#HUDzEFS=DbRG7`d^zb(evEDbG7$hS+2%z}crnBP=&`4Dq zjO%wjSfh}QQTPj^m@FPprBD^VgHmBt(Qxz+R0UE(-XOgMC=0p`U0pC)T19Gg!+);h z3_ewS_=01%(u$M{R-OIOb+Ea^3+}8hD7*)yE?`~3vYe};5Lk1OQZUGT@_$?fJpuE; z3e z?hCg{-dO!&ibnTY$5IQ<7j#qlTP%$+g^XE)Jwm(46=)H$xvWQ zpA{=X=YxHrMYy&6?;WKfL4Q=3RTYcuwhuKLBB`p8WG?2G+1u~B`{tHeK-h8ap1WI) z(;(y-3q88=Pj7$I<)tvQX>~bki}ur)kz+St9R+UhO6vCNg`s=-uhxvO-V z95M|60fg>hKDgI)nJFSRvMIo7qbXFtL;0A_O6fqy#>ljx10)osk$-mm%gO7*i#Lr2 za=_j*Q@GjElJDtvOHw~^gRDt!(XrW&M1Y}ZaZr4ucbQ5j|6wR-1^8d?#J z)@6VSr)Lq89H#{y=!$Syz|ph7DHuF)?4-O2L*zAddC5QqFc00vkHyaS#mn8|`|paK z-5(2pn74{y15+%3xqtBqbXGdIXLa_L%uf*TZjdbWQ}U|Ey3bAf$V>e`)=taZpIq<} zh*O*NCyy(q<|zWE;JJzj0iq-!`#V2PH1kWcD?}ncBuao|HK(oOM@l;2YM7G3ft!Q` z*DJnwoR>kG%}ICeIg{p_+=foF7zp|XLYe0X-$Zx79lyggpMQJ&l#b{8@-f}DgU-sC z_L};g+uU7F$i>-o;gD(uX)-g=v<~eAuIVg)&F^|m@9;RqUOPnx==d%6&uM%~26Z>d zu(-<(aC2OzC;s7OrhXmc7+{!m{GN7F*^94k|FJ}ge=hgvxV#d z%If~!s0%uY1Ai8jN_&0|&!c=b45`AzLOAvK$sMz?{Vj&n};rHKR%I=S3p^ID?bBN5b)ErCAvDBPO&3UOIW(_fm>|QxazJSNa zY3ousu777VFy+Acz=e{wvH@W?L(h+O|pNzml8)`%Bun;HJdHh0p70# zLI09(mmrb;q8|a@m<+n6LSW~pM2`@#y^sRs5?fBA`|O3K>66x?=eGylwDEk?GJzxy z!~v$W19~)FwlV0|S}aPZp|O{?Smt{(%~Hu}yniU1yJF$oJ?t)vxpN1scki9MhaZ>R z>csv6W~X-A9W;8|JAlGzZe`_jaMB>q&6=!5Ly2ta3IJ&y!1N%_rv~(?LsDYQ*HBvhBOX-hd(rxc<)ZmuNHd-1Kp)us!o%n; z-GB1XwbVN$NoB8y>chDGSe@$d_Akv$O{)_=;J?w;>MSlQXms~*$56nQr=Z)a)4I)=4)85DbbQ?O~l7KJ}9QZ_Aa?~2=TaXSs;;&$pS3V(lE zOs7Q{7qh&W`NeEe1Z-4{QZFdrKjMsL#WX8^`Lme(@@Emu!XlXY(;}G77H~Ew%RsJy zkmu?QID<-kTk4W#Q!=(HsKP!_XfBtQeK&#A26`wBP#qKmDQU1^i|ONT*vn|=?#+V` zwvz+ow@Ca6=c_XZ2?CA}=6R0Gxqt7)w;s7Gi51=_RBg_S6$H{3ECA{!bmj^0z!Bi- zAiykSf0zo=2lBTWh9gz$FdPZxUcz0K5@hEX@4z3}gN$FoAJ~VC{{?@(M>iy{{2Gta zWR8wW`4rDKc-Bb?;69v=N`6X6N-zjewKl-EM{GoU=P1N27wHu}eH zffUiUP~0~WPlAyDp`X*^NGyLu$)6Hvz2sh#u=vS5y+~s zl_OR=TjXBK;N9m#Vj0N^GK+BF1(rKlw1QEES<7osE+y8KG|? zyBrSrSY+2T=FcN9E#2{k4v|B4GfxNeh`UO`sliL%q+Spb1`UL~NPkPna-x6{C_SCu;~H5$onQ|*g*rypotaZAWYG(P+lz>8^*T`9P(7jX%Z*E ztb^k_Nkhr0MRE#2PihJpLYW{L7S)3T6FW;Ys<@f?V>V@_WU!sfaxE!k!5kVrO~hwk z$v~n%L@Dyg%pgmmx%lpV`pS{1n8wXj4l;b1{SlY}u6BSsBG!@`t&=d?A z8slZkM;2$R!i^HHJ1NnkP(Zy6fZl4VK9#>=7Lt7n34i2jMrR*K=a4=?fUDz!YQSJd z!!gPKHi1Z$?-F1jMS^~kM>!Q#3P!JKlX2uTxs1)cw9+GT>E)kel;qsccLpf$9Va}$RjB$A=MqvGFf_XY`xo{@;$4T1coIG9}PQ_Fv z!%d$rn}5}G@C?5%ryz*hO1XKuTIn#1A~6-KRFtgno6hO6WL~w6O`OdIYmG`4W3rOc zq`g8l6%`|;blA9vKGGB64JrH=)0@qhmvWc$%1R^#R#UpbLkb|8&HkNmfuILE<-GOr3^7nRuk**Kv}i- zpsY1_sVQGAndCXj<4f_CR!}M|BTd*M;v!eh!&!G-X0;`hT4{=(M3F~ZOqHUy0=g6a zSATCp{SVzx&5@&2i#C&aPCMpOsv2W@s(?`_CyJv^DYf!6a*I}qS09O|mPVUOTVJ}` zqgqaFJE|j2PhE4D+A&wkTX(0rD=KCAhD3{1TY+hnx-hL$Q}vaL)nAWR>!!p?<&;>d z9e|l=cPkR`$?Ha;=&!nQSh*D)tyHAiK7UB6{nj9({zHSD)w@(5M@8jVzi8ru&cgxo z6TW5&H7+J=OEJWUgytfGO(8*m1bE#U4<~5=`rA@IopE@CGpa!_8+!rbpI{{XlXp9` zqAB-S7?ay8iiR;432{+OQ$|y!UcDbNFXd-T{?p_~R}80f4ppNc81Zm5OCqu3sDDZ2 zt8ylDXK6AFRb9=L0SYcXP33UrC((SWb|*YKB(6yw8=;hdM^qz4tf@6vrip=|r-@h3 z)A-4o$I6)kEQv%&JOodjLRb6E(h|FxEsu7qSz?Mec0VPnb|1bDJT&WH4wo6A{R5oq%3EkW zi8g%}q5P4~qs`Ku4h4_u)8OL-_r3|tQ}|FA<>N%dOv3LKztw?9fIS$|n}-P?UJ&6? zs>0bgrSl@j?oyIn2`DCzV9Ak4F|9?s%7lo{lQ?gxp1|nPn}(5!F%OQ@(0}D1S;$2C zgdvHwhzfs>re@B88E~+K1Fq^QNS%3j<`Pk%8W!QE7yL2LWCOEllH}uZOsg!*Q_}0lw*8Pr>BTL7^d-O zfMBYsPLn(RLWq{GQlNk&TMrPK(B z3NSB|>4=i%V?BJgT@q=Ey|!8sMT#a{45v&SOTmdd1Ywnsmy{Fa-le>-gd7bHf5rQy zh&TaiG2Y>Nis*AnlUsQ`TDq#T$qqBMWq=_Y4(~PDibxUC6%pt16*bKXsU1D4+KNV8 zzAtB7yt&w~VMM+|Eq~U`awPf%*K#?js7O=E^<|Rw#cHX1b)9`VJy2=kMdqB zC5c`ViDECx!I<{K zF3wOEoO=1V8L$~|T0WM-8H(_s2a^BNR#H<1P^A1-S5*k*YfCx> zZHTCBb?u@o`-;8WCa1)PpwJ6Jm=7rkitz+tJj}#o5*dTNUL^IZ&0(lu9`*iAgrbfD zO%j%>co0g=Nq?Tgk0&Blt*V(OTwJO4T4E@Ij?lx&IGN}Cd*UHEYlygN7*7x{NW?|Q z3H6h%iBeCoR#}C|D>+%L>ab)A-&M-^LXX_@2yGi7l=*2AMa=;ld$*x%LnXD;oN8XF z3DwNSn)G{=#|Ht;#XKZ$Q;PEmrJN?F5}7LIKarr6dw->bN@bNPP$csKE23i1ia5=J zGmq<8Mts?j#~Ce2o@%r-dDNbHA?&pIXq$PY+Qnb3E{LgWSc-COQsrqeH3gx5Lt7~+ zF4jqk@l}#5KAY^8vxI4%hZG~r!6{Qp=RY)kt?n(+(gAUKN##lNN6NCC|H$XVhbf-?0?;8lLy`ha;Jh3t+AQ!;m5W*nt6)+a!PSI;A&F&NU6&a!=;iZhG02a4#c|T z+M84cWjT`%&2pmXwsI;BQ=X5w#JhzYEtd;=ip(V!-jO*hL6Qod@VkJQl zCZUZCxAVgU0N+T<`>INM^-=h?M~20=RDUJNXB3~Nv}wSPK2PTUxakQwUa3hU+$7&D z6_<;yN|LvMlu)mY7Ef($(o%)uUQbdwdM#DMGuKjAb>W!Fg&Z8rv^fZaP?%~=lYrhT z#-st_Em`2Q;ZVd$+9SU*Kx5aZ!BSjIg5^XZNGPZBiIEc3lgMOki*cKP(b&L?_P$(rt#X-6G^<}s4gh<}sZAVZ)sqD!s!H8{nRV@gx^P-(iG;Mcms9bb0! zxjwqSHtkhbCY_z9(=J}PPx92MGk4Eh74z=0uZ=J+2Z4le&L~+=Z$0MNF0SU_1=97T z^=K7&YmLo&edRic4tTYgTa$@`-LC87`g&#=n}c(xbi+5d!cslvQ!m`Jx_=FGE3RJb z*MmB%7q{{*+$DcM!n)nATDK!wXXWvaRv*9Y_k$lLev<{b(H3`2I&2i&YOs*=7pwy+ z1{Tkw)SHb%znB9%BMeLmHPdA3#XRgqv$0pmRi{wKoOzKavjX%P1+K~k-hfs!$bbOt zg^PU9OrwIfE@Y=ps$8B#hJWrnDU(ltjTezQgPdG4jG$dV`w3Z?BRG6tXq zIV&Y~nciGH? zXOHE>#3BG5$i*}af`}DU7KHO@5#?zSrOkX$zk_2)=2n*-o0v`ca zBS77nkH!>X&^IU_W@!P(jAFtVoEV~l?}myn_oC1*==cm1IS2@ zwqTriq>vp=aIr;zc8~=K)9Aa@g1m8B0*Ti36d0Ij!2pSVAq<8KZc|(Y%nxBO!0CYn zg`Xx_Hul1_5DO~=M4QNXz5oqL0p^x0Eqrf=FW^sBkVpmJ0Dl+4L{Y&%yg`zWive1C z;!i;x58EKA?<@u^O704wLM}iATX@l(cb^sT82mf(2B=zL^v&`w7u3wR$R4|r=WSz0BKagwZ=ZPY7wjx zg=&p;;RqH^S(cMC+O3!J8gISyo(?FtUdo@3a5KDsXybe`a+2EF zfMvdpGQxd7xX=apy9i8!bEf%;Mx7RcBL&%ubALM<&Q3?1YA6;KLMQy~d!vyIPrXIo z_sBqw4vQRIMVv6m*)b--mFbk_)HB${`Q3{A;rL@ zK$>Q={Ej?Xo-jHFWEeBJ=L~TChkS5K&*OWU?*%-A5L2mN0l=RBaJx7);`)SxULAju z6@Ng>W+?r|*(Dr?`Z)EYgzlGk5zBlg-?|8KKQm#s0em$fw-XCOY3vurqFf9JIAN?= zwQ(c4TcRNJ31h`i(v<(OYzB34vp*`GjfF)S{@!|Ne@0$*80OBEe#&@5VRM0IWdpJ< zo3egDq-e_7iHG{WmwQn%(o*=xDx3>UkPhVJNiW0&cl1EdT*H!dRi=hpprzP+Y zQ;&)d^l9~1+SMN>XblW(G!mdLh_X&EaR6qixZ z0vHu1GD3lbVS>2jUMIE`B^S(}6VKuzmwwO!7Jtl`{aY`hwM#PyOciw|xpyK}qc>%I zBn?x>6t5=u)8UU40}khWog=R;NxV5q$<9rTUi_u+41;9YbW15*=`c>vp%zhlXl^3H z&Nv8t4jjY{i!c=PB`Xom*4mT|v2^VUm(h!tVQ$c*7Qc`{kWkjh6$^i4N!%Cfgo@Bw zfq!xNhA<{qd@ZIG{ye7_juGK(ts?zd3d#$57)%x~FY-p*n{dtzfxMLeYOfe3QKDmv z$Tb^&%Jxd(gz=v~?zOn}5VbLig$JVe6DzjD0iinxYESWISpg_9$ExqSOAMkrh{`MvkbzBMR`b)VSV!g#*p

    0)7zmOl^-6KWDjG3lpEFNS|f0%Mo@=*>;c%CarR0 zL=p=RO`V2Rw;`Exm+R32B7b7#(J(@A@4TD>Q!1r|1OuL=6-B=ml#f5-;{|riEWr1u zZ=M0%oMwOc9%!orSmi7rPu^KR)OlB_nc#7nX`55Q@1kMH4H>-#EkYkh%0_A5yB#gG z@Vy9&Jpo2=w&0UBCZxL}ltrZXp~zyPg0iaYo`MncIIS>8vy!-W6n~%=64sz*yZ}mE z+*Uyl=U1Ao)8tNlY}|Y_oyXi#G4pPjUrS%F|W4m z&+fCFO=Uprw|I>Amc-V8-wgR|Fc#C_vEm4*IY)_uVxO;;GA?IET)JQ*%(v~jrcQ?C zbd&K$KvYB`N%p^IkfBB z@VWzhWoz!*3+NIQQyAh?qjQnL&o_KiWlY$;Isoz*Xn*ec&KBinvI;K0_hBAtil*uUm8bZQHF)rsrTXmTQ%9udLU3=2Gw~r8o zXa*oef!m_fZzE!z{b%+E-nSH-^7E%tl+9T1Y2)azFrE@CdjMoW^WR;x9!Y=H?~ zj}8AIF#;#crLxyJhbQIw0b_mj5o=AoS>r=z|3fQfXuaDYSVPlsvMtlH;QoWQ#nA*$ zJ8O97&Krv|-nwdv#~Kl{ALK4B+xuX{^1J7=Wq;>kDa8I4tm89*M*gAmp(uK;&D%F& zo&5;}bAQ}uJB1T$ZM3bT_@G|wgjwAQ+q+;G`n2&dVasCs)82$x%d(o}KhGbx6hV5{ z;uX-()G3&pLC?8!TDpqKV~r#G0iOxR8xUFHsqfXxr6s{*BZ6@B6{zEAS%Z;E^M>fu z%zs+ir}mY7VTX2trYU9BU9G7~Y#-~om5p7cDl%A?;I0kU=?MKvm+i3@`K}O-ti1G1 zrd>LZ8)l>1F>Pj>mT9R{^B9Jbb`N}Kac6&Hw-H+wSGnm|ye+GHdZNRcI}(qcH#6^a z1Q_18-{wSk0ce~_+Uax%jI7&Wt!xJFIDZXQkG*5rxwZG8^}|_OY9>%AG_WBpa7v%2 ztZSNkW_QoDw&)2$e1cFufiO^^cE2A3u;x6bkBZrC=We6j%2}2-nAL?u;NfXo_7!0H zyi1cjAw;Gdc*8E<;r-aM6Ii=IvL)cmRn{e3c^Kadgmc$k3<+y1NtF&G%Mw5Ia(@=^ zC8q-Y-u5yF$dA43S)1!G21%MGcbfCQF8wFbjo?j#XPpbr?QA|38&w4t5AyXmb_5DZ;_)+bzcNPmvk7#sp#U>hC*`w{}sn_T?Z#+_R1Me61vMT#*q z>2rwB8z-?A1D;t4k}uKm=WvL{{*IGa#o8bUQg;$!A-OP>P`fl3eb1k8pOPfY@qvWk zCH9%v0Rg@oKI=+%mU-o9EVqeuPAeo>=%iDc4*JEAeJ(2l`0KKO8vOnN!GG6jSpE_i zQr1SKc6K{Y@A35j8hTqt`R9~@?Kzx9S>QAP;2#>SQV8@PAA+Yz1j{>aUs=KY2UE;z z+0|u27B7c|Q9&v;_5SiVD z8GGw3O3{#uD0OwD9#yTJ@dm=4^<02%!sl@!kK-BpJm*s=%!=e@aYtSbci}Ma&*MR| z09uA?h<`EyYGIJt|t^Z~>e#O1((^Ki)c*#eT^ zW7)TJ-1NVb@Rcg7oPQ>Jm`g?u!BIVm?oU7}L=)JbAsk6Q?{?18M!VPUIL)nRo8P?J z|F-z}SJ!RZ?VdB=-ag!E;?EB%@lsD$tF?`!;f3V)A(fWN1r z*4^!$@0<9eQh-AbepJd!N_lxAQ@-25lz+XHDgXKxrN}ZnKY!s>FHcWJ%D=WT<%b{s zB~pI)={rbyt;(EGnUfzx%FFF-xYx_=?UyR$M@-q>mMJf{-L_@_f_?gVm!AG5R<-@| zdr{TS@e!o#9IM-HAJe1Blv8TJ>B+WA*`<`-Q@F=u*qO(0k6(6ebJMi!KlqCji~{oQ z{D^-K@$WJI#eX;OdKX{p>wn?jA1HFDUfB_)oWNgf@AmdFzQ*GpF>;ELQyvkm+}@$5 z-93eyy!-(l^!p!9;qQ;ofjd8LALHLs?1I;_Q|N(rPPId@7C<3iJJwByZy91Gv zmlXN#JNf3n`vFrxd+4Pk6k<6 zrr~t#Lw`aWN$=V+^&Hnu1V+R;8f+)QXapx!C{J5Ym}|jK{48zITqytsVLNKV@zrWu zr1baF0A@5v9>VvVM_v?h@di(Mi+afOB$?o*h^dJTG~(a`9*Ai2CvLyq{CXCnXoFXD z_WDq)lxMGx8EgP#PPOFaJg1%C+3RaInzKWtlYfpvaTx~-kCQkV`~iEUi|fM~oO*C2 zeM`9)*FYphO}n^$>&4y(x(+%Eu7Ylwd!k2pJ@7-%OZqo*0QxQUxFn8m;Ie*}Cn*d1 zqC_7y!I{)vy`G00^At#ogpLFHih}RYR+~GI6*RD>C_r;wSnJM&S0`)|`tukJCqUsF zaDR4U)3x8hAPFz8p|>ve9y!qt3scX>68O8m%n|Gy?z6lcTT}iH4oX-n53MPe-(n&7 zSyO@rt*Znkf0jwr8?YwPzNc08v1$zj~Sr?dpMcwcX~F7uM3A0Hx}}1r}UBN}BtXL2^;5U7Zrw zP~?A7HIt#F3H<~dxsKB+HXeT2U0Pe8pqGw2y(9a=wXcAq4y2|1O>4`v&v}&ddr;!( z-(l#L_XLLCE6LD%wZhPgDV&{GU9`PuY&@LH%g*Vt*}F$BYfEk*Y%da#Xjw=oBy-%Q z?XeCFMbor^Rb`dkd*xy91v+i}kKAtewjF;bk5lu73vaKcz}$Jz2~u|KAD7Ou4I|UC zqc7aK=5|hQ0K?N>&qm1S^Yd}IoKOf*PJn+RMRq7S4sZQ22VYP(n>P`6ox$QGxis!#70HJ>i z$iLwE7b5>cO&pLOuf@aE&Gyv&7_68119yEi)^%Qq~vZ}iXUCo^!w88 zvkpv_dK)+@*cP^ttmBZCXCTCv(Nt zIFgDlcEbs_+DkjmA1(LEaW*y{((UEer={)Zo!$+zub3U@u04?VZZ{5!=Z$%6buI7K z3vr9swyWw7;q|cMOj34Ul$wNhlXB-C+o$$~x!7s%**f(K^uvKz1kNXx{U(2F;PTYq zG*lK)hQ8~Rxt&Va&OK{szv=V_%zn@8cdk8^Px>Zn2+rw>nyKDHMAuP3?Ty0^#nEAL zbXpv}DUQyHqx0hEZE!XX}`q$$9S@GMi#lMd% z+VF3IL(aN;9QzfwRH1w8K68Jon<6=Px8PrWCVl|f*0;E&dXqJd{mPo`F74Oy%X`IN z?w1XKTnxt6>iPu*|_oA;@rZcTpQiE9nUuR)ZQZ7!`)B0 z;lsRBJFIF|f_mdPL+U8Ev6**FwiGvaH2NUq=hS>?ydC9_K0Oy-86gT^lAC7q2uNq?!l{Ftm(|!4!qz!C%R4b z81$7r%-ssGQYH6GqtyFz#hLq0l(yrJ4+ikT(t=uF&SEaha6CAX_VWG+|CtY zcMlhy_BU;JY5zOdcKCmTMt4t2)(bet!I#vfCT+qJ^Qz%mYdQEHy=mL&72mX%_C~C3 zD0ZFVktf41J|dhgev$8~*RE(PL`}#36Kc|$J7pRJD2&%L{0yRzOT%E!nYqce#JGKgX)y8l1dP=xhqC zwUDYbVmL&Nq8yR@b8Ne`%wN~uuf%Lv|LH_IeKl3@RbYj$y+G=~Ro#LKl?7@sSYowq zWybz_n4}GwVgJaDWN1uSOKdqTwHNv$SM%~PjDtpycJ_m`gDJOJD|khQ_k64JaEckU(qwy7QXZ|IY0^wNFy}b8K(K z_AfcV6e8+{i}FEX-e+Q06i0!*ku(Vz=8 zN2G@px-`_zM)y+XM%MV6x(W5eQfqjdAn+&GZz&Zv=>>nom{1z=X54y{s|bafTPdhX zNho!Snr2(_f=fpulF|{^x5k~$RpqFR8gYiRGW)tzY)b7icd20u9>)(7&5Fqk*0n$E zuKJsDlPo}t5Tscb-UV`T4`CVySIReVoup|yRbBX6x(8khM!mPBsyimV?uD=ud>xL? z;u5>^6&rt@pvCARH`qG`FQS*brRewHiRf9J|0u%m!)oLx@}@HuXoYva*CMJ`D6O&} zrt+wb>Xxs=5fo4fWMx1Ee!nqqLU9OD8kT!KACi5-_1H^!lVsWIMiGBzDV@EU)E*C^DO zC$~zadB!aZMEnZ;A7n%*HATYpeVls>5d(Fh2*1BNqo8a54)@Afex`h&GM3+jQ$}{q z$jhK*Vq9f&39%f`=%>bAsj^cTmuP&L(NLXvY4-mR_jb!|BU_``y`KWYc+-H{015sJ zG{k>Tk|kU7*|J8KN17oy5=1sBj3|HuG%d*jPo2wLo#ZN2sY+F-~smL%|kYHs1Kzu6I<6y8)YK(JSNlanlifqm&(!!mzp z8Um%CzWBNeed2&qs zlVLZQ81LXnexs);J`mk}`a*54HTn^?0p(;YF~g2tDjA_NnI}s#O;?MU-5F)t9Ti9V zUe>6csW}@bCNzIXy*;UR_{Si5?Nxuw=X(%LjMV~Vf!FyJy;Qmr1NA(w`g}#pB{v18zt>2{!^S6uI7mB96%HJeMu{ zG{o8B+1k%#gHL7gW%xZ2fMCLAWnT^kPg#ujr9usk6iRLg(g4|>-GlhVSE7(!aLu7# zNk1wF;n>}BzRQFJ;X=W8$HRZ!46YdaN>igG`ESwEh=)u+R$7|=_1|Pn=-ucq9M{{8 zoMO=d+4hzGcl9sHheAae-!$JXx#+Gks0>~YUK_nFWa&x~-ZT6U=`pY8{hPqQ!?LOW z0JmfujZ_bCd1-wU<1GWe0_4#L)V9V+uFAK}9nZ6_<5vCs1!d^}0C;~5mYKdZ^%WLM zu_AF2A?JG#4$VPQS-JWFlm8iA=#3K3;gTGBKN#1Y+>(8AOy1^hz}%h28BRA&E%xn+zgV}=4M{jcUttRxg3Hl zXcNQi*_u~Zy$BPV+MR#VC5VTf)pa6#jVzVVW%yjkznXciMlAArrHH9|u`0||ZuxMDT!MdhHMskq*Fa1G@-!hK zTqN|8I}y~$iVGF51Vrw5luZ{gHBJwOMg$!l*%2Z3ddpkV#euReH^WCcVD z2*Kk&2J)i-llO|t5HeHYhfB8AzcdTK4p&_A{_KDCKMURc+!x)S+*TO1t?_yEC#Z2L z$A;h`xKDp4yQjM!KVR;DIzHPyI{xg=DB1t8`}yA~T_>A~^-fqP7s`Hlo(bpPe$ zuJvtLV6amA^ZSFn-Q#zL#9GM>y~cp-+-%&{LsXs4A*P911LU8i5I(Ha2;`V%!gKHv zD4EznSzQL*)mL^&goS?;?mg&?l+SsJ{7twUA(ZF^SslkD zkU`_@&&|Z*2LkQ9JGeMmcOsl`b>6Fxt@>gVM52JujTERPE!M7#8v*`VauS%JB;Rw% zYoJQNzl6J`l8gWE5H9E^P}WJFlQplY<~^;AF|zR&yA#WlQ zllky12TL;b3^*iDb0+Wy@3<(7Jp$IKap>(FX5!el$cFvk0Z-fHXIMz@HI!T!`p6G4 z$N&`9Ah?0XB2Cz_>t=tWY3akLOX|(jUBV zMU;H(?Ht>9+5;~d&wqQ9+uLXwN;ow2eN-R{zJHrNcNLuLyo8kv7Xslo zIPxlk@smSp%R%9xv8;d?2A?ePLH>dWreX}nsnf4IwV`RagK^n>s*>7Ldu{A~Oiw-n zjhodB-|@i|#m9vq%%p+l8d;KG(SFiiE50FSBr-`wd;JqCDsRj?`kU{`-+W(ftE?m? zWRx|}weo*lhksm>hS%VydVD#2Z5RU^pUTDoN*?%&*)!8(g4sVkeEV-(KMi}8oaj^Q z`O@ouKKd{;)BZn7LsQLFdfy+6cdbKcxtGUv}Z% zp;(4gUcGu%A)xm0^dpSn%9}x@m+AZ=YxO~?4T*28Y=4u}ZH4?s-ShMh^Yq{&QavWt zRlFkp8YByUrtSGjFw*SfgEMuvf4v58*c<$GWL@7Cd9!OBr%jfdNC1C4=62DshD~Vx zbQ;qsc*lX@KpLdL?N{@K2wnqoGmT#mwQKl4rW5V@@vA@6z?U*!r34|2J<=eas~=PD z9!HakB!Ajyk+A4a6GyiuG*2{8D#7N*!JQ_Z&}!odl`1vvBUS?i7mZtQYcw2daK39a zhL+KQTf@+b8hr@h4;*m;9)_Sx1LV0jfp}VD4YWFO-F)@D&O*Y2ox7vSld`DA7Uq!#+ZSp^>aA$?$Q*f|GY7)-=g)(lVM& z<9`I57W+X$`T9r-O^Ye4c1$7Ahy+J4pSz=)I3BsNJa|jecwDmteq%b;;L@gXMIsHN zL~9UEPvbX>MJ#DeEm>S^0LWkgsWJbSY7i_&1DAq=2f!kY#gCc@+odhQk(q{CM;f@W z(a_I<#;<($;Eor+faiMn!-IKh5K)Tan16&iR7NerR)c6R(L{qVP<}Y}XGx@?DwM`u zYJdegKp{}W@cVE#%hx+dv>-u+CoM>TxC1{S3#x&yT7V)1jcCH50Rl|LS`_0(Veuo@ zEUd&HqF5Wjokkm7KQgL~z!{Y`0%u^_XnZ}>Mq@hCMnTB5(JY*5qv-liZItl5$$w20 zGi{VWdctCDw7`4|%(%d>I(O(JuT2^+2yI{!!lAV$q5=pf#M&$hry?ML#@pge!Tito zDuz>SA+s+qzid!a=cpH^@kt6J@Jl$=b559EsicF!Q#u$dQ*`){PNQUToeGSd-qUcL z{xKN5)YR*8NCGk@bMi>0K%hQD%zq1;qMx{?rHs9j!u+QkQKdBGN#Qt^eyX|KhcuX^ zF`WW;0A1z?ZNv@Onxjxr`3e@~Nq`Isl>^@5l_RVdZqN_I!X^17(ujgK?B=v$E|4|4 z6;kPfnQWJ>sPS`7iKKs36U<7ti{~s_TEZ1uIGK^g$k%wsjxSOD@jA{NAb%oUy7rD81PrPPbKG+&1^Gf_hBi1AkW289Mo%Qg)d;@CMJt!U1%a zSoDvR`-kRH+v#0|s+9b}o^pLl@g_rjXI`37aV*4=ng3(Nw;SEzDqd)nP9{nxrQ_Gk zyDFvGH|b7QI>w0Mh{b%vTz^5TUomPU5`O&~IdRtvH9hhdhAV8W!%?p2n>}B+~gt9c90kRhj0S05E=U(l8ku)rbCIER|^>J%1GyJ@ftTP*yDMG7A-(pzd=WgntwK4gJvQx$d)# z@yhVhHqIw5sMpe<)kr7xS~_VpOi-<`iy?au7*;P9>jou;s3H|rnm96}=la)4>W{}M zM7;tBuIXs@E#){-M4Y(nCyU{r5*R(VUQ1gIGX>{ZaL-Lg>{~jfcM+q53RFH0zQU-W zfZ0pwU*CFML4PdkfPYSaOV)Hsm5fuSyv)=uK-CyM*XX&O)M=&lnwfo3%iQ@7>0J<} zcZ*pXQXhh4ry-s8(kM>n(R>u5-GJz4y_Uw5E62x_4J!RfFwDmm(>ts+jU(2d#ltkY znWf21%oEXdfYO^@3f@OkIF!(hu3kuvm!g4;X(Q`Q>N--9xu!R8`+IP2lW>5{AoFwR^Z(TwJ=jup3g8q7jO_5)b z`-Az@hvnt4vLMWsgsV&jD-+uqzIH1MldPmcQsCu@LTPgWs7~c-2u7~Fb8;ntZd&>Y zmFV#4q3>H0{(n|k=Iq@HXyE*fNtVd@Q?JjhV|?qI!-H5B*(PwKO}v~@Kig<70rth7 z%FSPs%WYuTEWryypq?}=+J`1(yNUzvD6H3wCS(BH#=8w5^-TralgGk`+xX>Xcr$ps26omHH4tD;x|TP z2%m>(fA|w(D?B7){0pDwGvO;L0cs-oI(_DrFRmx2urZ3Vs%J^lH(LnAxxr$AJ+Ig$ zkue#OWV3npztWCcbpn+OL7GZZKGgTOEUQ8)7Px45QoFV^v#z|^8ua(y?w;)`eHsi` zhO#8l27mgT=kD!PHo(T4LGR&i_Uwd=6(D#sz%48N-*QMODhR5~B8*3k44!ny++Zpy z0EdUIR*~;~SPU5VW>CRQIMz&)vsw^v_E9O6pqvvFk)H1r@{WI?Tb#%Icm2X#-|yI; z{#QTqRLHOP*M9s%o!tRnvpbb%bL3~jn`e}0LVu#Dmccmj-yi`!8tJ5=(%X3sxOq@) zA=ocqBry~7?|^{PJMby+O#do7sMGULwx2^~(W^-j7cPspyZ42oRlO-c(nJd*;nzC! z^!{LQ|5LZpbN{X6m0uG3KfQisnEkhWJ-=4Rc^u;8X@x zRex>3hUrUFVTtM+GwaXfezh_8t72{*!%$rqdN&8BqE>#z*uWWoQj}8%OQq3&s9LK$ z=HaHJNL`8_47e!$CIElLmWDbjCiltTHuk+!PvO{q2Bh_U9qkAux;b@_Y-VV08kGF_ zYE=!He6Le*xRR%3AD~OJ)i>SMa%~C-h<^?k_^GipEPUeT4JZZD%ev7&W7=TAhWWRi z;i?&HOS1`fo7({l2K`JYy%5j+`xU(P`oXhoE&UJ6vOve=rM`01e#ai5KYtjc zJ3|AFPkL~ZHG6#xbXY<|E1Ug4v>`-64SC97P~&gq!659JuNBRiH~;AW!?T7x8S3^Q z*0A@F9lfaA6E7f|jmv==5(52KVJHy&Qfo&l|lQmi3+Q*#`Vw73<#%GhTXD zL+GXconqkT>_-b;FJ@@6D~ei(JAa30a8Q^kYQG26P%s==Q9NM^?I5|ZY`I+FlomI! z;;x-5K3%W03u=Qil7JsAbOn<+KR>nS(cCb}Dd#zkOmdq`@lU0c{I}hY&$zPBiStLG zKJq>CR$6WUdFws-@puu0k$bsKASAE5zfDH&*)|!?=kD<~8G+zo?(V`<;(zXKlW{Pa zxQE+h8o6J#iLjb+Uv85i4BWrA3D7|9d;ZI5A@_WnKp+?Qw{3D8u^A|nY?B#H64$>a zvxV<-Md>7yVF|&yUBfo--NkeZ%G0}Ji8?r zAlW^>CCOhiclVZ{Z$9_iEwcK>m=15r*Xxin_scE0_meyKmPrsw&<-HJ|pW4QzJL^FS9) ztCmWJOnO?H1Z;)*n@Hs;L4x+0M`*e^_O_g5!HV-~W2UM?qK4`^=W#bml_}9n{@uK* zn|bOROr5bp;tnQGi-|fW{>@xEyFU>%LtnD%=&cdO$%iw2ek_#$uQUtcn z$=nZjbTaeKy`8PCbFNNB63;j%Puw}ZyUz`zgNbo&^BW4#BSsb$)74FLZFZHfc>qmI zNAwl&)A?K(##xPk7wSV>UX`e1%K3z~dxd|rBFqc;_lNUT{gt5yA3r#ev z*k#L?Px_^9bB9R=3(JnVyfTMM7H7 zd|Bj*taf8*b}C1=Z|^#6D$D5S4zklVGZ+nVA0%`1LCB@+B#;-Ha3JI z0&E|?JW$sKRr}8?DXtj8!dhQ<9CK|%oND1kcV{ywofsSkbOnfHm3GbUl0{$!myZ$bSJXU7fK!StWkpE>{DCrmkVmT zu(8BU^HU<5Wtc(bYla7R!C@_QA!~7$nE+VIG+aT%jxyeD+USBOI!}AQ?{SBjl`kv-?-ZMN?&80X~AEiq5$~J-iqsf`v z9)Dbfk$C&GfQDdjo({2nBtY)-rHUD9i081bLiLh{WByruQ~+yvdFhYGpBL8>$;-8t zhZr2@02SQ}S}Ga4Ola(0Wd{VH_T`o6H*{&2nM)P{^E-!CAcjT=qJ{C9wm@P_RKp6s zt>Uwl%|Qx;!Biy-WTd8Z1t`zm^UaZ(9)FaQ$0APC`AVFo+toZS&+}T%wEs+|o|@?x z+)%1C3>=e$TDcH>Z< z3)ODMePv*)j>C#WB($~|GoO4`_J88Ai(b2y{6+D*QYB3IvoT2=7Qe|aWyW&nF($#c zesu`9g24cA3F|w<9#jqQWw9>Yy_JC<%`yv3*MKZon%4mp7&^>)xhx4Y&1Xw9n~^I6 zi8}L2zz5(W;abiEP2s%?Si0*XFvf5&xPPsKHMpLI%HT=26uNdj7>vunoqvO0v!|2a zJl(K*;J#WoEa5s2D2GXnVgXRXkVUy`Y^yx%@tRB>Y?HZjlxuKc4K5-ssi$wHTVA^5 zdFemWe|TvrEtS%emktK6;U7Gd;h&e1)JiQct$1lAb<=M8=1uzMjhDjzspokq{GaaZ z?7$x{#h;xN{!a%3=vH4SpnnRB$yxv5$@>x02y&>swv!Qk_7gYB{C%?bnYmEicj(pMOlIe$;{jB<10%{9Ez zUttaK``sEwip$e$!%uy%7^SHn)AZ=oT^c0eE0&HU{+Ht0J-%HpSo-H8VQF#`EoS2s zbU)Z*nt+P%V@g8^z?M#0`8je=#)pn1U8(dKbkM zwK>4sa&}z=oJcOH5`PkMmwEY0uK1ol$ARzDq6?f0YIb*5RrX{JD;`k#=o{TL&OW{U zlpY>kd^|{xKb;+J|6vkkiOsD|M&Fb;=*)IA!7Z}{FU$eM=^7(dTQw4 z(#)4p;*tox0Lob2=qHV<-eF7-hCxuz*(O-K2yg=rLRF@^u7_2O5Yy#`tGo2^z9)qRQnoOTLxkiuS%2(6TN+b+i#xZF5f3bKI$zkZm7Ly^8HrKHf7J``%KrJE3fz} z&z!Dr_P5@j29;F<7k@`9T|9m!-|>C3{E5yQjaYQAyn?TS+}VfGHQ;b@!T*O`dw(6SBcmkNX93tF`YD zg{kiTMh=eOUVh&FcyPJ<`Eu{z-O;g|Bb1C#D_-!=$&-rn`G56*<9uN9*_ZeQQN6cM z`_6EQX9OtSen0n?$MbW#t$1LVOR^?paNiLrynq$|xO=w${_^)3={_mJuUsN>55Ssg`V=6a6Y!CBqHNALG07cV* zZK884@LHFcIytv5FRv+t*T#dudl{1R+^ul_n5HX@e=_4y3ot8%UTGY5yioeU=<(L;}y;>_bMXl){aO|m%^uJWd# zWcjs`AAiC~uBo<2oW|6YPB@WG};sow80v+wg;Sf3-hbE_m@^LUIMtsJRj^Q)UXLz+_z$#=m zi#|Z)*L^>s2B=H6FR1t#Y!y%u}z_t<+{lFJyxI8VaKu8lT}2Hc!} ziM>qxq-2;yQ{(D<_w?A+P!u9^EXf1Sx9;yTh+xi_73B9{4b#*plz&ShSO7#;2CL3! zf>qfX&ux;`;@(!(T#_T|W&1o|uv?C12!B{6xfhaIvdg?`PQIV_FX#|h6;DfGh&*ve z^pKA+&*?Ju1{jW`P1a+~lEOBGCvReUe-f)JeD26IH&$|VKf75^#%|6PK-_kT8+ z6HU!aG$kkEbhLkE!8Mu43%W^GiNq87R-p-CDJ~a-x@h9{ zha}Im%RGm_&Z7jQSbJ3v*PJe9)L@KIWi^7zcP<9 z#Yc{D784qn-VP0Rs-@CdoZW>?BG;v`4!`EAe;SlY(Mgs3%FGoXER>Nb%*1=UY@_gd zF;gK2PN35ja2iF6ka^X$NmJHePUDoak9;a{GNrIttrkwng1CA0V?MR8y)J0CAUlWd z3g`KZ<+Pm9{}l#1glS~91anLSwRzx%3D6@O8wGHtY>5W^K{)+9^21&B*}tPFVn?jX zfAF(WZ%J`ZFsAYDY*qknIWH^wP8kZ{Hkdxdjsg~gJ2Yo$2w^HNLAjiRX%YO?X)m^I zb}4%vh=H=<;HZMFsN(sVD3!L1Usr48_iK*8%FY@*if-ZLWfQ3^f0RjnWy-O~LcrmVXmKpo|@ zzFb|(Bpm{4>jEz1UIlO|m;4sMXnYrhU}ty`jzPYa6<_^Go_KL!RxPotxXG`U63H^Z zc!e^I<-?0Zd3t5Q<1<)qs3F|ovMnu2@z$2-Uc22Z z9|2IjO?61xX45w;^eao#3Wgn4vaGcHIaXiEI{Cmcn8k5{~6kbfenPkgnKn0abf8s$g2jEo##hBG1GeZS59A}jw6^+PCgn5LlA8;1pQeLpVf+?=n#Tbe=! z$q;n66qZO>D4E5YE0wlje>aY9==j$x$yiPD@&ZNRP?$2Hp_^MJp}r5WK$aYX7OLy5 zCb&4R9#$c&_~a-cF&hxpnsK`_{b^p*bkCO{E&7>yy5rdJzb=#_<2Y0N!gb!HtIGAi z_N9U$7%g~cLVOgWW`7ilSXB5K%!j=TDwQ`MLoX!X0xuxHQ;!EMe>3zj5}aZ(4ZJpa z3_MlXlc^AF;0uz?+RdP7-XG%Gz(P1BcpY)*>H8q>&b>e5p5idwZ~`y zj<&d(rv`o(Dq>40;47S8^>kzq3AZ-83l@zj_>2PCFt_M90UHK%*rXTa+Fij;dg9WN zFCxCx7$y_Ko#3Whe@hLc_rdgL7EEv0DRnD<7-3}fZe)`AvJw#4!?)NG$TdK{Iiu%7 zve~nY{@~skmO)%E9OiNrUsu1G{h3%g7>9gD_bgEi?$^Qp8F(I*HFLlvm&mt*h)j=) zvwO;nLq!Xd*@Z6Kj3$^N-*dn60x%tO<{i>}kC$~#@6K5ne-b%&88qyAg*RI<)zQUL zpCA`?bsjh=Ip?(PNk;zcF>g0EPWatX^m>n_sqsr}#9^*${Bg1)PXtT2V(-I z=K0i_uJJ$(-KpAby9!3ovbzT4AP5#jgL5&SCZ=}dC)zcop*GIgwYF{&wTi2g83mtk z;7K>Hjn^a9Ezz2V`eXaW4P*1ftyHD~yIEYrF;>Y6fBl){&-R~5B}kG5O)9loyM1NC zI1VY;*G79fD3JrS3 z7l2ywOr5LQdG)KQa;WagH)Kje(959MG>l#pG088h#P2(>-Qt4&CzQRrztH6pGAhu2 zBqouJ1q>W9?+_ZnTRfz~jy~cn+jq1tb)<&!$eD%64nr{Osu_%g!9tQ*xMC(*Dj7%2 z+j7jMs2PEaHpgGq{3l|GGcU!>@Hm>f7hW}(*~uQ0mEnDuit zwc$`P#JmmRdOQowU}D??PwF;=r8;6JIbthQnwdg=ieUH4SsF8xUG`uc33H7xD@cNY zkAq)K0(gjQjhQMLE|s=66@dIEIgE`lBY;|pg?d#t!y5`U;XgA`b5=Ncq)oto5%azP z$|vtK5ZpS-7Wa1<({&blzfeHV61SB~%8T5WI5s>)$=X-bbe}JdyH1o314H=+12^4S zSe)KjmJ^Fe9Wh{JC5+!q)gf(O+P9G*7xo=QG|UtuknrdxY|HbaxSQa4 zB3e$E9O64_FG6l$D-!{WWZbZXc{?h`hDkztGY(J*Ba674onyi(6q+lQIE{n^3+!Zo zI4(4otb16CDrAU*HTg=Ci(CMql}d4tNi`$|S}KhhCxxCwD*(|bgpyf*GHSK#m}HAd zN>dZKfiuvqs^EoLD#>Yxh&1En(;)>jlrHDD9p(23I9=dAbHwaxAKCq097voaZ;s9q z2j(S!0#}A7XW)nh~&ZkAtv>93R_L2eI8aZaBVy8<(>^nm%YyGH<6zte0S$nV@Z2Dw6V z&BL;H_2P;+P|HEE%4$GffwBUBUwBBKjQ`}qlpj() zyS(yq-}@~zqL_!m{vEF`zfcdD8RxzXZ~|Ts$eB3B;dl}hD+>K!cVl5BGRU{ktCC6V zRmnl%RSBGkPM5`x0Q_K-i)GKETN)bP$xwm4=GGOe4PNQHxFg5ro5Lg|YNAfv3^kesTwp0hY#G7RJxFIIath@_ms7DNS|#BEA(03$fm87`5~CI>{6Ir(V+6VB9Z+X zaPE%zGzZXsOjCt>nWU+zwhFeS8IzNDpL4nFNic)9cVkmN-Ur;s(T}~Iv)J%sa!ARR zW18JPmYu^V8At0O^!AvYL@_fU?-wR?s{J@8G`2kN3sdM0Nq;M8mYToI%g-{><%_~x zBOU6ounGnlO(A*doPTV5%bjJ;UphAZkqCm=+YM+e1)Q}o+6u`3UFzz>}~(HU%{Y*U@EC3XsIT@uRd ztOOl@n+3gHT~q-vGQI}UBAGpEN@iO-i}H8PpZcIfItk*0X^Rm230-Yy;)-*^CBG+_ z7>A0F0%>s!S{{-+U66t(s=@bu92)wk5aWY@Jt8usIlCP~ns98WEo5;G%G=u64aTF; zd@VcZXOCJ!V;G^9(9t3e*rOH)$t|$2i>>{CGc4)*;5Mkp`qpYXyMYJ@e11_qwAfE~ zwE$8CAoN}n45<>@IR58n1lGe2!jQ&TTNlMJT?Nc-!14Ql>O%&s^%K^MLSv692iplF zMNtA;8@OKskXCfn`n`vcJ5+cE1U>zcCOY9;?gZM)H#8LCr-=8YutVS&XKAQN=DuRcT>#Z|kOVxw;0f@84Jb zDf|E)>3_HZ)UuC&g3Mh`06R2i@KMo!E}#Ud7GQ%rvd7QNXGrFQFqSiwccV1D55jSD z4;b|TBG>?Wr6G+C{q3iZ0$QPu5Oa~Ad;~fmogyr=HvU@B_)!Wxq0@}niU>5y9X23%85(R->9wrC({~qg}fPT{QgrQ_UO%3tn0iCc<`-gPsoxmw($Nv4>tPilo zuR+j=a@})~nAyVLV5u|-jr=v=@hEbI3u`_j*BoMnE);K0VR6kFU?r8B0mmZz0QEfY zkd{iBF%qOH9zP;u)$UVLLwP2DezB6-4yk7x!c|l^i!vcG4QaYh`xp4AG&Dh3YH^Hn zw#@wY92RxS52*(gAX<_za>AW#^iNi7e}>+?u)XhBXisr3%*e)4X=~2xHN^tOe22@; z4{3kxaG{aJ9YtvVWg@!nS?|k>>FqZE$V5}(1;W-0$0Xkd4&M=XG&gieA!D9EG z!I83e&wb82x4HHo=$it$Gn`94rDT_Jb=-oQ!o%u%2?oNa39j257!;O+ekFt|_M8z% zb=8wm`58fXNWrtgnl2b1j6K#rVZ&01|H9oz?JxCJSl=w8$9d4eUjxS{5TFdg&9NY+ zvqx~;_tmM{)fEA~zDYoTFUdfT$_b}YmunW-S$;PPNq!0}Smb^UhS zh`Uex@R1t^ewv&GcQkQtmkO3Xgt%tf1v90`F@~pxqC`s#o@RERF~z@N;GmQI9D9#} z@sve~)P9cbklr6a316j1!|)8`VB-B8lb>TdTgjh4ixd%D5#A0asJiy+tp|=*CEw@P z8!wDsZ%s0q0&na5))Q(>Iic}e!GHX_mvtBeJ%9X`yIisr05!%NG&I-Qg;#ocT-Gx+ z&AfBE2#ChF8F-D5jE#%if_B(L=HcYlQz&+0PM+3C|NOE!UgC}Q)>e2l>;26&^YZ&7 zUW%^XD>teuoji^FMMB*|b;~?-@omm+0e^7h*xScDhFs(0jE47ab8`Mzv53A^ZV&sa zYk$=-ndfU}%wf??WCl`}rjl{`;8_m2^{Ql_3(?Evhc{DcDJqK}d(-~IaKlO-o2Hbi zixIu`LgN@>4Vm4yIFM3_{}y;MQnzXK)6Ojj61~-AvhTgk7?=BLD(nSd!gQiI(!ZuNzKFs($=6`x4V?|X2$o(RA#@w8X4Ac9Fvxf?eDeG<;d+l#zRNWVy3V?_Y+H>VI;% z*_G-ra~Ub@!wM=!G7{t8cvd9=#3~ov>4&RaBydC+luEg*54CW60cSsyJ?4fhu2#i3 zxwbB61_RNqXxp$=*OkH(9T1fNlm|;jGTC78x8saFhO?G+w;T3?&3&hm$H4rl2yrm? zL!F4IwgCiJ&UI${A$Ss#3mPp=cYn27+qNUc|7=FS^2H&6#}t3cMZI^3_$^VwsF#!# zugN3R4JD+hUWUub(uC3e&3w$fe`&PAk`#OeM=N8Ut^3WL72W2pImT67g0A`#)pe=( z{`+k~eEmx?Qe2w;ev5J9k5@c1Ym-_nWlcEso%x`Pbvd`9=9 z8G3*euOB0f+m*ea#eSGfq8LhyX2JZ#XHfHm?#(FoC&_F0=(9L@5bt;MD5N3dWx=oa zr8~uioCP8E<9D$?4!D1#PI&TXNbhGAEO=dV3xu#6QGlF5?0O1rTcUq#cu~nyav#85n%aQf z&K~EmIYD^pxT5+S(Hf@KW$GBVF;DBrv`L<}ot2&{rJbrw9vv7m%~{PVq-7#mnFo>K zsc_d=nK=50c~m~~fw58q9PYy*p=8406mzL(7N4D8;Ti66O2Nr`!7L)c-<7>ty(6NY z3sOQ~PLF^w?0J9Q=ZT$so%Uu2ZgfB*nFR3HqnmFXZkA4FYW`9wDp2q}D(QC@BH2-3 zvF|b#2iFTmc{oIqj3kO4D=2!bP&1=`(jO0pE5!{+L|-}G%xg$QMua}%)k#$YcwC(X zTuY2YL;`sdMM~?1>$0S-f36u}KNJ3D`z?{7bSJn`d%Dn3=`D~oZ5 zz${87aKX&`d{;QbG94^##3)n&jC`V%6EW8yL`sBDzmR22f}$*glj+>27=56z7*xjI zjaeZDyc??O)oPN0pkd`9yq6H2UUM6i4}h1;p&>;o(IT(z7pK8eK+z% zE;8fU2;c+B7U%_KPX~WT;13ygW^eB4|M8#y551ffv79D}On07DQYDdz^*(lIWv7`j zE_#nymKlM6FJ>95a+P$G1Z8ht9+w|c&0?w;fCbq$oC=>mhysMKpQ)Win(+QYwkC@J zos((4kP(Ttgm>7Pd@!VB+?!dENv7&@GBRB*)ykAMu-W_zj(2*HtIMcRnQLiVcbGcO^%&}B>V^}wqb5L;zvR;J`yEu~a1SM2M50nx3O&Z@fMlH4D7PkKprMHpD3 z@4z#}c_sQAc!W5xS>g!E!5*`(qw!5&8UrTbismql=`h$mR)fzn1s>9a?glnt()EmC;nTDH|eyYB1 zcGA*M57pOoKP|z`g=QrXJ}mR0_QHN`2+}Nn7sOtbKV)bz@msaF%LB_EzJ4hLFd=vy z5=TYfS@m(EU&eIig9{XWNe_x6C?JkD=u=rihGAAq9U zo}b{k>jGyd*}w5)f5d3K7d^NK2LyN7D4HF{(cPDL=3X8w$>jt+Ud?CzBj~BQf_9UC zj4Cn~VYNXsPCupLn8sisd4sADyc}AM67Bvb7~N>|1xwHw*J>u@8)-%oX@fx!f?48N z8w`@g9R{74`SFwz)Sspg{@r{=O@diQG-NbZsssk<4b|qLc@5WSmS`yF3PNh@WHbw0 z-5L|kvP8oc^iE%rXu$xH^YJ~=aB=5<_H`f!cf;{VpWVPcM(5WV1X~EqzAO$ObOgak zA(h>UZx{32ix_ZVZm7l^%KCB$eSw>Y&$Ktt_>R6LGx}E;{m@)zb1bG4OUlWd(FwC4 z{8umu=vZU^G%;;`NhXl%pRay0TO>CcR0@^gzWc;|z-_3D-%YlvVwenu8AcX=ma?MI zqWnA=&4RZZ+QqMlW(0gS5ix1g;42Na>qpJvr7cX(d`R&Lj-z{U)#YpXsReG^>6~j~ z-_TH_A-ad9F=h;TV=yFqz&o)Vkc`(R$D{u=8da;+s$9JhDRxGCgFid#1KSmY;5CY| zh~bIle-%%!Ujf_fRUA*J*VnIqG$V|(pG;xK#8oJ?aS_URt-#CYQx@X{(lN{ve7Be= z$RX||jz%;|qS&S0j{cKgrb}|?kK!niLlzuDF~B<*AZF0sjOk>FuH-8NQPK{uK{xLvlS#3RLbA< zC4oISlDNJvkv&-6f$X43f8Haa(a^ttMi+?wY=(w}dH*0(B_2vwEgU#xIU>-gP-{|@#f*c+mvK9lYFz z-m2uS2mWVVrpVI6H8HdXuGtoM;HXA$k3-Hh&idx&@^(lT#a*dW$WsMSNcL})r;1}* zq&_Vrz5EqMg%qS0i%cSah4?FS*D?~Skj#(rV$#YHUHG_q zz^w4}0LiUzvlaT=xQpt$CNE9>4ma!EH{V#E}@#sg8vQ-sd_*tjirT?N}!Fon7tRPfk8}S1Q@gzU# z@Z=2=i7{SbMX!JZXM|;4>!Z+jsGB4(yVw$*vOUpnVf6k+U+AS$(PaK#X&g#!XxnuE zn>44F4IOOEM$t43zR|JvN-sxc9Xn8Lf4N+LB=j>@vIo>pkh86ScvVeM3(eP+t9Cv` za_OYL%gR#swj6iELie_+uCf}&|F_El=Gm_C|MQvi1#9o_Be6J6GIPozuUfz%@dKVDyeih!U*?XK2NB#cNhxh!u zzaTaB`%wVz`S&Oxb@ls64DVHQNRAJ@zP}(x0pWk6gq+0u?~v>sc>SY*WdEPU+5bcG z@xbfH33=&nDtUU~^C)~4%+wa$!Ez;a} zhNRzabx5bZJsd8H$UAzBgV`(?DcPNBtwEexA)6oH(J=7UTAg~WL!3sfu4KE8{hPZ& zWwm;>K^nDYQ^_@@Q9LbH)_^{LHtJ1B?b@HWYIC((r$d}tvr!+CzO%jE>X1ei21EL- zX01~tt!BO5Qp&^ib}>^MX;kYq(rnfo=*=*vZ{H$)D~T5fcgfx)H(3zNa@ouX8!x%( zGcc~Ic7r#e&FhvqgK}A(Q1X+|@T5Cl)UOUpCHlsTfMuOgywKOB5&;c=ciIK&_$4=a z6eVwm%En|G^L8#qnIRJBVJU{RkLx448091KY9{a1#i-ACx8SLq^(=D*vgKfpexG@r zD*3QkJfwfzZW6t0v}(07D~D$J>cz;a*=5MMvXYt7m3rZlX|;D#P6tznd;chwnLn1gIG1EK5BJv zER7m*>ez=`r%7s6Y_3t;elcoo!{-K+z|T(GZWD)RsMVYov#TUd?bw)OmY5_ShaeS zG&(IJANX>>xUFph-QOnd7bB|$D+JT;5DOZvCzi9_fN?-C#8YCmnxxSt{O8+?k5^@* zQ*S_TbWSGRh+gV{vX}ae#go42OLFoY(9~LWgqo@&phnEO(;@Y3yG@!k;yBwcM)d}1 z!_>4ow$o}8r`f>?t=Uadt9M5AhF#|wnvUIpBJc=bomv%20~A(a>`n0G00VASF?S80 z+u4eA8p?{`WVK;MU~(Mdz_L1GMbrgxs!q+W!*a9%%$m4=81ivs)uFc?Y`D`PP8;C7 z-XgV5ePqE1s~uu>w)x092nLSh;LvKYi)wbAG+I!)Ubj1>&9??XR@)g_4Y{h&1z6Zx zowO5NOR=QRi^OuO+r+9hp=$9ovf6x0wmO)IJFE%W0UBz}+Q`DW>Hu63hqvD#wH9`X zIBo0&?4DYGOMFQzc+CGL?G~|`+l2q%TdPX=4+kQr3M>DEQ1tCt)YGX08S7Lf>cQ2l z+fH3wOk%YVXb{{T9JIE+xn96x)@>)TI&~Wc1>aSl~A4v|BiD zoo%x1)QgMJ#>Hrm_6uB%DlP_8Au*d($D9sA8BFf40>z_#Yt)j^yJoy7&g{8e78`eIb$4b=Efvs!groNXt; zw<=GpCbepZv}-jYeqW55En+$CCgLibfciFoLk;#xtJNBHa1LsSC!Hn$t<~ssY^VY8 zWvkhLv70YOO+En#&>fB`TP?dowrd=zZrja&0ZjIu1CvIr0m!CS-|pmR6s`tt&3Xq9 zpmt4eKdb39Mphd)TaE7#I9i-K-=S5JMI2asw>S-)jv8rfb7*LDtll9`69?Obecglu zuuZnJZ!LrvI7D6~RucfHN!lZ;ApoU=ClEh>K;d5^ezW&BR!}42H)$gXIz;@wNNUw> z0`!vjRBr(U)|w3%LVJ7Uw6^(yhfCVz;9PCDF%#5Z?T8bsVb^N;7W?iFd$EEp z8+PkOVs)U2dK-_^TDxYqh~uq+S*0 zNR88Q@V8OpXos{K3FfKuto79csXz}$Z}dxht;fE+xVnewe5y~U9Z=zTBB;UYwej;b84E^YScBW)6|@H^~K0)0OqRK z0d>{uRvW8u>YC-$XO>fkpLW}FnoSLVTBBA=b6^M6>zY&F)*8+>6nRl|wzXEv(PlNL z0}tPD60Tr!->)xNyHSG~{cC+jL(if0*I}2>tgcJ(0u$l|5J`XZgHU&WBk#k35t83$ zW>>hymfwLn?m&iOj$pEF6x$J(}|0eW`Y+OHf~-`BRCZ`yMoglOeI{=5GzuLFR(s=aqQoh+-4cb{*cx2e=7axG_cFnM&4Yf z0sYwgLDuo#{}1X|YS>(}sdTx9W7+sY*72YJM`bL0>gv^e{%bgYr_X%A)oipo*vFca z&v&~63$giwzG}$F;stszMTYyhq1w5(d^h2}J@X4|f7wWihSXo!Z} zS`Ci>y4?Xp*@W6P$8KA7tzp-!y4JRviK8{`j)u=KM)+B4*)V{n#>;4}js}FA=F~cN zO>4C6js|y@rnc?aO|1zXZq)6zwq3V7+IGvXX-=(SH($ViBtb5_u5~o0i8a8Vj^@;F zS|g`s!8kfr1O7Nwt!mXQ=v~`tT29ld!5lVPa7uLAFDyQ>GYcpEroO%1u6Mp#jq3ZF z(;3xku&Uxe%+ncJa20{6ZZ++;)u>yJ)3#dMTFq|1uxeTh?n;_dwVGNTGCD1-U9;;} z9fw+NTdf9vEN};#>eQgAZ!hjTTJ>u~G+)KEyQ=1FH&zm_2nU-OxZe_Pc%QZUc5U_o z#_Lo^R=uIsTNVHhoLZfh=G5wN%xN79=HITvPfM$6FnzF~9j#W?IvRAc3iDaBaHauP z96WaGmZLT7js@`4hE!N7fF5g`FQS8kg;l9JZL8*gw6t2savB-{k5y}Goz4rZj*S7d zI2Np`Q^&&`*GH>c4ZEhn3EWI92l#FlzKq2BwO|&ZWh`X1Ixj4T18&EzSpYwH=WSX5 zKNi4`1@L15`~X2vvk-nP1Rr2uSsl3K)R7%wIZ$2w#sVU>romV`S_A$71ZfNtzrlGA^#6~Z`vivksJno&tDNp=>c8ph=kuq_0)`J zz#f7kKyn8o3|Gy1LUmT6D~YVkn#}BOT>akYnvBAI>U=dMuzCh^JDr;@noo zgTbp3jj3R~oK?C3+Nrn!%ZIFhc3^33WfV|h<*=g<-(;{CUW{=1oF<5XLh;u*XB*j& zp%8FpE6#Bcg3T%}bt9MnJSf<|jgatlzhb}7xqmaO)yl9L=UcufnK%_!A+T=o6G%UcVOl4`MI0ym` z#t1Y^%)kOqfnR~Vfx9S+WU4Et-{Rg>O2I$`bF)f$wuB|9;ES$AJAm7rDv zv*wMEw&GU7>dq?Nn93tplE{`DT#`nxiob69!$uFu(|6uvmD!laTyeYPVqqNYATW~R zi3imxRVm<8TvkXC(DbZ=X~DSy^KxwAxTJG~OM_GgCx+J8o|H7x1;}e!HQ-;Vge*o|(RIyQSbw!J7bid2E1d%-$(RPfo3HHo?VO5}KC$Qrt z({aTcS;;5TEv>1zcG$7tp1Ky?r=4G2>~D6vEiS$=7B(X6cwAdsafO8LC7#M&f;D1> zXBlS;foh-tuFoAv9hNwds|2zjw$Mwk2sFT}9h$;$3Sbii27=pt26bg6k(U8CLhfc7 z?iv#~8y*O{5}=S;Ji}an#m%x&qVf)THqy0Ku;jp7gm5S*DFgxs0{#X<2CQBK9Ffk! z53f06jWF2_9WZmVd3^X3*h!Ex`%$rzB4_p^VrPh)*=NGen+zxkJ_gO2q$oyGRPS4Y z#>?GPATkU5bBT5bgIc^pl^<|P0+EzxJn(QF@B@CB(f0~Rx%3Qw3uskwZQzKi!mKJu zr3g-~bNb=oKoS*IoK+k?T?#z#?bz z(7;g#JkM~rQ_^q2WP{jj)PXAmfp8!edIncB`ZdcIuF|08z!3~$hysPDTEdLsw9zPO z2;gKWem8g+1MY>X1ZxyS$pS0kf5VhkrPT=EhzLu|y4%RYJ}hl$jck-Qbu&l%uR{8dwX5cVZ0)m5uR@1%;At z8cVvKXPQ{F-q^;?mI6gxQE7pSWJSesv!IF!!oSp{1V~%41~wve*1!<18X;bz@d^-+ zXP`|>0aCCu{KIS3aI4}vdO`-U(u!*}ruN4~UU1b;!$_kJ#dq|7|8UcZFXG@a5Q8EKWR!UU@#6*U&t*n&59>I)) z=4wDB;A{zORN+AjOa)Lot2Ek|Q0oy$*&4KedERgjwrwC-VDG?%0gaX5qA?;M5M^Ol zV+r^1fV)|RXhdXIh+VKT3dS2#xdz!0+L6#=g9<({e8As;|SiVq|G9V~pLy~(C z$Kdx5XV((kM5v1;1I(+&u!U~~*9)-!xdo5EGqX@2TpKdC`0yJ-LHPw08|ngCz~BQE ze`gk_K<#nitJtzyxIR(@2ml%_af~g0_)@~XYsD(1ISf@&4gN>rcZ650@nWHXu2cif1(*a$PBQ=r$(9VSxCI>h3LF_0qCp`2 zerUj4%#FdH_-VETPR=R^f18TFn*&wi2EI^rLo@HugK@A}NDqp+1(?MZS|o{o9>VW^ zdVsQ#9wHd%K`_zM1O1-RgJqEhDur5qqz5ffAMMkFGYc1J0H8MfuB3$lHd-j$NIv`) z=wSx*036W;8i-J|085%2wFmSNTW;y0M-ZF|kY}`98YXd*ScW754s79f#C(N_2Q$M$ zRKlXoCxXG6XBe2PEovrMIsA@)(AOmAmEb@aifinzYJ2{0c5N+N&4 z7pAd@Q~docq1*xl9+)NlE+t!Vqzn*Fk^Cf7t}M6~(eDRpf_wT-&6x#ojl|yCmm-}9 zO#*frp8ZZ$7WF9TW7eQ}Ct-qcQUF5*MF8%KVBY|e;1B^-i_A++u(X1ISqab_Y_otE zu*UHUjUf_@vuODS8(Xl=qBbSa#0K9vC$lPE6Tr$sD~;k{9ka+iXO;8~+C8kqdvblS zFhavXOy$0@VAV2>s63vvzTy1P$i~Ad43j2_6DF-^jY7Ny8N$ z`2x+hyip`b;lZ*%W0-?~8O+ebqzUr^6L5o8LmhtsJ}5XEO9dy}p8&N4_^}}|8qP$R zys>a!fHxb`SAyxhz>fx*f#m1uU=pn1jSAW@N)AAkBm#mJ zssSq$*`Cn`)u0Uu9IQCdK%IUGj<>UP(AIB`!Z4v`FP}DvN#ILxG6Nu(Au#^SE(o$L^3Kadg0$Z8l za32I(EZWLIOk?zq0Gk-J8;oU3Kswj15gO>rHwrAV+`rC}9y^1jFe1}$)Arkhew$vu zWMpI&8jZv50@hbZ(hLAFYx9p7;6|YHZ=PIgTUh*gyRYX4!V+40iwC|e$=U_GoNLqUXilq z)r(jw`dA7cHZHjOQMS^YF9+vHcZ|>>0vz?AQ2KVWimq>#iZz$JL<1jx&*+XBPbR#N zTaaH5+#4Q#F&=EcC}5}T3igU2W^WcXLYVp9rcqi-MqeptQ95_6jNPdcQ3rS>$yzQI~(A-e(# z85m2z*Ry&XjgMr#{c!dv%7(65`&STbIC%P7KrUEWfHk6F3H~tlsu6P8kkJe7d-13T zdo`oGQJ`?Nuz&#)PBWAV@Fvyx3hWt-Z)~`s4u-D{ocl|x{rY+hg`Muoe760+h~LN0}B(8a)_(l3CJ;Z#D0D&&~Tj*Y%D!wim3ckr@e|5lt3 zgQmdQZMMKqWeS`nwm>fQ##a_EUMh)REE=5%n5;PLFSBHg3jPi#2ZAghVxu5yHwLd@ zOydn0#qc`Kazkz_PHlYSmq7A>Zv}c6s2@-ouHVVL2r`#{pP^R@{@N&U)hP7Pt+Yb# zi6EVYacvr3*SwHCmyMRJec-gWCAUB84S8 zuD9W4>5h}P?m4kUZ4Fn+q83vHT*Er8H2k4n45){wt~nK28x`ltXZJaZ`?u{yKo9h$p&XG8t!@cHKGknD^e*yn?Toi zg?HzFj4h0GAUsOgIjG2k9SHoB+=8LjYXlsU=)s5n7NqnW@dnsZ8R3%S{XYo9x{{3t zafNNTTd2mr#=%YqR&KD5IN)efyNWHjBIN`&qaamMh=b&+@n|DyK#c=iBP!dK-GHL9 z;An;hq$>1)fpZj$5svN*75D}Eg%PGfZyy7HOiKuI+cQGE<}{dIAy7_QQCSUq1PpQD zvj8d){SFB7=w;y1?gKZ1QiNY4P9@kcovE~c4LoDDuOtwSUI;LQ%lPXOX^ewZqBQ86 zxFe)}V*#ou&@gYnA1&NT%MJH%tG7_EK?I1Rdo=^0e1HswKbKOHa|P!pSKG=e6MiFq zKQIeY3>TF}n~12;4ul#Yulxp0GtBBGuiSze(=3b!+l5(ZuNwn@fxD#g;O@>BObc{^ zS)v)(!#z7!rZLRp9E1QoH`cn|pR=lwAOkgu!G(%hXbV;v6@QOdR1u4gu;>VjcA?OL zCH~sOqEtF!kx5NxRC0ty0Z6p3NeAM86<92!g`$8%`h}2KDduMtvJIv1XAgnG5(0(I zA<%lY1T%qSxlMJAfmhQ3RoG@IRsmo z0UrQ+AmX6GXwoKND>VC-#N!!%?6yGlU^nN%A%eyQ4pcB(xCENLE-+NXa=0oYXAXmr z8-flvJU3bBkm5mjm=W0VuB3)2BT)Hu2o`CShJUsH@SXbU`f_!3am4!~;VX8fHyVxv zu9xJ&plOX5XmDT|s3ZinK=5J1D-GueTypuVM!cb=s0@k;T*HTD1HP1hp02h0rqLw6 zp!)+Cz_VpH0<^_*$18RvHXM#ZZq3rrCeTPGd;>NK*m#uz1rO99Sof=jzX4%s*txER zfG_R-rAsxoffHC#_loK@oOk`pQU5}I#{S)B>0gce_n)nQsXlf8%d6G$y+8vsEFMw; zM40I%#9~6YnudQkTZRySbqckt8z6_87Pg)>5QzzK?GQ%S01a#!i0$_H6M`%S(FMf3 z`C4H)pp2RtFtvb)T35G}fg_8wQNz*_b|BEtEFowfg8MWDI|71gnq`fVw-{a6m4@Hl;EFrP?3si zAcA&K{<9Ctc7w8?a!?O1_7@L#PgWVwxvR$Lik!It-Lz1f239kAbI;Id zVZ_M%n>Arh*)@@TkeD7`>_5dNWxS)D{i$4128-$JPw9?-vXXxGr*%h(SCq5=S??$Z zSCk%faZhWfT2HmG?kf6LMfP#H+U<|2BeiC>CO^&^W8FiPGerF_Sq)Lo5N_<@Fo9b{ z2%pEVI;NOGkIq`Fnrl;Y$!hLREri2ZaEy|MM!}=A_BFS&rN9<-4JRRALgp+@|Dks$ z@K(&=ATV}++$;qp$$>KvBdmd?W5i5Nq4wzJS(Be9{JrQrMEFNYyAkq#tkey6OT5Oa@g55xAV)|l<$XPa z%d90OV!-+kijF-%yguaB31RP+WE0FU)P(gxPEf;VS7YXsnpU8O1%iwjX#K2}sNr_2 zhTE;$sG4?;p!5hp0j`GIt=fy4ZnuaPwHQbAMINWD?hvbkCukAyP8|{4o|+xS|Jl-e zLhiGF&q(hja-U7oyJt4iJCJ)It)``|_O}mbZ}zFZ=K>BJf%TT<1wjG-BeExo*AhrR zQheBj?Iw)B{QrXD6AZ%4JDLYjc1hlOR{#7hLGc@K)p)NL70%G2<*T=ys$Mt7RBtnk z-LBro&rdLeNc9B&xPsX1Cym22T@bR})lMsaVa9@3sf)kf4?2hD+Uksk08!wXuHiZg zz!DR3B^7%$3jIQw{WJuh>*|Siy6ppgU!rvY1Zz*f=vwfaYw(Yyy$D}Hi-Khb-hwT> zbGN}J@UJDsT*MS_^eg5>gP)Ylh=>M20AMx8@P9CvYK-|0BMWnZ->*~~22ZiO|DT(JAlX+Cp*o8@Y~`Sgqe+Jvn)yNx--f^m9jcQ3{oy3YFLuVB)X zoo#L|;JxqA+dFSMBWoWLTM5+DArlaPot)gpf60Fj|MDmeNFtlrvWap4YH^f6Ps0)4 z2+aNiDP-Wp2JiaXI8irBL*hOomXsri8`&n9lM2oxX=<%4NPq7@dSs_4T_9vOVl^Ju zhAjn{kx;m(C8ZrwwbBYwjX_30Fn6$e<`o=9%0eDBqw(-Gcs^k=HX3NmECk|z1&`7< zci|sMcBLSv64Y!wQLA~^tQw#+aZpFgYQevhw0qNQ_xPLA+%#wQz ziQ`l^-qm-kxiZKTnrofvm zX_q#%O=*`l;K&b?#584phSQ=o++Y#ftw_ES413yjG|3h2Rgx769UxItDo&w@FfrPw zZeWih0XDQrXANxA2KOlnxX6gQ#ECTEC1o+ge@LW7`Lp34zSE|P8NGVSpj# z=k>+=U*1u0z~J464>p60g^f3xBXl)iJnr7a2jh})%;&duboVUNdTgrS{8uSoIyJmK`}}qhzuoJnNN+p2 zzPPz_JPD7}0}VmVFJGK}H2>n8Z#7?8J%UHe=cgY@aryxEC;b0H==0iW_yB1RoTsl= z2qDgEV}<}@jIrmnv85WW1n`BXEE@qh=ls03eAJY+)}7aXR+Ip82$1mWd2Q9{g9V5F z5~x0}tuD1ynz|iL-7BS_snXP~&-NZVa;5n@K3Po^dUGQrXsxw5ubnE)$d#7gnG#@# z0~l^;`py>so)refnJvtL8#bU_9*NiLdclOo_Tf9Nxz^g(3*&`Qaj}ds=e1C*(1q|L zbZEr{kOZfHIu3hbHC{za1$0M?;Rl~>seB;qL3t0DS}+Ha)k5Qs{)%(P2*pY=ZR!Ps zQT3NmC>ps|-kE?QK!sCA7N+EsEA3>byemyLzFFVxrx>PJ2}b8iqBwlw8&eX^gWiu^JeEW)Ws!9k|r0UT7ht?}1zq1k{lM zEqK_@hV$DB_`&+&SHZGhd4K%YIu}|CEYBG+F?$hKqT3gUFrl-bCF{{$+lJ)XuZn0G z)H38FQm_dw4I-l@I2eM_5&(kfyEbVlofU0byXPUQFMVMte z;j>7zmKC{|Ws`i?S#AfEy->n)AP*}hn2{Ad+V8Dh+paLv{Yp99ejUh+K=CM+whTgU zB{jEwis-@fh!7DDhvY(Ncxz%|6-R{*Pn9r#D#f$)BMY*)8OQYzS$Z_1coxO2H`o9-4rs(lE`|l%G#xnH z$c7n_9OfNd-U6TIvDTSe;C31w1-A^HdY}j)ymmll@M8e)b?vzsPTk?rE3|eJ#5g^F z#H|C#a1!o%6sN5)L%5%YbCGRdBMjDEFl|k}u=rz8SKk6RlXT;N?M%YmByJt|({Le1 zM@|nYr+_ZU&wFq`4VP?sWRl+k7t@#p#yJH?J~n|_ZY-0)lAqSc`W09gWms1iUI08wl2G4&w#Ix9;jn6o26)rG^wWi<4* zL0bkD)l_GF5kwrhygpfc}`rz^LnA--$8eN3MyD`usoU~oEQAI0)P(bwDGVf;m4o{c?;Z3lDrr8 z7Py<_*&ffN2{z+su{mRGbRGxu13X$|iZF{} zT(M)$oa(PjCtZi5hBG|4%bOkwxLXP!X%rO%C}3_YV5fvRDBx}@fLqsp;ITGVGB~3n zd$IsvQaO0>YT@z6uz;8@fT#8pqC69~6d<9xq$)uHbz1>FJ(gX-9zqVoFeg0&?nV01B9?#exF%wgP}^#l7aB zy2FoQfyonBTWQ#B1^6w0U6|WuVQ!m+akmxlw-m5!YWqV0K2hLVJ%8sl%ztabH* zISD1L<qZmEd#z`O&K*Mja!SyIG%$=f29JGg^FoG=gXICcYn^0I5mD`$IGyHtXC zquU}*DJ^=Jyi~+WC+I?pplZ|)cgl*oy-5w%yElDveLA8b%H5Rx56XTN2Ga!x>t%>xGcz^_!e-nLq*6yOVhAeCnT$b!MZkcEm# zsGNk37Bm9%WHnQ&W+$tSP$IZt=DfzUmJ7pF4r;>XF!LVnIGi!b`j`>5N+VS7$mtO! zx28L6y6?_^N#FAW>Hvldt-6DyS3`2`y5po*!-#l@#2F>+aqc^3px(%whU3Z7U5R8O zg$gFSv!aJ}t(1!nL8H#J7Ls>|Lw}4;^^Sx-z_8wF&;s~^DGw*C%^*T>(eeKtKp5|d z>qm-6pcdLAhA*w-*}7ufsAIO{^9jK4j=v{!#iqu8NKhJx!I{j`0@_>-35XsyW_H|1 z-=N%FrSOvYb&ez$@tqQG7YV3k=NU?@D;T>EvD zps|?Yx=AU4;y52u2e;Twu+R6#$2s6Hy7Y7a3+mo^%>~O2JRI0u$qW8pp2=W$k>exI*uZv?pjjH0^22FG4q`kI~h>8+eDGsNYWvH zx~^5vip+7;N&jJwscv8(dv_AzxP~2!jK&aV-nB!YefD#Z3P+!L{A(|K)FffQS# zg=sr7G)gl9aEDFV5kNAnd$x%V;J8oYYzgkAF=X)=O|6~ zYQ)PCR;bcSlf62^G8d%Dw$1{K{*uluwCn|Gvd>q#wZidpriz{F!kL4@Wai*&JGJ8j z(|FaFD925Xuo8~MR-#m+0rD|_R%|Cvj+kj%P3FO2j8UHKAYj|$J$bSR(E$>6gH&E5 z$89T5W`@x}3%XTlhvBo}z&W`XVI5vtN3Tbs5puM+VcCV6oP{7SGcuEE)m|cz*vWK3_>giuMQ#FU zv@b(nnmoYAA`3Br7)di+I~z8w30$=MPT0`CJ9+85Dzu~X--RoDFGzHDllAz~A!m9^os1sb|pnSoVCz}^Hh1Kh+wquZB13F7a6;&5cu=u) zPxDcTAE&~}XhqPb6nb_9ET}OfG{$Na&CH0}BW9Hu%|D!1AgE=3x2RQ2ScIEWtD24Q z6J)Iztu@j&y4BKN<}qY@0bP^z^<;f>YyGT>>9lvbcKa3mbOFQIZ3Xx(1>|j`H?WJQ zODJ}034U7%ed{>1&8|YE0|UzJEiL%z7R;^lFhj{c*0n{s)mhdjYq&W;@3Dmuc&+Yi zMm}w;`hwBMwFlRKh=n`L^D1^=$_~$}DL9$>N~gNBLf!(;3Sl7{Dfb;IEL=!~pejwnh4^)#Gk&0Za4I10C&C=kN&3brrjgq?xTi|vYZcIsRabr2D1C`9~&KX|< zdB&#rAi8IN=D=~|dWu~yabX|a*m@i+hHDsKf>>6B@De15W8@3lSM5rU3;e85(#-W=(^CEV+97iQ~m)y}b1T4xXqXS2Rfm z@8lv5oV#R@>riJI-^MK%_<=>ocQv|sgO`7BSGI7Mbl^0(*aF+qgF84_Y^BNIfzbX% z+=6SzOi;z}GOV}z7EH<%*j~vkxC$P8+T-q|n>j}8js-u^P&O`>? zwB8GU#p!A<+|iAz+1zB}p!!#J3ogmb)I$9fG`YANL-{1MAH`NEA}1SAlTdwo^JcOE zI|)tAy~g-H^^EWTMny8$d>w%YVlcz~V?}QWCfz??us(U`?)3nDxHuf7d2!t$t7TKvR=U{le-DD)(&n0+tKg^vnYQg*MXzK z`aA};ijHLCrLoS6?ET=lzDV!eAk+(?B%-8<;=(9$5+k`l<<|#??R0)ljwKGm1ci%# z$sWC>mxmF`7hDIaKD&Iea!OO4{8J;;v+M#|hS!q)Ys3W)caKuz+0(x?1Vq!uiokSZ z8OwZc+l?*|y(2Ch;Wpt6UWq=p_34>p!-=IONk-`{kq)lc0h-<@0`D;Wj?-&JIPa3W zVsxGg9<*1LzgpxhZDGJ~(usBVi&7YWATQ}`kKGL}8L5nkXb(kf%I-MlN&hZ8d*SJ> zi1{qugDjBVJ9gke{d{21wgc3T?;tZ}ZpC177ulO2;}~wo)6+df4K0)lSBU*v$p!O< zVeLGXQ6@g zdU15p@M!D{TA7PO2xN8nn?!wOR2@CEb}3f;rFfBpdvVudMGM80AKZoG0iNYozu+;5ehkcPE<+HJ zY$b`2^we{8Xo|l>LdXrL(-xv^sji*nQHOrhj3HKG3ecfx9~>QWgtqzTeT<5f2M&vs zG^_Bzk77Ml4^dPubPtJV9T|ZIiN zKi5oKKfL+Y>F+=pCVV_OLb~{6v)0CO=QSQ42T8nW7<6Q|L#ib>J96xLIH&mI}H$^G?MQzg%FO7ELqd(y2995`Ryk9jR!` zK+B(zcCNIKlQ#0xB0T3PGjmD^3;aSmq5`x8KhqulR3!QKM9EQ1ixECFjPQ?AWn;ZP z&ALlobQZfLJwOPw@0Y4X6HBb{_4TRMkqc8Fel8X~Zb z^|L{a+`*%r8tq0p&|aG`mh#O?l`n^d{_8EvWn)HpF2xKg=C6S7cF;XN%1VqLo6x{J z54aF3lw{p>zaEtz6Wp-_jI!pxzLTpalzNaji`j_AubmJJ0`+|}PNTNsi!MZFA zbrodEm#M}zgd93fG%ILbSPkX=RvMr-RpJfSoXrr?`-H35cCQ(Pi1}wOCiDa_3qTMywq-QOl}{LAY=#`0iXuE2f?{N}JXigfJ{=Gx*l|?-~ItIsr{d2~R)E ztrx-?WxKj!yW)+aAWXH+S-akrjpeR@kL5F~jEVi_-fqMKa48-NZei$hMSlI68JuiO5>M)ud&e%<*CQg%h!rjzlG;5hCeig_>Ma?4UdX zt4MDNQLR4?zs>OHX~h9{N}n$+CH)tH|BW*RBgO`tTae^J+KmXk7JB6lHn}LA_-raz zud~NP%*_JX>vGeUy2h+E%(!UT3FFhyhQ>mS%>r2wEzN*uW1KpS!k=T_6XTJ`X@33U zs=Th-MUpNm{;O=^B_zLcyN8jmcJTQ>qGsP5eV`1b45x8W9jpfk4l^0cGHn0YhDb@E z;?|RMtMMDH9BOm?WHKg4@ZL2SIOtXu5-e1TVdJTZjrIQX!xi>A7)ty7$Ebqe40L() z0o(NWJ3B~dlU;`@MfXf(n~G$TS#YEk`&*_?3Bt1B!aAv;>dU`q5Cv)W5v(3&<1o)P z8Fop6rnXhY>t?2plwL>Ajb3KANb9RGtGVU5dm|jpO0L1v8RL*1&5tpRwvoJ z-gKq#Tw2ZNw!l&k(YB4~i8CbAChgZ~<1nVYQMkgy_hvNMU4g*6NRkw_cn^Dg!@REG zTil~LGmt1lY>5rTDIo@Q_bW7R8G8jMtoKsWe$lJ6GdU6vE-phOR8VOMN*<_5Zjl;o zHZ!JqugP2R1>$IBtps*QHwSRNQ7sa~j22xDBmJh2txSdh{eYUGZfi+ zFJFHrcp&e7COgiXGX4(xqbu!jEVp&Zd!1$-Xe0-1UbaAfvmlq*2)&L`Eo+!bvPdeG1IOfA^34=UcKe`5laz$V+*rbcVprm zk@p&r?U}z@GJiK_{;uoezGwM`s5~(s1^;R-nS+RN&2z4#1;am7HoH$(L z0^&0gX~&of^XiAMO1PB2nZB@lJX|UF8x~yvgHc6%(LL6zLeV{^-~Pv9|4EI^uBA=k zN?xuu7kK!>g zVnE}6qv8MQ1;APSM6#!hO=qgK7#;Wzm_i+M^4tz2*|ykN?ne>yTnQv?#h`K?9;1}qCf@qI7pUi)uL@CGj)X;=Z zhh0Vo(}ZG?g|tYw5r!NZ#R1>fSj~p)bR1cce_9tK>o8}OahRz#z)JdJ&e|D%aXoR| z3=S0^&;Qi_`x}k&#wfOgT5OfNjG;`}iLwAI)u$)5Aw%+6_y}C0o#4Iv!#-Dw0#--t zY<%-we5wJN5~Q`QA^m^U7ipAV#T?bgGBQGF8b=T4?uF14PB|<6z7_&oVQ1Hx4E5uM zUpSSz68O@}nf)D-TU7Fu!o15ALt@rx^`GWPdc759iqZ344S-RDv)qW%Evk8zZ)Iz9Uo@X0%rs9~>BOqc znNuwtx~+d#Iw7+PeTw~3do2>A;^xgWe##?SmW4j;Obu^*LJTA$NM3EuSa)LrL_~8W zp}2ef4kNBs_=0JM192j3JSzlIuifQxefrLsv)SFtUP-pxNAAs-Xc(ccvF9IRw`a(`cCBKGn(_ zTSwhECu;8;#sRaat!bOX);|7|?!C2y3p}~sGFUz6CjU@w4&#MYQLJ3<{-;z(o8QV^ zJLNxM;E^yuJ&HeyA5D^f(;)U)QMpS;%3E2A$%-zwott+on8bSVdcYjfphZHd&KNf6 zfQKVRo%!pTd&&>h*2@Ai7GZyviO1CE!v?si5=n-y9shtw&w_)eewEZlmNN9Fa97dpXhjRaKcjrp0E2Q73f2nHf|3ECB~c2^hE#3$80Fq>x@=? z8uUp46yj_tGym#Et}BaMNw0*%9TSGwY~k<7y2`MzI~G};DC53&GlWM+oI8l&Is13F z0f&Ah%v3<(f-Qc3f4hHu)E15eTIk#%PQPE=RNVDr^86+=nsE4)$KC`XLF45W#N!-k z)D};UKE@MVj7=eP=)%e!rmSM(c>kQkjCRy}35d!5Xa6v> zpTfSrpLJKUSNU?)StLJrr@UsS3?g-HLE@R*)dCN#l`D5<}ho46E(1z{iZ z0952Zlz&HEWue|(1ga(UMkM_;=@L}Zo!IX$Pm*VsA~W<) z$(lc5_&|(6e#eo$`mOB_1EmW&EZGmK9rpwa;3~o@)9!cXTO2d?Dd+8-@tHOu7gKo;qT5$F8Bq>Rzz7PSY{q?u;+j9M3CJVn%F z8liCIWQ>xg3Im4#Cu)Vo2_cDh@o5l|LGW>@lwnJ1J+ae(I%{3oXZLP5uaHHhGUopO z{&8)uEoXL`=!Od7D0!EVU7+$$;A_rgZni8{{V5oJ&trmnv_DWvOkGGUMlODa&*Hv=~5INy3$yjQL1%_7lUZ=785!ew@}=m%!V%RdpD_ zH%gvIph2mS5iFBs$wdi=z$T_3NH|r1?Y@cXQZK?hLl-iM(dkkn*xs;Qy#e8{`!C%= zDa9bQe=2*1J!p3&KFWSHv{}UYQ)g+KN661Sl~j+A*wygC$^YUXULFa0E`5OicIG=p zhmH=nbS*UQ(O&hDb(j2_1#;l zCVx>i1DBnuXF)7$^=-hs?xi`|DXTe8o`uBd3OIO|mg|^-Oax>M7$) z1b)GGC4N}yTdR8b=z>TZXL<@y8)1heX{cTD)wEBw-~;w6{@7LX@pD-B%(Z!0m=dSv^#k~Q&S1V zRsXU;L(Hv+x5j*bMTb^p15Ba_zy$c6=57q|)rJVzk_5&*4u8C+t=80lS#l8gdzrjS zdfPR9>#x6}NouO+cbM$qsM`xh@>0lhaxTU{0%K6}P++XhVkhdHq9b!{o(?_+qxeKW z;hYude9*Rb48q3iiOKu8n&GKJFepVJ@z>OH*y8Uo=kk`!a;&(CGmD7h=ar+TWsj|u zQF&XhGMggylry=9#3h&^VBlvRt6IphRnc@w!~J&*IAzZbg^g_YI)-1er`c_%!|5n6 z`94z!KV5oJP2vYc@4Cc2S~N;_!}u!2-<=FRY9m7EgLrV(Gf*%!((sGz=Z7jXVr<5wZW;#D7tLFXaki!u?Iax%ur3)0W9%AiznoG}tMPy~~W z-GwGCB^wV_j?AM;$vG$MfkjsE8JI*SYnQ%H*V?V^8u*X|lQ|%aGRsfFyHge`xx`gh z)hoj$At+NH$yr9j;2M+vmzAaBzWen@gGsae$3(}jZqD_;vp+ehXc=e+l{kXxUT31P z-24tOTdu-3m4Ivk*Gb$ctk(D4BwIeh;f1l9#CFGOD|?;>?VrRezGzZL?mWpTF1qo9h+cje_{s zvQwY`rKkH(sQ*Jgiu7Y*!O5_E|L5;JQQAY&f|X|QY-}?}T?Ns?or(gPjP-Vz~;|Rg%ktt(wd`%|TfMYoR=#c?q zv2RW0e!rs(-sq7wW3goo($|P2?25_M*h^&#m5Ktnns}0a8G3963x$dTo|^cHKAGds z3>JV)MFC}vLM5oC-Lg+c3zLCP+*%aRh>G)f5(lZ8lCOsJg&|c^@pxGBoQKvg`6ELK zADRN1F$?}#0^~tHbcRyyih^a6TFiT&)a20D%=Q_8VYR%HUDt%I*$#%u_XoJLSG z=Cw5xO${Y%f^l$g)cz@IC<+>yjtBj)qN)HYV(U{!=qW2oBLC)*^o-C;t!V1OA8{$l z!y7jJ0_6{2ody1&B{_w6L041bHnQx(Q{i5$FX|4LK^UdSXDru#1;zHecN2k&fQ8)XgDn`)KC2qZO}Y_} zGiVG?&S`DWweipMC(jUErHB%>aG7PG{xj?mzxc0OTE{6$5FIIU}|FGdymr!(=viF86+jv_*SLXKF_oe4`m<(+%&$DRJEcK z5(iyj*OL7%!pfw`nfzv5BfsiS@f*utgLA$5%YUa!q}@3Cp1W|e@aALtslf(CS#Y1V zdsqE5fA1x_eTu{CKakQ@`%6a(0$_MGsg_k-=w>|Rm>0g~QC%1=AMqn38r!1UIw5@i z!P*Od9{r^dX>E2Rv;VX`KFtvZkcGJg=bZL=uz!~u8#oYM9{V`;1IC9yN^e>08Ur@( zR*~wDsq!~HIL}06NsccPU9pk0`rP{z^dgnG1lL>B4da%)Xlk>Y*Uum7n*juW!f@@D z-04Up?;(L2T~dC=Zfk^n&-AnZaMeZR{w3^r_F;u~HMlR#nf}`SOrd0zrO}f05i19O9NF7I zVvx3I^BDK>U%a@Pq939wnVl(~9y_(al8aG?n5$yI5^E$geq8IMDk{FDuOgJ<0Vm8b zG9;g(X-L%86vx30o#6Y+8tw>2>Yzai2F;?<<1a_1>>OWY%O%Q0ASE}hr|uAGH_IP> zS+QvveJ}x;zUfHsNEeMXn{CR>?<>XKQW>kQf}C0smTFwUz)!r>ly4HIzvAL`QZ8fS zYv5(R_%sDy`^SfDgvAdiM-_08SaJ+eeEs3~71|<5e$21O$$H#1( zsZepl-#+19(K4y_0DUL}h7l5)H@#IZVusfuhJ=^n5e2-R}?ni8w?Z_RUhHo@H9Uo8+wI z48HBL{<+5g?RM2WDe$hjD}11?^S?o0eH24@<@LbfBlYVkKP84?(GP8#_}vY-0xPd- zvW*5{d-=D}Pr11Rr+retYkc-w!!A>aBoowfdsE7KaUTO>NSZ3sUuykY&BI*XsVP?S^ z`A-Fy8}S%6+jFlxgX!Xr`lwAWLY=u&$n1<@BIi2&Z?f&(3q|9(BtYjN&5<*?TYb}G zzHxrniiE%JXwQ{0U$V@B5{z^wOZGiA@TiO3TpSyGQRs?%Dtq1YAo>E|(%jJk@?j<- z%VXI(zxILqna9F8Vj-?+4)@>_0Nhvj8hhMN8#>hpA^^~H0R}p?gj)ls6Z>{WZYMn;cY}B-#TY!DJ23BgDC5VFowx1E0?65p)j!`# zcw^#2BbE?oqP_JtkpkhZ<`IWWxTRggJ@>=8C$stral_VBu)0AfF7b!O$UHym|3>>g zeB8ZZ`*GaeEUCrVLkGDva7l9iSOirMS%_TzH`RIh2;7n7;a1g=_I)Juk(v};{25FC zzUb=)TgfN$4(E|t;`^a13O}&a9{<7&4GR7x?YmN^GS?p&hPhi|pl5f2NY zxDW_VyH1DKmAljH$j|5e9ua@_cRz%ky7dbRdoTEQJBR+bvl;I2_}mHgFurK--EXhEqe)NdxgKcOETqbrTZowu zdfvUroB;Dw3qQRS-+(d}a$ZMgpKSGjYb&7mCc*MDPpEyx?!+3psy)Yg=c;9q> zEA)W8f@Wd>XTJCAcHg%cq;E` z&nG!K6C0TelkfdUuXj)}{p<Eb5-ZOpUgf1>IZi>fD_-l z0aT(%<$6K>K$ss|^nCPMJ#ljIe(vnc2~7?hy;?zUw3~?mP4NH`GH+avJARJZjlvkP z8{hi~RJ{v_naTIwqt`8H96YQ4e&yVm1Jwb`oBRQ^ z2|&IH^xhG^-Y?E?&U){c{I@%8JQE;a4F2aW<|%yuPx8K3zaR#{*uOkoZ;NRf-SKRy z?DTapR|TD%4NIQ5U9{isMIa8|B>j@D)4MN&qlhyHF7F<)xnMp4GCV0Aqpb2Z_ zYy|4LGK1SayCxP^j-RJTnKv|ZbcH=XfF>g5&W6PS43-AlRX zW1G*;`PY^JIev5pLk=BXoZ9);1Pu=EaIT`(V14zfS)1RSA3iKDdX*O!w;@9NLpZ2?v} zu4He4%t2oz2X5)8lQ_-p7eBXltmqc)4Dzt-Yb=m1)_Pb0^c-G2*G?h35FS zLee1TW(~q2!#1Fb$;ovMA&nfdoylL0IE~R1J}+-eusj}JkFVc?a@bx~8oF}$UO5{2 zavIedc&b6o%m1L-Zdz$)+*kdW-fVB@b}cCC`VL~o+)~2YbFC4lujV;do6j}8mWA)=nf`5AeeaVuZQ-khbBrlxYiP&bY19jE zNYUdP*zhXh{~xLBNy$V0`!^{RIB^m$paA|;;UIASfG*H=mM@91Jc%Z(5a%5U#<}z+ z@}b{;U4jn3mR?9I(%P>SeJ>VG*qo?a^`Y&4Q?qchoK@o%O*$@JM;4$ZPRBW1VwcdN zS(xM#^x$IH*ZHsa^2&YlFbu^$CUascnPv`L2D*a@#%T`3{tg=vLD&olj5XTx>n z5m5i9v;3wH=6(|&E~amnmab4PX`)C;g|;)MUc2-8>9k^KCXGg`&zP(1)KdP5Ec?#5 ztZW$}ah_+^i)jNqBm`5l(3|#BW~117=c^EDfdk4|sThvq#7qVMOn<6@T<@dpzm}{n z0{jDOOIEie(>sVU=aegL65hb$%GIS8ko55~7|$>R-3hYg`En>PK!>5QFu11*8a)|J zh)Ko^RV|di3KbH}9`?d`D35qJ-$E4II6JzE480KP6c)@*Glx922)By zSU3Nv3H}Qbo6gCFxY6iy+0LbC(4spkmKrYiLen7HZ`AZ zNy-)|3OfW-bc`51UGCeJq$EFtM0>jYEho_iInNwCf&g!GZ{*iq2*1q@4b6`6 z_EP>_+##b4PwW`p{vf3P-X3tO(YJ}YA$fsL+T26zff-L%b4!xkgUWjP`Z-!;r{$sy zYRqlI$9MPt!h`?b^=2p!?v_W_JezrpNceboj08JZJO0d08C9;z|#+<0H{cJ*Qos$3WL37caeeSCgqjRkx<-v7{Sx+p`f z!UnGY|LP9MD)kOmAk6~p&rGn$K#!ts}vCU{c2D$c31skGG|VI37zu0 zfcCekqGc86A<6s4V;CP^z@<$}%@Olaf)m7;G0T#1t5{ykUSh5pl*OnU3TeUkd*3J;t;FXok+{;gNJf?_B?2FTU zd|b@muvTdBGVV;_8x+db4RX4x-43e96waYPB76aJNXmpOY3~#U9?KdxPEM@!n zdeuHYt$BKx>y0a9&z)d=W-D1JiJ1{1Yy1$p5lFhJOBC~R*E={kw{@CtejE0$y^fS9 z|IKZFcy;J$vr9JU;BF8E+@k)ntsmQ>Mb%}tB>`2Pk5$KfK$T&d5@|ZPIP=@|`V23# z-IicJc^u&k3{F}2v}g-;>e<9Z?D@Jl=&Ws2Y*zCRCU|Kd-xf1fshdBzAtA^k)T*zn zNmn0KaC~P;9A1v6&Fn{Tvk4@&R6qbQGS;)&U$0#qG>F0Y5vxmRDF@wE3&Z=7Yb)N- zX3h=kI_K&u{bmh5&h4#DEnW{MZOzRsnl}IIhzArb297^L8>Mt@?|4F*Mp397?D~E> zhbZip3ch|)pY%QV8A#9Z$^m=5S$2$WdB3$ht?hJ}=$5*5O|8|b)4zIh0ek0=9U~Ww z9$sD@|Ae;cG5s*L=qG_d@Cq*6UsrRVhtoF)hr90aIZH1Xcm5WB;V49Writ^{^VjuP zV0WDlJ(`w_V3jYjsNc>ObBBq(dS@T*VR_-4Il8nQwf0MrgyyG6+L*Hyw9jjr^dYga|Xs+yZZL*jt~F9b1Q$)=YNyc_vKLh55G3y z7@QV&|GnnJ`E8&>STEw2%?k6Z#3IoZXJRv zLhIJ&cd<_%2=#iq0FLwWoMJ+A=c2Hd9XMe0wqc5`0jmd!vhZAdlQP;&Iq@5j~{LCmXUfaaIw<$sj0uxeYQ!$;KF z4|ZoEnU{4#x;55%7#kn+@onp$=Af&Z=i_pP93fqSq=>b*>GA;9`{S+Qd=K4aS>e0+ zEMvh?a$e8#QZ8Z0<$?O>&bAqt|GeuXr$(y$J)q zRNi2Qe$|@*NY1@1WSkj(Hm)|Ii~!*fQ+mXHZ0h*ozmknOBT)2Z^*M4)`#`b2)9YcE z=ilZ$FWR}(zn5K1;%D&|?u?In>pDc$v54rDgL5Q}Pz#eVf+V!C5eUCU?iibuO=-+$ zM*W0NWHIxZrZ|n9d&`zxe)F|p-K_6bM8Db_eMCJ4@bF-Ec6&&AD$fD-IP_gtnQ>ww zu6Gsh^E$h@J#%7!5dovR^{O}TG;t5*N#W4^mmIUvdsof>%C!+wLSwqR;dQjj?(Xf| z7n!@2;KVL#aeT#|ibC!@VfL=NoL`us}efBNTzTLe$cboHyZm>;1dQobiY9@%Fc?h&qpNBkgYt93=mOK?nww`T z*O^cVW{>TC`QE2neN1Qfb|U{y$RM1SJ_gwagwxWl0i5;#e&2jp_EpmdsXcdt?YzFg zf^V9P^oet~&o`pAS1(%Fc7MyhLF)VJm;|G`#w z;X`N$#XyNUnftrVJoN!%;bi>N{TlG?Z#9TSSfC<$XUXRQ)Cg}|3*UM3P}8-?1p5*U zME!MpXe(ZscJ2)Qd*%uEh-aAO%00ODi|5~V!e~VA+^6q1a8n7Qc2F5HAfmarBAfYF zFq7M0#_mgw_r(RNN!qtJ8Nqjy$0z2^Vs7;vyWGXm$D_RhjdyL@dm7;7!J?bHJi|oj z!^=JQCog`DOR`J5cR@Hk!S=5?C$)^2)rRkTsv)tSqJ39 zy}touk1B*4?}3dVj^17VJmUj~{*N2)y^Y0=3>yVF{<(=LKJUUbH`(~J!nQQR)5{<# z{8apvYOFQARic5x)ra?HVrA&K5{A%J?!5w*wzh2oysDc~H(o`w&5UCEaxj8z->~L_a9*fcgKav??I5BHQq^FR zz{d#6+aNK!IzCjx_Z`A|$+v6Qx@SIwtZ}SE;ln@r7h-R_h|X+wjXu2e&JToN4^Q&X ze+>bE$BjvYux)6R@5O@HxVRiTu`f**ldbK2+e7H)1r!2WxkiBFrJUGWVD$p>ox$Av z1=5S2dGX9&pPV)Vt!D%EkzeQRUh==z7aTCeuGEM;GW6(`%O4@n_`{q0hc^v`N5N&f zl^tInn6@vUC?u}!Vo)M$uGcOQ;9=p;uE>G+7`s>H^d;zqhI-LPmz@u@jlpUUbBwlB z#v^)`=!)bZq~v-7<+o2F)r$VU>t_S+JK_=Un+&x&=Z@#wS!5xRN>IgaRJ_41Z=M~4u3?pWYzcE;y&3StGrHna=0htZ|h4D;x}C~?BF68 ze_DMwTUC4OzHnPLU471Z-Na~B9dmh;Jv#-Vn<=oZ1LUO+Z7r=t-+YC?-K*0nJnsuf znfD)3qQieOU0G$7qw18%Fn#%68V1-+@2~k5F7|N)O@#g8lHf%b6{b7c_qfE%2%mz& z_s~{18%GLWbP4c=zW-4ji7OT-DGReBp>igbInQcRh?r;B-Z?T#rDrQjsvbwHh;tz9 zDXRQXZ39MS3`t{st!c+=a*m`@UxW9>6-Cc=lzktsAtV$6Lo=b!%*{|<29&oFia9o1 zM;UwYMwL}mRJf^`sTEoN_2v9QflnfMKp`+}O6Mzd^0e_5Zs4_Uc-1BT+}1q);fAQ@ zPyPX9bmLfa@cLsNheDn=$Sn}_X67gee(}G`QlkAPxSZe=i~O`^$tZa&_NQWjC(_6q z>^L8SrlPj>5|v3jhkHLYZ$JP^(riUoG?6Cm0E5BnH6NgbP2_vX2kn15+9~B`b$ET! z-&<0Jz4X#g#!Kb6p*E9zi)$vzsg*wTQgl%;+JTsGe8=>K**tKUHCeK_G8%L8eKXD< zv4e<@d_b73NxM-_Qa%q5#^$b9I+2(-Gg0Ftdr{WRY+zVAPy0xE$YsZ&+~wHdi2pr zPt|szdEU&yjxOW9VNYrcvV&M${rtMd4tJy2Y=-}z`%PKN|0#zn#QQyknjD7$d)2lHS#{XF>%8HBNY3UYcG`iW+i;Q^lVnD<=zYlud|P8sKui&%pS`3nfAxYPQ_+c*6L zRlH7&H_fe*bOfNCz8IcI(#^J>$BBGYNz3g~A3(+gipvc7}wpBB;wtR(Dylo{?Plw*8W)bxG;G z{bNFd#ek}$`t@5$yg7VF*w}mLST*Ha?*p9d6D6v~XY;M!s!-_Uop_zyY-(9YVU?N9 zQ|F$=&$K^cy$BW;u89j;`GsCB(XWD3ofjT}TNE>gV2UBoA?Y73dR6koamI;MhiwB7 zY=Pj|BQM34+lya2$jIeoNMw%1!K>A;Kcme`1DgdlV-jT`pL4r0aHf<92E8sy6_r`` zme(tPE+pp0&I;-w~9%=I+3TQ(pJWU*2ML?IV`5}raFrFumG zyZgh%uAFC2GP&2o`~3chBW-&+adJ$;!#bYU_)&A~!#czqhIFXHdfG0W&Iwmt6v+w2 zT7eA-Q5u{WHue)rY#=v@gGf%&PT>v+uyM1>?`)(BrVC|<1Rh)5iFA%JXRv`y%UqJ3 z$1(0a>byI_FN3^|d+Z0*<7t5hTA0A0c`%(K+TBLDQ{ER;3GzXlglz^(fh{mqr8dF6HEureim(~6_%1E7%JZY)riy- zY2((sK?pr_)I%F8 zi5Ew74`h3Yo+GeQPhx_5WCgEb+1GppuPugol-TLkKkJ=7w`paX475KZAM4hcSW~~3 zZZgRoKE$mV<~wkOM&yuDzty%U_DQ=UBNW!|>V2;O-osc@2y?#V_uc@$^vq&=jAW>^ z6E@kQZmEBGTF5T&-0uwbTKZ+TVIJjGgN3nrP8}yUMj4bPNazQ83&>HPJ|_0z-duYB zmO+U!e+%TXlqGQQh#nl{{VDw-s=Tz|Z%e@WW4+vAKc)^mp1%WPlb$12WS+;H^jN?Wd$=ZqZJ0^a4mHLF%6jaa$Xn6840@so_! zCjsi};QmaYyC_<+keqgwepS{)wfTp7`T?ZSpp3q@Z(K{SOcrKD7ZhK^$j1`}E7?%yJm8gsIHmj`0 z_&CV{?wiO=wA^dSn;%Xfx79r9E4i~VQiK+N)RaAX}sWk-&?D;8z6 zRxL_qoZ5i20z<<}9eqE@tiPs(ftgdwL5JBK!Z)s(jL?#h%skEeQ+QpaS zNTqwP5R(SyKu>+G+Dw%Lfb^!k@;*4vAQ6s{&GY;HqEcHo#+&%Ac2a47x@%BDF$W7+ zxmo-x46P2T&|?@2l-a@QT|5jCucc15clF&J@OmtJb-2Lz}gEb3+mw( zq!XR55=Q!wB**GWRZ|a}%G`{pU8n2)h|VVBGIX$YQ4$}O5$EF@cZ&xPJVFcI}k6l%MI9z4TOBf%v7HGl90EL;xv3E1|3E9FZ|C z%@dmfHag;7aFo%2 zG#%dPw#R=@jFBS}!HE^APXtoqN$2S;$r^F?*ZPSG2FCx$^)`t^lK#ty%0&S);g?)d4mW+tzT2`J(nWn#Vn2ys zL&1QjxRC@?v0wE^YF^78J4A3ZnLvt0(s66o-x*35G?KHiW&b>qG2%TD#wkubrF;L- zIQ;$4KPqW+T9aG$bvu>KN%eth&-O76IPRj93U~fPW~4-Y2@a2RoQN@%r#q3npf_$G z9DPnY!7!B;dJ4pjE)iC~$&paxk?(EC;rq}I%k@Eqc5|-4AzWDSM$GzD*4L$q<(R>& zr|NQRV?_H5E8E7JKsD`^j|l4ojH1m5q_eWaa{ue@^D;7f9gV>z_};qc`7b&j_|Jzl zjgFmwAQ?KFZ@yn#?5X*FM>v-&&el8|1V7o-96VE}Z=`6Mus`uzK2l3uEkSW6Pq$Kf zFg?qsYS#w|pJnX`HKCRvSTqfQgk|5_eR#1GW=Gi4$8^yUcz8p*>eQ`078i7S`t!;w zPa1>6{svm*OHdk^R4htPoN)k>R$G;Og4`lr3L|rNQrEa=8Z@2z*_0iSeR5;N=X^V3 z!_|1+TVlhVLUEPiA}z>p5Zeul-cY#KnY*?bQ?ZpC_66w-9^)jc+MlR-!ln!r`EP2G z7&cgY?#WVZwlKJ}W}U2T#y{#$&!@jW_TXJ!Br#mN847$2-qO)#K!*W}BGBm3!;2pV z^!yMa_rvVM%mA_MgJ)F797q8w!%e| z&kMKAeXmye_tK$cm@tgM3W*7p^I|b9oHH_0+qGXyQB=v3*tbKi0hG!!VfZD}YoxLF z0|WS_Y<|1wT^jG z4^GLSG?`;a5rj;Lvp|RldapcMXC$wd@z*DXsUr_##?)fUw1Dit) zjZ}8JVhdaj!)}m;Q@(cm%&#%`SA0r|RD}L1T^toS0^=J%=YRff=?%87TMn3eKN?6U zOi?k?U;P7VV1xC4@?6IfI?er#EIyt(2^=Y)vIzdj&MFo-D_@qkpcHl67c%Z3Y z>J*+v*nbm8A2X~#8-s++cBUxa+RYv+#7}2m*8H-wv9a!yYHZ}5&$xTR!ekRCw~GdDQMd*Eai12rCs3$DfM>&34kJE}e#j3+E@nZ#Nh3 zuQ|Xq+8IeE$HO!KTj}c5d1=Bvan5tGqgY#U4#&mgE-|E_d!s#mMd-}6g}E%DQ=(an z?#*EmWv5|98V>zW&R1c1YKs48obc^ejYJ|N8q}L-ClCDir1L&xWkCSKwkxr@r?6b! zom~a-yT`dJnk^n}ezQP24KkQiiN|;5cQ`-*62^`FyYHa8kI-y3%>Vd_RN?;tZ9tO0 zPGld$$E|yvPXdudfmWmTb7`&5T_Tlpfq!(oIPI`W*#E?EeBk<8{RMb)B0djrm?_Y( zNJU{{JR|7x8Ifh8IRAJ!AG5Pbn6YyAJsq$|I)i6fm<0-cCH?&@)Ur-ha3PwN;*1(8 zEr&$Ze-78Sw$@+VcCNOX>zj2uS>wExQ^!zADdoXPxv*rn>9!zPP^6fv4+wb~42uq3|X|K1C(Scg)&FeST1zOr# zZ{FBwA>N>fmqR+&T1|zwGw22!FMde8;Dj-vf0S}6w=OQ!_qLN0)RY7@dBHZ)vUy{B zN45}Z6Yx0TZ8LPptDt}$Clhes2p#PC8JqBEu@D(&)-1*R5sn}SF@`){U6o%A;Te%T zPc9}p5zazkGvk*8S#blcH*Pcb}}YD_s&jy z^vX^m2Zvq+)dE+mNyK)nskOQ)el*U!No-q3N#3g}Jg-%^EeOneg$F~q0&YjSlW9nZ zA;F7-8YuoDA>6eDc_iR&&mo~21nt;eUFE~9?NTImQAh%*Q;8EghHKhXl{RD0^d@pYqu|TeErUT8cc$;^UAAzIGOBMhXwff=1(|e8D;Zs9XAPKXBoYB; zi59=Zr-|A@FD?>2v@-@h4XD41pKh#2JRMWZB6GM;!zN~K<&;v+)2^=C-7%T>DC^h} zNgRiam7{&>ZT4td(L7J_HiqfzGn@(4NlxdW$^AjY>?WMIU+t%`*2_+y>5i(3IF1K2k`p< z5AJdi_87)NEX3ub(bfq(Hi>#_VMm?p`)c2~xLDa2@*hqBG~n4OI5a7EO3l2Q1E^;I z8Sx*O_zx~?+oBMsQ2244f5uFDa)xsac!02O!QnLWhU`9!)q;c_3VxCudK37QJBiIy ze1{dvqg|-q7Z;45nh<@Ie?WqEO7V2Sbtt1Z z)hfcMHttsY#Ey&BKMj$}t zjGZ<~NSh>~O(LQae^!Ph6g*<42O@0C@H;nS9yujd^ouY&H^FUpNK;9)5jp2(!Z{Tq z%~@nBq7TVrEFY4YT-SoUF{$8$FiHrlw3DVvm8?+tl^)0)dQ(8l1d=Q} zzTcSGFax?h5-XfZX(+>si()`k5x3cltsa-6i93pSJV_=2f1BJ+vMd?B#)A{R#_U9O zZDRCAizEvYbuR4%!7O|07 zC=pFaN?Yx8f5!-+C@!IEJa!_MjPQq6HlY5R9e2~UmPp3Fuj@%TquUXU>OKkam_O(H zxiw^y*uuEm!NQT5P>EcR=dv|!NH`)l!Ps8oc4gOa^B>h6bDBf!dznWDwAmh1tlNQ7 z=IQHp&}+}v=&X?mc9AiZel*;y5M7iv{tE>rKpostH7=9f&g((I&w- zqG83ve-FDU?|5GgKX$wjqWHXt?I_>TB4S*5Ngy|(#UiTHbrJjw*1I)^I4BuNrFu?# zEDvZV1Aj%i`e6?2@_;5}MiVlm3E88~_Fhi#o+S8)hTT0*@R2!Ronf6G@hO|nuYG#N z$tTD3sBs1kZP-KvQs;4+4)bIlu$sqnI{?~`e@S;t_IpGGYdLh<2Qrp*G6ve$qP5}L zzC6Z`L`*aA4Y;~$9~gnoAd%`2IXw4CSSo;#(F_=e%yH8=aLBMu_vLyN6f6c*MW-H+ zA>4@6c+JQV^#&fpQF#)DDGbWoX?v_RC^HdUi$Na`cAD7jkEqzddk*mnc)E{zPTMbI ze_9(Lsv{kKErk+7pkjUe#Sbl~=d>e6yBZ9adU}i;L0F$+y3cx!xHMK5>p4AM@6f0h zD2;O7Jz_b3yS}CQymB|`HhZT{e)&2z*Z39>*wQOZmE zJWq*_O8{BM0kpfVfDqh^o6wBUsz^0TUM^4i*6OOdYKED$8&OLWs<=9&%+pdu2?-rh zOUZtC@^6V5!wAfXk{dP)(R=e!~=N}>FYkA>2{QBUf{=~~LQ0Oispg$SZJInfZH1C%u$wQ8sr)&+7{ z&Nqlnxgt(4vFAnMVB(EWL%&cG^ix{VKI*CTvPIREXskcEw8nxmQ1(z||5SI3Sv1qT zc9g^kE@rJ{g+67mApJQ#f5LpXe`|p?Drsz9Z;{6J)@F}z`(-#QTW;6Ne^{jP;?{N2 z*nIIt4Q5@3S+_{5v3Y%+G`6=8$i$SRYBe^Rn>j8MnU3wn8;EQ{`V|VGE|;bDXnzi@ zddeC9ab$B10DN~6yPNjNIhV$}a1C(?ZjLTL&l0Ntww|iD&BW2M*YdpE>MG#|NH2Q$WA?7lIPfAH7A#2bJ|SJ$5tU^Ue56!RY)hH2*d_~ZU0a3}bA7gRPu zhoqU8g?{JB!0mTP`eXn+s@tEFY%3KXIov?|!L8fsC^LJ`3 zh##r+O+Eb`Po;0^>8Blf4w4-Ae3Xpcdoh7}Tt6Zs2nhC$M@ASgf9zhoAOL~D04KmJ zzyJfn0iI+m{xHOgfPS>VvIqzmlCk*1Vh!Uv-a@3EFUSakm^}-v7}snmxf~-TAV&DY z67Z#=!Os$!V#Nww1(F3B7dpAJD`m&Yep%Y=ym2~xW31|QPR|!4YQn!M{F}#Kn`97m zPTnE-Z8amYH)5$9e-m(hm%1T2_2M966E`8J*(ka_1uOrF8}T)g|4tW_5uYj~)6uQT zAa%or_#)n906Qln$xp%`YO>&!_|{~=Lm9d&D`BIQn!`anViN{Z@_aZfT`d=WD@EAu zO0$>;blx9BUwBbYubu*Npu@Ny&Um%_i-7l-FbkhBo~C}~e?7Gz0d2N}T+~pn2hQod z8+U_VkNRz3`Xl(S2@&sM-&|d_Q~o1$w5WhjA{ZZSZ1fR`3nLINC~HqL+Z@MkPYmR_ z9Fpcj5^6(u%EFDcbDB(}pcb{bi&Kmb@GV0K-1s3F^1f0vEw3=8W zjMrz4c#J;y3IAf~z8)DsA#OP?O9?O$7BGjLcj$Q#rsK$)yAQL*{wtjahqvCqAcqgL z#=|?WKP1Sqz|YR!@hcBxLNOWtU63boV?b#--wBXFf4&!p7V!N*(1m+~Y?Ig+6tQAs zkTlEf!NR}Rg0L#lX;FL3Usl6ETJ+xtF6}5;$SNcYEVeUX7Gf@s@lAnXE?i*|%p zuSTO02sMyM@g2~D^;x5E=)u{(q9SodLh=x!^rxu3n9~eIOmj7T((oNHr{5jZxN+(| z3F86qe@OP6%Mct_1I*^!@SYniC|KYq?0KgQUbZ9dhjK z>LrD?#ywtAV$>n~psm;|mvI*4JWc}U4i*H&e+#67$l%mIa>z0K-E+tx-Ctcj-rZ}c z;PlGuVRzi=?)SRKJ@-_8@bL0s-#(OMD^@fxMfM>Xm+Fre&T>58v6Px>tg`}SB)!Po}TFp!R> ze;sh1q->Ji>St`SYy7h3j%#O)Y0Slz_S_*rq_Y_BA_D5W(dw#RW6aJ_GRYBh7?^F^ zEObEjtkps70lhh+H_tPvDH_yUq)BGHvV~)(&H}qH8Qac+jOUd*vbnM^E{ud|UPw4; z0>+$d7Q;rxpX2RjeCQYCq*EA7X`MEne}fVNRM}E066PBS803f%{&< zlWC<9Ue|)`fJMMqu!+@^>gB9);-##^w=N;c)Z(YV?b)$}48c-{#U)u%*tb_Qe|Tk{l77w>W&_ZjuaA?}jr3FURB20@`ix(VO+h03X56m=~uB($Y6>ne>% zf)ci?G?M+D6!qp{K6f|CYW(=GCH>ko9>&R8EY#;<-Nk9=23#7q`>M~ya6(ka7YB5w z)guh$bUaB2tyQmvr8{M;^gp@sf0cjmm50#a1^@zn+={|M%*CM+7@(D6My_uz`%{GO z4Rk{JcBsiKTdS)Klw0zE$~=BJfl3-uy#ZnP30x5lLP*(!>ooCjaY)h*4FPD_ZS}N8 z2`C+Y@`t$iOQ(JptdyEwuTm~k#Hipjvnf0UDTFD|*V zE1)Yat=hE~5bxIw^5y2cy1G)zQ`=<#b^#TQWdCS79Bp(RXRvhPNML2WFwa-YXbu-7DS}2dHFFlc@wL{*F?5uIN(MTqPYpoYwym1Y# z$NZmrhgROkd?Sdt1&&#-bN}RR=4XxZB*_v0FWBQW;5KO&3X#|WmcZzMxY(23xaYL> zZqwJ1IAK9N%T1v)e=u$53Td+)?ij-%_pT84E+elS_B6KT)LUIGWY=3JK)y#Mw1}yZ zB_ODZV zyvcAnR%Y0A_A58$(ITF*LtvU=5^%WhL$S*O;{HLdUJ;Qa_X7>CeU&#Ib{H?}i+ zGgBk&G~Q0a*tUQx6?>g$?samXW3$ZMrywEAHwP(0H+Ki9g#)NEjy?xOp~7$|;;LH% zSR}P)ol0s^e?o8Wzru#!NJY4eWH)J}xwW;mwYr+^;OF*kp`7*c7r$b;0NL8!-bTRI z_V)D(xDOC+eSLi$;nvsJH!9%1Ea0wRzb@deU%ycWC#zmV(l{`FZ&i``@o)UG00#mE zG*DedYYq%l3Sc0;4E!5E(15RBzafFIU%y^SH3v4+fBAJqO`3eG6#34xOrspS{b?LY z`4wA<22Eat&k~EgCaUK@9fxEUGufnX*V-AuY`tmK&5XLWb4}#9S9Y-e8cs&*O5Loe ze#N?W^UAV?Eo(A=x>n6XyRSn=&!M5vd+upSWKX)M9_c`}{Kr{`#O!QebfeNOjphq- z7+&&#e<)TDh+^{rQQ`I1ih44zQeh3BIjgJOo>O~)u)3=M5~lK-WSd*cXYd#Z^Cesv z!g1s~rr3PP1jpGSYvgmvS&+9+t7XKxOc}(wC$1Yr=<0m1e`t_8eYYCXZiV{aU5c9{)b!zoay)O$ zGgB*jL??7P8VOwjycDk`Zm_X>m#`9ET&PD8!jMCCH(ZwW2R3c}1w}OxVet`5N!K7B zO<)gNZmGelh7$2c=84KQZ7vKbFz--s<sb79Th+X{900 zgvY0LL{gGCyC$5p@ZjfKE+IPZ!*vp-Sn*YaoC%}Qf)@n^4lvg~j(S_TfdfB9U6OL-Y85~n4AG})*EVp4$6o_&-X6c-y5 zs|FF#AoUIe(7ZmW5=EBvGzkqrZ`9NcJi^u0#HbDu%&xMN!r?EPORy!Rm_0GFxxjK^ zO1Ol0E7QeWLTSY*^rdDiD3X?%3A~{}POf;#mzGk3!g#2z#e zTo?%_7QV(c*Vs*1Vz$waiwh(~zH$zX1u-s;|JZDVJ-&?oahd-75nO6+JL&RmXHL_i zDgC^-Lg#9Q{*S-6_`_mt2^oD8f0{pI%aIa3v+09k6$4&{e(A;(Dl*j5Nv94()Q#kp zllQ=4AtaOmNJRrF`2$5?vW&5P`_z72O8J}x3F0zim#aRiIYNB*SY`sI8tjuS9NgHRsbGA9le;DsHQ#eNcShjo@W2JIp*eQmauv6*=69+c}z*~p8t z&`&MWO$o^4#}&kKP68eW!>aceNA!GBr~ z$#lybJA~I>j;1rcv+O0^tOM#VuJI2W$ut{Jm7|DJ&Wt}C@KOo9e|XI4=iU$t1-Sx4 zzE9kPVhOybkSn9=yPnT8G0yEU7BJ)X*w|NL};6rCYIvolYv+UH%T<~NI z-`vHm%iWH`PwqO`1wXpe{F58J%3u>N#-HHf(29w(Vgi)R<43p{GJ+Q^rOvA1xUYHF z`uJ-r#<~3r$6QrL0ILEwQWfPY9r}JFh8866Qhn^SK zSXY<~BKbYif4Fhu2H9F)C)VtqH{+)4y~2v_qH#`D31R%+D=pe*s;00>JWD;ZOX_Gf zHn(3aRZ<$qVzG@zceN&XQ}A2?WRfE@zaX68e8kz<#LC5 zldzcCFs+oLv$$c2hKZj`$H?5!p1%;^OUzszAK zVX0Jce^rv|Ljw9?axh6o5?cIHe^c>BugouX!QgP>og`0K^-ShjC*DM& zd9m5*Rq$6Dp7b=C=ySC&opKlrYd$#(NF|`6Fh|O>N|*~s7vy2*`5tWu(T3YvkfV}o z11TN}ZE2mo$+k#9vqt2duttI?pop-W0VjJuUS@)1WK%zkvV zvk93Sn`A~UPQa$+(eecBHLakyk_-Jue%2h6}nBb zzVRa2Xg0~lc9U$bH+jYoG8*ffq_Khjf4@x{o9jFasNQp2M7-Su~awOlE&K$C@-j^P!_oY^=+59z2 zdtf>&y(RVeJJLb*8`3z5nYECd{do$nH08w|A8~S@2K)T+#VCxqa~-!SD;zFzf5E^p z<^Ce}I^Cd0x(CE#Bw{^!NFLFMU2CxoaEovtW-~crWXKhdKFdY4Z_vZ;Mz4LuXxBOo z1GbiVA@CYQ@M4g=H04Fs9R%~DKHnGCxCuJZ63}oROXsYH&Nj3Sj2noNpV1N<{_t7M zL#~OR%Mpjck~)fS$iA?bOeW0Fe`;JV^2*2?uoNDy){;qWIu5*y)qHTWdCDfNmL)ae z2=)J{Rs7!<>OQMmH6{4Aa(liGqs?}Q?f9(SVNyHu7L$|epxXie5ob)zthZP`wKkie z`O)Uqqa|}IVwIC?$N+uas_{(-UMW9hv^cqrI8cvt&zO1#g)>5*k`GxvfB8HnNqV$s zKP)@dc;t{{6uUiS%4Li(u~8zWZmm6XY|L)G0&Z%8rIQ>Yk^t#~hIZMvRjM3QWg?ItmQRTVjezv}=bf6A#%guYb43R?5E8gR9b?sALg(7@65fidm&+SfkcJ(=}< zkibVTD0)DcjE!3b z<9i=`GJmVXUyTgklJ4 z`o!d9IXRpJ$zy?)e?)FS;DPV0i;K_PDH?4dh~w-sw0(fa+&wkD*uc_kGcXWPZy={* z(3kGoA%)T491^iSAI=Vth|v&}Fo(t@p`W{BGjx07$mGy*1%isQ^0g3)!T4J|v)r3_ zqcIEcwIzQb5jMh>IjzA z(e~-dz#co?bLIkR`C2cds(>bo1w}gOgmz` za%S|iUPH`&JA~IrGr8cniwOH%cNpdtoRJab{i`cn&T%JR1palS1t}-ai0Kc@?ON%R z!5VmP0f~d|e-+F1Z#H-3-r#$YZ^H&rTdQ&R9_v?qrw*}SbN^PZNM5*Tv%Bam!1+<` z_KobdjUEMuDZBO%8m#$3}I-84jD>cnN6}shr+iwJjfi>=}f0`A}7M!wcaBTry&#))_Tba{C z5C7IaT3xkg1=msnO@A6imO}reo8Za4&g46T3kwb)f9YC6kt;k*{0GqnKlJN7{*uM?%l(^g zn#viKq)T1ieDnTtpo(NEg%Uu9JS#7lOH4E(a=~}O6-P_C;&9C=I&E?zIG9{3_d3Y` zu?C#{VBH8|Ilu@uPr=O~7Zg3(Gq0!5u6bQS;y9Wfvhu%Z>fBKcQh8@P)MmZvNt9F)5hG{K{qd9+;#bz0g zZ2{Yhh$rBXs-X@q9p05zKE5l>UuJ+`MFDL-%BI>9ht^bo=4lm+G?xQwV1;F>#D34T z5YKT`+?dFo;7)cRC{At)Th%7skFma1LhPxe&{KtjNk1+^oh9Tc31#}xFWR!(e`u|{ z;H&$xQyv@!6W!6RtDKBqcH)PZvWZpfKvKmG#DWE^GL|yg0V$&IMm_#I!<y%a$53 zH#QJpp{=6XEu8xK0-BkNXI5Asf4Qz2e;yaIE7nv&n}%tIHY!glf+=l^q(oQamQuy4 zEvl))aU+F?cCdN%SwnY$Xya~|1arbKyxdhaGgLcGvhLtsH?wvvFuwEytDTq^s;F+Q zp-g=2sfgQc*5Fk0a06wEPEeRoO%aq+B&C;DChcX5J9Pj~&&|ufHw#m||JNH^S5>I1 z@nw)}8su7|)oQhE;U48?e`9?MI63V&C`~vXVrz{i(ltGoWoWXI!KFk>ah^_XozpPC z^rR_z?Z5PVNeFj;;*P6M4PRsiRhg;C9Egi1S48~BQ)ZlNbN$|0{oU#nqyhPH>X5>@ zh)(BbYf=vl;?~lcsIS551F>DSXd8tw9Mu+)i%kR|#c)Qj(m}#;!p4s^T!q&S!CFj!8rkYFgy!!+$yvo$7nwt zL9zrv0tkpf5_k&QwPVf4Lj$=XYBd|{AlM>C=ETe_G!T6sF2i>Tvexx7GVW)Q$5hJG z(AnbiS5U@|!^gt}RkppkwY6QsC*Gjfo3KH*@JU;|$XP|if3LHOGM<=}u%KT|A#Gui z`WB8d@rU$(iNAzHN%0HmuHaTQXQ@qtQE{I4iX~g2->xvtwMJGw6kNlWgS^YDF?f15 z=cHTK86-~OPBxzxt{qS2Vv*lzYK%Xt@@dL+tCn4+{F$3v9$CQ z?iy{gNRlv4@v*_#G*rl3ksFT>k03&A}E6LmW^h zKg~RUD5oLpyD7`^OFjH}u7*|2tyT{rFT(DWw!;<9K&eW1Q+lUqCytv^JcR|9C&=Z# zSfNJ}!7#)8PElNR=FQWWIT@YlxHCP0)Uh{VarR0oe^XinwSoL=ItJ^46a?^`D{+g& zXmc673JV&9LJsMOkPa?j@ByV83JHPitzFRdxe-aHC*{0Iyn=q#p-+({u!MQ)a0kz} zxbnF$9-ZhD0e(J%pze7lHLu~>W^}XXEWhy-&TixN4ZUAV&jgJM^ZUEYdZv zn2; z?RT`hcrB#&J6+b30bD{6z@^lWu9mF5dr(hPz*p9RG@^kQc0BM|7q&Baf{BG?*Ggo0 z=emM#FCfw?v=mNoL@0L~-cX@fHTqbdNU%{YAYMcX@$GP6acLe89kirov^f*);E-yg ze^Y0xQ7?{n0#}d7<}kZvTHN3T^IU3?$jpe3H7tFh~$AClz*= zn#mgVmk^A-HAYHhs2yYXDxj~Km>2oI;jok2s&sqYXpS`o(u_%3v-4Q#ha z9eMsFNnxV^Qv8V!M!oswghd`~1eN96e{JVnI9L{1j5{-aq(PHbT8P7Pw9+gNN_86; zrIxl28VB1^E=9N{*QyLQig+O|cAQQ}Ondh)vRMpKZnJKdJN(UGQa6=fV+?Fm^^{v# z@jz8zya>I5e*rgPjhM}{hslt|P9E$OEJYLA4SOnlrXDpj_duP&s_^O2z;;Bme*o#N zt{tE|!j)SCJB4S?j*o05?`58X))b+1kWPX+Z3zeHj|2!t7e)K7Ta$@5x7X37Cn(6D zVRiKuWZrE(E9i=C(Tj%F2;+dw?t{3H$ad9Xss~dt~f5f9Es0(joyJiZKlPHt(`w&vDG5ycee6W4h+mtooSS zlgileSyJntu^)_IyRZpkMjxYm~5x*Pp4ij;DYOT!5N<66E2=C=cH>^9c7K!cTu z@7%6i6a%aklQSVaM;L|z?-&H1Nw{`*lAPW3#;MDr8}x{uM5((Ee>&( zfqv54te;HSF^I$SmWXE5e_Ns-93kk>o+8%QMdvcevJmNq)dF<1kBL!$o-?4P+|v)%INTa2t${MIYm&yd5IGc# zofOPS;)J}yuPh%{#UyWdOXW!$}JyhMA~f8-}4G(kC#=bqB5 zHF;|-ELX<&BYtfC@x`xUV#{C?fyFPe=zn3z41 zb26u`NPVuUx@>;R8uVG(mtG7npO}?1PnI>OUR<7f;AC3J)%knwib-^-pBOQVwEAGl z>wT#BG)*Q0XTm4Af6dD9Oj>UeZbr++7x}>qI<^q6g%PzZ5t=2b2y*c_vlEDOk()-g zqdN7`MJt-k^2%O`GYGQ6gAj-5k#}SV#e-Eet9I(8wK$2_;^~M@LcivT&`t1y0^MZo zG%qfKo3zzxZMIsiC1}pV?6ek&pt<~!$T$fl7U^Vkhi-3ee{F2-jw#wlH$XqTCmdJW zf#cf28VA`X)gX3o^Jc5*)VH?QU)&~ku-a;_Z#ea>?TvMCo?aY6eR^`Xd%TcW;**kX zVSY7lKx6QA0mByu4)_r%SV5>696JkgRz&yurRYB4<)4dk0RgbEg0N7RCyLB`>T-Qy^01?eU0q zT@(=@YC*zadju^B_)M#b>V!RBVIU@^J%*?bq_^C_YIPO1Z(?yBuG4tpMN@X^IjQTE zjmX5gfBI0=!L?52VX0zRa31F!1kuA6Kg5985aO5vfU?zDcBn4n%nF{IO@a!XR2VZ1 z%RwrLD?%gB$elo!DHMBn|CFAC;c}Fma2GS`6PR<5{Rkl@rt$<*OsIqiSRc{B2az8} z3X>@_nKCP-L@YD$3tb4#_hK%lS_?9po60I7e?+V`wcT#7*UlS8iQ`rj|@I5W4Xfz*pd6 zR3|>RN5*+N=dbr>kkuZO*}3z`{{wYVr)d$$vd; zq)R-E4q|Q?DS}?eZ|Mh+MD+2gom^aC8UCg_D!=J|H5PsBG z@SR6^E8%Y?)Js?HWP(T!5^FRf(uo9(7UZ2O#}g!#!Z6v_d`Rt#SogdzUOQ%kX#}qo zBa(oA?zL0f=YC;R-7Wo{xgEd;;E-TJZg(n%x$xvgRRZ12r%GNEz%NK=et#*ow|QVu zjVkkk$NNZxHWpXJI(WkHdr(076RvKB$mMOh->)9*Hebv@d`l6&n^3HG4eD!IuJL#TaBydbYEp`9LhT~7L@BE8owM7-(kJ_}HHv*${< z*5z;{lKyBc(ax9e*cst3nSUqAG!E<*+qfD#{5^tnR1+ybHLo$){c>L%usfjkPCUr3 zBSgV<4d6_;`$~NG$wvII;x`*&!h{Odk-1e zb?N#d0Rdg=0OaR02mnFHbWSr8P!(8aSe*auOjy(jyPI7oVtxFzuUoFjdDYDM;zuec zx6Rpws*Ff^8eu|>oX7)Ybn{P(_C`l#~_s75e z6$_f^KNFQ$w207WVU**0-)Q?>;H}2TOBT>g3iN%L)A!T{SAWtsG3ooj=*GFgTaB+w zpRJ8F*8oave;@|syMo5=Eo(EDA~Sm9`j$ZIXd`VP{k}l@1A+91%LuJN`sSPGg;~iY z+CR5Q0DpfZ(Ef#}<}WMID{PCxbX3^>N?`kIf$eXe1zUt_KRGHt_*;#(XQydW!17-; znx2D}Q8ERS<$qJAFn#sMMS3pBWJ`i7Q2m5oW(~3~ZJe--c_X!bG6W}JM|4lVVcSF2 zknBVD7_*P*zRcbi*@uul$Ls@oD6{EYQF{hn5+M}rK{#W9z2Vd?-~>(>wI8zTyjOGAeQ~B( zB0rP-(0>Ul@;x)o^E_k=hdij_VqRsy2e%(cb_HuD4-9_A&s2f{J7MH4hPRM18}ne1 zONAuV{l1twjw<*Vtn{cpPwt8iuCHG`J#9<4SNist+$D*lg{2rVY9s8L$DC{H6(vRt ziWs$N!~_+@R1{wz4_dxN$I|)ZVrR&(tj72XxUS6M9^XRL(I3>g4|YY=2LDr zqiMw52Mw|wxouu<7G%FqZ#2napMr1W2mR%)2kGNGLPAe`IcqbL!eGhC=U?JBGA{7m zn@#w?npF=kXS9{y$f8pOIa>`MpfT$Kw1kzfLjf~-g%yAqzN5*e&Sv_^2m-3n(LN8w4@avnh zjz)898(_q)HmEq!Po1l96W^QDu4UtmOnT)}2rJ5y3qPT0K_*lYOY{D5? zB3DR{JjNTOxrI?p4qTiZ~yAml}v52_4UgWn`l0R^%zSCCSU!&Z$jq-8>OdsJ) z8!?$MVlq;MvtxvFQa$nzhGoYh5Py0cih)~IQAB=T6YjeFVH$ZU#_f!djsa}CpGmpn z>Bc%~HExjg#wOY5k(aAIyn+@9mf4QROJ_O5`6FvLqd6IZ54@QJ4>aJvurNGuPOC%o z`{o`ziMqJZlOFP?>0$%w%sFM?#E27DbRGV+p{qB4ruz<-NsN-Kg< z-Y~!$dAmcUYXRKvgz)BJ*68dVzw+QA{;OOJR0A!|hT~3U;wFZIlU8NwdS&WHW$Nal zM?U)>NIieLnJX$h-v}nR(vj`HP z7unRLRYk?dtssytxNH4nf`4-v9ntC$3%R@Q22AT)^9n(Cu7y^PP}j-Ensf*a^n!>p zYNtD)xN5AfhTXI$-T4ZtbvxBJQadA$IyQhg1g-jk8&Lj8e7sU|`&5Za>s;|NtIV)+ zU{09R4zAL1fdc|{b#OD5`v5>M)OVOaCt8qu%cP0F${i1%BezSN+<&f5-|o1j?Qqwp zM|0PwpY4dKi??tS^x-NbZEA~***lboe3<_F-+fFXOL=$vsc_^mRXz5`UUz$*7Zq-J275 z4#3@PVanakg4`Q$bF-6h5LxcI@?U_C7UW)pZvmG9taE^qaIh3&L!wv;u_`+)wtVfFIo z;GL#!0IPh*$7b-}*E&R)5~?u8^0?kDn#L)P6Ao!C1&1`+%JFkMcLjO{Y&G5G801;c zRCHnj(Iu3Dihm(@$!%2l7N)kUd`rr=9<{}&oh08d@_j)qf3?o@1)Z;4xkTk!REtin zt}5&N=knLuK^G9YVQW?UE8OAoWqfbY%{`{}@JJM^y^w{eXi4Sb&{ewyWepAk=WWB< zt#&WrxR1R?em2>n43W0Z!cjHRN)!zDbF&{oB zDkNg{Xn!km*)!5=Yp|mCSvjZ6#?z(8C35!yijsaUUIF%@o`0QkKDsXm~1y z1>iY^B)qWfN)4@8dbo)4g^RD=@#RWfu(MP6OuNJWC+@MlTFuLIyIv0)s;Yf{KSBGG zu);rr(_ULEf|C%8HJQ)*T@VbAc)fZIwyh;o>VINiE$d42ZHagx4Y^_?A{@$%#A@fi z_@gsdM9o>^z?e^ZWr|$%jzhYV_mm5tyEX@_d+=Ngj*$3c#aMr=7{etkOz(w35V75e zt~4QhNY30tw(z1Eg1FnvL0ww{p8u}Wtayz`%4yE5eDgDXCm-vCkNmurXlz$H*o?}W+a!F7J7(Nx#p&ed&ENSBza&Hi# zwR&yIWwl}7S}V?b53n!i0Ioyb@tn*@XFNmy~byk;TaH9e(5c-jVGYk ztj!&+bVy*O{P{1~-;XEEX91iJVuQ|Kvli%HdL&%NS0Sx9OEeD4iN$)5>ma7pynj_o zTjV{-Vw_{W5=JaTH(Ru0wTfw|=KbZq%{98jodwt@m3Cq69)>o*(x4-Mx6GlDW%K<|Wadlvg57!}t_DN6FY!$FY5m zPf*;(Hkrm@=33*-0-wTUY~*fiiEK8*?74Eyp(}mc2~Does}8mSl@9;wv9OH}>f^S2 z%L$$?!gz2fq6`hLt=9t_wpmdLIk=iYB&ApHK-kW`+(JWGxnc{+(78luj(@fwOBCm` zXH}d>^m3i~UXS|}FX=c3+|)Mj1hlibhQapaGbCr%zqlxQj^H5Z`WyWyg4hEtF@+FM z@gPE=V$Ml2%e|Bz(I(laO>#_|?Q18=EU()>&bgW5yn;}Mpo}G1Niz?2PF3J38`p9E z@pA%VA?`NaV~DzXWHjWmdVi}X{}LKDuO5?q-H4#EzO@$1z*??-ME%_c1g(JjNd%6o zwXlFT2#Al!K8zkle>_QMEV$@P;0u&g$qmwK=uV1K#<=0XK^hr*@9q%%P;k$ASpp2#Y{D`VJ|VTJd))W1fipB=R7jdG`?ZE4Knr<}JwU#WS>7amve6 zQ+iy>7d^OM+f{CjTz_jMR-KBBf`@Og!kII30Roc_(4&#>N3n~VNW$i zVfG2;er@?gUMN==rIVt{T~7{Vl@FJK_a3#V3*Kv>Uq7My9)8K_Jx7=s5?$`cC#~?~ zqv)=TMq@VFjZk)#7A}8yZxkbsy?$z3cPsj)zr1rco~d(wg`LE+TT8NinswKInLY$Z z-fF(f9`!$k9b_039ecqxV(vWL=kP-6M1eBb^81g z-Wn^z8(h*L;mIob<;R8}L!V@9TA$w7a2g6@!(*t->A4p7d*FY89z}Y;Zw6sQd|h0WUeLEHyi=*l7Q$n%65ZD8fpa|Xf`GP1{WiFl ziJIN4MDR zoFtPH8d5~zv_*gUQ|??zIh8mKycCn#y4z(Ubc7|SXQ?Z9AOwwh%Z#s4h>3~8)^%51 z1u^0|Xs}uQ1eM;4n(xK66IPq1+?6hd!PGr(fm$sehR>Y}wH?bjXBJox6ltyHsN-7Y zT@_jRfE^|Jo`0GS!!i1cy%h(yKriox()(zjmFu$qY}Uy+w5pM+5@x$60yf|;?5Q{!#iP>?oVK7I|b;tbTv7=aaY^iH1?(EH1)6nqR=#YKk zjGXQI4BZ}bPmT`ZU0_UUf_nfa?K$MYb}oN)@V|=g8lUdvUmXL_abvozDE7Toa~1p!XVIXbyir6)%4g zcFgG3)d8Bvm-+_mn@WjA`|+WAUZ);A;v)m6sArkc#`^W^H?G>lYemmD$oTEiCH3v? zTf3UNXG2%^#E#oqzd9gCbmQuP?9;5FPnoNGGLDcgK87nY$ti~cD*h< zDGeC}&yH9(>4B=)XFEfN@>twa0&3z-zk>s^wq9TEJ&Bw#oNVdEMYjiUxPcR5 zB+lS9Ha_{z8n83KB7nb*G#P)vbfGuz#tt`8!Lg0<@jPJ^=z)jXC!Er?w-|Xqu{>nl zm3#aHO6f{-_ijPi${uqM;pl)ic3ckDDuT(}QZk#^58(Ay(}C{4D?0B)^dh;Nzt>j5 z&9i0Cw(klRz^hD5X)r~KGd{sU^_T6O>C=qV>v8Uj9bROgF?f}JadCgXAh+pCvpv#I zU0mSF#A0uc($5MsgC#=fd(m3nLNQqFepXVn9ymm$R%y-V z)%mE!8)SJVdK#t{sE2=FWA@to+PbiY%jm({TdWnK|SLm?GF=KGEDc%FGEO)cGnJRB3hJ7OVU9jNX5Soe;K&*M-gc3o*Sd zSA6?*;av@qaOJ%&eXl{1SbDRiOYiketi2cFg|k@9vQMb>n`Ll1l38(<`bmUJ#WkxA zJ<7i?E-c@2dZf$!(aMgQWe%~E9e=I$90PRbc9ukmYT z{0Zyz`3#jpu17_8>~N>=k?tdKIIRpIl|589Adljnvmk$Obm+P7TIl+)W|0W(d21G# zxE7j^TMP1}$d?x)l9i3YvH%0xS)**SHi)L&DmS_ma1^bnGu#nIyC(CO>XE#B$^WH2 z5&#)v+H8+^dL$r~l%Ic~;t5YjW#x;u^)0RW_x_esGyrm`3p-X?wB8O|Vm79Mb(Bt;O&SCqYUU&Ao z2RQ;X_1LwCb^1o9%d>b=kL<%fTc=hGG0!ZpOoB)aMq#~n0nYQQ*_fK*V`|tJ<&NzTMf_0_~6GZ41H8m`*ka+{!e&mr@r)($g?j^A>%y$*IK{r665Nlh}A zSyeryAY~4jFUTwN!WoxIeNTnXW_9(gCb;X?&cF(AFB4`_az&^Jgi#L8=khoTGl+kx zb(F|h1`~451$ChoR8VSbqFveORaK zj<7X+oY}`QRRl0zx8#)zrZYwOU7 zAlO=_NdSE`i2J?T1VU|Fb=zW6`fSzxxqB z`kuI8QfmT7TeH2#!*#ZHkEj({?jA^`hO4XgKAhxXcTdG|H9~71$-tWXDy(L6u~1&Z z%iHi!tP#0FJXj5j_Q11EF}E}}5I?N|Kc2;00^dewLb4d|VlZmABGw(+D#Dw(kvGn# zi1H}`QE<^!h|~jFkw3}9&<1}H8A9oGtNJJ43nM_L!i38?5u!d_>%z)QxX)C=wH5_` zLa)ZdI5~@zCI~3u(GY7^JpjKHJSsQebdpM)z)Dl!JY?`c6>=I>)N=2N2+04{*5sJB zC3CM}L-E90ILiln! zs&i$i0k9e%7KkBY3ZZR7T4^S*(n1ALw`yL-3*54E4gNtIC*Cy-*GN z7*)V9%}5N*DY|6l(#wBDv-Lc#A;M=EFbW0bWHmZO@LC?E#f&YjlJ>3lx!Z#rqOmi1L|1fxeBxF`nB)^2C|I^c=#kG| zMVd6aOf7sJ(xe7&8JBq zU6(iH4l6S=w>K{8(d6fq&Z*`r`D@H1mnP(@rQ#8Bpz@qrohXD56VyGs3+GVM;9>X&!7gkF66l~9M ze4lodV`Yg~dQp%#aehi28YKr!c6{_^93RVeh>3Ehy#0QK^MZhszP0B%6Ti(~^`0IJ zRoKHXESz1bg^4C)pCPHnu|N!xd7TX1JtTkbOEgFJqdkC|0bxBe+qjfJYb({P zKrPWfbY`?u@fWX)^2A0^Lx=J5At$@ekvkus3I4G*GuuaJ`}Ivbe(*~8&?p*uqXcBV za#cGYRpPO;sa6fPI)U(9?O12~=nKgPin(A?0ckf(g-dJU$T6;p(x;>r)JRIG13L0k zRsVm0&t5a2Qg4Cw!Imhe`UIT+7_Cq-zJmgQ!t~Cz6=nw%x`o$VMMFH=m_hmHI*=Ru zQ0AKdfFBQR6lm7#Ucpe$t#l_$260MiEE8J=C+zl^$#yu6SD?vxW zk=4NjTy~x%mSDniIjJ^LwrieOc`?FgnOvF8BF5sF6Ij07M8g&2sXiLkKLvA3il~1w zVyUABLQKW>MyUqUsk|5XbGjFJX7Ekpi&)fdYK3UFpdj?$0B?bg#Vmg=*L*n$bjm8U3K*FjMw{|yhfq0=#Ob~U60ntr z@?1tL4dF9|*eR`rR#^AlyDcf7Tld{HW(BBffbH*7XD>cDf_2XbNh3XUUwIQCp|S#hksdT5q9xNEJGiO8rrcKfd;@@ zm(BMBF#%4O|MvqGOWjyow^j~ro9oumgWDxmU66Q9!>j^oc{a;*-P$(H^2im6`5Uvc z=JJ)NU^q?KAUapTRJ=`XE%8oYs-COWR}c1$yVe&MLC#}+mtXh;A1tRnK64>ME@pPV z%v{KXN1iE*jCthJ7PlQD9|$j4O3Rg&v@QtZ)K?Ey*O!;7_yZh&P6>Cyd!n|O7|hJs zvW6lu`EeQegQlq_QTeQrZico-Np{QTtb&%=EvZKzm8%Wfusg*}u%RYl;1z5`1|FD< z%Ugo4Dru?hfkDgyp^HW%6*j*1XigN~FuAEewLXs)%sHBm&Zth7cJKN~gPmn!lCzvs zgTQq=@bYPZ1B1wa@O-E0v-lQqoTV77!9d5gWk z_!GgiL0)JGMp}gKnp|ImvB=z-)7m4u$6!rnQiD;+^3D$H5VT-fiRu(%8qN`o2|MDt zKAJUEn6W;Sn?x>DlP@+r!=S`uvL&y0L-JrO>_3d{c$oz?y@#=O3Z8oLQ)qyYE_M=}W8S?yFj^YJxM) zjGwB<)xH<9ah@F*To%C2i3~_2hMh!1iN3LcjsSAsTYN&!whD9W z2T(kr1!(Ahg~s4q4|Y||kutdMgqu|&3{Cmc#1w0@uymw-u(q2qXxnJEd6cF?Y*UZr zgc7;vUz}i=3{E^%^u%hAd;uBpBq~0YxXiC>8Tyuv-bUn(K;;`rvtS(jk#OWWQ9lHB zUEzbG;zWai4f?+69t0DO9cJ51y>5w4(hMC>M4EAbqWr*APjPZ-;+8Vd1P5|sGoTq0 zHe)KMwdv4oGNCntXFPLAIg3TnjNGB{6M6e4-cUPWmrNOF#(=){vD>;SVl?Iv10P`o zHEh1gF@g0F$m_@A+4#$uN`qRnXk}*S%*bJb1X4CTi&$G4 z)|QI3zd4A6DVd4dxZvHP%2)E`;xRwjG$Tndar1hvZ?&_I=5HY4o`LH33LW}TU-Kl! zH%j1=UKmhO-rm6^91X3ITQCPSGlYAH`BO)K1Qo`{k;bIzk zdPhmmol!j0uR3Z5LuP<$xx?FpHx-QRV43}7V24|46EsZUw}#9n0i9U=I(y{0yCF4y zIB>C}TUOFu>99wE5;0p0*W@UKUdPi3k3kugOq_PxI@fP=Iofqr^G?d-{_%ZcOF^b8 z@)k$l2XkZm%;)?p(mB7v+$NG+^%X7`m+<`Un`H9^R%dsk%ibPuYS>|&GOjk~agEQ? zs9?}ao!((_dNWQ1wPN(cW=feNHg@=b@DL@Wd#z+@hGmxN3bT(-yWB*(srByS>8dn2 z$C_+N&wV%#94nB6d*5S6tcCH+vE#_G6YOt=#QC(ZVFVNCnZgxv=Gd;KsBMpj@wiLY zs$h9M9uL>>o-!T}^QU7713ikwFgZ%M1f|PRXaqA6j*_v*O^cqy#mg{E7Nv)O^7Eb- z!jmhcNy_|f^CKV%E8cI{V{QUFf&v|O*!1HEFMFf4sx$XGsppU41hQuLqI0eP%1fjz zZzTN4)tjYPOYBC8I(3mDh=P^E%}&lnc4iY8Y(+aa?@>J9c9`qUn!Kl;*RFW~?F#na zuExA}QD2-x%M1g{z9R#+l_Gzi&#Rrk|Ju2l3Fpc!(Kn@!U(B(L`|Ub#mj}*;ijCz=y0u_~c}5As(lSnL zgMxmqvJ5DEAh)y6Ht?7a`@9VclsfZP%Pq4$$r@uGdV*C(nzp9U4sOJMQRBtWGA%8f^GKNXGqz8~T%>CHWgitS~y!x%DKnjur8J{|>{3os_2 z<;M??#&P1g)hXZfo1&(4JPOPxHP*D|sFZ^cFu4kkosWdzCdLF~9;1PsJQ8wmyPmr~ zaH}FuV~(p5-JOzrhZTc=`$Xt$qF)}!2<>sqahk(2yc8+!azxuJei$cU?=vsI4fa zSL#J*A2wxK6$+9BkA&=qs52l{-hckwk>(b{{ET6F%wX_*La1J+a-Nl-HX6y2L9ot5 zQTW$cSArM0g>v?P1xw`~+;C)HxgBm;e&$gPQbPeoo%!e!=DL-M#H<{m0L4rTh|rYN z+ld#(Oak}5haWgs@kBtykHH!%SW^Jm%L+?{hFOJrHg&;JZsEo=qonDwJ(dNBeT_?J z#Ac#CL8{*jar&EX-FHhMQ zCsw01xl5Q_X-xz_>0870q$4mbCScg-XbYCLoY%90>nwC43BI(pyvY(CLeb5;QV*l$ zwux&e_7bV}Q3n_`yE|4NcIWIPEjALg@3TW2aqSS7WQerA zzmJP{CXox0dTMS)uz(j;Fs8xat5Qxu&bHtqE}&`xkoyx{Fq*MVQ1=G~w#HKAncF=) zcpp9n_FT+!tY0ZpTeXrwAQu4M4))#5F9>D54AD1#xh81jVKIzAc+Jq$E@}N<_0SXT zrCMjLa7q^~YMm%|CXBB;`iF4^=TEgXgqn>UmWT zR@?T{Go9&jY8qE z(bD~9P6m~Rxk1YB%>->4!eZxuq@bZK{I(re!{c&vhuijnD+9$fD7T`JTZxC3d)o_z zWeIBu+bV|_3$3yO@Fmv{5@I^mwz`Zlr!i)KWIfZXZfhQExThF1C`KhwVh18V*5Zpx zhB}6}ya^5=sN0+Yl&0K@#Owo}V{zLjjKuZ%eT<67T}Jk5 zgF5xVj+3)O6Ru7SgS)qxcd{r9UN~iJWe3MXyy~~Z4nU%oInj%R`@DcEL>Pms~@^^8u%0*<9Seh71p;+7?CDvU{6R|WH4qIlS zi<2YuIru2jU{b8tz+z%gkBYt_#5Cx91I&P0YWiv*Dp`_FR9!4vr=#LD`94 z-^bk>jNvL)rYzoysrLjp6I9%P+~qDBpFkdvccKgq{Bnv$0fq^!BlW{!?*3-0{M_sT zewIHWEL1xvNrM7Hqhoc3T9580nOjhWl>NvVh}C6`r{n_aEAv&VvmM_h#@t$=UfC>7GLxjx6? zyhfIq#3W0N`f5?mRCxaLYkbj|xm*Z$gD8KSUofQSk8C=@PWR4RQ2;J`4N+v6^qoB0 zbAjtX+yy_ts0XZ6Y*M&?gQF?|`v(Rw$(^tc_+>6r0>#5N*Cq{cx0vM}N!2^u*@>?@ zJK-0+Qe|qM*0g~frt@(Ss$5SMEtO?@(^4V&nZA4~<0lARz8QA@m7LVFDTV5}2vkd8 zfzDM91+|Ptydv$NVK6zV9L^j}jxWv}T1lyT_(X=i8TtByoDy1pp!VVN3-0}0bg0GY zYabFRyX(N`0CCenS##L;viw#TuVAa2RvL{EGl`11}JNzMO?C2QQydo6Bl-~5~ZR`F$DdBmSv zwbJ)}tRH1Q0l;Csi5C8iG6?}z{jyxVGW6k0^kG?{!)Z%`)%?RTMc;tByjvGrKm zi^3?-tC4s8ekEP6@oI6xR8m&Q6?JKWiN$7s4W`8BQ{#>4!A@c1o0~wA#CS>W=1?ijS1v= znSTA^;(}^W>CA)%X#B#!JG%ugQ|qEVGM%oGAgWTMB22U|ja1n|<2C8e2g7uJ%ONryrvnrjZS z?h?iYoHY{IqrSHUGVshm9)M^fSr-fLTW;M??7(*@&ZQVEGdmYA_p6f{wwu(d&Kg+; z%i#nh5nJxZr&yY~hA5d@+dR@)t2d4@J7$Tb8fZ_|Ce)SEabiRxEx?37Yn0VbCHr93 zhWgQchRY2 zvvvM1J{tIy`+2Z&&mGIKC!u&U;9cE5-=P89!NFZ;d0FdF7ULHUu-DJ`O;xv|wW2UQ z=M33}0Ol$lNKj}L$cy%XtFBmQxFDW!XnARWwpus6AcvSiqqUQhs*C%!gvRs&J!P2H zQ|N$!aVs&JECNnLVFOI4h)bd&aIjN`L0u!I+hd*eu+u1?%!<$vsN|jVLF*GH4pT{m ztA!kc@C1V>97Y_CzyDxqLxy*1B!R5g2i(kY~ zCz1hQ!hm=TdVssHJj-n4`28!M+xIZ&stCI5v4CqLz`={E-B+HI0b>k!Q|1!6QGy{d zlV})>V+@knM2SD}SlII-Y%I|}D}v&Gry+awycG>~xAnYDN@caCXc;RFP^u0OAVLdD z=Ei8FS5<3^+Ek^gR9!(Sb_MGJMs`MSMqx%)MoUIOMl?nzMj1woc{UCAb@eCB`$QTe ztVY(SF1Dl0#!r;H4`e$8l*!TtB?#xUxrq8~%w4^HdhEryBjEN}yPSv$7ot&r>80mQ zot3%OAN|NYt5MB4jcTfLnfqxNOzwYUlkewjiRb4o+BU;m5ATI0C-HyQAGp~ zcRlv)`fMI~+iY%Za^2%Cu8J6cFtmVGGlR%|x0Idsc}0h3GFJ*~z%ee2&Yw*3+g)_D^HhOk0c1A_#@$VeFek8BS^ofhY^s!zy zvaxZG--{D_G^^M=C5_f)=^`Stl{ibL0i4hd{Ma8118*(=CX zkv-g~*!t@usDW|+p_-mrxv}QvKJ@+zJugcr^twMs*&}=9<}WboBYQCPT=o!y%(P$^ zP!yHf28{M)_%D+yxLxhbu);!+6_H~+_}xH+4Q{#+8y zkwVf*%B5aHA?4GcAr}G#NX$+nbDInPvBUhHrRCRZMi$!kJ8FJvFwa}|z{kgbE7*N{ zx!2-wZy}+_N7Y9Jkqq|gKX-VPIuDj|MZTg;GD8TJi~8iFkw96jJ_;|FjKa(RBM z$1(~_McE32kZYF(%+*Ow#PkG-4*sSKO1=g5tkaDCI8ko3)I z1Zy@}q7uViSsgXOn~BhhjrO&F=KumTInqgSH7=d_F?(y%-np0j{V;%=TD@n80pj(6 zJK<8{HH1MPw>3j{?{d`Z#c+q^7bN$5@J#~HYJvLL@~;JOinaz zG!C?lDSnrXvl8W?y?lz4j7{}NH*iHmqeN#YWVUv4LzBjqG`+)R(e?3viEYwE1o4eE zT3iJ09JvwBltu#nC6q;G%T1RtF~M&b@$ia6jBcSXK%=~ZLjc!NP_Z?PKoD$&G?WpA zFM{1@}|MH zfk$yGI@qi?msbxK%om4$d&!2&R+p%C%xE<_5q7{o7I;POd(2l>)!5k~-2iOU(YVa| z>yTLjic4-0y0)_gCq?i~5`clxSqdAWQH)i!LJTQjH=9|+3d6tKYOFc4^NY|}&)j(G zly3l}5{om7+lch&50|T0R_>gLIhJkN_`CzfT{yz!%ut8REwAz zUdZipA(}AN9kEYtjXO$b3v!}FW*;Sk5tb!YJ{wc;>

    &)AmkBF zFfp1yKQykYoJxPGP-mo+Y?Mn33>;19FfJtJ4wpa*)Nez~{8Uofz=)fFIk)Ig5#R6< zOv(sCbcMv1s`ko|lrSzM>dvVmQG=BssEUj-(RY|H36l*z2E;ceemub2e~XBU1co0J zXiON~U5$og@f(5j?p>5KB9--#RX&Otco0S^*z2Lo!4Q*$y$}$zE4FOE_aEIklp{A3 z31S6~chY|#67`jo_S^&`QhN<_))==2CXCv!yTGUmMkOp0s1nGAq+;UQt6i0CivX1hXB z0S&ofpd^HS>@KIVy$O87q(vY_?ye?slAIcqrcZgewU!7{ArUcXAiy8HyOR5;^!8{E z0!>tzU)$k>UlfN^UxdcF>IXJcvS|NL477KT|M&R+BjA5e`k!P~QeOQGBbkH&G1nLY zx)^`12(9m@5ha0%Gb^}sxCB918{wfgvY;5$FleHgDWZJiVM!W*y~?1w{YXVPf{X|zI2d*J!4o?Q%%qJZEbv26=k>vZ%A=5B!~q&bksobC$4Jv6 zyn1sLpj5a6oUI$^z8!$?zkc$4(R|*6!){6AU7Pqljoj17FN{WdajX}~^oGPWE2C`> zgx@nJ?v*4vCD~5N4~Xay5z6TPCFg&XISdpCKHPlngy?s?n+ZYuAc{IDw^fqtlmLeH z@QXo$q`us5ORP9CY?_bvxHd9nzug}S3BkZPLZbml(*aW1{>r@%65iDv^1I6bzpK-+ zLFU_)nfF+IkJUdDtnTRr*O%+!>gJ8OSX^ATdmGa?rHCy_U>u#ZnW3SiCHAm_2L%uPT(D=H3y~&$RG%aq7bgDUJ(hYoQ$tXgcd*D z8umI)BW^RZ`WRc%2jY@~w@&tOV5yd49n)rXsp#M_#5+3@Huat2TO5ZeMNID>r8P^yIrnG3e3q5v7IwFi3 z;{fi5GIAIu;s_Y}rq{lbzBJZpl7X2Ohh%otWN7-AcbQZ~`X;?pC}^++OcN!Ba^ra) zDo9V&gQW==6B!#bE+Jz|50sb%MM`CML|_T~kVNN5*cEu~zyyDG7d@mj0YL+sO^^l= z6Aq(3QPOS}K!p%e%^VSt00Y22Dj+`5C^A-Pkz)QfCSdGrB`{wL6B6=pZ6N~MfpcC5 zMHLX+yT*e%C-SuPc?Z|ydqTY@)O$kx!3nh&QE+*6_8pYrdj!LP$Xsg%+k%IRXsxsLzA2Bg^J85H^OkA3?ZGu;gZ4S%{>DQr4rUdCk#xs92!8X1rP;O z9r+`u=Ipl)l{GMth~%B`qoVbB_M zSc-oUl3IxaqyguO5gCX`unxP%jI@9%8G`g1`6$VnXakUy(u6+~LP-dwLV}1XWE!mD zqpXvaa|kJ{FbEr$EzNMExRDF<5UtiFh1UG2btjdOGEQ2`WDxMiCo1w;D@{;DM%M3D zFv%daD8z~kf)33HZgLCp2sH)~h_e+9eg=P-&qzRpZ!BCWh=Y5OK;DZ+)+slTfcrf_ z#EuXCSSU>JgMj;eFraMT3R0NhTR^b!%!GoFHyzaHfW|gNrvmKgg9Qex0;Y9QNFxukr0@@mY*75~hg8-!8f`Y5M3PdaYww|$4hTzD~6%B*16)Y$t z4OQUp!#Rn39x2vJ6}WHo$tax%ccBi|*x;H{Nr%IQQgtNV_L%}^PK89Zj9ds;Wn-2E zIzUQPN0bm5aHn)=R zr)tuth3y(5Zxz}MHyxy>CY&YbL3d4Y8#52U&$wMs#2btV%^{{hvzo@=1Es@}=q+WR zA>NPIH#oF?Pya(CfV+s+?kVA(68<$Qp=W!imwA@V^zD0oNR6Q<#?$0B2c>^S`7km@ zHVh0rn`ajHZFptbK3SP4n6)1kgi;aG_Gk zeq%Hwen;QQ4YAQkI-J{J0X5Qho-q{=G>mMli#pFJ7>P7k-)~cq@Ry*|RH60Tg8bYy zK#(^HOgtCA0}e!QlUEx}c58oFV8ApZst;O7Skf^;nL9XwI!FK=0yXG@r~jY>o2>^e zV78h9sNC5Wt|o!I?fUn$a8C>WdbH5p^D{u`TSCu5<|rGHKA>Q558=y|41?O>1>za_ z*0fS`8Ew@toCs6qssFyDV?4i9`12^rB& zRxugaoYRU+=?FzZt{OC{NWfk-G$LCX;7s6hMx#)6O-&=4LDE-XMjtZtjtZK@!C-hJ zO+aZLKv1j@LIBfNri;2$E1r}T!bl2;6o*l5egU>FLYZ(opoNY4U_!MYWi}j7D9{k4 z3nldQ7Cyq6L2~hM3rK%>>5ioJ67fSwLfr)tUb|@%S`|Xl$v!gG%133B(u5RTk3~SZ zVwg&8+eI7AK&5f65L`jf&OXf1K!;0i3|k_l!am52#x+D$Hwz%1Ujy;Go;ijD?-26# zo)+(E@tzjH8!h%M%<_{Ht#8r9P*^k@eN)0^lEc+K^cw?p2)}AymVBN+<6D<+Pl(TqVXOCS+O81s4HcO%salfsLK5XvD8 zNWj5Xu@kXQ`X+xj8&^KiJ0oGhOd#f8fI&?Z`Nlp|gnSE-u_GjrR*D{>{c9JHu_HMF zC8EipX2j54IN~m*mLbL2yyIsf_Q1Ie58ORK|WDJQW^H-@w{$D24K7&f;V65eCHDx z8nHZ!BGfV?ccg+}LLvspfh3GE4D*pE>O#tDeUl^G79VKOo$>D zds(p+LQWgG9`_;Xh+#oPNe%U}ATY9;$vcN(H444@{vwtzcfvZn*(6IO~56 z`^l69J%f^{F2MDu6Wj@_8)tCW|QfRYw|Qf;GX7 zBNA~V#$Y~pRv?cjBd+|`wSw`$YL6i66sQCO!IX#^CsV>BYjZ#$Zy`jV3Bpi@s*Nsr zTQAet5u8Y2bEEJGdok1|{Ej~dH9++_=5p_2MvUQL%&r^A5LNd*#IQ!z#y)>8jJS|n zcwv$>8Otwf40T(q=-b)K_XtHPzh}5sq{};85AO-|oLTp&Sa zPi;~>V0r{3APQ*5u2<2(#==yBIh+KX3fZ!zT$v+U2qD0lHUd{Og#v(@H)YHw5eX>R z@KPU0ops`-K^4wx5e)>~l<0rA-AzHon6N&Kk4^3KfDbQA0LO>v2iG~jZTR%ghp$UIh*9WdmLts=A zAvDnkt$l7(_aW%rw;*d{_nECK2qC+02d>*j#`~xzygT*0kG;JmtUFu)?Y-_L6m#dnb$hA};cagJhA;Go(SCRq@F} zUwUPg6qS9DR(e^+Qw@I_iG#1#WtE;>^$X|8Dn^k+r7tTF->M&3tg}NrPl{o0)-H-N zMs+niPJ^8nHOl6zWSUl25HQ%5;BH`BJ4qIcNise2Cfv=>7Sq$ADWCAsWAkmNrO{|~ zuu5jLG@Hk@Kg5NHgYWdUzmsXdlWC`uZUfMT=OHM($vk%6Jyw6#d&66%m%DGV{#(qv z1<&TVsJ(@$n;LEYW2t9p;tZE*W|!FUC0$*Pt}U``A;mObte2TVl}ayHi==`*8Vbu8 zsyWLvTPBx7`0lckVzh-927Nl%fCZZkNzRihDadM(O!aBLnCarfS$f_eE8yO3SXbky zDc%qyhUV=wTcm%P_TI0NgQ`ffGR19lRpcvOR97)Bqr1JXZWsGpoZU=M^TO=Grt~CT zEZnjx@-yvPRB$KxrB|LNv;4xv&Q9`T2^A|{xZZ`HoTZfqea(t|<(cw-@~D2+qq)nP zRp~O#=HAIVo5JsSx}Ky{Z=%0VbulE;r7qzv8{PR8y6=CD+1gAihI98+!>v26eX%gN zN11-BV-f~fg1EU=%Mhct%)j+c(yGQxTih+NyOp=lCsj<8FW~LEu)sF!(pz_2pcmK5 zg#Q`4t|^}P8sEQr-L~nq9n5zc|DkvBX7fDF&vg;w=jfU&l8Ih8iTQo~JCSUlhqIhL86+oNj+5<|=cIhGo>JP~4(cNdX&2??Dgq zs(L$i8p<6xkDVKdx4#Fm-3{ml4)7g2p2hr;k3zPfa%Q_Lqr3Z@>;V~|VsYMSBdS^tfu zk8Xb@Sg?9BaZ{s!mCjF2N?p|x#jbu$DWxN~U+=aG=Go}~XYW0iR7aLZ(O2Q_6BBEC z(=?Jmu2;;&6~YORuwrK7guH|-G6?AC@Av(EAMrlP4H-bf_cG(oT08bMs;bQLSy@?W zK3l#QEt5yb<02pYvGtxJO_$E8%V%RmYzj|;>av_J3K3=fZU6O0^!d8~d`qZFL0pTb z=SJ%|26tCJ>xge@auhGe^fCVmyuSvD=!ic@Sk**=@|(A{#a{qVK(N0eISFTN{&;$N zzFsOMBi0|6)29SC7~ba`CMrpIdVk?rCmhvyXTCb3&W0z}=a(X=1Y>^}U$sIUFRH=H zVnOnCJtJQ@%YEfU$#TRCbF)YLpZ|Q`6;x9QhXQz1O7I<7kv+;66u#D*=hO6G{~KSN z<2j3ECa++8ZFa0+pKyln*LW&&sv+<9d?cDFSOtq)io6$I244A)v;2ITaE>gs%+7~< zKgJO7otUi`USUs!O_zVg299WptoJkYoQMv9NN(^H)0t!hqct7#2tq+9H)ynr~{{4y!PQ6SiI+A1ssFjBy0l?{PgG=PVt>k(zByjW}otp zO1ne2{q_=V3laI50s-_}3s+Xo5`!j#bU|lU{EyGM!H9w!H zWXi^i@YS@EU!O0hW?&B8NC@Pm#H9bxhJTBMTNy#v4M7?xWq@<>Z$)H<@aR0tkSr@a zGSA5lbOB35BrcJt3uTk-aslyvVT+~^C+(Lw5S?+C5$)GNW+bEby!wv*^MArO)Ys^{ z=&t4b#?YUCD)E2rTFym_;g?UPPq}!JL_ar%Zo0pGQasOoEf#|ycnL}`mLnG-002S^ zpMENdKb3f!^^{yFUi#%z6#4K|PWa`Mkppt2Tqjpb{tM*DTqzeGz~S%Gr(z5nqWP&L z{^#lQKdP~6_)u0zZn^Q46~agQ_^DcTesMhgkK8|Lp1=&l0FXH->NXf&75<)=jj9AQ zJU>?2pVin46amLy@So1e!9QV&{*^>K+#>NU`7JAUEEIS}yYyUmS>O>Bm*hv^2@j7C zDXuH|xFYZT5|{j{1btIqTI1VSt-t+*ALV5?8*&twU2o7+=ANFKjDrm0{d|y8izvMm zZVwxjz90hjCcQR}2yxu=-_s&GqncT;u`3YHwl(CLI{LPols{}~_ zHJ28w1V0t$LxUxCBMO&zuxK+|_ABrR82~@umujp8Uu!=%tNCRx^0{Rtkt&1%puX8ft;1l%+qYWM0bFU;5@s=C>-4GwR^3 zZ`+pRxw)TXOu5)J(NH%(t7gv9be4OLo(AIoK0Upq`hM1q3&3BfwfGL41D~F9IPg`ZV$*(o`dO)e( zwZkV1PV@p%G*48E_gc&mCCuoDfL|vyPMeoJ^!j_W+yYO{wxAPpR`{;v>BEX2>3#G) z%|(+Dj?Ws?H@MK>fAV3KsiqlDtF4*^&i6dae2$A34B=6TH-mL#HM+{b77~T=C?UB! zXVOwqjVS&7WofDAUDNjcgV%#YA7kB{i9{a}qNhDi8!Pyx75c6z`dwEG{+z*;%}W|^ ztU011uafGxUV&FsJyFSd)!9SBa}4v}2a2W;|8+e|weWghe{L1}s$T5da}=R})0SO8 zd$o;w&Hc+QO>(a-$(!xiw>E)#csLL1{A`X-+InH8v+Yea^jOvT3O$VKtwI;Z@{88s z`54}+^=u3uY~s^b&UaQ)5FebD^K+z@^WPL76c5~?wi7HVH1|+M$*J~VMHw+T4{!~q z=G>3@ddQsee~E2>(okK^`TEm^*S0`WJ+TmR3$LaXRhB(35+QZ^0MF!Htn7@NyU9?- z+}+djT_ksT#U&m50r9HlhO@(%Ej($jwxTu?j5`|9JuHGRl&4E zOL*N1MB%G>%75-$NxoUP&kg`M@iS8YV+iT3bAh&dmXYcr+t2;Z&wGV5LH%i6(M-ki0yQGAN3e| ze}gkh`h22(7<20}SA9{i_Klr>eKTS0sjIS!H@!=Dz)+e9y?nqQVOGTZ&ywe+F~xSE zVQ)ez5q3Z{y`RglAj;3F_T*sTr)S=G!rx?MA_W6KJw2bofu#uWwBRYGObd<TtoB|zi_yig{Me>s}Ovc1Wb?ag!YH=Td#&0@dge=~gDQ*B-R zV^0fWwXN{1W*w1zyX6rNyyXAn-@m5bUlNu6(?i9Rb?#xi+d@HMO+gov*T5XO4~vve z4rkogSuc874`)BdkuLI-`){rQMwC8kRh+l?;2j3>yF}zmf2w`&9S5V?r}or_e>$JD zOzy+-4>+UH{CdQi;xv9NI@d9StPU+}vHBt*6M1UHA-~ETM2^Jmn%)(r4gS zW|DL^l~0%S2QO+4{`++XlP>3yciI~-lJS~9sCU|qD2a|+a74l9MWJ9=(R#?4Rw2$3 zOPJq`?a08 zgYLp)_&hJV?$=I*pB+5^t0&_g!tJrNzpa=x-3qLZmsbiTG9cM&E>eS9D%!n3_#q2PL3U3`tS01ujfi9g3s z@^`_$F?(cw2UYFc{*x~ef9D64*KZv351jAjUZ46{<~w+!yxsXTMi)$&nAl3-?Zkn0 z;-dPRx9olPU*HrweL+ffAicM~!q(yK@f-L9kCd6q+&_1O6;t)JjRtjUbmaA+x<}8=bZH86|>oj*^hPlocsMim;IAI`yZDIF@O_h` z;RgCIJQzMoG4I>Be`W3skJ!3hzG!@anb*XvMyW;|j;74qg$~6oC>?*(C)0-+ z@IxdODAn$Sust7Aev3l5|FA_iV{NDY}%vvPumv4 z!oW$#;21cy1VW`|Pjn2`WW6tnD){&k{N4-c@>0Qp)9J=eM+Lj2UQ+l{moK;kN&>&? zmwUJb838YsmAC{f2Yt@}SY7`Mm&mvTCx7RQu^A6b%GQ1NTd%Oc-^iEZ0e*0N^Z&U$ zXeTbkLv8M-$DNixe;uBLKt*<}cm2ZOKzTCfCtg6_Z#!|njyIc_Xu4mk?8N(il6iI@ zo<)54hS3i)*Ru;s#%F5ENiesIns^tY5u7>HKW)^(e)H6^{sRSXIoN*${1)KH`~&TO z0he*P1S2{oCc@X?y*f+jli~tP0iSXd5cH2{DXuS{>!;||)$tXVgJ>B)?)uex!7&gcYH^=ZFrSx#20&i zkGF@ex?ak?QrOS%CAwg%ye2xKjl9Z-3ixjs;qbw8R^e34{V^?!R1cohAIai6l@ovd zf=WLBY4_!Hmr_^x!?Wkp-nrHBSu`$u@}06gxa8yFJHH-$k=XiH1(|rlQ-Yt=GI(k;*?ZZ;j6!6(NU&xDJl5ckXX0LB{_;z3aR^9yd z{$|+2ABBnG5y6^5ctIBnYgYHgx9BZ7a*dI7F7@ldCDXanrnFNz;KZREaM3wX0SsyC zJ-m^*cIfEm?GlcZLE3-bZG>kg=4kI}pIs5fS6xewMGEmR@%=@&*zgu|(O~IPNRiL1 zYlm^~Bhkbwm@DSMtflsCbFuRP;fc4F%;%pH8vOHzqN$Rp0Hsj*7@;)$9eYkV#2sC$ zVo`>`{h&I+@t`nI=n9C_|1PWpFrqSj^D!YID?S%Mv1OEt9;%bA5(`gC(R`P6?bg3xE=GYEYSA+rMrVaR9T z1xCTtFao#6F@zu4;o5e8`SemMh0m%8^88XNeGb2(-w5&yUL5f2yXtS=nH(oZGy-@5 z-FrQQfPZGspvyh`h6Mb_Hu$j({_)%3ct#k)TUsu27VSkeY;XJY_UC9EgL`MbDJNC? zhk;NCcJ6<3`;`+V%MmYV@gy>}M3Lb}Jolu;sTTeB@9R<1>2KrU`%&*+7o3TugrTv+ z?;rkIfB0wp2L4&2u0}4$IaUvyn0@1bRLN#|&cs4`i60LCgqmKzbxo>!4Bveo<0)&% z_RRxIeWStrippNULuDTr($g?wUl>j6BUbPquq~Hmy#yoyYnO_>1U!E&b;d6bdZO0) zB}0EHR7Wx!XJZ(v4?d;m0SG;N^O|oLY{fFg+)FZJckCm}A$yyi&d{Ox0Tt(1fg6Uh z%zhU<%L*NH=aS%c=SDjT#whegFRRak87>=&G|^8x5bvUWqNwYt?W*pj z7&_lokt7pZFA6?!41<57d(H>PaDPEbEz$G4N>o*{tt|UB)GEF3e!ky|=rscuI31S$ z9~=hbhRb#6rVrY~0SUY(@9Q{??LWys_Cl*I0uQu@J#e}T!-7dXbhF-u`k zfsj!UR93g-g!?*#9f+>AQJL5Mt>lU>Y|1b+1`pk|X4T@4XmfwZ=ruP(8Jw3(0wA1c z#3w{5{er$Yu$M0UYjs%DK`4HWCiAzLy8rR?KiSVjnKufgx+yqf7*9hCy|ZLalt^(Y zmNNYJ=wbB)>gg$SE@fb1N9?)64d9bzC&&43g6jO@c={jDhhMq>K7CQ2z_fktVR44` zxrcyK@9848W66Jx=(>fy*3~s*-OXzO486W3p);*Lw7d7Z~TpZst!?hlsbG-I;;YJ#1gMMk;nFr&W9~ze$ zpT9+WE-j+-i`$K$Nsl&k?;a^UFjA2UzEd=quZm-|XV24ZNUgU^OcwKVA&S&e2&LY7 zWQVm2&u)JJh*|q2bsq!d1&N35DkVEx-g}?Lsm&DaiX?zcl6$N>R?j|gE}8nD;;0qp zFc+8Uzhz0>o7G3B4*q*iUFNLzLJ`E7&$skyeF32`z~+gQw@W25&fZc5fY9Sx67lLH z>-bNwii+ExvV2N9beizP_GxeXbTVR>Y?V&N@Omndzkxl|S$A&t=P2cu|A7Dbk z?Gi~l8_8RR{sxrqLZ|Na=WbX2^j@S0g=dA{Tdz2}eZh~)Q@~~awW(;hX)2e50Q{nG zec&O4;9n&_OZI)-hA2ka+fCcaL)nA0E&jgaurqaLTpy?lWAuGA70##c(4DiK%W^No zFAL3h7`zM4F2A`pM(X&v+Nxp>x*4GG24DgD;?RZ{hCe$Xf8(a_8+=PZ`rVh%#1A2q_E#8d$;8?(*?Vo0c;P$MsZR+n(r~#T|%XLn7qz zj-~4bPE^>nYB@>B^aop%sIZ>+VtcwIObunJDW>WBRB!|BMK5v>j_75)5rVxoUSS)L zoZ=aJ?5z4??N4cdM5`~J0WKtlz86w#*JGnfL$5Qj>zTFacq-58ag_F>wjFPO>AqoJ zDw#|BtU0>DiT&cFVt_nfg8lDJkQC~-YcW57E73lHuaAdvueZmRo;v}$(%V}g7kYc4 z?ZDMm178~#<=A4ya@3t=dgtrFv4YD5zhoJtpucD#T%YbQ8J6n{<$jMJT?&~Ve{eBe zU;4L}$>qtuzw|?;Oi#6M1PjQ2_?Rjb5`e?gUMz(MV!3oAj7~0TNx4v$`#Bf!hKJg& zn-E@dcC`x~7mAzLqxs(98o2l9`#Jsf8X1ZH9GV#yJk>s{_W9+|HtW!oOY}c}oem2{ zWISSq!pG2*=5=E7nC)_Mt3K4bcQ*D=RToj+syjYkqqsodFiKFtv>YRUT>R^t9Y zI~e9aM@lA7Dg59$@)-A{$MmT;)_yeUooMlBmKzCe97#8uNCutnCTGYYq#?MK|A>`nRwvF3uw3#*f44MBEd1PXTcW^`!Z=o0X;hV)ZRg?^Y$f}u~lYxCdBL$GK z56QqK$*Gh>j}P~Mu<+Kg@BTq?;e6^**l@1^X9>VP*jRbsqctW_AFYaYe?EyWw>O!- zRpg^>i*(@jSL;8XGCh30Q!4IJN*bETD~~!R*n$SI%Xfcp&OG~)<^on%3AhJLK(Xd54; zpLh5D#c+-dP96mJ?RAcboTHgelu>@WG)~?TAaV`{?~}{vf-%H>7Nm^S=Sf`TdWWl6j(G zNB7?ZFqssQAWB7r#(CDjF5xEM15T2XP*M3IIt~ zEQ-ivmJgY(XsBnfNSGb9AlfULNQU23qkO+nqBPNQEawB2h)$*w(YaGeNPeA&aXl4` zMg9d3+3zp=O$z~UDYq1d#xd`w1C$;kUlEKf_ks~p@Agb)0oMl+C6JbVG4o|DW_ zU$JninD_!F5A=ybVl*D2kVSZZl$WOpo#VY+EUkQb6zzvQIlcp@Er@f8zG$}Gtpai0 z(|%B*J$5oMp{utX3`DJjPP-4mSNneo_jl;th2Dh?jG9`2&*wJ8irJeS72mYdg{LObZi+HodBljx~asF5IL;trIKWa-A32w)av{g3s9f`LO4NuASe*dim0EQciP%g;kztAD%MOirryW!=> z(Lz6rIR$)9eL(4tQu?Eu{(#dTNa`bUbGrYq(jU6|#_IZObm{O?_1lj4mgaqJM~vJY zpUSzjV(yf8hS1Ae;>9(8YyWku;-aW#ud5$knPcWSha#B0d3t*ZeNz^!Pz7V1ep!lc zgnl9l&Rr=5Cd%A~YN}@!#C=9$z;?2uLw7pJ6%2Ot3w?-Lh6P}5+Lgh=J`T`6&H&^=4&?3tXBh)YlJ1p1VG3{@zXx&Sz|J(}4nL4C`xrZrhPciRWZQ6+ z8AumP(^(8-MP7V=03yQ*@G#Et;61QwoItwFHAWzV=PDm?aR$`8!+Dhp$h7PN6CeSY zP`tbXx^hnDR{y6o?p7@Uq`{{u%h$x66cPYrp zd5n82zL7<}~oF!!t85*ACBr>|i{2Y0kFb+ToccYmZ!+uMnU|Z_F8L z^zel_Lqs0FFQ=hCc7C4hqz7-y3Dm#gvV4a<2)p=C;tn3aQD0>ZuH2|^@C8?H)c@x& z1=sHuH<^O#H|l>kQ*iwvbe}1BY^Z-Q1-Gtt--s!AY%0DdQ}EbA`Z!Z?Kkz(jzoUBDvai%1vx{r^t1vw1P2r`=}9;{1OT-O=Y14NMH9PqdLg zyAJ>AJ9^}E>NU(P29_j2pF_nVr2H%Zc%v&56(aoCfLwwoM?dq6@EaTxM2)zA^{-+O zgrX0rE@AN1N{u3|{*t3Qq(IalLLF9>lGRZNi>Q!9U2PIgO|5n15zo~z#)LXdch$_7 zx+QC4?6|tYj!WPi zT`$1CqzJg8QBuA#)DXgJG>UAVHoO|XvF4mGs?&JAzo`_bbEBhGu~KzaE7xnAS#Kqv zNY-gJyObEe^0 zC2TZ7#qvZe6;TLXc3c--Qi?+kHHO{RDuhxU)hLFV)ig>Bbc#~jB%y9Moq8o;^rdCa zJw!$OzO@1!78F>0&CLwIS>$lA=!yJnwJh2zmd}F>X-PdwqlZ*wjv}_V!hDV23A;e< zl9SoYvV^YI7%obZhnU-c_9~#zwpjN?usCW?JhmUK0(30bVb2s=q{WajQOqOIs#}s7 zi;|f>Yv2$D>-b#SXw9PPm$0V8(e-v~WOZB1N^42VwNXA_?2ns0q*|%?;TUsPvn zs!En%6m01gIHwt9J@=pBq_Dn5(i?r5_f6eR=`#*Vtu z0Y-uHo`ZJ~K$dHKS=5SaZ%{&KL#wp`7oe@66-ooi@Hwq z$GXxprCrTL>w2pNVJ&O4=C%e_hZ~AGTrzESD9KBIL286l6*REruS(R21IFMusC8JE z22162+71gVf#zLW5^~s>UWO{Z-M%lSV$}7%Y zwMuPr+%cSg(oj}R*M->zsdj5zI!t z7HN(b7$WxJNmoL4exI$>YZqgYSgChFI}c-sITdCt+(uz8 zKU0c}l{@vlQq#d#AiMnI3 z-NF_6#T{;RCcO&n6W#f6AkK+SKToem9S5tkwqy$%yt(a`QOZXAZnvQ^s?ywR;Pt}k6H&nMK@Yxdp*&b z6vWvLzoW6b*F@DhOL>bThrmjor&g3VMtFg#O-2i9v1H40R+y?3Wl@dFFreycO|drU z%o^6#psg@Pk3*1=iUwQQhOovu-*J0unV$$_b8a-vSy@yoGz={4j$5iXRZTH}5t{CPK74D8LhfnMVMFlVPFAez0$P;cqmWG6LF``W>mkf@ikAb%Co5}dedMcv2Ae! z86%+G@cNWK^*oNw3Dc`n#n=EcKYn4|=4$x(3VeZ)dakbX)=M{ykt!sw74gyXP zTNYiUM_pu8?Y9M1ZVsVfkU+Mf>Z|1LRgHgv46bP$9`LXHS{G>(t+>}K|yRPO|Tq>|;YaHog z7D6`ym0L6xHIVMNS+5I@{5DO|5^ImDl&Y8Q3S<-RmY;EBq&_Sy*O2UQ*o{@J0A+p< z!2ST(p(ehYVYOn9CR(C@RGlFUs=b{o>dPwSjzGT~XqE*BW34*rfHn(^W;#`Dsr{1L zVw?VAr^_x4w5f?*tsA{Yy*PxbUBjUlA_%~gRhO&2Jp@$Jujj4)I^QSfq&Ms^#mc-r zwluFlai}pfZPrL!tRq0D-JI5^4Z2!%w|$p1G1(_avlg^%43(jOOja9|--paHKZ0mlQJG_J)Fwk7mtY9bL~% zt1;p#Bb}Xp%|Q?F8~vT&70J$~<1i{wn;O-&;Z9s~wG!3E6s(Mv&D#oE}e8%36f?RuqaT9v8{!MiR)k6OKn&(-t#DhO1lwCODSI@1uw$a+rC zTk9Pn>kX8*>=B4|8WR8?Z2V$tX%6elVIFp9S8TI?%lQ~w^;K8z*vzae&^|*nxt{J# z`pbbX&ezpVfb(mP*bIAK9^c88C52X}_L?xd>ZletUd`Zy0ZKZXy0HRvMGcTT<&3sP z6O$3J!V>nVF{WU3ijS6?R)uaXw=PoRdL^|~wuewF=yB_%w+hhtZU}SGhQ?Hqsv2gu zPC!zBe}q>%qZ!^Bjdrc^5*CN+KA#`-`F5Vd9e?CIK)u3~i+*`C8R<-bSo4vpk>!q5 zlksM&(e+@OTw9>0s=d+3W_!e@V<=k<4f=GoqJb(+IkTX$BDMtVx|%=s2w79<^>{%R zJHW6Ft_Wo|WMFDVY{)jPRTp+cH)L4em_1^DL9`nqs8v&iIZ?N&Lu!OzGK01oz>J45 zr2>_fO>rc%#hS=_LqOFoav2*7S!oTrn$iy{WU?_g6%d|y@Pf0ldi59+s?QHXdc3Ks65cs zaA>!#39E)YMc~0&c2^;;(zOI^L2LIvxZJAfuZuKV;ff$f{K!fw3U1#r0ja zJes)HXxJ_zx{SG6Em#OOtRt>wP1AyZwk8E_-9=?R^&7x$%Zlu_>g&DEbir%hWXsXj z>5i|E0%^Njyj8WA9Ex?PlGhzpntX3LHyC5k?N9JJZkyC-02^$P=&>C}+Nnj|fMM9c z``)}qiZg8kv%6KlHFn3q7FCxR3FBzD*VXHKyT8=brlc~!vfhD!fi%O%iy>-%2^*(j zqSk6)7smi+n&Y)UMEzVXOe!n^UJv;yW*bGJoY9GuvKF@j3MdRJs5zqwlo%#SO{$#nscV@XZ!^gtJYV3AvW;bo9mfKfsph^;+4og$c zm8RT49KtKJZxF(?y>+JUf~m}Zl=XJwvJ_1+TcIW^z6T76e%=wbv<1Kb&sTBT6MSGg z7r>!sZkuyzL}`70RqxkT(b3AY zdPO7V-6kv#TSx=#jGO(L)a?!yL(&}vT4k>8M7u$@Dz$N)q~YCeV1eyPSpkbMVytTQ zWeH^z1Dz6Gwp5bvL5YR)i=BrkuV)iJGswtMhsjkHQ2E|*YgfYJJz z6#d1nJ)4;T3D|T{pA6N1#+LI4V~ouBrB&MD+nU03ZNsEoZ#rQ}i(WMiERX|H+LqTF zaz+y~VPd&ezvI}Rx8@bQOsyD;MmD&zm1PA0CdAksRD`s{BbmT0j5I8suzT z>3NN*?_wRmpDZNPtpf^gKokr?Z9KpLPG4&3x6O@d({rL!vToYqR7$rGk8GR<0}vEt>KTG|cgLW!()0SB64#6k_W zY^OtXx|4AYTTEzwzmo6tMXa(Kk+UEm{C1rgYU~y##>RX?d63*0EiF`TEZ37^n;?3l z#Q<70Wx8uJn%7^fwrh7d*$g!rgC>&Ip{u2i>5b)%CCS;b~`NOxQj9e3QBX5b{gGd-&6Bc%??Pxg54}5vtCdXK>!*6T@wZ|GbHb%0cl_j#+Yhl8sGAE%b4l z-Zk=n`GMMCq`=8qTA}0>Ji1cGHY;3r-x(ZrC+zpeHDflh#`t)rXRo? zsmkj9m z9ipTRHWIkB1IgPmYpBY0%hd+8>hwC4R-RB}f`FFtu1?cNTUamH<&FHLbp2^QWUpr_=IZ$6K_fv1Ba|A8?PlH^3bD7&XVL7Xm>L=yW?4ZcR1h7 z=E|aoAu=hlLZ!&Hr;XY|*k}?6V$C{P6L6ykGCH>ohEz>oYb?FqpxBsnk)p^N@Dv|1 zdWSLR<)(nzlijAs80E671ME(3uX(qvz?L;!;m8n}G|{;+sdvg)XE-JE`MlYqyS4QK zY_yqGe$@cErMHs0O2gmaeI4U}c0{pfX$!7_jw@}Dr8<>-zv5{uQKm?IN85vmW)&x- zMK`J|)~S~yr`lJ1#L-rjl_zSWrOSFWK3$JJJ+E*9#$)w-YoKF09bU~QTe{XTqy>r0 zs)7nHNWy~vqfx0CNV`mJIssRi>~O8vYL%+(a$aa*!`igjHX7CvUKFQ)_435%l9d^& z1pN``bo%4odHZ ziu9Is%IdT`SaJ&H8dTX`H4s{2)He=r`NTnGas~NrsfC@sZxdKrX_Mt&79{OOnrxc4u|7>6_PA^sK zE@rYr%o}c<6-aa|J&h&Hb!I{dlQlq(>rIYm8h+KORsfk6Dv-3T`%az~*W^H9tePzL zxhXvE!_LO56xYtKOmGu#>CkJXAy`Frty$%E2a$=j9?X#Gng$0vwd0A3HP9;cE$OhW zErR%#hFT6;uMgFK-DZprCQZIF0)?PFrE2A>GZ5MNqSUL8YfQ6(uj+PwOoPojJIHSb zetTw6)}k*N8p4e>8@yL7j!N95pLSW9;BcpW*xS@>0wb^cLrTp>nfYW8g^>Sh%t^2^y_p z3sxCc2gca4-38Z^)?xzG)JBWif+#BT%fWI0sM3`3M%I`QR#r<5_ZXtp%e8uY$6?h` zeZX6+GHwrQ3ZPYorpR@JrBSO4i{wVv@pfkKAG0mRA<+ppmn1s>5KdX$T#tB!w(^bp2^;ZkR4ZGm^%l3@<~+F)W0eXv(@ zgNoIW6nIr_`>Xb_T9dIFqZZ|wXD%nht_hCR=FnZ*wx_JhEpAAfs8OH6a}aL3jyvaO7q1x`SKFFI|Z%{6- z?H$2Qs=7-UW0Tg`bzwLw4#&t)hWZ|VzVJ1uOt#meSEfZ-A5e^|*6ooGbolY4rc`U7 z&cP10^PrYoBptO4SX;5Urt6JCwQ2yaDx=b%y(P;8Ri-?@YJi|>n$olc0kURqk+oET zO<<|AZG917fY@Y(#lq0BnpO+i<5hJjSKw}SGbCiai&sce5mv5NP(Rqz29^%4s@2i5;&iN)&-YkLk@91FGa2K7 zQqPli6YbJ7iRo2{S_6i+Mqd^c0mao_uxlvJ&aD%2iJfhyjw??QT?(*~26yyIYhk)5 z-`W-#7FO5bfSW5`8>1_OG1_Z?k9ELkO{pd{o@dY!VzcnJ;YCTw*Rvt!C zsO*f;wX~DUO&w|)gQB#V%qoDt81T#f8mQw9x7Qu?nJvD813joxHKsv-&DvGPu5^3$ zvNn--wPhffc}#GeakE_(Mmw=53}@8|)-vt+WL6*BN?BBAu0k4sgR6|zp#s0##zq#) z2P8d!{Cc^n^m^mD#IuYia6G~EJ32C{E*lk+;hYXt@5|znTlzHF0vUtzx@DraV#S$O z^en7S7fbUU(%nw2@wPU9LP4I}uIzBMR~i4GIcB2!&p&B}T<-i)S` zYL&6;5~_~J%XP1@WSc8TgjY+r+>smZbV&dv<{9;VyW~ULF;}vf>8@>eOLc!4pmWF` zwFVr`@T@v)TZRt|cI-@V8-OzFPlBSowwnf6o>w-#8sygd0t$(L&=i0b8D!Avz+CXA zM^|-E>bmpBTt-R`P5XKY@oPJN=+szdVu0ML(d#&*!#MT+S`h%B=L`%lcSyuwcO_#| zs}1L}xS6@mtX-U`-e_e@`OdH>&!GB{BaVBk^-8Tf;@UAJ z3lm3PRf7=JEmh`!&B<~?F>8^~*8BiCK*zt(6_#9}@_9;!L}=C`raO}(ma0Sh8eze$ znuOH?uE}U9JDAm_W|N=JVHgx!8tm-qVu_!vNtW++7Q8GGn}Hqp0A_SmOcYm}QQ0RU zjvLOomOXQ{>3Bm+WY3f2A^a7Sn3GrE5gb zTNG7^p39WRkO(D%JzQj#)LL9Dsl~#S%9VA$wj{|}ZMIpj@tG^QU87A-w_OS}1|z_S z=@E#`tU7Ks?HS*yf@7YULLT4k(+j8%L3(4Ole&ii4PgOx*~W;-lN)M-VKb*$gGa%X zV~y5et2EZtDbgEGf60O7PlxTcK4{e4Ds8W%iHK^=e6`wfIWTYMr-RD4>3hqm)P@Ef zv{@Msi<&ccv8CGZ+pc5CJKF#)iynWzw!Qq!Q3IPKco>5||PRIVz0s*QJ) z*4!6~5<8Gu6P(A@@))XH2HIHanq==bt;KYU?r=fJ^1cn{hwDzOQJXZ|9aG+pa9zpwsRsExl3g%RxsTGh=j_F9k)Wf-ovilRHAN`c`vglUkXu%gs7w z=MiE)pcXbbut#8iO>bCnZ0I81wQB1I2Xv}RPa89e22^`6KrLFHHyu;Zq7H91?5^70 zZBURQ+l@&dVQLs<&0BCA=C*T;T8erfZY%SR%WcJ}e{IbI-74e6ZpqZJ4maJ}lus!BXS(7` zD!%L&fh}Cq*gA*TM_7L{*_Nzn(_41MMqT8LN{=Wn*wSRT(&uH?tQqZ^q^vp%D!(Fh zpg8S-f3V!^m*%5wgY(q^}0&}Iu`>@@m2*PZSv#d)6V z(G`3);&D|+wX!u|HOkJsGe9O=Y(+OZ>lwMDjOj)YWZ$*>hTg071gFOvZgJey$StmM z1G3b>oNY-WVYln-%$^xcA=6*)@P#$V6NEJbe}N|2Dv>h=0HsyWtSEh!mfHHdxI++f zyBf30`T|(aXu=e%E-sZvHMl#R4KSc@b~SNqT9YOw=fQ#43)m?QHZKy?7f=x=sgy*%L(!d(Ku=M^uZ7&B36!}{ zd=0^Y`;c_s`y-brbq+rUgxv1sg`h;Pe{c5NZS37C2ob=NLK`vDVpYc-&Mlbzyc!=g zWg7Gx@pln~^{#`m-|xo(cCdu{BB}eAwWg$f4ltJT7xCMTZf1fzJ|Be~WZmv~;l`Gk zFi40pssJ*ix$dcbe@Lzi}p9muF~)AFsOi(7IYJ#t|wZ~G=WHOaMeuy9tJilu&O1~V2Q{V zB=CGWV}DIFGEvv5JRi9~T6`Cjj`Ltc0pWz%9hW_onon`q@1s!p-% z%l=@5CO)GG@m7$K-^6>OOh8(uJEKg9{H z>fPMoi^w4=^xjp{1KHp$p(!MrTQSgyF&~_;RM@P(yf2Iz`oE`Cw z1H`Pw_mPl@v+$0HCqN(>`EUiUaW#9@$=;zqhetT4;~o#N=}qbL-C21M*wJMJ)keot)csLQJxs zW06sv3g!ew_Fs(n;xZG7f68x7MW~%HZ<(s1>meW~K9~Q~^_cnnocXCKnoH%xZx=po z2Ga&6ouR;B4wh264DlzTudzWyS%_k?ePMuK_iA#%8KSYDa+Q5rF?{XvGn*c7e(>St zV?*;))W+okY+U0g-mI~N4;);b#p;;f0a>>}7d+0^6wuE7Rb2H;f6-S3N;JRt*+?{x zA~g%0kHm}TMengzZMt07>)YWr`m~-6p@)B*0#DdjUL?w;sI1ld%pX!2dw4tqNS0c|WJMx}cM4!SvfneGRP7{uHC>tVr=p`dHII1jUjc-t#&_5{x zJaUOstsEqIe<98wf4{d-t?fH&V@)k6W(PO=!jPuje$dG6L%tk=`;KWG%W!ylfMrVb z2|qH>6vbpldqsuh)@~V6GQS(H*x^?UB;7uX;ga-ubdz((yqXHF2>hC5sjuv|JBaG7 z8cor{RS1S(>ti^NI@pft%8O4IvtemkA&Xx~JGFwb2nSCpe;$2{SqAu^9&C?MYqVEs z&YI>2ROsr5tv|HH;IyG4k6Xt}bx8^RGlZRP>%onJNj{RD!wSj@J4(AW{j*KY5uMqpey6o1F%`cYzEBZgxe#x)7& zi(y!wLnda%Dd9M5=yd!Qj*2Tix1$5pyEe+VSQA%@R0iM(>-Ih8Z1z9Uf{wvAtqUNe z&K~T#7c9spI}vvGg!KoYsba!BHU_jQRkKNwf8D9lG#}3Smr4=FI0&k+P@s>fkB zf8>)|W^2H>1u0KpU`^(+>EI`e9+W}uM@tXcx;RYQd@no%0BE=v0-E4XprIlnSEc>j z22iOWseq?OZ^Xc%f!IT|VdK+gPzATq30z}h>qymjklL_viMoS_|<{7c=Hh`-9Op6QIxG5ASz8T32qq+eC ztv?4@%eO4qTBzeVrWUQa*99b5-@Qd@_{N11;Uzc=a=q4gb0Cr)40Cydl73i8}~!l9LcsS5yuSif6lbB+nJZ%7uGf(uegxb>?D4Qdd|NajpbYtfp4m8 ziB>Yl)3s|tH^n2TC#7qlbXAyD3lMq2G^c^}&?o(sg|-)R?w8Jf{&Z@ao^?d(#cQD| z#giPIK00jC18^4*rsfbwQ0kMk`#as4e#*$k-mOu$`xLmvmwrd zQ0W@c$-|i(sW?NP%Hh=lT^(Fd+>K%kLxGMZ=`64ngmlUwOAjJ6>Lk{iz;Xl+L@WAi zo6%P%LEQJw6ChNDj`4D|$vHTMM$JE#1Gzx2qgs}M`!x~^Yj{)#D2w7BhVlF&+q1OX z-1>gVTPh)e2jzGxnHl(-e*o4^!6JAK5>96A*Z2-GhoIH@ETV_s11|W;2TJ{1(Ce(blo;CNxrOxoQ_P zR%}<6B|)vOfP4O2it6;7=32tJA7WEO2hMMcYwhVu@R2v;&AUCPf2badDlZ(V$CGIN}$GW3SQGsYgDQWilIQ{i%KNR(ZV-gi7kJmdBaX?A*Ohv}_ z)1?|CNsZY+1X(Y~-JFWhQ4WeBL6L9d!Apa9sm~0Qhp`g2k_?%&9}O8=FS%k;90vWn zi+v4{8zn5d!)Hg{e@>TqzANU0duwhU!D-~9q|nr$*Z+SXSc{2!n0u{l>P?SVQH3!m z+Sr)sai>ITS?0R?A(ctur;gd4XX;wt!@?MV(>b%UjcOQT{gJ$ky9z~TaJT0IL0p?H zs$ub2-nKL#FUHXI_KPOl?&hHEZo$PEA|mOadoj3t@SFX@dojIf1%jRo13# z{0eumAaUERkh?dq2>}F-Y=ez-POfH(9}^i??jzan%k5S^heu9Yk6Zb*a~Xs)>}S-}P7&^23~#DMd!C z=lsy>L8{_mng){D1}6w9&kX^~5@)vX0;Yi%nl$(Spi->!a4GtUFXJxwcsX@*Lnly` zsTOPz#i5>sv?6UeeQ%&vclg1k1f~r9R&O!EBtj06e>g${=h9XR3=bRX1;7_lEACTx zB?I^sL3;)na6^-5$ z{IA=-e`?%OU|rQd#3Z&evH`Ql-;$(}?u|u-JbAUhzsT5%R9+7cX7G0-QD2&>A6uJe zvnp==o1vW04`j|@s%ebVblI$@6a+_;1X}1F_EA#+to1z?X(^)_luGhL_xm8}ZVf8f z4#p^jBivK-JG4+CqzJ$w!V~-#Xp@={D2q%zMsJONq6$DL ze<5bVhv&DC{kq2&2rn5RqQ;m%I;0-^p@#RTZx-kJg%NEaMH37anL>R?_ZB}Y7{u;QBxP_tB8-!ow2>wF6G58Lsy578s&K?j(;u2^XX`M zCa5R|$K}#{UhdBT^UQ;;)UC1u*vQy$fA2Tf(6vH});y%hhqk&t>O%}raB=(ZQs(Xf z&^G^8u6jldMQ8|8CnRyK{UpX+5%S6YS&PrVUEO^hX6zZ)>e0 zgfamysDvs_={X~M-$sq4!a(JTArSa`atE1E))d6>(|5_*MA<)9q4>NN$Yf4Ml> zp(xmR2dW5BI@PuO)30GX&dCV_>j5P9yL{v7G{b7lAFJ}sqH*Pvtp>Q)R!X2l5OPE+O|?(#uy5v5Og3pBPRTTQ zf^BOhkj3`ZbKVs>1@H#}0ZlsOe@++3@1U;mXPp%Qmz_q~3pw@ULPhH?gC*Vs@VAN) zR^*-dO;y+%F?lE(Q>}n)G8lMmHJJ6DNE(Yw8^MolYau-9CtDE^ttwut5q!nac4mm# zPJqV#hRMOmux~q+&~sE}tS2V+pvO+hsN06xN%gp0YcPDSoqjxt-hvB~f6Q^nAOx<~ z0s7P$i%I{{h}rn@TxJrFBMPu?=F5S)LjGtM0yk*}dSwcEmjjTo2`qF`LE2!@jijlD zwPe7r73S9|ax2H8yVyVX(McXQV-4?C6l;4@t^KoowAmING{__k`Mso+I*SM1XdLe+ zarDv2_cR;tR3zU!(#I=pf8m`DI#5G49Mn&;k>iGI<%ve;LNwm5jxNg7C41LF$NX+) z>WDc_Ertw#f7ci$t_qqDs(A3T2u$H;aTGO0(!rNy}G8-VOQjZMS<2HwbF@XbHeb zV0owld*w)j`od*esFhgmi?0S!(}PYFeeXU|OdX}h2rl2JUrk#He*>{ka2Y(j-$yTr zUeQE^%5)Ij*+3Hg+ku$EUtNo=d#h0&H|xGRtZcdbfHMv$WiR%}X21asbaZ<;IjlbW zgF+ZBXr%@!bu+-5_a*Vhw$=NJeX^>!$<^Ps4sY*__#8{l3KH11uAUF4c*{g#t9wWU zR*Y=&PM&mvWm_|Le}s`KGyK5kT)lHEBM0&iPx~!(+aR=8Yga~^NGSOU?OJhyWz&y4 zf2CYM=d=$TRsQwiZri|Su`(k(DV z#3$rqo&`mOtgUv9r)yftP$C_g4!d2s6I=Y480~SocLMxPe?fe98W-Ux{Q`C*UU#T; zk5bDD90rq{DnX7QIYqi1jh&E2gNwu6!+{C(`_0QxTFU)$p8QTGc^2OzfGQGXjgYL%lH&Dz=by zADQ|l58>3?vH25=)}P;9O|?lgXIovn+ANHhF3Atf4F?f?XzPgMQSJB<}7woVy*`MZ>E~j zjDX~BT%i)@&psl2#Kj+R`^fp0ZqRkC9F#b{JsItV`5<9CNFjK4XNwLv8$T$|haaI& zh{_z`C@zz;f0oORqyn1fCm}F+n40OZ()rve(k^G&FCgN~n6}v}7v=B)JF&Nn5Oo#W64wxIa)M^jqwe7Zj%nl5iPt zrHW(K*1QT?ApGt+xK7Bkk#|xD*V;qFe_ix@;0G0g=K7-{;p;9=}6nwMIqd@8jG3s&fa2N*~R>>pK{BAUy zD>(Wa@IVV%5xHpp_dB))eSEo}*bxfoB{s3wR;IWS{Btu6{>*|YXL*;Be|t|0U7@Ei zioU17NTb$~p}e+{zlw^!)9A~vjNW6sD++90)9arrWW(U(Wn(d~(Jk6hIJWSYsD z^zn4rvuodX63(T-pu~b*KE-b z0(4kebO?`voW6Q0K1FNHQ{jh`jbZ_Hxi7Z(*`V9ha8nvp0y*Hb)I!7QF(}kr=o4O? zI~_&_2?AySU1qT#LoO5j9-GsT>Ctf|C0xx&$HKid^z2UrE^C)*Tyfbj4R)*X%5fOh zsW$x}7?~hio06JaF`X!%8RDD23=D{HQ@TMUIWpDxe}$>7yR|nJa`QHwELK;%R>qdz zB>omKSjldWBEgL)|Go2wf$?CuH37%Vdgaki9)wS0mKKgd1K4IC85BCm1=S~w912s^ zR6iRz9l~|2Z7sVU-O1(gu8*^xWRjV-C}3Ogy;n5YT2su$AKh!|r0@_e-}7$$#ca6` zHNoIif8{GN1KZ{=ZrYSV9>KI&xsFPrWqxz^c^4}yBKw)LN&bvOO73@qDG1hAKMF{} zf-?Bc*1U2G)*zfp={qN)DY_SV@h~D>3WNv{;BU7%!w25qEv-&oPa6PHo^yvKSIua5 zesE#+kLh%?Nu45S$545D>_`H4SP}C!b{l!+(d4r~gsl%Ao11xaojfVO9`2ZCS%7%^U0G^DswyHiQ--h1*J z2-J8`VlX?1yRpH^-C)IYgaOdwqx7@i^Q0{@sdk{WG=CHPL@E&2+~ENrmn;@s^}+5i zfAIv9emp9@FNxzuQ8vcThHJIdd5f^a_=SAQMhg~|x>_%V9kM_YKS!e(*3M7D+2XQ1 zlsTuoepHl8ZPYEj3ecU0FSTDpFscPy!pd&xf7X>O{Q{j!U^)#tj||Eljt6hc2{i97v&Hc8 z@7Lmwg*V%k0};5(u<4*-gv^ssrd^OszzqB;NkMmhuRy}CGQQhr%Y%eZhJu)=9)XY` zz7FmccoVkCo1SODn=@|lAVv^1flm@JJL zoe>>9T&&V*(5L6a8v=jGhW>WBp1qdX3{ZnB*dWd_fV;~Q*y zZ{ZCaWDkY4ZxaRABiHv^%Em;mAfs>ykwk9`igU&C$_p8TbLjY)f51qe6vvgGBGki7 zBDq(Hs#W|KycC?L##-z}ocxLHe;l{@GzA!el@QA^$1J)@&Y0i5Tj^CqA}pJw2#z>A z4?Sl$KEnqlM31@?_Ntw^k)h=*f%%KxjSngbQ|izf4B;C`EC;(<^kXZ8CP8J-v+R$^E8v_aX4s|xrClS2=B63DOs{yYq$sdiL_Z2eZ!fLyhm(IcrZs&CXWi}gF~5Tn)7UiijgonI zLqq^-5pcYq^~SZ4H;$4x1>}C&6A^2-xmMjtn@#19c&gW%e{NhDe;SN-7r1vWDZC{{ zNf6yaS){et#OdeMjOp|HXU#ur%*O-+EUW0}L9N0pWX_4p8hXF28@qVy>o>o1B@Xf5 z(!yvLMkD_E+8eQXSsODvp91Md!Kcegi@7626IS?{u#nJ-94u5vQS?NIfLU&-0wawH z#+ba-kXouJ55W=)e@{z)9^!0rM-0SnN}6`Wq3wi+|vs~htT)M7h z{DgR|EsWmr%$?SqDWz;mykcmaJ{9!t>rA83m-6{mw3t@w^yUL%f6O4es+?#i_g0zh zNl&lRh@yUb3;|_hb_cd6wm8l&U?g+e}8F&X#3AfVr?hcc!O(hCGr>a4X@dvJ3^9Ce=0q>Qo~p)@b40}sS-i1 z+8s|}Hp;6&D|IQvLe;#>xWyF-{r3rvs{&NdIdnLpSk!5Bn`=r_xxsNvm1)l{N`XMv z2Jjd_AnrKXeQ~RmK;gaG17rdRH|1UUIrCzpKZ;;;f7lz?{hw!}Bw>2FDl==wv)2Q( z)%_?~1Nb2l9GDn6J{m8DpoJ$CF1s~;n4bmrd%&E@jH;b1UHAN!o5y3Ce+g9{TaNm9TO2%&Cu)tqJo()$G+BvOr(@!T#O64t>4#s#+kpM6aJwyR zggjrJe~d<{;yGtAzZN8=A?`~6dh_tiL-c6p;+*Ni%plHWQLfNbl^NZ*cg3RulAM& zBc`Yqjs(Nr2K=b11{ViQ4~{g-A05No@H)@AoRTCFX*uf>DbY6v1NhjtBA**YBR_)il~QOy0Z5NVd&d7Mu#E0;ER!#zk<-ZGPwCR5W>4TZeCbfSDet^R)pQ>@MY%|cbsLY7tR|G~EfY?pHaw5*M^5V6)-&E`RkHqG>z6*) z7%AR-Y|N015WKlJW=jF}t6#mH6OkdlWgHBB#n~xr2qaE~CCQk0V4=d0CDhmZ20@POoFIH*Rtt6vu9gWAMdTEv**>U=`oe%j9EVm_o+_EHE3R1;3(Z<~KTXog8JXiJw)Y@=A7zm{B7G>hn4%qLeo&`K7d|h|05#+h6 zWug@Esrocg);Q=pryK)|^HqZb{ThfcFsAm~fb7Xyn`kPz5w)jR8qKS3zSp}zB~6}w zrg0+j=diYb#5_x&R*2Dh6qibz!FOS3*ly}iHiPOSf5@j}phO0Pw#C~|6}Hk+=Z5&c z(ebV(f9EQ<#(>h9W&@mo#g87<$cyCXoe>y6E8=`PyFCVXVq0?DQ^u&t-sEQ3lQ402RQblc3(b;YYN^XbxtR9nJPR?Xv z*{(cjT@GGBbaRG5d*IQSMY;~Gh|jcNB;v&E^b#)|iN=Gt7E`%Ao+^#G^`pkr0v=dHjB#tz2kVIx52x(JWmzdz* zfA+0ssN6VkeWTKEDgGjKoka@Q#+LH5eK7j-vs{5nZ=ubORKd+l>g-yDn#m4l0stlQ z{uU1ybMG>^^+uL4)JKUr>W(mNNd=a|(b-XUMOQw*-lYePiD#$q+gaO+n9Qu+P*IjBVFXWL^Al)t=JuVy&$XUm zd+G^I+C^5&+X9i(Ty&knFXW~W?kNch6=bkVzi6dTGA*`*(!VyCe|(O%Q+Jh_Z;?huJO$aT8@o-isrWN(ki8<525|Z; z3c8$A_&}+Us@nc=S(1Up!t!-$iM9slRd~EIP?4n$j^XFqn$8ihwmG+nFi>iU=}5i5 z=e?q{Zdp$rVacmLaolPr_FXI&@V4O2>H&p=JgcuAN99)m#JPfK=T(ele<}8O0u=#q z{Vp}@%uL!l1tn zKBWgPbP0DxA=+vpsbe}cyV4RbP-QEBd~J4lUd#mYF9L_ujKWeY5foqJ4e>Y6xt~pV z5oh|v5dY-3n`Wd?I%^y*-xDt?*)cFS05Qi> zAmr(>Er|y3N}kQby!NTR1BUxR<4IbsPr2ZjbYHR=$8>V$3CJhMne_**MJvRSHt!of zjsxd`JwC#9Kc~#5{O6^69c;FGn0^9TXYJ>>U(^_7zq{38&6=@YeFGr)s?ozK6f0#kUQ?QE6pml#@X@1kvfk~?}XTFKWY|5yf%%LL2dRC|z63n^& ztJvMPBY7cQfR7TC?Z{1CQ+`{FSTxdKAt}|R{b)NXv~|_Jia@Gr;^gcXGx;K1@DH!d zbkA2EdS{%8U^mPmBCNsX$6DUXK3&YpB*ZR~i{CP?Gw=PQv7E&@awxc!{Y8ZkkVIpZpz}6SZ&TNs$J7bX0tX z7OI<;{^}>_$P4m~(yEVd+_^I))CIWr0<6EU7m@}qH= zhrL~sV|n-ue+oW$774?;a&HXSeKTyMZhj?VHX$5a9`eqgc{iPhrj}M@L zi$|}EBGA}dUQ4L!joy3;x@UXQmps|I?dk0f;WP9%f009W%#a4PP%3b`o`Z07?YiJb z#oB0B*iN1j&@z-fH{LB|_W!6;vZ3z-(EMYIS318r1iEJ%Bb7t1}Ivj2?ScRHLoMY3yW$-LO^9 zc-Hf#7SqL8aZ?pz+YBoeor=a#F_8|1dqGCg6Y&YqPHgUijRXtnU>?;!7oFBFe@~Ma{1CLE45o@FgO3OoMXAP984vq>(RWL7P%%b>K z1|i8T04psZa}fM+<##7)Lp7>!8d&-=2iR<1lB*SVmWmI2N)bw##i#$^seT1oq=DNf#{&T%>3Da}kF6 z0nk?|R$&94;4o^4o6nxCuyF$9M^u9_Q6Upapd{Au95{yN^dX+Qo9B-Je_*IrYlI6e zP<)htkTCMA$5fBFrM$f3W(d%l@;y){YUGsjJ2GhrfUfUe}|3NlSApMEwc3<(pL48U*wfVavn)CFFAz-#!btDl!jhv z_|4cXZ1&??Zkm$u)j+oDfrPGce(&bukHgP)Ku(g$UYfusrS+r;=}0_KpGa7^pfKMh zr5#t{p5K+2g6T!>E)EYtW^(>YdpltEN>1c&{U~17-U(rR@jDGaf8_%Tl}?K&OUV`= z629*QZtz!ymlV;WTMR|q9`APPm^DB35Xf7k#|`+&G<>k-wc8eYN5~i}JZoX_JlpyT z_m1K#L=DzM8-{&=eBMwW;NX085<`seT7!Wcch;3Y(>4wdtx;-o!?vgF;E%8C*x8un zj<*tJm0ynJ-V$ZuP2PL=&b{}?_3ESd%RX2_2*dcm0%4dlC*?gRl27$xKZs)H++Kle zdSGRED@Fje+v&xf~tDFyuqe|OZcyBbqly`@!l!=Li2V*R(Q8mma=sLj&O zkgK-1dMQr32>>vkEIu>^8t~F-6(2691-?dwB!0MV?&^5d%gN}4Z=Hs%+n|w`P$r*} z5fEJcViEP+b2J;3C>W84i(|i={|N(u(U;9iX)bZLk+!*`_53n?9))yBRrP$fe{V5O z3$z+fIeD2+)!^c-$d9)FodS&OPx|KPWS|k&o-Kjr80MB@w?UH3!O5rAVs3OI#j-Nz z$b^hqzs;rk##xgFSxxV7gS&JZVOi>DChh`hFHK>=;o3U4DqO9M7J;TeN7rlOiIus? zd z-p!Zzo*TtjO@We^WlVF18r2EaO9esr^7AB8EnJ|~8S-w}Mx2)^ua;K zM?l|0b7ptpC>guS{ZH~vqk6aXrrVW-Whf-)WW77m(^rOc7?IR^(OwcLc&#lrR|XsN zH(Vse|59@-g5e-n=@qtubN1^PHVuu-(}sc@vxz#3d|&qwWxTq;bzJxpW#1Z zV`79RMbe`4*_*MbNq-QE@tb}Im&s!y$4#a(n`YAsHLu`>k7uovaA;Neg50(4gl^ZqWzEH5syBX9m3fttH@@*h6 z%6@i|g#Ll$;H)xNe+3Dl{|WG*B&W>3LfsVTYr|yamzD(Fp^&wVJiNm^WcJawm6ch6&WsyLeBYcV8OusFw`?*af0NpcX=LDw;R z;om}#MFICP`GW?j6j0P*=?>r~&5%Bbvs9|MSaah;e4n1(f9-w_D$D4FRuO(!u?S!G~w_PGc287S!HN{IiPH}UOPq4|oMcg2+h-eN( zHna&u_=oubeijn^^YiR!_M+8%I<^YZ5MYn-c0cZ#M3A-7w11a;qZ|92b?dhims@W) zh1eKwFbfcmf0Cah&$DZrODkWF_Wa&6#f$C4f`7YzM=Wj%Jc=V^0{O71*tV)Go#M1v zIo?rmQ2Z=IV#zUiPe#y@@uMe4hV7StNgi&VQT2oGD85x_jXk*my8LQtt>rK_K6=_a z98=_rofB(hi8Ua3a7kzByU(kq31R{bj82QI+l<#UfB77N&6u(huOwRd*oF0@Qbw(; zY8>$tqt(^W0645q6d-Gk#3GMuRuwK&YR(qVupHgO)z5;`<{kk`A@XN|J#~nY3<+U( zTr$Oxk4xjJCez9W?=2#zNBl$CF_FKK!LP`J5@tgm1#_WNCXzbA*-6{7(Z@&ey*6Sd zy1db{f6Co(Oy@WH88Mx`pAVFEfbW~7GZ9KQr5ni`u{ObX<=EK??uGX-b5NHL8&}CA z{1n;n;)PmA*oP+ZS2cu4c|-lG330rR=pBlZeEVLlv0BH|^rTJz0=oQK5W=H-RvFn0 ze39aDpFy0|NSFFj)S2z;3JFhWAwE&UIguQwf5>e)(5ErmVjz$*W&T?X!!Un^J(B>= z{W!qW9S<=X1R!|MExjL)kcz;GUFEW}7$19yLNxWE$Dg#4+vW?w1*N@6F^dexlw6E< zdL8U(;0FLe-Qf^eS6$?EeowPQj+z*?utIxivS!z-8cc5RtSE`FPU8!WmyV?I8-nS} ze|jzX85tNdB9Ow+>Ju2Ux0M-KKSwhoavA%VYSob9Zz zZirsDe7^JM2bN(Y25l6$!e%(60@V$@)go!_=kJyoDZlb-^yN1`ui3SD}2w;!gv7pdNmXAx*TE1(yLEyM%Ude3@n-4del?2qH^|Td)6Mmg z=L!lqIis>6D|8*Z5_KFONKHC{6TcJ&xfk0+fA)v9YfO{1 zcv{0#mLZWzKN|1YSg+37%tDnxWs#hglyO^CgP22#V(rbVY+$+hfM~l4*rui@w<_0VZRSJ} zrYq*<1cLXzwU;uxVqI6D_ZM~*UC z6h9?|R70Ur&)PaTtF-$~L73Dv3MmVeXm1RT5rl;EK$1Y;=h0$>r*08+Mvo`TZh&IG z(CPpL)*=R>6-gYnWf;(4RJZGu=Y#0J#6i|P`|y^po2MFHfm`Wve{3cm7*2Bdn>+F~ ziEYQ*w6AM;O&JE6VJU#k9fqz#Oc5$(^K#|Bizs89&UkWxai}=yh2x-PMq@4}$lGG> z8NpFGmyK!G=iLS3I=6utTVXUB{lJP8Y=+^ng4)3dci3f3I9O%mh;wY55tQiQw}C+~ z^lReH!8_Gd%-Rz3e}w_)g_dGP*MNO42GyyToYxMxR6khR~l9dn7P7? zAEti43zK%S-IT_>H`|nds?=V;uy!$s0k+Odhk%DWmn#?s+InX_rd2d+>RtPpGe{L@K>s|5^U26%7Kk%FTKVjrWVZc zrb!G8F*`gWV}N(|id5gLThr00PQtT4Eb6697*q1lcdsNV#&T7;&zSAgTB7$B7Wc$O z5vIpYkYByBe_FD&{Ftg1fp#+ITJgm!5o+X<^9|V7bTgwx4AIZ7!#3JwC*{lqYxo#2 zoPXhf*X-|>4Ii%Em&54x(KM_6yVY_0$$Hz)LVSNNst4f!GeFG0s$IuRe)7YSQ~A+{ zgeKrCjw{BJb-8|qz(+UUCujNqg9{^w)U_LA#@7R+{(k$0G=Blr`qZm)+4yOMe`nHI zG(5i=H%Yv^r;82V|G>cfLR^4md2|C4>vo?PjnwFGjYB!$Y;GrINwCrvsJKPLiwRRw z9FgnIe1aE*%c|9RxlSK{%J__d0f)s1i|0{LJNk4S9@~cRPYes0Op6(E!svXv_nN_gMGhgzs z+?tNX6Y_1&N7d$3k`Zx?K$$!%L9`%Pp8M?-Dq;yPowKan%tcK7cuR@kwq?NRtd*`} z67xlE&Pg*F0QQc2B;4zCNJU6Usqx0r-~?D}-Plyt^napB(!ACESZx710z#346s}BY8FXbGHczoC1imnk0s^^WX6b^WmSlIDPJNwSJfU2(WodRtSrHwZ7o=d)W zQhQl_bjohHC1{SJ*a!DWrgh3EZVR=&lsDSU4uoXamrTkNOFSH|cgqQBi_uhPvTlpN z+dT<|b$?{nlF9t1O7ALu*?{vRPYT~Q)W&rq+AWKI3vtdl!Y--wblH@iNi>dmWbP>d zHklr$azMlgV8eh9{aT3;ge8^|S;UxC{D1M`$@hrMkrrw^j&eWvu4IGW=ZpPryj|CDr4AH(P zG2;|l{wa;3Y%cjq;qn^t4T?GpdCt}Fm47CsQF0bdB2u#B*@6K|ysO@Je~r|dZhYlj zm927F=F!4n^}Ko21S&g!CaNspu7n(4WRlqjhv2b8OzmKnO$^R@`R`1MHvSW1(+LV- zYqP!OvSM6D;5pDbw$3@yyIra(p^|SiF(?t0I=pqNYPW__8|wumN3ScV?)o^|ntua` zkm5eLJF2MW1Cg5qdrQi0*-=(<_0+v2ZG!ANnd=931$eRvtRPJ)7Y9T)>jHBU?*IAM zztzh=WxxOU_j{>pF87l8z)Rq-+cHANz*C_S`>R|3DchMIJo; zD+Uh3NTkLvyhE(xfaJd=jl_Xr*w_AR$tQm~62B_XMD3DB0>jA8KOpk`2Y>1a?%mj# z1VevD74zSyq(4)5;I|LTryeZY%i@=>KL+`9CAsvmxVr z5`U?~Fm?M^Dg8kUg@}-~k={9KLJY%#eOoEMh$X~`OCK{t3V;(tPIjU2?p7X&uFg()a6RZmZ-o0izsO8 z0+=rWhNPJ&Gf(I$SN99YP5yOg0$#)uTMj%GCcCNLJ_aNiWHcHPkbgzM(;g=3S+VV4 zkcvl>YMu{uoQ;-+o7P&o(t;Bwo2l;jP|5NI-A7!ZH0Bzy8)gteu=Zjuz~pG>70>NB z{M++2LQxXj4n^DeD6cUJ)UGs#hQ3PDm1yMwFWw#{so;Bc}vCRmcpp z`S|N)l05&djBbhd_kZ|I%@|D4y(sb;4u3^4*Lax6s>0ARoX4x~U7$jl@g!W~WP{>) zq?;5=b2r=OhK*7NbR8}`(*(FJG81CxTZunAFk1ai(tR2W<8>zyFzZY>#-yttf(*%@ zQNTGdlaLOh=$wacP}>NJojiF1FG8WvIuXDg^9^&lUQnXVaDN#sEVJy93`Xz_KXEMyL1*Z+U_pphY1Qh_H#m$( zwdu1>fq}4C&l28#XK1>dac#HpFv+g^{V)1S8fw(B>trPJp&#Y@BMnu-=zT?@9Lae~ z*LR#ytbbnJImZ>rt13mV`r~-zc%}|(SWD)qX!&KLXZ8s4nexsHKB&>SWpV1rB1el_ z>NG3<(#94+-YO05{3sQ3w$I-~???zmg*5|5ha1|kr)PlY`StSRR!CPgFVDPkWM*B< z51&vP;_H~XwGl_Zt?wZ-gjjFXos|TEA8gAaGJm0&K`*s-0wDgIWs>}W;|iHE#^O&5 zxV-nKmOC+)>jByQc^L-=d$}u~IzC(bOEY5t1%m0pVkDLZ4%>ucWP_-MXX0-oWfR(bCqD%Joth1ozg;R z|9`Q9u;o8|ICmmo0+EaWQ+u#*GEJNy!+=UH+?}M%K}dOD@T82>z32S6P#1P&vS3?O zr!_)B{aH7bjX#vGuR{J!aWje=d3_)5H8jmgEWMXmkGfQl-KKp>wJ&016$EZeUTEGU ztx>!&VnWxU)Ab3pq%;5AYVV>5>1Yzz1b>+%XL4ODqsY=uhb#9tddKLMSxRYQDZk&9 zvQa|wePD{KNYp&ffjW&8!ZxYG`I>G1^TS`OV!jY1feR>)3+ghyh2F|oVl2M^wKKLG zB9V4>NUyLjXSnQ}rIT)WP!xQ@42?s+uaJh0d0<(Br`Qz%M>VMEP zxqr@|3`U)6$$OK!Q7K4w$Fy}Q*IgWM!I&#j> ztEpK!fIKtt74-Q=gb#(JXxZ3EsTTZ!UGh&k`V|!Z;+a_=;Im^HP^>DqE(GZ+e2YpRLvLq%aG6BCYjhyLrt?p4! z=$IyHV!J;lGxAn4=2kdifmHn|Aosba*WU#!W>4K#XcgygZPX0+(Q%m<5l1mgOz)A3t-)q(yG-Dey}4EP;s^iV6S-w`Y{A2nDd;d`G19^`s&$k z1-s_fMkvos>tXhWv2dT8IdFEF9Z7M%1x=HZrAg9 z&l()`(|p9I+d;($dpdC(Kt?LdYC=?5jKA&V5y3{wB<;DCu2fB6PXpl)6LM;W+?!kj zZmzoCKSW`cuhvg2a6F2>*nhOCH~`~ZuSC0F@d=G;Y11AivfV%71>suYJ~ zkq&!XCFWUl3Y%KM`%?LO;f+Lx)Gv&{D90#@BpJ3l91n6`H2@1SOFgPmg63F>!0K6&D*3j23lm?s zxPoT`2x2*7%`l6*5inaO5&Kx)en3V8XiON!$q5NGR#nFoPJhQ^oSp^TH;oRYpNmYp zv!ZUta>Ph+Y082)VVKYYtS!W)Ampz#I@C;95MFONjJ@{k*lzXvG^% z=?Fv9sn&)lH#AzeVYs;{vHa2%fW{0l3!~S~Zd+Rek{oi2JX-7Cv2ayj-Oc*ax9Cnn zgrB^GOTv~_7=Okdy3H6oRdkpO+^4MZ?x(;ir++?;!&CbPJEIYp;01ldI7n4_eDs1X zn!euF6@^Bf8$ShZ#TnEJ+yrOlssXx*z}gN(A0PbQQhobu=;LvSI6ZHsChLO|qHh*w z`d#%kdtNQW&^s4#5`^IXiAiw~C8^JWDxacN&v3CRa1-m178v30y4O+u+c3oUgVXbo zCm+$puORXA&z^0aMTez_M}S%^#zqwDjWS zet-3^15(-&9M7ZwWGy;?M>iO#^T(`>EZ@R8rq_K5tPVPjIig1OayTHpG$^ZWaddy0 z-}eij{!DU`@Z9``GdfRc@a29!GfXo}SJ+~ebZyN?O^L&EJ(bU&Imji5+6XQ=ix?mm zaA6_kUR)18J36QQl`bKrx074K1Q1bkF(!R5B%WtPP zS@V&7;|?q&V85EODi+>ew?>@RY{%JFwiJ?#NJ4q&1N9Ua=Jehx8sc^qNcGSjhQ8S2 z!81>A#x2YzhbW3GxbvIt3}3+A`pkjY7%r5>+`{YAVsx1u#*v^Ln66XiRXKt@zkk&! zvfr2(IJ-{@6rb{Q>Cu66NHF2#+N1gw4fMG-NSJz|Jf8@Jk=pLfm4J0vWiE-8xq|?( zip>yARMc@8PorRCRL#8YII?+`obBFFgY?WeLlAgl^;<_V5xe{-?LK0&uuvMUsr12o zGe(=OWf--x)a5JDecqLe2MxDLsef+zdxqr6GJG6?+ zQIdRDBLb-4Pl(e|7KSCJi080|GfIYTabN>UVM;w;8q=eQ(Yi!Xs1SuF#W_nVRFd$0 zB04s^$6TTT^V&XF@>hqt#UI-XUD$X*jQWez<0?`ni+5zP!_=@KzJJsLH@;-?4_Mr0 z>9_OgFe`9xZf#)c$oX)k+<#vo<^d^R)k@eBM4rG11|(@nlsZc;l>Ls?a-I|J;HTuF z77r8>d#S?mcLc7xy7Lz{0X_g{R{{{Fd_u;NqyG&J3Wy;=iD_l`HTmHVAxIfLm!L-r+8@lq1ik9>YmXG1eX(}hlUOjW>c_P9 z3?t%FOP*fmZeiG}^emVjL_d=NmG@V${?dLYXV36!?1nZ7fni}q;+_Gv`lnfThQqZnHw#9U+JCHMnll4-VJEwrF9DscO5cFE!T+5SY%8$b>`cbTnJO6 z&ksOuTtGy3d=q)9RD<~{3$7*>bv~7=bx>FjqfcX{%g|NgCV!AaU!Z^kWXAQN*~Kq} zz!tGJV~>w`PCf09(Aj2Jbaic5UUY2c(zty&B$QZ151({n>J=YOnnHuL8yc_IBh@q@ z0C>|1L!5r7rpa{b|RNpsvx~Cv%JqU zJsv076o{E4QGbZNOZP|~|TBQk}vrem2O2@NWko7W&?=s>>=U>^)BXr@F&2KHXMr2FT2D8 z{`TO-MYuoUjtTY|t^55UkTuvBL!HGrHE zid#jKmXmbl3)_ZuWcq(OZA@mU#s>0z~iWO8KmJ^zCzPjCSFBVn7uB zDp-54MJY5T(3^;I%oD=#%p2fl!zfX#?&%j{S^}+5>7(1@s6Eb^C`3H|1th(t;JqAh z6dxp1s3#iDM!Wy764A}>WgVEp`y;9#Phzw$dv$GO`nJV1eM^7f!ONtUpp4jlwZk!? z#GcQ?^2y7-*7mC+y7{PE7xHFoNM#U`6}Z-pX?p|q0=Co@DAuXZNMs3S77&&V2ecHD z(et-qY8Q<2+q6P=#LJruOynTJ3Iqe~tfmcFt@~~p0hS0N;MH|^G`HW@tWYZ}nIX7E z-kWHLMUN)p>Y0BG{(Pa_9X9aFJn9xsJZY}W$bIPicQq2GkfbKSlMXL^Q8w?M8N#Z+ zd>n*EM4i+)w`(F)0PWyAzbq}upcH3OAju9r?~;_Mfn$rQ*_xSmyYOc)ajwxuGW~If zb1q_<6SbhIN}+I(nzjX6UkL=YAK9n#d0??d?QUDDk(Ph0Q&wm7a!jbW*H zb{b>Zxd+7Oj+|-9G~137bBkSZ=Ff%uMDT2f{S+!zpu>dzC>2Y5ZWd}Cw_pRb7^ zEE&pi8<>9vLI3*`=5ATHWK;lI?m=~L^@4{GMQ}ur(x$macWorXN6H_ zXr^)6_w!u0?`M3ccwoNywNBH>^B+NGM@4N3%D#{8 z^iF?$5f0}T!D|cf82O*K8H5$GpMv@`HbTJT>PfpW8UgS*R#B#N}p02K!C9-a58E3op*I!;OFWR?fG-e{{g_go(PK9m1=%;D<_FVbr`Z zo3A-)9T$%>o-+j+9@@chR2}{d^Bm z5K9zDbKQ#fF6fy*i{L7vpZ#-#di|;@xKm&Q=s{EqiiX7|<60NS+!0OAR>4Rly{UiI zdz({2rWqN|>q@_1CR*d^3kTjWe#e95$qWB^1tk#_$4osDgsZWfA$*{Ztq)7K6fQo* zl%aO6B?9v{Yk>Wel-nY67`q{6KM~#u1ZytV_>|%e;)w8XY-basQ zN&$pc)*OLmK88?}XihX?pk}T~_QqaE%^X1Xo%RZ!rDqDIqjS`a4*^@`Ey1-74lz*pKQNh4o~}hzk^SzzYHC{*Sra-2@Oj(bHj1@g zA%(k@ux?$>Z<(7%??(!QDh4WZy!B;sw$SRKsf_tFvmmQK|A-tp36O!*dZ;eX-WfnU28=#t^+8 zxo@m`Qo80)hI|pufc<-b z5L$&m!fz<}L8+wd74Nd9NY+Bn*CGDqXoSBojZ&6&#=nvI<;*CcmMO4rZ=BvXb7xPW z=pMGQyvXTBB%*)16csQ*BQBim6P~S)0>Hw18dIdxGWhrzhBw)kzSjVwBc|?#%ax*8U%(sX9om=(d@y!7Ep6saEr|FXQXnvsyp`?|-nMi}9J02-; za41Fm20?28ao4cVSa0sS8$pEy?_nsx273V*mL8Y)Dhhwuu2;!|Mpq&xbJ)K`3a1+L zE>Hdv{YbD4(ML7dhfdX(ABkK~Q4IVe3NJev!-FHrBTM^aBP+ zw?NnCF`9o%DS@d<|aGUZ>{Q)1|?+}>3@1eP{)b3MhX zQPaZME{&a+JpEUX236OUGdq%F)YO3iWvQ9=QO066C1|6WgNmFXEF0o0&^|1maRt;}7)SbuLFF1<<5n>J-3 z2sKd@S;qJmiwJd1c{)>snPlhXbsC6F_!YTC5&++YLvUrA@nKQ~l>z%tD-h@mi`N<9 z3;=&HO#)jht1%K%a$Tpy>Ie{xn`hk-GK-0OlFA26wUxQ6l#s={;q`xcxD1AfBq7eF z%;}%pWLBN{)xFn1>B)7Kh@!!C5wnozrG~x&P@r4U{>t7?97&q6vKYw-k*+e{r!*KV zt%QktMNpu8pDZhjYe{X{X3l#ySTj6j@)Uo8ijx$V1o@>D&=mAvBV69(bVkwE=xF7u zolo*cf`}jmNyaF2h`wo=P0RDiV+#3}Es>^chUp3O5-QRlOsc9{4pvuX;?M7H{9MPW z?t?80!*{VCJs0k(qSr?zUY4h z#wSjS+(?h5n!ksU^hl$-Co4tok7;Y(o3BZ8nFKvc%9KecF21#9?if6NX}l(<>q6*gA^y=v5&e*Arg zNmP$Qy#Sgz$f$*RQq20vZR=a^8@GSx8KNEqzxAuHOy#Nxm~&0utKUX=JB_~W`MXLg zEzS!8$Ej+o1+orh-xgwsE=~yFyNjoVXKDCKEL#p~-1S?oH@BOqI=B32D9}~AGg_~JvlmPmJmY`hRRIGL zJ|Sf-Um2-((q{0B0PC+~z4P~z+i8zq&T=eqE@yfUn82o4pknj>9mXRKQD&whhUjdi zHV;%N_7Tctt8rRB6@#B=$Y7F*lDl1oJ|Oboo`jD3%eLspUVzOMVo0e4T=1>u$A?m2 ztWC_pA{-BKrqqUEKb&f8X_tR3{Sa*^*B}eV&(`&RTeERfkwILT3TUZQ+foCmuhH|s z6h@94&~prTaIn9&vF}E&apU)7gD6Ae(73+J^t0~@#Zf#J)Bq!-rmN_6n+w04dtxj` zF+j9V(#PNsaA}-Drda$s09hMAYtrcHpo!x|k+_I=<8+O17Y|m|n;M0_h$$ z^nmTYIajUJkP&@|l6&X&80MsQg1JZhB6Nd)ZKQ)IR8!?zP`|3A0zvT*z@5sx^5zTY z8W)!qv6=53jm1fEMRT)Xrq+?Cb8QH>#yT7wFt2mu*NM-lRu_yoRo{;)7W zHmFtp*B@lC5HmL(hUu1L}o=}SBBM(1)-Z< z;K*O?w>Yi4dalhqA}Vomv~7|PbdVcq;uv*!RxK-YzWslcTq`fMu zjS7`(HZCXSpz(n3)oYYI7XljM*-Q!o_Wrid@Um9sxQYJpaog|hqMEd7890)_&=Ey} z2&XF`sKhl5K_F{{fp~$L(^TiI{mQay005Y_qwbpZ4034Q1HUyE`U{^ za=17L4Jm+Y!}n95HW3UuVx)hpgF(NtzLv4sRo{f0$UV_1l$8t3>W!jt(3Vh7#o7y? z5W4U53H??M0EzYOLdPV)Aq8-$w>K#j1`-R_b?oUCtVK9DV>dG_#XEq8DH&F!tz!Q! zTY-Pj2=SSe1JyzmG{(v1DNiiRnhA;zIV~QU;KKJ}`qc^sMrg(XG;nz6oKkipU$Zd5 z9b1s&;vb1feyTuzRqiXz7@(-d<+$Rct-~@6ukWlnfard4b)1z8cfka8WcdHijLZ8n zPBEOsLh{#s1;p&X>i=rQB8-0>$6THe_9A~XhaK$Tv?*waikg&{dN#*4GEQ$eE?qcR zJ^Vb>J*GPB7eMwB`d8;R1bxqF8wE?uGzQ&syx*c8-Qk!^Ms-yc0$SbA=%!|Sob8IA zxCzX-dYyY~a)ZwSMgv2&rV5>SKka}1adJXFAH=>y0Vz4#e~8K3uj5;^*D8eI9BO}F zGB=pQG^EU_X(nvxKCKoFaC=xF>4_{sL)Tdvew3Q+=KG`1Qok0;--#X7MH;W8v*D4d zQ5%upiv0j{J*$eoo*`Vu%}8mC1jowjHHPTwV81b9LsUWn65VbbI+AusTRftEUsN1h zbM{F~&%!}0a5pV;Mz!*z#t~kgdfI>1Png^+SN;fogZibdZ-dIx2yc7J-?Bv~fZ?V7 z4KIsuqB(;iD}VQeUtcS5P;1wPk6jRWfkiu6wc<^CER)kQp?2eXv=IF;J~FR`TucSL zY_0cg4}Ex{t3KKdK8#Y1O>=qeu3Bjw8TFzHId4Cf$pZhUvh2zOo8yopx`2OiJ3@XS zba{uc60F)n)!mqSYB1a4T~$FvqHl@R*geShNG|2aK>&9%oHcj^^C+AyDZuZ+x2`dR zBu1l#BdW{o`UUDVmQ*HlAL~A-b9D{?_OM=~x205E=k^U)m~F-c)0w61HA_%-Y~eU{e9P#lEsVdr>z=XQ@qyk)6+@%jL&*cyLvhy&TnD5G2e z6sb)OOH%3KxXg!#@~>O6N({B2CAOOiyTT;a(lReQje*9E=n zL5K(lZ&^CcfK+cr_IvW>uJ4ZwHznfvVoxt#R#>6Ji>+Ea~=^sOruRZ!uR-=(VaV7)+{f5`i@L8m|3lII5>$JAImsz{O&4ceg4%g;)6iecK1hu~ zpO>&`)TUlb502-0gx5r?Um&NIRK%mUhNd4F0=8$WGqfI9b_ahftth{kS`Oaa9d8V1 zN0I#Xm$v7lz4vwjd)SaAAS4M=V#f%?mm5<^i+i%+r1u?9O%qyF-D}0S!w<|~pZCiL zmjco#BgVlDx6Br?8Ammr=avvXLaFXbpIh77TdVPq@gx#^PqK!!VpF+?)37DMcKu7Yp3@Y6# zjax)AYCPvj!!@ujIC2gtE>YnjUF~lnR1Q_rU(&J@2}!@^FNdk`tesv<%_BZ=r2IPg zY@R9})kCaQpFla&;jdcgEmPGYeL;u|+k37kBJB58c<+DRC!q1;lyqjcXJOkV71G9R z!WD_GZ=OV8ons?WTQ8c6)L4ywp>4Jo`ocqIjzD&0m!xV#BZux%1m9tLu~pW;+`VzH zJECY2R9zGBj%{scE5oVxBnxN_+_v+iE)^s(C|Yi~*VatrVD$I~an|~{iuepk2ZeE! zL>Z5Q66t?C+N3DbniMf8qG&Nxd1b21oExlf@t1-?sFF=*0ZeLo!GW~ihS>(h@T^Xckj0c#2#XHVe4n z18N3c{-hM>J8Xo4S_~r0tGp@8uAlX^nU&ywTk$`(6%#67X)n)YqU>|j>l@h`XyO%~ z{SJSNFW{QrF~WBTMPg7Lcvjm0q{_;ZrD`ZnH-#g8%k`ZQypi6P0f>lVJMnK z0mJeqOzB}xF0Jl+@;w+%<!&>1_9?=Ts6iOq$7K*-xsnjf^=7iKvVBO$qFp z#OkiP3Rykc*VMy6RlpOkk1)cPfZw-hqJ}Lq6{~DJ)v!?eKzG2g(`9zml5Zxq-dlgU zRpX7P4%hSdymxX&9MlI4rdrba6P$mffMG!jrARVr;!uSt| zJEbD_u_-~RkOzoD0pe33VbBYe6&|^W26O&Ed{|5=g$TsU^^%z9wLtZ+!UH-j%j_k0 z7i%Dp21sVT;o;28lw5NNTyf-eyJ8KfzW>^XZG=c{*)EG7ibQY8p{y)|ir9Z7So~8> z|Fj{2!lH(BtB-Zy4(;u8IJ)cHR0ns{iz|6EmPeRQH5U6GwTAc?KAv7=rp?Mq7(jU# z7JfZhAT}S(hXgV2A#n_WHQfh8_K2Ws)uQW&Wjq?|5Z$7J4(73gjO4Wu1tyP@D56LX zW#hJrkXL4vIKNi<{S>4JEvkP$elO>`+djXQ5J;EQ|33*38&qI@?UuRa?csAJtpdzD zf5NFW3ZABkeDqH9F}f-QD?-2!dsjfe4F9tQ-~CN6Y~((0pM*WoZTgSX`-u8&cL{`M1t1osY>0vwbE_xD>3#P z0_)etrl~_U>gjNNEmIe3sm_h5mKkfzw{UtwK<+==qv)^aAKx05{VxXtc7rJ^4h#Jb z2lnOfe#V1jaL?HiWRHJ6UDt7M!Vl*Z(rmQnAwpoEi;XZB<6baa@GKbpcPeiLEG$M} zxEcC2z)rkh$Luv!;CwPigsFV=VDH9)8!Uj3iRUMq6m=lVgy{H`jFfJA@&WG>lS(bT zdrWD-jRlvA)`O?Ui0jL*FbA8RX7vo+unP}n5( zem#2+Q7+LRrHg+w){!oQ(z=c}IyJDp;Wm}?NpggVE&a`5fMZ)d?Ud?=&Fq%$zu8k} zwGQ1$w~?%6?xn=wquo*0ZJgm+Fr#D=$N!3f`vsH}WUlZsBOhy-xW>)O-assg8ZZ?q!Zux%^#!8`C`_SfKyJ1NO*VWe#EBLR|i#B=I6WDSD zeMNR2kQ>53(InJP6&uHjFMgQ(_ zc|Ki4QOkc-P5)w;vevbKBlV@^tg5RvER2f65C@E`35 zG2pSfwy2-3eqa4-WuX25BjM{G`e()IMb&KSY!!-c1aYzo&ZUc$lcp)j=_82o=!Hq_GmgtAN>Ue;OVxN5#f(lGqu z*K3I^i9@)plRxx%AIFgk^=cXgSllDrK+T@EYS}#UcWPIc(h?C6JpQe0H1EgJ8{~_f zW!V^edZ_$?j1=PqF_O3TS3^NAc@p7i`J|f|$=iNi7YJo=VZfB%9d8n`g(M6=iHA^0 zjUay}YeSFu6}Vs&p_ZR+GxBi!_S-X#AaA7YZ#Rur>G`M@Fe}98KF%x$*qgrXnzVfN)#Qfp9%GKh)2XHW6!asT0v^45ys(&SNLI z?I^McLgir&N%&cuN|SaK!G5=Qkfw9c4V-`TYt@EXT}~@a4fmp@Eb;1lnyjyF?Wv4w z3MNbDv~5so+DjNwq%f~!ZZijxM*?lq;5MpnYAdCTUSBQuKM!qxr|w8$@c>31Q?jZ` zR`(@W2UBl$_*KS{*2;@gn85ouH}FM>h}g|2R+edo6*^?qcfu!E?OwsW03fT*<^lf1r z@7YtbXCs^6X55HAPfwD*G6eT= zO&F}BK2po9f@nEcex>EGtQS#ac)jW=fX=dA9o;ti$WH&kD}uA5Ed(*>R4g}e~0mLT|7e~KTS05M%9itgMcvf?dAEJuGzs97is1SARwFOp16Ib?V;h3;M9i2&l|nb~6)CL7+f z;}JnUmR}Wip_Q4_8qnGWP*&n#!9H6Ujvq^(C0g4O14v2OZLctVDr_t{N+e*XmPvi5 zV!bBzs*P^hSWx!z@0sR5;Z}yoA2zD;u?eUi<5SEg$LU$h1Asm6xk!J$HQl{>!(62P zEq1n%&n#;x4dMzo`URwX{}SJehT&$G)o-Bs(5c1~@;gnV5K{RPHB1?x`x43{$l19Y zFNyRyWqLv3xE`zuY9j(d1LI{8JkZoL)fsv0T8kgJ1o$0 z$_Ygq5tIUI8*mQW>agc?kQAh$iT^~)qLskGkAk`pw=15Z61ab7wF;>u05s*sk~W}v z@hm%v?`oFk8P`ntoT4H{+A~XDa3 zwTjLkPWsGc>Q{iibryc}l($-bnE5V|(E($7^- zZs7M_N{5%~0r;dFck9p@W~=>pjVBqoO>)h-6xHT$&X{Lm1z3f^!24_Yssn zW6;JA*W7rXKUEuKL3+--WOZ>wg?y(0%qR3?q*ouX@@; zRo^+2TL{~I?)zp&QmX;|$yI?ta(ec#n@slthzk+QZ3~hG=L+3iM#DN)Nj;d17Slx( zuy5`}w8=AwpSm#UC(sM>rF`2$8q8Q#vz2g+0#JY9a6n&&LfaCdL!Pc}D!u3vudwTm zSve1$-{MZB9J*hN3~sXb@)%CVdZ(cSUNCXiC43GAo{%nHEkXMo1cyRxAWv|3K^BZh>jiKJ{UWH4ic9MEaNvD2bB7tPzJ{~H0m~py-EU~phAW2Q zvzveVi)Vf^6XHABm7lUTdDhJ}jb(eq)$ugdZF^6z{z>$zWy_iY_Zi3B;5c_Jo$-k9 z8UubK2i@fHPIvLEET1UL%6gken(-_sHG=_R!ZIRKUa?x<8|XFsZmz5HvlVWfT9EXR ze0e`ax`jKb&z}rjaMhpU`o7C`c73{7xF>)8#OjljNS_$fDm}FwFyvXZ6oxm3Qaw~| z2(dG=`mir77J%ao-#t%SfyHe-$E7U2ZOGc7N zgCUR;M7i&mBd1<)HG;*ISKV6c>#2V`{OGR?xUSja#_cd?Qm$)`QzDZTDSO+m?x-8~ zXl|Th7s=5B_=T5`uHU!$6gZCA#KXTMF>vfs;(1XLZRu3uihUsP&{pZf0&FS26y^ER zL*Et#C%0^O7#NEnk+LQ(1S|bSTr)Y4kMgX)nBq@Ox>R=rNagu)&fMLI= zpR&?|vIdQ9fn8d%(sBqP|79 zE$tVth|NI_yth!nOGKXh)zp6)jK>Mm5%keLAgyQwkE!iqeemop^7jcvRkM?7Kx^Ms zE1I|5Mj<%q`n>;{Q|S~b^~3dWTZbU{&f!ww=ZuUz*6p~(GXAxmDqQOX3xBwMdeC-7 z(m*q>x{8q6U=kB7iQgxamRL>c1Yg--fia4OM7;(atwLX}dZ0H282*18>;`-8t9$(P zcMczdeOB<)$%M!CA)hjr%=|a8glCGrOe2+u5c#lyVQ_PZJBn2>yr8T|91#!lW?S+1 zEb(}01&vic)FQg1W#pjTOmN|7X@s6z5=#Apd61~+ui42B|N9Sx)*A|2bUI}w*^>dP zj%UIjcf;eop}8ExF13FZ88EtsAdhmP;q{;XdSk%7Q;<6MaECW5KI5^9&#^Xjyo!Tr!e7Gj3ITuLvzOoH#%zh=ia@in zpeygGJ!MMmi2WUxz{Y_AGyO()7{00K++Qsfi{SUmr3(3u*!6KT^tC8ZFI{J=b`o_9 z?Yd0!pkW$VLxgp<8K!r{R}OES5ijo1&Gk^%e=%RRUVEjJIKmy%pL;B z3fF8+xM-!H=8%8;Vsn&HSwf(UEoik`-#a`Aa#GZBl!`m3oHq(1rk~7?l(Knd(~q5H z4UaMNT3_t&1^(=mLsIdDt21IChA<^vg_xLahIkM3WH~&pP!@$R-beM_c*E$_H(q_D zFn%7F#0?#w$s5#Rfw*wVVfRDOb?GNA<;9s%GsoI*t1o|S4o(Hogn!!v_CP1hwp{Vc z)(<5ns?#j^imr6a!mTN4XDC>3DR3qBELL6D6V1Us5bH3Ks9xiV1^6-1=9si-5~p6r za2kMQ$Z|qO6aKjGS~ut&;!!Mj_+#^BkwRcdz0j+&+hYPiVLX zfw#Zrh**D`Lc%ExGXn0>Y=6IB(pbn?&L89gm273t_`Bix!@E#0b=Cqzvrx(aDx5&` zSE3pqgx{PZ-JhkKGke*gqEy?Vk9K%A|CJqhC3VlxYVT=U7ClvF{g$;xgXPdO%dZV} zo#sU~DfGq8#->ke_4x1-tI^0H@)LfVp;%?Cx6glIDRab^mm5#`6r`gZs&GWQR&6&p zTFd&6ohJo{EF8Gw?!|r0RP*5ksAUgVG<$-dIdX55?FA+wO(IZ2qjFyxayL(5PnadH zRQ}2*M`(~r+jiL=s9^|r@E0>1ey$8e$QdP@Uz2lzm-e;-$oFUDuK(`O)A9c9%zf+?_!!jRQnB?c_OszATmYPZx`Xwd-WLZ-Tw@Af4=< zI#bg`>C&YfL1he{?jhjB;-9q2AF?!u(GwJhJ^?r2B~qLNB@GGX#8W?2f+L^YfRkO8 zPhEG_(*95r$KO<%9m>aThQL`&r@@x3m@L1njK?%6V9WLnZ*ePHhVV!{<-fBj#4q5p zFeNE})xJ?4+>P>plldR+n!m>7A83h=EIs`-YsYwS_Do|*^YOt_C1oUoR>uz(D3K%S zPDu4kHNUtIcV26t>0JX2u>Yd7n>>;|R(_r4PA#qtGj)v7g}nPRPUro|RsPedD#Js# zi_>zuB&a8>cWnk#6Q`ae0X@m}O{2W-X7l4>x@d715PK3Wd9qc&;g9g8H<4ZjVs||T2R4xL4 zO)l$l#8tJY50R-?!)00xf7-+6tjUq_D0N{Gfbu&3PzP~YiIb5n|I8~99$}BO3{=Jt zd)bj`Z%*NTbTdow3yb*b-F*A(LH_k>6^NVs4nGfKfpc4Bobkp z*`-t=@F$66aWTKidthrUtdlVflAim5|A^*qMTbS=g1gI{s$S2KrlZ z23zbf?jb&UE>h7*w7$`Oh}vWZ}TM`xYAjl~b4c8?aiY3h}=Hle3_ zi_djj)6Cc}9$yWwCB4l?iaZy8&SchNac7y74FChwG#I|OA^y^HjKq2LMI}cM3HOHE z)9{Z4&Dl>9Lrl`Mfei=*L-j`RC&8N}7ha$jf1$?tl{IV>@MnJH3e1S1Xx4h7WoMES_!y~wsES5>x@NK4 zDC@#jh_5s}iC#EFCbO{AV37AZf>auJ)Dg)<(WEC+fw!bbY8;X8rZ(n{K7(r9j3Xq)>qDjfq}O8wx360A#?5&Tg8vA#lV zT%cY1585-%?ag>89ZZpb3qA<8)inkOqr~QtAYxZBm4E`hf2@dVmR`{YajYs)9i~x% zG0gd@!zi4i^Rq4jb?_#N#jlB)BIQf~SiA(F_fh*-P0uhO;}^FGWPz6q8;N_g9BeiC zcjkaI7T@MzP5K+W`dZ3mC`)`RWw%!p3R%c!RVRBAKE?R9e@r%isCkA@XErDExJ zUg5vUHmU60MwvTn+1ZDXNTPStO*Ev3QQ*Q^2NuPU7AYU;?O5Ul*u@G6U5aztUsDe> zHOkC(5Kw9S(_E~f3a?e<5woK*iBqg0yD`C3eRADq7u^pfSbqQ&1IO;?d? zI}St_ltGw=VP?FUg_$|4|9Wb5Np>y3mTa9mprhxDr(o0*(c2`Cq*g0G7}E$EtMRM{ z{3~e~zJV-%++B+vl9*nSe(V>ns}Lm|c~f9$)C>z0&p}wo{cBqJYaD=a`<7ZcrSQ#E zmJH1#Ys{p{gHFIcgEcu8!$&1s9kbiDIW{ z2D;wC*F+VWC3@V50dTWKF-UNytdaVoG3rD`g>0A<aF?-=6r zhps0e+0KJHh6;27$A*m}H{(#79jQ7P zZU}c*19+l~EzGu@{H$p#?kk>(m63yuIM2ctVQeHeQc zH+Gq>W3$l!TySg3dLl)5vIqM~s+gH9llibqo|GrO2eS6v2grwClv8`Z&1!?vXgctQ zpe$A!y^PWN7llGkKo@I;m+9X7m!}z9^XhP!G=p|W5ntA9I)Q$27U@59h1EAh3@4m> zq~C=kGe0LvGQ{l}Cj+?W2ku3G6fTq&u?&v)NBUw+gLYwe$#ymWG~BO}1|t{?Z9-2g zYJwQUHxUS_^&}y=zf(gbH-Kwv9S`)Ng-{+VwR|?~!$S-=$%tqVz0!A22kCCnxnR|~ zK{e7*arB?ye}hOkwZF1W zgA4WvO}>dz7BS?PGXr7v8#ApeB2X9uPq12~_>>z%XZ~(8Fp;h}PhT4(jE4i0C!NE$ z89uI^oWonJQ)K@RC+(yO?TSWZ+0PZTAuW&A`fDdL1Y*NX=TG^8L;oXJK+c1{4u3&| zm)dZwl$EJBfp4+wrSXR_EzFeVBB>P8 zSmUdC?yG~shL6x&fY3dKV|RA%@X9B_$I&9xf(g#c6LH1gAgjQ>^Ss3F60oqKgJ^?o zfZ%N?%^4_+;)UAX5Ic4HSW)V;MHgfEI;44eJIeyoOo2AUd*D=mzo=!~T=1rwR@_jw zLlB~lgBz%8(m9A5}n*LA5Mj9_*IyvW{-q5ef%rm8p=2zW>JMxeO7x^`mSgHAB zFy1kwF^-wz`wn=a^5S7LbK(%jSzFb0B1g!U{52i$t&0Laj{P~a)|>|I6Tc4b{={tQ zVuV!BGiB)>SjcSrlrcUV(`FdB(s5P0qoOw8Se6ZN$}Tv6vX2G|Xt<)jcrX4i+gKH7 zxe4D-bqjTR3Hm5Yls{H#6BK@~i;##yrLt3qy4eX!MzK77#EpPbp{In2fKu>cKWwTW zLQB8OA;i_tztX~fRj+OrTwt|XRmX49bpozFOyP2&;?T*v@1$^St1zpLA8j$c&J-Y* zDih;fI&(omZ}oH&$Ylw)(I@^pW52i>kP(u8z~(|Pvx%Eg1QZc^!oQROuo#3fY1f)$f5Pji){51sSWQoh)oz_Wci}j`tJn7Wf ziOfm~-d06f=<*!O?$fC5QBczAEasgY-se1^r>$2tOX}44?x23Eyx+y3erlKUpjJ;1 zrf24dZh@D6XFSHe_khw?#Y?ni9}*ogHE*V=)3k)H0lChm!5bu3JF0e_sXKEMPET}X zDO%-!lBmd)3lOw1;&-LM<%|0lm1WvuxCbgb4)00h)YI8Rcrp#eEd8^1DN2D)$m*S= zaJN#6cq9jc&;rhUaAGQPOi~jZpb7&_BS}5MrRq0hCQWltBN}5J6E~{ig?4fPcTK`8 zsO}!5p}}C*Ojc{sZBTOZZc5sM`hH3d0#@!Z%{Rubd2H<*GAm|I zE`n|(dRiTK=|SDfg_Z<+nnf0Nbs#!~q|!Cz>f0ifVI_h7a`msk#e6}nGe@alil+t$Dxe&h+Rx-6FWpYu|&tWNLC>v zGBUhaAv)?Sf`LOSg6q<(LIb^c687_CC|Lrfpz+&|%Z02toLX0&MPeDC&X)ng8oTq_PIoK(QKi(NZp#(bihEZKu3=Mukif6NQ!@qM zQ>hCVV1EawE8eM5m+(row>2zRQVKO z(;l96%r_bC7SWVvIK4N2m8*ZD8Ci_z(%5MtNba)FP*QzPTT{$?QWB?dl3c|3de51o z_Hfqi3k3af&W5F{5M1icO{xtQW}h?X#ng4b&= zqZoHgs|GKNqei?`Q*MOCxAXw-FI+_kHd z_ZLBnxZtE!W-2_nob!&WUQI^6MMF>3*8OBSXXMb2!3lt8D}?vhB=F1zmi`RfeD)Pd zkSf`_s5mOLIjx?5US(+5L?3xl_%UDvg(7D<6mU0^=#spc+k~VmSD7goq5ulOcO~1# zK99!OO)mWwI_E~vuB^bhy0Kq)71z>cwAU6|8v^OprIavg0W?8l)s7%xn0l*s#FbP} zl>4;&F(CN?7;!1k4S=_w;sfrN@&{5?Y-Ro$;hATKyGTcWGa7BF=FVLuiE(0oJ$7V? zovf5*xgc@AwnBi%^D`gz1MqAik2@0Fqs(5tN0)-!$TH2m$oS)l=kVoWVdu|A^mXp1 zolDtV5rLVeK+T5bZOr(KX-cwy0$m|(&COF`k`+afT?ZMY3^z#0u|XCfkR($3^e$h5 zJC<9^{)fAN=^lxA@^mCM5lIuK_%9EfSX?1Nc<;NJ{N9$l5J}nQS}U@ipOo0(+>hPY z4WLY0brrX9b(Y--bjm2VD#i`{-yoZk-hCNAX}diVnNL30YSP>;{@D}(r3WC!8WXw*Jom?P7hUz{ESg=UO zlUbCyvgsP}Hr0)VPVN{k&Hh63eS7-c!cb#Qc@!i-JI{D`VnY)|(7bKhuKAi#*uGx_ zP|I%YCVfd2eEbz{{v&QnAC5x4@i4^FrYC3a`%8gaE?)&Z8|~8a z_w(%Js*N0pDzL~4q~0<%IBoJfzH2X@4*QO`LXibv$lKW{GZLxBhBAJFR!#P<&e?L8 zrp7ERc4CS2%^&*9B6H1~BC;Ibk_M(zLksGEvf6At$P=nxN|Lb)K^m!?hi))NVbO&^ z1OD`>c05p$-rf$R8;NP8y@Glb2$>(J;=n%24-4H4>U3IMtQQar;a%0y%Qd3^)VOfcq;sI57HYYV*|?3Tp_&AL z{;bS4PYYnKlQh&Yyu2A|uAXS2aZHX{g*eJ&K8KEeMA`R0hKU?ECYa~7Ar_!%L?9@I ztt^wKN$Br2Ed*4x@q?7>AA1^@gEu@%X!7gx#j7*|c>1;iLwGzQF~)i)7KgZmKW4M^Z#@HCzW$GlLv|twbDy z{c$zZ1a!xW=VcWC0y?lQNCYWSQR1R8^MRG&ahuVaKHx-v&gepueU!H)XZ4?@bBk>5 zZD5n+(lKcqg8R;yjj(O&I)JfISbd)m#P#4mQsvc!BK>FyL4qCF7@#@#@T&9T+sS`C zHdIVcKDt+#%-$jFc+kqujB})a%sIu;N4q!0dHVd@_%XLXHq)t?75FW%ce%Sa=jI_E1L3SSNt?*2o{()GN50;pnIg_s7G zWOHmo$C4jp9TzBvT3jOu21Q2vKdzPEZJ;?Qm*B(xJqQbct&hehw|!e)Dfc-{(t=!_-0V9Gh!9EPL=usxLDSs_+Ed7mElgVN=u?Jz2^9Tg04$z+ud4@dvGs6@X!BoS@Vq*sdj5bib86StFuXt)Zv@gX_J zelR?~4jA1k?nN*2qjE-`KAD7SgSe%UoD5cG!sheCeguw(jH{eJ&VEhb8i1r6z%Q6{#=wkJSHI^VhNBIyX4k(R`T5P*!!8Y7vF$0@UJ zoFQgH608VJLC&_=p$JJUnVgoItv~H z7bs%ZCefAWh!zC1q-(utLYIydVg%dU!4He+S6d80Y_j;Uwzy5>`Ac2a4r4K;K%Xn$ zSi#nvn&p6heCAN3j0~>)0i_HpI<@M=+!*2KuQQ)3|N6d>m}AB~Y!dF=b+nsO0UBXB zNRzK_OFUu_gpcXi7=L~Ar9Zo_sBZwjJ#q;>U*WjSSK&8(A-#p}1je!J<%@2zabd9u zi;g;eIVZ#fn22iNT&kEy+3rzmH z{z^Rvi%lcOL2@ccPASBX5$4)+J+J)%%P+@kW*QMx1LK*dt|spNhx(O04CW7qk()}A zbsi;Chw0mGWF5BCdUMJ;$-;FcQ(K58bssyg6>YT@oa|<3BWSH`InHu7+Rb81Wahx{dAM zzOERwXMVU^-k(4KD9-_w73n{U(VJnP6q8ba&Kg&S}6{`!jHVKi@K>|g-n6k^Ek>3_?hWIsp2 z1kNH0_5}RF^+6>)BJl0J{s&NTQ?_}<3c;q618TnJdp|}CS=LvOXx{;)p^DgGpkH4{Ymbt=K3s{`U1GAo z%IU{K0!@m%>Gm5d$sIo;pSqr&kYT`o$bpf54>KPou6fHWTsNjNh54Oo@5(P-65T$@ zDosBS@GBz~hTLd_1bELVYEXLSC?;S~`aTmIV%9g%7VQ|LfPtI6u-V{3>>OxQ#AQ&5 zT}Pd|fmg>@D*R^;e;d@7a;}5nOtqrP*{iPP>g}f9l?T-p66QFQKZR zH625Y>7i;+2oO7+WJ7#Zt?d5~de! zmjN!IG=8M~7-NZ>K#U*cQ8h7@X_0y-)lIL=l}TwuwKAdsO^nhfYWZv6fg)?i(;dbm zK|VmeLh&Ut8**%o#Dkamt|WxOr}&)jgBGQ@!b|cTgq(q85CY{zmx8u`lD(Hb0L`s5 zm|Pwv8F4a(cbr@sl-S8w^$d|3$?6_*U(fxO|Hj9esO|C{ZZyz z&vvoh=JSVSryyw8Uke*ch=1}VaEw$bfCJV}bwcKyi^_T~r_Q#uxe|)w4TfBW%kmd4 z8tdlnNhVCbfwE-QM9)vG@86;^4vk464CpAc0119egyExW0hu;`%93b$83|vPoH|Rt zAt(;XL(DE_!@DDl?*!}%O>I2@ltESZ0RaeIm|Hg~QGQaP(Cw)s&YZPr9&OndzQcnI z;%Q?mIm|pXvUy8Jx;Kgm-|gVelY6$S~ zk-9;YJ`9OYA>TA0a%5~o=2O;%t8QozeWU{FN6aqXu81*zu)g_L0t6oPQHcBg_*RAI zuxQTZt!c+Aa8}8#w)WxhK8a#+8?|dik0?3vU6@Gw{=%!duB)XI*?rV!d*H0#dd^vi zZ@VP6U*TwxvX6D3Y)pE(V0P7<~T1;O|VR7VYEY+49^*`Bn$*mk8m;U>~&dTlYZI5A3h zK9z{x9^$7(Y6Osw%-0Tbd|tnHbJGTZ3pj7Z2xku%=ay0q4NfQYv+qdWh|`h;sm>9y zv$?B4(Cu-XZ)-|4>ptk%f_uK!&)9bdcFr_Bc1P>J&ck$tk{!n~EroQqwCFMx$Pp46 zLa<+d{dtB$cjL&CEh(TEyd8UASR=P@sL^qh19&%23XHpf?m|=w9 zVwiyp3RgwzwhT50B6t<5I55r!0<|EP$YLG>9j946!*O+``ck7hT;t9}12c{~i@S!0@xP#EQ6>z+O-&1J|*2u3&M5#$7xy(dr;{qD*jmWKx9U&O6~Br483OBnU77fXVX={9PqzHpc((8 zaPXGZ_jsxovtKHho_8%MyJ%X7fYrpqd_6ok|gGg+p7M=A1o#Hnxo8@oN9ljhXnS8ysPf>_kGZ) zfLWCuG{6B7#|9@Xg2MxtnrWcR?0uQDl{F8mgf>@!sU z?(ub6Ro6}A_YO1K>nKPQFy*xPq}O5b25I6OYx?=!OIIy5NU*Im7n+NjI#uP)r%ZI7 z?)t-GZg1VgYmUcGqDs#gM8BG-0yjh{caeA30T@@BmJ2Vk|NG49=NGw0jG$ciQ4QW$F zomyv0I0vT)tc{|JYl`phCSZ`?K*=9=&aPk#)p%PFKf-G5?$PfW{yin{kV8SsP{qo% zc^@NQpxxDzND(U&i1}iFlrSi|$`Gd1(;$!PV^+Y1_La&Jr?~K(TWNP!{M@go^Q^pi|$t!W0=NiY!ilg3xn2wVD z9V0gYoJv*PEMUg$pw>+^xQ%eDr}*?YQ@4CcMEja_x%euPBz$Opo{u~BXuv5-{uTDK znrPcr+6a2QTM9?&Z#rN40~$~e<5&#GiE-o7d?(-wdP5R?Y=7EEcp1ThVeZ7rl3!_**bB%-aX42fUt2kZO8ri{ zt%!wuHh?nVcovQ7ft<;dh{%8 z$GAf0qF>Q}syK!RBsL!R#~%|Y;dMKPM2Tl;PtslrS;Z24$kw8R%VI|zev+JYmu12< zxM$uL6HX~qg&6BU!Dbn>=<~B+;&5z!OJot1UcC59xI}4^?qxgSNo36yga#@%+n$EA zXIh*ff+84sY~8a}Q&MfQi}Uuyk$k(5bq4-kgaMw~B)I<=E2kqf5hr-&0CJ4C4X5CV}-uXt{-nLk{5T z@)Md0w~7`y0?E;L*r78ZR_^V$WokRxWvtTh_Ag=6sXVK>EhCBjlLQlNGt zaJPql*<*7ZI%p{ksGxLq!(}-5aT*zL8Tz%T9*N6LHvtmFI%J5L*ZMHMjcQSJJVpqi zp1^9aY(-a^h<{6(P_zLL8N^qt_w0#^NYvE7Svy} z5|ryO`|bs6YLqx#+`&>N)*5Q&7hbN6yr(qlVx<6ZjJq4s2fbZTj~GmJ!-LiF*n}_v zRm$7AG?d_}fgCEVIXo7n;#1*F>?;ygR+uS;efUVdy1=ZnDL9(2+yOKx_&{j*xZPrZ zzp(eg7iK`lFcE_BC&U}|iLJ4wkHtHsz9Od3Th0`|Q(*rCTrj$uU#5U5NO9MADDMXt zv7Q!#pVhe?4{Xum*6wc*r1h?EmmOFw>48_){F~&5FESXQJe7u(bt^_>IW)C3egO!I zL1}jS2hN`;Gg*jE=y8I|2dt(r%3m9Q!2a4qjRA6WI;-xp{7k}dJAw5WC2+S^7(F5> zd7&aqMk5}(5?w&>7-J~i3_Tpcd>$>XxoWEzH(||d8U_E_%V|4~24s?E*-@m_Z7K!L z#~+$a2a;mfUHhOJ3c0H1Y~nC38w6eVRYsPN6r7M?UzDR{9Wn!PAn-sQX6~+k-r1SZ z5p9*KL1E`@s=%5LqMnt&@F`$mGD!tbNl2}0+q+4`@seX6bMPybzKs!&=Nmm*x;zUQ zdl!lN5E|m46Ml&5|hVfOpUCQq>-L>$Z3Z$yu zxMOx~C+u@gCYJ5yyj^}Bb;STJVguPeTNoXQL`D#9m92Z%7Px99;X_H#7JO;ikQfr~#2HO4+N0mq zidzln%0=V^IA}A^W!rq&Mze5tBBDkb#2j|(^w@VNKmrTL#_Z*TmDP}>NFUPJR7mg9 z5~5Om@FNE?Z)`HR7ZF>YplGH`I##+5OsgW3XH)S_^Y&i^4g6$(M2B>&pPdFQ`GQ^r zI~CgBpRvS+Kt~M-h@ZZ$HEY66z@+!o>PWb4A;IChQ5w8M+_}zCl@QynpB4iSzH4ux z`zG;|+DqeC#%|;dfP5tjcy8$q-vb=Y>PFXYrU9ssDwLI(I;U+bq?hw#0ZlZ1$GTKQ zn1qq&$$GKyDqVVihYeP=Uy;idUt9JiCh6xVh!=E_^wrk61Sw~Sh#!whZOP4&2{{O= zUmuqW9`)ibjt^61oMq^HuiTl?VIOvOo9XFpOed)Hwtyl?ioJC-aoA0dC&~8nx+qmg zr)MMCy~+#K0jf$*kLc>mRrZ1)`sDF3oopQI)wlK&cH*wZg06mhY%pt=4`9eqt;qUQRGD8z-$7uoV zOz|Ksas1_yhR~P(k`U%;RU|DmJy%#d9of26&N>#G;2Rzt=lPvYdO~QRzop;QoDIBx z@)i!nFrlM==!*@eTC>5A&cnU{-{1!kGdqz0&-OMiOoNTx*w-YmiG6MT2KgHtXib{M zRy{2P-V9~zqiqwGTr`W)^!s5C^Z5ZR_lWRN3qGh;I$mG4Nb5=Gv_I)LlL7gNqOer( zUkVG(p&$C-cWj%pnrh4Se3OkxlJ#aaWO9i^Ib&mgll&q*L4bua3+TF!>bBsuYVT%o zS{VR!)3CuY#6<*h4(>9*vfPR9Jko`YDr4EDtxVdvzy~M_=u|-dqy1Dza0}C5Oy~5oO}ge6-{Nz1 zAazi;{Hg_x27kYD{+7e+4kI!D75BGE3}^mzrN#{A|6&zJs2T6CZ;T``4a0r4b+K|U z8q6_POL!N!LuZ)GBH5te|w)S%30es?6Sb*1Vr={|e7VI=Su3 zt)V0t3aXg}DMdDF;uSo_@0*_|sY;o*eyh#Ob$8)wEeb&OHs>>HTo1ID*y_ei+*|A_ z1 zL=1kuYk!{C4Cj6#UYd+5f9+2LjL-Sr&N)r>2g`ian@x_?U6z^ej<$YvSu%rB->_Q6 zPQ?7SSx8NS-(A}RL!BQC+9d->9WL8T4!j-t+7AWBJg_^g$>ajxJ8sCjy(Bq#!HK;B zIup)ag8p;hg&!fic3D`61MhOJMbx3Fb(=z}f`6=Wze6s9^7i;dIfcIReyWNI3+o+& zo(LD_E1{`EsP1>BHTM+c(~l#BEFV~~D-Hq~M55aa(i-HrOTda9w73_GDj6(^cfooS zyrQ3r78G)L5Qp9sI%4R;u^kR(bc=Z(0^)BRdM=8?T8qqpFCaG5Vi5@@KJdyO zK!5Ki74`g&wRdY;R(ZYzzsf{kb%z}xDxPMpvWfx+Ll)=WV|{++x@DKLb`JgVP#P*yjhZ zPr6lqgq2gW}HnogKC|Ta40jb_eShvoL9P4RY7h-(~D_Qi(Sbr1O z(5y4OOeX9;nuYps=^l?W=`U{X2d$Fv^&r{<-u8~ij7eJz>rr#{wPyU75A@c=>0$Rt ztf@SonrPSCoheU$F8{Y{Z6}5iwvyS={BrKM(Q)ZkC)!`bUJEh?$x$`()#b< z)4b=F|9QO}d0_Xx^pk8|?IZx$=Up;HMPi{|Kkadv+@tyHJ?PKUv6qniv4k7AiwPQN zaP;oy#9B<+C`V$uMq+ne?0@*o0FC*XwVg@JpbZNFp)#>NeZdy>826ZcUtF)u5Yjao zu8b7>8R7M*+0FJ6gXZ;Z!=smL@hVIt65!zDI9Nzf?^m>gWqKooBOh(+=Wa9^j^C&8 zEqPr>=f!^e8cx+MslDm2-fw2(YG4og=zxdRKavx_i#c6u#n_(gaDUIGiTj#}x9__5 z3C8REvdXJ|CT8=yKa}iXl!yA{LtNshxbN%u%AS1X+?RML>cuWzy`1Y%3-k#mr_wRC zEJ~82hW$h*G3%9B%ZfK= zzR}}gT*xRM#XMyqI#wcDpR~HbtPUbpH*QtO+sc?PDr0V?a(}4F+o#C#$8NvDj}aOc z%lZW@e)#cXv0B_#bYnlo$*~(ge7aRVPn?~e4r+84^S#mZKcDv1+nn56f8HNN(|8t* z2HR6G?k!Gz%K9_gIo2OTw z>Gig(+e=byQ-5P^xXBV;H+>nV7sr#doZ3dVzrOpItA9U_#*e{xznW}=-Yz??(X75_ z)9qy~qM*M<{oweC-y>~t9R$PEda~iW{$@UVO{S@DkK6fl?GJj7kF%OAKG}BGp2FSy zyc%Si)ZN#?5j_9EfP$_SZVD=XAJyu0m<%;_YOu8mJm2^DRB8tmug>lmWlswg1~cKcgM=q4(cS5am@K8v=vEY`*NYMyxi z-h5Vc{eP@>GBZ4_Sk29MT+N^D@;Z-n!zi+3vb%cOWWa8h*(YFYaTq<#WLRHc)>}W6 z?n+w@*XPLCIZ0LZ4Wk^0Ry!>4nGFZ$mH$2&*Vp!dYJVsE{$5s#c+-BOx;zip*<|f| zeS9)TNpJh|+v~o9;rLaoAKu5dmaCD6fZVQ8uz&K3$zuxR`qFFavGyo>1iflx3j4p59d1G*V$J849D4fR!?5j{e+(M$-=bHJy*|x4lfpD znMO98qZeHL128+6k5UwLPK$betUk8?zT}@#c!!#Y%S+}?!sJ>yBhB8QAt<-c?R(RD z{eR1N5w^><6~%;j?7dxVle?A-KYL>l`8&Lk+cnr?_!#@%-P0%AO`bS?4C}!tz~g7y z&R_SiLfM9&TQryju4m#s`qULcXw|-4Mjko%f{fs0c;Yf5`O{v72eGm;e{-^y*(0T! z)uf=iHkS3ixKBjZGfY-G!^^PF(t`2Glz&rN);o8y^cHMGoujQ6E+xTAi#_)qa%XO5 z#pYNx&Gkj;)_T+Wats>#xjP&DWSjc}>rU2d`H743gXq(CUmgbf@G*0O-8(ihEZ29w zbw-?~TxTTv;7_G!yO`SGY${T?wJF<7W(v*Gbg-OuQq9y`i496m}G9?%%g*I zoZh#WDVJnNt32#^$zyf4$bR?Sm0_M{<6}EshFLnZo%nq&vb-rbvNs#?aWGt6%=2*- z7pG=aJtphJc+aqI>A z)bUk(u3K&$56%hgGnhP|a5T>@FHVhqBOzoqwCGwo;rL zeTVOxK-kU2J+_Rr;i}x{7iaGqtj+JU?UJ3}WzRQbh!*eb^}I|9f1ZEl25kFV@0gGM zps3O))91J1;RN$jyQO3^_FVrK-sd-bpWH6dd{e$Z=UduyvSK*cxN`Ffd$`|j*JHD7 zT30lK>_kw@KaZeip5IA&a(|1RH%`*q;Wtce6(_jnDgRWEzmS=3%+dchRBTE)Q9_FX#Ii z=C|hZ9QLO-$RgA?o5X$T%Z-_0Wbb?^0HRdQN zwDYrY=A-SE2z-qDPIP?KcWCyR%e%FB*ep1no^QXn z`l*bUi_cmIkFM|2d)nZ2zT01Jv}*3B6&o(Y%P8HgZsE)QYt9l0`hdzwUZ1&hdRWjWyUW<1(@f&mZ5| zBRP*y^3E=reE}z&W1h}Otv869wVO{(-`NMx=Qz-`I?+oy+JCx5Fkd^zy@mJpfp~0Yrvgu(muJ%tJ`?+VCgpZBUre5)Nza6au^IXNqH1t8{X*h1SWVM6$v>NWq;naIide=qb#gFu{ z_4lU}KD}Y~$bTLP;rq}@=o$*Q5KzhzC2erXZ)C>^F`VlSRwie3AKvemRUQ^AbG9y8 z_iTHbxbDi<_q1`bo{(noTzE#c;(0mT2f5XKw@^kCM%S0snx85YifCGu{V+;T}`eIOPK4t zeSeumW79>es>qif-{zIK|Crun&J5PF-aM4Ede$2?7iKL7{OI2=pXT{otye2Eu?I;V zhL88V#8Y#?v|gE;`T8@kui$T+wceXKyxvBd7L7xwc7N84KGCN%y?C^FuV#8QMx%Cb9rw@PN8cFE z{IR z^T_%%eEYjxBbz-QK4<+1q(lPQ1zZf%3V|w3l$#{i80&vgnzkho6dwTOmc5iyjg9<$eTyhE{&U&*zhyJEb<6e?x z94Ns7%feFXc9lXX z)nC`q{;}A;Y0jR`q$eJ3Se1{5`mxFTeK$NmCWlXAWyRid9^30U+pSlxWca#Gdqz24 zw)y0}f9uEFWqS0V<7US5Dt`eFz#dmZQR;kJJxWxiRg9`&voPK|p$ z+8^HSuIlxn41GJg%Sqas;(Pen>|f0!?e$UFocztRp3Hhv<38z`a*>`s{pk$rAqnEh ztZ`i2cfn=5+!SUIt}f;(9vt|$}x88EnthIh!C)xaXo}9eHO`AX4 zD7-!&+#lR)ebNNk`e97+dSamSN=CA}_imH)@O*Ew{%tUdQhT5et`S{Nt#bWZ<}zVx zYZDGJI%`&I=2F+(mzinHiD7?=n({3kY^$L|=8LtD?zb&k4nL-D`^6HySv(n8WOlk$yLU!UGrg>iKmzZ& zFwZN`bvBJIy1)PG#gno+;(0PZ)Rqr9sK2k#V>%Q(nIYQj+1l2TuX!inN-wtieF^^m z_`go&^SXQe_x~o#)9JtckN^38UjF-G_uBp6r}Fmdzh0;RXTN`|juZWV{wU4MPB4~8 zL<9C=#SX$;1Mo5>RzPBTuViCP>qXN77|Ewgso;FLae*e(egVLw>SSxVy`kHFP2asy z1>i~bHE?3?#L0A1_)xe&p@lE(<<-T6^i1>g>a|JB&dHFq$;gZN&1u*QiPSO!zBcP*V~WH^IlVb~GV;#p&})+s zIj2LjPC}BUYMYj2$<2hcW#?FP8l5zz239Qkw+T zb`b;9n z7>U$2>qMRNl?kb()(JKA=A2{AYiZ-1S+_nu(4i0X>KT0CxYtx}8 z;--tEoSn@eU7N7fK6PT?Ym$?5b;nZE0$F0dKoSRpX*vjQk)bUR=XB9% zhQIQAJ$r6jF-w4#_#r7mKsj&N$JMG8k~|1o=2%&5`DC_4KoCf&J_ANF4IqRezj!Rx z=!gPYVX0o%7X7EaGP5!;fG;`@npXswyGFp{9Jzx z$Q^k1c_S}4_Xcgv48ah_gY!B{kWpmWRsBL(aJfU5tP(48cv{#6+MwAh@?Scu51XUKGT`1~q}Fr(2a6NKlMmC7e1&0`!f+XSNOI zs$Ufd8|59>#3A^wu^McybN0GkozxH%SwYMV5H5mv8(-KKaqb{dJ=Z}R;J|-*#&~o` zfyuRF<^or-5UhsDcFUx0LjX4A@CQ#T;l99U?(RALT6nJzuO%l)2*H%g=6o%r+U(hOX59BmlL4%vyGJnj|1VesZl_3)>Jh zHWf8If7cwf3R~5>eRI*0%cpSO|NiTJPXBiQ-+TSAZ z4{!vx3=lTT^q|`}BS$1sYIxkhQ87k9F$hYQQK9k|(RdtSp<7N(zwa-|Vxhs?4Xaj~ zHOGiLv2>pxhzWtSSdH-^Dfk8nwLYAx`~XqL5R$)gk&_q;+U24rG*5rwvf)qP{r8Mv zSzw%j3w(x%Kh=-hSpXSRf0JoP@u28WKNP3MrFwazVQ5jHHoiO>O@axaO1slnc%1^K zKy|oCd;pk(;WXQV_S}3kZWbajzx6cf>Lv~0M$csM4@^F%nr9?4gcHpRldIFv+{xKb z4#DOpUcUL))ljuf($#-WB-2%gwdYu4?Jou=XL{z}ZR&W_)ra$+-0yPS6hxy9i_I+V zVo7~I*3!Wr%op*1*6G@mY@p()`wTT3sT#wi)3-UL@R^?bubBZnVZc)tg%Ez%nF_>9 z8@OD6q$P;d_{rr3h%E90skxjpB#E~}Ge$&3R2<-%t_j{cSYdxEmt+y~)Ey+LEecF{ z?DntPz~c!9k*I&wF%5`|G0s>ZKn9j%rit7)3TFu}&VV1nK!PI#gkaDO;B-MG5uWBm zk?9#;P7sk52F^SzWA(2Mmz8NkOt36WoR1C6IY#wZKgg|Zhp@&x++PKE76w^Ty2Ln( zZ4lgEAelLcQb>Op2B{mwpVs{kaCILf<#01{BO`LlXSNZ7IEuF*v;{oY$dL*!h>b-oi0CjNq!7ER z&SS~2EvOBWAjJ6+%Y;P{Pgz(H15$)FhamO&d1->6W)sDOt~Oi?OYH+Z?yzckPE1mp zeI$~|6tRCr0^2l=Y9@$!;MBvF#N{M3OFJr;sv?hD}uRgfM83g zjo$1ul>`*mEn_UmavZ5xU>L&c1Dtx~GhM|zwoP6bAVr``jq~Zh;VrQQ6oJKHC(s;V z0Qi=@cGp0xG)j;`pAgZ1t=k=!{|hsc6+!8mwN!uMhI(GDw~7T-^Ct);EmQG;S&r&% zt-mq##hor@yp}Zz6QIMW-};|y%$DCAK~&ven{{d+kv1LXwDIVRJ>=Z&R}=-7Ize2m zXKWcfUaebAx3VCn$l74^IvIi^$Y5&lYKcX0aw2#vn&gCe;hDL<1Wq@pzFqQ*^DyLg zv%Y^P0mI3gekRM6h!LuL$Bju`6btm0`o;)Yg0vEWq@J0Jg^E9ehrS8IggZ?!lmrEr zFs=&|A;kSqP^x|i(4TH>AP1yXJ=7{*lDgj*VM}tJBQB_eTd|{bwWG~&mt4sL2h{IB z`|g&L&46swQ~KRL$W2zUx` zUz2cqF4+_~YuuWtJXE=)*2VhvV{!YNw?Db9{qVevsp^j6YPG~Did1_~M`XHG^|9gFMkZBfdI$`lp{T|4ohIS1*4| z0%D|^m&gB#ADVNHzxBsRQW)1W2e@a5IH8DQI8g#M+kb0ImI_v z?AXKv35k#_&7b`d;w-XijtySnn_XQH{71E`75lE%Y+pzagoL&F=i@IAF_cgsstDP& zex!7PMXE3%QG5n*jUnjpBl8L`avp!ChGnVh1v5H4;skT;2V+7wZM2Oqii;uCHvbzq zR7=AUjQ2lwZ-zkKS2qfQK&J9jejooQuLB08g3cVPx)V#(0_213B`JZ^e~q^s<#dU7 zYLNgown!SqEzFj7J?Fp15<4f2FXEAx5$-g)rQ+gG?Lq0#YuMVXd2D8qsMvq}={r>{ zshH9{G&9NsE(o+G3JSxJv5O!z5U?-zl-xj)yNJ&WlT!~%H#;@S&$?G8$eD=+x7BvQ zRg5qKG9_5V{%;RC62#uiv^yRe&g|~E4|Q&1&j60P{bXz**!L#==>rLFA?vJk5(EGcde{AR_~Ah<`i>Ai?fW z6Ep_*1qO$RqZHsVT3rzK6i;ox8o)ntT&i9m{e$@(hdg)N#2rTe_JEwd;oHaGoGjJ4 zzM|{jVy^I>{nT($GtE24hKJMq2Mb!Sezw0wlH)&3={C@w^}z% znC(oisXoM^ihm&`*fW1e<-rl~5}X-4WXHeq(s}dDF8UVl_?O1Io+?#%r!e@p|NReh z;me`;&FR=+#Va_%mT=W|#e~SX!(9`){7U}~u5yA&4yKn&(glqHg_qgV1xf*B zmn72#EdiOAQqu)Vf6L3MWr_k3PC_I<^Lxw?%V;P0bo^`*<0nT~umzj}M;#9@d!D%6 zhxFRT^n6E&n-X0_A zR^J+J_ZHl~e;;g4K&m`VD9-p2M<3q*%s#;X@Gc8UZh};L5#17zsN)#@*SE?cArfQ-0Y^wuf<(IL3ub~muaP`g7=&loQE!x#iOp#Y zr)l1QQ;Q&uEW~W!>irU8^@k@ch}G%5K*9XQm^Mk=Y43`+{Vk3hPP;fEHQQQfmVzXLBckL;{^8Lck*8kkPdx;w;$z5?WCg4B zdQw**5tYJUf))v4_ky9%bk~1>M}D;Hn+5(zhyfR{6v_)f09+(*r1fe^E2E=64&_d#n9!S98MP^@wh@Y38Qr ze;jcl@fyNpX`V$Hg+;bRd}5mV9`jk&e;g<%-5ik15M?1T_;nkhx7wzavT88uGKGJA}M3UXkCf~$X)UW!>m1KmhZbrRI0+7K< zEjyqCS?#UMCApeBHjtTi=dT-rUQ+zzFaxvto9^JflEHWz1?4}_-?vXx8o-yr)&(bj zV=E2ycjolfAHKTj9wy-(D^qHC*VaviAY@Tv8EXt0(_Pa;qPmff7fedm|E>t+e6Y)Q=T zoQnPQ6F)pf>iB2fnx9$zPj3Hk548@z8uWkB`~2Iv)~^o0-k@|L++o0g7@aPVn}Kea z=x6R6{LHrv){rLoGk^Zksx;>tZ^N#CxBcp1=PvGQenuialR9m?Sj{Tq%}4|y3${?A z1-c-YI8O%DK)mCu#4OT>lYyFNos*FN9U~4W{P*66eS4qgr}tTQGcw%i2s|AxRa$@N zXr}7pS}CVqQ!of#>iS_D;ZztxAF*LUOdtmnm|+E)fu_7>P?fCFx;eDc7mPxGpI~%Y zxPjOrNvy=t6(+fch)v*X9!k)4d(e-PM_5usIy@odzvjoE`Ki;@6AMGNG=ntwND`oK zF7_|aA1;>C+94|+G@IC&q(>K36&sRvvrHJxj^M!~xTE1=f#m)HCHn4yrbKoDm+kU^*W*2-f-O*>#Ge3ocwYF&S^S6axSn&V7H@Q?@u zb6}9I5eNhE#OFf5bC%0eR|IWAn#nbT3-PjEH>YKyK#%z+CXK9_ybI#QTFbG|p@N#fv@ARS+S!>wedGwtua|*g)TsVYFb#;k>v|Hk24u76MXP!gNGb)o*nmz?b`ku27LZyP z_TB)-e#8rebC=@r6_x5|>&8;RP@lY=P|OKFu@ ze7!_Vu5l25e3`cTlN#YtX)Ajq4K|$NU|qcJ7+N!v9wegVOil~v4_Cko3}Jk{Mrj6Nv1Ebn8 z#-E(}=3%GxQf#UjdY&3TGu*DmfuLO-{LurtnePwgE#b?mf|cj+cG?0wKwN;#lcgKkNMMlU43UDnC;rs@aYxKBZ;* z@0yjNAp^PRiDig&1#44)OdSj<+O2hy1hI`}h5Nv5Whyd2XGp3(i@4TGGoXY5q4;Ph z?Y(v(?3uvJYD{ldOVbowdGC}>#YlL|H*a(VD6t7cJA*J`^+`(LVNV#|4BIaU$ zUU^9tTdKzn8~d`bI}TGk%(jaK)qmwq=OImpe{x;ToI77g8vgVI zt3UpxAHS*6iod;EYXAC$yO;~O{kx6#TThxUe$IcxpN*NZ7`5z|>kYh+efeE~okSj1 zy+OwT)@B`#eDymR`A@I8D%IEfDII@hN5n==bp#<#NdDl*2e2Vy;rY_3Jn9_(#WABClZ^c23Y-h=t+-5z!P6a8XSJxvmx} zx1cVzME+xrthjwZkn$IQo52yJP1Jl)dE0cSv^)Nha+5Q=q^W_BTmwJ%W}ulD zI^STn@tKMth=mcP7sScO7NrTF35B6oV4w^ukCB>>0;T)4G_W*(w~+Ib()moq&Tmrb zd>?gvKp;`r>3kP+5fm_DAvj9s`{(QlI&1(V?dtz82K?~E&Rd3L0!XIc*^=5e>U6vq z)ykii4oOQxa!$`mr-nLhoG2N3A&tA^wG8he{+%z)hrk023>`Te2U(5QkQ#=R7FKmx z?SG=G*FpV~8?f4cu8sKq!gAB&qTJgME6-WrN~`(6{c7^{=wYx{4cIV{49RLAd>BRi z6ny7P)F#03R_WBDc38muc)$o!=a45J^oL=`>BOrDGJy)7AT!{q2jcymhNvI|qoIzP zD@@hgAks19#KIZ;rK@J8()*8KGv)WE-#L;wJ8ptcg7wXR_k62aQ0LQ9c=}tfe{eUo z{<=;h9r?gi|N0k}{%Uas{mmEk+F$%4`|)?xUw-zlJv7whM{`vk4Q{EP3DvyQ-$1HUH$45}q!wh}?7BH7J;&vR4r;#9y0cls0K?-Bwc{dakD_j$oeDl=8 zQZr#tykNck;2?H^bMJ@CTd)R1B~46W`lt1H7D=*ywiI6oQnZ!SHhJarOJW_bF@k`F zr$EGn3)K}b6t3#ZQ$%ApOQ5f0VtX7g%d4!@IT|+j@%avDQyL4w2TrGbvG&_j{ov@A z+YNqjPx(U!OvOq|N34#!95qpTglvYHx*ns$$n@+*Z2k9bP{zMpQY5zO1I>tiAJ=?| zu<{Ro{PCT(tXBSzpPF2!LlBIgHWkMWnPTvvx!Y(wH=@MLFl8VZLEQ5~ zX}&;;rPZ()hvUt%;115{4Fijj^6fRO&$0eNNJf~8Fc~Ba$+>Sy?{EcXB1m|6N%r1t z&;#XHP_ZSLjkz8eO!Zv4zo@rtk!uXgin)@184E!cD892_zdKSskZiVI2daL?Ugz1F zs(e=4QoInv137@~C@LfasdLD60~+0Qc(_&itHBj7tar9l_v4?rf$4^lQ;_Tp z+eEE@L^99m=2QMWfHA{0ha(UIXMxWh4%~0OdbF&6 zTXzfn`bdI*)4zYQpwmgyD}>2Ue5u--j7X?|`B)qO^?u*?vdzEVzncYkXZkmf>0i28 z*W)Yi`S*QWGx%N$q`d7^{`8d*`|%oBo)wlg-s}b}&|Son-P~PQrx9$Ch~v<3Sp&t3 ztOB&Q6OK3y>at=qCoCOk2b+{Wh&bSX>wT{$-aRbEl1OzuS7LUgKVwUw>+40V$3#-< zH-(AnSz>^`Fc24tPgx+gCd~E-F=JEH&L^_q%4bdpu8BqD1kpwkCZ(E9D~!-N;#(3h z{E-PV^dR#MeQ%@qZQpNkI$W>pTwhsr-L8#sWC~(a{5WiCu_p~%=9QYiQ!mVaM7AEW zC9T=svQ~lk40MpJ?65{yK9U{oln9c$9aMrql6f#A2*n>SW}M`ji$v^LVdWsD6B9-^ zqIxKw$$C(_3gcWbM=VU8uV1dj8?!`+CjpUw0YCc_73VJjYp$*!RKCBez_RoGXa9xJN|ENWd-NR)7mLMxZ>``;CzWl=8xajO^`JP}-({zA5AZo??me znQP*i!VPmPg+oKKR%^T{FASqoZx)v^4e2~auvwkwrXdq-Xxv7H32q+m&05qTsJ)db zg^-Ad6O%%W;G^RJv8QQlNmy4W7wL~}6z411Q3l!X$Jmwb5KJA=(^J>|eAlUEKiUkt zI^XlIZmKU+*9d+4F2##~qrZ5=SKCRxzJiXge{&rDr6+W=mG4^2@XxxbwGF+t^9%4+a8 zXWc`h5LlF^SqgyDehmcy13X{p3+Txl5JYJRjbpF|p)RL?&ot*b!YucqXgHMr%jZfR z{|t9^CHay7R&mZYz`NuB;z@@I6vw|D_NzzW-+6S0ALK6&Y*+J&UPd@I$J8I9G>qmU z@DlZLg|Xspbs0Dz5(9g=l@?gY+B!y5R*!@C?cx*6=xW2NK8 z)~4g<(bC7 zcj-mpz2fAOq7r)>7l^AqCp-0L@ZzWvC{SND!+z#}D@djzU$9?1|9#H-x#shiPqgfh z?}Pq$0n#s?$UmCq=r1n@{c;AiuOK^U5^RK;g}Er5IN7|UzBP01g608Je$k9Kw(@D# z5F>>4UXuhSQrftsZ!X%_Y!U|*reW9d5rw9MfrX|}7xP*#KVWV|)V03x-fVRL$diR7 zqyvqAeDD!xm?x(4SRkI->RKGx{=1Y{bmx+78RYa&uX;qD&1So*_lxwng_% zxdy-;e2y6?ePq`vz5ZvGt~hh})j85D{HcpG7vXq`*0Y@j96ve1RFwZOYhTu$x{h@H zgSfCsoW!{-$+9g^AP<1KBPPgTz)74$z_0&*o{|m8K7G319=?n2lN}()8cH>+wTecP zOZsh%h!O!&g2D@6J$lEW)ZY_NjWKLH`YFj9*7o@i<^p;1m}8m3FZ<1Q#g^RiCpJxx z6Np^zf4n1RJ;$F9ejeI-9TN_Dmmx;PI8fJ5(eI1TSB^O>t$%96E9C-BdFGL_rJ@Ue zHP`R-$6c|8`R5y`DyAyAt4_cW!$K~feqwTmW>yA5M!v%)pvnXnD$XFMTYJyQ=^+Js zF;RR{$U9$3^};|+psI)~c!r66BF!dJ$_F{z8)AKdEtg2dB7*O^$){Y&03<$>SU8tt z6sP5c?#v5SVnVY|pNW2*qp9%@?*)B-SEL01td|^how~0B&s|j&*LvfS@4WwAYrf^h zPWs-R{qo(fHtWXayVgB_aks8(eZ7-!J`zz&aG-EF=I-|UC2}LEYF|Sk$!cGCo05X2{Pr}i> z-MV})=7_I@wP!vbsR|~s$zS{o_Q_$LhBel}7-}oLg5r7|wEo6M&`+uJkr`siSgy4Y zQpAX8?$ep2+_ok*v@ul`rO!@(>PU;!-$3%Z^&zhWh^OZj6ak28MHJhh%`=S1c*1%f zqJR~E`^UV;1IMb`-MDbLrghchnofOe#7^v57wb9pf(uZ^DVC-C_|*_NeP?EBu8HIk z1B^uqep$1%4;UAAOCl|XxG0eM=;4I9>B6PdzJQ44cS1}Y!94%WvZ#iCIZ-m9wFnWH z>2Fz}IT9+yycNbSRQomG#uSz<9VtK+6ym~w%Nsc8!P!DSz~Gl_dvSYSmfh}D^B1;D zbJ$_oKDb0Mikx{7BPN*o8Rx`b-Z27g0^D19Z`D%SYly}xo=f_QTp)@^ zl3~tJV>Af4a@DGylbXd>or4|+FyP(YQ%PSbED@s)veg(M=TsrT$tSTf^;6RW*$Jw- zn7h^ePSGuCcLc`qHcf?}OXcU4jHSvaCyRg~2&wIwZ7ty<*Il%Knt$%}{`=7wwKSCx z0jI$m7I~C&)LPLs?;no#9~|$&=f3|{x4Y?k@UxE94@p)z^rI9k=O6Wf14Z6U975hFHgDPG5OWfTz6uT+^KuE_k

    VW*l?reX2I9#WTr~)Re1(BWWo*N%ju*pwS`yU>A-e8U=Oe>eFxbj8JN%mPd@}Yng zn-Ix8s+wTxu%OomX)-Db3bF{|Tm^_t$9jh|7vB^QBl_uoPK)EOe~tggKd;5LCjN_C zrTv@%w?EkT^I+3=g?Ry1d59gyajEAb#M%>4Yf%6EdAw7T!i&IMS+b;d;L7HE2_{1f z=|6iO|01?D?Unlz1qM6;`4lih*z^F@clGl)Ls+R=TYK)q3mf%8%C5_l9N&l+$7m{xGoNr zEoF{5Jyn9y?>45cn`(}aSUN0=GpIpL)0k1o7`+pUg5vRkM+8_vQa*}t*AYP=Wsy5p z3lOoV_n#wSswN884^ykjMcnheuRGw`A?Q!bs=ls&Pds49d>Qxr_r8X)9oF`qi|cxq z$XUDZYnO5YtN1Qd7S@eJDtZ6>lASdf{Uj$u6mKPzi~w`#-hL;dn0QdFqmwi0F6BMu zK`AIQt)E<_ieMy7zbK#p^y|$yWA>N zuEX9Xmlp=7a=}RVLGH~UC*2o2bw?`L0kV#Pz`HshjW9>;5J=Jldoc zFxK~3-P@I4amP;o`!cLy2FF12p|#(7U3o&B zJYy1Bf*t*a4$(Z^4Qhd-%{>^xoJFL5F)ipt;IBs8h^>7c*-QM^4 ziTln>|9e=U@$WU@+0Pm@^e<~*m|wJiP%lv(zXYlYf>W1A^=C{)l>Iw%9-?Ah7W+ls z`P6|i$80)w>~VI+SWOx1YzT!G-5B^;L#%UVKUkEcn`iR(J-cDq?r%=S?M%7@L9fqpEApLH0Ur7_vr+v$$YebQk)X=glF&WGX#P!qxY z0BN}x?yleGq-K3Ccg)Y0ljP!Bv!DH1lRcM%wnJZQ>T3pPwp%GqJu!~F^tx8o8&T$z z~Fuv^xo7%+5QZIRY zWBGQBFd8Dd6Ezh0h2<>n@fE&9po*OJ;_xcq2oah?Y1?@Y(;szI2UTu$a z(YEav_dbY6&T(%u`n=n5h{4cz+zvR>8>WSyH6~@j&($&5?T;a~n0JwX{)SdZ0y2cl zDW$vW0UwMmyVA=`1*+wQR>uuGwMu;XP`-=1xQnECnZTN*Lb!Yz?*nEsDjT*J;T%*G z5@t3*7H^42-L(Jg!c|2#+C99{CLU2LkAPV4e4Nu=^%$Zy&O^8I^75o{E!(Q4&VxgD zRX9CY_Da1uQzI+?zI2c}Qx+s0APG8Dph zQtA5+2q;Xv-^nNZ5BG6%``j~A0&B(5(m|mtc`h*zA{v+NEX7Xoa?89Y_~a;Y0u1?$rymgW44E$;isF9XORHKbHw)qt4hp$=IPlK&dmNGvPzhczJDnqdiS;vW6l{VK0og4TIabC z2lQN#i-2dm7q58Kq>tnJdY;yIEN?#ZO-FF6b0R~`RQ;-540@EfwoN<(d5QKZY~5qM zdZ}YVFRssI%baWIn<$9BT#*k`J6sVWzwZY~TySJ{12K;6_kxBZIK)+4A~>+FmV5oOS} zy_@OiXIKpDy7;BFaLe)iXea8p?>_mh>o@2jYJcEb`#cGMIIH-%Z@~Xa_vAw-9oe1& z)zNT&YaSi37cUs{$meNDA9#^-wKyX0j724@5~6Ge>Uhi>k`)$tdOyKoRd+t9;)N>A z&2_FT<2(T|f!b~faK_G6hQA!F|q2;$k5{F?aLz+u1*6?P`anI_sr`;* z6M(euDr#P;$2ji&erg*Ri>$G(OLV-J#-}0YaI-hn%?^GO;fGB(bJ5?;+l~tEda}@5 zvrjlhfmEJ*szP_$1V$T;B=X+ziv+xX85av4;PyTD@renoD~KAB*H{;GWV`C_EUE%z zu_A&8k<->7*(TU^^j%&?`Sc;y*a~nBSXe=rmZ5v9xR+Ibd7et&V||^DrN8Xt)y}6Z zvQYa2bAH`q%P6GwVRA;ziSXTljv1{#lA?KYWxH^dTn3je#{9@9+Glz+rqXGD;G;yz z(_6j<@5SLhM4zO$TPG&ICFV~H>O%FNor*aKUQf0vfOpIj=cCMch#l4)9KaHGI}b%e zHqteapPyh3?+NHZ!7J#5-rvSKa`0d3Up^7(m6;mJ65 zjswbpvLLm%`qJiSDTTYB&B8r@!7`LBsljtWS|OR@XWAf4&55hRmu15!6v6iDUk~@= z5L^4t%ei_26+bK$-zC(F0BL+sQo6MsIQrL8=Z&2X!WZmyD)c=65i7~TF4ljEjgGjF zh|}NgL!6F2_BL?_Ey{krdL6=y9fR%WC&!?$Wo5xobip`&k2dfVGc6T=r%M^6?y1^N zGt|}0?Nz?bi^;AH)QIQ(rNqwfRkj=RsLADxXHL-w#imSe^&$i<#vIRc|k$;6dmX zAw7S3>}*k2m9HGUF_m3N6c6KqG0gLlbV*{-qQ*@^6qB)NO}GSwCrfYQ6^xBrlgM&4 zbGVa0d*b-K?$nsd=o8Obg4(wNc>q;<%_}B(aW~lSf#0pM`NcJVj$#6F+LbfdnB!HK z1cVh**#&glJaG_YS#V0|N%+EA7u0m8UPnl3J%zFjDAw^rYfkIJC9-m-I}vuuB94e- znpLliR0}@0TvD_jjKD`G-;SX^M`lYm7t;}J0;M3eXilDN^}KpJ7X00MGRzxhJ$J3o z5e_^Ce8wKbv>SAPF6$usy?+slEAEE8aLs9TNR;i+ExeLilBiC+P<7#gN=mbOMrHSr z*6|nAmFI7r(W~+B`u+O#8dQJS*iji#DI>ZQBh-i^X)#KmB}8izwC|IX<-)wPNjCm-jVU~Z zsJksp;%qK=G-oT27+~bOyQoYcB$UN@B|_>+i-c7Xa`>d)o6QV z7~JWQpGOLR4X*vC_9`##6RFp3Hrz))xnIm1?g~?>c;TvUrtbpf8v5RwKEy`T#S7#m zy%gkAoYuU5-#molNNyep0Yyf*erIMy&`^@LF4V$LCLP{CmlAc(GjXq9LoSfcOKl8W zP833wJ%Dc=XFKub#&JFs{hs=@-ykn)t>Fs4a=6!jT;R5%_NiKE-ShKo;!GbRqq*KE zKsKs*bH(>_3f>rAL~m>R`?hsuw{TUQrnI8~Jc7D#^Gx|Ja5O}eX3FB5$I89h(oEe4 zz$o33iEfQ8!7`q0T!$IzSS|2ju(o)Pf5y1I@wpGWmVU#c{f_&uxRVn3|CgJ_G2)Uh z@hhNzPUSVPur2!-k;c0p-QwyJOFy5;D5bl2M^X_Hm4>@>`$c8x(ZgyT$-<*2;qpry zLM}EOxaXwtBSN7Z)q2~qTLq6_9UzWb8KXnSY669Dn2D^6#q$$Tuud*DxXG^MpVV zt8=-}f%6~(2Wm84nvb)^`8`QK$36~~pqSfqlZ()}{OcS?@`fQB@+vz2bAR8VcX!qSvpF(e&D&-{#``@vPzNSDh7qj&-kj zRdZ;CL(GjS+6z}jK~?EyZ_q|3mM-7SFXY9$Q0!v$F}e_2f|OGx_!E>Nb>w5XJr&*i zV>f})zMh>^3b(UE=F0SQ(+sJSPHIXroTGq1e~wUB#cR_SiuGPNxruy>a=~$ZgkA$> zMpUrNQZkoZ&&|pDIX8mD#shPIItX@H?*&?TMAuM;IU9UeT}QbLI45N}?&&a0RFMOJZJ1wB^RPra zLdj)}_}D%^e_+l*;q%&#JDXmh5PJmIuVN_PmcWp{k^`u^)+I z{$=bSyMko) zp&Cy1>3oLwC72Y>_JxY(jb7|1-mOI_8V*~lGi4v!z{))elsG(p@N4~tW^aho3X)v7 zAxMSsn;V`@mt=7`5p5SSb4Nr$un;}eOHspAH)JA{7;ymKm~Wl_f52l9N8IzY#52&K z{lR<=uS}n&Y@+jwY4t`_Oc$nI;@%;3JzXg7kaM%Z9E(p(heGi&18Xt^H+Ljcjxt6j zc)x$H9a$pJIiR_JI$bdiOg38Eo#xD@qCvelN?_bXN|uy93r|+u1L+U{#ePx+FkqJQ z#sC$D>;?eb-Nu~>JO}vPb&&sWIFA`vBnEqGd;)3I?Pp5We~15gN?RW>Ddg1u_aae0 z7oWA6)9kr%1af)Hh>FIJ#V{`^zv4nR3FQHUKCk(xJR3}Z3lhXboXS2Yd8vN5);+gp z`_u2I$9~62lcu``_U{Aj`FZc~?)qPW-}7 zndI*N9{X>wjj?QrZ&~a7#4?cJ;JOYa*FV93is8_geOLCgu1uFHY zuHaLUBD|DjQgipKSu@A^Y^FQZ36&m6`NCI!JX!B!q3;zBLucQC7o2DG%ES$q%LmJ;O%PLa??f3OS0ODvm-5uH315!Ra&;e07XE$zqXW=k(3Jd5nk5J zy@+=PJcDg~ARg4jtxKn#kEviX+1U=MTi?OlL(MV}pEChvDrVIo61E4T6ilMVz^8P8 zxlS5$f2b{;7np%_fE6Q-1pg0vITI7eVd@x9;%c8TFFM(bcF1oqBm_Wg{2Qj`1%k#+ zf%a4L-qx_Jx3^4`+u+1%n8oXm*sttVyc%ES1hSS;*?uwFc z&5))sB{{##igAPY$ja;*|E!)VxF1>5+=W<^>Tv)7Zyo&D~QjCI6Fcz zHix%%YP=``vZ50x@qS7OxZbw#M>cY5e?^zLScg|8C9l^D#PO?gI1g zhSA~oo&`wMaRthH9%06M`y}BvUjc~f>q#-jK~SQ!WLxu_{4zwu#+Ch zf@9Ay<3qV1-X=`=OWc1WBA@;H_H|pfSRD2p5zwD_JU3unI4`DIAU zZ!9PwJfT4Dsphz5S|eIWm&kdie_VIwJPGpJ(s6L12+McXJ z2Rh#s7I~|IINuLTjjtbF3?kVZd|41;vZD8~FfWYz<@a54f3+EITK&mGd9Zy)v;pMU ze8)yDWsdrGC=*OYi?N?MkyStycSJM&aFj`WU@wvCmJZSm0ddU5^EC35e;}w`3nd9;}h$VAEnm% z=wBo6qKrtA=L9;B>c<>GEDQFE}Q@twSlf9fFqxX^YnD>Q%j z@7{x)4iwUS%p{|aP)20>G&a5qa~0h^>*vf4-rGglAX!X-d&FEv#@BQ&pzt(?MzJ{Y zwxe$$A|i{o`n|Fz$fF18ySujl(x*K(aRst5K4j>?x&kTy9f;xHf5CksFc0YT{HU8g z(f-%Z+W-})Pi@K1@tHrV7k(B?kH!*~YU`bvE$$pxb}PP(&-^?cMKpf#GFE$O+f3?i z$Ln^$Hmd-7JWY-Ge}%aX=$Ysm#N`jYfZor@#gFdCAGnQybNzEi-p?2?D&?zxX)X~_ z-(k_dxdLOBILsR0Zd!8x(wb3veNEzP(YyKZNr5@!3@xwOYMR=w^wc`EehXJ%p{~CS zX`FgHbK}eQ1$2{aS&J~??&@S*vKn2*sl@1FPl!*{yvf8U|!ovy15eiZ5n@I$_D zQKu!OY5SH@w&Bh*@N?Dh+U_R8LVX7h?VIl~BcL&PxeqcA``QTg8RquXxyT2W za^c9DDeLjN_sF0(rC*?4I{>?#J}x)|#A**{s4C*#m--daa;N-xTv&su_Y z4b({d%X>pl#Q`_hyPx;=^%Ye@OT$xNf~Pk}nNZo)J17E)$UYP03ioJ2e9*XMor zIfUJ4L*Z$T98O`MBZ(ZEJx>S^XdN>~s$jV-GncT_&uarO=`N{X)LF#0p_oau>RULX5Fk0-_j4+f)Q#6Od|>T~z4S%^b1HQFfm1 zrA65E@p)1|O)tuv+so6eziG?< zf10!M7j{+r#={Lc6^jn~XMg+JAN@IhYZ&)7jjGhmf~}o8mvp|^G2h8ff!6<0rcaA! z$Ehuz%$BdQsFH{^6Oe^(l-)mljfmKxV}6KEY#NW(>sSu2VgC75v?Z+yP|YcUf&{7Q zIT99fFItGpJQaPhgGWQ}@EN zVv}xByR+Yqu|S{MVl;0)T*9aW;v<&9b3_|&*sq)t#T<7H|M_QMp0{{rAoLlqe+e{V zGnp|#g%w405qhE_UTT|uYQ^sM1dfoz12^e^Z{E>*`i^Mc32P_xY|Uuh1dukHgqW`m zDLVnC?lqLBEr+yOC*O~#Yld|I8Ig0mfP;UsxpdZp}?ETbN|1!Vi93Nb? zD}S!bb+6-j+5KAI9rC0;)OtG2f7es;ZcnLPZcB{uc2(x>sx`ydq!NO~A^}<6v$`Am zHP-ycyHb{+^JqtDZPs~aretMLVAdt%QoC7>ahSrKNQMnULelJ*sN7E3E~x|XLAG8N zr$oi5*X8G1HW6vNB}4qK{iIigep)xw(JoiL!uX2Gx@p)B`xEmuHEIm9e`~$@FoDN! zV%pD%X?q{a4#^?dVSJdJ_$oP>Irf=65oCR$UVQA}o#aUO#SxX27L7@i850qEtbvMB zx#TD2lb3i(P%mCp%n_B3m&s&0DlorN3iK}pwuHs=1n5KF^cC}dUu&FER?aZz(sSCX zI*_QL91pQ2Cn77Q$i#+Ef1h$k2G5Rq8maFm$_UU6F!TqeMj1~)GzZRDTJ8Mra z<|gp%gs3W^N|L{hb?gl9L1zvOPleb$4r3?obPs_oU-Wq;+-pVs*SRIR z_{DWaf8Ex8Z!F9He*b==Mu+IMz#eMy&r(@5+wvDTwLQqtpXN+|e`Y)JaD0yFIEeYh zQHX_!ebiVaPsIyQ+l+u(@^h@6I_G<|g722jf9Cu9I^Wm$;a|=--qAlX71oO8(DJQU zH#>fbo*QM+jj$(17mZZ?m;KS$jaN5Yua2jT#4C{E%+I3qvVZvuD~uJ~+ICTB%>TX5 z=Qi-(pU!{g8~Y;Yf4qZ#yOYj+4h3r(y$?~cgjPFJiv{Mqan7k+TCbuiJy<`yLK*7u z){Y`4SrP*Ttk`FMHa~Nv_xNx|n{{5J7I_)RdjiPsj5x`Cxdrr%cK$Y7+;@n$&3;(Ctx1YWu8AihPp96wJX z8hv3bE|eb%f0i3x7X45<{}xeT+K%4vCwONlk{e@2KV@{n!yJeKm(<6!cpo#=eH?Uu ze10yPt45KI$*(oGy1L!JE>!d&N{5Qhhgmebe@xXDm?RKqWu6=M+9lY71Ck{vedMxSf#!Penv=?U}bv&c%UUZgI+?*Qg)%-Z&13k|Yr96>uMX zKfM}X;4v99$Q;Xfl|BetkK1^`d^P$2fy2P2_p(g3WrK`n7uo}dnq$6=ufJ{In4$lR z`V^aBf4Ij(NA&l2VLW{dWY>E+$i#7|H(-t8UiSn?Uhi(;M{9f> zzSpKdn5r1@I{liW9>QSLrix>&;@@At7h8O%``QZ+Z%ztiy|unkj|J=D{2avjaOZq4 zC+)kirbgR-p;iV3g|8<$nDj~Ugh5FHYd=W}e~%MUx=e%cVMWg%gCrN6KJyPXpdf$>0efH+8d|x#Q1NSc4A}<1yo_xofQ{3Nyqvz&v`{qbv zf3)3NXVT~EcDxzKj)-1~ybHDh*=7aRr6lZ8RMvNf+SmBJ`nlmh zShR+FVC zn)P*~>KD(F|Ej_FoD$tDsNc_b*EoRvfAnMZfBLnoTc)HRJ8?o0hb9V~ROz7}yW-JW zAl~a?Of^OUmRyTuRU4kQu|u!#XN^n$_WeabytKNDT`HMBS2ms0;uSvIoDlWpK)3a3 z6qW8mHx(~b2*)YE?g>%CQ<=3@CicXAaY=z#r`RRs{`Y6R6t17K2qV)byv#l3Y+wQ2>!<`zj2=Aane_z+B776M5cWRyfYn;^Ehg;nBr~cFL z^$^=0pN>70kWY#A3o&mVo@O8`AC!9r5g34aiU-t>t3)PBL1a)b&T0~aShVjkV$OZK zd9UwKz{d^s*J};@ zs++}rI81S$YrJrMhr3euP{#NG`ykNu+b|c#<7AkNl*sy>iuTBu)OW$Ue^H&NU#DLn z941%0V21PS&8d(flj1x(GYWfix+^OBAzwyk=^jB81rd`k5Ru*MxRnt+3zv}C=MsA{ zmhOZmi<+LCnd z#nzH(o#^snj;7_~6tQts%qQ)xFx$W;-)c0KnipGOZz~@fw*pyBe`tw)pv#y-$>6Pq zxnNIIutt= z-~;i=%rQfHieqnAN`F_D!MQD2)>xS1bjHjkH9v=EI489{hP6z?95u6}6-ZMM$&TB? z8jBKh@fZz$1fi8jl=t|e_IBsaEuLQH6|fdhi9fTz7W6NIk4ljmQ}#i6g7X^eEF9L*C!ngI zeuAnhhW>KLzGvkXU*BVNKBb&EO3aO~{ZXD`f0wF`xeO&gPML>&mrvZA%uel8Ry;Vr zmI;a7ms&&=c$DPxhjrSxw6;cBk`z2;sTY-B=qEU$^+8q3*KIZSAkN9|t(tDj81Qb( z-M4W2zHwzMvb-EmA$Evr3x_6lEQdJe4#yTSpq$g|t ze?#1OkG=J7zQMIO+&)eyVGhmeEu^g-A>Tp839FLtF)6VR66${?{@oHP&~xw=$~(ej z&DI;nJSa*cMB;5e-xGmay!=d|eVJYeHs+$Z$8@dkWZoYM_So)xH`~kPEEfG-97wns z2ZSW6)ECH+=S1MW#KP5X88}MyJfu@ce@z)xh{M79PVgu9s9#r&hrQ|JYu~ugVOcPW@9H%sO>0@!n{|rtl}V>m`VAaJw2~U zT^(L2_{Q?Tyf;2zL2G!9?y3^|QIS*W>)zd7^*t4K45!(2;;N$uoHOV2=}jLBe^1rZ z7lC}k^3|)mZ>N@?Q>br#CdrFZ#5={GgYAssGyBWDwlEjEZg%w@*`WP`+~~U>9!Yup zsoUSq**TH_+dbT(7Pt~e<7@d;f=wxO1*Ol?_t^BjZD4VMocuBbDUY*_8>P_BjaB(y z-Z%b~D(oT0z3lSVGH4h1rLKaLf235{PcK_Ob!F_bi}}4gVcy4o&dGmz2QipU?Q_b>Xy32L?cVWoxzqiTdW_gbiDY>w7_qG>73{RiPHBbt1MH0=D+={Xa0cNc?+4`k@l&$N#9}E0fjHi6 zUbmw_2mz`InA1EhDBkyoMUqZH$$KcL`tvaGm1Eh*zxkj05_^7Led%5!`PwgUd$oVI zGuZn9r*p&5EOaluAKxA0e@NX|Hs1a0!JTWT`Z-bm^Tni&#XphH07?o6%KbgqE+Lal zoXT4FbVFW)s@f@!mozRZf!24V?@x<$LhF(^s>8lx6i(bZSCSE>1S}XK+2@i|5mAXe zfTC{mmQiR}FpF3DmZL5sqlDR!S!{{4NsaxD_1s3ISH$BhNubYre-%TV(g17pId*r~ zyjFR|A>uwV;CdEscm(V#fVq}i?QDnrz%}>s#?`F;8b|IR8}J(SwKqw1W(PT03H9Z$ z&l7{fW#zi`_WQy+V?UNEa1?(`@^b3jpW3CGytiOZ+mLfd#YaMBs;X4YUYS~?bzN*E=Db~4pgn|b0vdO`vrtn5 zdjt~AD{?0}U=G6!sc^{xxbMhA0y269Q6KaAjd{EE%0->>*WxGs6Bz#qSc)F!GT6Sm zKd^dr@mJkvoY#Ljhrw>Zc=TL`9y_kJ&90i$@s0?cr@0Jwe_H?hy_g(U@iHbB$#m~9 z1WJ`I*~?d|inMmiWpeH$k8{^i_mfNgXAJ1bhiCs}lf~YeI`jHio~pAa+g${&)n0k` zuzy%X7L;OKEszgTKuH&*^$9c+_+Spj+~FZuVl84a7u#qXFJ`_-25S^UI- z$bHC6?3ar*f0u?lIkhzv$o(;0)Sa}R5u3t~lbKqjyUpuOqwLRK(m(t8s#U|TTB3jCGOO$R>OZiQy>`_x*k^U6Sni*zvlnzzXE|z_ zV16;T5)drRvBr~%X8UF*&ck6%?&eFTiri7mk1v8Df1VO^jeMI}uV7A%ui_Fbf!LfF zPAunl4_}UOSL+AV2SUDoqw&Cyrk@4K7f(u~Ox09lP(_0^I6_;pc8!PJ5!H935ThVH zP{qbqEWmnsbViJx;<`sn284<}pXiEk3`Ye^MU!jY{AL?(dc=L+Y;tK47jWzOHuPk_ z_6Ir!f3>a){e^d|bB>px9(nB_e8V&qYP39x_5P5?E=eb=B?{^bJ2|Rh9zm*eqadcn0=oA4ceB36r8!hR&L6ea3z)#c^eIT+SjH0uL1wHJ8KJhi zfBJYHqn0*mK?>jO>$^weLn0rkL{iBY<{Jue#l!zS@9vi-aUSo6-2R^&iTs9r{^2=q zgP%KVkecZi%sXr$7ffmA1+x5@LPJ45f4xrj{BcUFj|uQG6KpD)&oSylmF9lRlnC61 zK=%J`9*vi;?sdM*+;mMo{QaS*i-i!#cf`Nj#-3F8ck!&+p;9Y{EjQssK83Wirigp~ z>pb%ETKD_S_x>AwGTmqKM~m#tul?-jaE%Q$pAqMq-)YSg%<-#VwSg;dK~>kDe@|De zFn(^}3u_)B>I>ZejwyR=Yrm$Dwx7va@@4ZdcLg>$UB;_Z*+g0Tx+P4nOY-#g@WGhy zUH9<%uGb^UYcU~mc?Z~YK=C;U={N+r&oSx=SAO0(L{y0@S{k11nY)d5Q^i9UtN>Ju zjd>mHy$xwxG0iC}2^?&nfSfYbfBgPT0J|au@l3P(VxE!ZE-UPuK(lBL(EmICc>a1k zJuLTp=9=DZO<`dP7m|6fEe}D8eL`7ERl5(Nsc(eR8Jie?Cvnf3?r*zyKF7CX;SR1vWPwe-hCUn483(J~hTi zcl}=XkM9Ni-U|6Ei2u0++bY%C9BH{=qV(a+7Lb>y-;*W^i00rZf8=ayO`jw7QsDXe zXI!G<6socDYPulit51WKct=4IIxg)t9?oMa(wDl7YlI<)clK-C4*ed#Ay;-Q79;!J z8;$DS^PP%`V?2GqcFx_blZd;9wrQ(V8Evhqk5AM;taJ5u&qJ~T*R}bBU%@Ku%k2{t z7@DVRJQ2w9=TAJPf0Q>~iaqE5ac)BhYGY}P`Kep-j4eGi;Qaym{%fdf=O8AMABWx} zGp+9hRqZ0jQItNP*&3TnrHY91o*l`A(FpQQG#|=wl*WP15eGU}h)sE4_3y=c20miA zLcJ~8^X$-`B4-k9^MYY~aN}DI_x)h8?|)YA5yx~8|8z{xEBKioqtm>8+}p+(_k`|)1PYwu zHooMyR)4Q^avj&@Ppasn!u&-QNSY@Y7QIeQj| zatVSh;KfX+yQZK&Q+-Finm|gD@GJr%kRuio;VA0h{{LKep3|*CG*c?q+fuQBgSX0k zthcfS^4|11zqPo|#t-kUSR~iIm&a#t{^Yy;U~EbsZ9?Ctvh>~lXY7;ZlkG2#A@0W9pwEF~09|m=`#XMYRA2`Z_Wp4s5-BN9fFU zfBX}2lKc2{jCQ&1@P0sja#v%vvs+DS;3ij;QOM}UZQer&4e{M0pe~6wov>}eG)HP1R+hxI&KMKIT z*P1qn)4bPp9~*hAo&+>1`+s8|T;1R4+}~U9fp+?3WKEx5G3P=A?{I%QqH5vWiZ=e` z3BK>=+EzEQ0P)~E($iQxiaM@}`2_gX*x2NLVv@<_^By+8Z9h1;UTe(H9u-P>f2{Db z0q5VCuRk~(>pfM-^A$9KW^%15gWQZBKgu`k#XP#dj9%c1wPceCaxS)oFuqHdNHZiN z47R|w^O`n32_G7t6uR> zA6s1q-qTt9=nCsTlX4RcO`COw$KWeG z*!y*(3J{yo1U<%eF?CcFQ}NW=vJvRl(~WZ^N}?YYJ{Fg8OIySSn^bfGMg5?G_k9~D zG>CJCtA9D*mv0=*t0oToe^EQt*zSAexQ#V0&g$5dF@k+-k28}9D<;w}`mq)v1?58)k2H>rC)&wq2(f3)sXHfY)Je&ITQ z`&0wY`hXJT;Rb^?aU=2 z)Avyj0~Ns?yqXXI9Dnd5gRV|LuW54YL-K1b2d()aua0mmV=9WR{!WRU%qAkD42k+Z z7iqil>Xu9&+>cPLe~*!YPpRTAfuPmz#|sqdb~WF(l-_lke@G;04$#oG^gn=>5Mq5? z{B^81zv8NS_M4Bae+C_AS9C39Lb4!PdHZ+Fw`kq7yLG$ruy(3#gR?s9+dSh~duW3T znR#IUo>xlb^0@NqBay@BuWMpFoM9%9(YVBTY>fT%Oc{>#{Xycxur~=1#1{Y7d-(?} zF=|j+piGUae}TLuiQvSM6(_L5i3Xc58HCD!Znpv2%E>w`1pV9y;*nS zO40@TgZzLn*x<*=K_WFlN(e08jIdKeUdN(02`gG|X-B6z^y0!zdNS^zt9t^%Ii_yMjsxH~+~)%JSB-kq9xD0;F2VkdexR$p z^mO6GyyKizM#r_WebIBQ!76wqI_^s3z>(cfe_8N}lOH?M@M56Wr(mp|K2MIC`guSG zLu<>GaD&Ryj07Zq}56;P@dDIU%(JO1_LbDs%EAFPG@A$2cLI8~}AFj<3U2Isz?LL}xZ?xPq#f zfBXK`QaNWl4}u`DYAQv@qCV?In((7wELrUP%i!!^Yo)l~HFjt`ry#H7f8k#vCN}fZ zx6W;HNu%yKlknO9nO}{js`U^bgH1Rtvw?b7pSnNVcziGQFee7?UkIQtvk`fJP?9^@8YqeX;ge|}b`-*KEwKy$$WeFfbfA$%1`j0-u(WxXi6 zmm))5u0%FG<2MeAEC1SskM%sJ{>q1D(qLV8?&G;GTz2O%+3Yp(nX6#5o>_z0tQ?~r z%z7=wr(0g@?k{{^;fhASeN-EH&RuyjIsfkQj++bl4bA!4_(iav<;i49HUrcGqEUbSp5$>( z`J7z(gxY)z9yiY)R#p(@j2+~;f0BFfPDphq2Ie}-D+dx{%id^kavY`vacj8^mipwy zrE|*45la$c^W$_c!D-B-SL0B--pZF7}H{eZVRmosRuuu8=DsE4>qc z35xVU`X`sk&gi7R*f`m&eD&(r&>!vIEa8pSlPBN5chVaDQWp)srE-^W02nFMYsU7y zXwr)Yr*J9%I5^o$QQOOle@5BQwnxnRnU`+aU|ssuma_o!4@&Q%br-oyu|IoM;y%Xk zJ=?HsZwJxFCykD@%{~dn>%0+NJ1f|mURVdu6gWwzB4mAoLD-3?`n3JJ zX6pI4Ji*W(ckNa#hD;3(gjP9^GH>6@!i$~b5%pVgjg*eY=5O%Pe>7{(_Zn{d3nd36 z!q~?2rez3dA1H^d{dQDBCSYj|i7wCa>ykR~UJT+*>O|?gM>k`l9=wPrHdU1kkr)xB z8ocFq-}x@ecX13NZ>+!0_K(YYFf(>zA}+Dee~KqxyS4nZPgkBmD_lvJsEoK@bcFlJ z*1q@9g){~&i5TU&e;D-?al7wck$nt`as!GFS^C8zgO9;;e1nB42C0dY|EB7}qh;Td z4Ik#W9KXO)pvVsShmc*bP=5HMzjn|En}^3;A@Y{Cs$B^&X72ffgl^ zk^$<-qCE+@J`&uQB`*Ew?ONYmYBc5wX%h$-Qz>hGAg3EAe-YYKW$CsK-w!QiwCJb{ z2?revi5k7Mv|tb`Nwu_BUF1^law7T8^jX;zM2#MvTD=wWK>lyiG=7>^&*Cj9x`-1F z6^YJqi&j|6j;}5@sC$fobuh3mtOlWUq7W51O> zICTd8TU%rxf72nx4myf|fKF^OLfhJg!x)e!dH``SWT3~b!{DKX`+P{L{q_0Kb5cf> z?GR}ZHU1;OfDH+&a97?q5V!*}Qc%`JxAaPoI8kwL*%Q9`hm9?e>Drx+Qw*T;< zyaGBYI2u-$STY7Ee`D;psImG$C4X2)8_tMuTMOiF3~;|2fckvtLZAAGXU@>3Z3zP6 zlVIF>s}n;WP%49m?~yKA-%I3@U<(idMKDig_l9ai3H0N(6r3Z*N56_ziW(uPfuII^ zRU!u!Vuw5;M-#5PHbl8dWqXwXw;TNS(*tY;TPVbDbE9+0e{bb8ed%)EbK|CL{B(aV zN$%^f!EZEYukvEQb7?e2y8;v&Zsw##>Qbme_((40B20^KJfoa+mxdtFM&QDC*HxNB zhQ8ul@YVt;V>Se&MQj-sh*xVvVDKt!sRh+@!ln}ma{jWxI3II%c`ra(1FBN0$6?`V zVYA$yfpdxBj`I|>qD){W2Q%sFa&1{YpZ#s4`5q8;4g^Ffa(<<`@a zKy3w@>VJ-ZfVp<5{@R9RIbskbqtj&gDl5CSD*plgVPp2wxF+r~{*nYu{KHM3@fN## zG-r*~-L-8muQNPG#I~UTqlg`Y~oD;xj-pM6%JGW)$9kxXB=k$GcY_$QxgZ8i_R~&hFk3JC64Em zUe6bIe?GJZr}^jj@U^IeMhE`yVtbi*4C8EIujKF}+J6rqUzTyksoHS&c~#hw_IIPG&1LWYKc?4N+#B?Y96`zP6 zYod4wsdyvE2U-Ti#+Y4$e+cFz7@!R*pfEjof2-;MirY}Uyb8iICY2{n={hPB#u1}< zsuLlTger{L3rfMeTiYQY8hJf`ivb=iP?M4xJ;W>Svj7MAl?>0giGlh(*8B(7=0~43 zdds=E#9JqTvC_*Q_}tss`Y11F<-u=Q(Y73ZB>k~Z62|7zx{E~*UE6sg>g_m=VML*F zf4K@AcABmNnbj4jO@r60(G1_m` zQHa#s)I1i`N%32J@LA9MVROTOF8^)(e<%4BSCU=yim$qqjn36o=gj)a#fW~3JA6fh znZMwH{%w4yuV11Oq_G3)Q9}RGmtEJEyo*3dzm}7L@UIYgdi9gMOV?%uV|z=%n92sS z`gTd8&-fMksQwMkjJVt2Po(gN&|W4-JhM)|f~9a>tp{hU&z$o1o}sqWNrh>SfAQYP zp{V%VGN@^s1w^OenD&X-@%=_`i%8SDVqU1rF*8z35*fU zf34FF`jOZQ?$Ra5KkQJcf+5ihTK7wP43Rfh^j~07U!tPHRDr6xY#NSj$tPr5)`2QN z8Diy=>Lj8{jHlM6Z3_D&-asNIe>&O*r)&Elh6sMC_YF9G#5>+$7qMQ&ITrNZt`tnq zaR9gV{wMe3MF*e=$2aX|anm?%pY16Z&0n17+Q0ZRF1CbDsJurTK{*D8MM7Dh*99vK z`*!k4LAZ5bBc8EGufFN-kKroK;ZXBWZ$yv-o%`_yP5&HUk7Nt!P% z7)FOwKv(GnT`BJ^aL{poPro_;RX*vf%izOS2GBBf6|VE)=p4pR|I1a9gNd1 zGsTQkF?C7xJ#;YE2)IpjDHthjrUDcd zMPBw~SSAB8Kp~(WA3KlnfD81+=5{-K2>18g| z_Kg>`;r3QnzLiEV=vO`##6L9aXB3V#8!6OKLtlTFp*`e&Y>Bd)Mz6{IM)U9+%IPwQ~6~ zJ(avW;0j~Gl=3DoPcC)sMVxr=XoERE3BSx-i1=}(yEaEVYtEm&N#EsD*X63wm0RMu zf8%x2@cy59Gwyf+a6fdq!*!my>nmo<%5`eD$fuL>sLpH@!{e&E%S-EB)bc&~5@f2B z%{7nx*e!;2n^2|Dp5!=u_Q7;wbEk`TSJ7&7_JPYXsleArLJs8SSN!GOcbaCXS7iKS zc$RE9(=IV;XfH9G4?z>-ajx5BVCJLve+O>l90zBf|9RPmT=#`XKq}~vOp>Q$k{qWh4pghz~$v49m3&ZRW6bbHhS!hPbFnCa(tP_ z$a7`Nr@BLu=Le6;BV(HmrPjkVnM?vuG1mr9NC=1_le5>T&MfDchQm`Pd}h&Oehwbw`Yy}=XC&>AzlLs|4mHa23+k&lhZsu`!P z))wX!$q<)=UT-{dhi|e|76AYTY$u zx%mH_K1{o&99R5P4!>$KodWG7RN@}ol!H>=I~kqI=l3;C`iPO;whrl*NWm`}l7`B7 z)PI1s#&#Ls>q(!H^Lr<{!HJ)RGcU!TZu{Dh;N=tjx%Rl#1NQ}%edFS=vHua# zxtuR~j{BO$l67q&Aq~-2qBgkX%hKKiZ)i#f(u{y83e^2ZovPyQ)R`CLEJeZk0XCa<~~6C2}xOkZ3i z4DA$MbbTc*lPMKZf9hg>TZhrZv*b2w?8f)cnci@|NdmaP!M)2dutY5;JkL7eN{&EP zBh?ke=y!dO^X}J|!20<8`C z`4Z|IER8kzV+_7-c;7xy$9E6t7h9Y0*~AdvbXW@H-M`KQfARSoQuC#xLJKld)d)H@ zLAw;$*nQA)NFeMhnQ{W1)8oK>1UdM6mV+;-&>rctS}T_z{b&D#TD}yjQ9!hbF`U3_ z=OI<`X|Fk%gCO(o=Pi!IVDx}YV)P*4Wb%pqngtmDIkp=O)`{K3|Gg7}v5PkjRLncS zaV=tfk$Kx_e_uJ!?vnMV`ZDodgQu=tq%o60NMQ2x{`vKU{7SE%s~7YkgX+g0+VMeh z>v}Q8v-`U@=ZVF~%Bw%G1eeHpOX7r#?hoGPNsi&X&^Hxf6^Z?txE{ z|L5$Fyl}kU{3;ha(uEh{Y{Ne9lVf+B(i14bQCEKOe`bXwm7uE7Q~iL*a1g1$wsV~k z)h#@O^$$pIfI>TFl}tyE6UnoPK{ufg_TP=3pB~vHeMZg>DkT~2@bmXr1suC(uM4+@m?&DNjoR8Dxa)li7 z!Up~OZ}{pdWGW>UlCQ0UkP!apiYJU)r$jdEf9~D5!*%z}*>u5(WBjDiGBRr;SNwtM zKKl?HF&&P2)eK_^nsFyN`v~X}?c~+SM-Cez%fSgLh?StauLtJrdQ8a2CC=cfPdy#% zF=Ul%sBioSS;NgezK`2);t6;Y_jEoVh|9W)9Ij1YU1o%-fx)%33N3%c@sq$|?~)Rk ze{za-Ed7$n)Fs-c^Lp|)Cb-rGbE?nSkzIO?>2FPhi@x7Sjr>zM1~*QtNIVmFlkUF8 zqp9+g(W$2QxR?PL-uQo)$I|${3CWtd<2=8@SLvA8Uv0;&{X>}v6p=&j*(<&Wxs|9t zdTc=+ILKJfbz*}`37#`9b1{5w{td46e{R8*BNWD-NXdwl7oN&9KI-N1%*XPIaclpE zU!Yl^fAVfag6C4;E1Zvhj}uzHJVQn!l`_<8yz#JG9G0BE$-m)RFE^Acefi#K^ig&0 zZ+p}S@*fcQJCtO8UT3C;15wG#d9%E(X@!h~Tl_itFV6A0KVs)EXRVIS#2xQ-e|pgb z`uu^nFuFLXW8=*^l!D38OCN2Ukoa8AQFlE@u3ONe7RTd9ROr(v5&btC=WDe4xkXfL zPVi~|GIiI|#_b<|*vdtBaf+dp-wQF2j=A4YeLAh8s6b>%aN3JuEYZXQl zokC)jUd$yo_~VamSQgQ_ts^p-VrBCH$Y&b9*yDE{P$o3wrR- z{pig}#C;bA+tvrCQ}Vy9NxzSe=qrG?xXAh zPDVrIpJ_Zqn}gqQ8{z&W7bB2}vG!}rG3P2g)1dZ}PE=oY>HELq-;9J$t1}l{f13s*m({^Hd)E2X*uB_$B#2u9Lr=-{Q?bs|$=X z-tg~i3^HX7e1c|{_e({vWbk~3OZ+weB7RiygT zi_;*~;zc@nLI40=K%&3XDwVi(#8GihiIs3(X3)eflxRDJ7|+ zbjsW~`!PQdD4%$-*-el~Oe%MWc2||Wn)%>p;qw*Q1P;{6=Xs8}Z}(^9Em87c@c~2q z6T@_m_e&lxd_#X|*H_ea`+u8x_$T*Jo;ZzP()g~e+q`ei?EhrmE}}@vz)t5#KTA|K zU`f+n-ZHwiD^Ew9mTSf`I!egh+#T{uf8jY_f>It)M;Bz^`IG1}=MTcAYWfP3+n9h0 z(tjno9K%DOneqAh^$d{+i8{rdkCU0dO?xSC4SM5JSnrXej6^@%6n~VOUfOKL_?YO; zm_plr1?dhWvGv->UvSq~YFr#E#x83JQRjYAU%0Qmg38KoePjcWA3fx}rE>goGm;OPAE%`NG_dI`|^*Re#CR)2hiWcb1BY)o=hXB@(Kt1m#P!(zF zriW|AJL)_|7rnN5HH1=8B9%|2|MH^608|yzQ3r9rs5>1))Mahtty>_j5jEIHF<|T| z#+PP5$;V^)ml|Ww0?!E^K?pSnn-M|V5_M0sJzJmSP`|JH`xxu%m-u8{GaK85g4bq@ z|EoQRHzO(#41cbBd_grQMt2%9Zb#t%@>~P%GoaB*r-m~D$xjU)qvFk8|L@K-`g$Jg z;C_h%92Q#RmKOk^I-%NQXc2 zibu(hn15W_U@GxxE#5lzsV--;EjJ%9Ffja13>f2QX}cu2KO{N?H4$<%ES+ep+Hm)F>32HE%$ z@9d8{58StAX;ge|r6u7tJ^0A23@U^oD8~G(r+?Rb9B;jH$Xd^yzZ4FUDlqpG9l|@> za*=04aGc3T$7%{v($n%5=WNT&cTRm&8sV)}{5Kk3=6u;v&G|AwaX$A~9EH(xC~S7) z$iCN^cJKU2;RW-Yi~mlS_9*{LUE9mp(c7&yz5ttyMf)Qb0h4M(WVK5c5u2RQcD1p? z)qjk&ZN8&ASPwu36AN-TdU1sA?mfuHL6IFni-0NV4&y4}QUrunHGEo?CsVwCAo+i) zf1A=59r)zPHy;J+uuFpbnP9CpTq@5gMKwMmju6M2#gSi*m;FlSqnFi9Bl=*=O{3p@ zl8b@TQ=bFIs^D4otm`}vf9m8KJ>0+6$$!1tdAiNMmufUp1wZ2pjE1vy(Z#(#pP)bC z;;X%BD#-tD`^BRY*e7<7j%|7F6BIdRa12+w;UQiHGWDpIj}yzEByoj2L=XS(_lbE; zbryMp?g7y0hW%k$Za9wnQdGm(LQR6Hi388wk2|-WEG37#PxpaR!I-h?Y>IK~PEOS$|59t7jE8~f80_mMepY2WJ_6*L^kjjicdef3Kp>D=P# zf74&T#&KVG2h6;J9C}xtkL$eD>>>1x%cSw0EjARaz}sqvAJJcw)ZwdMGSCCA1K&IC z%C9kZsH%9lTF~|c`;(+2Gk<@W{;)~%E*+sg4|Fc%=36W;edfs@!Dc#QxIq&3fa?=E z2DN*_cr-mEaKr$Xx!Vptn6?s(twkL^$kU1zR(YAHXqi|vPkKX@#0cygGx;CSMXhZY zqwCInV$vsw&yy1>CH4o@a({w})Kr|*hYK5wZn|l_FLRGV)wUz+&40}`;@ifg*ry~4 zu)i#@j|NHLLlJl`>37fl@Vs7;q*a%VUMcNDsNmszKIsgVc>Iqz?*DYApZrH$8@IE& z>e2{#n&uTbb^a;0#>YdH|2(@lc-T?P!}Iw!2VX+_4)^#gANWlh;mT2CVl1t*U$^1Z zsMGjn!$_~1#ec~;|9`9^keL!};T!Imvg(MVPF%75U>Ljcd)aA6Vwt{LuTk#U?MF&Y zJs^Jqa#*Y3z!49eh%skRIpRv17|G9n~-ET7O^idxRWV8g1d?%Uii) zCR>iZQq!+(8p8Wa=TqUC1F?DB@Q&C{TT)WJQTB4J)3ct3r)=xBl?Nd?bz7+SgZ3xF zKsP!Gc!n@~36GH$lxl_Zhz<C{)Ot)Ag^EB?Zj)6Ivi=*#dS6VI-TlY=|8uFdh>Wa@u4hmk!iCZ?ra1N7v!k`yeI*NE4!hTu4r~$T7DLS2nj2 zuh?Kr`F}thb;wEJ5*3X3)A7YcjZhs&|E_>_d+UQx4byX*ALExAXoAON51OIf_ zUt22sBRXyGU0qJ8a%1Fc8Ee}$F|kdo3&IwiPk%S17@mpxitq-@Nv+V=Pk;?tE$|bV zi?cq(MH4w2gr_dMj?43R7oF)l9QAcSc4?Dl{NL@{PZwUMtT?wV&IQC#!5WmcN4$?m zCyw3AhE9~0RE12k;GIC41bN>F6-XPToSw2Zk7NdFLk{-S!ilmql`|rb{mDQjc`Oy< zd4Fm%2gq)0jcIudsZMO?m>pQBbm(ytP#IKS8CncVoEsUF$pB+jSdn%{l(&bB+eF#j z5wk+a=#OlwVC;GW(8#?gcw9YQ)}v?Ltj7Z&?mzS^n|e9hTPElI$K=A(aM$14VSMI) z9VfZ58LGf&&?eu#kFNn05CezfUEB)t^nVnm7Y%*n&<&771vC#iv>y8E@;7kMUku;9 zqv5oby@$T+{(GA9WAv%&JsgDxI&r*04;=OWW^~ePy(3V0P+OJZjQ+y4`-k1{70HVK zP5<(LR!6JAMV&8K@^_od%opyFT)tw5y$~of*gH9d^o$!TL8n>o!xO!S&S1V_00dE;|rs zjy;}jLE2UZnNN{ZI1gSpbKeGvkC=>F?{W9=m8PGDpL1J(JNeGp(5_9?#yKtElSYQO zHbkz1SlN97721gVQ4EvR?i<{Wvww0|UcC8UO#*J>CdL~&0;hLCei$vH_mi`OZ|575 zY-DsXPg$p0!pu`$%4DC9*xJdB{TY2F869)$rgQx%=EIyzrn+ti1Zg-JP)~!C=@M<` z$bFp{AGDpQH6!4xly`y&&h3d{`t_l+Jb19W@LeO0Dn%UC6rE&byhFv2h7aKmQkc)~^$bOyX+Cq#6+A zZYJx4W3U7eu2bMoSf@}Y>wnkDhQCQI337GsQ}U@&hsi8#?c2@{Y>f4XxYdhI{VFms z^~s5!nKyAP)e}~1F@`?{~$7g-^ z#|=5iga+zA(_LRmQBUnc<2C<&UIS9rn%{P5zG~;7lM%?O9eVTw-a3v`(BTvOXWHuq z?&cM%#9VvRj}35-DMcaS+@}^JQc1!~`tkY{#QHk#JmS2QDbhQ~k>2C^5PQ(Mmv~bK zE`P@%5vX6_km!B<_w&bAuF(Iaoz0*wZK5l|luclYJa)KeuN=umC+vK^fW|iZu*Tr6 z@**>H)z0g`In|r;mC=9?EtCUAZRRXJ?1SPOIiql&Du2j>+DP#{)Pcj{Xu&(ik6!fPk9cN8e4bCV zTRW#s#GyNcZ=euK3Q;YFy5Ye&U^|{PylL`s<-uv@`n9;6udeg|6{q=l;Y~qKo8W5C zJ=gzFUX|Co&o}+cU(aDzoUY+oI2n9s9gs85Zh5bJ8Jrdg+olCM2FR<%2Jqs{B7f;O zXSMt|m!+hTP;v}>dQ1X__$?Xug!Dh=9Xed0H_MumWEE{uC_^T(hHxz*!XkjCt z7S{=C8}$inJVLXLh(kA4gLyC((;$av+J!OES-hBlGgsX=Vs#@=Z9Gt!?Kl%Iitf^} zYqVEPg+^^aAxU1&K;j-cwQ-$!_0Kou03e4a1%)hj^BFY*$TPE8oq_t zvMB6veL8DrmWvHT5=X>~>^6;E%h_CX8=nDrWiRvE1zT|5f7@s7&t`1i+Vp&;`f+al z_45yV#+f_y>dzjZH8$(uj02b@o7aHwWB)VAi#gMeA6vo+Y1;?G|5NIYh<~j4+Y}0+ zrA=}{!Z=wIs7AACxSB~TM5e~gvkckX(V`GGk;#AoZ|_xiXhk;R?sm@PvPhx=T*_6tM>^!q}F8j-b&!_9$P*$!3USReVm`OnV zlS}$)wwP${W#vX=Tqa{2HMh@pKlMqLs)N`-P<`)13wvZIs0}&&P<;-wOFK0hsE*23 z5#qd+8hC0mB?eEAdtcWdOXQecoV8Lb7Z$B|P-(>BY)6^szzQUet0*i>V`quSiHGPn zT<;F5Ij3dt*ZjlR_Lm=51`h$!moHZa7k|^^rgTXELkE1svYj`2_}rrA=HMvDI1ni4 zn2%M%BNJo1W+_R&wCy33<{APk*v~`HOqo)udyMNP=rn zcGd}M>`=iNY~+9?akahh$50(iT>V5ZW68un95)01lsH7<{8w&4appwFVw6M7kALuR z(`Ie#SBT3ax{1$zZQ2KUZ0m3-{^i_fpMu|Fkgx6bhxO`*PHn@>*^IZdO?cwjCy0U% z_axG_zSs;G(Olu2{S|~YU)U$>=u^vXY=MG^!cHb&jcQKd%abG4dY<~Rb_n7#6fZ>b z^jiE$|a4b2fAOe+AugDs^poQ%w zM<1)~wc%Os4`7$j49+17pT+g*OI36A^gq4DG0&r5f_ z-9LG_<6=@R!nisGFTHovMDt>anu(m}Rc4dK^lMQBQAC^_Q$0thG$a~vk$)KHSJ*HJ1kz9p5WXe98m302GoY^%Tv7J$jiUh4svjx&{$UaL~%;d zmZxk9l2`tC=Kk)ZKs&ezB?XWY8B3k!3ox=O4j8>)5ULx%lYg57-KIVs444$5R!=cI!szozLq6TagUI{jkJb3?D7SW0 zP7ZncVy|Jki{_Di?7qH6AF1-o(^_0-amRT-HI1S?hC83 zXxsK@UDd6}fNiv5%nPJGPMPx=d}Q_Lrr+?LUxTv0A*={#{RK3PnJ`F}+*+aeT)uUs- z^~xS9NkYgtD`hkxQRz4`TTr_cE&2IGzqy~v_7_`QBNdT0e@)`=X!vFO*0hN7h4Ycj z^XRl6ERuH}vza*X@*_@Nx68jsykGe<{@opmN$6S~f4mHbn}4!zVFWOr5S9$vk-g%* zWNJtG%GoB_Q}o;xqojC>Uv0OSe9hwTwx1`pJm>f3NP7~sUq_QwIMsGm zP9E%Zy(;PSbuv3nHpz6mm8 z0GECY1n~k;z&b$0s|Qrz^CT1+2GW}Yh30yX`57UZDt}(b=6%|m14*fmBN}gke*mah zr9t8EJtZ<=;0``oCmLDja;5;I`4aOt#OE_kxP}0E$^(p}p&Fmpg|F{cUpf6&OJIIG z@Yx4f4=mt-hqh;|giwz*z!ZDN^yWHx1*Z5!6I0Ja%oGjmxrC?Y{=h;$ix+?nUUP}} zyIB6N$$t+xIg+AmBz(aehi|;5B|g7%dtb(+B;)2fBwz=>+&B3tuJ7min!h0=j~Fo? zDq`AW2%^ID=KXIL%uxf|mU#Vy`FX|mV5Yu1Sl=Py-B<=4MdlFATRm`4$CbPwZ^0MF zpJRJu`1?*%9{m07I@H(emY?B$xUY}PeSb2>>jdxHL38~fAy#+SdIRzG{@w4X z{=@I@?wj&9_a|_DUh>QPpy@A|pYmEicjY;k&s_q`n)>RL=DO7{Y1(SJut&W_KW_EQ zKbD8s*||MR96uc#525DM^Ycx9OHA*Y{I_rKSM&kM7{1l{dFS$cRe< ztq0xohp!3(Y3@U!$r%5}=C8b7Pr&d%H~Wu=?}tE=HyU*w+uyALjmzfyU;m9tKy5`- z)+Km8}Htww3CSx3zNs6m$*UIr3>Q$==2I`}2&0Tm`)?K`kXP4m$LTA;$1k3^j1 zLBYEs;dx1&Sj?*Yvv8Dn3-Vpz9K}4#54F=0naof1U@jI`-lc;=C|;0Qgd!J(O~q8? zR^b++EtXax3)c~)D&oQ&iB(WcYBv+>vY1o%BtEl3#hx#|pjg*lF1DLuS9@)Lu~Wr2 z^=(mQMVtC_ao|NS4wi19R*WXnjpWLqliGdbtE!$px^ru#>8t0eEo47;YhH_S zn#edVhSo!0I=U47G6aI?*t#M#)b;p@47{(AR5t!*A} zC*CgE9^;+gg}Guwn6_H2R=3q+t=_2h(R%5%I-^$b-Rrj|y_fjIDqH-!KU}oho#W|) zEPB1p_wv0TJ{O(s;C;P+PCH*8@2mausI%$47yD-*&8 zY0@cs@A3Z0?Uc)RxqpIx&T{nbFS^@K@$v5NyQ@y&zl%lpqmzHX^Lzg~1yXVF@87M)q=?KOkN;o-6OJwJqp_G7y<55h`4 z9(!|VQN2Fu-dj8T96Zjyr~SpO)m{&#)0f$}y?mW|FX^Wkc2$pmvdwvX{r~@N^Ogpx zRRv@+<&&A91UGfVBsplZ3g4zaSWXW%(E%ps7+yRY!yw1n#3DaHWjj_o)B9N@!=RivZ_F?lS!@;vd@#%Di!TApT?*< z6Q&jLol`qn+U2$F{5cGKA6!`2Y~mcS@k5SEMmrkL%3q&*<14+F1w z)HK$rDd!VQdy0c2lzFaCAhlQP3hZUd<8wJTRi(-y@X(gUCa*9?h&TO<*7YC8g02K5 zw(x;?^-u#JYCInO<^_zxipbj7*?QWLo93uG$ zs#qg}l`mxx>?{n(NX(&hKDj*EUh}`iJ6cbuSK;6XC5T)23m^(qR2M75`vHobW*{Grj^NPS$ z;Fy^5+jWHfm$1}7Kckp5-%r+>S>XW7A3ahH>)-p1@L_1i9gqV1A-0Wl$jtP^M`|#C zq{50dBts4C8BtB-+eB};RE->dcV*8W3H*2JIKYANWmhKV&7v5n)l@+n5Z zlgqfArhK1FX`uWakSxMErrcN5stjtvaZR08bf&Psh#C$R*tJvhd>TKZa1K<^-&GQ5 zG>OuR;0}%06?CyaLV^8*BkMKRO_Dmmf#Z6#`BGIRh1a9LcdQEYthhb`1uS2e9AyR+ z5b}M_oBLS5NpruhkEAg7HZ zgS|T+#<+YCr{Wj_pLhk|S1hdr(<-X}zWg|Mq+_+OSXTLY*7xVo#Wuov{@Zl|jbx9G zh9q1L>7yQEA5t)NV>k%@uEDH_Vil)bFRH@<&eu~%%*{F)hqPKM0#TA+yAF$519{W7 zS#Kq-iL2%}&XED373`V!IQ4)0B2a?Wo_SyW`)>q&8VD;};@*Pb+#xl{9r&`YkS$N* zK1SMhT4||B4D(9MHHp4vvNY?aXVzvbTSj%53u~q-m@!;O|Io)2uFaQy4B?@g(V-?u zn9s0&A!2ECz-wjJv^mf})O{{$e2!cTYnBg5wXEQaMLVGG96Wi>M@oOGCgt#t^KFXr z3Fe~*4$kcmA2f+4d1dw$?w|S$ch|hx=QwhS>tyOga!UDJ)b{L0e(Z8zF+Vue=od@1H1jO$ z|IBw8D8D?ny+biNZ~ezQtZ#i#QoR)P$-#ae?ci0mbLq6**_1+m8b2Qc;0exuGX5=H z?6k@6>59L59_H^2%k5NT!KSvDC-y28>^SpxTtg&x{+ux1hi8AU{xvPk&EE27TJHPM z^ST=>j~OqfZvItyu&>q6I>0%eCruqT<8OUw_jC?{u*A8aIGS3)PSnsBw7VSo*Y`WV z&pq6a+&$yJ8^iqlZ^jCak4hDrFWiT7`AAC916w}YW)Detm^xF_j(cz!VJV2XRfZgc;jNf{g} zhU+A8&3K8)pxL+LKATT$g7?#{KPTS5xbNe;A(G>YWq2LW_6rQDGa`{TH-YXIWl=Y-;CyZIc~HOfnx&jo*eK7Ze&AK>RnlfL=fq;Ebq z>6_1WeKO|Dy4qK^sw%Vt@&L;}^ZkGT5QgcdMSViv)h1nq=>QUwF7y2WK-wnVYmDiD zH0iokg`BBPx(^@EDjfH4M48Wt`JDT{Am{H3^SSwc*`!zaxopy#&#NZA`Mhq@o6nyn zA3!+fE=!O&*^aA34O1= z%^Ml+&zRm--1MCzw*816w8VYtKr~|%f72t>q{sd6_?q6rJ>M<;c}z=wO}{Ac`PS}k z?&z3z#0M=gPXp7hU%Vb6nEpL((pOwy+Gq3pYubORswBUr{lL7BuW4V*bMME@po`Cb z@OlJHddxfP?kdiA{+vRS{-9-1?JL3Y+1IoR^F6+%WhNijwB|h3y06FFS(E-;CxyxPHLYu&eNDT;eB9F3bxn11?yMl4SExB}P0%)jYt~E_2Th%+hB}h( z{%wEn=u5O!F_El_BS>dBoddb7d700FSYD0&%-fmICk6?j#r6oWpw%&8AUE4ef{?on zQf3o?3a?$c+rZ`efZQNJcoj=h;JnJInEEARY|F^!8%XnL>MH>vPcLbw(c$Q7M6m()DqK&HtT;wt9~6#9EobDh9kktO~`7HHsQb4|NI|IvFnlH%XJ7oBXx z$MmTr2TUg=zK5f}<_a7O-X-4W=yzIyt>9H7eYS=Es2KAzLupq>urJlW!i-y z;O_$PP#FaDIT3hOb)dQ5*$e=zw`{GEe|vB6+jFVL@Bg;nn)2AP z>8F2{KMGimN35T>^LR)V{x#+5VEP5the_AuOG*-*e`f*PV87X%$A`viU&A-PFV$E- zCH5x^ul-9t;jUe2AIt3^N%@;eCo%5!8Nc^6rVlXP7m|H1fShoAHS>R+zVE-^!239V z1h@Dh)>8TsKlI0cguppo3FmE!3m&Y2X2yk#eElAp&Zvd;_>JQ+kWk~e4gvWTQq&FU z0xMytp!fd}_Fg-(Gs}YBs|a&72ONQ(ES`Y?kwx#lyAkNU_Z;Zmh5q+my$&D&>MNqY zjz|-cnbquwNSeK9(_(*&pZ|yV)mNd7iJ`BbU+R7~Kc4=nQ-5*N&wbe6ic@LpCqI-k zesT*rnzL#9r$!3r(YD9%6&L<{(Y#Ij660OGNBbV*SA0nO6%$x| zLi-yN(tJ+)7!zK6MF$)cReVba786@PQHL3m(ELmX)6ZL(jSe6tt@x7;X(qGyhYmF+ zr+JtTBPPFij1GT0rl@%G=MX6^9-zYyQ_(-+vJz8WKhEkgbtQ;DPTN?5LPy)rCZf|3 z#)REf)MXMd0So=Qd~kTSgZ&I)~Gtdo)&q z|HY+S)+hfo?rg{^7p@R><`yjPL4wPNzr_ZgBbb_{F4n7nhpJ!h}Iuj z+_L`2_TN9UhUfj8^UQDGJnpw`%hi6lRsG>S`^!D+m)n2zd(!pGTka44>c8?5`pdh& z{I7nQ|I7c3ooBUQuD!qf{cWH9t6%%8|E+$x@!Nkto%!XE^vkX5mvevl@BFa#uYUVi z-sXPgIs141yx;NL?jJ5)+Mo7Y8u|+Uo{2V|AwKl)-+i<{e$q;}{zq5KA*{$eDaxGL zXwNBv*D3AS>5y;nuRiJi)zSX8_h{EI|Jz3E{c^qPp+xYXIEc?|`2GCVt^7>_cus{s zzAb+}8uZ&Y=5C2`KZO{C2@7I6sxc!&&fB7GI`R_XKxL?1q-0GMA&R@s> z>bHL9UuJ&wcRc@>|LXt#&R-|}&Y#s->^-3L0EWf1$!1-Q&-h-g z{ljh6pljD|>7O?fm-esc_rE+p|K%C_*Ym&h>2CI?zp(yFay;}~D)?PAP#DuyKl>i4 zT{)BueI29zpS1^DzisSdC_%HvnHaedDE3f^P|8s_Qf6lO;*I4>r-19E&-@a+P^!0x){r_y({Fnao{lEN)m;C%6|M9<- z(>f&kfB$d$dKCLJ7ys=){_p>@oW}isFO#hJU;lm@n*RAuC2@=J|N16*8ace$)5b@4+O8NERp}2CR;&Z|09Pe zL96~1;`;NK+RrPfpC9e#{wQYI|Nlbvn1QBky4t?+rTvtO-Ck#AEe@d@i!Fa%X8l|| z9d!kRR9fX^Dfi60Jq0@b(BBwOD-b%qdKXD}4IsYlO$C>dK1iJJ$O=yO|%T4=VYfhnb@D?L@wTB;*0iD zFGB!H9nsSxqij+I+LFdcO-W2>GNWiSIZ-8pQg&ANGWeVg1D3*nZ<2 zA%(P6m=^wid5K05OZN*Fhh;nWuWJ6_QI3){og0eNTln`2E%4cZi-Pnhj0>8ImQT9l zbgm_c1gY~po!U6X_VJX;2*`;HK_EocfRy*OFlk}DrpIa7^u~Wit&Ef-I0C1Oo`*Bp z7Tv?IJ-xJlY{}<{drd^#!DUr2zjXDn<^FS0fFp9TuL&)x8jG@B>J@m_LRKx-_+ zWt8n-WSf9`nZ`;R#(F7tYTZLOvDd>}EGcerel^<#%_-V`2y)Vxj|Ri?#iqr=pS-0B z?Y+q_{$Vh>basCV#Q8c1Cy%Udw^1}1Sv8*eerTRMWBP#c_%9JsE8YdBa^aNwgJg%I#*hmJ|$FbDlEesO!kA@sbvt({7eF*ECCZ2IXs1yyn)W^!ujzlzqIz^iIm1%~=Qp=`HyW)N zJsx5;KF%O+BfSAM;z2NB`tldeh?XW{T;lk6&HR)@3bSUaWjOgH(bCb97Et7;IO$wL zrwdtT9TmNTSR!j*LcME|zQYK>^XhWoQkmg;@t>4^kqd-}B`%h<4Q8=KpFG%}5t6$; zaFXuzp;CW}vZoR0=N{tktM{iRh)k#7c{R0D+*@d5S$BogB+Ft~Wl`AIVUWI$oLPO6 zBAcDvvFCI=GPz5v8vb^)P1m%9KHRfZ8!srcpsjWJGI-xx?qd1lA7ca_=IN9YudLZRo~=hLEq0k>^vhEp9rSHAj^O z8(-uLR9W-9Nk_L!)?+3Y5$r+h{?zJ?yH?Y;RbVG*mdr~pWI*$Z9HWQrS7@tJoJ^ff zI*dK+we!tKY~mT3rks|t_oK6SNK}S2xazmE9gO}YZ2Bd=6%@2ju4|G$PAc6@TlSet zUU7dgcR+OHWD~w>Y|JBS4adBWuV1NZ@T_O4I0|rD8TsQI_w>z$sc0LDUraJH@7Bkf z+;l5Yx+k1!S&NrwO%lC$6X)FPvQ%nC7IHVoRam7MkE`mL$d#y$_O>#tX(&{s$yL3C zY>+K|TO^h2<@T{6e;B!aaCSsey!|ZkO0IGka z?XyN&MuB!+?pmppyx9HsT@j)P2?*qxyd;V_N{DB}Rfig@ulz>JW75XAtGOx1RdvD%I0c zm91uqBwHR88t6ivQbkGyJB_ucyXm13EiDa?K!^xLfc}o~j7E_U10>#lXU+_lp(ZbouP{F6qMI z2g^*yUTlP}=1~>enAS+&ZxgzPHNvLL2uEIVy|>-9s_gXg_U4?@>J4FbJ`K8<&&Gog? zPMyQMY{ZvgBvq2OPTy%>34sx-0g*FY#neDE5;0uSrBK?ic}%!0oGV#Ux0jk~yKvXo zw^l`2atKRLhO~|=KaxkbG8S0$eWmGPwZni`d*>tF5r-;~6;Y#?c65JuA4LU+-hDuX zG^;}C)ah`_GQkdUb9mvl`|kDS1xKy|6ue!8p$K6H6=$qAu@g$PxWW%VID*K4)-lMH z@|@nbRFErm(D=+-i*ox-;Ia>MwIwV}A_|^5?dFhJ^lp41L4}Yd>r#&uY5hti)6#kY zG$UGIYwHR$-^zrGQVD-LcD^4)n?gASm8)VBkB_p@%0oPwK3t;a>9lDPom|g-SM}Pe z{3_#J(4osh1*|?QLWLh~Dg%zn-vhFJUnY<}nxHxe9r>0WNIeUSu+y}n%C65O&_$Q& zv%Yud*rDz1+J#$~x)#~wH~)qb1K`~FBv+iprF=3iVOGJVew=^4#Ztn+Pn3RrRP?!P zQhAcJmOfnSDgkk4wW)}m-X#Hdx{PSrT@uVf{AFFZEw5-pDlqkpe4aDM%DNbLBjd-P zWRe-m(qkE;(pGjt*<~TN5yOJzh3#5~Zx1Jk@T;y*vZks1bkcr9zIi+r8r}Ja&PnW* zA03dT;dB~}iZ*}rqNH5^;&N1(_nSRB<^Y!-5MGKouHdYpN!U!XS@pBF{qu?U|h-tNAB#82uZzYpxDKdj4`nr$19~Jn*k=7(GE!3vib9K`zm|br7PokKO@&nHJFsxcT znLXXrZtc!SH}&I7#$?gxgFUjoj>`9YFP zmbkGaob4wXgBhJ2Scj_8^pWmZj701Cv?C)1{_0b{TJLS0-^Q^wQt|gys;@OfFB?Uc znXtgx!^!1Wx8B1nAV)sXD0{B;LkSt?B7IZ!wh4Uq^7w@6CY7K&pQw z(qb^qB=bhgRa9&0T#CA=r<)Vf(AePR$j1#T^C~842YTuI`w>bJ3}ER)u0eSe0<%!D zzUA!LZ>2A`qe!2_`|RoWfcx=oQ-@U;zqmnSGwhiunneqHGM%`u<0r*Y*O{363Z@Br__IjzL|wqc@Q00QPBqBN@uYr$v6=RIW8UcSnveXK)#*M zx7+~T)imuXbhA(%$V`W9y>w$knq_Cd!IiWimmbaCR6dqw=*?DOQ|jib6Vz>V9xJQN zPB%dJvNQnA73}KRG?M7KFcP_D{l|j6D;Jc{!^9(6DfzQWVe+vS(9C}oWwGh^0oS1R zbg^ZaY}J+Y7MZ{8mW6}sR`SsP25ZytB!W$?}5O5!Xz$#U;D#- z@=`)d_VYQdtN*kd(BFT4fGa1Um8f@u%1Ay#)NRGw65%m$uLii8It8KdB`?r{-Lzv% zxwGs3VzcS`pzrVk1S$KEDw`a6oNZQC*rz$gZb`hf;Uibx+D$-;W-*WiD@n$GUo3OE zP9WzTg{EOa)14-J^@VvD-X%EbUAq~DxT7(lx|Yw9-zv^hjHiDzxfYN14Agu0MKPx( zaWF!Jz*yS>HfKyS5<_11lO&J?!$75(UGo_jPEgvHDCv^OaOPPB4nzaA@5#DlG2Q&j zQX{-GHq056hi-oXcG=DmwPJi00G&3Wjy^|j9B+HTcbs=;7tP-DeW}+|?r>?P@tY}j zaB$Klp6pN_+-`sQ&(F@4?iKop^0zDz@f0&G<&2E$JwF!FLnmB^S5$0Lr)H-JqN_Z< z;w76|8I|G_e0LNwc5EU&YMop#;q!^shOXGpw^6}Wr7C~WyYh|VvZz- z)@?2jiRO6xzBspT9~@2q3_FsTXWhvKTW#vk=AqXG^?6~S;SL`~QCg4Rw+uhi1D#6P zEP?oqpJ#tny+8_C6_^AZ4I+T=-3Yt`G*U-M+@?1i6zoe}1)7A{_Ld;|6_K&Q3QZmX z+)XwnZ+HJ(IBWOk0|?J88S)eJ^+Wp306ivA2Cw4KYm9Cia+&ScNW znhx!3nPW~tOxTL8FO4T$=D26{h5%4jg(0Z=DoWj(`HtfdE2V9E+|vd`MG()GKs>1gep~CziXJ{+4s1 zdg{7z$fr9ZJ7zrHsbB9T@&x_r>0&=q$zq&Efs2)G%me!caM(NaQb6zdk_7s7nsKKO zMI|$WdLZsSZg$}Cpg}mdNQHvq=65^AhbMn4AfWR3mO;c1)AA6fClUQDuBcD}&Da4c zQc)u>TplIoK4oyg9!o*w;(p;aqp>^brw6_v`$<86+Bk?W& z!>Qxx_z8FkntVG4K>!t8x9Hp}191)1fuhvC4hJh+S7RSp+7wSAQ~8y{B-(nhA8+sE~<)XHtS5EX>ot<6hu+iJ`Ygvq*jc2Rw11J6;hwt7BNfXul)td zp;xjehtg#&05U+$zj)KL0n4QT#%H>NXXSHy`Zg;c89ae(c&rZ9>H$hD_Z>H>oR$od z>~>z4F*}Xqz9XY6=H8RB6`XX*i^0tQ80?B8vB&d3xE1R?^wS-{_e@ZYe?l|CcLG>{ zw3JtvNtmM%V0b2moi@1mA~^{GxkC}jlT)3mdYRgG*+%Gqnb9t-=2g_G+o6R@2xPE# zfH{*&0DF~k8@SqfpCmy5?BjWt^li}#q{(^?F5UkY+pcvma==nJWFq9qA6zD3%P8 z#;c<+YA%51D9k5UQ@JE8BJlyV(2o9_T!`6ALs3j;Del`#!e(OI4Xcwto9La36q^0% z-LmNde?dME{PYS%Ksg`(dCAl9@U=EtHf@9WyMNg zW+es-uG701;^M?h;!YR3(R&I!%`@`wN%93QYAb~D3oe3BEK2dBe&!@@DHP~XZC}t) z#_(EF#6x|=-gIIJr4E+w6PKD%}Q5gJ;>(Z+$542=q3yDbiM{%JUIVNoIH;pJvu^tBCsQ!JVrb4I&hM zii4uby&635hXU_jCG4ZmlAq_iuI>wB30}zYIcUE+~Z54!PQTO??Qrrk#Lm* zSXAc@%P9qY0pwlcMBqV&b>Q69Za~8q-HO%dVm`=sH+1diDlK<>=5lBCrj~1Vx`Dm7 zOk%+?Epn_HPhOxemp;@yqi1ddPmt@{Hcq$=ETLGh+&RGb3PGlND zj3x@8h9%n`Xa)6jOC*Y`c8dN=OMLzsEBj`^FdA#gnwy? zBP4a2pijWMl@_4~b(FJ-zh9I}1iAIkj2yklP*4b6eSMvlq(3DfyTiUynsP|K010mI zUKLf|J|TjGLivoRw=s!`t`}2+f!ki6PLukgl12W8H4@B!a_H6}taC8;DFFhOZ!=Q> z!-xUmYntH%vtB5F)|K;R4Q0<0{`%kHJe|l zHq=kYz^PVB7i|fd<*Q+@}96bAWS_r7}P?k@>Ta8+o=1>-GdJA8Sy2(^O?rBb(M5WKq1c~aXy_T2o(9XgoRqm<@ z#9gqQWbd*0%A&B&rS-T55Ds^N1v_bI-x#rfec&5sP^#EKIR0w2!lGLDN=MJKV;DZE z+P#hwvfrGd1fn4%MANMo3^z(U;-hKX91a5CrQcWg*cx9MDPsK*^JO~{`5EYA`x(o$&fl!5%Y^W#o zuzZjKxAOdq>ETV8LqVD4LKm_vPe_~Puu(!3zoQE?71?*k^fg|wdADnV5PtexL2n;U zw~%4lqJ!&OGV%}|+mnYjCooiXQ+7c_2FVbfII)TrUavCJ>ZQHigI=35;9Ki|v7Rt& zI3A*aJ=B=Xt%k}9D>_(D*+i0aWrrxab0bGqKUyrmeGh*ZV>ev zh?y5bxn)@YH8A>b_|ZRMLI3SP{>T42Sm?jT3faH0LRj}3E5y_P-{t&&{zeIz-t+lM zi}>gEF8ZHbIi|8F{!tKpkA7-uGtNIbbNau^MbOQ~Z4nwnKZzCMKeYgVGH-s;la`X! z^&u1TpWq_zM=7)Z6St&A4M`SAUT5e({mcH=^82R@tQ8|S9#{YF1tb2HLyr4}YzY&d z;(-6Dm+YqhWO4jl2}>h?G0(G2P)~8d{geXyM~V28-utyA=rYbkfT7~2yTm`^`6*9G zmwX%32)?C={&PLCtNxejw1E&8&;ZxP&*e&trv24N(~m_2>A>%?MbjetPl4w4M{*pz zy!ygdit0bb2XpuzeLr1cCL>#36J?(GpZPofX{G%q%`oq*1<4A3eeMg|{~%CFO}4?THGun&xwtX&TGUzE#W1)q;&B9p1y$kEzR^%}m#Xf{(gt z1S5tj(4&@r;Xp-*yBBXermcN()KIITJOiH?6%RpddR13GQNWn0kS0E<#*OK&aq>cI zU1m4`5GzJ1DWd2A3u@MqR+>0f)xr!tHR;*Jzj8%t@S&cguF&=nONyIJL=A!9BeIclVNe9W)LyJ{cqn5G1NLru)5tR=sjP>_T=n@)|dHBje*ueGPSjj?SY$1a* z(IIMZ&)MMD#UQUl+YAH-c)Ym27`eVMBB0=nC6-_qLt^6rs>nPgb&%g3X^aEVtL{V6 zKubD*H;>nh(AT^_Nu$8l?ve0A3d960VV;DM_;eCCYL+uOw7c0EM$5XkGjj4KpNw%rtFp zTz4RsyA}L`%!NX48;3}yBl$>|xvOw7>ZX~02F#Ba!wZ;{EvESVT(M}%&AM+p_HlMScWLj%t4Hu&?MRXPqx&8tD{ zV3e#@W-##jZb8pUSVSN3%`f)o!Z8Ppaw{b8y7Nugd>2<=-ZzKq8W6^k4L}WEyGtH_ z;-cBWj}7O~>!UVDD>xeNYi~-HN<%w&VVft>m>LkT6uQiJ-l~+FI2ePS57kB*CKYz{ zwAmD07^MSyCpUfj^Y#gf;)OJ$1(Pxx7I>EhR7;fe0v_I*&B|y~G>iQ@DUa-V578-D#foy+*12z8dTS!4PJPM_`qM&Tw9r6o3NB zRv9=PsmvG7#kb-+fBNno_oSAtW&jb5di$lLeymP6;%LnS>md}>@r56JZoc?E0X#v| z!ncwKQ4oU6e#kG^MPy!EKKnuTJ9a|^y=c%9^-tf(g@{eI{>keTu?e%=Y~+rA^FktD z$nXj_hFErxoN2~V_O*%`9({Vx04)uC%%e_*9QjqP^H*^3DMe%J-J3i#eR-1WynW@N z?>1~uWHIeWP**5NL>pj%%%+l2`N(g0sVJQ6g|=cNkG`}9H_^J&#gt0b9yMD}D9=|X zsn5PsJnVN-HYouhNCO>WcG}0^Q_4XTUoH3M?@xKdNzqf&{J(nN*>971f^K_5cjEj zFB1YqTapjHI6C7i)Y$#KgoYe*0lCW0T^MDw+SkMs;{tEq#)$wP@0l)-CpiK@E-Dz~ z=r8?U^tc{Paf+M+m19Ar+5Y+L?9^bs?q6UzUMf9f=nPp>^>t2B@Hs`r5jXyq{uoO#Aqmfw8wAba7JcU6kfJs@ZfCf0P=TQgw%<8ot%l1cV4FFP2qM~deJ6rm3%H8V`P>R3Y=`SuTxDn*($J3% z}29(rRt-uHTH{qEjORP_LcOOIg%eL=bP9=4_MI?uF}tgHKJF@!@?=w{zNKSon$?K zu(P>)T1YW`2*eLpwhQ~Rc|oaS!+)o?hCJyg-az<}x`k4%MZ@~hFp7fnPBn zIJp6^@UCxPIkZV^b$TQXBUo-$56YsdpaxvXTi$Sgb2N^49K?{?3w|*UY_$EV7MA~A zHy1+kE3M#<`3c0`esp4?)Yy;YMKd0<>w~P-RBF>O8sop)XG^EdLj4%0EYP$zHx4w& zu_qZYW9VLs?!I3h=dhg1v~QVWt?KmXVAxUGL7F=~*O3O#=6TeaMYnHwSjG(OO4=Z{ zXaI(PkuS2}U}-SM2Qw+1zH0SY#}7(U@WCQkci+{cCVuV;jMLIEMLDBj#3+9bgRWU@ zqrN;D?#23UXGwiT`XamhOCmc9=HNXJhCVA}!fm31@~ReE%;ol7v2{=|Aix}yx)xa! zsh4RibNmOWWuqc5&$j$4pbR0S!x>sR8fTe*4KEgR0O*a?;?=|UzBmiVV;`)TKHkb? z-S(yBukH!X1=VxTQ4`?bJ8YF-!$VD)pynH_f}wXSWGhE7kq2@WKG z%C-8*h^`pII0zwk8GeV)T)1)^uKX7JYAj{}g9S2-D%R52L&;E>%$@gcPyz^yzu6wx zx=S5jok@Dy2clv-46F!Xi714J_a< zx4R_x70;pU`0IlhgD%|~fB0fcVT}ZT#UwEwW4qp1&R`3LqXtWcssEBbY#4ZHQe+|* z1c2C!hLfs*cOIz+NR*~5_#$q8ulaP;VHP)%m@~;vpa;HU2-b`!M36{L?+l(fuMO&n zT7r>8!2ElHbxjp1x8&leV#c|KpX#Ruk5u$iE~&`l#g-82k#f>Z5Vl8Q;AU@s=O0%J zQQ*50MHN5ENEcWXUEH`;Hz+WND|~BYTM#FT5Ap`X(_WBvSa(r-)72dFknB5jJpeIF zOiwgNl zGw@@I@AS=(?J&wA(#Sk+5=D}K05y9$v-TqGCqz-O5$I2GuCJBWkdK)>44w0Cp~fQ) zLRA{DlmNuMk()#7HG#!pTYtbaWea;OGz*6)#vNEY-LjB2zDwX^v~i@GQWXrKx3_`y zA3Ws13_L#ba+MKorWgfOA+YL&0_j12_OmM3+iPnB?9OL`=ZN z!H!W{&04P9$e#h15jAJc#81i7Krf?3dth9jJ`rcWTYHcD2!89yoH)x@3d8M4zG)gG zbVGaXI%|=#(Z$gt?FiSRZGDEJ)AuDE524N3pEs~uE zFA7t_@oVSBnGNHARG^|$ymv7XgjjH{@9PlIy^)&QMqXx8wq5Ap zvuo>-BSpM_nPMYa-tlGAr}|zivO-BEQ*nU47{wE892ToHe}n zS%;Z~A22PX26>NhqC`?^gBy*z-~}pBR@fbjcpRhzP(bPHQ3Xezi#zC%uKoE)1*zlE z27b1Eytfc}%+8%KHicyEEvFGTTC|PBRnYCG++1LPhhj5oNM^iC5kLCeli!idFtTVY z>@#+;Gq_f-aiSyU08}V}gD$-eJK)N4Hg`=_Av%fZ+>%AxLgYZ6uuK5LM>0*QN9`nz z%+$`Kq}#(M;lV_CUhX^-7mW;7WswF~Qlv=w~ex%p|bNZG^J`GlH(-E5Isx zV%P6~3LtN&pE>~VMvAAQ` zr^xcvACr;eX$p_UWz@1n?E<&675yA@`(o06rN{(Qgd-Eli-4_C?dwiP&8Y9x^xe@V z)0DAyM}G{gr5!TzN5LD!XGrCpHC|whIQqgX)W*S18FhyWJWps1?wsW8iNsovuxohk z8uuU%9LwN=kR0YmOknjazDb8tY@S{MSr?TORypFpufEk{CU9VZn~HLPK=P3=hkvzy z1qpMRPk9k=&Mov35{-kyG0X41=pu?G)t#6s6#zTKK(|ET6BN4mRXVFMf0cz1DDpkX z*EDvg_`DtMylRd;Dui(rK0uwV{Xe|DZLhjM*C_a1=H0|hM8&83WRh7e1t$cME3?I_XJ|^EQ1nbMxWpjU;z>G~?`9%j*V1@n+tyPXJs3w&pu(Deh z)PmLAk(dhMW2ok5K&%mm_`0a?JbV7r*{y~sAX>@=IR2n+9lZr55n@s1)0=WB$hkK0 z@LN!X_A$eQqVAt24d!u(>TguRq$m%Q*+t+(22;^J7q=WogQS-Wh;3vL@#o#kq{-UZ5NAqKI83LiO|$?e>JX+pvMm_!njQHce7hqCD5x2VGWyF9jp+Q|)ytu;s|3;wj`=@8CL4WQLH=(I(k<=q$~RwNJy zNEz)Y#7|llNEWqK=B=z!b}Nu;>Ssq=>@QN2+QU+8#|%_lUy%)A;S>b zoXqmc5jTSCiW_O!5Zt0rQExMDKwN#WBv!%Egha7nzq>>wMKT1R#6^IjynG^V1xnRs^sy2Ex5q^kn2D z#^W1vNj9QG^PMEkQ@L0@qtLUjKahPxBz~0}C()G(j@r8kwlZHCtj>o7MQ|XG-EGi8D1+2F}LfXtjv7& zdY)Qn^V{+UL7NZ50Edx_VXRB}X0dbTot@coD!8~SQGb8(b9{Xxiq-|30NP#jwr(-* z)wN*^_$<9a?R8UGOU$5 z`=<)p+XB4JK&QX_F#;@}vH%XqbhJw!*s{Ls9KmC_^EUFMnk0sWBy|qf@w4*XUI~Cl zX{hZuBk(2xad}Xj=fYx((Wb!h|G(szxfr(f? zXguQy-=3B8umKwOb~Vc?9;!5IXlgw`{)%VBPGo=g_eeVi7ciR@adZejt^NHl%GV1a zYakgOi8fME^fXo&{n2zA^@F89r8u?$K?tcT)cJ0{J`6v^%zXN5l2 z7XN>S9{xaj*8^36xR^)ad|dKI-jMNWWU5@YwJ&?g?Z z`Lln7cq&&-v*o*QO#WIK`-=8~0Vo~u4Of%8*DEzVKOjhggK^~Kmzz|{(a^f(QB5?& zxGAI;B>8#&i+BITx*tW|M9$n`$+$qGN6~*l#Dj1VL5&ZI>5J^Fk6O;SQSivX3;}ti zJw#-}y%SV2!2zLOSMcDUHI%34x0GXBEn@H_OywJNJL4 zYl>)iWU0|CA&?APNg&+Z9-EroOG8=q@!EAs2c;DDqCtf2G7eG}96;RRsy@Ix8{iYl zSf+tITJLr97yHUl7RfnqVFVMd4?EI2c$cHPu~-7G^txFH!^g)0CV;rz?%m$_=0dwq zxYbDtn*9+>B>{Fz&NMj`)MUlt#WpaBs(vI?Rpq@)E)lLw<7tqQFh|(DhK}LNY z^f5R=cby3GCQD?NwcBpf^xurZ7#V!hLShy&-~3Y z?|8OPb4sPQ@KtJ`;m;{BwFai(RKA6{8$AYmU!`cUV-+w%&~HOZ-@mvwP@pv+8%L-F zC`n!n5Py2#nZRV+FA!J_=Tm<=IuwTHYN5NZXB|XnQ{MSy0KkQDA);gUR^7Gi7nV6_N=;rI1Y4=M45~mJEw;3L4MBM^pfnjgE!7!w-LRUP(YaZ=9NW z+0ekq@>q#bR=Sc_AnmUVwZ;~SZd8b?^f;W}+T>TI*@znvj!h^5>(ut0r6$U`7fY#U z>|o?$8D#uwp#o`9YhyrO$`Ms?fYSzf>$p1^hiTIxjx90^O^96VFrg}ZEGc2C{zWIb zaI`vey(v1@WDm8U`(uCc#`Uo>&1At=Im^CRQDGbMRMl~t@d{Jp(V04;?Tma2RCDR2 zd&LQaPwS#7R#I0$y`{bWSRMRk2hFdpU&Ivmx`TOruJwuIwW? zJD470MBsX3|^Xa64!|K!Rg;9_Up7Lh{1{|#vD^^`#Av{sd8ITbJ1R8#nv>T4#!`E`>b*^;$6@6vzA*{TJ zlWWpOk&@*qW_x31+7H#vG-xfqt|0vFT-KZ-oRb|dJFSWbSQapj=l;E>WMIy8r3a(k zoMcHcoV}$A$v1zJCOOH%;l;eT2gm+IR&os#L-V5n5JfNP;8`vR-=)Zz%>ZQH4Soa! zl^6Td(o#RLtH5XX84NZ{_^XX|Lc49}0Le+AV7*qVsx0Dee_;mcsY8#>@<}-A`}uDD z#ZsA#RR4v5MFNCT=D=miaG^J)W?ureM_~Q-qZ6u6rYnD;SOw%yjE_cd70O0tsRnoC zv6*SrU3KR~b@{+_|~12b7X`1C)4jJtZyXUzZ(qj=6L(~M{{p4lfRd; znlI}iKYiy;*ljGLEc1ora5l@?qGbO!V1i7}K;(ZZwioPSzy9u~hCDn|yu{bRSXfxg zr#kI1<%{bmp)~lyV`@K@pEBV zu6a1RGO^A^Cop`h7c>f4R*^P%^yj--EN93MqN`sqJ}9X+$r`Hi#uBUNP@_qgoNv!@ z1*(4w`ld2=Rqo+iJ5hgxw4-T4lZVZEx1tN+yuO2cnwAkkv`^gpZ2+w}@wrZa)ZHBf zOp?mb&DhZLnEpGUzkgL*=EnVxi7f9ZwHYb9p9I|O8@pRdw+Ml$zm(8%{JwN{I=x4j zE4@Cf@@b1rlo4P-9!c)kBonr z-`hgeo!WHTdC=(wCwvyxK=_4!pwF+}ArxY{?}XD`0W^t&jEOyR6em4HleXcD5aKa`PEdE=!b@=`NPK_n$R8HK?PlHKaMSASVOnAfmqcd~b2CT;} zpSw(~;REN2pS;-gOM$ga=K&Lj&I6C@=BKGhUg8tU-ZNb~gF|waRihYEmG4a)i z{#l2e`6fobWxudRCXqm)UYt<`l{1FC&Va!X9hq?hAZ55KA~1UC`c;3R^G7BK)Rx=0 z;3Z`ihnMOe+@(mKR2(yFdxHo&(I^FOs(7L-3i4^g8ghdYJuAg*D$RU}<-i(l##iLVMl>6T76DTby*=bj z!?mg6&uTJ6SJ5YI2S$IT4e;>MQ9U>u8*8i&{8!bhbl_cnL>2M1U}<{IvB5O%ibi3c zA+Et5e7{VsxH2V`D9TxQ|Asf+8_+v4(~vKeB7_#`T?_j6DgGV#^Ot!XJ@JB5QHTux z0D3GZIZp!dEp)pEm>EiyR8y0PJ9RlFi;4cRY1GdJ2OOCRV&{K&5it=}C$}G@2~d-# zg5C*WL-|-j+EuPDWevQ-ds)@6C$(J3)LeDBmVsd}3)eH6PJaflcmxG6RiJ+^OgEAw2nMSHKIwkv} z_<;M9nw%0CgTc^z|90Xs>*KLj)`;5Qlt8>k7@^fj{J@+wSVL4lmGq=;_Gha7&WQCx z3pH09Ll}P)4%cyxIiSXe;u7_9Dc+-O-Rco|CMQtiH@#wNMZ}}Z=Ku;T#6U&H3KQC|EO%Fqxuy;2D9@VqHZFhCUku{ZmUuBLQ8Bk)uRTZ3W`Ani zJqp)uyw1FZuNQB7=1kwSVwn3)%m`f=eY6JEhD%V_hs}Ak9N&bHdnR9hl#M1I+Cow z!kl(NcZ6u&r4YG!F#u$QWWrmWI3tK-ogq?Ix140gRfeE+E#Q zSUg2FKWaje-fnmY1h9;nj}L^FM$xg<%@;SfCsvMZ!)GH2Z?w7Qj0EHHy)5&q+`;}A zGL~7aj-6nvgzt$a#VTrrR=4tHm1rIhW0?C)dHZ|Cc9X8`p=| zA^ML(g<;*~pVH(XnZfy|R7ubj`hyHe^nYw4|Eb_#V@8IjLMs1)MM3{Bvh;_A#*CyR z$qM-|r3#BS|DE^xx9g+h&yq&sUvAj{H$MMwDoB5VrhJC{ zQ>^?g;h!=^{!5W@j{G=Amq{>G{Y@A7$0rz3<4cs&e;HO8hFz>;`)u@2 z!SdHU|M3|5zph&OPsvi5YZ-qrVIon#>^pkvnhpxdTVCWwC+fQ3I$r|CNnc)T;2{9= z`X8GIjA#bGe-B186*nB@5?iphy&XIDiDed{Izn zD!dJ+dA;sTNPduAV`=9;;4T-UpXx*J6=11 zOh9ACTJ!-*UlQ;609^rGYqnfYU|u@SIh1j_D;!M#0Lsebm4|2}*i=X(NwA?m=E3HA zFkhUpp-~RUl2pP^{d~3KwR1+7_hY=>Jp;{H!o`tUK?$spB;3&SE2)-;Om;EfuO|+S}5rP`Qd^HXG$nzWUat7V~A4Xd<&qYFQ;o_)a>{s z_ha}%v0Wl}R(_>M;m6#|QM3*!)f4(mxVDJE+2yos-6 zagK*>0LaiTYS(`V6oVhZva&RLJ=b%hz-)W8Rh6`@QIIexgw@qG}=yIWazEFodjj`$gZxIxtnG1 z8;u{=wh;H1(?II?C^>AnDwmTO9hB0Q-)0Dmm!H(ZFr*;)$Y=vToqp956?0Wp;Z`su zIB^Ce=p%nVH0f0QisNV3d&i~)E3l?(_~QXOLc@qip#|pF&OAXe(lY-@$VI;Zu_5S# zlRi`qp^U3M+ePI}`<4l!NX9$Z@|7j^0SuM~8Mf>F@(0d(w9ZW3d$D+wCfv8DH_NxO z`;GT*(ufCD#9qrsZ?^5g<%1f*&#*kugSc1Iqs@Qpqc6BJhzo7DOZPBnBhgalde!T_ zxfQ}FP~)s=i@bv>N97uowfrF|f!zntNa*xgXP;s^+VFcRA&j;IYtUK9Evu&p;r(0bNX1s28a_h{o9J{#@XbTU+ z)C;UT9ZHObOUd}IU$L7c5Pa2u1%6L|H~~N65sGTdMV{M<4IFWWcu8z_LMN~n z5eo;QhY)Tl}#3e@CzyoMJ`Y&!`t5+WQB4xO=3VtSCZ z-Qs!!J4|Q{(dn3lV$7{~5FU0vkCt~g67lO)q=Szb8)|UH>{oftst;i|Q|5oXD8;e` ziES;Tl{vb({vfzS4>yu$1Xl7a$51VNkCG^^o$e`mcO-j;hCrj#a8aIsh-PXXdH%ud3=()^}3Es;uM3*@tmJw+zDHprO z-d&*Fy{x2i9LzJ2f3FnZ+X3Q^Ih1?0$~&319ed!;*Ky{7&!&?0x~_kbBnThD^oRf+q^XPFQkszMZ^X{9%8@`Dcr4MXb1jJQ3tkAG zbnsV-4j^_WenLR#f(_5xB=f*t3Uhjjs3C^x@bM8gEVsd0xMq)71`_F{*xoga?`aEg zDNlr^eu`>csFIUcEYN=;&*0BjHxpY)Ddo1GT*{_9+A6@4OR5!@%s;FHrhFX;AI2vQ zPQyRcCL<9`%{D`+{N6T*qtfGj*Xr{Cx5#}i8?Selec-Z_r$fTG`wKH5- z4XDQ-@t+h-I2=&qc-T)NS$_6z!};A6lGjR$!3yoIL%zZf$eVv9hp_9*{Q$G_r72Km zu`J=nBU}iCLU!nv-*h*7u%F0qHN&qU-8{IXGs}84H;5~wt?+)Trj?3=7&&^$CMS*b z!rVs+qLq^aBI#vpyCL;nS}|tLMcz0xV|#Tbd&J6%5}QPS^ia>4;u4T3B3iZTSD?HP zT!3u^aI!;}es6yl8K5t}<+}o?ecFT&Pu4AE{IzbmSaj1bb@NP_7;L#h3j4rug3BOo z%!zH|o_Hw-&W%s_691u~B1z_JJjQtR5yuq*EFMTPi;^dGZ_{vk1EtuH`Jp%jJd@59 z(Y-EfI9mQM)vDa8-(0NJVPUv@K}EyiGE%+)6Zx>y>Q$d+3p)& z{-&CTp*PhY_2?u`ZKN$09s%+CZ?@~V6+D4|P(?m6WpqoV&ymIiC1d??Vm~VWGPVbH zoHJ{bgnrFmEt;Ak4I8-nSF#fZX_Hyx;4VKx;9|Mj!H6e=>*{}~uj(T5E+=&}3 zFC)FIx`o01)O+P%FOiLtLVu8*855r|3yzWi=z4R-y>ATG(fWKNO@B0`7StpCK1XCW z&c#yh08#nJjXJgbRfjfGXG`xn?=7pcDc~ zde2LGtKwmPlU_ZGVvf-3H~~WEG?MQ)wS{*>XT2UD| zpg(^WKps1L(@d)F#FHfLt%+tmaE@VF2WQ^^=>1G8qy^(e%&uptGlGx_aVM0>45X*; zG$B}QH{iu#u2-_0gRJ6(u!NMhh9{ubK=?_<#DkM1;y8`An0zD6Q`;OesZ&`+PT?*@ z;A-M@5Gpm?)ZI*d@{7O&ZUi)T*Llp)ugHH7y1P5cIc_GE1Y&$}-N!l*0FTj6j=%fh zvt8;S@og|XdtmPzMR5J=pm#%KNh_11&iGQHYNw(#_%!HSslzE5xs zcefp*9{?z0**$ptQegOupU8~*;Z*QOTwjYeKSWK{+(Ig*)`6b7k2mUx@5#fo{TY8c zcWT#6>qV`F51oup{H1~R<^#Xid0S>~#7q=s-SqXO_S2hW){i;MD?im)GcqOfd?bLA zQU`nm5@EwZipVw(=VG`;MEZ>+nWjcXAeGhWBG!mI!Qs42iR_Z0q-!*4ol@i0!#%y- zJNJb6(gsKqzU()WLF0A+NycaKmt}v__q7NJkSn`R_`T|cT%S3BftB5M`;rex?2B0- z4K8#MTOaoivulToo%Xi3K;~RNDDf|6Vd}4|d#n zc{gqrpR5~oef8KPhHF=~nVXxsW{;*7=!JQ=UUwB*%4Oy$iDNLg1{+0`m#=@riePns z)ui3$rYNtX6eg-W4IgJBiY=wez2(l_yN$QH13EZVltlq00IFNSZ<0 zc0Z!LBng#?Gl(i8RZ?@Gg=Z2?rmd=eQ9oQwUD;C^YcC=Po?$2lf5djoG|g*fcwH8b z$k~6l>YjO?_NBBa0u1iBW_y2#X>u^gBj8A~D(pnq9WuRx&1ApWCrVw4ND;rBT->MKDb!%o`IP#8C-bLotg zL+zW7O1f)H&4<2q93fQPcr8sk3wTle;ptz>#L!pUL_}T20$HFcI&FWD52>@D_VF5g z_TY8uEm53-4&&H?w8CQ>&>Ef>eUlv{zg49d$o7bC8YoP`AGr8{kYmIT+u$fCzKej` zCnAg5pmQ-@+l)%la0jtIPlmnXH@K7lc=<_YhWZn_xn^wi+&UZ^0fJV7yBOXt5&2A9 zRKV-{7*-~<+VZF`%5#5y;apWjJi%x}`FJ?KKNH>Z+ZHTr@5>vZ2{!0Us~!p->txTP z`Vhh5Ou^!d+NR67KrcaXyfs}*hSVMAxwC@n$Eqz2O`6zsr~#Mus0RSHuF`Fa64Lay zy`xxWFH<)Y{NjBqgTkbJnAze<*wJQGjJ_ zvdz^)QC^8d6u}Ce6B3oDhG>Y1NEUz3T@{oV&p8AkV5JM2%KiL9ZNSGFFT<`|*u}_0 zjJQNo8soz8XF3RcKbbL(Tb>k|{oQup+7?h!AG4#hrKg4(MPS$LZ%YS`~QtzRIyPK7x zp5i+^x5`hkBzo2L5eE*g**;LWe*43|KzhaCbBp`52|i83*`)`6cP=t}oGdM~1r2g> zJK+|ASn)(cGXaVgc*B}8sYjXw5-8Tm8=ibdH=Be1m$KYh2s3Q6`c1O$9a7FWA_O&a zdTNtGGE~EGfqZ$aVQ(dW7iJzqRA=+E!7FT%>gfd_ z^9C#Y!i5aOIZ;}bI_0yyl5f;SH-<8P%XBE{`oMH4i?4R3Z9x3@>NV?+8a%#ZIwnX& z_7mn-RO{C-T3xdHBMpLB;0$(fsw5}IlEie!BK{K;Uv$5JS||cI@|Ee{VlM1H^to2p z)+F6B>hIZ&oI1CVp=DWCT5!WW8Rm~FJ22X!FZDw*z`~Wb< zwSJ!^C%P?v|FT^W%zz%=SU&=OuBN@FMAp}sU+K*mr&{Zti=I92%Q~7y#5>yDnjvJe zPsVK|eh}I-=+p+YTBq)jNjU}&S=qA!t$Z_%Wh1C54d9@L+i7Wu%?n}r$_nk!XeMkv z&t1`Fol9SRp4{r+Jw6WlpYP zd5Q6hi=taa=9Z|}eckd}Q+L6$6>H#IS2GMn@d94{41g8b>)lv^ughkLyd zklVjs5HWs>(EkBx4Kc~V0*zWRXTgV#y|GduKzG=(r6HA0jm9kDUo{J$ctu^;`gvcUgCsr*l*wcKZXvdqjPu%c0d=xH8r zNqBaue2BhJ%t@Q2ILud~3|X>;X?>CaNn?^u}wDVnM3gm zWZ12gQn$n_ui-H)yCjl{>4yH`9AL<+K1~MS+#1guiI`(KB{+}wX^;jox5?Dc{d#GC zxR@&IBoZTo7DNAI3;bt1w)w;DDr~|I%hOCt%rZjzF`TIQi_mtH%z_}@jGd*>HTZy( z4H6i;Ee^vFprIdepY38ESLbQ)EwWu<6^dHMOhvz&Nu@zFq(7vCpmj{X8ZxL=3DECu z-!ha2)`$W~qYtV|0_hnVit*n3%s_&dn=BC<0hO1xED{sMrxaV4^*wNI+kSMgeh zd~uOW;)%>7-QE3?Gj1&Tb6lVs=^|aX@0&0agOJ&K*eM;0DLD4l~v ziiD_4p~s~f`>(#|n?~#@_bGfL8S~0haqIKv>BsVO99G2#DD+cnyKbvU>*J_OZM9n_ zFB}KEtVmve842NQePc7aYNaG$6;aJC>CQJfp+bpDqDrT)B>x%JULimy16KFA7}#A( z!|!OkQhq)>NR3tsO!)16Z}%Q#^{dOfQx4Ue(rtvg?$mXESMa4;vLyZA8r^h-8# zB3!MUbijrY;aasQb@TWdSFYue#40ZcPhq{95p`GJd!P7naSNzi#|mL7^4=Bxx7zLVN6+S{r6MqE+fKBCZ>_YAPsPfnl0J|L z6WAJD!b7iv65(9t)VSPn1+&kBZCM$KWuaJq&jstQ|0y&s5#s8QRlLEg1JsKzB1$XB zV5Lr1hF>3PWAd~TvLTzW%t{p>fIlp;&Ay?&;7W3AB4z)~3Cj{eircHhzJQZ*!HdGW z{ec;N*qw%DN%m#@i91~(zv@pmsWsK8?)hzm&p@NA4pmT&shgl_$ePM24NQCyO2X<@2Rs?*n z$DyrG@2y(#01OFz$8gl{lGHrI_Yle8*Ze3Xyxf91-o-eR_+bi2Cx*IB$VrypRqCrq zR$uaK0~7Uhw2C%&GozLXpkSClef!$$-7@3msm4AzC!cD|(3UWnTQ&$stf6n0X)qBZ ze}4deE5Thq#l-CuKRz+j;9DmJnc|ncZcQKj1;WzRO2RG|5imhGHNX{Z`Rr7>d62|JCMZetUK?}4$!o^BAG?GICBp%olm&1F za1Pz{wkUhdv%hX{+d~yCBF`!#QD7~)V3U!)Q4=1V_kr}&CWXEueMw6ecevo^e+;r+ zQePPwW4Bz7|3*+o06}RQk)?4a;;lj*cJcHy6H)qUq_Fuc08FVdU{^ZN4Zf46oBA8NBU`3Q>MGuhbMk~NOAm0iS%jmoVGqZt|# z8Bm<8kJPg)^QP-9x&hBg(3YiPGOVIoi>vW2&sQqyv>D%5TB&%oMn}A#LrR07^M-Xu znsYFlpO;qv6SYWW|7!Z?}NO@8-f6sIHXZw#V(>hZF``JZ=kTBB z*NJT~$yt(#of64!SzU=4UlP8eYgo%=5#cs!=8+ZYZluD!xni9XpY1|LdLwr#B*SWQ zJ{gLTn3N+)13synOMB24F?u}(Yr08V;Mvm9B}%7sz%Sal4VnV-$DMQ;bV}-aUx0An z(G9!^dDk@zygm?#F%9hU?WH4~Oo@T+hJg0H+weBU=0Y^Em(B(eRj%76&yByg{QSZw{&923K+igIWkicegvG0J+;*|c($Re` z$M41Qjk2m;DCc_yy?>%h)T89=<>&0M$Ta2kK0rd!=Tu*2QEuI5!E&rc1CyZvlM zmeXfCCUAYwl$UJ7?T+(I=|+~-pJugM>|=HvPhNglb?d`pJ%5F#&0aTisqeOL(A-&i z;MV!>)JE>EokCwMjf%dT^7v9_&hF?c_g5ARUE-QNzA90N#Tvu8vZjyN;M zV-D7T?gjkYR)4_9^|HK@55)BOitPMfW?g2>cq4;;JLnf@70Udz7+t_c-V=z zHN?(5IPAm|@J2#@z{;B@$ngZzyHr$3!(?=TXN4{Z;2SkNEF zimW=^PJhK=iV*;>!zqWEf5ag5yDDi4* z?ZZlv5zloDWQ-u#s4UfWkd|Jlgn&5^ziDs`H`5?bS6aFys3#Gi6o0@o$KwUFQqW?s zrvgN;Jqi##5c~jV8vqVJV7Gq&i~j-}PG>(r3McNH{s4~~K=!`COjASq(0%0c2bl2} zh<_Z*1pxBJ96(M7@aYfeY4{Dc>y(h;PW2VfMvR1hdw^Y}XdQh!9rhiRdRm`)Y9Ox> zU=WpZ>cCb2vj;j?0)uBIjg8-Lj|@_86Dk{>(Dnt&sU1TATD?l4?$e3W%INu z#lb+4^rBG*+^_Zc0*VN63g!aLPowD{;NBm8egnUL$>TS0_b>48U!cZ<$KRl{pMSA4 z1qZy=R4WTGd4&*PHeq7;G&MnyIe_IC)#n)oxeWoabs%fH-@fdKPwk1h4l2jB7p;wo zhZ+TuTvS(d*1^;V0>IN5K?rUEtTO}z>UY{&KK8WEiOpHjX7PiEK0S!TCWp*WVdY2I9YF$7beVkh1WFIsInFMV7$gRP*KDw)Pz2)X+40({sveiU%-~X z^y4$uKY+krAf$Q!2PozbUHAn|ENtYLUd<$hnd#L8oukE_W9Xpe*arsE+${mp-BOiQ z?c~-hI%Y`RV2a?yniqCL`#dP&6#$yFzs!1??H~QDwTI8l{3V1OXc#zhb(uX zv;~}7Y9XtDO2@b-sa;(?3}qX^497T^&7iCOk9Q2lBcQGY@TlepFjrZn1DNYiko5(q z(%$(11V>-szryZU@EiDhur29EW8WZ`&M$!JpJVGAr2Pvlbn3oko+Q)ut!#fxorDT8z z5XftrPzR%)T9Bq?Y=55@sBd;a{GtK9$0ap4Kp-hF;UDUF09ozmkA9|qf>oE!08;z$ z3620v|Jd+%I)UGy!oSAi7YM5W)BOR2{<|!a$N-L;8Z|Bqvm^_`s0~Awszr0N>$ua) zy}gD2s=iVq&@O$>HaMUKW|0>D`smko=Y zxAVo5KR~`cP=7?x4_W*KW1R>ez==>W1y!-;3ukz`dlPL)J)2*5f7N0tIus=woS?^$l9IhN47T8L|ERSGE#A}dcv-) zHITY?L8yl48TN9mQi=>nL!)5W1STSVepgw20K7J^uYZiAfOx8}!91o=$G4RVlr5mG zs+-=v(y6IYu9_eUQP*mK$)<)H2^8RtE+_z;LS1_>m*-Tce73nL8^ACW0IaBJEkrTI zq!ot@i})8Y_H7?rpb0SgJ{olOOJ`?4z-xtUevkJmeSh(Mh%E&;JuIEy@zD=}Tc;8> z*AcS(_po`5$` z5MvtZq#ARYX79%VRPNf$a?9?Bv3@ISt;_cYRSp&_G$AKInS(k>RY62ZCEzVF1e6+x zOBv)Owbjx{!ft0#(VobHkgZYIV#ISZ=vfK@)_)ugy>G~Z>KCe}~OlzW&NT&``DX|bt6(2^o1fM&C8H)G?gg=0lKm8o|;=aEDm<8Z;YLMR`$=VQi zhxGhyA0bOW^O=tin*PmKe*uoWDE;<*B7Y7XbP~|k0tL`j*xnr|L!^qfGzeAB-2{>g zA>Cm}j_^zjMD;!54a9)!pzkXOMy}RFX42NNs@8-h2r;AKaROZMrY}na8VJDs6}7F= zNrl0zF<|o1)Avn7d=C`(fM^5_;%$#-(}%qF0Of%Oq7E>3f@r-~sRGfo<{FF~6MygP zrXw2Zfv<6eps;X(4;?SB=YTU|OQgb=eNZU5UdSR2VB=APDsDq@*g3pAV-pNF47{%y0P2!6#2;Fx2#nEZ%rV844IT@9Z3-*j-#b6;8fGNq5@h|ozeNA z2a6vYQjm`o8Ryhi#i91_?#AE~H%-6%^yY_-g@Ydb^8aMe!Oz%re#VgBynhqG>l0xw zzj*r5-7nB}vro@v|HnB0WgEW1*niDQ-zmHl0{5P3q@teMMH;B(@Gu4k0jaBk$c4S6 zL(u35rdzd$T&;}ukTzSDODbTP5K1&yq~=wUi-hGEGDcuuZ!v0IHK21hU?)NyqF1>N z3YkD2X}HfFgj|jKwT=pcUVogbfVaozl5}Qua<$ZfMnEBid7w(*F@BvjzauV$z)^i- zqt4L*Ra^_oJJ_|_#gwoaa4*%h&iYPM0 z+`fqPU-BAcmjBM1R1)(baS+HcETXf@@|eUp6VAVHk?MvG34<}=^w6ccqy zp#gSuCa~3j4G(AE4$jkw<=2U^nQsX=HPL1jVH!ciBOUvgU(d`Wq=Tm4?2pkE``YQD zLVOPjY8ygyg=u*)4~E?^VqT@>XeQ=WSQ7d+ALdn^m=EUCkOQi|VxHoAZuP^rvAq1U z1JmF9$j{_^UVrtm3j;rK;fz2Poqxq&Jk?EFTYzGu-QgV+^4%=k1GwZXlM&PPS+>5; z*BYZp?^zZbQI31MDt!s*Cw4`=d1y4Z)kG-)r7B?Vk9oSWLm-6^q{vU00VpP3Uw4qt zRq9KztI8^5B8KDvpc(f1sBjRL(P%l#J3w_2W3p5u6n}?+Du!8bp~JuD>=Zy?*yAI0 zLZMHIi2o&0e#QUiZ$I=yS48Zo|0Rd>m;Bk6&-!hTf9T$ydCc!QuZZ)1<${=1W!fVT zqbNm4I15#dZUSTl z-klfXf7eeDmwx8#KXmiUw|(-C>CgE8&z!Kxg@0Dj2vm7LjQP)eYRFG&^zShoE}Y3f z=`h}$0_S8#@syox>bUC?-?hp?pu;=M$Ei_}BLXn%|h{suVmP#6cp`Q z<1_@4mZ^Z0ZkBEUBZm~G+OHUH#d0oO8{#=t zXSfDIL;$6g#fXS6GcixVA`Wf{;sP`XY~L};hk3%c&-q~w2fOf1M zcc)7=wQDrMQ;6U3i3Pa}3iB4gyhDs2e0Dzz4OY(%33amU6m&va}Cx~~LK0um|6$);(D8EZ&ChkuX_ zpmybW$yG{iSAgokyAyuLX}{vJ&pb2+zxnmW0~rl`=m*~FKkdbjybPLGV2gb~y^e3P zsoCje6uPu^RnBS+kcB7a)YPnYR9b7mF>If@2q-&aMrLT#qFYZ)wV)wksRpP$qVdEw z3xW}kyr^bm@gxK=2|*Awm`8@{D}OrlL2Z^Df?O;Y8RO-6T!T9n^U6f%iwmM5pZTRD zf6SYH&l}Spb4T*Y-F;{xt;jz%l75Zpp&qLLp;Kl0WBub-PCUbrvskOqPVJqQ%A3lB zc^TNwSYy&2g664#8bh8EtU4==QH(%yP&=S)d>Uv){Q^Aewo)|5K=u)$7k|__W1A@^ z;!2`H3Hg8vuyct>X%mrVK>~7jVagYVbTc)gVOS%;X=L>K#Y%HEmFE~z*)#f7*I{Pd zD8_JZY8G!0V9nSr@r_>L)$Zg7gVxnPdC_dFw5S$&4|TAQSHK4QD1GJPKDtvmKm7Ax z@1!4dM&svN*_Y0I#%nKf=zlfW-|ao24FVMJ@boDr*$)CqqH^lkRf@H%=46_bK!zQ_ z>a|xWS4>F?b4J0F2h+&i@UD<=N3|M|E6e1BrKivGp#4tbt$ zyCQNE2>cPpMUu^k;6rC5_^uZs&P+|jvqutPG564HU;GRxN_?t7(8;sBYq8O_jz$$7 z7Qk@<$+#}3Do-KcY>t_uLi1i}Osp9)7gOorm23?zbwCW4=4W343=3T{>+JDDz|jRd zqd`+67#Tfgv<$cAQGfNp-VDhDQKIV_4X_oC##d*Jn5+ffZWFLnA3rW5rE8(al|YYI z^G#J8V)e6=dN>KD&&7DWit*@iHrVeUYknU-4YoRsweLJmNWXMQtj+uyv&PqY!*{-? z*M80yMV>g2(T~{ZGdE#>u0IU*2Ve3i9q1#{V2+)%Y7V$dEq^HT>I_3WBMvC)wbx0R z-Z_*+rKwY9K`LD!Wk(aV1kko{>Lyk@C4!&4A%wH}69JL~&Q!=-u61>8`Axk}r3V7c zG61mwGtZ`2&jZfYDS&UmZiGbqR&;xvQW3_j_Vu<|xb^ozp>}MYLEAReGw*^1QM~6H5{#tANl^-_y{O`G> zKk_8M;(y6u&CTKR_L{b-gB05Y#hmC-suuN?N>XkHjs-jEpm&as0t>7e4q-phd>F+x zkV7qg)H;)2Dj+qc4MJt(>&B{wfum#-`qlz$!BW{m#ywCFww|;WsrKhBQVx0%St>lZ zg+M4|Y@>2oY&%7)OW9hd#JGzP;`vEMD+5w&i+|j?Msc7glZyc`(paiuvA#|t9a0Bs zvuh~;b`fw|%XbRMi3u2&EiCpTG&W-V3p`Mq@FY%#B40nR82+b#3_k?fsvkP=b*NMD z`DF;QK}ccAjRiQ#Xu{nXj2fKrFoX%#B%V#+D>pX$7$dDED|YHp&Mcqba@5K8gL0cn3sXQAI^%{N#g4C&_{0OtUp zjA{ywfGI!)jtU?_7=Qz46@Zx;0>ssj0w0vdSZ3B4AO(#WIDk6=1Oy#`Ty@vu6X-bb zm?Z;9R7ZeTP6R*9gQ$-P@ZO6(UUKsUV1KR70LVaF16OROfeM<4JoBeZqQ9zkTy;Qe zaC;n=<0rs5{o=!4{tN-Jdl!hYKpPN)nKkVDseomHyO4_^z+WNwD@ddep{G#by&peL zIHOq|*J4`~5eSjJI`#nEf-VA;*zH4B2k;TKSl9n>mj2R*!Q*Jd|6yNMBnq9clYh&; zJzmMV5?4NoXVY&;&1J9aV!7ptZ-bHyOeK9~T^)|UX8JK)f%$AM<1Er9m$F4DP*on5d5hv9uVx6>V?K=-(c-RcP< zke*IirfXRX#6gmE)dENpJ%7%U0M;$cR3e#~y#tz!phj$f%|xq-4GcPgX$$g--T@&J z*^xlhp)q?0bo+~^pl?}bVY{&ySPe_v<7O;Hc{)bo2R0M!iK!p8vz33#+X9>zbB zthY)z08&g7gmUdK&=v2EzpV|_dZe>seFuu@Ztp;5+gSaSH*HJlr@Zew=nv(&ZDajYE`PNx&4+TM>^lU9@}q5Ib0|k` zEBO2^{}yB{nD}B&|7?M_7yd(xTO098=*;)H)Xvb3HC7R{;9!FX8)NUl!$uBkAuI=; zF2Ok!`hpy0-vP{vwjgo+Xb}*I%WG?i0jT3Bu|Ys2!u%cZ1V2xqYb>X(9fQkN*vhT#Yt-S-4ZDT}`=pSuc>I~&((033Gcf-9Vd|JZ6`5$;6IBbYHN>@)qwj)bmQEAITnQ&Bp zwIdPwo2mb3Z>OO8w8P`D7PcMVTM!9*E#eCFPdWopR(#?dSd%4C#eWey>Ah1Ve}A;6 z5VlCMZ2+`8*a+Jq#vKe&W`E1`&4u2qe$up{sb~op!D;0$McdrMLsPykd!egb>s{0W z=Op~fM^-uqnxEPwP(-XLp8qZryp2x6XMNMcGT|_Y&5vKIiI@|lQ>XgYt$#hvohRu5 zTa!i>e&61K@t^I!sszoyWDiBG!hf>+Kk?A{t%De}!p8f9o^QD(xoHF+K@u~z*6L>) z0;mc5hwnG2cl(Yx@TuSz5iyC^x5c1~Hqb>( z=-Hog(RTX|xOg`R#CVCAzkeL!RuN}I&8MM#2UDw@@}T_Rl#6cz9N{IHiThBvG4L{eoZ5jT0PQ>-sbLT_f zi);F!AFiVh{cxRr=!fgPZN>Z@JL21^Z9BspL-aoh`nQK=ddr4a2_l`p;^<3s<&6T(E@zi3O*?(i3_HMHXwZQHi2{meUUpzBM50dqBx zn-M-MiO9VL#i<2O9U0~C*fEfR$W`?s=Rq;7PG=6jeclXp0PIma?AO12t~tkmHn)}n zgWjD0Fl@Qla-nBsvOHE6kOl+sdn3AH3zup7b&qUrP**P8r*t6~oj^%_yGVK6^xaDyT z8N@#Lw3xS8-MbI7!4==f9DT}dk7E#&F^v~ z5E{DQ(SWEKW%Rx&m?l=fEIfOW{TA zN7)a_O9oO7uA(Il16;i6QSbmN5nSuUJrR>MJp#E{x6pI2V}OZ$>gjMFfgOXD_-~8) zaQ(SQGJp8L`|hxw_v5=veAgEADTU8ZK2>}NgABkE&?Ru7q8x)*+-nSMxcD9hJMd{| z&R_uy0ZSq79S||`UJmvYwt_BaP#5jnF(?6?f*pswD9v_D*ZGGp1`+jf z(ci7;CzAfDy~!X42H)a4JO1H|L%~z<&*OjrgMvMPXFSoC;D1IzE1m~m_cr^~zXosyj2!GacnZ0uL%SK+ z4uA4vz-I6z+LduI0;vA~pjZFYu2%4|5PVGkPX8R*!{7y|GO$EHK?C+;d>B-~qOdQ0 zcz?-nD*k5!-_nm94Jz8@4f;#rY2efUmA;7a1}s58$lGv4eZTelBaau*Q=uQTFMS%m zvlZ<^;A`ACA9)$W|5`lX5Aqt?(GT~7o_|N8{!onfaOlS%kCWifU%wmvt=^9w#Nzt~ z`LZj@$F8VY=tcKsKccVx{fqViHt4y~6EPkJ`4I7c=+__o*YT%a;`iC#?*Xt7a@qnY z3A$X|BL&!8$jK4>xeEKS2jctkS3CJneSC?2`wTiC{S);J_TnY#$%X!Z^pg+u3x7Q_ z9C6=@`o8Ur&EZ<;LHJ`mZ!A0C>v{je6)9iqX+-$H8lVW63|s~!px_3ONX3DN_>Rp# z)}8B z=l}WN#EAp{<+#lc|KscGT%P*>uz$AUVe{X$`?h{O@BeiuX&?RR@E;dZ#p&~eA4*m{ z{$nXg|Fym6|M7P`qv=mTBj+2?2zAjnpi$_F``7AgIqqL`=?nm)l&rxyS0N_oxd+m$ zg)SpCkWB^2Vr7vsm=+@_6Q&Y;(WfWq2ktG0G6|`%e@cF|yJYL}T{%VY+<)(1P*UA! z2kO!?OV)%J&$&MDSOUSVpU-4?(2s>*dQ6GK_k8)P?&}~Ew^p7mR?df znQxVFKfQPdx7LH{BRVYSv#}S1JChdsWwV*mk@|?YPs!V_^ZNnpo82qvDyrLCv8__c zQghqZ-IGa+BJz)Er*zw5)PKCkVb$Np<@UZ(jK_4tkz0q(=O}*fFi;s@ev~FYIqVXT+-R+`| zyF(mF`|HD^%GIzlRqC(1nlGcm40id_E)u>4(|1u^b}MYm#xN(5UVrQ!s~*^%{A$j- z`DheTN$K&-+_%wEC;hHn?`GE~xL>U%pGL_z-xO!_=BHQmvX#zgd^%0bdUP8f;`Ex$ zr7k0n>%?gHi8N!>w8^39R4R|h$BAi=)^lqXYWM34kbq&2xMun~U-i|ao6qy*Rtopa zez!hvAFixSTy&H$hY&PhN)@z613d%8z`JUcsbxH#Xr z%KFD*lhm%fKd)HCXh_!C>+;@3c)olrDi$mENWph@ycT`$C{k5jZwI)U8uF@o9^WJW z-p)J^9HvKmZ#(v>9*?SXoGT9t?A37`p2$9!hx<+Xw%k{=x^zy?9%uVJK2RtH#?ib@JzjTS_fBhzKJNig$*HJQU2^I= zC?2CbW&OxJtuGr%J-d0uchOWb!jV25g%1VUHziRo%YVBwH|)8kUb@tKRe2uD*+!$|>LH(RO>5>UFUJeA zD1qV37k_cb_RH(Wdw1L2e%wu(bAGI*QS%C)`~0ThgS80eZPM92Y0mG~90yM-5_IDeBV)8&jT*4e9y=h}HBX_$94ep~g$ z8~cW@_b)gaV}WcKkP2<~k9;|fZvM4!7+m*3A1~(DcH8hi-%3%(ChRV^OtJDseRp0D zbawT*o7^73f||;#Sj^H!xliBS*uN(sE!NB0f1MW(Ntv&M*D`>9SU>gkIflvA@*Mt{ zb$?U)fc1tK*RWQLThG#>JnfaqCSFGI!=8*U>)^0gZFSTcxt1!$RXTy$g88j|)|8H! zc`MIx`lPSxE?iB?O^zIOHKb;t>nVw(m3OJj7Z_YQB>yznv3bJrVp4L)8<_6a(brSF9GE)nmRa)$$C21Ct z?>2RjEsDUlwYXXO@R4ky*L1E=Yj?%&Q>)#t>iP&Y7rSV6P2&6P8MUS_7M$bV9Dl$3 zN4t>2(K4`4$yWQ_w!H)$(@Q_`R*0mw36) zZC2~kbCO;4=(1N?_RxEyxTKlmJLS`4B`p2RVROG}7uvM_20vqDzAto=uYcSm582*u zw)7Om$@#e%zXG@QD*Fu0;;gl2SRYSa!TbJk*iPn6zO6RJY%*y;Qa4FZK3==EIv*(! z$?#e?dvdlt$?%I=g`=wV;rlc(;^pRAtt4NKeEq(z=XpALOwTa%lhbN`&7Ofg&)9Nz zzfZzcd8|kKpqe$grv{Q;Nq+|ert^(ot@o9)wu=Qz+u4D>Rp__2pP_7=Zq3W0n%mOj zjNmk;`d-gsejI+GYNV# z$4}4l!D4+iSj~H9emTW}_suE3oq5nqV*U0G(re`1#$jqw$2vs1)HPhXZGA5wZKk}i z>73lR=2Vcd?4(n#)qk$guk^)nO?iI6N7#!119G-N1bjvC@#s_WdYO7cdX(@M9 zo#|_)9vXM;9*clYmB|R#WaOma)v@!q43+0;Gz%v9p)RfaVJ)-SrG6WGBhujOUUrLR z7)E%$wfrXU&HKZif%`3o3)61arCX0K+WIuVRJ`0TCsU@3uYXFVaBK@b{Z|$YgtfF}da4TI`oFW2VUF~&iWCQT@ia8BL zPN6s;=LpX+sDDvhdg<9^eN7`4xuDIhgX&N7&1oMFho$HjC&M@2lOH4cJiNT|{5@}u z6+hQkNDgLSOPj)k+r8aK<*j>;E|bSrJ7}>&vTHIgGJFGFbb7z{hIT;Qc2)K2mwn)l zGyTxsk>8u^lW86bt%#b@oNAhcXL^t(D+g6)bJJV#4S(3c+X5-1`J}n47f?Mr$BNV5 zn{Dz$S2EnimxC!uqs7(fR=bvuz3euhZx8bUZkg`maPtB z>s{VkqPd9HwYd&Vzd-q6b6hXSGTa>G&E8mMt>ChnZ91OcAecxf8T3uxS2_Z z$TqrUKkdeuI(`$3I{#HfIXTSyXeF((cQc9>H%U{K!??eyi+FVMnx$_*?e?5y@m}*% zB{`m#Ub!M_u)SPd1FE;^HhLK3LS}sBwzui3ntx}Bzn-lj59%?A^mXmNyiAe~akD;i2S>CMUQk%`jhZTrN z^V&1h;OHgoLh=o;Z39&Sq4faf@)f@(M|tY|W-^^Li`-rov0j*l*F`xpwy}jS^AqUr zY=5z7FX4^r^DTZ-XS#ijJZS-TQgYvqGW~WrUABb-Ux#~lP0_W$;pjMh?v)_3-p)pf z(&k{UiutMU9BJHdXV7Hx>=+e$t&NM(GB_r0mTx5@MY!9>b8W31Q+czS)H9VO8wo8B zSJkO0EDUc=uGhB#Vm!&%Up>g4|FQ6+muanp-k8niMx zW(G8^a$il?>-G8~wPh0H(X&-2WAhp)sm7ei44!XE;+*g6bEna=-C$*l>kAE}1 zLyIzCU@?lf2p;O>5=Ax-ru}3WKaUUco=m%2TW0TxwG1{6)l+U{%(}<($TsBjbP>Iv zvFk_7*Io%6cKP0R9CiSJwj)rY5?L#zL2ePBd{%y6IIL~nHw2@pq z%UN^IPTg#mY%j4rj&G;rS zw>zKjf)Sa?7WIYQ&k=#o^^>s-glt}qYzbhK`5mh?;88+UrP zzO8lKJ}+llytYkU`;*gZ6MwIE$J=4jhoQO2!&5VTA2#u2_AquR*=ZP|`SR@d=H014 zR!ez$T-;-{L*5K6%E{(6r?Y1en44Q-#AAQaP}#W^<9dJIz_IpnmUh0scO6qE=OXO- z11{I@uH8LGuNgdS-^xstVwGZ3ao_bLd0a2A@`+#X?9Qx9F>d=yD}QaDa__xQI}1`* zJ?=--{p9G39^loqP2AQ?@4Yt^qn^p_>hU<1K_2xwA3M|JdMPKXTyYcaJle0>;b6zr zLS(%veHKjBtAElxmWFxTNwxutx9{epjWEuS|pIJ!LUmgl`S|8m5K=or1zeYaa|Cz8G98)a^8 zP_13A`(-yxa{n~3eSf^T+0)Y88S#F4o?Mca@7^Y#JK@FcZb#QkWq7KStUVg1?ds*^ zJnhiy{2pC0{}GtSx!)HMGCG@QVRNdCFma0A_%`=hI$NC%l{>8iX~lofPp3GtZl(LF zZYfh3F4}p49HNY$1%iR6&9=wscHc^cn%cZOG(~}L@R}ZMAHUdqKHsI)42|c8`zRx2 zHgU{pu)IDj%Cmb1Hjw&h?RCh}~YZf=Vc zpIU%hdQ@h+Ub~5%?9YFxTee15P>)KUyIpz*XR?|l@e8I4HXjE_E#zd`GW}A)go6UB9!5h54-RtG^dC#soChpt8jgP|P zEuN>(eswLJz1&|WWwm>&8U}&XOvaX_UB{L)Sp-`&9?9!?W3YdDKYq-~#xL^MigUNv zM=3l9#61Rea6YeIrn*Gy(U{7Y<91T|?6%upPAa{sJ8<$3_3?gtZ`a{<>dBitLGNR^ z8AW>vdt=g`qs4T`?AmpvQW!X1AzwXrz6r8JG#aZ5C63Ek4R^RWjQXIH;3>t;d2&ze z#X=^NyES`An(Tis&pWS-F|H=Eg6LGaU$3%{G_O2i-(u@>^(^i5!QITyT3?6S3XP|u zzEAsQC~fEA^dVJVbZ+9ElZP+v&l;sNkYg-o_Q@L`+rx_1_uHm>*^fuUX0My|h#&IH z?8hW|&b66oUxW7LkcLbk-QcWYPSh+g^F@3-u^mg41NY;0fqvRs8DFCnYVWxs3;5=nWShi{o} zkpJ{ETMxP`#dbI;`BXn`aE^_*GP+Z97&p~jtIvP+#))q7q`*1};wZ)2t5-VHT-x~O zt;V-exz?n0v{*ZL|5B>6NZz_nyY#)7q4}-v0lHc{V}1`O??vPs@Pob=k?S@S?D=}U ze{R9)f3bDm%Gv_Wdc7b5ybBcIy?ODF;l1}4SqqEoO^w!_kA*~D0O&|$rcp+g?q`sW~ZZ$<) z!G4xfheqaD8`a=e#6a;NdA*1D;l+`oJD>Z>v|wI zZC;?1Uq*k>(W@-qAIOoqTkqTlM7_k|1hm1?T6#XOytx^9kO^s^9)6E5kLbh}plhJN zI8Pr+g#cGjpE$Eh6bY6-8z+8}Tb!nYOnK=w++n1#sGv07%ZZ)^X!fl3^d>x;Xc6|b9S2bkBEXg_&XwVwYD_y}cz@>WjRsT%3;`BQAhg*I zv>TATMySre{@&JGseT7Ex5Lzi8D$vEheL(es#5<^trss%CXaZsY&R@t%F2HNKNHC! zTJWt}aS2lKX7Fl~=21OpBf=5=69efB_RF5%Mq3QK`k@UmCHHjxunJ)HU z7CH27A0NS&xbvhP0?SQr6oh}L#~plui4>KKhBApj)*d^{xgCy$5S>F(Ny84D?((*k z)WrG(b2I*6pRW^#Rq7FOABM%6=}y~4s0q7QqFtX$&Pk;GtLH^{+{`@D{7llP!WJLm z==C^z49x$otChhvzc0T-AA7QGjE8A-@h$<$fZN!Q`@FF5CRs z*oKaJxx}WUHmGIaaR;`L1uZSn+(F)(-Z;*PsZ+dL$L+Vs9k@;FY|}cvcu#2rz?JF97-!s(VVyA9vMaNjw(F^?pY}E)%V~@~lx2}& zuCqe|nW3L&3SWOvLd%ll!*s|s9~ehg@ad7)UGIjbSG0J`&cSw5>MF?TYo5am()0>A z^RtnnZ3&1`uG_gJat*8ddP<7&w@8dpNEb*^d|B@>(FEKJh4{8`gH^a?fTm9|U6JRq z85^KnHhd4DhvV;D09744vBq|2k8%X`Th>tGE(J}A5T0>#N>iR zMqV<4@?%5fv<}kg+BtTHBYiD52b*i}6La+m;43ZEcFr;O>4Q_6^9>)dJS=b*N2F7a z;C6pn>=^~>+{^S83|DqNkOK}b?X6%}5-*NT9*S^8O7X%d9;dx2ow~BPE;WQ@AM&DH z2gzxdJ=7vdi2MaTA_=~&1ww57l01m3o|QR+$eZOmE*nfu9mKx$B4s2EQb`<%#+Ecl zuaV@imAPeYo&B(uv}G1x# za9_IUJa3ub1S-~9io3Ct;SU~0W)bD3=Tfr=pyxaU(7(Hg9h+=iXR9TjkO*$SYbYN` zIis~jSVEH+A1|L~1=2h>>j0NiLS(h_VjpA^>gt>a6i**%sEPtq1u@P7-^+EWv-pTPuXI z7oG2>=cWne<<#Rc16po-=7%^`5X?0eMwF8xsmh*E&>kF( zxh0c??DmqdId>eQUo&c)KMQ}Dwg-oU2V5ou+zGj-G44HRZimuzHBps0QDtz|l!OtF zx%e975w-l!E9@#a-aM+S)=wQ!+|#Lm*U8;7!0M3V<=>e`1#o)7F^f~YdLzEIbWj7o zqEArUe29Shfz*b!IGAqZ1CpCz2x4^)ph2h9`^VEOF?C0bd`P)QK2CoPCn(~4@KLd? zC#v9nkm!#><4%h$Fm$%9pq)5CW5|brgt`K)dq&<8g2jX#Uh``dop&5D{QN$Cd3hAs z^F78q-T(-&qI&VIsCaMnG8uEhF@VB15WpV=_3hMEp-LT0ScG`_180n2~`EbfpZpzfB1L@nFjT z{GJ}9?6JchpReC}s?Q|2_xQ{PadV*U2})Q+fvWUA6=9Z4^)Amso+a-vJoP^B|VE(6EhD2jSUAQMe* z4L_52YSK*dRWLpT7K;RH9{a0mwfbS|`KYQCJiPOxbQwhZHW`va?&%-Mz=Swga^g(ZiU`eea>idWhdHAH-ESq(U|Ara&1u9Dyi0_&?6jO$1=MH$%b=chskene2M5zI&`U_lBvp3^MRL6JhN4kCB>JPm4dWQON8lL zF3pl^<8f`F{MtZqMTS;O6D-(h82YY|`2O`!#k{kv8U0lByCIo3jWjHw-}#8VA?ac2 z^*=O<&hwUTU&*`@U)3am>xV#G6zGCUQ=W>?*@RGL+MuA2(OSK%K%3$nA%MQ&?xZh{ z3dw(hQ3Gr9E_trJvc7T{efh=2W!7o^GzBP9i*1cXCp+wX__@{|?I=9hb)S(Z8)vb~ zhi_3_cY?*9;*qCL@nHUJ&|Ejg4j83>^-FaDn+zyoch8H}89FE8;7D1UVKuY#$S6Ux zYsO#Ncr|Y~EUgb;B!~skxR(WVRS*#=m_2`7E9-xe>*PPGGec%2aZ2t`GM(l@oqFAk z`xUHkDqlR=bK=WG0AGSj&4Qm~U4+v~#)G`Y91^+uH5t;xKNS5Cl|tOC-W<}o-e0+F z`tsUZxnyea(sKsrlzbK)-BcEGXfEY%918jkkOC~>NXx|vj2r{ zR)QyQTk8TqIZ(o?U`-!*yZEhO9vbw0nK6%*ocLV$VIqa9%8(w$3C}#Qq>NAkvW*vr zdR@gS)(XnWtPV**j;c9el8L{O^nQO#AnI|(MjsS&p?}ufBF!3fWwC_FIBfoE)GqW< zQx|5#Vg|L#rNc>_j3N?EP38jVS0=D=`c6S+_M3enup~qYYdgczrjDS52E(_K#FI7$ z0We(}9?TqS1{90EJnPZOB zc8#oSd~cm>6x`%Uu@-gZL&0HjWdK=fYz3Vt>3-XLR*;Mwx(0l~+RtEb&mLliaCv+i zKAfUAADPs`sx0)xjA0G9MT>uO%2c=L!}`(`%D;H|Ee@EMu0%8v#8a6%AHW3GArxFs zasfIw5?!9p?i({e4z54%*Ljao^_`6F&wlLvrID-@ieMf8~)PO|`K^ za>c{Qsb5Q-9<=U+94nRyAtxDN;erpb#0ZKP*t;~;J&oW7#>hp=q*`ZKAGC6Mz2a-hg;2ghyR+tBlAX^ZrU1x=^K5Vp>fI%~aLBTex+< zre>yl_!BBiVt2BNF~$;`C@2}teDsZuuaJMj-p*@^C-px5O_<*5%Wi;J!~zUv8#)l+&Bl>5J3e5B9wDKM!jBdWLz1DLbU^{h_H1H84o=+*7G zCxN^x+jf~{NZ?39MvTVvhMjdD^dNPA#N^agwaO*&^O;4NuGFj7X*1WyyK$Gj5?Hjm zZ!Y@drOt9L)q_8JdncNMD1itAwZgT+y4AiU;_HhPVcI8BF_?c^FP-(xDyN^~XP8~? zs62l~xik}rQe&Ja&JXHf;z2qggs$T?vr5ljxv&rp=?<5T3FlU}VTMh3O(t3bsN?CK zZBf}8&^&7+sCCaVI}okGtmcyky7^~?y!3YDjiHeSbyn7%MJPCJ+Q)t^REn;JKyy(? zb)Br>;YvAsr$>Jh7z+}Pyb#fMD)MqJ-?|mpkWzm5GGxOvU!Z3eH3iYw?X9l+X@A6$cCu+IoZ|(QI}D`(9yn)z*9>wc3gqs zrw1zCc*{E0>I)~LzgStNa$=nK>(0b zf`Z1S|A>G5%Q3bJzdW?mX~&8udCIArq)_~3fSV1_kiZy?UW=kkGTBzWs$+@AfZ)2T zwQ5x6;8eyK3+xFF`&{lIr~GuhkN$At->S?XIvgR4G#;MC%W1e3WAJjth~x(rmbf!x zNbj{qkZCvO@WTc=CXd5WiHqF7r%IU)Ey{BC65)S*nJld{1bu8vj#&~1Haj-*Xrq+H z+30nByLseF3r_m64BARsioyZ6>Q?j zSBKEiHbX5YU5bpj2-tP6{Pbd$I15QiS-(}i2S|~*(FA(z#E2Qelwsk{iQof2099b0 z<%)l{(=7R2AbF{-|14pI>K{I=QCT@D9btdC%jK2#hsUtvs%u%_!0U`e>M*!mYekQ3?!0w=p|t&K1zQ*f2@l=KwF_KGWvJ)j!U!jXS+=C157P)L9LwD^Nk~` z9K!tGqG8lMyRz4EX~B5e&QCkL<;f>9>HUbJ7PK9rBlCL+4ufDe<%@w&BL6(IK*E2b zX4zcZ;%V~8(DI&Z7vl5+Vi?0t;VfX%iZF@uB3dY>YrI0*)e}H5wSeDzZP<7sf5XDj z_5nj_QE_acc}bbQzh(DvCTIh0i)t%+w4@yUFA1vl7nI6 z=<+4_6Y;dSu+8zG<3quct;ug3&Ch?Hf;ORmQ9At)EMUk*w+-VWn32B$#8115Qr=z% zEC%Lnr}vKjc)*qA-^w@ zZA1zAy>P#UgB7lb;u{%DX3NoPPI7W0bwD$*UA}pRyHoMuf%lv3yx)0n!|#9QZ%@!= zE$UPdZN_yYXY^N>Myi1|pOBCTstS?j^XYw)3|7lTo6_ZH`xZEO8Sam`A!xiFnm98V zEU90CQ98>#G#v1Wp`tH+1ojv_9E#zA$lNe8lA-p}@jB$2=*OA5O8#Y|V~$6~-COcQ zF_`0Ewwy=#f$zY#go@1%PbPoBdP8Umx*F~wYO*Mpyl<9mne4AxJ zGMqbQ&xgh3D+g_2UA-WkbbWkZwhOBT)Z4gB0wFX)7X5}04P^_{z1D+t76!L6sti6+ zF7K8sT=4OU`O(dAAjnLy7~jM`h|?hosT28PQU-U2Q^g)jEtCZLkfkmpXuu#wS$T-E*9h!Gmc>2Q70&V{a&B*yJF?PG{avW*&>ej>MpK zx7OTlF1)^;_H*cpLn-KjUc{w1z@KQq0z9rZ7d=PsUG@lsk-**f(w+ph2&gw;>}7Do z)dk#WmdH^d_F7VZPHLU@S8{i|myX>LHW3L{ej^#T`L!Ih|N4JKALq1D=#F;kA|t}K z;&7fQ$|idQg(J|hx;mI49@Qau^1Dnun%+{7w`>N$=J)z4#3$7p&v|4NkB6Q(k5VdT zJs~sA%1uk@4b}@-Isk;HHl#*cQL;7ARAKNlfao?nfNrF&6w*DxP>^VNa4OR78MZ*R zO}kI34f`6aPcVOSymp(gM$A3@B+?xaG@MMA2!2R58gfz~Wr9y%t0SyIh42lee(|gG z-o?H_Dy9TNhIvW;3MqdW(kU=5(h0>JRxVaZf-=6R zqR8^51rJw+9_}!q)%ajr7bhY1*DWnaQL8>a>9_a1fElw51yUPsCq4AZ6Lv=8d2&2c zD%t4#oz;>#n?A{_JFDztm8~?Yr8fD6yMz##*jk}v>olAyi{vl|48N8`38^aZ%S(-l z4C4W1Nx^@Ck!&lGUon&`j1*g7>rB9!9>!Y05j*;_Z-M0P(GxAbVV0*4+RxC-<_5X8 zj30zEm_^?ly|_iv>2XAZI!um?+|`L1kk)Amr8=$!OFBl-$HPP?g=hSch4h!l&;%|7 zr?{PsWo*0)9@5ZmJ}znAXB@@Yl-RJvw7kFbVnTmOJUsoZT--Wmh}~5-*7TBMG5WX( zUOAfsQNA5Vi$S{)^Eev*Du7i0bHU$YA-yR~ zP?`^K;-WEG{Pit`Hq=#f^RLdCM$x1?4*3RSgTl$Cvk^a0?t#Q9yY3EUIg+)!Fv*W5;Wkxqqh1QFc zy(&5Chc2gQf#goST8_eP720%s!5xEW{DeRX%F>wXG?qX^RHbyloTBCvPV<-M+FTw8 zmYLJ5_En!Y+79-EvtNU=u|ca9WF2LlOkaN<9^2vqB~`TCA^}Yk=I;{q@RIy!k<}v6 zIjYu74J1u&ct%gln`2TlYo`Z`2wn6?k^jdA{fUkhVOqD z$1$2ff2ED^UxvB;M<`|xi0R-lmx@0u@A{_)XSYA|IA+8xNtVce>BRZ}u!6b&!t4HZ zm>>_H)c>$F48w&8!!Tde@SCOoDa6H|VVLQkiJbrIk3nL<D#1|sN9SPwnii(NRw_$)_z{2KGsJBE>iyG~8~>qqn0>>$;eT}F43|;=4F={I zmS>1N8^z=A`HlGRCQ|oTv>K(Oa)LWU&A<2a{VS!{|3K1_;kyL-ryu_-|DUG2ze`#% zyZf4#<{FfBP$O`Q948Uz&M*k-l=RDRJTZBw3dut}1G%1f_-Arj4^o!7MgD)N-5_t2 zH~y3|pnb+vnzR+QA^s>=%`#{HNzfFFUbPuXhVx2FLQ9zvcZ(1YTh;5KUJCn57abDS z!1Go3&XbUgP7+BAV1iOQI`hruRR-59FK;~lDf%Q6{j zP$PgCb1p2bx9|uc^{y#K@z#Hm2@Z=b5EHB$kSo~w+K42I!x|2IXPdbo9`^65Bu6oJ z&GYl+ynwp)>)6UIzSuAtXA|V4w8?%H)?COiF;!6Pt@nXDu(?5|8x;?ZLn|hGi|-O# z8;#)0UG@QzN8ozF+Ts|1S6L7!iD_*jYLe_HcKPJ4Hi&eY2&7}#0nkR`h`9{EKGd_GOG8*%-YvEnv)uDab{m#j7EK3U13Bocp}~bz zNKr#r5!oL+YgvEmO*P5kD;adx}5D`tG7oXH3e0@Qf8s5AA8)QYB(C=u5~}B ztPr{SlfRl*@vTm+T*xVJf)okc>B(PcyCeU5yezbGoMlH&jcXszA;? z)Nwo_=TeH1zB7IYAuL==0sIXY!=KWOhV>>vF&U%NG}wO*rGwz0+Q*)O@JH8R{1pwZ zCl`VXgJ==wGwJ~%Ko8*3P(*qG@7p)~v@F?L^~UVB!-@(dz-QwB0)UCd3ke8%-okbO z^!RB68l=d?5JAO656x6|cJuSp=bVdy!E3f(tN+a`5c0!E1C=cc3^DlmLD!~1IU*&3 zhz2C~)j5CfW6CHn4Fyn<8z6c@2!3V%IoxbClv&0MHMBJ|m-qeCkmx-JwTP=n7DaoB z2g8(624Fw@@eN|>LO9+o3K-4jD3q(s2l5+0d&e0`Nl>jLKe26BA=}i0BiM>VN47j% zYD6ge5LX8g7O|D{wO&uoP=guYHB{QJ!n@J#MgY-@dc7++>Z*A@&=8~gk2m{Vq{ zpRQedhpR2_m3eMj{p9OW%0u{q`%!-|S?qtgqzamMahX}}eA?mJ$REl0)`W7FBY_=` zT(%CCI1A`p97U|qiEE-*2VRlh_w-mgSPPPM1`wYXL_F;E7IndKr(6XfZ$;D_4i z>omlpEk(4LV0%o|jw|#)u+vt;NbE5dl1}>y@t!XeRk`BJ)3#U*W`CCilzz< z^pWP4(-7TN2s;$x^tE_ZUO-{P1POntQf-J*%80g80|DQq=J(v|2WO$KAy)KUg*81- z2m@y(hVeZ~DRlGDVSQ8! zxQA*FfS;3Gvrj+DlBpS@v9FY)*mQAIWo&0&4YkUM*+NwgSssC_BYaz6f8FSO0SYh|<&^f3qojnG#YS@iJc@MU0hSR+Liy9&_41QYPY+0( z068_$iNlg^cm~U+Swd~xCLJI=cKXEVGge%;y`>_)0xvn35ukvslvOb*rs(mgkUHc^ zA-DXfd(EYA?aaR2@`-=O+YO=6*)w)4I7Z5YUPfa4&{NmF7x4(ZHP!Ux7+{Bz%k4vwX{X>CFKe8c-^+ zV4-?BonT3R;=n!Cs=?1s?j@B)_WqX$JkST6w#zZ@b1#2$pgbI(eKDlK0P2EPB0L=e zKrX?2W`Y$66nPZ9iH$B00E%w41Ky~YQ#VJRnBT5MM}6iaJEoE++)v`IRTF_^WAoM- zzOGU7dn`2iR@iK$*-~%h9BDDB_!Lm>k_o1=FR}~cz#4*UyYKR#x1s^p7~lGoRUM)o zCIq0(;MIRmr;f8-9Tcxj0J+;F^ZfCR)E{%>T6S;6-D2fqpsLQs4uQ4U;18sy8P_Ka zU#>#R4Eegp)n%?oc67!hH;UbRiHsd&6SyQ_ajG&rGh~pcd~xtP^&Aqws{(B{3g969 z;HRG)sg|t#K#!~6#3oG;=E591woL_CV^Omq3B7-D$Z}o9TuN|Nk{>O#a=X;+;NrsR zE>$qy{Xk;cwli|{?27qAe6+mXA1L?8gmgDy`4ENmfl$qeaLzT=AI>@njol^x`oG|C{)9iosZ^ zrUgbxM865}pmKoDr~KT}BSjzTd&Ubvvb=wY#2U0&^TQNRB=}f;+BP>RmuJ^a7wF#5 zrB--F$6qA0Q=pS%TR==muV4uhn!-0cZ~{X5%W6oMV^hFtBlALrI+H!3Hw(+3JSjW; z6#nIL2XC_qcJ@o~{F|yvP|D{-tG|p9!r+({O^`y^&zswAYH<)2-SF8W@ZEu9<|Tjh z;2YB902G@JpPhOSW!E3lxR`T9)R=^h}S4cnr;{ml0!UV?B0~mQ5&@F#V3_VRacwC)# zh{(%EAUqr36pdVR{@jf4PWpBW3J!|k8}J0>zbx$gxIDpdtQvN@{g`C5zGHue(RSZP zyK{zxxwSV%M`gV~^B&MC?Yj&D-%40sX@>_;PHzMlduT>AEJc#DTjo;qyQz9YD7R0!;4J?1fj&U! zuap-tR*@Fb2Dyl42wN~qEQ#}=I@QC7rwqm5^-+^R#iS=hpu7P!q zNW{Cq>Z)3B1KxxoFW}GL=8aroOy`>IPIF|O7~IOhznHzS`5d?uO}Y(5hsQP?0F(b{ z2r;76tw2sNk##7w(?sYQvpFmYjZK!zkGsRQx(@C`sO+fnV7?`|)0KaZcJvyu;h%TG zedv>>Y^fMB`YJq6MZ4k{h_c-`xFKaaiEIm%BKJ63p-4kP)<&=U6 zNm><8`mIFSWIYn&LxUU!7a%Ath8JTQ`F5mUG@`0jcvH$9ghVxc-2VE4OZ4{cD zy7$gUoP@^D>UmMN0&4c)0U4f?EH|L{q;Co_03f3=m)PeKhnEn#lycagV}ZJ3+;?>Xe-htCv@t)jaP+O85* z^VrjHMSwL|p}C&2*X3S0a#W24?9VhT7g^be->h){8z9heJwjJBC zDLLi}7BZpAi<-x;Bb}K_4>lB@WkW4E#rUteLoZxLIFmr?ZO@;WSTNx3nbtK?d>jz} zkrsI`4nDd=<@X^{7r#@f#A5~(;o85Bs8%)oQ{c= zrffJn;-xyOMP9a9gN=QGhFJm}+g{$E2$segjn8N0nmKwmf6nNz-BLQj+D~h4hEY0; zgs00p!}x)Rs`VO$cy<`WmyEhPxOAJJaTQOQA)SAVmM{c=pmV_4or(csh1XB_Kp-0& zz8fW3FpL6WqwC`a?TON6UR-kungzxajVcbS>#heg#6t zd?sc`b9{26>ekxjf&H1n?g0_L5gXuQTjwj*aJ*g!xx13XVhK#r36?`j`4$^8^Jno{CC5&E1qqD!(!aoEp2*d(q+hj@`$6T&6t zMv@o+@l5Qhak{jE`pYx{9)#qY@>YsOCLWXbBOrIqyCE5>ct>WPZKH3IKJ`x=`+_DjCO$XEc*03dx-Z9=fy82V( zE}CP-LQ8rvkwQP68gkdSj*`=$lm35zn^jKykb&RFCIWav4+0d1zMD3ZO}I~QEUoVR zf!6O_IY${l5^Lc!igaS$l@WkG9?o5AWiqsCT z#x1bdZ z01AqQQeMnIKY+e50I-m{n=^k$jtO8gibph59xqwh1J#DDz!HVzVyLY^>~*~^XE$tHRTkhT=vva&r@*) zO=Oq|uhAIQ4Pv0c^uK@U`mQZkm1WBh(gPmBW8%H{Z#dyWfDq>Ed!V)UKIiuBu9vDo zm5R*Bh&g8jWTeva98M#THOP0o@KbGVw;ta!Br4>`aMT(Sf8cZ|60t<5BM#-c$3$8& z$Q)ppvgCdb8^{Mg&T^oeY4T^{F~w-4W-ZluAqveOaxJIz{bYZQh0WmNdZU{z5!;GF zqMsz>>+IT^fIprnHN!I}ON!@hRjr>dF`T>43U{CEp}4a@e#q%5WK&ncwr?)kGb}SA z6I~7C0QAyNG^1jNroyYio;A8~2~%s%*KDbZX?Kr-OMQO?gL!2Y4);Z&#xt+&{^+2^!cxjqnD?8y4-AoL#S#>%8gjYkD3DaP|bn4C? z8*lG=l{RWerBS6kg+8KIN};~?h&9AFohZwpn~H_5MU?;6HzqQuM%bkdf{+e#YrFincx1 z>BR#$3avJcu%g<78^1M_yMu#Pr;(629>EyHY32ZZ*+Xip)Z15YFETaS_(^M%)o?Pmh^Ad6apV@ zQWrdN?L~j%WwJY?;TaGy8*3qRg3uoIgZ)Y%n;i)TeJM4FTDim#z=qmweZsnJ;LTq# z%jt4?#{>$GvqFs{ct{8Td=1%Wi-s&RJn48)2xDEfO`4Iq@*jtVw?!?j)pM-ZG+7dw zODsDmsA;{)$t$=eNEYYuPzOx1^RYW=gOk(Zd7XbQf2tOjL1$hJ-jQKD6s6P*kK!u% zE|;`DaWz?H_99z{sO95&;aCe*_#KxI(v#XnAsKw;RVT07$QbW!QJnnfHlkRu+D}ag z{q$1T<7D+#kqj|4ef>mT4gBEwi?SQtU7 z?+kGM)8d6r0yKezhQWQNs z3Xi7vuSh^NmtvAZyC+SK;iqUYynOf%+Mj<3=R5~RU+wz|(JOpx&%40{C0Qhi@cH9m z=hoc6I`hHpxzoH7azn3R0&4j5WqD;INPc`EjAJ7R#VEXlbe1j0PPwp4+GFY*MjhuP zVFG>F+X3r;qwaH(ERVsvtnD;yE3Rx)JDHy|;hN ztg%m$J68|O+{X|EV<4AcoMMvf{SADom*v6>B2%CGehWYyeVp`Ef% z)fct9Z^N4y65+u4Ic&i~M*xx{GGJd_qRpp2cuA|rB|=P zPVN^=-RHA+%aTC)&fOuUC+vu?+MV05p2`Ksc=`B7Hvx3Io9rU#MhTkpc@QRIZtMCo zx~J6HTzIY0M=%am#e6tb9khSf6B8bm^C?yRK(F7B)AA_?-?@zqIYaPex;(4vgk*oE z@5YW8!=)w}WlK&qJ>w}UD zT}85Yi7QW0(hv^h@Z4-}J{ohLb-iFMDD?o;cP?SD1F1QTca4C2@63Np#(_+(lqfPH6-!Veqw&YCjqy=F*Yf*cCRaJ*xvks7t-JVCU z6L!etK<(JQZG6uNlcVHfH}V8N(iTD(jr}4M_!=fM+6OgHRp(rZnkvph(R?+R%QT7G zL|GC}h<&Vd9cMn;zKnlXKriiM1olZ}e)>sYX*M-UjeJZZJ%m-OhU8OlDLv*UcR9yv zoP+twLk&hE^0Sb%-M)HunTd`T%kTC`yJESl?5nNz*Z|h-xs-o*$8(pt10UascF1PV zBmwP!)M+Lw&g`<%R2ytllA3MV`qj`DA{M}NBwGG!6Z(jO)g^xbA%uAP(^#<3T#tfl z6IfYr$mX;7;L65pq?cJkFY?jn(F1toV8{r zmulvre!VD5-k?9A)9ecogQU-E8<$l6LSeEEMI?Q)At#c-sYNsH80niqu<~WTAAwTb za~bG}AP=`!?6ZG<+53Dn0pyN;+xBP5>1m`9tqG=&|4|ET;LG=WCi{LYvhpn&Z4kXO z{i*4cW8%YrWgh{hSc;inno}C+O3==?!So!CT|P6O@BZw`*ZYHwL(S;4mz0jRlvZKU z?Sde(IV@^;LJO&$s7U)S<2zE)Vvo-}T*f|~=_`e*?HPa3<%AfVe!kP!nx-`}09jM0 z*Fk5moR+L0=bt{gU2lfy@8bcqZ2r0ew2R*(q9pSpkno~h??G?xR}hn(of(u@GejfM z?-PAyQJ>~Mf?Hulnp|(pJ(AW4D6I`9-=o{N9OcO=Ff^ZY5jPm2C%x?`GSJ-^LY%Go^Hm>PdFaF=XVCi9Of8631&p4*!x zW7ae3=T7Z{Hoy11(NrPl92&EWN7IEU_7t6{!zm{oT^4ehS6@BHC|X?Mo0p5#F{0IO zFNEy0j5N}ai}bK!rz^}J%PT5f`lQ3BsU#`CA47li^j6MAaL`o!ov@st~mrw+!e^`4{Kl;=|}lZQoVLLdNareTIZ~@lr==j_08;o#ss% z#NbsMSL`H1_%jZyXU*{l4FPV0#Ir4Ft}=hU#4B(S$LKFz3J!*UrB;NV08ci{9GxQ` zti_6~>b0t`FRVvMk{8VP;hm-~9(?e!)w6II0t*x8ZMQ3=*LdJApU83#^#pr~_@u$T z<@e~vLhb}TC8*C4K1?wPwiH$%8rly`ov}d+$PF!HnLR4MvEeBX=YeM3vPQr3Q;~mo zB7*ml*giA?mcpgga~hp+L!7=)>r6vNWP^Z4312EbU1jFw3QM$6-PSQ2U*cs!RD=#N zVABrtO3{MsK^xD)B)4x_lhg39v?LWSjU*4$B-ju^H_OjMe9KzrX?@cD5gQ3*NQm?C zfN_9%UrE+Uo%XT#9g?_|T9s9etVl zUJ6`ggi$pBM?kp0vk|t0q8u5QB2niBOV%&nsv{)(7H}E(s&Fi7Ih7TcIq{Lcrnx-D zi%?S4QYmZdJ$*xD2_O)ZuGDc?X5>Sv(%d4MgLYVY0ei7oAy3ha9$E zs&AlB5CD48%AHzVWcFIx4f804s5U{FBiO9$qC-B@!*hIrlC3fCj8q;|c`dRUV6PUo zrasws)hM1K?ik(ySqCML>sZ@aB*aI5lf9o?k{LoeFO?L~HKK8yRRhf-ty6CznB5^R zH)3vB8!=pnhDp$p@rbW#(!RJ`bn>)@KDUUm5eBFR@8`9&93_CvF3E%h<$^7Bk4R2+ z0?}1-KDqRfU~_|!!udJ3NpE>&rhLop?X3*3qwX;osrRy~#azafUJf+#!nEOk!tZKz z>(`0He2v;~xI(_Hz|a0OwPIhphLiG(Y<_DVZR7aC!Sri!gQtuWS@On_P4vPvs11tg zgGX9A!S+mOsdH8WXISDwLO#NK^qiZ? zqFn=I?QzM$aiBi*gVV`E45WU4MA9?@EmjJi_%N_HNG?57Ry2V5Im-OK_usZnSlKhs z8YjUJc^;+(NMd}9iaft-^LWp?0u6doR{AR-^D(p+XU3kAE0 zV;o66+Yg*YcDUI2_#r*n%EUDNehBr7jzazvi1IWJvrh;d?y#xct{leecwUe&n^>lM zU`UZe>$Z`;sxh=?9Y>;ntzTEcWhglwAktg&?2ginpQwjCvm$Z#M{*7cBV6qBYjt7 zSvIlS$g=J5Oo@udn@4rVc8-9|9LL z2w^8K8lBvHnM=A5!)ZP!O1=4^_%9z3z=B89$N|GmfuxpUp&zfIb};_V8{4>XhB zn#SaRWj;5aJrsf51Ail!m2#Y(I}YQpiiR(Eicox?d-1fAo#^=J=(&kim?odfHy_J! zc9RTVjBtbOXD5?FE2?3Qq2HY|015v&llI=?B2b|D9UBY&-C838*7)s<0l5C3?LIQ# zE8yPW&HGqM!v;^A_kcGsIS>ty@yuRHAt-TwRsxC)#-06CS_6)li)@1iPn>($cERvC zpX~0+*ptxN(R3f2JMMYJ;Fa{;Yrrac*8Avy=0QUyjqVi{Y*-KO3|Mq4PXCSXyf*LlB_PLpX-ONSDJf3#BNpux_Z`)RT%6d><*qot0pZUK>6~UaoPhsqTRT>^cvC4eOjq38Fz#NzAep=S2dDC1#4GQQuYQgyZSml=^tMvB*s= zs*B|JDHZ^o99d*|n=#i~K>=WJv_2Kb=~$a#h1YSIo3@F&IxsK}uGFo68By4WQ#2td z>QqMN?M~8Tin#_>apcW)%=+NHWWM5j({i={@_en?@y2G1vzklOC%dV-9`m5-Lh(2n zfgk?0KzO9pm1Z0j4+`2Yw|m=~)TWD&MdjzCb2U0|K^X{Y3WmI0jJQ-`8rF`djbg9p zd8%lql*NNS7i*W`#5a3??15HVrzp=y@(FZVKX+~fFt64?4$;Nh#YG+^3)5 z9c@BQv-X`dVx5-UeY%WKjCjv+Czdd>a}S+32&FU@5vV6jr83=5W#3+}ecWl(8ZwU} zh9@w#ED`25JC3$5F56}lqJsl3G3RYJ^W?RzWud;U8f3X`N&uf)U{RykMopY>vTK zb(R&@auxj*E!uHC-|<4jv#tAl`;OhM8c1J{40m>#6HSlrFqpecd4p!+F}^ zqM=%d#)9rAe4$lOi9yBS#=Ln~>XqgY-~;>GE8$bX_LgFQI+UgU0^$Icsl9mdZtK{? zo>3VC>>z)-Eu&5qhcm{m*;lL-hddvMKgNPHc?fp2Q#!H&QodPnMRYwR)N+zv)#0gZ z@h;XlQaGMJa`RO6u-V$yy@ZpE>W9KM3?CE%=gxk|z8pLeuvuT; zJ1iYXNPvxhjbAG)!Lm%&rb(fsRFua2W&LpJ8N1^8*%#dcW+h%Wc@Z|f=8bI^;ie+Z zdl%pN7IEg{SuWP+OOHmcCc=Y_0!&wSGsG30M15w-3a1K(1S0_j=}5=nbAhRR{r z%^y7ms*6&OO%uag)_*Trj_TJY!0anvWVdr)KITS$8yO}gtTSSx$=$aN)jmFI5uMSQ zpGLWi)`!{D&lW9y!K1S=a=ZU@^iM@ zBLMvQe7~n_P&g=FAbS^Is25F}2Pc}uyNjaZa?Hib@VP!Zw zvk}(7hy8AVwE3#Hb58Z0qblpe+S%|jb#)5jv1a_B$18eQi)bFRb=Y%BkWsui>|I`e zkH{PFp~F(q8)JVUtq=m8LcLS^`yh7N-(N>QuuIAHM+7QjZbUzoy@J!s;X*E-pPJWm zK|DJSK8^1eAH6Q%#E?mXeHCa@%V6xS)5qpDwe2rj5M%%p$Ozac*%Ow1HYl>`T z?N4{vwBjHn0;UxSM~vK@Z%>|z03**2vbhfSL>x9G5t6iD4>qrhW1*7Sg+&k=kSNm2 zOCNJqdZ(;&F;jq265`Wwkv}#c$>5!yCwYlD=G!cFy()r9W@)L%GNOhXVgT`f>Q(+A zIPZ~M>LAm5s7*=15H{<7?M|`^>M8=gZ>H_=&Ja$?hhG?aXsWD%x^Mm!G|_NW16g=U zC^OktI(vHhpmnycn>KPEV|TY^Qr^V_e^w$wY&h6RBBj1?_Na0EP{japA2Y_5j`kh+ z&?4o`2>F=)eDFmYTIxm7Naw_VPXRdg!-%~C^!nfkVWn+PV<3&~%C_&jLAgL3)xkqC zY#*PMrA}#O+jG%`tWYWJTL!AGf>4K}_nhAd*lN8Y&&z&_9j|e^xr(a2g4xT8rnS>9%>l_3b@fVx;*)lNE_`H*j-tz< zIh=wv(X}{ILeHaqV3VE_mbbBW8)?Sb7-IveafOm?P@{z-4a$I6)0K1x-@Zn3G`RI+ zYfei@%y-Irn!8j4F(u*PIFziI=PNauOxsYadw3WO0fB_(`Z`wLqfL;MU$)5t^ZW{h zI4(4E(T|!qC^9(?%B~)Nj|se9ek!@&;kc9_6kZtgD7M<_E?o3U7}+ufehv_ln;Dh_ z*z?{&fA6gld8CAF zPE5ncpjC2k2-||RNiNGiMHUXIa4iQ|u5_TP+-aRR*}=$g~U$zM#{B zRIP7XGzGbuaH9Kcx|H~b$w&rXl&=Ooq%q$TYSldAB0G*i0H%4GY{|kBgZZ)l@{M)} zv06Dm9=HtpQX4#crQwBJ5YwHFD;);QP=NP`L)g&J@$-~ z1p>iKu?O~?lh!JKW4ZS}0{$G5+83p+(v(C@_P9uy)o7d}n&b6o_i4Z(rnWk962j9J z@v51mDTG#k-PcRMjH_t6OQ)%aMxFVi372;`AeMznLOkEEJ0V^o`-Ux(O`-JCkl8xy zzvL;^cL}-AgzIC;Z*&wLWBKW&Xs+oml>hchm_d$H7FyYVzW1FeM0er9A-pW}iPzSV z6#42u1$F*}QRc1n`=pAk;e>yI#a!G$pR@3U;=ZoQjRMASRQ-!8P+v|Bo{jps%dF?qx zYFCDB*)m^$N^4c30co>Y+mZC#OL=7;xLdR%P39O;*U~v4i1YH4hs@9N5J$s|amJbS zs260|D^!cBU00P*3*Otl)AH@XUFYx(f}T?D`NVMNm_86asNW)K#4d|p%BFt5Vfmw013#*~*&S#D9hM#ok;&|{Kx?Xmtu{#xijBtT<;yF--aERnU zz0WS6g>Z}}eb9;-CTaHVKJ)VXT43>Frc(p=8{ztJG4FG+ll775;gjjmy%`Gom(Ist$Y==<|(em8r$JI#O-$idxu zh1mhn5B}oa{Ni2x><9bQD>;D5^#mCYK*j731i%xhCjdF%cOZXkN@XgU)tePHxcC%m zTa;^^V|INoJR8+7ITWP3h+=mJ@6Y^y33%y%L!q`|g|%S?$TT#;EQqK$6>VjJ#Wltf8+DHN|uB=j*Qlal}2y)Gai{L z4g@B?^h!q!&mQDc76r#S@f=Y zIh?-xNCGq-;3^}sncEd*AAh2M&>qAlA3&%dV86Rw>~0Z&x4?FpXlJb98AsqVmEM6f zbaz~L;l{Z5T)^ttI6u1PT@MsHzk&y^p0qOkr4(WdV7dEs?*Tp#VW3d#X9p?(>BZbZ z@n>D&-@F}LV$doC_2ekPe}T+nL}s#OyQ;R^-Zm>jaLk0e)&Rx^%%ZD*z~)bN_XDN* z$9^#;*f=jZb|h3n*0Xyc96{)x-BUVbUU1cwt0OJOA7U&dxKQh^7X(=74RNZ*zj3B7 zS^$2`Vmt8fak7B_jni;qM*z9VR5^0r$7p@TVDk+OV<9q*A9;Y}SwCWrjs)xKD-aQP zXdk3Cuz7c#<4;~Sq5S=S#VFu`0we}F6Z@?|ao>mA|JXymVvMQs@%9Pj%0uC@M}3ba zqs=3x%0pq#2jxg99dwMv;IZjWZ=>EDZSQurvSE=S1nJBnnej1DNsQUbi|kghUI#%Fbh;h9#&4BLgIZt2B#) zjwkEL-0RnlN(Sl+LO|^^^leY31FYsdf86a-t-Ju|&%EVR{^6al$9F8>Sn&XN%vU@S zV*`~5#iwiVO$!-O_4JS#V}`DK(hp6t^pSb0sVHm>uB`J-WoZ8On=iOKZ?OuH-hi3z z`RbtJPxa78#f}5Pu`j)iFXQlQ!_!Zp)lZ}Kf0&}6JPTZ4Yqsw*;`hPVauhR_f5rni zS9Xb$_L3_!I|MdA>p(ZyRnRof7jAoLzxG%*L+;@9^XKa4&$aC4*grVNocRO%#W-e< zMP|=M!MQOpk3c;>gvw;<1|HO)ay2iAu!9hj1pOwj|TSry2^Iz%A?+6HDke7#8L(GQ?ezV+ff1f?7Y8{Bu;M`rqP!5E zpw?5?A{Aigf$Zg-)e@O=b!=|Sk;>6Qz^qmq=GtDpixY_2|dUSfa%yyL6y?cwdGm*ZIif`;%1 z_XOI>x=OHGVlrz-R9u>88$~v6|79@9vgIC>6%Z=W-s>kjyUpI~cl!U}8D_NkjqUfC zMEUYj!iAIMOsC6KqsXrr3RSf9pI7r!^)kI;6OrBIm00 zn)<1A@~1QHTmS9yU;XFmb?h^ZJQ$4r2lwx> za!GBm(`>PCS!&irs^LZ5rSj$iwY?tK%?Ia7snxH^Aqt&2fPd?s1M$|E&g%R>{{QU@ z;~%|w`*9xre`2f3rI`D}GeT$UPE~z*C%1Sv-c4I)u=CHopg8&e@g5797`%b}1p$2X zo^;!vscaU2U;Na@$xAm+gC?RWdu6$5`TLz;^rbs*UQ!@v0%^`}x4*Ib%}WJ;rzoww z7&q?y@h#Y>b?FaJ{}YS3#MHho9R8o{SN`8RG@2cJe^C#-P|wxy8b?k%gN?eARh&aK z&Kh?-{H<`dU;b-TvCtfT(|EtJufjG_yOfkY>i}FX8^~{O0@+nKo|1}<|lJ7B(e`4YnE2!}=_vEK=#Tf795wpjRz~s@b$#2~S zy4NJJe+474NGz<)p2FSNuwlty)0~HW#AxuTzk;OVF2)YJAysix-z(nqe1RSPij&Sv zYW!ZUMUas-NZqO9YF*XN-EsZSQd!?IE<9J+wYNh-kHoq3LC=5`d2^nAKpVjup&tlU zVe`9wuQLukcy8a-&z-h<6_ek2M$pl!<4)|Ie`hH6FP|9xwZj$v@xdl;R{|LiBDeu_Fl>S{$Nn&EAZ7R^6;p#LST1$XNmS(s;IZZ0pAKbLveJQI{;!c zcw+@Tt({<099au^T@}S#;DkUl5oZDo)r5V}u*XiymmP?}u+2el?Hayyq69qvi`r@r zxbDXH9qd1zjEweA9w*qp*cg=khhevWf0^K?)qmJ_+tdWV82M=$OxXYAd4J+1MuFEQ z%~XYEMTK!Q`5Gg;YE#vMpRd|s4_0#@1l_qwnDXjjz>FGym8E{=C*@T@al{}GQ#(#n zHZP>A`|_ilTh8vvHL$p9ddRHFiaRbd9IdLvI}EB0713Pv2ek}^)pGw-BTDF@e{&U3 zKY;t`!$GqD%<~uDe_}oSGlzftNN<=umXzynj`|ndnR{;c@2dFcTff}avu_SyMkjt4 zURs;$y21)*=-z49RHnAvvzHzV9Kwn) z{$#z}aQf;q2!`)m?~%cY*hd4Sf5z%>VfIziar1PygkpHV^)4qJMbpUthSd|MK@U|M9b6D}!_? ziE8T|3v!in&0T16imN-(;x$mCK{;GIJ`KXD`oyK8O)h!BmMW70l`Zj3fBaO~n7O>F z=D0itbtkg^3Ud{8N}Rs94(#*QL0H(xoAqa85uqMM!!o(uwVGpNj&%agTNjS*RXMjm z0|!E37OT`12)7P4c>d2iV?v!vf8)3WKW_R*PyGD&zCQkomA6maIPizR2j#@2pB_8^ z`MV!K{PmY#j`}A)fQa^uf804>H$?~J%4siJfZA?Xt6aS%0=x4WU!{uEE$X!&N@EvN zr59{H>3mg5F|1YwLX~1295F*Xs>duM)9ak1cY0UX!2Mt%tW&(hTe2G;?y){HSdgqb za3sdBgd?YACNArG?~SIb97ldCtjx;*e(}!k3;VPN2d!2I6BSHsLJKTR^AgBG9hc@^ zEUOq5dib0NR2Z-o@BolMUUf)2#s2cZE5`ojZIN;1gL3QU`d=HI|8V~8BU&Z@@RmRS zaMbOWSe*x^fAJHml5Z_^&vmw$ir|*(G9qJaS`uP^P-p0f_;6pdyqaNO}8YlH#MMN%M%mc`Z zV+Ycq>x?a094jNbOF*3$k9?!|npZVa{V|*CmP(a=={HyZ;f~wa{&LO_m)M>MjDN$=KlPANY4Qy;w~mR~1SJlo2}JF{525B= z@e(4PdBkd?J^C(Y{L9`g5EzNmd-@xByAIW}d#s8rXxMOPV<$&s1|*6DXX1{3?e;xD zz5HNYU2i-2tK}3T)xpcLd)2vAS`SW9tO=krdcr%({kQ}O7BqkpX6K6GfdBxj1%JBZ zR_d>gW8d2TuQvG06Mxq9r;l!4X_PC!{?So?Yn1=sTlmpss+IZ=KO|fA~ja z?s~&dZ|3Gb$G>qP{;j=4|4Xy~Z1sne1v~#+Gyhwk_-Pj32O)LXd+K?p$bZU*u8Zfv zX;KIe6wpYnoWa8=Feo}B6=@%CIfVjBBem?~xA>NH@-0yh8DTq>^Y^>E<4H{31~ zUE7!|+0lPfwR0#n2mJ6*{6a6r0PIq+%8iHV zzxhi3i|0Rn{#)O_esF87JAYof8z+Btz&{)vi~r+q&iujN&oTaLd9Bj_@t;4n)aq}} z^H+2J(fGeS^k-Y?j>lWqRCnC@zEs`unDfWeu$g8j_^J}6#!hUWmz{JvWd`?nh8M+Z z3Rs6%c>r511MlLo^Q`0VvI83ah|PdrKbqSu+-_IQv8$p@MS{K0I)9_vPBmq!Z!ZcL z(`MUI8lxx~lYd%RVAHxEeZE2~aWA-P$`z&yMIIj%o9EesF9lXlJrVgnJqIYeZLs{b zST2lzvANeAU~+q2ut%)$AXdPxzvWSI?2`d*gPUBw>&oUTUt)mH0-I-;mhpcgpJj5`M%0c+g#j1Y+Aai!@I#ibqM9<5 zGuCY;c}b^ZO5S>RrKIfuhNY_GMf;~CJm}=P_0HG$^*+Hv{)CF9?)=TS!i|}R`;Qj? zi=Vf~`WG8-?7&&({4eeG)As-1f&SBB+-+axZ(eujHHYo+PX4bx{ZoU@H-`I*yOn=L zHT9Yq4I{V}1xcJu2#4*+u>H}S6*rlkl3Cfd9K4n0WVXXR?Z$ZGtI1>?D&dYvl3kmW zgxymI{OJh8S9kVEjPyx9fz8&F!ZvZEwhzOv2>Wv+F^}x|y}dy}1Y2*Wf^SzysH;_D zZZD4C+-Vn@lhZ96lP^HYdP5D&p<91^!&f{TP|`VNRzQs&kRfQQAl}(M>CL|c&o4w< zX4#4=1b6+a`!7${AQI<#TMPf@Pb{C^Oy%zn=1J1ws z(VsQAdHp}neo^E4)p$$B$xCqj>hSEv`Oi(>yJZz(E;0dw2uB`sOL zUy!fZFDWqKD;O{!n^XDkDH+C`bM0?^Yv<%t@|5v1Y}A%pt-h(dC7Xx}j^bLSr1aE7 z4_6L&^ekX%A3W9nM||1(a;F~^3IYewlmHL2QleP0eW05Bmf4qYN0uZ|qXRjG1O~VK zr3alI0e82W!v|Ob0iUBmkC(#92QUIZH@Ec32QLC0m;d$u{NMRk zV*a<&EalP@@JRQb;7?x2)yI)O=C2Lhn6 zV09{&@q*i;Bu=;Ba0sacn{K7s)^h*1_ffg+2fBAjOSZ+xO3+v30ilYa&}8s0tZiB#Q^DjwZ53-#agA58zgP#{#BiwIT4= zzg-jh3Fjn1HdpR0DkWqZ`O<_rX$b%U;3AFff}Op8c=_RZ#3G?AUFV{|ab6#<3kh7AG3nIQ{m_2GkBEYO4Mf(QVk2vU zs>STQ!RAUU?s!+=-y=pP9z1+?E)(o9^sgM_F4&mP#u9i zFIZB4T)}ZS&z{(386m$!FmXt6y!6e$#_m?h#+U{H*WauGx9p_BxmAlp2?fzsj#kLl zGw#u>3Inu^P$YK$w#BdNBz$9qEwm+v(Xr_?`l}-Ex?lQH%rC{U7e>$Mo1Mnue*#YN z_SZBC)fiG_*!(?e4_`s0BOi(vULh{>xGN0AHB=`-jd=r!DW5qkfjwzChB)EOsXZ?~F`@sOG4<+qCb*#04Dgo|qAFdJfr_aR_i zbXwv~hw6Sv6veCDfqEdRYvc0;XUc$h!Md0iQ5SpMi8Q1@UEDu!A#UBN9uVaW@kCvJ z(F|h=(^cHLnTIub5dx~G5t%;(Z48NP7$0xV#FBBP7|kWf=vA8b`1(ZM$y{VO6_XO# z&s(l)(zsd<&W1Af`Uiz%XB=|pNJoKvI`aiSmsp90jfvMFhCJ)HG9?0v$RG7oj)F5> zv`C%iF@?xT!q@+R=h2;eiE`_go-Uhzd$d-n1aZHk-$v}&@$z;$-@*P)P0RBcaug2t zBY&!}#7i77&M#w)1~r?24H}G`kj|nFjvb5LrNa6RDWx!itC&0|j$M7MP6i^Q@$d6s`(T0GVu9{wLj2@ zlzVN{h{}U5kPL4a9G8Gc!y}pH@Sep<4N%M#nQJmIB!FkfmG&&-n7*BVdMTuCz+5Ed zo8SP-$F(82)f{mZ3D5agiI;(71L)(jv&zFDoFn&sp``~|+-VKh@?d3SpcGK<1Hh~0 zpu~uZz?$CGP=%u?$6w^Y65uyQsJ}9K3Ix{$l;BME@WNmSIv#zJ-RAdRo;Yru6o`y_ zbi#xv%0x8uM1YTw^2g184Bi_X_NK&dZw(Tu5xN@pm--xvx5wg;W$iRdaX*AK)z%&L z)(o<@Nb8>jLV&p?i~iNxm8k5;uj)c>^dg&-xayS`zA_oHV zpxuTmLl6dM>tK+7E@L%}13o`b#)Z?PCwM;pI^44bp)Nx!uyJ*j!6a@M!c^!(E{+f4 zO+##8C4F10_&f>$C?OKQ*+9E4r~Ij+kIX<=2^clZQMn7ZZY(J!zM|nR`yFey(yLvj zQmWx(Smwq}qNk`9 zo17NrqbiXQNKrq4>Vmjr<7J>|#hRy`mueV;g32g?W2aPK7i|;spiRRtIMQ@baI>>8 z-!@(-PoI-UUm7H*w$(+qA_F%%sBC+I)QmQ408p}OZ7Ml<`w)Ylj-R&%y}yZF?6JVq zHb!M1x7IU%2SQ(Y@7R-6XATkK8?jO{S=oH>Df1;HrUvR0=l|tV%W-8E#syy zkR>V5HS)Esf+liie38V~vk1cxwm{d={n_9~hqyI=pgyOWQ;sTqMcKd(ghnGZhZByC z-F2HhobKs|Q(uud$>KXVy{iq<28YY{T`(RBbns;OEjpo>MT{ z^DDQ17YaXUrfK-ev!zojhbUz<1j35O9xaE@#>T;%)RCGdL-O0wFueGmtKmn6tvXKF zkqzTPTPS1N0`~cX7I4MoRr0=2(A{*rA@1l%818>GX7+dK84G{QlQxqVaaPGh4|9av z<`!;bP(DfU71BKB9uWD@zEjvSl>>RuX+vy(J{qBE0Sw5#ys6U3t%$_qh*n4W)6USy z=PN_oNgYFdM`Z!)p-V_ZKxD$8gzBw7;%Lv{V#iN{t$n7RBRr-E+7705lQQ*CkEu{E z8#$Pt90BD`{hl7B;Sl{T0}j(3^seiamHY@!`*fTSmO)q)(R9lg%j=Mpj=(Z!JyW4I`68iFKdJ za8y;IEN^&ek)=DDz@jrV2pjw`d^Hb$_hoxD>sF~`GCpllbk9o=ex7lX)=MVreyFo` zg8w+=H!t;CVmda7g_dDx0k$gxf%cj9CcEcp$%@(RMBg^60G0aGxldHPus;Rbt7z$( z0N^$&+=PjYk%l@GerUAclb<|o;08tMOVJ*f>xbKPiBPcgAzFqzp_8Jt-Gg6$_T%;v zWwt>ACB9u(UsUr!VGd>-K>%G^T-X5=NoB4)a(<7$p}@M$buDn8vh@M1N~hjTug#Xr zDgBm}*ZjRp17M5QRX~%B^#%tHj;Uf4EJRn(PuZ`?5k!viK#(Vd9X(eC!D<4B0QUOi z=--xw9kgnkKi^K-;dCkJfLLCCKvI{L(_MoFZRm$z$Wt1dNmeKt!c^IBe{spX)|%hx zNA!d?tklZFnOksnS$$$=>Q7&8&>ZAy;u`9(yhY&`fz`qUj?*F!cip(SglO@g)A0k$ z0_c)2lVjb3=+i!8o{UwdCbvQ*s-g6|wAXZxe8C7^IQ(|gJ+UDTlw%2hD$3A7J!XWS zb(peb=`^%0>uuCbT^m)LEV_5fvT|R06V+Jr8KE+v(?JVI949!JcrvpBfwiANz+wTq zc=J%>Nt`AR`tc`1YciWr4G6RO63(L)v>&G~VC|1rAa>5(Gd>5_RUAsU%(#02z{;*_ zais}g+)ExOD7A{OEzIhF3jgHP0XpTT>`{McR;oZa?_lH-J8u^w(SLSOfh)lo0IF?m zg}25qM@Gluw&g5=soo|aasu~+2V@X`C80d&2xH5lgE6So(8f= zBk`S4p5*FsQHsx$>&oW)V0#KkERLLI3WNJE2OxSb%UByrcbog-P~EIpie#_+7^{2Y zQHVHYNt3S{L!{Y%sTmrV9ux%`@MXFkeuesk##}rS0Tn|xFsjC6Sq|BXcnYW5KGZbj zoPKMTn!0*NCueJP>HY))qHc_qH-(mbO9auY2PeXA@|M^ZE3H?`-Q;BIB5r*=w5H}+ zRcSmuw{z>ILUtRR9LS08>Zks0!fZ{BbO;ql$EP|F_=pI9ngFfBx*gm77L#xFM?bK9 z4`TYS6B)!8_)t+=MDg$p19H-MJ5F23c0Z9?)g^DPSIBL~#d}I3CKJ*-DxU2kQvB8${V}rW#s~i+ zadqNCBZ4Q@>$9Xx6<~dHZhAH0JG8|3nf>v>AVTFdx1(ad7WY{-F2MxFP4q1c(gMsj zh0hp&=m@uUiyt-nlLPep%m$C(XPCcdg!_P(c)EE}?4qqcDVv+CAIH)^KVU_?a&pgN zTUcBkDr=j1s@y7{=!DBJ=BZYM{;9>+eb}_-YF00hKohvPgB2nn&4|FMM}_4?(Vdf5 zd72k<)M7`408&e$NvG_8(Mk&UN}>n!SUhe$zV2zFJ`+N|0d=?D z##e};dt$}B1uT6|J$@4FJSP-25#n?^cs$m{x_7g2sqexkTw7W~k5_>#zsA7tf!O?# zxDc=({Vd=J{+Iy!rf9@how|T=*=Z3&-W9pcMe$>(rJ~WT!AVlF$P=OSKwptCtE{?z zbeqL;vL8%I1%6sU*ZmUNKz`@|-|{qzG~a4P%{v9-*upR|W!#}wG8MbcBpdo@dP(Et z<+`;}4nb0O#)MCOw@_3dDAg1Q+ZTwe9Qqr+8NI3m&aD11r?SRwGS-^AP2gssUcV`5 z@+~7Z&j91}V`ZPCNjf7*dNJK}G~h3P0OjkyY}~~@EV3ov+i78oBfa53Zc_(P>2Z*=W%}d@{ZV#<3>j4BnP&3@ z*7nHa8u8sSyR(YdQkxs7=}i>nH#6C5cOhVVh@^8^)iu>_Xxv`6!Qah03x=LV9bw?{c01z8MRE1RT91(mOMbVe* zz5#({;b{muCJ|d{VE5h6zTW#+K2V?#;zT{eK_x%lrOn8ZY6j0i%YL$MGkq;<=ti1U zmy&I8+!vg8POn$ahhp4dz3-BylsMifaVw=4aafeI-X5KTGX``->U4HkC4GFL$5fYdnvChg{yHSPR*pO zOn<8c{~e~!C2Ki~Xk=x|vyi;(9jbUYWms|PCk2|ujTzDe@86ZO{sJRQ1~DmN>9j5q3$PH<|j!F$ulS2m>YY;{@WGo z1D+c zgDmKq=4WzUFYwK^2Vhm1UbQ!R#2LQef{6hSJf#P;(LBBbkxxwi{7s#6*rm3k?rMY8 ztioY`H5mL{u$#w~#Wv(Xlk(k_d_nPBn0(~ib*+n&a{*{Q&V)(2?E5RxRq(x1K^on zXd^BsSj>}eOuwpjZEoK0ZFJ&CJro)fcY6C&IRpUNm{L&DtDAVDP zvT4<(tvHH6J_<$VrdW`jzJD^NbFGgs8(VT}8lKH7w24q?a&D}-IcAMmTr!0YGg(iIF<`0nmUUX_54 zbV9BPxrir~H5uJRBf6fa6dJ9c*LUUs)12&go?~6waPiVTCUPVveu2-Le8=8LgRB+9 zn;>){2YReB_oJx(?z`gnw2AN*7d&f!M}Yf%gHBeH!5;jIsg&gWCOZXy`N4|q*v?f< zIT-9T!uw#9yw^mT=qg;c;SNC($~{AR$>qW|bZmkk{1*ja6Vn@0&IO8_cgr=tj9Mlu z=q%*17=&-7CuDqnF7I74`#{MIEGf03iW*zjEDMRwgnBA_bG5Rx4kt2=Tjz3rUrt!< zu&00p6y5eo^HLvRu4mFpjKpAWUNu{7;bSfBwV>N>IavD=lub(uIUb!U9QiQOFkC2}otzge*q#cN+DV{_pR zeqrsnj_)dU&2*{kxcI!ZDq$~JC!k!CS`m|L1+KHIfu8BRy6ylo^V*4uRZFT;*JTs6 z5;KNF;Wb%(N_YAzMXzx^dQ%xjNm<1UqMPu$Y+Pu*BlJ#jV)w^%3)eV*lv5YR^B1`^ zHAKWwrf1)vnq(A&jkooOhdmi#VhH~oB!<2gnAIA?)RD3PF+k40rdvwjaf&3wc=aXm zLrxU>*;(M3J@=ujh`7nyg|&as8Bn`%5{Gy|(-^^b@IsDCz;6I3LBh@ipQQ-RPwW_; zNu3GqJ}G0(_NL3Lf+7DXNNkjxe{Gn3{t4d{D=;%Rti;TSnFV}q&3!Pk4>#QkmNT>C zxLSUXEV@x^2e_aOb{jIc_922d&BzNP?BHt4LGgUFJ5Em1{*_+>GSP-?blTh)G6X2@+n9ekaWEFKzW!?adWc-=&mGHnu{t6RNWO8>He{E)N{x9?d z3u91|!(T=y$>7z}#du0Yq2#BK=;DbjeI~~(d+8+pY6h)nk##x|(SrJ8^e~2dDKxO2 zPwVqQ5tDU0*RmJ~6Jz~=w83EwAm>i|f!E<)$6wKotcfIBjcF4`>|eV;OD$X2LYMQm z41jE&ki`9J$iEj|2Y{y~f7&Zh3z9opNK&~ve>5#+>f=x+FH#XQAPb8jOrW1Jms_Xk zEv)qpwIRuLFUFpz4jaI!Eg3vnNb-Jysehi?k7QlNWNJ}SrgM#wIj*6RthXckZ;UqI zqNBlU&6IqrLFBF_HJd~1es_W(*4aDZhNd)K=hD^Q8VC0W5wb?ze`eZL*n`;*ZbMk^ zOhMrZ5pJH(d>~hSr)#+y@?8|F+Y*&Su6wBT+JwN%Y~{U7pf836YJ%dy9iy3wS-Bf3n8(n=FShgsV+E&QB(E3M z#Lv&dfez^*XP{3cb?P;kn_F>wgf-FvoxAtQyz@!^C|iHHf4g0UMhOMi_ktH{v^0>m zY+>p=r{ty<_+I==-u>0HRx57TL-qtdRKB~R*;3~dyKLpU_wB}%%b;JLmVr~DW{&vpgpA%gCr@N`$0yoCc zJmTn&wH^Ke*U=w|Ex+?60$7YD(4TV1|{AEFb?N%$|1e(L|t?Vqj~iQj~K z!)8uG;ZIDx|1*pHy^wQc*D?E4f}xUO7?t`fDs=xQrGqm6gZB(E|DuV;e~u#KZx%ck zSh(Vhe{fU&X^y#c|3CWTzd^FRL(JJI9*M&+#y^L*^jA;lB?Yam<<3y^FE`>v{+`$K zzgz2GBMTw3Z}PW(Jo}GH@BW6O74vUj^U_?+vJM$+=)xkvIiY17rW13aB!6qx&m`Fe?TTcq`06kK8hX`=_g<4WZQOG0^-B? z+K*AFF7(}x6I_*Ruds>QDa9${bDdfqA`sA-saB)Sc7b2jhd`&dOz#SoxpnFwK)&|+ z>+Ey4D!;0xx%n}WURf%nV7|Xma#r0V3^Cv8x)eL>H>eW8X=HEkdi(uaoQ$hR_R-oc ze_fq0s5ZF}%iLGy=Fj)Hxp{q6eVJ((^_p_uX!7q#OZ(mGcIic3?w&KK=d??s>WaSo z5Eru!jtF9F+MX<%kif3OIKL02o}C0xx4SJ&D+F_bzLAV}m09YC}sAOMYe~el! z-I8EWQBWvnwHOxpa@+4a-RL9t58W4&rULcDxOy;-mC9Tm9 zXqkUUbF;6`sjiwesF6fGEqbfMR$!I5Si9PAk=`=fph6bmNg`6oU95wpdhJ?~#>mzw#%`YTu zKEJp5I{jFmZZUKfFK(MK!+eB$2nnWY2pdfJYdl}r4|KL#U|v9`qf*AiZC+>+{JpAu z5s^_M+L)7-^2ax{N`EOTe^VED6b^@KKuh4Ki$%sex4{Nc0~|@%xf7o`p)jh*{5Fi; zaQgQ!d;#;bNlaj+lzR!l@md6~kKo^@gD^NzE`gC4!l4*``d=Iv>s zn<^r|eJY6R{1F1cU5|c@JPIQ3p^p?r8hQz5=ypwNgYx`UhX619e;}Usjap9mk_FM2 za9>>yt!KuhZRi(=`z7^NKX1i&zhmOh-157I#nxVEWAVd`O1c~>Y02aZ)wqW+=C15A z?#QaSHM@c0oS}rOh9BXIltMCdS1{J!jJjAqcqI`GC&5f4Pns*hcPQbBF*+_et8(%SKf3u*5Ki|fH^6))Ue>x?fxV$00Jx|fdCVZxP09sy~xAoL;efDE~sTs3Pm?j!X5a1WUD68(HU&j;1mOj z+iUfK(i^`I+v#~Af(a*lIr`$WLI|{Q_bb0I*&sm>$R+ z0?_NaLGBJU!8L&l&zEZkGysbnHOh9;RbH6`ovpp|5x*KiYh?;POj8xg+GwO4hr^H~ zuot>aIF(wU| zS zq#@Rsf0!WA#7Oef3u-NwM-Z`l?Q7#>AawYo!Z~9}e#= zu$>Y(oB^d#KR@hmdw%lE5{R8opIuyy=;=NbMS19mt`w{=nggail)-M4B<_^)Gy9}f zIJ3O?aN>r=>jWvaLdikyOmH?8YKsOCe*l-qf27M9=m|>~LqYSbvr2?=B0W%xgCeLj=XG=mYI>aSp6aClEJbp}~9OE*&TS2hN zf+SX)Hj6|L7B|Hr*30+h)0rq{JB(4xYAN=47@S4oH?AqLqLGKoGlY47n7Li`L>^Kt ze>TQ;gf!plOo-YQ1*ejQOgSLu@ssgqXqXt}QJ+No9Uleclk$Pd+uB!eL_VIC&zj># z{1_+C+_p0(DpH=G0-M>J=@6s^?{W)Bf`F2TH`7gOc;FI%yP;@ z*X%HQ(c07f*c$z>|e}6F&@dvN<)qv}+)h`)<5h zA)}ic|5fvKqb7;Zgu0-4m}?IV;Qh}C_mnqVa1+*Pee=m1vc}}nEJ4&pX#yi0f5|YW zbk>DRX_XTUn|r#cFn550(qWQrvX7=Bu|7oc4w83<74_GSd8pJdP z?=UXCrDBC3q7c6+<+;H;9>+I~e@cVKH}xm8f!S9UOj5@HuGi?d5|`6{H-h#DJ?4}_ z&yf9!96}MFPNAO~NR>IY@5Ha7KpjWNb+suW6d? z3~{7P;NCPNU3bVlZGt23V2_jD1}pjdhN)Gk(8iUr%HP>YXlOfQzWZP!EpcuozSGsI zH~pDP|K7jba|+CFnUCO3lkGc69(k2~!}F^Xc|c9`Tj_Wgkv)cyqOx`7=GJAKI?3*~YelYNt4pH{~ zXk^pZM}c8ZB52v(^Q;>T1rvPv@B@|_n~=KJ5&o?B%+8-%;z(6+rl0R>DZFQ-xZC1RcB%3+>RH7>PWbo~J^#8Iq3Q zkbZu`s3Q^o0x8!oew=@T<`{m26E21?dx^p-aL7*A3Y@ zXhz)#;&Xr;Pd-|xdc%W?=v;7FRY`dIwI&;b3Zjrq#d|h9qd8?NPRL)Z*C~ji-wO$$ zYq)gLJ~t(zQQ&D*m(`$X@V;lMgi60fjaMjIRp~{0#?|m*e|@tswKP>T2t8);5G8@a}ycvOL`1T0$(G+I_`^fBR0@ZUSx^@c`sEJ#7FDKHs$@3GU(YMkDlL`Hb=9{q1%AM0Rl;H z>OC#I9P_P%26)*nU88;^A%ty?{L}F*_u$|pq{q`9(cd_2iogR^^s>t^Pl}U3R+iOz z@qis)DXsNj>*33G+?r`6|R9dXoIA} zdvJUzAXi%Rfb;Y5+QSY<0&Tc^zA52nIA%U1kCV}5#=04EvTgkwZhZ{H7dgO6>H zn@&gme~v7dTsE^{Pwlr5Qn#t``7#$oramUt#HC&tC58yj|=QD_I_p6+CLqn6o8crS$W ze`2WD9Xn)x(vyW<`*)7&6dp1-n?~7?8j|Aj{JG$V{H9Qvl%YPFwVHXUf%TpO`T_jJpp%Zf}MGv+)-{& zR9fuXq(WIxX8JiC@QOWT7m%rMomeAXxIRAGx9>?z{hN?Jo#=r{5IuU@bD5{*2E;ZP zeX$1Q$E~PYmw5r%F*36abY|)9671%U=XgbTNYN@;v8-qKT1DyZKrm zmdEnYI1M1RCThbRJPO`z7k5awe+5z@wj zBjzr5zrE>gwEo8jo8m~3OGGz`0b>i}NlD*unOfmKI%tqW%aaBur`dBB#JMNXVhajh zmN>^W*~E{D@M75X30Q44uo;-!3^@frK3h3WJI+Y+e(p|BD{LoMi`-vC`rU zl0ckI`xYN#kh+4>z4qLk?O{{j3SX3T=`d;nd}0p~zEG}OXweNpe_qq`Q4G&dB5{Cu zqiwJ}4J{sf^35xd`kfucYe4n=l28Fz91s^nf(yt z4w|{(LT%cAkyhi~4{zqPyPN!lo^Y2KmP_6=dU+%+c}+2XNt(F(^6Q-`{Zz(c?qcWS zV%8eOStW%4BvM75f8vq+9*o~}m)GgC0wXYesJ9wn=k-f(l2;gq4&m0{*j8zal3WL? z;d%7^JrziK!Yd*Aq)$WPjq{F5JI<)9i4J}c8`b=q2>hrAI+C0GrH|H}5O*+)HhQdT zGss0YMa>wO+mAJZViw-*(yG-orz6;tT>+4JjAXgJS``>Z;j`{JeBRvu+s_>2oWl{7XpJb-2N*6eHfiyQe%CDTSJ6k0FQB}7Ksia3y_j#xhyO?^c2Z6xq7V_vzo z&^gkkl^QV+e-XX9NYeSDDBBLXS}+zh zX7IT$Gc9*&f1o^~09Kh72b=RSb`06~C{UF;q|yY)-p45J{_x62jWoP@JC;1rm_0Yv zsft*&?<4wLRv9GD?r%2W%f)chwvvQQtdWX$7lhyP4Zhs2H4o;9R~2x=*IaN&4D(bn%^}WlzA@9;yvWwE1PY!HO${BV|FG zfmTT*iF}7hF4z_r$XP9uN=*!EPlGI{*V6B4-g zWqEf$i6CW|GqEjbJOE{m$R;*70{Sp?fwaIMZ+dKpp*?`882&e9X~bf1{>tq1;Wj6p z9|uC2P|~&WE6u4LDEkN_Ir66m*h-Icl3$*14Wh#*K6yg)?j13NOekDugraPfS9Ig9 ze`tNflD2OeCN)1Hiesty-;}+{)}u<3E%-nh00JZ!)WDs-@7o!D4-yE`JpElL;+*{R zX4a}zqloxIAGn)s+qTV+8?~$*?;%Q4EZPb5GiwNyZPi{@H;?e+g{gxoPCPyQPFSv7|-Kno|rd>^Ab(<}#*(n6VMH%;z zXy6qq?!+Q`COQqZ4>QoiRObvU=u^q4H{Y#75rc7;AXMH6cURBPSS=SJ8Y8nellweB zkidcT0KTz{FK6?%^LH*GfvYi<+qI8v6Vunn55p=f9#Dxf%u5i2rHE&SKTRoAf4-B1 z?|pAIuSpxhE4J>j6bR-Nvc2ti{^UI8cR%9eGbEeHz|efQ89gcrOD;wHrtFVp{(gEP z^kK%l;QFnLb+m(32 zxNLdzbQUmDC3RyLk2RymhHtu#f3W#ao4OG?&%el`^H`{MPrrs8hUGEhHTNn^U-m9N zgZ)7hmGufk&zt7YmwmVh(}ast(@$r8&N|X6=@)rc_z-T0`Y`ug@IPNQLj2>KC9eGG zXC*;V4_6+~I5ci>Te=L1Ts0jZh=?n)_UbiB7;vso*U1cZ_Rn=S8%#@Wp#>5rgTqTeOOGsE$^S!;2fI zqeJxZWV6ofxvJDKS5XVh#co7Fm&NNj7jUJ)( zMyfB_j-ztFt!FS6ocD=Vf7DiHer_FiL23PkY|UH8UW9_K!tPPz+Ts2(eeG6#J`5O{ zpMmn{$T;)PDbs#WhyBDzU*uoTe|Ij~-EE#==696NsjGBqY=Bc``2+)_hYj0ptb$`E!Hoi z@IKb$+o)jOKEi(Vh&FJHs0p#iCxX0~Es~0w$OPa1)jqe(oN1Ke-1(j(@fSmhK8lLR zS>kiLb@#)Br2ox?e|kLk9m9`&v$vOg`#ca@pbKUcC*kSDfui>ew_?HFPC{E5PsEtc zBQvqdYLnWZ@1Gt!P6bc-2D|$bV%s>mxs~f7R`L17Pv`cPt-^@Kqj(Vab|SW&5Q8M% zalZ19+A75d%j!7&A)RA=qX(JB$i(J6^)mOo!KiR!dU+MPe+@G?c_SDjQ{dvaOR0@s zprGDNQ(0^%d#A7W2lEw7&gbduaA<=fn>94XjNwYO-GSB0TTgd$qJB9ysU68uF@g$x zX`TGriL7P0Jv!4M_SyJ6U$`_-FC;T?h3(%jOMSQKyQe*sKd9F$*Fx04aXx~K1)MMp`7Y8wFK}lD>+`T-5o5iU><#HA-AlYWoTI6> zCm#j6TU4F>^ZVL(`F(uM8zE41EVnT-m){f362GLm(d5}$X5_QJ0pbS422{4TsheEe z(pg%Vf22AqSTe*fT2ZwTgMHcBbKp$(DYMS-{kqw(ztbo77O;KyOwaahR}~s>?@v zKSu#RFJwy&T_G~-5EWmH+IUu~zjG)1u{0oMe})ee&De*gqXZHRqbW{y)S|+25}@3g z`6=abf#y#>m;g+ejFUbVui6!@_F{^Wc9#?#J4Ayh;}FF_9;Hw&M{;#3ec+D{1bJkf z0TMP}H<){0`6xUSq2=C5p8?T6Ncnm=aBoRu-*^F)sD@jPcz45l7r%G~e>+-l+iFQM ze>Lxi>drMD;~>$_bYNZY74E0^r-99at8d<6t2ah?9C)>fxq;^o#2%hr`E|RGa8Ol@ zrtJ;oC#<{Y3p+I&i^uFMrudK;bw#yJcYIO(gc_x14j$Xx`WjD+V`A{o54~gR*U5dO z&dGOQ=eS1`A4p&Q37f8K@C%1u_nrNt}Ep?G*=S>ZB%?B-kgieBd- z-mW?FDL%7h(bQf*6WzI;CF>w&?-<2*3Z87M6twxu{G79f;^}S9*Q$0~Up)^_lCDx*5o#4%GDqk{ltJKnJEZMxq2vO0Cj)?DDs0 zFxcUH<%)O{&#AsqJ#M7R1-{<*FD?*WWor5C1nd6zte(sr^(JE2*uJ8=rTMK1Y>!Tr z4h{Wh{H5~O?W^B3Z{}2XnA8|Ue;&SDy&hxHo;8Mj$4muti1E3-qImXl7Yzi7(&ZDp zErmdznlDmWe1ZgSELWD^n|n=d_{bX|KfUc4wa+y*#lCqyROg|6?8jCk$gQ^bNARTj zkkspTc^V>Nx*pGYGAKib$|E()CH-+f_V&foi9wYMP`ErDaYuSl^vu1Ce+sR1hDH4J zHGWKPPr+k@{}c={uyMD%6Evav2_ijb91KF^ReACcA@5*s}f z(ondBUzgYCS-rmd+hOCAe(v8rlJ>8kx)1R-Q*sWCx$mPA{eH7!dRDWXmJv8rn|@R7 z!cpg{*o(Z{C*96A_Slj4f6UfAe&=Y@M51$i&V>fPUwce%f{Bh4{7RD%QxjX-l!`Tv zxm=PU5jxO2${qRAkD7UW!qxXDc0`b<58{tGXN|BqY5(*yy^dZitl~GR9~IGtoPX^5 zFlOJN1YAFA-4^5MD1U&zJ-EU1^*GwG#{*G<>eP_b{VYO!yJ^KXe{4+)TX~sjqrS-> z7oRI4d}SomOMM)Pr(S2Duy2myjy{?16yI0leBEmKRkPfhRyZ|4f(t)27w}ARa?I|( zW&Fiw_Z^m|uv;H}0!+%pnrufDJZv{?nv|%)-lx2Kc$?Fw?Ss2Re}CGevYtFC2*o|_ zGA0YCX?5R}QW)>&f2?%Jt8CK!@fPjuacpH}A@OL6gp5}-&CXIKW?fM+p4%q9Lq2-V za5^qksfnJ-c5^ha7kE90FkfNiwsA5)r_Rj-;&yvJpQtVMD?bo~W_-q%qd)EK+^77b z5TQWS@4&6NXH|c~FqH2iip1a5dr@g9WM zh60KT0BbtY9?(_pSK=noA4iqO(Q`LX{K}ZxsoO)%HJnn>)JPd#2K&3vwjmo??)i#_ z?L%X@jp*n;TI|hiOTSd`eJ>Xi1nuLtaGOJ>HFf1oB-hAVC~o6`2cJv{7lpEeLDQwXI^RQi!*@15EBvvFth>k#>{l#5we_o2;6 zV?{^xE`%gY2URAxdq$Lyb51(XnNMB_Ssm5U?-}=lyfrmFWb!t7byn< zN^@Dgov3}Rp7Vtpkr1PXcKCpXjYb^k%|!~UV6|pa#=1Y&Iz&RG!C5#yO(om1K$VMu ze~@O|xXm$zJ<3y~k=xhUWj8xTWzz~~^7j|}LK>A2h@)z`lQH}zc$rA~Ry#DM)!+#$ zv?gX}qg8q;=}GtXJo6G|$Aj#1U%nL}C=_-bYu8;yz+4QvnCe=C+^WPHt=Hp(jg%DJ zch|$axTpr|9!j*<%sPm26`JCPKl=sNfB60w7$T^eRitZ`zjbEZVnoQi%--MGfE-|@ z1|&xwoBz_-S>UkD0Y}0(jcOwrIqw_sPL0E?iZ7fU$Ooo@e3fLq?FB}U?>zFtl2lE( zqfeNg=vf-K-rb5~B}$d0+e*^hW~J&g*=54$t2DV~;!H|RxC3dZh&`De*e>#Wf5K8P zdavBd7D~xH?Nr}bIdi3K4*W*TH;+1<+;svge7v-S5kx2HFwL&(wbtPL-~t&eEzb=f z=Annxb9TFSZL6NdnLMsuTRSU8)7-SW8%>q%oHJphs2jDHw6#D%YdE}HT@Gqv!8r5n zYwvq`qg8hpEy%onby`|z;Ua(8_9>p?z zw4XgX7dNDqZ%%eSf19s!&_Q<}%#j(4 zHy)H~z`~nO-(KXfZdyS2<{_B}FXLacd~logaEC5TrlUtGI}xgC@%NCbT(RB1WRhmV!PYGqt_9U!RsvB9wEqzL)SRa8g%!2UX?vM17qB}4$C&OtKuBDp#Yb5JZDAI)Q|C{ik({PfL!8># z_>IRZp&EFds^YzR^>|Z{!tFXr-fZHIOCKuH((m&b6Zf=0f33z3NlRcUkPXvAJbUDZ z@~XV3nZcq7^QoJg4!?Fm*U#wd+B98LwP&!b_2vKCODurZI?A|?0!;tQbC09w{ zHpGM+TzMv*<{V}B#CiYWyhE&?(A0hjkrN>OUPB1}J?a80&|XJK z;AXJ@bqL1+P`Y05GVbtf9w&N%?=z+psWTm6{lJjWpgAKc=l@-ORmnG1S%7^O0H1r6Gu=kWm^O2u?Ib05P?=F1;0|EQ5P&vXyMB)RRIu@W7khX^&$jzhu~N7_^-Ww=a*is>cBmHxpiMTg+e^a|FRc1pGrw;rx;OS?^Q#Dz!cUzGb z&x}oII3vRk0{tD#=dCIP?bsX-6x(_%woXlP;^PbC-(%e*nWM1JL^s_hXB4B@An#8b z!fb@@xC_|sDw{)6ew_JE#;NOzAQfWL$(erL&TvugiX^DW^i5GqzZYjzBA}Qj0;Q-~ zf7-}*fj&`pqAclNKtk--z4_ecyFKTAM&vz%z4w(qH*1WEVrb;&tmB89+>7_vu=Lo6 zSpB)9&7L}9#7T{qey6^*v-`o%Dk*n#MBely%N6`n-?7hVXv$2npD)xmYM~K-S;nY% zhlNfHc#rj;OPJwLQ|V`sAw&jmTSJ+3f74jv$lm1%Cg(Jr*g~tqjoBXGGImZ;iSzFH zw6SLzSN=k7h$l**&|hsN*i|IXkLO9Vy!7>uqeQ}|C;`;BEK36GB|0id+y}jiUMFuz z0_-CJzN62J-89zPZd^Q`AktcCmjEZ!RfJc|_7yvo*otmVqAQ3dQ;0|ZTYop3f8}Y} zvAsxulY29Squ1`?nVHW`oQr2yQ#JxVH$_g0E0s_hxA*gB@I0b6U>fk`pVmaIm zl|d11?wsG)UE@+{wc^|T9GlMPT3Vco81EU*;_r1q9bf<6?0Xp!ey(?~xcq*{ZuE=d z79cY?zcHgQuK;i$lC>74hk_DC_a(hP@hU+?HH&fZak2hHsX#EZpmQz1cb23^SAiP7&+8e|LA^^TLio_*BKwdAo!q;2dYDZC^hu_^I&YQ3c;{vIhY; z&y~Aq2ibl4 zX0+RcUbK%!PjU(S2o!8GchtSZ+gD%6)ksM&4RcN%KBsxV_sS;lIb&YuN4N>~D_ZE+e71#}F+yT!Xf1faqr32a5PHp3fvu3rI z(XAxHuoc+$bCb4SNe*0ZwP&4!5wjfb#f_Mb_RqB+!Ve4VsX1hoz?xT}MfT_YdDn`3 zi2c*-NQFPQtal!UBdAJUvd=SKg#p5zNT*f9Kt1eS--kqs+}D&PnpGHtZwh;GgbgaB`e;w*1a^$F8*Ba8&Io)D6W}=#blRdg< zowA)LZ!2+gBKm|DvC?S;&^j5{{*>fIeyu^)oJvo7XGAJtzQ2N8-2z*i==Kzzpi^TF zH61T6K9-{D{)@iNT}l>9W9NGD%+s#RZ?tFi>!(;d5vfKg%o0RMx(}@Qw%qG)f}b)D zf9gk2b=j8i`4`f@8Hk}d_j*8UX7XkfY}-=eO++bUyX$ZtTY=G?e*U-?Tu7(Z3JXPs z?07Y1#y1Zkv;+09@T$@_1lZC%ePI{=H1&Cv8X4PNM1+p?l&NK z?$oPH)O-q_s@jm-)6KYfdNWB_t_A@=e?__ibzOt6N(;zT7@AY}CHj4Twy|@bWX-p? zMmYUiFw?Y9c5*^Q#LQzZFm{Xc%r@X~OLpbX_9y1Kp7GkI%l$b&qECf6ujAFxADWVH z1be)NiQ|+p>Z>|pthWQ0mZ|tw=kH2pPn%_FLy|$y2V4D7hQSYH#rDEIakaOWf8h=j zzgvyM-N8;LS6Y_^HTD-|wv(M<)2Tq|?sznz!b=!|QQ=V}-s><8Uff3u9;g0!Qx0k! z`P5Sn)V`O=>(ufGE37X1tfmM%&TML9;(H8|zD`(_kDg67@yC5=_gJ?joIGEr^;A6u zjc)dqK18{+sN<8n^`2Q9y-uLOfA4KH9M4ZnnjKA>8!Wz$DWg)zA#Npv^EM5byUcrN zzaEH{A>~wLr7>h#$w}t-4MZFhYIG)-?C||Mq!DCHCzsT2us1$>Ivlw{e(8`M=)0fa zHl6D>o&#c==vlY3lMFa1taU;n-fs@o(r6Gm5+0%*2-P|f7AUqX;p_!re^%{&$OsG` zgWLK#@`Pw%($7AahU`SOy87@XKZ~yqBZY8|tH3M18dONeo0JU^`vQ87=>XOi%U~>; z_IV$pv6$J9;PPO2*lXO)Cz#NXE!#U{>Nmw4&O9OW(yKY(JB*-DQz4Ud<`Mrb$~$;BIzM z`Kc_;b)!Wo8$5S6?&BdI(&2dyhLDNsvs|@<*sR4sUVXbCpLcRMf4un>_Swe3yE{l1uM!XUFmP1F1#5Dc~bNX$;wIdfy#ec z-Fw!ZF1;TwJBV^L=a&gH?=r|g32$`5$9XKA(T5p8V|^7{B6w zgO!VEU#$xZSMC6pku3-mf7^VtDmIDW6{Gg}8%RRMBOKE}T5tFQ09*d9%?f{~mi*?Z zKrQ|KwH{Nk9~uVr;q|N6h$&Vh62Y%yK^v^)y1(~pN$A6GO3803$u+s;*9>CpDGhDn zOJ%xh0igey7y?KIx{Z8ybvOwC*8Xj7`!ki~H-!WMfXct$hiepIe|^*z;86i!&%Y*; zl?n8jtCIdMDge08uN|U{*Ax=?n>qpjY2=D0nkb_cVvU^TG#!2-pWq-h%lYi4!-t0xMd9wyoQCIAH#^3x5)*z1{b-S zBqvgs)R7E=dcM^Tea{SV zXpMYif^vzrQgH1RR*}fp*`!s5)O~3{jKkaMp%2^bvs9Iwu|x(N^bf|@`3$I94AB6-FCSyr{%`7RGS(=+gN2y zuu3q=9pSTqe@{woVBM5vi9|>V+4+#5;~^%aJt4c{W*L`^Og- z7A{}p5m!k*yAzyH3Fu~5I$KdSTfZN_Zl5S3w#9wGfBNfw3d=S@;7T*Jy+Lk%cE$BJ z^I^?-Vx>XV%Uupio)X?9u!B@jvMeEYtn78>cYQuZtc~-)3IK2Y z*Y&y63pm>{AjmMiy?o`N@Ixl~Hj{3s7^4uXc zrKxW;f0vej_kkPS0Q&{A1;pQdDDb<_q22W@66G_(q7itIUQ=_LM3>f*T7EnXy{#kTL;o9e+}_bZ&(NDPKG) zOSFm7+yDg36Zups54s63+qIsBzt#gBPLym;f8-nvmuwCP(FP4Gyq!#YJE{z-M+JGN zy`feH^`?Rvhybu4iyQ>NDn_~bvvAfF0JuZhvr%s^yZtx(3)g{g){8`Ww_4q09k$Ol zp@uGFbFLUtgZ!H?#R0Wq7qCQ-zv2%hC#Zh-=_NO-!NSsMl`&nhO+Wi`I6_&h3&iZ= ze<@+vBIeqU-dguRoFnV$SP)4nSQZtLfW%fh@$1xA;DsO>Jb?HCV7c&f@tp=OaPb}FFYWXm7hyTR% z70GqonI(69%Qa#zDKWad%@pMspmCgu$*_~GaKdcKI@!!aFK z5OjnFk!J)PF(bB2%dTraVn4tZ1{@?J)r4%{sSqUiE&j&FZ-STwz6e^uf7KEd0Ex>! z#xEW>PLv$U=k4s@SiEwv{3rkXm)~{{zU=mf*qm&lvXSsL<+$2|Wv!1p_^jD>2Um%T z2aXNeWa)O;TiLsq7ktg^Eu7i-Uw&3gaV~(XG55V!rY*l_GDxCS?sO`Rk5b%C=QIZA}e-pWb3}vW`NEIBts08r1)xl&<|t>7 zBAFmNH;59w1%1xH)K6TM66!4u;3I)wP-9SB-l<|~LkGQ!V&(kAdH>R7mpCNb*F#UX z#0sOMbMYOfJFObTe}UIscuuc)!+;oEI!oGs^l`Y1v!)Bry}U)O{9A8!*Y_y=qp_}d z`*F}eYrf+D@+XY9I5Pb>)sFaX*Q=b}9jxv;O6+Jyb!fq4$pLbeko7RrjtGaMcEAIMcoe3KR?O8;}9iE}Ypc$e0Ufe?R`euzuOJlK$w;+Pbva zA=OnVIb0|((!bi1Z-_=}hz3O3DmBt7{o(lyK_#$kqI~IE@fXkDKLA-kroXz@`A=QT z9e;J*@y8zj##sygy$9T=FY8D$vk{KKP>oRWqK z+e>0a(YjNm{PD-WcmopE&MJWev)yDb=cp_1 z@?h?Od*#1UjME=PpqUX z-hLdcGuaA$`@25#4<`xM(rmK^?ko%`OWHCES`sVOlCExvCD~8a?5(%C!wnvjBimle zTh$zO-P^W_R)6fgjpKmII8?~(mLr53<^&luAJ(h^-UFUjL;+C6Y6}K_w_}*ef!$dd zYd8H+O$_Go5}-0}??UFRU(T@H6^j}r26YuK3Yo)NSU~C8F{RuQgGZwVf+5O`% zO9$(Zs((RW_B>2=?j))*iMe*K z_5nl=Of|=PZUHxw-FVJ-8Q(=OlRE_qNun-;QsuI!PfsHt42Xog5SM#t>C3GcoG6IOg8Dh26X6w&cn- zNfKBh>#9%vmu3!&T4B~#tF5TRdc0GJ)_>rE<($)@HeS!Rn;KMq>cj_7Ab1nt`c}Yl zX7eBZ`Q^1M=Uug%50C6G&ac|%PfT4r%3eGJ|J3>aw{P_ye`UcRzVZZkL~z}$oU!vk za(33(Dv^g$Fxa3`Yp#MU&j>sWn-i<`ewt;{_k8A+It>(>dJ~w9iUe+1gi1c)5q|}O zjqx^1tfJKoO68d`%q&<_uHL|#SaT-uY+sXo%Re_yWhcq!jVoPc}{1lDv-9w z{PXRbFA`hw=dy!;*#Xt`pB>yR;I}_}|L+>+@+ZMa$B$?J&4)jm{iDyDXjH50r*r=D z(fv=3`NhJI)30924_koWil)<*y?@L2mJj(PZ;i7PVYNwclM>1DEp6`-wnl65#3X5- za7&9ZoBy5s;{n^?);0FjT`m$ zf8~&WIrpl+z?<0w=<}*u=R*rxE2KuB*+b`m8`FA;c2bM%Zh{TDyI5xd(ovRtdIdNk z6t43fess5k9SCshRZA|0N`LFs&PRiLWvfl@&Dbk956g14<)hl{~A+QEOx ziUU2PGWtUVlm$@es^snl&PHY@>BGM2)k}w$2!Y@WQ(H<3I1RvAhw`r;_kOk9Pm8$L zf8~}c68{@s&Itc&hbzu*kj=thoc&X8ozlPBRI2z*gVMwwlFe9(;(zp9#?*btx>YSY zo+fZot}>_(?jdh-mcUe%$?TKD22`mH(XMm-l6G0D;MAbpKR|Z0uj_gvulu#_dIRgo zFXpr+wpkFpo@V8GJNGnEVAE8_wj}nxp_)rfskhzq09xZ6Vt-~yp9pD5{uN(50CxZy zOgkon*`&YxnyrxekAFSyzdC6&E9v5&-@Y#m^Dl0;fBEer{^rACML1k3*F{s zx8s(ykzD0VX3!#VrP4~aMknzgB4(EuhqE-xqMmUG-g>2!9Ouojj#So4YzEgS-Dc@} z2cFq1thEuj%uYASWgHP~fyYduab|0C$eVHZHUL!Q`K2*XO@CLubO2jjI(C&U?VrBw zf5yHKt8DoBxYEXdbmu>M;&1QoKXT5WTypj4q(kP}eUEjU6l{CzEirqE!PF{ly+A)Y zguvvX>gHP!8})`YF;c79e?SGqBM-5wTu52v|^2Qk5?Pk8_ zM0vMG%UWQ_ZIA;W!8)Ue^Ycw~vXS42Kt13?+$_v$2}e4(K+n9ItM;a>h)HD7JM?VP zMLF7>yj*O+0i&p3>L_5{DiQ-Uz4Vo2B0ZQceFeEoUrCnrx8HR6SsMRxz>Q(E^FO%o z{`c9!uYd2vTUqm`cKI*u&+pNn+Uzf1g3+@$mm{=5AZ56VNyeFG5;YeD#g1E8p&@_$S}~@DTpd zQvanf|Jm1{e`n(#y_NlX<-dLZa`WGM=%-OG-n@D)9;!2cG57~#|IsJ=>C~%k`j1+W zV1Bmy#p|y(&i<(vr9X8-CG;Xzt#(ZeNz3JuwPXUcVvC_Frn)LrsGEbN`b`8}_mRwnH` zYXB;dEA;ANo7&DT-jQwI?y;clCJoa*cGDe-)tQn*A@#6SJQMlc9lOn}uWHT&#U`3W zUzX(8J?xvs;x=-U>w54#D-dmpl$<5vynkv^7L4d!@68F*gNIY>qI9D8g{Z(O)edgv zodprWVyJecaMtt|n8;O6&6XP7)V4oQGsO~V!AV@WE-zdQ)eqP37sEfU`wvaH{jvFv z6R&!W&940Rm+SuV*_2tMA6_AZzc#z{y6~?(tE&e=u>bJ2T1jx> zqg<=Afq_7&woGCzUx-{S2*~{Oxs8`=tz_M;;}~j@S&ju&#WI_u18XW+>wjYy&=(>` z>%{BmYcCy+Bi-F&@+6nmKyR`9o;jk!DQvZ6OvOQ9MXS?W7~0)c2}l;)GK^!1g=M1_ zR)oTV%z@B?>@hj~)}i4csoH^7qa$U;;l&Kd3Jy?*f7c%FeIxko*Y^^T!((Njpa6Ct zxt?#hUdC71Pp@*WRzUWzhJXLtH~E7#>9WZW!{WsU|1(FAJk0*!(7E`85HJ1)=fy>T z`1$_>s|j#)MLaa0Wu$M^fI?_J7StcXYOGd~az?)1rDadL0KC=fK7<1nFRvEa-@Vj; zmCV_k&Vk8Xl@_ov^kcsGV{`Ye5LskGaZ7}K7y$hrJySRU+^3M z!SC$}HtND}df_+dxL^MJ|AA3atN#ygZ`Q0jtgVavm%OE@lnt2f+?Nc=a^E1gn3oh7 zkc|xn47f?6>R?i9CkeAt}cOkt} z);Y=O)@=3M)S?T{Lk1aWs1NBnZ)_*GRhd;3wK`FNtj$c{;D5k7>8$Q8A0PDM;9#2E zNniE!x?S&Owt+U?D72n)KSgHAuT1ojew4FhtG4xi-7Y`z>`^GY%zW_`?T*KD{W`$k zyq$ASIUm1x9v^+y8rN^kA{xf?6IWExu%1YNZS7~R`@8<=@tgRy$Gok9Kmr&FnHn8?-e1Mt4zfcV^H?s_!Yc z0-2kpS`%dZ2*jt?#ZOfUBceuQM5wn#;+afWt{R7-<9}CF$ylYxRaagQE4N+V%&rNx zt@K#8jV0$|%;5xP#xA5=_EAYc`9$vDy(z9KLW6$J0R736ao_AG7W?Dx{lrZqZ5VqV zSIU3pm62>+ceQ%$O)M8ICM{E6L9 z`G0yqqk8zOyBbYPv-)J+$wg_GW{8T~h@-1ZZ&5^7r7kTKL#b>;ZvsK}9zkg9UV6}G zNIGQ^n60vdT)M0~Q#A8zqI)o+zTen>b4ul#ueV6)ZfJ+Ir|sgU)-mzz%X+ND^^mX8 zc>KK|*13C7D!$HbwaixdI@h2){`p<6bAOw6rcmu z2KL$Xvb3cg4jIbe6bJV8iqczzwvik}KoQc+Q<-EW&34{@r7xr*ud1Ac0-+@z*A3tr zjttDFD8BW!cA?>U4KmgL&OvbhjO*;5SUZs4HQzTkseQ*yCH*Id`CZ2hn2Nt*+Ul1t zRn+O~Uu(~a)ARV0J|5(D(Cz;Nj%j}759vaUZdoWvVS&U4Oz)S zi43JKEj+^*g@xZsWzmJTmaoR^Tt9-JmTrrXBcpH=9Fi;yk8Ypt{tN%cAM_yi&+Cxu z|6zS^G}V9;`{a^SZYQ~}m%nzFuVuM@s8W8 z2X(6JUKmo0mZ=Iimz?d@145=u^%qglE9+@3d(6|D)?K5er+@jR^*a{CByCMb#_Ptr zt+V=s^Bd#-F#%mMaW>vjUG;)we!3p^xJz!jSFI@$d&3j^aLis2X`&CF@+>fpZ+nG+ z2OMIme)H&_(uO%|No+S{bC@0;4n9awGe=nKd>thxjHlu?K@dhV7!saQFh00W`oVSf zVdu2uqW9tSaDP2WM(;fijk%CLrU-_ls33lKaccyQOFV6Oh_34JR;h(rK#{pLdiBNJdQH9XB=xIP@MR&3YRW53YXN${J#bk{bwS6COG)^5g zH!dC%dw&@WOUGei%V@S4)7o#JMzxyQh)hSLgW?&1Bxwb^5hM{lFaBCML!2U!+f~^0 z7{2Sw?k5j5Z&FoI5U>@q7Kjcx9e6|!@&a;Ao-HMFxjve3sgP;$`> zBn8u(jYtNmN#|>)DETK-F_66=QKg21&H3=i`A|ctH}GL=bN$PGw~dk|8=K=u+Ryy* z@qbf37++EEjIhdQ`Z;6@6yX6pTlQJB4Hd{yIP;ye_=c=P`%5^trF14!(Q+CQgP&3_ zP)swgmr+TUZpZPN1Agr&v^w&@yaLv_9~~gd@wc|dZNLiMcZfWXR>VK6$SCFRL5DD(CBFh*KJNMuR0k-wDb*R^6 zDTLvhblC|bz2trxSufv^rfAG@qB#8G-|CBw!~|20f8)7|UkrEro8$bn>Bo+122Hfz zTydX@&JRAmO{hI~bObAT(>*mGk`2)Cupwz8E6=OmkVT?8)Wdpp-hgp9S3dH5S$`L^ zt%sd5=_E<>phEdvSClX&?RJ!kp&p@iOUDs){BZA%O_CtkTTw{|%rFoL*2_8UZCq2h z4*p7;ayi*J-#Gf$j%tJM`O39(wucxP`Ue|*Dky4Bh*qu9krfKIzSGpIjIuF8MA8|# z?){sI+Iqfwu&JW9dk@>YtfTE%9DiH!62FQG!GXdoM`_0UY~~o*q3{;BeH7_D&fyB9 z8p&XufunmlZqJjp1Z@1#*F#%Ac&RM?#58{JYfUo!=p{~5JN7v}tCYD)tDjMSAP%#M z;$7CRS7az}Op_|D>=&Zp8A^()hl|oZ>8TjVQM@UZ2|iX!rAt&3n&BI~f`4!l93~UE z6ONDl$dOkCz_=WJ+cAvikKe{|vRuyjxY^o(mA_~5CWDFQ(GWpJ6qYE^ita8;?}+J& z%cN@U)pb}jzUfKf{;~}Ztz@gz4&-R*8EbQAgRxpe)4KkE+f9T~f_9Dh0-SPw)t|mt zk!BdL&$^tiA^bs(u1+gqC4U%ltRKAoCvVcD&W@BU9YLOcc$Q-ysdAKkQ{6RN6$g?= zC^b-2RYXn;M5Fl{9pX(hKj{I{Ei^Aok-g2fe97%>ziGw8(_b(AhFV4|A}`}#J7iO& z*mDb2o49jD-M5K?v~Haj%tR4(1{^TTgi(VO%Hv%2fNO<+!0Dv%uYZmF@+r`e(oDoM zl6-~T!ETsJJc1ugoFxg`K9Rmbp6>0$jy~^ghls@Wk0y)~yrSBkn|JiEFYwVn114sW zZLYt)!Cq{B?Oi9SI$wKzV;Nrz9{1s+?{7c*2R?JZ%Y=1bdbX0B5lYq?vh5RNJ#K7- z5jZN;JMhDlKP&*zA+fA#dRZtU!|n z9$YVIjQp-+P!?`9MnBlD-+m=m)TCV3aen^F-H?-ul?A5ud4H)RAO>QEfvB!)O)EGJ zQ_S-dAxNvJP4>8CW>V*0~~BEqm_j1 zIm?xn+HnSlkAGzeGf)q3Bp_4Rb9q#l#G&h49xvZK;;+wQ3^6t?3qQJeS9)ma2N%Qf znlL^xq7u|lSAP*N=v<|4zGFEa>kdeQeRWPjCI$M%;fC`@H=<~eVZWIzbR5^O?*{oB z@G<0}0MF3Zn@_&r8|(HZmD`OkmOlIky*jK5?a2P!W0k7+h+{C59quI;F-gai5{PV< zKh!t$VaK>^U)6@@M zcs;3Kcw?jciA?N-+`^1RxB(3EThKwxxrS4Q^PTds+J5ejG0_GIKC&IMkAJ zSmM6(w|_wQR8Pjx4qrS6Sr8tO)O;UDHt=g}|M2T4?(q{#YzN%YuiWa*Afs2#*UyW1 zRhxPbL2W5H^cu~Kn3}#BNGb+mC^)gVKIE{OZMZw(C((r2TF7CWqG`hy47=W-1#j2%FKqb_?tf>*^H1D=`Ro%I{J{^ew%@qf(AMvI ztVRnJgG|(F67=M1vx4ZEo#~!l>SKbStwfS37sX4|i$ZKUt6r#Vd4O6{MXJ7vZQ?W^ITyUYO>E9 z6Sk@Y>>_(JM`=^O*1yVrg^5Ql@?^;rAb-1U#0WH6h%QGJNogqeMGFO76`2Ku=?CH^ z#7~wdL$+N7Zi&H`mf5^cn*bJcY`7ugafDuo-D8xt*K#BoqP-y`?&W54nr z{qhF;>e41vm7n-DaaEPdw0|;k z4uNhJFBSDmdQ}SJrp4hQ&BZoIr$}$>+E>WUIg5MT4E|Ovj3|EcxPlyl&;5r`pO}sK z$t!)^D6@}_`fG0zUo#AH{{8(I1N*y{4F=t?(HV!Gd~Nq<-^kTqYJNV?05w#`4fSFF zB$P45�mLlz_jobX1$rkSYX;T7RW}bHsAZ_e{!Jh=JKS%*Bx`14`Aj(DFmBbj8l3 zb%#`X*Oi&?ok=feJe@2Sj(oG6tc19eR$3!Ii0@E@ib&}xbA5K!JzJG&K_RS|(N(2? zXf9SZ_z9*UQj{`n3-aKPv>N;M6ZpFF`WJq`cRsfAM}9Xv_V>-#PyYDXe1C5nADa{G zJ=)Ti+`uOk(QvAG6t5dM(q+2x0|ipwK53B3T2m0c(H)~OG(;6C_TlU{><%8?sF)mS zD9sU&oLq{+I2SJyGX?u>ziydew97Vr_hDyNp&0WlJ{%FzdUO8 z!y5Kq^J$a(<>P+%Q)g+2et-5^+D_Oy1%j^)k9p;2ly>6J4Z*?2kg`!u24D}M_! zp(~Kfv<+YYi|6}&mMLtozY51oeDcg($A*7$Y~2tuA8H7oWAdN(xw=qP6qtPj zCZ6S-+KOz4iqBBq)G`X2v)rtn8WoRxqjjy_XgC#j|B9WUi(=qfA*<- zbi)x?dj0k1|HMboy|GQQ_{2keZ&?_fvX+rF+l;F+^HUGC_J5ADrH;-zdUlZ+T;O>Y zNoV}1fSCT%a8^6m5B5(Nt^L)FU``z`lB$%y5`)n`4TaK7Gc021I7tAk|;i zmE3V_r=*!}Tz_2G)uoXFIh|=i$r4;FKKq6iBcV|Oj>8-dTkt~d4|dYl^cJ+CZ5}y0 zz5O$P8|AD|A#ya$CwqgSu^gv<((3I7R%=Y^r{<>wgN-j93iNilbK>GGT*ds=9OImk znO;^e=TaRI*q``JBk}mmjmJR%*)n3O%jb>fjGPZHk$=CR<7oIe4Ss;0^V~<;qZKH? zn#gQZtT4WhG{GAWIcjH*tTRMNw6AWTkOMSmBP>iGC%3Q@31pt*G9qdHqjGcIV_73n zp9Dy(9bJ;ndh@ZfCR3Mi820tO06_W4pEw^3*)WHwC!+)LJ(VG$&4WY_u9uP%u4=lRorM}2Y z6wYj!-t28;S4N-Cj(C{o!{FzEfw`Xtc;L0a+oAIJ>g{*$L!+d5Bq6>x$EQYso7tFL z`1Wy0T}R$6%ufq_LK_RbaEI4aky$?Af!4wWn9lw7kA4`|*7Rqu#IWA^=!>t9!R=}{ zoPRCw!JM?;cwnQJs9D}+pQ&ac>ME*^K~^M+0$Y>m9`Td)zJn0F+c{<-fdf?!*Hp_| zy4H(Ac5~`F$PS3C>;KxHBd_%(?DlLpwG|S;n8yV_fBNSqo#zA~m09z9a_ex)?dP=H z1$X|&jogA4qBSTTBun&YG=Kv|3(!L-7=J!ea}2LgRvKpm(^PJn^$1~`MAUmBiz-GF zzYr_bE$lVaOWK_=3#*_%S<&GwF=_KKu?UvPT2FQphc=T!KBevvmKRy7Dap{A=!{kf zi)2ka*HcQeOTB5UZ^(uDcVy+Ht%4nL*F+$pi@dvH72&cCYh_bMH*fP&&U$>!_J41s zuQseb%E~<4bS-mLQY6PVAFJzTYiDJmZkwM|Zimxk*1i^%tr)!cy?;C0C;Z;z_ttPP^81qCo1jeq)qFEU2Mz-pED_`c z?HD|vmOx>S2r|K23JbpCQyX&wN`e!9Zb(ajGK&fV!zPa-=VxKT%#>UD3j>Bw1#05} zAW?QXecu3L_5z@m*8qFyMR3790Ae_M3m52@pivqF3US}S9z+5vGj;$q?|+{J-kzVc zz?l-Q_&G0(K*Ur92t&&T9M}@@n4$ub(1_0vDp0QN;x$q7~&r#`A4fDZG~uGrAK-fiP$E^gS2pKm8#F zprZ>@DGb|gO%onNxYmZM;`{iANUco?{?AWx7F)Cc3s?N}qk=A8OgS4fsYzvhy_7(y z@Qf0~rT;vZ_yN~w)>zxGMnJ!nee^^KANvV_1~B}<{|e*3*u1qlKYtB-O+f&qh7Ch= zFoCH@TN8xkrzV_y%6V#@qHHL4Z4d-OkOYAVD?ymPg@dr_g~@v~mSpJ=|Fq-3`m+Em zj2lp*#ES~1la~ZA^ZW&<;rN!+GTqlW+a_wl@-j+l(92|B3=70$VxN<&&h$M%ya%A2 zLu#%ciNk04TlyfFZ_lg!P^JsO zbnHpG1-8zwMJL-@Kte6=fa#b{5U#OOYFKTcaO9dv5rlfGTYzcj;5xGr5qTTj1{9QR zqf5vAe*T=b2Vz+un+u5On4NXs&T6!kY4<~0>~)p0+D5bndVh%@dIfYWd$FU`uvKFI z&Mw-9

    Sh>@{~wu_hQjx}U{ZBcPFAqG6A zNHG!&>=Gjofm2y0e*etwoFNfKB=&GoJb7^1sS7*~g$czNCXi!B?CTWF?7}G7_qjMT zm#*Z!xG27VF?VSPhs)-ui24~?o}6#v=&TfSifh@F9?p%-><)Vk$rlmnMMT*#oV}HN zyl3z^2elb@Y`3hhlidQVJrT}7aqA=v=bpdwia1Y?H1!hv-p@?l{|>lh5@4Z)IS^1} zk^4M@mEZfec~g${Uq3$_n4jV!&gvk_l#6W73W+6~UEoV_+@gS;i*wHJh-)&yD)U>6O=E$S@ED?*>a)yfGxUEd|!AbtgRPH?_%le zdzX%?Yho9tjli|@))3m!G(|lew(S%VD<%h)xlUN&+VKKxk?O05hQj6Ip;NGYi?PQ~ zrDrTA+XtrK;8&fuG7Kiu;$l^Q`O>vT$|6u@q*aac+9qc}_30d1HHd`{2&06J?88Cz z(upPYS10s5b)2-FD z{SBI1D0FEucW>h-e!;@JqT&^gy9Nh_;&P7_-x@A14s{+nl)IjPGDg3%lS`qr93o0L zE;BL`+m*H0;+1ACfKM@F_4dX3#|pH^OyTedsf*~#UkwZtuNi70Ak zr>rEUC9)Mt6CBY52fXfRO>p*Vf>YjvTuxdOobo0(`)GpN4?~j9ke!NYzz&*n&xS6A z=3Ap;dZEn3h;Xidi3-4IQw2Q{b#7iD50xX8ETv_i%(qe352aO~`#1UK5hJ3&U1BRc zDP9h063arrUCnn0rwg)Hko@z8Hui>;(KEH`psOa}Bzf;kCF+}s6rs@Hl**9&nDg|+ znht!E@OIEaXbSOailXWAQ{7{uLK<9O>|sFc!I6u%m@>hC&GnF9(nCV*f_KHvj;Ol4 z5m~UTrzh9tu1e}RUAc6OKrgQj9zLC2$(<8uNrhl{ZwP;9fR(Pn%I5B z>?jX&v#i`T8`v~Susi$6HJH}a<;Zs{4RK#Q8W6_6M}zDRxyQreo(xM89+o6BjB{X_ zKBZ4*1$9Lx-7J5Tfy=`NE=g&4Nh13qcR$O0Jmg`pE28)HbFvy9Uahb#ISk1=`@mVH zXG*x!h2abLx@U0J=NORU>DjoC;YpBVVD60JdxPO7$&x_X7dP#pWJZ2M*7Te(x++>= zm@{Mjgy?S`nSNtQlJHVH=fEj%l#lzsl>Mu8tJ0gfPNaWXw<*0vv;I-Kg$jDyoHG{9 zkLPWe&n-UT7G2UPidyKMW{&pGeX1EcDdGd?$9Ydsym3Aa=j~zcE10E|!ZG^GH=bki zx=mT&&B&hb!>x@{T7hUq$69m5jtg2%GUtz3t*K~&ES%|OPt#D^uP)OJy{B*=Do$!Q z{qNCn(p^!%?N`Z(r<$SZbQw?}UQv(WLGxL`Y@b2Rn;EyV{M#F~bBqr~R(8+_lKmuwmYQvu4Cz#0T0f7tWt z$!M}34Z42l!9!RKlIhVtSczmBNV9Tno9x&-Yx?WT;)5uR_=cVF04VXPzU5hWv(c96 zz*i3SDcXS>t&!a?{+O6}R-oIFwK<{pO{^!|m5tHA{GZN~baaWiL$539q?8mZk08V@X6$ggJXkXkjmR3kgY zsF7-WVQFKdy|K|*+I{wo-DiKc^nbth!GHhc*`@#f;DZjSVHO7gai^WtfA!>=Lxj0}EfC$VUGVzewifKzi4(b=j?1!4a=C?tj1(%z zmni9j**`7QDfw4oHV@;4e?xO6O=B-vXK_jc1a>>=SYaA?+f_Eq6Dz9>4Xs?^nST_q zRTn<#)r06Mze-L&c!uEls^?cB6B^Rckb-r&Toq?VaBB(AqVVF6l0*xZmG^J-NXYDM zdv3)MPuR@QQ9A_BF|6QIeigp+m8dlh>6^=MAL;Du`0bU>&Q8^De=m2|*li7|Fp*-N zQV|I7HxCOzhD6GAKlv|D(wT(H1QsmIT57IUXJSfP>O-LDe*t{YcwurPISmRTBC&oq>!rJNxNR(5AOwG^eUfVSv+_A znHs4Vx&lXxE(J~6OKB9wd)E%&+1{<-h63+wT#r!(oMlw(eij^n<8t z2N`m!G7LN$54`A*14^#gamB85$9=x1Qb|VOMsC*<=3l2l5*d1B=P^Q1!*jjSq{k$!v{Q^3_>HDTALfA|zBvVu4Aq@IjCJ2??w0Bu%y z&8Ac2PO;feIyAF7ChWqnq;?OFNBgY`OY~sqc-6g2xCf=Bqh4|)DlY+;<9Ogfvg&ds zzRB`LwbPPJW6y~~_ttZ5mgegBLT@{P#dz;bSHMCl4?x>han)$aJH~M`9(ftfpc$GW zDF{H{e|79>C`4dwM*Ykze0fH_VIwW<`iM4E4o)@lPw!}^DdA~g`~TqR0W|KsZ~7J9 z%4T$vIh)Xs|D%0t+AtfmX*TJyxlC8g6?)h_OjpfSdc-_JkD5p6G4mKbZXTz$*=*1g z@TY4woAf07am?m2U57ue*<7Kg;ExAe#nbSoe`hvV=^3-RnzEM8?#^s%1C!(CwiEeW zcYqk7b&Td_fe_h{c<|N`TFoPSB9bQwhZ%=SqnX9v(_39paWh(#9_og`*J#Lo)}&%>wO0?+W*acp$FeypaTC(p6orVpj&X zGm0vTMZ&;}#8d|9)w&;_@_K&gxz&JafAlRbE+lgbo03{kBdOT^;Py4Dj64@}0e@;i zjkGGT$pte7Swdnk-8d*9EEJ$(iXyITdZBP*T){B(a#jI(p!dG6Z(zif2n~K zZXNH8s)wG~h!5gReQhH?c;}&64quru2{RERu!>UIYnslLoR?!%a*4r6Y0}&qk zU9S`m{u6iuFOF@{W%l?)$us(ym}Dzt6p2 zZvT!<#d7M25Wc{EniM1;IA-uPp|BG04u-D-;>)%%*{EWRkP|0($O#bwf7NlK;bg?M zCpCxGQy%Y+oWOIZPMS{nlJM8JV`nryDZ}5^!cWWaceL;`GW-=S{5~1}MJ@cS4F9qgeolt}RttYX zhJQy3zaYbZDZ_gsduzHbehe9QeCZMcO3f++1-b+I zfrGX5Box@gjg5Q4D3Qa4tgIYM4~RUxr%MK3mPZ&R7N*RCaOmy6fAj;TB23>6@?xFy z(hu|i%!eDhE-z^crlV7(D-Ll_vCY9aEv>+dFNq55zW5U78BhkV!8hj=W-%AF)1{_2 zNHZf<(?MQ$1$ecs^F zF!zW!LA7WoILq=%(@7<7k&P+KV>BlZH_IcKQ;%R?Jv=3kIrRwU)uWf?aoLT~&+&9) zCs-iu3TM zm&d^%73Z8iQ|P)mBo?M(J?`rmkNaxuL%G<8wAhP&Ki?V*QgI#`#(ofmIu;MqO4`g9 zwW-%6&d0}E{DT9ppQ{T7NgU9(NYD@$Y(e3sej1OvHkfuwr%rlDmmnnsDu01InCU#R zH;%UXRIoc;D_6kXZaXi4+| zs3FEBN#>xt-N}qH^7yXHoajuMdFa@KE|AOWQlsAVw$rINeOcPqUE0^(+Siiybx`{Sn^IEhNKEsfZWqOF-(xl!UhUC2J!1dX0z=v2rHa-kzH=%N!u zHSj`N;P%WqY|o+;4C1pg%om=vav@PM$S(^j4J!#mun-J&d0d8Vi_0LkFc&YWc~nGb zDDS*Zz-!GJ^%_+QcEjW5z5TVsx)+z5CIoT@^fj1&+=-VHCj=xCvm9M>d>>5{|HXD1 z+qSL7PQ%8wt&45jY1ptq8{2AZHMX6+dES3+X7?7e`&r!H?0n}p6VuL58cHA2i*Y=4 zAI)c?bQx+$b~>q>xG!r<-Awy&`-EdwOq^y1@M%(ZQQz~uZ-x_#H%Qt(R&M*M5w&01 z(tl7x5&_Psr2^g@nz$D9a`Y{n2-Yj5*I^<01`cCE=*8+X`)WEqY4JKpk<2I61wOKo z)3kT$4$3NIExzYfNF)*CcQdmls}rbM1D=FNzXJlI_$#?n`p-8p0&ag0)}hRyXij^9 zDDlpJ!MAaftD|{gwq=^@ST5_a$GmR{eqMkx3j(rN17L7s>_--BJ5TN;*v_aF^yN@m zr=yBx3Wl{CXSy3L8x?!W^V*IFS9x$CWj0yfvN)IN z`~-U$HuRNZ&(`HOh~2-|CvFh=Q~!CuHdot*pEVN^onLG9S@O|UY05l?c@AGT)!Iq^ zCI|@5X>MOS@}=U8Rc<xjxrDvEBAhOSu=^RHQP^UK{8!^n}v zSBp{962UJ|)R4et=V7R8@fN8k&gEszN%PGkMye~8kSx&I#IoZa3k)PR%+9{~VEcBB z?;W0D`gZAGnh!=$Ft>(7nJ0X@8tjoAR0RSXTNZlA7{90fLqd4X%%sBm?Qa=RwbRb@ zTNwVpqRjKnUI1@j)m&=Ez8hoWxBk@i0-O0QUmO9cxQrZAMw(`vIktSnp5WUpBldE2 zUv+~lf{tL$mtURvL6BhfH5sYJJE1*{9H(5XsiDlJlxDR{4d@elV&5IADtuC^f9U|s zt3J<19fJv+ZR3KHlgHK?(?_dx(i~ke<@QI?cX=B=_NEkVdsH<9n%R$6n7n26(_W3# zKLiQlx^0H-;*8QJUax!}wd`Ndi%&a-F2|FauBy^vaR)y!IJxi%UxJ?)^_`p&13 zvcK#9SXii@TgP7Bt^4;Za4Y!3%k-)7DfqI|ci3iS~n(f{{*^g*G zq{AB2sfWwnrHj&7{LR6wC3x1(P_za@(WCG)Ha`a}q|02SA6YN^>NOIL1=&*E2%*U5 zWG$C1>aD++C&Ya>=} zbwUk$mfZPN3IX0^6qpQ=~T+I?U}Jn?})xJpfwMU zHJ-XYQ!F9L{lAcn*C?Lv4mPcgHz+BusT^149J0`z-?Jnoe$0Nlwo8(O*l!BzMlh$w z#7TH|qr7Xy_Zg`+A6at!^Ku<+UAevXt&d?|?wH5;DHQ>2y?9W3G+j;&IM2lV`~Lbx z@XRq@HC(%5kM==6s7wQpd??hS* zN8~R=?&aaF9~XvTeF=s8O3TTUjL-y8J9NP!$ikQsI+!fVx+B4|HXb4?8GIBe-}~)L zH!f01>)D{B$N0J_Si(f8+t+S0jtrq1*x0x>Ul^q6rF5AbM}dg>u<9|16EshlO{RDO zvxk@0>v1<+hCizD$07YxfF@Y6Lzs-2Uw2uk1h-rzvSdyrv(XB@< zc*nWf$w+a|yrg-z@N|5LB5moU8u69@%Tpx`(Jo!ifK?ec^$tlZ1GybRLfVBb>! z6HT0P$LRYB5D}ibsK8tg72ReVfB9WP|H?PPI<%Z#mE~QH_x;D^v;Lr|=a^VjayvAOI22i@X7}M*B15nRycRzjprb9c&3pU~mjTFN!UUN8F?o-< zw3PH@As1*yKg)Cr$C(=mwX+pLFwL;ODHy<M#Bc2=rnH;I3n z{uyMR0jQ3YXvwrh=@^ZVd-76JYxM5L)(JT^>D4Ym=+rKTUrN=(QQS<8<4Lc#$ov>B zo!5yO^KmrX5Uwy^u|>s;R<3AsXT$FkSpG(wSC?K=XXm=!EzZt6G6$^_|3GJ!s^S=RiYSgF2(rKsKW(fXPV{erPb@S@$@Q~lZi&xQcc>KT63 zAfQ#B;x5W(tq~!j3_fDw#~IH6Umg<)S59%q?ht2xe`Bz*NgVO{op%*fzp+VLyl2gK z2?nkoGp^Z+jRBsOkTXL5?HMNZtYfhz6UCGeMN3)diTbtQgS7v0yE{B-HAZOLPR;Zw z)>hvpZ5%DH&-Z;DPdaysXRbXYFD8aMj|7rvceze?q56|4Bi7Gn%)y*GAg@{*Rmba> z@M@jr?6}|~1_qa;G|vXD{dLlR4b;aUtgv0nW;>r=*6#`E%gDtM^EMyvU&-P*pVqah z(zGna)inB+RHXCQNvU9_xydcw;Zc^G{p9UDC7=~$An!@DWuQxfrN(JBP7?`(x8a__^?2#Av2>SIe z7mmdNp3SWwj&u~}6Ufa5r8GMojjV8#+M0hX7mVwI1a^74R>PB`-`Oy1Wb;G<&*XF~ zvx$E^YT}q}(zb*jV#zlI(pD|sqc{;G)=)1NJ56GK-?H-|XAH3mb1RMlpsH*wCClc{ zjkReEJZZrW>RX$Y8I9At^ni*CyEWDU!vrIfF^g8YNm*I7*LX~R zVM3wauyMt*Ok;4mMiPxaATz}eUofSK8qSfsDJ#nlQ*fGe z7?32)&}e&@40M8`5m_2DlaYiwa$`D8uOSqdm=)Fkb;$co_4}GH1nvuB<&(IoVx=)8h|E~bD3w&(u7B$7*FBM zqMVmU9w@LRWgveVUlm|4J**nUP!A4^iezP3n2+N_NE@KZyYMr|`T}p}_fvCFX$O8K zLo9ihr(9Jf6~`Pp#^#`=WN*$I`XMkm|Ao{wNX^E*za#Oqg2&BpkB@PIH8uAr`n$ps z*?jQD!o(2rDPU&~VpV#gsYE<=R86cfX$(Rk^WRYqq7kExC_MOWs6K|0i=hH@lE2fl}b(2Qk!JK>?s!?);2A;0|;dTju5I~91dSZWS5XH`-y^#GGpY^vW z)48zH!%H{z_?0SN^3JxTF)fP51WyZ0Q)nEz)db49vpJ{jr=*S52y5#_cfpzKbaZvj zum_9#VaigHAU@xte}?B54w_Efz|5PdTvH9es5^MVNldRAKc(*1Ki6^%5t5|@HJiWY z=1mvV7{J;7->b>owVCv>H%zlbSKJ!WUH90pjGX*XA34f1H!7<2^y5nz98^PEpl=Kc zvwwx6mZN$iQqvCotg|uAw)XF#t$O6Mw3M2D56|jgWw@zp>bNncE_PJLs&I%#-Hy*) zSSIb~q+DAIfO3<#mB$`@$5GF-RH>aV7^)5N34vfAQd!QBHRHU^we7<(HEcceJ=BYP zxr&Bqd@P>- zARw$e-CFY53(}5bWTzCp7-d4Z$+wi8eMY$SGk*(znij3P)2A&v<)ViX^@QuWgdU_g zCjp9KTVuSx7|2r8NAOnr@u}!zj3m;aI^`4eZa5P&DN-cIjTD);7HGeZ7LA{Ee{PD$ zeN2sCs%X1=jb{yybR>Ub(GW!E#}%y|Mf_n|3rp*p#QqsnFSalsF5ctRhy>fP97)+s zTH{Lqr#j<kWj8LU(>GA# z%kbEoRMalJIAsV~0k=xyv~qGX;D#rS9#I;9t0Xh9a7+32{xK5x==ukHBb zbLP^vDAeV1w!Chm=~1h<+Ht>j-gMwm>u3J3V&PHSQRj0uux`VArvI{Y-t->%Iea`Sd$#0)efUd>PcAv9V?wSwJhZPU^TEB=#ivpDP2N?dT7%Eor@b;>q3Od4{im4&Y=^A8G3hO@ zaT1+aK8zr9NFF7X6^UzV22c_?&1sxu7LrF!r7e2sWt`Lp>lc<``lWCRJ()oC5Xv~I z7`8)N-WjnlJwhgq{PwHkRmAisD-hBtDGy%c|0!Zb2BfL4`{ok{>xdf8i<@y0V@Ri{ zJUYp(f}>H)bW3+Sg_r>qRV3;&7wZV>4MYnOIpPa4t4z$aM~@(YKlmP!VHz{76e+`G za;h02)69DMSwy^l@D@Jk9jk{sQs(`HF317wkrNVt-a<&|nSX#cKD3ZCX4cqSZP&$0U{ zs8OH40IVZ2AqCTFupQ#^&UmCWXZj%pzg|#-{!WrXmimEzM)-e5xWI~9p^!jA^2DQZ zyoiPr)T5Bzg0YT3mEguShoc3}+{51UB^;2{&RHeF=FVTD1*0)As|yk?KAkk?{4nYk(jl+wLD6t44aL(NC~4-$byQrng4AZD7GI|iM(4>L;dl&{6E z$kng`H&H<@#9{4)Av_ccN7TT}!mWcaN95a7Rml4p3`jukaMK^WJ38cjmlN{+NuUsT zVGWoi%Vjnh*E*n7x7MCucsc%4?vY%NYyE!A?XvG-j z@ev2V3IG2UKXjMKp>SOsv*A~k<+0DC@vw%^9{KDt*r>w3Ko12nf;U_NH|YJ+<-a1r zaol64Q4GMgpu|4zp^ZOl#+Kg!pN#b|06T;V=!Bfc|CEz4*;BfR8w`a0Mi78q_y>T7 zg}wO#$_sbh6p2vZwL$bxn?^x{4C;6h4A=lY_FaHrL!NL!(iteIz5jqukL4ue#KL`s zGzfB@K;Seml+^*|)oGqY^!NCW%yacAF*u~&9Uds3@|M^E1WI`%0*S!&gN?r7fX0uI zl5NFS-`_YvpqDa`$oj?`tB7iw81oIA@?h`l0828V6b(<2QFx0b*Scpu;OEiQVFV^oWBh z*GSNPw@;h_EV3J-ps{|_S`l6Y_K}3tN2Gv)pPw9T(EB?hC;=H(bd4M289dzu3{kMk zJ*b7S+=qjVgSb(9yDz{1*i=81V$wrSP=U$DN0r+g4-<5mH(j<{>A+XC)wXc6~Opz!073&P-2L2nkis9Lxa2Gx~$P&EG?3^e|M05W>t z1c$YL>6FRqxqpWO72LcSfWv0pQiBS*tEhMbUH!2@G#6f=fXPiM(9pdLs25X`A)|K@ z)Xll22@9~Iw@(;S^j_c~+I(7pK_(f~RbMEf8dvCBu>5$&4K7T@LG(R?fi?Up1VK^_ zc0u0Vy3U!LBcXV&om|~mfDLAB(_IAux9Mit@@0kq#R6dUuy2<)RJ;QV0qJ03Tn}FZ z3~4$|lV+uhus;WW2RO)mEguBl>x&EeM5~KK1*!%$aS?4mL9Nif*d)!+zW5>}<6|Ha zqxT?Ge26JF9E*n=#IS}s$c%LdW5tAn2L)$nt2_SWwXxB6Fp+8XO&E~m{Rs#}*@*yB z26qZ@76u2oc&}4DkhVPwfdbwzKnWNhup|_d*&r|{A&aC&=^RZk(2g(IXZ8M}sO~^- zHb`t8fCkMSV}A~LRAS3ZFHnJo0|cn)IafxgN7jFp;$b%izyOucUw{R4`b<$A)Wrxx zfK&iT5dIBzvOqV_LL&$)+-s8JfVIp6EC3#W18G6%gFsI+Hz2Tk3s8W&{{RJwF@F~# zh~kA0BvPJF8qjxR10rkl`a%k5+#lc~wsw8aZM}XGrV#CG&aG&~Fiuc1SZF6CNcpe| zw1pR7>6_x4*~25-Icb<7)CG>7DV7p7aWQEKpU*lm_Ibh-&eno1ep(T zk%Ajd5QxuYzncnx0$u)X0Tbzc8=;72n*=5(;#r~Z8AOIV^I%}v-UvZFg~AA^&ikJt z^O)c>LqHN7`cMH+6ek0lAjtK1CeZK&T2M_t&tcalM8I!ysU1VFTa=BT6!4pmq2;BM;N5o(}oM-;jc)(4S{&<*#Rm zwYDx$L%I#J{}n%cvS_|((;wcrh1j2IqY=eIP&&sMgbUVCIzxz+&tVXc8bjU}t3Jzs zNLy`#6|?`Gfq-8aRA)R1pA|CMz*sz-Mm{!3t4-F2`guenFg6& zcTR~pm*^?^drQ7X-k1I|hr(xk{*56Bf}KeR!r!EX<*TZv_Y~1kG9Ir8{ToeBysxn@ zK(8qtt1-N%_DHXz4$lkwJ3R)^Q7`K{WJTQ}te5$3)o;U4{jUc+l5xk4p+4XKEOJ(3 z67GC+2~u_2FGbsPYyAhQR1FWY=Nnigu&C`j0^-LaS0v6;*3=Zy)k8;|1FS$u{p&H+2?IIHS_z!9fge5WB>iLbgSB*J z8XYg`&j<{P*Bo^j^UuypdL;o_e76$folKS|D_jR#&bIZ{^7TF@|MN{c49VCtPul@h zL3C<1FuaIOHQke^@9r+2fw%-2&{?}CZ^HhgT?YDN(5U+08lVI zd%=9|oFemnj?T!)dq|!v4}JCN=ICOK2E)geQ_fjK{2QnG9C$f5FI)l*MMHLoW=vC4 zV)NaBToNUrEE5J#su&?187K1U8xZ>KD=JbI9#fmA{L!o~6Z!W25>A3=u34(LSl^te6otvTSOewcNQWe{^c>y=v;A(9GMrbk|cV_+^y# z<;RUp`pzZOJ3g_QhF200_?JW-C2=P1toREK19{J<-G^8#$m%^;h}@S70QBI9_`vAh z?E$BBTRBgzpI2|Gnq0DDMc;+psZ`-7DWl z5Jg53SI8|c#%I;l(qaP6%E|N9gE>sld1>=wqWTq!zzSvdg}(2CFY~vWrf?+rHdGe} zfh&#wyD#a#o{Ynzfl6Qg72e{@3lsMG#NRC2U=`{Ht<8#b;_f(**Z)8(KCNRZebl4d zoM}zRI+|2uNI2DA@=p~m!v&QZGR1gbx*j_aN+U%SGLl!H>C=2~pG)>D>ku`L=<6En zCtOHdpeESF4qCRUyzz{xRwEZwWR#A^G^K_zM^!3~8|=f-&eBGcJbVY4sXK6=zgXyY0)77C;j9aJHJ2>;De`YI ziI>US-PH?WOzac^@vh)LUo7`R=0h(^QbI!a9V>P^)h)5vW?pjKyBZtG;XjX>X2aZs z$2(ukVg1lf#=pXBW)RUYXHl8~e$F?)LfST*dXFm)_ae?btY4?1 zqQXih{P!Z@?gAWspBj4elq^z{cxSV|vx0_IOgVK^)<`k{Ppghm95+8yBw2V0t~QwE ze22`I1;^uS`l@B*vdfe#?c_27zmTiBD`Lmh=F-~x-LYb6Lg(U2~^ITbEqV;z9n;0phf$9(CvYY#Ec8n;> zy1mq${h1gVsCanJ_opP z@9SJ9rz&njVFK4r(%UUmwmJw{E2r=~Bz*|U9(O4qscv0{{h+_D6mCn&mQ&|S&z~NP zZ87f4!kNm)_Q-U!Q3a+W|GDyF)CCk6voZzGZR9|(%<#EO$ZZmW>V}eCy;BcU6 z7zM8ZL*|HJc=JrXqc~ARol-jr8T=Bf!e-rpsCDW9kq(m~qRpAiEJ%iu{-5}(kx!Bc zDm^NyaQk0BaFwJ8;)BrJr6dF6!d&JhP;tdsBXOJs2yK(l{2KA?FkG@W{SRiI6jki2 ze^caa4UbTO3SkD5$jv5fqzI#$$DW#bIIm;@*l40vwc$`Dng~L+o+1ti(?@Q*_<)$!oDX zKTJWXv6BA8H5OXoJI8$_?lUbGn|O?iC(FAetT)$p-;G=;|9t^NPg|g3T-2spXHb_0 z#)j?C;Bzw!Tn5Z?Dj5dPxnFa1nHbt|HOWJue^&Z`D)pT!Rbn0f-T08}v1wOt+0KNY zKxmvH=x?rSSm){y?0Fz?1U&cuv=%56Du@{1a%;-)N#G`<{~o>eP3H?QcEj!)agkiW_SzrEmEygOd)^!dm#I%I^@)G0>j*u7HLH z+zP)(79;RpoDp21{M4}{MD8_)+`WIINg0Rk!GPxF&?+nYem&whFJ-x}(iJXY#xyl8r{Np+?i*9M8AY zZA(cM@jG)@M?;0o=Lqd8$usjZ0IMmTADS@S-&pc2RWS-To@(AOZOTU?bUc_Z;29}F zD?dT#x0B;C@?ajWvak^D(stU>4H_x7=2cR6TLvw7zkb7{D_G?}130^y!-tixB}y18 zy8HSA=6A%^*T;9`Yv?E!W&K|+zX;nP-;k2d!*X#|p7Ec*E)FOecpGQA1BlbKt&K9y z9Q}Oz(mX^^Nw&{qzleY2kDES^+Cz>eGAoWkD3+@g8s5`o+~tTYSIO8ch|6v_h15Rz z5MBijv#QW%*!i+egeQ$kLC31OebgP)r_gVJ87wbHE^&To&QzD|s!Dd-r(KYaikoUr zSnS|Tcb$~~TUZp~epjZ-3gmqi#XF7m7BsL{Y=iC+D68kbG`aLU&KxkmICVs-Jf)MZ zXcNE)!~7s5wtN*QG1f3Cg=_pQ579$s%8TOpY z_Rd%RhhqoM6$N*)CbXwm_od2t--^Te`)#dZE+WP?bA}$|(z8WOJHQ0C^0{eWk2G3G zqj;e*QW@v=2tLhwlUft_Zy|N`PMY)U((`!hZ!3^BClO=+ zE+g7zcYOQFHVFC=rUX!^&mEA%z&XPn3XC0iiog|m_$_|!vq`2sNm_CfYopH;RT=|v zCJrI#ID)Gv0xb={Zkbb)fLrmqR0aNR>!&!uI8i0sxaD=Hj}hDaEcHYt&b=Q8@pb_6^|LgOJh|E_ioCig}z7WGs zp)e;pyB_~5B?k6qhL=;O)(V|keyKX&eB-yE_HUgNaE+GOwfWXbJ6=lf#>lIN5sX|o z^Wn}XY)cS4F&Vk{E6mu=INtQPTg881cAqCnm{!}-aNpl(w7Crf@;lAOHW`A-+nBoX zx+rDa>l!Xk87do4^NvYU2haV)Q%fCRkF$NS!qX9B-~eM*@~Oz@!jBj>h0!Q}<*Fpv zFO#r3**5(FxtT`OODo**iik3PcW2&%g8n3_HZK~*^IZr`tOMU24RmwuzoV|5Hcsddz594?o9 zeDgN`MFFI0aqc_(LjQF5u!u0ucszuij8?6R;Zg`^aEZqOR;Bh>gDyAm&Z7_w&Z3cB zGG5)fWPBYz?#@wYn&_H<-XW)<o3~ThC}AfahiOcTzXUy?;U? zS1w^2=`OE=*Au19;%M5hWGllfrF}@9!a&R8%@bz{BV*&YQAnpIy=Kvj#(4Q|_?I|6 zuqoAPw97R@Z9CS1Q~wcJf|>sFZ2$XP>KI&_rXw6HYPCE2D@8YQm?>Y6L{@>?>smgo zjyeFNoK-io+Ms<&cL;UMnt^ouFU_SE-@8heaeObG*EmtA)2rvujGnmkCt#vDM`x1N5)=4YPR;@=A6Z;aB-(!}4XCCPnMA5302I?mq}RY z)hW)nEQIpG-h859=(-Nz|6GlES%A?eSoQOah#L(29;k`0LAUy63xfY)fIpNLn|g+Z zD--InT>L7>Siis%-(2#}2(uW87Cz5@>d!tep42YdFUNm&ieVe*lTwz)6P!tQiSj!S zeU^}oYBsKF9NvGJmg@+*+N0kEB056oBeh`E!vy99&ZBB>eJ|^FjJgOeRDoEGO?l^xP zkiKLdWVB!LwK>8TL+LrHXge!28>8l>t{J1})l1_}J^43t zy}|{v@+Ki}MIkq3vmfkh$Q+p*XSZZbaDWFf>0IcFBDeMz!-c;SZkh9S$v+H7Wtu6_ z*sj-|Y7b&7WifFKjqUyci$Ub^7Zu?PL|Ak^{o|&GGe(4TmcO-sbkd+yYGd~c;}3S8 zoSlAas+`u2IJ6Zp$F0aMROe`e1d9H!>a4ZLG*P`%Ny*k&v3&(xkCJ>h^|MZz^n$b1U%n>UTLy zWHOY^q>7v4>>yGMTuWa1Zn}B2EWR{D*}-twVcBL`P#>*TXIsHn$D+bz!_B)f&04p} zTAch5*@uc7vKE+g*wlBBERiQQ>Kl48PQHZFe37#R}zn&THAW&jF<4CTH zSy_IJVz_Yu1arRgTP{r=zoU2T@HY0p~p#8^{aGX0l@oEuO!S>d=Sh-PTzZj2&9L z00e%jRHGEC;^*g6Ugl6wvDD3`LL_Oa&YBto@|IoR9D2T-aedQOlNHZ4?S{#0=(|8g zHz@#vgWm^fA{@TUPCgX+d$(7Jm>7@h&`=WeEB^qMs}oNMMn$OD5LlzX#}PtBAr^WN z@ZiehBx0D7ig)yH?&f-dbwzVqYx=BNb3M6kKMxuOdhlPMu!31C6+(>E2H@&)?_I9f&Xz?KqmMC<^-O?B9DmH5Ycp<~ z6HFD?w81K}pO1~#9-0?m6p*4TEnyZk5g-^n7)Atafo9=X(>tk00s<*ydt?T`dnY5(_$86t}(-0!c{uK(UsYFW0)Hz>Y);e4PVWPVNk7b*Nnq{)rXr|6K;x>!7#i^CW@Mvn-R`HhT zL8cJ%7?c+W!gK9jYT`b_3AS_mYodgmH&Yhp{0FexS0>Pk{VrY5YQBox%3e5 zy|=dp_JtO||MUOhL$ZF>8F-vYB)GgeV))NL2x-8aAsgA@+4Ul9Jpvnr4@P>~Rd=Bk zzL}UX1kzGy!kn{yaZCkhPT8!g$8oErv%5heV@PtSjZ;n9!|p>1)QrzJohEl(&FmWs z>zX|8H)!cR7|GT|D-M z6PqfHZr`W~jPpm-&%f?nK9#p%+^=f+m}z$#=t6G&Rvd@phL>H8_A zA>7`-e(_X(qmN>9-?>)hWVOFaOe$>+P5w|Vyxb_5?Dw*Oxb}XR_~+Eq+r~1x|1SE4 zh3Dk#`U13cIztXTIATr>?q1!F*d?s^Lvon%af_uo0haC!gtA`izatmgYpK_58IElU z#r;wU9U6_@%{DvN{Vk2|FSnJ6)mu6=&U_}8SV~PeG@tcCjjHZcR;pGEPEPdYCi|zj1hYUbJf0HX0{z>ti$sM!iV$k1LG!HfAheufRkj*treHe05 z_GiiQO8(&p9OZPt@N#un84=ic+GsdjhvVeFFWckLJ`!EM$Ifl8RT7!>W_*3Tz8j?3 z+lH~9&jy~~uL!;go)JB%D2)VThasS}heJW8e|Y(FS| zqL9#|`KVH0D;@W~tahmH1W|(dk7m~~cbc6f7vbe-2b)-+4LN$qtr)=3%12N$xI$Ev zrZ68{WHkrWk!1{o{-dTYt&vp9Q}$p5h>&Z~*Sf#hqOC8E*@32_aAb&b#jUB@T#+Z9J}RS)7Thts-j#5*fMP+bG< zcS;u?tnH2bup$}q=ET}ANn00AMW?7;I@?HJHmbRyQjk70>Ad4)q3yZGdsjZvV}8f1 zpJw5W3b4$%(FGv$a23bd50dN<*Zd(wp&GHR$(2d<8sm_Iu~!h^GygY9B*byy`}&;b zk+avk+kqVq7XRVMd9G^u;vEWDMjd*k{64A=T^*yVaOED2JMuP!t!U9n&9(@ScJ;<0 z&-q{oh)^4*sdu_LZS>VUw&3GkeB=q`;{SGM>X=chD)?fEit>K&@woYceZXR%hh*4K z+~xleQ9IK`%C96l_scA-=2`qb%vY7d#r3w`{<-7CmaPioG05I7bOIbWVN!TVN*hndyr6;6>UC9>zINW>`)_zC zmBx1C!nVcGH*HA0k>3sO?7t~3j4h}u-_OO_$6hrc&JBO}vZJQ>FzF_%F@P&gxhqv1tX1li=B9wU0 zrW8w<>+AM(Wuofo$Fh`xz+3GiN=@U>nst4tTA~7)*O|llV0VBl{^P3h)$YXd&hqx3 z`qQdTKyG|q$(6_H@yO#m;+v-kQP*9oJ+wf(B(eWe92I}>%Y7N{tzY|%hDEN2{!e%P zH>`M1!S{}*)u4}6qDRJdgHAdd&*V)ZFLp5+#FOjKW_h~_R2qSG6AqnEMwZ3tjHl8Dmxg0na<@%32sE zUseOD)hW%BeO}+QUoRsk+fA&$rU$onX0>&V&YzHQ80OjF?9{(0A+aLE>^=Ezt%MNB zX#cpe_4NC#YicII^RrU@_3>(80O~g?1@nG9Z|N;-C*YLCgmnc%p?wgT$L^Y9-UV|j z?-WLs#7Rf@Qr;C5+ks;eWR&jul5k?HhKSh~L z5ds}qvgWIZyA9sy(i4eNjK*+_Q|P}1yTg8;h!|OrX!5~9=*)G;0|&mXtb~Tk8t+)x zp1@KluoE6QB;73*e<|&mv-Ls3#^^KMKsam2UitgHs}p+i9|bd0lJUJ*KBDmF=IS8O zS_Ty5Cq;?vxa_gyp`y_|I)G5-wv@}%UV7>Z>xce>8qE|YS{WfYxttl7~O7# zG+&U!U)u&*=OW*D3G^jKXahxFmBbA6Pz%_?oCuo!GKwUG-b_$UI=K-fH4sn;xgQ4| z#lnOg#Q6i$HOln z?MtsTU{Ot-eA1m(0wP&d$H>QH&k63}qL zHfbo~-YmD>wsz6*&?pPDl*o>>QS*9_2*LW0QN!M~{(;4G;6cdK>(53_1u7&Ep~XN% zADaL~aKZi5QT&fHF*6X3eI&f{jMswbj;#cc=GH{u>(#yf1=$1F0O+)}GSH*1k}u>X zFOVO6m>qQo?Up(cGZ4m*x(-sMA3a|JToNH4>y#)cR~gS!A_NqLDiX*8!R!>Q^5f_& zBb;1oO-Nb91f>V%<{s@)f`(WGWgYBauDcg18f>bFgwR^9Z(9v18hB?&PTLZTWqtkK;i2tKX;jHcN}djl zMPuAJ8hg?o#68)b|0{~_`Dlvm0ZxoBH!oNpsRs;YY1lFXWknJoRx2A?z8}N&rif1s zpm%!i5L}v8;q8P~3`XdzXcBA*ZRwBBtCCQomowL6w|?1N>Q9&V+s6>;`96e!j#b_d z(|#Xm7sNLKT&Q)wu&4yISBknyB?R!gJ%%7vyX*5`on&l z6eoTq{Zzy^^2;mopiGpJ2Uwq$k@E1g5FjU&yVn@7b>70E5E$O~##1CF1{0@+_>!gc z%_S+1s#J>5UL?#JG|L=c=p{W*`6NBl%8c$L*FxF?$)ES)rTX?>{I?QmZn0y~C<*f; zIuMZxj4(kSuHQLp?Jnd_>)WP|`xr|er`_^!ZKu--IL* z`opEB)u}{^OTB zCZAUT5S#>9c|I?a^JiWc?5}T>2oJ6kw6G}Y(~U61fa$hf@&=(+gbV-OJqo^v;g60M z_18mm7AONaM>(j+OnO@8x+H0yZjFbE_Yy!jm24(P&3p zl9+li(#YY9W6nf2E%MZ;zR#J8iX_xCWsPh=&2ev^n~E=A8)SZucFrF!XeX^1XF0$; z2wJ4%;`E5&=kX>&9RhFYB@eR#zSGf#aFu@!ubpe;3J&U7wzJz+e zZ#qLi;rU&eTYi>uRQ#osQ1u4CCzxQaYt^c$T%^})XS(Bab z70dh4=6KX)zD47IFKE6H#9_R}iO#|%ZgKdN zztmDketf^__vpBEyXjCgE4J={q1Kc)+i1Qlbj0+%h&wya@89W&0+%fWz5^VZ@9ME_ z?Z7IgGM4=ER&+R*OK>Yx_n)~suWj*&EmPvy6Dd9Z^6y9|b)h5P;Dv5!3nK2*fV*v; zC#wD^M#?#2p@oFUo#^ z)9e;(+~US5J8Cy$y<^%`=^LR#%-EBm9vli-vqlnA%BJkmo~-mWCL0>KB7^gwR@dxVf5xtm z_VZ~YZme_tT5>r3Q&?iY9_|JcUFcpbsh7OmGj?a1sR6+u>RLPK0j|EQc5Qzmh;duO zfBZGC+x>K@YlES7bbrV>jLYk)4bT3vIhuUpQOJH!IOB*;2HOmPRlB6s0YvlgqFHiO z%b0GHz@b7kgv#pEBpj_0M+-JY(989pKZ18qQ(zp=frWO{)nDqq=scWcp_)~aLzzO9!rbpcCSYuqjX*F~8Z z-YQ2YXXhl(Di0!NPx8ggoL+zB(y1}dr`^s5FIQU6VrSr9l6?##sq#+)a;Pa}9~SPiN;s{W6)Tq=j%?*IGm*M!4%_|VX4K=U4j(=& zhlktD^YH7wlgFtioeO_Yc22#0r+W5?xdZ3112+9;DUNSt^WJytXIo=GgR!6OjQz`- z$8OHm9u#Z0mi>Ha-?;^LTQ9e}<~F0-pyJ52fdOXm_LZ{htMPyRui&czm;6C*RPCW# z4{a~A2tgRX3dk~99;75G?oAPKmL#Z?zA$DE^l-Me4n+DOoSto_AaqhvjMqjQs*o3>o$Aem|HgFe*<2S!bmk?+AB_=p6hX{=_e*Abk%gEQ|OfS;&f@HdSyt>L~=}bR9 zr-TQDKR#J67Ih){8bS*H>Q@+%r<(nFU44|zo)(2Oc8-6JZMm)bLMO8;IO)?;J7c_S zd~Ti$uYcvh+xfbRogs)$N6z@eUVFwr1|PYmvM!1{Ro>LYOBMx;;H9o$L7cI3k}OJn z@T*^ydT~PX%*GdW#!gaRWm9K}e$m*t9pBFE9Ov<4XW2v@uCG;T>CEy>|H^S}fHE^p zX!#;Z_bz|PK~wqKNoTIJx~fj|%yld~>^e<+8v{*ed7=*N$prkIopU37; z9l947?W5TdJ$A+pJ^0n;;8$-@_B*B4vG1#L{MFSH2uAnUG^;`r$r*m`>~fG; z!-%y1b~VO>cuL>C9Zo66v;HbCARyphUp#-aIurxFzrJ{et^IzHPliW}!{ctAn8#ly z=ZVb%y%~*0qs_rG1JpP~Wx#N>-%a^%SHs#Zu{<0@*c}*5r!*U59Ks^WX8CeB`YqET zq?m3JN_|y6d^r57gp@@PwFWj6yH~_8R{MQe;uPEMYdmedX$=TwZWoO4%B_*^YBPVH zn$5$c0<^zKI)l(0SEK8y@x3*fEz05S$*dD=JYEqvBrysh2nuN*1=3lZxd!*(7&I6g zr|uTm4W<}@2 zudYU$(ZLG!H_vM{{djD4nacKLs<8lV`t zp~CUV-F+Dd77`xqxOP9m%&Ira#*$*zlkHx3z}`5g=8q9#c$bU8yR{+MmIo;mZkG0J zUtGUlmsNUl@{3!Wy~qO+wx+J%_iL8^I6=g}P!wdmthSnLVnNP~?aG+e9nO z>EC9m;kHC=zlth_{I}^0Zux(uC-7P>?NaWUNJoEjH>FJ3XAqj; zY`-#QFEHx190UuBDyI3u0dWPa_iYlqZJ2JuaPOrjWJMUp097}wIOXocuz@JLW10175O$~S;iR_==Vg_z1|5OME?$4nFTz0@Yh6BOEbzO~8N4vr z%}lEBO;3XS{@7~Kbe0rno=v-J@U(P+k^XbGxWXg|9r2sJjZY89x{2aZx!E2YgPZr> zI^08UgS22Sq~Q>90^)y-s1LLStkE*bW|&{D)Rl=e)+Em`FCE}D0i#CN`LF8+Y|3M| ztOKN!#2)pXuN_rO9fG3~^tZ81>e?v1ImxH%ayZ&Fr_4o~&GHM9%w|vF+VdTX3loI}fGRqk}5lj3Q&%)~!2`}oFva5AnH zB29FEjXW|i!?x>1K1Go>0pe=oi;+?%#q{*^q)3+blC&I*o;=z2;G2OB?NgpY){d7t zn+-dbT(@-BZ|r|GDY3DQnb~l3{ka$(4VD9U@Xg?O)MfZ)hsbR=Ayk$bIY}3lE`ER2 zS`T3yOboX^K00=DocU+T3e86>?mPL3qeqxdXIPSS=9V>2v={@|Z9+iiIM31;o*Yae zK2a0a-(dX*<1sxL)ipbhlM2!@K6&CbVR?qrK5i(s~Ktu7o%;ft67uk16B!ku_jTH?RPPLCWf%$ z)NnL-_0KQzm6ws} zBGp-C9vy!SKhSUAroU5k^zb2UaOqO#YeVbpTM+zr755v3MMN-!AFmd z;bquO;ikKXYd<%5jQ8^A2f&gq)C7KHXGt-Mh$|sM;%;LX6D9&1FO^A z=<$DWc&5i^mV&pA)H6+%<-DEofw@3c!$S!8JJs%$`|;@Q+fF|)o7Hf%ai7_#Ott%5 zqZR?uHNn8sSu+&VTsdr3)}3lg{H6n=(|13%qqgsXX53yXaUq+g&8v56-smZA?f^{W zK3MF5>w^>0Jpi7(7>?p;ea5?h!Dd(AxJQ3eQs|Qg>I*%7{_x>&_~lKj{bjw_AB@M( zNjiISgge;teLL9E@%G+DJ4T-$(9Ln&29UZ{?}=BP#XN=^PUsjuwMa4(8p}HplrGC- zqjAFv(KKI_-Vr+~UelA}7~kv{vyf`((i|uAk zvZP8adrQ(15<}(?&HL@!sRrKr&}6~oJgQ5O8%O+j_z-{seg8>c?R)$|=lOrbhq%`j zs5ALkpCs!=H5|DwhOloTE|+r;Gg=Sa)>06hlI{8MUa6?u%|?V8dHqByY3?vwOi#0L)Ws<;o~MKlLIobL}7xEZZj07)ZY7g~c++@Q)=ug`z%Ur5lqOf&O0 zxqOYG@5{llyMp&-S6Kq@zQ#1c;IvW~=t~o77n-?Rf#oMQaam1YuZnzVm!D&F{A{G2i*&JYCfcb5eW2Q1=#$EU^dMIw+ggcl*_Q?zIR1Z55ApPac+dTTXnaL) z;K*_BKI{M5GwYYhas`RHZku*p$N6Th!5_+uw|)_ay83~K`j=DlDw{%O;& z9m1cp7y6sQ=DlE+9qz8(N3(PAXqfF;I$`>quFEP1bG|wvc-)cO>CSDimNVd$W@s)p zCy@PoICwnM%N(p&!4%O*oxUz1XZQ&J4o9HmHKMm+2CRQtM*x$e0H(PCkCqupR&ba9 ztej(IkjlL~&@!Cv?P1F@9aXi!fSAdi|+jA%RfJNj2oOY{3I`&WnO?;!b$ErXp`iCf5pJ5%}n20G(TD7 zlgG=XtaN|zSYP6fQ9dqpajuKUFP?t%@n=uTvZvoN&zjuhjScQ#gc#JtnD;}@zwROI zrq%1a-$h@ZA$#<31m%oAPwOfIeE0=De}oeFH3*dG4>Kp^1Xm-^)8vdcmjOJBB}Hk zLOD&=b8nfJQ*yQJDhda`U(cN(J{TFDk>MtQC?F1->CE|jp_5WO$)e1ia;2x~$(1wM zZ>D{rC+qaF?m`jU%`J~c7GwhH!lL>@y&N`gUH=)S6I1_FR7_aq4Uyrg*Z*r)R|ZTfTOX`zkfm0TYRV zV>a&^N!seJlj)?u!T|=lw>8Osv-dFE4fdD^!A{i4cRxQIUc(Lz87Q*DMPBMQtwx<> zB6G{+Tp#u`PVCyA(0!&SuxXntTjr}t(~zEAK~@q-w}7Xm)xuzZ#d0|KsL;40plN?2 ze7Q)n3YHbPcQB`fMXD-vYldt$)8PeNeOBYn**hQK+;=(Z9;%mM+cvMY8fWAA(Iq$_ ze|a*z$d4|MNAC0SpPKI@CA06md_JNx%c+%2Y)!hzb`sc z<#!DtTQ7ByPTNtyY{$=AIF!Q9j<1UCX0N&BY|AzGfaP-=nn9k z2Ug32erkmqgrOwvQtsZgcJ)mcZ&fj`R?AW>R6z{ml{g}Y~K|y%D8QpIBm0y22;B*3sz`QFX z(_dZo?bpnr8jiU;*JjUu1GyVK+>Kz5o!3sGbky5+&JSICD*j1d#dDDQ)|NF#`0Kbe z1iKny$;k3bbAo?2?%Al<1MBy2_0AqTC&YVq#`~R3ZtU!X55Dt+#Ath`KHs@pbWNYE z*T|eScNQ=QGO2u@bV`5Y&iCQ^t023_g<)#0#%X4;}y)dDTED3yP18Uvkr3Gu zJ+FWha-Wsso|VUT8UbumKYjaFe=kEjaQ^CQsNPrwPwYInzQ~XC@!Pk?ZEW*qciuVJ zMdjPJqCL&U`TMM$^{5+vB98R7HhKRgEx*OT6)zf2dr;!x{0@KW{(5d*Q&;c0#N~bR z*#mo2lR5a=LIHnV0X$%5j5_>A&)~@&ZE0`tt-le??&3(_vV|=_h6}uljCtb?`few9 zWCU(L_uuUmpZC;uRtwqjLD{2P|IX;X0CZpMLH8cUPrk=)@ji;|8+MNV5=>9^^lXw} z_JjN6^SKF#p{Hkhb~8oW3ZvUBjhBlaJPZ!>7Ube^*q?vsa3yx*QoLj8vm&{GFe~%e zd^0zrZFA;7?Kb?i+LadmY3g3vWNx@GG`sd7L>u&=xjr#019yc$$CDoq%ez(i9{vdiXsXdg5t6I}#yAd__ zty9m4@|=H*`inJ%L2k8WVGT2kb^#q*#GR%6vNNFJH_8T7u)pVr?zULRI8%W03@+Y( zom)S&7kOR{Z#0KQ3(XN^Ap<|*q1Bye0vzbH`V9Ojw-M&TD&R*4zk6()*FHo~z29L( zAE(d@VDGLLx;)jo8aUG;FH1;&nr0vV){Q6nBrkuo8z(1~F0MVVy09BPctcDfM9^jI z(1U#su9^pE5d|{dPUW_&tyknArD}ebXZoV^2L5a-m(yf{4YKax(kS(+t_W-yDSJk>MrTV3RJnUxcHC+U1$=Ur;-;8>tGCM*$rXC>_e)nvn(Y*t*kaBmj_YpjH=6s6=6<8O-)MfMjb?|hndE<$ z&^DlUEo9Fx5A0WiueAPY8Y%NdI;)Eee*Ri39q7qsAJ0^~wmE~r=CoQa+!#c**C77z zPSM8%0!trw7kYA*R-Tc6rQd2VnSoe0W|aQsz+2|udQ~zp%ODTDy!)&36>qmE$ueDB z#p~27ldSYgU8Ehp$cm2}rR2ONrIW4Yj2?ULR_wAD!T43#JV9HjPn`AO6;)_#V=;416%DIn6 z*F7-I^fWgC-Qp%Xk2A}wO3x17Rbg|=+*9tJ3injHr^-F`uT9K8fKxpzEV<}UBDwAl zeY>N&-UI^@W%KK9BNG$`Y%sja*<|K|xg7+FzTUzMaC3$oUAr$o`7F7fBVXimLCFVLp^ppIJVB^b68q%-gew03RmO#r z_*H?9{u#W+h?g*n?vGXa{1gR4rrG6!$0FP~R$CosOfvUZo(H(^^G*Q3Q4yBKTNv+L46%hy%^S-qR# zDD3pnA@+FR>|yOuv36ibZ`q>QuPZPSS<;)+?~UsmFHn`UA=} z+mzM3UeD5UwMedF@Df51@dkyi->maWyR(Hmo4GqXrh8hrh8ur%%`R5s-^M5qb#=W5 zBNyP2GrMjJ@`bxzxFDdt#@@{_4l~?`(8ivj^Lj(8g+{WffxI!uX|SUsifxE{_tACD z3o%PrzmBGfNKWde_P27E1}riiVZcL6Sl1?IY;0r3ZsOXMynbNyRhx7sgw+=}CW)EVZkb6FqZtjFi-X5&^iq!WTc~T5L?<83);E^|BZhp3y zo(@g6zgvOqg<~r>KGbY4(-jv)TDpEfh)pjbd5tZC%9KmVQew- zGY6gaSF!VJ88M9)owjruJHMXE*`!}w;+06UY0xX3nwwH|GVK;8G2Bw7CzDyPbc)4{ z(q3^EJHO@=AtmjV;xK~QOv{7b`k4+-`lFlh*a7C(*M6^9>@SA4*$IzCZ!&3y)jAA1o29&F=5phCU}52&I(3wH4!VAGQwJYW&!jP1dLV50Wm0o8Q6g9nM=V!DF| ziJ#0SlN~%@GcI_tg9j{Pn#&zLUE$rZ%ueEVB3r2vv0S>+- zArwP0E?6&E>`;<`Vcp0H1qlJ2VO0Z00%(JC=0IxT@W4Z)4& zL-IINo@D9L@ZY97S7oo%oaYYVPz?4O$Xa?aLFh_crm6z<#5FzRVfPqDPW&~^WH8v><^Jo7*nh%60s9; zBAFjS8^05+)*s z&B4+JlK3(ROby@RKIGt5kuNPCb43LA@i|i=A|c>yp#0FHHW!5Z)YK3m4IHinVU+sV z2Vgp9gvroSi?47HGz=n9WBQGQ*ld3~jP4jWVp7plK>fXGomf-uAyRI<=fK4h8qQOu3phageN z&GWz)q2n_mWyDc(iM$9SD)fJmN1}enn4<)dk}Aq;UgXv>^fsSn!Mpnb-`x*_LND*S z@7LtExbH{yB8gJu4>btrkk4Ygo{|4m{+kL_nl8ayx zR$K8J&Eb4E2d~3JMxyB7m>&uw!{KlyINTJG!-4%qgd{>-l73TO#!i1oi7@<=bOK6( zdh-!LAc%TlQw}g>ga-)NpoKx~FirvlLC^s~_v(7r&AG+LcWlr1h@?Z79#&v7+`l0rBPPB%p!xO`mQFosE_U=J-UzZ*j=~I8hL-F8|OV1 z|9fL`PePFwD=^snxpAF%FXO}t%B2DkOGH==C_!j|XA)&JQVf4(=zbVd)zz*<0(k+> zgD7&C4?KJrejft33;u#2%$}E=hwZ3QJ|=vlT~IPm{??!o$0Of}(3prK*&8$5gg6ml z6dC?TQ04F-BsAcN;D};y4K=b#8mcfv6pNT*LhVPfTU+drJK<^{!85*}!0Z+e2;(N_Xz&Z-MPHVs?&?)T<=!|qAZ)j8HTcpz~_uPJcMIK`82Zl%t8|Q zzG1zoMDDv&V`UT>QgUWiDWU=e^^`^;6s-WvS5hFh37EI40RMr;Z>DUp+w;D z&?kxpkss8k9*{^FsL_x`j-ZSP6$)9aqB$cWXGjYni)6S3_;)pddz%j&2Hw>;4+QRS zppBDrNX*W5^ZM=lF?4G!3Y z$W5uwrErMiJoI@Z%?qSauq&>>a6v_cM$)ID-BWIPADB2`{unZGTuG%I#Yv=uB1HBUOBN=kihH|5*!LUBJD);T#azCpo_b{sRJ=C;@ z?%t`Sy(hqX0=y@{_aeZaB<^aWi`6Rm#VRVV?M z>-yo3t%PG?V+W_gwp+`)~rrgIj#VcJ4T5V-$SVJ z)9egCMJNoj`$&SM#c9+mXbf;df&!c8b~O=Zh!UIye!G;$tili&Shx8R5v->E+Ax1F zh&uSiZg|im3c5`Ybe9pNCOP2$F}!6@f>7zk;=j4!99R3mPZ|;s)yb3h#f`Hd;ioHhfcXbwk4_ zbJw*<%9Sv-wleY=Iz)s#4DtT=6LNozz%y6^ll#wf_VW;VfPfuL;LQDBi6D^ubw^~4 z0&6nIqpd~9$J{rLE`bcG?60|Hpr}GJkU~iA2o-|yTqqhzJ0(1zX2}Cixg&&6A_Y$O zEtdPC#GQ%z0m!Dh*aiB_fCy?v5i%y21D*Pk`#%{M7l0N8cYt6Q;ycEA-^qW3wWY(m zoJ#iuc~6k{1o__sl(3%0$NN|oWFCk#}&p-h3FE7#$h&0AI5){mS!x=}!kAxAHOFw}9$|pe( zNoM6KQjrK>lwk|5q~X{(7Y=_DKIqeB5bZJD;lM^SWA^CNum@C>=SvaYhgKY2=qgBi%D%o)oj41ER-``9w-Qb1CJNF;JDVD3Gj{QU(!;L};u{ zSm^hj|CpizOJTmTTj88V0Saj;_?y&1UdF)I-LP>qgecYuSU<|b;NO4Q2bB|vI2VMD zAffPuUz12{7zFeX5*ZoXV>AlY*80X^xXuGZQA+YC=r6CK2&PnI_CV0QhY_gpRpas% zfmEKEKmeTiqK{teT@{Sz2VxHx{PTSY6$Gb#c-OPQlE7VVCik>(PYeH=w9wu2AxwH% z-Fzuv0VtA;BVa%y_#J;Srh>4x$ngoM!d76sU*X4=4u^sSzI9_{J_v|Aom->ONEssK z0Sy9&`5_TJV2$RFNze%z0fPu5hlPp+Ot7fo&V2BtH_I4=6zG@{;Y;6|p8Tu&q28S+ zzy>BX0$`;e_J9mYq{4{Tyi}JoAofgPG`&KJP;IR^>SBG4S89L8BB|KcipP$i#E;CK zTSVa9+FNk6SPe-isX_g43+49QjqkemTT*!UCE%VW?rGv*ohG_xK=RExx%mW;iU{dj zyFeeTGJQr^$o!zz4oXHMVN^sk@D=H0;+X%>|ZO13%6J